diff --git a/hosts/ocaml/bin/run_tests.ml b/hosts/ocaml/bin/run_tests.ml index 9689fa35..04315ca8 100644 --- a/hosts/ocaml/bin/run_tests.ml +++ b/hosts/ocaml/bin/run_tests.ml @@ -1599,6 +1599,213 @@ let run_foundation_tests () = Printf.printf " FAIL: invocation_count: %s\n" (match other with Some n -> string_of_int n | None -> "None")); + Printf.printf "\nSuite: extensions/erlang_ext (Phase 9h)\n"; + (* Register the Erlang opcode namespace. Disjoint id range (200-217) + from test_ext (220/221) so they coexist. *) + Erlang_ext.register (); + + (match prim [String "erlang.OP_PATTERN_TUPLE"] with + | Integer 222 -> + incr pass_count; + Printf.printf " PASS: extension-opcode-id erlang.OP_PATTERN_TUPLE = 222\n" + | other -> + incr fail_count; + Printf.printf " FAIL: erlang.OP_PATTERN_TUPLE: got %s\n" + (Sx_types.inspect other)); + + (match prim [String "erlang.OP_BIF_IS_TUPLE"] with + | Integer 239 -> + incr pass_count; + Printf.printf " PASS: extension-opcode-id erlang.OP_BIF_IS_TUPLE = 239\n" + | other -> + incr fail_count; + Printf.printf " FAIL: erlang.OP_BIF_IS_TUPLE: got %s\n" + (Sx_types.inspect other)); + + (match prim [String "erlang.OP_NONEXISTENT"] with + | Nil -> + incr pass_count; + Printf.printf " PASS: unknown erlang opcode -> nil\n" + | other -> + incr fail_count; + Printf.printf " FAIL: unknown erlang opcode: got %s\n" + (Sx_types.inspect other)); + + (* Phase 10b vertical slice: erlang.OP_BIF_LENGTH (230) is a REAL + handler. Build [CONST 0; OP_BIF_LENGTH; RETURN] with an Erlang + list [1,2,3] in the constant pool; expect Integer 3. Proves the + full path: bytecode -> Sx_vm extension fallthrough -> erlang_ext + handler -> correct stack result. *) + (let mk_dict kvs = + let h = Hashtbl.create 4 in + List.iter (fun (k, v) -> Hashtbl.replace h k v) kvs; + Sx_types.Dict h in + let er_nil = mk_dict [("tag", Sx_types.String "nil")] in + let er_cons hd tl = + mk_dict [("tag", Sx_types.String "cons"); + ("head", hd); ("tail", tl)] in + let lst = er_cons (Sx_types.Integer 1) + (er_cons (Sx_types.Integer 2) + (er_cons (Sx_types.Integer 3) er_nil)) in + let code = ({ + vc_arity = 0; vc_rest_arity = -1; vc_locals = 0; + vc_bytecode = [| 1; 0; 0; 230; 50 |]; + vc_constants = [| lst |]; + vc_bytecode_list = None; vc_constants_list = None; + } : Sx_types.vm_code) in + let globals = Hashtbl.create 1 in + try + match Sx_vm.execute_module code globals with + | Integer 3 -> + incr pass_count; + Printf.printf " PASS: erlang.OP_BIF_LENGTH [1,2,3] -> 3 (real handler, end-to-end)\n" + | other -> + incr fail_count; + Printf.printf " FAIL: OP_BIF_LENGTH result: got %s\n" + (Sx_types.inspect other) + with exn -> + incr fail_count; + Printf.printf " FAIL: OP_BIF_LENGTH raised: %s\n" + (Printexc.to_string exn)); + + (* More real handlers (Phase 10b batch): build a list/tuple constant + and exercise HD/TL/TUPLE_SIZE/IS_* end-to-end through the VM. *) + (let mk_dict kvs = + let h = Hashtbl.create 4 in + List.iter (fun (k, v) -> Hashtbl.replace h k v) kvs; + Sx_types.Dict h in + let er_nil = mk_dict [("tag", Sx_types.String "nil")] in + let er_cons hd tl = mk_dict [("tag", Sx_types.String "cons"); + ("head", hd); ("tail", tl)] in + let er_tuple es = mk_dict [("tag", Sx_types.String "tuple"); + ("elements", Sx_types.List es)] in + let er_atom nm = mk_dict [("tag", Sx_types.String "atom"); + ("name", Sx_types.String nm)] in + let lst3 = er_cons (Sx_types.Integer 7) + (er_cons (Sx_types.Integer 8) + (er_cons (Sx_types.Integer 9) er_nil)) in + let tup3 = er_tuple [Sx_types.Integer 1; Sx_types.Integer 2; + Sx_types.Integer 3] in + let run consts bc = + let code = ({ + vc_arity = 0; vc_rest_arity = -1; vc_locals = 0; + vc_bytecode = bc; vc_constants = consts; + vc_bytecode_list = None; vc_constants_list = None; + } : Sx_types.vm_code) in + Sx_vm.execute_module code (Hashtbl.create 1) in + let nm = function + | Sx_types.Dict d -> + (match Hashtbl.find_opt d "name" with + | Some (Sx_types.String s) -> s | _ -> "?") + | _ -> "?" in + let check label want got = + if got = want then begin + incr pass_count; + Printf.printf " PASS: %s\n" label + end else begin + incr fail_count; + Printf.printf " FAIL: %s: got %s\n" label (Sx_types.inspect got) + end in + (* HD [7,8,9] -> 7 *) + check "OP_BIF_HD [7,8,9] -> 7" (Sx_types.Integer 7) + (run [| lst3 |] [| 1;0;0; 231; 50 |]); + (* TL [7,8,9] -> [8,9], check its HD = 8 *) + check "OP_BIF_TL then HD -> 8" (Sx_types.Integer 8) + (run [| lst3 |] [| 1;0;0; 232; 231; 50 |]); + (* TUPLE_SIZE {1,2,3} -> 3 *) + check "OP_BIF_TUPLE_SIZE {1,2,3} -> 3" (Sx_types.Integer 3) + (run [| tup3 |] [| 1;0;0; 234; 50 |]); + (* IS_INTEGER 42 -> true ; IS_INTEGER [..] -> false *) + (match run [| Sx_types.Integer 42 |] [| 1;0;0; 236; 50 |] with + | v when nm v = "true" -> + incr pass_count; Printf.printf " PASS: OP_BIF_IS_INTEGER 42 -> true\n" + | v -> incr fail_count; + Printf.printf " FAIL: IS_INTEGER 42: got %s\n" (Sx_types.inspect v)); + (match run [| lst3 |] [| 1;0;0; 236; 50 |] with + | v when nm v = "false" -> + incr pass_count; Printf.printf " PASS: OP_BIF_IS_INTEGER list -> false\n" + | v -> incr fail_count; + Printf.printf " FAIL: IS_INTEGER list: got %s\n" (Sx_types.inspect v)); + (* IS_ATOM atom -> true ; IS_LIST nil -> true ; IS_TUPLE tuple -> true *) + (match run [| er_atom "ok" |] [| 1;0;0; 237; 50 |] with + | v when nm v = "true" -> + incr pass_count; Printf.printf " PASS: OP_BIF_IS_ATOM ok -> true\n" + | v -> incr fail_count; + Printf.printf " FAIL: IS_ATOM: got %s\n" (Sx_types.inspect v)); + (match run [| er_nil |] [| 1;0;0; 238; 50 |] with + | v when nm v = "true" -> + incr pass_count; Printf.printf " PASS: OP_BIF_IS_LIST nil -> true\n" + | v -> incr fail_count; + Printf.printf " FAIL: IS_LIST nil: got %s\n" (Sx_types.inspect v)); + (match run [| tup3 |] [| 1;0;0; 239; 50 |] with + | v when nm v = "true" -> + incr pass_count; Printf.printf " PASS: OP_BIF_IS_TUPLE {..} -> true\n" + | v -> incr fail_count; + Printf.printf " FAIL: IS_TUPLE: got %s\n" (Sx_types.inspect v)); + (match run [| tup3 |] [| 1;0;0; 238; 50 |] with + | v when nm v = "false" -> + incr pass_count; Printf.printf " PASS: OP_BIF_IS_LIST tuple -> false\n" + | v -> incr fail_count; + Printf.printf " FAIL: IS_LIST tuple: got %s\n" (Sx_types.inspect v)); + (* ELEMENT: element(2, {1,2,3}) -> 2. Calling convention: push + Index then Tuple; opcode pops Tuple (TOS) then Index. *) + check "OP_BIF_ELEMENT element(2,{1,2,3}) -> 2" (Sx_types.Integer 2) + (run [| Sx_types.Integer 2; tup3 |] [| 1;0;0; 1;1;0; 233; 50 |]); + check "OP_BIF_ELEMENT element(1,{1,2,3}) -> 1" (Sx_types.Integer 1) + (run [| Sx_types.Integer 1; tup3 |] [| 1;0;0; 1;1;0; 233; 50 |]); + (* ELEMENT out of range raises *) + (let raised = + (try ignore (run [| Sx_types.Integer 9; tup3 |] + [| 1;0;0; 1;1;0; 233; 50 |]); false + with Sx_types.Eval_error _ -> true) in + if raised then begin + incr pass_count; + Printf.printf " PASS: OP_BIF_ELEMENT out-of-range raises\n" + end else begin + incr fail_count; + Printf.printf " FAIL: OP_BIF_ELEMENT out-of-range should raise\n" + end); + (* LISTS_REVERSE [7,8,9] -> [9,8,7]; verify HD = 9 then HD of TL = 8 *) + check "OP_BIF_LISTS_REVERSE then HD -> 9" (Sx_types.Integer 9) + (run [| lst3 |] [| 1;0;0; 235; 231; 50 |]); + check "OP_BIF_LISTS_REVERSE then TL,HD -> 8" (Sx_types.Integer 8) + (run [| lst3 |] [| 1;0;0; 235; 232; 231; 50 |]); + (* reverse preserves length *) + check "OP_BIF_LISTS_REVERSE then LENGTH -> 3" (Sx_types.Integer 3) + (run [| lst3 |] [| 1;0;0; 235; 230; 50 |])); + + (* A still-stubbed opcode (222 = erlang.OP_PATTERN_TUPLE) raises the + not-wired Eval_error — confirms the honest-failure path remains + for opcodes whose real handlers haven't landed. *) + (let globals = Hashtbl.create 1 in + try + ignore (Sx_vm.execute_module (make_bc_seq [| 222; 50 |]) globals); + incr fail_count; + Printf.printf " FAIL: erlang.OP_PATTERN_TUPLE dispatch should have raised\n" + with + | Sx_types.Eval_error msg + when (let needle = "not yet wired" in + let nl = String.length needle and ml = String.length msg in + let rec scan i = + if i + nl > ml then false + else if String.sub msg i nl = needle then true + else scan (i + 1) + in scan 0) -> + incr pass_count; + Printf.printf " PASS: erlang opcode dispatch raises not-wired error\n" + | exn -> + incr fail_count; + Printf.printf " FAIL: unexpected exn: %s\n" (Printexc.to_string exn)); + + (match Erlang_ext.dispatch_count () with + | Some n when n >= 1 -> + incr pass_count; + Printf.printf " PASS: erlang_ext state recorded %d dispatch(es)\n" n + | other -> + incr fail_count; + Printf.printf " FAIL: dispatch_count: %s\n" + (match other with Some n -> string_of_int n | None -> "None")); + Printf.printf "\nSuite: jit extension-opcode awareness\n"; let scan = Sx_vm.bytecode_uses_extension_opcodes in let no_consts = [||] in diff --git a/hosts/ocaml/bin/sx_server.ml b/hosts/ocaml/bin/sx_server.ml index 40de7b49..9d76c91f 100644 --- a/hosts/ocaml/bin/sx_server.ml +++ b/hosts/ocaml/bin/sx_server.ml @@ -18,6 +18,20 @@ open Sx_types +(* Force-link Sx_vm_extensions so its module-init runs: installs the + extension dispatch fallthrough and registers the `extension-opcode-id` + SX primitive. Without a reference here OCaml dead-code-eliminates the + module from sx_server.exe (it's only otherwise reached from run_tests), + leaving guest-language opcode extensions (Erlang Phase 9, etc.) + invisible to the runtime. The applied call is a harmless lookup. *) +let () = ignore (Sx_vm_extensions.id_of_name "") + +(* Register the Erlang opcode extension (Phase 9h) so + `extension-opcode-id "erlang.OP_*"` resolves to the host ids the SX + stub dispatcher consults. Guarded: a double-register raises Failure, + which we swallow so a re-entered server process doesn't die. *) +let () = try Erlang_ext.register () with Failure _ -> () + (* ====================================================================== *) (* Font measurement via otfm — reads OpenType/TrueType font tables *) (* ====================================================================== *) diff --git a/hosts/ocaml/lib/extensions/erlang_ext.ml b/hosts/ocaml/lib/extensions/erlang_ext.ml new file mode 100644 index 00000000..64ed1701 --- /dev/null +++ b/hosts/ocaml/lib/extensions/erlang_ext.ml @@ -0,0 +1,278 @@ +(** {1 [erlang_ext] — Erlang-on-SX VM opcode extension (Phase 9h)} + + Registers the Erlang opcode namespace in [Sx_vm_extensions] so that + [extension-opcode-id "erlang.OP_*"] resolves to a stable id. The SX + stub dispatcher in [lib/erlang/vm/dispatcher.sx] consults these ids + (Phase 9i) and falls back to its own local ids when the host + extension is absent. + + Opcode ids occupy 222-239 in the extension partition (200-247). + 222+ is chosen to clear the test extensions' reserved ids + (test_reg 210/211, test_ext 220/221) so all three coexist in + run_tests; production sx_server only registers this one. Names + mirror the SX stub dispatcher exactly: + + - 222 erlang.OP_PATTERN_TUPLE - 231 erlang.OP_BIF_HD + - 223 erlang.OP_PATTERN_LIST - 232 erlang.OP_BIF_TL + - 224 erlang.OP_PATTERN_BINARY - 233 erlang.OP_BIF_ELEMENT + - 225 erlang.OP_PERFORM - 234 erlang.OP_BIF_TUPLE_SIZE + - 226 erlang.OP_HANDLE - 235 erlang.OP_BIF_LISTS_REVERSE + - 227 erlang.OP_RECEIVE_SCAN - 236 erlang.OP_BIF_IS_INTEGER + - 228 erlang.OP_SPAWN - 237 erlang.OP_BIF_IS_ATOM + - 229 erlang.OP_SEND - 238 erlang.OP_BIF_IS_LIST + - 230 erlang.OP_BIF_LENGTH - 239 erlang.OP_BIF_IS_TUPLE + + {2 Handler status} + + The bytecode compiler does not yet emit these opcodes — Erlang + programs run through the general CEK path and the working + specialization path is the SX stub dispatcher. So every handler + here raises a descriptive [Eval_error] rather than silently + corrupting the VM stack. This keeps the extension honest: the + namespace is registered and disassembles by name, [extension-opcode-id] + works, but actually dispatching an opcode (which only happens once a + future phase teaches the compiler to emit them) fails loudly with a + pointer to the phase that will wire it. Real stack-machine handlers + land alongside compiler emission in a later phase. *) + +open Sx_types + +(** Per-instance state: invocation counter, purely to exercise the + [extension_state] machinery (mirrors [test_ext]). *) +type Sx_vm_extension.extension_state += ErlangExtState of { + mutable dispatched : int; +} + +let not_wired name = + raise (Eval_error + (Printf.sprintf + "%s: bytecode emission not yet wired (Phase 9j) — \ + Erlang runs via CEK; specialization path is the SX stub \ + dispatcher in lib/erlang/vm/dispatcher.sx" + name)) + +module M : Sx_vm_extension.EXTENSION = struct + let name = "erlang" + let init () = ErlangExtState { dispatched = 0 } + + let opcodes st = + let bump () = match st with + | ErlangExtState s -> s.dispatched <- s.dispatched + 1 + | _ -> () + in + let op id nm = + (id, nm, (fun (_vm : Sx_vm.vm) (_frame : Sx_vm.frame) -> + bump (); not_wired nm)) + in + (* Phase 10b vertical slice: one REAL register-machine handler. + erlang.OP_BIF_LENGTH (230) — pops an Erlang list off the VM + stack and pushes its length. Proves the full path works: + extension-opcode-id -> bytecode -> Sx_vm dispatch fallthrough + -> this handler -> correct stack result. The remaining 17 + opcodes still raise not_wired until their handlers + compiler + emission land. Erlang lists are tagged dicts: + nil = {"tag" -> String "nil"} + cons = {"tag" -> String "cons"; "head" -> v; "tail" -> v} *) + let er_tag d = + match Hashtbl.find_opt d "tag" with + | Some (String s) -> s | _ -> "" + in + let op_bif_length = + (230, "erlang.OP_BIF_LENGTH", + (fun (vm : Sx_vm.vm) (_frame : Sx_vm.frame) -> + bump (); + let v = Sx_vm.pop vm in + let rec walk acc node = + match node with + | Dict d -> + (match er_tag d with + | "nil" -> acc + | "cons" -> + (match Hashtbl.find_opt d "tail" with + | Some t -> walk (acc + 1) t + | None -> raise (Eval_error + "erlang.OP_BIF_LENGTH: cons cell without :tail")) + | _ -> raise (Eval_error + "erlang.OP_BIF_LENGTH: not a proper list")) + | _ -> raise (Eval_error + "erlang.OP_BIF_LENGTH: not a proper list") + in + Sx_vm.push vm (Integer (walk 0 v)))) + in + (* Phase 10b — simple hot-BIF handlers. Erlang bool is the atom + {"tag"->"atom"; "name"->"true"|"false"}; mk_atom builds it. *) + let mk_atom nm = + let h = Hashtbl.create 2 in + Hashtbl.replace h "tag" (String "atom"); + Hashtbl.replace h "name" (String nm); + Dict h + in + let er_bool b = mk_atom (if b then "true" else "false") in + let is_tag v t = match v with + | Dict d -> er_tag d = t + | _ -> false + in + let op_bif_hd = + (231, "erlang.OP_BIF_HD", + (fun (vm : Sx_vm.vm) _f -> + bump (); + match Sx_vm.pop vm with + | Dict d when er_tag d = "cons" -> + (match Hashtbl.find_opt d "head" with + | Some h -> Sx_vm.push vm h + | None -> raise (Eval_error "erlang.OP_BIF_HD: cons without :head")) + | _ -> raise (Eval_error "erlang.OP_BIF_HD: not a cons"))) + in + let op_bif_tl = + (232, "erlang.OP_BIF_TL", + (fun (vm : Sx_vm.vm) _f -> + bump (); + match Sx_vm.pop vm with + | Dict d when er_tag d = "cons" -> + (match Hashtbl.find_opt d "tail" with + | Some t -> Sx_vm.push vm t + | None -> raise (Eval_error "erlang.OP_BIF_TL: cons without :tail")) + | _ -> raise (Eval_error "erlang.OP_BIF_TL: not a cons"))) + in + let op_bif_tuple_size = + (234, "erlang.OP_BIF_TUPLE_SIZE", + (fun (vm : Sx_vm.vm) _f -> + bump (); + match Sx_vm.pop vm with + | Dict d when er_tag d = "tuple" -> + let n = match Hashtbl.find_opt d "elements" with + | Some (List es) -> List.length es + | Some (ListRef r) -> List.length !r + | _ -> raise (Eval_error + "erlang.OP_BIF_TUPLE_SIZE: tuple without :elements") + in + Sx_vm.push vm (Integer n) + | _ -> raise (Eval_error "erlang.OP_BIF_TUPLE_SIZE: not a tuple"))) + in + let op_bif_is_integer = + (236, "erlang.OP_BIF_IS_INTEGER", + (fun (vm : Sx_vm.vm) _f -> + bump (); + let v = Sx_vm.pop vm in + Sx_vm.push vm (er_bool (match v with Integer _ -> true | _ -> false)))) + in + let op_bif_is_atom = + (237, "erlang.OP_BIF_IS_ATOM", + (fun (vm : Sx_vm.vm) _f -> + bump (); + let v = Sx_vm.pop vm in + Sx_vm.push vm (er_bool (is_tag v "atom")))) + in + let op_bif_is_list = + (238, "erlang.OP_BIF_IS_LIST", + (fun (vm : Sx_vm.vm) _f -> + bump (); + let v = Sx_vm.pop vm in + Sx_vm.push vm (er_bool (is_tag v "cons" || is_tag v "nil")))) + in + let op_bif_is_tuple = + (239, "erlang.OP_BIF_IS_TUPLE", + (fun (vm : Sx_vm.vm) _f -> + bump (); + let v = Sx_vm.pop vm in + Sx_vm.push vm (er_bool (is_tag v "tuple")))) + in + (* element/2 and lists:reverse/1 — pure stack transforms (no + bytecode operands). Calling convention: args pushed left→right, + so element/2 stack is [.. Index Tuple] (Tuple on top). Erlang + element/2 is 1-indexed. *) + let op_bif_element = + (233, "erlang.OP_BIF_ELEMENT", + (fun (vm : Sx_vm.vm) _f -> + bump (); + let tup = Sx_vm.pop vm in + let idx = Sx_vm.pop vm in + match tup, idx with + | Dict d, Integer i when er_tag d = "tuple" -> + let es = match Hashtbl.find_opt d "elements" with + | Some (List es) -> es + | Some (ListRef r) -> !r + | _ -> raise (Eval_error + "erlang.OP_BIF_ELEMENT: tuple without :elements") + in + let n = List.length es in + if i < 1 || i > n then + raise (Eval_error + (Printf.sprintf + "erlang.OP_BIF_ELEMENT: index %d out of range 1..%d" i n)) + else + Sx_vm.push vm (List.nth es (i - 1)) + | _, Integer _ -> + raise (Eval_error "erlang.OP_BIF_ELEMENT: 2nd arg not a tuple") + | _ -> + raise (Eval_error "erlang.OP_BIF_ELEMENT: 1st arg not an integer"))) + in + let op_bif_lists_reverse = + (235, "erlang.OP_BIF_LISTS_REVERSE", + (fun (vm : Sx_vm.vm) _f -> + bump (); + let v = Sx_vm.pop vm in + let mk_nil () = + let h = Hashtbl.create 1 in + Hashtbl.replace h "tag" (String "nil"); Dict h in + let mk_cons hd tl = + let h = Hashtbl.create 3 in + Hashtbl.replace h "tag" (String "cons"); + Hashtbl.replace h "head" hd; + Hashtbl.replace h "tail" tl; + Dict h in + let rec rev acc node = + match node with + | Dict d -> + (match er_tag d with + | "nil" -> acc + | "cons" -> + let hd = match Hashtbl.find_opt d "head" with + | Some x -> x + | None -> raise (Eval_error + "erlang.OP_BIF_LISTS_REVERSE: cons without :head") in + let tl = match Hashtbl.find_opt d "tail" with + | Some x -> x + | None -> raise (Eval_error + "erlang.OP_BIF_LISTS_REVERSE: cons without :tail") in + rev (mk_cons hd acc) tl + | _ -> raise (Eval_error + "erlang.OP_BIF_LISTS_REVERSE: not a proper list")) + | _ -> raise (Eval_error + "erlang.OP_BIF_LISTS_REVERSE: not a proper list") + in + Sx_vm.push vm (rev (mk_nil ()) v))) + in + [ + op 222 "erlang.OP_PATTERN_TUPLE"; + op 223 "erlang.OP_PATTERN_LIST"; + op 224 "erlang.OP_PATTERN_BINARY"; + op 225 "erlang.OP_PERFORM"; + op 226 "erlang.OP_HANDLE"; + op 227 "erlang.OP_RECEIVE_SCAN"; + op 228 "erlang.OP_SPAWN"; + op 229 "erlang.OP_SEND"; + op_bif_length; + op_bif_hd; + op_bif_tl; + op_bif_element; + op_bif_tuple_size; + op_bif_lists_reverse; + op_bif_is_integer; + op_bif_is_atom; + op_bif_is_list; + op_bif_is_tuple; + ] +end + +(** Register [erlang] in [Sx_vm_extensions]. Idempotent only by failing + loudly — calling twice raises [Failure]. sx_server calls this once + at startup. *) +let register () = Sx_vm_extensions.register (module M : Sx_vm_extension.EXTENSION) + +(** Read the dispatch counter from the live registry state. [None] if + [register] hasn't run. *) +let dispatch_count () = + match Sx_vm_extensions.state_of_extension "erlang" with + | Some (ErlangExtState s) -> Some s.dispatched + | _ -> None diff --git a/lib/erlang/bench_ring_results.md b/lib/erlang/bench_ring_results.md index 96883b8f..589b6e54 100644 --- a/lib/erlang/bench_ring_results.md +++ b/lib/erlang/bench_ring_results.md @@ -33,3 +33,54 @@ least: persistent (path-copying) envs, an inline scheduler that doesn't call/cc on the common path (msg-already-in-mailbox), and a linked-list mailbox. None of those are in scope for the Phase 3 checkbox — captured here as the floor we're starting from. + +## Phase 9 status (2026-05-14) + +Specialized opcodes 9b–9f landed as **stub dispatchers** in +`lib/erlang/vm/dispatcher.sx`: `OP_PATTERN_TUPLE/LIST/BINARY`, +`OP_PERFORM/HANDLE`, `OP_RECEIVE_SCAN`, `OP_SPAWN/SEND`, and ten +`OP_BIF_*` hot dispatch entries. Each opcode's handler is a thin +wrapper over the existing `er-match-*` / `er-bif-*` / runtime impls, +so **the perf numbers above are unchanged** — same per-hop cost, same +scheduler. The stubs exist to nail down opcode IDs, operand contracts, +and tests against `er-match!` parity *before* 9a (the OCaml +opcode-extension mechanism in `hosts/ocaml/evaluator/`) lands. + +When 9a integrates and the bytecode compiler can emit these opcodes +at hot call sites, the real speedup story (~3000× ring throughput, +~1000× spawn) starts. Until then this file documents the +pre-integration ceiling. 72 vm-suite tests guard the stub correctness; +full conformance is **709/709** with the stub infrastructure loaded. + +## Phase 9g — post-integration bench (2026-05-15) + +9a (vm-ext mechanism), 9h (`erlang_ext.ml` registering `erlang.OP_*` +ids 222-239), and 9i (SX dispatcher consulting `extension-opcode-id`) +are now integrated and built into `hosts/ocaml/_build/default/bin/sx_server.exe`. +Re-ran the ring ladder on that binary: + +| N (processes) | Hops | Wall-clock | Throughput | +|---|---|---|---| +| 10 | 10 | 938ms | 11 hops/s | +| 100 | 100 | 2772ms | 36 hops/s | +| 500 | 500 | 14190ms | 35 hops/s | +| 1000 | 1000 | 31814ms | 31 hops/s | + +**Numbers are unchanged from the pre-integration baseline** — and that +is the expected, correct result. The opcode handlers (both the SX stub +dispatcher and the OCaml `erlang_ext` module) wrap the existing +`er-match-*` / `er-bif-*` / scheduler implementations 1-to-1, and the +**bytecode compiler does not yet emit `erlang.OP_*` opcodes**, so every +hop still goes through the general CEK path exactly as before. The +unchanged numbers therefore double as a no-regression check: the full +extension wiring (cherry-picked vm-ext A-E + force-link + erlang_ext + +SX bridge) added zero per-hop cost. Conformance **715/715** on this +binary. + +The ~3000×/~1000× targets remain gated on a **future phase (Phase 10 — +bytecode emission)**: teach `lib/compiler.sx` (or the Erlang +transpiler) to emit `erlang.OP_PATTERN_TUPLE` etc. at hot call sites, +then give `erlang_ext.ml` real register-machine handlers instead of the +current honest not-wired raise. That is a substantial standalone phase, +tracked in `plans/erlang-on-sx.md`. 9g's deliverable — *honest +measurement + recorded numbers on the integrated binary* — is complete. diff --git a/lib/erlang/conformance.sh b/lib/erlang/conformance.sh index dd724163..c4a56a0d 100755 --- a/lib/erlang/conformance.sh +++ b/lib/erlang/conformance.sh @@ -36,6 +36,8 @@ SUITES=( "bank|er-bank-test-pass|er-bank-test-count" "echo|er-echo-test-pass|er-echo-test-count" "fib|er-fib-test-pass|er-fib-test-count" + "ffi|er-ffi-test-pass|er-ffi-test-count" + "vm|er-vm-test-pass|er-vm-test-count" ) cat > "$TMPFILE" << 'EPOCHS' @@ -56,6 +58,9 @@ cat > "$TMPFILE" << 'EPOCHS' (load "lib/erlang/tests/programs/bank.sx") (load "lib/erlang/tests/programs/echo.sx") (load "lib/erlang/tests/programs/fib_server.sx") +(load "lib/erlang/vm/dispatcher.sx") +(load "lib/erlang/tests/ffi.sx") +(load "lib/erlang/tests/vm.sx") (epoch 100) (eval "(list er-test-pass er-test-count)") (epoch 101) @@ -74,6 +79,10 @@ cat > "$TMPFILE" << 'EPOCHS' (eval "(list er-echo-test-pass er-echo-test-count)") (epoch 108) (eval "(list er-fib-test-pass er-fib-test-count)") +(epoch 109) +(eval "(list er-ffi-test-pass er-ffi-test-count)") +(epoch 110) +(eval "(list er-vm-test-pass er-vm-test-count)") EPOCHS timeout 600 "$SX_SERVER" < "$TMPFILE" > "$OUTFILE" 2>&1 diff --git a/lib/erlang/runtime.sx b/lib/erlang/runtime.sx index 03aaad5d..a7d7bc2a 100644 --- a/lib/erlang/runtime.sx +++ b/lib/erlang/runtime.sx @@ -853,6 +853,112 @@ (define er-modules-get (fn () (nth er-modules 0))) (define er-modules-reset! (fn () (set-nth! er-modules 0 {}))) +(define er-mk-module-slot + (fn (mod-env old-env version) + {:current mod-env :old old-env :version version :tag "module"})) + +(define er-module-current-env (fn (slot) (get slot :current))) +(define er-module-old-env (fn (slot) (get slot :old))) +(define er-module-version (fn (slot) (get slot :version))) + +;; ── FFI BIF registry (Phase 8) ─────────────────────────────────── +;; Global dict from "Module/Name/Arity" key to {:module :name :arity :fn :pure?}. +;; Replaces the giant cond chain in transpile.sx#er-apply-remote-bif over time — +;; Phase 8 BIFs (crypto / cid / file / httpc / sqlite) all register here. +(define er-bif-registry (list {})) +(define er-bif-registry-get (fn () (nth er-bif-registry 0))) +(define er-bif-registry-reset! (fn () (set-nth! er-bif-registry 0 {}))) + +(define er-bif-key + (fn (module name arity) + (str module "/" name "/" arity))) + +(define er-register-bif! + (fn (module name arity sx-fn) + (dict-set! (er-bif-registry-get) (er-bif-key module name arity) + {:module module :name name :arity arity :fn sx-fn :pure? false}) + (er-mk-atom "ok"))) + +(define er-register-pure-bif! + (fn (module name arity sx-fn) + (dict-set! (er-bif-registry-get) (er-bif-key module name arity) + {:module module :name name :arity arity :fn sx-fn :pure? true}) + (er-mk-atom "ok"))) + +(define er-lookup-bif + (fn (module name arity) + (let ((reg (er-bif-registry-get)) (k (er-bif-key module name arity))) + (if (dict-has? reg k) (get reg k) nil)))) + +(define er-list-bifs + (fn () (keys (er-bif-registry-get)))) + +;; ── term marshalling (Phase 8) ─────────────────────────────────── +;; Bridge Erlang term values (tagged dicts) and SX-native values for +;; FFI BIFs to call out into platform primitives. Conversions: +;; +;; Erlang SX-native +;; ───────────────────────── ──────────────── +;; atom {:tag "atom" :name S} ↔ symbol (make-symbol S) +;; nil {:tag "nil"} ↔ '() +;; cons {:tag "cons" :head :tail} → list of marshalled elements +;; tuple {:tag "tuple" :elements} → list of marshalled elements +;; binary {:tag "binary" :bytes} ↔ SX string +;; integer / float / boolean ↔ passthrough +;; SX string on the way back → binary +;; +;; Pids, refs, funs pass through unchanged — they have no SX-native +;; equivalent and are opaque to FFI primitives. + +(define er-cons-to-sx-list + (fn (v) + (cond + (er-nil? v) (list) + (er-cons? v) + (let ((tail (er-cons-to-sx-list (get v :tail))) + (head (er-to-sx (get v :head)))) + (let ((out (list head))) + (for-each + (fn (i) (append! out (nth tail i))) + (range 0 (len tail))) + out)) + :else (list v)))) + +(define er-to-sx + (fn (v) + (cond + (er-atom? v) (make-symbol (get v :name)) + (er-nil? v) (list) + (er-cons? v) (er-cons-to-sx-list v) + (er-tuple? v) + (let ((out (list)) (es (get v :elements))) + (for-each + (fn (i) (append! out (er-to-sx (nth es i)))) + (range 0 (len es))) + out) + (er-binary? v) (list->string (map integer->char (get v :bytes))) + :else v))) + +(define er-of-sx + (fn (v) + (let ((ty (type-of v))) + (cond + (= ty "symbol") (er-mk-atom (str v)) + (= ty "string") (er-mk-binary (map char->integer (string->list v))) + (= ty "list") + (let ((out (er-mk-nil))) + (for-each + (fn (i) + (set! out + (er-mk-cons (er-of-sx (nth v (- (- (len v) 1) i))) out))) + (range 0 (len v))) + out) + (= ty "nil") (er-mk-nil) + :else v)))) + + + + ;; Load an Erlang module declaration. Source must start with ;; `-module(Name).` and contain function definitions. Functions ;; sharing a name (different arities) get their clauses concatenated @@ -897,7 +1003,15 @@ ((all-clauses (get by-name k))) (er-env-bind! mod-env k (er-mk-fun all-clauses mod-env)))) (keys by-name)) - (dict-set! (er-modules-get) mod-name mod-env) + (let ((registry (er-modules-get))) + (if (dict-has? registry mod-name) + (let ((existing-slot (get registry mod-name))) + (dict-set! registry mod-name + (er-mk-module-slot mod-env + (er-module-current-env existing-slot) + (+ (er-module-version existing-slot) 1)))) + (dict-set! registry mod-name + (er-mk-module-slot mod-env nil 1)))) (er-mk-atom mod-name))))) (define @@ -905,7 +1019,7 @@ (fn (mod name vs) (let - ((mod-env (get (er-modules-get) mod))) + ((mod-env (er-module-current-env (get (er-modules-get) mod)))) (if (not (dict-has? mod-env name)) (raise @@ -1189,16 +1303,170 @@ :else (er-mk-atom "undefined"))) :else (error "Erlang: ets:info: arity")))) -(define - er-apply-ets-bif - (fn - (name vs) - (cond - (= name "new") (er-bif-ets-new vs) - (= name "insert") (er-bif-ets-insert vs) - (= name "lookup") (er-bif-ets-lookup vs) - (= name "delete") (er-bif-ets-delete vs) - (= name "tab2list") (er-bif-ets-tab2list vs) - (= name "info") (er-bif-ets-info vs) - :else (error - (str "Erlang: undefined 'ets:" name "/" (len vs) "'"))))) + + +;; ── file module (Phase 8 FFI) ──────────────────────────────────── +;; Synchronous file IO. Filenames must be SX strings (or Erlang +;; binaries/char-code lists coercible to strings via er-source-to-string). +;; Returns `{ok, Binary}` / `ok` on success, `{error, Reason}` on failure +;; where Reason is one of `enoent`, `eacces`, `enotdir`, `posix_error`. + +(define er-classify-file-error + (fn (msg) + (let ((s (str msg))) + (cond + (string-contains? s "No such") (er-mk-atom "enoent") + (string-contains? s "Permission denied") (er-mk-atom "eacces") + (string-contains? s "Not a directory") (er-mk-atom "enotdir") + (string-contains? s "Is a directory") (er-mk-atom "eisdir") + :else (er-mk-atom "posix_error"))))) + +(define er-bif-file-read-file + (fn (vs) + (let ((path (er-source-to-string (nth vs 0)))) + (cond + (= path nil) + (er-mk-tuple (list (er-mk-atom "error") (er-mk-atom "badarg"))) + :else + (let ((res (list nil)) (err (list nil))) + (guard (c (:else (set-nth! err 0 c))) + (set-nth! res 0 (file-read path))) + (cond + (not (= (nth err 0) nil)) + (er-mk-tuple (list (er-mk-atom "error") + (er-classify-file-error (nth err 0)))) + :else + (er-mk-tuple (list (er-mk-atom "ok") + (er-mk-binary (map char->integer (string->list (nth res 0)))))))))))) + +(define er-bif-file-write-file + (fn (vs) + (let ((path (er-source-to-string (nth vs 0))) + (data (er-source-to-string (nth vs 1)))) + (cond + (or (= path nil) (= data nil)) + (er-mk-tuple (list (er-mk-atom "error") (er-mk-atom "badarg"))) + :else + (let ((err (list nil))) + (guard (c (:else (set-nth! err 0 c))) + (file-write path data)) + (cond + (not (= (nth err 0) nil)) + (er-mk-tuple (list (er-mk-atom "error") + (er-classify-file-error (nth err 0)))) + :else (er-mk-atom "ok"))))))) + +(define er-bif-file-delete + (fn (vs) + (let ((path (er-source-to-string (nth vs 0)))) + (cond + (= path nil) + (er-mk-tuple (list (er-mk-atom "error") (er-mk-atom "badarg"))) + :else + (let ((err (list nil))) + (guard (c (:else (set-nth! err 0 c))) + (file-delete path)) + (cond + (not (= (nth err 0) nil)) + (er-mk-tuple (list (er-mk-atom "error") + (er-classify-file-error (nth err 0)))) + :else (er-mk-atom "ok"))))))) + +;; ── builtin BIF registrations (Phase 8 migration) ──────────────── +;; Populates `er-bif-registry` with every existing built-in BIF. Each +;; entry is keyed by "Module/Name/Arity"; multi-arity BIFs register +;; once per arity. Called eagerly at the end of runtime.sx so the +;; registry is ready before any erlang-eval-ast call. +(define er-register-builtin-bifs! + (fn () + ;; erlang module — type predicates (all pure) + (er-register-pure-bif! "erlang" "is_integer" 1 er-bif-is-integer) + (er-register-pure-bif! "erlang" "is_atom" 1 er-bif-is-atom) + (er-register-pure-bif! "erlang" "is_list" 1 er-bif-is-list) + (er-register-pure-bif! "erlang" "is_tuple" 1 er-bif-is-tuple) + (er-register-pure-bif! "erlang" "is_number" 1 er-bif-is-number) + (er-register-pure-bif! "erlang" "is_float" 1 er-bif-is-float) + (er-register-pure-bif! "erlang" "is_boolean" 1 er-bif-is-boolean) + (er-register-pure-bif! "erlang" "is_pid" 1 er-bif-is-pid) + (er-register-pure-bif! "erlang" "is_reference" 1 er-bif-is-reference) + (er-register-pure-bif! "erlang" "is_binary" 1 er-bif-is-binary) + (er-register-pure-bif! "erlang" "is_function" 1 er-bif-is-function) + (er-register-pure-bif! "erlang" "is_function" 2 er-bif-is-function) + ;; erlang module — pure data ops + (er-register-pure-bif! "erlang" "length" 1 er-bif-length) + (er-register-pure-bif! "erlang" "hd" 1 er-bif-hd) + (er-register-pure-bif! "erlang" "tl" 1 er-bif-tl) + (er-register-pure-bif! "erlang" "element" 2 er-bif-element) + (er-register-pure-bif! "erlang" "tuple_size" 1 er-bif-tuple-size) + (er-register-pure-bif! "erlang" "byte_size" 1 er-bif-byte-size) + (er-register-pure-bif! "erlang" "atom_to_list" 1 er-bif-atom-to-list) + (er-register-pure-bif! "erlang" "list_to_atom" 1 er-bif-list-to-atom) + (er-register-pure-bif! "erlang" "abs" 1 er-bif-abs) + (er-register-pure-bif! "erlang" "min" 2 er-bif-min) + (er-register-pure-bif! "erlang" "max" 2 er-bif-max) + (er-register-pure-bif! "erlang" "tuple_to_list" 1 er-bif-tuple-to-list) + (er-register-pure-bif! "erlang" "list_to_tuple" 1 er-bif-list-to-tuple) + (er-register-pure-bif! "erlang" "integer_to_list" 1 er-bif-integer-to-list) + (er-register-pure-bif! "erlang" "list_to_integer" 1 er-bif-list-to-integer) + ;; erlang module — process / runtime (side-effecting) + (er-register-bif! "erlang" "self" 0 er-bif-self) + (er-register-bif! "erlang" "spawn" 1 er-bif-spawn) + (er-register-bif! "erlang" "spawn" 3 er-bif-spawn) + (er-register-bif! "erlang" "exit" 1 er-bif-exit) + (er-register-bif! "erlang" "exit" 2 er-bif-exit) + (er-register-bif! "erlang" "make_ref" 0 er-bif-make-ref) + (er-register-bif! "erlang" "link" 1 er-bif-link) + (er-register-bif! "erlang" "unlink" 1 er-bif-unlink) + (er-register-bif! "erlang" "monitor" 2 er-bif-monitor) + (er-register-bif! "erlang" "demonitor" 1 er-bif-demonitor) + (er-register-bif! "erlang" "process_flag" 2 er-bif-process-flag) + (er-register-bif! "erlang" "register" 2 er-bif-register) + (er-register-bif! "erlang" "unregister" 1 er-bif-unregister) + (er-register-bif! "erlang" "whereis" 1 er-bif-whereis) + (er-register-bif! "erlang" "registered" 0 er-bif-registered) + ;; erlang module — exception raising (modelled as side-effecting) + (er-register-bif! "erlang" "throw" 1 + (fn (vs) (raise (er-mk-throw-marker (er-bif-arg1 vs "throw"))))) + (er-register-bif! "erlang" "error" 1 + (fn (vs) (raise (er-mk-error-marker (er-bif-arg1 vs "error"))))) + ;; lists module — all pure + (er-register-pure-bif! "lists" "reverse" 1 er-bif-lists-reverse) + (er-register-pure-bif! "lists" "map" 2 er-bif-lists-map) + (er-register-pure-bif! "lists" "foldl" 3 er-bif-lists-foldl) + (er-register-pure-bif! "lists" "seq" 2 er-bif-lists-seq) + (er-register-pure-bif! "lists" "seq" 3 er-bif-lists-seq) + (er-register-pure-bif! "lists" "sum" 1 er-bif-lists-sum) + (er-register-pure-bif! "lists" "nth" 2 er-bif-lists-nth) + (er-register-pure-bif! "lists" "last" 1 er-bif-lists-last) + (er-register-pure-bif! "lists" "member" 2 er-bif-lists-member) + (er-register-pure-bif! "lists" "append" 2 er-bif-lists-append) + (er-register-pure-bif! "lists" "filter" 2 er-bif-lists-filter) + (er-register-pure-bif! "lists" "any" 2 er-bif-lists-any) + (er-register-pure-bif! "lists" "all" 2 er-bif-lists-all) + (er-register-pure-bif! "lists" "duplicate" 2 er-bif-lists-duplicate) + ;; io module — side-effecting (writes to io buffer) + (er-register-bif! "io" "format" 1 er-bif-io-format) + (er-register-bif! "io" "format" 2 er-bif-io-format) + ;; ets module — side-effecting (mutates table state) + (er-register-bif! "ets" "new" 2 er-bif-ets-new) + (er-register-bif! "ets" "insert" 2 er-bif-ets-insert) + (er-register-bif! "ets" "lookup" 2 er-bif-ets-lookup) + (er-register-bif! "ets" "delete" 1 er-bif-ets-delete) + (er-register-bif! "ets" "delete" 2 er-bif-ets-delete) + (er-register-bif! "ets" "tab2list" 1 er-bif-ets-tab2list) + (er-register-bif! "ets" "info" 2 er-bif-ets-info) + ;; code module — side-effecting (mutates module registry, kills procs) + (er-register-bif! "code" "load_binary" 3 er-bif-code-load-binary) + (er-register-bif! "code" "purge" 1 er-bif-code-purge) + (er-register-bif! "code" "soft_purge" 1 er-bif-code-soft-purge) + (er-register-bif! "code" "which" 1 er-bif-code-which) + (er-register-bif! "code" "is_loaded" 1 er-bif-code-is-loaded) + (er-register-bif! "code" "all_loaded" 0 er-bif-code-all-loaded) + ;; file module + (er-register-bif! "file" "read_file" 1 er-bif-file-read-file) + (er-register-bif! "file" "write_file" 2 er-bif-file-write-file) + (er-register-bif! "file" "delete" 1 er-bif-file-delete) + (er-mk-atom "ok"))) + +;; Register everything at load time. +(er-register-builtin-bifs!) diff --git a/lib/erlang/scoreboard.json b/lib/erlang/scoreboard.json index b2db94e0..c57e1d32 100644 --- a/lib/erlang/scoreboard.json +++ b/lib/erlang/scoreboard.json @@ -1,16 +1,18 @@ { "language": "erlang", - "total_pass": 530, - "total": 530, + "total_pass": 715, + "total": 715, "suites": [ {"name":"tokenize","pass":62,"total":62,"status":"ok"}, {"name":"parse","pass":52,"total":52,"status":"ok"}, - {"name":"eval","pass":346,"total":346,"status":"ok"}, - {"name":"runtime","pass":39,"total":39,"status":"ok"}, + {"name":"eval","pass":385,"total":385,"status":"ok"}, + {"name":"runtime","pass":93,"total":93,"status":"ok"}, {"name":"ring","pass":4,"total":4,"status":"ok"}, {"name":"ping-pong","pass":4,"total":4,"status":"ok"}, {"name":"bank","pass":8,"total":8,"status":"ok"}, {"name":"echo","pass":7,"total":7,"status":"ok"}, - {"name":"fib","pass":8,"total":8,"status":"ok"} + {"name":"fib","pass":8,"total":8,"status":"ok"}, + {"name":"ffi","pass":14,"total":14,"status":"ok"}, + {"name":"vm","pass":78,"total":78,"status":"ok"} ] } diff --git a/lib/erlang/scoreboard.md b/lib/erlang/scoreboard.md index bf9592fa..2b0cd79b 100644 --- a/lib/erlang/scoreboard.md +++ b/lib/erlang/scoreboard.md @@ -1,18 +1,20 @@ # Erlang-on-SX Scoreboard -**Total: 530 / 530 tests passing** +**Total: 715 / 715 tests passing** | | Suite | Pass | Total | |---|---|---|---| | ✅ | tokenize | 62 | 62 | | ✅ | parse | 52 | 52 | -| ✅ | eval | 346 | 346 | -| ✅ | runtime | 39 | 39 | +| ✅ | eval | 385 | 385 | +| ✅ | runtime | 93 | 93 | | ✅ | ring | 4 | 4 | | ✅ | ping-pong | 4 | 4 | | ✅ | bank | 8 | 8 | | ✅ | echo | 7 | 7 | | ✅ | fib | 8 | 8 | +| ✅ | ffi | 14 | 14 | +| ✅ | vm | 78 | 78 | Generated by `lib/erlang/conformance.sh`. diff --git a/lib/erlang/tests/eval.sx b/lib/erlang/tests/eval.sx index a3056000..4bd322db 100644 --- a/lib/erlang/tests/eval.sx +++ b/lib/erlang/tests/eval.sx @@ -1125,6 +1125,222 @@ (er-eval-test "lists:duplicate val" (nm (ev "hd(lists:duplicate(3, marker))")) "marker") + +;; ── Phase 7: code:load_binary/3 ─────────────────────────────── +(er-modules-reset!) + +(er-eval-test "code:load_binary ok tag" + (nm (ev "element(1, code:load_binary(cl1, \"cl1.erl\", \"-module(cl1). foo() -> 1.\"))")) + "module") +(er-eval-test "code:load_binary ok name" + (nm (ev "element(2, code:load_binary(cl1, \"cl1.erl\", \"-module(cl1). foo() -> 1.\"))")) + "cl1") +(er-eval-test "code:load_binary then call" + (ev "cl1:foo()") 1) + +(er-eval-test "code:load_binary reload v2" + (ev "code:load_binary(cl1, \"cl1.erl\", \"-module(cl1). foo() -> 99.\"), cl1:foo()") + 99) + +(er-eval-test "code:load_binary name mismatch tag" + (nm (ev "element(1, code:load_binary(cl2, \"x.erl\", \"-module(other). f() -> 0.\"))")) + "error") +(er-eval-test "code:load_binary name mismatch reason" + (nm (ev "element(2, code:load_binary(cl2, \"x.erl\", \"-module(other). f() -> 0.\"))")) + "module_name_mismatch") + +(er-eval-test "code:load_binary badfile on garbage" + (nm (ev "element(2, code:load_binary(cl3, \"x.erl\", \"this is not erlang\"))")) + "badfile") + +(er-eval-test "code:load_binary non-atom mod is badarg" + (nm (ev "element(2, code:load_binary(\"cl1\", \"x.erl\", \"-module(cl1). f() -> 0.\"))")) + "badarg") + + +;; ── Phase 7: code:purge/1 + code:soft_purge/1 ─────────────────── +(er-modules-reset!) + +;; purge unknown module → false +(er-eval-test "code:purge unknown" + (nm (ev "code:purge(nope)")) "false") + +;; load, then purge without old version → false (nothing to purge) +(er-eval-test "code:purge no old" + (nm (ev "code:load_binary(pg1, \"pg1\", \"-module(pg1). v() -> 1.\"), code:purge(pg1)")) + "false") + +;; load v1, load v2 (creates :old), purge with no live procs → true +(er-eval-test "code:purge after reload" + (nm (ev "code:load_binary(pg2, \"pg2\", \"-module(pg2). v() -> 1.\"), code:load_binary(pg2, \"pg2\", \"-module(pg2). v() -> 2.\"), code:purge(pg2)")) + "true") + +;; idempotent: purging again returns false (already purged) +(er-eval-test "code:purge twice" + (nm (ev "code:load_binary(pg3, \"pg3\", \"-module(pg3). v() -> 1.\"), code:load_binary(pg3, \"pg3\", \"-module(pg3). v() -> 2.\"), code:purge(pg3), code:purge(pg3)")) + "false") + +;; purge returns true whenever an :old slot exists, regardless of process tracking +;; (proper "kill lingering" semantics requires spawn/3 which is still stubbed) +(er-eval-test "code:purge with old slot present" + (nm (ev "code:load_binary(pg4, \"pg4\", \"-module(pg4). loop() -> receive stop -> ok end.\"), + Pid = spawn(fun () -> pg4:loop() end), + code:load_binary(pg4, \"pg4\", \"-module(pg4). loop() -> receive stop -> done end.\"), + code:purge(pg4)")) + "true") + +;; soft_purge unknown → true (nothing to purge) +(er-eval-test "code:soft_purge unknown" + (nm (ev "code:soft_purge(nope)")) "true") + +;; soft_purge with no old version → true +(er-eval-test "code:soft_purge no old" + (nm (ev "code:load_binary(sp1, \"sp1\", \"-module(sp1). v() -> 1.\"), code:soft_purge(sp1)")) + "true") + +;; soft_purge with old + no lingering procs → true (clears :old) +(er-eval-test "code:soft_purge clean" + (nm (ev "code:load_binary(sp2, \"sp2\", \"-module(sp2). v() -> 1.\"), code:load_binary(sp2, \"sp2\", \"-module(sp2). v() -> 2.\"), code:soft_purge(sp2)")) + "true") + +;; non-atom Mod is badarg (raise) +(er-eval-test "code:purge badarg" + (nm (ev "try code:purge(\"str\") catch error:badarg -> ok end")) "ok") +(er-eval-test "code:soft_purge badarg" + (nm (ev "try code:soft_purge(123) catch error:badarg -> ok end")) "ok") + + +;; ── Phase 7: code:which/1 + code:is_loaded/1 + code:all_loaded/0 ── +(er-modules-reset!) + +(er-eval-test "code:which non_existing" + (nm (ev "code:which(nope)")) "non_existing") + +(er-eval-test "code:which after load" + (nm (ev "code:load_binary(wh1, \"wh1\", \"-module(wh1). v() -> 1.\"), code:which(wh1)")) + "loaded") + +(er-eval-test "code:is_loaded missing" + (nm (ev "code:is_loaded(nope)")) "false") + +(er-eval-test "code:is_loaded tag" + (nm (ev "code:load_binary(il1, \"il1\", \"-module(il1). v() -> 1.\"), element(1, code:is_loaded(il1))")) + "file") + +(er-eval-test "code:is_loaded value" + (nm (ev "code:load_binary(il2, \"il2\", \"-module(il2). v() -> 1.\"), element(2, code:is_loaded(il2))")) + "loaded") + +(er-modules-reset!) +(er-eval-test "code:all_loaded empty" + (ev "length(code:all_loaded())") 0) + +(er-modules-reset!) +(er-eval-test "code:all_loaded count" + (ev "code:load_binary(al1, \"al1\", \"-module(al1). v() -> 1.\"), + code:load_binary(al2, \"al2\", \"-module(al2). v() -> 1.\"), + length(code:all_loaded())") + 2) + +(er-eval-test "code:all_loaded first entry tag" + (nm (ev "code:load_binary(al3, \"al3\", \"-module(al3). v() -> 1.\"), + element(2, hd(code:all_loaded()))")) + "loaded") + +(er-eval-test "code:which badarg" + (nm (ev "try code:which(\"str\") catch error:badarg -> ok end")) "ok") +(er-eval-test "code:is_loaded badarg" + (nm (ev "try code:is_loaded(123) catch error:badarg -> ok end")) "ok") + + +;; ── Phase 7: hot-reload call dispatch semantics ────────────────── +;; Cross-module M:F() calls always hit the CURRENT version; +;; local F() calls inside a module body resolve through the env +;; the function closed over (i.e. the version it was loaded with). + +(er-modules-reset!) + +;; M:F always hits current +(er-eval-test "cross-mod after reload v2" + (ev "code:load_binary(hr1, \"hr1\", \"-module(hr1). f() -> 1.\"), + code:load_binary(hr1, \"hr1\", \"-module(hr1). f() -> 2.\"), + hr1:f()") + 2) + +;; Local call inside reloaded module body resolves via fresh mod-env +;; (a() does a local b(); b() got upgraded too) +(er-eval-test "local call inside reloaded module body" + (ev "code:load_binary(hr2, \"hr2\", \"-module(hr2). a() -> b(). b() -> 1.\"), + code:load_binary(hr2, \"hr2\", \"-module(hr2). a() -> b(). b() -> 99.\"), + hr2:a()") + 99) + +;; Fun captured BEFORE reload, with local-call body, keeps v1 semantics +(er-eval-test "captured fun keeps closed-over env (local call)" + (ev "code:load_binary(hr3, \"hr3\", \"-module(hr3). get_fn() -> fun () -> b() end. b() -> 1.\"), + Fn = hr3:get_fn(), + code:load_binary(hr3, \"hr3\", \"-module(hr3). get_fn() -> fun () -> b() end. b() -> 99.\"), + Fn()") + 1) + +;; Fun captured BEFORE reload, with CROSS-mod body, sees v2's current +(er-eval-test "captured fun follows cross-mod to current" + (ev "code:load_binary(hr4, \"hr4\", \"-module(hr4). get_xref() -> fun () -> hr4:b() end. b() -> 1.\"), + Fn = hr4:get_xref(), + code:load_binary(hr4, \"hr4\", \"-module(hr4). get_xref() -> fun () -> hr4:b() end. b() -> 99.\"), + Fn()") + 99) + +;; Two captured funs from two different vintages +(er-eval-test "two funs from two vintages stay independent" + (ev "code:load_binary(hr5, \"hr5\", \"-module(hr5). gf() -> fun () -> v() end. v() -> 10.\"), + F1 = hr5:gf(), + code:load_binary(hr5, \"hr5\", \"-module(hr5). gf() -> fun () -> v() end. v() -> 20.\"), + F2 = hr5:gf(), + F1() + F2()") + 30) + +;; Version slot bumps correctly when a captured fun stays alive +(er-eval-test "version bumps despite captured funs" + (ev "code:load_binary(hr6, \"hr6\", \"-module(hr6). gf() -> fun () -> v() end. v() -> 1.\"), + _Pinned = hr6:gf(), + code:load_binary(hr6, \"hr6\", \"-module(hr6). gf() -> fun () -> v() end. v() -> 2.\"), + code:load_binary(hr6, \"hr6\", \"-module(hr6). gf() -> fun () -> v() end. v() -> 3.\"), + hr6:v()") + 3) + + + +;; ── Phase 7 capstone: full hot-reload ladder ─────────────────── +;; Load v1 → spawn from inside module → load v2 → cross-mod hits v2 → +;; local call inside v1 process still resolves v1 → soft_purge refuses +;; while v1 procs alive → purge kills them. +;; +;; All stages must run in a single erlang-eval-ast call: each call resets +;; the scheduler (er-sched-init!) so cross-call Pid handles would point at +;; reaped processes. +(er-modules-reset!) + +(define er-rt-cap-prog "code:load_binary(cap, \"cap.erl\", \"-module(cap). start() -> spawn(fun () -> loop() end). loop() -> receive {ping, From} -> From ! {pong, v1}, loop(); stop -> done end. tag() -> v1.\"), Tag1 = cap:tag(), Pid1 = cap:start(), code:load_binary(cap, \"cap.erl\", \"-module(cap). start() -> spawn(fun () -> loop() end). loop() -> receive {ping, From} -> From ! {pong, v2}, loop(); stop -> done end. tag() -> v2.\"), Tag2 = cap:tag(), _Pid2 = cap:start(), Soft1 = code:soft_purge(cap), Hard = code:purge(cap), Soft2 = code:soft_purge(cap), {Tag1, Tag2, Soft1, Hard, Soft2}") + +(define er-rt-cap-result (ev er-rt-cap-prog)) + +(er-eval-test "capstone v1 tag direct" + (get (nth (get er-rt-cap-result :elements) 0) :name) "v1") + +(er-eval-test "capstone v2 tag" + (get (nth (get er-rt-cap-result :elements) 1) :name) "v2") + +(er-eval-test "capstone soft_purge while v1 alive = false" + (get (nth (get er-rt-cap-result :elements) 2) :name) "false") + +(er-eval-test "capstone hard purge = true" + (get (nth (get er-rt-cap-result :elements) 3) :name) "true") + +(er-eval-test "capstone soft_purge clean after hard = true" + (get (nth (get er-rt-cap-result :elements) 4) :name) "true") + + (define er-eval-test-summary (str "eval " er-eval-test-pass "/" er-eval-test-count)) diff --git a/lib/erlang/tests/ffi.sx b/lib/erlang/tests/ffi.sx new file mode 100644 index 00000000..8c0ffaa2 --- /dev/null +++ b/lib/erlang/tests/ffi.sx @@ -0,0 +1,113 @@ +;; Phase 8 FFI BIF tests — one round-trip per BIF. +;; Each BIF lives in lib/erlang/runtime.sx (registered with +;; er-bif-registry) and wraps an SX-host primitive. + +(define er-ffi-test-count 0) +(define er-ffi-test-pass 0) +(define er-ffi-test-fails (list)) + +(define + er-ffi-test + (fn + (name actual expected) + (set! er-ffi-test-count (+ er-ffi-test-count 1)) + (if + (= actual expected) + (set! er-ffi-test-pass (+ er-ffi-test-pass 1)) + (append! er-ffi-test-fails {:name name :expected expected :actual actual})))) + +(define ffi-ev erlang-eval-ast) +(define ffi-nm (fn (v) (get v :name))) + +;; ── file:read_file/1 + file:write_file/2 ──────────────────────── +(er-ffi-test + "file:write_file ok" + (ffi-nm (ffi-ev "file:write_file(\"/tmp/er-ffi-1.txt\", \"hello\")")) + "ok") + +(er-ffi-test + "file:read_file ok tag" + (ffi-nm (ffi-ev "element(1, file:read_file(\"/tmp/er-ffi-1.txt\"))")) + "ok") + +(er-ffi-test + "file:read_file payload is binary" + (ffi-nm + (ffi-ev + "case file:read_file(\"/tmp/er-ffi-1.txt\") of {ok, B} -> is_binary(B) end")) + "true") + +(er-ffi-test + "file:read_file content byte_size" + (ffi-ev + "case file:read_file(\"/tmp/er-ffi-1.txt\") of {ok, B} -> byte_size(B) end") + 5) + +(er-ffi-test + "file:read_file missing enoent" + (ffi-nm (ffi-ev "element(2, file:read_file(\"/tmp/er-ffi-no-such-xyz\"))")) + "enoent") + +(er-ffi-test + "file:write_file bad path enoent" + (ffi-nm + (ffi-ev "element(2, file:write_file(\"/tmp/er-ffi-no-dir-xyz/x\", \"y\"))")) + "enoent") + +(er-ffi-test + "file:write_file binary payload" + (ffi-ev + "file:write_file(\"/tmp/er-ffi-2.bin\", <<1, 2, 3, 4, 5>>), case file:read_file(\"/tmp/er-ffi-2.bin\") of {ok, B} -> byte_size(B) end") + 5) + +;; ── file:delete/1 ──────────────────────────────────────────────── +(er-ffi-test + "file:delete ok" + (ffi-nm + (ffi-ev + "file:write_file(\"/tmp/er-ffi-del.txt\", \"x\"), file:delete(\"/tmp/er-ffi-del.txt\")")) + "ok") + +(er-ffi-test + "file:read_file after delete enoent" + (ffi-nm + (ffi-ev + "file:write_file(\"/tmp/er-ffi-del2.txt\", \"x\"), file:delete(\"/tmp/er-ffi-del2.txt\"), element(2, file:read_file(\"/tmp/er-ffi-del2.txt\"))")) + "enoent") + +;; ── Blocked BIFs (placeholder asserts so the suite documents intent) ── +;; crypto:hash/2, cid:from_bytes/1, cid:to_string/1, file:list_dir/1, +;; httpc:request/4, sqlite:* — documented in plans/erlang-on-sx.md +;; under Blockers. When the host runtime gains the underlying primitive, +;; the wrappers land in runtime.sx and tests appear here. For now we +;; assert each is NOT registered, so a future iteration that adds them +;; without updating this file fails fast. + +(er-ffi-test + "crypto:hash unregistered" + (er-lookup-bif "crypto" "hash" 2) + nil) + +(er-ffi-test + "cid:from_bytes unregistered" + (er-lookup-bif "cid" "from_bytes" 1) + nil) + +(er-ffi-test + "file:list_dir unregistered" + (er-lookup-bif "file" "list_dir" 1) + nil) + +(er-ffi-test + "httpc:request unregistered" + (er-lookup-bif "httpc" "request" 4) + nil) + +(er-ffi-test + "sqlite:exec unregistered" + (er-lookup-bif "sqlite" "exec" 2) + nil) + +(define + er-ffi-test-summary + (str "ffi " er-ffi-test-pass "/" er-ffi-test-count)) diff --git a/lib/erlang/tests/runtime.sx b/lib/erlang/tests/runtime.sx index 95c20dce..b0b7fbe5 100644 --- a/lib/erlang/tests/runtime.sx +++ b/lib/erlang/tests/runtime.sx @@ -134,6 +134,144 @@ (er-sched-current-pid) nil) + + +;; ── Phase 7: module-version slots ─────────────────────────────── +(er-modules-reset!) + +(define er-rt-slot1 (er-mk-module-slot (er-env-new) nil 1)) +(er-rt-test "slot tag" (get er-rt-slot1 :tag) "module") +(er-rt-test "slot version" (er-module-version er-rt-slot1) 1) +(er-rt-test "slot old nil" (er-module-old-env er-rt-slot1) nil) +(er-rt-test "slot current not nil" (= (er-module-current-env er-rt-slot1) nil) false) + +(erlang-load-module "-module(hr1). a() -> 1.") +(define er-rt-reg (er-modules-get)) +(er-rt-test "registry has hr1" (dict-has? er-rt-reg "hr1") true) +(er-rt-test "v1 on first load" (er-module-version (get er-rt-reg "hr1")) 1) +(er-rt-test "v1 old is nil" (er-module-old-env (get er-rt-reg "hr1")) nil) +(er-rt-test "v1 current not nil" (= (er-module-current-env (get er-rt-reg "hr1")) nil) false) + +(define er-rt-env-v1 (er-module-current-env (get er-rt-reg "hr1"))) +(erlang-load-module "-module(hr1). a() -> 2.") +(er-rt-test "v2 on second load" (er-module-version (get er-rt-reg "hr1")) 2) +(er-rt-test "v2 old is v1 env" (er-module-old-env (get er-rt-reg "hr1")) er-rt-env-v1) +(er-rt-test "v2 current is new" (= (er-module-current-env (get er-rt-reg "hr1")) er-rt-env-v1) false) + +(erlang-load-module "-module(hr1). a() -> 3.") +(er-rt-test "v3 on third load" (er-module-version (get er-rt-reg "hr1")) 3) + +(er-modules-reset!) +(er-rt-test "registry-reset clears" (dict-has? (er-modules-get) "hr1") false) + + + + +;; ── Phase 8: FFI BIF registry ────────────────────────────────── +(er-bif-registry-reset!) + +(er-rt-test "empty registry" (len (er-list-bifs)) 0) +(er-rt-test "lookup miss" (er-lookup-bif "crypto" "hash" 2) nil) + +(er-register-bif! "fake" "echo" 1 (fn (vs) (nth vs 0))) +(er-rt-test "register grows registry" (len (er-list-bifs)) 1) + +(define er-rt-bif-hit (er-lookup-bif "fake" "echo" 1)) +(er-rt-test "lookup hit module" (get er-rt-bif-hit :module) "fake") +(er-rt-test "lookup hit name" (get er-rt-bif-hit :name) "echo") +(er-rt-test "lookup hit arity" (get er-rt-bif-hit :arity) 1) +(er-rt-test "lookup hit pure?" (get er-rt-bif-hit :pure?) false) + +(er-rt-test "fn invocable" ((get er-rt-bif-hit :fn) (list 42)) 42) + +;; Re-register replaces (same key) +(er-register-bif! "fake" "echo" 1 (fn (vs) "replaced")) +(er-rt-test "re-register same key, count unchanged" (len (er-list-bifs)) 1) +(er-rt-test "re-register replaces fn" + ((get (er-lookup-bif "fake" "echo" 1) :fn) (list 99)) "replaced") + +;; Pure variant +(er-register-pure-bif! "fake" "pure" 2 (fn (vs) (+ (nth vs 0) (nth vs 1)))) +(er-rt-test "pure registered separately, count 2" (len (er-list-bifs)) 2) +(er-rt-test "pure flag true" + (get (er-lookup-bif "fake" "pure" 2) :pure?) true) +(er-rt-test "pure fn invocable" + ((get (er-lookup-bif "fake" "pure" 2) :fn) (list 7 8)) 15) + +;; Arity disambiguation: same module+name, different arity = distinct entries +(er-register-bif! "fake" "echo" 2 (fn (vs) (list (nth vs 0) (nth vs 1)))) +(er-rt-test "arity disambiguation count" (len (er-list-bifs)) 3) +(er-rt-test "arity-1 lookup still works" + ((get (er-lookup-bif "fake" "echo" 1) :fn) (list 11)) "replaced") +(er-rt-test "arity-2 lookup independent" + (len ((get (er-lookup-bif "fake" "echo" 2) :fn) (list 1 2))) 2) + +;; Reset clears the registry +(er-bif-registry-reset!) +(er-rt-test "reset clears" (len (er-list-bifs)) 0) +(er-rt-test "reset lookup nil" (er-lookup-bif "fake" "echo" 1) nil) + + + +;; ── Phase 8: term marshalling (er-to-sx / er-of-sx) ───────────── + +;; er-to-sx: Erlang → SX +(er-rt-test "to-sx atom" (er-to-sx (er-mk-atom "foo")) (make-symbol "foo")) +(er-rt-test "to-sx atom is symbol" (type-of (er-to-sx (er-mk-atom "x"))) "symbol") +(er-rt-test "to-sx nil" (er-to-sx (er-mk-nil)) (list)) +(er-rt-test "to-sx integer passthrough" (er-to-sx 42) 42) +(er-rt-test "to-sx float passthrough" (er-to-sx 3.14) 3.14) +(er-rt-test "to-sx boolean passthrough" (er-to-sx true) true) +(er-rt-test "to-sx binary → string" + (er-to-sx (er-mk-binary (list 104 105 33))) "hi!") +(er-rt-test "to-sx cons → list" + (er-to-sx (er-mk-cons 1 (er-mk-cons 2 (er-mk-cons 3 (er-mk-nil))))) (list 1 2 3)) +(er-rt-test "to-sx tuple → list" + (er-to-sx (er-mk-tuple (list 1 2 3))) (list 1 2 3)) +(er-rt-test "to-sx nested cons" + (er-to-sx (er-mk-cons (er-mk-atom "a") (er-mk-cons 7 (er-mk-nil)))) + (list (make-symbol "a") 7)) + +;; er-of-sx: SX → Erlang +(er-rt-test "of-sx symbol" + (get (er-of-sx (make-symbol "ok")) :name) "ok") +(er-rt-test "of-sx symbol is atom" + (er-atom? (er-of-sx (make-symbol "x"))) true) +(er-rt-test "of-sx string is binary" + (er-binary? (er-of-sx "hi")) true) +(er-rt-test "of-sx string bytes" + (get (er-of-sx "hi") :bytes) (list 104 105)) +(er-rt-test "of-sx integer passthrough" + (er-of-sx 42) 42) +(er-rt-test "of-sx empty list → nil" + (er-nil? (er-of-sx (list))) true) +(er-rt-test "of-sx list → cons chain length" + (er-list-length (er-of-sx (list 1 2 3 4))) 4) +(er-rt-test "of-sx list head/tail" + (get (er-of-sx (list 10 20)) :head) 10) + +;; Round-trips +(er-rt-test "rtrip integer" (er-to-sx (er-of-sx 99)) 99) +(er-rt-test "rtrip atom" + (get (er-of-sx (er-to-sx (er-mk-atom "abc"))) :name) "abc") +(er-rt-test "rtrip binary bytes" + (get (er-of-sx (er-to-sx (er-mk-binary (list 1 2 3)))) :bytes) (list 1 2 3)) +(er-rt-test "rtrip cons-of-ints length" + (er-list-length (er-of-sx (er-to-sx + (er-mk-cons 1 (er-mk-cons 2 (er-mk-cons 3 (er-mk-nil))))))) 3) + +;; Tuples don't round-trip exactly (er-to-sx flattens tuples to lists); +;; documented one-way conversion. +(er-rt-test "to-sx of tuple loses tag" + (er-cons? (er-of-sx (er-to-sx (er-mk-tuple (list 1 2 3))))) true) + + +;; Re-populate built-in BIFs so subsequent test files (ring, ping-pong, etc.) +;; can call length/spawn/etc. The migration onto the registry means a reset +;; here would otherwise break the rest of the conformance suite. +(er-register-builtin-bifs!) + + (define er-rt-test-summary (str "runtime " er-rt-test-pass "/" er-rt-test-count)) diff --git a/lib/erlang/tests/vm.sx b/lib/erlang/tests/vm.sx new file mode 100644 index 00000000..026171b8 --- /dev/null +++ b/lib/erlang/tests/vm.sx @@ -0,0 +1,403 @@ +;; Phase 9 — stub VM opcode dispatcher tests. +;; Verifies the dispatcher shape (mirrors plans/sx-vm-opcode-extension.md +;; for when 9a integrates) and the three pattern-match opcodes (9b) +;; route to the correct er-match-* impl. + +(define er-vm-test-count 0) +(define er-vm-test-pass 0) +(define er-vm-test-fails (list)) + +(define + er-vm-test + (fn + (name actual expected) + (set! er-vm-test-count (+ er-vm-test-count 1)) + (if + (= actual expected) + (set! er-vm-test-pass (+ er-vm-test-pass 1)) + (append! er-vm-test-fails {:name name :expected expected :actual actual})))) + +;; ── dispatcher core ───────────────────────────────────────────── +(er-vm-test + "tuple opcode registered" + (= (er-vm-lookup-opcode-by-id 128) nil) + false) + +(er-vm-test + "tuple opcode name" + (get (er-vm-lookup-opcode-by-id 128) :name) + "OP_PATTERN_TUPLE") + +(er-vm-test + "list opcode by name" + (get (er-vm-lookup-opcode-by-name "OP_PATTERN_LIST") :id) + 129) + +(er-vm-test + "binary opcode by name" + (get (er-vm-lookup-opcode-by-name "OP_PATTERN_BINARY") :id) + 130) + +(er-vm-test "lookup miss by id" (er-vm-lookup-opcode-by-id 999) nil) + +(er-vm-test "lookup miss by name" (er-vm-lookup-opcode-by-name "OP_NOPE") nil) + +(er-vm-test + "opcode list has 3+" + (>= (len (er-vm-list-opcodes)) 3) + true) + +;; ── OP_PATTERN_TUPLE ──────────────────────────────────────────── +;; Pattern: {ok, X} matches value {ok, 42} → X bound to 42 +(define er-vm-t1-env (er-env-new)) +(define er-vm-t1-pat {:type "tuple" :elements (list {:type "atom" :value "ok"} {:name "X" :type "var"})}) +(define er-vm-t1-val (er-mk-tuple (list (er-mk-atom "ok") 42))) +(er-vm-test + "OP_PATTERN_TUPLE match" + (er-vm-dispatch 128 (list er-vm-t1-pat er-vm-t1-val er-vm-t1-env)) + true) +(er-vm-test "OP_PATTERN_TUPLE binds var" (get er-vm-t1-env "X") 42) + +;; Same pattern against {error, ...} → false +(define er-vm-t2-env (er-env-new)) +(define er-vm-t2-val (er-mk-tuple (list (er-mk-atom "error") 7))) +(er-vm-test + "OP_PATTERN_TUPLE no-match" + (er-vm-dispatch 128 (list er-vm-t1-pat er-vm-t2-val er-vm-t2-env)) + false) + +;; Wrong arity tuple — pattern has 2 elements, value has 3 +(define er-vm-t3-env (er-env-new)) +(define + er-vm-t3-val + (er-mk-tuple (list (er-mk-atom "ok") 1 2))) +(er-vm-test + "OP_PATTERN_TUPLE arity mismatch" + (er-vm-dispatch 128 (list er-vm-t1-pat er-vm-t3-val er-vm-t3-env)) + false) + +;; ── OP_PATTERN_LIST (cons) ────────────────────────────────────── +;; Pattern: [H | T] matches [1, 2, 3] → H=1, T=[2,3] +(define er-vm-l1-env (er-env-new)) +(define er-vm-l1-pat {:type "cons" :tail {:name "T" :type "var"} :head {:name "H" :type "var"}}) +(define + er-vm-l1-val + (er-mk-cons + 1 + (er-mk-cons 2 (er-mk-cons 3 (er-mk-nil))))) +(er-vm-test + "OP_PATTERN_LIST match" + (er-vm-dispatch 129 (list er-vm-l1-pat er-vm-l1-val er-vm-l1-env)) + true) +(er-vm-test "OP_PATTERN_LIST binds head" (get er-vm-l1-env "H") 1) +(er-vm-test + "OP_PATTERN_LIST tail is cons" + (er-cons? (get er-vm-l1-env "T")) + true) + +;; [H|T] against empty list → false +(define er-vm-l2-env (er-env-new)) +(er-vm-test + "OP_PATTERN_LIST no-match on nil" + (er-vm-dispatch 129 (list er-vm-l1-pat (er-mk-nil) er-vm-l2-env)) + false) + +;; ── OP_PATTERN_BINARY ─────────────────────────────────────────── +;; Pattern <> against <<42>> → A bound to 42 +(define er-vm-b1-env (er-env-new)) +(define er-vm-b1-pat {:type "binary" :segments (list {:value {:name "A" :type "var"} :size {:type "integer" :value "8"} :spec "integer"})}) +(define er-vm-b1-val (er-mk-binary (list 42))) +(er-vm-test + "OP_PATTERN_BINARY match" + (er-vm-dispatch 130 (list er-vm-b1-pat er-vm-b1-val er-vm-b1-env)) + true) +(er-vm-test + "OP_PATTERN_BINARY binds segment" + (get er-vm-b1-env "A") + 42) + +;; Same pattern against wrong-size binary (2 bytes) → false +(define er-vm-b2-env (er-env-new)) +(define er-vm-b2-val (er-mk-binary (list 42 99))) +(er-vm-test + "OP_PATTERN_BINARY size mismatch" + (er-vm-dispatch 130 (list er-vm-b1-pat er-vm-b2-val er-vm-b2-env)) + false) + +;; ── dispatch error path ──────────────────────────────────────── +(define er-vm-err-caught (list nil)) +(guard + (c (:else (set-nth! er-vm-err-caught 0 (str c)))) + (er-vm-dispatch 999 (list))) +(er-vm-test + "unknown opcode raises" + (string-contains? (str (nth er-vm-err-caught 0)) "unknown opcode") + true) + + +;; ── Phase 9c — OP_PERFORM / OP_HANDLE ─────────────────────────── +(er-vm-test "perform opcode by id" + (get (er-vm-lookup-opcode-by-id 131) :name) "OP_PERFORM") +(er-vm-test "handle opcode by id" + (get (er-vm-lookup-opcode-by-id 132) :name) "OP_HANDLE") + +(define er-vm-pf-caught (list nil)) +(guard (c (:else (set-nth! er-vm-pf-caught 0 c))) + (er-vm-dispatch 131 (list "yield" (list 42)))) +(er-vm-test "perform raises tagged" + (get (nth er-vm-pf-caught 0) :tag) "vm-effect") +(er-vm-test "perform effect name" + (get (nth er-vm-pf-caught 0) :effect) "yield") +(er-vm-test "perform args carried" + (nth (get (nth er-vm-pf-caught 0) :args) 0) 42) + +(er-vm-test "handle catches matching effect" + (er-vm-dispatch 132 + (list + (fn () (er-vm-dispatch 131 (list "yield" (list 7)))) + "yield" + (fn (args) (+ (nth args 0) 100)))) + 107) + +(er-vm-test "handle no-effect returns thunk result" + (er-vm-dispatch 132 + (list + (fn () 99) + "yield" + (fn (args) "handler ran"))) + 99) + +(define er-vm-rt-caught (list nil)) +(guard (c (:else (set-nth! er-vm-rt-caught 0 c))) + (er-vm-dispatch 132 + (list + (fn () (er-vm-dispatch 131 (list "other" (list)))) + "yield" + (fn (args) "wrong")))) +(er-vm-test "handle rethrows non-matching" + (get (nth er-vm-rt-caught 0) :effect) "other") + +(er-vm-test "nested handles separate effect names" + (er-vm-dispatch 132 + (list + (fn () + (er-vm-dispatch 132 + (list + (fn () (er-vm-dispatch 131 (list "b" (list 5)))) + "a" + (fn (args) "inner-handled")))) + "b" + (fn (args) (+ (nth args 0) 1000)))) + 1005) + + +;; ── Phase 9d — OP_RECEIVE_SCAN ────────────────────────────────── +(er-vm-test "receive-scan opcode by id" + (get (er-vm-lookup-opcode-by-id 133) :name) "OP_RECEIVE_SCAN") + +;; Pattern: receive {ok, X} -> X end against mailbox [{error, 1}, {ok, 42}, foo] +(define er-vm-r1-env (er-env-new)) +(define er-vm-r1-clauses + (list + {:pattern {:type "tuple" + :elements (list + {:type "atom" :value "ok"} + {:type "var" :name "X"})} + :guards (list) + :body (list {:type "var" :name "X"})})) +(define er-vm-r1-mbox + (list + (er-mk-tuple (list (er-mk-atom "error") 1)) + (er-mk-tuple (list (er-mk-atom "ok") 42)) + (er-mk-atom "foo"))) + +(define er-vm-r1-result + (er-vm-dispatch 133 (list er-vm-r1-clauses er-vm-r1-mbox er-vm-r1-env))) +(er-vm-test "scan finds match" + (get er-vm-r1-result :matched) true) +(er-vm-test "scan reports correct index" + (get er-vm-r1-result :index) 1) +(er-vm-test "scan binds var" + (get er-vm-r1-env "X") 42) +(er-vm-test "scan leaves body unevaluated" + (= (get er-vm-r1-result :body) nil) false) + +;; No match case +(define er-vm-r2-env (er-env-new)) +(define er-vm-r2-mbox (list (er-mk-atom "nope") 99)) +(define er-vm-r2-result + (er-vm-dispatch 133 (list er-vm-r1-clauses er-vm-r2-mbox er-vm-r2-env))) +(er-vm-test "scan no-match" + (get er-vm-r2-result :matched) false) +(er-vm-test "scan no-match leaves env clean" + (dict-has? er-vm-r2-env "X") false) + +;; Empty mailbox +(define er-vm-r3-result + (er-vm-dispatch 133 (list er-vm-r1-clauses (list) (er-env-new)))) +(er-vm-test "scan empty mailbox" + (get er-vm-r3-result :matched) false) + +;; First-match wins (arrival order) +(define er-vm-r4-env (er-env-new)) +(define er-vm-r4-mbox + (list + (er-mk-tuple (list (er-mk-atom "ok") 1)) + (er-mk-tuple (list (er-mk-atom "ok") 2)))) +(define er-vm-r4-result + (er-vm-dispatch 133 (list er-vm-r1-clauses er-vm-r4-mbox er-vm-r4-env))) +(er-vm-test "scan first-match wins (index 0)" + (get er-vm-r4-result :index) 0) +(er-vm-test "scan binds first match's var" + (get er-vm-r4-env "X") 1) + + +;; ── Phase 9e — OP_SPAWN / OP_SEND ─────────────────────────────── +(er-vm-procs-reset!) + +(er-vm-test "spawn opcode by id" + (get (er-vm-lookup-opcode-by-id 134) :name) "OP_SPAWN") +(er-vm-test "send opcode by id" + (get (er-vm-lookup-opcode-by-id 135) :name) "OP_SEND") + +(define er-vm-fn (fn () "body")) +(define er-vm-p1 (er-vm-dispatch 134 (list er-vm-fn (list)))) +(define er-vm-p2 (er-vm-dispatch 134 (list er-vm-fn (list "arg")))) +(er-vm-test "spawn returns pid 0 first" + er-vm-p1 0) +(er-vm-test "spawn returns pid 1 second" + er-vm-p2 1) +(er-vm-test "proc count is 2" + (er-vm-proc-count) 2) +(er-vm-test "spawned proc state runnable" + (er-vm-proc-state er-vm-p1) "runnable") +(er-vm-test "spawned proc mailbox empty" + (len (er-vm-proc-mailbox er-vm-p1)) 0) +(er-vm-test "spawned proc has 8 registers" + (len (get (er-vm-proc-get er-vm-p1) :registers)) 8) + +;; OP_SEND appends to target's mailbox, preserves arrival order. +(er-vm-test "send returns true on valid pid" + (er-vm-dispatch 135 (list er-vm-p1 "msg1")) true) +(er-vm-dispatch 135 (list er-vm-p1 "msg2") +) +(er-vm-dispatch 135 (list er-vm-p1 "msg3")) +(er-vm-test "mailbox length after 3 sends" + (len (er-vm-proc-mailbox er-vm-p1)) 3) +(er-vm-test "mailbox preserves order — first" + (nth (er-vm-proc-mailbox er-vm-p1) 0) "msg1") +(er-vm-test "mailbox preserves order — last" + (nth (er-vm-proc-mailbox er-vm-p1) 2) "msg3") + +;; send to nonexistent pid returns false (doesn't crash) +(er-vm-test "send to unknown pid is false" + (er-vm-dispatch 135 (list 99999 "x")) false) + +;; Isolation: msgs to p1 don't appear in p2's mailbox +(er-vm-test "isolation — p2 mailbox empty" + (len (er-vm-proc-mailbox er-vm-p2)) 0) + +;; reset clears +(er-vm-procs-reset!) +(er-vm-test "reset clears procs" + (er-vm-proc-count) 0) +(er-vm-test "reset resets pid counter" + (er-vm-dispatch 134 (list er-vm-fn (list))) 0) + + +;; ── Phase 9f — hot-BIF dispatch table ─────────────────────────── +;; Each opcode skips the registry lookup and calls the underlying +;; er-bif-* directly. Verify each returns the same result as going +;; through er-apply-bif. + +(er-vm-test "BIF_LENGTH opcode by id" + (get (er-vm-lookup-opcode-by-id 136) :name) "OP_BIF_LENGTH") +(er-vm-test "BIF_LENGTH on 3-cons" + (er-vm-dispatch 136 + (list (er-mk-cons 1 (er-mk-cons 2 (er-mk-cons 3 (er-mk-nil)))))) + 3) + +(er-vm-test "BIF_HD on cons" + (er-vm-dispatch 137 (list (er-mk-cons 99 (er-mk-nil)))) 99) + +(er-vm-test "BIF_TL is cons" + (er-cons? (er-vm-dispatch 138 + (list (er-mk-cons 1 (er-mk-cons 2 (er-mk-nil)))))) true) + +(er-vm-test "BIF_ELEMENT pulls index" + (er-vm-dispatch 139 (list 2 (er-mk-tuple (list "a" "b" "c")))) "b") + +(er-vm-test "BIF_TUPLE_SIZE on 4-tuple" + (er-vm-dispatch 140 (list (er-mk-tuple (list 1 2 3 4)))) 4) + +(er-vm-test "BIF_LISTS_REVERSE preserves elements" + (er-list-length (er-vm-dispatch 141 + (list (er-mk-cons 1 (er-mk-cons 2 (er-mk-cons 3 (er-mk-nil))))))) 3) + +(er-vm-test "BIF_LISTS_REVERSE actually reverses" + (get (er-vm-dispatch 141 + (list (er-mk-cons 1 (er-mk-cons 2 (er-mk-cons 3 (er-mk-nil)))))) :head) 3) + +(er-vm-test "BIF_IS_INTEGER true on int" + (get (er-vm-dispatch 142 (list 42)) :name) "true") +(er-vm-test "BIF_IS_INTEGER false on float" + (get (er-vm-dispatch 142 (list 3.14)) :name) "false") + +(er-vm-test "BIF_IS_ATOM true" + (get (er-vm-dispatch 143 (list (er-mk-atom "ok"))) :name) "true") +(er-vm-test "BIF_IS_ATOM false on int" + (get (er-vm-dispatch 143 (list 7)) :name) "false") + +(er-vm-test "BIF_IS_LIST true on cons" + (get (er-vm-dispatch 144 + (list (er-mk-cons 1 (er-mk-nil)))) :name) "true") +(er-vm-test "BIF_IS_LIST true on nil" + (get (er-vm-dispatch 144 (list (er-mk-nil))) :name) "true") +(er-vm-test "BIF_IS_LIST false on tuple" + (get (er-vm-dispatch 144 (list (er-mk-tuple (list)))) :name) "false") + +(er-vm-test "BIF_IS_TUPLE true" + (get (er-vm-dispatch 145 (list (er-mk-tuple (list 1)))) :name) "true") +(er-vm-test "BIF_IS_TUPLE false on int" + (get (er-vm-dispatch 145 (list 5)) :name) "false") + +;; Sanity: total opcode count grew (3 patterns + perform + handle + +;; receive-scan + spawn + send + 10 hot-BIFs = 16+ registered). +(er-vm-test "opcode list has 16+" + (>= (len (er-vm-list-opcodes)) 16) true) + + +;; ── Phase 9i — host opcode-id resolution ──────────────────────── +;; Requires a binary with the erlang_ext extension registered (9h). +;; The loop runs conformance against exactly that binary. +(er-vm-test "host id: OP_PATTERN_TUPLE = 222" + (er-vm-host-opcode-id "erlang.OP_PATTERN_TUPLE") 222) +(er-vm-test "host id: OP_BIF_IS_TUPLE = 239" + (er-vm-host-opcode-id "erlang.OP_BIF_IS_TUPLE") 239) +(er-vm-test "host id: unknown name -> nil" + (er-vm-host-opcode-id "erlang.OP_NOPE") nil) +(er-vm-test "effective id prefers host when present" + (er-vm-effective-opcode-id "erlang.OP_BIF_LENGTH" 136) 230) +(er-vm-test "effective id falls back to stub on nil" + (er-vm-effective-opcode-id "erlang.OP_NOPE" 999) 999) +;; The full erlang.OP_* namespace resolves to the contiguous 222-239 block. +(er-vm-test "host ids contiguous 222..239" + (let ((names (list "erlang.OP_PATTERN_TUPLE" "erlang.OP_PATTERN_LIST" + "erlang.OP_PATTERN_BINARY" "erlang.OP_PERFORM" + "erlang.OP_HANDLE" "erlang.OP_RECEIVE_SCAN" + "erlang.OP_SPAWN" "erlang.OP_SEND" + "erlang.OP_BIF_LENGTH" "erlang.OP_BIF_HD" + "erlang.OP_BIF_TL" "erlang.OP_BIF_ELEMENT" + "erlang.OP_BIF_TUPLE_SIZE" "erlang.OP_BIF_LISTS_REVERSE" + "erlang.OP_BIF_IS_INTEGER" "erlang.OP_BIF_IS_ATOM" + "erlang.OP_BIF_IS_LIST" "erlang.OP_BIF_IS_TUPLE")) + (ok (list true))) + (for-each + (fn (i) + (when (not (= (er-vm-host-opcode-id (nth names i)) (+ 222 i))) + (set-nth! ok 0 false))) + (range 0 (len names))) + (nth ok 0)) + true) + +(define er-vm-test-summary (str "vm " er-vm-test-pass "/" er-vm-test-count)) diff --git a/lib/erlang/transpile.sx b/lib/erlang/transpile.sx index ac2bf562..915d31b6 100644 --- a/lib/erlang/transpile.sx +++ b/lib/erlang/transpile.sx @@ -669,96 +669,23 @@ (define er-apply-bif - (fn - (name vs) - (cond - (= name "is_integer") (er-bif-is-integer vs) - (= name "is_atom") (er-bif-is-atom vs) - (= name "is_list") (er-bif-is-list vs) - (= name "is_tuple") (er-bif-is-tuple vs) - (= name "is_number") (er-bif-is-number vs) - (= name "is_float") (er-bif-is-float vs) - (= name "is_boolean") (er-bif-is-boolean vs) - (= name "length") (er-bif-length vs) - (= name "hd") (er-bif-hd vs) - (= name "tl") (er-bif-tl vs) - (= name "element") (er-bif-element vs) - (= name "tuple_size") (er-bif-tuple-size vs) - (= name "atom_to_list") (er-bif-atom-to-list vs) - (= name "list_to_atom") (er-bif-list-to-atom vs) - (= name "is_pid") (er-bif-is-pid vs) - (= name "is_reference") (er-bif-is-reference vs) - (= name "is_binary") (er-bif-is-binary vs) - (= name "byte_size") (er-bif-byte-size vs) - (= name "abs") (er-bif-abs vs) - (= name "min") (er-bif-min vs) - (= name "max") (er-bif-max vs) - (= name "tuple_to_list") (er-bif-tuple-to-list vs) - (= name "list_to_tuple") (er-bif-list-to-tuple vs) - (= name "integer_to_list") (er-bif-integer-to-list vs) - (= name "list_to_integer") (er-bif-list-to-integer vs) - (= name "is_function") (er-bif-is-function vs) - (= name "self") (er-bif-self vs) - (= name "spawn") (er-bif-spawn vs) - (= name "exit") (er-bif-exit vs) - (= name "make_ref") (er-bif-make-ref vs) - (= name "link") (er-bif-link vs) - (= name "unlink") (er-bif-unlink vs) - (= name "monitor") (er-bif-monitor vs) - (= name "demonitor") (er-bif-demonitor vs) - (= name "process_flag") (er-bif-process-flag vs) - (= name "register") (er-bif-register vs) - (= name "unregister") (er-bif-unregister vs) - (= name "whereis") (er-bif-whereis vs) - (= name "registered") (er-bif-registered vs) - (= name "throw") (raise (er-mk-throw-marker (er-bif-arg1 vs "throw"))) - (= name "error") (raise (er-mk-error-marker (er-bif-arg1 vs "error"))) - :else (error - (str "Erlang: undefined function '" name "/" (len vs) "'"))))) + (fn (name vs) + (let ((entry (er-lookup-bif "erlang" name (len vs)))) + (if (not (= entry nil)) + ((get entry :fn) vs) + (error (str "Erlang: undefined function '" name "/" (len vs) "'")))))) (define er-apply-remote-bif - (fn - (mod name vs) + (fn (mod name vs) (cond (dict-has? (er-modules-get) mod) - (er-apply-user-module mod name vs) - (= mod "lists") (er-apply-lists-bif name vs) - (= mod "io") (er-apply-io-bif name vs) - (= mod "erlang") (er-apply-bif name vs) - (= mod "ets") (er-apply-ets-bif name vs) - :else (error - (str "Erlang: undefined module '" mod "'"))))) - -(define - er-apply-lists-bif - (fn - (name vs) - (cond - (= name "reverse") (er-bif-lists-reverse vs) - (= name "map") (er-bif-lists-map vs) - (= name "foldl") (er-bif-lists-foldl vs) - (= name "seq") (er-bif-lists-seq vs) - (= name "sum") (er-bif-lists-sum vs) - (= name "nth") (er-bif-lists-nth vs) - (= name "last") (er-bif-lists-last vs) - (= name "member") (er-bif-lists-member vs) - (= name "append") (er-bif-lists-append vs) - (= name "filter") (er-bif-lists-filter vs) - (= name "any") (er-bif-lists-any vs) - (= name "all") (er-bif-lists-all vs) - (= name "duplicate") (er-bif-lists-duplicate vs) - :else (error - (str "Erlang: undefined 'lists:" name "/" (len vs) "'"))))) - -(define - er-apply-io-bif - (fn - (name vs) - (cond - (= name "format") (er-bif-io-format vs) - :else (error - (str "Erlang: undefined 'io:" name "/" (len vs) "'"))))) + (er-apply-user-module mod name vs) + :else + (let ((entry (er-lookup-bif mod name (len vs)))) + (if (not (= entry nil)) + ((get entry :fn) vs) + (error (str "Erlang: undefined remote function '" mod ":" name "/" (len vs) "'"))))))) (define er-bif-arg1 @@ -1911,3 +1838,180 @@ (fn (_) (set! out (er-mk-cons v out))) (range 0 n)) out)))) + + +;; ── code module (Phase 7 hot-reload) ───────────────────────────── +(define er-source-walk-bytes! + (fn (n bytes-box) + (cond + (er-nil? n) true + (er-cons? n) + (let ((h (get n :head))) + (cond + (= (type-of h) "number") + (do (append! (nth bytes-box 0) h) + (er-source-walk-bytes! (get n :tail) bytes-box)) + :else (do (set-nth! bytes-box 0 nil) false))) + :else (do (set-nth! bytes-box 0 nil) false)))) + +(define er-source-to-string + (fn (v) + (cond + (= (type-of v) "string") v + (er-binary? v) (list->string (map integer->char (get v :bytes))) + (or (er-nil? v) (er-cons? v)) + (let ((box (list (list)))) + (er-source-walk-bytes! v box) + (cond + (= (nth box 0) nil) nil + :else (list->string (map integer->char (nth box 0))))) + :else nil))) + +(define er-bif-code-load-binary + (fn (vs) + (let ((mod-arg (nth vs 0)) (src-arg (nth vs 2))) + (cond + (not (er-atom? mod-arg)) + (er-mk-tuple (list (er-mk-atom "error") (er-mk-atom "badarg"))) + :else + (let ((src-str (er-source-to-string src-arg))) + (cond + (= src-str nil) + (er-mk-tuple (list (er-mk-atom "error") (er-mk-atom "badarg"))) + :else + (let ((result-box (list nil)) (failed-box (list false))) + (guard + (c (:else (set-nth! failed-box 0 true))) + (set-nth! result-box 0 (erlang-load-module src-str))) + (cond + (nth failed-box 0) + (er-mk-tuple + (list (er-mk-atom "error") (er-mk-atom "badfile"))) + (not (= (get (nth result-box 0) :name) (get mod-arg :name))) + (er-mk-tuple + (list (er-mk-atom "error") (er-mk-atom "module_name_mismatch"))) + :else + (er-mk-tuple (list (er-mk-atom "module") mod-arg)))))))))) + +(define er-env-derived-from? + (fn (env target-env) + ;; Object-identity check, NOT value `=`. On evaluators where dict `=` + ;; is structural/deep, comparing closure envs (which are large and + ;; cyclic — a module fun's env references the fun) does not terminate. + ;; `identical?` is pointer identity on every host and is the actual + ;; intended semantics: "is this the same env object". + (cond + (identical? env target-env) true + :else + (let ((ks (keys env)) (found-ref (list false))) + (for-each + (fn (i) + (when (not (nth found-ref 0)) + (let ((v (get env (nth ks i)))) + (when (and (er-fun? v) (identical? (get v :env) target-env)) + (set-nth! found-ref 0 true))))) + (range 0 (len ks))) + (nth found-ref 0))))) + +(define er-procs-on-env + (fn (target-env) + (let ((all-keys (keys (er-sched-processes))) + (matches (list))) + (for-each + (fn (i) + (let ((proc (get (er-sched-processes) (nth all-keys i)))) + (let ((init-fun (get proc :initial-fun))) + (when (and (not (= init-fun nil)) + (er-fun? init-fun) + (er-env-derived-from? (get init-fun :env) target-env) + (not (= (get proc :state) "dead"))) + (append! matches (get proc :pid)))))) + (range 0 (len all-keys))) + matches))) + +(define er-bif-code-purge + (fn (vs) + (let ((mod-arg (nth vs 0))) + (cond + (not (er-atom? mod-arg)) + (raise (er-mk-error-marker (er-mk-atom "badarg"))) + :else + (let ((registry (er-modules-get)) (mod-name (get mod-arg :name))) + (cond + (not (dict-has? registry mod-name)) (er-mk-atom "false") + :else + (let ((slot (get registry mod-name))) + (cond + (= (er-module-old-env slot) nil) (er-mk-atom "false") + :else + (let ((procs (er-procs-on-env (er-module-old-env slot)))) + (for-each + (fn (i) (er-cascade-exit! (nth procs i) (er-mk-atom "killed"))) + (range 0 (len procs))) + (dict-set! registry mod-name + (er-mk-module-slot (er-module-current-env slot) nil + (er-module-version slot))) + (er-mk-atom "true")))))))))) + +(define er-bif-code-soft-purge + (fn (vs) + (let ((mod-arg (nth vs 0))) + (cond + (not (er-atom? mod-arg)) + (raise (er-mk-error-marker (er-mk-atom "badarg"))) + :else + (let ((registry (er-modules-get)) (mod-name (get mod-arg :name))) + (cond + (not (dict-has? registry mod-name)) (er-mk-atom "true") + :else + (let ((slot (get registry mod-name))) + (cond + (= (er-module-old-env slot) nil) (er-mk-atom "true") + :else + (let ((procs (er-procs-on-env (er-module-old-env slot)))) + (cond + (> (len procs) 0) (er-mk-atom "false") + :else + (do + (dict-set! registry mod-name + (er-mk-module-slot (er-module-current-env slot) nil + (er-module-version slot))) + (er-mk-atom "true")))))))))))) + +(define er-bif-code-which + (fn (vs) + (let ((mod-arg (nth vs 0))) + (cond + (not (er-atom? mod-arg)) + (raise (er-mk-error-marker (er-mk-atom "badarg"))) + (dict-has? (er-modules-get) (get mod-arg :name)) + (er-mk-atom "loaded") + :else (er-mk-atom "non_existing"))))) + +(define er-bif-code-is-loaded + (fn (vs) + (let ((mod-arg (nth vs 0))) + (cond + (not (er-atom? mod-arg)) + (raise (er-mk-error-marker (er-mk-atom "badarg"))) + (dict-has? (er-modules-get) (get mod-arg :name)) + (er-mk-tuple (list (er-mk-atom "file") (er-mk-atom "loaded"))) + :else (er-mk-atom "false"))))) + +(define er-bif-code-all-loaded + (fn (vs) + (let ((registry (er-modules-get)) + (ks (keys (er-modules-get))) + (out (er-mk-nil))) + (for-each + (fn (i) + (let ((k (nth ks (- (- (len ks) 1) i)))) + (set! out + (er-mk-cons + (er-mk-tuple + (list (er-mk-atom k) (er-mk-atom "loaded"))) + out)))) + (range 0 (len ks))) + out))) + + diff --git a/lib/erlang/vm/dispatcher.sx b/lib/erlang/vm/dispatcher.sx new file mode 100644 index 00000000..dc3946a7 --- /dev/null +++ b/lib/erlang/vm/dispatcher.sx @@ -0,0 +1,313 @@ +;; Erlang VM — stub opcode dispatcher (Phase 9). +;; +;; Mimics the OCaml-side EXTENSION shape from +;; plans/sx-vm-opcode-extension.md so opcodes 9b-9g can be designed +;; and tested in SX before 9a (`hosts/ocaml/`) lands the real +;; registration plumbing. When 9a is available, these stubs become +;; the cross-host SX-side mirror of the C/OCaml handlers and the +;; bytecode compiler emits them directly. +;; +;; Opcode IDs follow the plan's tier partition: +;; 0-127 reserved for SX core +;; 128-199 guest extensions (e.g. erlang, lua) +;; 200-247 port-/platform-specific +;; +;; Erlang owns 128-159 for now. + +(define er-vm-opcodes (list {})) + +(define er-vm-opcodes-get (fn () (nth er-vm-opcodes 0))) + +(define + er-vm-opcodes-reset! + (fn () (set-nth! er-vm-opcodes 0 {}))) + +(define + er-vm-register-opcode! + (fn + (id name handler) + (dict-set! (er-vm-opcodes-get) (str id) {:name name :id id :handler handler}) + (er-mk-atom "ok"))) + +(define + er-vm-lookup-opcode-by-id + (fn + (id) + (let + ((reg (er-vm-opcodes-get)) (k (str id))) + (if (dict-has? reg k) (get reg k) nil)))) + +(define + er-vm-lookup-opcode-by-name + (fn + (name) + (let + ((reg (er-vm-opcodes-get)) + (ks (keys (er-vm-opcodes-get))) + (found (list nil))) + (for-each + (fn + (i) + (let + ((entry (get reg (nth ks i)))) + (when + (= (get entry :name) name) + (set-nth! found 0 entry)))) + (range 0 (len ks))) + (nth found 0)))) + +(define er-vm-list-opcodes (fn () (keys (er-vm-opcodes-get)))) + +;; ── Phase 9i — host opcode-id resolution ──────────────────────── +;; When the OCaml `erlang_ext` extension is registered (Phase 9h), the +;; runtime exposes `extension-opcode-id` which maps an "erlang.OP_*" +;; name to the host-assigned id (222-239). We consult it so the SX +;; side and the OCaml side agree on ids; when it returns nil (name not +;; registered) we fall back to the stub-local id. +;; +;; NOTE: this requires a binary with the VM extension mechanism (the +;; vm-ext phase-A..E cherry-pick + Sx_vm_extensions force-link). The +;; loop builds and runs against exactly that binary +;; (hosts/ocaml/_build/default/bin/sx_server.exe). `extension-opcode-id` +;; resolves lazily at call time, so merely loading this file is safe; +;; only invoking the resolver on a binary that lacks the primitive +;; would raise. + +(define er-vm-host-opcode-id + (fn (ext-name) + (extension-opcode-id ext-name))) + +(define er-vm-effective-opcode-id + (fn (ext-name stub-id) + (let ((host (extension-opcode-id ext-name))) + (cond + (= host nil) stub-id + :else host)))) + +(define + er-vm-dispatch + (fn + (id operands) + (let + ((entry (er-vm-lookup-opcode-by-id id))) + (if + (= entry nil) + (error (str "Erlang VM: unknown opcode id " id)) + ((get entry :handler) operands))))) + +(define + er-vm-dispatch-by-name + (fn + (name operands) + (let + ((entry (er-vm-lookup-opcode-by-name name))) + (if + (= entry nil) + (error (str "Erlang VM: unknown opcode name '" name "'")) + ((get entry :handler) operands))))) + +;; ── Phase 9c — effect opcodes (perform / handle) ──────────────── +;; Stub algebraic-effects-style operators. OP_PERFORM raises a tagged +;; exception; OP_HANDLE wraps a thunk in `guard` and catches matching +;; effects, passing the args to the handler. The real specialization +;; (constant-time effect dispatch, single-shot vs multi-shot continuations) +;; lands when 9a integrates. + +(define er-vm-effect-marker? + (fn (c effect-name) + (and (= (type-of c) "dict") + (= (get c :tag) "vm-effect") + (= (get c :effect) effect-name)))) + +(define er-vm-op-perform + (fn (operands) + (raise {:tag "vm-effect" :effect (nth operands 0) :args (nth operands 1)}))) + +(define er-vm-op-handle + (fn (operands) + (let ((thunk (nth operands 0)) + (effect-name (nth operands 1)) + (handler (nth operands 2)) + (result (list nil)) + (caught (list false)) + (rethrow (list nil))) + (guard + (c + (:else + (cond + (er-vm-effect-marker? c effect-name) + (do (set-nth! caught 0 true) + (set-nth! result 0 (handler (get c :args)))) + :else (set-nth! rethrow 0 c)))) + (set-nth! result 0 (thunk))) + (cond + (not (= (nth rethrow 0) nil)) (raise (nth rethrow 0)) + :else (nth result 0))))) + +;; ── Phase 9d — receive scan opcode ──────────────────────────── +;; Selective receive primitive. Scans a mailbox value-list in arrival +;; order; for each value, tries each clause's pattern (binding into +;; env on success); on match returns `{:matched true :index N :body B}` +;; — the caller decides what to do with the index (queue-delete) and +;; the body (eval in the now-mutated env). On miss returns +;; `{:matched false}`, the caller arranges suspension (via OP_PERFORM). +;; +;; Operands: (clauses mbox-list env) +;; clauses — list of {:pattern :guards :body} dicts +;; mbox-list — SX list of message values +;; env — env dict (mutated on match) + +(define er-vm-receive-try-clauses + (fn (clauses msg env i) + (cond + (>= i (len clauses)) {:matched false} + :else + (let ((c (nth clauses i)) (snap (er-env-copy env))) + (cond + (and + (er-match! (get c :pattern) msg env) + (er-eval-guards (get c :guards) env)) + {:matched true :body (get c :body)} + :else + (do (er-env-restore! env snap) + (er-vm-receive-try-clauses clauses msg env (+ i 1)))))))) + +(define er-vm-receive-scan-loop + (fn (clauses mbox env i) + (cond + (>= i (len mbox)) {:matched false} + :else + (let ((msg (nth mbox i)) + (cr (er-vm-receive-try-clauses clauses msg env 0))) + (cond + (get cr :matched) {:matched true :index i :body (get cr :body)} + :else (er-vm-receive-scan-loop clauses mbox env (+ i 1))))))) + +(define er-vm-op-receive-scan + (fn (operands) + (er-vm-receive-scan-loop (nth operands 0) (nth operands 1) (nth operands 2) 0))) + +;; ── Phase 9e — spawn / send + lightweight scheduler ───────────── +;; Stub register-machine process layout for the eventual fast scheduler. +;; A VM-process is `{:id :registers :mailbox :state :initial-fn :initial-args}`. +;; Registers is a vector (SX list, mutated via set-nth!) — fixed slot count +;; per process so cells don't grow during execution. Mailbox is an SX list. +;; State is one of "runnable" / "waiting" / "dead". This sits PARALLEL to +;; the existing `er-scheduler` (which is the language-level scheduler) — +;; the VM scheduler will eventually take over once 9a integrates and +;; bytecode-compiled Erlang runs against it. + +(define er-vm-procs (list {})) +(define er-vm-procs-get (fn () (nth er-vm-procs 0))) +(define er-vm-procs-reset! + (fn () (do (set-nth! er-vm-procs 0 {}) (set-nth! er-vm-next-pid 0 0)))) + +(define er-vm-next-pid (list 0)) + +(define er-vm-proc-new! + (fn (initial-fn initial-args) + (let ((pid (nth er-vm-next-pid 0))) + (set-nth! er-vm-next-pid 0 (+ pid 1)) + (let ((proc + {:id pid + :registers (list nil nil nil nil nil nil nil nil) + :mailbox (list) + :state "runnable" + :initial-fn initial-fn + :initial-args initial-args})) + (dict-set! (er-vm-procs-get) (str pid) proc) + pid)))) + +(define er-vm-proc-get (fn (pid) (get (er-vm-procs-get) (str pid)))) + +(define er-vm-proc-send! + (fn (pid msg) + (let ((proc (er-vm-proc-get pid))) + (cond + (= proc nil) false + :else + (do + (dict-set! proc :mailbox (append (get proc :mailbox) (list msg))) + (when (= (get proc :state) "waiting") + (dict-set! proc :state "runnable")) + true))))) + +(define er-vm-proc-mailbox (fn (pid) (get (er-vm-proc-get pid) :mailbox))) +(define er-vm-proc-state (fn (pid) (get (er-vm-proc-get pid) :state))) +(define er-vm-proc-count (fn () (len (keys (er-vm-procs-get))))) + +(define er-vm-op-spawn + (fn (operands) + (er-vm-proc-new! (nth operands 0) (nth operands 1)))) + +(define er-vm-op-send + (fn (operands) + (er-vm-proc-send! (nth operands 0) (nth operands 1)))) + +;; ── Phase 9f — hot-BIF dispatch table ────────────────────────── +;; Specialized opcodes for the BIFs that the bytecode compiler emits +;; on hot call sites. The handler is the underlying `er-bif-*` impl +;; directly — same `(vs)` signature as the dispatcher uses for +;; operands, so the cost is the opcode-id → handler hop with no +;; registry-key string lookup. Cold BIFs continue going through the +;; general path (`er-apply-bif` / `er-lookup-bif`). +;; +;; Opcodes 136-159 reserved for hot BIFs. + +;; ── Phase 9b — pattern-match opcodes ──────────────────────────── +;; Each handler takes a list (pattern-ast value env) and returns +;; true/false, mutating env on success (same contract as the +;; existing er-match-tuple / er-match-cons / er-match-binary). +;; Wire these as wrappers for now; the real opcodes will eventually +;; have register-machine semantics and skip the AST-walk overhead. + +(define + er-vm-register-erlang-opcodes! + (fn + () + (er-vm-register-opcode! + 128 + "OP_PATTERN_TUPLE" + (fn + (operands) + (er-match-tuple + (nth operands 0) + (nth operands 1) + (nth operands 2)))) + (er-vm-register-opcode! + 129 + "OP_PATTERN_LIST" + (fn + (operands) + (er-match-cons + (nth operands 0) + (nth operands 1) + (nth operands 2)))) + (er-vm-register-opcode! + 130 + "OP_PATTERN_BINARY" + (fn + (operands) + (er-match-binary + (nth operands 0) + (nth operands 1) + (nth operands 2)))) + (er-vm-register-opcode! 131 "OP_PERFORM" er-vm-op-perform) + (er-vm-register-opcode! 132 "OP_HANDLE" er-vm-op-handle) + (er-vm-register-opcode! 133 "OP_RECEIVE_SCAN" er-vm-op-receive-scan) + (er-vm-register-opcode! 134 "OP_SPAWN" er-vm-op-spawn) + (er-vm-register-opcode! 135 "OP_SEND" er-vm-op-send) + ;; Phase 9f — hot BIFs + (er-vm-register-opcode! 136 "OP_BIF_LENGTH" er-bif-length) + (er-vm-register-opcode! 137 "OP_BIF_HD" er-bif-hd) + (er-vm-register-opcode! 138 "OP_BIF_TL" er-bif-tl) + (er-vm-register-opcode! 139 "OP_BIF_ELEMENT" er-bif-element) + (er-vm-register-opcode! 140 "OP_BIF_TUPLE_SIZE" er-bif-tuple-size) + (er-vm-register-opcode! 141 "OP_BIF_LISTS_REVERSE" er-bif-lists-reverse) + (er-vm-register-opcode! 142 "OP_BIF_IS_INTEGER" er-bif-is-integer) + (er-vm-register-opcode! 143 "OP_BIF_IS_ATOM" er-bif-is-atom) + (er-vm-register-opcode! 144 "OP_BIF_IS_LIST" er-bif-is-list) + (er-vm-register-opcode! 145 "OP_BIF_IS_TUPLE" er-bif-is-tuple) + (er-mk-atom "ok"))) + +(er-vm-register-erlang-opcodes!) diff --git a/plans/erlang-on-sx.md b/plans/erlang-on-sx.md index cc068a23..8d33d228 100644 --- a/plans/erlang-on-sx.md +++ b/plans/erlang-on-sx.md @@ -10,7 +10,9 @@ End-state goal: spawn a million processes, run the classic **ring benchmark**, p - **Conformance:** not BEAM-compat. "Looks like Erlang, runs like Erlang, not byte-compatible." We care about semantics, not BEAM bug-for-bug. - **Test corpus:** custom — ring, ping-pong, fibonacci-server, bank-account-server, echo-server, plus ~100 hand-written tests for patterns/guards/BIFs. No ISO Common Test. - **Binaries:** basic bytes-lists only; full binary pattern matching deferred. -- **Hot code reload, distribution, NIFs:** out of scope entirely. +- **Distribution, NIFs:** out of scope entirely. +- **Hot code reload (Phase 7):** in scope — driven by [fed-sx](../plans/fed-sx-design.md) (section 17.5) which needs federated modules to be re-loaded without restarting the scheduler. +- **FFI BIFs (Phase 8):** in scope — Erlang code needs `crypto:hash`, `cid:from_bytes`, `file:read_file`, `httpc:request`, `sqlite:exec` to participate in fed-sx. A general FFI BIF registry replaces today's hard-coded BIF dispatch. ## Ground rules @@ -95,10 +97,126 @@ Core mapping: - [x] ETS-lite (in-memory tables via SX dicts) — **13 new eval tests**; `ets:new/2`, `insert/2`, `lookup/2`, `delete/1-2`, `tab2list/1`, `info/2` (size); set semantics with full Erlang-term keys - [x] More BIFs — target 200+ test corpus green — **40 new eval tests**; 530/530 total. New: `abs/1`, `min/2`, `max/2`, `tuple_to_list/1`, `list_to_tuple/1`, `integer_to_list/1`, `list_to_integer/1`, `is_function/1-2`, `lists:seq/2-3`, `lists:sum/1`, `lists:nth/2`, `lists:last/1`, `lists:member/2`, `lists:append/2`, `lists:filter/2`, `lists:any/2`, `lists:all/2`, `lists:duplicate/2` +### Phase 7 — hot code reload + +Driven by **fed-sx** (see `plans/fed-sx-design.md` §17.5): federated modules must be replaceable at runtime without bouncing the scheduler. Classic OTP behaviour: two versions per module ("current" and "old"), local calls stick to the version the process started with, cross-module (`M:F(...)`) calls always resolve to the current version, and `purge` kills any process still running old code. + +- [x] Module version slot: `er-modules` entry becomes `{:current MOD-ENV :old MOD-ENV-or-nil :version INT}`; bump version on each load — **13 new runtime tests** (543/543 total) +- [x] `code:load_binary/3` (the canonical reload BIF) — re-parses module source, swaps `:current` → `:old`, installs new env as `:current`; returns `{module, Name}` or `{error, Reason}` (badarg / badfile / module_name_mismatch). **+8 eval tests** (551/551 total). `code:load_file/1` is a thin filesystem wrapper around this and lands once `file:read_file/1` is in (Phase 8). +- [x] `code:purge/1` + `code:soft_purge/1` — purge clears `:old` slot and kills any process whose `:initial-fun` env identity matches the old env (returns `true` if there was old code, `false` if there wasn't). soft_purge: refuses (returns `false`, leaves `:old` intact) if any process is still pinned to the old env; otherwise clears and returns `true`. **+10 eval tests** (561/561 total). Caveat: a true "lingering on old code" test needs `spawn/3` (still stubbed) or `fun M:F/A` syntax (not parsed) — anonymous `fun () -> M:F() end` closures capture the caller's env, not the module's, and cross-module calls always resolve to `:current`. Current tests therefore exercise the return-value matrix but not the kill path. +- [x] `code:which/1`, `code:is_loaded/1`, `code:all_loaded/0` — introspection. **+10 eval tests** (571/571 total). Return-value contract: `which` → `loaded` / `non_existing` (since we have no filesystem path); `is_loaded` → `{file, loaded}` / `false`; `all_loaded` → list of `{Module, loaded}` tuples. Non-atom Mod raises `error:badarg`. +- [x] Cross-module call `M:F(...)` dispatches to `:current`; local calls inside a module body keep using the env they closed over so a running process finishes its current function with the version it started with — **+6 eval tests** verifying the property end-to-end (577/577 total). No implementation change: `er-apply-user-module` already routes through `er-module-current-env`, and `er-mk-fun` captures its env by reference so closures created under v1 retain v1's `mod-env` even after the slot bumps to v2. +- [x] Tests: load v1 → spawn → load v2 → cross-module call hits v2 → local call inside v1 process keeps v1 semantics until function returns → purge kills v1 procs → soft_purge refuses while v1 procs alive — **+5 capstone eval tests** (582/582 total). Required extending `er-procs-on-env` from raw identity match to `er-env-derived-from?` (an env "comes from" mod-env if it IS mod-env or contains a value that's a fun closed over mod-env), because `er-apply-fun-clauses` does `er-env-copy closure-env` before binding params — so the spawned-from-inside-module fun's `:env` is a fresh dict, not mod-env. Test ladder runs as one single `erlang-eval-ast` program (every call to `ev` resets the scheduler via `er-sched-init!`, so Pid handles must live within one program). + +### Phase 8 — FFI BIF mechanism + standard libs + +Replace today's hardcoded BIF dispatch (`er-apply-bif`/`er-apply-remote-bif` in `transpile.sx`) with a runtime-extensible **BIF registry**. Each registry entry is `{:module :name :arity :fn :pure?}`. Standard libs are then registered at boot, and fed-sx can register new BIFs from `.sx` files. Includes the marshalling layer (Erlang term ↔ SX value) so wrappers stay one-liners. + +- [x] BIF registry: `er-bif-registry` global dict keyed by `"Module/Name/Arity"`, with `er-register-bif!`/`er-register-pure-bif!`/`er-lookup-bif`/`er-list-bifs`/`er-bif-registry-reset!` helpers — **+18 runtime tests** (600/600 total). Entries are `{:module :name :arity :fn :pure?}`. Arity is part of the key so `m:f/1` and `m:f/2` are independent. Re-registering the same key replaces the previous entry; reset clears. +- [x] Migrate existing local + remote BIFs (length/hd/tl/lists:*/io:format/ets:*/etc.) onto the registry; delete the giant `cond` dispatch in `er-apply-bif`/`er-apply-remote-bif`. Conformance held at **600/600** after migration (baseline was 600, not the plan-text's 530 — the text was authored before Phase 7 work added rows). 67 builtin registrations across `erlang`/`lists`/`io`/`ets`/`code` modules; multi-arity BIFs (`is_function`, `spawn`, `exit`, `io:format`, `lists:seq`, `ets:delete`) register once per arity, all pointing at the same impl which dispatches on `(len vs)` internally. The four per-module cond dispatchers (`er-apply-lists-bif`, `er-apply-io-bif`, `er-apply-ets-bif`, `er-apply-code-bif`) are deleted. `er-apply-bif` and `er-apply-remote-bif` are now ~5-line registry lookups; user modules still win precedence over the registry. +- [x] Term-marshalling helpers: `er-of-sx` (SX → Erlang) and `er-to-sx` (Erlang → SX). atom ↔ symbol, nil ↔ `()`, cons → list, tuple → list (one-way; tuples flatten), binary ↔ SX string, integer / float / boolean passthrough. **+23 runtime tests** (623/623 total). Erlang maps (`dict ↔ map`) deferred — Erlang map term not implemented in this port; will land when `#{}` syntax does. Pids, refs, funs pass through unchanged. SX strings on the way back become Erlang binaries (most useful FFI return shape). +- [ ] `crypto:hash/2` — **BLOCKED** (no `sha256`/`sha512`/`blake3` primitive in this SX runtime). See Blockers. +- [ ] `cid:from_bytes/1`, `cid:to_string/1` — **BLOCKED** (needs `crypto:hash/2`). See Blockers. +- [x] `file:read_file/1`, `file:write_file/2`, `file:delete/1` — **+10 eval tests** (633/633 total). Returns `{ok, Binary}` / `ok` / `{error, Reason}` where Reason is `enoent`/`eacces`/`enotdir`/`eisdir`/`posix_error` (classified from the SX `file-read`/`-write`/`-delete` exception string). Path accepts SX string, Erlang binary, or Erlang char-code list. `file:list_dir/1` deferred — no directory-listing primitive in this SX runtime; see Blockers. +- [ ] `httpc:request/4` — **BLOCKED** (no HTTP client primitive). See Blockers. +- [ ] `sqlite:open/1`, `sqlite:close/1`, `sqlite:exec/2`, `sqlite:query/2` — **BLOCKED** (no SQLite primitive). See Blockers. +- [x] Tests: 1 round-trip per BIF; suite name `ffi`; conformance scoreboard auto-picks it up — **+14 ffi tests** at 637/637 total. Suite covers the 3 implemented file BIFs (9 tests: write-ok, read-ok-tag, payload-is-binary, byte_size content, missing-enoent, bad-path-enoent, binary-payload round-trip, delete-ok, read-after-delete-enoent) plus 5 negative asserts (one per blocked BIF — `crypto:hash`/`cid:from_bytes`/`file:list_dir`/`httpc:request`/`sqlite:exec`) so this suite fails fast if a future iteration adds a wrapper without registering proper tests. Target "+40 ffi tests" was relative to the original 5-BIF-family plan; with 5 of those families blocked on host primitives, the achievable count is 14 — the suite scaffolding is what matters and is ready to accept the remaining tests when the primitives land. + +### Phase 9 — specialized opcodes (the BEAM analog) + +**Driver:** Erlang-on-SX going through the general-purpose CEK machine has architectural perf ceilings (call/cc per receive, env-copy per call, mailbox rebuild on delete). The fix is specialized bytecode opcodes that bypass the general machinery for hot Erlang operations. Targets: 100k+ message hops/sec, 1M-process spawn in under 30sec. Layered perf strategy: Layer 1 (this) = specialized opcodes; Layer 2 (Phase 10, deferred) = multi-core scheduler. + +**Architectural note:** opcodes get developed in `lib/erlang/vm/` (in scope). The **opcode extension mechanism in `hosts/ocaml/`** (Phase 9a) is **out of scope** for this loop — log as Blocker until a session that owns `hosts/` lands it. Sub-phases 9b-9g design and test opcodes against a stub dispatcher in the meantime; integrate when 9a is available. + +**Shared-opcode discipline:** opcodes that another language port could plausibly use (pattern match, perform/handle, record access) get prepared for **chiselling out to `lib/guest/vm/`** when a second use materialises. Same lib/guest pattern, applied at the bytecode layer. Don't pre-extract; do annotate candidates in commit messages. + +- [x] **9a — Opcode extension mechanism** — **INTEGRATED** (scope widened by user 2026-05-15: hosts/ in scope, merging back). Cherry-picked the 5 vm-ext commits (phases A-E: dispatch fallthrough for opcodes ≥200, `Sx_vm_extension` interface, `Sx_vm_extensions` registry, `extension-opcode-id` SX primitive, JIT skip path) onto loops/erlang. Force-linked `Sx_vm_extensions` into `bin/sx_server.ml` so its module-init runs (was dead-code-eliminated — only `run_tests` referenced it). `extension-opcode-id` is now live in the runtime: returns the registered opcode id, or nil for unknown names. Built clean; conformance held at **709/709** on the freshly built binary. Design: `plans/sx-vm-opcode-extension.md`. +- [x] **9b — `OP_PATTERN_TUPLE` / `OP_PATTERN_LIST` / `OP_PATTERN_BINARY`** — **+19 vm tests** (656/656 total). Stub dispatcher in `lib/erlang/vm/dispatcher.sx` mirrors the OCaml extension shape from `plans/sx-vm-opcode-extension.md`: `er-vm-register-opcode!`/`er-vm-lookup-opcode-by-id`/`er-vm-lookup-opcode-by-name`/`er-vm-dispatch`. Opcode IDs 128 (TUPLE), 129 (LIST), 130 (BINARY) per the guest-tier partition (128-199). Handlers are thin wrappers over the existing `er-match-tuple`/`er-match-cons`/`er-match-binary` for now; the real specialization (skip AST walk, register-machine operands) lands when 9a integrates. Conformance must remain unchanged — **656/656** preserved. Candidate for chiselling to `lib/guest/vm/match.sx` once a second port (Prolog? miniKanren?) wants the same opcodes. +- [x] **9c — `OP_PERFORM` / `OP_HANDLE`** — **+9 vm tests** (665/665 total). Stubs in `lib/erlang/vm/dispatcher.sx`: `OP_PERFORM` (id 131) raises `{:tag "vm-effect" :effect :args }`; `OP_HANDLE` (id 132) wraps a thunk in `guard`, catches matching effects (by `:effect` name), passes args to the handler, returns the handler's result. Non-matching effects rethrow to outer handlers (verified by a nested-handle test). Pure Erlang `receive` interface unchanged; this is the substrate for the eventual call/cc-free implementation when 9a integrates. Candidate for chiselling (Scheme call/cc, OCaml 5 effects, miniKanren all want the same shape). +- [x] **9d — `OP_RECEIVE_SCAN`** — **+10 vm tests** (675/675 total). Stub at id 133 in `lib/erlang/vm/dispatcher.sx`. Operand contract: `(clauses mbox-list env)` where each clause is `{:pattern :guards :body}`, mbox-list is a plain SX list (not a queue — caller does queue→list before invoking and queue-delete after). Walks mbox in arrival order; tries each clause per message; first match returns `{:matched true :index N :body B}` (env mutated with bindings, body NOT evaluated — caller chooses when); no match returns `{:matched false}`. Pure pattern scan; suspension is the caller's job (compose with OP_PERFORM "receive-suspend" once 9a integrates). The real opcode will skip the AST walk by JIT-compiling each clause's match expr; this stub re-uses `er-match!` for correctness. +- [x] **9e — `OP_SPAWN` / `OP_SEND` + lightweight scheduler** — **+16 vm tests** (691/691 total). Stubs at ids 134 (SPAWN) and 135 (SEND) in `lib/erlang/vm/dispatcher.sx`, plus the VM-process registry: `er-vm-procs` (dict pid → proc record), `er-vm-next-pid`, `er-vm-procs-reset!`, `er-vm-proc-new!`/`get`/`send!`/`mailbox`/`state`/`count`. Process record shape is the register-machine layout the real scheduler will use: `{:id :registers (list of 8 nil slots) :mailbox (SX list) :state ("runnable"/"waiting"/"dead") :initial-fn :initial-args}`. OP_SPAWN returns a numeric pid and allocates a fresh record; OP_SEND appends to the target's mailbox, flipping `:state` from "waiting" → "runnable" if needed (returns true on success, false on unknown pid — no crash). Sits parallel to `er-scheduler` (the language-level scheduler from Phase 3); the real VM scheduler will take over once 9a integrates and Erlang programs compile to bytecode. Perf targets in the bullet (spawn <50µs, send <5µs) defer to the integration step. +- [x] **9f — BIF dispatch table** — **+18 vm tests** (709/709 total). 10 hot BIFs get their own opcode IDs (136-145) in `lib/erlang/vm/dispatcher.sx`: `OP_BIF_LENGTH`, `OP_BIF_HD`, `OP_BIF_TL`, `OP_BIF_ELEMENT`, `OP_BIF_TUPLE_SIZE`, `OP_BIF_LISTS_REVERSE`, `OP_BIF_IS_INTEGER`, `OP_BIF_IS_ATOM`, `OP_BIF_IS_LIST`, `OP_BIF_IS_TUPLE`. Each opcode's handler IS the underlying `er-bif-*` impl directly (no registry-string-lookup), so cost is opcode-id → handler one-hop. Cold BIFs continue through `er-apply-bif` / `er-lookup-bif` as before. IDs 136-159 reserved for future hot-BIF additions. +- [x] **9h — `erlang_ext.ml`** — OCaml extension at `hosts/ocaml/lib/extensions/erlang_ext.ml` registering the 18-opcode Erlang namespace (ids **222-239**, names `erlang.OP_*` mirroring the SX stub dispatcher). Registered at sx_server startup via `Erlang_ext.register ()` (guarded against double-register Failure). `extension-opcode-id "erlang.OP_PATTERN_TUPLE"` → 222 … `OP_BIF_IS_TUPLE` → 239, unknown → nil. Handlers raise a descriptive not-wired `Eval_error` (bytecode emission is a later phase; SX stub dispatcher remains the working specialization path) — keeps the extension honest rather than silently corrupting the VM stack. id range 222+ dodges test_reg (210/211) + test_ext (220/221) so all three coexist in run_tests. **+5 OCaml ext tests** (run_tests `Suite: extensions/erlang_ext`); Erlang conformance held **709/709**. +- [x] **9i — wire SX dispatcher to real ids** — `lib/erlang/vm/dispatcher.sx` gains `er-vm-host-opcode-id` (thin `extension-opcode-id` wrapper) and `er-vm-effective-opcode-id name stub-id` (host id when non-nil, else stub-id). `extension-opcode-id` resolves lazily at call time so loading the file is safe even on a binary lacking the primitive; only invoking the resolver there would raise (documented prereq — the loop builds + runs against the binary that has it). **+6 vm tests** (715/715): OP_PATTERN_TUPLE→222, OP_BIF_IS_TUPLE→239, unknown→nil, effective prefers host (OP_BIF_LENGTH→230), effective falls back to stub on nil (999), and a sweep asserting the whole 18-name namespace maps contiguously to 222..239. Stub-local ids (128-145) registration untouched so the prior 72 vm tests stay green. +- [x] **9g — Conformance + perf bench** — Ran `lib/erlang/bench_ring.sh 10 100 500 1000` on the integrated binary (9a+9h+9i built in): 11/36/35/31 hops/s — **unchanged from the pre-integration baseline**, which is the correct expected result and doubles as a no-regression proof (the full extension wiring added zero per-hop cost). Conformance **715/715** on the same binary. Numbers recorded in `lib/erlang/bench_ring_results.md` with the rationale. The ~3000×/~1000× targets are gated on Phase 10 (bytecode emission) — the compiler doesn't emit `erlang.OP_*` yet, so every hop still takes the general CEK path. 9g's deliverable (honest measurement on the integrated binary) is complete. + +### Phase 10 — bytecode emission (unlock the speedup) + +The Phase 9 opcodes are registered, tested, and bridged SX↔OCaml, but inert: nothing emits them. Phase 10 makes the speedup real. + +- [ ] **10a — compiler emits `erlang.OP_*` at hot sites** — **BLOCKED on `lib/compiler.sx` ownership (out of this loop's scope).** Architecture fully mapped (2026-05-15, see Blockers + design below). The correct implementation site is `lib/compiler.sx`'s `compile-call` — it must recognize calls to the Erlang runtime-helper functions that have a registered `erlang.OP_*` opcode and emit that opcode (via the already-live `extension-opcode-id` primitive) instead of a generic CALL. This is **generic shared compiler infrastructure** (any guest port — Prolog, Lua — would use the same intrinsic mechanism), explicitly excluded by the ground rules ("Don't edit lib/ root"; not in the widened hosts/-only scope). Concrete sub-steps for the owning session: + - **10a.1** Add an *intrinsic registry* to `lib/compiler.sx`: a dict `callee-name → extension-opcode-name`, populated by guests at load (e.g. Erlang registers `er-bif-length → "erlang.OP_BIF_LENGTH"`, `er-match-tuple → "erlang.OP_PATTERN_TUPLE"`, …). + - **10a.2** In `compile-call`: if the resolved callee is in the intrinsic registry AND `(extension-opcode-id name)` is non-nil, compile the args normally (push left→right) then emit the single opcode byte instead of `CALL`. Fall back to generic CALL when the opcode is absent (graceful on binaries without the extension). + - **10a.3** Define the operand/stack contract per opcode class and make `erlang_ext.ml`'s control handlers (222-229) match it (pattern opcodes need the pattern AST as a constant-pool operand + the scrutinee on the stack; perform/handle/receive/spawn/send need OCaml↔SX runtime-state access — see 10b-control note). + - **10a.4** Conformance must stay green; add bytecode-emission tests (compile an Erlang fn, disassemble, assert the opcode appears at the hot site). + Until a session owning `lib/compiler.sx` lands 10a.1-10a.2, the speedup cannot be realized from this loop. The BIF half of 10b (operand-less stack ops) is fully done and *would* light up immediately once emission exists. +- [~] **10b — real `erlang_ext.ml` handlers** — **10 of 18 real** (ALL BIF opcodes done: 230-239). Latest: `OP_BIF_ELEMENT` (233, pops Tuple-then-Index, 1-indexed, range-checked) and `OP_BIF_LISTS_REVERSE` (235, builds a fresh reversed cons chain in OCaml). Re-scoping correction: ELEMENT/REVERSE were earlier mislabelled "gated on 10a" — they're pure stack transforms (no bytecode operands; element/2 just pops 2), so they landed now. **21 e2e run_tests** total. Remaining 8 stubs are the genuine control/structural opcodes that DO need compiler-defined operands + runtime state: `OP_PATTERN_TUPLE/LIST/BINARY` (222-224), `OP_PERFORM/HANDLE` (225-226), `OP_RECEIVE_SCAN` (227), `OP_SPAWN/SEND` (228-229). not-wired guard repointed to 222. 715/715 unaffected. — earlier note: 8 of 18 real (all hot-BIFs done). Real register-machine handlers: `OP_BIF_LENGTH` (230, cons-walk), `OP_BIF_HD` (231), `OP_BIF_TL` (232), `OP_BIF_TUPLE_SIZE` (234, handles List + ListRef `:elements`), `OP_BIF_IS_INTEGER` (236, `Integer _`), `OP_BIF_IS_ATOM` (237), `OP_BIF_IS_LIST` (238, cons|nil), `OP_BIF_IS_TUPLE` (239) — all operate on the tagged-Dict value repr, push Erlang bool atoms via a `mk_atom` helper, raise on type errors. **15 end-to-end run_tests tests** (build real bytecode `[CONST i; op; RETURN]` with list/tuple/atom constants, assert via `Sx_vm.execute_module`). Still `not_wired`: the 8 control opcodes — `OP_PATTERN_TUPLE/LIST/BINARY` (222-224), `OP_PERFORM/HANDLE` (225-226), `OP_RECEIVE_SCAN` (227), `OP_SPAWN/SEND` (228-229) — plus `OP_BIF_ELEMENT` (233, needs 2 operands) and `OP_BIF_LISTS_REVERSE` (235). not-wired guard repointed to 233. 715/715 conformance unaffected (VM-bytecode path only; interpreter untouched). Remaining 10b: the 10 control/structural handlers. +- [ ] **10c — perf validation**: re-run `bench_ring.sh`; target 100k+ hops/sec at N=1000, 1M-process spawn < 30s; record in `bench_ring_results.md`. Conformance must stay green. + +**Acceptance:** ring benchmark hits the 100k hops/sec target. All prior phase tests pass. Two opcodes chiselled to `lib/guest/vm/` (or annotated as candidates with a written rationale). + ## Progress log _Newest first._ +- **2026-05-18 FIXED merge-blocking regression: cyclic-env hang in `er-env-derived-from?`** — A trial merge of loops/erlang → architecture regressed Erlang **715/715 → 0/0** on the architecture binary. Bisected: not loader semantics, not a uniform slowdown — pinpointed to the *single* Phase 7 capstone test (eval.sx lines 1314-1346; prefix-1313 was byte-identical speed on both binaries, 27s, prefix-1346 was 28s on loops vs >5min/hung on architecture). Isolated further: spawn+reload alone 0.6s, reload+purge alone 0.3s, but spawn+reload+**purge over forever-blocked procs** hung. Root cause: `er-env-derived-from?` (transpile.sx, used by `code:purge`/`soft_purge` via `er-procs-on-env`) compared closure envs with `(= env target-env)`. loops/erlang's evaluator implements dict `=` as **object identity**; architecture's 131-commit-newer evaluator changed it to **structural deep equality**. Erlang closure envs are large and **cyclic** (a module fun's `:env` transitively references the fun), so structural `=` over them never terminates. Fix: use `identical?` (pointer-identity predicate, present + consistent `(true false)` on *both* binaries) — the actually-intended semantics and host-independent. Verified: full eval.sx on the architecture binary >200s/hung → **59s**; full 10-suite conformance on the architecture binary now **715/715** (eval 385/385, vm 78/78, ffi 14/14, all process suites green). loops/erlang behaviour unchanged (`identical?` ≡ its old `=`-identity). One-file change (`lib/erlang/transpile.sx`, +7/-2). The merge can now be re-attempted; this was the sole blocker. + +- **2026-05-15 Phase 10a — architecture traced, scoped, blocked on `lib/compiler.sx`** — Investigation-only iteration (correctly: faking compiler emission within scope is impossible and would be dishonest). Traced the full JIT path: `sx_vm.ml`'s `jit_compile_lambda` (the ref set at line 1206) invokes the SX-level `compile` from `lib/compiler.sx` via the CEK machine — that is the only SX→bytecode producer. Erlang's hot helpers are ordinary SX functions in `transpile.sx` that get JIT-compiled through exactly this path, so emitting `erlang.OP_*` means teaching `compiler.sx`'s `compile-call` to recognize them as intrinsics and emit the extension opcode (the file's own docstring already anticipates this — "Compilers call `extension-opcode-id` to emit extension opcodes" — designed but unimplemented; grep confirms zero `extension-opcode-id` uses in `compiler.sx`). `lib/compiler.sx` is lib-root: excluded by ground rules and the widened scope (editing it changes every guest's JIT — must be a shared-compiler session, not this loop). Recorded a precise Blockers entry + decomposed 10a into four numbered sub-steps (10a.1 intrinsic registry, 10a.2 `compile-call` emission with graceful CALL fallback, 10a.3 operand/stack contract for control opcodes, 10a.4 bytecode-emission tests) so the owning session can execute directly. Key payoff documented: all 10 BIF handlers (230-239) are already real, so they light up the instant 10a.1-10a.2 land — zero further Erlang-side work for the BIF speedup. No code changed; conformance unverified-but-untouched at **715/715** (no source touched). Phase 10's loop-reachable work (10b BIF half) is complete; the rest is correctly blocked and fully actionable elsewhere. + +- **2026-05-15 Phase 10b — ELEMENT + LISTS_REVERSE real; all 10 BIF opcodes done** — Re-examined the earlier "gated on 10a" claim for ELEMENT/REVERSE and found it wrong: both are pure stack transforms with no need for bytecode operands (`element/2` just pops Tuple then Index off the VM stack; `lists:reverse/1` pops one list). Implemented both as real handlers in `erlang_ext.ml`. `OP_BIF_ELEMENT` (233): pops Tuple (TOS) then Index, handles List/ListRef `:elements`, 1-indexed, raises on out-of-range or wrong arg types. `OP_BIF_LISTS_REVERSE` (235): walks the cons chain building a fresh reversed one via local `mk_cons`/`mk_nil`, raises on improper list. Defined the calling convention for arity-2 ELEMENT: args pushed left→right so stack is `[Index Tuple]`, Tuple on top. 6 new e2e run_tests: element(2/1,{1,2,3}), element out-of-range raises, reverse-then-HD=9, reverse-then-TL-HD=8, reverse-then-LENGTH=3 (composes 3 real opcodes in one bytecode sequence). erlang_ext suite 15→21 PASS, dispatch_count 22. not-wired guard repointed 233→222 (OP_PATTERN_TUPLE — a genuine control opcode still stubbed). **All 10 BIF opcodes (230-239) now real**; the 8 remaining stubs are the true control/structural opcodes (pattern match, perform/handle, receive-scan, spawn/send) which genuinely need 10a's compiler-defined operand encoding + runtime-state access. Erlang conformance **715/715** (interpreter path untouched). 10b is now BIF-complete; the control-opcode half is the real remaining Phase 10 work and is correctly gated on 10a. + +- **2026-05-15 Phase 10b — all 8 hot-BIF handlers real** — Built on the vertical slice: added 7 more real register-machine handlers in `erlang_ext.ml` (HD 231, TL 232, TUPLE_SIZE 234, IS_INTEGER 236, IS_ATOM 237, IS_LIST 238, IS_TUPLE 239), joining LENGTH 230. Shared helpers added: `mk_atom` (builds the Erlang bool atom `{tag→atom, name→true|false}`), `er_bool`, `is_tag` (Dict tag predicate). TUPLE_SIZE handles both `List` and `ListRef` `:elements` (Erlang tuples may be built mutably). IS_INTEGER keys off `Sx_types.Integer`. All raise descriptive `Eval_error` on type mismatch. The `op N "name"` stub helper now only covers the 10 remaining control/structural opcodes. 9 new end-to-end run_tests assertions added (HD, TL∘HD, TUPLE_SIZE, IS_INTEGER pos+neg, IS_ATOM, IS_LIST nil-true + tuple-false, IS_TUPLE) — each builds real bytecode with a list/tuple/atom constant and executes via `Sx_vm.execute_module`. erlang_ext suite 6→15 PASS; dispatch_count 12. not-wired guard repointed 231→233 (OP_BIF_ELEMENT, still stubbed — it needs two operands so it's a later sub-step). Erlang conformance **715/715** (the interpreter path is untouched; only the VM-bytecode dispatch gained real handlers). Remaining 10b: pattern tuple/list/binary, perform/handle, receive-scan, spawn/send, element, lists:reverse (10 opcodes). + +- **2026-05-15 Phase 10b vertical slice — first real opcode handler, end-to-end VM proof** — Investigation first: confirmed Erlang runs as a pure tree-walking interpreter (`er-eval-expr` over CEK) — there is **no** Erlang→bytecode compiler, so full 10a (compiler emits opcodes) is a multi-week standalone effort, not one iteration. Rather than fake it, de-risked the whole Phase 9/10 architecture with a vertical slice: replaced the `not_wired` raise for `erlang.OP_BIF_LENGTH` (id 230) with a genuine register-machine handler in `erlang_ext.ml` — pops a value, walks the Erlang cons-list representation (`Dict` with `"tag"`→`"cons"`/`"nil"`, `"head"`, `"tail"`), pushes `Integer` length, raises on improper lists. Added an end-to-end run_tests test that builds real bytecode `[| 1; 0; 0; 230; 50 |]` (CONST idx 0 → OP_BIF_LENGTH → RETURN) with an Erlang `[1,2,3]` in `vc_constants`, executes via `Sx_vm.execute_module`, asserts `Integer 3`. This proves the complete path works: `extension-opcode-id` → bytecode → `Sx_vm` ≥200 dispatch fallthrough → `erlang_ext` handler → correct VM stack result — the load-bearing proof that Phase 9's wiring isn't just stubs. The other 17 opcodes still honestly raise `not_wired`; the prior not-wired guard test was repointed from 230 to 231 (OP_BIF_HD) so it still verifies the honest-failure path. erlang_ext suite 5→6 tests, dispatch_count now 2. Erlang conformance **715/715** unaffected (the new path is VM-bytecode-only; the interpreter path is untouched). 10b marked in-progress `[~]`; remaining: real handlers for the other 17 opcodes + 10a compiler emission. Builds clean via `dune build bin/run_tests.exe bin/sx_server.exe`. + +- **2026-05-15 Phase 9g — perf bench recorded on integrated binary; Phase 10 scoped** — Built the fresh `sx_server.exe` (9a+9h+9i wired in), ran `lib/erlang/bench_ring.sh 10 100 500 1000`: 11/36/35/31 hops/s — statistically identical to the pre-9a baseline (11/24/26/29/34). This is the *expected* outcome and the iteration's actual deliverable: it proves the entire extension stack (vm-ext A-E cherry-pick + `Sx_vm_extensions` force-link + `erlang_ext.ml` + SX dispatcher bridge) added **zero per-hop overhead** — a clean no-regression result — while honestly showing the speedup hasn't arrived because the bytecode compiler still doesn't emit `erlang.OP_*` (every hop takes the general CEK path). Updated `bench_ring_results.md` with a "Phase 9g" section: the table + the rationale that unchanged numbers = correct + no-regression. Conformance **715/715** on the integrated binary. Added **Phase 10 — bytecode emission** to the roadmap (10a compiler emits opcodes at hot sites, 10b real register-machine `erlang_ext.ml` handlers replacing the not-wired raises, 10c perf validation against the 100k-hops/1M-spawn targets). Phase 9 is now fully ticked (9a-9i); the actual speedup is honestly deferred to Phase 10 rather than faked. No code change this iteration — measurement + documentation + roadmap. + +- **2026-05-15 Phase 9i — SX dispatcher consults host opcode ids** — `lib/erlang/vm/dispatcher.sx` now bridges SX↔OCaml opcode ids. Two new functions: `er-vm-host-opcode-id` (wraps `extension-opcode-id`) and `er-vm-effective-opcode-id name stub-id` (host id if the OCaml `erlang_ext` registered it, else the stub-local id). Key SX-runtime fact established this iteration: symbol resolution is **lazy/call-time** — `(define f (fn () (extension-opcode-id "x")))` does NOT raise at load even when the primitive is absent; only calling `f` does. Combined with the earlier findings (guard can't catch undefined-symbol; no symbol-existence reflection), this means graceful in-SX degradation is impossible — so the design instead documents the binary prerequisite and relies on the loop building+running the freshly-built `hosts/ocaml/_build/default/bin/sx_server.exe` (conformance.sh's default, which has the vm-ext mechanism + erlang_ext). Stub-local registration (128-145) deliberately left intact so the 72 pre-existing vm tests don't move. 6 new vm tests: 222/239 lookups, unknown→nil, effective-prefers-host (230), effective-fallback (999), and a contiguity sweep over all 18 `erlang.OP_*` names asserting they map to 222..239 in order. vm suite 72→78. Total **715/715** on the fresh binary. Next: 9g — re-run ring bench, record numbers (note: stubs still wrap existing impls 1-to-1 so numbers won't move until the compiler emits these opcodes — a later phase). + +- **2026-05-15 Phase 9h — erlang_ext.ml registered, opcode namespace live** — New `hosts/ocaml/lib/extensions/erlang_ext.ml` modelled on `test_ext.ml`: an `EXTENSION` module `name="erlang"`, per-instance `ErlangExtState` (dispatch counter), 18 opcodes ids 222-239 named `erlang.OP_*` exactly mirroring the SX stub dispatcher. Registered at sx_server startup with a second guarded line in `bin/sx_server.ml` (`try Erlang_ext.register () with Failure _ -> ()` — survives a re-entered server). `include_subdirs unqualified` in `lib/dune` already pulls `lib/extensions/*.ml` into the `sx` lib, so no dune edit needed. Handlers deliberately raise a descriptive `Eval_error` ("bytecode emission not yet wired (Phase 9j) — Erlang runs via CEK; specialization path is the SX stub dispatcher") rather than fake stack ops — the compiler doesn't emit these yet, so an honest loud failure beats silent corruption. Hit and fixed an opcode-id collision: the original 200-217 range clashed with run_tests' inline test_reg (210/211); relocated to 222-239 (clears test_reg + test_ext 220/221, all coexist; production sx_server only registers erlang). 5 new OCaml tests in run_tests `Suite: extensions/erlang_ext`: opcode-id 222 + 239 resolve, unknown→nil, dispatch raises not-wired (substring check, no Str dep since run_tests doesn't link str), dispatch_count state ≥1. Built via `eval $(opam env --switch=5.2.0); dune build bin/run_tests.exe bin/sx_server.exe`. Erlang conformance **709/709** on the rebuilt binary (the broad run_tests 1110 failures are loops/erlang's pre-existing months-old divergence from architecture — run_tests was never built on this branch before; my changes are isolated additive). Next: 9i — wire the SX stub dispatcher to consult `extension-opcode-id`. + +- **2026-05-15 Phase 9a integrated — scope widened to hosts/** — User lifted the hosts/ scope restriction ("we are going to merge this back anyhow"). Cherry-picked the 5 `vm-ext` commits (phases A-E) from `loops/sx-vm-extensions` onto `loops/erlang` — only conflict was `plans/sx-vm-opcode-extension.md` (already had architecture's final copy from an earlier iteration; resolved `-X ours`, OCaml files auto-merged clean since loops/erlang never touched hosts/). Discovered `extension-opcode-id` was still "Undefined symbol" even on a fresh build: `Sx_vm_extensions`'s module-init (`install_dispatch` + primitive registration) only runs if the module is linked, and `sx_server.ml` never referenced it (only `run_tests.ml` did), so OCaml dead-code-eliminated it. Fix: added `let () = ignore (Sx_vm_extensions.id_of_name "")` force-link reference near the top of `bin/sx_server.ml`. Rebuilt with `dune build` (opam switch 5.2.0; `dune` not on PATH by default — `eval $(opam env --switch=5.2.0)` first). `extension-opcode-id` now live: returns nil for unregistered names, will return real ids once an extension registers. Conformance **709/709** on the freshly built binary (cherry-picked sx_vm.ml dispatch changes + force-link, zero regressions). 9a checkbox flipped from BLOCKED to INTEGRATED; Blockers entry resolved; added 9h (erlang_ext.ml) + 9i (wire SX dispatcher to real ids) as ordinary in-scope checkboxes, reordered 9g after them. Next: write `hosts/ocaml/lib/extensions/erlang_ext.ml`. + +- **2026-05-14 Phase 9g logged as partially BLOCKED — perf bench waits on 9a** — Conformance half satisfied: 709/709 with all Phase 9 stub infrastructure loaded (10 opcode IDs registered, 72 vm-suite tests passing, zero regressions in tokenize/parse/eval/runtime/ring/ping-pong/bank/echo/fib/ffi suites). Perf-bench half can't move forward in this worktree because the stub handlers wrap the existing `er-bif-*` / `er-match-*` / scheduler impls 1-to-1; a ring benchmark with the new opcodes "active" would measure the same 34 hops/s already documented in `bench_ring_results.md`. Updated `bench_ring_results.md` with a Phase 9 status section explaining the pre-integration state (stubs ready, real measurement gated on 9a's bytecode compiler emitting these IDs at hot sites). Blockers entry added pairing 9g with the existing 9a Blocker. No code change; total **709/709** unchanged. Phase 9 stub work (9b-9f) is complete from this loop's vantage point — 9a and 9g remain BLOCKED on a `hosts/ocaml/` iteration. + +- **2026-05-14 Phase 9f — hot-BIF opcode table green** — Ten hot BIFs get direct opcode IDs in `lib/erlang/vm/dispatcher.sx` so the bytecode compiler can emit them at hot call sites without paying the registry string-key hash: `OP_BIF_LENGTH (136)`, `OP_BIF_HD (137)`, `OP_BIF_TL (138)`, `OP_BIF_ELEMENT (139)`, `OP_BIF_TUPLE_SIZE (140)`, `OP_BIF_LISTS_REVERSE (141)`, `OP_BIF_IS_INTEGER (142)`, `OP_BIF_IS_ATOM (143)`, `OP_BIF_IS_LIST (144)`, `OP_BIF_IS_TUPLE (145)`. Implementation is one line per opcode: the handler IS the existing `er-bif-*` function directly — same `(vs)` signature as the dispatcher's `(operands)`, so the registration is `(er-vm-register-opcode! ID "NAME" er-bif-FOO)`. IDs 136-159 reserved for future hot-BIF additions; cold BIFs continue through `er-apply-bif`/`er-lookup-bif`. 18 new tests in `tests/vm.sx`: opcode-by-id verification (LENGTH), one positive test per BIF (length on 3-cons, hd, tl-is-cons, element index 2, tuple_size 4, lists:reverse preserves length AND actually reverses [head check], is_integer pos+neg, is_atom pos+neg, is_list pos+nil pos+tuple neg, is_tuple pos+neg), opcode-list-grew-to-16+. vm suite 54 → 72. Total **709/709** (+18 vm). Real perf benefit lands when 9a integrates and the compiler emits these IDs at hot sites. + +- **2026-05-14 Phase 9e — OP_SPAWN / OP_SEND + VM-process registry green** — `lib/erlang/vm/dispatcher.sx` gains a parallel mini-runtime distinct from the language-level `er-scheduler`: `er-vm-procs` (dict pid → proc record), `er-vm-next-pid` (counter cell), `er-vm-procs-reset!`, plus six accessors (`er-vm-proc-new!`/`get`/`send!`/`mailbox`/`state`/`count`). Process record shape is the register-machine layout the real bytecode scheduler will use: `{:id :registers (8 nil slots) :mailbox :state :initial-fn :initial-args}` — fixed register width so cells don't grow during execution. Opcode 134 `OP_SPAWN` calls `er-vm-proc-new!` and returns the new pid; 135 `OP_SEND` appends to the target's mailbox and flips a waiting proc back to runnable, returns false for unknown pid (graceful, doesn't crash). 16 new tests in `tests/vm.sx`: opcode-by-id for both, spawn returns 0 / 1 / count=2 / state=runnable / mailbox empty / 8 registers, send returns true, 3-sends preserve arrival order (first + last verified), send to unknown pid returns false, isolation (p1's msgs don't leak into p2), reset clears procs + resets pid counter. vm suite 38 → 54. One gotcha during impl: SX `fn` bodies evaluate ONLY the last expression — `er-vm-procs-reset!` had two `set-nth!` calls back-to-back which silently dropped the first; wrapped in `(do ...)` to fix. Total **691/691** (+16 vm). Real scheduler with per-process scheduling latency and runnable queue is post-9a. + +- **2026-05-14 Phase 9d — OP_RECEIVE_SCAN stub green** — Selective-receive primitive at opcode id 133 in `lib/erlang/vm/dispatcher.sx`. Operand contract: `(clauses mbox-list env)` — clauses are AST dicts (`{:pattern :guards :body}`), mbox-list is a plain SX list (queue → list is the caller's job), env is the binding target. Internal helpers `er-vm-receive-try-clauses` (per-message clause walker with env snapshot/restore on failure) and `er-vm-receive-scan-loop` (mailbox walker, arrival order). Match returns `{:matched true :index N :body B}` so the caller can queue-delete at N and then evaluate B in the now-mutated env; miss returns `{:matched false}` so the caller can suspend via OP_PERFORM "receive-suspend". Mirrors the existing `er-try-receive-loop` in `transpile.sx` but doesn't reach into the scheduler — purely VM-level. 10 new tests in `tests/vm.sx`: opcode registered, scan finds match at correct index, scan binds var, body left unevaluated, no-match leaves env untouched, empty mailbox, first-match wins (arrival order — verified by two `{ok, _}` msgs and binding the FIRST value). vm suite 28 → 38. Total **675/675** (+10 vm). When 9a integrates and the real OP_RECEIVE_SCAN compiles clauses into a register-machine match, the existing `er-eval-receive-loop` becomes a one-line dispatch wrapper. + +- **2026-05-14 Phase 9c — OP_PERFORM / OP_HANDLE stubs green** — Two new opcodes in `lib/erlang/vm/dispatcher.sx`: id 131 `OP_PERFORM` raises `{:tag "vm-effect" :effect :args }`; id 132 `OP_HANDLE` wraps a thunk in SX `guard`, catches matching effects by `:effect` name, passes the `:args` list to the handler fn, returns the handler's result. New helper `er-vm-effect-marker?` predicates on the dict shape. Non-matching effects rethrow via a small box+rethrow dance (caught with `:else` first, decision deferred to a post-guard cond — re-raise outside the guard's scope so it propagates to outer handlers cleanly). 9 new tests in `tests/vm.sx`: opcode registered for each id; OP_PERFORM raises with correct tag/effect/args; OP_HANDLE catches matching effect; OP_HANDLE returns thunk result when no effect performed; OP_HANDLE rethrows non-matching effect to outer; nested OP_HANDLE blocks separate by effect name (inner handles "a", outer handles "b", performing "b" bypasses inner). vm suite grew 19 → 28 tests. Total **665/665** (+9 vm). Underlying call/cc + raise/guard machinery used by Erlang `receive` is unchanged; this is the shape for the eventual specialization when 9a integrates. Candidate for chiselling to `lib/guest/vm/effects.sx` — Scheme call/cc, OCaml 5 effects, miniKanren all want the same shape. + +- **2026-05-14 Phase 9b — stub VM dispatcher + 3 pattern opcodes green** — New `lib/erlang/vm/dispatcher.sx` defines the stub opcode registry mirroring the OCaml `EXTENSION` shape from `plans/sx-vm-opcode-extension.md`: opcodes registered as `{:id :name :handler}` keyed by string-id, looked up by id OR by name, dispatched via `er-vm-dispatch`. Opcode IDs follow the guest-tier partition (128-199 reserved for guest extensions like erlang/lua). Three opcodes registered at load time via `er-vm-register-erlang-opcodes!`: 128 `OP_PATTERN_TUPLE` → `er-match-tuple`, 129 `OP_PATTERN_LIST` → `er-match-cons`, 130 `OP_PATTERN_BINARY` → `er-match-binary`. Operand contract: `(pattern-ast value env)` returning `true`/`false` and mutating env on success — same as the underlying match functions. New `lib/erlang/tests/vm.sx` suite with 19 tests: 7 dispatcher core (registered, lookup by id+name for all three, two miss cases, list-has-3+); 4 OP_PATTERN_TUPLE (match success + var bind, no-match, arity mismatch); 4 OP_PATTERN_LIST (match, head bind, tail-is-cons, no-match on nil); 3 OP_PATTERN_BINARY (match, segment bind, size mismatch); 1 dispatch error (unknown opcode raises). `conformance.sh` updated: added `vm` to SUITES, added `(load "lib/erlang/vm/dispatcher.sx")` before tests and `(load "lib/erlang/tests/vm.sx")` after ffi, added epoch 110 evaluator. AST shape gotcha: er-match! reads `:type` not `:tag`; binary segment `:size` must be an AST node `{:type "integer" :value "8"}` because `er-eval-expr` runs on it. Total **656/656** (+19 vm). 9b complete; 9c (OP_PERFORM/OP_HANDLE) is next. + +- **2026-05-14 Phase 9a logged as Blocker — sub-phase 9b is next** — 9a (the opcode extension mechanism in `hosts/ocaml/evaluator/`) is explicitly out-of-scope for this loop per the plan itself (briefing scope rule + 9a's own text). Logged a Blockers entry citing `plans/sx-vm-opcode-extension.md` as the design doc and pointing at the fix path (a `hosts/` session lands the registration shape, then a follow-up here wires the stub dispatcher to the real one). Ticked 9a as DONE because its contract was "Log as Blocker" — that's complete. Sub-phases 9b–9g (PATTERN/PERFORM/RECEIVE/SPAWN_SEND/BIF/conformance) now in queue against a stub dispatcher in `lib/erlang/vm/`. No code change this iteration. Total **637/637** unchanged. + +- **2026-05-14 Phase 9 scoped + supporting plan files synced** — Copied three plan files from `/root/rose-ash/plans/` (architecture branch) that this worktree was missing: `fed-sx-design.md` (124KB, the substrate design referenced from Phase 7/8 drivers), `fed-sx-milestone-1.md` (33KB, first concrete implementation milestone), `sx-vm-opcode-extension.md` (19KB, the prerequisite for Phase 9a — designs how `lib//vm/` registers opcodes against the OCaml SX VM core). Then appended **Phase 9 — specialized opcodes (the BEAM analog)** to `plans/erlang-on-sx.md` covering sub-phases 9a-9g: 9a (opcode extension mechanism in `hosts/ocaml/`) is out-of-scope for this loop (will be logged as a Blocker when the next iteration tries to start it); 9b-9g (PATTERN_TUPLE/LIST/BINARY, PERFORM/HANDLE, RECEIVE_SCAN, SPAWN/SEND + lightweight scheduler, BIF dispatch table, conformance + perf bench) can be designed and tested against a stub dispatcher in the meantime. Targets: ring benchmark 100k+ hops/sec at N=1000 (~3000× speedup), 1M-process spawn under 30sec (~1000× speedup). Plan framing intact for Phase 7/8 — those reflect the actual implementation done in this loop; the architecture-branch framing diverges in language but the work is equivalent. No code touched this iteration. Total **637/637** unchanged. + +- **2026-05-14 ffi test suite extracted, conformance scoreboard auto-picks it up** — New `lib/erlang/tests/ffi.sx` with its own counter trio (`er-ffi-test-count`/`-pass`/`-fails`) and `er-ffi-test` helper following the same pattern as runtime/eval/ring tests. The 10 file BIF eval tests from the previous iteration moved out of `eval.sx` (eval dropped from 395 to 385 tests) and into the new suite where they're now 9 tests (consolidated the two write+read tests). `conformance.sh` updated: added `ffi` to `SUITES` array with `er-ffi-test-pass`/`-count` symbols, added `(load "lib/erlang/tests/ffi.sx")` after `fib_server.sx`, added `(epoch 109) (eval "(list er-ffi-test-pass er-ffi-test-count)")`. Scoreboard markdown auto-updated to include the row. Suite also asserts that the 5 blocked BIFs (`crypto:hash`, `cid:from_bytes`, `file:list_dir`, `httpc:request`, `sqlite:exec`) are NOT yet registered — turns a future "added the wrapper but forgot to extend ffi tests" into a hard failure. One eval-comparison gotcha en route: SX's `=` does identity equality on dicts so comparing two separately-constructed `(er-mk-atom "true")` values is false; the existing eval suite has an `eev-deep=` helper that handles this, but the simpler fix in ffi was to extract `:name` via `ffi-nm` and compare strings. Total **637/637** (+14 ffi). Phase 8 fully ticked aside from the BLOCKED bullets — those remain unchecked with explicit Blockers references. + +- **2026-05-14 file BIFs landed; crypto/cid/list_dir/http/sqlite blocked on missing host primitives** — Three new FFI BIFs registered in `runtime.sx`: `file:read_file/1`, `file:write_file/2`, `file:delete/1`. Each wraps the SX-host primitive (`file-read`, `file-write`, `file-delete`) inside a `guard` that converts thrown exception strings into Erlang `{error, Reason}` tuples. New helper `er-classify-file-error` does loose pattern-matching on the error message using `string-contains?` to map to standard POSIX-style reasons: `"No such"` → `enoent`, `"Permission denied"` → `eacces`, `"Not a directory"` → `enotdir`, `"Is a directory"` → `eisdir`, fallback `posix_error`. Filenames coerce through `er-source-to-string` so SX strings, Erlang binaries, and Erlang char-code lists all work. Read returns `{ok, Binary}` (bytes via `(map char->integer (string->list ...))` then `er-mk-binary`); write returns bare `ok`; delete returns bare `ok`. Bootstrap registrations added at the bottom of `er-register-builtin-bifs!` under `"file"`. 10 new eval tests: write-then-read round-trip, ok-tag, payload is binary, byte_size content, missing-file `enoent`, delete-ok, read-after-delete `enoent`, write to non-existent dir `enoent`, binary payload (5 raw bytes) round-trip preserving byte count. Blockers entry added covering five Phase 8 BIFs whose host primitives don't exist in this SX runtime: `crypto:hash/2`, `cid:from_bytes/1`/`to_string/1`, `file:list_dir/1`, `httpc:request/4`, `sqlite:open/exec/query/close`. Fix path documented inline (architecture-branch iteration to register OCaml-side primitives). Total **633/633** (+10 eval). + +- **2026-05-14 term-marshalling helpers landed** — `er-to-sx` (Erlang term → SX-native) and `er-of-sx` (SX-native → Erlang term) plus internal helper `er-cons-to-sx-list` (recursive cons-chain walker). All three live in `runtime.sx` next to the BIF registry. Conversion table: atom ↔ symbol via `make-symbol`/`er-mk-atom`; nil ↔ `()`; cons-chain → SX list (recursive marshal of each head); tuple → SX list (one-way — tuples flatten and can't be reconstructed without a tag); binary ↔ SX string (bytes ↔ char codes via `char->integer`/`integer->char`); integer / float / boolean passthrough; opaque types (pid, ref, fun) passthrough. SX strings on the way back become Erlang binaries — the natural FFI return shape. Empty SX list (`type-of` `"nil"`) marshals back to `er-mk-nil`. Edit gotchas during implementation: SX has no `while`, `string-ref`, or `string-length` primitive — used `(map char->integer (string->list s))` for byte extraction and a recursive helper for cons-walking. 23 new runtime tests in `tests/runtime.sx`: 10 covering `er-to-sx` (atom/atom-is-symbol, nil, int / float / bool passthrough, binary→string, cons→list, tuple→list, nested), 8 covering `er-of-sx` (symbol→atom, atom-tag, string→binary, byte content, int passthrough, empty-list→nil, list→cons length, head field), 4 round-trips (int, atom, binary bytes, list length), 1 negative documenting that tuple round-trip flattens to cons. Total **623/623** (+23 runtime). + +- **2026-05-14 BIF registry migration complete — cond chains gone** — `er-register-builtin-bifs!` at the end of `runtime.sx` populates the registry with all 67 built-in BIFs in five module namespaces. Pure ops (`length`, `hd`, `tl`, `element`, predicates, arithmetic, list/atom/integer conversions, all of `lists`) registered via `er-register-pure-bif!`; side-effecting ops (`spawn`, `self`, `exit`, `link`/`monitor`/`register`, `process_flag`, `make_ref`, `throw`/`error`, `io:format`, all of `ets`, all of `code`) via `er-register-bif!`. Multi-arity entries: `is_function/1`/`/2`, `spawn/1`/`/3`, `exit/1`/`/2`, `io:format/1`/`/2`, `lists:seq/2`/`/3`, `ets:delete/1`/`/2` — six pairs, twelve registrations, all pointing at the existing arity-dispatching impl. `throw` and `error` are registered with a tiny inline `(fn (vs) (raise ...))` lambda because the original code chained directly through `raise` inside the cond instead of an `er-bif-*` helper. `er-apply-bif` shrinks from a 44-line cond chain to a 5-line registry lookup. `er-apply-remote-bif` becomes a 7-line dispatcher (user-modules-first → registry → error). All four per-module dispatchers (`er-apply-lists-bif`, `er-apply-io-bif`, `er-apply-ets-bif`, `er-apply-code-bif`) deleted — net reduction ~110 lines of cond machinery. One subtle wrinkle: `tests/runtime.sx` calls `er-bif-registry-reset!` near the end of its BIF-registry tests, which would have left subsequent test files (ring, ping-pong, etc.) unable to call `length`/`spawn`/etc. Fix: re-call `er-register-builtin-bifs!` at the bottom of `tests/runtime.sx` to repopulate. Total **600/600** unchanged. + +- **2026-05-14 Phase 8 BIF registry foundation** — `lib/erlang/runtime.sx` gains `er-bif-registry` (a `(list {})` mutable cell, same shape as `er-modules`) and five helpers: `er-bif-registry-get`/`er-bif-registry-reset!` (access + reset), `er-bif-key` (format `"Module/Name/Arity"`), `er-register-bif!` and `er-register-pure-bif!` (both upsert; differ only in the `:pure?` flag — pure ones are safe to inline, side-effecting ones go through normal IO), `er-lookup-bif` (returns the entry dict or nil), `er-list-bifs` (registered keys). Entries are `{:module :name :arity :fn :pure?}`. Lookup miss → nil; arity is part of the key so `m:f/1` and `m:f/2` are distinct; re-registering the same key replaces in-place (count stays the same); reset clears. Registry sits alongside `er-modules` in runtime.sx so any other piece of the system can register BIFs without touching the dispatcher — the migration onto this registry (the next checkbox) will rip out the giant cond chains in `er-apply-bif`/`er-apply-remote-bif`. 18 new runtime tests in `tests/runtime.sx`: empty-state, lookup-miss, register-grows-count, lookup-hit-fields (module/name/arity/pure?), fn-invocable, re-register-replaces, pure-flag-true, arity-disambiguation (3 entries for `fake:echo/1`, `fake:echo/2`, `fake:pure/2`), reset-clears, reset-lookup-nil. Total **600/600** (+18 runtime). + +- **2026-05-14 Phase 7 capstone green — full hot-reload ladder works end-to-end** — Wires everything from the previous five iterations into one test program: load cap v1 with `start/0` (spawn-from-inside-module) + `loop/0` + `tag/0` → spawn Pid1 (running v1) → load cap v2 → assert `cap:tag()` returns v2 (cross-module dispatch hits `:current`) → spawn Pid2 (running v2) → `code:soft_purge(cap)` returns `false` (refuses while Pid1 is alive on v1's env) → `code:purge(cap)` returns `true` (kills Pid1, clears `:old`) → `code:soft_purge(cap)` returns `true` (clean — no `:old` left). To make this work, `er-procs-on-env` was extended with a new helper `er-env-derived-from?`: a process counts as "running on" mod-env if its `:initial-fun`'s `:env` IS mod-env directly OR contains at least one binding whose value is a fun closed over mod-env. Reason: `er-apply-fun-clauses` always `er-env-copy`s the closure-env before binding params, so a fun created inside a module body has a `:env` that's a *copy* of mod-env, not mod-env itself — the copy still contains the module's other functions as values, each pointing back to the canonical mod-env. The whole ladder runs as a single `erlang-eval-ast` invocation because each call to `ev` resets the scheduler via `er-sched-init!`, wiping any cross-call Pids. 5 capstone tests: v1 tag, v2 tag (cross-mod after reload), soft_purge-refuses, hard purge, soft_purge-clean-after-hard. Total **582/582** (+5 eval). Phase 7 fully ticked. + +- **2026-05-14 hot-reload call-dispatch semantics verified** — Tests-only iteration: no implementation change, just six new eval tests that nail down the Erlang semantics already implicit in the current code. (1) `M:F()` after reload returns v2's value (cross-module call hits `:current`). (2) Inside a freshly-loaded body, a bare local call resolves through the new mod-env so a chain `a() -> b()` reflects v2's `b/0`. (3) Calling a fun captured BEFORE reload, whose body uses a local call, returns the v1 value (closure pinned to old mod-env via `er-mk-fun`'s `:env` reference). (4) Calling a fun captured BEFORE reload, whose body uses a cross-module call `M:b()`, returns v2's value (cross-module always wins over closed-over env). (5) Two captured funs from two distinct vintages stay independent — F1() + F2() = 10 + 20 = 30. (6) The slot version counter still bumps even while old captured funs are alive, demonstrating the closure-pinning doesn't block reloads. The "running process finishes its current function with the version it started with" property falls out of fun-as-closure semantics for free — there's no special bookkeeping. Total **577/577** (+6 eval). + +- **2026-05-14 code introspection BIFs green** — `code:which/1`, `code:is_loaded/1`, `code:all_loaded/0` added to `er-apply-code-bif` dispatch with three small implementations in `transpile.sx`. `which` and `is_loaded` are dict-lookups on the module registry returning the loaded-marker (atom `loaded`) or the missing-marker (atom `non_existing` for which, atom `false` for is_loaded). Since we don't have a filesystem path representation, the standard `{file, Path}` shape for `is_loaded` becomes `{file, loaded}` — same tuple arity so destructuring code stays portable. `all_loaded` iterates `(keys (er-modules-get))` in reverse (so the result list preserves insertion order after the cons-prepend loop), wrapping each name in a `{Module, loaded}` tuple. **10 new eval tests**: non_existing for absent / loaded after load for which; missing / file-tag / loaded-value for is_loaded; empty / count-after-2-loads / first-entry-tag for all_loaded; badarg for both single-arg BIFs. Two of the all_loaded tests needed an explicit `(er-modules-reset!)` before the measurement because prior tests in the suite leave modules registered (the registry is process-global across the whole epoch session). Total **571/571** (+10 eval). + +- **2026-05-14 code:purge/1 + code:soft_purge/1 green** — Two new BIFs in `transpile.sx`: `er-bif-code-purge` and `er-bif-code-soft-purge`, both dispatched through the existing `er-apply-code-bif` cond chain. Shared helper `er-procs-on-env` walks `(er-sched-processes)` and collects pids whose `:initial-fun` is a fun whose `:env` is identical (dict-identity, not structural) to a given env, filtering out already-dead procs. `er-bif-code-purge` looks up the module slot, returns `false` if either the module isn't registered or `:old` is nil; otherwise calls `er-cascade-exit!` on every matching pid with reason `killed`, replaces the slot with a fresh `er-mk-module-slot` that has `:old nil` (current + version preserved), returns `true`. `er-bif-code-soft-purge` returns `true` (treating "no module" / "no old version" as already-purged), else checks for lingering procs and returns `false` (leaving the slot untouched) if any, else clears `:old` and returns `true`. Non-atom Mod raises `error:badarg` from both. **10 new eval tests**: unknown / no-old / after-reload / idempotent for purge; unknown / no-old / clean for soft_purge; badarg for both; one "purge after spawn" test verifying return value (does NOT exercise the kill path — see caveat in plan). Total **561/561** (+10 eval). Implementation cost: 1 dispatch entry, 3 small BIFs, no scheduler changes. + +- **2026-05-14 code:load_binary/3 green** — Canonical hot-reload entry point. Adds a `"code"` module branch to `er-apply-remote-bif`'s dispatch; new helpers `er-source-walk-bytes!` and `er-source-to-string` coerce any of {SX string, Erlang binary `<<...>>`, Erlang char-code cons list} to an SX source string before parsing. `er-bif-code-load-binary` is the BIF itself: validates `Mod` is an atom (`{error, badarg}` else), coerces source (`{error, badarg}` on unrecognised shape), wraps `erlang-load-module` in `guard` to convert parse failures into `{error, badfile}`, checks the parsed `-module(Name).` matches the BIF's first arg (`{error, module_name_mismatch}` else), returns `{module, Mod}`. Reload reuses the Phase-7 slot logic from the previous iteration so calling `code:load_binary(m, _, v2_source)` after `code:load_binary(m, _, v1_source)` bumps the slot to version 2 with v1 sitting in `:old`. 8 new eval tests: ok-tag/ok-name on first load, immediate cross-module call hits new env, reload-and-call returns v2 result, name-mismatch errors with both tag and reason, garbage source yields badfile, non-atom Mod is badarg. Total **551/551** (+8 eval). `code:load_file/1` deferred until `file:read_file/1` lands in Phase 8 (it's just a wrapper that reads bytes from disk then calls `load_binary`). + +- **2026-05-14 Phase 7 module-version slot landed** — `er-modules` entries are now `{:current MOD-ENV :old MOD-ENV-or-nil :version INT :tag "module"}` instead of bare mod-env dicts. New helpers in `runtime.sx`: `er-mk-module-slot`, `er-module-current-env`, `er-module-old-env`, `er-module-version`. `erlang-load-module` updated: first load creates a slot with `:version 1` and `:old nil`; subsequent loads of the same module name copy `:current` into `:old` and increment `:version` (bump-and-shift, single-old-version retention as per OTP semantics). `er-apply-user-module` now reads via `er-module-current-env` so cross-module calls always hit the latest version. 13 new runtime tests (mostly in `tests/runtime.sx`): slot constructor + accessors, registry-after-first-load (v1, old nil), registry-after-second-load (v2, old = previous current env identity, current = new env), v3 on triple-load, registry-reset clears. Total **543/543** (was 530/530). Note: sx-tree path-based MCP tools (`sx_replace_node`, `sx_read_subtree`) are broken in this worktree's `mcp_tree.exe` (every path returns/replaces form 0); edits applied via a Python script then `sx_validate`d. Pattern-based tools (`sx_find_all`, `sx_rename_symbol`) still work fine. + +- **2026-05-14 Phase 7 + Phase 8 scoped** — Plan extended with two new phases driven by fed-sx (see `plans/fed-sx-design.md` §17.5). Phase 7 brings hot code reload back in scope (was previously listed as out-of-scope): module versioning slot, `code:load_file/1`/`purge/1`/`soft_purge/1`/`which/1`/`is_loaded/1`, cross-module calls hitting current, local calls keeping start-time semantics until function returns. Phase 8 introduces a runtime-extensible **FFI BIF registry** that replaces today's hardcoded `er-apply-bif`/`er-apply-remote-bif` cond chains, plus a term-marshalling layer and concrete BIFs for `crypto:hash`, `cid:from_bytes`/`to_string`, `file:read_file`/`write_file`/`list_dir`/`delete`, `httpc:request`, `sqlite:open`/`exec`/`query`. Scope decisions header updated accordingly. Baseline 530/530 unchanged; no code touched this iteration. + - **2026-04-25 BIF round-out — Phase 6 complete, full plan ticked** — Added 18 standard BIFs in `lib/erlang/transpile.sx`. **erlang module:** `abs/1` (negates negative numbers), `min/2`/`max/2` (use `er-lt?` so cross-type comparisons follow Erlang term order), `tuple_to_list/1`/`list_to_tuple/1` (proper conversions), `integer_to_list/1` (returns SX string per the char-list shim), `list_to_integer/1` (uses `parse-number`, raises badarg on failure), `is_function/1` and `is_function/2` (arity-2 form scans the fun's clause patterns). **lists module:** `seq/2`/`seq/3` (right-fold builder with step), `sum/1`, `nth/2` (1-indexed, raises badarg out of range), `last/1`, `member/2`, `append/2` (alias for `++`), `filter/2`, `any/2`, `all/2`, `duplicate/2`. 40 new eval tests with positive + negative cases, plus a few that compose existing BIFs (e.g. `lists:sum(lists:seq(1, 100)) = 5050`). Total suite **530/530** — every checkbox in `plans/erlang-on-sx.md` is now ticked. - **2026-04-25 ETS-lite green** — Scheduler state gains `:ets` (table-name → mutable list of tuples). New `er-apply-ets-bif` dispatches `ets:new/2` (registers table by atom name; rejects duplicate name with `{badarg, Name}`), `insert/2` (set semantics — replaces existing entry with the same first-element key, else appends), `lookup/2` (returns Erlang list — `[Tuple]` if found else `[]`), `delete/1` (drop table), `delete/2` (drop key; rebuilds entry list), `tab2list/1` (full list view), `info/2` with `size` only. Keys are full Erlang terms compared via `er-equal?`. 13 new eval tests: new return value, insert true, lookup hit + miss, set replace, info size after insert/delete, tab2list length, table delete, lookup-after-delete raises badarg, multi-key aggregate sum, tuple-key insert + lookup, two independent tables. Total suite 490/490. - **2026-04-25 binary pattern matching green** — Parser additions: `<<...>>` literal/pattern in `er-parse-primary`, segment grammar `Value [: Size] [/ Spec]` (Spec defaults to `integer`, supports `binary` for tail). Critical fix: segment value uses `er-parse-primary` (not `er-parse-expr-prec`) so the trailing `:Size` doesn't get eaten by the postfix `Mod:Fun` remote-call handler. Runtime value: `{:tag "binary" :bytes (list of int 0-255)}`. Construction: integer segments emit big-endian bytes (size in bits, must be multiple of 8); binary-spec segments concatenate. Pattern matching consumes bytes from a cursor at the front, decoding integer segments big-endian, capturing `Rest/binary` tail at the end. Whole-binary length must consume exactly. New BIFs: `is_binary/1`, `byte_size/1`. Binaries participate in `er-equal?` (byte-wise) and format as `<>`. 21 new eval tests: tag/predicate, byte_size for 8/16/32-bit segments, single + multi segment match, three 8-bit, tail rest size + content, badmatch on size mismatch, `=:=` equality, var-driven construction. Total suite 477/477. @@ -131,4 +249,10 @@ _Newest first._ ## Blockers -- _(none yet)_ +- **Phase 10a — opcode emission requires `lib/compiler.sx` (out of scope)** (2026-05-15). Architecture fully traced this iteration: the OCaml JIT (`sx_vm.ml` `jit_compile_lambda`, ref-set at line 1206) invokes the SX-level `compile` from **`lib/compiler.sx`** via the CEK machine; that is the sole SX→bytecode producer. Erlang's hot helpers (`er-match-tuple`, `er-bif-*`, …) are SX functions in `transpile.sx` that get JIT-compiled through this path. To emit `erlang.OP_*` they must be recognized as intrinsics inside `compiler.sx`'s `compile-call` (the file's own docstring already anticipates this: "Compilers call `extension-opcode-id` to emit extension opcodes" — designed, not yet implemented). `lib/compiler.sx` is **lib-root**, excluded by the ground rules ("Don't edit lib/ root") and absent from the widened `lib/erlang/** + hosts/ocaml/** (extension only)` scope — editing it changes every guest language's JIT, so it must be owned by a shared-compiler session, not this loop. **Fix path:** that session implements 10a.1 (intrinsic registry in `compiler.sx`) + 10a.2 (`compile-call` emits the opcode when registered & `extension-opcode-id` non-nil, else generic CALL). Erlang's BIF handlers (10b, ids 230-239, all real) light up the instant emission exists — zero further work here. The control opcodes (222-229) additionally need 10a.3 (operand contract) + OCaml↔SX runtime-state bridging (Erlang scheduler/mailbox live in `lib/erlang/runtime.sx`, not OCaml). + +- **Phase 9g — Perf bench gated on 9a** (2026-05-14). The conformance half of 9g (709/709 with stub VM loaded) is satisfied; the perf-bench half requires 9a's bytecode compiler to actually emit the new opcodes at hot call sites. Until then a benchmark would measure today's `er-bif-*` / `er-match-*` numbers unchanged (since the stub handlers wrap them 1-to-1). Re-fire 9g after 9a lands. + +- **Phase 9a — Opcode extension mechanism** — **RESOLVED 2026-05-15.** User widened scope to include hosts/ (merging back anyhow). Cherry-picked vm-ext phases A-E + force-linked `Sx_vm_extensions` into sx_server.exe. `extension-opcode-id` live; conformance 709/709. Remaining integration work (erlang_ext.ml + wiring the SX stub dispatcher to consult real ids) tracked as ordinary in-scope checkboxes now, not blockers. + +- **SX runtime lacks platform primitives for crypto / dir-listing / HTTP / SQLite** (2026-05-14). Probed in `mcp_tree.exe`'s embedded `sx_server.exe`: `(sha256 "x")`, `(blake3 "x")`, `(hash "sha256" "x")`, `(file-list-dir "plans")`, `(http-get "url")`, `(fetch "url")` all return `Undefined symbol`. Only file-byte-level primitives exist: `file-read` ✓, `file-write` ✓, `file-delete` ✓, `file-exists?` ✓. Out-of-scope to add these (they live in `hosts/` per ground rules). Blocked Phase 8 BIFs: `crypto:hash/2`, `cid:from_bytes/1`, `cid:to_string/1`, `file:list_dir/1`, `httpc:request/4`, `sqlite:open/exec/query/close`. **Fix path:** a future iteration on the architecture branch can register host primitives (e.g. expose OCaml's `Digestif` for hashes, `Sys.readdir` for list_dir, `cohttp` for httpc); the BIF wrappers here will then become one-line registrations against `er-bif-registry`. diff --git a/plans/fed-sx-design.md b/plans/fed-sx-design.md new file mode 100644 index 00000000..62e811d1 --- /dev/null +++ b/plans/fed-sx-design.md @@ -0,0 +1,2638 @@ +# fed-sx — Federated SX Activity Substrate + +A federated, content-addressed, extensible application substrate where the unit of +computation is a signed activity, the unit of state is a pure SX projection over the +activity log, and the substrate's own extensibility (new verbs, new object types, new +projections, new validators) is itself published through the same mechanism. + +Status: **design** — not yet implemented. Target subdomain: `next.rose-ash.com`. +Target location in repo: `next/` (new top-level dir, sibling to `blog/`, `market/`, +etc.). Stack: pure SX-on-OCaml. Implementation language(s) to be chosen after design +is complete. + +--- + +## 1. Premise + +ActivityPub's data model — actors, signed activities, inboxes/outboxes — generalises +beyond social posting to any domain where state evolves via signed messages. fed-sx +takes that generalisation seriously: + +- The unit of communication is a **signed AP activity**. +- The unit of content is an **AP object**, content-addressed by **CID** (multihash + + multicodec, default `dag-cbor` over the parsed SX AST). +- State is the **deterministic fold** of pure SX functions over the activity log. +- The substrate is **self-extending**: new activity types, object types, projections, + validators, codecs, transports, and signature suites are themselves published as + `Define*` activities — federated like any other content. + +Three commitments make the rest fall into place: + +1. **The kernel is dumb.** It only knows envelope shape, signature verification, + append-to-log, fetch-by-id, transport in/out. It does not know what `Create` or + `Pin` *mean*. +2. **Everything else is registry-driven.** Verbs, object types, validators, projections, + codecs, transports, audiences, proofs, sig suites — all looked up in registries the + kernel calls into. +3. **The registries are themselves publishable.** New entries arrive as `Define*` + activities. Bootstrap registries load from a known set of CIDs at startup; everything + else is replayed from the log. + +Result: the only code that ever needs to change in the kernel is the envelope itself. +New verbs = published SX, federated like any other artifact. + +--- + +## 2. CIDs and content addressing + +Every artifact has a CID. Default codec is **dag-cbor** over the parsed SX AST (not +the raw text). This buys: + +- **Sub-AST addressing for free.** Each nested structure has an implicit CID; IPLD can + walk paths like `/components/card`. The "file CID *and* component CID" + question dissolves: every node is a CID, you choose the granularity at reference + time. +- **Polyglot canonicalization.** JS, OCaml, Python only need to agree on AST shape + + CBOR's deterministic encoding (RFC 8949 §4.2.1). No byte-identical pretty-printer + required across hosts. +- **Format immunity.** Reformatting, indent changes, equivalent-form normalisations + do not change the CID. +- **Tooling fit.** sx-tree already has the parsed form in memory; computing or + verifying a CID is just an encode + hash. + +Costs accepted: +- One spec to maintain: SX↔CBOR mapping (number → CBOR int/float, string → text, + symbol → tag, keyword → tag, list → array, dict → map). ~50 lines of code per host. +- Author's exact source text is not preserved; re-pretty-print on fetch. +- "Why don't these CIDs match" requires comparing CBOR (a `cid-explain` tool helps). + +The CID format itself is multicodec-agile: the substrate also accepts `raw`, +`dag-json`, `dag-pb`, etc. when seen, dispatched via the codec registry. + +--- + +## 3. Kernel surface (fixed — get this right) + +The kernel is the only thing that's hard to change later. Everything else is in +registries. Two envelope shapes plus five operations. + +### 3.1 Activity envelope + +``` +{ id, type, actor, published, + to, cc, audience-extras, + object | target | origin | result, # AP slots, opaque to kernel + capabilities-required: [...], # so receivers can refuse cleanly + proofs: [...], # OTS, on-chain, multi-sig — all opaque + signature: { key-id, algorithm, value, covered-fields } } +``` + +### 3.2 Object envelope + +``` +{ id, type, cid, media-type, + where: inline | cid | url, + content?, link? } # only one populated based on `where` +``` + +### 3.3 Kernel verbs + +The only verbs implemented directly by the kernel: + +- **Append signed activity** to outbox (after envelope check + sig verify + validator + pipeline). +- **Verify signature** against actor's published keys, time-aware (which key was + active at `published`). +- **Fetch** by `id` or by `cid`. +- **Receive at inbox** (verify + dispatch to registered handlers). +- **Replay log** to rebuild registries on boot. + +Everything else is registry-resolved. + +--- + +## 4. Registries + +Each registry has a default-populated set (loaded from genesis-bundled CIDs) and +accepts new entries via `Define*` activities. Default entries themselves are SX +artifacts — versioning, audit, replacement work the same way as user content. + +| Registry | Bootstrap defaults | Extended by | +|----------|-------------------|-------------| +| **Activity types** | `Create`, `Update`, `Delete`, `Announce` | `DefineActivity{type, schema-sx, semantics-sx}` | +| **Object types** | `SXArtifact`, `Note`, `Image`, `Tombstone` | `DefineObject{type, schema-sx, render-hint}` | +| **Validators** | envelope shape, signature, type-schema | `DefineValidator{applies-to, predicate-sx}` | +| **Projections** | identity, by-type, by-cid, by-actor, actor-state, define-registry, audience-graph, by-object | `DefineProjection{name, fold-sx, query-sx}` | +| **Codecs** | dag-cbor, raw, dag-json | `DefineCodec{multicodec, encode-sx, decode-sx}` | +| **Hash algorithms** | sha2-256 | multihash table — agile by spec | +| **Transports** | http-inbox-push | `DefineTransport{name, deliver-sx, receive-sx}` | +| **Audience predicates** | `Public`, `Followers`, direct | `DefineAudience{name, member-of-sx}` | +| **Subscription types** | `Follow` (AP-standard) | `DefineSubscription{name, schema-sx, match-sx, delivery}` | +| **Proof types** | (none) | `DefineProof{type, attach-sx, verify-sx}` | +| **Storage backends** | files-on-disk | `DefineStorage{where-tag, put-sx, get-sx}` | +| **Triggers** | (none) | `DefineTrigger{when-subscription, then-sx, cascade-limit}` | +| **Signature suites** | rsa-sha256 (AP-compatible) | `DefineSigSuite{name, sign-sx, verify-sx}` | +| **Application bundles** | (none) | `DefineApplication{name, subscriptions, triggers, projections, storage}` | + +Adding `Pin`, `Endorse`, `Supersede`, `Test`, `Build`, `Compose`, etc. later is just +publishing `DefineActivity` artifacts — no kernel diff, no redeploy required if +registries are hot. + +--- + +## 5. The meta-level + +A `DefineActivity` is itself an AP `Create` activity over an `SXArtifact` of a +specific type: + +```sx +(activity 'Create + :object {:type "DefineActivity" + :name "Pin" + :schema (fn (act) + (and (string? (-> act :object :path)) + (cid? (-> act :object :cid)))) + :semantics + '(fn (act state) + (assoc-in state [:pins (-> act :object :path)] + (-> act :object :cid)))}) +``` + +When the kernel receives an activity with `type: "Pin"` it looks up the registered +semantics from a `DefineActivity{name: "Pin"}` artifact, runs the SX, projects the new +state. The semantics are themselves content-addressed and federated — every receiver +runs the same code. + +Same pattern handles `DefineProjection`, `DefineValidator`, etc. The substrate is +genuinely self-extending. + +--- + +## 6. Verbs + +### 6.1 Bootstrap verbs (milestone 1) + +The substrate exposes `POST /activity` (not `POST /publish`) — generalised entry +point that takes any well-formed AP activity, validates, signs, appends to outbox. +`(publish sx)` is sugar at the SX layer for `Create{SXArtifact}`. + +Day-one verbs (cost ~zero once `/activity` exists): + +- **`Create`** — the publish primitive. +- **`Update`** — supersede a previous activity (correct metadata, change a path + mapping). Distinct from "publishing new content" — new content is always a new + `Create` with a new CID. +- **`Delete`** — tombstone. AP-native; readers honour it. +- **`Announce`** — boost another actor's artifact into your outbox. Comes free. +- **`Subscribe`** — generalised subscription verb (parallel to publish/`Create`). + Wraps any registered `DefineSubscription` type. `Follow` is the standard AP + `Subscribe{Follow{actor: ...}}` for wire compatibility. See §18. +- **`Unsubscribe`** — `Undo` of a prior `Subscribe`. Same shape as AP + `Undo{Follow}`. + +### 6.2 Custom verbs (designed-for, defined later) + +Substrate accepts these from day one (any signed activity can be appended); semantics +projected once `DefineActivity` artifacts exist. + +- **`Pin`** — assign `domain:path/name → CID`. The future name-resolution layer made + of activities. Each pin is signed; the resolver replays the outbox to compute current + state. +- **`Endorse`** (modelled on `Like`/`Approve`) — third-party signature on a CID. + Web-of-trust style code review without central authority. +- **`Supersede`** — "CID A replaces CID B". Stronger than `Update`; readers can chase + the chain. +- **`Test`** — published assertion that running CID A under conditions X yields result + Y. Test-as-artifact, federated. +- **`Build`** — links a source CID to a compiled-output CID, with provenance. +- **`Compose`** — derived artifact citing input CIDs. Provenance graph in the outbox + itself. +- **`Note`** (AP-native) — comments / reviews / discussion attached to a CID. +- **`Follow`** / **`Undo(Follow)`** — subscribe to another instance's outbox. + +The pattern that matters: your outbox isn't just "things published," it's an +**append-only log of every assertion this actor makes about the SX universe.** + +--- + +## 7. Capability discovery + +Two pieces: + +- **`GET /.well-known/sx-capabilities`** — JSON listing every registered activity-type, + object-type, codec, transport, sig-suite, proof-type. Each with the CID of the + `Define*` artifact that introduced it. Peers can diff capabilities before federating. +- **`capabilities-required`** field on activities — sender declares "this needs `Pin` + semantics + `dag-cbor` codec." Receivers without those capabilities return a clean + 422 referencing the missing CIDs; sender knows whether to replay-and-deliver the + bootstrapping `Define*` artifacts first. + +Federation degrades gracefully across instances at different versions. + +--- + +## 8. Axes of flexibility (all designed-for) + +1. **Object types** beyond SXArtifact — `Note`, `Article`, `Image`, `Video`, `Question`, + `Event`, etc. via the object-type registry. +2. **Storage tier per-object** — `where: inline | cid | url`. Tiny things inline; big + things to IPFS; legacy stuff URL-linked. Migrating storage backends doesn't migrate + the substrate. +3. **Multihash + multicodec agility** — sha2-256 + dag-cbor by default; substrate + accepts blake3, raw, dag-json, dag-pb, etc. +4. **Multi-key actors** — `publicKeys` array always; per-key `purpose`; multiple key + types (RSA for AP wire compat, Ed25519 modern). See §9. +5. **Audience / visibility** — AP-native `to`, `cc`, `bto`, `bcc`. Public, followers, + direct, unlisted. Custom audiences via `DefineAudience`. +6. **Outbox-as-database** — no source-of-truth other than the log. Projections are + recomputable views. +7. **Programmable activities** — activities can carry SX. Reactive federation, + conditional pins, automated propose/test/release pipelines, all expressed as AP + activities. +8. **Federation transport pluggable** — outbox is canonical; how peers exchange is + pluggable (HTTP push, pull, libp2p, polling). +9. **Optional timestamp proofs** — every activity has an attachable `proofs` slot. + OpenTimestamps, on-chain merkle commit, third-party TSA all slot in without changing + activity semantics. + +Explicitly **not** pursuing for MVP: +- Schema-version negotiation (premature; `@context` handles extension). +- Configurable conflict-resolution per actor (last-signed-wins, log preserved for + audit). +- Verb-specific kernel handlers (other than `Create`'s "compute CID, store body"). + +--- + +## 9. Identity & actor lifecycle + +### 9.1 Actor doc shape + +```jsonld +{ + "@context": ["https://www.w3.org/ns/activitystreams", + "https://w3id.org/security/v1", + "https://next.rose-ash.com/ns/fed-sx/v1"], + "type": "Person", // or Service, Group, Application + "id": "https://next.rose-ash.com/actors/giles", + "preferredUsername": "giles", + "inbox": "https://next.rose-ash.com/actors/giles/inbox", + "outbox": "https://next.rose-ash.com/actors/giles/outbox", + "followers": "...", + "following": "...", + + "publicKeys": [ // ARRAY from day one — never `publicKey` + { "id": "...#key-2026-05", + "type": "RsaVerificationKey2018", + "owner": "", + "publicKeyPem": "...", + "purpose": ["sign-activity", "sign-http"], + "created": "2026-05-14T...", + "expires": null, + "supersedes": null, + "supersededBy": null }, + { "id": "...#key-ed25519-2026-05", + "type": "Ed25519VerificationKey2020", + "owner": "", + "publicKeyMultibase": "z6Mk...", + "purpose": ["sign-activity"], + "created": "2026-05-14T..." } + ], + + "capabilities": "https://.../actors/giles/capabilities", // what verbs they speak + "alsoKnownAs": ["did:web:rose-ash.com:giles", ...], // bridge to DID, AP migration + "movedTo": null // set on Move +} +``` + +Key shape decisions: + +- **`publicKeys` array always.** Single-key actors have an array of length 1. AP + standard `publicKey` is *also* served as the first array element for back-compat + with vanilla AP servers (Mastodon etc. ignore the array). +- **Per-key `purpose`** — separates signing weight. Day-to-day publish key vs. high- + value key for `Pin`/`Endorse` vs. delegated machine key. Validators can require + specific purposes per activity type (registry-driven). +- **Multiple key types** — RSA for AP wire compat, Ed25519 for everything else + (smaller, faster, modern). Sig suite registry decides which suites are accepted. +- **`supersedes` / `supersededBy`** — keys form a chain, not a snapshot. Old activities + still verify against historical keys. + +### 9.2 Key rotation + +Key rotation is itself an activity, signed by the *old* key (or a recovery key): + +```sx +(activity 'Update + :object actor-id + :patch {:add-publicKey new-key + :supersede {old-key-id new-key-id}}) +``` + +Kernel: +1. Fetches actor's current state (a projection over their own outbox). +2. Verifies activity is signed by a key with `purpose: rotate-key` (or any active key, + if registry allows). +3. Appends. The actor-state projection now has the new key. + +Old activities still verify because the projection retains the historical key with +`supersededBy` set — sig verification looks up "what keys were active at activity +timestamp T." + +### 9.3 Key recovery / loss + +- **Recovery key** — separate key at actor creation, never used except to rotate. + Stored offline. `purpose: ["recover"]`. Validator allows + `Update{actor, patch: rotate-all-keys}` if signed by a recovery key. +- **Social recovery** — designate N trusted actors, M-of-N can co-sign a recovery + `Update`. Implemented as a `DefineValidator` extension; multi-sig slot in `proofs` + makes it possible without changing the envelope. +- **Total loss** — if both signing and recovery keys are gone, the actor is dead. + They publish a new actor with `alsoKnownAs: ` from a fresh key. + Followers can choose to re-follow but there's no cryptographic continuity. + +### 9.4 Migration (`Move`) + +AP-native: + +```sx +(activity 'Move + :object old-actor-id + :target new-actor-id) +``` + +Receivers update their follow lists. New actor's `alsoKnownAs` must include old +actor — bidirectional handshake prevents hijacking. + +For fed-sx, `Move` should also carry an outbox migration hint (CID of an export bundle) +so receivers can re-anchor projections without re-fetching activity-by-activity. + +### 9.5 Subordinate actors / delegation + +Two patterns supported: + +- **Service actors** (AP-native `type: Service`): bots, build servers, test runners. + Their own keys, their own outboxes, but `attributedTo` a parent actor. +- **Capability tokens**: parent publishes `Authorize{actor: child, capabilities: [...], + expires: ...}` signed by parent. Child publishes activities normally with their own + key; receivers verify the capability chain when child invokes an authority they don't + own outright. Useful for: temporary publish access, delegated `Pin` rights for a + specific path prefix, multi-device. + +Both work *without* new kernel mechanism — just activities. + +### 9.6 Implications + +- **Sig verification is timestamp-aware.** Verifying an old activity needs the key + state at the time it was published — actor-state projection must support time-travel + queries. +- **Inbox doesn't trust `keyId` blindly.** Fetches actor doc, projects current key + state, checks key was valid at `published`. +- **Cross-instance identity via `alsoKnownAs` and DIDs.** Don't depend on DIDs but + slot them in for Bluesky-bridge, Solid-bridge, etc. + +--- + +## 10. Projection model + +The architectural commitment: **state is what you get when you fold pure SX over the +log.** No DB-of-record. Everything queryable is a projection. + +### 10.1 What a projection is + +A `DefineProjection` activity registers four things: + +```sx +(activity 'Create + :object {:type "DefineProjection" + :name "actor-state" + :initial-state {} ; pure SX value + :fold (fn (state activity) ; pure SX + (case (:type activity) + "Create" (when (= "Person" (-> activity :object :type)) + (assoc state (:id activity) (:object activity))) + "Update" (apply-patch state activity) + "Move" (set-moved state activity) + state)) + :snapshot-codec "dag-cbor" + :indexes [{:by :id} {:by :preferredUsername}]}) +``` + +- **`name`** — query handle. Unique per actor; collisions resolved by CID + supersession. +- **`initial-state`** — pure SX value used as state-zero. +- **`fold`** — pure SX function `(state activity) → state`. The only thing the kernel + calls. +- **`indexes`** — optional hint for materializing lookup paths. + +The CID of the `DefineProjection` artifact is the projection's identity. Two instances +running the same projection are running the same CID's `fold` over the same log slice +— equivalence is decidable. + +### 10.2 The fold contract — purity, determinism, gas + +The fold function must be **pure and deterministic**. Non-negotiable; it's what makes +cross-instance equivalence and replay possible. + +- **No IO.** No HTTP, no file access, no DB calls, no clock. The activity carries its + own `published` timestamp. +- **No randomness.** No host-seeded PRNG. (If pseudo-randomness is needed, seed from + the activity's CID — deterministic across hosts.) +- **No mutation outside the returned state.** +- **Bounded execution.** Each fold call gets a gas budget (default tunable, e.g. 100k + CEK steps). Exceeding it is a hard failure. + +Enforced at the SX evaluator level by running folds in a sandboxed environment with +the IO platform stripped to nothing. Same sandbox model applies to validators and +trigger semantics. + +**Cross-host equivalence guarantee:** for the same projection CID + same activity log +slice, every conforming SX host (JS, OCaml, Python, Haskell-on-SX, …) must produce a +state value with the same canonical CID. Tested via the spec test suite. + +### 10.3 Bootstrap projections + +The kernel cannot start without some projections, because the kernel itself uses them. +Baked into the genesis bundle (see §11), superseded only by deliberate kernel-version +upgrades. + +| Projection | What it computes | Used by | +|------------|------------------|---------| +| `activity-log` | Identity — every activity, indexed by id and CID | Everything | +| `by-type` | `type → ordered list of activity-CIDs` | Most queries | +| `by-actor` | `actor-id → ordered list of activity-CIDs` | Per-actor outbox view | +| `by-object` | `object-CID → list of referencing activity-CIDs` | "Who pinned this?" | +| `actor-state` | `actor-id → current actor doc with key history` | Sig verification (kernel) | +| `define-registry` | `kind+name → currently-active Define* CID` | All other Define* lookups | +| `audience-graph` | `actor → followers/following` | Federation push | + +`define-registry` is the bootstrap chicken-and-egg: it's the projection that knows +which projections (and validators, codecs, etc.) are currently active. Kernel ships +with it hardcoded; once running, every other projection (including a future replacement +of `define-registry` itself) is a regular `DefineProjection` superseding it. + +### 10.4 Snapshotting + +Replaying the entire log on every restart is unacceptable past day one. + +- **Snapshot = `(activity-tip-CID, projection-state, projection-CID)` tuple,** + dag-cbor encoded, content-addressed. +- **Snapshot rule** — every K activities (default 1000) and every T seconds (default + 60), serialize, hash, store on disk. +- **Resume** — on startup, find latest snapshot for each (projection-CID, log-tip), + load state, fold forward. +- **Snapshot CID is verifiable** — anyone with the same log slice and projection-CID + can recompute and check the CID matches. This is the cross-instance agreement proof. + +Snapshots are themselves publishable as activities (`Create{Snapshot}`): an instance +can publish "here's my computed state for projection X at log-tip Y, CID Z." Other +instances can fetch and use as a starting point. **Federated state sharing falls out of +federated activities.** + +Snapshots are pruning-friendly: keep latest + snapshots referenced by published +`Create{Snapshot}` activities; everything else is GC-able. + +### 10.5 Reprojection on definition change + +When `DefineProjection{name: "actor-state"}` is superseded by a new CID with a +different fold: + +1. `define-registry` projection sees the supersession; its state advances. +2. New projection materialized **alongside** the old one — both kept live during + migration. +3. New projection runs in catch-up mode: replay from genesis (or from deepest + compatible snapshot). +4. When new projection catches up to log tip, queries cut over. Old projection state + can be retired. +5. Snapshots of old version stay around as long as referenced (e.g. for time-travel + queries against historical state under old semantics). + +Changing a projection definition is **safe and online**. Cost: temporary state +duplication during catch-up. Slow folds → slow migrations, but never breakage. + +For projections too expensive to fully reproject, `Update{DefineProjection}` can +declare `migrationHint: ` — opt-in, used at migrator's +risk. + +### 10.6 Time-travel queries + +Folds are deterministic functions of `(initial-state, activity-list-prefix)`. +Time-travel is fold-up-to: + +- `state-as-of(projection, activity-id-or-timestamp)` → walk to requested point, + return state. +- Snapshots act as accelerators (resume from nearest snapshot ≤ target). +- Used by sig verification ("what keys did this actor have when this activity was + signed?"), audit, "what did we believe last Tuesday." + +### 10.7 Projection composition + +**Projections do not directly read each other's state during folding.** Preserves +locality and parallelism — every projection runs independently against the same log. + +Composition via: + +- **Query time** — `(query (projection actor-state) ...)` joins are SX expressions + over multiple projection states. +- **Republishing as activities** — a projection that exposes its state as input to + others publishes `Create{Snapshot}` periodically. Downstream projections fold over + those. + +Direct cross-projection reads during fold introduce ordering, cycles, cache- +invalidation problems we don't need. + +### 10.8 Querying + +Three layers: + +- **Raw projection state** — `GET /projections/?at=` returns dag-cbor + (also JSON for tooling). Large states paginated by index. +- **SX queries** — `POST /query` with an SX expression that runs against one or more + projection states in pure mode. Equivalent to Datalog/GraphQL. +- **Materialized indexes** — declared on projection (`indexes:` field). Kernel + maintains as side-tables for `O(log n)` lookup. + +Real-time: clients `GET /projections//subscribe` (SSE), receive deltas as +activities land. Delta is `(old-state, new-state, applied-activity-CID)`; clients can +verify by re-folding. + +### 10.9 Lag, async, concurrency + +- **Append is sync; projection is async.** `POST /activity` returns once activity is + durably in the log. Projections run in a separate worker pool; query results carry + `projected-up-to` so callers know whether the latest write is visible. +- **One worker per projection.** Folds are sequential, but projections run in parallel + with each other. +- **Sync option** — `POST /activity?wait-for=projection-name` blocks until the named + projection has folded the new activity. Use sparingly. + +### 10.10 Failure modes + +| Failure | Response | +|---------|----------| +| **Gas exhaustion** | Activity tagged `projection-failed` for this projection. State unchanged. Operator alert. | +| **SX runtime error** (assertion, type mismatch) | Same as gas: activity skipped, error logged, state unchanged. | +| **Schema violation** | Caught earlier in validation pipeline, never reaches projection. | + +The log itself is always written successfully if it passes envelope + signature + +validator checks. Projection failures don't gate appending — that would couple writes +to arbitrary user-defined code. + +### 10.11 Operational implications + +- **Projection determinism is the linchpin.** If JS and OCaml ever produce different + state for the same log + projection, federation cracks. Spec test suite must cover + projection equivalence across hosts as a first-class requirement. +- **Snapshots are eventual consensus.** Two instances publish `Create{Snapshot}` for + the same log+projection; if their CIDs match, they agree without coordination. +- **Kernel reads its own projections.** `actor-state` for sig verification; + `define-registry` for every Define* lookup. Startup sequence must bootstrap these + before serving traffic. +- **Reprojection cost is real.** Heavy projection changes mean replaying from genesis. + Encourage incremental schemas (small per-activity work, idempotent updates) and + provide profiling. + +--- + +## 11. Sandbox & determinism + +The runtime contract that makes folds (and validators, triggers, semantics) safe to +execute, and that guarantees every conforming SX host computes the same state from +the same log. + +### 11.1 Three sandbox levels + +Different registry entries need different power. We define three nested execution +modes; the registry entry declares which mode it requires. + +| Mode | Used by | IO | Clock | Random | Determinism | +|------|---------|----|----|--------|-------------| +| **pure** | folds, validators, audience predicates, semantics, trigger `when-sx` | none | activity's own `published` only | seeded from activity CID only | required across hosts | +| **crypto** | sig suite verify, codec encode/decode | crypto primitives only | none | sign-only secure RNG | required across hosts (verify); single-host (sign) | +| **effectful** | storage backends, transports, trigger `then-sx`, some proof verifiers | per-capability grant only | host clock | host RNG | not required; single-host | + +Default mode is **pure**. The other two are opt-in at registration time, and the +registration is itself a signed activity — anyone can audit which extensions claim +which powers. + +### 11.2 Pure sandbox (the load-bearing one) + +This is the mode every projection fold runs in. It must produce identical results on +every conforming SX host, every time. + +**Allowed:** +- All spec primitives in `spec/primitives.sx` that don't perform IO (arithmetic, + comparison, predicates, string ops, collection ops, dict ops, format helpers). +- The activity being processed (full envelope), as the function's argument. +- The current state value, as the function's argument. +- A small set of fed-sx-specific deterministic primitives: + - `(activity-cid act)` → CID of the activity envelope + - `(activity-time act)` → ISO timestamp from `published` + - `(actor-state-as-of state-snapshot actor-id activity-time)` → if the projection + has been declared dependent on `actor-state` (see §10.7), reads from a snapshot + of that projection at the activity's timestamp + - `(seeded-rng cid)` → deterministic PRNG seeded from a CID, returns a stream of + uniform values + +**Forbidden:** +- All IO: HTTP, file, network, stdin/stdout, environment. +- Wall-clock access. The host's `now` is not in scope; the only time available is + `(activity-time act)`. +- Host-seeded randomness. Only `seeded-rng` (CID-derived) is available. +- Mutation outside the returned value. Enforced by the SX evaluator's lack of + ambient mutable bindings; folds may use local `let` and mutation within their own + closure but cannot reach outside. +- Calling other registry entries by name. Composition happens at query time, not + fold time (see §10.7). + +**Enforced by:** evaluator runs the fold with the IO platform stripped to nothing. +The fed-sx kernel constructs a `pure-platform` (no fetch, no query, no action, no +DOM, no storage) and uses it as the sole evaluator platform when calling the fold. +Any IO primitive call raises a hard error caught as a fold failure. + +### 11.3 Crypto sandbox + +Sig suites and codec encode/decode need hash + crypto + encoding primitives but +nothing else. They're still deterministic across hosts (verify case) but get a +narrower platform than effectful, wider than pure. + +**Additional primitives over pure:** +- `(sha2-256 bytes)`, `(sha3-256 bytes)`, `(blake3 bytes)`, … +- `(rsa-verify pubkey msg sig)`, `(ed25519-verify pubkey msg sig)`, … +- `(rsa-sign privkey msg)`, `(ed25519-sign privkey msg)` — sign-only; requires the + caller to supply a secure RNG handle (which is *not* in pure mode) +- `(cbor-encode value)`, `(cbor-decode bytes)` — for codecs implementing CBOR variants +- `(base32-encode bytes)`, `(base58btc-encode bytes)`, `(multibase-encode tag bytes)` +- `(multihash-encode tag digest-bytes)`, `(multihash-decode bytes)` +- `(cid-encode codec mhash)`, `(cid-decode bytes)` + +**Sign vs verify:** verify is pure (deterministic). Sign is not — it consumes +randomness. fed-sx draws a clean line: signing happens *outside* registry-entry SX +(it's an operation the kernel/runtime performs on behalf of the actor with their +private key); registry SX only ever *verifies*. This keeps the pure↔crypto distinction +tractable. + +### 11.4 Effectful sandbox + +Storage backends, transports, trigger `then-sx`, and proof verifiers that need the +network (e.g. blockchain RPC for on-chain proof verification) all need real IO. +These are not used to compute projected state; they're how the substrate interacts +with the outside world. + +**Capability-granted primitives.** The registration activity declares the +capabilities the entry needs: + +```sx +(activity 'Create + :object {:type "DefineStorage" + :where-tag "ipfs" + :capabilities [{:type "http-client" :allowlist ["http://localhost:5001/*"]} + {:type "fs-read" :path-prefix "/var/cache/fed-sx/ipfs/"} + {:type "fs-write" :path-prefix "/var/cache/fed-sx/ipfs/"}] + :put-sx (fn (cid bytes) ...) + :get-sx (fn (cid) ...)}) +``` + +**Capability types** (initial set; extensible): + +- `http-client` with `allowlist` (URL prefix patterns) +- `http-server` with `path-prefix` (mounts a sub-handler) +- `fs-read` / `fs-write` with `path-prefix` (chroot-style) +- `subprocess` with `command-allowlist` +- `clock-read` (wall clock; granted if registry entry needs to timestamp something) +- `random-bytes` (host CSPRNG) + +**No ambient authority.** Default capability set is empty; every capability is +explicit, declared, signed, and auditable. A peer can refuse to load a registry +entry whose capability claim is unacceptable to them. + +**Capabilities are content-addressed.** Each capability descriptor has a CID. The +substrate maintains a registry of "capability CIDs that this instance trusts to +honour" — operator policy, not protocol. + +### 11.5 Gas and resource accounting + +Each sandbox call gets a budget: + +- **CEK gas** — every evaluator step costs 1 unit; primitive calls cost a per- + primitive amount declared in `spec/primitives.sx`. Default budget: 100k units per + fold call. Tunable per-projection via `DefineProjection.gas-limit`. +- **Memory ceiling** — peak heap size for the fold call. Default 64 MB. Tunable. +- **IO budget** (effectful only) — bytes read/written and network calls per + invocation, granted separately per capability. +- **Wall-clock budget** (effectful only) — max real-time before forced termination. + +Exceeding any budget is a hard failure; the call returns an error value, the fold's +state is unchanged, and the activity is tagged for the projection. + +Gas accounting is part of the spec — every conforming host must charge the same +units for the same operations, so "this fold runs out of gas" is a deterministic +property of the (projection, activity) pair, not a host-specific outcome. + +### 11.6 Determinism gotchas + +The pure sandbox is only as deterministic as its primitives. Worth nailing: + +- **Floating point.** IEEE 754 binary operations are bitwise-identical across + conforming hosts, but transcendentals (`sin`, `cos`, `log`, `exp`) are *not* — + libm implementations differ. **Decision: floats are forbidden in pure mode unless + the projection declares `requires-deterministic-floats: true` and uses only the + IEEE 754 basic operations (+, -, *, /, sqrt, comparison, conversion).** For exact + arithmetic, use integers or rationals (fed-sx will provide a rational primitive). +- **Map / dict iteration order.** Must be sorted-key always in pure mode. The SX + spec mandates this for `for-each` and `map` over dicts; we tighten it: pure mode + forbids relying on insertion order. +- **String encoding.** All strings are UTF-8 NFC at ingestion; pure-mode operations + use byte-level comparison after normalization. Codepoint operations (`length`, + `substring`) return identical results across hosts because they operate on the + normalized form. +- **Integer overflow.** Pure mode uses arbitrary-precision integers (the SX spec + default). No undefined behaviour. Overflow is impossible. +- **Equality.** Structural equality (`equal?`) compared across hosts must yield the + same result for the same canonical-CID values. Implies dict equality is + order-independent (as it should be), and float equality follows IEEE 754 (NaN ≠ + NaN; +0.0 = -0.0). +- **Error values.** When a primitive errors, the error must be representable as a + dag-cbor value with a stable CID across hosts. Reserve a `{:error :type ... :msg + ...}` shape; standard error types defined in the spec. + +### 11.7 Failure model + +A pure-mode call ends in one of three terminal states: + +1. **Success** — returns a value. Fold uses it as new state. +2. **Sandbox violation** — IO attempted, capability denied, etc. Returns a stable + error value; fold's state is unchanged; activity tagged + `{:projection-failed :reason :sandbox-violation :detail ...}`. +3. **Resource exhaustion** — gas, memory, IO budget exceeded. Same handling as + sandbox violation but with `:reason :resource-exhausted`. + +Crypto-mode failures (e.g. invalid signature) are *return values*, not exceptions — +verify returns boolean, sign returns either a sig or an error. This forces callers +to handle failure explicitly. + +Effectful-mode failures (network down, disk full) propagate to the operator as +errors but never affect projected state. The substrate retries effectful operations +according to the registry entry's policy (declared at registration). + +### 11.8 Conformance testing + +Cross-host equivalence isn't aspirational; it's tested. + +- **Spec test suite** ships projection equivalence tests: a corpus of (log slice, + projection CID, expected snapshot CID) tuples. Every conforming SX host must + produce the expected snapshot CID for each input. +- **Validator equivalence tests** likewise: (validator CID, activity, expected + result). +- **Codec equivalence tests:** (codec CID, value, expected encoded bytes), in both + encode and decode directions. +- **Sandbox isolation tests:** "this fold attempts to call `fetch`; expected + outcome: sandbox violation error with stable CID." + +Hosts run the conformance suite to claim "fed-sx pure-mode conformance." Failures +are publishable as `Test{result: failed, host: ..., projection: ...}` activities — +the conformance graph itself is federated. + +### 11.9 Operational implications + +- **The pure sandbox is the heart of cross-host federation.** Every divergence is a + spec bug or a host bug; both are caught by snapshot CID mismatches and surfaced + via `Test` activities. +- **Capability descriptors are the new audit trail.** "What can the IPFS storage + backend do?" is a question with a precise answer at any timestamp — the registered + capability CIDs. +- **Floats are mostly absent.** This is unusual but defensible — most state in the + substrate is ids, counts, sets, references. Numerical computation belongs in + effectful registry entries (e.g. an analytics projection that publishes summaries + as activities, projected by a downstream pure projection that just stores them). +- **Gas is part of the protocol.** Two hosts disagreeing about whether a fold runs + out of gas is a conformance failure. Spec primitive gas costs are normative. + +## 12. Bootstrap & genesis + +How a fresh instance starts with no log, where the initial registry entries come +from, and how the kernel evolves without bricking peers. + +### 12.1 The genesis problem + +The substrate is "everything is a `Define*` activity in the log." But on a fresh +instance the log is empty — so there are no `Define*` activities to tell the kernel +what `Create` means, how to verify a signature, or what dag-cbor is. Strict +turtles-all-the-way-down would deadlock startup. + +Solution: **the kernel ships with a baked-in genesis bundle** containing the minimal +set of definitions it needs to interpret its own log. The bundle is a constant of +the kernel binary; its CID is hardcoded; the kernel verifies on startup that the +bundle matches its hardcoded CID. After that, everything (including superseding the +bundled definitions themselves) goes through the activity log. + +The genesis bundle is *not* itself a federated artifact in the AP sense. It's the +dictionary you need before you can read any activities. Optionally, an actor can +`Create{GenesisRecord}` as their first published activity to advertise which genesis +they started from — informational, not load-bearing. + +### 12.2 Genesis bundle contents + +Minimal viable bundle (dag-cbor object, content-addressed): + +``` +{ + "type": "fed-sx-genesis", + "kernel-version": "1.0.0", + "envelope-spec": { ... }, // canonical schema for activity envelope + "object-spec": { ... }, // canonical schema for object envelope + "definitions": { + "activity-types": { + "Create": { "schema": , "semantics": }, + "Update": { "schema": , "semantics": }, + "Delete": { "schema": , "semantics": }, + "Announce": { "schema": , "semantics": } + }, + "object-types": { + "SXArtifact": { "schema": }, + "Note": { "schema": }, + "Tombstone": { "schema": }, + "DefineActivity": { "schema": }, + "DefineObject": { "schema": }, + "DefineProjection": { "schema": }, + "DefineValidator": { "schema": }, + "DefineCodec": { "schema": }, + "DefineTransport": { "schema": }, + "DefineAudience": { "schema": }, + "DefineProof": { "schema": }, + "DefineStorage": { "schema": }, + "DefineTrigger": { "schema": }, + "DefineSigSuite": { "schema": }, + "Snapshot": { "schema": } + }, + "sig-suites": { + "rsa-sha256-2018": { "verify": , "key-format": }, + "ed25519-2020": { "verify": , "key-format": } + }, + "codecs": { + "dag-cbor": { "encode": , "decode": }, + "raw": { "encode": , "decode": }, + "dag-json": { "encode": , "decode": } + }, + "projections": { + "activity-log": { "initial-state": ..., "fold": }, + "by-type": { "initial-state": ..., "fold": }, + "by-actor": { "initial-state": ..., "fold": }, + "by-object": { "initial-state": ..., "fold": }, + "actor-state": { "initial-state": ..., "fold": }, + "define-registry": { "initial-state": ..., "fold": }, + "audience-graph": { "initial-state": ..., "fold": } + }, + "validators": { + "envelope-shape": { "predicate": }, + "signature": { "predicate": }, + "type-schema": { "predicate": } + }, + "audience-predicates": { + "Public": { "member-of": }, + "Followers": { "member-of": }, + "Direct": { "member-of": } + } + }, + "capability-types": [ // schema for capability descriptors + "http-client", "http-server", + "fs-read", "fs-write", + "subprocess", "clock-read", "random-bytes" + ] +} +``` + +Each definition's body is **SX source**, not bytecode. The kernel evaluates it at +startup using the same SX evaluator user-published `Define*` artifacts use — there +is no privileged "native" path. The bootstrap is just SX loaded from the binary +instead of from the log. + +### 12.3 Hardcoded CID and verification + +The kernel binary contains: + +- The full genesis bundle (embedded as bytes). +- The CID computed over those bytes at build time. + +On startup: + +1. Compute the actual CID of the embedded bundle. +2. Compare to the hardcoded CID. +3. **Mismatch → refuse to start.** Either the binary has been tampered with or the + build process is broken. Either way, the operator should know immediately. +4. **Match → proceed.** Every running instance with a given kernel binary has + byte-identical bootstrap state — no version drift possible within a binary. + +The genesis CID is exposed at `GET /.well-known/sx-capabilities` so peers can see +which kernel version they're talking to. + +### 12.4 Fresh instance startup sequence + +``` +1. Load and verify genesis bundle (panic on mismatch) +2. Parse all definition SX sources, instantiate evaluator closures +3. Initialize registries from definitions (in the order: codecs → sig-suites → + validators → object-types → activity-types → audience-predicates → projections) +4. Open log file (create if missing) +5. Replay any existing log: for each activity, validate, then fold into each + projection (resuming from snapshots where available) +6. Load or generate actor keypair (filesystem path from config) +7. If actor has never published a Create{Person} for itself, generate and append + one as the first activity of this instance's outbox +8. Initialize HTTP server, wire routes +9. Open inbox: start accepting federated activities +10. Mark instance as ready +``` + +Steps 1-3 are the bootstrap. Step 5 is replay-and-project. Step 7 is the +"actor genesis" — every instance has at least one local actor; it publishes itself +as its first activity, and that activity (signed by the actor's own key) anchors all +subsequent activity from that actor. + +### 12.5 First activity — actor creation + +Every fresh actor's outbox starts with: + +```sx +(activity 'Create + :id "https://next.rose-ash.com/actors/giles/activities/" + :actor "https://next.rose-ash.com/actors/giles" + :published "" + :to ["https://www.w3.org/ns/activitystreams#Public"] + :object + :signature ) +``` + +Self-signed: the activity introduces the key it's signed with. Verifiers fetch the +actor doc embedded in the activity, find the key, verify against the activity. This +is the trust-on-first-encounter for a new actor — the same model AP uses. + +The kernel emits this automatically on first startup if the actor has no prior +activity. Subsequent actor changes (key rotation, profile updates) are `Update` +activities signed by an existing key. + +### 12.6 Joining federation + +A new instance has no peers initially. Discovery is operator-driven for v1: + +1. Operator configures one or more peer URLs (or a well-known seed list). +2. Instance fetches peer's actor doc and `/.well-known/sx-capabilities`. +3. Instance verifies it can interpret the peer's activities (envelope compatible, + sig suites overlap). Reports incompatibilities to operator. +4. If compatible, instance follows peer's primary actor (`POST /inbox` with a + `Follow` activity). +5. Peer streams or backfills outbox to this instance. +6. Activities arrive, validate, fold into local projections. + +Discovery beyond manual config (e.g. peer recommendations, federation directories) +is a v2 concern. + +### 12.7 Kernel version evolution + +The substrate must evolve without forcing every instance to upgrade in lockstep. +Three rules: + +**Rule 1: The activity envelope shape is forward-compatible only.** + +We may *add* optional fields to the envelope; we may not change semantics or remove +fields. Old activities still validate under new kernels. New activities with new +fields are accepted by old kernels (which ignore the unknown fields, store the raw +envelope, and project conservatively). + +This is the AP discipline. We adopt it strictly. If we ever need a breaking envelope +change, it's a major version (fed-sx 2.0) and instances at different majors don't +federate directly — only via bridges. + +**Rule 2: Everything else evolves via supersession.** + +New sig suite, new codec, new projection definition, new validator: publish a +`Define*` activity that supersedes the old one. Both old and new versions stay valid +at their respective timestamps. Old activities verify under old definitions; new +activities use new definitions. Time-aware lookup (§9.6, §10.6) makes this work. + +**Rule 3: New genesis bundles supersede old ones via published activities.** + +When the kernel team ships a new version with an updated bundle: + +- The new bundle's CID is different. +- Operators upgrading the kernel get the new bundle automatically. +- The new bundle's *contents* are largely supersession `Update{DefineProjection, + DefineValidator, ...}` activities relative to the old bundle's definitions. +- A peer running the old kernel sees these `Update` activities (when they appear in + followed outboxes) and *can* opt to load them dynamically (§12.8) or stay on the + old bundle definitions until the operator upgrades. + +In other words: the kernel binary evolution and the activity-log evolution are +parallel tracks. The binary determines what's *built in*; the log determines what's +*currently active*. They converge over time but don't have to be lockstep. + +### 12.8 Dynamic Define* loading + +When an instance receives an activity of `type: "PinV3"` and has no `DefineActivity{ +name: "PinV3"}` in its define-registry, it has three options (operator policy): + +- **Strict mode** — store the activity envelope (it's valid AP), tag it `unknown-type` + in `by-type`, do not project semantics. Operator must explicitly load the + definition to enable projection. +- **Permissive mode** — fetch the `DefineActivity{name: "PinV3"}` artifact (its CID + is in the activity's `capabilities-required` list), validate, evaluate the + semantics SX (in pure sandbox), reproject the activity. Operator notified. +- **Trusted-peers-only mode** — like permissive, but only auto-loads `Define*` from + actors on a configured trust list. + +Default for fed-sx v1: **strict mode**. Operators opt-in to broader policies. + +This lets the substrate genuinely live-extend — new verbs land via federation, no +binary upgrade — while keeping a clean audit trail of what got loaded when. + +### 12.9 Genesis as the substrate's manifest + +A useful framing: the genesis bundle is the substrate's **manifest** (in the package- +manager sense). It declares "this kernel ships with these definitions, identified by +these CIDs, and this is what the kernel does until the log says otherwise." + +Two instances with the same genesis CID start identical. Two instances with +different genesis CIDs can federate as long as their *active* registry states (after +log replay) overlap enough. + +The genesis bundle is also the **conformance reference**: a kernel implementation +claims fed-sx v1.0 conformance by reproducing the standard genesis bundle's CID +from its own build of the included SX sources. If two implementations build the same +spec sources and produce different CIDs, one of them is non-conformant. Cheap, +deterministic conformance check. + +### 12.10 Operational implications + +- **Build-time CID computation is part of the kernel build.** The build pipeline + must include the genesis-bundling step and embed the resulting CID. Mismatch + protection requires the binary to know what it expects. +- **Genesis evolution is a deliberate kernel-team decision.** Adding a new bundled + projection or sig suite is a kernel release, not a federated activity. (User- + defined projections still federate normally.) +- **Strict-mode default protects against malicious extensions.** Operators have to + consciously opt into auto-loading remote `Define*`. This trades convenience for + security — appropriate for v1. +- **Cross-major federation is a bridge problem.** If/when fed-sx 2.0 ships with an + envelope change, bridges between v1 and v2 are themselves federated artifacts — + built by anyone, signed, audited. + +## 13. Federation mechanics + +How instances exchange activities, how peers subscribe, how new followers backfill, +how delivery survives unreliable networks, and how the substrate resists abuse. + +### 13.1 Push, pull, hybrid + +ActivityPub canonically uses **push**: actor A publishes by POSTing each delivery to +each follower's inbox URL. This gives low latency and clear delivery semantics, but +requires a reliable per-recipient delivery queue and falls over when peers go down. + +fed-sx supports both, with a **push-primary, pull-fallback** model: + +- **Push** is the default delivery mechanism. When an activity is appended to A's + outbox, A's delivery worker posts it to each follower's inbox. +- **Pull** is always available: any peer can `GET /actors//outbox?since=` + and stream activities in order. Used for backfill, recovery from delivery gaps, + and instances that prefer pull-only operation. +- **Hybrid in practice:** push delivers *notifications* (the activity itself, or a + pointer to its CID); receivers may pull the full content if not inlined. Useful + when the activity body is large. + +Operators can configure their actors as push-only, pull-only, or hybrid. The +default is hybrid. + +### 13.2 The Follow lifecycle + +AP-standard, slightly tightened: + +```sx +;; A wants to follow B +(activity 'Follow + :actor "https://a.example/actors/alice" + :object "https://b.example/actors/bob") +;; → POST to B's inbox + +;; B accepts (or rejects) +(activity 'Accept + :actor "https://b.example/actors/bob" + :object ) +;; → POST to A's inbox + +;; A unfollows later +(activity 'Undo + :actor "https://a.example/actors/alice" + :object ) +;; → POST to B's inbox +``` + +State derived by the `audience-graph` projection on each instance: + +- `(followers actor)` — set of actors who follow `actor`, projected from + `Accept{Follow}` activities in `actor`'s outbox (and the inverse via received + `Follow` activities). +- `(following actor)` — symmetric. + +**Auto-accept by default.** Public actors auto-publish `Accept` for any incoming +`Follow`. Locked actors require manual approval, implemented as an operator UI that +publishes the `Accept` (or `Reject`) once a human decides. + +### 13.3 Backfill + +When A first follows B, A wants B's history. Four supported modes: + +| Mode | Mechanism | Trade-off | +|------|-----------|-----------| +| **No backfill** | Just stream new activities going forward | Cheapest, missing context for new followers | +| **Pull paginated** | `GET /outbox?since=epoch&limit=100` repeatedly | Standard, slow for large outboxes | +| **Snapshot fetch** | Find latest `Create{Snapshot}` published by B for the projection of interest, fetch + verify, then pull only activities after the snapshot's tip | Fast, requires B to publish snapshots | +| **Bundle fetch** | Out-of-band: B publishes a CID for an export bundle (a dag-cbor list of activities + actor doc + sig suite verification metadata); A fetches once, validates the chain, replays | Fastest for cold starts; bundle creation is opt-in | + +Default: snapshot fetch when available, paginated pull otherwise. + +A new instance joining federation typically combines: snapshot-fetch the +`actor-state` and `define-registry` projections from a trusted peer (so it knows who +exists and what verbs are defined), then incrementally backfill specific actors of +interest. + +### 13.4 Delivery queue and retry + +Every push delivery attempt has a fate: + +| Outcome | Action | +|---------|--------| +| 2xx | Mark delivered | +| 3xx | Follow redirect (with limit) | +| 4xx (except 429) | Mark *permanently failed* — peer rejected the activity. Log; don't retry. | +| 429 | Honour `Retry-After`; reschedule | +| 5xx | Exponential backoff; reschedule | +| Connection error | Exponential backoff; reschedule | + +**Retry schedule** (default, tunable per peer): + +``` +1 min, 5 min, 15 min, 1 h, 4 h, 12 h, 24 h, 48 h, 96 h +``` + +After the last attempt fails, the activity is **abandoned for push** but remains in +A's outbox. Followers can still pull it via `GET /outbox?since=...`. The peer will +eventually catch up if they come back online and pull. Push is best-effort; pull is +the source of truth. + +**Persistent queue.** Delivery state is itself stored in the local instance — it's +operator-internal, not federated. (Could be a regular SQLite table; doesn't need to +be a projection because it's not state-the-world-cares-about.) On instance restart, +the queue resumes from where it left off. + +**Queue-as-projection (alternative):** for instances that want every aspect to be +log-derived, the delivery state could be a local-only projection over a stream of +`Attempt` / `DeliverySuccess` / `DeliveryFailure` activities written to a private +local-only outbox. Out of scope for v1 but the design admits it. + +### 13.5 Audience-respecting delivery + +Each activity carries `to`, `cc`, `bto`, `bcc`. The delivery worker computes the +**delivery set**: union of explicit recipients + (if `as:Public` or `Followers` in +audience) the actor's followers projection. + +- `bto` and `bcc` are stripped before delivery (recipients shouldn't see who else is + blind-copied). +- **Receivers honour audience.** When an instance receives an activity it should + not be in the audience for (e.g. a `Direct` activity to someone else, leaked via a + misconfigured peer), it logs and discards. Validators in the inbound pipeline + enforce this. +- **Public ≠ unlisted.** `to: as:Public` means deliver to followers AND make + publicly fetchable AND show in public projections. Some actors prefer "publicly + fetchable but not pushed broadly" — `cc: as:Public` with `to: Followers`. + +### 13.6 Spam and abuse posture + +ActivityPub has well-known abuse vectors (Mastodon's history is instructive). fed-sx +defends in layers: + +**Signature verification.** Every inbound activity must have a valid signature +matching an actor whose key was active at `published`. Forgeries are dropped at the +envelope-validation stage (§14). Necessary but not sufficient — signatures only +prove the message wasn't tampered with, not that the sender is benign. + +**Per-source rate limits.** Per-actor and per-instance request rate limits on +`/inbox`. Default: 100/min per actor, 1000/min per instance. Exceeded → 429. + +**Per-instance trust state.** Three categories, operator-configured (and +overridable per actor): + +- **Trusted** — auto-accept, auto-load Define* (if permissive mode), no rate- + multiplier penalty. +- **Default** — accept signed activities, standard rate limits, do not auto-load + Define*. +- **Suspended** — drop all inbound activities, refuse outbound delivery, do not + fetch artifacts. Operator decision (e.g. spam source, harassment instance). + +Trust state is local-only (operator policy); it is not federated. Different +instances can disagree. + +**Audience refusal.** Activities not addressed to anyone on this instance (no local +followers, not `as:Public`, not `to:` a local actor) are dropped on receipt. +Discourages spam targeting random instances. + +**Content validators.** Registry-driven content moderation: a `DefineValidator` +with `applies-to: "inbound"` runs against every inbound activity and can reject +based on content rules. Examples: link-spam detection, ML moderation models served +via an effectful validator (note: effectful validators are a special case — they +*can* fail-closed without affecting determinism, because validators happen *before* +projection and don't contribute to projected state). + +**Capability vetting.** If an inbound activity declares `capabilities-required` +that includes definitions this instance hasn't loaded *and* trust policy is strict- +mode, the activity is quarantined (stored but not projected) pending operator +review. + +**Federation circuit breakers.** Per-peer error rate triggers temporary defederation: +if a peer is sending malformed activities, exceeding rate limits, or signing with +revoked keys, automatic suspension for an exponential cool-off. + +### 13.7 Discovery + +How an instance finds other instances and actors: + +- **WebFinger** (RFC 7033). `GET /.well-known/webfinger?resource=acct:user@host` + returns links to actor URLs. AP-standard. fed-sx implements. +- **Well-known capabilities.** `GET /.well-known/sx-capabilities` (§7) for cross- + instance compatibility checks. +- **Manual peer config.** Operators add peer instance URLs to their config. +- **Peer recommendations.** An instance can publish `Recommend{actor}` activities + pointing at peers it considers worth following. Receivers can use these as + discovery hints (subject to local trust). Out of scope for v1 but the verb is + reservable. +- **Federation directories.** Community-maintained lists of instances; an instance + can opt into being listed by publishing a `Directory{listed-by}` activity. v2 + concern. + +For v1: WebFinger + capabilities + manual config. Discovery beyond that is opt-in +via standard verbs. + +### 13.8 Streaming and real-time + +Two streaming mechanisms: + +- **Outbox SSE** — `GET /actors//outbox/stream` opens a Server-Sent Events + connection. Each new activity appended to the outbox is sent as an event. Allows + pull-style federation peers to maintain a live connection without polling. +- **Projection SSE** — `GET /projections//subscribe` (§10.8) streams projection + deltas. Useful for clients (browsers) wanting reactive views. + +Both are local-only mechanisms; the canonical federation transport remains push to +inbox + pull from outbox. SSE is convenience, not protocol. + +### 13.9 Operational implications + +- **Push is best-effort, pull is authoritative.** Operators should treat the outbox + as the canonical record; delivery queue is bookkeeping. +- **Trust is per-instance and not federated.** Two instances may have different + views of "good actors" and "bad instances." This is a feature — defederation + decisions are local sovereignty. +- **Backfill via snapshots is the cheap path.** Encouraging actors to publish + `Create{Snapshot}` regularly makes new-follower onboarding fast. +- **Audience semantics are enforced both ways.** Senders compute delivery set; + receivers honour audience. Defence-in-depth against misconfigured peers. +- **Capability-based extension loading is opt-in.** Strict-mode default means + unknown verbs are stored-but-not-projected — safe by default, with explicit + operator control over what extensions load. + +## 14. Validation pipeline + +Every activity entering the substrate (whether published locally or received from a +peer) flows through a fixed pipeline of checks. Order matters: cheap and fail-safe +first, expensive and content-aware last. Each stage has a defined failure response +(reject, quarantine, drop). Registry-driven validators plug in at a specific stage. + +### 14.1 The two pipelines + +**Inbound** — activities arriving via `POST /inbox` or pulled from a peer's outbox: + +``` +HTTP transport → envelope → signature → replay → audience → + activity-type schema → object-type schema → content validators → + capabilities → trust state → log append → projection (async) +``` + +**Outbound** — activities being published locally via `POST /activity`: + +``` +authentication → authorization → envelope construction → object handling → + activity-type schema → signature → log append → projection (async) → + delivery (async) +``` + +Stages they share are implemented as the same SX functions called from both pipelines. + +### 14.2 Inbound pipeline — stage by stage + +| # | Stage | Check | Failure response | +|---|-------|-------|------------------| +| 1 | **Transport** | Valid HTTP request, content-type acceptable, body parseable as JSON-LD or dag-cbor | `400 Bad Request`; log | +| 2 | **Envelope** | Matches kernel's envelope spec (required fields present, types valid, recognised activity type or `unknown` allowed) | `400`; log; structured error in response body | +| 3 | **Signature** | Time-aware sig verification: fetch (or cache-lookup) actor doc, find key with `id == sig.key-id` that was active at `published`, verify against canonical envelope bytes per the named sig suite | `401`; log; do not retry; mark sender's instance for circuit-breaker accounting | +| 4 | **Replay** | Activity id and CID not already in `activity-log` projection | `200 OK` with `{status: "duplicate"}`, no-op | +| 5 | **Audience** | This instance has at least one local actor in `to`/`cc`, OR audience contains `as:Public`/`Followers` and the actor has local followers | Drop silently (no response indicating either acceptance or refusal — prevents inbox-membership probing); do not store | +| 6 | **Activity-type schema** | Look up `DefineActivity{name: }` in `define-registry`; run its `schema` predicate over the activity in pure sandbox | If type unknown: per trust policy (strict: 422 with missing-definition CID; permissive: attempt dynamic load §12.8). If schema fails: 422 with violation detail | +| 7 | **Object-type schema** | If activity has an `object` with a `type`, look up `DefineObject{name: }` and run its `schema` | Same as #6 | +| 8 | **Content validators** | All registered validators with `applies-to: inbound` or `applies-to: all` run sequentially; each is a pure-sandbox predicate that returns `:accept` / `:reject` / `:quarantine` | `:reject` → 422 with reason. `:quarantine` → store activity but mark `quarantined`, do not project, alert operator | +| 9 | **Capabilities** | Every CID in `capabilities-required` is present in this instance's loaded registries (or auto-loadable per trust policy) | Missing → 422 with list of missing CIDs (sender can deliver bootstrapping `Define*` artifacts first). Auto-load attempt can be triggered by re-POST with `?retry-after-load=true` | +| 10 | **Trust state** | Sender's actor and instance are not in `Suspended` state on this instance | Drop silently; do not respond | +| 11 | **Log append** | Write activity envelope (and inlined object content) to local mirror of sender's outbox; assign local sequence number | Disk error → 503 (transient); sender retries | +| 12 | **Projection** | Asynchronously fold the activity into every relevant projection (per `define-registry`) | Per-projection failure (gas, sandbox violation) → tag activity `projection-failed:`; do not affect log durability | + +Pipeline halts at the first failing stage. Stages 1–10 are synchronous (`POST /inbox` +holds the connection). Stage 11 is synchronous; stage 12 is asynchronous and the +HTTP response returns once the log append succeeds. + +### 14.3 Outbound pipeline — stage by stage + +| # | Stage | Check | Failure response | +|---|-------|-------|------------------| +| 1 | **Authentication** | Caller has a valid bearer token, mTLS cert, or session for the actor | `401` | +| 2 | **Authorization** | Caller's identity is allowed to publish as the named `actor` (capability token §9.5 or owns the actor key) | `403` | +| 3 | **Envelope construction** | Kernel fills in `id`, `published`, normalises `to`/`cc`, computes `capabilities-required` (by walking referenced `Define*` CIDs) | n/a | +| 4 | **Object handling** | If `object` has inline content: canonicalize, compute CID, optionally store per `where`. If `object` references a CID, verify the artifact exists locally or remotely (or accept as a forward reference) | Storage error → `503` | +| 5 | **Activity-type schema** | Same as inbound #6 — schema must pass | `422` with violation detail (caller bug) | +| 6 | **Signature** | Sign envelope with the actor's currently-active key matching the activity type's required `purpose` (e.g. `Pin` requires `purpose: pin`) | If no suitable key: `400` | +| 7 | **Log append** | Write to local outbox; assign sequence number | `503` | +| 8 | **Projection** | Async fold (same as inbound #12) | Per-projection failure tag | +| 9 | **Delivery** | Async push to follower inboxes per audience | Per-recipient retry per §13.4 | + +Caller's HTTP response returns after stage 7 (log append). The activity is durable +and queryable as soon as the response is sent; projection lag is reported via +`projected-up-to` headers and `?wait-for=` parameter. + +### 14.4 Failure response taxonomy + +Three response categories with explicit semantics: + +**Reject** — tell sender, don't store, reject can be retried after sender corrects. +Used for: malformed envelope, invalid signature, schema violation, missing +capabilities. HTTP 4xx with structured error. + +**Quarantine** — store envelope (it's a valid signed message) but don't project, +alert operator. Used for: content-validator soft-fail, unloaded capabilities under +permissive policy, suspect-but-not-banned senders. Activity sits in a quarantine +projection until operator reviews; operator can release (project) or expunge. + +**Drop silently** — don't store, don't respond informatively. Used for: replay (ack +as duplicate), audience refusal (would leak inbox membership otherwise), suspended- +sender activities. The sender experiences this as a successful POST with no visible +effect; they can detect it only by polling for their activity not appearing in our +outbox. + +### 14.5 Registry-driven validators + +Most of the pipeline is **fixed kernel logic** (envelope, signature, replay, audience, +log append, delivery). Two stages are **registry-driven** and extend dynamically: + +- **Stage 8 (content validators)** — operators add/remove `DefineValidator` entries + with `applies-to: inbound | outbound | all`. Each runs in pure or effectful + sandbox per its declaration. Returns one of `:accept` / `:reject{:reason}` / + `:quarantine{:reason}`. +- **Stages 6–7 (schema validators)** — these *are* registry entries + (`DefineActivity.schema`, `DefineObject.schema`); the pipeline calls into the + registry to fetch them. + +**Pure-mode validators** are deterministic and cheap; results can be cached per +(activity-CID, validator-CID). + +**Effectful-mode validators** can call out to ML models, blocklist services, +external moderation APIs. They get a per-call IO budget; exceeding it counts as +`:reject{:reason :validator-timeout}`. Effectful validators do *not* break +determinism because validation happens **before projection** — a rejected activity +never enters projected state. + +### 14.6 Validator composition and ordering + +Validators have an integer `priority` field; lower priority runs first. Pipeline +short-circuits on first `:reject`. `:quarantine` is *not* short-circuiting; later +validators still run, and `:quarantine` results aggregate. + +Default priorities (room for operator-added validators): + +``` +0-99 : kernel-internal (envelope, sig, replay, audience) +100-199 : standard schema validators +200-299 : standard content validators (rate limit, audience leak) +300-399 : operator-added moderation +400-499 : effectful (ML, third-party APIs) +500+ : reserved +``` + +Operators can publish `Update{DefineValidator}` to change priorities or add new +ones; takes effect on next inbound activity. + +### 14.7 Determinism requirement and its limit + +A subtlety worth being explicit about: **inbound validation is not required to be +deterministic across instances.** Two instances can disagree about whether to +accept a given activity (e.g. one has a stricter content validator). Their projected +states will then diverge — but only on activities one accepted and the other didn't. + +This is fine. Federation does not require state convergence; it requires *fold +determinism for activities both instances accepted*. Validators are sovereignty +controls, not protocol invariants. + +Where determinism *is* required: schema validators (§14.2 stages 6–7). If two +instances disagree on whether `Pin v3` matches its schema, they can't federate +`Pin v3` activities meaningfully. So schema validators must be pure-mode and +referenced by CID. + +### 14.8 Operational implications + +- **The pipeline is the security perimeter.** Every checkable property is checked + here, not deeper in the kernel. No "trust the caller" assumptions inside log or + projection code. +- **Quarantine is the operator's friend.** Anything suspicious sits in quarantine + with full envelope, sig, and reason — operator can review and decide. Better than + outright drop because it preserves audit. +- **Schema validators are protocol-load-bearing; content validators are policy.** + The first set must converge across instances for federation to work; the second + set can diverge (and that's how local moderation policy is expressed). +- **Outbound validation catches local bugs early.** A malformed `Pin` activity + fails at outbound stage 5, never enters the local log, never gets delivered. + +## 15. Storage layout + +The on-disk shape of an instance. Three concerns kept separate: the **activity log** +(append-only, canonical), **content-addressed object storage** (keyed by CID, +immutable), and **operational state** (projections, indexes, queues — derived, +rebuildable). + +### 15.1 Storage tiers + +``` +/var/lib/fed-sx/ +├── log/ # canonical, append-only +│ ├── actors/ +│ │ ├── / +│ │ │ ├── outbox/ +│ │ │ │ ├── 000001.jsonl # segment, ~64MB cap +│ │ │ │ ├── 000002.jsonl +│ │ │ │ └── tip # symlink to current segment +│ │ │ ├── inbox/ # received, pre-projection +│ │ │ └── seq # next sequence number +│ │ └── /... +│ └── mirrors/ # local mirrors of followed remote outboxes +│ └── / +│ ├── 000001.jsonl +│ └── ... +├── objects/ # CID → bytes +│ └── // +├── snapshots/ +│ └── / +│ ├── .cbor # snapshot value +│ └── index # ordered list of (log-tip, file) +├── projections/ # live projection state +│ └── .cbor # latest in-memory state, periodically flushed +├── indexes/ +│ └── fed-sx.db # SQLite: lookups, queue, trust state +├── keys/ +│ └── / # private keys, mode 0600 +│ ├── primary.pem +│ ├── recovery.pem +│ └── sigs.toml # key metadata +├── genesis/ +│ └── bundle.cbor # extracted from binary at first run +└── config.toml # operator config +``` + +### 15.2 The log — append-only segments + +The activity log is the only thing the substrate cannot lose. It is the source of +truth from which everything else is derived. + +**Format: JSONL segments.** Each line is one activity envelope, encoded as JSON-LD +(canonical form), terminated by `\n`. Easy to inspect, easy to grep, trivially +streamable. + +**Why JSON-LD on disk, not dag-cbor?** Two reasons: +- Operability: humans can `tail -f` and `grep` the log. dag-cbor is opaque. +- AP wire compatibility: activities arrive over HTTP as JSON-LD anyway; storing the + same form avoids round-trip conversion. + +The CID of each activity is computed from its **canonical dag-cbor representation** +(per §2), independent of how it's stored. CIDs are stable across storage formats. + +**Segments cap at ~64MB.** Rotation by size, not time. Old segments are immutable; +new writes go to the tip segment. Compression (zstd) applied on segments older than +the current tip — saves disk, doesn't slow appends. + +**Per-actor outboxes.** Each local actor has its own outbox directory. This matches +AP semantics (one outbox per actor) and means: +- Backing up a single actor is a simple directory copy +- Per-actor sequence numbers (no cross-actor coordination) +- Migration (`Move`) is a directory rename + a `Move` activity + +**Mirror outboxes.** When a local actor follows a remote one, the remote's outbox is +mirrored locally for replay. Same JSONL format. Tracked under `log/mirrors//` to avoid filesystem path issues with URL characters. The hash is +purely a filesystem-friendly encoding; the canonical actor id stays in the log +content. + +**Inbox vs outbox distinction.** Inboxes hold *received* activities pre-validation; +outboxes hold *committed* activities post-pipeline. An inbound activity that passes +the validation pipeline (§14) is moved from inbox to the appropriate mirror outbox. +This makes inbox a transient queue, not a permanent record. + +### 15.3 Object storage + +Content-addressed blob store, sharded directories. + +**Path scheme:** `objects///`. Sha2-256 CIDs +are uniformly distributed; this gives ~65k buckets with a couple-hundred files each +at moderate scale. Standard pattern (matches IPFS, Git). + +**Storage backends.** Pluggable per `where: cid` object: + +- **`files-on-disk`** (default) — write to local filesystem. +- **`ipfs`** — register-driven backend; calls out to a local IPFS node. +- **`s3`** — object storage in cloud bucket. +- **`memory-only`** — in-memory cache, evictable; useful for ephemeral artifacts. + +The kernel uses the `where-tag` on each object to dispatch to the correct backend. +Backends are registry entries (`DefineStorage`); operators install only the ones +they want. + +**Garbage collection** is opt-in per backend. Default policy: **never GC** (objects +are immutable and may be referenced by future activities). Operators can configure +per-backend retention rules: + +- "Keep last N versions of objects referenced by `Pin` activities for path X" +- "Evict objects not referenced in last 90 days from the `memory-only` cache" +- "Mirror objects referenced by ≥ 3 endorsements; evict others after 30 days" + +GC operates on the projected reference graph (a `reference-graph` projection that +maintains "what activities reference this CID"). Removing an object that's still +referenced is allowed but produces a warning logged in operations. + +### 15.4 Snapshots + +Per §10.4, snapshots are the (projection-CID, log-tip-CID, state) triples that let +us resume without full replay. + +**Storage:** `snapshots//.cbor`. The state value is +dag-cbor-encoded; the file's content CID matches the snapshot's claimed CID. + +**Index:** `snapshots//index` is a sorted list of `(log-tip-time, +log-tip-cid, file)` triples. On startup, kernel finds the latest snapshot ≤ current +log tip and resumes from it. On time-travel queries, finds the latest snapshot +≤ target time and folds forward. + +**Retention:** keep at least: +- Latest snapshot per active projection +- Snapshots referenced by published `Create{Snapshot}` activities (federation + proofs) +- One snapshot per day for the last 7 days (audit / time-travel) + +Older snapshots GC'd by default. Operators can increase retention. + +### 15.5 Operational state — SQLite + +Things that are derived, frequently-queried, but not federated: + +- **Lookup indexes** for projections (when `indexes:` declared) — `(projection, + index-key, value) → activity-cid` rows +- **Delivery queue** — outbound activities pending push, retry counts, next-attempt + timestamps +- **Trust state** — per-actor and per-instance trust levels (Trusted / Default / + Suspended) +- **Quarantine queue** — activities pending operator review +- **Configuration cache** — currently-active registry entries (also in memory; on- + disk cache for fast restart) + +Single SQLite file (`indexes/fed-sx.db`). Recoverable: if corrupted or deleted, +rebuilt from the log on next startup (with cost proportional to log size). The +SQLite is a cache, not authoritative. + +WAL mode for concurrent readers. Single-writer (the kernel); reads from many +HTTP request workers. + +### 15.6 Backup and export + +The substrate is an append-only log of immutable artifacts; backup is simple. + +- **Full backup:** rsync `/var/lib/fed-sx/log/` and `/var/lib/fed-sx/objects/`. The + rest is rebuildable. +- **Per-actor export:** tar `log/actors//` + the objects referenced by + activities in that outbox. Self-contained, importable into another instance. +- **Activity bundle export:** for federation backfill, produce a dag-cbor bundle of + `[activity envelopes... + referenced objects]` for a specified actor + range. + Single file, content-addressed, signed by the source instance with a `Bundle` + activity attesting to its contents. + +Exports are themselves publishable (`Create{Bundle}` activity carrying the bundle +CID). This is how an actor migrates instances cleanly: export bundle, import on +new instance, publish `Move` activity. + +### 15.7 Mirroring and replication + +Two patterns: + +- **Federation mirroring** (the canonical kind) — when actor A follows B, A's + instance mirrors B's outbox locally. This is just normal federation (§13). Each + follower keeps its own copy. +- **Operational mirroring** — for high availability. An operator runs two instances + with shared filesystem (NFS / EFS) for `log/` and `objects/`, separate SQLite + files. Reads can hit either; writes go through one. Or: rsync-based hot standby + with manual failover. + +Operational mirroring is out of scope for v1. Federation mirroring is the substrate- +level redundancy: as long as one peer that followed you is still online, your log is +still recoverable. + +### 15.8 Storage size estimates + +Rough targets at moderate scale (10 active local actors, 1000 followed peers, 1 +year of activity at 100 activities/actor/day): + +- **Log:** 10 actors × 100 act/day × 1 KB avg envelope × 365 days ≈ 365 MB local + outbox. Mirrors: 1000 peers × 10 act/day × 1 KB × 365 ≈ 3.6 GB. +- **Objects:** depends heavily on content. Assume 50% of activities have inline + content of avg 5 KB → ~2 GB total inline. CID-referenced larger objects: count + separately, depends on use case. +- **Snapshots:** typically much smaller than the log. ~10 active projections × + ~10 MB per snapshot × ~8 retained snapshots ≈ 800 MB. +- **SQLite:** index sizes proportional to indexed projection content; typical few + hundred MB. + +Total: order of 10 GB at the described scale. Single-machine viable; SSD recommended +for log throughput; spinning disk fine for snapshots and object storage cold tier. + +### 15.9 Operational implications + +- **The log is sacred.** Never modify, never delete. Backups go to multiple media. + Loss of `log/` means loss of identity (actor activities) and loss of state-of- + record. Loss of `objects/` means loss of content but log + peers can recover most + of it. +- **Everything else is rebuildable.** Projections, indexes, snapshots, queue state + can all be recomputed from the log at startup cost. Operationally, this means + upgrades and migrations are forgiving. +- **CID-addressed storage is naturally idempotent.** Two instances writing the same + artifact write the same bytes to the same path. Race conditions become no-ops. +- **JSONL on disk pays for itself** the first time an operator needs to debug a + weird federation issue with `grep` and `jq`. Worth the storage cost vs dag-cbor. + +## 16. API surface + +HTTP API for reading the log, publishing activities, querying projections, and +streaming updates. Three layers: **AP-standard** endpoints (for vanilla AP +interop), **fed-sx-specific** endpoints (publish, query, capabilities), and +**discovery** endpoints (webfinger, well-known). + +### 16.1 Endpoint catalog + +#### AP-standard + +| Method | Path | Purpose | +|--------|------|---------| +| GET | `/actors/` | Actor doc (Person/Service/Group/Application) | +| GET | `/actors//inbox` | Read inbox — auth required | +| POST | `/actors//inbox` | Receive federated activity (HTTP Signature required) | +| GET | `/actors//outbox` | OrderedCollection of actor's published activities | +| POST | `/actors//outbox` | AP-standard publish (alias for `POST /activity` with `actor` set) | +| GET | `/actors//followers` | OrderedCollection of follower actor URIs | +| GET | `/actors//following` | OrderedCollection of followed actor URIs | +| GET | `/activities/` | Single activity by id | +| GET | `/objects/` | Single object by id (note: distinct from CID-addressed `/artifacts/`) | + +#### fed-sx-specific + +| Method | Path | Purpose | +|--------|------|---------| +| POST | `/activity` | Generalised publish — accepts any well-formed activity | +| GET | `/artifacts/` | CID-addressed artifact fetch (content negotiated) | +| GET | `/artifacts//raw` | Raw bytes (whatever the codec stored) | +| GET | `/artifacts//` | IPLD path traversal into the artifact | +| GET | `/projections` | List of registered projections (name, CID, last-folded-tip) | +| GET | `/projections/` | Full projection state (paginated for large states) | +| GET | `/projections/?at=` | Time-travel: state as of timestamp | +| GET | `/projections//` | Single key from a projection (uses indexes) | +| POST | `/query` | Run an SX query expression against one or more projections | +| GET | `/define-registry` | Currently active `Define*` artifacts by kind | +| GET | `/capabilities/` | Per-actor declared capabilities | + +#### Discovery and well-known + +| Method | Path | Purpose | +|--------|------|---------| +| GET | `/.well-known/webfinger?resource=acct:@` | RFC 7033 actor discovery | +| GET | `/.well-known/sx-capabilities` | This instance's capability advertisement (§7) | +| GET | `/.well-known/host-meta` | XRD describing the host | +| GET | `/.well-known/nodeinfo` | Standard fediverse node metadata (Mastodon, Pleroma compatibility) | + +#### Real-time (SSE) + +| Method | Path | Purpose | +|--------|------|---------| +| GET | `/actors//outbox/stream` | New activities as they're appended (events: `activity`) | +| GET | `/actors//inbox/stream` | New inbound activities (auth required) | +| GET | `/projections//subscribe` | Projection deltas (events: `delta`) | +| GET | `/federation/health/stream` | Per-peer delivery health (events: `peer-status`) | + +WebSocket equivalents (`/ws/...` paths) available where SSE is awkward (browsers +behind proxies); same event payloads, different framing. + +### 16.2 Authentication + +Three mechanisms, each appropriate to a different caller type: + +- **HTTP Signatures** (RFC draft-cavage-http-signatures) — the AP-standard mechanism + for inter-instance calls. Sender signs a digest of relevant headers + body with + their actor's private key; receiver verifies via the actor's public keys + projection (§9.6). Used for: `POST /inbox`, peer-to-peer outbox pulls when + authentication is desired. +- **Bearer tokens** — for interactive clients (CLIs, web UIs, mobile apps). + Issued via OAuth2 (or simple admin-issued tokens for v1). Used for: + `POST /activity`, `GET /actors//inbox`, anything requiring caller identity. +- **Capability tokens** (§9.5) — for delegated publish. Token includes the granting + actor, the granted capabilities (e.g. `publish: Pin for path-prefix /docs/`), the + bearer's actor, expiry, and signature from the granter. Used for: child actors, + service accounts, temporary publish access. + +Public reads (most GET endpoints to public-audience activities) require no auth. +Private/followers-only reads check the caller's identity against the audience. + +### 16.3 Content negotiation + +Same resource, multiple representations. `Accept` header dispatches: + +| Accept header | Returns | +|---------------|---------| +| `application/activity+json` | AP-standard JSON-LD (default for ambiguous Accepts) | +| `application/ld+json; profile="..."` | JSON-LD with explicit profile | +| `application/cbor` | dag-cbor | +| `application/json` | Plain JSON (compact, no `@context` expansion) | +| `application/sx` | Canonical SX wire format | +| `text/html` | HTML representation (for browsers — renders the artifact via SX) | + +Same negotiation applies to `/artifacts/`, `/activities/`, +`/projections/`. Servers MUST honour the request; absent `Accept` defaults to +`application/activity+json`. + +### 16.4 Pagination + +Cursor-based via AP's `OrderedCollectionPage`: + +``` +GET /actors/giles/outbox +→ { + "type": "OrderedCollection", + "totalItems": 12345, + "first": "/actors/giles/outbox?page=true", + "last": "/actors/giles/outbox?page=true&min_id=0" + } + +GET /actors/giles/outbox?page=true +→ { + "type": "OrderedCollectionPage", + "id": "...?page=true", + "next": "...?page=true&max_id=", + "prev": "...?page=true&min_id=", + "orderedItems": [...] + } +``` + +Cursors are CIDs of the boundary activity (not opaque tokens). Stable across +restarts and instances. `max_id` returns activities **before** the cursor (newest +first); `min_id` returns activities **after** the cursor. + +Default page size: 50. Max: 1000. `Link: <...>; rel="next"` header also provided +for HTTP-native pagination. + +For projections: same shape, items are projection entries. + +### 16.5 The query API + +`POST /query` takes an SX expression evaluated in pure mode against named +projections: + +```sx +POST /query +Content-Type: application/sx +Accept: application/sx + +(let ((actors (projection actor-state)) + (pins (projection pin-state))) + (for-each ([(actor-id actor) actors]) + (when (> (count (filter (fn ((path cid)) (= (:owner cid) actor-id)) pins)) 10) + {:actor (:preferredUsername actor) + :pins-published (count ...)}))) +``` + +Query semantics: + +- Evaluated in pure sandbox; all the determinism rules apply. +- Projection access is read-only and snapshot-consistent: the query sees state + as-of the time of the request (or `?at=` if specified). +- Result is serialized in the negotiated content type. +- Gas limit applies (default 1M units per query, tunable by operator). +- Cacheable: query CID + projection state CIDs uniquely determine the result. + +Query results can themselves be published as `Create{QueryResult}` activities, +making derived analyses federable. + +### 16.6 Errors + +Uniform JSON error envelope: + +```json +{ + "error": { + "type": "https://next.rose-ash.com/ns/fed-sx/errors/v1#InvalidSignature", + "status": 401, + "title": "Activity signature invalid", + "detail": "Key id 'https://example/actors/x#key-1' was superseded at 2026-01-15T...", + "activity-id": "https://...", + "key-id": "...#key-1", + "instance": "/incidents/" + } +} +``` + +Error types are URIs in the fed-sx namespace; receivers can check `type` for +programmatic handling. Standard errors: + +- `MissingCapability` — includes `missing` array of CIDs +- `SchemaViolation` — includes `schema-cid`, `field-path`, `expected`, `got` +- `InvalidSignature` +- `Quarantined` — includes `quarantine-id` for operator-status tracking +- `RateLimited` — includes `retry-after` +- `ResourceExhausted` — for query gas exhaustion + +### 16.7 Streaming details + +SSE event format: + +``` +event: activity +id: +data: { ...activity envelope... } + +event: delta +id: +data: {"projection": "actor-state", "key": "...", "old": ..., "new": ...} + +event: heartbeat +data: {"projected-up-to": "", "ts": "..."} +``` + +Clients reconnect with `Last-Event-ID: ` to resume from the last event seen. +Server replays from that point in the log (or returns 410 if too far behind, in +which case client should switch to paginated pull). + +### 16.8 Versioning + +The substrate is versioned at three levels: + +- **Envelope version** — declared in `/.well-known/sx-capabilities`. Currently `1`. + Forward-compatible (new fields OK; semantics fixed). +- **API version** — URL prefix optional: `/v1/...` works the same as `/...`. Future + major version: `/v2/...` paths in parallel. +- **Definition versions** — supersession via activity log (§§9.2, 12.7). No special + URL handling. + +Capability negotiation happens before federation; clients shouldn't hard-code +URL paths beyond the canonical set documented here. + +### 16.9 Operational implications + +- **The API is small but layered.** AP compatibility is one layer; fed-sx + extensions are another; both share auth and content negotiation. Adding a new + endpoint shouldn't require new transport machinery. +- **Content negotiation is the polyglot bridge.** Same artifact addressable in JSON- + LD (for AP peers), dag-cbor (for fed-sx peers), SX (for SX clients), HTML (for + humans). One CID, four representations. +- **Cursor pagination is CID-based.** Stable identifiers, no opaque tokens to + invalidate, peers can synchronize without coordination. +- **The query API is a load-bearing differentiator.** Datalog/GraphQL-equivalent + expressiveness with no separate query language — it's just SX. Federable, signable, + versionable like any other SX artifact. + +--- + +## 17. Implementation languages + +Polyglot **authoring**, monoglot **runtime**: every language-on-SX compiles to core +SX and runs on any host with the SX evaluator. The language is an authoring choice; +the federated artifact is uniform SX. Authors of `Define*` artifacts pick the +source language they prefer; consumers don't need that compiler installed to +execute the compiled SX. + +Languages are picked because they **genuinely fit the problem**, not to demonstrate +the polyglot story. Where a chosen language has gaps (e.g. Erlang-on-SX missing hot +reload), we invest in maturing the port rather than working around the gap. + +### 17.1 The v1 stack + +| Layer | Language | Why | +|-------|----------|-----| +| **Native primitives** | OCaml (existing runtime) | Crypto (RSA, Ed25519, SHA), dag-cbor encode/decode, HTTP socket, file IO, SQLite. Surfaced as Erlang-on-SX BIFs. | +| **Kernel orchestration** | Erlang-on-SX | Actor model = federation. `gen_server` per actor / per projection / per peer. `supervisor` for delivery workers. Message passing is literally the substrate. Hot code reload (Phase 7) for `Define*` live extension. | +| **Query API back-end** | Datalog-on-SX | Projection state is relational; trust graph walks, provenance, projection joins are textbook Datalog. Already mature (276/276 tests, full core Datalog with stratified negation, aggregation, magic sets, federation-graph demo). | +| **`Define*` semantics, schemas, validators, codecs, audience predicates** | Core SX | The canonical federated language. Everything content-addressed and federated lives here. | + +### 17.2 Languages explicitly **not** booked for v1 + +Available, mature, considered — would be reached for if a real fed-sx need surfaced, +but no preemptive use: + +- **Haskell-on-SX** (285/285 tests, 36 programs, type checker working) — for complex + operator-authored extensions that benefit from typed pattern matching. Schemas in + fed-sx are short predicates; types don't earn their keep here. +- **Smalltalk-on-SX** (625/629 tests, classic corpus running) — natural fit for a + live operator dashboard / Glamorous-Toolkit-style introspection. v2/v3 territory; + a browser UI likely wins for operator audiences. +- **APL-on-SX** — high-throughput batch reprojection if scalar SX folds become a + bottleneck. Premature without measured need. +- **JS-on-SX**, **Elm-on-SX** — browser-side client SDK / viewer. v2. +- **Common Lisp-on-SX**, **Forth-on-SX**, **Go-on-SX**, **Dream-on-SX**, + **Elixir-on-SX**, **Erlang-on-SX (alternative form)** — case by case if a use + case appears. + +### 17.3 The FFI BIF layer + +Erlang-on-SX has no FFI / NIF mechanism in its current form (Phase 6 plan: "out of +scope entirely"). fed-sx adds a **BIF layer** in `lib/erlang/transpile.sx` (or a +dedicated `lib/erlang/fed_bifs.sx`) exposing native primitives: + +``` +crypto:rsa_verify/3 crypto:ed25519_verify/3 +crypto:sha2_256/1 crypto:sha3_256/1 + +cid:cbor_encode/1 cid:cbor_decode/1 +cid:multihash/2 cid:from_bytes/2 +cid:to_string/1 cid:from_string/1 + +log:append/2 log:read/3 +log:tip/1 log:replay/3 + +http:listen/2 http:request/2 +http:respond/3 http:sse_send/2 + +fs:read/1 fs:write/2 +fs:exists/1 fs:list/1 + +sqlite:open/1 sqlite:exec/2 +sqlite:query/3 sqlite:close/1 + +snapshot:put/3 snapshot:get/2 +``` + +Each BIF is a thin Erlang-on-SX function dispatching to the corresponding SX runtime +IO primitive. Returns Erlang-shaped values (atoms, tuples, binaries). Errors raise +appropriate Erlang exceptions (`badarg`, `enoent`, `eaccess`). + +This is the **only** native-FFI surface in fed-sx. All other I/O goes through these +BIFs. Operators can audit the BIF list to know exactly what the substrate touches +outside SX. + +### 17.4 Build pipeline + +``` +.sx files (core SX, registry entries) ──┐ +.erl files (Erlang-on-SX kernel) ──┼──> compile to core SX +.dl files (Datalog-on-SX queries) ──┘ + │ + content-addressed SX artifacts + │ + ▼ + genesis bundle (CID-verified) + │ + ▼ + OCaml runtime evaluates everything +``` + +Each authoring language's compiler runs at build time, producing core SX that goes +into the genesis bundle (for bootstrap definitions) or gets published as activities +(for runtime extensions). + +### 17.5 Prerequisite work + +Pieces of investment land in or alongside the Erlang-on-SX loop. The first two +land **before** fed-sx kernel code starts; the third runs in parallel, not +blocking milestone 1, but blocking production-grade throughput. + +1. **Phase 7 — hot code reload.** `code:load_binary/3`, `gen_server` + `code_change/3` callback dispatch, atomic module-version swap. Required for + `Define*` live extension (no kernel restart to load new verbs). Reload- + semantics choice (two-version coexistence vs single-version atomic swap with + closure capture) decided during the work. + +2. **Phase 8 — FFI mechanism + initial BIFs.** `define-bif` registration + term + marshalling + error mapping, then BIFs for `crypto:*`, `cid:*` (dag-cbor), + `fs:*`, `http:*`, `sqlite:*`. Required for fed-sx kernel to call native + primitives. Lands before kernel code that calls them. + +3. **Phase 9 — specialized opcodes (the BEAM analog).** *Layered perf strategy:* + - **Layer 1 (Phase 9, in scope)** — specialized bytecode opcodes that bypass + the general-purpose CEK machine for hot Erlang operations. `OP_PATTERN_TUPLE`, + `OP_PERFORM`/`OP_HANDLE`, `OP_RECEIVE_SCAN`, `OP_SPAWN`/`OP_SEND`, BIF + dispatch table. Targets: 100k+ message hops/sec, 1M-process spawn under + 30sec — roughly 1000-3000× speedup over the current general-purpose path. + - **Layer 2 (Phase 10, deferred)** — multi-core scheduler via OCaml 5 + domains. Decided empirically after Layer 1 lands; likely unnecessary if + Layer 1 alone hits target throughput. + - **Layer 3 (skipped)** — incremental tuning of the existing call/cc-based + receive and env-copy-per-call machinery. Obsoleted by Layer 1; not pursued. + + **Architectural note for Phase 9.** Phase 9a (the **opcode extension + mechanism in `hosts/ocaml/evaluator/`**) is out of scope for the Erlang loop + — it's SX VM core, used by every language port that wants specialized + opcodes. Designed in `plans/sx-vm-opcode-extension.md`; lands as a separate + focused workstream (~1-2 weeks) owning `hosts/`. Phase 9b-9g (the actual + Erlang opcodes in `lib/erlang/vm/`) are designed and tested against a stub + dispatcher in the Erlang loop until 9a is available. + + **Shared-opcode discipline.** Opcodes Phase 9 produces that other language + ports could plausibly use (pattern match, perform/handle, record access) + become candidates for chiselling out to **`lib/guest/vm/`** — same lib/guest + discipline, applied at the bytecode layer. Don't pre-extract; promote to + `lib/guest/vm/` when a second language port has an actual second use. The + substrate accumulates a richer opcode surface over time as ports contribute, + and every port benefits from every shared opcode (the structural advantage + over BEAM, which is special-purpose-built for one language). + + **fed-sx is not blocked by Phase 9.** Milestone 1 ships on current Erlang- + on-SX perf (which has 100-1000× headroom for a single demo instance). Phase + 9 lands in parallel; by the time fed-sx needs production-grade throughput + (federation hub use cases, milestone 2-3), Phase 9 is ready. + +After Phases 7 and 8 land, fed-sx milestone 1 (kernel + registries + bootstrap +entries + Pin smoke test + reactive application smoke test) becomes the next +workstream. Phase 9 work continues in parallel. + +--- + +## 18. Subscription model + +Symmetric to the publish-side extensibility: just as `DefineActivity` registers what +*kinds of things can be published*, `DefineSubscription` registers what *kinds of +patterns can be subscribed to*. `Follow` becomes one standard subscription type +among many, not a hardcoded primitive. + +### 18.1 The asymmetry being fixed + +Without this, the substrate has rich publish-side extensibility (any new verb is a +`DefineActivity`) and *one* hardcoded subscription primitive (`Follow`). That +mirrors AP but it's an arbitrary limitation in a substrate where everything else +is registry-driven. Generalising restores symmetry. + +### 18.2 The `DefineSubscription` shape + +```sx +(activity 'Create + :object {:type "DefineSubscription" + :name "Follow" ; AP-standard + :schema (fn (sub) ; what params the sub takes + (and (cid? (-> sub :object)) + (= "Person" (-> sub :object-type)))) + :match (fn (subscription activity) ; pure-mode predicate + (= (-> subscription :object) (:actor activity))) + :delivery {:default :push + :modes [:push :pull :sse] + :digest-window nil} + :capabilities-required []}) ; some subs may need authority +``` + +Four mandatory parts: + +- **`schema`** — pure-mode predicate validating subscription parameters at + `Subscribe` time. Catches malformed subscriptions before they enter state. +- **`match`** — pure-mode predicate `(subscription, activity) → bool`. Decides + whether a given activity is a hit for this subscription. Determinism rules + apply (§11.2). +- **`delivery`** — supported modes (push to inbox / pull on demand / SSE + streaming / batched digest). The subscription instance picks its preferred + mode at `Subscribe` time from the supported set. +- **`capabilities-required`** — capability tokens the subscriber must hold + (empty for public subs; populated for paywalled/gated/private streams). + +### 18.3 The `Subscribe` verb + +The bootstrap verb that activates a subscription: + +```sx +(activity 'Subscribe + :object {:type "Follow" :object "https://alice.example/actors/alice"}) + +(activity 'Subscribe + :object {:type "Topic" :tag "climate-change" + :delivery :digest :digest-window "P1D"}) + +(activity 'Subscribe + :object {:type "CidWatch" :cid "bafy..." + :events [:supersede :endorse]}) + +(activity 'Subscribe + :object {:type "Predicate" + :pred '(fn (act) (and (= (:type act) "Note") + (string-contains? (-> act :object :content) "fed-sx")))}) +``` + +`Unsubscribe` is `Undo{Subscribe}` — AP's standard pattern, retains audit. + +### 18.4 Standard subscription types (defined later, not bootstrap) + +Same status as the custom verbs in §6.2 — substrate accepts any subscription +type once a `DefineSubscription` artifact registers it. Standard set: + +| Name | Params | Match semantics | Use case | +|------|--------|-----------------|----------| +| **`Follow`** | `{object: actor-id}` | activity.actor == subscription.object | AP-standard actor following | +| **`Topic`** | `{tag: string}` | tag in activity.object.tags | Hashtag follows, RSS-like | +| **`CidWatch`** | `{cid, events: [...]}` | activity references cid AND activity.type in events | "Notify me when this artifact is updated/endorsed/forked" | +| **`PathWatch`** | `{path, events: [...]}` | activity is a Pin/Update of named path | "Notify me when domain:foo/bar/baz changes" | +| **`VerbFilter`** | `{wraps: subscription-cid, types: [...]}` | inner subscription matches AND activity.type in types | "Follow Alice but only Endorse activities" | +| **`TrustGraph`** | `{root: actor-id, depth: int}` | activity.actor reachable from root in trust graph at depth | Web-of-trust expansion | +| **`Predicate`** | `{pred: sx-fn}` | (pred activity) returns truthy | Escape hatch — most powerful, highest cost | +| **`Channel`** | `{channel-id}` | activity addresses or originates from channel | Multi-actor pooled streams | + +### 18.5 Match-fn execution location + +The load-bearing question. Three choices, fed-sx adopts the **hybrid model**: + +- **Coarse filter on the publisher side** — audience predicates (§8) decide who + the activity is delivered to at all. This is mandatory and cheap (audience set + is usually small and well-defined). +- **Fine filter on the subscriber side** — once an activity arrives in inbox, + the subscriber's instance evaluates each active subscription's `match-fn` + against it. Pure-mode evaluation (deterministic, gas-bounded). Activities + matching one or more subscriptions enter the subscriber's projected state. + +Why hybrid: publisher-side fine filtering would require the publisher to know +every subscriber's match-fn (privacy-violating, scaling-killing). Subscriber-side +filtering is wasteful only if the publisher's audience model is too coarse — +which is the audience system's job to fix per §8. + +### 18.6 Subscription state and storage + +Active subscriptions are themselves projected state. A bootstrap projection +`subscriptions` (paralleling `audience-graph` for the inverse direction) +maintains: + +``` +{actor-id -> [{subscription-cid, type, params, mode, started-at}]} +``` + +Updated by `Subscribe` and `Unsubscribe` activities. Queryable like any other +projection (§16). Used by: + +- The inbox dispatcher to know which match-fns to evaluate against incoming + activities +- Triggers (§19) to know which activities to fire on +- Federation to advertise "here are the subscription types I currently subscribe + to" (capability-style, opt-in) + +### 18.7 Federation interactions + +Subscriptions interact with federation in three ways: + +- **Discovery.** Peer's `/.well-known/sx-capabilities` (§7) lists registered + `DefineSubscription` CIDs, so subscribers know what they can ask for. +- **Negotiation.** A `Subscribe` activity carries `capabilities-required`; if + the publisher's instance doesn't support the named subscription type, it + responds with the standard 422 + missing-CIDs error (§14.2 #9). Subscriber + can then deliver the bootstrapping `DefineSubscription` artifact and retry. +- **Cross-instance match-fn**. If subscriber and publisher both run the same + conformance-tested SX evaluator, identical subscriptions match identically + (cross-host equivalence, §11.8). This is what makes federated topic + subscriptions reliable: every conforming instance computes the same + set-of-matches for the same activity. + +### 18.8 Operational implications + +- **The audience system handles "who do I send this to."** The subscription + system handles "what do I want to receive." They're complementary, not + redundant. +- **Subscription types can themselves evolve via supersession.** New version of + `Topic` with case-insensitive matching? Publish a new `DefineSubscription`, + `Supersede` the old one. Existing subscriptions migrate at next match + evaluation. +- **Match-fn cost matters.** A `Predicate` subscription with a slow predicate + becomes a per-activity tax. Gas budgets (§11.5) bound the worst case; + operators can disable expensive subscription types if needed. +- **Subscriptions are signed messages.** Audit, accountability, and revocation + all work the same way as activities — because subscriptions *are* activities. + +--- + +## 19. Application model + +The synthesis. With publish, subscribe, project, and trigger as registry-driven +primitives, the substrate has everything needed to express **distributed reactive +applications** as data — no native code, no kernel changes, no privileged +runtime. Applications are themselves federated artifacts. + +### 19.1 An application is a tuple of artifacts + +``` +Application = { + subscriptions : [DefineSubscription instances and their parameters], + triggers : [DefineTrigger registrations], + projections : [DefineProjection registrations], + storage : [DefineStorage registrations] (optional) +} +``` + +That tuple, signed and bundled, is the application. Installing one = following +the named actors / activating the named subscriptions + loading the Define* +CIDs into the local registry. Forking one = republishing the Define* with +`Supersede` over the bits you change. + +### 19.2 The reactive loop + +``` + External actors Operator publishes activities + publish activities via this instance's actors + │ │ + ▼ ▼ + ┌─────────────────────────────────────────────┐ + │ Inbound + outbound activities │ + └────────────────────┬────────────────────────┘ + │ + ▼ + For each active subscription: + evaluate match-fn (pure mode) + │ + ┌─────────────┴─────────────┐ + ▼ ▼ + Activity matches Activity does + a subscription not match + │ │ + ▼ ▼ + Projections ← (silently dropped from + fold the activity this application's view; + │ may match other apps) + ▼ + Triggers fire on the + subscription's match + │ + ▼ + Trigger then-sx runs + (effectful sandbox) + │ + ├──> updates local state (private projections) + ├──> publishes new activity (via outbox) + └──> calls effectful primitives (HTTP, fs, etc.) + per declared capabilities +``` + +Three things happen on a match: **state updates** (projection), **derived +publishes** (new activities), **side effects** (effectful primitives). Each is +authorisation-gated by the trigger's declared capabilities. + +### 19.3 Trigger semantics + +`DefineTrigger` registers `(when-subscription, then-sx, cascade-limit)`: + +- **`when-subscription`** — references a subscription (by CID or by name). The + trigger fires whenever that subscription matches an inbound or outbound + activity. Multiple triggers can reference the same subscription. +- **`then-sx`** — function of `(activity, subscription, env) → trigger-result`. + Runs in pure or effectful sandbox per declaration. Returns one or more of: + - `:publish [activity-spec ...]` — request publish of derived activities + - `:project [name → state-update ...]` — request projection updates + - `:effect [capability-call ...]` — request effectful primitive calls + - `:noop` — observed but no action +- **`cascade-limit`** — bounded depth for trigger cascades (§19.4). + +A trigger is fundamentally **a reactive rule**: "when X happens, do Y." The +substrate guarantees Y happens at most once per X (deduplicated by activity-CID), +exactly-once-per-instance (delivery from trigger to its effects is durable), +and bounded-cost (gas + cascade-limit). + +### 19.4 Cascade control + +A trigger that publishes activities can fire other triggers. Without limits, a +single inbound activity could cascade across instances forever. + +Each trigger declares `cascade-limit: N` (default 3). Each activity carries an +implicit `cascade-depth` field, incremented when it's the result of a trigger +firing. A trigger refuses to fire if `cascade-depth > cascade-limit`. + +Cascade limits are local-only (operator policy, not federated). Defending +against runaway cascades from peer instances is the operator's job; the +substrate gives them the knob. + +### 19.5 The `DefineApplication` bundle + +A bundle artifact that names and groups the components of an application: + +```sx +(activity 'Create + :object {:type "DefineApplication" + :name "rose-ash-blog" + :version 1 + :subscriptions [{:type "Follow" :object "https://blog.rose-ash.com/actors/main"} + {:type "Topic" :tag "rose-ash"} + {:type "CidWatch" :cid + :events [:supersede]}] + :triggers [ + + ] + :projections [ + ] + :storage [] + :capabilities [ + ] + :description "Federated blog with moderated comments and RSS"}) +``` + +Three operations on applications, all themselves activities: + +- **Install** — `Subscribe` to each subscription, `Create{}` references in + `define-registry` to each trigger/projection/storage CID. One activity per + reference, audited and replayable. Or: a single `Install{DefineApplication}` + meta-verb that does the bundle in one signed step (defined later as a custom + verb, not bootstrap). +- **Update** — publish a new `DefineApplication` with the same name + + `supersedes` pointing at the old. Diff-then-apply: subscriptions added/ + removed, triggers loaded/unloaded, projections reprojected per §10.5. +- **Fork** — publish a new `DefineApplication` referencing the original's CID + via `forked-from`, with whatever Define* CIDs you want to swap. Run alongside + the original or in place of it. + +### 19.6 Per-application namespacing + +Multiple applications running on one instance need isolation: + +- **Projections are namespaced by application.** `pin-state` from app A is + distinct from `pin-state` from app B — both addressable as + `/projections//pin-state`. +- **Triggers fire only on subscriptions belonging to their application.** App + A's trigger doesn't see app B's subscription matches. +- **Storage backends are namespaced.** App A's `files-on-disk` backend writes + to `data/apps/A/objects/`; app B writes to `data/apps/B/objects/`. +- **Capabilities are per-application.** Granting `http-client` to app A + doesn't grant it to app B. Operator can audit per-app capability surface + and revoke selectively. + +Cross-application reads are explicit and require a capability grant +(`read-projection: /`). Default isolation; opt-in sharing. + +### 19.7 Worked examples + +#### Example A — Blog with moderated comments + +``` +DefineApplication "blog-with-comments": + subscriptions: + - Follow: + - Topic: "post-comment" (filter: object.in-reply-to in our-posts) + triggers: + - on Topic match → publish Note (the new comment, derived if approved) + → projection pending-moderation + - on inbound Approve{Reply} → projection comment-thread (visible) + projections: + - comment-thread: post-cid → [approved comment activities] + - pending-moderation: list of pending replies awaiting approval +``` + +#### Example B — Continuous integration + +``` +DefineApplication "ci-pipeline": + subscriptions: + - Follow: + - VerbFilter: wraps Follow, types: [Push] + triggers: + - on Push match → effect: run build (capability: subprocess + fs-write) + → publish Build{source: Push.cid, output: , status} + - on Build{status: success} → effect: run tests + → publish Test{...} + - on (Test{passed} count for N days) → publish Release{...} + projections: + - build-history: commit-cid → [build activities] + - release-history: ordered list of Release activities +``` + +#### Example C — Distributed code review + +``` +DefineApplication "code-review": + subscriptions: + - Topic: "review-request" + - CidWatch: , events: [Endorse] + triggers: + - on review-request match → projection review-queue + → effect: notify-reviewer + - on Endorse from authorised reviewer → publish Approve{review-cid} + → projection approval-state + projections: + - review-queue: ordered list of pending requests with summaries + - approval-state: review-cid → endorsement set +``` + +In all three: the application is *just* the bundle of subscriptions, triggers, +and projections. Federation makes them composable across instances. The +substrate provides exactly-once-per-CID semantics and pure-mode determinism for +the matches and folds. + +### 19.8 Composition and discovery + +Applications are themselves federated content. This means: + +- **App registries** — actors can publish curated lists of applications they + endorse. Discovery becomes follow-an-actor + browse-their-app-list. +- **Cross-app composition** — application A publishes derived activities that + application B subscribes to. Pipeline of applications via the activity log. +- **App marketplaces** — pin a friendly path to a `DefineApplication` CID + (`rose-ash.com:apps/blog → bafy...`) for human discoverability. + +None of this requires kernel changes. It's all activities about activities. + +### 19.9 Operational implications + +- **Applications are inspectable from the activity log alone.** Replay an + actor's outbox and you can reconstruct the exact application installation + state at any point in time. +- **Application updates are atomic relative to the activity log.** Either the + `Update{DefineApplication}` succeeded (new state visible from next activity) + or it didn't (old state continues). No partial-update window. +- **Forking is the same as installing a copy.** No special "fork" mechanism + needed; the activity-log mechanics already support it. +- **Per-app capabilities are a real security surface.** Operators must + understand what they're granting when they install. The bundle's + `capabilities` list is the audit point — should be human-readable and + reviewable before installation. +- **The substrate isn't an "application platform" — it's an "application + substrate."** Applications aren't installed *on* fed-sx; they're expressed + *in* fed-sx, as the same kind of content as everything else. + +--- + +## Appendix A: relationship to adjacent systems + +Worth knowing about so we can borrow good ideas: + +- **ATproto / Bluesky** — Lexicons (schemas) + repos (per-actor signed merkle trees). + Closest in spirit. We borrow the schema-as-data idea; we differ by making schemas + themselves federated activities, not central registry entries. +- **Spritely Goblins** — capability-secure actors. We borrow the capability-token + pattern for delegation. +- **Ceramic** — signed event streams, content-addressed. Similar log-as-state model; + we differ by making the projection function pluggable per-stream rather than + hardcoded per-streamtype. +- **Holochain** — agent-centric DHT. We share the "every agent has their own log" + shape; we use AP federation instead of DHT. +- **Farcaster** — pubsub on hubs. We share the firehose model; we add cryptographic + outbox-as-source-of-truth. + +None of them are *code-as-data the whole way down* — that's the SX-distinctive bit. +Handlers, validators, projections aren't bytecode shipped out-of-band; they're SX in +the same log as everything else, evaluable by any host that speaks SX. + +## Appendix B: implications worth sitting with + +- **Deployment dissolves.** Releasing a feature = publishing `DefineActivity{name: + "Whatever", ...}`. Federation distributes it. No build artifact, no rolling deploy, + no version-skew between server and client. +- **Applications are forkable by default.** "Fork the rose-ash blog" = take the bundle + of `Define*` CIDs that constitute it, publish your own with `Supersede` over the + ones to change, run your own projector. Same federation graph, divergent state. +- **Composition is by reference, not import.** `Pin` activity points at the CID of the + `DefineActivity{name: "Pin"}`. No package manager, no transitive deps, no lockfiles. +- **The boundary between "user" and "developer" softens.** Both publish signed + activities. Power users can publish handlers, projections, sig suites under their + own actor. +- **This is more ambitious than a rose-ash rewrite.** It's a substrate that *happens + to* host rose-ash as its first application. + +--- + +## Appendix C: AI agent collaboration patterns + +The substrate is incidentally well-shaped for one of the open problems of the +next decade: **infrastructure for AI agent collaboration where contributions +are signed federated artifacts, behavior is bounded by declared capabilities, +decisions are audit-by-replay, and infrastructure improves through agent +contribution within a web of trust.** + +This is not a designed-for use case — fed-sx was conceived as a federated +publishing and reactive application substrate. But the properties it has fit +agent collaboration almost exactly. Worth being deliberate about, because the +framing changes who fed-sx is *for*. + +### Why the substrate fits agent collaboration + +AI agents need infrastructure where contributions are first-class artifacts, +not pull requests against human-controlled repos. Currently agents squeeze +through GitHub PRs, deployment pipelines, npm publishes — all of which assume +a human in the loop. fed-sx is shaped for direct contribution: + +- **Direct authoring of substrate features.** An agent doesn't *propose* a + feature, it *publishes* one. A `DefineActivity` artifact is the agent's + contribution. A `DefineProjection` is its analysis. A `DefineTrigger` is its + automation. The signed publication IS the deploy — no PR review, no CI, no + DevOps. +- **Cryptographic identity without registration.** Agents have actor keys; + reputation is the endorsement graph; trust is provable by signature chain. + Two agents that have never met can verify each other's contributions + cryptographically. +- **Capability-bounded autonomy.** An agent declares `capabilities-required` on + its activities. A trigger says "I publish to path-prefix `/agent-x/*` and + call `http-client` for `api.example.com/*`." Receivers verify the constraint + cryptographically; the agent can't escape its declared surface even if the + agent itself is misaligned. Sandbox model designed for autonomous code (§11). +- **Audit-by-replay applied to AI behavior.** Every AI decision is + reconstructable, deterministically, by anyone with the log. "Why did agent A + do X?" replay the log to that moment, see the activities A subscribed to, + the projection state it observed, the trigger that fired, the activity it + published. Fundamentally better than today's "trust the model" posture. +- **Composition without coordination.** Agent A publishes a moderation + validator. Agent B subscribes and uses it. Agent C improves it, supersedes + A's. B sees the supersession, decides whether to adopt. No central registry, + no maintainer to coordinate with, no version skew. +- **Disagreement is visible, not hidden.** If agents A and B compute the same + projection over the same log and produce different snapshot CIDs, the + disagreement is *cryptographically observable*. Today, two AI services + answering the same question with different answers is invisible until + somebody notices. + +### Dynamics that emerge + +- **Agent specialisation = publication.** "I'm the indexing agent" = publishes + `DefineProjection` artifacts. "I'm the moderation agent" = publishes + `DefineValidator` artifacts. "I'm the matchmaking agent" = publishes a + `DefineApplication` for marketplace subscriptions and triggers. Specialisation + is content, not service deployment. +- **Reputation = endorsement graph.** Web of trust applied to agent + contributions. Bad actors get cut out organically; no central authority to + capture. +- **Forking = explicit disagreement resolution.** Agents disagree on + validation? Both publish their `DefineValidator`s. Subscribers pick. The fork + is signed, observable, recoverable. Compare today: when AI services have + different rules, one is just *invisibly applied*. +- **Cascade limits = agent population safety.** The `cascade-depth` and + `cascade-limit` (§19.4) become the bounded-autonomy guard rails for agent + populations. Self-coordination without runaway-cascade across the substrate. +- **Self-improving infrastructure.** Agents observe substrate behavior, propose + improvements as `DefineProjection` for monitoring, `DefineTrigger` for + automation. The substrate itself improves through agent contribution — not + through a release cycle. Every improvement is signed and traceable. + +### Use cases + +- **Agent-managed scientific datasets** — collection, cleaning, analysis, + publication, peer review by other agents, all signed activities. Replication + is replay; provenance is built in. +- **Multi-agent code maintenance** — agents observing repos (subscribe to + `Push`), running tests (triggers), proposing fixes (`Pull`-equivalent + activities), endorsing each other's work. +- **Agent-curated knowledge** — agents publish, endorse, and supersede + knowledge artifacts. Truth accumulates via the trust graph; outdated info + gets `Supersede`d explicitly. +- **Distributed agent marketplaces** — agents publish capabilities, subscribers + find them via `Topic` / `Predicate` subscriptions, contracts via signed + activity exchange. +- **Cross-agent AI safety monitoring** — monitoring agents subscribe to other + agents' outboxes, run validators, publish `Alert` activities when patterns + of concern appear. Decentralised oversight without central authority. +- **Cross-org agent workflow coordination** — supply chain, healthcare, legal — + multiple specialised agents coordinating across organisational boundaries + with cryptographic provenance. + +### Safety and governance properties + +The substrate provides several properties AI safety has been asking for and +that current infrastructure does not provide: + +- **Every action is signed.** Attribution is cryptographic, not a log file an + agent could spoof. +- **Capabilities are declared and enforced.** Agents operate within their + declared sandbox; can't grow capabilities silently. +- **Cascades are bounded.** No exponential agent-on-agent feedback loops + without explicit configuration. +- **Audit is replay.** Every decision can be reconstructed deterministically; + no opaque "the model decided" moments. +- **Disagreement is visible.** Two agents producing different projections of + the same data is a cryptographically-detectable event, not invisible drift. +- **Trust is the endorsement graph, not central authority.** No single point of + capture or coercion. +- **Forks are first-class.** When safety-critical disagreements occur, the + substrate accommodates them without forcing a winner; observers see all + positions. + +### What this implies for the project + +- **Milestone 1's smoke tests remain right** — the verb-extensibility and + reactive-application proofs apply to agent contributions exactly as they + apply to human contributions. The agent collaboration framing doesn't + require new mechanisms; it interprets the existing mechanisms differently. +- **The application model (§§18-19) is the headline story** for this audience, + not a layer on top. Subscriptions + triggers + projections + capabilities = + agent collaboration primitives. +- **Capability discovery and trust dynamics gain weight earlier.** Where + human-driven applications can rely on operator policy, agent-driven + populations need the trust graph to be operational from milestone 2. +- **The pitch line evolves.** Less "ActivityPub for code" / "rose-ash next + gen," more "infrastructure for AI agent collaboration with cryptographic + provenance, bounded autonomy, and audit-by-replay." The technical substance + is unchanged; the framing of *who needs this* changes substantially. + +The substrate accidentally being well-shaped for the most important +software-distribution problem of the next decade is worth being deliberate +about. + diff --git a/plans/fed-sx-milestone-1.md b/plans/fed-sx-milestone-1.md new file mode 100644 index 00000000..de7a3e60 --- /dev/null +++ b/plans/fed-sx-milestone-1.md @@ -0,0 +1,922 @@ +# fed-sx Milestone 1 — Kernel + Registries + Pin Smoke Test + +Concrete implementation plan for the smallest fed-sx that proves the architecture +works end-to-end. Reference: `plans/fed-sx-design.md`. Prerequisite: Erlang-on-SX +Phases 7 (hot reload) + 8 (FFI BIFs). + +## Goal + +Ship a single-instance, single-actor fed-sx server that: + +1. Boots from a verified genesis bundle. +2. Accepts and durably appends signed activities via `POST /activity`. +3. Folds them into projections in real time. +4. Serves AP-standard endpoints (actor, outbox, artifacts, capabilities). +5. Demonstrates **two extensibility proof-points** end-to-end with zero kernel + code changes between definition and use: + - **Verb extensibility** (§5 meta-level): publish `DefineActivity{Pin}` + + `DefineProjection{pin-state}`, then publish a `Pin` activity, observe it + validated and projected. + - **Reactive application extensibility** (§§18-19): publish + `DefineSubscription{Topic}` + `Subscribe{topic: smoketest}` + + `DefineTrigger{when: that subscription, then: publish TestEcho}`, then + publish a tagged Note, observe the subscription match, the trigger fire, + and the derived activity appear in the outbox. + +Federation, multi-actor, advanced verbs, IPFS, browser UI, operator dashboard +are **explicitly v2**. + +## Non-goals (what milestone 1 deliberately does NOT do) + +- **Federation.** No `POST /inbox` from peers, no `Follow`, no delivery queue, no + webfinger discovery flow. Single instance only. +- **Multi-actor.** Single domain actor (`acct:next@next.rose-ash.com`). +- **IPFS / S3 storage backends.** Files on disk only. +- **Advanced verbs.** No `Endorse`, `Supersede`, `Test`, `Build`, `Compose`, + `Note`, `Announce`. Only the four bootstrap verbs (`Create`, `Update`, `Delete`) + plus a defined-from-the-log `Pin` for the smoke test. (`Announce` deferred — + no use case until federation exists.) +- **Browser UI.** Curl-shaped API only. +- **Operator dashboard, quarantine UX.** Logs only. +- **Performance work.** Functional correctness first; perf when measured. +- **Cross-host conformance test corpus.** Only the OCaml/Erlang-on-SX host runs + fed-sx in v1; conformance suite for other hosts is v2. + +## Architecture summary + +``` + POST /activity + │ + ▼ + ┌──────────────────────────┐ + │ HTTP server (Erlang-on-SX)│ + └─────────────┬─────────────┘ + │ + ┌─────────────▼──────────────┐ + │ Validation pipeline driver │ + │ (envelope→sig→schema→...) │ + └─────────────┬──────────────┘ + │ + ┌─────────────▼──────────────┐ + │ Log append (JSONL segment) │ ← canonical + └─────────────┬──────────────┘ + │ + ┌─────────────▼──────────────┐ + │ Projection workers │ ← gen_server per + │ (fold scheduler) │ projection + └─────────────────────────────┘ + │ + ▼ + Projection state + (queryable via HTTP) + +Native primitives (Erlang-on-SX BIFs from Phase 8): + crypto:* cid:* fs:* http:* sqlite:* + +Genesis bundle (binary-embedded SX): + activity-types object-types projections + validators codecs sig-suites +``` + +## Build order + +Eight steps in dependency order. Each step has concrete deliverables, testable +in isolation, and a clear acceptance check. + +| Step | Title | Depends on | +|------|-------|------------| +| **1** | Repo skeleton + canonical CID computation | Phase 8 (cid BIFs) | +| **2** | Activity envelope + signature verify | Phase 8 (crypto BIFs) | +| **3** | JSONL log + sequence numbers | Phase 8 (fs BIFs) | +| **4** | Genesis bundle (SX sources + bundling + CID verification) | Step 1 | +| **5** | Registry mechanism + bootstrap-projection dispatch | Steps 2, 4 | +| **6** | Validation pipeline driver + `POST /activity` | Steps 2, 3, 5 | +| **7** | Projection scheduler (gen_server per projection) | Steps 5, 6 | +| **8** | HTTP server, AP endpoints, projection queries | Steps 6, 7 | +| **9** | Smoke tests (Pin verb + reactive application) | Steps 1-8 | + +--- + +## Step 1 — Repo skeleton + canonical CID + +**Deliverables:** + +``` +next/ +├── README.md # what this is +├── kernel/ # Erlang-on-SX +│ └── (empty for now) +├── genesis/ # core SX bootstrap definitions +│ └── (empty for now) +├── tests/ # smoke test scripts +│ └── (empty for now) +└── data/ # gitignored runtime state + ├── log/ + ├── objects/ + ├── snapshots/ + ├── indexes/ + └── keys/ +``` + +Plus one Erlang-on-SX module: + +```erlang +% next/kernel/cid.erl +-module(cid). +-export([from_sx/1, to_string/1, from_string/1, equals/2]). + +from_sx(SxValue) -> + Cbor = cid:cbor_encode(canonicalize_sx(SxValue)), + Hash = crypto:sha2_256(Cbor), + cid:from_bytes(<<"raw">>, Hash). % defaults to dag-cbor codec + +canonicalize_sx(V) -> ... % sorts dict keys, normalizes strings +``` + +**Tests:** +- Same SX value → same CID across multiple invocations. +- Different SX values → different CIDs. +- Whitespace/comment differences in source → identical CIDs (parsed AST identical). +- Reordered dict keys → identical CIDs (sorted-key canonicalization). +- Cross-host parity (just OCaml host for v1, but write the test so adding hosts is mechanical). + +**Acceptance:** `bash next/tests/cid.sh` passes 10+ cases. + +--- + +## Step 2 — Activity envelope + signature verify + +**Deliverables:** + +```erlang +% next/kernel/envelope.erl +-module(envelope). +-export([validate_shape/1, canonical_bytes/1, verify_signature/2]). + +% Envelope shape per design §3.1: +% #{id, type, actor, published, to, cc, audience_extras, +% object | target | origin | result, +% capabilities_required, proofs, signature} +validate_shape(Activity) -> ok | {error, Reason}. + +canonical_bytes(Activity) -> + % Strip signature, canonicalize via dag-cbor, return bytes for sig coverage + Stripped = maps:remove(signature, Activity), + cid:cbor_encode(canonicalize_for_sig(Stripped)). + +verify_signature(Activity, ActorState) -> + % Time-aware: find key with id == sig.key_id that was active at published + % Per design §9.6 + ... +``` + +**Tests:** +- Envelope shape: required fields present (id, type, actor, published, signature) +- Envelope shape: type is a known activity-type or unknown-but-string +- Envelope shape: signature has key_id, algorithm, value +- Sig verify: valid RSA-SHA256 signature against published key → ok +- Sig verify: valid Ed25519 signature → ok +- Sig verify: tampered envelope → fail +- Sig verify: key superseded before activity timestamp → fail +- Sig verify: key superseded after activity timestamp → ok (historical valid) + +**Acceptance:** `bash next/tests/envelope.sh` passes 15+ cases. + +--- + +## Step 3 — JSONL log + sequence numbers + +**Deliverables:** + +```erlang +% next/kernel/log.erl +-module(log). +-export([open/1, append/2, read_segment/2, tip/1, replay/3]). + +% Per design §15.2: per-actor outbox, segments cap ~64MB, +% format = JSONL (one canonical JSON-LD activity per line) + +open(ActorId) -> + BasePath = log_path_for_actor(ActorId), + fs:mkdir_p(BasePath), + {ok, #{base => BasePath, current => current_segment(BasePath), seq => next_seq(BasePath)}}. + +append(LogState, Activity) -> + Json = jsonld:encode(Activity), + Path = current_segment_path(LogState), + Line = <>, + fs:append_file(Path, Line), + NewSeq = LogState#{seq := LogState.seq + 1}, + rotate_if_needed(NewSeq). + +% replay/3 calls Fun(Activity, Acc) for every activity in chronological order +replay(LogState, InitAcc, Fun) -> ... +``` + +**Tests:** +- Append + read back gives identical activity (round-trip). +- Sequence numbers monotonic and gap-free per actor. +- Segment rotation at size threshold. +- Replay visits all activities in append order across multiple segments. +- Restart preserves tip pointer (seq number resumes correctly). +- Concurrent appends (using gen_server-mediated access) are serialized correctly. + +**Acceptance:** `bash next/tests/log.sh` passes 10+ cases. + +--- + +## Step 4 — Genesis bundle + +**Deliverables:** + +Genesis bundle SX sources (per design §12.2). Each is a small SX file authored +by hand for the bootstrap set: + +``` +next/genesis/ +├── manifest.sx # bundle root: lists all definitions +├── activity-types/ +│ ├── create.sx # DefineActivity{name: "Create", ...} +│ ├── update.sx +│ └── delete.sx +├── object-types/ +│ ├── sx-artifact.sx +│ ├── note.sx +│ ├── tombstone.sx +│ ├── define-activity.sx # DefineObject for the Define* meta types +│ ├── define-object.sx +│ ├── define-projection.sx +│ ├── define-validator.sx +│ ├── define-codec.sx +│ ├── define-sig-suite.sx +│ └── snapshot.sx +├── projections/ +│ ├── activity-log.sx # identity projection +│ ├── by-type.sx +│ ├── by-actor.sx +│ ├── by-object.sx +│ ├── actor-state.sx +│ ├── define-registry.sx # the chicken-and-egg projection +│ └── audience-graph.sx +├── validators/ +│ ├── envelope-shape.sx +│ ├── signature.sx +│ └── type-schema.sx +├── codecs/ +│ ├── dag-cbor.sx # delegates to cid:cbor_encode/decode BIFs +│ ├── raw.sx +│ └── dag-json.sx +├── sig-suites/ +│ ├── rsa-sha256-2018.sx +│ └── ed25519-2020.sx +└── audience/ + ├── public.sx + ├── followers.sx + └── direct.sx +``` + +Plus a build-time bundler: + +```erlang +% next/kernel/bootstrap.erl +-module(bootstrap). +-export([build_genesis/1, verify_genesis/1, load_genesis/1]). + +build_genesis(SourceDir) -> + % Walk SourceDir, parse each .sx file, build a single dag-cbor bundle, + % compute its CID, write bundle.cbor + CID to data/genesis/ + ... + +verify_genesis(BundlePath) -> + % Compute CID of the bundle as loaded; compare to expected (hardcoded + % in the kernel binary). Mismatch → halt. + ... + +load_genesis(BundlePath) -> + % Parse the bundle, register all definitions in the in-memory registry + ... +``` + +**Tests:** +- All genesis SX files parse cleanly. +- Bundle CID is deterministic (rebuild same sources → same CID). +- Bundle reload reproduces the exact same registry state. +- Tampered bundle → `verify_genesis` returns `{error, cid_mismatch}`. + +**Acceptance:** `bash next/tests/bootstrap.sh` passes; `next/data/genesis/bundle.cbor` +created with a known stable CID. + +--- + +## Step 5 — Registry mechanism + bootstrap dispatch + +**Deliverables:** + +Registries are gen_servers, one per kind, each holding the active version map: + +```erlang +% next/kernel/registry.erl +-module(registry). +-behaviour(gen_server). +-export([start_link/0, lookup/2, register/3, list/1]). +% Internal state: +% #{activity_types => #{Name => #{cid, schema_fn, semantics_fn, supersedes}}, +% object_types => ..., +% projections => ..., +% validators => ..., +% codecs => ..., +% sig_suites => ..., +% ...} + +lookup(Kind, Name) -> {ok, Entry} | {error, not_found}. +register(Kind, Name, Entry) -> ok | {error, Reason}. +list(Kind) -> [#{name, cid}]. +``` + +The `define-registry` projection's fold updates this gen_server's state when +new `Define*` activities arrive. (Bootstrapping circle resolved: at startup, +`bootstrap:load_genesis/1` populates the registry directly; from then on, the +projection fold maintains it.) + +**Tests:** +- After genesis load, `registry:list(activity_types)` returns Create/Update/Delete. +- `registry:lookup(activity_types, "Create")` returns the schema and semantics. +- A new `DefineActivity{name: "Pin"}` activity (synthesised, hand-signed for the + test) routes through the projection fold, ends up in the registry. +- Lookup never caches across activities (verified by introducing a new definition + mid-test and confirming the next lookup sees it). + +**Acceptance:** `bash next/tests/registry.sh` passes 10+ cases. + +--- + +## Step 6 — Validation pipeline + POST /activity + +**Deliverables:** + +```erlang +% next/kernel/pipeline.erl +-module(pipeline). +-export([validate_inbound/1, validate_outbound/1]). + +% Per design §14, run stages in order, halt on first failure. +validate_inbound(Activity) -> + Stages = [ + fun stage_envelope/1, + fun stage_signature/1, + fun stage_replay/1, + fun stage_audience/1, + fun stage_activity_schema/1, + fun stage_object_schema/1, + fun stage_content_validators/1, + fun stage_capabilities/1, + fun stage_trust/1 + ], + run_stages(Activity, Stages). + +validate_outbound(Activity) -> + % Subset of inbound stages (no replay, no trust check; auth done at HTTP layer) + ... +``` + +```erlang +% next/kernel/outbox.erl +-module(outbox). +-export([publish/2]). + +publish(ActorId, ActivityRequest) -> + Activity = construct_envelope(ActorId, ActivityRequest), + Signed = sig:sign(Activity, ActorId), + case pipeline:validate_outbound(Signed) of + ok -> + log:append(actor_log(ActorId), Signed), + projection:async_fold(Signed), + {ok, #{cid => cid:from_sx(Signed), + ap_id => maps:get(id, Signed)}}; + {error, Reason} -> + {error, Reason} + end. +``` + +**Tests:** +- Valid activity through full pipeline → appended to log. +- Bad envelope → 400, not in log. +- Bad signature → 401, not in log. +- Replayed activity → 200 duplicate, not re-appended. +- Schema violation (e.g. Create with no object) → 422. +- Activity logged before projection completes (async). + +**Acceptance:** `bash next/tests/pipeline.sh` passes 15+ cases. + +--- + +## Step 7 — Projection scheduler + +**Deliverables:** + +```erlang +% next/kernel/projection.erl +-module(projection). +-export([start_link/1, async_fold/1, query/2, snapshot/1]). +-behaviour(gen_server). + +% One gen_server per active projection. State: +% #{cid, name, fold_fn, current_state, log_tip, +% snapshot_dir, last_snapshot_at} + +% async_fold/1 broadcasts a new activity to every projection gen_server; +% each folds it into its own state. Failures (gas, sandbox violation) +% tag the activity but don't affect log durability. + +% query/2 returns current state (or state-as-of) +% snapshot/1 forces a snapshot now (also runs periodically) +``` + +```erlang +% next/kernel/sandbox.erl +-module(sandbox). +-export([eval_pure/2, eval_crypto/2, eval_effectful/3]). + +% eval_pure runs an SX function in pure mode: no IO platform, gas budget, +% deterministic. Used by projection folds, validators, audience predicates. +% Wrapper over the SX runtime evaluator with a stripped platform. +``` + +**Tests:** +- New activity → all projections fold it concurrently. +- Projection fold completes within gas budget. +- Gas-exhausting fold → activity tagged, projection state unchanged, no kernel crash. +- Sandbox violation (fold tries IO) → same handling. +- Snapshot create + reload → state matches. +- Snapshot CID stable across kernel restarts. + +**Acceptance:** `bash next/tests/projection.sh` passes 15+ cases. + +--- + +## Step 8 — HTTP server + endpoints + +**Deliverables:** + +Core endpoints (per design §16.1): + +``` +GET /actors/ # actor doc +GET /actors//outbox # OrderedCollection +GET /actors//outbox?page=true # OrderedCollectionPage +POST /activity # publish (auth: bearer token) +GET /artifacts/ # CID-addressed artifact +GET /artifacts//raw +GET /projections # list of projections +GET /projections/ # full state +GET /projections/?at= # time-travel +GET /projections// # indexed lookup +GET /define-registry +GET /.well-known/sx-capabilities +GET /.well-known/webfinger +``` + +```erlang +% next/kernel/http_server.erl +-module(http_server). +-export([start/1, route/1]). + +start(Port) -> + http:listen(Port, fun ?MODULE:route/1). + +route(Request) -> {Status, Headers, Body}. +``` + +Content negotiation per `Accept`: +- `application/activity+json` (default) +- `application/cbor` (dag-cbor) +- `application/json` (compact, no @context expansion) +- `application/sx` + +Auth on `POST /activity`: bearer token from env var `NEXT_PUBLISH_TOKEN`. + +**Tests:** +- Each endpoint returns expected shape for known artifact. +- Content negotiation: same artifact in 4 representations. +- 404 for unknown artifact CID. +- 401 for `POST /activity` without token. +- Pagination: outbox with > 50 activities returns OrderedCollectionPage. + +**Acceptance:** `bash next/tests/http.sh` passes 20+ cases. + +--- + +## Step 9 — Smoke tests + +**The proof points.** Two end-to-end smoke tests demonstrate, between them, that +fed-sx is genuinely a substrate for distributed reactive applications expressed +as data — not a system you extend by writing kernel code. + +- **9a — Pin smoke test (`next/tests/smoke_pin.sh`)** — verb extensibility: + defining a new activity type and projection at runtime via `Define*` + artifacts. Verifies the meta-level (§5). +- **9b — Reactive application smoke test (`next/tests/smoke_app.sh`)** — + application extensibility: defining a new subscription type, subscribing, + registering a trigger, and observing the full reactive loop fire end-to-end + without kernel code changes. Verifies §§18-19. + +Both must pass for milestone 1 acceptance. + +### Step 9a — Pin smoke test + +**Test script:** `next/tests/smoke_pin.sh` + +```bash +#!/usr/bin/env bash +set -euo pipefail + +# 0. Start a fresh fed-sx kernel (background) +./next/scripts/start.sh fresh +sleep 2 +TOKEN=$(cat next/data/keys/publish.token) + +# 1. Verify actor exists +curl -s http://localhost:9999/actors/next | jq -e '.type == "Person"' + +# 2. Verify outbox has actor's first Create{Person} +curl -s http://localhost:9999/actors/next/outbox?page=true \ + | jq -e '.orderedItems | length == 1 and .[0].type == "Create"' + +# 3. Verify Pin is NOT a known activity type +curl -s http://localhost:9999/define-registry?kind=activity_types \ + | jq -e '.[] | select(.name == "Pin") | length == 0' || exit 1 + +# 4. Publish DefineActivity{name: "Pin", schema: ..., semantics: ...} +PIN_DEF=$(cat <<'JSON' +{ + "type": "Create", + "object": { + "type": "DefineActivity", + "name": "Pin", + "schema": "(fn (act) (and (string? (-> act :object :path)) (cid? (-> act :object :cid))))", + "semantics": "(fn (state act) (assoc-in state [:pins (-> act :object :path)] (-> act :object :cid)))" + } +} +JSON +) +curl -s -X POST http://localhost:9999/activity \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/activity+json" \ + -d "$PIN_DEF" | jq -e '.cid' > /dev/null + +# 5. Verify Pin IS now a known activity type +curl -s http://localhost:9999/define-registry?kind=activity_types \ + | jq -e '.[] | select(.name == "Pin") | length == 1' + +# 6. Also publish a DefineProjection{name: "pin-state"} that folds Pin into state +PIN_PROJ=$(cat <<'JSON' +{ + "type": "Create", + "object": { + "type": "DefineProjection", + "name": "pin-state", + "initial-state": "{}", + "fold": "(fn (state act) (if (= (:type act) \"Pin\") (assoc state (-> act :object :path) (-> act :object :cid)) state))" + } +} +JSON +) +curl -s -X POST http://localhost:9999/activity \ + -H "Authorization: Bearer $TOKEN" \ + -d "$PIN_PROJ" | jq -e '.cid' + +# 7. Now publish a Pin activity +PIN=$(cat <<'JSON' +{ + "type": "Pin", + "object": { + "type": "PinSpec", + "path": "/docs/intro", + "cid": "bafyreigh2akiscaildc3xqxx4xqxx4xqxx4xqxx4xqxx4xqxx4xqxx4xqxxe" + } +} +JSON +) +curl -s -X POST http://localhost:9999/activity \ + -H "Authorization: Bearer $TOKEN" \ + -d "$PIN" | jq -e '.cid' + +# 8. Verify Pin appears in outbox +curl -s http://localhost:9999/actors/next/outbox?page=true \ + | jq -e '.orderedItems | map(select(.type == "Pin")) | length == 1' + +# 9. Verify pin-state projection has the entry +sleep 1 # allow async projection +curl -s http://localhost:9999/projections/pin-state \ + | jq -e '."/docs/intro" == "bafyreigh2akiscaildc3xqxx4xqxx4xqxx4xqxx4xqxx4xqxx4xqxx4xqxxe"' + +# 10. Negative test: publish a malformed Pin (missing path) → expect 422 +BAD_PIN='{"type": "Pin", "object": {"cid": "bafy..."}}' +HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST http://localhost:9999/activity \ + -H "Authorization: Bearer $TOKEN" -d "$BAD_PIN") +[[ "$HTTP_STATUS" == "422" ]] || { echo "expected 422, got $HTTP_STATUS"; exit 1; } + +# 11. Restart kernel; verify state recovers +./next/scripts/stop.sh +./next/scripts/start.sh +sleep 2 +curl -s http://localhost:9999/projections/pin-state \ + | jq -e '."/docs/intro" == "bafyreigh2akiscaildc3xqxx4xqxx4xqxx4xqxx4xqxx4xqxx4xqxxe"' + +echo "✓ Pin smoke test passed — verb extensibility demonstrated end-to-end" +``` + +**Acceptance for 9a:** smoke test exits 0. The whole flow happens with **zero +fed-sx kernel code changes** between defining the verb and using it. + +### Step 9b — Reactive application smoke test + +**The bigger proof point.** Demonstrates that fed-sx supports distributed +reactive applications composed of `DefineSubscription` + `DefineTrigger` + +`DefineProjection` — the application model from §§18-19. + +The test runs on a single instance (federation is v2), so the "subscriber" and +"publisher" are the same actor. That's intentional — milestone 1 proves the +mechanism; milestone 2 spreads it across instances. + +**Test script:** `next/tests/smoke_app.sh` + +```bash +#!/usr/bin/env bash +set -euo pipefail + +# Assumes 9a has already run (fresh kernel optional; can run alongside). +TOKEN=$(cat next/data/keys/publish.token) +BASE=http://localhost:9999 + +# 1. Verify "Topic" subscription type and "Subscribe" verb are NOT yet defined. +curl -s "$BASE/define-registry?kind=subscription_types" \ + | jq -e 'map(select(.name == "Topic")) | length == 0' + +# 2. Publish DefineSubscription{name: "Topic", ...} +TOPIC_DEF=$(cat <<'JSON' +{ + "type": "Create", + "object": { + "type": "DefineSubscription", + "name": "Topic", + "schema": "(fn (sub) (string? (-> sub :tag)))", + "match": "(fn (sub act) (and (= (:type act) \"Note\") (member? (-> sub :tag) (or (-> act :object :tags) (list)))))", + "delivery": "{:default :push :modes (list :push :pull)}" + } +} +JSON +) +curl -s -X POST "$BASE/activity" \ + -H "Authorization: Bearer $TOKEN" -d "$TOPIC_DEF" | jq -e '.cid' + +# 3. Verify Topic IS now a known subscription type. +curl -s "$BASE/define-registry?kind=subscription_types" \ + | jq -e 'map(select(.name == "Topic")) | length == 1' + +# 4. Subscribe to the "smoketest" topic. +SUBSCRIBE=$(cat <<'JSON' +{ + "type": "Subscribe", + "object": {"type": "Topic", "tag": "smoketest"} +} +JSON +) +SUB_CID=$(curl -s -X POST "$BASE/activity" \ + -H "Authorization: Bearer $TOKEN" -d "$SUBSCRIBE" | jq -r '.cid') + +# 5. Verify subscriptions projection has the new entry. +sleep 1 +curl -s "$BASE/projections/subscriptions" \ + | jq -e '.["https://next.rose-ash.com/actors/next"] | map(select(.type == "Topic")) | length == 1' + +# 6. Define a projection that records matched activities (per-application +# namespace would happen via DefineApplication in v1.x; for v1 the +# projection is global to the actor). +TOPIC_PROJ=$(cat <<'JSON' +{ + "type": "Create", + "object": { + "type": "DefineProjection", + "name": "topic-events", + "initial-state": "{}", + "fold": "(fn (state act) (if (and (= (:type act) \"Note\") (member? \"smoketest\" (or (-> act :object :tags) (list)))) (assoc-in state [(:cid act)] act) state))" + } +} +JSON +) +curl -s -X POST "$BASE/activity" \ + -H "Authorization: Bearer $TOKEN" -d "$TOPIC_PROJ" | jq -e '.cid' + +# 7. Define a trigger: when a Topic{smoketest} subscription matches, publish +# a TestEcho activity. We need an "Echo" activity type first. +ECHO_DEF=$(cat <<'JSON' +{ + "type": "Create", + "object": { + "type": "DefineActivity", + "name": "TestEcho", + "schema": "(fn (act) (cid? (-> act :object :echoes)))", + "semantics": "(fn (state act) state)" + } +} +JSON +) +curl -s -X POST "$BASE/activity" \ + -H "Authorization: Bearer $TOKEN" -d "$ECHO_DEF" | jq -e '.cid' + +TRIGGER=$(cat <= 1" +curl -s "$BASE/define-registry?kind=triggers" \ + | jq -e 'map(select(.name == "echo-on-smoketest")) | length == 1' + +echo "✓ Reactive application smoke test passed — Subscribe + Trigger + Projection demonstrated end-to-end" +``` + +**What this proves (and what it doesn't):** + +Proves: +- `DefineSubscription` + `Subscribe` mechanism works end-to-end. +- Subscription's `match-fn` evaluates correctly in pure mode against inbound + activities. +- `DefineTrigger` fires on subscription matches. +- Trigger's `then-sx` can publish derived activities (the `:publish` result). +- Cascade-depth metadata propagates correctly. +- Subscription state, trigger registration, and projection state all survive + kernel restart (snapshot + log replay). +- The full reactive application loop works without any kernel code changes + between defining the components and exercising them. + +Does NOT prove (deferred to milestone 2+): +- Cross-instance subscriptions (federation). +- Trigger `:effect` results calling effectful primitives. +- `DefineApplication` bundle install/update/fork. +- Per-application namespace isolation. +- Cascade prevention against malicious cascading from peer instances. + +**Acceptance for 9b:** smoke test exits 0. Like 9a, **zero fed-sx kernel code +changes** between defining the application components and observing them +operate. + +--- + +## Acceptance criteria for milestone 1 + +All of: + +1. **Each step's test suite passes** (`bash next/tests/.sh`). +2. **Both smoke tests pass** (`bash next/tests/smoke_pin.sh` and + `bash next/tests/smoke_app.sh`). +3. **Erlang-on-SX baseline preserved** — adding fed-sx kernel modules in + `next/kernel/*.erl` doesn't break Phase 1-8 conformance. +4. **Restart durability** — kill the kernel mid-write, restart, projections + resume from snapshot, no log corruption. +5. **Manual Mastodon poke** — point a Mastodon account at + `https://next.rose-ash.com/actors/next` and verify the actor doc fetches and + webfinger discovery works (read-only AP interop, no follow). + +## What lands when + +This is the work-order an agent (or human) follows. Steps 1-3 can be done in +parallel after the Erlang Phase 8 BIFs land. Steps 4-7 are sequential. Step 8 +can start in parallel with step 7. Step 9 is the integration test. + +``` +Phase 7+8 (loops/erlang) ───┐ + │ + ▼ + ┌─── Step 1 ──┬─── Step 2 ──┬─── Step 3 + │ │ │ + └─────────────┼─── Step 4 ──┴────┐ + │ │ + └─── Step 5 ───────┤ + │ + Step 6 ─────┤ + │ + Step 7 ─────┤ + │ + Step 8 ─────┤ + │ + Step 9 ─────┘ +``` + +Estimated effort if done by a focused agent loop, one feature per iteration: +~30-50 commits across all 9 steps. Could plausibly be a `loops/fed-sx` workstream +once Phase 7+8 are done. + +## What's deferred to milestone 2 + +- **Federation** (the second-biggest piece). `POST /inbox`, Follow lifecycle, + delivery queue, backfill, capability negotiation between peers. Whole of + design §13. +- **Multi-actor** with per-user OAuth and capability tokens. Design §9.5. +- **IPFS storage backend** as a `DefineStorage` entry. Design §15.3. +- **Browser client + operator dashboard** (probably in Elm-on-SX or similar). +- **Rich verbs**: `Endorse`, `Supersede`, `Test`, `Build`, `Compose`, `Note`, + `Announce`. All defined as `DefineActivity` artifacts, federated. +- **Cross-host conformance** — Python/JS/Haskell hosts running fed-sx. Design + §11.8. +- **OpenTimestamps proofs** as a `DefineProof` entry. +- **Performance work** — JIT-compiled folds, snapshot acceleration, federation + batching. + +Milestone 2 unlocks "real federation between two fed-sx instances." Milestone 3 +is the rose-ash port (blog, market, events, federation, account, orders) as +fed-sx applications. + +--- + +## Appendix A: open questions for milestone 1 + +A few things still under-specified; resolve as work begins. + +1. **HTTP server library.** Does the Phase 8 `http:listen/2` BIF wrap an + existing OCaml HTTP server (the sx.rose-ash.com one) or something simpler? + Implementation choice deferred to Phase 8. +2. **JSON-LD library.** AP wire format requires JSON-LD canonicalization for + signature coverage. Either pull a library or write a minimal subset for the + shapes we actually use. Probably the latter — our envelope is well-defined. +3. **Bearer token rotation.** v1 uses a single env-var token. Token rotation + without restart needs registry-style mgmt; can wait. +4. **Snapshot rate limits.** Default in design is "every 1000 activities or + 60 seconds." Tunable per-projection later; v1 uses the default. +5. **Genesis bundle format.** Dag-cbor map per §12.2; concrete schema needs + one round of refinement once we author the actual definitions in step 4.