Merge loops/erlang into architecture: Phases 7-10 (hot reload, FFI BIFs, BIF registry, VM opcode extension + erlang_ext); fixes cyclic-env identity hang

# Conflicts:
#	hosts/ocaml/bin/run_tests.ml
#	plans/sx-vm-opcode-extension.md
This commit is contained in:
2026-05-18 20:46:04 +00:00
17 changed files with 5912 additions and 110 deletions

View File

@@ -1599,6 +1599,213 @@ let run_foundation_tests () =
Printf.printf " FAIL: invocation_count: %s\n"
(match other with Some n -> string_of_int n | None -> "None"));
Printf.printf "\nSuite: extensions/erlang_ext (Phase 9h)\n";
(* Register the Erlang opcode namespace. Disjoint id range (200-217)
from test_ext (220/221) so they coexist. *)
Erlang_ext.register ();
(match prim [String "erlang.OP_PATTERN_TUPLE"] with
| Integer 222 ->
incr pass_count;
Printf.printf " PASS: extension-opcode-id erlang.OP_PATTERN_TUPLE = 222\n"
| other ->
incr fail_count;
Printf.printf " FAIL: erlang.OP_PATTERN_TUPLE: got %s\n"
(Sx_types.inspect other));
(match prim [String "erlang.OP_BIF_IS_TUPLE"] with
| Integer 239 ->
incr pass_count;
Printf.printf " PASS: extension-opcode-id erlang.OP_BIF_IS_TUPLE = 239\n"
| other ->
incr fail_count;
Printf.printf " FAIL: erlang.OP_BIF_IS_TUPLE: got %s\n"
(Sx_types.inspect other));
(match prim [String "erlang.OP_NONEXISTENT"] with
| Nil ->
incr pass_count;
Printf.printf " PASS: unknown erlang opcode -> nil\n"
| other ->
incr fail_count;
Printf.printf " FAIL: unknown erlang opcode: got %s\n"
(Sx_types.inspect other));
(* Phase 10b vertical slice: erlang.OP_BIF_LENGTH (230) is a REAL
handler. Build [CONST 0; OP_BIF_LENGTH; RETURN] with an Erlang
list [1,2,3] in the constant pool; expect Integer 3. Proves the
full path: bytecode -> Sx_vm extension fallthrough -> erlang_ext
handler -> correct stack result. *)
(let mk_dict kvs =
let h = Hashtbl.create 4 in
List.iter (fun (k, v) -> Hashtbl.replace h k v) kvs;
Sx_types.Dict h in
let er_nil = mk_dict [("tag", Sx_types.String "nil")] in
let er_cons hd tl =
mk_dict [("tag", Sx_types.String "cons");
("head", hd); ("tail", tl)] in
let lst = er_cons (Sx_types.Integer 1)
(er_cons (Sx_types.Integer 2)
(er_cons (Sx_types.Integer 3) er_nil)) in
let code = ({
vc_arity = 0; vc_rest_arity = -1; vc_locals = 0;
vc_bytecode = [| 1; 0; 0; 230; 50 |];
vc_constants = [| lst |];
vc_bytecode_list = None; vc_constants_list = None;
} : Sx_types.vm_code) in
let globals = Hashtbl.create 1 in
try
match Sx_vm.execute_module code globals with
| Integer 3 ->
incr pass_count;
Printf.printf " PASS: erlang.OP_BIF_LENGTH [1,2,3] -> 3 (real handler, end-to-end)\n"
| other ->
incr fail_count;
Printf.printf " FAIL: OP_BIF_LENGTH result: got %s\n"
(Sx_types.inspect other)
with exn ->
incr fail_count;
Printf.printf " FAIL: OP_BIF_LENGTH raised: %s\n"
(Printexc.to_string exn));
(* More real handlers (Phase 10b batch): build a list/tuple constant
and exercise HD/TL/TUPLE_SIZE/IS_* end-to-end through the VM. *)
(let mk_dict kvs =
let h = Hashtbl.create 4 in
List.iter (fun (k, v) -> Hashtbl.replace h k v) kvs;
Sx_types.Dict h in
let er_nil = mk_dict [("tag", Sx_types.String "nil")] in
let er_cons hd tl = mk_dict [("tag", Sx_types.String "cons");
("head", hd); ("tail", tl)] in
let er_tuple es = mk_dict [("tag", Sx_types.String "tuple");
("elements", Sx_types.List es)] in
let er_atom nm = mk_dict [("tag", Sx_types.String "atom");
("name", Sx_types.String nm)] in
let lst3 = er_cons (Sx_types.Integer 7)
(er_cons (Sx_types.Integer 8)
(er_cons (Sx_types.Integer 9) er_nil)) in
let tup3 = er_tuple [Sx_types.Integer 1; Sx_types.Integer 2;
Sx_types.Integer 3] in
let run consts bc =
let code = ({
vc_arity = 0; vc_rest_arity = -1; vc_locals = 0;
vc_bytecode = bc; vc_constants = consts;
vc_bytecode_list = None; vc_constants_list = None;
} : Sx_types.vm_code) in
Sx_vm.execute_module code (Hashtbl.create 1) in
let nm = function
| Sx_types.Dict d ->
(match Hashtbl.find_opt d "name" with
| Some (Sx_types.String s) -> s | _ -> "?")
| _ -> "?" in
let check label want got =
if got = want then begin
incr pass_count;
Printf.printf " PASS: %s\n" label
end else begin
incr fail_count;
Printf.printf " FAIL: %s: got %s\n" label (Sx_types.inspect got)
end in
(* HD [7,8,9] -> 7 *)
check "OP_BIF_HD [7,8,9] -> 7" (Sx_types.Integer 7)
(run [| lst3 |] [| 1;0;0; 231; 50 |]);
(* TL [7,8,9] -> [8,9], check its HD = 8 *)
check "OP_BIF_TL then HD -> 8" (Sx_types.Integer 8)
(run [| lst3 |] [| 1;0;0; 232; 231; 50 |]);
(* TUPLE_SIZE {1,2,3} -> 3 *)
check "OP_BIF_TUPLE_SIZE {1,2,3} -> 3" (Sx_types.Integer 3)
(run [| tup3 |] [| 1;0;0; 234; 50 |]);
(* IS_INTEGER 42 -> true ; IS_INTEGER [..] -> false *)
(match run [| Sx_types.Integer 42 |] [| 1;0;0; 236; 50 |] with
| v when nm v = "true" ->
incr pass_count; Printf.printf " PASS: OP_BIF_IS_INTEGER 42 -> true\n"
| v -> incr fail_count;
Printf.printf " FAIL: IS_INTEGER 42: got %s\n" (Sx_types.inspect v));
(match run [| lst3 |] [| 1;0;0; 236; 50 |] with
| v when nm v = "false" ->
incr pass_count; Printf.printf " PASS: OP_BIF_IS_INTEGER list -> false\n"
| v -> incr fail_count;
Printf.printf " FAIL: IS_INTEGER list: got %s\n" (Sx_types.inspect v));
(* IS_ATOM atom -> true ; IS_LIST nil -> true ; IS_TUPLE tuple -> true *)
(match run [| er_atom "ok" |] [| 1;0;0; 237; 50 |] with
| v when nm v = "true" ->
incr pass_count; Printf.printf " PASS: OP_BIF_IS_ATOM ok -> true\n"
| v -> incr fail_count;
Printf.printf " FAIL: IS_ATOM: got %s\n" (Sx_types.inspect v));
(match run [| er_nil |] [| 1;0;0; 238; 50 |] with
| v when nm v = "true" ->
incr pass_count; Printf.printf " PASS: OP_BIF_IS_LIST nil -> true\n"
| v -> incr fail_count;
Printf.printf " FAIL: IS_LIST nil: got %s\n" (Sx_types.inspect v));
(match run [| tup3 |] [| 1;0;0; 239; 50 |] with
| v when nm v = "true" ->
incr pass_count; Printf.printf " PASS: OP_BIF_IS_TUPLE {..} -> true\n"
| v -> incr fail_count;
Printf.printf " FAIL: IS_TUPLE: got %s\n" (Sx_types.inspect v));
(match run [| tup3 |] [| 1;0;0; 238; 50 |] with
| v when nm v = "false" ->
incr pass_count; Printf.printf " PASS: OP_BIF_IS_LIST tuple -> false\n"
| v -> incr fail_count;
Printf.printf " FAIL: IS_LIST tuple: got %s\n" (Sx_types.inspect v));
(* ELEMENT: element(2, {1,2,3}) -> 2. Calling convention: push
Index then Tuple; opcode pops Tuple (TOS) then Index. *)
check "OP_BIF_ELEMENT element(2,{1,2,3}) -> 2" (Sx_types.Integer 2)
(run [| Sx_types.Integer 2; tup3 |] [| 1;0;0; 1;1;0; 233; 50 |]);
check "OP_BIF_ELEMENT element(1,{1,2,3}) -> 1" (Sx_types.Integer 1)
(run [| Sx_types.Integer 1; tup3 |] [| 1;0;0; 1;1;0; 233; 50 |]);
(* ELEMENT out of range raises *)
(let raised =
(try ignore (run [| Sx_types.Integer 9; tup3 |]
[| 1;0;0; 1;1;0; 233; 50 |]); false
with Sx_types.Eval_error _ -> true) in
if raised then begin
incr pass_count;
Printf.printf " PASS: OP_BIF_ELEMENT out-of-range raises\n"
end else begin
incr fail_count;
Printf.printf " FAIL: OP_BIF_ELEMENT out-of-range should raise\n"
end);
(* LISTS_REVERSE [7,8,9] -> [9,8,7]; verify HD = 9 then HD of TL = 8 *)
check "OP_BIF_LISTS_REVERSE then HD -> 9" (Sx_types.Integer 9)
(run [| lst3 |] [| 1;0;0; 235; 231; 50 |]);
check "OP_BIF_LISTS_REVERSE then TL,HD -> 8" (Sx_types.Integer 8)
(run [| lst3 |] [| 1;0;0; 235; 232; 231; 50 |]);
(* reverse preserves length *)
check "OP_BIF_LISTS_REVERSE then LENGTH -> 3" (Sx_types.Integer 3)
(run [| lst3 |] [| 1;0;0; 235; 230; 50 |]));
(* A still-stubbed opcode (222 = erlang.OP_PATTERN_TUPLE) raises the
not-wired Eval_error — confirms the honest-failure path remains
for opcodes whose real handlers haven't landed. *)
(let globals = Hashtbl.create 1 in
try
ignore (Sx_vm.execute_module (make_bc_seq [| 222; 50 |]) globals);
incr fail_count;
Printf.printf " FAIL: erlang.OP_PATTERN_TUPLE dispatch should have raised\n"
with
| Sx_types.Eval_error msg
when (let needle = "not yet wired" in
let nl = String.length needle and ml = String.length msg in
let rec scan i =
if i + nl > ml then false
else if String.sub msg i nl = needle then true
else scan (i + 1)
in scan 0) ->
incr pass_count;
Printf.printf " PASS: erlang opcode dispatch raises not-wired error\n"
| exn ->
incr fail_count;
Printf.printf " FAIL: unexpected exn: %s\n" (Printexc.to_string exn));
(match Erlang_ext.dispatch_count () with
| Some n when n >= 1 ->
incr pass_count;
Printf.printf " PASS: erlang_ext state recorded %d dispatch(es)\n" n
| other ->
incr fail_count;
Printf.printf " FAIL: dispatch_count: %s\n"
(match other with Some n -> string_of_int n | None -> "None"));
Printf.printf "\nSuite: jit extension-opcode awareness\n";
let scan = Sx_vm.bytecode_uses_extension_opcodes in
let no_consts = [||] in

View File

@@ -18,6 +18,20 @@
open Sx_types
(* Force-link Sx_vm_extensions so its module-init runs: installs the
extension dispatch fallthrough and registers the `extension-opcode-id`
SX primitive. Without a reference here OCaml dead-code-eliminates the
module from sx_server.exe (it's only otherwise reached from run_tests),
leaving guest-language opcode extensions (Erlang Phase 9, etc.)
invisible to the runtime. The applied call is a harmless lookup. *)
let () = ignore (Sx_vm_extensions.id_of_name "")
(* Register the Erlang opcode extension (Phase 9h) so
`extension-opcode-id "erlang.OP_*"` resolves to the host ids the SX
stub dispatcher consults. Guarded: a double-register raises Failure,
which we swallow so a re-entered server process doesn't die. *)
let () = try Erlang_ext.register () with Failure _ -> ()
(* ====================================================================== *)
(* Font measurement via otfm — reads OpenType/TrueType font tables *)
(* ====================================================================== *)

View File

@@ -0,0 +1,278 @@
(** {1 [erlang_ext] — Erlang-on-SX VM opcode extension (Phase 9h)}
Registers the Erlang opcode namespace in [Sx_vm_extensions] so that
[extension-opcode-id "erlang.OP_*"] resolves to a stable id. The SX
stub dispatcher in [lib/erlang/vm/dispatcher.sx] consults these ids
(Phase 9i) and falls back to its own local ids when the host
extension is absent.
Opcode ids occupy 222-239 in the extension partition (200-247).
222+ is chosen to clear the test extensions' reserved ids
(test_reg 210/211, test_ext 220/221) so all three coexist in
run_tests; production sx_server only registers this one. Names
mirror the SX stub dispatcher exactly:
- 222 erlang.OP_PATTERN_TUPLE - 231 erlang.OP_BIF_HD
- 223 erlang.OP_PATTERN_LIST - 232 erlang.OP_BIF_TL
- 224 erlang.OP_PATTERN_BINARY - 233 erlang.OP_BIF_ELEMENT
- 225 erlang.OP_PERFORM - 234 erlang.OP_BIF_TUPLE_SIZE
- 226 erlang.OP_HANDLE - 235 erlang.OP_BIF_LISTS_REVERSE
- 227 erlang.OP_RECEIVE_SCAN - 236 erlang.OP_BIF_IS_INTEGER
- 228 erlang.OP_SPAWN - 237 erlang.OP_BIF_IS_ATOM
- 229 erlang.OP_SEND - 238 erlang.OP_BIF_IS_LIST
- 230 erlang.OP_BIF_LENGTH - 239 erlang.OP_BIF_IS_TUPLE
{2 Handler status}
The bytecode compiler does not yet emit these opcodes — Erlang
programs run through the general CEK path and the working
specialization path is the SX stub dispatcher. So every handler
here raises a descriptive [Eval_error] rather than silently
corrupting the VM stack. This keeps the extension honest: the
namespace is registered and disassembles by name, [extension-opcode-id]
works, but actually dispatching an opcode (which only happens once a
future phase teaches the compiler to emit them) fails loudly with a
pointer to the phase that will wire it. Real stack-machine handlers
land alongside compiler emission in a later phase. *)
open Sx_types
(** Per-instance state: invocation counter, purely to exercise the
[extension_state] machinery (mirrors [test_ext]). *)
type Sx_vm_extension.extension_state += ErlangExtState of {
mutable dispatched : int;
}
let not_wired name =
raise (Eval_error
(Printf.sprintf
"%s: bytecode emission not yet wired (Phase 9j) — \
Erlang runs via CEK; specialization path is the SX stub \
dispatcher in lib/erlang/vm/dispatcher.sx"
name))
module M : Sx_vm_extension.EXTENSION = struct
let name = "erlang"
let init () = ErlangExtState { dispatched = 0 }
let opcodes st =
let bump () = match st with
| ErlangExtState s -> s.dispatched <- s.dispatched + 1
| _ -> ()
in
let op id nm =
(id, nm, (fun (_vm : Sx_vm.vm) (_frame : Sx_vm.frame) ->
bump (); not_wired nm))
in
(* Phase 10b vertical slice: one REAL register-machine handler.
erlang.OP_BIF_LENGTH (230) — pops an Erlang list off the VM
stack and pushes its length. Proves the full path works:
extension-opcode-id -> bytecode -> Sx_vm dispatch fallthrough
-> this handler -> correct stack result. The remaining 17
opcodes still raise not_wired until their handlers + compiler
emission land. Erlang lists are tagged dicts:
nil = {"tag" -> String "nil"}
cons = {"tag" -> String "cons"; "head" -> v; "tail" -> v} *)
let er_tag d =
match Hashtbl.find_opt d "tag" with
| Some (String s) -> s | _ -> ""
in
let op_bif_length =
(230, "erlang.OP_BIF_LENGTH",
(fun (vm : Sx_vm.vm) (_frame : Sx_vm.frame) ->
bump ();
let v = Sx_vm.pop vm in
let rec walk acc node =
match node with
| Dict d ->
(match er_tag d with
| "nil" -> acc
| "cons" ->
(match Hashtbl.find_opt d "tail" with
| Some t -> walk (acc + 1) t
| None -> raise (Eval_error
"erlang.OP_BIF_LENGTH: cons cell without :tail"))
| _ -> raise (Eval_error
"erlang.OP_BIF_LENGTH: not a proper list"))
| _ -> raise (Eval_error
"erlang.OP_BIF_LENGTH: not a proper list")
in
Sx_vm.push vm (Integer (walk 0 v))))
in
(* Phase 10b — simple hot-BIF handlers. Erlang bool is the atom
{"tag"->"atom"; "name"->"true"|"false"}; mk_atom builds it. *)
let mk_atom nm =
let h = Hashtbl.create 2 in
Hashtbl.replace h "tag" (String "atom");
Hashtbl.replace h "name" (String nm);
Dict h
in
let er_bool b = mk_atom (if b then "true" else "false") in
let is_tag v t = match v with
| Dict d -> er_tag d = t
| _ -> false
in
let op_bif_hd =
(231, "erlang.OP_BIF_HD",
(fun (vm : Sx_vm.vm) _f ->
bump ();
match Sx_vm.pop vm with
| Dict d when er_tag d = "cons" ->
(match Hashtbl.find_opt d "head" with
| Some h -> Sx_vm.push vm h
| None -> raise (Eval_error "erlang.OP_BIF_HD: cons without :head"))
| _ -> raise (Eval_error "erlang.OP_BIF_HD: not a cons")))
in
let op_bif_tl =
(232, "erlang.OP_BIF_TL",
(fun (vm : Sx_vm.vm) _f ->
bump ();
match Sx_vm.pop vm with
| Dict d when er_tag d = "cons" ->
(match Hashtbl.find_opt d "tail" with
| Some t -> Sx_vm.push vm t
| None -> raise (Eval_error "erlang.OP_BIF_TL: cons without :tail"))
| _ -> raise (Eval_error "erlang.OP_BIF_TL: not a cons")))
in
let op_bif_tuple_size =
(234, "erlang.OP_BIF_TUPLE_SIZE",
(fun (vm : Sx_vm.vm) _f ->
bump ();
match Sx_vm.pop vm with
| Dict d when er_tag d = "tuple" ->
let n = match Hashtbl.find_opt d "elements" with
| Some (List es) -> List.length es
| Some (ListRef r) -> List.length !r
| _ -> raise (Eval_error
"erlang.OP_BIF_TUPLE_SIZE: tuple without :elements")
in
Sx_vm.push vm (Integer n)
| _ -> raise (Eval_error "erlang.OP_BIF_TUPLE_SIZE: not a tuple")))
in
let op_bif_is_integer =
(236, "erlang.OP_BIF_IS_INTEGER",
(fun (vm : Sx_vm.vm) _f ->
bump ();
let v = Sx_vm.pop vm in
Sx_vm.push vm (er_bool (match v with Integer _ -> true | _ -> false))))
in
let op_bif_is_atom =
(237, "erlang.OP_BIF_IS_ATOM",
(fun (vm : Sx_vm.vm) _f ->
bump ();
let v = Sx_vm.pop vm in
Sx_vm.push vm (er_bool (is_tag v "atom"))))
in
let op_bif_is_list =
(238, "erlang.OP_BIF_IS_LIST",
(fun (vm : Sx_vm.vm) _f ->
bump ();
let v = Sx_vm.pop vm in
Sx_vm.push vm (er_bool (is_tag v "cons" || is_tag v "nil"))))
in
let op_bif_is_tuple =
(239, "erlang.OP_BIF_IS_TUPLE",
(fun (vm : Sx_vm.vm) _f ->
bump ();
let v = Sx_vm.pop vm in
Sx_vm.push vm (er_bool (is_tag v "tuple"))))
in
(* element/2 and lists:reverse/1 — pure stack transforms (no
bytecode operands). Calling convention: args pushed left→right,
so element/2 stack is [.. Index Tuple] (Tuple on top). Erlang
element/2 is 1-indexed. *)
let op_bif_element =
(233, "erlang.OP_BIF_ELEMENT",
(fun (vm : Sx_vm.vm) _f ->
bump ();
let tup = Sx_vm.pop vm in
let idx = Sx_vm.pop vm in
match tup, idx with
| Dict d, Integer i when er_tag d = "tuple" ->
let es = match Hashtbl.find_opt d "elements" with
| Some (List es) -> es
| Some (ListRef r) -> !r
| _ -> raise (Eval_error
"erlang.OP_BIF_ELEMENT: tuple without :elements")
in
let n = List.length es in
if i < 1 || i > n then
raise (Eval_error
(Printf.sprintf
"erlang.OP_BIF_ELEMENT: index %d out of range 1..%d" i n))
else
Sx_vm.push vm (List.nth es (i - 1))
| _, Integer _ ->
raise (Eval_error "erlang.OP_BIF_ELEMENT: 2nd arg not a tuple")
| _ ->
raise (Eval_error "erlang.OP_BIF_ELEMENT: 1st arg not an integer")))
in
let op_bif_lists_reverse =
(235, "erlang.OP_BIF_LISTS_REVERSE",
(fun (vm : Sx_vm.vm) _f ->
bump ();
let v = Sx_vm.pop vm in
let mk_nil () =
let h = Hashtbl.create 1 in
Hashtbl.replace h "tag" (String "nil"); Dict h in
let mk_cons hd tl =
let h = Hashtbl.create 3 in
Hashtbl.replace h "tag" (String "cons");
Hashtbl.replace h "head" hd;
Hashtbl.replace h "tail" tl;
Dict h in
let rec rev acc node =
match node with
| Dict d ->
(match er_tag d with
| "nil" -> acc
| "cons" ->
let hd = match Hashtbl.find_opt d "head" with
| Some x -> x
| None -> raise (Eval_error
"erlang.OP_BIF_LISTS_REVERSE: cons without :head") in
let tl = match Hashtbl.find_opt d "tail" with
| Some x -> x
| None -> raise (Eval_error
"erlang.OP_BIF_LISTS_REVERSE: cons without :tail") in
rev (mk_cons hd acc) tl
| _ -> raise (Eval_error
"erlang.OP_BIF_LISTS_REVERSE: not a proper list"))
| _ -> raise (Eval_error
"erlang.OP_BIF_LISTS_REVERSE: not a proper list")
in
Sx_vm.push vm (rev (mk_nil ()) v)))
in
[
op 222 "erlang.OP_PATTERN_TUPLE";
op 223 "erlang.OP_PATTERN_LIST";
op 224 "erlang.OP_PATTERN_BINARY";
op 225 "erlang.OP_PERFORM";
op 226 "erlang.OP_HANDLE";
op 227 "erlang.OP_RECEIVE_SCAN";
op 228 "erlang.OP_SPAWN";
op 229 "erlang.OP_SEND";
op_bif_length;
op_bif_hd;
op_bif_tl;
op_bif_element;
op_bif_tuple_size;
op_bif_lists_reverse;
op_bif_is_integer;
op_bif_is_atom;
op_bif_is_list;
op_bif_is_tuple;
]
end
(** Register [erlang] in [Sx_vm_extensions]. Idempotent only by failing
loudly — calling twice raises [Failure]. sx_server calls this once
at startup. *)
let register () = Sx_vm_extensions.register (module M : Sx_vm_extension.EXTENSION)
(** Read the dispatch counter from the live registry state. [None] if
[register] hasn't run. *)
let dispatch_count () =
match Sx_vm_extensions.state_of_extension "erlang" with
| Some (ErlangExtState s) -> Some s.dispatched
| _ -> None