From bf298684fdc5691bc75e1ad19b0381b59b2e5c86 Mon Sep 17 00:00:00 2001 From: giles Date: Fri, 19 Jun 2026 22:22:40 +0000 Subject: [PATCH] vm-ext: gate serving-JIT behind SX_SERVING_JIT + fix continuation-guest regressions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Enabling the epoch serving-mode JIT globally regressed continuation-based guest interpreters (the epoch mode is the shared command channel every loop's conformance runner uses). Two-part fix: 1. SAFE DEFAULT GATE. register_jit_hook in the persistent server branch is now opt-in via SX_SERVING_JIT=1 (default OFF). Default behaviour is unchanged (no JIT in epoch serving) → zero regression for sibling loops. The content/Smalltalk page server opts in. 2. GENERAL FIXES + per-guest interpret-only declarations: - callable? (sx_server/run_tests/integration_tests/mcp_tree) now accepts VmClosure. A JIT-compiled higher-order function returns its inner closure as a VmClosure; callable? previously rejected it, so scheme-apply's (callable? proc) guard failed with "not a procedure: ". - jit-exclude! gains a trailing-"*" namespace-prefix form (Sx_types.jit_excluded_prefixes), the robust way to mark a whole guest interpreter interpret-only (a name-list misses functions in extra files — it left erlang's vm/dispatcher JIT'd and 13 tests short). - Per-guest exclusions in each guest's runtime.sx: scheme "scheme-*" "scm-*" erlang "er-*" "erlang-*" prolog "pl-*" common-lisp "cl-*" "clos-*" js "js-*" haskell "hk-*" Verified under opt-in JIT (== CEK, no hang): smalltalk 847/847, scheme/flow 166/166, erlang 530/530, prolog 590/590, apl 152/152, js 147/148. Residual (documented, protected by the default gate): common-lisp 6 fails in advanced suites (parser-recovery/debugger/CLOS/MOP). lua (0/16) and tcl (3/4) fail identically on CEK — pre-existing, not JIT. run_tests --jit/no-jit unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) --- hosts/ocaml/bin/integration_tests.ml | 2 +- hosts/ocaml/bin/mcp_tree.ml | 2 +- hosts/ocaml/bin/run_tests.ml | 2 +- hosts/ocaml/bin/sx_server.ml | 61 +++++++++++++++++----------- hosts/ocaml/lib/sx_primitives.ml | 19 ++++++--- hosts/ocaml/lib/sx_types.ml | 15 +++++++ hosts/ocaml/lib/sx_vm.ml | 8 ++-- lib/common-lisp/runtime.sx | 8 +++- lib/erlang/runtime.sx | 8 ++++ lib/haskell/runtime.sx | 6 +++ lib/js/runtime.sx | 6 +++ lib/prolog/runtime.sx | 7 ++++ lib/scheme/runtime.sx | 8 ++++ plans/jit-bytecode-correctness.md | 47 +++++++++++++++++++++ 14 files changed, 163 insertions(+), 36 deletions(-) diff --git a/hosts/ocaml/bin/integration_tests.ml b/hosts/ocaml/bin/integration_tests.ml index 54ec5b19..256afb46 100644 --- a/hosts/ocaml/bin/integration_tests.ml +++ b/hosts/ocaml/bin/integration_tests.ml @@ -263,7 +263,7 @@ let make_integration_env () = (* Type predicates — needed by adapter-sx.sx *) bind "callable?" (fun args -> - match args with [NativeFn _] | [Lambda _] | [Component _] | [Island _] -> Bool true | _ -> Bool false); + match args with [NativeFn _] | [Lambda _] | [Component _] | [Island _] | [VmClosure _] -> Bool true | _ -> Bool false); bind "lambda?" (fun args -> match args with [Lambda _] -> Bool true | _ -> Bool false); bind "macro?" (fun args -> match args with [Macro _] -> Bool true | _ -> Bool false); bind "island?" (fun args -> match args with [Island _] -> Bool true | _ -> Bool false); diff --git a/hosts/ocaml/bin/mcp_tree.ml b/hosts/ocaml/bin/mcp_tree.ml index 8591d00a..a0142191 100644 --- a/hosts/ocaml/bin/mcp_tree.ml +++ b/hosts/ocaml/bin/mcp_tree.ml @@ -477,7 +477,7 @@ let setup_env () = bind "number?" (fun args -> match args with | [Number _] -> Bool true | _ -> Bool false); bind "callable?" (fun args -> match args with - | [NativeFn _ | Lambda _ | Component _ | Island _] -> Bool true | _ -> Bool false); + | [NativeFn _ | Lambda _ | Component _ | Island _ | VmClosure _] -> Bool true | _ -> Bool false); bind "empty?" (fun args -> match args with | [List []] | [ListRef { contents = [] }] -> Bool true | [Nil] -> Bool true | _ -> Bool false); diff --git a/hosts/ocaml/bin/run_tests.ml b/hosts/ocaml/bin/run_tests.ml index 9689fa35..65c297ff 100644 --- a/hosts/ocaml/bin/run_tests.ml +++ b/hosts/ocaml/bin/run_tests.ml @@ -595,7 +595,7 @@ let make_test_env () = (* regex-find-all now provided by sx_primitives.ml *) bind "callable?" (fun args -> match args with - | [NativeFn _] | [Lambda _] | [Component _] | [Island _] -> Bool true + | [NativeFn _] | [Lambda _] | [Component _] | [Island _] | [VmClosure _] -> Bool true | _ -> Bool false); bind "make-sx-expr" (fun args -> match args with [String s] -> SxExpr s | _ -> raise (Eval_error "make-sx-expr: expected string")); bind "sx-expr-source" (fun args -> match args with [SxExpr s] -> String s | [String s] -> String s | _ -> raise (Eval_error "sx-expr-source: expected sx-expr or string")); diff --git a/hosts/ocaml/bin/sx_server.ml b/hosts/ocaml/bin/sx_server.ml index 1b2ba6bf..1547ab1a 100644 --- a/hosts/ocaml/bin/sx_server.ml +++ b/hosts/ocaml/bin/sx_server.ml @@ -789,7 +789,11 @@ let setup_introspection env = bind "component?" (fun args -> match args with [Component _] | [Island _] -> Bool true | _ -> Bool false); bind "callable?" (fun args -> - match args with [NativeFn _] | [Lambda _] | [Component _] | [Island _] -> Bool true | _ -> Bool false); + (* VmClosure must count as callable: a JIT-compiled higher-order function + returns its inner closure as a VmClosure, and downstream code (e.g. + scheme-apply's `(callable? proc)` guard) must recognize it — it is + invocable via the normal call path. *) + match args with [NativeFn _] | [Lambda _] | [Component _] | [Island _] | [VmClosure _] -> Bool true | _ -> Bool false); bind "spread?" (fun args -> match args with [Spread _] -> Bool true | _ -> Bool false); bind "continuation?" (fun args -> match args with [Continuation _] -> Bool true | [_] -> Bool false | _ -> Bool false); @@ -4556,29 +4560,38 @@ let () = else begin (* Normal persistent server mode *) let env = make_server_env () in - (* JIT needs the SX bytecode compiler (lib/compiler.sx) as its `compile` - binding — the native Sx_compiler.compile is an incomplete stub that - miscompiles parameters (emits arity-0 bytecode with params as - GLOBAL_GET). http/cli/site modes already load compiler.sx; the - persistent (epoch) serving mode must too before enabling the hook, - or every JIT-compiled function fails at runtime with "VM undefined: - " and falls back to CEK (with double-executed side effects). *) - (_import_env := Some env; - let project_dir = try Sys.getenv "SX_PROJECT_DIR" with Not_found -> - try Sys.getenv "SX_ROOT" with Not_found -> - if Sys.file_exists "/app/spec" then "/app" else Sys.getcwd () in - let lib_base = try Sys.getenv "SX_LIB_DIR" with Not_found -> - project_dir ^ "/lib" in - let compiler_path = lib_base ^ "/compiler.sx" in - let compiler_path = - if Sys.file_exists compiler_path then compiler_path - else if Sys.file_exists "lib/compiler.sx" then "lib/compiler.sx" - else compiler_path in - try load_library_file compiler_path; rebind_host_extensions env - with exn -> - Printf.eprintf "[sx-server] WARNING: failed to load compiler.sx for JIT (%s) — JIT disabled\n%!" - (Printexc.to_string exn)); - register_jit_hook env; + (* JIT in the epoch serving mode is OPT-IN via SX_SERVING_JIT=1. + Default OFF: this mode is the shared command channel used by every + loop's conformance runner, and enabling JIT globally regresses + continuation-based guest interpreters (Scheme/Erlang/Prolog/CL: their + eval/dispatch cores capture call/cc continuations the stack VM can't + escape, and deep AST recursion can miscompile into a non-terminating + loop). Guests that are safe declare their interpret-only namespace with + `(jit-exclude! "-*")`; until every guest is validated, the safe + default is no JIT here. Opt in (SX_SERVING_JIT=1) for validated + workloads — e.g. the content/Smalltalk page server. *) + (match Sys.getenv_opt "SX_SERVING_JIT" with + | Some ("1" | "true" | "yes" | "on") -> + (* Load the SX bytecode compiler (lib/compiler.sx) as `compile` — the + native Sx_compiler.compile is an incomplete stub (arity-0 bytecode, + params as GLOBAL_GET). http/cli/site modes already load it. *) + (_import_env := Some env; + let project_dir = try Sys.getenv "SX_PROJECT_DIR" with Not_found -> + try Sys.getenv "SX_ROOT" with Not_found -> + if Sys.file_exists "/app/spec" then "/app" else Sys.getcwd () in + let lib_base = try Sys.getenv "SX_LIB_DIR" with Not_found -> + project_dir ^ "/lib" in + let compiler_path = lib_base ^ "/compiler.sx" in + let compiler_path = + if Sys.file_exists compiler_path then compiler_path + else if Sys.file_exists "lib/compiler.sx" then "lib/compiler.sx" + else compiler_path in + try load_library_file compiler_path; rebind_host_extensions env + with exn -> + Printf.eprintf "[sx-server] WARNING: failed to load compiler.sx for JIT (%s) — JIT disabled\n%!" + (Printexc.to_string exn)); + register_jit_hook env + | _ -> ()); send "(ready)"; (* Main command loop *) try diff --git a/hosts/ocaml/lib/sx_primitives.ml b/hosts/ocaml/lib/sx_primitives.ml index 5c438547..39a65874 100644 --- a/hosts/ocaml/lib/sx_primitives.ml +++ b/hosts/ocaml/lib/sx_primitives.ml @@ -4154,17 +4154,26 @@ let () = Queue.clear Sx_types.jit_cache_queue; Nil); register "jit-exclude!" (fun args -> - (* Mark one or more function names as interpret-only (never JIT-compiled). - A guest interpreter calls this for its continuation-using dispatch core. - Accepts any number of string/symbol names. *) + (* Mark function names as interpret-only (never JIT-compiled). A guest + interpreter calls this for its continuation-using dispatch core. + Accepts string/symbol names; a trailing "*" makes it a namespace prefix + (e.g. "er-*" excludes every function whose name starts with "er-") — + the robust way to declare a whole guest interpreter core. *) List.iter (fun a -> match a with - | String n | Symbol n -> Hashtbl.replace Sx_types.jit_excluded n () + | String n | Symbol n -> + let len = String.length n in + if len > 0 && n.[len - 1] = '*' then begin + let prefix = String.sub n 0 (len - 1) in + if not (List.mem prefix !Sx_types.jit_excluded_prefixes) then + Sx_types.jit_excluded_prefixes := prefix :: !Sx_types.jit_excluded_prefixes + end else + Hashtbl.replace Sx_types.jit_excluded n () | _ -> ()) args; Nil); register "jit-excluded?" (fun args -> match args with - | [String n] | [Symbol n] -> Bool (Hashtbl.mem Sx_types.jit_excluded n) + | [String n] | [Symbol n] -> Bool (Sx_types.jit_name_excluded n) | _ -> Bool false); register "jit-reset-counters!" (fun _args -> Sx_types.jit_compiled_count := 0; diff --git a/hosts/ocaml/lib/sx_types.ml b/hosts/ocaml/lib/sx_types.ml index 40ffc230..599232ba 100644 --- a/hosts/ocaml/lib/sx_types.ml +++ b/hosts/ocaml/lib/sx_types.ml @@ -487,6 +487,21 @@ let jit_threshold_skipped_count = ref 0 points: the CEK call hook and the in-VM tiered-compilation path. *) let jit_excluded : (string, unit) Hashtbl.t = Hashtbl.create 64 +(** Namespace-prefix exclusions. A guest interpreter declares its whole + function namespace interpret-only with one entry (e.g. ["er-"], ["scm-"]), + which is far more robust than enumerating every function — a name-list + misses functions in extra files (the erlang VM dispatcher, etc.) and + silently regresses. Set via [jit-exclude!] with a trailing ["*"] + (e.g. [(jit-exclude! "er-*")]). Checked via [jit_name_excluded]. *) +let jit_excluded_prefixes : string list ref = ref [] + +(** True if [name] is excluded from JIT — by exact name or by namespace prefix. *) +let jit_name_excluded name = + Hashtbl.mem jit_excluded name + || List.exists (fun p -> + String.length name >= String.length p + && String.sub name 0 (String.length p) = p) !jit_excluded_prefixes + (** {2 JIT cache LRU eviction — Phase 2} Once a lambda crosses the threshold, its [l_compiled] slot is filled. diff --git a/hosts/ocaml/lib/sx_vm.ml b/hosts/ocaml/lib/sx_vm.ml index 49f310bb..bce21648 100644 --- a/hosts/ocaml/lib/sx_vm.ml +++ b/hosts/ocaml/lib/sx_vm.ml @@ -1144,10 +1144,12 @@ let jit_compile_lambda (l : lambda) globals = None ) else if _jit_is_broken_name fn_name then ( None - ) else if Hashtbl.mem Sx_types.jit_excluded fn_name then ( + ) else if Sx_types.jit_name_excluded fn_name then ( (* Guest-declared interpret-only function (continuation-using dispatch - core). Run on the CEK; the stack VM can't escape through a CEK - continuation. See Sx_types.jit_excluded. *) + core, or a whole namespace via prefix). Run on the CEK; the stack VM + can't escape through a CEK continuation and may miscompile deep AST + recursion into a non-terminating loop. See Sx_types.jit_excluded / + jit_excluded_prefixes. *) None ) else try diff --git a/lib/common-lisp/runtime.sx b/lib/common-lisp/runtime.sx index a43d2905..9656c3ef 100644 --- a/lib/common-lisp/runtime.sx +++ b/lib/common-lisp/runtime.sx @@ -757,4 +757,10 @@ "format-arguments" args)))) (cl-restart-case (fn () (cl-signal-obj obj cl-handler-stack)) - (list "continue" (list) (fn () nil)))))) \ No newline at end of file + (list "continue" (list) (fn () nil)))))) +;; ── JIT interpret-only boundary ─────────────────────────────────────────── +;; The Common-Lisp evaluator implements block/return-from, catch/throw, and +;; the condition system via non-local control (host continuations); under JIT +;; a compiled frame can't transfer control through a CEK continuation. Exclude +;; the cl-/clos- namespaces from JIT. See Sx_types.jit_excluded_prefixes. +(jit-exclude! "cl-*" "clos-*") diff --git a/lib/erlang/runtime.sx b/lib/erlang/runtime.sx index 03aaad5d..85101d83 100644 --- a/lib/erlang/runtime.sx +++ b/lib/erlang/runtime.sx @@ -1202,3 +1202,11 @@ (= name "info") (er-bif-ets-info vs) :else (error (str "Erlang: undefined 'ets:" name "/" (len vs) "'"))))) + +;; ── JIT interpret-only boundary ─────────────────────────────────────────── +;; The Erlang evaluator (er-eval-* in transpile.sx + the vm/dispatcher) recurses +;; over the AST and the scheduler/receive path captures call/cc continuations. +;; Under JIT the recursive eval miscompiles into a non-terminating loop and the +;; continuation path cannot transfer control. Exclude the whole er-/erlang- +;; namespace (covers transpile, runtime, and vm/dispatcher in one declaration). +(jit-exclude! "er-*" "erlang-*") diff --git a/lib/haskell/runtime.sx b/lib/haskell/runtime.sx index 84e3b51e..5e5bb6f0 100644 --- a/lib/haskell/runtime.sx +++ b/lib/haskell/runtime.sx @@ -148,3 +148,9 @@ (fn (acc i) (str acc (char-at buf i))) "" (range off (string-length buf))))))) + +;; ── JIT interpret-only boundary ─────────────────────────────────────────── +;; The Haskell evaluator (hk-eval and the lazy-thunk forcer) recurses deeply +;; over the AST/graph; under JIT the recursive eval can miscompile into a +;; non-terminating loop. Exclude the hk- namespace from JIT. +(jit-exclude! "hk-*") diff --git a/lib/js/runtime.sx b/lib/js/runtime.sx index a6576ace..94896a5f 100644 --- a/lib/js/runtime.sx +++ b/lib/js/runtime.sx @@ -6994,3 +6994,9 @@ (set! js-global-this js-global) (dict-set! js-global "globalThis" js-global) + +;; ── JIT interpret-only boundary ─────────────────────────────────────────── +;; The JS evaluator (transpile.sx) uses call/cc for control flow (exceptions, +;; early return); a JIT-compiled frame can't escape through a CEK continuation. +;; Exclude the js- namespace from JIT. See Sx_types.jit_excluded_prefixes. +(jit-exclude! "js-*") diff --git a/lib/prolog/runtime.sx b/lib/prolog/runtime.sx index 257894a0..d173e156 100644 --- a/lib/prolog/runtime.sx +++ b/lib/prolog/runtime.sx @@ -2792,3 +2792,10 @@ {:cut false} (fn () (begin (dict-set! box :n (+ (dict-get box :n) 1)) false))) (dict-get box :n)))) + +;; ── JIT interpret-only boundary ─────────────────────────────────────────── +;; The Prolog resolution engine (pl-solve! and friends) recurses deeply over +;; goals/clauses with backtracking; under JIT it miscompiles into a +;; non-terminating loop (the suite never completes). Exclude the whole pl- +;; namespace from JIT. See Sx_types.jit_excluded_prefixes. +(jit-exclude! "pl-*") diff --git a/lib/scheme/runtime.sx b/lib/scheme/runtime.sx index d8473171..35b1a6de 100644 --- a/lib/scheme/runtime.sx +++ b/lib/scheme/runtime.sx @@ -647,3 +647,11 @@ (raise (get outcome :value))) (:else outcome)))))))))) env))) + +;; ── JIT interpret-only boundary ─────────────────────────────────────────── +;; The Scheme evaluator uses call/cc, dynamic-wind, guard/raise and applies +;; user procedures (which may be continuations or JIT-returned closures); a +;; JIT-compiled frame cannot transfer control through a CEK continuation. +;; Exclude the whole scheme-/scm- namespace from JIT (robust vs a name list, +;; which misses functions in extra files). See Sx_types.jit_excluded_prefixes. +(jit-exclude! "scheme-*" "scm-*") diff --git a/plans/jit-bytecode-correctness.md b/plans/jit-bytecode-correctness.md index c82ab1b0..d506a4d8 100644 --- a/plans/jit-bytecode-correctness.md +++ b/plans/jit-bytecode-correctness.md @@ -121,3 +121,50 @@ Five distinct root causes were found and fixed (not one "miscompile"): - A debug aid was added to the serving hook: `SX_JIT_DENY=name,...` / `SX_JIT_ONLY=name,...` env vars to bisect which named lambda the VM mishandles (hook-path only). + +--- + +## Guest-loop regression sweep + safe-default gate (2026-06-19, follow-up) + +Host-loop verification found that enabling serving-mode JIT **globally** +regresses continuation-based guest interpreters (the epoch serving mode is the +shared command channel for every loop's conformance runner). Failure modes: +- **VmClosure not callable** — a JIT'd higher-order function returns its inner + closure as a `VmClosure`; the native `callable?` predicate didn't list + `VmClosure`, so `scheme-apply`'s `(callable? proc)` guard rejected it + ("scheme-eval: not a procedure: "). FIXED generally: `callable?` + (all 4 bindings) now accepts `VmClosure`. +- **Continuation escape** — Scheme `call/cc`, Erlang receive, CL conditions, + JS exceptions: a JIT'd frame can't transfer control through a CEK + continuation. +- **Non-terminating miscompile (HANG)** — Erlang/Prolog/Haskell recursive + evaluators miscompiled into an infinite loop (worse than an error: can't + fall back). + +### Mechanism +- `jit-exclude!` now accepts a trailing `*` wildcard → namespace-prefix + exclusion (`Sx_types.jit_excluded_prefixes`, checked in + `jit_compile_lambda` for both JIT entry points). One declaration per guest, + robust vs name-lists (which missed e.g. the erlang `vm/dispatcher`). + +### Per-guest exclusions added (in each guest's runtime, loaded with it) +| Guest | Declaration | Status under opt-in JIT | +|-------|-------------|--------------------------| +| smalltalk | name-list (dispatch core) + `pharo-test-class` | 847/847 == CEK | +| scheme | `(jit-exclude! "scheme-*" "scm-*")` | flow 166/166 == CEK | +| erlang | `(jit-exclude! "er-*" "erlang-*")` | 530/530 == CEK, no hang | +| prolog | `(jit-exclude! "pl-*")` | 590/590 == CEK | +| common-lisp | `(jit-exclude! "cl-*" "clos-*")` | residual: 6 fail (advanced suites) | +| js | `(jit-exclude! "js-*")` | (verifying) | +| haskell | `(jit-exclude! "hk-*")` | (verifying) | + +Not JIT-related (fail identically on CEK and JIT, pre-existing): lua 0/16, +tcl 3/4. apl/datalog/forth/ocaml: clean under JIT as-is (no continuations). + +### Safe-default gate +Serving-mode JIT is now **opt-in via `SX_SERVING_JIT=1` (default OFF)** in +`sx_server.ml`. Default behavior is unchanged (no JIT in epoch serving) ⇒ +**zero regression** for every sibling loop's conformance. The content/Smalltalk +page server opts in. This bounds risk: guests are validated and excluded +incrementally; until then the default protects them. Common-Lisp's advanced +suites still need investigation before CL is opt-in-clean.