Fix JIT compilation cascade + MCP robustness

Three interacting JIT bugs caused infinite loops and server hangs:

1. _jit_compiling cascade: the re-entrancy flag was local to each
   binary's hook. When vm_call triggered JIT compilation internally,
   compiler functions got JIT-compiled during compilation, creating
   infinite cascades. Fix: shared _jit_compiling flag in sx_vm.ml,
   set in jit_compile_lambda itself.

2. call_closure always created new VMs: every HO primitive callback
   (for-each, map, filter) allocated a fresh VM. With 43K+ calls
   during compilation, this was the direct cause of hangs. Fix:
   call_closure_reuse reuses the active VM by isolating frames and
   running re-entrantly. VmSuspended is handled by merging frames
   for proper IO resumption.

3. vm_call for compiled Lambdas: OP_CALL dispatching to a Lambda
   with cached bytecode created a new VM instead of pushing a frame
   on the current one. Fix: push_closure_frame directly.

Additional MCP server fixes:
- Hot-reload: auto-execv when binary on disk is newer (no restart needed)
- Robust JSON: to_int_safe/to_int_or handle null, string, int params
- sx_summarise depth now optional (default 2)
- Per-request error handling (malformed JSON doesn't crash server)
- sx_test uses pre-built binary (skips dune rebuild overhead)
- Timed module loading for startup diagnostics

sx_server.ml fixes:
- Uses shared _jit_compiling flag
- Marks lambdas as jit_failed_sentinel on compile failure (no retry spam)
- call_closure_reuse with VmSuspended frame merging for IO support

Compiled compiler bytecode bug: deeply nested cond/case/let forms
(e.g. tw-resolve-style) cause the compiled compiler to loop.
Workaround: _jit_compiling guard prevents compiled function execution
during compilation. Compilation uses CEK (slower but correct).
Test suite: 3127/3127 passed.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-04-07 21:59:31 +00:00
parent 75130876c7
commit 03278c640d
3 changed files with 193 additions and 124 deletions

View File

@@ -908,7 +908,7 @@ let sx_render_to_html expr env =
into the kernel env. The hook handles both cached execution (bytecode
already compiled) and first-call compilation (invoke compiler.sx via
CEK, cache result). cek_call checks this before CEK dispatch. *)
let _jit_compiling = ref false (* re-entrancy guard *)
(* Re-entrancy guard lives in Sx_vm._jit_compiling (shared with vm_call path) *)
(* JIT compilation is lazy-only: every named lambda gets one compile
attempt on first call. Failures are sentineled (never retried). *)
@@ -936,9 +936,11 @@ let register_jit_hook env =
| Lambda l ->
(match l.l_compiled with
| Some cl when not (Sx_vm.is_jit_failed cl) ->
(* Cached bytecode — run on VM, fall back to CEK on runtime error.
Log once per function name, then stay quiet. Don't disable. *)
(try Some (Sx_vm.call_closure cl args cl.vm_env_ref)
(* Cached bytecode — run via VM. Skip during compilation to avoid
compiled-compiler bytecode bugs on complex nested forms. *)
if !(Sx_vm._jit_compiling) then None
else
(try Some (Sx_vm.call_closure_reuse cl args)
with
| Sx_vm.VmSuspended (request, saved_vm) ->
Some (make_vm_suspend_marker request saved_vm)
@@ -948,33 +950,34 @@ let register_jit_hook env =
Hashtbl.replace _jit_warned fn_name true;
Printf.eprintf "[jit] %s runtime fallback to CEK: %s\n%!" fn_name (Printexc.to_string e)
end;
l.l_compiled <- Some Sx_vm.jit_failed_sentinel;
None)
| Some _ -> None (* compile failed or disabled — CEK handles *)
| Some _ -> None
| None ->
(* Only block NEW compilations during _jit_compiling, not execution *)
let fn_name = match l.l_name with Some n -> n | None -> "?" in
if !_jit_compiling then None
if !(Sx_vm._jit_compiling) then None
else if Hashtbl.mem _jit_warned fn_name then None
else begin
_jit_compiling := true;
let t0 = Unix.gettimeofday () in
let compiled = Sx_vm.jit_compile_lambda l (env_to_vm_globals env) in
let dt = Unix.gettimeofday () -. t0 in
_jit_compiling := false;
if dt > 0.5 || (match compiled with None -> true | _ -> false) then
Printf.eprintf "[jit] %s compile %s in %.3fs\n%!"
fn_name (match compiled with Some _ -> "OK" | None -> "FAIL") dt;
Printf.eprintf "[jit] %s compile in %.3fs\n%!" fn_name dt;
match compiled with
| Some cl ->
l.l_compiled <- Some cl;
(try Some (Sx_vm.call_closure cl args cl.vm_env_ref)
(try Some (Sx_vm.call_closure_reuse cl args)
with
| Sx_vm.VmSuspended (request, saved_vm) ->
Some (make_vm_suspend_marker request saved_vm)
| e ->
Printf.eprintf "[jit] %s first-call fallback to CEK: %s\n%!" fn_name (Printexc.to_string e);
Hashtbl.replace _jit_warned fn_name true;
l.l_compiled <- Some Sx_vm.jit_failed_sentinel;
None)
| None -> None
| None ->
l.l_compiled <- Some Sx_vm.jit_failed_sentinel;
None
end)
| _ -> None)