Refactor MCP tree server: dispatch table, caching, validation, subprocess cleanup

Break up the 1735-line handle_tool match into 45 individual handler functions
with hashtable-based dispatch. Add mtime-based file parse caching (AST + CST),
consolidated run_command helper replacing 9 bare open_process_in patterns,
require_file/require_dir input validation, and pagination (limit/offset) for
sx_find_across, sx_comp_list, sx_comp_usage. Also includes pending VM changes:
rest-arity support, hyperscript parser, compiler/transpiler updates.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

hosts/ocaml/lib/sx_primitives.ml

@@ -941,7 +941,19 @@ let () =
     | [f; Nil] -> call f []
     | _ -> raise (Eval_error "apply: function and list"));
   register "identical?" (fun args ->
-    match args with [a; b] -> Bool (a == b) | _ -> raise (Eval_error "identical?: 2 args"));
+    match args with
+    | [a; b] ->
+      (* Physical identity for reference types, structural for values.
+         Numbers/strings/booleans from different constant pools must
+         compare equal when their values match. *)
+      let identical = match a, b with
+        | Number x, Number y -> x = y
+        | String x, String y -> x = y  (* String.equal *)
+        | Bool x, Bool y -> x = y
+        | Nil, Nil -> true
+        | _ -> a == b  (* reference identity for dicts, lists, etc. *)
+      in Bool identical
+    | _ -> raise (Eval_error "identical?: 2 args"));
   register "make-spread" (fun args ->
     match args with
     | [Dict d] ->

hosts/ocaml/lib/sx_ref.ml

@@ -895,7 +895,7 @@ and step_continue state =
 
 (* continue-with-call *)
 and continue_with_call f args env raw_args kont =
-  (if sx_truthy ((parameter_p (f))) then (let uid = (parameter_uid (f)) in let frame = (kont_find_provide (kont) (uid)) in (make_cek_value ((if sx_truthy (frame) then (get (frame) ((String "value"))) else (parameter_default (f)))) (env) (kont))) else (if sx_truthy ((callcc_continuation_p (f))) then (let arg = (if sx_truthy ((empty_p (args))) then Nil else (first (args))) in let captured = (callcc_continuation_data (f)) in (make_cek_value (arg) (env) (captured))) else (if sx_truthy ((continuation_p (f))) then (let arg = (if sx_truthy ((empty_p (args))) then Nil else (first (args))) in let cont_data = (continuation_data (f)) in (let captured = (get (cont_data) ((String "captured"))) in (let result' = (cek_run ((make_cek_value (arg) (env) (captured)))) in (make_cek_value (result') (env) (kont))))) else (if sx_truthy ((let _and = (is_callable (f)) in if not (sx_truthy _and) then _and else (let _and = (Bool (not (sx_truthy ((is_lambda (f)))))) in if not (sx_truthy _and) then _and else (let _and = (Bool (not (sx_truthy ((is_component (f)))))) in if not (sx_truthy _and) then _and else (Bool (not (sx_truthy ((is_island (f)))))))))) then (let result' = (sx_apply_cek f args) in (if sx_truthy ((Bool (is_eval_error result'))) then (make_cek_value ((get (result') ((String "message")))) (env) ((kont_push ((make_raise_eval_frame (env) ((Bool false)))) (kont)))) else (make_cek_value (result') (env) (kont)))) else (if sx_truthy ((is_lambda (f))) then (let params = (lambda_params (f)) in let local = (env_merge ((lambda_closure (f))) (env)) in (let () = ignore ((if sx_truthy ((Bool (not (sx_truthy ((bind_lambda_params (params) (args) (local))))))) then (let () = ignore ((if sx_truthy ((prim_call ">" [(len (args)); (len (params))])) then (raise (Eval_error (value_to_str (String (sx_str [(let _or = (lambda_name (f)) in if sx_truthy _or then _or else (String "lambda")); (String " expects "); (len (params)); (String " args, got "); (len (args))]))))) else Nil)) in (let () = ignore ((List.iter (fun pair -> ignore ((env_bind local (sx_to_string (first (pair))) (nth (pair) ((Number 1.0)))))) (sx_to_list (prim_call "zip" [params; args])); Nil)) in (List.iter (fun p -> ignore ((env_bind local (sx_to_string p) Nil))) (sx_to_list (prim_call "slice" [params; (len (args))])); Nil))) else Nil)) in (let jit_result = (jit_try_call (f) (args)) in (if sx_truthy ((is_nil (jit_result))) then (make_cek_state ((lambda_body (f))) (local) (kont)) else (if sx_truthy ((let _and = (dict_p (jit_result)) in if not (sx_truthy _and) then _and else (get (jit_result) ((String "__vm_suspended"))))) then (make_cek_suspended ((get (jit_result) ((String "request")))) (env) ((kont_push ((make_vm_resume_frame ((get (jit_result) ((String "resume")))) (env))) (kont)))) else (make_cek_value (jit_result) (local) (kont))))))) else (if sx_truthy ((let _or = (is_component (f)) in if sx_truthy _or then _or else (is_island (f)))) then (let parsed = (parse_keyword_args (raw_args) (env)) in let kwargs = (first (parsed)) in let children = (nth (parsed) ((Number 1.0))) in let local = (env_merge ((component_closure (f))) (env)) in (let () = ignore ((List.iter (fun p -> ignore ((env_bind local (sx_to_string p) (let _or = (dict_get (kwargs) (p)) in if sx_truthy _or then _or else Nil)))) (sx_to_list (component_params (f))); Nil)) in (let () = ignore ((if sx_truthy ((component_has_children (f))) then (env_bind local (sx_to_string (String "children")) children) else Nil)) in (make_cek_state ((component_body (f))) (local) ((kont_push ((make_comp_trace_frame ((component_name (f))) ((component_file (f))))) (kont))))))) else (raise (Eval_error (value_to_str (String (sx_str [(String "Not callable: "); (inspect (f))])))))))))))
+  (if sx_truthy ((parameter_p (f))) then (let uid = (parameter_uid (f)) in let frame = (kont_find_provide (kont) (uid)) in (make_cek_value ((if sx_truthy (frame) then (get (frame) ((String "value"))) else (parameter_default (f)))) (env) (kont))) else (if sx_truthy ((callcc_continuation_p (f))) then (let arg = (if sx_truthy ((empty_p (args))) then Nil else (first (args))) in let captured = (callcc_continuation_data (f)) in (make_cek_value (arg) (env) (captured))) else (if sx_truthy ((continuation_p (f))) then (let arg = (if sx_truthy ((empty_p (args))) then Nil else (first (args))) in let cont_data = (continuation_data (f)) in (let captured = (get (cont_data) ((String "captured"))) in (let result' = (cek_run ((make_cek_value (arg) (env) (captured)))) in (make_cek_value (result') (env) (kont))))) else (if sx_truthy ((let _and = (is_callable (f)) in if not (sx_truthy _and) then _and else (let _and = (Bool (not (sx_truthy ((is_lambda (f)))))) in if not (sx_truthy _and) then _and else (let _and = (Bool (not (sx_truthy ((is_component (f)))))) in if not (sx_truthy _and) then _and else (Bool (not (sx_truthy ((is_island (f)))))))))) then (let result' = (sx_apply_cek f args) in (if sx_truthy ((Bool (is_eval_error result'))) then (make_cek_value ((get (result') ((String "message")))) (env) ((kont_push ((make_raise_eval_frame (env) ((Bool false)))) (kont)))) else (make_cek_value (result') (env) (kont)))) else (if sx_truthy ((is_lambda (f))) then (let params = (lambda_params (f)) in let local = (env_merge ((lambda_closure (f))) (env)) in (let () = ignore ((if sx_truthy ((Bool (not (sx_truthy ((bind_lambda_params (params) (args) (local))))))) then (let () = ignore ((if sx_truthy ((prim_call ">" [(len (args)); (len (params))])) then (raise (Eval_error (value_to_str (String (sx_str [(let _or = (lambda_name (f)) in if sx_truthy _or then _or else (String "lambda")); (String " expects "); (len (params)); (String " args, got "); (len (args))]))))) else Nil)) in (let () = ignore ((List.iter (fun pair -> ignore ((env_bind local (sx_to_string (first (pair))) (nth (pair) ((Number 1.0)))))) (sx_to_list (prim_call "zip" [params; args])); Nil)) in (List.iter (fun p -> ignore ((env_bind local (sx_to_string p) Nil))) (sx_to_list (prim_call "slice" [params; (len (args))])); Nil))) else Nil)) in (let jit_result = (jit_try_call (f) (args)) in (if sx_truthy ((jit_skip_p (jit_result))) then (make_cek_state ((lambda_body (f))) (local) (kont)) else (if sx_truthy ((let _and = (dict_p (jit_result)) in if not (sx_truthy _and) then _and else (get (jit_result) ((String "__vm_suspended"))))) then (make_cek_suspended ((get (jit_result) ((String "request")))) (env) ((kont_push ((make_vm_resume_frame ((get (jit_result) ((String "resume")))) (env))) (kont)))) else (make_cek_value (jit_result) (local) (kont))))))) else (if sx_truthy ((let _or = (is_component (f)) in if sx_truthy _or then _or else (is_island (f)))) then (let parsed = (parse_keyword_args (raw_args) (env)) in let kwargs = (first (parsed)) in let children = (nth (parsed) ((Number 1.0))) in let local = (env_merge ((component_closure (f))) (env)) in (let () = ignore ((List.iter (fun p -> ignore ((env_bind local (sx_to_string p) (let _or = (dict_get (kwargs) (p)) in if sx_truthy _or then _or else Nil)))) (sx_to_list (component_params (f))); Nil)) in (let () = ignore ((if sx_truthy ((component_has_children (f))) then (env_bind local (sx_to_string (String "children")) children) else Nil)) in (make_cek_state ((component_body (f))) (local) ((kont_push ((make_comp_trace_frame ((component_name (f))) ((component_file (f))))) (kont))))))) else (raise (Eval_error (value_to_str (String (sx_str [(String "Not callable: "); (inspect (f))])))))))))))
 
 (* sf-case-step-loop *)
 and sf_case_step_loop match_val clauses env kont =

hosts/ocaml/lib/sx_runtime.ml

@@ -186,6 +186,7 @@ let get_val container key =
        Hashtbl.replace d "vc-bytecode" (List bc);
        Hashtbl.replace d "vc-constants" (List consts);
        Hashtbl.replace d "vc-arity" (Number (float_of_int c.vc_arity));
+       Hashtbl.replace d "vc-rest-arity" (Number (float_of_int c.vc_rest_arity));
        Hashtbl.replace d "vc-locals" (Number (float_of_int c.vc_locals));
        Dict d
      | "vm-upvalues" ->
@@ -496,13 +497,28 @@ let _jit_hit = ref 0
 let _jit_miss = ref 0
 let _jit_skip = ref 0
 let jit_reset_counters () = _jit_hit := 0; _jit_miss := 0; _jit_skip := 0
+(* Sentinel value for "JIT skipped — fall back to CEK".
+   Must be distinguishable from any legitimate return value including Nil.
+   We use a unique tagged dict that is_jit_skip can identify. *)
+let _jit_skip_sentinel =
+  let d = Hashtbl.create 1 in
+  Hashtbl.replace d "__jit_skip" (Bool true);
+  Dict d
+
+let is_jit_skip v = match v with
+  | Dict d -> Hashtbl.mem d "__jit_skip"
+  | _ -> false
+
+(* Platform function for the spec: (jit-skip? v) → transpiles to jit_skip_p *)
+let jit_skip_p v = Bool (is_jit_skip v)
+
 let jit_try_call f args =
   match !_jit_try_call_fn with
-  | None -> incr _jit_skip; Nil
+  | None -> incr _jit_skip; _jit_skip_sentinel
   | Some hook ->
     match f with
     | Lambda l when l.l_name <> None ->
       let arg_list = match args with List a | ListRef { contents = a } -> a | _ -> [] in
-      (match hook f arg_list with Some result -> incr _jit_hit; result | None -> incr _jit_miss; Nil)
-    | _ -> incr _jit_skip; Nil
+      (match hook f arg_list with Some result -> incr _jit_hit; result | None -> incr _jit_miss; _jit_skip_sentinel)
+    | _ -> incr _jit_skip; _jit_skip_sentinel
 

hosts/ocaml/lib/sx_types.ml

@@ -178,6 +178,7 @@ and parameter = {
 (** Compiled function body — bytecode + constant pool. *)
 and vm_code = {
   vc_arity     : int;
+  vc_rest_arity : int;  (** -1 = no &rest; >= 0 = number of positional params before &rest *)
   vc_locals    : int;
   vc_bytecode  : int array;
   vc_constants : value array;

hosts/ocaml/lib/sx_vm.ml

@@ -50,7 +50,7 @@ let jit_compile_ref : (lambda -> (string, value) Hashtbl.t -> vm_closure option)
 (** Sentinel closure indicating JIT compilation was attempted and failed.
     Prevents retrying compilation on every call. *)
 let jit_failed_sentinel = {
-  vm_code = { vc_arity = -1; vc_locals = 0; vc_bytecode = [||]; vc_constants = [||];
+  vm_code = { vc_arity = -1; vc_rest_arity = -1; vc_locals = 0; vc_bytecode = [||]; vc_constants = [||];
               vc_bytecode_list = None; vc_constants_list = None };
   vm_upvalues = [||]; vm_name = Some "__jit_failed__"; vm_env_ref = Hashtbl.create 0; vm_closure_env = None
 }
@@ -142,34 +142,61 @@ let _jit_compiling = ref false
     This is the fast path for intra-VM closure calls. *)
 let push_closure_frame vm cl args =
   let frame = { closure = cl; ip = 0; base = vm.sp; local_cells = Hashtbl.create 4 } in
-  List.iter (fun a -> push vm a) args;
-  for _ = List.length args to cl.vm_code.vc_locals - 1 do push vm Nil done;
+  let rest_arity = cl.vm_code.vc_rest_arity in
+  if rest_arity >= 0 then begin
+    (* &rest function: push positional args, collect remainder into a list.
+       For (fn (a b &rest c) body) with rest_arity=2:
+         slots: 0=a, 1=b, 2=c (the rest list) *)
+    let nargs = List.length args in
+    let rec push_args i = function
+      | [] ->
+        for _ = i to rest_arity - 1 do push vm Nil done;
+        push vm (List [])
+      | a :: remaining ->
+        if i < rest_arity then (push vm a; push_args (i + 1) remaining)
+        else push vm (List (a :: remaining))
+    in
+    push_args 0 args;
+    let used = (if nargs > rest_arity then rest_arity + 1 else nargs + 1) in
+    for _ = used to cl.vm_code.vc_locals - 1 do push vm Nil done
+  end else begin
+    List.iter (fun a -> push vm a) args;
+    for _ = List.length args to cl.vm_code.vc_locals - 1 do push vm Nil done
+  end;
   vm.frames <- frame :: vm.frames
 
 (** Convert compiler output (SX dict) to a vm_code object. *)
 let code_from_value v =
   match v with
   | Dict d ->
-    let bc_list = match Hashtbl.find_opt d "bytecode" with
+    (* Accept both compiler output keys (bytecode/constants/arity) and
+       SX vm-code keys (vc-bytecode/vc-constants/vc-arity) *)
+    let find2 k1 k2 = match Hashtbl.find_opt d k1 with
+      | Some _ as r -> r | None -> Hashtbl.find_opt d k2 in
+    let bc_list = match find2 "bytecode" "vc-bytecode" with
       | Some (List l | ListRef { contents = l }) ->
         Array.of_list (List.map (fun x -> match x with Number n -> int_of_float n | _ -> 0) l)
       | _ -> [||]
     in
-    let entries = match Hashtbl.find_opt d "constants" with
+    let entries = match find2 "constants" "vc-constants" with
       | Some (List l | ListRef { contents = l }) -> Array.of_list l
       | _ -> [||]
     in
     let constants = Array.map (fun entry ->
       match entry with
-      | Dict ed when Hashtbl.mem ed "bytecode" -> entry  (* nested code — convert lazily *)
+      | Dict ed when Hashtbl.mem ed "bytecode" || Hashtbl.mem ed "vc-bytecode" -> entry
       | _ -> entry
     ) entries in
-    let arity = match Hashtbl.find_opt d "arity" with
+    let arity = match find2 "arity" "vc-arity" with
       | Some (Number n) -> int_of_float n | _ -> 0
     in
-    { vc_arity = arity; vc_locals = arity + 16; vc_bytecode = bc_list; vc_constants = constants;
+    let rest_arity = match find2 "rest-arity" "vc-rest-arity" with
+      | Some (Number n) -> int_of_float n | _ -> -1
+    in
+    { vc_arity = arity; vc_rest_arity = rest_arity; vc_locals = arity + 16;
+      vc_bytecode = bc_list; vc_constants = constants;
       vc_bytecode_list = None; vc_constants_list = None }
-  | _ -> { vc_arity = 0; vc_locals = 16; vc_bytecode = [||]; vc_constants = [||];
+  | _ -> { vc_arity = 0; vc_rest_arity = -1; vc_locals = 16; vc_bytecode = [||]; vc_constants = [||];
            vc_bytecode_list = None; vc_constants_list = None }
 
 (** JIT-compile a component or island body.

hosts/ocaml/lib/sx_vm_ref.ml

@@ -292,7 +292,7 @@ let vm_create_closure vm_val frame_val code_val =
 
 (* --- JIT sentinel --- *)
 let _jit_failed_sentinel = {
-  vm_code = { vc_arity = -1; vc_locals = 0; vc_bytecode = [||]; vc_constants = [||];
+  vm_code = { vc_arity = -1; vc_rest_arity = -1; vc_locals = 0; vc_bytecode = [||]; vc_constants = [||];
               vc_bytecode_list = None; vc_constants_list = None };
   vm_upvalues = [||]; vm_name = Some "__jit_failed__"; vm_env_ref = Hashtbl.create 0; vm_closure_env = None
 }