diff --git a/hosts/ocaml/bin/sx_server.ml b/hosts/ocaml/bin/sx_server.ml index 6690edc..6fa25ec 100644 --- a/hosts/ocaml/bin/sx_server.ml +++ b/hosts/ocaml/bin/sx_server.ml @@ -61,21 +61,26 @@ let rec serialize_value = function | RawHTML s -> "\"" ^ escape_sx_string s ^ "\"" | _ -> "nil" +(** Request epoch — monotonically increasing, set by (epoch N) from Python. + All responses are tagged with the current epoch so Python can discard + stale messages from previous requests. Makes pipe desync impossible. *) +let current_epoch = ref 0 + let send line = print_string line; print_char '\n'; flush stdout -let send_ok () = send "(ok)" -let send_ok_value v = send (Printf.sprintf "(ok %s)" (serialize_value v)) -let send_error msg = send (Printf.sprintf "(error \"%s\")" (escape_sx_string msg)) +let send_ok () = send (Printf.sprintf "(ok %d)" !current_epoch) +let send_ok_value v = send (Printf.sprintf "(ok %d %s)" !current_epoch (serialize_value v)) +let send_error msg = send (Printf.sprintf "(error %d \"%s\")" !current_epoch (escape_sx_string msg)) (** Length-prefixed binary send — handles any content without escaping. - Sends: (ok-len N)\n followed by exactly N bytes of raw data, then \n. + Sends: (ok-len EPOCH N)\n followed by exactly N bytes of raw data, then \n. Python reads the length line, then reads exactly N bytes. *) let send_ok_blob s = let n = String.length s in - Printf.printf "(ok-len %d)\n" n; + Printf.printf "(ok-len %d %d)\n" !current_epoch n; print_string s; print_char '\n'; flush stdout @@ -125,171 +130,11 @@ let io_batch_mode = ref false let io_queue : (int * string * value list) list ref = ref [] let io_counter = ref 0 -(* Request cookies — set by Python bridge before each page render. - get-cookie reads from here on the server; set-cookie is a no-op - (server can't set response cookies from SX — that's the framework's job). *) -let _request_cookies : (string, string) Hashtbl.t = Hashtbl.create 8 - -let () = Sx_primitives.register "get-cookie" (fun args -> - match args with - | [String name] -> - (match Hashtbl.find_opt _request_cookies name with - | Some v -> String v - | None -> Nil) - | _ -> Nil) - -let () = Sx_primitives.register "set-cookie" (fun _args -> - (* No-op on server — cookies are set via HTTP response headers *) - Nil) - -(* Module-level scope stacks — shared between make_server_env (aser - scope-push!/pop!) and step-sf-context (via get-primitive "scope-peek"). *) -let _scope_stacks : (string, value list) Hashtbl.t = Hashtbl.create 8 - -let () = Sx_primitives.register "scope-push!" (fun args -> - match args with - | [String name; value] -> - let stack = try Hashtbl.find _scope_stacks name with Not_found -> [] in - Hashtbl.replace _scope_stacks name (value :: stack); Nil - | _ -> Nil) - -let () = Sx_primitives.register "scope-pop!" (fun args -> - match args with - | [String name] -> - let stack = try Hashtbl.find _scope_stacks name with Not_found -> [] in - (match stack with _ :: rest -> Hashtbl.replace _scope_stacks name rest | [] -> ()); Nil - | _ -> Nil) - -let () = Sx_primitives.register "scope-peek" (fun args -> - match args with - | [String name] -> - let stack = try Hashtbl.find _scope_stacks name with Not_found -> [] in - (match stack with v :: _ -> v | [] -> Nil) - | _ -> Nil) - -let () = Sx_primitives.register "context" (fun args -> - match args with - | [String name] | [String name; _] -> - let stack = try Hashtbl.find _scope_stacks name with Not_found -> [] in - (match stack, args with - | v :: _, _ -> v - | [], [_; default_val] -> default_val - | [], _ -> Nil) - | _ -> Nil) - -(** collect! — lazy scope accumulator. Creates root scope if missing, - emits value (deduplicates). Used by cssx and spread components. *) -let () = Sx_primitives.register "collect!" (fun args -> - match args with - | [String name; value] -> - let stack = try Hashtbl.find _scope_stacks name with Not_found -> [] in - (match stack with - | List items :: rest -> - if not (List.mem value items) then - Hashtbl.replace _scope_stacks name (List (items @ [value]) :: rest) - | [] -> - (* Lazy root scope — create with the value *) - Hashtbl.replace _scope_stacks name [List [value]] - | _ :: _ -> ()); - Nil - | _ -> Nil) - -let () = Sx_primitives.register "collected" (fun args -> - match args with - | [String name] -> - let stack = try Hashtbl.find _scope_stacks name with Not_found -> [] in - (match stack with List items :: _ -> List items | _ -> List []) - | _ -> List []) - -let () = Sx_primitives.register "clear-collected!" (fun args -> - match args with - | [String name] -> - let stack = try Hashtbl.find _scope_stacks name with Not_found -> [] in - (match stack with - | _ :: rest -> Hashtbl.replace _scope_stacks name (List [] :: rest) - | [] -> Hashtbl.replace _scope_stacks name [List []]); - Nil - | _ -> Nil) - -(* emit!/emitted — adapter-html.sx uses these for spread attr collection *) -let () = Sx_primitives.register "scope-emit!" (fun args -> - match args with - | [String name; value] -> - let stack = try Hashtbl.find _scope_stacks name with Not_found -> [] in - (match stack with - | List items :: rest -> - Hashtbl.replace _scope_stacks name (List (items @ [value]) :: rest) - | v :: rest -> - (* Non-list top — wrap current entries as list + new value *) - Hashtbl.replace _scope_stacks name (List [value] :: v :: rest) - | [] -> - Hashtbl.replace _scope_stacks name [List [value]]); - Nil - | _ -> Nil) - -let () = Sx_primitives.register "emitted" (fun args -> - match args with - | [String name] -> - let stack = try Hashtbl.find _scope_stacks name with Not_found -> [] in - (match stack with List items :: _ -> List items | _ -> List []) - | _ -> List []) - -let () = Sx_primitives.register "scope-emitted" (fun args -> - match args with - | [String name] -> - let stack = try Hashtbl.find _scope_stacks name with Not_found -> [] in - (match stack with List items :: _ -> List items | _ -> List []) - | _ -> List []) - -let () = Sx_primitives.register "scope-collected" (fun args -> - match args with - | [String name] -> - let stack = try Hashtbl.find _scope_stacks name with Not_found -> [] in - (match stack with List items :: _ -> List items | _ -> List []) - | _ -> List []) - -let () = Sx_primitives.register "scope-clear-collected!" (fun args -> - match args with - | [String name] -> - let stack = try Hashtbl.find _scope_stacks name with Not_found -> [] in - (match stack with - | _ :: rest -> Hashtbl.replace _scope_stacks name (List [] :: rest) - | [] -> Hashtbl.replace _scope_stacks name [List []]); - Nil - | _ -> Nil) - -let () = Sx_primitives.register "provide-push!" (fun args -> - match Sx_primitives.get_primitive "scope-push!" with - | NativeFn (_, fn) -> fn args | _ -> Nil) -let () = Sx_primitives.register "provide-pop!" (fun args -> - match Sx_primitives.get_primitive "scope-pop!" with - | NativeFn (_, fn) -> fn args | _ -> Nil) - -let () = Sx_primitives.register "scope-emit!" (fun args -> - match args with - | [String name; value] -> - let stack = try Hashtbl.find _scope_stacks name with Not_found -> [] in - (match stack with - | List items :: rest -> - Hashtbl.replace _scope_stacks name (List (items @ [value]) :: rest) - | Nil :: rest -> - Hashtbl.replace _scope_stacks name (List [value] :: rest) - | [] -> - (* Lazy root scope *) - Hashtbl.replace _scope_stacks name [List [value]] - | _ :: _ -> ()); - Nil - | _ -> Nil) - -let () = Sx_primitives.register "clear-collected!" (fun args -> - match args with - | [String name] -> - let stack = try Hashtbl.find _scope_stacks name with Not_found -> [] in - (match stack with - | _ :: rest -> Hashtbl.replace _scope_stacks name (List [] :: rest) - | [] -> ()); - Nil - | _ -> Nil) +(* Scope stacks and cookies — all primitives registered in sx_scope.ml. + We just reference the shared state for the IO bridge. *) +module Sx_scope = Sx.Sx_scope +let _request_cookies = Sx_scope.request_cookies +let _scope_stacks = Sx_scope.scope_stacks (** Helpers safe to defer — pure functions whose results are only used as rendering output (inlined into SX wire format), not in control flow. *) @@ -303,6 +148,29 @@ let is_batchable name args = | String h :: _ -> List.mem h batchable_helpers | _ -> false +(** Read an io-response from stdin, discarding stale messages from old epochs. *) +let rec read_io_response () = + match read_line_blocking () with + | None -> raise (Eval_error "IO bridge: stdin closed while waiting for io-response") + | Some line -> + let exprs = Sx_parser.parse_all line in + match exprs with + (* Epoch-tagged: (io-response EPOCH value) *) + | [List [Symbol "io-response"; Number n; value]] + when int_of_float n = !current_epoch -> value + | [List (Symbol "io-response" :: Number n :: values)] + when int_of_float n = !current_epoch -> + (match values with [v] -> v | _ -> List values) + (* Legacy untagged: (io-response value) — accept for backwards compat *) + | [List [Symbol "io-response"; value]] -> value + | [List (Symbol "io-response" :: values)] -> + (match values with [v] -> v | _ -> List values) + (* Stale epoch or unexpected — discard and retry *) + | _ -> + Printf.eprintf "[io] discarding stale message (%d chars, epoch=%d)\n%!" + (String.length line) !current_epoch; + read_io_response () + (** Send an io-request — batch mode returns placeholder, else blocks. *) let io_request name args = if !io_batch_mode && is_batchable name args then begin @@ -313,20 +181,36 @@ let io_request name args = SxExpr (Printf.sprintf "(\xc2\xabIO:%d\xc2\xbb)" id) end else begin let args_str = String.concat " " (List.map serialize_value args) in - send (Printf.sprintf "(io-request \"%s\" %s)" name args_str); - (* Block on stdin for io-response *) + send (Printf.sprintf "(io-request %d \"%s\" %s)" !current_epoch name args_str); + read_io_response () + end + +(** Read a batched io-response, discarding stale epoch messages. *) +let read_batched_io_response () = + let rec loop () = match read_line_blocking () with - | None -> raise (Eval_error "IO bridge: stdin closed while waiting for io-response") + | None -> raise (Eval_error "IO batch: stdin closed") | Some line -> let exprs = Sx_parser.parse_all line in match exprs with - | [List [Symbol "io-response"; value]] -> value - | [List (Symbol "io-response" :: values)] -> - (match values with - | [v] -> v - | _ -> List values) - | _ -> raise (Eval_error ("IO bridge: unexpected response: " ^ line)) - end + (* Epoch-tagged: (io-response EPOCH value) *) + | [List [Symbol "io-response"; Number n; String s]] + when int_of_float n = !current_epoch -> s + | [List [Symbol "io-response"; Number n; SxExpr s]] + when int_of_float n = !current_epoch -> s + | [List [Symbol "io-response"; Number n; v]] + when int_of_float n = !current_epoch -> serialize_value v + (* Legacy untagged *) + | [List [Symbol "io-response"; String s]] + | [List [Symbol "io-response"; SxExpr s]] -> s + | [List [Symbol "io-response"; v]] -> serialize_value v + (* Stale — discard and retry *) + | _ -> + Printf.eprintf "[io-batch] discarding stale message (%d chars)\n%!" + (String.length line); + loop () + in + loop () (** Flush batched IO: send all requests, read all responses, replace placeholders. *) let flush_batched_io result_str = @@ -335,44 +219,35 @@ let flush_batched_io result_str = io_counter := 0; if queue = [] then result_str else begin - (* Send all batched requests with IDs *) + (* Send all batched requests with IDs, tagged with epoch *) List.iter (fun (id, name, args) -> let args_str = String.concat " " (List.map serialize_value args) in - send (Printf.sprintf "(io-request %d \"%s\" %s)" id name args_str) + send (Printf.sprintf "(io-request %d %d \"%s\" %s)" !current_epoch id name args_str) ) queue; - send (Printf.sprintf "(io-done %d)" (List.length queue)); + send (Printf.sprintf "(io-done %d %d)" !current_epoch (List.length queue)); (* Read all responses and replace placeholders *) let final = ref result_str in List.iter (fun (id, _, _) -> - match read_line_blocking () with - | Some line -> - let exprs = Sx_parser.parse_all line in - let value_str = match exprs with - | [List [Symbol "io-response"; String s]] - | [List [Symbol "io-response"; SxExpr s]] -> s - | [List [Symbol "io-response"; v]] -> serialize_value v - | _ -> "nil" - in - let placeholder = Printf.sprintf "(\xc2\xabIO:%d\xc2\xbb)" id in - (* Replace all occurrences of this placeholder *) - let plen = String.length placeholder in - let buf = Buffer.create (String.length !final) in - let pos = ref 0 in - let s = !final in - let slen = String.length s in - while !pos <= slen - plen do - if String.sub s !pos plen = placeholder then begin - Buffer.add_string buf value_str; - pos := !pos + plen - end else begin - Buffer.add_char buf s.[!pos]; - incr pos - end - done; - if !pos < slen then - Buffer.add_substring buf s !pos (slen - !pos); - final := Buffer.contents buf - | None -> raise (Eval_error "IO batch: stdin closed") + let value_str = read_batched_io_response () in + let placeholder = Printf.sprintf "(\xc2\xabIO:%d\xc2\xbb)" id in + (* Replace all occurrences of this placeholder *) + let plen = String.length placeholder in + let buf = Buffer.create (String.length !final) in + let pos = ref 0 in + let s = !final in + let slen = String.length s in + while !pos <= slen - plen do + if String.sub s !pos plen = placeholder then begin + Buffer.add_string buf value_str; + pos := !pos + plen + end else begin + Buffer.add_char buf s.[!pos]; + incr pos + end + done; + if !pos < slen then + Buffer.add_substring buf s !pos (slen - !pos); + final := Buffer.contents buf ) queue; !final end @@ -519,56 +394,21 @@ let make_server_env () = (* Scope stack — platform primitives for render-time dynamic scope. Used by aser for spread/provide/emit patterns. Module-level so step-sf-context can check it via get-primitive. *) - let scope_stacks = _scope_stacks in - bind "scope-push!" (fun args -> - match args with - | [String name; value] -> - let stack = try Hashtbl.find scope_stacks name with Not_found -> [] in - Hashtbl.replace scope_stacks name (value :: stack); Nil - | _ -> Nil); - bind "scope-pop!" (fun args -> - match args with - | [String name] -> - let stack = try Hashtbl.find scope_stacks name with Not_found -> [] in - (match stack with - | _ :: rest -> Hashtbl.replace scope_stacks name rest - | [] -> ()); Nil - | _ -> Nil); - bind "scope-peek" (fun args -> - match args with - | [String name] -> - let stack = try Hashtbl.find scope_stacks name with Not_found -> [] in - (match stack with v :: _ -> v | [] -> Nil) - | _ -> Nil); - (* scope-emit! / scope-peek — Hashtbl-based scope primitives for aser. - Different names from emit!/emitted to avoid CEK special form conflict. *) - bind "scope-emit!" (fun args -> - match args with - | [String name; value] -> - let stack = try Hashtbl.find scope_stacks name with Not_found -> [] in - (match stack with - | List items :: rest -> - Hashtbl.replace scope_stacks name (List (items @ [value]) :: rest) - | Nil :: rest -> - Hashtbl.replace scope_stacks name (List [value] :: rest) - | _ :: _ -> () - | [] -> ()); Nil - | _ -> Nil); - - (* context — scope lookup. The CEK handles this as a special form - by walking continuation frames, but compiled VM code needs it as - a function that reads from the scope_stacks hashtable. *) - let context_impl = NativeFn ("context", fun args -> - match args with - | [String name] | [String name; _] -> - let stack = try Hashtbl.find scope_stacks name with Not_found -> [] in - (match stack, args with - | v :: _, _ -> v - | [], [_; default_val] -> default_val - | [], _ -> Nil) - | _ -> Nil) in - ignore (env_bind env "sx-context" context_impl); - ignore (env_bind env "context" context_impl); + (* Scope primitives are registered globally in sx_scope.ml. + Bind them into the env so the JIT VM can find them via vm.globals + (OP_GLOBAL_GET checks env.bindings before the primitives table). *) + List.iter (fun name -> + try ignore (env_bind env name (Sx_primitives.get_primitive name)) + with _ -> () + ) ["scope-push!"; "scope-pop!"; "scope-peek"; "context"; + "collect!"; "collected"; "clear-collected!"; + "scope-emit!"; "emit!"; "emitted"; "scope-emitted"; + "scope-collected"; "scope-clear-collected!"; + "provide-push!"; "provide-pop!"; + "get-cookie"; "set-cookie"]; + (* sx-context is an env alias for context *) + let context_prim = Sx_primitives.get_primitive "context" in + ignore (env_bind env "sx-context" context_prim); (* qq-expand-runtime — quasiquote expansion at runtime. The bytecode compiler emits CALL_PRIM "qq-expand-runtime" for @@ -1667,9 +1507,17 @@ let () = | Some line -> let line = String.trim line in if line = "" then () (* skip blank lines *) + (* Discard stale io-responses from previous requests. *) + else if String.length line > 14 + && String.sub line 0 14 = "(io-response " then + Printf.eprintf "[sx-server] discarding stale io-response (%d chars)\n%!" + (String.length line) else begin let exprs = Sx_parser.parse_all line in match exprs with + (* Epoch marker: (epoch N) — set current epoch, read next command *) + | [List [Symbol "epoch"; Number n]] -> + current_epoch := int_of_float n | [cmd] -> dispatch env cmd | _ -> send_error ("Expected single command, got " ^ string_of_int (List.length exprs)) end diff --git a/hosts/ocaml/lib/sx_primitives.ml b/hosts/ocaml/lib/sx_primitives.ml index 2a27dcb..f821212 100644 --- a/hosts/ocaml/lib/sx_primitives.ml +++ b/hosts/ocaml/lib/sx_primitives.ml @@ -36,10 +36,11 @@ let as_string = function | String s -> s | v -> raise (Eval_error ("Expected string, got " ^ type_of v)) -let as_list = function +let rec as_list = function | List l -> l | ListRef r -> !r | Nil -> [] + | Thunk _ as t -> as_list (!_sx_trampoline_fn t) | v -> raise (Eval_error ("Expected list, got " ^ type_of v)) let as_bool = function @@ -316,8 +317,16 @@ let () = | [List l] | [ListRef { contents = l }] -> Number (float_of_int (List.length l)) | [String s] -> Number (float_of_int (String.length s)) | [Dict d] -> Number (float_of_int (Hashtbl.length d)) - | [Nil] -> Number 0.0 - | _ -> raise (Eval_error "len: 1 arg")); + | [Nil] | [Bool false] -> Number 0.0 + | [Bool true] -> Number 1.0 + | [Number _] -> Number 1.0 + | [RawHTML s] -> Number (float_of_int (String.length s)) + | [SxExpr s] -> Number (float_of_int (String.length s)) + | [Spread pairs] -> Number (float_of_int (List.length pairs)) + | [Component _] | [Island _] | [Lambda _] | [NativeFn _] + | [Macro _] | [Thunk _] | [Keyword _] | [Symbol _] -> Number 0.0 + | _ -> raise (Eval_error (Printf.sprintf "len: %d args" + (List.length args)))); register "first" (fun args -> match args with | [List (x :: _)] | [ListRef { contents = x :: _ }] -> x diff --git a/hosts/ocaml/lib/sx_scope.ml b/hosts/ocaml/lib/sx_scope.ml new file mode 100644 index 0000000..a5159c6 --- /dev/null +++ b/hosts/ocaml/lib/sx_scope.ml @@ -0,0 +1,154 @@ +(** Scope stacks — dynamic scope for render-time effects. + + Provides scope-push!/pop!/peek, collect!/collected/clear-collected!, + scope-emit!/emitted/scope-emitted, context, and cookie access. + + All functions are registered as primitives so both the CEK evaluator + and the JIT VM can find them in the same place. *) + +open Sx_types + +(** The shared scope stacks hashtable. Each key maps to a stack of values. + Used by aser for spread/provide/emit patterns, CSSX collect/flush, etc. *) +let scope_stacks : (string, value list) Hashtbl.t = Hashtbl.create 8 + +(** Request cookies — set by the Python bridge before each render. + get-cookie reads from here; set-cookie is a no-op on the server. *) +let request_cookies : (string, string) Hashtbl.t = Hashtbl.create 8 + +(** Clear all scope stacks. Called between requests if needed. *) +let clear_all () = Hashtbl.clear scope_stacks + +let () = + let register = Sx_primitives.register in + + (* --- Cookies --- *) + + register "get-cookie" (fun args -> + match args with + | [String name] -> + (match Hashtbl.find_opt request_cookies name with + | Some v -> String v + | None -> Nil) + | _ -> Nil); + + register "set-cookie" (fun _args -> Nil); + + (* --- Core scope stack operations --- *) + + register "scope-push!" (fun args -> + match args with + | [String name; value] -> + let stack = try Hashtbl.find scope_stacks name with Not_found -> [] in + Hashtbl.replace scope_stacks name (value :: stack); Nil + | _ -> Nil); + + register "scope-pop!" (fun args -> + match args with + | [String name] -> + let stack = try Hashtbl.find scope_stacks name with Not_found -> [] in + (match stack with _ :: rest -> Hashtbl.replace scope_stacks name rest | [] -> ()); Nil + | _ -> Nil); + + register "scope-peek" (fun args -> + match args with + | [String name] -> + let stack = try Hashtbl.find scope_stacks name with Not_found -> [] in + (match stack with v :: _ -> v | [] -> Nil) + | _ -> Nil); + + (* --- Context (scope lookup with optional default) --- *) + + register "context" (fun args -> + match args with + | [String name] | [String name; _] -> + let stack = try Hashtbl.find scope_stacks name with Not_found -> [] in + (match stack, args with + | v :: _, _ -> v + | [], [_; default_val] -> default_val + | [], _ -> Nil) + | _ -> Nil); + + (* --- Collect / collected / clear-collected! --- *) + + register "collect!" (fun args -> + match args with + | [String name; value] -> + let stack = try Hashtbl.find scope_stacks name with Not_found -> [] in + (match stack with + | List items :: rest -> + if not (List.mem value items) then + Hashtbl.replace scope_stacks name (List (items @ [value]) :: rest) + | [] -> + Hashtbl.replace scope_stacks name [List [value]] + | _ :: _ -> ()); + Nil + | _ -> Nil); + + register "collected" (fun args -> + match args with + | [String name] -> + let stack = try Hashtbl.find scope_stacks name with Not_found -> [] in + (match stack with List items :: _ -> List items | _ -> List []) + | _ -> List []); + + register "clear-collected!" (fun args -> + match args with + | [String name] -> + let stack = try Hashtbl.find scope_stacks name with Not_found -> [] in + (match stack with + | _ :: rest -> Hashtbl.replace scope_stacks name (List [] :: rest) + | [] -> Hashtbl.replace scope_stacks name [List []]); + Nil + | _ -> Nil); + + (* --- Emit / emitted (for spread attrs in adapter-html.sx) --- *) + + register "scope-emit!" (fun args -> + match args with + | [String name; value] -> + let stack = try Hashtbl.find scope_stacks name with Not_found -> [] in + (match stack with + | List items :: rest -> + Hashtbl.replace scope_stacks name (List (items @ [value]) :: rest) + | Nil :: rest -> + Hashtbl.replace scope_stacks name (List [value] :: rest) + | [] -> + Hashtbl.replace scope_stacks name [List [value]] + | _ :: _ -> ()); + Nil + | _ -> Nil); + + register "emit!" (fun args -> + (* Alias for scope-emit! *) + match Sx_primitives.get_primitive "scope-emit!" with + | NativeFn (_, fn) -> fn args | _ -> Nil); + + register "emitted" (fun args -> + match args with + | [String name] -> + let stack = try Hashtbl.find scope_stacks name with Not_found -> [] in + (match stack with List items :: _ -> List items | _ -> List []) + | _ -> List []); + + register "scope-emitted" (fun args -> + match Sx_primitives.get_primitive "emitted" with + | NativeFn (_, fn) -> fn args | _ -> List []); + + register "scope-collected" (fun args -> + match Sx_primitives.get_primitive "collected" with + | NativeFn (_, fn) -> fn args | _ -> List []); + + register "scope-clear-collected!" (fun args -> + match Sx_primitives.get_primitive "clear-collected!" with + | NativeFn (_, fn) -> fn args | _ -> Nil); + + (* --- Provide aliases --- *) + + register "provide-push!" (fun args -> + match Sx_primitives.get_primitive "scope-push!" with + | NativeFn (_, fn) -> fn args | _ -> Nil); + + register "provide-pop!" (fun args -> + match Sx_primitives.get_primitive "scope-pop!" with + | NativeFn (_, fn) -> fn args | _ -> Nil) diff --git a/hosts/ocaml/lib/sx_vm.ml b/hosts/ocaml/lib/sx_vm.ml index 0115919..3f9ec63 100644 --- a/hosts/ocaml/lib/sx_vm.ml +++ b/hosts/ocaml/lib/sx_vm.ml @@ -345,6 +345,13 @@ and run vm = let argc = read_u8 frame in let name = match consts.(idx) with String s -> s | _ -> "" in let args = List.init argc (fun _ -> pop vm) |> List.rev in + (* Resolve thunks — the CEK evaluator does this automatically + via trampoline, but the VM must do it explicitly before + passing args to primitives. *) + let args = List.map (fun v -> + match v with + | Thunk _ -> !Sx_primitives._sx_trampoline_fn v + | _ -> v) args in let result = try (* Check primitives FIRST (native implementations of map/filter/etc.), diff --git a/shared/static/scripts/sx-browser.js b/shared/static/scripts/sx-browser.js index fe97fa3..2c23d11 100644 --- a/shared/static/scripts/sx-browser.js +++ b/shared/static/scripts/sx-browser.js @@ -14,7 +14,7 @@ // ========================================================================= var NIL = Object.freeze({ _nil: true, toString: function() { return "nil"; } }); - var SX_VERSION = "2026-03-24T14:17:24Z"; + var SX_VERSION = "2026-03-24T15:37:52Z"; function isNil(x) { return x === NIL || x === null || x === undefined; } function isSxTruthy(x) { return x !== false && !isNil(x); } diff --git a/shared/sx/async_eval.py b/shared/sx/async_eval.py index 305a03b..e4cd21d 100644 --- a/shared/sx/async_eval.py +++ b/shared/sx/async_eval.py @@ -423,23 +423,39 @@ async def _asf_define(expr, env, ctx): async def _asf_defcomp(expr, env, ctx): - from .ref.sx_ref import sf_defcomp - return sf_defcomp(expr[1:], env) + # Component definitions are handled by OCaml kernel at load time. + # Python-side: just store a minimal Component in env for reference. + from .types import Component + name_sym = expr[1] + name = name_sym.name if hasattr(name_sym, 'name') else str(name_sym) + env[name] = Component(name=name.lstrip("~"), params=[], has_children=False, + body=expr[-1], closure={}) + return NIL async def _asf_defstyle(expr, env, ctx): - from .ref.sx_ref import sf_defstyle - return sf_defstyle(expr[1:], env) + # Style definitions handled by OCaml kernel. + return NIL async def _asf_defmacro(expr, env, ctx): - from .ref.sx_ref import sf_defmacro - return sf_defmacro(expr[1:], env) + # Macro definitions handled by OCaml kernel. + from .types import Macro + name_sym = expr[1] + name = name_sym.name if hasattr(name_sym, 'name') else str(name_sym) + params_form = expr[2] if len(expr) > 3 else [] + param_names = [p.name for p in params_form if isinstance(p, Symbol) and not p.name.startswith("&")] + rest_param = None + for i, p in enumerate(params_form): + if isinstance(p, Symbol) and p.name == "&rest" and i + 1 < len(params_form): + rest_param = params_form[i + 1].name if isinstance(params_form[i + 1], Symbol) else None + env[name] = Macro(name=name, params=param_names, rest_param=rest_param, body=expr[-1]) + return NIL async def _asf_defhandler(expr, env, ctx): - from .ref.sx_ref import sf_defhandler - return sf_defhandler(expr[1:], env) + # Handler definitions handled by OCaml kernel. + return NIL async def _asf_begin(expr, env, ctx): @@ -601,9 +617,12 @@ async def _asf_reset(expr, env, ctx): from .types import NIL _ASYNC_RESET_RESUME.append(value if value is not None else NIL) try: - # Sync re-evaluation; the async caller will trampoline - from .ref.sx_ref import eval_expr as sync_eval, trampoline as _trampoline - return _trampoline(sync_eval(body, env)) + # Continuations are handled by OCaml kernel. + # Python-side cont_fn should not be called in normal operation. + raise RuntimeError( + "Python-side continuation invocation not supported — " + "use OCaml bridge for shift/reset" + ) finally: _ASYNC_RESET_RESUME.pop() k = Continuation(cont_fn) diff --git a/shared/sx/deps.py b/shared/sx/deps.py index 0de5557..81f1ec4 100644 --- a/shared/sx/deps.py +++ b/shared/sx/deps.py @@ -152,18 +152,11 @@ def transitive_deps(name: str, env: dict[str, Any]) -> set[str]: Returns the set of all component names (with ~ prefix) that *name* can transitively render, NOT including *name* itself. """ - if _use_ref(): - from .ref.sx_ref import transitive_deps as _ref_td - return set(_ref_td(name, env)) return _transitive_deps_fallback(name, env) def compute_all_deps(env: dict[str, Any]) -> None: """Compute and cache deps for all Component entries in *env*.""" - if _use_ref(): - from .ref.sx_ref import compute_all_deps as _ref_cad - _ref_cad(env) - return _compute_all_deps_fallback(env) @@ -172,9 +165,6 @@ def scan_components_from_sx(source: str) -> set[str]: Returns names with ~ prefix, e.g. {"~card", "~shared:layout/nav-link"}. """ - if _use_ref(): - from .ref.sx_ref import scan_components_from_source as _ref_sc - return set(_ref_sc(source)) return _scan_components_from_sx_fallback(source) @@ -183,18 +173,11 @@ def components_needed(page_sx: str, env: dict[str, Any]) -> set[str]: Returns names with ~ prefix. """ - if _use_ref(): - from .ref.sx_ref import components_needed as _ref_cn - return set(_ref_cn(page_sx, env)) return _components_needed_fallback(page_sx, env) def compute_all_io_refs(env: dict[str, Any], io_names: set[str]) -> None: """Compute and cache transitive IO refs for all Component entries in *env*.""" - if _use_ref(): - from .ref.sx_ref import compute_all_io_refs as _ref_cio - _ref_cio(env, list(io_names)) - return _compute_all_io_refs_fallback(env, io_names) diff --git a/shared/sx/handlers.py b/shared/sx/handlers.py index fb77c77..d627e14 100644 --- a/shared/sx/handlers.py +++ b/shared/sx/handlers.py @@ -219,11 +219,8 @@ async def execute_handler( result_sx = await bridge.aser(sx_text, ctx=ocaml_ctx) return SxExpr(result_sx or "") else: - # Python fallback - if os.environ.get("SX_USE_REF") == "1": - from .ref.async_eval_ref import async_eval_to_sx - else: - from .async_eval import async_eval_to_sx + # Python fallback (async_eval) + from .async_eval import async_eval_to_sx env = dict(get_component_env()) env.update(get_page_helpers(service_name)) diff --git a/shared/sx/helpers.py b/shared/sx/helpers.py index 4c4546b..1021114 100644 --- a/shared/sx/helpers.py +++ b/shared/sx/helpers.py @@ -385,10 +385,7 @@ async def _render_to_sx_with_env(__name: str, extra_env: dict, **kwargs: Any) -> ocaml_ctx = {"_helper_service": _get_request_context().get("_helper_service", "")} if isinstance(_get_request_context(), dict) else {} return SxExpr(await bridge.aser_slot(sx_text, ctx=ocaml_ctx)) - if os.environ.get("SX_USE_REF") == "1": - from .ref.async_eval_ref import async_eval_slot_to_sx - else: - from .async_eval import async_eval_slot_to_sx + from .async_eval import async_eval_slot_to_sx env = dict(get_component_env()) env.update(extra_env) @@ -421,10 +418,7 @@ async def _render_to_sx(__name: str, **kwargs: Any) -> str: # symbols like `title` that were bound during the earlier expansion. return SxExpr(await bridge.aser_slot(sx_text)) - if os.environ.get("SX_USE_REF") == "1": - from .ref.async_eval_ref import async_eval_to_sx - else: - from .async_eval import async_eval_to_sx + from .async_eval import async_eval_to_sx env = dict(get_component_env()) ctx = _get_request_context() @@ -442,15 +436,23 @@ async def render_to_html(__name: str, **kwargs: Any) -> str: Same as render_to_sx() but produces HTML output instead of SX wire format. Used by route renders that need HTML (full pages, fragments). - Note: does NOT use OCaml bridge — the shell render is a pure HTML - template with no IO, so the Python renderer handles it reliably. - The OCaml path is used for _render_to_sx and _eval_slot (IO-heavy). + Routes through the OCaml bridge (render mode) which handles component + parameter binding, scope primitives, and all evaluation. """ - from .jinja_bridge import get_component_env, _get_request_context import os - from .async_eval import async_render ast = _build_component_ast(__name, **kwargs) + + if os.environ.get("SX_USE_OCAML") == "1": + from .ocaml_bridge import get_bridge + from .parser import serialize + bridge = await get_bridge() + sx_text = serialize(ast) + return await bridge.render(sx_text) + + # Fallback: Python async_eval (requires working evaluator) + from .jinja_bridge import get_component_env, _get_request_context + from .async_eval import async_render env = dict(get_component_env()) ctx = _get_request_context() return await async_render(ast, env, ctx) diff --git a/shared/sx/html.py b/shared/sx/html.py index 14361c1..686b12c 100644 --- a/shared/sx/html.py +++ b/shared/sx/html.py @@ -28,20 +28,163 @@ import contextvars from typing import Any from .types import Component, Island, Keyword, Lambda, Macro, NIL, Symbol -# sx_ref.py removed — these stubs exist so the module loads. -# With SX_USE_OCAML=1, rendering goes through the OCaml bridge; these -# are only called if a service falls back to Python-side rendering. -def _not_available(*a, **kw): - raise RuntimeError("sx_ref.py has been removed — use SX_USE_OCAML=1") -_raw_eval = _raw_call_component = _expand_macro = _trampoline = _not_available + def _eval(expr, env): - """Evaluate and unwrap thunks — all html.py _eval calls are non-tail.""" - return _trampoline(_raw_eval(expr, env)) + """Minimal Python evaluator for sync html.py rendering. -def _call_component(comp, raw_args, env): - """Call component and unwrap thunks — non-tail in html.py.""" - return _trampoline(_raw_call_component(comp, raw_args, env)) + Handles: literals, symbols, keywords, dicts, special forms (if, when, + cond, let, begin/do, and, or, str, not, list), lambda calls, and + primitive function calls. Enough for the sync sx() Jinja function. + """ + from .primitives import _PRIMITIVES + + # Literals + if isinstance(expr, (int, float, str, bool)): + return expr + if expr is None or expr is NIL: + return NIL + + # Symbol lookup + if isinstance(expr, Symbol): + name = expr.name + if name in env: + return env[name] + if name in _PRIMITIVES: + return _PRIMITIVES[name] + if name == "true": + return True + if name == "false": + return False + if name == "nil": + return NIL + from .types import EvalError + raise EvalError(f"Undefined symbol: {name}") + + # Keyword + if isinstance(expr, Keyword): + return expr.name + + # Dict + if isinstance(expr, dict): + return {k: _eval(v, env) for k, v in expr.items()} + + # List — dispatch + if not isinstance(expr, list): + return expr + if not expr: + return [] + + head = expr[0] + if isinstance(head, Symbol): + name = head.name + + # Special forms + if name == "if": + cond = _eval(expr[1], env) + if cond and cond is not NIL: + return _eval(expr[2], env) + return _eval(expr[3], env) if len(expr) > 3 else NIL + + if name == "when": + cond = _eval(expr[1], env) + if cond and cond is not NIL: + result = NIL + for body in expr[2:]: + result = _eval(body, env) + return result + return NIL + + if name == "let": + bindings = expr[1] + local = dict(env) + if isinstance(bindings, list): + if bindings and isinstance(bindings[0], list): + for b in bindings: + vname = b[0].name if isinstance(b[0], Symbol) else b[0] + local[vname] = _eval(b[1], local) + elif len(bindings) % 2 == 0: + for i in range(0, len(bindings), 2): + vname = bindings[i].name if isinstance(bindings[i], Symbol) else bindings[i] + local[vname] = _eval(bindings[i + 1], local) + result = NIL + for body in expr[2:]: + result = _eval(body, local) + return result + + if name in ("begin", "do"): + result = NIL + for body in expr[1:]: + result = _eval(body, env) + return result + + if name == "and": + result = True + for arg in expr[1:]: + result = _eval(arg, env) + if not result or result is NIL: + return result + return result + + if name == "or": + for arg in expr[1:]: + result = _eval(arg, env) + if result and result is not NIL: + return result + return NIL + + if name == "not": + val = _eval(expr[1], env) + return val is NIL or val is False or val is None + + if name == "lambda" or name == "fn": + params_form = expr[1] + param_names = [p.name if isinstance(p, Symbol) else str(p) for p in params_form] + return Lambda(params=param_names, body=expr[2], closure=dict(env)) + + if name == "define": + var_name = expr[1].name if isinstance(expr[1], Symbol) else str(expr[1]) + env[var_name] = _eval(expr[2], env) + return NIL + + if name == "quote": + return expr[1] + + if name == "str": + parts = [] + for arg in expr[1:]: + val = _eval(arg, env) + if val is NIL or val is None: + parts.append("") + else: + parts.append(str(val)) + return "".join(parts) + + if name == "list": + return [_eval(arg, env) for arg in expr[1:]] + + # Primitive or function call + fn = _eval(head, env) + else: + fn = _eval(head, env) + + # Evaluate args + args = [_eval(a, env) for a in expr[1:]] + + # Call + if callable(fn): + return fn(*args) + if isinstance(fn, Lambda): + local = dict(fn.closure) + local.update(env) + for p, v in zip(fn.params, args): + local[p] = v + return _eval(fn.body, local) + return NIL + + +def _expand_macro(*a, **kw): + raise RuntimeError("Macro expansion requires OCaml bridge") # ContextVar for collecting CSS class names during render. # Set to a set[str] to collect; None to skip. diff --git a/shared/sx/ocaml_bridge.py b/shared/sx/ocaml_bridge.py index fc3b747..b15c228 100644 --- a/shared/sx/ocaml_bridge.py +++ b/shared/sx/ocaml_bridge.py @@ -46,6 +46,7 @@ class OcamlBridge: self._components_loaded = False self._helpers_injected = False self._io_cache: dict[tuple, Any] = {} # (name, args...) → cached result + self._epoch: int = 0 # request epoch — monotonically increasing async def start(self) -> None: """Launch the OCaml subprocess and wait for (ready).""" @@ -77,7 +78,7 @@ class OcamlBridge: self._started = True # Verify engine identity - await self._send("(ping)") + await self._send_command("(ping)") kind, engine = await self._read_response() engine_name = engine if kind == "ok" else "unknown" _logger.info("OCaml SX kernel ready (pid=%d, engine=%s)", self._proc.pid, engine_name) @@ -95,24 +96,36 @@ class OcamlBridge: self._proc = None self._started = False + async def _restart(self) -> None: + """Kill and restart the OCaml subprocess to recover from pipe desync.""" + _logger.warning("Restarting OCaml SX kernel (pipe recovery)") + if self._proc and self._proc.returncode is None: + self._proc.kill() + await self._proc.wait() + self._proc = None + self._started = False + self._components_loaded = False + self._helpers_injected = False + await self.start() + async def ping(self) -> str: """Health check — returns engine name (e.g. 'ocaml-cek').""" async with self._lock: - await self._send("(ping)") + await self._send_command("(ping)") kind, value = await self._read_response() return value or "" if kind == "ok" else "" async def load(self, path: str) -> int: """Load an .sx file for side effects (defcomp, define, defmacro).""" async with self._lock: - await self._send(f'(load "{_escape(path)}")') + await self._send_command(f'(load "{_escape(path)}")') value = await self._read_until_ok(ctx=None) return int(float(value)) if value else 0 async def load_source(self, source: str) -> int: """Evaluate SX source for side effects.""" async with self._lock: - await self._send(f'(load-source "{_escape(source)}")') + await self._send_command(f'(load-source "{_escape(source)}")') value = await self._read_until_ok(ctx=None) return int(float(value)) if value else 0 @@ -124,7 +137,7 @@ class OcamlBridge: """ await self._ensure_components() async with self._lock: - await self._send('(eval-blob)') + await self._send_command('(eval-blob)') await self._send_blob(source) return await self._read_until_ok(ctx) @@ -136,14 +149,14 @@ class OcamlBridge: """Render SX to HTML, handling io-requests via Python async IO.""" await self._ensure_components() async with self._lock: - await self._send(f'(render "{_escape(source)}")') + await self._send_command(f'(render "{_escape(source)}")') return await self._read_until_ok(ctx) async def aser(self, source: str, ctx: dict[str, Any] | None = None) -> str: """Evaluate SX and return SX wire format, handling io-requests.""" await self._ensure_components() async with self._lock: - await self._send('(aser-blob)') + await self._send_command('(aser-blob)') await self._send_blob(source) return await self._read_until_ok(ctx) @@ -159,7 +172,7 @@ class OcamlBridge: # a separate lock acquisition could let another coroutine # interleave commands between injection and aser-slot. await self._inject_helpers_locked() - await self._send('(aser-slot-blob)') + await self._send_command('(aser-slot-blob)') await self._send_blob(source) return await self._read_until_ok(ctx) @@ -182,7 +195,7 @@ class OcamlBridge: var = f"__shell-{key.replace('_', '-')}" defn = f'(define {var} "{_escape(str(val))}")' try: - await self._send(f'(load-source "{_escape(defn)}")') + await self._send_command(f'(load-source "{_escape(defn)}")') await self._read_until_ok(ctx=None) except OcamlBridgeError as e: _logger.warning("Shell static inject failed for %s: %s", key, e) @@ -198,7 +211,7 @@ class OcamlBridge: else: defn = f'(define {var} "{_escape(str(val))}")' try: - await self._send(f'(load-source "{_escape(defn)}")') + await self._send_command(f'(load-source "{_escape(defn)}")') await self._read_until_ok(ctx=None) except OcamlBridgeError as e: _logger.warning("Shell static inject failed for %s: %s", key, e) @@ -221,7 +234,7 @@ class OcamlBridge: if pairs: cmd = f'(set-request-cookies {{{" ".join(pairs)}}})' try: - await self._send(cmd) + await self._send_command(cmd) await self._read_until_ok(ctx=None) except OcamlBridgeError as e: _logger.debug("Cookie inject failed: %s", e) @@ -277,7 +290,7 @@ class OcamlBridge: parts.append(f' :{k} "{_escape(str(val))}"') parts.append(")") cmd = "".join(parts) - await self._send(cmd) + await self._send_command(cmd) # Send page source as binary blob (avoids string-escape issues) await self._send_blob(page_source) html = await self._read_until_ok(ctx) @@ -312,7 +325,7 @@ class OcamlBridge: arg_list = " ".join(chr(97 + i) for i in range(nargs)) sx_def = f'(define {name} (fn ({param_names}) (helper "{name}" {arg_list})))' try: - await self._send(f'(load-source "{_escape(sx_def)}")') + await self._send_command(f'(load-source "{_escape(sx_def)}")') await self._read_until_ok(ctx=None) count += 1 except OcamlBridgeError: @@ -325,70 +338,11 @@ class OcamlBridge: async def _compile_adapter_module(self) -> None: """Compile adapter-sx.sx to bytecode and load as a VM module. - All aser functions become NativeFn VM closures in the kernel env. - Subsequent aser-slot calls find them as NativeFn → VM executes - the entire render path compiled, no CEK steps. + Previously used Python's sx_ref.py evaluator for compilation. + Now the OCaml kernel handles JIT compilation natively — this method + is a no-op. The kernel's own JIT hook compiles functions on first call. """ - from .parser import parse_all, serialize - from .ref.sx_ref import eval_expr, trampoline, PRIMITIVES - - # Ensure compiler primitives are available - if 'serialize' not in PRIMITIVES: - PRIMITIVES['serialize'] = lambda x: serialize(x) - if 'primitive?' not in PRIMITIVES: - PRIMITIVES['primitive?'] = lambda name: isinstance(name, str) and name in PRIMITIVES - if 'has-key?' not in PRIMITIVES: - PRIMITIVES['has-key?'] = lambda *a: isinstance(a[0], dict) and str(a[1]) in a[0] - if 'set-nth!' not in PRIMITIVES: - from .types import NIL - PRIMITIVES['set-nth!'] = lambda *a: (a[0].__setitem__(int(a[1]), a[2]), NIL)[-1] - if 'init' not in PRIMITIVES: - PRIMITIVES['init'] = lambda *a: a[0][:-1] if isinstance(a[0], list) else a[0] - if 'concat' not in PRIMITIVES: - PRIMITIVES['concat'] = lambda *a: (a[0] or []) + (a[1] or []) - if 'slice' not in PRIMITIVES: - PRIMITIVES['slice'] = lambda *a: a[0][int(a[1]):int(a[2])] if len(a) == 3 else a[0][int(a[1]):] - from .types import Symbol - if 'make-symbol' not in PRIMITIVES: - PRIMITIVES['make-symbol'] = lambda name: Symbol(name) - from .types import NIL - for ho in ['map', 'filter', 'for-each', 'reduce', 'some', 'every?', 'map-indexed']: - if ho not in PRIMITIVES: - PRIMITIVES[ho] = lambda *a: NIL - - # Load compiler - compiler_env = {} - spec_dir = os.path.join(os.path.dirname(__file__), "../../spec") - for f in ["bytecode.sx", "compiler.sx"]: - path = os.path.join(spec_dir, f) - if os.path.isfile(path): - with open(path) as fh: - for expr in parse_all(fh.read()): - trampoline(eval_expr(expr, compiler_env)) - - # Compile adapter-sx.sx - web_dir = os.path.join(os.path.dirname(__file__), "../../web") - adapter_path = os.path.join(web_dir, "adapter-sx.sx") - if not os.path.isfile(adapter_path): - _logger.warning("adapter-sx.sx not found at %s", adapter_path) - return - - with open(adapter_path) as f: - adapter_exprs = parse_all(f.read()) - - compiled = trampoline(eval_expr( - [Symbol('compile-module'), [Symbol('quote'), adapter_exprs]], - compiler_env)) - - code_sx = serialize(compiled) - _logger.info("Compiled adapter-sx.sx: %d bytes bytecode", len(code_sx)) - - # Load the compiled module into the OCaml VM - async with self._lock: - await self._send(f'(vm-load-module {code_sx})') - await self._read_until_ok(ctx=None) - - _logger.info("Loaded adapter-sx.sx as VM module") + _logger.info("Adapter module compilation delegated to OCaml kernel JIT") async def _ensure_components(self) -> None: """Load all .sx source files into the kernel on first use. @@ -455,7 +409,7 @@ class OcamlBridge: async with self._lock: for filepath in all_files: try: - await self._send(f'(load "{_escape(filepath)}")') + await self._send_command(f'(load "{_escape(filepath)}")') value = await self._read_until_ok(ctx=None) # Response may be a number (count) or a value — just count files count += 1 @@ -468,14 +422,14 @@ class OcamlBridge: # reactive loops during island SSR — effects are DOM side-effects) try: noop_dispose = '(fn () nil)' - await self._send(f'(load-source "(define effect (fn (f) {noop_dispose}))")') + await self._send_command(f'(load-source "(define effect (fn (f) {noop_dispose}))")') await self._read_until_ok(ctx=None) except OcamlBridgeError: pass # Register JIT hook — lambdas compile on first call try: - await self._send('(vm-compile-adapter)') + await self._send_command('(vm-compile-adapter)') await self._read_until_ok(ctx=None) _logger.info("JIT hook registered — lambdas compile on first call") except OcamlBridgeError as e: @@ -499,7 +453,7 @@ class OcamlBridge: if callable(fn) and not name.startswith("~"): sx_def = f'(define {name} (fn (&rest args) (apply helper (concat (list "{name}") args))))' try: - await self._send(f'(load-source "{_escape(sx_def)}")') + await self._send_command(f'(load-source "{_escape(sx_def)}")') await self._read_until_ok(ctx=None) count += 1 except OcamlBridgeError: @@ -510,7 +464,7 @@ class OcamlBridge: async def reset(self) -> None: """Reset the kernel environment to pristine state.""" async with self._lock: - await self._send("(reset)") + await self._send_command("(reset)") kind, value = await self._read_response() if kind == "error": raise OcamlBridgeError(f"reset: {value}") @@ -531,6 +485,20 @@ class OcamlBridge: self._proc.stdin.write((line + "\n").encode()) await self._proc.stdin.drain() + async def _send_command(self, line: str) -> None: + """Send a command with a fresh epoch prefix. + + Increments the epoch counter and sends (epoch N) before the + actual command. The OCaml kernel tags all responses with this + epoch so stale messages from previous requests are discarded. + """ + self._epoch += 1 + assert self._proc and self._proc.stdin + _logger.debug("EPOCH %d SEND: %s", self._epoch, line[:120]) + self._proc.stdin.write(f"(epoch {self._epoch})\n".encode()) + self._proc.stdin.write((line + "\n").encode()) + await self._proc.stdin.drain() + async def _send_blob(self, data: str) -> None: """Send a length-prefixed binary blob to the subprocess. @@ -562,16 +530,45 @@ class OcamlBridge: """Read a single (ok ...) or (error ...) response. Returns (kind, value) where kind is "ok" or "error". + Discards stale epoch messages. """ - line = await self._readline() - # Length-prefixed blob - if line.startswith("(ok-len "): - n = int(line[8:-1]) - assert self._proc and self._proc.stdout - data = await self._proc.stdout.readexactly(n) - await self._proc.stdout.readline() # trailing newline - return ("ok", data.decode()) - return _parse_response(line) + while True: + line = await self._readline() + if not self._is_current_epoch(line): + _logger.debug("Discarding stale response: %s", line[:80]) + if line.startswith("(ok-len "): + parts = line[1:-1].split() + if len(parts) >= 3: + n = int(parts[-1]) + assert self._proc and self._proc.stdout + await self._proc.stdout.readexactly(n) + await self._proc.stdout.readline() + continue + # Length-prefixed blob: (ok-len EPOCH N) or (ok-len N) + if line.startswith("(ok-len "): + parts = line[1:-1].split() + n = int(parts[-1]) + assert self._proc and self._proc.stdout + data = await self._proc.stdout.readexactly(n) + await self._proc.stdout.readline() # trailing newline + return ("ok", data.decode()) + return _parse_response(line) + + def _is_current_epoch(self, line: str) -> bool: + """Check if a response line belongs to the current epoch. + + Lines tagged with a stale epoch are discarded. Untagged lines + (from a kernel that predates the epoch protocol) are accepted. + """ + # Extract epoch number from known tagged formats: + # (ok EPOCH ...), (error EPOCH ...), (ok-len EPOCH N), + # (io-request EPOCH ...), (io-done EPOCH N) + import re + m = re.match(r'\((?:ok|error|ok-len|ok-raw|io-request|io-done)\s+(\d+)\b', line) + if m: + return int(m.group(1)) == self._epoch + # Untagged (legacy) — accept + return True async def _read_until_ok( self, @@ -583,6 +580,9 @@ class OcamlBridge: - Legacy (blocking): single io-request → immediate io-response - Batched: collect io-requests until (io-done N), process ALL concurrently with asyncio.gather, send responses in order + + Lines tagged with a stale epoch are silently discarded, making + pipe desync from previous failed requests impossible. """ import asyncio pending_batch: list[str] = [] @@ -590,20 +590,53 @@ class OcamlBridge: while True: line = await self._readline() + # Discard stale epoch messages + if not self._is_current_epoch(line): + _logger.debug("Discarding stale epoch message: %s", line[:80]) + # If it's a stale ok-len, drain the blob bytes too + if line.startswith("(ok-len "): + parts = line[1:-1].split() + if len(parts) >= 3: + n = int(parts[2]) + assert self._proc and self._proc.stdout + await self._proc.stdout.readexactly(n) + await self._proc.stdout.readline() + continue + if line.startswith("(io-request "): - # Check if batched (has numeric ID after "io-request ") + # New format: (io-request EPOCH ...) or (io-request EPOCH ID ...) + # Strip epoch from the line for IO dispatch after = line[len("(io-request "):].lstrip() + # Skip epoch number if present if after and after[0].isdigit(): - # Batched mode — collect, don't respond yet - pending_batch.append(line) - continue + # Could be epoch or batch ID — check for second number + parts = after.split(None, 2) + if len(parts) >= 2 and parts[1][0].isdigit(): + # (io-request EPOCH ID "name" args...) — batched with epoch + pending_batch.append(line) + continue + elif len(parts) >= 2 and parts[1].startswith('"'): + # (io-request EPOCH "name" args...) — legacy with epoch + try: + result = await self._handle_io_request(line, ctx) + await self._send( + f"(io-response {self._epoch} {_serialize_for_ocaml(result)})") + except Exception as e: + _logger.warning("IO request failed, sending nil: %s", e) + await self._send(f"(io-response {self._epoch} nil)") + continue + else: + # Old format: (io-request ID "name" ...) — batched, no epoch + pending_batch.append(line) + continue # Legacy blocking mode — respond immediately try: result = await self._handle_io_request(line, ctx) - await self._send(f"(io-response {_serialize_for_ocaml(result)})") + await self._send( + f"(io-response {self._epoch} {_serialize_for_ocaml(result)})") except Exception as e: _logger.warning("IO request failed, sending nil: %s", e) - await self._send("(io-response nil)") + await self._send(f"(io-response {self._epoch} nil)") continue if line.startswith("(io-done "): @@ -614,16 +647,17 @@ class OcamlBridge: for result in results: if isinstance(result, BaseException): _logger.warning("Batched IO failed: %s", result) - await self._send("(io-response nil)") + await self._send(f"(io-response {self._epoch} nil)") else: await self._send( - f"(io-response {_serialize_for_ocaml(result)})") + f"(io-response {self._epoch} {_serialize_for_ocaml(result)})") pending_batch = [] continue - # Length-prefixed blob: (ok-len N) + # Length-prefixed blob: (ok-len EPOCH N) or (ok-len N) if line.startswith("(ok-len "): - n = int(line[8:-1]) + parts = line[1:-1].split() # ["ok-len", epoch, n] or ["ok-len", n] + n = int(parts[-1]) # last number is always byte count assert self._proc and self._proc.stdout data = await self._proc.stdout.readexactly(n) # Read trailing newline @@ -829,25 +863,50 @@ def _escape(s: str) -> str: def _parse_response(line: str) -> tuple[str, str | None]: """Parse an (ok ...) or (error ...) response line. + Handles epoch-tagged responses: (ok EPOCH), (ok EPOCH value), + (error EPOCH "msg"), as well as legacy untagged responses. + Returns (kind, value) tuple. """ line = line.strip() - if line == "(ok)": + # (ok EPOCH) — tagged no-value + if line == "(ok)" or (line.startswith("(ok ") and line[4:-1].isdigit()): return ("ok", None) if line.startswith("(ok-raw "): - # Raw SX wire format — no unescaping needed - return ("ok", line[8:-1]) + # (ok-raw EPOCH value) or (ok-raw value) + inner = line[8:-1] + # Strip epoch if present + if inner and inner[0].isdigit(): + space = inner.find(" ") + if space > 0: + inner = inner[space + 1:] + else: + return ("ok", None) + return ("ok", inner) if line.startswith("(ok "): - value = line[4:-1] # strip (ok and ) + inner = line[4:-1] # strip (ok and ) + # Strip epoch number if present: (ok 42 "value") → "value" + if inner and inner[0].isdigit(): + space = inner.find(" ") + if space > 0: + inner = inner[space + 1:] + else: + # (ok EPOCH) with no value + return ("ok", None) # If the value is a quoted string, unquote it - if value.startswith('"') and value.endswith('"'): - value = _unescape(value[1:-1]) - return ("ok", value) + if inner.startswith('"') and inner.endswith('"'): + inner = _unescape(inner[1:-1]) + return ("ok", inner) if line.startswith("(error "): - msg = line[7:-1] - if msg.startswith('"') and msg.endswith('"'): - msg = _unescape(msg[1:-1]) - return ("error", msg) + inner = line[7:-1] + # Strip epoch number if present: (error 42 "msg") → "msg" + if inner and inner[0].isdigit(): + space = inner.find(" ") + if space > 0: + inner = inner[space + 1:] + if inner.startswith('"') and inner.endswith('"'): + inner = _unescape(inner[1:-1]) + return ("error", inner) return ("error", f"Unexpected response: {line}") diff --git a/shared/sx/ocaml_sync.py b/shared/sx/ocaml_sync.py index a8002d5..1418e05 100644 --- a/shared/sx/ocaml_sync.py +++ b/shared/sx/ocaml_sync.py @@ -52,6 +52,7 @@ class OcamlSync: def __init__(self, binary: str | None = None): self._binary = binary or os.environ.get("SX_OCAML_BIN") or _DEFAULT_BIN self._proc: subprocess.Popen | None = None + self._epoch: int = 0 def _ensure(self): if self._proc is not None and self._proc.poll() is None: @@ -62,13 +63,17 @@ class OcamlSync: stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) + self._epoch = 0 # Wait for (ready) line = self._readline() if line != "(ready)": raise OcamlSyncError(f"Expected (ready), got: {line}") def _send(self, command: str): + """Send a command with epoch prefix.""" assert self._proc and self._proc.stdin + self._epoch += 1 + self._proc.stdin.write(f"(epoch {self._epoch})\n".encode()) self._proc.stdin.write((command + "\n").encode()) self._proc.stdin.flush() @@ -79,12 +84,26 @@ class OcamlSync: raise OcamlSyncError("OCaml subprocess died unexpectedly") return data.decode().rstrip("\n") + def _strip_epoch(self, inner: str) -> str: + """Strip leading epoch number from a response value: '42 value' → 'value'.""" + if inner and inner[0].isdigit(): + space = inner.find(" ") + if space > 0: + return inner[space + 1:] + return "" # epoch only, no value + return inner + def _read_response(self) -> str: - """Read a single response. Returns the value string or raises on error.""" + """Read a single response. Returns the value string or raises on error. + + Handles epoch-tagged responses: (ok EPOCH), (ok EPOCH value), + (ok-len EPOCH N), (error EPOCH "msg"). + """ line = self._readline() - # Length-prefixed blob: (ok-len N) + # Length-prefixed blob: (ok-len N) or (ok-len EPOCH N) if line.startswith("(ok-len "): - n = int(line[8:-1]) + parts = line[1:-1].split() # ["ok-len", ...] + n = int(parts[-1]) # last number is always byte count assert self._proc and self._proc.stdout data = self._proc.stdout.read(n) self._proc.stdout.readline() # trailing newline @@ -93,17 +112,18 @@ class OcamlSync: if value.startswith('"') and value.endswith('"'): value = _sx_unescape(value[1:-1]) return value - if line == "(ok)": + if line == "(ok)" or (line.startswith("(ok ") and line[4:-1].isdigit()): return "" if line.startswith("(ok-raw "): - return line[8:-1] + inner = self._strip_epoch(line[8:-1]) + return inner if line.startswith("(ok "): - value = line[4:-1] + value = self._strip_epoch(line[4:-1]) if value.startswith('"') and value.endswith('"'): value = _sx_unescape(value[1:-1]) return value if line.startswith("(error "): - msg = line[7:-1] + msg = self._strip_epoch(line[7:-1]) if msg.startswith('"') and msg.endswith('"'): msg = _sx_unescape(msg[1:-1]) raise OcamlSyncError(msg) diff --git a/shared/sx/pages.py b/shared/sx/pages.py index 41f1794..221e1a0 100644 --- a/shared/sx/pages.py +++ b/shared/sx/pages.py @@ -313,10 +313,7 @@ async def _eval_slot(expr: Any, env: dict, ctx: Any) -> str: sx_text = _wrap_with_env(expr, env) service = ctx.get("_helper_service", "") if isinstance(ctx, dict) else "" return await bridge.aser_slot(sx_text, ctx={"_helper_service": service}) - if os.environ.get("SX_USE_REF") == "1": - from .ref.async_eval_ref import async_eval_slot_to_sx - else: - from .async_eval import async_eval_slot_to_sx + from .async_eval import async_eval_slot_to_sx return await async_eval_slot_to_sx(expr, env, ctx) diff --git a/shared/sx/parser.py b/shared/sx/parser.py index ed5eaad..c10a28e 100644 --- a/shared/sx/parser.py +++ b/shared/sx/parser.py @@ -38,10 +38,11 @@ def _resolve_sx_reader_macro(name: str): If a file like z3.sx defines (define z3-translate ...), then #z3 is automatically available as a reader macro without any Python registration. Looks for {name}-translate as a Lambda in the component env. + + Uses the synchronous OCaml bridge (ocaml_sync) when available. """ try: from .jinja_bridge import get_component_env - from .ref.sx_ref import trampoline as _trampoline, call_lambda as _call_lambda from .types import Lambda except ImportError: return None @@ -49,10 +50,18 @@ def _resolve_sx_reader_macro(name: str): fn = env.get(f"{name}-translate") if fn is None or not isinstance(fn, Lambda): return None - # Return a Python callable that invokes the SX lambda - def _sx_handler(expr): - return _trampoline(_call_lambda(fn, [expr], env)) - return _sx_handler + # Use sync OCaml bridge to invoke the lambda + try: + from .ocaml_sync import OcamlSync + _sync = OcamlSync() + _sync.start() + def _sx_handler(expr): + from .parser import serialize as _ser + result = _sync.eval(f"({name}-translate {_ser(expr)})") + return parse(result) if result else expr + return _sx_handler + except Exception: + return None # --------------------------------------------------------------------------- diff --git a/shared/sx/primitives.py b/shared/sx/primitives.py index 7c98f7e..de88f48 100644 --- a/shared/sx/primitives.py +++ b/shared/sx/primitives.py @@ -579,26 +579,54 @@ def prim_json_encode(value) -> str: # (shared global state between transpiled and hand-written evaluators) # --------------------------------------------------------------------------- -def _lazy_scope_primitives(): - """Register scope/provide/collect primitives from sx_ref.py. +def _register_scope_primitives(): + """Register scope/provide/collect primitive stubs. - Called at import time — if sx_ref.py isn't built yet, silently skip. - These are needed by the hand-written _aser in async_eval.py when - expanding components that use scoped effects (e.g. ~cssx/flush). + The OCaml kernel provides the real implementations. These stubs exist + so _PRIMITIVES contains the names for dependency analysis, and so + any Python-side code that checks for their existence finds them. """ - try: - from .ref.sx_ref import ( - sx_collect, sx_collected, sx_clear_collected, - sx_emitted, sx_emit, sx_context, - ) - _PRIMITIVES["collect!"] = sx_collect - _PRIMITIVES["collected"] = sx_collected - _PRIMITIVES["clear-collected!"] = sx_clear_collected - _PRIMITIVES["emitted"] = sx_emitted - _PRIMITIVES["emit!"] = sx_emit - _PRIMITIVES["context"] = sx_context - except ImportError: - pass + import threading + _scope_data = threading.local() -_lazy_scope_primitives() + def _collect(channel, value): + if not hasattr(_scope_data, 'collected'): + _scope_data.collected = {} + _scope_data.collected.setdefault(channel, []).append(value) + return NIL + + def _collected(channel): + if not hasattr(_scope_data, 'collected'): + return [] + return list(_scope_data.collected.get(channel, [])) + + def _clear_collected(channel): + if hasattr(_scope_data, 'collected'): + _scope_data.collected.pop(channel, None) + return NIL + + def _emit(channel, value): + if not hasattr(_scope_data, 'emitted'): + _scope_data.emitted = {} + _scope_data.emitted.setdefault(channel, []).append(value) + return NIL + + def _emitted(channel): + if not hasattr(_scope_data, 'emitted'): + return [] + return list(_scope_data.emitted.get(channel, [])) + + def _context(key): + if not hasattr(_scope_data, 'context'): + return NIL + return _scope_data.context.get(key, NIL) if isinstance(_scope_data.context, dict) else NIL + + _PRIMITIVES["collect!"] = _collect + _PRIMITIVES["collected"] = _collected + _PRIMITIVES["clear-collected!"] = _clear_collected + _PRIMITIVES["emitted"] = _emitted + _PRIMITIVES["emit!"] = _emit + _PRIMITIVES["context"] = _context + +_register_scope_primitives() diff --git a/shared/sx/query_executor.py b/shared/sx/query_executor.py index 6a69414..ba231d8 100644 --- a/shared/sx/query_executor.py +++ b/shared/sx/query_executor.py @@ -49,10 +49,7 @@ async def execute_query(query_def: QueryDef, params: dict[str, str]) -> Any: result = None return _normalize(result) - if os.environ.get("SX_USE_REF") == "1": - from .ref.async_eval_ref import async_eval - else: - from .async_eval import async_eval + from .async_eval import async_eval ctx = _get_request_context() result = await async_eval(query_def.body, env, ctx) @@ -91,10 +88,7 @@ async def execute_action(action_def: ActionDef, payload: dict[str, Any]) -> Any: result = None return _normalize(result) - if os.environ.get("SX_USE_REF") == "1": - from .ref.async_eval_ref import async_eval - else: - from .async_eval import async_eval + from .async_eval import async_eval ctx = _get_request_context() result = await async_eval(action_def.body, env, ctx) diff --git a/shared/sx/templates/cssx.sx b/shared/sx/templates/cssx.sx index 6dc8df5..b15d955 100644 --- a/shared/sx/templates/cssx.sx +++ b/shared/sx/templates/cssx.sx @@ -468,7 +468,7 @@ ;; (div (~cssx/tw "bg-red-500") (~cssx/tw "p-4") "content") ;; ========================================================================= -(defcomp ~cssx/tw (tokens) +(defcomp ~cssx/tw (&key tokens) (let ((token-list (filter (fn (t) (not (= t ""))) (split (or tokens "") " "))) (results (map cssx-process-token token-list)) diff --git a/shared/sx/tests/test_post_removal_bugs.py b/shared/sx/tests/test_post_removal_bugs.py new file mode 100644 index 0000000..c53859c --- /dev/null +++ b/shared/sx/tests/test_post_removal_bugs.py @@ -0,0 +1,558 @@ +"""Tests exposing bugs after sx_ref.py removal. + +These tests document all known breakages from removing the Python SX evaluator. +Each test targets a specific codepath that was depending on sx_ref.py and is now +broken. + +Usage: + pytest shared/sx/tests/test_post_removal_bugs.py -v +""" + +import asyncio +import os +import sys +import unittest + +_project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../..")) +if _project_root not in sys.path: + sys.path.insert(0, _project_root) + +from shared.sx.parser import parse, parse_all, serialize +from shared.sx.types import Component, Symbol, Keyword, NIL + + +# --------------------------------------------------------------------------- +# Helper: load shared components fresh (no cache) +# --------------------------------------------------------------------------- + +def _load_components_fresh(): + """Load shared components, clearing cache to force re-parse.""" + from shared.sx.jinja_bridge import _COMPONENT_ENV + _COMPONENT_ENV.clear() + from shared.sx.components import load_shared_components + load_shared_components() + return _COMPONENT_ENV + + +# =========================================================================== +# 1. register_components() loses all parameter information +# =========================================================================== + +class TestComponentRegistration(unittest.TestCase): + """register_components() hardcodes params=[] and has_children=False + for every component, losing all parameter metadata.""" + + @classmethod + def setUpClass(cls): + cls.env = _load_components_fresh() + + def test_shell_component_should_have_params(self): + """~shared:shell/sx-page-shell has 17+ &key params but gets params=[].""" + comp = self.env.get("~shared:shell/sx-page-shell") + self.assertIsNotNone(comp, "Shell component not found") + self.assertIsInstance(comp, Component) + # BUG: params is [] — should include title, meta-html, csrf, etc. + self.assertGreater( + len(comp.params), 0, + f"Shell component has params={comp.params} — expected 17+ keyword params" + ) + + def test_cssx_tw_should_have_tokens_param(self): + """~cssx/tw needs a 'tokens' parameter.""" + comp = self.env.get("~cssx/tw") + self.assertIsNotNone(comp, "~cssx/tw component not found") + self.assertIn( + "tokens", comp.params, + f"~cssx/tw has params={comp.params} — expected 'tokens'" + ) + + def test_cart_mini_should_have_params(self): + """~shared:fragments/cart-mini has &key params.""" + comp = self.env.get("~shared:fragments/cart-mini") + self.assertIsNotNone(comp, "cart-mini component not found") + self.assertGreater( + len(comp.params), 0, + f"cart-mini has params={comp.params} — expected keyword params" + ) + + def test_has_children_flag(self): + """Components with &rest children should have has_children=True.""" + comp = self.env.get("~shared:shell/sx-page-shell") + self.assertIsNotNone(comp) + # Many components accept children but has_children is always False + # Check any component that is known to accept &rest children + # e.g. a layout component + for name, val in self.env.items(): + if isinstance(val, Component): + # Every component has has_children=False — at least some should be True + pass + # Count how many have has_children=True + with_children = sum( + 1 for v in self.env.values() + if isinstance(v, Component) and v.has_children + ) + total = sum(1 for v in self.env.values() if isinstance(v, Component)) + # BUG: with_children is 0 — at least some components accept children + self.assertGreater( + with_children, 0, + f"0/{total} components have has_children=True — at least some should" + ) + + def test_all_components_have_empty_params(self): + """Show the scale of the bug — every single component has params=[].""" + components_with_params = [] + components_without = [] + for name, val in self.env.items(): + if isinstance(val, Component): + if val.params: + components_with_params.append(name) + else: + components_without.append(name) + # BUG: ALL components have empty params + self.assertGreater( + len(components_with_params), 0, + f"ALL {len(components_without)} components have params=[] — none have parameters parsed" + ) + + +# =========================================================================== +# 2. Sync html.py rendering is completely broken +# =========================================================================== + +class TestSyncHtmlRendering(unittest.TestCase): + """html.py render() stubs _raw_eval/_trampoline — any evaluation crashes.""" + + def test_html_render_simple_element(self): + """Even simple elements with keyword attrs need _eval, which is stubbed.""" + from shared.sx.html import render + # This should work — (div "hello") needs no eval + result = render(parse('(div "hello")'), {}) + self.assertIn("hello", result) + + def test_html_render_with_keyword_attr(self): + """Keyword attrs go through _eval, which raises RuntimeError.""" + from shared.sx.html import render + try: + result = render(parse('(div :class "test" "hello")'), {}) + # If it works, great + self.assertIn("test", result) + except RuntimeError as e: + self.assertIn("sx_ref.py has been removed", str(e)) + self.fail(f"html.py render crashes on keyword attrs: {e}") + + def test_html_render_symbol_lookup(self): + """Symbol lookup goes through _eval, which is stubbed.""" + from shared.sx.html import render + try: + result = render(parse('(div title)'), {"title": "Hello"}) + self.assertIn("Hello", result) + except RuntimeError as e: + self.assertIn("sx_ref.py has been removed", str(e)) + self.fail(f"html.py render crashes on symbol lookup: {e}") + + def test_html_render_component(self): + """Component rendering needs _eval for kwarg evaluation.""" + from shared.sx.html import render + env = _load_components_fresh() + try: + result = render( + parse('(~shared:fragments/cart-mini :cart-count 0 :blog-url "" :cart-url "")'), + env, + ) + self.assertIn("cart-mini", result) + except RuntimeError as e: + self.assertIn("sx_ref.py has been removed", str(e)) + self.fail(f"html.py render crashes on component calls: {e}") + + def test_sx_jinja_function_broken(self): + """The sx() Jinja helper is broken — it uses html_render internally.""" + from shared.sx.jinja_bridge import sx + env = _load_components_fresh() + try: + result = sx('(div "hello")') + self.assertIn("hello", result) + except RuntimeError as e: + self.assertIn("sx_ref.py has been removed", str(e)) + self.fail(f"sx() Jinja function is broken: {e}") + + +# =========================================================================== +# 3. Async render_to_html uses Python path, not OCaml +# =========================================================================== + +class TestAsyncRenderToHtml(unittest.IsolatedAsyncioTestCase): + """helpers.py render_to_html() deliberately uses Python async_eval, + not the OCaml bridge. But Python eval is now broken.""" + + async def test_render_to_html_uses_python_path(self): + """render_to_html goes through async_render, not OCaml bridge.""" + from shared.sx.helpers import render_to_html + env = _load_components_fresh() + # The shell component has many &key params — none are bound because params=[] + try: + html = await render_to_html( + "shared:shell/sx-page-shell", + title="Test", csrf="abc", asset_url="/static", + sx_js_hash="abc123", + ) + self.assertIn("Test", html) + except Exception as e: + # Expected: either RuntimeError from stubs or EvalError from undefined symbols + self.fail( + f"render_to_html (Python path) failed: {type(e).__name__}: {e}\n" + f"This should go through OCaml bridge instead" + ) + + async def test_async_render_component_no_params_bound(self): + """async_eval.py _arender_component can't bind params because comp.params=[].""" + from shared.sx.async_eval import async_render + from shared.sx.primitives_io import RequestContext + env = _load_components_fresh() + # Create a simple component manually with correct params + test_comp = Component( + name="test/greeting", + params=["name"], + has_children=False, + body=parse('(div (str "Hello " name))'), + ) + env["~test/greeting"] = test_comp + try: + result = await async_render( + parse('(~test/greeting :name "World")'), + env, + RequestContext(), + ) + self.assertIn("Hello World", result) + except Exception as e: + self.fail( + f"async_render failed even with correct params: {type(e).__name__}: {e}" + ) + + +# =========================================================================== +# 4. Dead imports from removed sx_ref.py +# =========================================================================== + +class TestDeadImports(unittest.TestCase): + """Files that import from sx_ref.py will crash when their codepaths execute.""" + + def test_async_eval_defcomp(self): + """async_eval.py _asf_defcomp should work as a stub (no sx_ref import).""" + from shared.sx.async_eval import _asf_defcomp + env = {} + asyncio.run(_asf_defcomp( + [Symbol("defcomp"), Symbol("~test"), [], [Symbol("div")]], + env, None + )) + # Should register a minimal component in env + self.assertIn("~test", env) + + def test_async_eval_defmacro(self): + """async_eval.py _asf_defmacro should work as a stub (no sx_ref import).""" + from shared.sx.async_eval import _asf_defmacro + env = {} + asyncio.run(_asf_defmacro( + [Symbol("defmacro"), Symbol("test"), [], [Symbol("div")]], + env, None + )) + self.assertIn("test", env) + + def test_async_eval_defstyle(self): + """async_eval.py _asf_defstyle should be a no-op (no sx_ref import).""" + from shared.sx.async_eval import _asf_defstyle + result = asyncio.run(_asf_defstyle( + [Symbol("defstyle"), Symbol("test"), [], [Symbol("div")]], + {}, None + )) + # Should return NIL without crashing + self.assertIsNotNone(result) + + def test_async_eval_defhandler(self): + """async_eval.py _asf_defhandler should be a no-op (no sx_ref import).""" + from shared.sx.async_eval import _asf_defhandler + result = asyncio.run(_asf_defhandler( + [Symbol("defhandler"), Symbol("test"), [], [Symbol("div")]], + {}, None + )) + self.assertIsNotNone(result) + + def test_async_eval_continuation_reset(self): + """async_eval.py _asf_reset imports eval_expr/trampoline from sx_ref.""" + # The cont_fn inside _asf_reset will crash when invoked + from shared.sx.async_eval import _ASYNC_RENDER_FORMS + reset_fn = _ASYNC_RENDER_FORMS.get("reset") + # reset is defined in async_eval — the import is deferred to execution + # Just verify the module doesn't have the import available + try: + from shared.sx.ref.sx_ref import eval_expr + self.fail("sx_ref.py should not exist") + except (ImportError, ModuleNotFoundError): + pass # Expected + + def test_ocaml_bridge_jit_compile(self): + """ocaml_bridge.py _compile_adapter_module imports from sx_ref.""" + try: + from shared.sx.ref.sx_ref import eval_expr, trampoline, PRIMITIVES + self.fail("sx_ref.py should not exist — JIT compilation path is broken") + except (ImportError, ModuleNotFoundError): + pass # Expected — confirms the bug + + def test_parser_reader_macro(self): + """parser.py _try_reader_macro imports trampoline/call_lambda from sx_ref.""" + try: + from shared.sx.ref.sx_ref import trampoline, call_lambda + self.fail("sx_ref.py should not exist — reader macros are broken") + except (ImportError, ModuleNotFoundError): + pass # Expected — confirms the bug + + def test_primitives_scope_prims(self): + """primitives.py _lazy_scope_primitives silently fails to load scope prims.""" + from shared.sx.primitives import _PRIMITIVES + # collect!, collected, clear-collected!, emitted, emit!, context + # These are needed for CSSX but the import from sx_ref silently fails + missing = [] + for name in ("collect!", "collected", "clear-collected!", "emitted", "emit!", "context"): + if name not in _PRIMITIVES: + missing.append(name) + if missing: + self.fail( + f"Scope primitives missing from _PRIMITIVES (sx_ref import failed silently): {missing}\n" + f"CSSX components depend on these for collect!/collected" + ) + + def test_deps_transitive_deps_ref_path(self): + """deps.py transitive_deps imports from sx_ref when SX_USE_REF=1.""" + # The fallback path should still work + from shared.sx.deps import transitive_deps + env = _load_components_fresh() + # Should work via fallback, not crash + try: + result = transitive_deps("~cssx/tw", env) + self.assertIsInstance(result, set) + except (ImportError, ModuleNotFoundError) as e: + self.fail(f"transitive_deps crashed: {e}") + + def test_handlers_python_fallback(self): + """handlers.py eval_handler Python fallback imports async_eval_ref.""" + # When not using OCaml, handler evaluation falls through to async_eval + # The ref path (SX_USE_REF=1) would crash + try: + from shared.sx.ref.async_eval_ref import async_eval_to_sx + self.fail("async_eval_ref.py should not exist") + except (ImportError, ModuleNotFoundError): + pass # Expected + + +# =========================================================================== +# 5. ~cssx/tw signature mismatch +# =========================================================================== + +class TestCssxTwSignature(unittest.TestCase): + """~cssx/tw changed from (&key tokens) to (tokens) positional, + but callers use :tokens keyword syntax.""" + + @classmethod + def setUpClass(cls): + cls.env = _load_components_fresh() + + def test_cssx_tw_source_uses_positional(self): + """Verify the current source has positional (tokens) not (&key tokens).""" + import os + cssx_path = os.path.join( + os.path.dirname(__file__), "..", "templates", "cssx.sx" + ) + with open(cssx_path) as f: + source = f.read() + # Check if it's positional or keyword + if "(defcomp ~cssx/tw (tokens)" in source: + # Positional — callers using :tokens will break + self.fail( + "~cssx/tw uses positional (tokens) but callers use :tokens keyword syntax.\n" + "Should be: (defcomp ~cssx/tw (&key tokens) ...)" + ) + elif "(defcomp ~cssx/tw (&key tokens)" in source: + pass # Correct + else: + # Unknown signature + for line in source.split("\n"): + if "defcomp ~cssx/tw" in line: + self.fail(f"Unexpected ~cssx/tw signature: {line.strip()}") + + def test_cssx_tw_callers_use_keyword(self): + """Scan for callers that use :tokens keyword syntax.""" + import glob as glob_mod + sx_dir = os.path.join(os.path.dirname(__file__), "../../..") + keyword_callers = [] + positional_callers = [] + for fp in glob_mod.glob(os.path.join(sx_dir, "**/*.sx"), recursive=True): + try: + with open(fp) as f: + content = f.read() + except Exception: + continue + if "~cssx/tw" not in content: + continue + for line_no, line in enumerate(content.split("\n"), 1): + if "~cssx/tw" in line and "defcomp" not in line: + if ":tokens" in line: + keyword_callers.append(f"{fp}:{line_no}") + elif "(~cssx/tw " in line: + positional_callers.append(f"{fp}:{line_no}") + + if keyword_callers: + # If signature is positional but callers use :tokens, that's a bug + import os as os_mod + cssx_path = os.path.join( + os.path.dirname(__file__), "..", "templates", "cssx.sx" + ) + with open(cssx_path) as f: + source = f.read() + if "(defcomp ~cssx/tw (tokens)" in source: + self.fail( + f"~cssx/tw uses positional params but {len(keyword_callers)} callers use :tokens:\n" + + "\n".join(keyword_callers[:5]) + ) + + +# =========================================================================== +# 6. OCaml bridge rendering (should work — this is the good path) +# =========================================================================== + +class TestOcamlBridgeRendering(unittest.IsolatedAsyncioTestCase): + """The OCaml bridge should handle all rendering correctly.""" + + @classmethod + def setUpClass(cls): + from shared.sx.ocaml_bridge import _DEFAULT_BIN + bin_path = os.path.abspath(_DEFAULT_BIN) + if not os.path.isfile(bin_path): + raise unittest.SkipTest("OCaml binary not found") + + async def asyncSetUp(self): + from shared.sx.ocaml_bridge import OcamlBridge + self.bridge = OcamlBridge() + await self.bridge.start() + + async def asyncTearDown(self): + if hasattr(self, 'bridge'): + await self.bridge.stop() + + async def test_simple_element(self): + result = await self.bridge.render('(div "hello")') + self.assertIn("hello", result) + + async def test_element_with_keyword_attrs(self): + result = await self.bridge.render('(div :class "test" "hello")') + self.assertIn('class="test"', result) + self.assertIn("hello", result) + + async def test_component_with_params(self): + """OCaml should handle component parameter binding correctly.""" + # Use load_source to define a component (bypasses _ensure_components lock) + await self.bridge.load_source('(defcomp ~test/greet (&key name) (div (str "Hello " name)))') + result = await self.bridge.render('(~test/greet :name "World")') + self.assertIn("Hello World", result) + + async def test_let_binding(self): + result = await self.bridge.render('(let ((x "hello")) (div x))') + self.assertIn("hello", result) + + async def test_conditional(self): + result = await self.bridge.render('(if true (div "yes") (div "no"))') + self.assertIn("yes", result) + self.assertNotIn("no", result) + + async def test_cssx_tw_keyword_call(self): + """Test that ~cssx/tw works when called with :tokens keyword. + Components are loaded by _ensure_components() automatically.""" + try: + result = await self.bridge.render('(div (~cssx/tw :tokens "bg-red-500") "content")') + # Should produce a spread with CSS class, not an error + self.assertNotIn("error", result.lower()) + except Exception as e: + self.fail(f"~cssx/tw :tokens keyword call failed: {e}") + + async def test_cssx_tw_positional_call(self): + """Test that ~cssx/tw works when called positionally.""" + try: + result = await self.bridge.render('(div (~cssx/tw "bg-red-500") "content")') + self.assertNotIn("error", result.lower()) + except Exception as e: + self.fail(f"~cssx/tw positional call failed: {e}") + + async def test_repeated_renders_dont_crash(self): + """Verify OCaml bridge handles multiple sequential renders.""" + for i in range(5): + result = await self.bridge.render(f'(div "iter-{i}")') + self.assertIn(f"iter-{i}", result) + + +# =========================================================================== +# 7. Scope primitives missing (collect!, collected, etc.) +# =========================================================================== + +class TestScopePrimitives(unittest.TestCase): + """Scope primitives needed by CSSX are missing because the import + from sx_ref.py silently fails.""" + + def test_python_primitives_have_scope_ops(self): + """Check that collect!/collected/etc. are in _PRIMITIVES.""" + from shared.sx.primitives import _PRIMITIVES + required = ["collect!", "collected", "clear-collected!", + "emitted", "emit!", "context"] + missing = [p for p in required if p not in _PRIMITIVES] + if missing: + self.fail( + f"Missing Python-side scope primitives: {missing}\n" + f"These were provided by sx_ref.py — need OCaml bridge or Python stubs" + ) + + +# =========================================================================== +# 8. Query executor fallback path +# =========================================================================== + +class TestQueryExecutorFallback(unittest.TestCase): + """query_executor.py imports async_eval for its fallback path.""" + + def test_query_executor_import(self): + """query_executor can be imported without crashing.""" + try: + import shared.sx.query_executor + except Exception as e: + self.fail(f"query_executor import crashed: {e}") + + +# =========================================================================== +# 9. End-to-end: sx_page shell rendering +# =========================================================================== + +class TestShellRendering(unittest.IsolatedAsyncioTestCase): + """The shell template needs to render through some path that works.""" + + async def test_sx_page_shell_via_python(self): + """render_to_html('shared:shell/sx-page-shell', ...) uses Python path. + This is the actual failure from the production error log.""" + from shared.sx.helpers import render_to_html + _load_components_fresh() + try: + html = await render_to_html( + "shared:shell/sx-page-shell", + title="Test Page", + csrf="test-csrf", + asset_url="/static", + sx_js_hash="abc", + ) + # Should produce full HTML document + self.assertIn("", html.lower()) + self.assertIn("Test Page", html) + except Exception as e: + self.fail( + f"Shell rendering via Python path failed: {type(e).__name__}: {e}\n" + f"This is the exact error seen in production — " + f"render_to_html should use OCaml bridge" + ) + + +if __name__ == "__main__": + unittest.main()