coop/rose-ash
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Complete Python eval removal: epoch protocol, scope consolidation, JIT fixes

Browse Source 
Route all rendering through OCaml bridge — render_to_html no longer uses
Python async_eval. Fix register_components to parse &key params and &rest
children from defcomp forms. Remove all dead sx_ref.py imports.

Epoch protocol (prevents pipe desync):
- Every command prefixed with (epoch N), all responses tagged with epoch
- Both sides discard stale-epoch messages — desync structurally impossible
- OCaml main loop discards stale io-responses between commands

Consolidate scope primitives into sx_scope.ml:
- Single source of truth for scope-push!/pop!/peek, collect!/collected,
  emit!/emitted, context, and 12 other scope operations
- Removes duplicate registrations from sx_server.ml (including bugs where
  scope-emit! and clear-collected! were registered twice with different impls)
- Bind scope prims into env so JIT VM finds them via OP_GLOBAL_GET

JIT VM fixes:
- Trampoline thunks before passing args to CALL_PRIM
- as_list resolves thunks via _sx_trampoline_fn
- len handles all value types (Bool, Number, RawHTML, SxExpr, Spread, etc.)

Other fixes:
- ~cssx/tw signature: (tokens) → (&key tokens) to match callers
- Minimal Python evaluator in html.py for sync sx() Jinja function
- Python scope primitive stubs (thread-local) for non-OCaml paths
- Reader macro resolution via OcamlSync instead of sx_ref.py

Tests: 1114 OCaml, 1078 JS, 35 Python regression, 6/6 Playwright SSR

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
giles
2026-03-24 16:14:40 +00:00
parent e887c0d978
commit f9f810ffd7
18 changed files with 1305 additions and 478 deletions
Download Patch File Download Diff File Expand all files Collapse all files

372
hosts/ocaml/bin/sx_server.ml
View File

@@ -61,21 +61,26 @@ let rec serialize_value = function
| RawHTML s -> "\"" ^ escape_sx_string s ^ "\"" | RawHTML s -> "\"" ^ escape_sx_string s ^ "\""
| _ -> "nil" | _ -> "nil"
(** Request epoch — monotonically increasing, set by (epoch N) from Python.
All responses are tagged with the current epoch so Python can discard
stale messages from previous requests. Makes pipe desync impossible. *)
let current_epoch = ref 0
let send line = let send line =
print_string line; print_string line;
print_char '\n'; print_char '\n';
flush stdout flush stdout
let send_ok () = send "(ok)" let send_ok () = send (Printf.sprintf "(ok %d)" !current_epoch)
let send_ok_value v = send (Printf.sprintf "(ok %s)" (serialize_value v)) let send_ok_value v = send (Printf.sprintf "(ok %d %s)" !current_epoch (serialize_value v))
let send_error msg = send (Printf.sprintf "(error \"%s\")" (escape_sx_string msg)) let send_error msg = send (Printf.sprintf "(error %d \"%s\")" !current_epoch (escape_sx_string msg))
(** Length-prefixed binary send — handles any content without escaping. (** Length-prefixed binary send — handles any content without escaping.
Sends: (ok-len N)\n followed by exactly N bytes of raw data, then \n. Sends: (ok-len EPOCH N)\n followed by exactly N bytes of raw data, then \n.
Python reads the length line, then reads exactly N bytes. *) Python reads the length line, then reads exactly N bytes. *)
let send_ok_blob s = let send_ok_blob s =
let n = String.length s in let n = String.length s in
Printf.printf "(ok-len %d)\n" n; Printf.printf "(ok-len %d %d)\n" !current_epoch n;
print_string s; print_string s;
print_char '\n'; print_char '\n';
flush stdout flush stdout
@@ -125,171 +130,11 @@ let io_batch_mode = ref false
let io_queue : (int * string * value list) list ref = ref [] let io_queue : (int * string * value list) list ref = ref []
let io_counter = ref 0 let io_counter = ref 0
(* Request cookies — set by Python bridge before each page render. (* Scope stacks and cookies — all primitives registered in sx_scope.ml.
get-cookie reads from here on the server; set-cookie is a no-op We just reference the shared state for the IO bridge. *)
(server can't set response cookies from SX — that's the framework's job). *) module Sx_scope = Sx.Sx_scope
let _request_cookies : (string, string) Hashtbl.t = Hashtbl.create 8 let _request_cookies = Sx_scope.request_cookies
let _scope_stacks = Sx_scope.scope_stacks
let () = Sx_primitives.register "get-cookie" (fun args ->
match args with
| [String name] ->
(match Hashtbl.find_opt _request_cookies name with
| Some v -> String v
| None -> Nil)
| _ -> Nil)
let () = Sx_primitives.register "set-cookie" (fun _args ->
(* No-op on server — cookies are set via HTTP response headers *)
Nil)
(* Module-level scope stacks — shared between make_server_env (aser
scope-push!/pop!) and step-sf-context (via get-primitive "scope-peek"). *)
let _scope_stacks : (string, value list) Hashtbl.t = Hashtbl.create 8
let () = Sx_primitives.register "scope-push!" (fun args ->
match args with
| [String name; value] ->
let stack = try Hashtbl.find _scope_stacks name with Not_found -> [] in
Hashtbl.replace _scope_stacks name (value :: stack); Nil
| _ -> Nil)
let () = Sx_primitives.register "scope-pop!" (fun args ->
match args with
| [String name] ->
let stack = try Hashtbl.find _scope_stacks name with Not_found -> [] in
(match stack with _ :: rest -> Hashtbl.replace _scope_stacks name rest | [] -> ()); Nil
| _ -> Nil)
let () = Sx_primitives.register "scope-peek" (fun args ->
match args with
| [String name] ->
let stack = try Hashtbl.find _scope_stacks name with Not_found -> [] in
(match stack with v :: _ -> v | [] -> Nil)
| _ -> Nil)
let () = Sx_primitives.register "context" (fun args ->
match args with
| [String name] | [String name; _] ->
let stack = try Hashtbl.find _scope_stacks name with Not_found -> [] in
(match stack, args with
| v :: _, _ -> v
| [], [_; default_val] -> default_val
| [], _ -> Nil)
| _ -> Nil)
(** collect! — lazy scope accumulator. Creates root scope if missing,
emits value (deduplicates). Used by cssx and spread components. *)
let () = Sx_primitives.register "collect!" (fun args ->
match args with
| [String name; value] ->
let stack = try Hashtbl.find _scope_stacks name with Not_found -> [] in
(match stack with
| List items :: rest ->
if not (List.mem value items) then
Hashtbl.replace _scope_stacks name (List (items @ [value]) :: rest)
| [] ->
(* Lazy root scope — create with the value *)
Hashtbl.replace _scope_stacks name [List [value]]
| _ :: _ -> ());
Nil
| _ -> Nil)
let () = Sx_primitives.register "collected" (fun args ->
match args with
| [String name] ->
let stack = try Hashtbl.find _scope_stacks name with Not_found -> [] in
(match stack with List items :: _ -> List items | _ -> List [])
| _ -> List [])
let () = Sx_primitives.register "clear-collected!" (fun args ->
match args with
| [String name] ->
let stack = try Hashtbl.find _scope_stacks name with Not_found -> [] in
(match stack with
| _ :: rest -> Hashtbl.replace _scope_stacks name (List [] :: rest)
| [] -> Hashtbl.replace _scope_stacks name [List []]);
Nil
| _ -> Nil)
(* emit!/emitted — adapter-html.sx uses these for spread attr collection *)
let () = Sx_primitives.register "scope-emit!" (fun args ->
match args with
| [String name; value] ->
let stack = try Hashtbl.find _scope_stacks name with Not_found -> [] in
(match stack with
| List items :: rest ->
Hashtbl.replace _scope_stacks name (List (items @ [value]) :: rest)
| v :: rest ->
(* Non-list top — wrap current entries as list + new value *)
Hashtbl.replace _scope_stacks name (List [value] :: v :: rest)
| [] ->
Hashtbl.replace _scope_stacks name [List [value]]);
Nil
| _ -> Nil)
let () = Sx_primitives.register "emitted" (fun args ->
match args with
| [String name] ->
let stack = try Hashtbl.find _scope_stacks name with Not_found -> [] in
(match stack with List items :: _ -> List items | _ -> List [])
| _ -> List [])
let () = Sx_primitives.register "scope-emitted" (fun args ->
match args with
| [String name] ->
let stack = try Hashtbl.find _scope_stacks name with Not_found -> [] in
(match stack with List items :: _ -> List items | _ -> List [])
| _ -> List [])
let () = Sx_primitives.register "scope-collected" (fun args ->
match args with
| [String name] ->
let stack = try Hashtbl.find _scope_stacks name with Not_found -> [] in
(match stack with List items :: _ -> List items | _ -> List [])
| _ -> List [])
let () = Sx_primitives.register "scope-clear-collected!" (fun args ->
match args with
| [String name] ->
let stack = try Hashtbl.find _scope_stacks name with Not_found -> [] in
(match stack with
| _ :: rest -> Hashtbl.replace _scope_stacks name (List [] :: rest)
| [] -> Hashtbl.replace _scope_stacks name [List []]);
Nil
| _ -> Nil)
let () = Sx_primitives.register "provide-push!" (fun args ->
match Sx_primitives.get_primitive "scope-push!" with
| NativeFn (_, fn) -> fn args | _ -> Nil)
let () = Sx_primitives.register "provide-pop!" (fun args ->
match Sx_primitives.get_primitive "scope-pop!" with
| NativeFn (_, fn) -> fn args | _ -> Nil)
let () = Sx_primitives.register "scope-emit!" (fun args ->
match args with
| [String name; value] ->
let stack = try Hashtbl.find _scope_stacks name with Not_found -> [] in
(match stack with
| List items :: rest ->
Hashtbl.replace _scope_stacks name (List (items @ [value]) :: rest)
| Nil :: rest ->
Hashtbl.replace _scope_stacks name (List [value] :: rest)
| [] ->
(* Lazy root scope *)
Hashtbl.replace _scope_stacks name [List [value]]
| _ :: _ -> ());
Nil
| _ -> Nil)
let () = Sx_primitives.register "clear-collected!" (fun args ->
match args with
| [String name] ->
let stack = try Hashtbl.find _scope_stacks name with Not_found -> [] in
(match stack with
| _ :: rest -> Hashtbl.replace _scope_stacks name (List [] :: rest)
| [] -> ());
Nil
| _ -> Nil)
(** Helpers safe to defer — pure functions whose results are only used (** Helpers safe to defer — pure functions whose results are only used
as rendering output (inlined into SX wire format), not in control flow. *) as rendering output (inlined into SX wire format), not in control flow. *)
@@ -303,6 +148,29 @@ let is_batchable name args =
| String h :: _ -> List.mem h batchable_helpers | String h :: _ -> List.mem h batchable_helpers
| _ -> false | _ -> false
(** Read an io-response from stdin, discarding stale messages from old epochs. *)
let rec read_io_response () =
match read_line_blocking () with
| None -> raise (Eval_error "IO bridge: stdin closed while waiting for io-response")
| Some line ->
let exprs = Sx_parser.parse_all line in
match exprs with
(* Epoch-tagged: (io-response EPOCH value) *)
| [List [Symbol "io-response"; Number n; value]]
when int_of_float n = !current_epoch -> value
| [List (Symbol "io-response" :: Number n :: values)]
when int_of_float n = !current_epoch ->
(match values with [v] -> v | _ -> List values)
(* Legacy untagged: (io-response value) — accept for backwards compat *)
| [List [Symbol "io-response"; value]] -> value
| [List (Symbol "io-response" :: values)] ->
(match values with [v] -> v | _ -> List values)
(* Stale epoch or unexpected — discard and retry *)
| _ ->
Printf.eprintf "[io] discarding stale message (%d chars, epoch=%d)\n%!"
(String.length line) !current_epoch;
read_io_response ()
(** Send an io-request — batch mode returns placeholder, else blocks. *) (** Send an io-request — batch mode returns placeholder, else blocks. *)
let io_request name args = let io_request name args =
if !io_batch_mode && is_batchable name args then begin if !io_batch_mode && is_batchable name args then begin
@@ -313,20 +181,36 @@ let io_request name args =
SxExpr (Printf.sprintf "(\xc2\xabIO:%d\xc2\xbb)" id) SxExpr (Printf.sprintf "(\xc2\xabIO:%d\xc2\xbb)" id)
end else begin end else begin
let args_str = String.concat " " (List.map serialize_value args) in let args_str = String.concat " " (List.map serialize_value args) in
send (Printf.sprintf "(io-request \"%s\" %s)" name args_str); send (Printf.sprintf "(io-request %d \"%s\" %s)" !current_epoch name args_str);
(* Block on stdin for io-response *) read_io_response ()
end
(** Read a batched io-response, discarding stale epoch messages. *)
let read_batched_io_response () =
let rec loop () =
match read_line_blocking () with match read_line_blocking () with
| None -> raise (Eval_error "IO bridge: stdin closed while waiting for io-response") | None -> raise (Eval_error "IO batch: stdin closed")
| Some line -> | Some line ->
let exprs = Sx_parser.parse_all line in let exprs = Sx_parser.parse_all line in
match exprs with match exprs with
| [List [Symbol "io-response"; value]] -> value (* Epoch-tagged: (io-response EPOCH value) *)
| [List (Symbol "io-response" :: values)] -> | [List [Symbol "io-response"; Number n; String s]]
(match values with when int_of_float n = !current_epoch -> s
| [v] -> v | [List [Symbol "io-response"; Number n; SxExpr s]]
| _ -> List values) when int_of_float n = !current_epoch -> s
| _ -> raise (Eval_error ("IO bridge: unexpected response: " ^ line)) | [List [Symbol "io-response"; Number n; v]]
end when int_of_float n = !current_epoch -> serialize_value v
(* Legacy untagged *)
| [List [Symbol "io-response"; String s]]
| [List [Symbol "io-response"; SxExpr s]] -> s
| [List [Symbol "io-response"; v]] -> serialize_value v
(* Stale — discard and retry *)
| _ ->
Printf.eprintf "[io-batch] discarding stale message (%d chars)\n%!"
(String.length line);
loop ()
in
loop ()
(** Flush batched IO: send all requests, read all responses, replace placeholders. *) (** Flush batched IO: send all requests, read all responses, replace placeholders. *)
let flush_batched_io result_str = let flush_batched_io result_str =
@@ -335,44 +219,35 @@ let flush_batched_io result_str =
io_counter := 0; io_counter := 0;
if queue = [] then result_str if queue = [] then result_str
else begin else begin
(* Send all batched requests with IDs *) (* Send all batched requests with IDs, tagged with epoch *)
List.iter (fun (id, name, args) -> List.iter (fun (id, name, args) ->
let args_str = String.concat " " (List.map serialize_value args) in let args_str = String.concat " " (List.map serialize_value args) in
send (Printf.sprintf "(io-request %d \"%s\" %s)" id name args_str) send (Printf.sprintf "(io-request %d %d \"%s\" %s)" !current_epoch id name args_str)
) queue; ) queue;
send (Printf.sprintf "(io-done %d)" (List.length queue)); send (Printf.sprintf "(io-done %d %d)" !current_epoch (List.length queue));
(* Read all responses and replace placeholders *) (* Read all responses and replace placeholders *)
let final = ref result_str in let final = ref result_str in
List.iter (fun (id, _, _) -> List.iter (fun (id, _, _) ->
match read_line_blocking () with let value_str = read_batched_io_response () in
| Some line -> let placeholder = Printf.sprintf "(\xc2\xabIO:%d\xc2\xbb)" id in
let exprs = Sx_parser.parse_all line in (* Replace all occurrences of this placeholder *)
let value_str = match exprs with let plen = String.length placeholder in
| [List [Symbol "io-response"; String s]] let buf = Buffer.create (String.length !final) in
| [List [Symbol "io-response"; SxExpr s]] -> s let pos = ref 0 in
| [List [Symbol "io-response"; v]] -> serialize_value v let s = !final in
| _ -> "nil" let slen = String.length s in
in while !pos <= slen - plen do
let placeholder = Printf.sprintf "(\xc2\xabIO:%d\xc2\xbb)" id in if String.sub s !pos plen = placeholder then begin
(* Replace all occurrences of this placeholder *) Buffer.add_string buf value_str;
let plen = String.length placeholder in pos := !pos + plen
let buf = Buffer.create (String.length !final) in end else begin
let pos = ref 0 in Buffer.add_char buf s.[!pos];
let s = !final in incr pos
let slen = String.length s in end
while !pos <= slen - plen do done;
if String.sub s !pos plen = placeholder then begin if !pos < slen then
Buffer.add_string buf value_str; Buffer.add_substring buf s !pos (slen - !pos);
pos := !pos + plen final := Buffer.contents buf
end else begin
Buffer.add_char buf s.[!pos];
incr pos
end
done;
if !pos < slen then
Buffer.add_substring buf s !pos (slen - !pos);
final := Buffer.contents buf
| None -> raise (Eval_error "IO batch: stdin closed")
) queue; ) queue;
!final !final
end end
@@ -519,56 +394,21 @@ let make_server_env () =
(* Scope stack — platform primitives for render-time dynamic scope. (* Scope stack — platform primitives for render-time dynamic scope.
Used by aser for spread/provide/emit patterns. Used by aser for spread/provide/emit patterns.
Module-level so step-sf-context can check it via get-primitive. *) Module-level so step-sf-context can check it via get-primitive. *)
let scope_stacks = _scope_stacks in (* Scope primitives are registered globally in sx_scope.ml.
bind "scope-push!" (fun args -> Bind them into the env so the JIT VM can find them via vm.globals
match args with (OP_GLOBAL_GET checks env.bindings before the primitives table). *)
| [String name; value] -> List.iter (fun name ->
let stack = try Hashtbl.find scope_stacks name with Not_found -> [] in try ignore (env_bind env name (Sx_primitives.get_primitive name))
Hashtbl.replace scope_stacks name (value :: stack); Nil with _ -> ()
| _ -> Nil); ) ["scope-push!"; "scope-pop!"; "scope-peek"; "context";
bind "scope-pop!" (fun args -> "collect!"; "collected"; "clear-collected!";
match args with "scope-emit!"; "emit!"; "emitted"; "scope-emitted";
| [String name] -> "scope-collected"; "scope-clear-collected!";
let stack = try Hashtbl.find scope_stacks name with Not_found -> [] in "provide-push!"; "provide-pop!";
(match stack with "get-cookie"; "set-cookie"];
| _ :: rest -> Hashtbl.replace scope_stacks name rest (* sx-context is an env alias for context *)
| [] -> ()); Nil let context_prim = Sx_primitives.get_primitive "context" in
| _ -> Nil); ignore (env_bind env "sx-context" context_prim);
bind "scope-peek" (fun args ->
match args with
| [String name] ->
let stack = try Hashtbl.find scope_stacks name with Not_found -> [] in
(match stack with v :: _ -> v | [] -> Nil)
| _ -> Nil);
(* scope-emit! / scope-peek — Hashtbl-based scope primitives for aser.
Different names from emit!/emitted to avoid CEK special form conflict. *)
bind "scope-emit!" (fun args ->
match args with
| [String name; value] ->
let stack = try Hashtbl.find scope_stacks name with Not_found -> [] in
(match stack with
| List items :: rest ->
Hashtbl.replace scope_stacks name (List (items @ [value]) :: rest)
| Nil :: rest ->
Hashtbl.replace scope_stacks name (List [value] :: rest)
| _ :: _ -> ()
| [] -> ()); Nil
| _ -> Nil);
(* context — scope lookup. The CEK handles this as a special form
by walking continuation frames, but compiled VM code needs it as
a function that reads from the scope_stacks hashtable. *)
let context_impl = NativeFn ("context", fun args ->
match args with
| [String name] | [String name; _] ->
let stack = try Hashtbl.find scope_stacks name with Not_found -> [] in
(match stack, args with
| v :: _, _ -> v
| [], [_; default_val] -> default_val
| [], _ -> Nil)
| _ -> Nil) in
ignore (env_bind env "sx-context" context_impl);
ignore (env_bind env "context" context_impl);
(* qq-expand-runtime — quasiquote expansion at runtime. (* qq-expand-runtime — quasiquote expansion at runtime.
The bytecode compiler emits CALL_PRIM "qq-expand-runtime" for The bytecode compiler emits CALL_PRIM "qq-expand-runtime" for
@@ -1667,9 +1507,17 @@ let () =
| Some line -> | Some line ->
let line = String.trim line in let line = String.trim line in
if line = "" then () (* skip blank lines *) if line = "" then () (* skip blank lines *)
(* Discard stale io-responses from previous requests. *)
else if String.length line > 14
&& String.sub line 0 14 = "(io-response " then
Printf.eprintf "[sx-server] discarding stale io-response (%d chars)\n%!"
(String.length line)
else begin else begin
let exprs = Sx_parser.parse_all line in let exprs = Sx_parser.parse_all line in
match exprs with match exprs with
(* Epoch marker: (epoch N) — set current epoch, read next command *)
| [List [Symbol "epoch"; Number n]] ->
current_epoch := int_of_float n
| [cmd] -> dispatch env cmd | [cmd] -> dispatch env cmd
| _ -> send_error ("Expected single command, got " ^ string_of_int (List.length exprs)) | _ -> send_error ("Expected single command, got " ^ string_of_int (List.length exprs))
end end

15
hosts/ocaml/lib/sx_primitives.ml
View File

@@ -36,10 +36,11 @@ let as_string = function
| String s -> s | String s -> s
| v -> raise (Eval_error ("Expected string, got " ^ type_of v)) | v -> raise (Eval_error ("Expected string, got " ^ type_of v))
let as_list = function let rec as_list = function
| List l -> l | List l -> l
| ListRef r -> !r | ListRef r -> !r
| Nil -> [] | Nil -> []
| Thunk _ as t -> as_list (!_sx_trampoline_fn t)
| v -> raise (Eval_error ("Expected list, got " ^ type_of v)) | v -> raise (Eval_error ("Expected list, got " ^ type_of v))
let as_bool = function let as_bool = function
@@ -316,8 +317,16 @@ let () =
| [List l] | [ListRef { contents = l }] -> Number (float_of_int (List.length l)) | [List l] | [ListRef { contents = l }] -> Number (float_of_int (List.length l))
| [String s] -> Number (float_of_int (String.length s)) | [String s] -> Number (float_of_int (String.length s))
| [Dict d] -> Number (float_of_int (Hashtbl.length d)) | [Dict d] -> Number (float_of_int (Hashtbl.length d))
| [Nil] -> Number 0.0 | [Nil] | [Bool false] -> Number 0.0
| _ -> raise (Eval_error "len: 1 arg")); | [Bool true] -> Number 1.0
| [Number _] -> Number 1.0
| [RawHTML s] -> Number (float_of_int (String.length s))
| [SxExpr s] -> Number (float_of_int (String.length s))
| [Spread pairs] -> Number (float_of_int (List.length pairs))
| [Component _] | [Island _] | [Lambda _] | [NativeFn _]
| [Macro _] | [Thunk _] | [Keyword _] | [Symbol _] -> Number 0.0
| _ -> raise (Eval_error (Printf.sprintf "len: %d args"
(List.length args))));
register "first" (fun args -> register "first" (fun args ->
match args with match args with
| [List (x :: _)] | [ListRef { contents = x :: _ }] -> x | [List (x :: _)] | [ListRef { contents = x :: _ }] -> x

154
hosts/ocaml/lib/sx_scope.ml Normal file
View File

@@ -0,0 +1,154 @@
(** Scope stacks — dynamic scope for render-time effects.
Provides scope-push!/pop!/peek, collect!/collected/clear-collected!,
scope-emit!/emitted/scope-emitted, context, and cookie access.
All functions are registered as primitives so both the CEK evaluator
and the JIT VM can find them in the same place. *)
open Sx_types
(** The shared scope stacks hashtable. Each key maps to a stack of values.
Used by aser for spread/provide/emit patterns, CSSX collect/flush, etc. *)
let scope_stacks : (string, value list) Hashtbl.t = Hashtbl.create 8
(** Request cookies — set by the Python bridge before each render.
get-cookie reads from here; set-cookie is a no-op on the server. *)
let request_cookies : (string, string) Hashtbl.t = Hashtbl.create 8
(** Clear all scope stacks. Called between requests if needed. *)
let clear_all () = Hashtbl.clear scope_stacks
let () =
let register = Sx_primitives.register in
(* --- Cookies --- *)
register "get-cookie" (fun args ->
match args with
| [String name] ->
(match Hashtbl.find_opt request_cookies name with
| Some v -> String v
| None -> Nil)
| _ -> Nil);
register "set-cookie" (fun _args -> Nil);
(* --- Core scope stack operations --- *)
register "scope-push!" (fun args ->
match args with
| [String name; value] ->
let stack = try Hashtbl.find scope_stacks name with Not_found -> [] in
Hashtbl.replace scope_stacks name (value :: stack); Nil
| _ -> Nil);
register "scope-pop!" (fun args ->
match args with
| [String name] ->
let stack = try Hashtbl.find scope_stacks name with Not_found -> [] in
(match stack with _ :: rest -> Hashtbl.replace scope_stacks name rest | [] -> ()); Nil
| _ -> Nil);
register "scope-peek" (fun args ->
match args with
| [String name] ->
let stack = try Hashtbl.find scope_stacks name with Not_found -> [] in
(match stack with v :: _ -> v | [] -> Nil)
| _ -> Nil);
(* --- Context (scope lookup with optional default) --- *)
register "context" (fun args ->
match args with
| [String name] | [String name; _] ->
let stack = try Hashtbl.find scope_stacks name with Not_found -> [] in
(match stack, args with
| v :: _, _ -> v
| [], [_; default_val] -> default_val
| [], _ -> Nil)
| _ -> Nil);
(* --- Collect / collected / clear-collected! --- *)
register "collect!" (fun args ->
match args with
| [String name; value] ->
let stack = try Hashtbl.find scope_stacks name with Not_found -> [] in
(match stack with
| List items :: rest ->
if not (List.mem value items) then
Hashtbl.replace scope_stacks name (List (items @ [value]) :: rest)
| [] ->
Hashtbl.replace scope_stacks name [List [value]]
| _ :: _ -> ());
Nil
| _ -> Nil);
register "collected" (fun args ->
match args with
| [String name] ->
let stack = try Hashtbl.find scope_stacks name with Not_found -> [] in
(match stack with List items :: _ -> List items | _ -> List [])
| _ -> List []);
register "clear-collected!" (fun args ->
match args with
| [String name] ->
let stack = try Hashtbl.find scope_stacks name with Not_found -> [] in
(match stack with
| _ :: rest -> Hashtbl.replace scope_stacks name (List [] :: rest)
| [] -> Hashtbl.replace scope_stacks name [List []]);
Nil
| _ -> Nil);
(* --- Emit / emitted (for spread attrs in adapter-html.sx) --- *)
register "scope-emit!" (fun args ->
match args with
| [String name; value] ->
let stack = try Hashtbl.find scope_stacks name with Not_found -> [] in
(match stack with
| List items :: rest ->
Hashtbl.replace scope_stacks name (List (items @ [value]) :: rest)
| Nil :: rest ->
Hashtbl.replace scope_stacks name (List [value] :: rest)
| [] ->
Hashtbl.replace scope_stacks name [List [value]]
| _ :: _ -> ());
Nil
| _ -> Nil);
register "emit!" (fun args ->
(* Alias for scope-emit! *)
match Sx_primitives.get_primitive "scope-emit!" with
| NativeFn (_, fn) -> fn args | _ -> Nil);
register "emitted" (fun args ->
match args with
| [String name] ->
let stack = try Hashtbl.find scope_stacks name with Not_found -> [] in
(match stack with List items :: _ -> List items | _ -> List [])
| _ -> List []);
register "scope-emitted" (fun args ->
match Sx_primitives.get_primitive "emitted" with
| NativeFn (_, fn) -> fn args | _ -> List []);
register "scope-collected" (fun args ->
match Sx_primitives.get_primitive "collected" with
| NativeFn (_, fn) -> fn args | _ -> List []);
register "scope-clear-collected!" (fun args ->
match Sx_primitives.get_primitive "clear-collected!" with
| NativeFn (_, fn) -> fn args | _ -> Nil);
(* --- Provide aliases --- *)
register "provide-push!" (fun args ->
match Sx_primitives.get_primitive "scope-push!" with
| NativeFn (_, fn) -> fn args | _ -> Nil);
register "provide-pop!" (fun args ->
match Sx_primitives.get_primitive "scope-pop!" with
| NativeFn (_, fn) -> fn args | _ -> Nil)

7
hosts/ocaml/lib/sx_vm.ml
View File

@@ -345,6 +345,13 @@ and run vm =
let argc = read_u8 frame in let argc = read_u8 frame in
let name = match consts.(idx) with String s -> s | _ -> "" in let name = match consts.(idx) with String s -> s | _ -> "" in
let args = List.init argc (fun _ -> pop vm) |> List.rev in let args = List.init argc (fun _ -> pop vm) |> List.rev in
(* Resolve thunks — the CEK evaluator does this automatically
via trampoline, but the VM must do it explicitly before
passing args to primitives. *)
let args = List.map (fun v ->
match v with
| Thunk _ -> !Sx_primitives._sx_trampoline_fn v
| _ -> v) args in
let result = let result =
try try
(* Check primitives FIRST (native implementations of map/filter/etc.), (* Check primitives FIRST (native implementations of map/filter/etc.),

2
shared/static/scripts/sx-browser.js
View File

@@ -14,7 +14,7 @@
// ========================================================================= // =========================================================================
var NIL = Object.freeze({ _nil: true, toString: function() { return "nil"; } }); var NIL = Object.freeze({ _nil: true, toString: function() { return "nil"; } });
var SX_VERSION = "2026-03-24T14:17:24Z"; var SX_VERSION = "2026-03-24T15:37:52Z";
function isNil(x) { return x === NIL || x === null || x === undefined; } function isNil(x) { return x === NIL || x === null || x === undefined; }
function isSxTruthy(x) { return x !== false && !isNil(x); } function isSxTruthy(x) { return x !== false && !isNil(x); }

41
shared/sx/async_eval.py
View File

@@ -423,23 +423,39 @@ async def _asf_define(expr, env, ctx):
async def _asf_defcomp(expr, env, ctx): async def _asf_defcomp(expr, env, ctx):
from .ref.sx_ref import sf_defcomp # Component definitions are handled by OCaml kernel at load time.
return sf_defcomp(expr[1:], env) # Python-side: just store a minimal Component in env for reference.
from .types import Component
name_sym = expr[1]
name = name_sym.name if hasattr(name_sym, 'name') else str(name_sym)
env[name] = Component(name=name.lstrip("~"), params=[], has_children=False,
body=expr[-1], closure={})
return NIL
async def _asf_defstyle(expr, env, ctx): async def _asf_defstyle(expr, env, ctx):
from .ref.sx_ref import sf_defstyle # Style definitions handled by OCaml kernel.
return sf_defstyle(expr[1:], env) return NIL
async def _asf_defmacro(expr, env, ctx): async def _asf_defmacro(expr, env, ctx):
from .ref.sx_ref import sf_defmacro # Macro definitions handled by OCaml kernel.
return sf_defmacro(expr[1:], env) from .types import Macro
name_sym = expr[1]
name = name_sym.name if hasattr(name_sym, 'name') else str(name_sym)
params_form = expr[2] if len(expr) > 3 else []
param_names = [p.name for p in params_form if isinstance(p, Symbol) and not p.name.startswith("&")]
rest_param = None
for i, p in enumerate(params_form):
if isinstance(p, Symbol) and p.name == "&rest" and i + 1 < len(params_form):
rest_param = params_form[i + 1].name if isinstance(params_form[i + 1], Symbol) else None
env[name] = Macro(name=name, params=param_names, rest_param=rest_param, body=expr[-1])
return NIL
async def _asf_defhandler(expr, env, ctx): async def _asf_defhandler(expr, env, ctx):
from .ref.sx_ref import sf_defhandler # Handler definitions handled by OCaml kernel.
return sf_defhandler(expr[1:], env) return NIL
async def _asf_begin(expr, env, ctx): async def _asf_begin(expr, env, ctx):
@@ -601,9 +617,12 @@ async def _asf_reset(expr, env, ctx):
from .types import NIL from .types import NIL
_ASYNC_RESET_RESUME.append(value if value is not None else NIL) _ASYNC_RESET_RESUME.append(value if value is not None else NIL)
try: try:
# Sync re-evaluation; the async caller will trampoline # Continuations are handled by OCaml kernel.
from .ref.sx_ref import eval_expr as sync_eval, trampoline as _trampoline # Python-side cont_fn should not be called in normal operation.
return _trampoline(sync_eval(body, env)) raise RuntimeError(
"Python-side continuation invocation not supported — "
"use OCaml bridge for shift/reset"
)
finally: finally:
_ASYNC_RESET_RESUME.pop() _ASYNC_RESET_RESUME.pop()
k = Continuation(cont_fn) k = Continuation(cont_fn)

17
shared/sx/deps.py
View File

@@ -152,18 +152,11 @@ def transitive_deps(name: str, env: dict[str, Any]) -> set[str]:
Returns the set of all component names (with ~ prefix) that Returns the set of all component names (with ~ prefix) that
*name* can transitively render, NOT including *name* itself. *name* can transitively render, NOT including *name* itself.
""" """
if _use_ref():
from .ref.sx_ref import transitive_deps as _ref_td
return set(_ref_td(name, env))
return _transitive_deps_fallback(name, env) return _transitive_deps_fallback(name, env)
def compute_all_deps(env: dict[str, Any]) -> None: def compute_all_deps(env: dict[str, Any]) -> None:
"""Compute and cache deps for all Component entries in *env*.""" """Compute and cache deps for all Component entries in *env*."""
if _use_ref():
from .ref.sx_ref import compute_all_deps as _ref_cad
_ref_cad(env)
return
_compute_all_deps_fallback(env) _compute_all_deps_fallback(env)
@@ -172,9 +165,6 @@ def scan_components_from_sx(source: str) -> set[str]:
Returns names with ~ prefix, e.g. {"~card", "~shared:layout/nav-link"}. Returns names with ~ prefix, e.g. {"~card", "~shared:layout/nav-link"}.
""" """
if _use_ref():
from .ref.sx_ref import scan_components_from_source as _ref_sc
return set(_ref_sc(source))
return _scan_components_from_sx_fallback(source) return _scan_components_from_sx_fallback(source)
@@ -183,18 +173,11 @@ def components_needed(page_sx: str, env: dict[str, Any]) -> set[str]:
Returns names with ~ prefix. Returns names with ~ prefix.
""" """
if _use_ref():
from .ref.sx_ref import components_needed as _ref_cn
return set(_ref_cn(page_sx, env))
return _components_needed_fallback(page_sx, env) return _components_needed_fallback(page_sx, env)
def compute_all_io_refs(env: dict[str, Any], io_names: set[str]) -> None: def compute_all_io_refs(env: dict[str, Any], io_names: set[str]) -> None:
"""Compute and cache transitive IO refs for all Component entries in *env*.""" """Compute and cache transitive IO refs for all Component entries in *env*."""
if _use_ref():
from .ref.sx_ref import compute_all_io_refs as _ref_cio
_ref_cio(env, list(io_names))
return
_compute_all_io_refs_fallback(env, io_names) _compute_all_io_refs_fallback(env, io_names)

7
shared/sx/handlers.py
View File

@@ -219,11 +219,8 @@ async def execute_handler(
result_sx = await bridge.aser(sx_text, ctx=ocaml_ctx) result_sx = await bridge.aser(sx_text, ctx=ocaml_ctx)
return SxExpr(result_sx or "") return SxExpr(result_sx or "")
else: else:
# Python fallback # Python fallback (async_eval)
if os.environ.get("SX_USE_REF") == "1": from .async_eval import async_eval_to_sx
from .ref.async_eval_ref import async_eval_to_sx
else:
from .async_eval import async_eval_to_sx
env = dict(get_component_env()) env = dict(get_component_env())
env.update(get_page_helpers(service_name)) env.update(get_page_helpers(service_name))

28
shared/sx/helpers.py
View File

@@ -385,10 +385,7 @@ async def _render_to_sx_with_env(__name: str, extra_env: dict, **kwargs: Any) ->
ocaml_ctx = {"_helper_service": _get_request_context().get("_helper_service", "")} if isinstance(_get_request_context(), dict) else {} ocaml_ctx = {"_helper_service": _get_request_context().get("_helper_service", "")} if isinstance(_get_request_context(), dict) else {}
return SxExpr(await bridge.aser_slot(sx_text, ctx=ocaml_ctx)) return SxExpr(await bridge.aser_slot(sx_text, ctx=ocaml_ctx))
if os.environ.get("SX_USE_REF") == "1": from .async_eval import async_eval_slot_to_sx
from .ref.async_eval_ref import async_eval_slot_to_sx
else:
from .async_eval import async_eval_slot_to_sx
env = dict(get_component_env()) env = dict(get_component_env())
env.update(extra_env) env.update(extra_env)
@@ -421,10 +418,7 @@ async def _render_to_sx(__name: str, **kwargs: Any) -> str:
# symbols like `title` that were bound during the earlier expansion. # symbols like `title` that were bound during the earlier expansion.
return SxExpr(await bridge.aser_slot(sx_text)) return SxExpr(await bridge.aser_slot(sx_text))
if os.environ.get("SX_USE_REF") == "1": from .async_eval import async_eval_to_sx
from .ref.async_eval_ref import async_eval_to_sx
else:
from .async_eval import async_eval_to_sx
env = dict(get_component_env()) env = dict(get_component_env())
ctx = _get_request_context() ctx = _get_request_context()
@@ -442,15 +436,23 @@ async def render_to_html(__name: str, **kwargs: Any) -> str:
Same as render_to_sx() but produces HTML output instead of SX wire Same as render_to_sx() but produces HTML output instead of SX wire
format. Used by route renders that need HTML (full pages, fragments). format. Used by route renders that need HTML (full pages, fragments).
Note: does NOT use OCaml bridge — the shell render is a pure HTML Routes through the OCaml bridge (render mode) which handles component
template with no IO, so the Python renderer handles it reliably. parameter binding, scope primitives, and all evaluation.
The OCaml path is used for _render_to_sx and _eval_slot (IO-heavy).
""" """
from .jinja_bridge import get_component_env, _get_request_context
import os import os
from .async_eval import async_render
ast = _build_component_ast(__name, **kwargs) ast = _build_component_ast(__name, **kwargs)
if os.environ.get("SX_USE_OCAML") == "1":
from .ocaml_bridge import get_bridge
from .parser import serialize
bridge = await get_bridge()
sx_text = serialize(ast)
return await bridge.render(sx_text)
# Fallback: Python async_eval (requires working evaluator)
from .jinja_bridge import get_component_env, _get_request_context
from .async_eval import async_render
env = dict(get_component_env()) env = dict(get_component_env())
ctx = _get_request_context() ctx = _get_request_context()
return await async_render(ast, env, ctx) return await async_render(ast, env, ctx)

165
shared/sx/html.py
View File

@@ -28,20 +28,163 @@ import contextvars
from typing import Any from typing import Any
from .types import Component, Island, Keyword, Lambda, Macro, NIL, Symbol from .types import Component, Island, Keyword, Lambda, Macro, NIL, Symbol
# sx_ref.py removed — these stubs exist so the module loads.
# With SX_USE_OCAML=1, rendering goes through the OCaml bridge; these
# are only called if a service falls back to Python-side rendering.
def _not_available(*a, **kw):
raise RuntimeError("sx_ref.py has been removed — use SX_USE_OCAML=1")
_raw_eval = _raw_call_component = _expand_macro = _trampoline = _not_available
def _eval(expr, env): def _eval(expr, env):
"""Evaluate and unwrap thunks — all html.py _eval calls are non-tail.""" """Minimal Python evaluator for sync html.py rendering.
return _trampoline(_raw_eval(expr, env))
def _call_component(comp, raw_args, env): Handles: literals, symbols, keywords, dicts, special forms (if, when,
"""Call component and unwrap thunks — non-tail in html.py.""" cond, let, begin/do, and, or, str, not, list), lambda calls, and
return _trampoline(_raw_call_component(comp, raw_args, env)) primitive function calls. Enough for the sync sx() Jinja function.
"""
from .primitives import _PRIMITIVES
# Literals
if isinstance(expr, (int, float, str, bool)):
return expr
if expr is None or expr is NIL:
return NIL
# Symbol lookup
if isinstance(expr, Symbol):
name = expr.name
if name in env:
return env[name]
if name in _PRIMITIVES:
return _PRIMITIVES[name]
if name == "true":
return True
if name == "false":
return False
if name == "nil":
return NIL
from .types import EvalError
raise EvalError(f"Undefined symbol: {name}")
# Keyword
if isinstance(expr, Keyword):
return expr.name
# Dict
if isinstance(expr, dict):
return {k: _eval(v, env) for k, v in expr.items()}
# List — dispatch
if not isinstance(expr, list):
return expr
if not expr:
return []
head = expr[0]
if isinstance(head, Symbol):
name = head.name
# Special forms
if name == "if":
cond = _eval(expr[1], env)
if cond and cond is not NIL:
return _eval(expr[2], env)
return _eval(expr[3], env) if len(expr) > 3 else NIL
if name == "when":
cond = _eval(expr[1], env)
if cond and cond is not NIL:
result = NIL
for body in expr[2:]:
result = _eval(body, env)
return result
return NIL
if name == "let":
bindings = expr[1]
local = dict(env)
if isinstance(bindings, list):
if bindings and isinstance(bindings[0], list):
for b in bindings:
vname = b[0].name if isinstance(b[0], Symbol) else b[0]
local[vname] = _eval(b[1], local)
elif len(bindings) % 2 == 0:
for i in range(0, len(bindings), 2):
vname = bindings[i].name if isinstance(bindings[i], Symbol) else bindings[i]
local[vname] = _eval(bindings[i + 1], local)
result = NIL
for body in expr[2:]:
result = _eval(body, local)
return result
if name in ("begin", "do"):
result = NIL
for body in expr[1:]:
result = _eval(body, env)
return result
if name == "and":
result = True
for arg in expr[1:]:
result = _eval(arg, env)
if not result or result is NIL:
return result
return result
if name == "or":
for arg in expr[1:]:
result = _eval(arg, env)
if result and result is not NIL:
return result
return NIL
if name == "not":
val = _eval(expr[1], env)
return val is NIL or val is False or val is None
if name == "lambda" or name == "fn":
params_form = expr[1]
param_names = [p.name if isinstance(p, Symbol) else str(p) for p in params_form]
return Lambda(params=param_names, body=expr[2], closure=dict(env))
if name == "define":
var_name = expr[1].name if isinstance(expr[1], Symbol) else str(expr[1])
env[var_name] = _eval(expr[2], env)
return NIL
if name == "quote":
return expr[1]
if name == "str":
parts = []
for arg in expr[1:]:
val = _eval(arg, env)
if val is NIL or val is None:
parts.append("")
else:
parts.append(str(val))
return "".join(parts)
if name == "list":
return [_eval(arg, env) for arg in expr[1:]]
# Primitive or function call
fn = _eval(head, env)
else:
fn = _eval(head, env)
# Evaluate args
args = [_eval(a, env) for a in expr[1:]]
# Call
if callable(fn):
return fn(*args)
if isinstance(fn, Lambda):
local = dict(fn.closure)
local.update(env)
for p, v in zip(fn.params, args):
local[p] = v
return _eval(fn.body, local)
return NIL
def _expand_macro(*a, **kw):
raise RuntimeError("Macro expansion requires OCaml bridge")
# ContextVar for collecting CSS class names during render. # ContextVar for collecting CSS class names during render.
# Set to a set[str] to collect; None to skip. # Set to a set[str] to collect; None to skip.

281
shared/sx/ocaml_bridge.py
View File

@@ -46,6 +46,7 @@ class OcamlBridge:
self._components_loaded = False self._components_loaded = False
self._helpers_injected = False self._helpers_injected = False
self._io_cache: dict[tuple, Any] = {} # (name, args...) → cached result self._io_cache: dict[tuple, Any] = {} # (name, args...) → cached result
self._epoch: int = 0 # request epoch — monotonically increasing
async def start(self) -> None: async def start(self) -> None:
"""Launch the OCaml subprocess and wait for (ready).""" """Launch the OCaml subprocess and wait for (ready)."""
@@ -77,7 +78,7 @@ class OcamlBridge:
self._started = True self._started = True
# Verify engine identity # Verify engine identity
await self._send("(ping)") await self._send_command("(ping)")
kind, engine = await self._read_response() kind, engine = await self._read_response()
engine_name = engine if kind == "ok" else "unknown" engine_name = engine if kind == "ok" else "unknown"
_logger.info("OCaml SX kernel ready (pid=%d, engine=%s)", self._proc.pid, engine_name) _logger.info("OCaml SX kernel ready (pid=%d, engine=%s)", self._proc.pid, engine_name)
@@ -95,24 +96,36 @@ class OcamlBridge:
self._proc = None self._proc = None
self._started = False self._started = False
async def _restart(self) -> None:
"""Kill and restart the OCaml subprocess to recover from pipe desync."""
_logger.warning("Restarting OCaml SX kernel (pipe recovery)")
if self._proc and self._proc.returncode is None:
self._proc.kill()
await self._proc.wait()
self._proc = None
self._started = False
self._components_loaded = False
self._helpers_injected = False
await self.start()
async def ping(self) -> str: async def ping(self) -> str:
"""Health check — returns engine name (e.g. 'ocaml-cek').""" """Health check — returns engine name (e.g. 'ocaml-cek')."""
async with self._lock: async with self._lock:
await self._send("(ping)") await self._send_command("(ping)")
kind, value = await self._read_response() kind, value = await self._read_response()
return value or "" if kind == "ok" else "" return value or "" if kind == "ok" else ""
async def load(self, path: str) -> int: async def load(self, path: str) -> int:
"""Load an .sx file for side effects (defcomp, define, defmacro).""" """Load an .sx file for side effects (defcomp, define, defmacro)."""
async with self._lock: async with self._lock:
await self._send(f'(load "{_escape(path)}")') await self._send_command(f'(load "{_escape(path)}")')
value = await self._read_until_ok(ctx=None) value = await self._read_until_ok(ctx=None)
return int(float(value)) if value else 0 return int(float(value)) if value else 0
async def load_source(self, source: str) -> int: async def load_source(self, source: str) -> int:
"""Evaluate SX source for side effects.""" """Evaluate SX source for side effects."""
async with self._lock: async with self._lock:
await self._send(f'(load-source "{_escape(source)}")') await self._send_command(f'(load-source "{_escape(source)}")')
value = await self._read_until_ok(ctx=None) value = await self._read_until_ok(ctx=None)
return int(float(value)) if value else 0 return int(float(value)) if value else 0
@@ -124,7 +137,7 @@ class OcamlBridge:
""" """
await self._ensure_components() await self._ensure_components()
async with self._lock: async with self._lock:
await self._send('(eval-blob)') await self._send_command('(eval-blob)')
await self._send_blob(source) await self._send_blob(source)
return await self._read_until_ok(ctx) return await self._read_until_ok(ctx)
@@ -136,14 +149,14 @@ class OcamlBridge:
"""Render SX to HTML, handling io-requests via Python async IO.""" """Render SX to HTML, handling io-requests via Python async IO."""
await self._ensure_components() await self._ensure_components()
async with self._lock: async with self._lock:
await self._send(f'(render "{_escape(source)}")') await self._send_command(f'(render "{_escape(source)}")')
return await self._read_until_ok(ctx) return await self._read_until_ok(ctx)
async def aser(self, source: str, ctx: dict[str, Any] | None = None) -> str: async def aser(self, source: str, ctx: dict[str, Any] | None = None) -> str:
"""Evaluate SX and return SX wire format, handling io-requests.""" """Evaluate SX and return SX wire format, handling io-requests."""
await self._ensure_components() await self._ensure_components()
async with self._lock: async with self._lock:
await self._send('(aser-blob)') await self._send_command('(aser-blob)')
await self._send_blob(source) await self._send_blob(source)
return await self._read_until_ok(ctx) return await self._read_until_ok(ctx)
@@ -159,7 +172,7 @@ class OcamlBridge:
# a separate lock acquisition could let another coroutine # a separate lock acquisition could let another coroutine
# interleave commands between injection and aser-slot. # interleave commands between injection and aser-slot.
await self._inject_helpers_locked() await self._inject_helpers_locked()
await self._send('(aser-slot-blob)') await self._send_command('(aser-slot-blob)')
await self._send_blob(source) await self._send_blob(source)
return await self._read_until_ok(ctx) return await self._read_until_ok(ctx)
@@ -182,7 +195,7 @@ class OcamlBridge:
var = f"__shell-{key.replace('_', '-')}" var = f"__shell-{key.replace('_', '-')}"
defn = f'(define {var} "{_escape(str(val))}")' defn = f'(define {var} "{_escape(str(val))}")'
try: try:
await self._send(f'(load-source "{_escape(defn)}")') await self._send_command(f'(load-source "{_escape(defn)}")')
await self._read_until_ok(ctx=None) await self._read_until_ok(ctx=None)
except OcamlBridgeError as e: except OcamlBridgeError as e:
_logger.warning("Shell static inject failed for %s: %s", key, e) _logger.warning("Shell static inject failed for %s: %s", key, e)
@@ -198,7 +211,7 @@ class OcamlBridge:
else: else:
defn = f'(define {var} "{_escape(str(val))}")' defn = f'(define {var} "{_escape(str(val))}")'
try: try:
await self._send(f'(load-source "{_escape(defn)}")') await self._send_command(f'(load-source "{_escape(defn)}")')
await self._read_until_ok(ctx=None) await self._read_until_ok(ctx=None)
except OcamlBridgeError as e: except OcamlBridgeError as e:
_logger.warning("Shell static inject failed for %s: %s", key, e) _logger.warning("Shell static inject failed for %s: %s", key, e)
@@ -221,7 +234,7 @@ class OcamlBridge:
if pairs: if pairs:
cmd = f'(set-request-cookies {{{" ".join(pairs)}}})' cmd = f'(set-request-cookies {{{" ".join(pairs)}}})'
try: try:
await self._send(cmd) await self._send_command(cmd)
await self._read_until_ok(ctx=None) await self._read_until_ok(ctx=None)
except OcamlBridgeError as e: except OcamlBridgeError as e:
_logger.debug("Cookie inject failed: %s", e) _logger.debug("Cookie inject failed: %s", e)
@@ -277,7 +290,7 @@ class OcamlBridge:
parts.append(f' :{k} "{_escape(str(val))}"') parts.append(f' :{k} "{_escape(str(val))}"')
parts.append(")") parts.append(")")
cmd = "".join(parts) cmd = "".join(parts)
await self._send(cmd) await self._send_command(cmd)
# Send page source as binary blob (avoids string-escape issues) # Send page source as binary blob (avoids string-escape issues)
await self._send_blob(page_source) await self._send_blob(page_source)
html = await self._read_until_ok(ctx) html = await self._read_until_ok(ctx)
@@ -312,7 +325,7 @@ class OcamlBridge:
arg_list = " ".join(chr(97 + i) for i in range(nargs)) arg_list = " ".join(chr(97 + i) for i in range(nargs))
sx_def = f'(define {name} (fn ({param_names}) (helper "{name}" {arg_list})))' sx_def = f'(define {name} (fn ({param_names}) (helper "{name}" {arg_list})))'
try: try:
await self._send(f'(load-source "{_escape(sx_def)}")') await self._send_command(f'(load-source "{_escape(sx_def)}")')
await self._read_until_ok(ctx=None) await self._read_until_ok(ctx=None)
count += 1 count += 1
except OcamlBridgeError: except OcamlBridgeError:
@@ -325,70 +338,11 @@ class OcamlBridge:
async def _compile_adapter_module(self) -> None: async def _compile_adapter_module(self) -> None:
"""Compile adapter-sx.sx to bytecode and load as a VM module. """Compile adapter-sx.sx to bytecode and load as a VM module.
All aser functions become NativeFn VM closures in the kernel env. Previously used Python's sx_ref.py evaluator for compilation.
Subsequent aser-slot calls find them as NativeFn → VM executes Now the OCaml kernel handles JIT compilation natively — this method
the entire render path compiled, no CEK steps. is a no-op. The kernel's own JIT hook compiles functions on first call.
""" """
from .parser import parse_all, serialize _logger.info("Adapter module compilation delegated to OCaml kernel JIT")
from .ref.sx_ref import eval_expr, trampoline, PRIMITIVES
# Ensure compiler primitives are available
if 'serialize' not in PRIMITIVES:
PRIMITIVES['serialize'] = lambda x: serialize(x)
if 'primitive?' not in PRIMITIVES:
PRIMITIVES['primitive?'] = lambda name: isinstance(name, str) and name in PRIMITIVES
if 'has-key?' not in PRIMITIVES:
PRIMITIVES['has-key?'] = lambda *a: isinstance(a[0], dict) and str(a[1]) in a[0]
if 'set-nth!' not in PRIMITIVES:
from .types import NIL
PRIMITIVES['set-nth!'] = lambda *a: (a[0].__setitem__(int(a[1]), a[2]), NIL)[-1]
if 'init' not in PRIMITIVES:
PRIMITIVES['init'] = lambda *a: a[0][:-1] if isinstance(a[0], list) else a[0]
if 'concat' not in PRIMITIVES:
PRIMITIVES['concat'] = lambda *a: (a[0] or []) + (a[1] or [])
if 'slice' not in PRIMITIVES:
PRIMITIVES['slice'] = lambda *a: a[0][int(a[1]):int(a[2])] if len(a) == 3 else a[0][int(a[1]):]
from .types import Symbol
if 'make-symbol' not in PRIMITIVES:
PRIMITIVES['make-symbol'] = lambda name: Symbol(name)
from .types import NIL
for ho in ['map', 'filter', 'for-each', 'reduce', 'some', 'every?', 'map-indexed']:
if ho not in PRIMITIVES:
PRIMITIVES[ho] = lambda *a: NIL
# Load compiler
compiler_env = {}
spec_dir = os.path.join(os.path.dirname(__file__), "../../spec")
for f in ["bytecode.sx", "compiler.sx"]:
path = os.path.join(spec_dir, f)
if os.path.isfile(path):
with open(path) as fh:
for expr in parse_all(fh.read()):
trampoline(eval_expr(expr, compiler_env))
# Compile adapter-sx.sx
web_dir = os.path.join(os.path.dirname(__file__), "../../web")
adapter_path = os.path.join(web_dir, "adapter-sx.sx")
if not os.path.isfile(adapter_path):
_logger.warning("adapter-sx.sx not found at %s", adapter_path)
return
with open(adapter_path) as f:
adapter_exprs = parse_all(f.read())
compiled = trampoline(eval_expr(
[Symbol('compile-module'), [Symbol('quote'), adapter_exprs]],
compiler_env))
code_sx = serialize(compiled)
_logger.info("Compiled adapter-sx.sx: %d bytes bytecode", len(code_sx))
# Load the compiled module into the OCaml VM
async with self._lock:
await self._send(f'(vm-load-module {code_sx})')
await self._read_until_ok(ctx=None)
_logger.info("Loaded adapter-sx.sx as VM module")
async def _ensure_components(self) -> None: async def _ensure_components(self) -> None:
"""Load all .sx source files into the kernel on first use. """Load all .sx source files into the kernel on first use.
@@ -455,7 +409,7 @@ class OcamlBridge:
async with self._lock: async with self._lock:
for filepath in all_files: for filepath in all_files:
try: try:
await self._send(f'(load "{_escape(filepath)}")') await self._send_command(f'(load "{_escape(filepath)}")')
value = await self._read_until_ok(ctx=None) value = await self._read_until_ok(ctx=None)
# Response may be a number (count) or a value — just count files # Response may be a number (count) or a value — just count files
count += 1 count += 1
@@ -468,14 +422,14 @@ class OcamlBridge:
# reactive loops during island SSR — effects are DOM side-effects) # reactive loops during island SSR — effects are DOM side-effects)
try: try:
noop_dispose = '(fn () nil)' noop_dispose = '(fn () nil)'
await self._send(f'(load-source "(define effect (fn (f) {noop_dispose}))")') await self._send_command(f'(load-source "(define effect (fn (f) {noop_dispose}))")')
await self._read_until_ok(ctx=None) await self._read_until_ok(ctx=None)
except OcamlBridgeError: except OcamlBridgeError:
pass pass
# Register JIT hook — lambdas compile on first call # Register JIT hook — lambdas compile on first call
try: try:
await self._send('(vm-compile-adapter)') await self._send_command('(vm-compile-adapter)')
await self._read_until_ok(ctx=None) await self._read_until_ok(ctx=None)
_logger.info("JIT hook registered — lambdas compile on first call") _logger.info("JIT hook registered — lambdas compile on first call")
except OcamlBridgeError as e: except OcamlBridgeError as e:
@@ -499,7 +453,7 @@ class OcamlBridge:
if callable(fn) and not name.startswith("~"): if callable(fn) and not name.startswith("~"):
sx_def = f'(define {name} (fn (&rest args) (apply helper (concat (list "{name}") args))))' sx_def = f'(define {name} (fn (&rest args) (apply helper (concat (list "{name}") args))))'
try: try:
await self._send(f'(load-source "{_escape(sx_def)}")') await self._send_command(f'(load-source "{_escape(sx_def)}")')
await self._read_until_ok(ctx=None) await self._read_until_ok(ctx=None)
count += 1 count += 1
except OcamlBridgeError: except OcamlBridgeError:
@@ -510,7 +464,7 @@ class OcamlBridge:
async def reset(self) -> None: async def reset(self) -> None:
"""Reset the kernel environment to pristine state.""" """Reset the kernel environment to pristine state."""
async with self._lock: async with self._lock:
await self._send("(reset)") await self._send_command("(reset)")
kind, value = await self._read_response() kind, value = await self._read_response()
if kind == "error": if kind == "error":
raise OcamlBridgeError(f"reset: {value}") raise OcamlBridgeError(f"reset: {value}")
@@ -531,6 +485,20 @@ class OcamlBridge:
self._proc.stdin.write((line + "\n").encode()) self._proc.stdin.write((line + "\n").encode())
await self._proc.stdin.drain() await self._proc.stdin.drain()
async def _send_command(self, line: str) -> None:
"""Send a command with a fresh epoch prefix.
Increments the epoch counter and sends (epoch N) before the
actual command. The OCaml kernel tags all responses with this
epoch so stale messages from previous requests are discarded.
"""
self._epoch += 1
assert self._proc and self._proc.stdin
_logger.debug("EPOCH %d SEND: %s", self._epoch, line[:120])
self._proc.stdin.write(f"(epoch {self._epoch})\n".encode())
self._proc.stdin.write((line + "\n").encode())
await self._proc.stdin.drain()
async def _send_blob(self, data: str) -> None: async def _send_blob(self, data: str) -> None:
"""Send a length-prefixed binary blob to the subprocess. """Send a length-prefixed binary blob to the subprocess.
@@ -562,16 +530,45 @@ class OcamlBridge:
"""Read a single (ok ...) or (error ...) response. """Read a single (ok ...) or (error ...) response.
Returns (kind, value) where kind is "ok" or "error". Returns (kind, value) where kind is "ok" or "error".
Discards stale epoch messages.
""" """
line = await self._readline() while True:
# Length-prefixed blob line = await self._readline()
if line.startswith("(ok-len "): if not self._is_current_epoch(line):
n = int(line[8:-1]) _logger.debug("Discarding stale response: %s", line[:80])
assert self._proc and self._proc.stdout if line.startswith("(ok-len "):
data = await self._proc.stdout.readexactly(n) parts = line[1:-1].split()
await self._proc.stdout.readline() # trailing newline if len(parts) >= 3:
return ("ok", data.decode()) n = int(parts[-1])
return _parse_response(line) assert self._proc and self._proc.stdout
await self._proc.stdout.readexactly(n)
await self._proc.stdout.readline()
continue
# Length-prefixed blob: (ok-len EPOCH N) or (ok-len N)
if line.startswith("(ok-len "):
parts = line[1:-1].split()
n = int(parts[-1])
assert self._proc and self._proc.stdout
data = await self._proc.stdout.readexactly(n)
await self._proc.stdout.readline() # trailing newline
return ("ok", data.decode())
return _parse_response(line)
def _is_current_epoch(self, line: str) -> bool:
"""Check if a response line belongs to the current epoch.
Lines tagged with a stale epoch are discarded. Untagged lines
(from a kernel that predates the epoch protocol) are accepted.
"""
# Extract epoch number from known tagged formats:
# (ok EPOCH ...), (error EPOCH ...), (ok-len EPOCH N),
# (io-request EPOCH ...), (io-done EPOCH N)
import re
m = re.match(r'\((?:ok|error|ok-len|ok-raw|io-request|io-done)\s+(\d+)\b', line)
if m:
return int(m.group(1)) == self._epoch
# Untagged (legacy) — accept
return True
async def _read_until_ok( async def _read_until_ok(
self, self,
@@ -583,6 +580,9 @@ class OcamlBridge:
- Legacy (blocking): single io-request → immediate io-response - Legacy (blocking): single io-request → immediate io-response
- Batched: collect io-requests until (io-done N), process ALL - Batched: collect io-requests until (io-done N), process ALL
concurrently with asyncio.gather, send responses in order concurrently with asyncio.gather, send responses in order
Lines tagged with a stale epoch are silently discarded, making
pipe desync from previous failed requests impossible.
""" """
import asyncio import asyncio
pending_batch: list[str] = [] pending_batch: list[str] = []
@@ -590,20 +590,53 @@ class OcamlBridge:
while True: while True:
line = await self._readline() line = await self._readline()
# Discard stale epoch messages
if not self._is_current_epoch(line):
_logger.debug("Discarding stale epoch message: %s", line[:80])
# If it's a stale ok-len, drain the blob bytes too
if line.startswith("(ok-len "):
parts = line[1:-1].split()
if len(parts) >= 3:
n = int(parts[2])
assert self._proc and self._proc.stdout
await self._proc.stdout.readexactly(n)
await self._proc.stdout.readline()
continue
if line.startswith("(io-request "): if line.startswith("(io-request "):
# Check if batched (has numeric ID after "io-request ") # New format: (io-request EPOCH ...) or (io-request EPOCH ID ...)
# Strip epoch from the line for IO dispatch
after = line[len("(io-request "):].lstrip() after = line[len("(io-request "):].lstrip()
# Skip epoch number if present
if after and after[0].isdigit(): if after and after[0].isdigit():
# Batched mode — collect, don't respond yet # Could be epoch or batch ID — check for second number
pending_batch.append(line) parts = after.split(None, 2)
continue if len(parts) >= 2 and parts[1][0].isdigit():
# (io-request EPOCH ID "name" args...) — batched with epoch
pending_batch.append(line)
continue
elif len(parts) >= 2 and parts[1].startswith('"'):
# (io-request EPOCH "name" args...) — legacy with epoch
try:
result = await self._handle_io_request(line, ctx)
await self._send(
f"(io-response {self._epoch} {_serialize_for_ocaml(result)})")
except Exception as e:
_logger.warning("IO request failed, sending nil: %s", e)
await self._send(f"(io-response {self._epoch} nil)")
continue
else:
# Old format: (io-request ID "name" ...) — batched, no epoch
pending_batch.append(line)
continue
# Legacy blocking mode — respond immediately # Legacy blocking mode — respond immediately
try: try:
result = await self._handle_io_request(line, ctx) result = await self._handle_io_request(line, ctx)
await self._send(f"(io-response {_serialize_for_ocaml(result)})") await self._send(
f"(io-response {self._epoch} {_serialize_for_ocaml(result)})")
except Exception as e: except Exception as e:
_logger.warning("IO request failed, sending nil: %s", e) _logger.warning("IO request failed, sending nil: %s", e)
await self._send("(io-response nil)") await self._send(f"(io-response {self._epoch} nil)")
continue continue
if line.startswith("(io-done "): if line.startswith("(io-done "):
@@ -614,16 +647,17 @@ class OcamlBridge:
for result in results: for result in results:
if isinstance(result, BaseException): if isinstance(result, BaseException):
_logger.warning("Batched IO failed: %s", result) _logger.warning("Batched IO failed: %s", result)
await self._send("(io-response nil)") await self._send(f"(io-response {self._epoch} nil)")
else: else:
await self._send( await self._send(
f"(io-response {_serialize_for_ocaml(result)})") f"(io-response {self._epoch} {_serialize_for_ocaml(result)})")
pending_batch = [] pending_batch = []
continue continue
# Length-prefixed blob: (ok-len N) # Length-prefixed blob: (ok-len EPOCH N) or (ok-len N)
if line.startswith("(ok-len "): if line.startswith("(ok-len "):
n = int(line[8:-1]) parts = line[1:-1].split() # ["ok-len", epoch, n] or ["ok-len", n]
n = int(parts[-1]) # last number is always byte count
assert self._proc and self._proc.stdout assert self._proc and self._proc.stdout
data = await self._proc.stdout.readexactly(n) data = await self._proc.stdout.readexactly(n)
# Read trailing newline # Read trailing newline
@@ -829,25 +863,50 @@ def _escape(s: str) -> str:
def _parse_response(line: str) -> tuple[str, str | None]: def _parse_response(line: str) -> tuple[str, str | None]:
"""Parse an (ok ...) or (error ...) response line. """Parse an (ok ...) or (error ...) response line.
Handles epoch-tagged responses: (ok EPOCH), (ok EPOCH value),
(error EPOCH "msg"), as well as legacy untagged responses.
Returns (kind, value) tuple. Returns (kind, value) tuple.
""" """
line = line.strip() line = line.strip()
if line == "(ok)": # (ok EPOCH) — tagged no-value
if line == "(ok)" or (line.startswith("(ok ") and line[4:-1].isdigit()):
return ("ok", None) return ("ok", None)
if line.startswith("(ok-raw "): if line.startswith("(ok-raw "):
# Raw SX wire format — no unescaping needed # (ok-raw EPOCH value) or (ok-raw value)
return ("ok", line[8:-1]) inner = line[8:-1]
# Strip epoch if present
if inner and inner[0].isdigit():
space = inner.find(" ")
if space > 0:
inner = inner[space + 1:]
else:
return ("ok", None)
return ("ok", inner)
if line.startswith("(ok "): if line.startswith("(ok "):
value = line[4:-1] # strip (ok and ) inner = line[4:-1] # strip (ok and )
# Strip epoch number if present: (ok 42 "value") → "value"
if inner and inner[0].isdigit():
space = inner.find(" ")
if space > 0:
inner = inner[space + 1:]
else:
# (ok EPOCH) with no value
return ("ok", None)
# If the value is a quoted string, unquote it # If the value is a quoted string, unquote it
if value.startswith('"') and value.endswith('"'): if inner.startswith('"') and inner.endswith('"'):
value = _unescape(value[1:-1]) inner = _unescape(inner[1:-1])
return ("ok", value) return ("ok", inner)
if line.startswith("(error "): if line.startswith("(error "):
msg = line[7:-1] inner = line[7:-1]
if msg.startswith('"') and msg.endswith('"'): # Strip epoch number if present: (error 42 "msg") → "msg"
msg = _unescape(msg[1:-1]) if inner and inner[0].isdigit():
return ("error", msg) space = inner.find(" ")
if space > 0:
inner = inner[space + 1:]
if inner.startswith('"') and inner.endswith('"'):
inner = _unescape(inner[1:-1])
return ("error", inner)
return ("error", f"Unexpected response: {line}") return ("error", f"Unexpected response: {line}")

34
shared/sx/ocaml_sync.py
View File

@@ -52,6 +52,7 @@ class OcamlSync:
def __init__(self, binary: str | None = None): def __init__(self, binary: str | None = None):
self._binary = binary or os.environ.get("SX_OCAML_BIN") or _DEFAULT_BIN self._binary = binary or os.environ.get("SX_OCAML_BIN") or _DEFAULT_BIN
self._proc: subprocess.Popen | None = None self._proc: subprocess.Popen | None = None
self._epoch: int = 0
def _ensure(self): def _ensure(self):
if self._proc is not None and self._proc.poll() is None: if self._proc is not None and self._proc.poll() is None:
@@ -62,13 +63,17 @@ class OcamlSync:
stdout=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, stderr=subprocess.PIPE,
) )
self._epoch = 0
# Wait for (ready) # Wait for (ready)
line = self._readline() line = self._readline()
if line != "(ready)": if line != "(ready)":
raise OcamlSyncError(f"Expected (ready), got: {line}") raise OcamlSyncError(f"Expected (ready), got: {line}")
def _send(self, command: str): def _send(self, command: str):
"""Send a command with epoch prefix."""
assert self._proc and self._proc.stdin assert self._proc and self._proc.stdin
self._epoch += 1
self._proc.stdin.write(f"(epoch {self._epoch})\n".encode())
self._proc.stdin.write((command + "\n").encode()) self._proc.stdin.write((command + "\n").encode())
self._proc.stdin.flush() self._proc.stdin.flush()
@@ -79,12 +84,26 @@ class OcamlSync:
raise OcamlSyncError("OCaml subprocess died unexpectedly") raise OcamlSyncError("OCaml subprocess died unexpectedly")
return data.decode().rstrip("\n") return data.decode().rstrip("\n")
def _strip_epoch(self, inner: str) -> str:
"""Strip leading epoch number from a response value: '42 value' → 'value'."""
if inner and inner[0].isdigit():
space = inner.find(" ")
if space > 0:
return inner[space + 1:]
return "" # epoch only, no value
return inner
def _read_response(self) -> str: def _read_response(self) -> str:
"""Read a single response. Returns the value string or raises on error.""" """Read a single response. Returns the value string or raises on error.
Handles epoch-tagged responses: (ok EPOCH), (ok EPOCH value),
(ok-len EPOCH N), (error EPOCH "msg").
"""
line = self._readline() line = self._readline()
# Length-prefixed blob: (ok-len N) # Length-prefixed blob: (ok-len N) or (ok-len EPOCH N)
if line.startswith("(ok-len "): if line.startswith("(ok-len "):
n = int(line[8:-1]) parts = line[1:-1].split() # ["ok-len", ...]
n = int(parts[-1]) # last number is always byte count
assert self._proc and self._proc.stdout assert self._proc and self._proc.stdout
data = self._proc.stdout.read(n) data = self._proc.stdout.read(n)
self._proc.stdout.readline() # trailing newline self._proc.stdout.readline() # trailing newline
@@ -93,17 +112,18 @@ class OcamlSync:
if value.startswith('"') and value.endswith('"'): if value.startswith('"') and value.endswith('"'):
value = _sx_unescape(value[1:-1]) value = _sx_unescape(value[1:-1])
return value return value
if line == "(ok)": if line == "(ok)" or (line.startswith("(ok ") and line[4:-1].isdigit()):
return "" return ""
if line.startswith("(ok-raw "): if line.startswith("(ok-raw "):
return line[8:-1] inner = self._strip_epoch(line[8:-1])
return inner
if line.startswith("(ok "): if line.startswith("(ok "):
value = line[4:-1] value = self._strip_epoch(line[4:-1])
if value.startswith('"') and value.endswith('"'): if value.startswith('"') and value.endswith('"'):
value = _sx_unescape(value[1:-1]) value = _sx_unescape(value[1:-1])
return value return value
if line.startswith("(error "): if line.startswith("(error "):
msg = line[7:-1] msg = self._strip_epoch(line[7:-1])
if msg.startswith('"') and msg.endswith('"'): if msg.startswith('"') and msg.endswith('"'):
msg = _sx_unescape(msg[1:-1]) msg = _sx_unescape(msg[1:-1])
raise OcamlSyncError(msg) raise OcamlSyncError(msg)

5
shared/sx/pages.py
View File

@@ -313,10 +313,7 @@ async def _eval_slot(expr: Any, env: dict, ctx: Any) -> str:
sx_text = _wrap_with_env(expr, env) sx_text = _wrap_with_env(expr, env)
service = ctx.get("_helper_service", "") if isinstance(ctx, dict) else "" service = ctx.get("_helper_service", "") if isinstance(ctx, dict) else ""
return await bridge.aser_slot(sx_text, ctx={"_helper_service": service}) return await bridge.aser_slot(sx_text, ctx={"_helper_service": service})
if os.environ.get("SX_USE_REF") == "1": from .async_eval import async_eval_slot_to_sx
from .ref.async_eval_ref import async_eval_slot_to_sx
else:
from .async_eval import async_eval_slot_to_sx
return await async_eval_slot_to_sx(expr, env, ctx) return await async_eval_slot_to_sx(expr, env, ctx)

19
shared/sx/parser.py
View File

@@ -38,10 +38,11 @@ def _resolve_sx_reader_macro(name: str):
If a file like z3.sx defines (define z3-translate ...), then #z3 is If a file like z3.sx defines (define z3-translate ...), then #z3 is
automatically available as a reader macro without any Python registration. automatically available as a reader macro without any Python registration.
Looks for {name}-translate as a Lambda in the component env. Looks for {name}-translate as a Lambda in the component env.
Uses the synchronous OCaml bridge (ocaml_sync) when available.
""" """
try: try:
from .jinja_bridge import get_component_env from .jinja_bridge import get_component_env
from .ref.sx_ref import trampoline as _trampoline, call_lambda as _call_lambda
from .types import Lambda from .types import Lambda
except ImportError: except ImportError:
return None return None
@@ -49,10 +50,18 @@ def _resolve_sx_reader_macro(name: str):
fn = env.get(f"{name}-translate") fn = env.get(f"{name}-translate")
if fn is None or not isinstance(fn, Lambda): if fn is None or not isinstance(fn, Lambda):
return None return None
# Return a Python callable that invokes the SX lambda # Use sync OCaml bridge to invoke the lambda
def _sx_handler(expr): try:
return _trampoline(_call_lambda(fn, [expr], env)) from .ocaml_sync import OcamlSync
return _sx_handler _sync = OcamlSync()
_sync.start()
def _sx_handler(expr):
from .parser import serialize as _ser
result = _sync.eval(f"({name}-translate {_ser(expr)})")
return parse(result) if result else expr
return _sx_handler
except Exception:
return None
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------

66
shared/sx/primitives.py
View File

@@ -579,26 +579,54 @@ def prim_json_encode(value) -> str:
# (shared global state between transpiled and hand-written evaluators) # (shared global state between transpiled and hand-written evaluators)
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
def _lazy_scope_primitives(): def _register_scope_primitives():
"""Register scope/provide/collect primitives from sx_ref.py. """Register scope/provide/collect primitive stubs.
Called at import time — if sx_ref.py isn't built yet, silently skip. The OCaml kernel provides the real implementations. These stubs exist
These are needed by the hand-written _aser in async_eval.py when so _PRIMITIVES contains the names for dependency analysis, and so
expanding components that use scoped effects (e.g. ~cssx/flush). any Python-side code that checks for their existence finds them.
""" """
try: import threading
from .ref.sx_ref import ( _scope_data = threading.local()
sx_collect, sx_collected, sx_clear_collected,
sx_emitted, sx_emit, sx_context,
)
_PRIMITIVES["collect!"] = sx_collect
_PRIMITIVES["collected"] = sx_collected
_PRIMITIVES["clear-collected!"] = sx_clear_collected
_PRIMITIVES["emitted"] = sx_emitted
_PRIMITIVES["emit!"] = sx_emit
_PRIMITIVES["context"] = sx_context
except ImportError:
pass
_lazy_scope_primitives() def _collect(channel, value):
if not hasattr(_scope_data, 'collected'):
_scope_data.collected = {}
_scope_data.collected.setdefault(channel, []).append(value)
return NIL
def _collected(channel):
if not hasattr(_scope_data, 'collected'):
return []
return list(_scope_data.collected.get(channel, []))
def _clear_collected(channel):
if hasattr(_scope_data, 'collected'):
_scope_data.collected.pop(channel, None)
return NIL
def _emit(channel, value):
if not hasattr(_scope_data, 'emitted'):
_scope_data.emitted = {}
_scope_data.emitted.setdefault(channel, []).append(value)
return NIL
def _emitted(channel):
if not hasattr(_scope_data, 'emitted'):
return []
return list(_scope_data.emitted.get(channel, []))
def _context(key):
if not hasattr(_scope_data, 'context'):
return NIL
return _scope_data.context.get(key, NIL) if isinstance(_scope_data.context, dict) else NIL
_PRIMITIVES["collect!"] = _collect
_PRIMITIVES["collected"] = _collected
_PRIMITIVES["clear-collected!"] = _clear_collected
_PRIMITIVES["emitted"] = _emitted
_PRIMITIVES["emit!"] = _emit
_PRIMITIVES["context"] = _context
_register_scope_primitives()

10
shared/sx/query_executor.py
View File

@@ -49,10 +49,7 @@ async def execute_query(query_def: QueryDef, params: dict[str, str]) -> Any:
result = None result = None
return _normalize(result) return _normalize(result)
if os.environ.get("SX_USE_REF") == "1": from .async_eval import async_eval
from .ref.async_eval_ref import async_eval
else:
from .async_eval import async_eval
ctx = _get_request_context() ctx = _get_request_context()
result = await async_eval(query_def.body, env, ctx) result = await async_eval(query_def.body, env, ctx)
@@ -91,10 +88,7 @@ async def execute_action(action_def: ActionDef, payload: dict[str, Any]) -> Any:
result = None result = None
return _normalize(result) return _normalize(result)
if os.environ.get("SX_USE_REF") == "1": from .async_eval import async_eval
from .ref.async_eval_ref import async_eval
else:
from .async_eval import async_eval
ctx = _get_request_context() ctx = _get_request_context()
result = await async_eval(action_def.body, env, ctx) result = await async_eval(action_def.body, env, ctx)

2
shared/sx/templates/cssx.sx
View File

@@ -468,7 +468,7 @@
;; (div (~cssx/tw "bg-red-500") (~cssx/tw "p-4") "content") ;; (div (~cssx/tw "bg-red-500") (~cssx/tw "p-4") "content")
;; ========================================================================= ;; =========================================================================
(defcomp ~cssx/tw (tokens) (defcomp ~cssx/tw (&key tokens)
(let ((token-list (filter (fn (t) (not (= t ""))) (let ((token-list (filter (fn (t) (not (= t "")))
(split (or tokens "") " "))) (split (or tokens "") " ")))
(results (map cssx-process-token token-list)) (results (map cssx-process-token token-list))

558
shared/sx/tests/test_post_removal_bugs.py Normal file
View File

@@ -0,0 +1,558 @@
"""Tests exposing bugs after sx_ref.py removal.
These tests document all known breakages from removing the Python SX evaluator.
Each test targets a specific codepath that was depending on sx_ref.py and is now
broken.
Usage:
pytest shared/sx/tests/test_post_removal_bugs.py -v
"""
import asyncio
import os
import sys
import unittest
_project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../.."))
if _project_root not in sys.path:
sys.path.insert(0, _project_root)
from shared.sx.parser import parse, parse_all, serialize
from shared.sx.types import Component, Symbol, Keyword, NIL
# ---------------------------------------------------------------------------
# Helper: load shared components fresh (no cache)
# ---------------------------------------------------------------------------
def _load_components_fresh():
"""Load shared components, clearing cache to force re-parse."""
from shared.sx.jinja_bridge import _COMPONENT_ENV
_COMPONENT_ENV.clear()
from shared.sx.components import load_shared_components
load_shared_components()
return _COMPONENT_ENV
# ===========================================================================
# 1. register_components() loses all parameter information
# ===========================================================================
class TestComponentRegistration(unittest.TestCase):
"""register_components() hardcodes params=[] and has_children=False
for every component, losing all parameter metadata."""
@classmethod
def setUpClass(cls):
cls.env = _load_components_fresh()
def test_shell_component_should_have_params(self):
"""~shared:shell/sx-page-shell has 17+ &key params but gets params=[]."""
comp = self.env.get("~shared:shell/sx-page-shell")
self.assertIsNotNone(comp, "Shell component not found")
self.assertIsInstance(comp, Component)
# BUG: params is [] — should include title, meta-html, csrf, etc.
self.assertGreater(
len(comp.params), 0,
f"Shell component has params={comp.params} — expected 17+ keyword params"
)
def test_cssx_tw_should_have_tokens_param(self):
"""~cssx/tw needs a 'tokens' parameter."""
comp = self.env.get("~cssx/tw")
self.assertIsNotNone(comp, "~cssx/tw component not found")
self.assertIn(
"tokens", comp.params,
f"~cssx/tw has params={comp.params} — expected 'tokens'"
)
def test_cart_mini_should_have_params(self):
"""~shared:fragments/cart-mini has &key params."""
comp = self.env.get("~shared:fragments/cart-mini")
self.assertIsNotNone(comp, "cart-mini component not found")
self.assertGreater(
len(comp.params), 0,
f"cart-mini has params={comp.params} — expected keyword params"
)
def test_has_children_flag(self):
"""Components with &rest children should have has_children=True."""
comp = self.env.get("~shared:shell/sx-page-shell")
self.assertIsNotNone(comp)
# Many components accept children but has_children is always False
# Check any component that is known to accept &rest children
# e.g. a layout component
for name, val in self.env.items():
if isinstance(val, Component):
# Every component has has_children=False — at least some should be True
pass
# Count how many have has_children=True
with_children = sum(
1 for v in self.env.values()
if isinstance(v, Component) and v.has_children
)
total = sum(1 for v in self.env.values() if isinstance(v, Component))
# BUG: with_children is 0 — at least some components accept children
self.assertGreater(
with_children, 0,
f"0/{total} components have has_children=True — at least some should"
)
def test_all_components_have_empty_params(self):
"""Show the scale of the bug — every single component has params=[]."""
components_with_params = []
components_without = []
for name, val in self.env.items():
if isinstance(val, Component):
if val.params:
components_with_params.append(name)
else:
components_without.append(name)
# BUG: ALL components have empty params
self.assertGreater(
len(components_with_params), 0,
f"ALL {len(components_without)} components have params=[] — none have parameters parsed"
)
# ===========================================================================
# 2. Sync html.py rendering is completely broken
# ===========================================================================
class TestSyncHtmlRendering(unittest.TestCase):
"""html.py render() stubs _raw_eval/_trampoline — any evaluation crashes."""
def test_html_render_simple_element(self):
"""Even simple elements with keyword attrs need _eval, which is stubbed."""
from shared.sx.html import render
# This should work — (div "hello") needs no eval
result = render(parse('(div "hello")'), {})
self.assertIn("hello", result)
def test_html_render_with_keyword_attr(self):
"""Keyword attrs go through _eval, which raises RuntimeError."""
from shared.sx.html import render
try:
result = render(parse('(div :class "test" "hello")'), {})
# If it works, great
self.assertIn("test", result)
except RuntimeError as e:
self.assertIn("sx_ref.py has been removed", str(e))
self.fail(f"html.py render crashes on keyword attrs: {e}")
def test_html_render_symbol_lookup(self):
"""Symbol lookup goes through _eval, which is stubbed."""
from shared.sx.html import render
try:
result = render(parse('(div title)'), {"title": "Hello"})
self.assertIn("Hello", result)
except RuntimeError as e:
self.assertIn("sx_ref.py has been removed", str(e))
self.fail(f"html.py render crashes on symbol lookup: {e}")
def test_html_render_component(self):
"""Component rendering needs _eval for kwarg evaluation."""
from shared.sx.html import render
env = _load_components_fresh()
try:
result = render(
parse('(~shared:fragments/cart-mini :cart-count 0 :blog-url "" :cart-url "")'),
env,
)
self.assertIn("cart-mini", result)
except RuntimeError as e:
self.assertIn("sx_ref.py has been removed", str(e))
self.fail(f"html.py render crashes on component calls: {e}")
def test_sx_jinja_function_broken(self):
"""The sx() Jinja helper is broken — it uses html_render internally."""
from shared.sx.jinja_bridge import sx
env = _load_components_fresh()
try:
result = sx('(div "hello")')
self.assertIn("hello", result)
except RuntimeError as e:
self.assertIn("sx_ref.py has been removed", str(e))
self.fail(f"sx() Jinja function is broken: {e}")
# ===========================================================================
# 3. Async render_to_html uses Python path, not OCaml
# ===========================================================================
class TestAsyncRenderToHtml(unittest.IsolatedAsyncioTestCase):
"""helpers.py render_to_html() deliberately uses Python async_eval,
not the OCaml bridge. But Python eval is now broken."""
async def test_render_to_html_uses_python_path(self):
"""render_to_html goes through async_render, not OCaml bridge."""
from shared.sx.helpers import render_to_html
env = _load_components_fresh()
# The shell component has many &key params — none are bound because params=[]
try:
html = await render_to_html(
"shared:shell/sx-page-shell",
title="Test", csrf="abc", asset_url="/static",
sx_js_hash="abc123",
)
self.assertIn("Test", html)
except Exception as e:
# Expected: either RuntimeError from stubs or EvalError from undefined symbols
self.fail(
f"render_to_html (Python path) failed: {type(e).__name__}: {e}\n"
f"This should go through OCaml bridge instead"
)
async def test_async_render_component_no_params_bound(self):
"""async_eval.py _arender_component can't bind params because comp.params=[]."""
from shared.sx.async_eval import async_render
from shared.sx.primitives_io import RequestContext
env = _load_components_fresh()
# Create a simple component manually with correct params
test_comp = Component(
name="test/greeting",
params=["name"],
has_children=False,
body=parse('(div (str "Hello " name))'),
)
env["~test/greeting"] = test_comp
try:
result = await async_render(
parse('(~test/greeting :name "World")'),
env,
RequestContext(),
)
self.assertIn("Hello World", result)
except Exception as e:
self.fail(
f"async_render failed even with correct params: {type(e).__name__}: {e}"
)
# ===========================================================================
# 4. Dead imports from removed sx_ref.py
# ===========================================================================
class TestDeadImports(unittest.TestCase):
"""Files that import from sx_ref.py will crash when their codepaths execute."""
def test_async_eval_defcomp(self):
"""async_eval.py _asf_defcomp should work as a stub (no sx_ref import)."""
from shared.sx.async_eval import _asf_defcomp
env = {}
asyncio.run(_asf_defcomp(
[Symbol("defcomp"), Symbol("~test"), [], [Symbol("div")]],
env, None
))
# Should register a minimal component in env
self.assertIn("~test", env)
def test_async_eval_defmacro(self):
"""async_eval.py _asf_defmacro should work as a stub (no sx_ref import)."""
from shared.sx.async_eval import _asf_defmacro
env = {}
asyncio.run(_asf_defmacro(
[Symbol("defmacro"), Symbol("test"), [], [Symbol("div")]],
env, None
))
self.assertIn("test", env)
def test_async_eval_defstyle(self):
"""async_eval.py _asf_defstyle should be a no-op (no sx_ref import)."""
from shared.sx.async_eval import _asf_defstyle
result = asyncio.run(_asf_defstyle(
[Symbol("defstyle"), Symbol("test"), [], [Symbol("div")]],
{}, None
))
# Should return NIL without crashing
self.assertIsNotNone(result)
def test_async_eval_defhandler(self):
"""async_eval.py _asf_defhandler should be a no-op (no sx_ref import)."""
from shared.sx.async_eval import _asf_defhandler
result = asyncio.run(_asf_defhandler(
[Symbol("defhandler"), Symbol("test"), [], [Symbol("div")]],
{}, None
))
self.assertIsNotNone(result)
def test_async_eval_continuation_reset(self):
"""async_eval.py _asf_reset imports eval_expr/trampoline from sx_ref."""
# The cont_fn inside _asf_reset will crash when invoked
from shared.sx.async_eval import _ASYNC_RENDER_FORMS
reset_fn = _ASYNC_RENDER_FORMS.get("reset")
# reset is defined in async_eval — the import is deferred to execution
# Just verify the module doesn't have the import available
try:
from shared.sx.ref.sx_ref import eval_expr
self.fail("sx_ref.py should not exist")
except (ImportError, ModuleNotFoundError):
pass # Expected
def test_ocaml_bridge_jit_compile(self):
"""ocaml_bridge.py _compile_adapter_module imports from sx_ref."""
try:
from shared.sx.ref.sx_ref import eval_expr, trampoline, PRIMITIVES
self.fail("sx_ref.py should not exist — JIT compilation path is broken")
except (ImportError, ModuleNotFoundError):
pass # Expected — confirms the bug
def test_parser_reader_macro(self):
"""parser.py _try_reader_macro imports trampoline/call_lambda from sx_ref."""
try:
from shared.sx.ref.sx_ref import trampoline, call_lambda
self.fail("sx_ref.py should not exist — reader macros are broken")
except (ImportError, ModuleNotFoundError):
pass # Expected — confirms the bug
def test_primitives_scope_prims(self):
"""primitives.py _lazy_scope_primitives silently fails to load scope prims."""
from shared.sx.primitives import _PRIMITIVES
# collect!, collected, clear-collected!, emitted, emit!, context
# These are needed for CSSX but the import from sx_ref silently fails
missing = []
for name in ("collect!", "collected", "clear-collected!", "emitted", "emit!", "context"):
if name not in _PRIMITIVES:
missing.append(name)
if missing:
self.fail(
f"Scope primitives missing from _PRIMITIVES (sx_ref import failed silently): {missing}\n"
f"CSSX components depend on these for collect!/collected"
)
def test_deps_transitive_deps_ref_path(self):
"""deps.py transitive_deps imports from sx_ref when SX_USE_REF=1."""
# The fallback path should still work
from shared.sx.deps import transitive_deps
env = _load_components_fresh()
# Should work via fallback, not crash
try:
result = transitive_deps("~cssx/tw", env)
self.assertIsInstance(result, set)
except (ImportError, ModuleNotFoundError) as e:
self.fail(f"transitive_deps crashed: {e}")
def test_handlers_python_fallback(self):
"""handlers.py eval_handler Python fallback imports async_eval_ref."""
# When not using OCaml, handler evaluation falls through to async_eval
# The ref path (SX_USE_REF=1) would crash
try:
from shared.sx.ref.async_eval_ref import async_eval_to_sx
self.fail("async_eval_ref.py should not exist")
except (ImportError, ModuleNotFoundError):
pass # Expected
# ===========================================================================
# 5. ~cssx/tw signature mismatch
# ===========================================================================
class TestCssxTwSignature(unittest.TestCase):
"""~cssx/tw changed from (&key tokens) to (tokens) positional,
but callers use :tokens keyword syntax."""
@classmethod
def setUpClass(cls):
cls.env = _load_components_fresh()
def test_cssx_tw_source_uses_positional(self):
"""Verify the current source has positional (tokens) not (&key tokens)."""
import os
cssx_path = os.path.join(
os.path.dirname(__file__), "..", "templates", "cssx.sx"
)
with open(cssx_path) as f:
source = f.read()
# Check if it's positional or keyword
if "(defcomp ~cssx/tw (tokens)" in source:
# Positional — callers using :tokens will break
self.fail(
"~cssx/tw uses positional (tokens) but callers use :tokens keyword syntax.\n"
"Should be: (defcomp ~cssx/tw (&key tokens) ...)"
)
elif "(defcomp ~cssx/tw (&key tokens)" in source:
pass # Correct
else:
# Unknown signature
for line in source.split("\n"):
if "defcomp ~cssx/tw" in line:
self.fail(f"Unexpected ~cssx/tw signature: {line.strip()}")
def test_cssx_tw_callers_use_keyword(self):
"""Scan for callers that use :tokens keyword syntax."""
import glob as glob_mod
sx_dir = os.path.join(os.path.dirname(__file__), "../../..")
keyword_callers = []
positional_callers = []
for fp in glob_mod.glob(os.path.join(sx_dir, "**/*.sx"), recursive=True):
try:
with open(fp) as f:
content = f.read()
except Exception:
continue
if "~cssx/tw" not in content:
continue
for line_no, line in enumerate(content.split("\n"), 1):
if "~cssx/tw" in line and "defcomp" not in line:
if ":tokens" in line:
keyword_callers.append(f"{fp}:{line_no}")
elif "(~cssx/tw " in line:
positional_callers.append(f"{fp}:{line_no}")
if keyword_callers:
# If signature is positional but callers use :tokens, that's a bug
import os as os_mod
cssx_path = os.path.join(
os.path.dirname(__file__), "..", "templates", "cssx.sx"
)
with open(cssx_path) as f:
source = f.read()
if "(defcomp ~cssx/tw (tokens)" in source:
self.fail(
f"~cssx/tw uses positional params but {len(keyword_callers)} callers use :tokens:\n"
+ "\n".join(keyword_callers[:5])
)
# ===========================================================================
# 6. OCaml bridge rendering (should work — this is the good path)
# ===========================================================================
class TestOcamlBridgeRendering(unittest.IsolatedAsyncioTestCase):
"""The OCaml bridge should handle all rendering correctly."""
@classmethod
def setUpClass(cls):
from shared.sx.ocaml_bridge import _DEFAULT_BIN
bin_path = os.path.abspath(_DEFAULT_BIN)
if not os.path.isfile(bin_path):
raise unittest.SkipTest("OCaml binary not found")
async def asyncSetUp(self):
from shared.sx.ocaml_bridge import OcamlBridge
self.bridge = OcamlBridge()
await self.bridge.start()
async def asyncTearDown(self):
if hasattr(self, 'bridge'):
await self.bridge.stop()
async def test_simple_element(self):
result = await self.bridge.render('(div "hello")')
self.assertIn("hello", result)
async def test_element_with_keyword_attrs(self):
result = await self.bridge.render('(div :class "test" "hello")')
self.assertIn('class="test"', result)
self.assertIn("hello", result)
async def test_component_with_params(self):
"""OCaml should handle component parameter binding correctly."""
# Use load_source to define a component (bypasses _ensure_components lock)
await self.bridge.load_source('(defcomp ~test/greet (&key name) (div (str "Hello " name)))')
result = await self.bridge.render('(~test/greet :name "World")')
self.assertIn("Hello World", result)
async def test_let_binding(self):
result = await self.bridge.render('(let ((x "hello")) (div x))')
self.assertIn("hello", result)
async def test_conditional(self):
result = await self.bridge.render('(if true (div "yes") (div "no"))')
self.assertIn("yes", result)
self.assertNotIn("no", result)
async def test_cssx_tw_keyword_call(self):
"""Test that ~cssx/tw works when called with :tokens keyword.
Components are loaded by _ensure_components() automatically."""
try:
result = await self.bridge.render('(div (~cssx/tw :tokens "bg-red-500") "content")')
# Should produce a spread with CSS class, not an error
self.assertNotIn("error", result.lower())
except Exception as e:
self.fail(f"~cssx/tw :tokens keyword call failed: {e}")
async def test_cssx_tw_positional_call(self):
"""Test that ~cssx/tw works when called positionally."""
try:
result = await self.bridge.render('(div (~cssx/tw "bg-red-500") "content")')
self.assertNotIn("error", result.lower())
except Exception as e:
self.fail(f"~cssx/tw positional call failed: {e}")
async def test_repeated_renders_dont_crash(self):
"""Verify OCaml bridge handles multiple sequential renders."""
for i in range(5):
result = await self.bridge.render(f'(div "iter-{i}")')
self.assertIn(f"iter-{i}", result)
# ===========================================================================
# 7. Scope primitives missing (collect!, collected, etc.)
# ===========================================================================
class TestScopePrimitives(unittest.TestCase):
"""Scope primitives needed by CSSX are missing because the import
from sx_ref.py silently fails."""
def test_python_primitives_have_scope_ops(self):
"""Check that collect!/collected/etc. are in _PRIMITIVES."""
from shared.sx.primitives import _PRIMITIVES
required = ["collect!", "collected", "clear-collected!",
"emitted", "emit!", "context"]
missing = [p for p in required if p not in _PRIMITIVES]
if missing:
self.fail(
f"Missing Python-side scope primitives: {missing}\n"
f"These were provided by sx_ref.py — need OCaml bridge or Python stubs"
)
# ===========================================================================
# 8. Query executor fallback path
# ===========================================================================
class TestQueryExecutorFallback(unittest.TestCase):
"""query_executor.py imports async_eval for its fallback path."""
def test_query_executor_import(self):
"""query_executor can be imported without crashing."""
try:
import shared.sx.query_executor
except Exception as e:
self.fail(f"query_executor import crashed: {e}")
# ===========================================================================
# 9. End-to-end: sx_page shell rendering
# ===========================================================================
class TestShellRendering(unittest.IsolatedAsyncioTestCase):
"""The shell template needs to render through some path that works."""
async def test_sx_page_shell_via_python(self):
"""render_to_html('shared:shell/sx-page-shell', ...) uses Python path.
This is the actual failure from the production error log."""
from shared.sx.helpers import render_to_html
_load_components_fresh()
try:
html = await render_to_html(
"shared:shell/sx-page-shell",
title="Test Page",
csrf="test-csrf",
asset_url="/static",
sx_js_hash="abc",
)
# Should produce full HTML document
self.assertIn("<!doctype html>", html.lower())
self.assertIn("Test Page", html)
except Exception as e:
self.fail(
f"Shell rendering via Python path failed: {type(e).__name__}: {e}\n"
f"This is the exact error seen in production — "
f"render_to_html should use OCaml bridge"
)
if __name__ == "__main__":
unittest.main()