Complete Python eval removal: epoch protocol, scope consolidation, JIT fixes

Route all rendering through OCaml bridge — render_to_html no longer uses
Python async_eval. Fix register_components to parse &key params and &rest
children from defcomp forms. Remove all dead sx_ref.py imports.

Epoch protocol (prevents pipe desync):
- Every command prefixed with (epoch N), all responses tagged with epoch
- Both sides discard stale-epoch messages — desync structurally impossible
- OCaml main loop discards stale io-responses between commands

Consolidate scope primitives into sx_scope.ml:
- Single source of truth for scope-push!/pop!/peek, collect!/collected,
  emit!/emitted, context, and 12 other scope operations
- Removes duplicate registrations from sx_server.ml (including bugs where
  scope-emit! and clear-collected! were registered twice with different impls)
- Bind scope prims into env so JIT VM finds them via OP_GLOBAL_GET

JIT VM fixes:
- Trampoline thunks before passing args to CALL_PRIM
- as_list resolves thunks via _sx_trampoline_fn
- len handles all value types (Bool, Number, RawHTML, SxExpr, Spread, etc.)

Other fixes:
- ~cssx/tw signature: (tokens) → (&key tokens) to match callers
- Minimal Python evaluator in html.py for sync sx() Jinja function
- Python scope primitive stubs (thread-local) for non-OCaml paths
- Reader macro resolution via OcamlSync instead of sx_ref.py

Tests: 1114 OCaml, 1078 JS, 35 Python regression, 6/6 Playwright SSR

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-03-24 16:14:40 +00:00
parent e887c0d978
commit f9f810ffd7
18 changed files with 1305 additions and 478 deletions

View File

@@ -14,7 +14,7 @@
// =========================================================================
var NIL = Object.freeze({ _nil: true, toString: function() { return "nil"; } });
var SX_VERSION = "2026-03-24T14:17:24Z";
var SX_VERSION = "2026-03-24T15:37:52Z";
function isNil(x) { return x === NIL || x === null || x === undefined; }
function isSxTruthy(x) { return x !== false && !isNil(x); }

View File

@@ -423,23 +423,39 @@ async def _asf_define(expr, env, ctx):
async def _asf_defcomp(expr, env, ctx):
from .ref.sx_ref import sf_defcomp
return sf_defcomp(expr[1:], env)
# Component definitions are handled by OCaml kernel at load time.
# Python-side: just store a minimal Component in env for reference.
from .types import Component
name_sym = expr[1]
name = name_sym.name if hasattr(name_sym, 'name') else str(name_sym)
env[name] = Component(name=name.lstrip("~"), params=[], has_children=False,
body=expr[-1], closure={})
return NIL
async def _asf_defstyle(expr, env, ctx):
from .ref.sx_ref import sf_defstyle
return sf_defstyle(expr[1:], env)
# Style definitions handled by OCaml kernel.
return NIL
async def _asf_defmacro(expr, env, ctx):
from .ref.sx_ref import sf_defmacro
return sf_defmacro(expr[1:], env)
# Macro definitions handled by OCaml kernel.
from .types import Macro
name_sym = expr[1]
name = name_sym.name if hasattr(name_sym, 'name') else str(name_sym)
params_form = expr[2] if len(expr) > 3 else []
param_names = [p.name for p in params_form if isinstance(p, Symbol) and not p.name.startswith("&")]
rest_param = None
for i, p in enumerate(params_form):
if isinstance(p, Symbol) and p.name == "&rest" and i + 1 < len(params_form):
rest_param = params_form[i + 1].name if isinstance(params_form[i + 1], Symbol) else None
env[name] = Macro(name=name, params=param_names, rest_param=rest_param, body=expr[-1])
return NIL
async def _asf_defhandler(expr, env, ctx):
from .ref.sx_ref import sf_defhandler
return sf_defhandler(expr[1:], env)
# Handler definitions handled by OCaml kernel.
return NIL
async def _asf_begin(expr, env, ctx):
@@ -601,9 +617,12 @@ async def _asf_reset(expr, env, ctx):
from .types import NIL
_ASYNC_RESET_RESUME.append(value if value is not None else NIL)
try:
# Sync re-evaluation; the async caller will trampoline
from .ref.sx_ref import eval_expr as sync_eval, trampoline as _trampoline
return _trampoline(sync_eval(body, env))
# Continuations are handled by OCaml kernel.
# Python-side cont_fn should not be called in normal operation.
raise RuntimeError(
"Python-side continuation invocation not supported — "
"use OCaml bridge for shift/reset"
)
finally:
_ASYNC_RESET_RESUME.pop()
k = Continuation(cont_fn)

View File

@@ -152,18 +152,11 @@ def transitive_deps(name: str, env: dict[str, Any]) -> set[str]:
Returns the set of all component names (with ~ prefix) that
*name* can transitively render, NOT including *name* itself.
"""
if _use_ref():
from .ref.sx_ref import transitive_deps as _ref_td
return set(_ref_td(name, env))
return _transitive_deps_fallback(name, env)
def compute_all_deps(env: dict[str, Any]) -> None:
"""Compute and cache deps for all Component entries in *env*."""
if _use_ref():
from .ref.sx_ref import compute_all_deps as _ref_cad
_ref_cad(env)
return
_compute_all_deps_fallback(env)
@@ -172,9 +165,6 @@ def scan_components_from_sx(source: str) -> set[str]:
Returns names with ~ prefix, e.g. {"~card", "~shared:layout/nav-link"}.
"""
if _use_ref():
from .ref.sx_ref import scan_components_from_source as _ref_sc
return set(_ref_sc(source))
return _scan_components_from_sx_fallback(source)
@@ -183,18 +173,11 @@ def components_needed(page_sx: str, env: dict[str, Any]) -> set[str]:
Returns names with ~ prefix.
"""
if _use_ref():
from .ref.sx_ref import components_needed as _ref_cn
return set(_ref_cn(page_sx, env))
return _components_needed_fallback(page_sx, env)
def compute_all_io_refs(env: dict[str, Any], io_names: set[str]) -> None:
"""Compute and cache transitive IO refs for all Component entries in *env*."""
if _use_ref():
from .ref.sx_ref import compute_all_io_refs as _ref_cio
_ref_cio(env, list(io_names))
return
_compute_all_io_refs_fallback(env, io_names)

View File

@@ -219,11 +219,8 @@ async def execute_handler(
result_sx = await bridge.aser(sx_text, ctx=ocaml_ctx)
return SxExpr(result_sx or "")
else:
# Python fallback
if os.environ.get("SX_USE_REF") == "1":
from .ref.async_eval_ref import async_eval_to_sx
else:
from .async_eval import async_eval_to_sx
# Python fallback (async_eval)
from .async_eval import async_eval_to_sx
env = dict(get_component_env())
env.update(get_page_helpers(service_name))

View File

@@ -385,10 +385,7 @@ async def _render_to_sx_with_env(__name: str, extra_env: dict, **kwargs: Any) ->
ocaml_ctx = {"_helper_service": _get_request_context().get("_helper_service", "")} if isinstance(_get_request_context(), dict) else {}
return SxExpr(await bridge.aser_slot(sx_text, ctx=ocaml_ctx))
if os.environ.get("SX_USE_REF") == "1":
from .ref.async_eval_ref import async_eval_slot_to_sx
else:
from .async_eval import async_eval_slot_to_sx
from .async_eval import async_eval_slot_to_sx
env = dict(get_component_env())
env.update(extra_env)
@@ -421,10 +418,7 @@ async def _render_to_sx(__name: str, **kwargs: Any) -> str:
# symbols like `title` that were bound during the earlier expansion.
return SxExpr(await bridge.aser_slot(sx_text))
if os.environ.get("SX_USE_REF") == "1":
from .ref.async_eval_ref import async_eval_to_sx
else:
from .async_eval import async_eval_to_sx
from .async_eval import async_eval_to_sx
env = dict(get_component_env())
ctx = _get_request_context()
@@ -442,15 +436,23 @@ async def render_to_html(__name: str, **kwargs: Any) -> str:
Same as render_to_sx() but produces HTML output instead of SX wire
format. Used by route renders that need HTML (full pages, fragments).
Note: does NOT use OCaml bridge — the shell render is a pure HTML
template with no IO, so the Python renderer handles it reliably.
The OCaml path is used for _render_to_sx and _eval_slot (IO-heavy).
Routes through the OCaml bridge (render mode) which handles component
parameter binding, scope primitives, and all evaluation.
"""
from .jinja_bridge import get_component_env, _get_request_context
import os
from .async_eval import async_render
ast = _build_component_ast(__name, **kwargs)
if os.environ.get("SX_USE_OCAML") == "1":
from .ocaml_bridge import get_bridge
from .parser import serialize
bridge = await get_bridge()
sx_text = serialize(ast)
return await bridge.render(sx_text)
# Fallback: Python async_eval (requires working evaluator)
from .jinja_bridge import get_component_env, _get_request_context
from .async_eval import async_render
env = dict(get_component_env())
ctx = _get_request_context()
return await async_render(ast, env, ctx)

View File

@@ -28,20 +28,163 @@ import contextvars
from typing import Any
from .types import Component, Island, Keyword, Lambda, Macro, NIL, Symbol
# sx_ref.py removed — these stubs exist so the module loads.
# With SX_USE_OCAML=1, rendering goes through the OCaml bridge; these
# are only called if a service falls back to Python-side rendering.
def _not_available(*a, **kw):
raise RuntimeError("sx_ref.py has been removed — use SX_USE_OCAML=1")
_raw_eval = _raw_call_component = _expand_macro = _trampoline = _not_available
def _eval(expr, env):
"""Evaluate and unwrap thunks — all html.py _eval calls are non-tail."""
return _trampoline(_raw_eval(expr, env))
"""Minimal Python evaluator for sync html.py rendering.
def _call_component(comp, raw_args, env):
"""Call component and unwrap thunks — non-tail in html.py."""
return _trampoline(_raw_call_component(comp, raw_args, env))
Handles: literals, symbols, keywords, dicts, special forms (if, when,
cond, let, begin/do, and, or, str, not, list), lambda calls, and
primitive function calls. Enough for the sync sx() Jinja function.
"""
from .primitives import _PRIMITIVES
# Literals
if isinstance(expr, (int, float, str, bool)):
return expr
if expr is None or expr is NIL:
return NIL
# Symbol lookup
if isinstance(expr, Symbol):
name = expr.name
if name in env:
return env[name]
if name in _PRIMITIVES:
return _PRIMITIVES[name]
if name == "true":
return True
if name == "false":
return False
if name == "nil":
return NIL
from .types import EvalError
raise EvalError(f"Undefined symbol: {name}")
# Keyword
if isinstance(expr, Keyword):
return expr.name
# Dict
if isinstance(expr, dict):
return {k: _eval(v, env) for k, v in expr.items()}
# List — dispatch
if not isinstance(expr, list):
return expr
if not expr:
return []
head = expr[0]
if isinstance(head, Symbol):
name = head.name
# Special forms
if name == "if":
cond = _eval(expr[1], env)
if cond and cond is not NIL:
return _eval(expr[2], env)
return _eval(expr[3], env) if len(expr) > 3 else NIL
if name == "when":
cond = _eval(expr[1], env)
if cond and cond is not NIL:
result = NIL
for body in expr[2:]:
result = _eval(body, env)
return result
return NIL
if name == "let":
bindings = expr[1]
local = dict(env)
if isinstance(bindings, list):
if bindings and isinstance(bindings[0], list):
for b in bindings:
vname = b[0].name if isinstance(b[0], Symbol) else b[0]
local[vname] = _eval(b[1], local)
elif len(bindings) % 2 == 0:
for i in range(0, len(bindings), 2):
vname = bindings[i].name if isinstance(bindings[i], Symbol) else bindings[i]
local[vname] = _eval(bindings[i + 1], local)
result = NIL
for body in expr[2:]:
result = _eval(body, local)
return result
if name in ("begin", "do"):
result = NIL
for body in expr[1:]:
result = _eval(body, env)
return result
if name == "and":
result = True
for arg in expr[1:]:
result = _eval(arg, env)
if not result or result is NIL:
return result
return result
if name == "or":
for arg in expr[1:]:
result = _eval(arg, env)
if result and result is not NIL:
return result
return NIL
if name == "not":
val = _eval(expr[1], env)
return val is NIL or val is False or val is None
if name == "lambda" or name == "fn":
params_form = expr[1]
param_names = [p.name if isinstance(p, Symbol) else str(p) for p in params_form]
return Lambda(params=param_names, body=expr[2], closure=dict(env))
if name == "define":
var_name = expr[1].name if isinstance(expr[1], Symbol) else str(expr[1])
env[var_name] = _eval(expr[2], env)
return NIL
if name == "quote":
return expr[1]
if name == "str":
parts = []
for arg in expr[1:]:
val = _eval(arg, env)
if val is NIL or val is None:
parts.append("")
else:
parts.append(str(val))
return "".join(parts)
if name == "list":
return [_eval(arg, env) for arg in expr[1:]]
# Primitive or function call
fn = _eval(head, env)
else:
fn = _eval(head, env)
# Evaluate args
args = [_eval(a, env) for a in expr[1:]]
# Call
if callable(fn):
return fn(*args)
if isinstance(fn, Lambda):
local = dict(fn.closure)
local.update(env)
for p, v in zip(fn.params, args):
local[p] = v
return _eval(fn.body, local)
return NIL
def _expand_macro(*a, **kw):
raise RuntimeError("Macro expansion requires OCaml bridge")
# ContextVar for collecting CSS class names during render.
# Set to a set[str] to collect; None to skip.

View File

@@ -46,6 +46,7 @@ class OcamlBridge:
self._components_loaded = False
self._helpers_injected = False
self._io_cache: dict[tuple, Any] = {} # (name, args...) → cached result
self._epoch: int = 0 # request epoch — monotonically increasing
async def start(self) -> None:
"""Launch the OCaml subprocess and wait for (ready)."""
@@ -77,7 +78,7 @@ class OcamlBridge:
self._started = True
# Verify engine identity
await self._send("(ping)")
await self._send_command("(ping)")
kind, engine = await self._read_response()
engine_name = engine if kind == "ok" else "unknown"
_logger.info("OCaml SX kernel ready (pid=%d, engine=%s)", self._proc.pid, engine_name)
@@ -95,24 +96,36 @@ class OcamlBridge:
self._proc = None
self._started = False
async def _restart(self) -> None:
"""Kill and restart the OCaml subprocess to recover from pipe desync."""
_logger.warning("Restarting OCaml SX kernel (pipe recovery)")
if self._proc and self._proc.returncode is None:
self._proc.kill()
await self._proc.wait()
self._proc = None
self._started = False
self._components_loaded = False
self._helpers_injected = False
await self.start()
async def ping(self) -> str:
"""Health check — returns engine name (e.g. 'ocaml-cek')."""
async with self._lock:
await self._send("(ping)")
await self._send_command("(ping)")
kind, value = await self._read_response()
return value or "" if kind == "ok" else ""
async def load(self, path: str) -> int:
"""Load an .sx file for side effects (defcomp, define, defmacro)."""
async with self._lock:
await self._send(f'(load "{_escape(path)}")')
await self._send_command(f'(load "{_escape(path)}")')
value = await self._read_until_ok(ctx=None)
return int(float(value)) if value else 0
async def load_source(self, source: str) -> int:
"""Evaluate SX source for side effects."""
async with self._lock:
await self._send(f'(load-source "{_escape(source)}")')
await self._send_command(f'(load-source "{_escape(source)}")')
value = await self._read_until_ok(ctx=None)
return int(float(value)) if value else 0
@@ -124,7 +137,7 @@ class OcamlBridge:
"""
await self._ensure_components()
async with self._lock:
await self._send('(eval-blob)')
await self._send_command('(eval-blob)')
await self._send_blob(source)
return await self._read_until_ok(ctx)
@@ -136,14 +149,14 @@ class OcamlBridge:
"""Render SX to HTML, handling io-requests via Python async IO."""
await self._ensure_components()
async with self._lock:
await self._send(f'(render "{_escape(source)}")')
await self._send_command(f'(render "{_escape(source)}")')
return await self._read_until_ok(ctx)
async def aser(self, source: str, ctx: dict[str, Any] | None = None) -> str:
"""Evaluate SX and return SX wire format, handling io-requests."""
await self._ensure_components()
async with self._lock:
await self._send('(aser-blob)')
await self._send_command('(aser-blob)')
await self._send_blob(source)
return await self._read_until_ok(ctx)
@@ -159,7 +172,7 @@ class OcamlBridge:
# a separate lock acquisition could let another coroutine
# interleave commands between injection and aser-slot.
await self._inject_helpers_locked()
await self._send('(aser-slot-blob)')
await self._send_command('(aser-slot-blob)')
await self._send_blob(source)
return await self._read_until_ok(ctx)
@@ -182,7 +195,7 @@ class OcamlBridge:
var = f"__shell-{key.replace('_', '-')}"
defn = f'(define {var} "{_escape(str(val))}")'
try:
await self._send(f'(load-source "{_escape(defn)}")')
await self._send_command(f'(load-source "{_escape(defn)}")')
await self._read_until_ok(ctx=None)
except OcamlBridgeError as e:
_logger.warning("Shell static inject failed for %s: %s", key, e)
@@ -198,7 +211,7 @@ class OcamlBridge:
else:
defn = f'(define {var} "{_escape(str(val))}")'
try:
await self._send(f'(load-source "{_escape(defn)}")')
await self._send_command(f'(load-source "{_escape(defn)}")')
await self._read_until_ok(ctx=None)
except OcamlBridgeError as e:
_logger.warning("Shell static inject failed for %s: %s", key, e)
@@ -221,7 +234,7 @@ class OcamlBridge:
if pairs:
cmd = f'(set-request-cookies {{{" ".join(pairs)}}})'
try:
await self._send(cmd)
await self._send_command(cmd)
await self._read_until_ok(ctx=None)
except OcamlBridgeError as e:
_logger.debug("Cookie inject failed: %s", e)
@@ -277,7 +290,7 @@ class OcamlBridge:
parts.append(f' :{k} "{_escape(str(val))}"')
parts.append(")")
cmd = "".join(parts)
await self._send(cmd)
await self._send_command(cmd)
# Send page source as binary blob (avoids string-escape issues)
await self._send_blob(page_source)
html = await self._read_until_ok(ctx)
@@ -312,7 +325,7 @@ class OcamlBridge:
arg_list = " ".join(chr(97 + i) for i in range(nargs))
sx_def = f'(define {name} (fn ({param_names}) (helper "{name}" {arg_list})))'
try:
await self._send(f'(load-source "{_escape(sx_def)}")')
await self._send_command(f'(load-source "{_escape(sx_def)}")')
await self._read_until_ok(ctx=None)
count += 1
except OcamlBridgeError:
@@ -325,70 +338,11 @@ class OcamlBridge:
async def _compile_adapter_module(self) -> None:
"""Compile adapter-sx.sx to bytecode and load as a VM module.
All aser functions become NativeFn VM closures in the kernel env.
Subsequent aser-slot calls find them as NativeFn → VM executes
the entire render path compiled, no CEK steps.
Previously used Python's sx_ref.py evaluator for compilation.
Now the OCaml kernel handles JIT compilation natively — this method
is a no-op. The kernel's own JIT hook compiles functions on first call.
"""
from .parser import parse_all, serialize
from .ref.sx_ref import eval_expr, trampoline, PRIMITIVES
# Ensure compiler primitives are available
if 'serialize' not in PRIMITIVES:
PRIMITIVES['serialize'] = lambda x: serialize(x)
if 'primitive?' not in PRIMITIVES:
PRIMITIVES['primitive?'] = lambda name: isinstance(name, str) and name in PRIMITIVES
if 'has-key?' not in PRIMITIVES:
PRIMITIVES['has-key?'] = lambda *a: isinstance(a[0], dict) and str(a[1]) in a[0]
if 'set-nth!' not in PRIMITIVES:
from .types import NIL
PRIMITIVES['set-nth!'] = lambda *a: (a[0].__setitem__(int(a[1]), a[2]), NIL)[-1]
if 'init' not in PRIMITIVES:
PRIMITIVES['init'] = lambda *a: a[0][:-1] if isinstance(a[0], list) else a[0]
if 'concat' not in PRIMITIVES:
PRIMITIVES['concat'] = lambda *a: (a[0] or []) + (a[1] or [])
if 'slice' not in PRIMITIVES:
PRIMITIVES['slice'] = lambda *a: a[0][int(a[1]):int(a[2])] if len(a) == 3 else a[0][int(a[1]):]
from .types import Symbol
if 'make-symbol' not in PRIMITIVES:
PRIMITIVES['make-symbol'] = lambda name: Symbol(name)
from .types import NIL
for ho in ['map', 'filter', 'for-each', 'reduce', 'some', 'every?', 'map-indexed']:
if ho not in PRIMITIVES:
PRIMITIVES[ho] = lambda *a: NIL
# Load compiler
compiler_env = {}
spec_dir = os.path.join(os.path.dirname(__file__), "../../spec")
for f in ["bytecode.sx", "compiler.sx"]:
path = os.path.join(spec_dir, f)
if os.path.isfile(path):
with open(path) as fh:
for expr in parse_all(fh.read()):
trampoline(eval_expr(expr, compiler_env))
# Compile adapter-sx.sx
web_dir = os.path.join(os.path.dirname(__file__), "../../web")
adapter_path = os.path.join(web_dir, "adapter-sx.sx")
if not os.path.isfile(adapter_path):
_logger.warning("adapter-sx.sx not found at %s", adapter_path)
return
with open(adapter_path) as f:
adapter_exprs = parse_all(f.read())
compiled = trampoline(eval_expr(
[Symbol('compile-module'), [Symbol('quote'), adapter_exprs]],
compiler_env))
code_sx = serialize(compiled)
_logger.info("Compiled adapter-sx.sx: %d bytes bytecode", len(code_sx))
# Load the compiled module into the OCaml VM
async with self._lock:
await self._send(f'(vm-load-module {code_sx})')
await self._read_until_ok(ctx=None)
_logger.info("Loaded adapter-sx.sx as VM module")
_logger.info("Adapter module compilation delegated to OCaml kernel JIT")
async def _ensure_components(self) -> None:
"""Load all .sx source files into the kernel on first use.
@@ -455,7 +409,7 @@ class OcamlBridge:
async with self._lock:
for filepath in all_files:
try:
await self._send(f'(load "{_escape(filepath)}")')
await self._send_command(f'(load "{_escape(filepath)}")')
value = await self._read_until_ok(ctx=None)
# Response may be a number (count) or a value — just count files
count += 1
@@ -468,14 +422,14 @@ class OcamlBridge:
# reactive loops during island SSR — effects are DOM side-effects)
try:
noop_dispose = '(fn () nil)'
await self._send(f'(load-source "(define effect (fn (f) {noop_dispose}))")')
await self._send_command(f'(load-source "(define effect (fn (f) {noop_dispose}))")')
await self._read_until_ok(ctx=None)
except OcamlBridgeError:
pass
# Register JIT hook — lambdas compile on first call
try:
await self._send('(vm-compile-adapter)')
await self._send_command('(vm-compile-adapter)')
await self._read_until_ok(ctx=None)
_logger.info("JIT hook registered — lambdas compile on first call")
except OcamlBridgeError as e:
@@ -499,7 +453,7 @@ class OcamlBridge:
if callable(fn) and not name.startswith("~"):
sx_def = f'(define {name} (fn (&rest args) (apply helper (concat (list "{name}") args))))'
try:
await self._send(f'(load-source "{_escape(sx_def)}")')
await self._send_command(f'(load-source "{_escape(sx_def)}")')
await self._read_until_ok(ctx=None)
count += 1
except OcamlBridgeError:
@@ -510,7 +464,7 @@ class OcamlBridge:
async def reset(self) -> None:
"""Reset the kernel environment to pristine state."""
async with self._lock:
await self._send("(reset)")
await self._send_command("(reset)")
kind, value = await self._read_response()
if kind == "error":
raise OcamlBridgeError(f"reset: {value}")
@@ -531,6 +485,20 @@ class OcamlBridge:
self._proc.stdin.write((line + "\n").encode())
await self._proc.stdin.drain()
async def _send_command(self, line: str) -> None:
"""Send a command with a fresh epoch prefix.
Increments the epoch counter and sends (epoch N) before the
actual command. The OCaml kernel tags all responses with this
epoch so stale messages from previous requests are discarded.
"""
self._epoch += 1
assert self._proc and self._proc.stdin
_logger.debug("EPOCH %d SEND: %s", self._epoch, line[:120])
self._proc.stdin.write(f"(epoch {self._epoch})\n".encode())
self._proc.stdin.write((line + "\n").encode())
await self._proc.stdin.drain()
async def _send_blob(self, data: str) -> None:
"""Send a length-prefixed binary blob to the subprocess.
@@ -562,16 +530,45 @@ class OcamlBridge:
"""Read a single (ok ...) or (error ...) response.
Returns (kind, value) where kind is "ok" or "error".
Discards stale epoch messages.
"""
line = await self._readline()
# Length-prefixed blob
if line.startswith("(ok-len "):
n = int(line[8:-1])
assert self._proc and self._proc.stdout
data = await self._proc.stdout.readexactly(n)
await self._proc.stdout.readline() # trailing newline
return ("ok", data.decode())
return _parse_response(line)
while True:
line = await self._readline()
if not self._is_current_epoch(line):
_logger.debug("Discarding stale response: %s", line[:80])
if line.startswith("(ok-len "):
parts = line[1:-1].split()
if len(parts) >= 3:
n = int(parts[-1])
assert self._proc and self._proc.stdout
await self._proc.stdout.readexactly(n)
await self._proc.stdout.readline()
continue
# Length-prefixed blob: (ok-len EPOCH N) or (ok-len N)
if line.startswith("(ok-len "):
parts = line[1:-1].split()
n = int(parts[-1])
assert self._proc and self._proc.stdout
data = await self._proc.stdout.readexactly(n)
await self._proc.stdout.readline() # trailing newline
return ("ok", data.decode())
return _parse_response(line)
def _is_current_epoch(self, line: str) -> bool:
"""Check if a response line belongs to the current epoch.
Lines tagged with a stale epoch are discarded. Untagged lines
(from a kernel that predates the epoch protocol) are accepted.
"""
# Extract epoch number from known tagged formats:
# (ok EPOCH ...), (error EPOCH ...), (ok-len EPOCH N),
# (io-request EPOCH ...), (io-done EPOCH N)
import re
m = re.match(r'\((?:ok|error|ok-len|ok-raw|io-request|io-done)\s+(\d+)\b', line)
if m:
return int(m.group(1)) == self._epoch
# Untagged (legacy) — accept
return True
async def _read_until_ok(
self,
@@ -583,6 +580,9 @@ class OcamlBridge:
- Legacy (blocking): single io-request → immediate io-response
- Batched: collect io-requests until (io-done N), process ALL
concurrently with asyncio.gather, send responses in order
Lines tagged with a stale epoch are silently discarded, making
pipe desync from previous failed requests impossible.
"""
import asyncio
pending_batch: list[str] = []
@@ -590,20 +590,53 @@ class OcamlBridge:
while True:
line = await self._readline()
# Discard stale epoch messages
if not self._is_current_epoch(line):
_logger.debug("Discarding stale epoch message: %s", line[:80])
# If it's a stale ok-len, drain the blob bytes too
if line.startswith("(ok-len "):
parts = line[1:-1].split()
if len(parts) >= 3:
n = int(parts[2])
assert self._proc and self._proc.stdout
await self._proc.stdout.readexactly(n)
await self._proc.stdout.readline()
continue
if line.startswith("(io-request "):
# Check if batched (has numeric ID after "io-request ")
# New format: (io-request EPOCH ...) or (io-request EPOCH ID ...)
# Strip epoch from the line for IO dispatch
after = line[len("(io-request "):].lstrip()
# Skip epoch number if present
if after and after[0].isdigit():
# Batched mode — collect, don't respond yet
pending_batch.append(line)
continue
# Could be epoch or batch ID — check for second number
parts = after.split(None, 2)
if len(parts) >= 2 and parts[1][0].isdigit():
# (io-request EPOCH ID "name" args...) — batched with epoch
pending_batch.append(line)
continue
elif len(parts) >= 2 and parts[1].startswith('"'):
# (io-request EPOCH "name" args...) — legacy with epoch
try:
result = await self._handle_io_request(line, ctx)
await self._send(
f"(io-response {self._epoch} {_serialize_for_ocaml(result)})")
except Exception as e:
_logger.warning("IO request failed, sending nil: %s", e)
await self._send(f"(io-response {self._epoch} nil)")
continue
else:
# Old format: (io-request ID "name" ...) — batched, no epoch
pending_batch.append(line)
continue
# Legacy blocking mode — respond immediately
try:
result = await self._handle_io_request(line, ctx)
await self._send(f"(io-response {_serialize_for_ocaml(result)})")
await self._send(
f"(io-response {self._epoch} {_serialize_for_ocaml(result)})")
except Exception as e:
_logger.warning("IO request failed, sending nil: %s", e)
await self._send("(io-response nil)")
await self._send(f"(io-response {self._epoch} nil)")
continue
if line.startswith("(io-done "):
@@ -614,16 +647,17 @@ class OcamlBridge:
for result in results:
if isinstance(result, BaseException):
_logger.warning("Batched IO failed: %s", result)
await self._send("(io-response nil)")
await self._send(f"(io-response {self._epoch} nil)")
else:
await self._send(
f"(io-response {_serialize_for_ocaml(result)})")
f"(io-response {self._epoch} {_serialize_for_ocaml(result)})")
pending_batch = []
continue
# Length-prefixed blob: (ok-len N)
# Length-prefixed blob: (ok-len EPOCH N) or (ok-len N)
if line.startswith("(ok-len "):
n = int(line[8:-1])
parts = line[1:-1].split() # ["ok-len", epoch, n] or ["ok-len", n]
n = int(parts[-1]) # last number is always byte count
assert self._proc and self._proc.stdout
data = await self._proc.stdout.readexactly(n)
# Read trailing newline
@@ -829,25 +863,50 @@ def _escape(s: str) -> str:
def _parse_response(line: str) -> tuple[str, str | None]:
"""Parse an (ok ...) or (error ...) response line.
Handles epoch-tagged responses: (ok EPOCH), (ok EPOCH value),
(error EPOCH "msg"), as well as legacy untagged responses.
Returns (kind, value) tuple.
"""
line = line.strip()
if line == "(ok)":
# (ok EPOCH) — tagged no-value
if line == "(ok)" or (line.startswith("(ok ") and line[4:-1].isdigit()):
return ("ok", None)
if line.startswith("(ok-raw "):
# Raw SX wire format — no unescaping needed
return ("ok", line[8:-1])
# (ok-raw EPOCH value) or (ok-raw value)
inner = line[8:-1]
# Strip epoch if present
if inner and inner[0].isdigit():
space = inner.find(" ")
if space > 0:
inner = inner[space + 1:]
else:
return ("ok", None)
return ("ok", inner)
if line.startswith("(ok "):
value = line[4:-1] # strip (ok and )
inner = line[4:-1] # strip (ok and )
# Strip epoch number if present: (ok 42 "value") → "value"
if inner and inner[0].isdigit():
space = inner.find(" ")
if space > 0:
inner = inner[space + 1:]
else:
# (ok EPOCH) with no value
return ("ok", None)
# If the value is a quoted string, unquote it
if value.startswith('"') and value.endswith('"'):
value = _unescape(value[1:-1])
return ("ok", value)
if inner.startswith('"') and inner.endswith('"'):
inner = _unescape(inner[1:-1])
return ("ok", inner)
if line.startswith("(error "):
msg = line[7:-1]
if msg.startswith('"') and msg.endswith('"'):
msg = _unescape(msg[1:-1])
return ("error", msg)
inner = line[7:-1]
# Strip epoch number if present: (error 42 "msg") → "msg"
if inner and inner[0].isdigit():
space = inner.find(" ")
if space > 0:
inner = inner[space + 1:]
if inner.startswith('"') and inner.endswith('"'):
inner = _unescape(inner[1:-1])
return ("error", inner)
return ("error", f"Unexpected response: {line}")

View File

@@ -52,6 +52,7 @@ class OcamlSync:
def __init__(self, binary: str | None = None):
self._binary = binary or os.environ.get("SX_OCAML_BIN") or _DEFAULT_BIN
self._proc: subprocess.Popen | None = None
self._epoch: int = 0
def _ensure(self):
if self._proc is not None and self._proc.poll() is None:
@@ -62,13 +63,17 @@ class OcamlSync:
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
self._epoch = 0
# Wait for (ready)
line = self._readline()
if line != "(ready)":
raise OcamlSyncError(f"Expected (ready), got: {line}")
def _send(self, command: str):
"""Send a command with epoch prefix."""
assert self._proc and self._proc.stdin
self._epoch += 1
self._proc.stdin.write(f"(epoch {self._epoch})\n".encode())
self._proc.stdin.write((command + "\n").encode())
self._proc.stdin.flush()
@@ -79,12 +84,26 @@ class OcamlSync:
raise OcamlSyncError("OCaml subprocess died unexpectedly")
return data.decode().rstrip("\n")
def _strip_epoch(self, inner: str) -> str:
"""Strip leading epoch number from a response value: '42 value''value'."""
if inner and inner[0].isdigit():
space = inner.find(" ")
if space > 0:
return inner[space + 1:]
return "" # epoch only, no value
return inner
def _read_response(self) -> str:
"""Read a single response. Returns the value string or raises on error."""
"""Read a single response. Returns the value string or raises on error.
Handles epoch-tagged responses: (ok EPOCH), (ok EPOCH value),
(ok-len EPOCH N), (error EPOCH "msg").
"""
line = self._readline()
# Length-prefixed blob: (ok-len N)
# Length-prefixed blob: (ok-len N) or (ok-len EPOCH N)
if line.startswith("(ok-len "):
n = int(line[8:-1])
parts = line[1:-1].split() # ["ok-len", ...]
n = int(parts[-1]) # last number is always byte count
assert self._proc and self._proc.stdout
data = self._proc.stdout.read(n)
self._proc.stdout.readline() # trailing newline
@@ -93,17 +112,18 @@ class OcamlSync:
if value.startswith('"') and value.endswith('"'):
value = _sx_unescape(value[1:-1])
return value
if line == "(ok)":
if line == "(ok)" or (line.startswith("(ok ") and line[4:-1].isdigit()):
return ""
if line.startswith("(ok-raw "):
return line[8:-1]
inner = self._strip_epoch(line[8:-1])
return inner
if line.startswith("(ok "):
value = line[4:-1]
value = self._strip_epoch(line[4:-1])
if value.startswith('"') and value.endswith('"'):
value = _sx_unescape(value[1:-1])
return value
if line.startswith("(error "):
msg = line[7:-1]
msg = self._strip_epoch(line[7:-1])
if msg.startswith('"') and msg.endswith('"'):
msg = _sx_unescape(msg[1:-1])
raise OcamlSyncError(msg)

View File

@@ -313,10 +313,7 @@ async def _eval_slot(expr: Any, env: dict, ctx: Any) -> str:
sx_text = _wrap_with_env(expr, env)
service = ctx.get("_helper_service", "") if isinstance(ctx, dict) else ""
return await bridge.aser_slot(sx_text, ctx={"_helper_service": service})
if os.environ.get("SX_USE_REF") == "1":
from .ref.async_eval_ref import async_eval_slot_to_sx
else:
from .async_eval import async_eval_slot_to_sx
from .async_eval import async_eval_slot_to_sx
return await async_eval_slot_to_sx(expr, env, ctx)

View File

@@ -38,10 +38,11 @@ def _resolve_sx_reader_macro(name: str):
If a file like z3.sx defines (define z3-translate ...), then #z3 is
automatically available as a reader macro without any Python registration.
Looks for {name}-translate as a Lambda in the component env.
Uses the synchronous OCaml bridge (ocaml_sync) when available.
"""
try:
from .jinja_bridge import get_component_env
from .ref.sx_ref import trampoline as _trampoline, call_lambda as _call_lambda
from .types import Lambda
except ImportError:
return None
@@ -49,10 +50,18 @@ def _resolve_sx_reader_macro(name: str):
fn = env.get(f"{name}-translate")
if fn is None or not isinstance(fn, Lambda):
return None
# Return a Python callable that invokes the SX lambda
def _sx_handler(expr):
return _trampoline(_call_lambda(fn, [expr], env))
return _sx_handler
# Use sync OCaml bridge to invoke the lambda
try:
from .ocaml_sync import OcamlSync
_sync = OcamlSync()
_sync.start()
def _sx_handler(expr):
from .parser import serialize as _ser
result = _sync.eval(f"({name}-translate {_ser(expr)})")
return parse(result) if result else expr
return _sx_handler
except Exception:
return None
# ---------------------------------------------------------------------------

View File

@@ -579,26 +579,54 @@ def prim_json_encode(value) -> str:
# (shared global state between transpiled and hand-written evaluators)
# ---------------------------------------------------------------------------
def _lazy_scope_primitives():
"""Register scope/provide/collect primitives from sx_ref.py.
def _register_scope_primitives():
"""Register scope/provide/collect primitive stubs.
Called at import time — if sx_ref.py isn't built yet, silently skip.
These are needed by the hand-written _aser in async_eval.py when
expanding components that use scoped effects (e.g. ~cssx/flush).
The OCaml kernel provides the real implementations. These stubs exist
so _PRIMITIVES contains the names for dependency analysis, and so
any Python-side code that checks for their existence finds them.
"""
try:
from .ref.sx_ref import (
sx_collect, sx_collected, sx_clear_collected,
sx_emitted, sx_emit, sx_context,
)
_PRIMITIVES["collect!"] = sx_collect
_PRIMITIVES["collected"] = sx_collected
_PRIMITIVES["clear-collected!"] = sx_clear_collected
_PRIMITIVES["emitted"] = sx_emitted
_PRIMITIVES["emit!"] = sx_emit
_PRIMITIVES["context"] = sx_context
except ImportError:
pass
import threading
_scope_data = threading.local()
_lazy_scope_primitives()
def _collect(channel, value):
if not hasattr(_scope_data, 'collected'):
_scope_data.collected = {}
_scope_data.collected.setdefault(channel, []).append(value)
return NIL
def _collected(channel):
if not hasattr(_scope_data, 'collected'):
return []
return list(_scope_data.collected.get(channel, []))
def _clear_collected(channel):
if hasattr(_scope_data, 'collected'):
_scope_data.collected.pop(channel, None)
return NIL
def _emit(channel, value):
if not hasattr(_scope_data, 'emitted'):
_scope_data.emitted = {}
_scope_data.emitted.setdefault(channel, []).append(value)
return NIL
def _emitted(channel):
if not hasattr(_scope_data, 'emitted'):
return []
return list(_scope_data.emitted.get(channel, []))
def _context(key):
if not hasattr(_scope_data, 'context'):
return NIL
return _scope_data.context.get(key, NIL) if isinstance(_scope_data.context, dict) else NIL
_PRIMITIVES["collect!"] = _collect
_PRIMITIVES["collected"] = _collected
_PRIMITIVES["clear-collected!"] = _clear_collected
_PRIMITIVES["emitted"] = _emitted
_PRIMITIVES["emit!"] = _emit
_PRIMITIVES["context"] = _context
_register_scope_primitives()

View File

@@ -49,10 +49,7 @@ async def execute_query(query_def: QueryDef, params: dict[str, str]) -> Any:
result = None
return _normalize(result)
if os.environ.get("SX_USE_REF") == "1":
from .ref.async_eval_ref import async_eval
else:
from .async_eval import async_eval
from .async_eval import async_eval
ctx = _get_request_context()
result = await async_eval(query_def.body, env, ctx)
@@ -91,10 +88,7 @@ async def execute_action(action_def: ActionDef, payload: dict[str, Any]) -> Any:
result = None
return _normalize(result)
if os.environ.get("SX_USE_REF") == "1":
from .ref.async_eval_ref import async_eval
else:
from .async_eval import async_eval
from .async_eval import async_eval
ctx = _get_request_context()
result = await async_eval(action_def.body, env, ctx)

View File

@@ -468,7 +468,7 @@
;; (div (~cssx/tw "bg-red-500") (~cssx/tw "p-4") "content")
;; =========================================================================
(defcomp ~cssx/tw (tokens)
(defcomp ~cssx/tw (&key tokens)
(let ((token-list (filter (fn (t) (not (= t "")))
(split (or tokens "") " ")))
(results (map cssx-process-token token-list))

View File

@@ -0,0 +1,558 @@
"""Tests exposing bugs after sx_ref.py removal.
These tests document all known breakages from removing the Python SX evaluator.
Each test targets a specific codepath that was depending on sx_ref.py and is now
broken.
Usage:
pytest shared/sx/tests/test_post_removal_bugs.py -v
"""
import asyncio
import os
import sys
import unittest
_project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../.."))
if _project_root not in sys.path:
sys.path.insert(0, _project_root)
from shared.sx.parser import parse, parse_all, serialize
from shared.sx.types import Component, Symbol, Keyword, NIL
# ---------------------------------------------------------------------------
# Helper: load shared components fresh (no cache)
# ---------------------------------------------------------------------------
def _load_components_fresh():
"""Load shared components, clearing cache to force re-parse."""
from shared.sx.jinja_bridge import _COMPONENT_ENV
_COMPONENT_ENV.clear()
from shared.sx.components import load_shared_components
load_shared_components()
return _COMPONENT_ENV
# ===========================================================================
# 1. register_components() loses all parameter information
# ===========================================================================
class TestComponentRegistration(unittest.TestCase):
"""register_components() hardcodes params=[] and has_children=False
for every component, losing all parameter metadata."""
@classmethod
def setUpClass(cls):
cls.env = _load_components_fresh()
def test_shell_component_should_have_params(self):
"""~shared:shell/sx-page-shell has 17+ &key params but gets params=[]."""
comp = self.env.get("~shared:shell/sx-page-shell")
self.assertIsNotNone(comp, "Shell component not found")
self.assertIsInstance(comp, Component)
# BUG: params is [] — should include title, meta-html, csrf, etc.
self.assertGreater(
len(comp.params), 0,
f"Shell component has params={comp.params} — expected 17+ keyword params"
)
def test_cssx_tw_should_have_tokens_param(self):
"""~cssx/tw needs a 'tokens' parameter."""
comp = self.env.get("~cssx/tw")
self.assertIsNotNone(comp, "~cssx/tw component not found")
self.assertIn(
"tokens", comp.params,
f"~cssx/tw has params={comp.params} — expected 'tokens'"
)
def test_cart_mini_should_have_params(self):
"""~shared:fragments/cart-mini has &key params."""
comp = self.env.get("~shared:fragments/cart-mini")
self.assertIsNotNone(comp, "cart-mini component not found")
self.assertGreater(
len(comp.params), 0,
f"cart-mini has params={comp.params} — expected keyword params"
)
def test_has_children_flag(self):
"""Components with &rest children should have has_children=True."""
comp = self.env.get("~shared:shell/sx-page-shell")
self.assertIsNotNone(comp)
# Many components accept children but has_children is always False
# Check any component that is known to accept &rest children
# e.g. a layout component
for name, val in self.env.items():
if isinstance(val, Component):
# Every component has has_children=False — at least some should be True
pass
# Count how many have has_children=True
with_children = sum(
1 for v in self.env.values()
if isinstance(v, Component) and v.has_children
)
total = sum(1 for v in self.env.values() if isinstance(v, Component))
# BUG: with_children is 0 — at least some components accept children
self.assertGreater(
with_children, 0,
f"0/{total} components have has_children=True — at least some should"
)
def test_all_components_have_empty_params(self):
"""Show the scale of the bug — every single component has params=[]."""
components_with_params = []
components_without = []
for name, val in self.env.items():
if isinstance(val, Component):
if val.params:
components_with_params.append(name)
else:
components_without.append(name)
# BUG: ALL components have empty params
self.assertGreater(
len(components_with_params), 0,
f"ALL {len(components_without)} components have params=[] — none have parameters parsed"
)
# ===========================================================================
# 2. Sync html.py rendering is completely broken
# ===========================================================================
class TestSyncHtmlRendering(unittest.TestCase):
"""html.py render() stubs _raw_eval/_trampoline — any evaluation crashes."""
def test_html_render_simple_element(self):
"""Even simple elements with keyword attrs need _eval, which is stubbed."""
from shared.sx.html import render
# This should work — (div "hello") needs no eval
result = render(parse('(div "hello")'), {})
self.assertIn("hello", result)
def test_html_render_with_keyword_attr(self):
"""Keyword attrs go through _eval, which raises RuntimeError."""
from shared.sx.html import render
try:
result = render(parse('(div :class "test" "hello")'), {})
# If it works, great
self.assertIn("test", result)
except RuntimeError as e:
self.assertIn("sx_ref.py has been removed", str(e))
self.fail(f"html.py render crashes on keyword attrs: {e}")
def test_html_render_symbol_lookup(self):
"""Symbol lookup goes through _eval, which is stubbed."""
from shared.sx.html import render
try:
result = render(parse('(div title)'), {"title": "Hello"})
self.assertIn("Hello", result)
except RuntimeError as e:
self.assertIn("sx_ref.py has been removed", str(e))
self.fail(f"html.py render crashes on symbol lookup: {e}")
def test_html_render_component(self):
"""Component rendering needs _eval for kwarg evaluation."""
from shared.sx.html import render
env = _load_components_fresh()
try:
result = render(
parse('(~shared:fragments/cart-mini :cart-count 0 :blog-url "" :cart-url "")'),
env,
)
self.assertIn("cart-mini", result)
except RuntimeError as e:
self.assertIn("sx_ref.py has been removed", str(e))
self.fail(f"html.py render crashes on component calls: {e}")
def test_sx_jinja_function_broken(self):
"""The sx() Jinja helper is broken — it uses html_render internally."""
from shared.sx.jinja_bridge import sx
env = _load_components_fresh()
try:
result = sx('(div "hello")')
self.assertIn("hello", result)
except RuntimeError as e:
self.assertIn("sx_ref.py has been removed", str(e))
self.fail(f"sx() Jinja function is broken: {e}")
# ===========================================================================
# 3. Async render_to_html uses Python path, not OCaml
# ===========================================================================
class TestAsyncRenderToHtml(unittest.IsolatedAsyncioTestCase):
"""helpers.py render_to_html() deliberately uses Python async_eval,
not the OCaml bridge. But Python eval is now broken."""
async def test_render_to_html_uses_python_path(self):
"""render_to_html goes through async_render, not OCaml bridge."""
from shared.sx.helpers import render_to_html
env = _load_components_fresh()
# The shell component has many &key params — none are bound because params=[]
try:
html = await render_to_html(
"shared:shell/sx-page-shell",
title="Test", csrf="abc", asset_url="/static",
sx_js_hash="abc123",
)
self.assertIn("Test", html)
except Exception as e:
# Expected: either RuntimeError from stubs or EvalError from undefined symbols
self.fail(
f"render_to_html (Python path) failed: {type(e).__name__}: {e}\n"
f"This should go through OCaml bridge instead"
)
async def test_async_render_component_no_params_bound(self):
"""async_eval.py _arender_component can't bind params because comp.params=[]."""
from shared.sx.async_eval import async_render
from shared.sx.primitives_io import RequestContext
env = _load_components_fresh()
# Create a simple component manually with correct params
test_comp = Component(
name="test/greeting",
params=["name"],
has_children=False,
body=parse('(div (str "Hello " name))'),
)
env["~test/greeting"] = test_comp
try:
result = await async_render(
parse('(~test/greeting :name "World")'),
env,
RequestContext(),
)
self.assertIn("Hello World", result)
except Exception as e:
self.fail(
f"async_render failed even with correct params: {type(e).__name__}: {e}"
)
# ===========================================================================
# 4. Dead imports from removed sx_ref.py
# ===========================================================================
class TestDeadImports(unittest.TestCase):
"""Files that import from sx_ref.py will crash when their codepaths execute."""
def test_async_eval_defcomp(self):
"""async_eval.py _asf_defcomp should work as a stub (no sx_ref import)."""
from shared.sx.async_eval import _asf_defcomp
env = {}
asyncio.run(_asf_defcomp(
[Symbol("defcomp"), Symbol("~test"), [], [Symbol("div")]],
env, None
))
# Should register a minimal component in env
self.assertIn("~test", env)
def test_async_eval_defmacro(self):
"""async_eval.py _asf_defmacro should work as a stub (no sx_ref import)."""
from shared.sx.async_eval import _asf_defmacro
env = {}
asyncio.run(_asf_defmacro(
[Symbol("defmacro"), Symbol("test"), [], [Symbol("div")]],
env, None
))
self.assertIn("test", env)
def test_async_eval_defstyle(self):
"""async_eval.py _asf_defstyle should be a no-op (no sx_ref import)."""
from shared.sx.async_eval import _asf_defstyle
result = asyncio.run(_asf_defstyle(
[Symbol("defstyle"), Symbol("test"), [], [Symbol("div")]],
{}, None
))
# Should return NIL without crashing
self.assertIsNotNone(result)
def test_async_eval_defhandler(self):
"""async_eval.py _asf_defhandler should be a no-op (no sx_ref import)."""
from shared.sx.async_eval import _asf_defhandler
result = asyncio.run(_asf_defhandler(
[Symbol("defhandler"), Symbol("test"), [], [Symbol("div")]],
{}, None
))
self.assertIsNotNone(result)
def test_async_eval_continuation_reset(self):
"""async_eval.py _asf_reset imports eval_expr/trampoline from sx_ref."""
# The cont_fn inside _asf_reset will crash when invoked
from shared.sx.async_eval import _ASYNC_RENDER_FORMS
reset_fn = _ASYNC_RENDER_FORMS.get("reset")
# reset is defined in async_eval — the import is deferred to execution
# Just verify the module doesn't have the import available
try:
from shared.sx.ref.sx_ref import eval_expr
self.fail("sx_ref.py should not exist")
except (ImportError, ModuleNotFoundError):
pass # Expected
def test_ocaml_bridge_jit_compile(self):
"""ocaml_bridge.py _compile_adapter_module imports from sx_ref."""
try:
from shared.sx.ref.sx_ref import eval_expr, trampoline, PRIMITIVES
self.fail("sx_ref.py should not exist — JIT compilation path is broken")
except (ImportError, ModuleNotFoundError):
pass # Expected — confirms the bug
def test_parser_reader_macro(self):
"""parser.py _try_reader_macro imports trampoline/call_lambda from sx_ref."""
try:
from shared.sx.ref.sx_ref import trampoline, call_lambda
self.fail("sx_ref.py should not exist — reader macros are broken")
except (ImportError, ModuleNotFoundError):
pass # Expected — confirms the bug
def test_primitives_scope_prims(self):
"""primitives.py _lazy_scope_primitives silently fails to load scope prims."""
from shared.sx.primitives import _PRIMITIVES
# collect!, collected, clear-collected!, emitted, emit!, context
# These are needed for CSSX but the import from sx_ref silently fails
missing = []
for name in ("collect!", "collected", "clear-collected!", "emitted", "emit!", "context"):
if name not in _PRIMITIVES:
missing.append(name)
if missing:
self.fail(
f"Scope primitives missing from _PRIMITIVES (sx_ref import failed silently): {missing}\n"
f"CSSX components depend on these for collect!/collected"
)
def test_deps_transitive_deps_ref_path(self):
"""deps.py transitive_deps imports from sx_ref when SX_USE_REF=1."""
# The fallback path should still work
from shared.sx.deps import transitive_deps
env = _load_components_fresh()
# Should work via fallback, not crash
try:
result = transitive_deps("~cssx/tw", env)
self.assertIsInstance(result, set)
except (ImportError, ModuleNotFoundError) as e:
self.fail(f"transitive_deps crashed: {e}")
def test_handlers_python_fallback(self):
"""handlers.py eval_handler Python fallback imports async_eval_ref."""
# When not using OCaml, handler evaluation falls through to async_eval
# The ref path (SX_USE_REF=1) would crash
try:
from shared.sx.ref.async_eval_ref import async_eval_to_sx
self.fail("async_eval_ref.py should not exist")
except (ImportError, ModuleNotFoundError):
pass # Expected
# ===========================================================================
# 5. ~cssx/tw signature mismatch
# ===========================================================================
class TestCssxTwSignature(unittest.TestCase):
"""~cssx/tw changed from (&key tokens) to (tokens) positional,
but callers use :tokens keyword syntax."""
@classmethod
def setUpClass(cls):
cls.env = _load_components_fresh()
def test_cssx_tw_source_uses_positional(self):
"""Verify the current source has positional (tokens) not (&key tokens)."""
import os
cssx_path = os.path.join(
os.path.dirname(__file__), "..", "templates", "cssx.sx"
)
with open(cssx_path) as f:
source = f.read()
# Check if it's positional or keyword
if "(defcomp ~cssx/tw (tokens)" in source:
# Positional — callers using :tokens will break
self.fail(
"~cssx/tw uses positional (tokens) but callers use :tokens keyword syntax.\n"
"Should be: (defcomp ~cssx/tw (&key tokens) ...)"
)
elif "(defcomp ~cssx/tw (&key tokens)" in source:
pass # Correct
else:
# Unknown signature
for line in source.split("\n"):
if "defcomp ~cssx/tw" in line:
self.fail(f"Unexpected ~cssx/tw signature: {line.strip()}")
def test_cssx_tw_callers_use_keyword(self):
"""Scan for callers that use :tokens keyword syntax."""
import glob as glob_mod
sx_dir = os.path.join(os.path.dirname(__file__), "../../..")
keyword_callers = []
positional_callers = []
for fp in glob_mod.glob(os.path.join(sx_dir, "**/*.sx"), recursive=True):
try:
with open(fp) as f:
content = f.read()
except Exception:
continue
if "~cssx/tw" not in content:
continue
for line_no, line in enumerate(content.split("\n"), 1):
if "~cssx/tw" in line and "defcomp" not in line:
if ":tokens" in line:
keyword_callers.append(f"{fp}:{line_no}")
elif "(~cssx/tw " in line:
positional_callers.append(f"{fp}:{line_no}")
if keyword_callers:
# If signature is positional but callers use :tokens, that's a bug
import os as os_mod
cssx_path = os.path.join(
os.path.dirname(__file__), "..", "templates", "cssx.sx"
)
with open(cssx_path) as f:
source = f.read()
if "(defcomp ~cssx/tw (tokens)" in source:
self.fail(
f"~cssx/tw uses positional params but {len(keyword_callers)} callers use :tokens:\n"
+ "\n".join(keyword_callers[:5])
)
# ===========================================================================
# 6. OCaml bridge rendering (should work — this is the good path)
# ===========================================================================
class TestOcamlBridgeRendering(unittest.IsolatedAsyncioTestCase):
"""The OCaml bridge should handle all rendering correctly."""
@classmethod
def setUpClass(cls):
from shared.sx.ocaml_bridge import _DEFAULT_BIN
bin_path = os.path.abspath(_DEFAULT_BIN)
if not os.path.isfile(bin_path):
raise unittest.SkipTest("OCaml binary not found")
async def asyncSetUp(self):
from shared.sx.ocaml_bridge import OcamlBridge
self.bridge = OcamlBridge()
await self.bridge.start()
async def asyncTearDown(self):
if hasattr(self, 'bridge'):
await self.bridge.stop()
async def test_simple_element(self):
result = await self.bridge.render('(div "hello")')
self.assertIn("hello", result)
async def test_element_with_keyword_attrs(self):
result = await self.bridge.render('(div :class "test" "hello")')
self.assertIn('class="test"', result)
self.assertIn("hello", result)
async def test_component_with_params(self):
"""OCaml should handle component parameter binding correctly."""
# Use load_source to define a component (bypasses _ensure_components lock)
await self.bridge.load_source('(defcomp ~test/greet (&key name) (div (str "Hello " name)))')
result = await self.bridge.render('(~test/greet :name "World")')
self.assertIn("Hello World", result)
async def test_let_binding(self):
result = await self.bridge.render('(let ((x "hello")) (div x))')
self.assertIn("hello", result)
async def test_conditional(self):
result = await self.bridge.render('(if true (div "yes") (div "no"))')
self.assertIn("yes", result)
self.assertNotIn("no", result)
async def test_cssx_tw_keyword_call(self):
"""Test that ~cssx/tw works when called with :tokens keyword.
Components are loaded by _ensure_components() automatically."""
try:
result = await self.bridge.render('(div (~cssx/tw :tokens "bg-red-500") "content")')
# Should produce a spread with CSS class, not an error
self.assertNotIn("error", result.lower())
except Exception as e:
self.fail(f"~cssx/tw :tokens keyword call failed: {e}")
async def test_cssx_tw_positional_call(self):
"""Test that ~cssx/tw works when called positionally."""
try:
result = await self.bridge.render('(div (~cssx/tw "bg-red-500") "content")')
self.assertNotIn("error", result.lower())
except Exception as e:
self.fail(f"~cssx/tw positional call failed: {e}")
async def test_repeated_renders_dont_crash(self):
"""Verify OCaml bridge handles multiple sequential renders."""
for i in range(5):
result = await self.bridge.render(f'(div "iter-{i}")')
self.assertIn(f"iter-{i}", result)
# ===========================================================================
# 7. Scope primitives missing (collect!, collected, etc.)
# ===========================================================================
class TestScopePrimitives(unittest.TestCase):
"""Scope primitives needed by CSSX are missing because the import
from sx_ref.py silently fails."""
def test_python_primitives_have_scope_ops(self):
"""Check that collect!/collected/etc. are in _PRIMITIVES."""
from shared.sx.primitives import _PRIMITIVES
required = ["collect!", "collected", "clear-collected!",
"emitted", "emit!", "context"]
missing = [p for p in required if p not in _PRIMITIVES]
if missing:
self.fail(
f"Missing Python-side scope primitives: {missing}\n"
f"These were provided by sx_ref.py — need OCaml bridge or Python stubs"
)
# ===========================================================================
# 8. Query executor fallback path
# ===========================================================================
class TestQueryExecutorFallback(unittest.TestCase):
"""query_executor.py imports async_eval for its fallback path."""
def test_query_executor_import(self):
"""query_executor can be imported without crashing."""
try:
import shared.sx.query_executor
except Exception as e:
self.fail(f"query_executor import crashed: {e}")
# ===========================================================================
# 9. End-to-end: sx_page shell rendering
# ===========================================================================
class TestShellRendering(unittest.IsolatedAsyncioTestCase):
"""The shell template needs to render through some path that works."""
async def test_sx_page_shell_via_python(self):
"""render_to_html('shared:shell/sx-page-shell', ...) uses Python path.
This is the actual failure from the production error log."""
from shared.sx.helpers import render_to_html
_load_components_fresh()
try:
html = await render_to_html(
"shared:shell/sx-page-shell",
title="Test Page",
csrf="test-csrf",
asset_url="/static",
sx_js_hash="abc",
)
# Should produce full HTML document
self.assertIn("<!doctype html>", html.lower())
self.assertIn("Test Page", html)
except Exception as e:
self.fail(
f"Shell rendering via Python path failed: {type(e).__name__}: {e}\n"
f"This is the exact error seen in production — "
f"render_to_html should use OCaml bridge"
)
if __name__ == "__main__":
unittest.main()