coop/rose-ash
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Complete Python eval removal: epoch protocol, scope consolidation, JIT fixes

Browse Source 
Route all rendering through OCaml bridge — render_to_html no longer uses
Python async_eval. Fix register_components to parse &key params and &rest
children from defcomp forms. Remove all dead sx_ref.py imports.

Epoch protocol (prevents pipe desync):
- Every command prefixed with (epoch N), all responses tagged with epoch
- Both sides discard stale-epoch messages — desync structurally impossible
- OCaml main loop discards stale io-responses between commands

Consolidate scope primitives into sx_scope.ml:
- Single source of truth for scope-push!/pop!/peek, collect!/collected,
  emit!/emitted, context, and 12 other scope operations
- Removes duplicate registrations from sx_server.ml (including bugs where
  scope-emit! and clear-collected! were registered twice with different impls)
- Bind scope prims into env so JIT VM finds them via OP_GLOBAL_GET

JIT VM fixes:
- Trampoline thunks before passing args to CALL_PRIM
- as_list resolves thunks via _sx_trampoline_fn
- len handles all value types (Bool, Number, RawHTML, SxExpr, Spread, etc.)

Other fixes:
- ~cssx/tw signature: (tokens) → (&key tokens) to match callers
- Minimal Python evaluator in html.py for sync sx() Jinja function
- Python scope primitive stubs (thread-local) for non-OCaml paths
- Reader macro resolution via OcamlSync instead of sx_ref.py

Tests: 1114 OCaml, 1078 JS, 35 Python regression, 6/6 Playwright SSR

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
giles
2026-03-24 16:14:40 +00:00
parent e887c0d978
commit f9f810ffd7
18 changed files with 1305 additions and 478 deletions
Download Patch File Download Diff File Expand all files Collapse all files

558
shared/sx/tests/test_post_removal_bugs.py Normal file
View File

@@ -0,0 +1,558 @@
"""Tests exposing bugs after sx_ref.py removal.
These tests document all known breakages from removing the Python SX evaluator.
Each test targets a specific codepath that was depending on sx_ref.py and is now
broken.
Usage:
pytest shared/sx/tests/test_post_removal_bugs.py -v
"""
import asyncio
import os
import sys
import unittest
_project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../.."))
if _project_root not in sys.path:
sys.path.insert(0, _project_root)
from shared.sx.parser import parse, parse_all, serialize
from shared.sx.types import Component, Symbol, Keyword, NIL
# ---------------------------------------------------------------------------
# Helper: load shared components fresh (no cache)
# ---------------------------------------------------------------------------
def _load_components_fresh():
"""Load shared components, clearing cache to force re-parse."""
from shared.sx.jinja_bridge import _COMPONENT_ENV
_COMPONENT_ENV.clear()
from shared.sx.components import load_shared_components
load_shared_components()
return _COMPONENT_ENV
# ===========================================================================
# 1. register_components() loses all parameter information
# ===========================================================================
class TestComponentRegistration(unittest.TestCase):
"""register_components() hardcodes params=[] and has_children=False
for every component, losing all parameter metadata."""
@classmethod
def setUpClass(cls):
cls.env = _load_components_fresh()
def test_shell_component_should_have_params(self):
"""~shared:shell/sx-page-shell has 17+ &key params but gets params=[]."""
comp = self.env.get("~shared:shell/sx-page-shell")
self.assertIsNotNone(comp, "Shell component not found")
self.assertIsInstance(comp, Component)
# BUG: params is [] — should include title, meta-html, csrf, etc.
self.assertGreater(
len(comp.params), 0,
f"Shell component has params={comp.params} — expected 17+ keyword params"
)
def test_cssx_tw_should_have_tokens_param(self):
"""~cssx/tw needs a 'tokens' parameter."""
comp = self.env.get("~cssx/tw")
self.assertIsNotNone(comp, "~cssx/tw component not found")
self.assertIn(
"tokens", comp.params,
f"~cssx/tw has params={comp.params} — expected 'tokens'"
)
def test_cart_mini_should_have_params(self):
"""~shared:fragments/cart-mini has &key params."""
comp = self.env.get("~shared:fragments/cart-mini")
self.assertIsNotNone(comp, "cart-mini component not found")
self.assertGreater(
len(comp.params), 0,
f"cart-mini has params={comp.params} — expected keyword params"
)
def test_has_children_flag(self):
"""Components with &rest children should have has_children=True."""
comp = self.env.get("~shared:shell/sx-page-shell")
self.assertIsNotNone(comp)
# Many components accept children but has_children is always False
# Check any component that is known to accept &rest children
# e.g. a layout component
for name, val in self.env.items():
if isinstance(val, Component):
# Every component has has_children=False — at least some should be True
pass
# Count how many have has_children=True
with_children = sum(
1 for v in self.env.values()
if isinstance(v, Component) and v.has_children
)
total = sum(1 for v in self.env.values() if isinstance(v, Component))
# BUG: with_children is 0 — at least some components accept children
self.assertGreater(
with_children, 0,
f"0/{total} components have has_children=True — at least some should"
)
def test_all_components_have_empty_params(self):
"""Show the scale of the bug — every single component has params=[]."""
components_with_params = []
components_without = []
for name, val in self.env.items():
if isinstance(val, Component):
if val.params:
components_with_params.append(name)
else:
components_without.append(name)
# BUG: ALL components have empty params
self.assertGreater(
len(components_with_params), 0,
f"ALL {len(components_without)} components have params=[] — none have parameters parsed"
)
# ===========================================================================
# 2. Sync html.py rendering is completely broken
# ===========================================================================
class TestSyncHtmlRendering(unittest.TestCase):
"""html.py render() stubs _raw_eval/_trampoline — any evaluation crashes."""
def test_html_render_simple_element(self):
"""Even simple elements with keyword attrs need _eval, which is stubbed."""
from shared.sx.html import render
# This should work — (div "hello") needs no eval
result = render(parse('(div "hello")'), {})
self.assertIn("hello", result)
def test_html_render_with_keyword_attr(self):
"""Keyword attrs go through _eval, which raises RuntimeError."""
from shared.sx.html import render
try:
result = render(parse('(div :class "test" "hello")'), {})
# If it works, great
self.assertIn("test", result)
except RuntimeError as e:
self.assertIn("sx_ref.py has been removed", str(e))
self.fail(f"html.py render crashes on keyword attrs: {e}")
def test_html_render_symbol_lookup(self):
"""Symbol lookup goes through _eval, which is stubbed."""
from shared.sx.html import render
try:
result = render(parse('(div title)'), {"title": "Hello"})
self.assertIn("Hello", result)
except RuntimeError as e:
self.assertIn("sx_ref.py has been removed", str(e))
self.fail(f"html.py render crashes on symbol lookup: {e}")
def test_html_render_component(self):
"""Component rendering needs _eval for kwarg evaluation."""
from shared.sx.html import render
env = _load_components_fresh()
try:
result = render(
parse('(~shared:fragments/cart-mini :cart-count 0 :blog-url "" :cart-url "")'),
env,
)
self.assertIn("cart-mini", result)
except RuntimeError as e:
self.assertIn("sx_ref.py has been removed", str(e))
self.fail(f"html.py render crashes on component calls: {e}")
def test_sx_jinja_function_broken(self):
"""The sx() Jinja helper is broken — it uses html_render internally."""
from shared.sx.jinja_bridge import sx
env = _load_components_fresh()
try:
result = sx('(div "hello")')
self.assertIn("hello", result)
except RuntimeError as e:
self.assertIn("sx_ref.py has been removed", str(e))
self.fail(f"sx() Jinja function is broken: {e}")
# ===========================================================================
# 3. Async render_to_html uses Python path, not OCaml
# ===========================================================================
class TestAsyncRenderToHtml(unittest.IsolatedAsyncioTestCase):
"""helpers.py render_to_html() deliberately uses Python async_eval,
not the OCaml bridge. But Python eval is now broken."""
async def test_render_to_html_uses_python_path(self):
"""render_to_html goes through async_render, not OCaml bridge."""
from shared.sx.helpers import render_to_html
env = _load_components_fresh()
# The shell component has many &key params — none are bound because params=[]
try:
html = await render_to_html(
"shared:shell/sx-page-shell",
title="Test", csrf="abc", asset_url="/static",
sx_js_hash="abc123",
)
self.assertIn("Test", html)
except Exception as e:
# Expected: either RuntimeError from stubs or EvalError from undefined symbols
self.fail(
f"render_to_html (Python path) failed: {type(e).__name__}: {e}\n"
f"This should go through OCaml bridge instead"
)
async def test_async_render_component_no_params_bound(self):
"""async_eval.py _arender_component can't bind params because comp.params=[]."""
from shared.sx.async_eval import async_render
from shared.sx.primitives_io import RequestContext
env = _load_components_fresh()
# Create a simple component manually with correct params
test_comp = Component(
name="test/greeting",
params=["name"],
has_children=False,
body=parse('(div (str "Hello " name))'),
)
env["~test/greeting"] = test_comp
try:
result = await async_render(
parse('(~test/greeting :name "World")'),
env,
RequestContext(),
)
self.assertIn("Hello World", result)
except Exception as e:
self.fail(
f"async_render failed even with correct params: {type(e).__name__}: {e}"
)
# ===========================================================================
# 4. Dead imports from removed sx_ref.py
# ===========================================================================
class TestDeadImports(unittest.TestCase):
"""Files that import from sx_ref.py will crash when their codepaths execute."""
def test_async_eval_defcomp(self):
"""async_eval.py _asf_defcomp should work as a stub (no sx_ref import)."""
from shared.sx.async_eval import _asf_defcomp
env = {}
asyncio.run(_asf_defcomp(
[Symbol("defcomp"), Symbol("~test"), [], [Symbol("div")]],
env, None
))
# Should register a minimal component in env
self.assertIn("~test", env)
def test_async_eval_defmacro(self):
"""async_eval.py _asf_defmacro should work as a stub (no sx_ref import)."""
from shared.sx.async_eval import _asf_defmacro
env = {}
asyncio.run(_asf_defmacro(
[Symbol("defmacro"), Symbol("test"), [], [Symbol("div")]],
env, None
))
self.assertIn("test", env)
def test_async_eval_defstyle(self):
"""async_eval.py _asf_defstyle should be a no-op (no sx_ref import)."""
from shared.sx.async_eval import _asf_defstyle
result = asyncio.run(_asf_defstyle(
[Symbol("defstyle"), Symbol("test"), [], [Symbol("div")]],
{}, None
))
# Should return NIL without crashing
self.assertIsNotNone(result)
def test_async_eval_defhandler(self):
"""async_eval.py _asf_defhandler should be a no-op (no sx_ref import)."""
from shared.sx.async_eval import _asf_defhandler
result = asyncio.run(_asf_defhandler(
[Symbol("defhandler"), Symbol("test"), [], [Symbol("div")]],
{}, None
))
self.assertIsNotNone(result)
def test_async_eval_continuation_reset(self):
"""async_eval.py _asf_reset imports eval_expr/trampoline from sx_ref."""
# The cont_fn inside _asf_reset will crash when invoked
from shared.sx.async_eval import _ASYNC_RENDER_FORMS
reset_fn = _ASYNC_RENDER_FORMS.get("reset")
# reset is defined in async_eval — the import is deferred to execution
# Just verify the module doesn't have the import available
try:
from shared.sx.ref.sx_ref import eval_expr
self.fail("sx_ref.py should not exist")
except (ImportError, ModuleNotFoundError):
pass # Expected
def test_ocaml_bridge_jit_compile(self):
"""ocaml_bridge.py _compile_adapter_module imports from sx_ref."""
try:
from shared.sx.ref.sx_ref import eval_expr, trampoline, PRIMITIVES
self.fail("sx_ref.py should not exist — JIT compilation path is broken")
except (ImportError, ModuleNotFoundError):
pass # Expected — confirms the bug
def test_parser_reader_macro(self):
"""parser.py _try_reader_macro imports trampoline/call_lambda from sx_ref."""
try:
from shared.sx.ref.sx_ref import trampoline, call_lambda
self.fail("sx_ref.py should not exist — reader macros are broken")
except (ImportError, ModuleNotFoundError):
pass # Expected — confirms the bug
def test_primitives_scope_prims(self):
"""primitives.py _lazy_scope_primitives silently fails to load scope prims."""
from shared.sx.primitives import _PRIMITIVES
# collect!, collected, clear-collected!, emitted, emit!, context
# These are needed for CSSX but the import from sx_ref silently fails
missing = []
for name in ("collect!", "collected", "clear-collected!", "emitted", "emit!", "context"):
if name not in _PRIMITIVES:
missing.append(name)
if missing:
self.fail(
f"Scope primitives missing from _PRIMITIVES (sx_ref import failed silently): {missing}\n"
f"CSSX components depend on these for collect!/collected"
)
def test_deps_transitive_deps_ref_path(self):
"""deps.py transitive_deps imports from sx_ref when SX_USE_REF=1."""
# The fallback path should still work
from shared.sx.deps import transitive_deps
env = _load_components_fresh()
# Should work via fallback, not crash
try:
result = transitive_deps("~cssx/tw", env)
self.assertIsInstance(result, set)
except (ImportError, ModuleNotFoundError) as e:
self.fail(f"transitive_deps crashed: {e}")
def test_handlers_python_fallback(self):
"""handlers.py eval_handler Python fallback imports async_eval_ref."""
# When not using OCaml, handler evaluation falls through to async_eval
# The ref path (SX_USE_REF=1) would crash
try:
from shared.sx.ref.async_eval_ref import async_eval_to_sx
self.fail("async_eval_ref.py should not exist")
except (ImportError, ModuleNotFoundError):
pass # Expected
# ===========================================================================
# 5. ~cssx/tw signature mismatch
# ===========================================================================
class TestCssxTwSignature(unittest.TestCase):
"""~cssx/tw changed from (&key tokens) to (tokens) positional,
but callers use :tokens keyword syntax."""
@classmethod
def setUpClass(cls):
cls.env = _load_components_fresh()
def test_cssx_tw_source_uses_positional(self):
"""Verify the current source has positional (tokens) not (&key tokens)."""
import os
cssx_path = os.path.join(
os.path.dirname(__file__), "..", "templates", "cssx.sx"
)
with open(cssx_path) as f:
source = f.read()
# Check if it's positional or keyword
if "(defcomp ~cssx/tw (tokens)" in source:
# Positional — callers using :tokens will break
self.fail(
"~cssx/tw uses positional (tokens) but callers use :tokens keyword syntax.\n"
"Should be: (defcomp ~cssx/tw (&key tokens) ...)"
)
elif "(defcomp ~cssx/tw (&key tokens)" in source:
pass # Correct
else:
# Unknown signature
for line in source.split("\n"):
if "defcomp ~cssx/tw" in line:
self.fail(f"Unexpected ~cssx/tw signature: {line.strip()}")
def test_cssx_tw_callers_use_keyword(self):
"""Scan for callers that use :tokens keyword syntax."""
import glob as glob_mod
sx_dir = os.path.join(os.path.dirname(__file__), "../../..")
keyword_callers = []
positional_callers = []
for fp in glob_mod.glob(os.path.join(sx_dir, "**/*.sx"), recursive=True):
try:
with open(fp) as f:
content = f.read()
except Exception:
continue
if "~cssx/tw" not in content:
continue
for line_no, line in enumerate(content.split("\n"), 1):
if "~cssx/tw" in line and "defcomp" not in line:
if ":tokens" in line:
keyword_callers.append(f"{fp}:{line_no}")
elif "(~cssx/tw " in line:
positional_callers.append(f"{fp}:{line_no}")
if keyword_callers:
# If signature is positional but callers use :tokens, that's a bug
import os as os_mod
cssx_path = os.path.join(
os.path.dirname(__file__), "..", "templates", "cssx.sx"
)
with open(cssx_path) as f:
source = f.read()
if "(defcomp ~cssx/tw (tokens)" in source:
self.fail(
f"~cssx/tw uses positional params but {len(keyword_callers)} callers use :tokens:\n"
+ "\n".join(keyword_callers[:5])
)
# ===========================================================================
# 6. OCaml bridge rendering (should work — this is the good path)
# ===========================================================================
class TestOcamlBridgeRendering(unittest.IsolatedAsyncioTestCase):
"""The OCaml bridge should handle all rendering correctly."""
@classmethod
def setUpClass(cls):
from shared.sx.ocaml_bridge import _DEFAULT_BIN
bin_path = os.path.abspath(_DEFAULT_BIN)
if not os.path.isfile(bin_path):
raise unittest.SkipTest("OCaml binary not found")
async def asyncSetUp(self):
from shared.sx.ocaml_bridge import OcamlBridge
self.bridge = OcamlBridge()
await self.bridge.start()
async def asyncTearDown(self):
if hasattr(self, 'bridge'):
await self.bridge.stop()
async def test_simple_element(self):
result = await self.bridge.render('(div "hello")')
self.assertIn("hello", result)
async def test_element_with_keyword_attrs(self):
result = await self.bridge.render('(div :class "test" "hello")')
self.assertIn('class="test"', result)
self.assertIn("hello", result)
async def test_component_with_params(self):
"""OCaml should handle component parameter binding correctly."""
# Use load_source to define a component (bypasses _ensure_components lock)
await self.bridge.load_source('(defcomp ~test/greet (&key name) (div (str "Hello " name)))')
result = await self.bridge.render('(~test/greet :name "World")')
self.assertIn("Hello World", result)
async def test_let_binding(self):
result = await self.bridge.render('(let ((x "hello")) (div x))')
self.assertIn("hello", result)
async def test_conditional(self):
result = await self.bridge.render('(if true (div "yes") (div "no"))')
self.assertIn("yes", result)
self.assertNotIn("no", result)
async def test_cssx_tw_keyword_call(self):
"""Test that ~cssx/tw works when called with :tokens keyword.
Components are loaded by _ensure_components() automatically."""
try:
result = await self.bridge.render('(div (~cssx/tw :tokens "bg-red-500") "content")')
# Should produce a spread with CSS class, not an error
self.assertNotIn("error", result.lower())
except Exception as e:
self.fail(f"~cssx/tw :tokens keyword call failed: {e}")
async def test_cssx_tw_positional_call(self):
"""Test that ~cssx/tw works when called positionally."""
try:
result = await self.bridge.render('(div (~cssx/tw "bg-red-500") "content")')
self.assertNotIn("error", result.lower())
except Exception as e:
self.fail(f"~cssx/tw positional call failed: {e}")
async def test_repeated_renders_dont_crash(self):
"""Verify OCaml bridge handles multiple sequential renders."""
for i in range(5):
result = await self.bridge.render(f'(div "iter-{i}")')
self.assertIn(f"iter-{i}", result)
# ===========================================================================
# 7. Scope primitives missing (collect!, collected, etc.)
# ===========================================================================
class TestScopePrimitives(unittest.TestCase):
"""Scope primitives needed by CSSX are missing because the import
from sx_ref.py silently fails."""
def test_python_primitives_have_scope_ops(self):
"""Check that collect!/collected/etc. are in _PRIMITIVES."""
from shared.sx.primitives import _PRIMITIVES
required = ["collect!", "collected", "clear-collected!",
"emitted", "emit!", "context"]
missing = [p for p in required if p not in _PRIMITIVES]
if missing:
self.fail(
f"Missing Python-side scope primitives: {missing}\n"
f"These were provided by sx_ref.py — need OCaml bridge or Python stubs"
)
# ===========================================================================
# 8. Query executor fallback path
# ===========================================================================
class TestQueryExecutorFallback(unittest.TestCase):
"""query_executor.py imports async_eval for its fallback path."""
def test_query_executor_import(self):
"""query_executor can be imported without crashing."""
try:
import shared.sx.query_executor
except Exception as e:
self.fail(f"query_executor import crashed: {e}")
# ===========================================================================
# 9. End-to-end: sx_page shell rendering
# ===========================================================================
class TestShellRendering(unittest.IsolatedAsyncioTestCase):
"""The shell template needs to render through some path that works."""
async def test_sx_page_shell_via_python(self):
"""render_to_html('shared:shell/sx-page-shell', ...) uses Python path.
This is the actual failure from the production error log."""
from shared.sx.helpers import render_to_html
_load_components_fresh()
try:
html = await render_to_html(
"shared:shell/sx-page-shell",
title="Test Page",
csrf="test-csrf",
asset_url="/static",
sx_js_hash="abc",
)
# Should produce full HTML document
self.assertIn("<!doctype html>", html.lower())
self.assertIn("Test Page", html)
except Exception as e:
self.fail(
f"Shell rendering via Python path failed: {type(e).__name__}: {e}\n"
f"This is the exact error seen in production — "
f"render_to_html should use OCaml bridge"
)
if __name__ == "__main__":
unittest.main()