Restore hyperscript work on stable site base (908f4f80)

Reset to last known-good state (908f4f80) where links, stepper, and
islands all work, then recovered all hyperscript implementation,
conformance tests, behavioral tests, Playwright specs, site sandbox,
IO-aware server loading, and upstream test suite from f271c88a.

Excludes runtime changes (VM resolve hook, VmSuspended browser handler,
sx_ref.ml guard recovery) that need careful re-integration.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-04-09 19:29:56 +00:00
parent 908f4f80d4
commit 7492ceac4e
55 changed files with 32933 additions and 437 deletions

View File

@@ -894,7 +894,8 @@ let handle_sx_build args =
let cmd = match target with
| "ocaml" ->
let abs_project = if Filename.is_relative pd then Sys.getcwd () ^ "/" ^ pd else pd in
Printf.sprintf "cd %s/hosts/ocaml && eval $(opam env 2>/dev/null) && dune build 2>&1 && cp _build/default/browser/sx_browser.bc.wasm.js %s/shared/static/wasm/sx_browser.bc.wasm.js && cp _build/default/browser/sx_browser.bc.js %s/shared/static/wasm/sx_browser.bc.js && cp -r _build/default/browser/sx_browser.bc.wasm.assets %s/shared/static/wasm/" abs_project abs_project abs_project abs_project
(* Remove assets dir that conflicts with dune's browser target, then build *)
Printf.sprintf "cd %s/hosts/ocaml && rm -rf browser/sx_browser.bc.wasm.assets && eval $(opam env 2>/dev/null) && dune build 2>&1 && cp _build/default/browser/sx_browser.bc.wasm.js %s/shared/static/wasm/sx_browser.bc.wasm.js && cp _build/default/browser/sx_browser.bc.js %s/shared/static/wasm/sx_browser.bc.js && rm -rf %s/shared/static/wasm/sx_browser.bc.wasm.assets && cp -r _build/default/browser/sx_browser.bc.wasm.assets %s/shared/static/wasm/ && cp -r _build/default/browser/sx_browser.bc.wasm.assets browser/" abs_project abs_project abs_project abs_project abs_project
| "wasm" ->
let abs_project = if Filename.is_relative pd then Sys.getcwd () ^ "/" ^ pd else pd in
Printf.sprintf "cd %s && bash hosts/ocaml/browser/build-all.sh 2>&1" abs_project

View File

@@ -315,12 +315,12 @@ let resolve_library_path lib_spec =
The file should contain a define-library form that registers itself. *)
let _import_env : env option ref = ref None
let load_library_file path =
(* Use eval_expr which has the cek_run import patch — handles nested imports *)
let rec load_library_file path =
(* Use eval_expr_io for IO-aware loading (handles nested imports) *)
let env = match !_import_env with Some e -> e | None -> Sx_types.make_env () in
let exprs = Sx_parser.parse_file path in
List.iter (fun expr ->
try ignore (Sx_ref.eval_expr expr (Env env))
try ignore (eval_expr_io expr (Env env))
with Eval_error msg ->
Printf.eprintf "[load-library] %s: %s\n%!" (Filename.basename path) msg
) exprs
@@ -328,7 +328,7 @@ let load_library_file path =
(** IO-aware CEK run — handles suspension by dispatching IO requests.
Import requests are handled locally (load .sx file).
Other IO requests are sent to the Python bridge. *)
let cek_run_with_io state =
and cek_run_with_io state =
let s = ref state in
let is_terminal s = match Sx_ref.cek_terminal_p s with Bool true -> true | _ -> false in
let is_suspended s = match Sx_runtime.get_val s (String "phase") with String "io-suspended" -> true | _ -> false in
@@ -368,7 +368,7 @@ let cek_run_with_io state =
loop ()
(** IO-aware eval_expr — like eval_expr but handles IO suspension. *)
let _eval_expr_io expr env =
and eval_expr_io expr env =
let state = Sx_ref.make_cek_state expr env (List []) in
cek_run_with_io state
@@ -1009,7 +1009,7 @@ let rec dispatch env cmd =
ignore (Sx_types.env_bind env "*current-file*" (String path));
let count = ref 0 in
List.iter (fun expr ->
ignore (Sx_ref.eval_expr expr (Env env));
ignore (eval_expr_io expr (Env env));
incr count
) exprs;
(* Rebind host extension points after .sx load — evaluator.sx
@@ -2223,7 +2223,7 @@ let http_load_files env files =
try
let exprs = Sx_parser.parse_file path in
List.iter (fun expr ->
try ignore (Sx_ref.eval_expr expr (Env env))
try ignore (eval_expr_io expr (Env env))
with e -> Printf.eprintf "[http-load] %s: %s\n%!" (Filename.basename path) (Printexc.to_string e)
) exprs
with e -> Printf.eprintf "[http-load] parse error %s: %s\n%!" path (Printexc.to_string e)
@@ -3175,6 +3175,162 @@ let http_mode port =
Array.iter Domain.join workers)
(* --site mode: full site env (same setup as HTTP) + epoch protocol on stdin/stdout.
No HTTP server, no ports. Used by Playwright sandbox tests to render pages
as a pure function: URL → HTML via the render-page epoch command. *)
let site_mode () =
let env = make_server_env () in
http_setup_declarative_stubs env;
http_setup_platform_constructors env;
http_setup_page_helpers env;
(* Load all .sx files — same as http_mode *)
let project_dir = try Sys.getenv "SX_PROJECT_DIR" with Not_found ->
Sys.getcwd () in
let spec_base = project_dir ^ "/spec" in
let lib_base = project_dir ^ "/lib" in
let web_base = project_dir ^ "/web" in
let shared_sx = try Sys.getenv "SX_SHARED_DIR" with Not_found ->
let docker_path = project_dir ^ "/shared_sx" in
let dev_path = project_dir ^ "/shared/sx/templates" in
if Sys.file_exists docker_path then docker_path else dev_path in
let sx_sx = try Sys.getenv "SX_COMPONENTS_DIR" with Not_found ->
let docker_path = project_dir ^ "/components" in
let dev_path = project_dir ^ "/sx/sx" in
if Sys.file_exists docker_path then docker_path else dev_path in
let static_dir = try Sys.getenv "SX_STATIC_DIR" with Not_found ->
let docker_path = project_dir ^ "/static" in
let dev_path = project_dir ^ "/shared/static" in
if Sys.file_exists docker_path then docker_path else dev_path in
ignore (env_bind env "_project-dir" (String project_dir));
ignore (env_bind env "_spec-dir" (String spec_base));
ignore (env_bind env "_lib-dir" (String lib_base));
ignore (env_bind env "_web-dir" (String web_base));
_import_env := Some env;
let core_files = [
spec_base ^ "/parser.sx"; spec_base ^ "/render.sx"; spec_base ^ "/signals.sx";
lib_base ^ "/compiler.sx";
web_base ^ "/adapter-html.sx"; web_base ^ "/adapter-sx.sx";
web_base ^ "/io.sx"; web_base ^ "/web-forms.sx"; web_base ^ "/engine.sx";
web_base ^ "/request-handler.sx"; web_base ^ "/page-helpers.sx";
] in
http_load_files env core_files;
let skip_files = ["primitives.sx"; "types.sx"; "boundary.sx";
"harness.sx"; "eval-rules.sx"; "vm-inline.sx"] in
let skip_dirs = ["tests"; "test"; "plans"; "essays"; "spec"; "client-libs"] in
let rec load_dir dir =
if Sys.file_exists dir && Sys.is_directory dir then begin
let entries = Sys.readdir dir in
Array.sort String.compare entries;
Array.iter (fun f ->
let path = dir ^ "/" ^ f in
if Sys.is_directory path then begin
if not (List.mem f skip_dirs) then load_dir path
end
else if Filename.check_suffix f ".sx"
&& not (List.mem f skip_files)
&& not (String.length f > 5 && String.sub f 0 5 = "test-")
&& not (Filename.check_suffix f ".test.sx") then
http_load_files env [path]
) entries
end
in
load_dir lib_base;
load_dir shared_sx;
let sx_sxc = try Sys.getenv "SX_SXC_DIR" with Not_found ->
let docker_path = project_dir ^ "/sxc" in
let dev_path = project_dir ^ "/sx/sxc" in
if Sys.file_exists docker_path then docker_path else dev_path in
load_dir sx_sxc;
load_dir sx_sx;
(* IO registry + app config *)
(try match env_get env "__io-registry" with
| Dict registry ->
let batchable = Hashtbl.fold (fun name entry acc ->
match entry with
| Dict d -> (match Hashtbl.find_opt d "batchable" with
| Some (Bool true) -> name :: acc | _ -> acc)
| _ -> acc) registry [] in
if batchable <> [] then batchable_helpers := batchable
| _ -> ()
with _ -> ());
(try match env_get env "__app-config" with
| Dict d -> _app_config := Some d
| _ -> ()
with _ -> ());
(* SSR overrides *)
let bind name fn = ignore (env_bind env name (NativeFn (name, fn))) in
bind "effect" (fun _args -> Nil);
bind "register-in-scope" (fun _args -> Nil);
rebind_host_extensions env;
ignore (env_bind env "assoc" (NativeFn ("assoc", fun args ->
match args with
| Dict d :: rest ->
let d2 = Hashtbl.copy d in
let rec go = function
| [] -> Dict d2
| String k :: v :: rest -> Hashtbl.replace d2 k v; go rest
| Keyword k :: v :: rest -> Hashtbl.replace d2 k v; go rest
| _ -> raise (Eval_error "assoc: pairs")
in go rest
| _ -> raise (Eval_error "assoc: dict + pairs"))));
ignore (env_bind env "expand-components?" (NativeFn ("expand-components?", fun _args -> Bool true)));
(* Shell statics for render-page *)
http_inject_shell_statics env static_dir sx_sxc;
(* No JIT in site mode — the lazy JIT hook can loop on complex ASTs
(known bug: project_jit_bytecode_bug.md). Pure CEK is slower but
correct. First renders take ~2-5s, subsequent ~0.5-1s with caching. *)
Printf.eprintf "[site] Ready — epoch protocol on stdin/stdout\n%!";
send "(ready)";
(* nav-urls helper — walk sx-nav-tree, collect (href label) pairs *)
let nav_urls () =
let tree = env_get env "sx-nav-tree" in
let urls = ref [] in
let rec walk node = match node with
| Dict d ->
let href = match Hashtbl.find_opt d "href" with Some (String s) -> s | _ -> "" in
let label = match Hashtbl.find_opt d "label" with Some (String s) -> s | _ -> "" in
if href <> "" then urls := (href, label) :: !urls;
(match Hashtbl.find_opt d "children" with
| Some (List items) | Some (ListRef { contents = items }) ->
List.iter walk items
| _ -> ())
| _ -> ()
in
walk tree;
let items = List.rev !urls in
"(" ^ String.concat " " (List.map (fun (h, l) ->
Printf.sprintf "(\"%s\" \"%s\")" (escape_sx_string h) (escape_sx_string l)
) items) ^ ")"
in
(* Epoch protocol loop *)
(try
while true do
match read_line_blocking () with
| None -> exit 0
| Some line ->
let line = String.trim line in
if line = "" then ()
else begin
let exprs = Sx_parser.parse_all line in
match exprs with
| [List [Symbol "epoch"; Number n]] ->
current_epoch := int_of_float n
(* render-page: full SSR pipeline — URL → complete HTML *)
| [List [Symbol "render-page"; String path]] ->
(try match http_render_page env path [] with
| Some html -> send_ok_blob html
| None -> send_error ("render-page: no route for " ^ path)
with e -> send_error ("render-page: " ^ Printexc.to_string e))
(* nav-urls: flat list of (href label) from nav tree *)
| [List [Symbol "nav-urls"]] ->
(try send_ok_raw (nav_urls ())
with e -> send_error ("nav-urls: " ^ Printexc.to_string e))
| [cmd] -> dispatch env cmd
| _ -> send_error ("Expected single command, got " ^ string_of_int (List.length exprs))
end
done
with End_of_file -> ())
let () =
(* Check for CLI mode flags *)
let args = Array.to_list Sys.argv in
@@ -3182,6 +3338,7 @@ let () =
else if List.mem "--render" args then cli_mode "render"
else if List.mem "--aser-slot" args then cli_mode "aser-slot"
else if List.mem "--aser" args then cli_mode "aser"
else if List.mem "--site" args then site_mode ()
else if List.mem "--http" args then begin
(* Extract port: --http PORT *)
let port = ref 8014 in

View File

@@ -71,6 +71,11 @@ cp "$ROOT/shared/sx/templates/tw-layout.sx" "$DIST/sx/"
cp "$ROOT/shared/sx/templates/tw-type.sx" "$DIST/sx/"
cp "$ROOT/shared/sx/templates/tw.sx" "$DIST/sx/"
# 10. Hyperscript
for f in tokenizer parser compiler runtime integration; do
cp "$ROOT/lib/hyperscript/$f.sx" "$DIST/sx/hs-$f.sx"
done
# Summary
WASM_SIZE=$(du -sh "$DIST/sx_browser.bc.wasm.assets" | cut -f1)
JS_SIZE=$(du -sh "$DIST/sx_browser.bc.js" | cut -f1)

View File

@@ -80,7 +80,11 @@ const FILES = [
'dom.sx', 'browser.sx', 'adapter-html.sx', 'adapter-sx.sx', 'adapter-dom.sx',
'tw-layout.sx', 'tw-type.sx', 'tw.sx',
'boot-helpers.sx', 'hypersx.sx', 'harness.sx', 'harness-reactive.sx',
'harness-web.sx', 'engine.sx', 'orchestration.sx', 'boot.sx',
'harness-web.sx', 'engine.sx', 'orchestration.sx',
// Hyperscript modules — loaded on demand via transparent lazy loader
'hs-tokenizer.sx', 'hs-parser.sx', 'hs-compiler.sx', 'hs-runtime.sx',
'hs-integration.sx',
'boot.sx',
];
@@ -339,6 +343,18 @@ function libKey(spec) {
return spec.replace(/^\(/, '').replace(/\)$/, '');
}
// Extract top-level (define name ...) symbols from a non-library file
function extractDefines(source) {
const names = [];
const re = /^\(define\s+(\S+)/gm;
let m;
while ((m = re.exec(source)) !== null) {
const name = m[1];
if (name && !name.startsWith('(') && !name.startsWith(':')) names.push(name);
}
return names;
}
const manifest = {};
let entryFile = null;
@@ -360,6 +376,18 @@ for (const file of FILES) {
} else if (deps.length > 0) {
// Entry point (no define-library, has imports)
entryFile = { file: sxbcFile, deps: deps.map(libKey) };
} else {
// Non-library file (e.g. hyperscript modules) — extract top-level defines
// as exports so the transparent lazy loader can resolve symbols to files.
const defines = extractDefines(src);
if (defines.length > 0) {
const key = file.replace(/\.sx$/, '');
manifest[key] = {
file: sxbcFile,
deps: [],
exports: defines,
};
}
}
}

View File

@@ -0,0 +1,229 @@
#!/usr/bin/env node
// test_hs_repeat.js — Debug hyperscript repeat+wait continuation bug
//
// Runs the exact expression that fails in the browser:
// on click repeat 3 times add .active to me then wait 300ms
// then remove .active then wait 300ms end
//
// Uses the real WASM kernel with perform/resume_vm, NOT mock IO.
// Waits are shortened to 1ms. All IO suspensions are logged.
//
// Usage: node hosts/ocaml/browser/test_hs_repeat.js
const fs = require('fs');
const path = require('path');
const PROJECT_ROOT = path.resolve(__dirname, '../../..');
const WASM_DIR = path.join(PROJECT_ROOT, 'shared/static/wasm');
// --- DOM stubs with class tracking ---
function makeElement(tag) {
const el = {
tagName: tag, _attrs: {}, _children: [], _classes: new Set(),
style: {}, childNodes: [], children: [], textContent: '',
nodeType: 1,
classList: {
add(c) { el._classes.add(c); console.log(` [dom] classList.add("${c}") → {${[...el._classes]}}`); },
remove(c) { el._classes.delete(c); console.log(` [dom] classList.remove("${c}") → {${[...el._classes]}}`); },
contains(c) { return el._classes.has(c); },
toggle(c) { if (el._classes.has(c)) el._classes.delete(c); else el._classes.add(c); },
},
setAttribute(k, v) { el._attrs[k] = String(v); },
getAttribute(k) { return el._attrs[k] || null; },
removeAttribute(k) { delete el._attrs[k]; },
appendChild(c) { el._children.push(c); el.childNodes.push(c); el.children.push(c); return c; },
insertBefore(c) { el._children.push(c); el.childNodes.push(c); el.children.push(c); return c; },
removeChild(c) { return c; },
replaceChild(n) { return n; },
cloneNode() { return makeElement(tag); },
addEventListener() {}, removeEventListener() {}, dispatchEvent() {},
get innerHTML() {
return el._children.map(c => {
if (c._isText) return c.textContent || '';
if (c._isComment) return '<!--' + (c.textContent || '') + '-->';
return c.outerHTML || '';
}).join('');
},
set innerHTML(v) { el._children = []; el.childNodes = []; el.children = []; },
get outerHTML() {
let s = '<' + tag;
for (const k of Object.keys(el._attrs).sort()) s += ` ${k}="${el._attrs[k]}"`;
s += '>';
if (['br','hr','img','input','meta','link'].includes(tag)) return s;
return s + el.innerHTML + '</' + tag + '>';
},
dataset: new Proxy({}, {
get(_, k) { return el._attrs['data-' + k.replace(/[A-Z]/g, c => '-' + c.toLowerCase())]; },
set(_, k, v) { el._attrs['data-' + k.replace(/[A-Z]/g, c => '-' + c.toLowerCase())] = v; return true; }
}),
querySelectorAll() { return []; },
querySelector() { return null; },
};
return el;
}
global.window = global;
global.document = {
createElement: makeElement,
createDocumentFragment() { return makeElement('fragment'); },
head: makeElement('head'), body: makeElement('body'),
querySelector() { return null; }, querySelectorAll() { return []; },
createTextNode(s) { return { _isText: true, textContent: String(s), nodeType: 3 }; },
addEventListener() {},
createComment(s) { return { _isComment: true, textContent: s || '', nodeType: 8 }; },
getElementsByTagName() { return []; },
};
global.localStorage = { getItem() { return null; }, setItem() {}, removeItem() {} };
global.CustomEvent = class { constructor(n, o) { this.type = n; this.detail = (o || {}).detail || {}; } };
global.MutationObserver = class { observe() {} disconnect() {} };
global.requestIdleCallback = fn => setTimeout(fn, 0);
global.matchMedia = () => ({ matches: false });
global.navigator = { serviceWorker: { register() { return Promise.resolve(); } } };
global.location = { href: '', pathname: '/', hostname: 'localhost' };
global.history = { pushState() {}, replaceState() {} };
global.fetch = () => Promise.resolve({ ok: true, text() { return Promise.resolve(''); } });
global.XMLHttpRequest = class { open() {} send() {} };
async function main() {
// Load WASM kernel
require(path.join(WASM_DIR, 'sx_browser.bc.js'));
const K = globalThis.SxKernel;
if (!K) { console.error('FATAL: SxKernel not found'); process.exit(1); }
console.log('WASM kernel loaded');
// Register FFI primitives
K.registerNative('host-global', args => {
const name = args[0];
return (name in globalThis) ? globalThis[name] : null;
});
K.registerNative('host-get', args => {
const [obj, prop] = args;
if (obj == null) return null;
const v = obj[prop];
return v === undefined ? null : v;
});
K.registerNative('host-set!', args => { if (args[0] != null) args[0][args[1]] = args[2]; return args[2]; });
K.registerNative('host-call', args => {
const [obj, method, ...rest] = args;
if (obj == null || typeof obj[method] !== 'function') return null;
const r = obj[method].apply(obj, rest);
return r === undefined ? null : r;
});
K.registerNative('host-new', args => new (Function.prototype.bind.apply(args[0], [null, ...args.slice(1)])));
K.registerNative('host-callback', args => {
const fn = args[0];
return function() { return K.callFn(fn, Array.from(arguments)); };
});
K.registerNative('host-typeof', args => typeof args[0]);
K.registerNative('host-await', args => args[0]);
K.eval('(define SX_VERSION "test-hs-1.0")');
K.eval('(define SX_ENGINE "ocaml-vm-wasm-test")');
K.eval('(define parse sx-parse)');
K.eval('(define serialize sx-serialize)');
// Stub DOM primitives that HS runtime calls
// dom-listen fires handler immediately (simulates the event)
K.eval('(define dom-add-class (fn (el cls) (dict-set! (get el "classes") cls true) nil))');
K.eval('(define dom-remove-class (fn (el cls) (dict-delete! (get el "classes") cls) nil))');
K.eval('(define dom-has-class? (fn (el cls) (dict-has? (get el "classes") cls)))');
K.eval('(define dom-listen (fn (target event-name handler) (handler {:type event-name :target target})))');
// Load hyperscript modules
const hsFiles = [
'lib/hyperscript/tokenizer.sx',
'lib/hyperscript/parser.sx',
'lib/hyperscript/compiler.sx',
'lib/hyperscript/runtime.sx',
];
for (const f of hsFiles) {
const src = fs.readFileSync(path.join(PROJECT_ROOT, f), 'utf8');
const r = K.load(src);
if (typeof r === 'string' && r.startsWith('Error')) {
console.error(`Load failed: ${f}: ${r}`);
process.exit(1);
}
}
console.log('Hyperscript modules loaded');
// Compile the expression
const compiled = K.eval('(hs-to-sx-from-source "on click repeat 3 times add .active to me then wait 300ms then remove .active then wait 300ms end")');
console.log('Compiled:', K.eval(`(inspect '${typeof compiled === 'string' ? compiled : '?'})`));
// Actually get it as a string
const compiledStr = K.eval('(inspect (hs-to-sx-from-source "on click repeat 3 times add .active to me then wait 300ms then remove .active then wait 300ms end"))');
console.log('Compiled SX:', compiledStr);
// Create handler function (same as hs-handler does)
K.eval('(define _test-me {:tag "button" :id "test" :classes {} :_hs-activated true})');
// Build the handler — wraps compiled SX in (fn (me) (let ((it nil) (event ...)) <sx>))
const handlerSrc = K.eval('(inspect (hs-to-sx-from-source "on click repeat 3 times add .active to me then wait 300ms then remove .active then wait 300ms end"))');
K.eval(`(define _test-handler
(eval-expr
(list 'fn '(me)
(list 'let '((it nil) (event {:type "click" :target _test-me}))
(hs-to-sx-from-source "on click repeat 3 times add .active to me then wait 300ms then remove .active then wait 300ms end")))))`);
console.log('\n=== Invoking handler (simulates click event) ===');
console.log('Expected: 3 iterations × (add .active, wait 300, remove .active, wait 300)');
console.log('Expected: 6 IO suspensions total\n');
// Call the handler — this will suspend on the first hs-wait (perform)
let suspensionCount = 0;
let result;
try {
result = K.callFn(K.eval('_test-handler'), [K.eval('_test-me')]);
} catch(e) {
console.error('Initial call error:', e.message);
process.exit(1);
}
// Drive async suspension chain with real timeouts (1ms instead of 300ms)
function driveAsync(res) {
return new Promise((resolve) => {
function step(r) {
if (!r || !r.suspended) {
console.log(`\n=== Done. Total suspensions: ${suspensionCount} (expected: 6) ===`);
console.log(`Result: ${r === null ? 'null' : typeof r === 'object' ? JSON.stringify(r) : r}`);
resolve();
return;
}
suspensionCount++;
const req = r.request;
const items = req && (req.items || req);
const op = items && items[0];
const opName = typeof op === 'string' ? op : (op && op.name) || String(op);
const arg = items && items[1];
console.log(`Suspension #${suspensionCount}: op=${opName} arg=${arg}`);
if (opName === 'io-sleep' || opName === 'wait') {
// Resume after 1ms (not real 300ms)
setTimeout(() => {
try {
const resumed = r.resume(null);
console.log(` Resumed: suspended=${resumed && resumed.suspended}, type=${typeof resumed}`);
step(resumed);
} catch(e) {
console.error(` Resume error: ${e.message}`);
resolve();
}
}, 1);
} else {
console.log(` Unhandled IO op: ${opName}`);
resolve();
}
}
step(res);
});
}
await driveAsync(result);
// Check final element state
const classes = K.eval('(get _test-me "classes")');
console.log('\nFinal element classes:', JSON.stringify(classes));
}
main().catch(e => { console.error('FATAL:', e.message); process.exit(1); });