WIP: bytecode module pre-compilation and loading infrastructure

- compile-modules.js: Node.js build tool, all 23 .sx files compile to .sxbc.json
- api_load_module with shared globals (beginModuleLoad/endModuleLoad batch API)
- api_compile_module for runtime compilation
- sx-platform.js: bytecode-first loader with source fallback, JSON deserializer
- Deferred JIT enable (setTimeout after boot)

Known issues:
- WASM browser: loadModule loads but functions not accessible (env writeback
  issue with interned keys)
- WASM browser: compileModule fails ("Not callable: nil" — compile-module
  function from bytecode not working correctly in WASM context)
- Node.js js_of_ocaml: full roundtrip works (compile → load → call)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-03-26 17:54:55 +00:00
parent cb7bbc9557
commit 00de248ee9
32 changed files with 351 additions and 28 deletions

View File

@@ -197,6 +197,70 @@
}
})();
/**
* Deserialize type-tagged JSON constant back to JS value for loadModule.
*/
function deserializeConstant(c) {
if (!c || !c.t) return null;
switch (c.t) {
case 's': return c.v;
case 'n': return c.v;
case 'b': return c.v;
case 'nil': return null;
case 'sym': return { _type: 'symbol', name: c.v };
case 'kw': return { _type: 'keyword', name: c.v };
case 'list': return { _type: 'list', items: (c.v || []).map(deserializeConstant) };
case 'code': return {
_type: 'dict',
bytecode: { _type: 'list', items: c.v.bytecode },
constants: { _type: 'list', items: (c.v.constants || []).map(deserializeConstant) },
arity: c.v.arity || 0,
'upvalue-count': c.v['upvalue-count'] || 0,
locals: c.v.locals || 0,
};
case 'dict': {
var d = { _type: 'dict' };
for (var k in c.v) d[k] = deserializeConstant(c.v[k]);
return d;
}
default: return null;
}
}
/**
* Try loading a pre-compiled .sxbc.json bytecode module.
* Returns true on success, null on failure (caller falls back to .sx source).
*/
function loadBytecodeFile(path) {
var bcPath = path.replace(/\.sx$/, '.sxbc.json');
var url = _baseUrl + bcPath + _cacheBust;
try {
var xhr = new XMLHttpRequest();
xhr.open("GET", url, false);
xhr.send();
if (xhr.status !== 200) return null;
var json = JSON.parse(xhr.responseText);
if (!json.module || json.magic !== 'SXBC') return null;
var module = {
_type: 'dict',
bytecode: { _type: 'list', items: json.module.bytecode },
constants: { _type: 'list', items: json.module.constants.map(deserializeConstant) },
};
var result = K.loadModule(module);
if (typeof result === 'string' && result.indexOf('Error') === 0) {
console.warn("[sx-platform] bytecode FAIL " + path + ":", result);
return null;
}
console.log("[sx-platform] ok " + path + " (bytecode)");
return true;
} catch(e) {
return null;
}
}
/**
* Load an .sx file synchronously via XHR (boot-time only).
* Returns the number of expressions loaded, or an error string.
@@ -227,7 +291,7 @@
/**
* Load all web adapter .sx files in dependency order.
* Called after the 8 FFI primitives are registered.
* Tries pre-compiled bytecode first, falls back to source.
*/
function loadWebStack() {
var files = [
@@ -264,12 +328,19 @@
"sx/boot.sx",
];
var loaded = 0;
var loaded = 0, bcCount = 0, srcCount = 0;
if (K.beginModuleLoad) K.beginModuleLoad();
for (var i = 0; i < files.length; i++) {
var r = loadSxFile(files[i]);
if (typeof r === "number") loaded += r;
var r = loadBytecodeFile(files[i]);
if (r) { bcCount++; continue; }
// Bytecode not available — end batch, load source, restart batch
if (K.endModuleLoad) K.endModuleLoad();
r = loadSxFile(files[i]);
if (typeof r === "number") { loaded += r; srcCount++; }
if (K.beginModuleLoad) K.beginModuleLoad();
}
console.log("[sx-platform] Loaded " + loaded + " expressions from " + files.length + " files");
if (K.endModuleLoad) K.endModuleLoad();
console.log("[sx-platform] Loaded " + files.length + " files (" + bcCount + " bytecode, " + srcCount + " source, " + loaded + " exprs)");
return loaded;
}
@@ -306,8 +377,8 @@
// Debug islands
console.log("[sx] ~home/stepper defined?", K.eval("(type-of ~home/stepper)"));
console.log("[sx] ~layouts/header defined?", K.eval("(type-of ~layouts/header)"));
// Try manual island query
console.log("[sx] manual island query:", K.eval("(len (dom-query-all (dom-body) \"[data-sx-island]\"))"));
// Island count (JS-side, avoids VM overhead)
console.log("[sx] manual island query:", document.querySelectorAll("[data-sx-island]").length);
// Try hydrating again
console.log("[sx] retry hydrate-islands...");
K.eval("(sx-hydrate-islands nil)");
@@ -339,7 +410,8 @@
var _doInit = function() {
loadWebStack();
Sx.init();
K.eval('(enable-jit!)');
// Enable JIT after all boot code has run
setTimeout(function() { K.eval('(enable-jit!)'); }, 0);
};
if (document.readyState === "loading") {

View File

@@ -231,17 +231,46 @@ let api_load src_js =
| Eval_error msg -> Js.Unsafe.inject (Js.string ("Error: " ^ msg))
| Parse_error msg -> Js.Unsafe.inject (Js.string ("Parse error: " ^ msg))
(* Shared globals table for batch module loading.
Created by beginModuleLoad, accumulated across loadModule calls,
flushed to env by endModuleLoad. Ensures closures from early
modules can see definitions from later modules. *)
let _module_globals : (string, value) Hashtbl.t option ref = ref None
let api_begin_module_load () =
let g = Hashtbl.create 512 in
Hashtbl.iter (fun id v -> Hashtbl.replace g (unintern id) v) global_env.bindings;
_module_globals := Some g;
Js.Unsafe.inject true
let api_end_module_load () =
(match !_module_globals with
| Some g ->
Hashtbl.iter (fun k v ->
Hashtbl.replace global_env.bindings (intern k) v
) g;
_module_globals := None
| None -> ());
Js.Unsafe.inject true
let api_load_module module_js =
try
let code_val = js_to_value module_js in
let code = Sx_vm.code_from_value code_val in
let globals = Hashtbl.create 256 in
Hashtbl.iter (fun id v -> Hashtbl.replace globals (unintern id) v) global_env.bindings;
let globals = match !_module_globals with
| Some g -> g (* use shared table *)
| None ->
(* standalone mode: create temp table *)
let g = Hashtbl.create 256 in
Hashtbl.iter (fun id v -> Hashtbl.replace g (unintern id) v) global_env.bindings;
g
in
let _result = Sx_vm.execute_module code globals in
(* Copy all globals back into env — new defines + unchanged values *)
Hashtbl.iter (fun k v ->
Hashtbl.replace global_env.bindings (intern k) v
) globals;
(* If standalone (no batch), copy back immediately *)
if !_module_globals = None then
Hashtbl.iter (fun k v ->
Hashtbl.replace global_env.bindings (intern k) v
) globals;
Js.Unsafe.inject (Hashtbl.length globals)
with
| Eval_error msg -> Js.Unsafe.inject (Js.string ("Error: " ^ msg))
@@ -653,6 +682,8 @@ let () =
Js.Unsafe.set sx (Js.string "renderToHtml") (Js.wrap_callback api_render_to_html);
Js.Unsafe.set sx (Js.string "load") (Js.wrap_callback api_load);
Js.Unsafe.set sx (Js.string "loadModule") (Js.wrap_callback api_load_module);
Js.Unsafe.set sx (Js.string "beginModuleLoad") (Js.wrap_callback (fun () -> api_begin_module_load ()));
Js.Unsafe.set sx (Js.string "endModuleLoad") (Js.wrap_callback (fun () -> api_end_module_load ()));
Js.Unsafe.set sx (Js.string "compileModule") (wrap api_compile_module);
Js.Unsafe.set sx (Js.string "typeOf") (Js.wrap_callback api_type_of);
Js.Unsafe.set sx (Js.string "inspect") (Js.wrap_callback api_inspect);