WIP: bytecode module pre-compilation and loading infrastructure
- compile-modules.js: Node.js build tool, all 23 .sx files compile to .sxbc.json
- api_load_module with shared globals (beginModuleLoad/endModuleLoad batch API)
- api_compile_module for runtime compilation
- sx-platform.js: bytecode-first loader with source fallback, JSON deserializer
- Deferred JIT enable (setTimeout after boot)
Known issues:
- WASM browser: loadModule loads but functions not accessible (env writeback
issue with interned keys)
- WASM browser: compileModule fails ("Not callable: nil" — compile-module
function from bytecode not working correctly in WASM context)
- Node.js js_of_ocaml: full roundtrip works (compile → load → call)
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -197,6 +197,70 @@
|
||||
}
|
||||
})();
|
||||
|
||||
/**
|
||||
* Deserialize type-tagged JSON constant back to JS value for loadModule.
|
||||
*/
|
||||
function deserializeConstant(c) {
|
||||
if (!c || !c.t) return null;
|
||||
switch (c.t) {
|
||||
case 's': return c.v;
|
||||
case 'n': return c.v;
|
||||
case 'b': return c.v;
|
||||
case 'nil': return null;
|
||||
case 'sym': return { _type: 'symbol', name: c.v };
|
||||
case 'kw': return { _type: 'keyword', name: c.v };
|
||||
case 'list': return { _type: 'list', items: (c.v || []).map(deserializeConstant) };
|
||||
case 'code': return {
|
||||
_type: 'dict',
|
||||
bytecode: { _type: 'list', items: c.v.bytecode },
|
||||
constants: { _type: 'list', items: (c.v.constants || []).map(deserializeConstant) },
|
||||
arity: c.v.arity || 0,
|
||||
'upvalue-count': c.v['upvalue-count'] || 0,
|
||||
locals: c.v.locals || 0,
|
||||
};
|
||||
case 'dict': {
|
||||
var d = { _type: 'dict' };
|
||||
for (var k in c.v) d[k] = deserializeConstant(c.v[k]);
|
||||
return d;
|
||||
}
|
||||
default: return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Try loading a pre-compiled .sxbc.json bytecode module.
|
||||
* Returns true on success, null on failure (caller falls back to .sx source).
|
||||
*/
|
||||
function loadBytecodeFile(path) {
|
||||
var bcPath = path.replace(/\.sx$/, '.sxbc.json');
|
||||
var url = _baseUrl + bcPath + _cacheBust;
|
||||
try {
|
||||
var xhr = new XMLHttpRequest();
|
||||
xhr.open("GET", url, false);
|
||||
xhr.send();
|
||||
if (xhr.status !== 200) return null;
|
||||
|
||||
var json = JSON.parse(xhr.responseText);
|
||||
if (!json.module || json.magic !== 'SXBC') return null;
|
||||
|
||||
var module = {
|
||||
_type: 'dict',
|
||||
bytecode: { _type: 'list', items: json.module.bytecode },
|
||||
constants: { _type: 'list', items: json.module.constants.map(deserializeConstant) },
|
||||
};
|
||||
|
||||
var result = K.loadModule(module);
|
||||
if (typeof result === 'string' && result.indexOf('Error') === 0) {
|
||||
console.warn("[sx-platform] bytecode FAIL " + path + ":", result);
|
||||
return null;
|
||||
}
|
||||
console.log("[sx-platform] ok " + path + " (bytecode)");
|
||||
return true;
|
||||
} catch(e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Load an .sx file synchronously via XHR (boot-time only).
|
||||
* Returns the number of expressions loaded, or an error string.
|
||||
@@ -227,7 +291,7 @@
|
||||
|
||||
/**
|
||||
* Load all web adapter .sx files in dependency order.
|
||||
* Called after the 8 FFI primitives are registered.
|
||||
* Tries pre-compiled bytecode first, falls back to source.
|
||||
*/
|
||||
function loadWebStack() {
|
||||
var files = [
|
||||
@@ -264,12 +328,19 @@
|
||||
"sx/boot.sx",
|
||||
];
|
||||
|
||||
var loaded = 0;
|
||||
var loaded = 0, bcCount = 0, srcCount = 0;
|
||||
if (K.beginModuleLoad) K.beginModuleLoad();
|
||||
for (var i = 0; i < files.length; i++) {
|
||||
var r = loadSxFile(files[i]);
|
||||
if (typeof r === "number") loaded += r;
|
||||
var r = loadBytecodeFile(files[i]);
|
||||
if (r) { bcCount++; continue; }
|
||||
// Bytecode not available — end batch, load source, restart batch
|
||||
if (K.endModuleLoad) K.endModuleLoad();
|
||||
r = loadSxFile(files[i]);
|
||||
if (typeof r === "number") { loaded += r; srcCount++; }
|
||||
if (K.beginModuleLoad) K.beginModuleLoad();
|
||||
}
|
||||
console.log("[sx-platform] Loaded " + loaded + " expressions from " + files.length + " files");
|
||||
if (K.endModuleLoad) K.endModuleLoad();
|
||||
console.log("[sx-platform] Loaded " + files.length + " files (" + bcCount + " bytecode, " + srcCount + " source, " + loaded + " exprs)");
|
||||
return loaded;
|
||||
}
|
||||
|
||||
@@ -306,8 +377,8 @@
|
||||
// Debug islands
|
||||
console.log("[sx] ~home/stepper defined?", K.eval("(type-of ~home/stepper)"));
|
||||
console.log("[sx] ~layouts/header defined?", K.eval("(type-of ~layouts/header)"));
|
||||
// Try manual island query
|
||||
console.log("[sx] manual island query:", K.eval("(len (dom-query-all (dom-body) \"[data-sx-island]\"))"));
|
||||
// Island count (JS-side, avoids VM overhead)
|
||||
console.log("[sx] manual island query:", document.querySelectorAll("[data-sx-island]").length);
|
||||
// Try hydrating again
|
||||
console.log("[sx] retry hydrate-islands...");
|
||||
K.eval("(sx-hydrate-islands nil)");
|
||||
@@ -339,7 +410,8 @@
|
||||
var _doInit = function() {
|
||||
loadWebStack();
|
||||
Sx.init();
|
||||
K.eval('(enable-jit!)');
|
||||
// Enable JIT after all boot code has run
|
||||
setTimeout(function() { K.eval('(enable-jit!)'); }, 0);
|
||||
};
|
||||
|
||||
if (document.readyState === "loading") {
|
||||
|
||||
@@ -231,17 +231,46 @@ let api_load src_js =
|
||||
| Eval_error msg -> Js.Unsafe.inject (Js.string ("Error: " ^ msg))
|
||||
| Parse_error msg -> Js.Unsafe.inject (Js.string ("Parse error: " ^ msg))
|
||||
|
||||
(* Shared globals table for batch module loading.
|
||||
Created by beginModuleLoad, accumulated across loadModule calls,
|
||||
flushed to env by endModuleLoad. Ensures closures from early
|
||||
modules can see definitions from later modules. *)
|
||||
let _module_globals : (string, value) Hashtbl.t option ref = ref None
|
||||
|
||||
let api_begin_module_load () =
|
||||
let g = Hashtbl.create 512 in
|
||||
Hashtbl.iter (fun id v -> Hashtbl.replace g (unintern id) v) global_env.bindings;
|
||||
_module_globals := Some g;
|
||||
Js.Unsafe.inject true
|
||||
|
||||
let api_end_module_load () =
|
||||
(match !_module_globals with
|
||||
| Some g ->
|
||||
Hashtbl.iter (fun k v ->
|
||||
Hashtbl.replace global_env.bindings (intern k) v
|
||||
) g;
|
||||
_module_globals := None
|
||||
| None -> ());
|
||||
Js.Unsafe.inject true
|
||||
|
||||
let api_load_module module_js =
|
||||
try
|
||||
let code_val = js_to_value module_js in
|
||||
let code = Sx_vm.code_from_value code_val in
|
||||
let globals = Hashtbl.create 256 in
|
||||
Hashtbl.iter (fun id v -> Hashtbl.replace globals (unintern id) v) global_env.bindings;
|
||||
let globals = match !_module_globals with
|
||||
| Some g -> g (* use shared table *)
|
||||
| None ->
|
||||
(* standalone mode: create temp table *)
|
||||
let g = Hashtbl.create 256 in
|
||||
Hashtbl.iter (fun id v -> Hashtbl.replace g (unintern id) v) global_env.bindings;
|
||||
g
|
||||
in
|
||||
let _result = Sx_vm.execute_module code globals in
|
||||
(* Copy all globals back into env — new defines + unchanged values *)
|
||||
Hashtbl.iter (fun k v ->
|
||||
Hashtbl.replace global_env.bindings (intern k) v
|
||||
) globals;
|
||||
(* If standalone (no batch), copy back immediately *)
|
||||
if !_module_globals = None then
|
||||
Hashtbl.iter (fun k v ->
|
||||
Hashtbl.replace global_env.bindings (intern k) v
|
||||
) globals;
|
||||
Js.Unsafe.inject (Hashtbl.length globals)
|
||||
with
|
||||
| Eval_error msg -> Js.Unsafe.inject (Js.string ("Error: " ^ msg))
|
||||
@@ -653,6 +682,8 @@ let () =
|
||||
Js.Unsafe.set sx (Js.string "renderToHtml") (Js.wrap_callback api_render_to_html);
|
||||
Js.Unsafe.set sx (Js.string "load") (Js.wrap_callback api_load);
|
||||
Js.Unsafe.set sx (Js.string "loadModule") (Js.wrap_callback api_load_module);
|
||||
Js.Unsafe.set sx (Js.string "beginModuleLoad") (Js.wrap_callback (fun () -> api_begin_module_load ()));
|
||||
Js.Unsafe.set sx (Js.string "endModuleLoad") (Js.wrap_callback (fun () -> api_end_module_load ()));
|
||||
Js.Unsafe.set sx (Js.string "compileModule") (wrap api_compile_module);
|
||||
Js.Unsafe.set sx (Js.string "typeOf") (Js.wrap_callback api_type_of);
|
||||
Js.Unsafe.set sx (Js.string "inspect") (Js.wrap_callback api_inspect);
|
||||
|
||||
Reference in New Issue
Block a user