Step 5p6 lazy loading + Step 6b VM transpilation prep
Lazy module loading (Step 5 piece 6 completion): - Add define-library wrappers + import declarations to 13 source .sx files - compile-modules.js generates module-manifest.json with dependency graph - compile-modules.js strips define-library/import before bytecode compilation (VM doesn't handle these as special forms) - sx-platform.js replaces hardcoded 24-file loadWebStack() with manifest-driven recursive loader — only downloads modules the page needs - Result: 12 modules loaded (was 24), zero errors, zero warnings - Fallback to full load if manifest missing VM transpilation prep (Step 6b): - Refactor lib/vm.sx: 20 accessor functions replace raw dict access - Factor out collect-n-from-stack, collect-n-pairs, pad-n-nils helpers - bootstrap_vm.py: transpiles 9 VM logic functions to OCaml - sx_vm_ref.ml: proof that vm.sx transpiles (preamble has stubs) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -249,8 +249,51 @@
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a parsed SX code form ({_type:"list", items:[symbol"code", ...]})
|
||||
* into the dict format that K.loadModule / js_to_value expects.
|
||||
* Mirrors the OCaml convert_code/convert_const in sx_browser.ml.
|
||||
*/
|
||||
function convertCodeForm(form) {
|
||||
if (!form || form._type !== "list" || !form.items || !form.items.length) return null;
|
||||
var items = form.items;
|
||||
if (!items[0] || items[0]._type !== "symbol" || items[0].name !== "code") return null;
|
||||
|
||||
var d = { _type: "dict", arity: 0, "upvalue-count": 0 };
|
||||
for (var i = 1; i < items.length; i++) {
|
||||
var item = items[i];
|
||||
if (item && item._type === "keyword" && i + 1 < items.length) {
|
||||
var val = items[i + 1];
|
||||
if (item.name === "arity" || item.name === "upvalue-count") {
|
||||
d[item.name] = (typeof val === "number") ? val : 0;
|
||||
} else if (item.name === "bytecode" && val && val._type === "list") {
|
||||
d.bytecode = val; // {_type:"list", items:[numbers...]}
|
||||
} else if (item.name === "constants" && val && val._type === "list") {
|
||||
d.constants = { _type: "list", items: (val.items || []).map(convertConst) };
|
||||
}
|
||||
i++; // skip value
|
||||
}
|
||||
}
|
||||
return d;
|
||||
}
|
||||
|
||||
function convertConst(c) {
|
||||
if (!c || typeof c !== "object") return c; // number, string, boolean, null pass through
|
||||
if (c._type === "list" && c.items && c.items.length > 0) {
|
||||
var head = c.items[0];
|
||||
if (head && head._type === "symbol" && head.name === "code") {
|
||||
return convertCodeForm(c);
|
||||
}
|
||||
if (head && head._type === "symbol" && head.name === "list") {
|
||||
return { _type: "list", items: c.items.slice(1).map(convertConst) };
|
||||
}
|
||||
}
|
||||
return c; // symbols, keywords, etc. pass through
|
||||
}
|
||||
|
||||
/**
|
||||
* Try loading a pre-compiled .sxbc bytecode module (SX text format).
|
||||
* Uses K.loadModule which handles VM suspension (import requests).
|
||||
* Returns true on success, null on failure (caller falls back to .sx source).
|
||||
*/
|
||||
function loadBytecodeFile(path) {
|
||||
@@ -262,20 +305,90 @@
|
||||
xhr.send();
|
||||
if (xhr.status !== 200) return null;
|
||||
|
||||
window.__sxbcText = xhr.responseText;
|
||||
var result = K.eval('(load-sxbc (first (parse (host-global "__sxbcText"))))');
|
||||
delete window.__sxbcText;
|
||||
// Parse the sxbc text to get the SX tree
|
||||
var parsed = K.parse(xhr.responseText);
|
||||
if (!parsed || !parsed.length) return null;
|
||||
var sxbc = parsed[0]; // (sxbc version hash (code ...))
|
||||
if (!sxbc || sxbc._type !== "list" || !sxbc.items) return null;
|
||||
|
||||
// Extract the code form — 3rd or 4th item (after sxbc, version, optional hash)
|
||||
var codeForm = null;
|
||||
for (var i = 1; i < sxbc.items.length; i++) {
|
||||
var item = sxbc.items[i];
|
||||
if (item && item._type === "list" && item.items && item.items.length > 0 &&
|
||||
item.items[0] && item.items[0]._type === "symbol" && item.items[0].name === "code") {
|
||||
codeForm = item;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!codeForm) return null;
|
||||
|
||||
// Convert the SX code form to a dict for loadModule
|
||||
var moduleDict = convertCodeForm(codeForm);
|
||||
if (!moduleDict) return null;
|
||||
|
||||
// Load via K.loadModule which handles VmSuspended
|
||||
var result = K.loadModule(moduleDict);
|
||||
|
||||
// Handle import suspensions — fetch missing libraries on demand
|
||||
while (result && result.suspended && result.op === "import") {
|
||||
var req = result.request;
|
||||
var libName = req && req.library;
|
||||
if (libName) {
|
||||
// Try to find and load the library from the manifest
|
||||
var loaded = handleImportSuspension(libName);
|
||||
if (!loaded) {
|
||||
console.warn("[sx-platform] lazy import: library not found:", libName);
|
||||
}
|
||||
}
|
||||
// Resume the suspended module (null = library is now in env)
|
||||
result = result.resume(null);
|
||||
}
|
||||
|
||||
if (typeof result === 'string' && result.indexOf('Error') === 0) {
|
||||
console.warn("[sx-platform] bytecode FAIL " + path + ":", result);
|
||||
return null;
|
||||
}
|
||||
return true;
|
||||
} catch(e) {
|
||||
delete window.__sxbcText;
|
||||
console.warn("[sx-platform] bytecode FAIL " + path + ":", e.message || e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle an import suspension by finding and loading the library.
|
||||
* The library name may be an SX value (list/string) — normalize to manifest key.
|
||||
*/
|
||||
function handleImportSuspension(libSpec) {
|
||||
// libSpec from the kernel is the library name spec, e.g. {_type:"list", items:[{name:"sx"},{name:"dom"}]}
|
||||
// or a string like "sx dom"
|
||||
var key;
|
||||
if (typeof libSpec === "string") {
|
||||
key = libSpec;
|
||||
} else if (libSpec && libSpec._type === "list" && libSpec.items) {
|
||||
key = libSpec.items.map(function(item) {
|
||||
return (item && item.name) ? item.name : String(item);
|
||||
}).join(" ");
|
||||
} else if (libSpec && libSpec._type === "dict") {
|
||||
// Dict with key/name fields
|
||||
key = libSpec.key || libSpec.name || "";
|
||||
} else {
|
||||
key = String(libSpec);
|
||||
}
|
||||
|
||||
if (_loadedLibs[key]) return true; // already loaded
|
||||
|
||||
if (!_manifest) loadManifest();
|
||||
if (!_manifest || !_manifest[key]) {
|
||||
console.warn("[sx-platform] lazy import: unknown library key '" + key + "'");
|
||||
return false;
|
||||
}
|
||||
|
||||
// Load the library (and its deps) on demand
|
||||
return loadLibrary(key, {});
|
||||
}
|
||||
|
||||
/**
|
||||
* Load an .sx file synchronously via XHR (boot-time only).
|
||||
* Returns the number of expressions loaded, or an error string.
|
||||
@@ -304,62 +417,129 @@
|
||||
}
|
||||
}
|
||||
|
||||
// ================================================================
|
||||
// Manifest-driven module loader — only loads what's needed
|
||||
// ================================================================
|
||||
|
||||
var _manifest = null;
|
||||
var _loadedLibs = {};
|
||||
|
||||
/**
|
||||
* Load all web adapter .sx files in dependency order.
|
||||
* Tries pre-compiled bytecode first, falls back to source.
|
||||
* Fetch and parse the module manifest (library deps + file paths).
|
||||
*/
|
||||
function loadManifest() {
|
||||
if (_manifest) return _manifest;
|
||||
try {
|
||||
var xhr = new XMLHttpRequest();
|
||||
xhr.open("GET", _baseUrl + "sx/module-manifest.json" + _cacheBust, false);
|
||||
xhr.send();
|
||||
if (xhr.status === 200) {
|
||||
_manifest = JSON.parse(xhr.responseText);
|
||||
return _manifest;
|
||||
}
|
||||
} catch(e) {}
|
||||
console.warn("[sx-platform] No manifest found, falling back to full load");
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Load a single library and all its dependencies (recursive).
|
||||
* Cycle-safe: tracks in-progress loads to break circular deps.
|
||||
* Functions in cyclic modules resolve symbols at call time via global env.
|
||||
*/
|
||||
function loadLibrary(name, loading) {
|
||||
if (_loadedLibs[name]) return true;
|
||||
if (loading[name]) return true; // cycle — skip
|
||||
loading[name] = true;
|
||||
|
||||
var info = _manifest[name];
|
||||
if (!info) {
|
||||
console.warn("[sx-platform] Unknown library: " + name);
|
||||
return false;
|
||||
}
|
||||
|
||||
// Resolve deps first
|
||||
for (var i = 0; i < info.deps.length; i++) {
|
||||
loadLibrary(info.deps[i], loading);
|
||||
}
|
||||
|
||||
// Load this module
|
||||
var ok = loadBytecodeFile("sx/" + info.file);
|
||||
if (!ok) {
|
||||
var sxFile = info.file.replace(/\.sxbc$/, '.sx');
|
||||
ok = loadSxFile("sx/" + sxFile);
|
||||
}
|
||||
_loadedLibs[name] = true;
|
||||
return !!ok;
|
||||
}
|
||||
|
||||
/**
|
||||
* Load web stack using the module manifest.
|
||||
* Only downloads libraries that the entry point transitively depends on.
|
||||
*/
|
||||
function loadWebStack() {
|
||||
var files = [
|
||||
// Spec modules
|
||||
"sx/render.sx",
|
||||
"sx/core-signals.sx",
|
||||
"sx/signals.sx",
|
||||
"sx/deps.sx",
|
||||
"sx/router.sx",
|
||||
"sx/page-helpers.sx",
|
||||
// Freeze scope (signal persistence) + highlight (syntax coloring)
|
||||
"sx/freeze.sx",
|
||||
"sx/highlight.sx",
|
||||
// Bytecode compiler + VM
|
||||
"sx/bytecode.sx",
|
||||
"sx/compiler.sx",
|
||||
"sx/vm.sx",
|
||||
// Web libraries (use 8 FFI primitives)
|
||||
"sx/dom.sx",
|
||||
"sx/browser.sx",
|
||||
// Web adapters
|
||||
"sx/adapter-html.sx",
|
||||
"sx/adapter-sx.sx",
|
||||
"sx/adapter-dom.sx",
|
||||
// Boot helpers (platform functions in pure SX)
|
||||
"sx/boot-helpers.sx",
|
||||
"sx/hypersx.sx",
|
||||
// Test harness (for inline test runners)
|
||||
"sx/harness.sx",
|
||||
"sx/harness-reactive.sx",
|
||||
"sx/harness-web.sx",
|
||||
// Web framework
|
||||
"sx/engine.sx",
|
||||
"sx/orchestration.sx",
|
||||
"sx/boot.sx",
|
||||
];
|
||||
var manifest = loadManifest();
|
||||
if (!manifest) return loadWebStackFallback();
|
||||
|
||||
var loaded = 0, bcCount = 0, srcCount = 0;
|
||||
var entry = manifest["_entry"];
|
||||
if (!entry) {
|
||||
console.warn("[sx-platform] No _entry in manifest, falling back");
|
||||
return loadWebStackFallback();
|
||||
}
|
||||
|
||||
var loading = {};
|
||||
var t0 = performance.now();
|
||||
if (K.beginModuleLoad) K.beginModuleLoad();
|
||||
|
||||
// Load all entry point deps recursively
|
||||
for (var i = 0; i < entry.deps.length; i++) {
|
||||
loadLibrary(entry.deps[i], loading);
|
||||
}
|
||||
|
||||
// Load entry point itself (boot.sx — not a library, just defines + init)
|
||||
loadBytecodeFile("sx/" + entry.file) || loadSxFile("sx/" + entry.file.replace(/\.sxbc$/, '.sx'));
|
||||
|
||||
if (K.endModuleLoad) K.endModuleLoad();
|
||||
var count = Object.keys(_loadedLibs).length + 1; // +1 for entry
|
||||
var dt = Math.round(performance.now() - t0);
|
||||
console.log("[sx-platform] Loaded " + count + " modules in " + dt + "ms (manifest-driven)");
|
||||
}
|
||||
|
||||
/**
|
||||
* Fallback: load all files in hardcoded order (pre-manifest compat).
|
||||
*/
|
||||
function loadWebStackFallback() {
|
||||
var files = [
|
||||
"sx/render.sx", "sx/core-signals.sx", "sx/signals.sx", "sx/deps.sx",
|
||||
"sx/router.sx", "sx/page-helpers.sx", "sx/freeze.sx", "sx/highlight.sx",
|
||||
"sx/bytecode.sx", "sx/compiler.sx", "sx/vm.sx", "sx/dom.sx", "sx/browser.sx",
|
||||
"sx/adapter-html.sx", "sx/adapter-sx.sx", "sx/adapter-dom.sx",
|
||||
"sx/boot-helpers.sx", "sx/hypersx.sx", "sx/harness.sx",
|
||||
"sx/harness-reactive.sx", "sx/harness-web.sx",
|
||||
"sx/engine.sx", "sx/orchestration.sx", "sx/boot.sx",
|
||||
];
|
||||
if (K.beginModuleLoad) K.beginModuleLoad();
|
||||
for (var i = 0; i < files.length; i++) {
|
||||
var r = loadBytecodeFile(files[i]);
|
||||
if (r) { bcCount++; continue; }
|
||||
// Bytecode not available — end batch, load source, restart batch
|
||||
if (K.endModuleLoad) K.endModuleLoad();
|
||||
r = loadSxFile(files[i]);
|
||||
if (typeof r === "number") { loaded += r; srcCount++; }
|
||||
if (K.beginModuleLoad) K.beginModuleLoad();
|
||||
if (!loadBytecodeFile(files[i])) loadSxFile(files[i]);
|
||||
}
|
||||
if (K.endModuleLoad) K.endModuleLoad();
|
||||
console.log("[sx-platform] Loaded " + files.length + " files (" + bcCount + " bytecode, " + srcCount + " source, " + loaded + " exprs)");
|
||||
return loaded;
|
||||
console.log("[sx-platform] Loaded " + files.length + " files (fallback)");
|
||||
}
|
||||
|
||||
/**
|
||||
* Load an optional library on demand (e.g., highlight, harness).
|
||||
* Can be called after boot for pages that need extra modules.
|
||||
*/
|
||||
globalThis.__sxLoadLibrary = function(name) {
|
||||
if (!_manifest) loadManifest();
|
||||
if (!_manifest) return false;
|
||||
if (_loadedLibs[name]) return true;
|
||||
if (K.beginModuleLoad) K.beginModuleLoad();
|
||||
var ok = loadLibrary(name, {});
|
||||
if (K.endModuleLoad) K.endModuleLoad();
|
||||
return ok;
|
||||
};
|
||||
|
||||
// ================================================================
|
||||
// Compatibility shim — expose Sx global matching current JS API
|
||||
// ================================================================
|
||||
|
||||
Reference in New Issue
Block a user