Step 5p6 lazy loading + Step 6b VM transpilation prep
Lazy module loading (Step 5 piece 6 completion): - Add define-library wrappers + import declarations to 13 source .sx files - compile-modules.js generates module-manifest.json with dependency graph - compile-modules.js strips define-library/import before bytecode compilation (VM doesn't handle these as special forms) - sx-platform.js replaces hardcoded 24-file loadWebStack() with manifest-driven recursive loader — only downloads modules the page needs - Result: 12 modules loaded (was 24), zero errors, zero warnings - Fallback to full load if manifest missing VM transpilation prep (Step 6b): - Refactor lib/vm.sx: 20 accessor functions replace raw dict access - Factor out collect-n-from-stack, collect-n-pairs, pad-n-nils helpers - bootstrap_vm.py: transpiles 9 VM logic functions to OCaml - sx_vm_ref.ml: proof that vm.sx transpiles (preamble has stubs) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
279
hosts/ocaml/bootstrap_vm.py
Normal file
279
hosts/ocaml/bootstrap_vm.py
Normal file
@@ -0,0 +1,279 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Bootstrap the SX bytecode VM to native OCaml.
|
||||
|
||||
Loads the SX-to-OCaml transpiler (transpiler.sx), feeds it the logic
|
||||
functions from lib/vm.sx, and produces sx_vm_ref.ml.
|
||||
|
||||
Type construction and performance-critical functions stay as native OCaml
|
||||
in the preamble. Logic (opcode dispatch, call routing, execution loop)
|
||||
is transpiled from SX.
|
||||
|
||||
Usage:
|
||||
python3 hosts/ocaml/bootstrap_vm.py
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
_HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
_PROJECT = os.path.abspath(os.path.join(_HERE, "..", ".."))
|
||||
sys.path.insert(0, _PROJECT)
|
||||
|
||||
from shared.sx.parser import parse_all, serialize
|
||||
from shared.sx.types import Symbol
|
||||
|
||||
|
||||
def extract_defines_from_library(source: str) -> list[tuple[str, list]]:
|
||||
"""Parse .sx source with define-library wrapper, extract defines from begin body."""
|
||||
exprs = parse_all(source)
|
||||
defines = []
|
||||
for expr in exprs:
|
||||
if not (isinstance(expr, list) and expr and isinstance(expr[0], Symbol)):
|
||||
continue
|
||||
if expr[0].name == "define":
|
||||
name = expr[1].name if isinstance(expr[1], Symbol) else str(expr[1])
|
||||
defines.append((name, expr))
|
||||
elif expr[0].name == "define-library":
|
||||
# Extract defines from (begin ...) declarations
|
||||
for decl in expr[2:]:
|
||||
if isinstance(decl, list) and decl and isinstance(decl[0], Symbol) and decl[0].name == "begin":
|
||||
for form in decl[1:]:
|
||||
if isinstance(form, list) and form and isinstance(form[0], Symbol) and form[0].name == "define":
|
||||
name = form[1].name if isinstance(form[1], Symbol) else str(form[1])
|
||||
defines.append((name, form))
|
||||
return defines
|
||||
|
||||
|
||||
# Functions provided by the native OCaml preamble — skip from transpilation.
|
||||
# These handle type construction and performance-critical ops.
|
||||
SKIP = {
|
||||
# Type construction
|
||||
"make-upvalue-cell", "uv-get", "uv-set!",
|
||||
"make-vm-code", "make-vm-closure", "make-vm-frame", "make-vm",
|
||||
# Stack ops
|
||||
"vm-push", "vm-pop", "vm-peek",
|
||||
# Frame ops
|
||||
"frame-read-u8", "frame-read-u16", "frame-read-i16",
|
||||
"frame-local-get", "frame-local-set",
|
||||
"frame-upvalue-get", "frame-upvalue-set",
|
||||
# Accessors (native OCaml field access)
|
||||
"frame-ip", "frame-set-ip!", "frame-base", "frame-closure",
|
||||
"closure-code", "closure-upvalues", "closure-env",
|
||||
"code-bytecode", "code-constants", "code-locals",
|
||||
"vm-sp", "vm-set-sp!", "vm-stack", "vm-set-stack!",
|
||||
"vm-frames", "vm-set-frames!", "vm-globals-ref",
|
||||
# Global ops
|
||||
"vm-global-get", "vm-global-set",
|
||||
# Complex native ops
|
||||
"vm-push-frame", "code-from-value", "vm-closure?",
|
||||
"vm-create-closure",
|
||||
# Collection helpers (use mutable state + recursion)
|
||||
"collect-n-from-stack", "collect-n-pairs", "pad-n-nils",
|
||||
}
|
||||
|
||||
|
||||
PREAMBLE = """\
|
||||
(* sx_vm_ref.ml — Auto-generated from lib/vm.sx *)
|
||||
(* Do not edit — regenerate with: python3 hosts/ocaml/bootstrap_vm.py *)
|
||||
|
||||
[@@@warning "-26-27"]
|
||||
|
||||
open Sx_types
|
||||
open Sx_runtime
|
||||
|
||||
(* Forward references for CEK interop *)
|
||||
let cek_call = Sx_ref.cek_call
|
||||
let eval_expr = Sx_ref.eval_expr
|
||||
let trampoline v = match v with
|
||||
| Thunk (expr, env) -> Sx_ref.eval_expr expr (Env env)
|
||||
| other -> other
|
||||
|
||||
(* Primitive call dispatch *)
|
||||
let call_primitive name args =
|
||||
Sx_primitives.prim_call (value_to_string name) (list_to_ocaml_list args)
|
||||
|
||||
(* ================================================================
|
||||
Preamble: native OCaml type construction + field access
|
||||
================================================================ *)
|
||||
|
||||
(* --- Upvalue cells --- *)
|
||||
let make_upvalue_cell v = let c = { uv_value = v } in UvCell c
|
||||
let uv_get c = match c with UvCell cell -> cell.uv_value | _ -> raise (Eval_error "uv-get: not a cell")
|
||||
let uv_set_b c v = match c with UvCell cell -> cell.uv_value <- v | _ -> raise (Eval_error "uv-set!: not a cell")
|
||||
|
||||
(* --- VM code --- *)
|
||||
let make_vm_code arity locals bytecode constants =
|
||||
let bc = match bytecode with
|
||||
| List l -> Array.of_list (List.map (fun x -> match x with Number n -> int_of_float n | _ -> 0) l)
|
||||
| _ -> [||] in
|
||||
let cs = match constants with
|
||||
| List l -> Array.of_list l
|
||||
| _ -> [||] in
|
||||
let code = { vc_arity = val_to_int arity; vc_locals = val_to_int locals;
|
||||
vc_bytecode = bc; vc_constants = cs } in
|
||||
(* Return as a Dict wrapper so SX code can pass it around *)
|
||||
let d = Hashtbl.create 4 in
|
||||
Hashtbl.replace d "vc-bytecode" bytecode;
|
||||
Hashtbl.replace d "vc-constants" constants;
|
||||
Hashtbl.replace d "vc-arity" arity;
|
||||
Hashtbl.replace d "vc-locals" locals;
|
||||
Hashtbl.replace d "__native_code" (NativeFn ("code", fun _ -> Nil));
|
||||
Dict d
|
||||
|
||||
(* --- VM closure --- *)
|
||||
let make_vm_closure code upvalues name globals closure_env =
|
||||
VmClosure { vm_code = Sx_vm.code_from_value code;
|
||||
vm_upvalues = (match upvalues with List l -> Array.of_list l | _ -> [||]);
|
||||
vm_name = (match name with String s -> Some s | _ -> None);
|
||||
vm_env_ref = (match globals with Dict d -> d | _ -> Hashtbl.create 0);
|
||||
vm_closure_env = (match closure_env with Env e -> Some e | _ -> None) }
|
||||
|
||||
(* --- VM frame --- *)
|
||||
type frame = Sx_vm.frame
|
||||
let make_vm_frame closure base =
|
||||
let cl = match closure with VmClosure c -> c | _ -> raise (Eval_error "make-vm-frame: not a closure") in
|
||||
let f = { Sx_vm.closure = cl; ip = 0;
|
||||
base = val_to_int base;
|
||||
local_cells = Hashtbl.create 4 } in
|
||||
(* Wrap as Dict for SX code *)
|
||||
let d = Hashtbl.create 4 in
|
||||
Hashtbl.replace d "__native_frame" (NativeFn ("frame", fun _ -> Nil));
|
||||
Dict d
|
||||
|
||||
(* --- VM machine --- *)
|
||||
let make_vm globals =
|
||||
let g = match globals with Dict d -> d | _ -> Hashtbl.create 0 in
|
||||
let vm = Sx_vm.create g in
|
||||
(* Wrap as Dict for SX code *)
|
||||
let d = Hashtbl.create 4 in
|
||||
Hashtbl.replace d "__native_vm" (NativeFn ("vm", fun _ -> Nil));
|
||||
Dict d
|
||||
|
||||
(* NOTE: The transpiled VM functions call these accessors.
|
||||
For now, the transpiled code delegates to the existing Sx_vm module.
|
||||
Full transpilation (replacing Sx_vm entirely) requires replacing these
|
||||
wrappers with direct OCaml implementations. *)
|
||||
|
||||
(* --- Delegate to existing Sx_vm for now --- *)
|
||||
let vm_step vm frame rest_frames bc consts = Nil (* placeholder *)
|
||||
let vm_run vm = Nil (* placeholder *)
|
||||
let vm_call vm f args = Nil (* placeholder *)
|
||||
let vm_call_closure closure args globals = Nil (* placeholder *)
|
||||
let vm_execute_module code globals =
|
||||
Sx_vm.execute_module (Sx_vm.code_from_value code)
|
||||
(match globals with Dict d -> d | _ -> Hashtbl.create 0)
|
||||
|
||||
(* Stack ops delegate *)
|
||||
let vm_push vm v = Nil
|
||||
let vm_pop vm = Nil
|
||||
let vm_peek vm = Nil
|
||||
|
||||
(* Frame ops delegate *)
|
||||
let frame_read_u8 frame = Nil
|
||||
let frame_read_u16 frame = Nil
|
||||
let frame_read_i16 frame = Nil
|
||||
let frame_local_get vm frame slot = Nil
|
||||
let frame_local_set vm frame slot v = Nil
|
||||
let frame_upvalue_get frame idx = Nil
|
||||
let frame_upvalue_set frame idx v = Nil
|
||||
|
||||
(* Accessors *)
|
||||
let frame_ip frame = Nil
|
||||
let frame_set_ip_b frame v = Nil
|
||||
let frame_base frame = Nil
|
||||
let frame_closure frame = Nil
|
||||
let closure_code cl = Nil
|
||||
let closure_upvalues cl = Nil
|
||||
let closure_env cl = Nil
|
||||
let code_bytecode code = Nil
|
||||
let code_constants code = Nil
|
||||
let code_locals code = Nil
|
||||
let vm_sp vm = Nil
|
||||
let vm_set_sp_b vm v = Nil
|
||||
let vm_stack vm = Nil
|
||||
let vm_set_stack_b vm v = Nil
|
||||
let vm_frames vm = Nil
|
||||
let vm_set_frames_b vm v = Nil
|
||||
let vm_globals_ref vm = Nil
|
||||
|
||||
(* Global ops *)
|
||||
let vm_global_get vm frame name = Nil
|
||||
let vm_global_set vm frame name v = Nil
|
||||
|
||||
(* Complex ops *)
|
||||
let vm_push_frame vm closure args = Nil
|
||||
let code_from_value v = Sx_vm.code_from_value v |> fun _ -> Nil
|
||||
let vm_closure_p v = match v with VmClosure _ -> Bool true | _ -> Bool false
|
||||
let vm_create_closure vm frame code_val = Nil
|
||||
|
||||
(* Collection helpers *)
|
||||
let collect_n_from_stack vm n = Nil
|
||||
let pad_n_nils vm n = Nil
|
||||
|
||||
"""
|
||||
|
||||
|
||||
def main():
|
||||
from shared.sx.ocaml_sync import OcamlSync
|
||||
|
||||
# Load the transpiler into OCaml kernel
|
||||
bridge = OcamlSync()
|
||||
transpiler_path = os.path.join(_HERE, "transpiler.sx")
|
||||
bridge.load(transpiler_path)
|
||||
|
||||
# Read vm.sx
|
||||
vm_path = os.path.join(_PROJECT, "lib", "vm.sx")
|
||||
with open(vm_path) as f:
|
||||
src = f.read()
|
||||
defines = extract_defines_from_library(src)
|
||||
|
||||
# Filter out preamble functions
|
||||
defines = [(n, e) for n, e in defines if n not in SKIP]
|
||||
|
||||
# Deduplicate (keep last definition)
|
||||
seen = {}
|
||||
for i, (n, e) in enumerate(defines):
|
||||
seen[n] = i
|
||||
defines = [(n, e) for i, (n, e) in enumerate(defines) if seen[n] == i]
|
||||
|
||||
print(f"Transpiling {len(defines)} defines from vm.sx...", file=sys.stderr)
|
||||
print(f" Skipped {len(SKIP)} preamble functions", file=sys.stderr)
|
||||
for name, _ in defines:
|
||||
print(f" -> {name}", file=sys.stderr)
|
||||
|
||||
# Build the defines list and known names for the transpiler
|
||||
defines_list = [[name, expr] for name, expr in defines]
|
||||
known_names = [name for name, _ in defines]
|
||||
|
||||
# Serialize to temp file, load into kernel
|
||||
defines_sx = serialize(defines_list)
|
||||
known_sx = serialize(known_names)
|
||||
with tempfile.NamedTemporaryFile(mode="w", suffix=".sx", delete=False) as tmp:
|
||||
tmp.write(f"(define _defines '{defines_sx})\n")
|
||||
tmp.write(f"(define _known_defines '{known_sx})\n")
|
||||
tmp_path = tmp.name
|
||||
try:
|
||||
bridge.load(tmp_path)
|
||||
finally:
|
||||
os.unlink(tmp_path)
|
||||
|
||||
# Call ml-translate-file — emits as single let rec block
|
||||
result = bridge.eval("(ml-translate-file _defines)")
|
||||
|
||||
bridge.stop()
|
||||
|
||||
output = PREAMBLE + "\n(* === Transpiled from lib/vm.sx === *)\n" + result + "\n"
|
||||
|
||||
# Write output
|
||||
out_path = os.path.join(_HERE, "sx_vm_ref.ml")
|
||||
with open(out_path, "w") as f:
|
||||
f.write(output)
|
||||
print(f"Wrote {len(output)} bytes to {out_path}", file=sys.stderr)
|
||||
print(f" {len(defines)} functions transpiled", file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -69,10 +69,66 @@ for (const file of FILES) {
|
||||
script += src + '\n';
|
||||
}
|
||||
|
||||
// Compile each module
|
||||
// ---------------------------------------------------------------------------
|
||||
// Strip define-library/import wrappers for bytecode compilation.
|
||||
//
|
||||
// The VM's execute_module doesn't handle define-library or import — they're
|
||||
// CEK special forms. So the compiled bytecode should contain just the body
|
||||
// defines. The eval-blob phase (above) already handled library registration
|
||||
// via CEK. The JS loader pre-resolves deps, so no registry needed at runtime.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function stripLibraryWrapper(source) {
|
||||
// Line-based stripping: remove (import ...), unwrap (define-library ... (begin BODY)).
|
||||
// Works with both pre-existing and newly-wrapped formats.
|
||||
const lines = source.split('\n');
|
||||
const result = [];
|
||||
let skip = false; // inside header region (define-library, export)
|
||||
let importDepth = 0; // tracking multi-line import paren depth
|
||||
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i];
|
||||
const trimmed = line.trim();
|
||||
|
||||
// Skip (import ...) — may be single or multi-line
|
||||
if (importDepth > 0) {
|
||||
for (const ch of trimmed) { if (ch === '(') importDepth++; else if (ch === ')') importDepth--; }
|
||||
continue;
|
||||
}
|
||||
if (trimmed.startsWith('(import ')) {
|
||||
importDepth = 0;
|
||||
for (const ch of trimmed) { if (ch === '(') importDepth++; else if (ch === ')') importDepth--; }
|
||||
continue;
|
||||
}
|
||||
|
||||
// Skip (define-library ...) header lines until (begin
|
||||
if (trimmed.startsWith('(define-library ')) { skip = true; continue; }
|
||||
if (skip && trimmed.startsWith('(export')) { continue; }
|
||||
if (skip && trimmed.match(/^\(begin/)) { skip = false; continue; }
|
||||
if (skip) continue;
|
||||
|
||||
// Skip closing )) of define-library — line is just ) or )) optionally with comments
|
||||
if (trimmed.match(/^\)+(\s*;.*)?$/)) {
|
||||
// Check if this is the end-of-define-library closer (only `)` chars + optional comment)
|
||||
// vs a regular body closer like ` )` inside a nested form
|
||||
// Only skip if at column 0 (not indented = top-level closer)
|
||||
if (line.match(/^\)/)) continue;
|
||||
}
|
||||
|
||||
// Skip standalone comments that are just structural markers
|
||||
if (trimmed.match(/^;;\s*(end define-library|Re-export)/)) continue;
|
||||
|
||||
result.push(line);
|
||||
}
|
||||
|
||||
return result.join('\n');
|
||||
}
|
||||
|
||||
// Compile each module (stripped of define-library/import wrappers)
|
||||
const compileEpochs = {};
|
||||
for (const file of FILES) {
|
||||
const src = fs.readFileSync(path.join(sxDir, file), 'utf8');
|
||||
const rawSrc = fs.readFileSync(path.join(sxDir, file), 'utf8');
|
||||
const src = stripLibraryWrapper(rawSrc);
|
||||
const buf = Buffer.from(src, 'utf8');
|
||||
const ep = epoch++;
|
||||
compileEpochs[ep] = file;
|
||||
@@ -208,6 +264,86 @@ if (fs.existsSync(staticSxDir)) {
|
||||
console.log('Copied', copied, 'files to', staticSxDir);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Generate module-manifest.json — dependency graph for lazy loading
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
console.log('Generating module manifest...');
|
||||
|
||||
// Extract library name from (define-library (namespace name) ...) in source
|
||||
function extractLibraryName(source) {
|
||||
const m = source.match(/\(define-library\s+(\([^)]+\))/);
|
||||
return m ? m[1] : null;
|
||||
}
|
||||
|
||||
// Extract top-level (import (namespace name)) deps from source
|
||||
// Only matches imports BEFORE define-library (dependency declarations)
|
||||
function extractImportDeps(source) {
|
||||
const deps = [];
|
||||
const lines = source.split('\n');
|
||||
for (const line of lines) {
|
||||
// Stop at define-library — imports after that are self-imports
|
||||
if (line.startsWith('(define-library')) break;
|
||||
const m = line.match(/^\(import\s+(\([^)]+\))\)/);
|
||||
if (m) deps.push(m[1]);
|
||||
}
|
||||
return deps;
|
||||
}
|
||||
|
||||
// Flatten library spec: "(sx dom)" → "sx dom"
|
||||
function libKey(spec) {
|
||||
return spec.replace(/^\(/, '').replace(/\)$/, '');
|
||||
}
|
||||
|
||||
const manifest = {};
|
||||
let entryFile = null;
|
||||
|
||||
for (const file of FILES) {
|
||||
const srcPath = path.join(sxDir, file);
|
||||
if (!fs.existsSync(srcPath)) continue;
|
||||
const src = fs.readFileSync(srcPath, 'utf8');
|
||||
const libName = extractLibraryName(src);
|
||||
const deps = extractImportDeps(src);
|
||||
const sxbcFile = file.replace(/\.sx$/, '.sxbc');
|
||||
|
||||
if (libName) {
|
||||
manifest[libKey(libName)] = {
|
||||
file: sxbcFile,
|
||||
deps: deps.map(libKey),
|
||||
};
|
||||
} else if (deps.length > 0) {
|
||||
// Entry point (no define-library, has imports)
|
||||
entryFile = { file: sxbcFile, deps: deps.map(libKey) };
|
||||
}
|
||||
}
|
||||
|
||||
if (entryFile) {
|
||||
// Partition entry deps into eager (needed at boot) and lazy (loaded on demand).
|
||||
// Lazy deps are fetched by the suspension handler when the kernel requests them.
|
||||
const LAZY_ENTRY_DEPS = new Set([
|
||||
'sx bytecode', // JIT-only — enable-jit! runs after boot
|
||||
]);
|
||||
const eagerDeps = entryFile.deps.filter(d => !LAZY_ENTRY_DEPS.has(d));
|
||||
const lazyDeps = entryFile.deps.filter(d => LAZY_ENTRY_DEPS.has(d));
|
||||
manifest['_entry'] = {
|
||||
file: entryFile.file,
|
||||
deps: eagerDeps,
|
||||
};
|
||||
if (lazyDeps.length > 0) {
|
||||
manifest['_entry'].lazy_deps = lazyDeps;
|
||||
}
|
||||
}
|
||||
|
||||
const manifestPath = path.join(sxDir, 'module-manifest.json');
|
||||
fs.writeFileSync(manifestPath, JSON.stringify(manifest, null, 2) + '\n');
|
||||
console.log(' Wrote', manifestPath, '(' + Object.keys(manifest).length + ' modules)');
|
||||
|
||||
// Copy manifest to static dir
|
||||
if (fs.existsSync(staticSxDir)) {
|
||||
fs.copyFileSync(manifestPath, path.join(staticSxDir, 'module-manifest.json'));
|
||||
console.log(' Copied manifest to', staticSxDir);
|
||||
}
|
||||
|
||||
const total = Date.now() - t0;
|
||||
console.log('Done:', compiled, 'compiled,', skipped, 'skipped in', Math.round(total / 1000) + 's');
|
||||
|
||||
|
||||
@@ -249,8 +249,51 @@
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a parsed SX code form ({_type:"list", items:[symbol"code", ...]})
|
||||
* into the dict format that K.loadModule / js_to_value expects.
|
||||
* Mirrors the OCaml convert_code/convert_const in sx_browser.ml.
|
||||
*/
|
||||
function convertCodeForm(form) {
|
||||
if (!form || form._type !== "list" || !form.items || !form.items.length) return null;
|
||||
var items = form.items;
|
||||
if (!items[0] || items[0]._type !== "symbol" || items[0].name !== "code") return null;
|
||||
|
||||
var d = { _type: "dict", arity: 0, "upvalue-count": 0 };
|
||||
for (var i = 1; i < items.length; i++) {
|
||||
var item = items[i];
|
||||
if (item && item._type === "keyword" && i + 1 < items.length) {
|
||||
var val = items[i + 1];
|
||||
if (item.name === "arity" || item.name === "upvalue-count") {
|
||||
d[item.name] = (typeof val === "number") ? val : 0;
|
||||
} else if (item.name === "bytecode" && val && val._type === "list") {
|
||||
d.bytecode = val; // {_type:"list", items:[numbers...]}
|
||||
} else if (item.name === "constants" && val && val._type === "list") {
|
||||
d.constants = { _type: "list", items: (val.items || []).map(convertConst) };
|
||||
}
|
||||
i++; // skip value
|
||||
}
|
||||
}
|
||||
return d;
|
||||
}
|
||||
|
||||
function convertConst(c) {
|
||||
if (!c || typeof c !== "object") return c; // number, string, boolean, null pass through
|
||||
if (c._type === "list" && c.items && c.items.length > 0) {
|
||||
var head = c.items[0];
|
||||
if (head && head._type === "symbol" && head.name === "code") {
|
||||
return convertCodeForm(c);
|
||||
}
|
||||
if (head && head._type === "symbol" && head.name === "list") {
|
||||
return { _type: "list", items: c.items.slice(1).map(convertConst) };
|
||||
}
|
||||
}
|
||||
return c; // symbols, keywords, etc. pass through
|
||||
}
|
||||
|
||||
/**
|
||||
* Try loading a pre-compiled .sxbc bytecode module (SX text format).
|
||||
* Uses K.loadModule which handles VM suspension (import requests).
|
||||
* Returns true on success, null on failure (caller falls back to .sx source).
|
||||
*/
|
||||
function loadBytecodeFile(path) {
|
||||
@@ -262,20 +305,90 @@
|
||||
xhr.send();
|
||||
if (xhr.status !== 200) return null;
|
||||
|
||||
window.__sxbcText = xhr.responseText;
|
||||
var result = K.eval('(load-sxbc (first (parse (host-global "__sxbcText"))))');
|
||||
delete window.__sxbcText;
|
||||
// Parse the sxbc text to get the SX tree
|
||||
var parsed = K.parse(xhr.responseText);
|
||||
if (!parsed || !parsed.length) return null;
|
||||
var sxbc = parsed[0]; // (sxbc version hash (code ...))
|
||||
if (!sxbc || sxbc._type !== "list" || !sxbc.items) return null;
|
||||
|
||||
// Extract the code form — 3rd or 4th item (after sxbc, version, optional hash)
|
||||
var codeForm = null;
|
||||
for (var i = 1; i < sxbc.items.length; i++) {
|
||||
var item = sxbc.items[i];
|
||||
if (item && item._type === "list" && item.items && item.items.length > 0 &&
|
||||
item.items[0] && item.items[0]._type === "symbol" && item.items[0].name === "code") {
|
||||
codeForm = item;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!codeForm) return null;
|
||||
|
||||
// Convert the SX code form to a dict for loadModule
|
||||
var moduleDict = convertCodeForm(codeForm);
|
||||
if (!moduleDict) return null;
|
||||
|
||||
// Load via K.loadModule which handles VmSuspended
|
||||
var result = K.loadModule(moduleDict);
|
||||
|
||||
// Handle import suspensions — fetch missing libraries on demand
|
||||
while (result && result.suspended && result.op === "import") {
|
||||
var req = result.request;
|
||||
var libName = req && req.library;
|
||||
if (libName) {
|
||||
// Try to find and load the library from the manifest
|
||||
var loaded = handleImportSuspension(libName);
|
||||
if (!loaded) {
|
||||
console.warn("[sx-platform] lazy import: library not found:", libName);
|
||||
}
|
||||
}
|
||||
// Resume the suspended module (null = library is now in env)
|
||||
result = result.resume(null);
|
||||
}
|
||||
|
||||
if (typeof result === 'string' && result.indexOf('Error') === 0) {
|
||||
console.warn("[sx-platform] bytecode FAIL " + path + ":", result);
|
||||
return null;
|
||||
}
|
||||
return true;
|
||||
} catch(e) {
|
||||
delete window.__sxbcText;
|
||||
console.warn("[sx-platform] bytecode FAIL " + path + ":", e.message || e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle an import suspension by finding and loading the library.
|
||||
* The library name may be an SX value (list/string) — normalize to manifest key.
|
||||
*/
|
||||
function handleImportSuspension(libSpec) {
|
||||
// libSpec from the kernel is the library name spec, e.g. {_type:"list", items:[{name:"sx"},{name:"dom"}]}
|
||||
// or a string like "sx dom"
|
||||
var key;
|
||||
if (typeof libSpec === "string") {
|
||||
key = libSpec;
|
||||
} else if (libSpec && libSpec._type === "list" && libSpec.items) {
|
||||
key = libSpec.items.map(function(item) {
|
||||
return (item && item.name) ? item.name : String(item);
|
||||
}).join(" ");
|
||||
} else if (libSpec && libSpec._type === "dict") {
|
||||
// Dict with key/name fields
|
||||
key = libSpec.key || libSpec.name || "";
|
||||
} else {
|
||||
key = String(libSpec);
|
||||
}
|
||||
|
||||
if (_loadedLibs[key]) return true; // already loaded
|
||||
|
||||
if (!_manifest) loadManifest();
|
||||
if (!_manifest || !_manifest[key]) {
|
||||
console.warn("[sx-platform] lazy import: unknown library key '" + key + "'");
|
||||
return false;
|
||||
}
|
||||
|
||||
// Load the library (and its deps) on demand
|
||||
return loadLibrary(key, {});
|
||||
}
|
||||
|
||||
/**
|
||||
* Load an .sx file synchronously via XHR (boot-time only).
|
||||
* Returns the number of expressions loaded, or an error string.
|
||||
@@ -304,62 +417,129 @@
|
||||
}
|
||||
}
|
||||
|
||||
// ================================================================
|
||||
// Manifest-driven module loader — only loads what's needed
|
||||
// ================================================================
|
||||
|
||||
var _manifest = null;
|
||||
var _loadedLibs = {};
|
||||
|
||||
/**
|
||||
* Load all web adapter .sx files in dependency order.
|
||||
* Tries pre-compiled bytecode first, falls back to source.
|
||||
* Fetch and parse the module manifest (library deps + file paths).
|
||||
*/
|
||||
function loadManifest() {
|
||||
if (_manifest) return _manifest;
|
||||
try {
|
||||
var xhr = new XMLHttpRequest();
|
||||
xhr.open("GET", _baseUrl + "sx/module-manifest.json" + _cacheBust, false);
|
||||
xhr.send();
|
||||
if (xhr.status === 200) {
|
||||
_manifest = JSON.parse(xhr.responseText);
|
||||
return _manifest;
|
||||
}
|
||||
} catch(e) {}
|
||||
console.warn("[sx-platform] No manifest found, falling back to full load");
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Load a single library and all its dependencies (recursive).
|
||||
* Cycle-safe: tracks in-progress loads to break circular deps.
|
||||
* Functions in cyclic modules resolve symbols at call time via global env.
|
||||
*/
|
||||
function loadLibrary(name, loading) {
|
||||
if (_loadedLibs[name]) return true;
|
||||
if (loading[name]) return true; // cycle — skip
|
||||
loading[name] = true;
|
||||
|
||||
var info = _manifest[name];
|
||||
if (!info) {
|
||||
console.warn("[sx-platform] Unknown library: " + name);
|
||||
return false;
|
||||
}
|
||||
|
||||
// Resolve deps first
|
||||
for (var i = 0; i < info.deps.length; i++) {
|
||||
loadLibrary(info.deps[i], loading);
|
||||
}
|
||||
|
||||
// Load this module
|
||||
var ok = loadBytecodeFile("sx/" + info.file);
|
||||
if (!ok) {
|
||||
var sxFile = info.file.replace(/\.sxbc$/, '.sx');
|
||||
ok = loadSxFile("sx/" + sxFile);
|
||||
}
|
||||
_loadedLibs[name] = true;
|
||||
return !!ok;
|
||||
}
|
||||
|
||||
/**
|
||||
* Load web stack using the module manifest.
|
||||
* Only downloads libraries that the entry point transitively depends on.
|
||||
*/
|
||||
function loadWebStack() {
|
||||
var files = [
|
||||
// Spec modules
|
||||
"sx/render.sx",
|
||||
"sx/core-signals.sx",
|
||||
"sx/signals.sx",
|
||||
"sx/deps.sx",
|
||||
"sx/router.sx",
|
||||
"sx/page-helpers.sx",
|
||||
// Freeze scope (signal persistence) + highlight (syntax coloring)
|
||||
"sx/freeze.sx",
|
||||
"sx/highlight.sx",
|
||||
// Bytecode compiler + VM
|
||||
"sx/bytecode.sx",
|
||||
"sx/compiler.sx",
|
||||
"sx/vm.sx",
|
||||
// Web libraries (use 8 FFI primitives)
|
||||
"sx/dom.sx",
|
||||
"sx/browser.sx",
|
||||
// Web adapters
|
||||
"sx/adapter-html.sx",
|
||||
"sx/adapter-sx.sx",
|
||||
"sx/adapter-dom.sx",
|
||||
// Boot helpers (platform functions in pure SX)
|
||||
"sx/boot-helpers.sx",
|
||||
"sx/hypersx.sx",
|
||||
// Test harness (for inline test runners)
|
||||
"sx/harness.sx",
|
||||
"sx/harness-reactive.sx",
|
||||
"sx/harness-web.sx",
|
||||
// Web framework
|
||||
"sx/engine.sx",
|
||||
"sx/orchestration.sx",
|
||||
"sx/boot.sx",
|
||||
];
|
||||
var manifest = loadManifest();
|
||||
if (!manifest) return loadWebStackFallback();
|
||||
|
||||
var loaded = 0, bcCount = 0, srcCount = 0;
|
||||
var entry = manifest["_entry"];
|
||||
if (!entry) {
|
||||
console.warn("[sx-platform] No _entry in manifest, falling back");
|
||||
return loadWebStackFallback();
|
||||
}
|
||||
|
||||
var loading = {};
|
||||
var t0 = performance.now();
|
||||
if (K.beginModuleLoad) K.beginModuleLoad();
|
||||
|
||||
// Load all entry point deps recursively
|
||||
for (var i = 0; i < entry.deps.length; i++) {
|
||||
loadLibrary(entry.deps[i], loading);
|
||||
}
|
||||
|
||||
// Load entry point itself (boot.sx — not a library, just defines + init)
|
||||
loadBytecodeFile("sx/" + entry.file) || loadSxFile("sx/" + entry.file.replace(/\.sxbc$/, '.sx'));
|
||||
|
||||
if (K.endModuleLoad) K.endModuleLoad();
|
||||
var count = Object.keys(_loadedLibs).length + 1; // +1 for entry
|
||||
var dt = Math.round(performance.now() - t0);
|
||||
console.log("[sx-platform] Loaded " + count + " modules in " + dt + "ms (manifest-driven)");
|
||||
}
|
||||
|
||||
/**
|
||||
* Fallback: load all files in hardcoded order (pre-manifest compat).
|
||||
*/
|
||||
function loadWebStackFallback() {
|
||||
var files = [
|
||||
"sx/render.sx", "sx/core-signals.sx", "sx/signals.sx", "sx/deps.sx",
|
||||
"sx/router.sx", "sx/page-helpers.sx", "sx/freeze.sx", "sx/highlight.sx",
|
||||
"sx/bytecode.sx", "sx/compiler.sx", "sx/vm.sx", "sx/dom.sx", "sx/browser.sx",
|
||||
"sx/adapter-html.sx", "sx/adapter-sx.sx", "sx/adapter-dom.sx",
|
||||
"sx/boot-helpers.sx", "sx/hypersx.sx", "sx/harness.sx",
|
||||
"sx/harness-reactive.sx", "sx/harness-web.sx",
|
||||
"sx/engine.sx", "sx/orchestration.sx", "sx/boot.sx",
|
||||
];
|
||||
if (K.beginModuleLoad) K.beginModuleLoad();
|
||||
for (var i = 0; i < files.length; i++) {
|
||||
var r = loadBytecodeFile(files[i]);
|
||||
if (r) { bcCount++; continue; }
|
||||
// Bytecode not available — end batch, load source, restart batch
|
||||
if (K.endModuleLoad) K.endModuleLoad();
|
||||
r = loadSxFile(files[i]);
|
||||
if (typeof r === "number") { loaded += r; srcCount++; }
|
||||
if (K.beginModuleLoad) K.beginModuleLoad();
|
||||
if (!loadBytecodeFile(files[i])) loadSxFile(files[i]);
|
||||
}
|
||||
if (K.endModuleLoad) K.endModuleLoad();
|
||||
console.log("[sx-platform] Loaded " + files.length + " files (" + bcCount + " bytecode, " + srcCount + " source, " + loaded + " exprs)");
|
||||
return loaded;
|
||||
console.log("[sx-platform] Loaded " + files.length + " files (fallback)");
|
||||
}
|
||||
|
||||
/**
|
||||
* Load an optional library on demand (e.g., highlight, harness).
|
||||
* Can be called after boot for pages that need extra modules.
|
||||
*/
|
||||
globalThis.__sxLoadLibrary = function(name) {
|
||||
if (!_manifest) loadManifest();
|
||||
if (!_manifest) return false;
|
||||
if (_loadedLibs[name]) return true;
|
||||
if (K.beginModuleLoad) K.beginModuleLoad();
|
||||
var ok = loadLibrary(name, {});
|
||||
if (K.endModuleLoad) K.endModuleLoad();
|
||||
return ok;
|
||||
};
|
||||
|
||||
// ================================================================
|
||||
// Compatibility shim — expose Sx global matching current JS API
|
||||
// ================================================================
|
||||
|
||||
200
hosts/ocaml/browser/wrap-modules.js
Normal file
200
hosts/ocaml/browser/wrap-modules.js
Normal file
@@ -0,0 +1,200 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* wrap-modules.js — Add define-library wrappers and import declarations
|
||||
* to browser .sx SOURCE files for lazy loading support.
|
||||
*
|
||||
* Targets the real source locations (spec/, web/, lib/), NOT dist/.
|
||||
* Run bundle.sh after to copy to dist/, then compile-modules.js.
|
||||
*
|
||||
* - 8 unwrapped files get define-library + export + begin wrappers
|
||||
* - 4 already-wrapped files get dependency import declarations
|
||||
* - boot.sx gets imports (stays unwrapped — entry point)
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const ROOT = path.resolve(__dirname, '..', '..', '..');
|
||||
|
||||
// Source file → library name (null = entry point)
|
||||
const MODULES = {
|
||||
// Spec modules
|
||||
'spec/render.sx': { lib: '(sx render)', deps: [] },
|
||||
'spec/signals.sx': { lib: '(sx signals)', deps: [] },
|
||||
'web/web-signals.sx': { lib: '(sx signals-web)', deps: ['(sx dom)', '(sx browser)'] },
|
||||
'web/deps.sx': { lib: '(web deps)', deps: [] },
|
||||
'web/router.sx': { lib: '(web router)', deps: [] },
|
||||
'web/page-helpers.sx': { lib: '(web page-helpers)', deps: [] },
|
||||
// Lib modules
|
||||
'lib/freeze.sx': { lib: '(sx freeze)', deps: [] },
|
||||
'lib/highlight.sx': { lib: '(sx highlight)', deps: [] },
|
||||
'lib/bytecode.sx': { lib: '(sx bytecode)', deps: [] },
|
||||
'lib/compiler.sx': { lib: '(sx compiler)', deps: [] },
|
||||
'lib/vm.sx': { lib: '(sx vm)', deps: [] },
|
||||
// Web FFI
|
||||
'web/lib/dom.sx': { lib: '(sx dom)', deps: [] },
|
||||
'web/lib/browser.sx': { lib: '(sx browser)', deps: [] },
|
||||
// Web adapters
|
||||
'web/adapter-html.sx': { lib: '(web adapter-html)', deps: ['(sx render)'] },
|
||||
'web/adapter-sx.sx': { lib: '(web adapter-sx)', deps: ['(web boot-helpers)'] },
|
||||
'web/adapter-dom.sx': { lib: '(web adapter-dom)', deps: ['(sx dom)', '(sx render)'] },
|
||||
// Web framework
|
||||
'web/lib/boot-helpers.sx': { lib: '(web boot-helpers)', deps: ['(sx dom)', '(sx browser)', '(web adapter-dom)'] },
|
||||
'web/lib/hypersx.sx': { lib: '(sx hypersx)', deps: [] },
|
||||
'web/engine.sx': { lib: '(web engine)', deps: ['(web boot-helpers)', '(sx dom)', '(sx browser)'] },
|
||||
'web/orchestration.sx': { lib: '(web orchestration)', deps: ['(web boot-helpers)', '(sx dom)', '(sx browser)', '(web adapter-dom)', '(web engine)'] },
|
||||
'web/boot.sx': { lib: null, deps: ['(sx dom)', '(sx browser)', '(web boot-helpers)', '(web adapter-dom)',
|
||||
'(sx signals)', '(sx signals-web)', '(web router)', '(web page-helpers)',
|
||||
'(web orchestration)', '(sx render)',
|
||||
'(sx bytecode)', '(sx compiler)', '(sx vm)'] },
|
||||
// Test harness
|
||||
'spec/harness.sx': { lib: '(sx harness)', deps: [] },
|
||||
'web/harness-reactive.sx': { lib: '(sx harness-reactive)', deps: [] },
|
||||
'web/harness-web.sx': { lib: '(sx harness-web)', deps: [] },
|
||||
};
|
||||
|
||||
// Extract top-level define names from source.
|
||||
// Handles both `(define name ...)` and `(define\n name ...)` formats.
|
||||
function extractDefineNames(source) {
|
||||
const names = [];
|
||||
const lines = source.split('\n');
|
||||
let depth = 0;
|
||||
let expectName = false;
|
||||
for (const line of lines) {
|
||||
if (depth === 0) {
|
||||
const m = line.match(/^\(define\s+\(?(\S+)/);
|
||||
if (m) {
|
||||
names.push(m[1]);
|
||||
expectName = false;
|
||||
} else if (line.match(/^\(define\s*$/)) {
|
||||
expectName = true;
|
||||
}
|
||||
} else if (depth === 1 && expectName) {
|
||||
const m = line.match(/^\s+(\S+)/);
|
||||
if (m) {
|
||||
names.push(m[1]);
|
||||
expectName = false;
|
||||
}
|
||||
}
|
||||
for (const ch of line) {
|
||||
if (ch === '(') depth++;
|
||||
else if (ch === ')') depth--;
|
||||
}
|
||||
}
|
||||
return names;
|
||||
}
|
||||
|
||||
function processFile(relPath, info) {
|
||||
const filePath = path.join(ROOT, relPath);
|
||||
if (!fs.existsSync(filePath)) {
|
||||
console.log(' SKIP', relPath, '(not found)');
|
||||
return;
|
||||
}
|
||||
|
||||
let source = fs.readFileSync(filePath, 'utf8');
|
||||
const { lib, deps } = info;
|
||||
const hasWrapper = source.includes('(define-library');
|
||||
const hasDepImports = deps.length > 0 && source.match(/^\(import\s+\(/m) &&
|
||||
!source.match(/^\(import\s+\(\w+ \w+\)\)\s*$/m); // more than just self-import
|
||||
|
||||
// Skip files with no deps and already wrapped (or no wrapper needed)
|
||||
if (deps.length === 0 && (hasWrapper || !lib)) {
|
||||
console.log(' ok', relPath, '(no changes needed)');
|
||||
return;
|
||||
}
|
||||
|
||||
// Build import lines for deps
|
||||
const importLines = deps.map(d => `(import ${d})`).join('\n');
|
||||
|
||||
// CASE 1: Entry point (boot.sx) — just add imports at top
|
||||
if (!lib) {
|
||||
if (deps.length > 0 && !source.startsWith('(import')) {
|
||||
source = importLines + '\n\n' + source;
|
||||
fs.writeFileSync(filePath, source);
|
||||
console.log(' +imports', relPath, `(${deps.length} deps, entry point)`);
|
||||
} else {
|
||||
console.log(' ok', relPath, '(entry point, already has imports)');
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// CASE 2: Already wrapped — add imports before define-library
|
||||
if (hasWrapper) {
|
||||
if (deps.length > 0) {
|
||||
// Check if imports already present
|
||||
const firstImportCheck = deps[0].replace(/[()]/g, '\\$&');
|
||||
if (source.match(new RegExp('\\(import ' + firstImportCheck))) {
|
||||
console.log(' ok', relPath, '(already has dep imports)');
|
||||
return;
|
||||
}
|
||||
const dlIdx = source.indexOf('(define-library');
|
||||
source = source.slice(0, dlIdx) + importLines + '\n\n' + source.slice(dlIdx);
|
||||
fs.writeFileSync(filePath, source);
|
||||
console.log(' +imports', relPath, `(${deps.length} deps)`);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// CASE 3: Needs full wrapping
|
||||
if (deps.length === 0 && !hasWrapper) {
|
||||
// Wrap with no deps
|
||||
const names = extractDefineNames(source);
|
||||
if (names.length === 0) {
|
||||
console.log(' WARN', relPath, '— no defines found, skipping');
|
||||
return;
|
||||
}
|
||||
const wrapped = buildWrapped(lib, names, source, '');
|
||||
fs.writeFileSync(filePath, wrapped);
|
||||
console.log(' wrapped', relPath, `as ${lib} (${names.length} exports)`);
|
||||
return;
|
||||
}
|
||||
|
||||
// Wrap with deps
|
||||
const names = extractDefineNames(source);
|
||||
if (names.length === 0) {
|
||||
console.log(' WARN', relPath, '— no defines found, skipping');
|
||||
return;
|
||||
}
|
||||
const wrapped = buildWrapped(lib, names, source, importLines);
|
||||
fs.writeFileSync(filePath, wrapped);
|
||||
console.log(' wrapped', relPath, `as ${lib} (${names.length} exports, ${deps.length} deps)`);
|
||||
}
|
||||
|
||||
function buildWrapped(libName, exportNames, bodySource, importSection) {
|
||||
const parts = [];
|
||||
|
||||
// Dependency imports (top-level, before define-library)
|
||||
if (importSection) {
|
||||
parts.push(importSection);
|
||||
parts.push('');
|
||||
}
|
||||
|
||||
// define-library header
|
||||
parts.push(`(define-library ${libName}`);
|
||||
parts.push(` (export ${exportNames.join(' ')})`);
|
||||
parts.push(' (begin');
|
||||
parts.push('');
|
||||
|
||||
// Body (original source, indented)
|
||||
parts.push(bodySource);
|
||||
parts.push('');
|
||||
|
||||
// Close begin + define-library
|
||||
parts.push('))');
|
||||
parts.push('');
|
||||
|
||||
// Self-import for backward compat
|
||||
parts.push(`;; Re-export to global env`);
|
||||
parts.push(`(import ${libName})`);
|
||||
parts.push('');
|
||||
|
||||
return parts.join('\n');
|
||||
}
|
||||
|
||||
console.log('Processing source .sx files...\n');
|
||||
for (const [relPath, info] of Object.entries(MODULES)) {
|
||||
processFile(relPath, info);
|
||||
}
|
||||
console.log('\nDone! Now run:');
|
||||
console.log(' bash hosts/ocaml/browser/bundle.sh');
|
||||
console.log(' node hosts/ocaml/browser/compile-modules.js');
|
||||
175
hosts/ocaml/sx_vm_ref.ml
Normal file
175
hosts/ocaml/sx_vm_ref.ml
Normal file
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user