Native bytecode compiler: 8x faster, compile-blob command
Rewrite compile-modules.js to use the native OCaml sx_server binary instead of the js_of_ocaml kernel in Node.js. Compiles 23 modules in 23s (was 3+ minutes). Uses batch epoch protocol with latin1 encoding to preserve byte positions for multi-byte UTF-8 content. - Add compile-blob server command: parse source natively, compile via SX compile-module, return bytecode dict - Fix orchestration.sxbc.json and boot.sxbc.json — never compiled successfully with the old JS kernel, now work with native compiler - Auto-copy compiled bytecode to shared/static/wasm/sx/ for serving Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -829,6 +829,29 @@ let rec dispatch env cmd =
|
||||
let src = read_blob () in
|
||||
dispatch env (List [Symbol "eval"; String src])
|
||||
|
||||
| List [Symbol "compile-blob"] ->
|
||||
(* Read source as blob, parse natively in OCaml, compile via SX compile-module.
|
||||
Returns the bytecode dict as SX text. Much faster than JS kernel. *)
|
||||
let src = read_blob () in
|
||||
(try
|
||||
let exprs = Sx_parser.parse_all src in
|
||||
let compile_module = env_get env "compile-module" in
|
||||
let result = Sx_ref.cek_call compile_module (List [List exprs]) in
|
||||
let rec raw_serialize = function
|
||||
| Nil -> "nil"
|
||||
| Bool true -> "true" | Bool false -> "false"
|
||||
| Number n -> if Float.is_integer n then string_of_int (int_of_float n) else Printf.sprintf "%g" n
|
||||
| String s -> "\"" ^ escape_sx_string s ^ "\""
|
||||
| Symbol s -> s | Keyword k -> ":" ^ k
|
||||
| List items | ListRef { contents = items } -> "(" ^ String.concat " " (List.map raw_serialize items) ^ ")"
|
||||
| Dict d -> let pairs = Hashtbl.fold (fun k v acc -> (Printf.sprintf ":%s %s" k (raw_serialize v)) :: acc) d [] in "{" ^ String.concat " " pairs ^ "}"
|
||||
| SxExpr s -> s | _ -> "nil"
|
||||
in
|
||||
send_ok_raw (raw_serialize result)
|
||||
with
|
||||
| Eval_error msg -> send_error msg
|
||||
| exn -> send_error (Printexc.to_string exn))
|
||||
|
||||
| List [Symbol "eval"; String src] ->
|
||||
(try
|
||||
let exprs = Sx_parser.parse_all src in
|
||||
|
||||
@@ -2,8 +2,9 @@
|
||||
/**
|
||||
* compile-modules.js — Pre-compile .sx files to bytecode s-expressions.
|
||||
*
|
||||
* Uses the js_of_ocaml kernel in Node.js to compile each .sx module,
|
||||
* then serializes the bytecode as .sxbc (s-expression format) for browser loading.
|
||||
* Uses the native OCaml sx_server binary for compilation (~5x faster than
|
||||
* the js_of_ocaml kernel). Sends source via the blob protocol, receives
|
||||
* compiled bytecode as SX text.
|
||||
*
|
||||
* Usage: node compile-modules.js [dist-dir]
|
||||
*/
|
||||
@@ -11,6 +12,7 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const crypto = require('crypto');
|
||||
const { execSync, spawnSync } = require('child_process');
|
||||
|
||||
const distDir = process.argv[2] || path.join(__dirname, 'dist');
|
||||
const sxDir = path.join(distDir, 'sx');
|
||||
@@ -20,15 +22,16 @@ if (!fs.existsSync(sxDir)) {
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Load the js_of_ocaml kernel
|
||||
const kernelPath = path.join(__dirname, '..', '_build', 'default', 'browser', 'sx_browser.bc.js');
|
||||
if (!fs.existsSync(kernelPath)) {
|
||||
console.error('Kernel not found:', kernelPath);
|
||||
// Find the native OCaml binary
|
||||
const binPaths = [
|
||||
path.join(__dirname, '..', '_build', 'default', 'bin', 'sx_server.exe'),
|
||||
'/app/bin/sx_server',
|
||||
];
|
||||
const binPath = binPaths.find(p => fs.existsSync(p));
|
||||
if (!binPath) {
|
||||
console.error('sx_server binary not found at:', binPaths.join(', '));
|
||||
process.exit(1);
|
||||
}
|
||||
require(kernelPath);
|
||||
const K = globalThis.SxKernel;
|
||||
if (!K) { console.error('SxKernel not initialized'); process.exit(1); }
|
||||
|
||||
const FILES = [
|
||||
'render.sx', 'core-signals.sx', 'signals.sx', 'deps.sx', 'router.sx',
|
||||
@@ -38,155 +41,162 @@ const FILES = [
|
||||
'harness-web.sx', 'engine.sx', 'orchestration.sx', 'boot.sx',
|
||||
];
|
||||
|
||||
// Load all files to build up the env (need compiler loaded)
|
||||
console.log('Loading SX environment...');
|
||||
for (const file of FILES) {
|
||||
const r = K.load(fs.readFileSync(path.join(sxDir, file), 'utf8'));
|
||||
if (typeof r === 'string' && r.startsWith('Error')) {
|
||||
console.error(' FAIL', file, r);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
console.log(' ' + FILES.length + ' files loaded');
|
||||
// ---------------------------------------------------------------------------
|
||||
// Build the full input script — all commands in one batch
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// Compile each file to bytecode
|
||||
console.log('Compiling bytecode modules...');
|
||||
const t0 = Date.now();
|
||||
console.log('Building compilation script...');
|
||||
|
||||
let epoch = 1;
|
||||
let script = '';
|
||||
|
||||
// Load compiler
|
||||
script += `(epoch ${epoch++})\n(load "lib/compiler.sx")\n`;
|
||||
|
||||
// JIT pre-compile the compiler
|
||||
script += `(epoch ${epoch++})\n(vm-compile-adapter)\n`;
|
||||
|
||||
// Load all modules into env
|
||||
for (const file of FILES) {
|
||||
const src = fs.readFileSync(path.join(sxDir, file), 'utf8');
|
||||
const buf = Buffer.from(src, 'utf8');
|
||||
script += `(epoch ${epoch++})\n(eval-blob)\n(blob ${buf.length})\n`;
|
||||
script += src + '\n';
|
||||
}
|
||||
|
||||
// Compile each module
|
||||
const compileEpochs = {};
|
||||
for (const file of FILES) {
|
||||
const src = fs.readFileSync(path.join(sxDir, file), 'utf8');
|
||||
const buf = Buffer.from(src, 'utf8');
|
||||
const ep = epoch++;
|
||||
compileEpochs[ep] = file;
|
||||
script += `(epoch ${ep})\n(compile-blob)\n(blob ${buf.length})\n`;
|
||||
script += src + '\n';
|
||||
}
|
||||
|
||||
// Write script to temp file and pipe to server
|
||||
const tmpFile = '/tmp/sx-compile-script.txt';
|
||||
fs.writeFileSync(tmpFile, script);
|
||||
|
||||
console.log('Running native OCaml compiler (' + FILES.length + ' files)...');
|
||||
const t1 = Date.now();
|
||||
|
||||
const result = spawnSync(binPath, [], {
|
||||
input: fs.readFileSync(tmpFile),
|
||||
maxBuffer: 100 * 1024 * 1024, // 100MB
|
||||
timeout: 300000, // 5 min
|
||||
stdio: ['pipe', 'pipe', 'pipe'],
|
||||
});
|
||||
|
||||
if (result.error) {
|
||||
console.error('Server error:', result.error);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const stderr = result.stderr.toString();
|
||||
process.stderr.write(stderr);
|
||||
|
||||
// Use latin1 to preserve byte positions (UTF-8 multi-byte chars stay as-is in length)
|
||||
const stdoutBuf = result.stdout;
|
||||
const stdout = stdoutBuf.toString('latin1');
|
||||
const dt = Date.now() - t1;
|
||||
console.log('Server finished in ' + Math.round(dt / 1000) + 's');
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Parse responses — extract compiled bytecode for each file
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// Parse responses — stdout is latin1 so byte positions match string positions
|
||||
let compiled = 0, skipped = 0;
|
||||
let pos = 0;
|
||||
|
||||
for (const file of FILES) {
|
||||
const srcPath = path.join(sxDir, file);
|
||||
const src = fs.readFileSync(srcPath, 'utf8');
|
||||
const hash = crypto.createHash('sha256').update(src).digest('hex').slice(0, 16);
|
||||
function nextLine() {
|
||||
const nl = stdout.indexOf('\n', pos);
|
||||
if (nl === -1) return null;
|
||||
const line = stdout.slice(pos, nl);
|
||||
pos = nl + 1;
|
||||
return line;
|
||||
}
|
||||
|
||||
try {
|
||||
const code = K.eval('(compile-module (sx-parse ' + JSON.stringify(src) + '))');
|
||||
while (pos < stdout.length) {
|
||||
const line = nextLine();
|
||||
if (line === null) break;
|
||||
const trimmed = line.trim();
|
||||
|
||||
if (typeof code === 'string' && code.startsWith('Error')) {
|
||||
console.error(' SKIP', file, '—', code);
|
||||
skipped++;
|
||||
continue;
|
||||
// ok-len EPOCH LEN — read LEN bytes as value
|
||||
const lenMatch = trimmed.match(/^\(ok-len (\d+) (\d+)\)$/);
|
||||
if (lenMatch) {
|
||||
const ep = parseInt(lenMatch[1]);
|
||||
const len = parseInt(lenMatch[2]);
|
||||
// Read exactly len bytes — latin1 encoding preserves byte positions
|
||||
const rawValue = stdout.slice(pos, pos + len);
|
||||
// Re-encode to proper UTF-8
|
||||
const value = Buffer.from(rawValue, 'latin1').toString('utf8');
|
||||
pos += len;
|
||||
// skip trailing newline
|
||||
if (pos < stdout.length && stdout.charCodeAt(pos) === 10) pos++;
|
||||
|
||||
const file = compileEpochs[ep];
|
||||
if (file) {
|
||||
if (value === 'nil' || value.startsWith('(error')) {
|
||||
console.error(' SKIP', file, '—', value.slice(0, 60));
|
||||
skipped++;
|
||||
} else {
|
||||
const hash = crypto.createHash('sha256')
|
||||
.update(fs.readFileSync(path.join(sxDir, file), 'utf8'))
|
||||
.digest('hex').slice(0, 16);
|
||||
|
||||
const sxbc = '(sxbc 1 "' + hash + '"\n (code\n ' +
|
||||
value.replace(/^\{/, '').replace(/\}$/, '').trim() + '))\n';
|
||||
|
||||
const outPath = path.join(sxDir, file.replace(/\.sx$/, '.sxbc'));
|
||||
fs.writeFileSync(outPath, sxbc);
|
||||
|
||||
const size = fs.statSync(outPath).size;
|
||||
console.log(' ok', file, '→', Math.round(size / 1024) + 'K');
|
||||
compiled++;
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
const sx = serializeModuleToSx(code, hash);
|
||||
|
||||
// Write .sxbc (s-expression format)
|
||||
const outPath = srcPath.replace(/\.sx$/, '.sxbc');
|
||||
fs.writeFileSync(outPath, sx);
|
||||
|
||||
// Also write .sxbc.json for backwards compatibility during transition
|
||||
const json = {
|
||||
magic: 'SXBC',
|
||||
version: 1,
|
||||
hash: hash,
|
||||
module: serializeModuleToJson(code),
|
||||
};
|
||||
const jsonPath = srcPath.replace(/\.sx$/, '.sxbc.json');
|
||||
fs.writeFileSync(jsonPath, JSON.stringify(json));
|
||||
|
||||
const size = fs.statSync(outPath).size;
|
||||
console.log(' ok', file, '→', Math.round(size / 1024) + 'K');
|
||||
compiled++;
|
||||
} catch (e) {
|
||||
console.error(' SKIP', file, '—', e.message || e);
|
||||
skipped++;
|
||||
// Simple ok or error — skip
|
||||
if (trimmed.match(/^\(ok \d+/) || trimmed.match(/^\(error \d+/)) {
|
||||
if (trimmed.match(/^\(error/)) {
|
||||
const epMatch = trimmed.match(/^\(error (\d+)/);
|
||||
if (epMatch) {
|
||||
const ep = parseInt(epMatch[1]);
|
||||
const file = compileEpochs[ep];
|
||||
if (file) {
|
||||
console.error(' SKIP', file, '—', trimmed.slice(0, 80));
|
||||
skipped++;
|
||||
}
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
console.log('Done:', compiled, 'compiled,', skipped, 'skipped');
|
||||
|
||||
// --- S-expression serialization ---
|
||||
|
||||
function serializeModuleToSx(code, hash) {
|
||||
return '(sxbc 1 "' + hash + '"\n ' + serializeCodeToSx(code, 2) + ')\n';
|
||||
}
|
||||
|
||||
function serializeCodeToSx(code, indent) {
|
||||
const pad = ' '.repeat(indent);
|
||||
const bc = extractList(code.bytecode);
|
||||
const consts = extractList(code.constants);
|
||||
const arity = code.arity || code['arity'] || 0;
|
||||
const uvc = code['upvalue-count'] || 0;
|
||||
|
||||
let parts = ['(code'];
|
||||
if (arity) parts.push(' :arity ' + arity);
|
||||
if (uvc) parts.push(' :upvalue-count ' + uvc);
|
||||
parts.push('\n' + pad + ' :bytecode (' + bc.join(' ') + ')');
|
||||
parts.push('\n' + pad + ' :constants (');
|
||||
|
||||
const constStrs = consts.map(c => serializeConstToSx(c, indent + 4));
|
||||
if (constStrs.length > 0) {
|
||||
parts.push('\n' + constStrs.map(s => pad + ' ' + s).join('\n'));
|
||||
parts.push(')');
|
||||
} else {
|
||||
parts[parts.length - 1] += ')';
|
||||
// Copy compiled files to shared/static/wasm/sx/ for web serving
|
||||
const staticSxDir = path.resolve(__dirname, '..', '..', '..', 'shared', 'static', 'wasm', 'sx');
|
||||
if (fs.existsSync(staticSxDir)) {
|
||||
let copied = 0;
|
||||
for (const file of FILES) {
|
||||
for (const ext of ['.sxbc', '.sxbc.json']) {
|
||||
const src = path.join(sxDir, file.replace(/\.sx$/, ext));
|
||||
const dst = path.join(staticSxDir, file.replace(/\.sx$/, ext));
|
||||
if (fs.existsSync(src)) {
|
||||
fs.copyFileSync(src, dst);
|
||||
copied++;
|
||||
}
|
||||
}
|
||||
}
|
||||
parts.push(')');
|
||||
return parts.join('');
|
||||
console.log('Copied', copied, 'files to', staticSxDir);
|
||||
}
|
||||
|
||||
function serializeConstToSx(c, indent) {
|
||||
if (c === null || c === undefined) return 'nil';
|
||||
if (typeof c === 'number') return String(c);
|
||||
if (typeof c === 'string') return '"' + c.replace(/\\/g, '\\\\').replace(/"/g, '\\"') + '"';
|
||||
if (typeof c === 'boolean') return c ? 'true' : 'false';
|
||||
if (c._type === 'symbol') return "'" + c.name;
|
||||
if (c._type === 'keyword') return ':' + c.name;
|
||||
if (c._type === 'list') {
|
||||
const items = extractList(c).map(x => serializeConstToSx(x, indent));
|
||||
return '(list ' + items.join(' ') + ')';
|
||||
}
|
||||
// Code object (nested lambda bytecode)
|
||||
if (c.bytecode) return serializeCodeToSx(c, indent);
|
||||
if (c._type === 'dict') {
|
||||
const bc = c.get ? c.get('bytecode') : c.bytecode;
|
||||
if (bc) return serializeCodeToSx(c, indent);
|
||||
// Regular dict — serialize as {:key val ...}
|
||||
const entries = [];
|
||||
if (c.forEach) c.forEach((v, k) => { entries.push(':' + k + ' ' + serializeConstToSx(v, indent)); });
|
||||
return '{' + entries.join(' ') + '}';
|
||||
}
|
||||
return 'nil';
|
||||
}
|
||||
const total = Date.now() - t0;
|
||||
console.log('Done:', compiled, 'compiled,', skipped, 'skipped in', Math.round(total / 1000) + 's');
|
||||
|
||||
// --- JSON serialization (backwards compat) ---
|
||||
|
||||
function serializeModuleToJson(code) {
|
||||
const result = {
|
||||
bytecode: extractList(code.bytecode),
|
||||
constants: extractList(code.constants).map(serializeConstantJson),
|
||||
};
|
||||
const arity = code.arity || code['arity'];
|
||||
const uvc = code['upvalue-count'];
|
||||
const locals = code.locals || code['locals'];
|
||||
if (arity) result.arity = typeof arity === 'number' ? arity : 0;
|
||||
if (uvc) result['upvalue-count'] = typeof uvc === 'number' ? uvc : 0;
|
||||
if (locals) result.locals = typeof locals === 'number' ? locals : 0;
|
||||
return result;
|
||||
}
|
||||
|
||||
function serializeConstantJson(c) {
|
||||
if (c === null || c === undefined) return { t: 'nil' };
|
||||
if (typeof c === 'number') return { t: 'n', v: c };
|
||||
if (typeof c === 'string') return { t: 's', v: c };
|
||||
if (typeof c === 'boolean') return { t: 'b', v: c };
|
||||
if (c._type === 'symbol') return { t: 'sym', v: c.name };
|
||||
if (c._type === 'keyword') return { t: 'kw', v: c.name };
|
||||
if (c._type === 'list') return { t: 'list', v: extractList(c).map(serializeConstantJson) };
|
||||
if (c.bytecode) return { t: 'code', v: serializeModuleToJson(c) };
|
||||
if (c._type === 'dict') {
|
||||
const bc = c.get ? c.get('bytecode') : c.bytecode;
|
||||
if (bc) return { t: 'code', v: serializeModuleToJson(c) };
|
||||
const entries = {};
|
||||
if (c.forEach) c.forEach((v, k) => { entries[k] = serializeConstantJson(v); });
|
||||
return { t: 'dict', v: entries };
|
||||
}
|
||||
return { t: 'nil' };
|
||||
}
|
||||
|
||||
function extractList(v) {
|
||||
if (!v) return [];
|
||||
if (Array.isArray(v)) return v;
|
||||
if (v._type === 'list' && v.items) return v.items;
|
||||
if (v.items) return v.items;
|
||||
return [];
|
||||
}
|
||||
fs.unlinkSync(tmpFile);
|
||||
|
||||
Reference in New Issue
Block a user