rose-ash/hosts/ocaml/browser/compile-modules.js

#!/usr/bin/env node
/**
 * compile-modules.js — Pre-compile .sx files to bytecode s-expressions.
 *
 * Uses the native OCaml sx_server binary for compilation (~5x faster than
 * the js_of_ocaml kernel). Sends source via the blob protocol, receives
 * compiled bytecode as SX text.
 *
 * Usage: node compile-modules.js [dist-dir]
 */

const fs = require('fs');
const path = require('path');
const crypto = require('crypto');
const { execSync, spawnSync } = require('child_process');

const distDir = process.argv[2] || path.join(__dirname, 'dist');
const sxDir = path.join(distDir, 'sx');

if (!fs.existsSync(sxDir)) {
  console.error('sx dir not found:', sxDir);
  process.exit(1);
}

// Find the native OCaml binary
const binPaths = [
  path.join(__dirname, '..', '_build', 'default', 'bin', 'sx_server.exe'),
  '/app/bin/sx_server',
];
const binPath = binPaths.find(p => fs.existsSync(p));
if (!binPath) {
  console.error('sx_server binary not found at:', binPaths.join(', '));
  process.exit(1);
}

const FILES = [
  'render.sx', 'core-signals.sx', 'signals.sx', 'deps.sx', 'router.sx',
  'page-helpers.sx', 'freeze.sx', 'bytecode.sx', 'compiler.sx', 'vm.sx',
  'dom.sx', 'browser.sx', 'adapter-html.sx', 'adapter-sx.sx', 'adapter-dom.sx',
  'tw-layout.sx', 'tw-type.sx', 'tw.sx',
  'boot-helpers.sx', 'hypersx.sx', 'harness.sx', 'harness-reactive.sx',
  'harness-web.sx', 'engine.sx', 'orchestration.sx', 'boot.sx',
];


// ---------------------------------------------------------------------------
// Build the full input script — all commands in one batch
// ---------------------------------------------------------------------------

const t0 = Date.now();
console.log('Building compilation script...');

let epoch = 1;
let script = '';

// Load compiler
script += `(epoch ${epoch++})\n(load "lib/compiler.sx")\n`;

// JIT pre-compile the compiler
script += `(epoch ${epoch++})\n(vm-compile-adapter)\n`;

// Load all modules into env
for (const file of FILES) {
  const src = fs.readFileSync(path.join(sxDir, file), 'utf8');
  const buf = Buffer.from(src, 'utf8');
  script += `(epoch ${epoch++})\n(eval-blob)\n(blob ${buf.length})\n`;
  script += src + '\n';
}

// Compile each module
const compileEpochs = {};
for (const file of FILES) {
  const src = fs.readFileSync(path.join(sxDir, file), 'utf8');
  const buf = Buffer.from(src, 'utf8');
  const ep = epoch++;
  compileEpochs[ep] = file;
  script += `(epoch ${ep})\n(compile-blob)\n(blob ${buf.length})\n`;
  script += src + '\n';
}

// Write script to temp file and pipe to server
const tmpFile = '/tmp/sx-compile-script.txt';
fs.writeFileSync(tmpFile, script);

console.log('Running native OCaml compiler (' + FILES.length + ' files)...');
const t1 = Date.now();

const result = spawnSync(binPath, [], {
  input: fs.readFileSync(tmpFile),
  maxBuffer: 100 * 1024 * 1024, // 100MB
  timeout: 600000, // 10 min
  stdio: ['pipe', 'pipe', 'pipe'],
});

if (result.error) {
  console.error('Server error:', result.error);
  process.exit(1);
}

const stderr = result.stderr.toString();
process.stderr.write(stderr);

// Use latin1 to preserve byte positions (UTF-8 multi-byte chars stay as-is in length)
const stdoutBuf = result.stdout;
const stdout = stdoutBuf.toString('latin1');
const dt = Date.now() - t1;
console.log('Server finished in ' + Math.round(dt / 1000) + 's');

// ---------------------------------------------------------------------------
// Parse responses — extract compiled bytecode for each file
// ---------------------------------------------------------------------------

// Parse responses — stdout is latin1 so byte positions match string positions
let compiled = 0, skipped = 0;
let pos = 0;

function nextLine() {
  const nl = stdout.indexOf('\n', pos);
  if (nl === -1) return null;
  const line = stdout.slice(pos, nl);
  pos = nl + 1;
  return line;
}

while (pos < stdout.length) {
  const line = nextLine();
  if (line === null) break;
  const trimmed = line.trim();

  // ok-len EPOCH LEN — read LEN bytes as value
  const lenMatch = trimmed.match(/^\(ok-len (\d+) (\d+)\)$/);
  if (lenMatch) {
    const ep = parseInt(lenMatch[1]);
    const len = parseInt(lenMatch[2]);
    // Read exactly len bytes — latin1 encoding preserves byte positions
    const rawValue = stdout.slice(pos, pos + len);
    // Re-encode to proper UTF-8
    const value = Buffer.from(rawValue, 'latin1').toString('utf8');
    pos += len;
    // skip trailing newline
    if (pos < stdout.length && stdout.charCodeAt(pos) === 10) pos++;

    const file = compileEpochs[ep];
    if (file) {
      if (value === 'nil' || value.startsWith('(error')) {
        console.error('  SKIP', file, '—', value.slice(0, 60));
        skipped++;
      } else {
        const hash = crypto.createHash('sha256')
          .update(fs.readFileSync(path.join(sxDir, file), 'utf8'))
          .digest('hex').slice(0, 16);

        const sxbc = '(sxbc 1 "' + hash + '"\n  (code\n    ' +
          value.replace(/^\{/, '').replace(/\}$/, '').trim() + '))\n';

        const outPath = path.join(sxDir, file.replace(/\.sx$/, '.sxbc'));
        fs.writeFileSync(outPath, sxbc);

        const size = fs.statSync(outPath).size;
        console.log('  ok', file, '→', Math.round(size / 1024) + 'K');
        compiled++;
      }
    }
    continue;
  }

  // Simple ok or error — skip
  if (trimmed.match(/^\(ok \d+/) || trimmed.match(/^\(error \d+/)) {
    if (trimmed.match(/^\(error/)) {
      const epMatch = trimmed.match(/^\(error (\d+)/);
      if (epMatch) {
        const ep = parseInt(epMatch[1]);
        const file = compileEpochs[ep];
        if (file) {
          console.error('  SKIP', file, '—', trimmed.slice(0, 80));
          skipped++;
        }
      }
    }
    continue;
  }
}

// Copy compiled files to shared/static/wasm/sx/ for web serving
const staticSxDir = path.resolve(__dirname, '..', '..', '..', 'shared', 'static', 'wasm', 'sx');
if (fs.existsSync(staticSxDir)) {
  let copied = 0;
  for (const file of FILES) {
    // Copy bytecode
    for (const ext of ['.sxbc', '.sxbc.json']) {
      const src = path.join(sxDir, file.replace(/\.sx$/, ext));
      const dst = path.join(staticSxDir, file.replace(/\.sx$/, ext));
      if (fs.existsSync(src)) {
        fs.copyFileSync(src, dst);
        copied++;
      }
    }
    // Also sync .sx source files (fallback when .sxbc missing)
    const sxSrc = path.join(sxDir, file);
    const sxDst = path.join(staticSxDir, file);
    if (fs.existsSync(sxSrc) && !fs.lstatSync(sxSrc).isSymbolicLink()) {
      fs.copyFileSync(sxSrc, sxDst);
      copied++;
    }
  }
  console.log('Copied', copied, 'files to', staticSxDir);
}

const total = Date.now() - t0;
console.log('Done:', compiled, 'compiled,', skipped, 'skipped in', Math.round(total / 1000) + 's');

fs.unlinkSync(tmpFile);