From a25427cb7986b711bc7c9e4d4c6017641a28770e Mon Sep 17 00:00:00 2001 From: giles Date: Tue, 30 Jun 2026 22:20:25 +0000 Subject: [PATCH] =?UTF-8?q?host:=20warm-conf.sh=20=E2=80=94=20persistent?= =?UTF-8?q?=20conformance=20server=20for=20fast=20iteration?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit conformance.sh cold-loads all ~57 modules every run (a multi-minute tax, worst under box contention). warm-conf.sh keeps a long-lived sx_server with the 44 heavy dependency modules (datalog/acl/relations/persist/dream) loaded ONCE, and per run reloads only the 16 lib/host/* modules + the suite's test file — the things you actually edit — then evals the runner. Reads the MODULES + SUITES arrays straight from conformance.sh (no duplication/drift). Safe across runs: each test file re-opens a fresh persist store, and (since blog typing now reads direct KV edges, not lib/relations) the warm Datalog DB no longer feeds blog results, so stale facts can't pollute a re-run. Usage: warm-conf.sh start | run [suite] | stop. Co-Authored-By: Claude Opus 4.8 --- lib/host/warm-conf.sh | 128 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100755 lib/host/warm-conf.sh diff --git a/lib/host/warm-conf.sh b/lib/host/warm-conf.sh new file mode 100755 index 00000000..e239d5da --- /dev/null +++ b/lib/host/warm-conf.sh @@ -0,0 +1,128 @@ +#!/usr/bin/env bash +# warm-conf.sh — a WARM, persistent conformance server for fast iteration. +# +# conformance.sh cold-loads all ~57 modules (datalog/acl/relations/persist/dream + host) +# on EVERY run — a fixed multi-minute tax, worst under box contention. This keeps a +# long-lived sx_server with the heavy dependency modules loaded ONCE, and per run reloads +# only the lib/host/* modules + the suite's test file (the things you actually edit), +# then evals the runner. Cross-run state is safe: each test file re-opens a fresh persist +# store at its top, and (since host/blog typing now reads direct KV edges, not lib/relations) +# the warm Datalog DB no longer feeds blog results, so stale facts can't pollute a re-run. +# +# Usage: +# lib/host/warm-conf.sh start # boot server, load the heavy dep modules once +# lib/host/warm-conf.sh run blog # reload host modules + tests/blog.sx, run the suite +# lib/host/warm-conf.sh run # run every suite +# lib/host/warm-conf.sh stop # kill the warm server +# lib/host/warm-conf.sh restart # stop + start +# +# It reads the MODULES + SUITES arrays straight from conformance.sh (no duplication, no +# drift). Heavy deps are everything NOT under lib/host/; those host modules + the test +# files are what `run` reloads. +set -u + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$HERE" || exit 1 +CONF="lib/host/conformance.sh" + +SX_SERVER="${SX_SERVER:-hosts/ocaml/_build/default/bin/sx_server.exe}" +[ -x "$SX_SERVER" ] || SX_SERVER="/root/rose-ash/hosts/ocaml/_build/default/bin/sx_server.exe" +if [ ! -x "$SX_SERVER" ]; then echo "ERROR: sx_server.exe not found" >&2; exit 1; fi + +D="${WARM_CONF_DIR:-/tmp/warm-conf-host}" +FIFO="$D/in"; LOG="$D/out"; SPID="$D/server.pid"; HPID="$D/holder.pid"; EPF="$D/epoch" + +# All module load paths from conformance.sh's MODULES=( ... ) array (in order). +mapfile -t ALL_MODULES < <(awk '/^MODULES=\(/{f=1;next} f&&/^\)/{f=0} f' "$CONF" | grep -oE '"[^"]+\.sx"' | tr -d '"') +# Heavy deps = everything that is NOT a lib/host module (loaded once, kept warm). +DEPS=(); HOSTMODS=() +for m in "${ALL_MODULES[@]}"; do + case "$m" in lib/host/*) HOSTMODS+=("$m") ;; *) DEPS+=("$m") ;; esac +done +# Suites: "NAME RUNNER FILE" lines from conformance.sh's SUITES=( ... ) array. +mapfile -t SUITES < <(awk '/^SUITES=\(/{f=1;next} f&&/^\)/{f=0} f' "$CONF" | grep -oE '"[^"]+"' | tr -d '"') + +_running() { [ -f "$SPID" ] && kill -0 "$(cat "$SPID")" 2>/dev/null; } + +_send() { printf '%s\n' "$1" > "$FIFO"; } + +# wait until a line matching $1 appears in the log AFTER byte-offset $2, or $3 seconds pass. +_wait_for() { + local pat="$1" from="$2" timeout="${3:-1200}" waited=0 + while true; do + if tail -c +"$((from+1))" "$LOG" | grep -qE "$pat"; then return 0; fi + if tail -c +"$((from+1))" "$LOG" | grep -qE 'Undefined symbol|Unhandled exception|: error |expected list, got'; then + echo " ! error in server output:" >&2 + tail -c +"$((from+1))" "$LOG" | grep -nE 'Undefined symbol|Unhandled exception|: error |expected list, got' | head -5 >&2 + return 2 + fi + sleep 1; waited=$((waited+1)) + [ "$waited" -ge "$timeout" ] && { echo " ! timeout after ${timeout}s waiting for /$pat/" >&2; return 1; } + done +} + +_emit_loads() { # $@ = module paths; uses + bumps the epoch counter in $EPF + local e; e="$(cat "$EPF")" + { for m in "$@"; do e=$((e+1)); printf '(epoch %d)\n(load "%s")\n' "$e" "$m"; done; } > "$FIFO" + echo "$e" > "$EPF"; echo "$e" # echo the last epoch used +} + +cmd_start() { + cmd_stop >/dev/null 2>&1 + mkdir -p "$D"; : > "$LOG"; echo 0 > "$EPF" + rm -f "$FIFO"; mkfifo "$FIFO" + "$SX_SERVER" < "$FIFO" > "$LOG" 2>&1 & + echo $! > "$SPID" + sleep infinity > "$FIFO" & # holder: keeps the write end open so the server never EOFs + echo $! > "$HPID" + echo "warm: loading ${#DEPS[@]} dependency modules (once)..." + local last; last="$(_emit_loads "${DEPS[@]}")" + if _wait_for "^\(ok $last " 0 900; then + echo "warm: ready — ${#DEPS[@]} deps loaded, server pid $(cat "$SPID")" + else + echo "warm: FAILED to load deps" >&2; return 1 + fi +} + +cmd_stop() { + [ -f "$HPID" ] && kill "$(cat "$HPID")" 2>/dev/null + [ -f "$SPID" ] && kill "$(cat "$SPID")" 2>/dev/null + rm -f "$FIFO" "$SPID" "$HPID" "$EPF" + echo "warm: stopped" +} + +cmd_run() { + if ! _running; then echo "warm: server not running — starting it first"; cmd_start || return 1; fi + local filter="${1:-}" any=0 totp=0 totf=0 + for s in "${SUITES[@]}"; do + read -r name runner file <<< "$s" + [ -n "$filter" ] && [ "$name" != "$filter" ] && continue + any=1 + # reload the host modules (what changes) + this suite's test file, then eval the runner. + local off; off="$(wc -c < "$LOG")" + _emit_loads "${HOSTMODS[@]}" "$file" >/dev/null + local e; e="$(cat "$EPF")"; e=$((e+1)) + _send "(epoch $e)"; _send "(eval \"($runner)\")" + echo "$e" > "$EPF" + if ! _wait_for '^\{:' "$off" 1800; then echo "X $name — no result"; continue; fi + local dict; dict="$(tail -c +"$((off+1))" "$LOG" | grep -E '^\{:' | tail -1)" + local p f; p="$(echo "$dict" | grep -oE ':passed [0-9]+' | awk '{print $2}')"; f="$(echo "$dict" | grep -oE ':failed [0-9]+' | awk '{print $2}')" + p="${p:-0}"; f="${f:-0}"; totp=$((totp+p)); totf=$((totf+f)) + if [ "$f" -gt 0 ]; then + printf 'X %-12s %d/%d\n' "$name" "$p" "$((p+f))" + echo "$dict" | grep -oE ':name "[^"]*"' | sed 's/:name / fail: /' + else + printf 'ok %-12s %d passed\n' "$name" "$p" + fi + done + [ "$any" = 0 ] && { echo "no suite matched '$filter'"; return 1; } + if [ "$totf" -eq 0 ]; then echo "ok $totp passed (warm)"; else echo "FAIL $totp passed, $totf failed (warm)"; return 1; fi +} + +case "${1:-}" in + start) cmd_start ;; + stop) cmd_stop ;; + restart) cmd_stop; cmd_start ;; + run) shift; cmd_run "${1:-}" ;; + *) echo "usage: $0 {start|run [suite]|stop|restart}" >&2; exit 1 ;; +esac