giles
c361946974
Phase 1 of the jit-perf-regression plan reproduced and quantified the alleged
30× substrate slowdown across 5 guests (tcl, lua, erlang, prolog, haskell). On
a quiet machine all five suites pass cleanly:
tcl test.sh 57.8s wall, 16.3s user, 376/376 ✓
lua test.sh 27.3s wall, 4.2s user, 185/185 ✓
erlang conformance 3m25s wall, 36.8s user, 530/530 ✓ (needs ≥600s budget)
prolog conformance 3m54s wall, 1m08s user, 590/590 ✓
haskell conformance 6m59s wall, 2m37s user, 156/156 ✓
Per-test user-time at architecture HEAD vs pre-substrate-merge baseline
(83dbb595) is essentially flat (tcl 0.83×, lua 1.4×, prolog 0.82×). The
symptoms reported in the plan (test timeouts, OOMs, 30-min hangs) were heavy
CPU contention from concurrent loops + one undersized internal `timeout 120`
in erlang's conformance script. There is no substrate regression to bisect.
Changes:
* lib/tcl/test.sh: `timeout 2400` → `timeout 300`. The original 180s deadline
is comfortable on a quiet machine (3.1× headroom); 300s gives some safety
margin for moderate contention without masking real regressions.
* lib/erlang/conformance.sh: `timeout 120` → `timeout 600`. The 120s budget
was actually too tight for the full 9-suite chain even before this work.
* lib/erlang/scoreboard.{json,md}: 0/0 → 530/530 — populated by a successful
conformance run with the new deadline. The previous 0/0 was a stale
artefact of the run timing out before parsing any markers.
* plans/jit-perf-regression.md: full Phase 1 progress log including
per-guest perf table, quiet-machine re-measurement, and conclusion.
Phases 2–4 (bisect, diagnose, fix) skipped — there is no substrate regression
to find. Phase 6 (perf-regression alarm) still planned to catch the next
quadratic blow-up early instead of via watchdog bumps.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
117 lines
4.6 KiB
Bash
Executable File
117 lines
4.6 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Tcl-on-SX test runner — epoch protocol to sx_server.exe
|
|
set -uo pipefail
|
|
cd "$(git rev-parse --show-toplevel)"
|
|
|
|
SX_SERVER="${SX_SERVER:-hosts/ocaml/_build/default/bin/sx_server.exe}"
|
|
if [ ! -x "$SX_SERVER" ]; then
|
|
SX_SERVER="/root/rose-ash/hosts/ocaml/_build/default/bin/sx_server.exe"
|
|
fi
|
|
if [ ! -x "$SX_SERVER" ]; then echo "ERROR: sx_server.exe not found"; exit 1; fi
|
|
|
|
VERBOSE="${1:-}"
|
|
TMPFILE=$(mktemp)
|
|
HELPER=$(mktemp --suffix=.sx)
|
|
trap "rm -f $TMPFILE $HELPER" EXIT
|
|
|
|
# Helper file: run all test suites and format a parseable summary string
|
|
cat > "$HELPER" << 'HELPER_EOF'
|
|
(define __pr (tcl-run-parse-tests))
|
|
(define __er (tcl-run-eval-tests))
|
|
(define __xr (tcl-run-error-tests))
|
|
(define __nr (tcl-run-namespace-tests))
|
|
(define __cr (tcl-run-coro-tests))
|
|
(define __ir (tcl-run-idiom-tests))
|
|
(define tcl-test-summary
|
|
(str "PARSE:" (get __pr "passed") ":" (get __pr "failed")
|
|
" EVAL:" (get __er "passed") ":" (get __er "failed")
|
|
" ERROR:" (get __xr "passed") ":" (get __xr "failed")
|
|
" NAMESPACE:" (get __nr "passed") ":" (get __nr "failed")
|
|
" CORO:" (get __cr "passed") ":" (get __cr "failed")
|
|
" IDIOM:" (get __ir "passed") ":" (get __ir "failed")))
|
|
HELPER_EOF
|
|
|
|
cat > "$TMPFILE" << EPOCHS
|
|
(epoch 1)
|
|
(load "lib/guest/lex.sx")
|
|
(load "lib/guest/prefix.sx")
|
|
(load "lib/tcl/tokenizer.sx")
|
|
(epoch 2)
|
|
(load "lib/tcl/parser.sx")
|
|
(epoch 3)
|
|
(load "lib/tcl/tests/parse.sx")
|
|
(epoch 4)
|
|
(load "lib/fiber.sx")
|
|
(load "lib/tcl/runtime.sx")
|
|
(epoch 5)
|
|
(load "lib/tcl/tests/eval.sx")
|
|
(epoch 6)
|
|
(load "lib/tcl/tests/error.sx")
|
|
(epoch 7)
|
|
(load "lib/tcl/tests/namespace.sx")
|
|
(epoch 8)
|
|
(load "lib/tcl/tests/coro.sx")
|
|
(epoch 9)
|
|
(load "lib/tcl/tests/idioms.sx")
|
|
(epoch 10)
|
|
(load "$HELPER")
|
|
(epoch 11)
|
|
(eval "tcl-test-summary")
|
|
EPOCHS
|
|
|
|
OUTPUT=$(timeout 300 "$SX_SERVER" < "$TMPFILE" 2>&1)
|
|
[ "$VERBOSE" = "-v" ] && echo "$OUTPUT"
|
|
|
|
# Extract summary line from epoch 11 output
|
|
SUMMARY=$(echo "$OUTPUT" | grep -A1 "^(ok-len 11 " | tail -1 | tr -d '"')
|
|
|
|
if [ -z "$SUMMARY" ]; then
|
|
echo "ERROR: no summary from test run"
|
|
echo "$OUTPUT" | tail -20
|
|
exit 1
|
|
fi
|
|
|
|
# Parse PARSE:N:M EVAL:N:M ERROR:N:M NAMESPACE:N:M CORO:N:M IDIOM:N:M
|
|
PARSE_PART=$(echo "$SUMMARY" | grep -o 'PARSE:[0-9]*:[0-9]*')
|
|
EVAL_PART=$(echo "$SUMMARY" | grep -o 'EVAL:[0-9]*:[0-9]*')
|
|
ERROR_PART=$(echo "$SUMMARY" | grep -o 'ERROR:[0-9]*:[0-9]*')
|
|
NAMESPACE_PART=$(echo "$SUMMARY" | grep -o 'NAMESPACE:[0-9]*:[0-9]*')
|
|
CORO_PART=$(echo "$SUMMARY" | grep -o 'CORO:[0-9]*:[0-9]*')
|
|
IDIOM_PART=$(echo "$SUMMARY" | grep -o 'IDIOM:[0-9]*:[0-9]*')
|
|
|
|
PARSE_PASSED=$(echo "$PARSE_PART" | cut -d: -f2)
|
|
PARSE_FAILED=$(echo "$PARSE_PART" | cut -d: -f3)
|
|
EVAL_PASSED=$(echo "$EVAL_PART" | cut -d: -f2)
|
|
EVAL_FAILED=$(echo "$EVAL_PART" | cut -d: -f3)
|
|
ERROR_PASSED=$(echo "$ERROR_PART" | cut -d: -f2)
|
|
ERROR_FAILED=$(echo "$ERROR_PART" | cut -d: -f3)
|
|
NAMESPACE_PASSED=$(echo "$NAMESPACE_PART" | cut -d: -f2)
|
|
NAMESPACE_FAILED=$(echo "$NAMESPACE_PART" | cut -d: -f3)
|
|
CORO_PASSED=$(echo "$CORO_PART" | cut -d: -f2)
|
|
CORO_FAILED=$(echo "$CORO_PART" | cut -d: -f3)
|
|
IDIOM_PASSED=$(echo "$IDIOM_PART" | cut -d: -f2)
|
|
IDIOM_FAILED=$(echo "$IDIOM_PART" | cut -d: -f3)
|
|
|
|
PARSE_PASSED=${PARSE_PASSED:-0}; PARSE_FAILED=${PARSE_FAILED:-1}
|
|
EVAL_PASSED=${EVAL_PASSED:-0}; EVAL_FAILED=${EVAL_FAILED:-1}
|
|
ERROR_PASSED=${ERROR_PASSED:-0}; ERROR_FAILED=${ERROR_FAILED:-1}
|
|
NAMESPACE_PASSED=${NAMESPACE_PASSED:-0}; NAMESPACE_FAILED=${NAMESPACE_FAILED:-1}
|
|
CORO_PASSED=${CORO_PASSED:-0}; CORO_FAILED=${CORO_FAILED:-1}
|
|
IDIOM_PASSED=${IDIOM_PASSED:-0}; IDIOM_FAILED=${IDIOM_FAILED:-1}
|
|
|
|
TOTAL_PASSED=$((PARSE_PASSED + EVAL_PASSED + ERROR_PASSED + NAMESPACE_PASSED + CORO_PASSED + IDIOM_PASSED))
|
|
TOTAL_FAILED=$((PARSE_FAILED + EVAL_FAILED + ERROR_FAILED + NAMESPACE_FAILED + CORO_FAILED + IDIOM_FAILED))
|
|
TOTAL=$((TOTAL_PASSED + TOTAL_FAILED))
|
|
|
|
if [ "$TOTAL_FAILED" = "0" ]; then
|
|
echo "ok $TOTAL_PASSED/$TOTAL tcl tests passed (parse: $PARSE_PASSED, eval: $EVAL_PASSED, error: $ERROR_PASSED, namespace: $NAMESPACE_PASSED, coro: $CORO_PASSED, idiom: $IDIOM_PASSED)"
|
|
exit 0
|
|
else
|
|
echo "FAIL $TOTAL_PASSED/$TOTAL passed, $TOTAL_FAILED failed (parse: $PARSE_PASSED/$((PARSE_PASSED+PARSE_FAILED)), eval: $EVAL_PASSED/$((EVAL_PASSED+EVAL_FAILED)), error: $ERROR_PASSED/$((ERROR_PASSED+ERROR_FAILED)), namespace: $NAMESPACE_PASSED/$((NAMESPACE_PASSED+NAMESPACE_FAILED)), coro: $CORO_PASSED/$((CORO_PASSED+CORO_FAILED)), idiom: $IDIOM_PASSED/$((IDIOM_PASSED+IDIOM_FAILED)))"
|
|
if [ -z "$VERBOSE" ]; then
|
|
echo "--- output ---"
|
|
echo "$OUTPUT" | tail -30
|
|
fi
|
|
exit 1
|
|
fi
|