conformance: migrate common-lisp onto shared driver (counters, 487/487)

Extend the shared driver's MODE=counters with a backward-compatible SUITES format: name:file[:pass-var:fail-var[:extra-preload ...]]. Optional per-suite counter symbols (override the global COUNTERS_PASS/COUNTERS_FAIL) and per-suite preload chains (loaded after the global PRELOADS). Plain name:file entries are unchanged — verified against haskell (fib/sieve/quicksort 2/2/5, matches committed scoreboard). common-lisp has 8 distinct per-suite counter pairs and a different preload chain per suite, so it could not fit the single-counter/fixed-preload model; the extended format expresses it directly. conformance.conf keeps the historical scoreboard schema; conformance.sh becomes the 3-line shim. Result 487/487 (0 fail) vs the old 305/0 baseline — higher and explained: the old per-suite 'timeout 30' was too tight for the slow eval suite (~15-25s under contention), silently recording it as 0; the driver's 180s budget recovers its true 182. geometry/mop-trace stay 0/0 (pre-existing refl-class-chain-depth-with load error; counter vars defined as 0 -> clean gc-result, no fail-fallback). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-07 09:55:44 +00:00
parent bfdd0fe65a
commit 2e7a08309c
6 changed files with 152 additions and 214 deletions
--- a/lib/common-lisp/conformance.sh
+++ b/lib/common-lisp/conformance.sh
@@ -1,161 +1,3 @@
 #!/usr/bin/env bash
-# lib/common-lisp/conformance.sh — CL-on-SX conformance test runner
-#
-# Runs all Common Lisp test suites and writes scoreboard.json + scoreboard.md.
-#
-# Usage:
-#   bash lib/common-lisp/conformance.sh
-#   bash lib/common-lisp/conformance.sh -v
-
-set -uo pipefail
-cd "$(git rev-parse --show-toplevel)"
-
-SX_SERVER="${SX_SERVER:-hosts/ocaml/_build/default/bin/sx_server.exe}"
-if [ ! -x "$SX_SERVER" ]; then
-  SX_SERVER="/root/rose-ash/hosts/ocaml/_build/default/bin/sx_server.exe"
-fi
-if [ ! -x "$SX_SERVER" ]; then
-  echo "ERROR: sx_server.exe not found."
-  exit 1
-fi
-
-VERBOSE="${1:-}"
-TOTAL_PASS=0; TOTAL_FAIL=0
-SUITE_NAMES=()
-SUITE_PASS=()
-SUITE_FAIL=()
-
-# run_suite NAME "file1 file2 ..." PASS_VAR FAIL_VAR FAILURES_VAR
-run_suite() {
-  local name="$1" load_files="$2" pass_var="$3" fail_var="$4" failures_var="$5"
-  local TMP; TMP=$(mktemp)
-  {
-    printf '(epoch 1)\n(load "spec/stdlib.sx")\n(load "lib/guest/prefix.sx")\n'
-    local i=2
-    for f in $load_files; do
-      printf '(epoch %d)\n(load "%s")\n' "$i" "$f"
-      i=$((i+1))
-    done
-    printf '(epoch 100)\n(eval "%s")\n' "$pass_var"
-    printf '(epoch 101)\n(eval "%s")\n' "$fail_var"
-  } > "$TMP"
-  local OUT; OUT=$(timeout 30 "$SX_SERVER" < "$TMP" 2>/dev/null)
-  rm -f "$TMP"
-  local P F
-  P=$(echo "$OUT" | grep -A1 "^(ok-len 100 " | tail -1 | tr -d ' ()' || true)
-  F=$(echo "$OUT" | grep -A1 "^(ok-len 101 " | tail -1 | tr -d ' ()' || true)
-  # Also try plain (ok 100 N) format
-  [ -z "$P" ] && P=$(echo "$OUT" | grep "^(ok 100 " | awk '{print $3}' | tr -d ')' || true)
-  [ -z "$F" ] && F=$(echo "$OUT" | grep "^(ok 101 " | awk '{print $3}' | tr -d ')' || true)
-  [ -z "$P" ] && P=0; [ -z "$F" ] && F=0
-  SUITE_NAMES+=("$name")
-  SUITE_PASS+=("$P")
-  SUITE_FAIL+=("$F")
-  TOTAL_PASS=$((TOTAL_PASS + P))
-  TOTAL_FAIL=$((TOTAL_FAIL + F))
-  if [ "$F" = "0" ] && [ "${P:-0}" -gt 0 ] 2>/dev/null; then
-    echo "  PASS  $name ($P tests)"
-  else
-    echo "  FAIL  $name ($P passed, $F failed)"
-  fi
-}
-
-echo "=== Common Lisp on SX — Conformance Run ==="
-echo ""
-
-run_suite "Phase 1: tokenizer/reader" \
-  "lib/common-lisp/reader.sx lib/common-lisp/tests/read.sx" \
-  "cl-test-pass" "cl-test-fail" "cl-test-fails"
-
-run_suite "Phase 1: parser/lambda-lists" \
-  "lib/common-lisp/reader.sx lib/common-lisp/parser.sx lib/common-lisp/tests/lambda.sx" \
-  "cl-test-pass" "cl-test-fail" "cl-test-fails"
-
-run_suite "Phase 2: evaluator" \
-  "lib/common-lisp/reader.sx lib/common-lisp/parser.sx lib/common-lisp/eval.sx lib/common-lisp/tests/eval.sx" \
-  "cl-test-pass" "cl-test-fail" "cl-test-fails"
-
-run_suite "Phase 3: condition system" \
-  "lib/common-lisp/runtime.sx lib/common-lisp/tests/conditions.sx" \
-  "passed" "failed" "failures"
-
-run_suite "Phase 3: restart-demo" \
-  "lib/common-lisp/runtime.sx lib/common-lisp/tests/programs/restart-demo.sx" \
-  "demo-passed" "demo-failed" "demo-failures"
-
-run_suite "Phase 3: parse-recover" \
-  "lib/common-lisp/runtime.sx lib/common-lisp/tests/programs/parse-recover.sx" \
-  "parse-passed" "parse-failed" "parse-failures"
-
-run_suite "Phase 3: interactive-debugger" \
-  "lib/common-lisp/runtime.sx lib/common-lisp/tests/programs/interactive-debugger.sx" \
-  "debugger-passed" "debugger-failed" "debugger-failures"
-
-run_suite "Phase 4: CLOS" \
-  "lib/common-lisp/runtime.sx lib/common-lisp/clos.sx lib/common-lisp/tests/clos.sx" \
-  "passed" "failed" "failures"
-
-run_suite "Phase 4: geometry" \
-  "lib/common-lisp/runtime.sx lib/common-lisp/clos.sx lib/common-lisp/tests/programs/geometry.sx" \
-  "geo-passed" "geo-failed" "geo-failures"
-
-run_suite "Phase 4: mop-trace" \
-  "lib/common-lisp/runtime.sx lib/common-lisp/clos.sx lib/common-lisp/tests/programs/mop-trace.sx" \
-  "mop-passed" "mop-failed" "mop-failures"
-
-run_suite "Phase 5: macros+LOOP" \
-  "lib/common-lisp/reader.sx lib/common-lisp/parser.sx lib/common-lisp/eval.sx lib/common-lisp/loop.sx lib/common-lisp/tests/macros.sx" \
-  "macro-passed" "macro-failed" "macro-failures"
-
-run_suite "Phase 6: stdlib" \
-  "lib/common-lisp/reader.sx lib/common-lisp/parser.sx lib/common-lisp/eval.sx lib/common-lisp/tests/stdlib.sx" \
-  "stdlib-passed" "stdlib-failed" "stdlib-failures"
-
-echo ""
-echo "=== Total: $TOTAL_PASS passed, $TOTAL_FAIL failed ==="
-
-# ── write scoreboard.json ─────────────────────────────────────────────────
-
-SCORE_DIR="lib/common-lisp"
-JSON="$SCORE_DIR/scoreboard.json"
-{
-  printf '{\n'
-  printf '  "generated": "%s",\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
-  printf '  "total_pass": %d,\n' "$TOTAL_PASS"
-  printf '  "total_fail": %d,\n' "$TOTAL_FAIL"
-  printf '  "suites": [\n'
-  first=true
-  for i in "${!SUITE_NAMES[@]}"; do
-    if [ "$first" = "true" ]; then first=false; else printf ',\n'; fi
-    printf '    {"name": "%s", "pass": %d, "fail": %d}' \
-      "${SUITE_NAMES[$i]}" "${SUITE_PASS[$i]}" "${SUITE_FAIL[$i]}"
-  done
-  printf '\n  ]\n'
-  printf '}\n'
-} > "$JSON"
-
-# ── write scoreboard.md ───────────────────────────────────────────────────
-
-MD="$SCORE_DIR/scoreboard.md"
-{
-  printf '# Common Lisp on SX — Scoreboard\n\n'
-  printf '_Generated: %s_\n\n' "$(date -u '+%Y-%m-%d %H:%M UTC')"
-  printf '| Suite | Pass | Fail | Status |\n'
-  printf '|-------|------|------|--------|\n'
-  for i in "${!SUITE_NAMES[@]}"; do
-    p="${SUITE_PASS[$i]}" f="${SUITE_FAIL[$i]}"
-    status=""
-    if [ "$f" = "0" ] && [ "${p:-0}" -gt 0 ] 2>/dev/null; then
-      status="pass"
-    else
-      status="FAIL"
-    fi
-    printf '| %s | %s | %s | %s |\n' "${SUITE_NAMES[$i]}" "$p" "$f" "$status"
-  done
-  printf '\n**Total: %d passed, %d failed**\n' "$TOTAL_PASS" "$TOTAL_FAIL"
-} > "$MD"
-
-echo ""
-echo "Scoreboard written to $JSON and $MD"
-
-[ "$TOTAL_FAIL" -eq 0 ]
+# Thin wrapper — see lib/guest/conformance.sh and lib/common-lisp/conformance.conf.
+exec bash "$(dirname "$0")/../guest/conformance.sh" "$(dirname "$0")/conformance.conf" "$@"