rose-ash/lib/common-lisp/conformance.sh

#!/usr/bin/env bash
# lib/common-lisp/conformance.sh — CL-on-SX conformance test runner
#
# Runs all Common Lisp test suites and writes scoreboard.json + scoreboard.md.
#
# Usage:
#   bash lib/common-lisp/conformance.sh
#   bash lib/common-lisp/conformance.sh -v

set -uo pipefail
cd "$(git rev-parse --show-toplevel)"

SX_SERVER="${SX_SERVER:-hosts/ocaml/_build/default/bin/sx_server.exe}"
if [ ! -x "$SX_SERVER" ]; then
  SX_SERVER="/root/rose-ash/hosts/ocaml/_build/default/bin/sx_server.exe"
fi
if [ ! -x "$SX_SERVER" ]; then
  echo "ERROR: sx_server.exe not found."
  exit 1
fi

VERBOSE="${1:-}"
TOTAL_PASS=0; TOTAL_FAIL=0
SUITE_NAMES=()
SUITE_PASS=()
SUITE_FAIL=()

# run_suite NAME "file1 file2 ..." PASS_VAR FAIL_VAR FAILURES_VAR
run_suite() {
  local name="$1" load_files="$2" pass_var="$3" fail_var="$4" failures_var="$5"
  local TMP; TMP=$(mktemp)
  {
    printf '(epoch 1)\n(load "spec/stdlib.sx")\n'
    local i=2
    for f in $load_files; do
      printf '(epoch %d)\n(load "%s")\n' "$i" "$f"
      i=$((i+1))
    done
    printf '(epoch 100)\n(eval "%s")\n' "$pass_var"
    printf '(epoch 101)\n(eval "%s")\n' "$fail_var"
  } > "$TMP"
  local OUT; OUT=$(timeout 30 "$SX_SERVER" < "$TMP" 2>/dev/null)
  rm -f "$TMP"
  local P F
  P=$(echo "$OUT" | grep -A1 "^(ok-len 100 " | tail -1 | tr -d ' ()' || true)
  F=$(echo "$OUT" | grep -A1 "^(ok-len 101 " | tail -1 | tr -d ' ()' || true)
  # Also try plain (ok 100 N) format
  [ -z "$P" ] && P=$(echo "$OUT" | grep "^(ok 100 " | awk '{print $3}' | tr -d ')' || true)
  [ -z "$F" ] && F=$(echo "$OUT" | grep "^(ok 101 " | awk '{print $3}' | tr -d ')' || true)
  [ -z "$P" ] && P=0; [ -z "$F" ] && F=0
  SUITE_NAMES+=("$name")
  SUITE_PASS+=("$P")
  SUITE_FAIL+=("$F")
  TOTAL_PASS=$((TOTAL_PASS + P))
  TOTAL_FAIL=$((TOTAL_FAIL + F))
  if [ "$F" = "0" ] && [ "${P:-0}" -gt 0 ] 2>/dev/null; then
    echo "  PASS  $name ($P tests)"
  else
    echo "  FAIL  $name ($P passed, $F failed)"
  fi
}

echo "=== Common Lisp on SX — Conformance Run ==="
echo ""

run_suite "Phase 1: tokenizer/reader" \
  "lib/common-lisp/reader.sx lib/common-lisp/tests/read.sx" \
  "cl-test-pass" "cl-test-fail" "cl-test-fails"

run_suite "Phase 1: parser/lambda-lists" \
  "lib/common-lisp/reader.sx lib/common-lisp/parser.sx lib/common-lisp/tests/lambda.sx" \
  "cl-test-pass" "cl-test-fail" "cl-test-fails"

run_suite "Phase 2: evaluator" \
  "lib/common-lisp/reader.sx lib/common-lisp/parser.sx lib/common-lisp/eval.sx lib/common-lisp/tests/eval.sx" \
  "cl-test-pass" "cl-test-fail" "cl-test-fails"

run_suite "Phase 3: condition system" \
  "lib/common-lisp/runtime.sx lib/common-lisp/tests/conditions.sx" \
  "passed" "failed" "failures"

run_suite "Phase 3: restart-demo" \
  "lib/common-lisp/runtime.sx lib/common-lisp/tests/programs/restart-demo.sx" \
  "demo-passed" "demo-failed" "demo-failures"

run_suite "Phase 3: parse-recover" \
  "lib/common-lisp/runtime.sx lib/common-lisp/tests/programs/parse-recover.sx" \
  "parse-passed" "parse-failed" "parse-failures"

run_suite "Phase 3: interactive-debugger" \
  "lib/common-lisp/runtime.sx lib/common-lisp/tests/programs/interactive-debugger.sx" \
  "debugger-passed" "debugger-failed" "debugger-failures"

echo ""
echo "=== Total: $TOTAL_PASS passed, $TOTAL_FAIL failed ==="

# ── write scoreboard.json ─────────────────────────────────────────────────

SCORE_DIR="lib/common-lisp"
JSON="$SCORE_DIR/scoreboard.json"
{
  printf '{\n'
  printf '  "generated": "%s",\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
  printf '  "total_pass": %d,\n' "$TOTAL_PASS"
  printf '  "total_fail": %d,\n' "$TOTAL_FAIL"
  printf '  "suites": [\n'
  first=true
  for i in "${!SUITE_NAMES[@]}"; do
    if [ "$first" = "true" ]; then first=false; else printf ',\n'; fi
    printf '    {"name": "%s", "pass": %d, "fail": %d}' \
      "${SUITE_NAMES[$i]}" "${SUITE_PASS[$i]}" "${SUITE_FAIL[$i]}"
  done
  printf '\n  ]\n'
  printf '}\n'
} > "$JSON"

# ── write scoreboard.md ───────────────────────────────────────────────────

MD="$SCORE_DIR/scoreboard.md"
{
  printf '# Common Lisp on SX — Scoreboard\n\n'
  printf '_Generated: %s_\n\n' "$(date -u '+%Y-%m-%d %H:%M UTC')"
  printf '| Suite | Pass | Fail | Status |\n'
  printf '|-------|------|------|--------|\n'
  for i in "${!SUITE_NAMES[@]}"; do
    p="${SUITE_PASS[$i]}" f="${SUITE_FAIL[$i]}"
    status=""
    if [ "$f" = "0" ] && [ "${p:-0}" -gt 0 ] 2>/dev/null; then
      status="pass"
    else
      status="FAIL"
    fi
    printf '| %s | %s | %s | %s |\n' "${SUITE_NAMES[$i]}" "$p" "$f" "$status"
  done
  printf '\n**Total: %d passed, %d failed**\n' "$TOTAL_PASS" "$TOTAL_FAIL"
} > "$MD"

echo ""
echo "Scoreboard written to $JSON and $MD"

[ "$TOTAL_FAIL" -eq 0 ]