cl: Phase 3 all complete — conformance.sh runner, 363/363 tests green
conformance.sh runs all 7 test suites (reader/parser/eval/conditions/ restart-demo/parse-recover/interactive-debugger), writes scoreboard.json and scoreboard.md. 363 total tests: 79 tokenizer, 31 parser/lambda-lists, 174 evaluator (including unwind-protect), 59 conditions, 20 classic programs. Phase 3 fully complete — all roadmap boxes ticked. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
141
lib/common-lisp/conformance.sh
Executable file
141
lib/common-lisp/conformance.sh
Executable file
@@ -0,0 +1,141 @@
|
||||
#!/usr/bin/env bash
|
||||
# lib/common-lisp/conformance.sh — CL-on-SX conformance test runner
|
||||
#
|
||||
# Runs all Common Lisp test suites and writes scoreboard.json + scoreboard.md.
|
||||
#
|
||||
# Usage:
|
||||
# bash lib/common-lisp/conformance.sh
|
||||
# bash lib/common-lisp/conformance.sh -v
|
||||
|
||||
set -uo pipefail
|
||||
cd "$(git rev-parse --show-toplevel)"
|
||||
|
||||
SX_SERVER="${SX_SERVER:-hosts/ocaml/_build/default/bin/sx_server.exe}"
|
||||
if [ ! -x "$SX_SERVER" ]; then
|
||||
SX_SERVER="/root/rose-ash/hosts/ocaml/_build/default/bin/sx_server.exe"
|
||||
fi
|
||||
if [ ! -x "$SX_SERVER" ]; then
|
||||
echo "ERROR: sx_server.exe not found."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
VERBOSE="${1:-}"
|
||||
TOTAL_PASS=0; TOTAL_FAIL=0
|
||||
SUITE_NAMES=()
|
||||
SUITE_PASS=()
|
||||
SUITE_FAIL=()
|
||||
|
||||
# run_suite NAME "file1 file2 ..." PASS_VAR FAIL_VAR FAILURES_VAR
|
||||
run_suite() {
|
||||
local name="$1" load_files="$2" pass_var="$3" fail_var="$4" failures_var="$5"
|
||||
local TMP; TMP=$(mktemp)
|
||||
{
|
||||
printf '(epoch 1)\n(load "spec/stdlib.sx")\n'
|
||||
local i=2
|
||||
for f in $load_files; do
|
||||
printf '(epoch %d)\n(load "%s")\n' "$i" "$f"
|
||||
i=$((i+1))
|
||||
done
|
||||
printf '(epoch 100)\n(eval "%s")\n' "$pass_var"
|
||||
printf '(epoch 101)\n(eval "%s")\n' "$fail_var"
|
||||
} > "$TMP"
|
||||
local OUT; OUT=$(timeout 30 "$SX_SERVER" < "$TMP" 2>/dev/null)
|
||||
rm -f "$TMP"
|
||||
local P F
|
||||
P=$(echo "$OUT" | grep -A1 "^(ok-len 100 " | tail -1 | tr -d ' ()' || true)
|
||||
F=$(echo "$OUT" | grep -A1 "^(ok-len 101 " | tail -1 | tr -d ' ()' || true)
|
||||
# Also try plain (ok 100 N) format
|
||||
[ -z "$P" ] && P=$(echo "$OUT" | grep "^(ok 100 " | awk '{print $3}' | tr -d ')' || true)
|
||||
[ -z "$F" ] && F=$(echo "$OUT" | grep "^(ok 101 " | awk '{print $3}' | tr -d ')' || true)
|
||||
[ -z "$P" ] && P=0; [ -z "$F" ] && F=0
|
||||
SUITE_NAMES+=("$name")
|
||||
SUITE_PASS+=("$P")
|
||||
SUITE_FAIL+=("$F")
|
||||
TOTAL_PASS=$((TOTAL_PASS + P))
|
||||
TOTAL_FAIL=$((TOTAL_FAIL + F))
|
||||
if [ "$F" = "0" ] && [ "${P:-0}" -gt 0 ] 2>/dev/null; then
|
||||
echo " PASS $name ($P tests)"
|
||||
else
|
||||
echo " FAIL $name ($P passed, $F failed)"
|
||||
fi
|
||||
}
|
||||
|
||||
echo "=== Common Lisp on SX — Conformance Run ==="
|
||||
echo ""
|
||||
|
||||
run_suite "Phase 1: tokenizer/reader" \
|
||||
"lib/common-lisp/reader.sx lib/common-lisp/tests/read.sx" \
|
||||
"cl-test-pass" "cl-test-fail" "cl-test-fails"
|
||||
|
||||
run_suite "Phase 1: parser/lambda-lists" \
|
||||
"lib/common-lisp/reader.sx lib/common-lisp/parser.sx lib/common-lisp/tests/lambda.sx" \
|
||||
"cl-test-pass" "cl-test-fail" "cl-test-fails"
|
||||
|
||||
run_suite "Phase 2: evaluator" \
|
||||
"lib/common-lisp/reader.sx lib/common-lisp/parser.sx lib/common-lisp/eval.sx lib/common-lisp/tests/eval.sx" \
|
||||
"cl-test-pass" "cl-test-fail" "cl-test-fails"
|
||||
|
||||
run_suite "Phase 3: condition system" \
|
||||
"lib/common-lisp/runtime.sx lib/common-lisp/tests/conditions.sx" \
|
||||
"passed" "failed" "failures"
|
||||
|
||||
run_suite "Phase 3: restart-demo" \
|
||||
"lib/common-lisp/runtime.sx lib/common-lisp/tests/programs/restart-demo.sx" \
|
||||
"demo-passed" "demo-failed" "demo-failures"
|
||||
|
||||
run_suite "Phase 3: parse-recover" \
|
||||
"lib/common-lisp/runtime.sx lib/common-lisp/tests/programs/parse-recover.sx" \
|
||||
"parse-passed" "parse-failed" "parse-failures"
|
||||
|
||||
run_suite "Phase 3: interactive-debugger" \
|
||||
"lib/common-lisp/runtime.sx lib/common-lisp/tests/programs/interactive-debugger.sx" \
|
||||
"debugger-passed" "debugger-failed" "debugger-failures"
|
||||
|
||||
echo ""
|
||||
echo "=== Total: $TOTAL_PASS passed, $TOTAL_FAIL failed ==="
|
||||
|
||||
# ── write scoreboard.json ─────────────────────────────────────────────────
|
||||
|
||||
SCORE_DIR="lib/common-lisp"
|
||||
JSON="$SCORE_DIR/scoreboard.json"
|
||||
{
|
||||
printf '{\n'
|
||||
printf ' "generated": "%s",\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
printf ' "total_pass": %d,\n' "$TOTAL_PASS"
|
||||
printf ' "total_fail": %d,\n' "$TOTAL_FAIL"
|
||||
printf ' "suites": [\n'
|
||||
first=true
|
||||
for i in "${!SUITE_NAMES[@]}"; do
|
||||
if [ "$first" = "true" ]; then first=false; else printf ',\n'; fi
|
||||
printf ' {"name": "%s", "pass": %d, "fail": %d}' \
|
||||
"${SUITE_NAMES[$i]}" "${SUITE_PASS[$i]}" "${SUITE_FAIL[$i]}"
|
||||
done
|
||||
printf '\n ]\n'
|
||||
printf '}\n'
|
||||
} > "$JSON"
|
||||
|
||||
# ── write scoreboard.md ───────────────────────────────────────────────────
|
||||
|
||||
MD="$SCORE_DIR/scoreboard.md"
|
||||
{
|
||||
printf '# Common Lisp on SX — Scoreboard\n\n'
|
||||
printf '_Generated: %s_\n\n' "$(date -u '+%Y-%m-%d %H:%M UTC')"
|
||||
printf '| Suite | Pass | Fail | Status |\n'
|
||||
printf '|-------|------|------|--------|\n'
|
||||
for i in "${!SUITE_NAMES[@]}"; do
|
||||
p="${SUITE_PASS[$i]}" f="${SUITE_FAIL[$i]}"
|
||||
status=""
|
||||
if [ "$f" = "0" ] && [ "${p:-0}" -gt 0 ] 2>/dev/null; then
|
||||
status="pass"
|
||||
else
|
||||
status="FAIL"
|
||||
fi
|
||||
printf '| %s | %s | %s | %s |\n' "${SUITE_NAMES[$i]}" "$p" "$f" "$status"
|
||||
done
|
||||
printf '\n**Total: %d passed, %d failed**\n' "$TOTAL_PASS" "$TOTAL_FAIL"
|
||||
} > "$MD"
|
||||
|
||||
echo ""
|
||||
echo "Scoreboard written to $JSON and $MD"
|
||||
|
||||
[ "$TOTAL_FAIL" -eq 0 ]
|
||||
14
lib/common-lisp/scoreboard.json
Normal file
14
lib/common-lisp/scoreboard.json
Normal file
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"generated": "2026-05-05T11:24:34Z",
|
||||
"total_pass": 363,
|
||||
"total_fail": 0,
|
||||
"suites": [
|
||||
{"name": "Phase 1: tokenizer/reader", "pass": 79, "fail": 0},
|
||||
{"name": "Phase 1: parser/lambda-lists", "pass": 31, "fail": 0},
|
||||
{"name": "Phase 2: evaluator", "pass": 174, "fail": 0},
|
||||
{"name": "Phase 3: condition system", "pass": 59, "fail": 0},
|
||||
{"name": "Phase 3: restart-demo", "pass": 7, "fail": 0},
|
||||
{"name": "Phase 3: parse-recover", "pass": 6, "fail": 0},
|
||||
{"name": "Phase 3: interactive-debugger", "pass": 7, "fail": 0}
|
||||
]
|
||||
}
|
||||
15
lib/common-lisp/scoreboard.md
Normal file
15
lib/common-lisp/scoreboard.md
Normal file
@@ -0,0 +1,15 @@
|
||||
# Common Lisp on SX — Scoreboard
|
||||
|
||||
_Generated: 2026-05-05 11:24 UTC_
|
||||
|
||||
| Suite | Pass | Fail | Status |
|
||||
|-------|------|------|--------|
|
||||
| Phase 1: tokenizer/reader | 79 | 0 | pass |
|
||||
| Phase 1: parser/lambda-lists | 31 | 0 | pass |
|
||||
| Phase 2: evaluator | 174 | 0 | pass |
|
||||
| Phase 3: condition system | 59 | 0 | pass |
|
||||
| Phase 3: restart-demo | 7 | 0 | pass |
|
||||
| Phase 3: parse-recover | 6 | 0 | pass |
|
||||
| Phase 3: interactive-debugger | 7 | 0 | pass |
|
||||
|
||||
**Total: 363 passed, 0 failed**
|
||||
@@ -78,7 +78,7 @@ Core mapping:
|
||||
- [x] `restart-demo.sx` — division with `use-zero` and `retry` restarts (7 tests)
|
||||
- [x] `parse-recover.sx` — parser with skipped-token restart (6 tests)
|
||||
- [x] `interactive-debugger.sx` — policy-driven debugger hook, *debugger-hook* global (7 tests)
|
||||
- [ ] `lib/common-lisp/conformance.sh` + runner, `scoreboard.json` + `scoreboard.md`
|
||||
- [x] `lib/common-lisp/conformance.sh` + runner, `scoreboard.json` + `scoreboard.md` (363 total tests)
|
||||
|
||||
### Phase 4 — CLOS
|
||||
- [ ] `defclass` with `:initarg`/`:initform`/`:accessor`/`:reader`/`:writer`/`:allocation`
|
||||
@@ -124,6 +124,7 @@ data; format for string templating.
|
||||
|
||||
_Newest first._
|
||||
|
||||
- 2026-05-05: Phase 3 fully complete — conformance.sh runner + scoreboard.json/scoreboard.md; 363 total tests across all suites (79 reader, 31 parser, 174 eval, 59 conditions, 7+6+7 classic programs).
|
||||
- 2026-05-05: Phase 3 complete — cl-debugger-hook/cl-invoke-debugger in runtime.sx (cl-error routes through hook), cl-break-on-signals (fires hook before handlers on type match), cl-invoke-restart-interactively (calls fn with no args); 4 new tests (147 total). Phase 3 all boxes ticked.
|
||||
- 2026-05-05: Phase 3 interactive-debugger.sx — cl-debugger-hook global, cl-invoke-debugger, cl-error-with-debugger, make-policy-debugger; 7 tests (143 total). Tests wired into test.sh program suite runner. Phase 3 condition core complete.
|
||||
- 2026-05-05: Phase 3 classic programs — `tests/programs/restart-demo.sx` (7 tests: safe-divide with use-zero + retry restarts) and `tests/programs/parse-recover.sx` (6 tests: token parser with skip-token + use-zero restarts, handler-case abort). Key gotcha: use `=` not `equal?` for list comparison in sx_server.
|
||||
|
||||
Reference in New Issue
Block a user