giles
2e7a08309c
Some checks failed
Test, Build, and Deploy / test-build-deploy (push) Failing after 14m2s
Extend the shared driver's MODE=counters with a backward-compatible SUITES format: name:file[:pass-var:fail-var[:extra-preload ...]]. Optional per-suite counter symbols (override the global COUNTERS_PASS/COUNTERS_FAIL) and per-suite preload chains (loaded after the global PRELOADS). Plain name:file entries are unchanged — verified against haskell (fib/sieve/quicksort 2/2/5, matches committed scoreboard). common-lisp has 8 distinct per-suite counter pairs and a different preload chain per suite, so it could not fit the single-counter/fixed-preload model; the extended format expresses it directly. conformance.conf keeps the historical scoreboard schema; conformance.sh becomes the 3-line shim. Result 487/487 (0 fail) vs the old 305/0 baseline — higher and explained: the old per-suite 'timeout 30' was too tight for the slow eval suite (~15-25s under contention), silently recording it as 0; the driver's 180s budget recovers its true 182. geometry/mop-trace stay 0/0 (pre-existing refl-class-chain-depth-with load error; counter vars defined as 0 -> clean gc-result, no fail-fallback). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
68 lines
3.4 KiB
Plaintext
68 lines
3.4 KiB
Plaintext
# Common-Lisp-on-SX conformance config — sourced by lib/guest/conformance.sh.
|
|
#
|
|
# CL suites run their tests at *load* time, mutating per-suite global counters
|
|
# (different variable names per suite), and each suite needs a different
|
|
# preload chain. Both are expressed via the extended MODE=counters SUITES
|
|
# format: "name:file:pass-var:fail-var:extra-preload ...".
|
|
|
|
LANG_NAME=common-lisp
|
|
MODE=counters
|
|
# No global counter defaults — every suite names its own pair below.
|
|
COUNTERS_PASS=
|
|
COUNTERS_FAIL=
|
|
TIMEOUT_PER_SUITE=180
|
|
|
|
# Base preloads common to every suite (loaded before each suite's own chain).
|
|
PRELOADS=(
|
|
spec/stdlib.sx
|
|
lib/guest/prefix.sx
|
|
)
|
|
|
|
# name:file:pass-var:fail-var:extra-preloads(space-separated)
|
|
SUITES=(
|
|
"read:lib/common-lisp/tests/read.sx:cl-test-pass:cl-test-fail:lib/common-lisp/reader.sx"
|
|
"lambda:lib/common-lisp/tests/lambda.sx:cl-test-pass:cl-test-fail:lib/common-lisp/reader.sx lib/common-lisp/parser.sx"
|
|
"eval:lib/common-lisp/tests/eval.sx:cl-test-pass:cl-test-fail:lib/common-lisp/reader.sx lib/common-lisp/parser.sx lib/common-lisp/eval.sx"
|
|
"conditions:lib/common-lisp/tests/conditions.sx:passed:failed:lib/common-lisp/runtime.sx"
|
|
"restart-demo:lib/common-lisp/tests/programs/restart-demo.sx:demo-passed:demo-failed:lib/common-lisp/runtime.sx"
|
|
"parse-recover:lib/common-lisp/tests/programs/parse-recover.sx:parse-passed:parse-failed:lib/common-lisp/runtime.sx"
|
|
"interactive-debugger:lib/common-lisp/tests/programs/interactive-debugger.sx:debugger-passed:debugger-failed:lib/common-lisp/runtime.sx"
|
|
"clos:lib/common-lisp/tests/clos.sx:passed:failed:lib/common-lisp/runtime.sx lib/common-lisp/clos.sx"
|
|
"geometry:lib/common-lisp/tests/programs/geometry.sx:geo-passed:geo-failed:lib/common-lisp/runtime.sx lib/common-lisp/clos.sx"
|
|
"mop-trace:lib/common-lisp/tests/programs/mop-trace.sx:mop-passed:mop-failed:lib/common-lisp/runtime.sx lib/common-lisp/clos.sx"
|
|
"macros:lib/common-lisp/tests/macros.sx:macro-passed:macro-failed:lib/common-lisp/reader.sx lib/common-lisp/parser.sx lib/common-lisp/eval.sx lib/common-lisp/loop.sx"
|
|
"stdlib:lib/common-lisp/tests/stdlib.sx:stdlib-passed:stdlib-failed:lib/common-lisp/reader.sx lib/common-lisp/parser.sx lib/common-lisp/eval.sx"
|
|
)
|
|
|
|
# Preserve the historical scoreboard schema (total_pass/total_fail, suites with
|
|
# name/pass/fail) so any consumer of lib/common-lisp/scoreboard.json keeps working.
|
|
emit_scoreboard_json() {
|
|
local n=${#GC_NAMES[@]} i
|
|
printf '{\n'
|
|
printf ' "generated": "%s",\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
printf ' "total_pass": %d,\n' "$GC_TOTAL_PASS"
|
|
printf ' "total_fail": %d,\n' "$GC_TOTAL_FAIL"
|
|
printf ' "suites": [\n'
|
|
for ((i=0; i<n; i++)); do
|
|
[ "$i" -gt 0 ] && printf ',\n'
|
|
printf ' {"name": "%s", "pass": %d, "fail": %d}' \
|
|
"${GC_NAMES[$i]}" "${GC_PASS[$i]}" "${GC_FAIL[$i]}"
|
|
done
|
|
printf '\n ]\n'
|
|
printf '}\n'
|
|
}
|
|
|
|
emit_scoreboard_md() {
|
|
local n=${#GC_NAMES[@]} i p f status
|
|
printf '# Common Lisp on SX — Scoreboard\n\n'
|
|
printf '_Generated: %s_\n\n' "$(date -u '+%Y-%m-%d %H:%M UTC')"
|
|
printf '| Suite | Pass | Fail | Status |\n'
|
|
printf '|-------|------|------|--------|\n'
|
|
for ((i=0; i<n; i++)); do
|
|
p="${GC_PASS[$i]}"; f="${GC_FAIL[$i]}"
|
|
if [ "$f" = "0" ] && [ "${p:-0}" -gt 0 ] 2>/dev/null; then status="pass"; else status="FAIL"; fi
|
|
printf '| %s | %s | %s | %s |\n' "${GC_NAMES[$i]}" "$p" "$f" "$status"
|
|
done
|
|
printf '\n**Total: %d passed, %d failed**\n' "$GC_TOTAL_PASS" "$GC_TOTAL_FAIL"
|
|
}
|