GUEST: step 1 — lib/guest/conformance.{sx,sh} config-driven driver
Extracted the duplicated conformance plumbing into a single driver:
- lib/guest/conformance.sx — two helper fns that emit (gc-result NAME P F T)
lines for the bash side to grep: gc-dict-result for runners returning
a {:passed :failed :total} dict, and gc-counters-result for guests that
bump a global pass/fail counter from a test file load.
- lib/guest/conformance.sh — config-driven bash driver. Sources a per-lang
conf, locates sx_server, runs sx_server in either single-session "dict"
mode (one preload + many suite evals) or per-suite "counters" mode
(fresh sx_server per suite, with shared preloads). Aggregates and writes
scoreboard.{json,md} via per-lang emit_scoreboard_* functions.
- Ported lib/prolog/conformance.sh and lib/haskell/conformance.sh down to
one-line wrappers that exec the shared driver against their .conf file.
Verification:
- Prolog: 590/590 — diff vs baseline is timestamp-only.
- Haskell: 156/156 — significantly higher than the 0/18 in baseline. The
old conformance.sh was buggy (its `(ok-len 3 ...)` grep never matched,
defaulting every program to 0 pass / 1 fail). Updated baseline to the
true count; no actual test regressed. Plan baseline cell updated.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -1,140 +1,3 @@
|
||||
#!/usr/bin/env bash
|
||||
# lib/haskell/conformance.sh — run the classic-program test suites.
|
||||
# Writes lib/haskell/scoreboard.json and lib/haskell/scoreboard.md.
|
||||
#
|
||||
# Usage:
|
||||
# bash lib/haskell/conformance.sh # run + write scoreboards
|
||||
# bash lib/haskell/conformance.sh --check # run only, exit 1 on failure
|
||||
|
||||
set -euo pipefail
|
||||
cd "$(git rev-parse --show-toplevel)"
|
||||
|
||||
SX_SERVER="hosts/ocaml/_build/default/bin/sx_server.exe"
|
||||
if [ ! -x "$SX_SERVER" ]; then
|
||||
MAIN_ROOT=$(git worktree list | head -1 | awk '{print $1}')
|
||||
if [ -x "$MAIN_ROOT/$SX_SERVER" ]; then
|
||||
SX_SERVER="$MAIN_ROOT/$SX_SERVER"
|
||||
else
|
||||
echo "ERROR: sx_server.exe not found. Run: cd hosts/ocaml && dune build"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
PROGRAMS=(fib sieve quicksort nqueens calculator collatz palindrome maybe fizzbuzz anagram roman binary either primes zipwith matrix wordcount powers)
|
||||
PASS_COUNTS=()
|
||||
FAIL_COUNTS=()
|
||||
|
||||
run_suite() {
|
||||
local prog="$1"
|
||||
local FILE="lib/haskell/tests/program-${prog}.sx"
|
||||
local TMPFILE
|
||||
TMPFILE=$(mktemp)
|
||||
cat > "$TMPFILE" <<EPOCHS
|
||||
(epoch 1)
|
||||
(load "lib/haskell/tokenizer.sx")
|
||||
(load "lib/haskell/layout.sx")
|
||||
(load "lib/haskell/parser.sx")
|
||||
(load "lib/haskell/desugar.sx")
|
||||
(load "lib/haskell/runtime.sx")
|
||||
(load "lib/haskell/match.sx")
|
||||
(load "lib/haskell/eval.sx")
|
||||
(load "lib/haskell/testlib.sx")
|
||||
(epoch 2)
|
||||
(load "$FILE")
|
||||
(epoch 3)
|
||||
(eval "(list hk-test-pass hk-test-fail)")
|
||||
EPOCHS
|
||||
local OUTPUT
|
||||
OUTPUT=$(timeout 120 "$SX_SERVER" < "$TMPFILE" 2>&1 || true)
|
||||
rm -f "$TMPFILE"
|
||||
|
||||
local LINE
|
||||
LINE=$(echo "$OUTPUT" | awk '/^\(ok-len 3 / {getline; print; exit}')
|
||||
if [ -z "$LINE" ]; then
|
||||
LINE=$(echo "$OUTPUT" | grep -E '^\(ok 3 \([0-9]+ [0-9]+\)\)' | tail -1 \
|
||||
| sed -E 's/^\(ok 3 //; s/\)$//' || true)
|
||||
fi
|
||||
if [ -z "$LINE" ]; then
|
||||
echo "0 1"
|
||||
else
|
||||
local P F
|
||||
P=$(echo "$LINE" | sed -E 's/^\(([0-9]+) ([0-9]+)\).*/\1/' || echo "0")
|
||||
F=$(echo "$LINE" | sed -E 's/^\(([0-9]+) ([0-9]+)\).*/\2/' || echo "1")
|
||||
echo "$P $F"
|
||||
fi
|
||||
}
|
||||
|
||||
for prog in "${PROGRAMS[@]}"; do
|
||||
RESULT=$(run_suite "$prog")
|
||||
P=$(echo "$RESULT" | cut -d' ' -f1)
|
||||
F=$(echo "$RESULT" | cut -d' ' -f2)
|
||||
PASS_COUNTS+=("$P")
|
||||
FAIL_COUNTS+=("$F")
|
||||
T=$((P + F))
|
||||
if [ "$F" -eq 0 ]; then
|
||||
printf '✓ %-14s %d/%d\n' "${prog}.hs" "$P" "$T"
|
||||
else
|
||||
printf '✗ %-14s %d/%d\n' "${prog}.hs" "$P" "$T"
|
||||
fi
|
||||
done
|
||||
|
||||
TOTAL_PASS=0
|
||||
TOTAL_FAIL=0
|
||||
PROG_PASS=0
|
||||
for i in "${!PROGRAMS[@]}"; do
|
||||
TOTAL_PASS=$((TOTAL_PASS + PASS_COUNTS[i]))
|
||||
TOTAL_FAIL=$((TOTAL_FAIL + FAIL_COUNTS[i]))
|
||||
[ "${FAIL_COUNTS[$i]}" -eq 0 ] && PROG_PASS=$((PROG_PASS + 1))
|
||||
done
|
||||
PROG_TOTAL=${#PROGRAMS[@]}
|
||||
|
||||
echo ""
|
||||
echo "Classic programs: ${TOTAL_PASS}/$((TOTAL_PASS + TOTAL_FAIL)) tests | ${PROG_PASS}/${PROG_TOTAL} programs passing"
|
||||
|
||||
if [[ "${1:-}" == "--check" ]]; then
|
||||
[ $TOTAL_FAIL -eq 0 ]
|
||||
exit $?
|
||||
fi
|
||||
|
||||
DATE=$(date '+%Y-%m-%d')
|
||||
|
||||
# scoreboard.json
|
||||
{
|
||||
printf '{\n'
|
||||
printf ' "date": "%s",\n' "$DATE"
|
||||
printf ' "total_pass": %d,\n' "$TOTAL_PASS"
|
||||
printf ' "total_fail": %d,\n' "$TOTAL_FAIL"
|
||||
printf ' "programs": {\n'
|
||||
last=$((${#PROGRAMS[@]} - 1))
|
||||
for i in "${!PROGRAMS[@]}"; do
|
||||
prog="${PROGRAMS[$i]}"
|
||||
if [ $i -lt $last ]; then
|
||||
printf ' "%s": {"pass": %d, "fail": %d},\n' "$prog" "${PASS_COUNTS[$i]}" "${FAIL_COUNTS[$i]}"
|
||||
else
|
||||
printf ' "%s": {"pass": %d, "fail": %d}\n' "$prog" "${PASS_COUNTS[$i]}" "${FAIL_COUNTS[$i]}"
|
||||
fi
|
||||
done
|
||||
printf ' }\n'
|
||||
printf '}\n'
|
||||
} > lib/haskell/scoreboard.json
|
||||
|
||||
# scoreboard.md
|
||||
{
|
||||
printf '# Haskell-on-SX Scoreboard\n\n'
|
||||
printf 'Updated %s · Phase 6 (prelude extras + 18 programs)\n\n' "$DATE"
|
||||
printf '| Program | Tests | Status |\n'
|
||||
printf '|---------|-------|--------|\n'
|
||||
for i in "${!PROGRAMS[@]}"; do
|
||||
prog="${PROGRAMS[$i]}"
|
||||
P=${PASS_COUNTS[$i]}
|
||||
F=${FAIL_COUNTS[$i]}
|
||||
T=$((P + F))
|
||||
[ "$F" -eq 0 ] && STATUS="✓" || STATUS="✗"
|
||||
printf '| %s | %d/%d | %s |\n' "${prog}.hs" "$P" "$T" "$STATUS"
|
||||
done
|
||||
printf '| **Total** | **%d/%d** | **%d/%d programs** |\n' \
|
||||
"$TOTAL_PASS" "$((TOTAL_PASS + TOTAL_FAIL))" "$PROG_PASS" "$PROG_TOTAL"
|
||||
} > lib/haskell/scoreboard.md
|
||||
|
||||
echo "Wrote lib/haskell/scoreboard.json and lib/haskell/scoreboard.md"
|
||||
[ $TOTAL_FAIL -eq 0 ]
|
||||
# Thin wrapper — see lib/guest/conformance.sh and lib/haskell/conformance.conf.
|
||||
exec bash "$(dirname "$0")/../guest/conformance.sh" "$(dirname "$0")/conformance.conf" "$@"
|
||||
|
||||
Reference in New Issue
Block a user