GUEST: step 1 — lib/guest/conformance.{sx,sh} config-driven driver
Extracted the duplicated conformance plumbing into a single driver:
- lib/guest/conformance.sx — two helper fns that emit (gc-result NAME P F T)
lines for the bash side to grep: gc-dict-result for runners returning
a {:passed :failed :total} dict, and gc-counters-result for guests that
bump a global pass/fail counter from a test file load.
- lib/guest/conformance.sh — config-driven bash driver. Sources a per-lang
conf, locates sx_server, runs sx_server in either single-session "dict"
mode (one preload + many suite evals) or per-suite "counters" mode
(fresh sx_server per suite, with shared preloads). Aggregates and writes
scoreboard.{json,md} via per-lang emit_scoreboard_* functions.
- Ported lib/prolog/conformance.sh and lib/haskell/conformance.sh down to
one-line wrappers that exec the shared driver against their .conf file.
Verification:
- Prolog: 590/590 — diff vs baseline is timestamp-only.
- Haskell: 156/156 — significantly higher than the 0/18 in baseline. The
old conformance.sh was buggy (its `(ok-len 3 ...)` grep never matched,
defaulting every program to 0 pass / 1 fail). Updated baseline to the
true count; no actual test regressed. Plan baseline cell updated.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -1,129 +1,3 @@
|
||||
#!/usr/bin/env bash
|
||||
# Run every Prolog test suite via sx_server and refresh scoreboard.{json,md}.
|
||||
# Exit 0 if all green, 1 if any failures.
|
||||
set -euo pipefail
|
||||
|
||||
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||
ROOT="$(cd "$HERE/../.." && pwd)"
|
||||
SX="${SX_SERVER:-/root/rose-ash/hosts/ocaml/_build/default/bin/sx_server.exe}"
|
||||
|
||||
if [[ ! -x "$SX" ]]; then
|
||||
echo "sx_server not found at $SX (set SX_SERVER env to override)" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
cd "$ROOT"
|
||||
|
||||
# name : test-file : runner-fn
|
||||
SUITES=(
|
||||
"parse:lib/prolog/tests/parse.sx:pl-parse-tests-run!"
|
||||
"unify:lib/prolog/tests/unify.sx:pl-unify-tests-run!"
|
||||
"clausedb:lib/prolog/tests/clausedb.sx:pl-clausedb-tests-run!"
|
||||
"solve:lib/prolog/tests/solve.sx:pl-solve-tests-run!"
|
||||
"operators:lib/prolog/tests/operators.sx:pl-operators-tests-run!"
|
||||
"dynamic:lib/prolog/tests/dynamic.sx:pl-dynamic-tests-run!"
|
||||
"findall:lib/prolog/tests/findall.sx:pl-findall-tests-run!"
|
||||
"term_inspect:lib/prolog/tests/term_inspect.sx:pl-term-inspect-tests-run!"
|
||||
"append:lib/prolog/tests/programs/append.sx:pl-append-tests-run!"
|
||||
"reverse:lib/prolog/tests/programs/reverse.sx:pl-reverse-tests-run!"
|
||||
"member:lib/prolog/tests/programs/member.sx:pl-member-tests-run!"
|
||||
"nqueens:lib/prolog/tests/programs/nqueens.sx:pl-nqueens-tests-run!"
|
||||
"family:lib/prolog/tests/programs/family.sx:pl-family-tests-run!"
|
||||
"atoms:lib/prolog/tests/atoms.sx:pl-atom-tests-run!"
|
||||
"query_api:lib/prolog/tests/query_api.sx:pl-query-api-tests-run!"
|
||||
"iso_predicates:lib/prolog/tests/iso_predicates.sx:pl-iso-predicates-tests-run!"
|
||||
"meta_predicates:lib/prolog/tests/meta_predicates.sx:pl-meta-predicates-tests-run!"
|
||||
"list_predicates:lib/prolog/tests/list_predicates.sx:pl-list-predicates-tests-run!"
|
||||
"meta_call:lib/prolog/tests/meta_call.sx:pl-meta-call-tests-run!"
|
||||
"set_predicates:lib/prolog/tests/set_predicates.sx:pl-set-predicates-tests-run!"
|
||||
"char_predicates:lib/prolog/tests/char_predicates.sx:pl-char-predicates-tests-run!"
|
||||
"io_predicates:lib/prolog/tests/io_predicates.sx:pl-io-predicates-tests-run!"
|
||||
"assert_rules:lib/prolog/tests/assert_rules.sx:pl-assert-rules-tests-run!"
|
||||
"string_agg:lib/prolog/tests/string_agg.sx:pl-string-agg-tests-run!"
|
||||
"advanced:lib/prolog/tests/advanced.sx:pl-advanced-tests-run!"
|
||||
"compiler:lib/prolog/tests/compiler.sx:pl-compiler-tests-run!"
|
||||
"cross_validate:lib/prolog/tests/cross_validate.sx:pl-cross-validate-tests-run!"
|
||||
"integration:lib/prolog/tests/integration.sx:pl-integration-tests-run!"
|
||||
"hs_bridge:lib/prolog/tests/hs_bridge.sx:pl-hs-bridge-tests-run!"
|
||||
)
|
||||
|
||||
SCRIPT='(epoch 1)
|
||||
(load "lib/prolog/tokenizer.sx")
|
||||
(load "lib/prolog/parser.sx")
|
||||
(load "lib/prolog/runtime.sx")
|
||||
(load "lib/prolog/query.sx")
|
||||
(load "lib/prolog/compiler.sx")
|
||||
(load "lib/prolog/hs-bridge.sx")'
|
||||
for entry in "${SUITES[@]}"; do
|
||||
IFS=: read -r _ file _ <<< "$entry"
|
||||
SCRIPT+=$'\n(load "'"$file"$'")'
|
||||
done
|
||||
for entry in "${SUITES[@]}"; do
|
||||
IFS=: read -r _ _ fn <<< "$entry"
|
||||
SCRIPT+=$'\n(eval "('"$fn"$')")'
|
||||
done
|
||||
|
||||
OUTPUT="$(printf '%s\n' "$SCRIPT" | "$SX" 2>&1)"
|
||||
|
||||
mapfile -t LINES < <(printf '%s\n' "$OUTPUT" | grep -E '^\{:failed')
|
||||
|
||||
if [[ ${#LINES[@]} -ne ${#SUITES[@]} ]]; then
|
||||
echo "Expected ${#SUITES[@]} suite results, got ${#LINES[@]}" >&2
|
||||
echo "---- raw output ----" >&2
|
||||
printf '%s\n' "$OUTPUT" >&2
|
||||
exit 3
|
||||
fi
|
||||
|
||||
TOTAL_PASS=0
|
||||
TOTAL_FAIL=0
|
||||
TOTAL=0
|
||||
JSON_SUITES=""
|
||||
MD_ROWS=""
|
||||
|
||||
for i in "${!SUITES[@]}"; do
|
||||
IFS=: read -r name _ _ <<< "${SUITES[$i]}"
|
||||
line="${LINES[$i]}"
|
||||
passed=$(grep -oE ':passed [0-9]+' <<< "$line" | grep -oE '[0-9]+')
|
||||
total=$(grep -oE ':total [0-9]+' <<< "$line" | grep -oE '[0-9]+')
|
||||
failed=$(grep -oE ':failed [0-9]+' <<< "$line" | grep -oE '[0-9]+')
|
||||
TOTAL_PASS=$((TOTAL_PASS + passed))
|
||||
TOTAL_FAIL=$((TOTAL_FAIL + failed))
|
||||
TOTAL=$((TOTAL + total))
|
||||
status="ok"
|
||||
[[ "$failed" -gt 0 ]] && status="FAIL"
|
||||
[[ -n "$JSON_SUITES" ]] && JSON_SUITES+=","
|
||||
JSON_SUITES+="\"$name\":{\"passed\":$passed,\"total\":$total,\"failed\":$failed}"
|
||||
MD_ROWS+="| $name | $passed | $total | $status |"$'\n'
|
||||
done
|
||||
|
||||
WHEN="$(date -Iseconds 2>/dev/null || date)"
|
||||
|
||||
cat > "$HERE/scoreboard.json" <<JSON
|
||||
{
|
||||
"total_passed": $TOTAL_PASS,
|
||||
"total_failed": $TOTAL_FAIL,
|
||||
"total": $TOTAL,
|
||||
"suites": {$JSON_SUITES},
|
||||
"generated": "$WHEN"
|
||||
}
|
||||
JSON
|
||||
|
||||
cat > "$HERE/scoreboard.md" <<MD
|
||||
# Prolog scoreboard
|
||||
|
||||
**$TOTAL_PASS / $TOTAL passing** ($TOTAL_FAIL failure(s)).
|
||||
Generated $WHEN.
|
||||
|
||||
| Suite | Passed | Total | Status |
|
||||
|-------|--------|-------|--------|
|
||||
$MD_ROWS
|
||||
Run \`bash lib/prolog/conformance.sh\` to refresh. Override the binary
|
||||
with \`SX_SERVER=path/to/sx_server.exe bash …\`.
|
||||
MD
|
||||
|
||||
if [[ "$TOTAL_FAIL" -gt 0 ]]; then
|
||||
echo "$TOTAL_FAIL failure(s) across $TOTAL tests" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "All $TOTAL tests pass."
|
||||
# Thin wrapper — see lib/guest/conformance.sh and lib/prolog/conformance.conf.
|
||||
exec bash "$(dirname "$0")/../guest/conformance.sh" "$(dirname "$0")/conformance.conf" "$@"
|
||||
|
||||
Reference in New Issue
Block a user