GUEST: step 1 — lib/guest/conformance.{sx,sh} config-driven driver
Extracted the duplicated conformance plumbing into a single driver:
- lib/guest/conformance.sx — two helper fns that emit (gc-result NAME P F T)
lines for the bash side to grep: gc-dict-result for runners returning
a {:passed :failed :total} dict, and gc-counters-result for guests that
bump a global pass/fail counter from a test file load.
- lib/guest/conformance.sh — config-driven bash driver. Sources a per-lang
conf, locates sx_server, runs sx_server in either single-session "dict"
mode (one preload + many suite evals) or per-suite "counters" mode
(fresh sx_server per suite, with shared preloads). Aggregates and writes
scoreboard.{json,md} via per-lang emit_scoreboard_* functions.
- Ported lib/prolog/conformance.sh and lib/haskell/conformance.sh down to
one-line wrappers that exec the shared driver against their .conf file.
Verification:
- Prolog: 590/590 — diff vs baseline is timestamp-only.
- Haskell: 156/156 — significantly higher than the 0/18 in baseline. The
old conformance.sh was buggy (its `(ok-len 3 ...)` grep never matched,
defaulting every program to 0 pass / 1 fail). Updated baseline to the
true count; no actual test regressed. Plan baseline cell updated.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
80
lib/prolog/conformance.conf
Normal file
80
lib/prolog/conformance.conf
Normal file
@@ -0,0 +1,80 @@
|
||||
# Prolog conformance config — sourced by lib/guest/conformance.sh.
|
||||
|
||||
LANG_NAME=prolog
|
||||
MODE=dict
|
||||
|
||||
PRELOADS=(
|
||||
lib/prolog/tokenizer.sx
|
||||
lib/prolog/parser.sx
|
||||
lib/prolog/runtime.sx
|
||||
lib/prolog/query.sx
|
||||
lib/prolog/compiler.sx
|
||||
lib/prolog/hs-bridge.sx
|
||||
)
|
||||
|
||||
SUITES=(
|
||||
"parse:lib/prolog/tests/parse.sx:(pl-parse-tests-run!)"
|
||||
"unify:lib/prolog/tests/unify.sx:(pl-unify-tests-run!)"
|
||||
"clausedb:lib/prolog/tests/clausedb.sx:(pl-clausedb-tests-run!)"
|
||||
"solve:lib/prolog/tests/solve.sx:(pl-solve-tests-run!)"
|
||||
"operators:lib/prolog/tests/operators.sx:(pl-operators-tests-run!)"
|
||||
"dynamic:lib/prolog/tests/dynamic.sx:(pl-dynamic-tests-run!)"
|
||||
"findall:lib/prolog/tests/findall.sx:(pl-findall-tests-run!)"
|
||||
"term_inspect:lib/prolog/tests/term_inspect.sx:(pl-term-inspect-tests-run!)"
|
||||
"append:lib/prolog/tests/programs/append.sx:(pl-append-tests-run!)"
|
||||
"reverse:lib/prolog/tests/programs/reverse.sx:(pl-reverse-tests-run!)"
|
||||
"member:lib/prolog/tests/programs/member.sx:(pl-member-tests-run!)"
|
||||
"nqueens:lib/prolog/tests/programs/nqueens.sx:(pl-nqueens-tests-run!)"
|
||||
"family:lib/prolog/tests/programs/family.sx:(pl-family-tests-run!)"
|
||||
"atoms:lib/prolog/tests/atoms.sx:(pl-atom-tests-run!)"
|
||||
"query_api:lib/prolog/tests/query_api.sx:(pl-query-api-tests-run!)"
|
||||
"iso_predicates:lib/prolog/tests/iso_predicates.sx:(pl-iso-predicates-tests-run!)"
|
||||
"meta_predicates:lib/prolog/tests/meta_predicates.sx:(pl-meta-predicates-tests-run!)"
|
||||
"list_predicates:lib/prolog/tests/list_predicates.sx:(pl-list-predicates-tests-run!)"
|
||||
"meta_call:lib/prolog/tests/meta_call.sx:(pl-meta-call-tests-run!)"
|
||||
"set_predicates:lib/prolog/tests/set_predicates.sx:(pl-set-predicates-tests-run!)"
|
||||
"char_predicates:lib/prolog/tests/char_predicates.sx:(pl-char-predicates-tests-run!)"
|
||||
"io_predicates:lib/prolog/tests/io_predicates.sx:(pl-io-predicates-tests-run!)"
|
||||
"assert_rules:lib/prolog/tests/assert_rules.sx:(pl-assert-rules-tests-run!)"
|
||||
"string_agg:lib/prolog/tests/string_agg.sx:(pl-string-agg-tests-run!)"
|
||||
"advanced:lib/prolog/tests/advanced.sx:(pl-advanced-tests-run!)"
|
||||
"compiler:lib/prolog/tests/compiler.sx:(pl-compiler-tests-run!)"
|
||||
"cross_validate:lib/prolog/tests/cross_validate.sx:(pl-cross-validate-tests-run!)"
|
||||
"integration:lib/prolog/tests/integration.sx:(pl-integration-tests-run!)"
|
||||
"hs_bridge:lib/prolog/tests/hs_bridge.sx:(pl-hs-bridge-tests-run!)"
|
||||
)
|
||||
|
||||
emit_scoreboard_json() {
|
||||
local n=${#GC_NAMES[@]} i sep
|
||||
printf '{\n'
|
||||
printf ' "total_passed": %d,\n' "$GC_TOTAL_PASS"
|
||||
printf ' "total_failed": %d,\n' "$GC_TOTAL_FAIL"
|
||||
printf ' "total": %d,\n' "$GC_TOTAL"
|
||||
printf ' "suites": {'
|
||||
for ((i=0; i<n; i++)); do
|
||||
sep=","; [ $i -eq $((n-1)) ] && sep=""
|
||||
printf '"%s":{"passed":%d,"total":%d,"failed":%d}%s' \
|
||||
"${GC_NAMES[$i]}" "${GC_PASS[$i]}" "${GC_TOTAL_S[$i]}" "${GC_FAIL[$i]}" "$sep"
|
||||
done
|
||||
printf '},\n'
|
||||
printf ' "generated": "%s"\n' "$(date -Iseconds 2>/dev/null || date)"
|
||||
printf '}\n'
|
||||
}
|
||||
|
||||
emit_scoreboard_md() {
|
||||
local n=${#GC_NAMES[@]} i status when
|
||||
when="$(date -Iseconds 2>/dev/null || date)"
|
||||
printf '# Prolog scoreboard\n\n'
|
||||
printf '**%d / %d passing** (%d failure(s)).\n' \
|
||||
"$GC_TOTAL_PASS" "$GC_TOTAL" "$GC_TOTAL_FAIL"
|
||||
printf 'Generated %s.\n\n' "$when"
|
||||
printf '| Suite | Passed | Total | Status |\n'
|
||||
printf '|-------|--------|-------|--------|\n'
|
||||
for ((i=0; i<n; i++)); do
|
||||
status="ok"; [ "${GC_FAIL[$i]}" -gt 0 ] && status="FAIL"
|
||||
printf '| %s | %d | %d | %s |\n' \
|
||||
"${GC_NAMES[$i]}" "${GC_PASS[$i]}" "${GC_TOTAL_S[$i]}" "$status"
|
||||
done
|
||||
printf '\nRun `bash lib/prolog/conformance.sh` to refresh. Override the binary\n'
|
||||
printf 'with `SX_SERVER=path/to/sx_server.exe bash …`.\n'
|
||||
}
|
||||
@@ -1,129 +1,3 @@
|
||||
#!/usr/bin/env bash
|
||||
# Run every Prolog test suite via sx_server and refresh scoreboard.{json,md}.
|
||||
# Exit 0 if all green, 1 if any failures.
|
||||
set -euo pipefail
|
||||
|
||||
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||
ROOT="$(cd "$HERE/../.." && pwd)"
|
||||
SX="${SX_SERVER:-/root/rose-ash/hosts/ocaml/_build/default/bin/sx_server.exe}"
|
||||
|
||||
if [[ ! -x "$SX" ]]; then
|
||||
echo "sx_server not found at $SX (set SX_SERVER env to override)" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
cd "$ROOT"
|
||||
|
||||
# name : test-file : runner-fn
|
||||
SUITES=(
|
||||
"parse:lib/prolog/tests/parse.sx:pl-parse-tests-run!"
|
||||
"unify:lib/prolog/tests/unify.sx:pl-unify-tests-run!"
|
||||
"clausedb:lib/prolog/tests/clausedb.sx:pl-clausedb-tests-run!"
|
||||
"solve:lib/prolog/tests/solve.sx:pl-solve-tests-run!"
|
||||
"operators:lib/prolog/tests/operators.sx:pl-operators-tests-run!"
|
||||
"dynamic:lib/prolog/tests/dynamic.sx:pl-dynamic-tests-run!"
|
||||
"findall:lib/prolog/tests/findall.sx:pl-findall-tests-run!"
|
||||
"term_inspect:lib/prolog/tests/term_inspect.sx:pl-term-inspect-tests-run!"
|
||||
"append:lib/prolog/tests/programs/append.sx:pl-append-tests-run!"
|
||||
"reverse:lib/prolog/tests/programs/reverse.sx:pl-reverse-tests-run!"
|
||||
"member:lib/prolog/tests/programs/member.sx:pl-member-tests-run!"
|
||||
"nqueens:lib/prolog/tests/programs/nqueens.sx:pl-nqueens-tests-run!"
|
||||
"family:lib/prolog/tests/programs/family.sx:pl-family-tests-run!"
|
||||
"atoms:lib/prolog/tests/atoms.sx:pl-atom-tests-run!"
|
||||
"query_api:lib/prolog/tests/query_api.sx:pl-query-api-tests-run!"
|
||||
"iso_predicates:lib/prolog/tests/iso_predicates.sx:pl-iso-predicates-tests-run!"
|
||||
"meta_predicates:lib/prolog/tests/meta_predicates.sx:pl-meta-predicates-tests-run!"
|
||||
"list_predicates:lib/prolog/tests/list_predicates.sx:pl-list-predicates-tests-run!"
|
||||
"meta_call:lib/prolog/tests/meta_call.sx:pl-meta-call-tests-run!"
|
||||
"set_predicates:lib/prolog/tests/set_predicates.sx:pl-set-predicates-tests-run!"
|
||||
"char_predicates:lib/prolog/tests/char_predicates.sx:pl-char-predicates-tests-run!"
|
||||
"io_predicates:lib/prolog/tests/io_predicates.sx:pl-io-predicates-tests-run!"
|
||||
"assert_rules:lib/prolog/tests/assert_rules.sx:pl-assert-rules-tests-run!"
|
||||
"string_agg:lib/prolog/tests/string_agg.sx:pl-string-agg-tests-run!"
|
||||
"advanced:lib/prolog/tests/advanced.sx:pl-advanced-tests-run!"
|
||||
"compiler:lib/prolog/tests/compiler.sx:pl-compiler-tests-run!"
|
||||
"cross_validate:lib/prolog/tests/cross_validate.sx:pl-cross-validate-tests-run!"
|
||||
"integration:lib/prolog/tests/integration.sx:pl-integration-tests-run!"
|
||||
"hs_bridge:lib/prolog/tests/hs_bridge.sx:pl-hs-bridge-tests-run!"
|
||||
)
|
||||
|
||||
SCRIPT='(epoch 1)
|
||||
(load "lib/prolog/tokenizer.sx")
|
||||
(load "lib/prolog/parser.sx")
|
||||
(load "lib/prolog/runtime.sx")
|
||||
(load "lib/prolog/query.sx")
|
||||
(load "lib/prolog/compiler.sx")
|
||||
(load "lib/prolog/hs-bridge.sx")'
|
||||
for entry in "${SUITES[@]}"; do
|
||||
IFS=: read -r _ file _ <<< "$entry"
|
||||
SCRIPT+=$'\n(load "'"$file"$'")'
|
||||
done
|
||||
for entry in "${SUITES[@]}"; do
|
||||
IFS=: read -r _ _ fn <<< "$entry"
|
||||
SCRIPT+=$'\n(eval "('"$fn"$')")'
|
||||
done
|
||||
|
||||
OUTPUT="$(printf '%s\n' "$SCRIPT" | "$SX" 2>&1)"
|
||||
|
||||
mapfile -t LINES < <(printf '%s\n' "$OUTPUT" | grep -E '^\{:failed')
|
||||
|
||||
if [[ ${#LINES[@]} -ne ${#SUITES[@]} ]]; then
|
||||
echo "Expected ${#SUITES[@]} suite results, got ${#LINES[@]}" >&2
|
||||
echo "---- raw output ----" >&2
|
||||
printf '%s\n' "$OUTPUT" >&2
|
||||
exit 3
|
||||
fi
|
||||
|
||||
TOTAL_PASS=0
|
||||
TOTAL_FAIL=0
|
||||
TOTAL=0
|
||||
JSON_SUITES=""
|
||||
MD_ROWS=""
|
||||
|
||||
for i in "${!SUITES[@]}"; do
|
||||
IFS=: read -r name _ _ <<< "${SUITES[$i]}"
|
||||
line="${LINES[$i]}"
|
||||
passed=$(grep -oE ':passed [0-9]+' <<< "$line" | grep -oE '[0-9]+')
|
||||
total=$(grep -oE ':total [0-9]+' <<< "$line" | grep -oE '[0-9]+')
|
||||
failed=$(grep -oE ':failed [0-9]+' <<< "$line" | grep -oE '[0-9]+')
|
||||
TOTAL_PASS=$((TOTAL_PASS + passed))
|
||||
TOTAL_FAIL=$((TOTAL_FAIL + failed))
|
||||
TOTAL=$((TOTAL + total))
|
||||
status="ok"
|
||||
[[ "$failed" -gt 0 ]] && status="FAIL"
|
||||
[[ -n "$JSON_SUITES" ]] && JSON_SUITES+=","
|
||||
JSON_SUITES+="\"$name\":{\"passed\":$passed,\"total\":$total,\"failed\":$failed}"
|
||||
MD_ROWS+="| $name | $passed | $total | $status |"$'\n'
|
||||
done
|
||||
|
||||
WHEN="$(date -Iseconds 2>/dev/null || date)"
|
||||
|
||||
cat > "$HERE/scoreboard.json" <<JSON
|
||||
{
|
||||
"total_passed": $TOTAL_PASS,
|
||||
"total_failed": $TOTAL_FAIL,
|
||||
"total": $TOTAL,
|
||||
"suites": {$JSON_SUITES},
|
||||
"generated": "$WHEN"
|
||||
}
|
||||
JSON
|
||||
|
||||
cat > "$HERE/scoreboard.md" <<MD
|
||||
# Prolog scoreboard
|
||||
|
||||
**$TOTAL_PASS / $TOTAL passing** ($TOTAL_FAIL failure(s)).
|
||||
Generated $WHEN.
|
||||
|
||||
| Suite | Passed | Total | Status |
|
||||
|-------|--------|-------|--------|
|
||||
$MD_ROWS
|
||||
Run \`bash lib/prolog/conformance.sh\` to refresh. Override the binary
|
||||
with \`SX_SERVER=path/to/sx_server.exe bash …\`.
|
||||
MD
|
||||
|
||||
if [[ "$TOTAL_FAIL" -gt 0 ]]; then
|
||||
echo "$TOTAL_FAIL failure(s) across $TOTAL tests" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "All $TOTAL tests pass."
|
||||
# Thin wrapper — see lib/guest/conformance.sh and lib/prolog/conformance.conf.
|
||||
exec bash "$(dirname "$0")/../guest/conformance.sh" "$(dirname "$0")/conformance.conf" "$@"
|
||||
|
||||
@@ -3,5 +3,5 @@
|
||||
"total_failed": 0,
|
||||
"total": 590,
|
||||
"suites": {"parse":{"passed":25,"total":25,"failed":0},"unify":{"passed":47,"total":47,"failed":0},"clausedb":{"passed":14,"total":14,"failed":0},"solve":{"passed":62,"total":62,"failed":0},"operators":{"passed":19,"total":19,"failed":0},"dynamic":{"passed":11,"total":11,"failed":0},"findall":{"passed":11,"total":11,"failed":0},"term_inspect":{"passed":14,"total":14,"failed":0},"append":{"passed":6,"total":6,"failed":0},"reverse":{"passed":6,"total":6,"failed":0},"member":{"passed":7,"total":7,"failed":0},"nqueens":{"passed":6,"total":6,"failed":0},"family":{"passed":10,"total":10,"failed":0},"atoms":{"passed":34,"total":34,"failed":0},"query_api":{"passed":16,"total":16,"failed":0},"iso_predicates":{"passed":29,"total":29,"failed":0},"meta_predicates":{"passed":25,"total":25,"failed":0},"list_predicates":{"passed":33,"total":33,"failed":0},"meta_call":{"passed":15,"total":15,"failed":0},"set_predicates":{"passed":15,"total":15,"failed":0},"char_predicates":{"passed":27,"total":27,"failed":0},"io_predicates":{"passed":24,"total":24,"failed":0},"assert_rules":{"passed":15,"total":15,"failed":0},"string_agg":{"passed":25,"total":25,"failed":0},"advanced":{"passed":21,"total":21,"failed":0},"compiler":{"passed":17,"total":17,"failed":0},"cross_validate":{"passed":17,"total":17,"failed":0},"integration":{"passed":20,"total":20,"failed":0},"hs_bridge":{"passed":19,"total":19,"failed":0}},
|
||||
"generated": "2026-05-06T21:34:26+00:00"
|
||||
"generated": "2026-05-06T22:23:38+00:00"
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# Prolog scoreboard
|
||||
|
||||
**590 / 590 passing** (0 failure(s)).
|
||||
Generated 2026-05-06T21:34:26+00:00.
|
||||
Generated 2026-05-06T22:23:38+00:00.
|
||||
|
||||
| Suite | Passed | Total | Status |
|
||||
|-------|--------|-------|--------|
|
||||
|
||||
Reference in New Issue
Block a user