giles
ae94a24de5
Some checks failed
Test, Build, and Deploy / test-build-deploy (push) Has been cancelled
100 lines
3.2 KiB
Bash
Executable File
100 lines
3.2 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Smalltalk-on-SX conformance runner.
|
|
#
|
|
# Runs the full test suite once with per-file detail, pulls out the
|
|
# classic-corpus numbers, and writes:
|
|
# lib/smalltalk/scoreboard.json — machine-readable summary
|
|
# lib/smalltalk/scoreboard.md — human-readable summary
|
|
#
|
|
# Usage: bash lib/smalltalk/conformance.sh
|
|
|
|
set -uo pipefail
|
|
cd "$(git rev-parse --show-toplevel)"
|
|
|
|
OUT_JSON="lib/smalltalk/scoreboard.json"
|
|
OUT_MD="lib/smalltalk/scoreboard.md"
|
|
|
|
DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
|
|
|
# Catalog .st programs in the corpus.
|
|
PROGRAMS=()
|
|
for f in lib/smalltalk/tests/programs/*.st; do
|
|
[ -f "$f" ] || continue
|
|
PROGRAMS+=("$(basename "$f" .st)")
|
|
done
|
|
NUM_PROGRAMS=${#PROGRAMS[@]}
|
|
|
|
# Run the full test suite with per-file detail.
|
|
RUNNER_OUT=$(bash lib/smalltalk/test.sh -v 2>&1)
|
|
RC=$?
|
|
|
|
# Final summary line: "OK 403/403 ..." or "FAIL 400/403 ...".
|
|
ALL_SUM=$(echo "$RUNNER_OUT" | grep -E '^(OK|FAIL) [0-9]+/[0-9]+' | tail -1)
|
|
ALL_PASS=$(echo "$ALL_SUM" | grep -oE '[0-9]+/[0-9]+' | head -1 | cut -d/ -f1)
|
|
ALL_TOTAL=$(echo "$ALL_SUM" | grep -oE '[0-9]+/[0-9]+' | head -1 | cut -d/ -f2)
|
|
|
|
# Per-file pass counts (verbose lines look like "OK <path> N passed").
|
|
get_pass () {
|
|
local fname="$1"
|
|
echo "$RUNNER_OUT" | awk -v f="$fname" '
|
|
$0 ~ f { for (i=1; i<=NF; i++) if ($i ~ /^[0-9]+$/) { print $i; exit } }'
|
|
}
|
|
|
|
PROG_PASS=$(get_pass "tests/programs.sx")
|
|
PROG_PASS=${PROG_PASS:-0}
|
|
|
|
# scoreboard.json
|
|
{
|
|
printf '{\n'
|
|
printf ' "date": "%s",\n' "$DATE"
|
|
printf ' "programs": [\n'
|
|
for i in "${!PROGRAMS[@]}"; do
|
|
sep=","; [ "$i" -eq "$((NUM_PROGRAMS - 1))" ] && sep=""
|
|
printf ' "%s.st"%s\n' "${PROGRAMS[$i]}" "$sep"
|
|
done
|
|
printf ' ],\n'
|
|
printf ' "program_count": %d,\n' "$NUM_PROGRAMS"
|
|
printf ' "program_tests_passed": %s,\n' "$PROG_PASS"
|
|
printf ' "all_tests_passed": %s,\n' "$ALL_PASS"
|
|
printf ' "all_tests_total": %s,\n' "$ALL_TOTAL"
|
|
printf ' "exit_code": %d\n' "$RC"
|
|
printf '}\n'
|
|
} > "$OUT_JSON"
|
|
|
|
# scoreboard.md
|
|
{
|
|
printf '# Smalltalk-on-SX Scoreboard\n\n'
|
|
printf '_Last run: %s_\n\n' "$DATE"
|
|
|
|
printf '## Totals\n\n'
|
|
printf '| Suite | Passing |\n'
|
|
printf '|-------|---------|\n'
|
|
printf '| All Smalltalk-on-SX tests | **%s / %s** |\n' "$ALL_PASS" "$ALL_TOTAL"
|
|
printf '| Classic-corpus tests (`tests/programs.sx`) | **%s** |\n\n' "$PROG_PASS"
|
|
|
|
printf '## Classic-corpus programs (`lib/smalltalk/tests/programs/`)\n\n'
|
|
printf '| Program | Status |\n'
|
|
printf '|---------|--------|\n'
|
|
for prog in "${PROGRAMS[@]}"; do
|
|
printf '| `%s.st` | present |\n' "$prog"
|
|
done
|
|
printf '\n'
|
|
|
|
printf '## Per-file test counts\n\n'
|
|
printf '```\n'
|
|
echo "$RUNNER_OUT" | grep -E '^(OK|X) lib/smalltalk/tests/' | sort
|
|
printf '```\n\n'
|
|
|
|
printf '## Notes\n\n'
|
|
printf -- '- The spec interpreter is correct but slow (call/cc + dict-based ivars per send).\n'
|
|
printf -- '- Larger Life multi-step verification, the 8-queens canonical case, and the glider-gun pattern are deferred to the JIT path.\n'
|
|
printf -- '- Generated by `bash lib/smalltalk/conformance.sh`. Both files are committed; the runner overwrites them on each run.\n'
|
|
} > "$OUT_MD"
|
|
|
|
echo "Scoreboard updated:"
|
|
echo " $OUT_JSON"
|
|
echo " $OUT_MD"
|
|
echo "Programs: $NUM_PROGRAMS Corpus tests: $PROG_PASS All: $ALL_PASS/$ALL_TOTAL"
|
|
|
|
exit $RC
|