giles
3e77dd4ded
Some checks failed
Test, Build, and Deploy / test-build-deploy (push) Failing after 51s
114 lines
3.3 KiB
Bash
Executable File
114 lines
3.3 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# lib/apl/conformance.sh — run APL test suites, emit scoreboard.json + scoreboard.md.
|
|
|
|
set -uo pipefail
|
|
cd "$(git rev-parse --show-toplevel)"
|
|
|
|
SX_SERVER="${SX_SERVER:-/root/rose-ash/hosts/ocaml/_build/default/bin/sx_server.exe}"
|
|
if [ ! -x "$SX_SERVER" ]; then
|
|
SX_SERVER="hosts/ocaml/_build/default/bin/sx_server.exe"
|
|
fi
|
|
if [ ! -x "$SX_SERVER" ]; then
|
|
echo "ERROR: sx_server.exe not found." >&2
|
|
exit 1
|
|
fi
|
|
|
|
SUITES=(structural operators dfn tradfn valence programs system)
|
|
|
|
OUT_JSON="lib/apl/scoreboard.json"
|
|
OUT_MD="lib/apl/scoreboard.md"
|
|
|
|
run_suite() {
|
|
local suite=$1
|
|
local file="lib/apl/tests/${suite}.sx"
|
|
local TMP
|
|
TMP=$(mktemp)
|
|
cat > "$TMP" << EPOCHS
|
|
(epoch 1)
|
|
(load "spec/stdlib.sx")
|
|
(load "lib/apl/runtime.sx")
|
|
(load "lib/apl/transpile.sx")
|
|
(epoch 2)
|
|
(eval "(define apl-test-pass 0)")
|
|
(eval "(define apl-test-fail 0)")
|
|
(eval "(define apl-test (fn (name got expected) (if (= got expected) (set! apl-test-pass (+ apl-test-pass 1)) (set! apl-test-fail (+ apl-test-fail 1)))))")
|
|
(epoch 3)
|
|
(load "${file}")
|
|
(epoch 4)
|
|
(eval "(list apl-test-pass apl-test-fail)")
|
|
EPOCHS
|
|
|
|
local OUTPUT
|
|
OUTPUT=$(timeout 180 "$SX_SERVER" < "$TMP" 2>/dev/null)
|
|
rm -f "$TMP"
|
|
|
|
local LINE
|
|
LINE=$(echo "$OUTPUT" | awk '/^\(ok-len 4 / {getline; print; exit}')
|
|
if [ -z "$LINE" ]; then
|
|
LINE=$(echo "$OUTPUT" | grep -E '^\(ok 4 \([0-9]+ [0-9]+\)\)' | tail -1 \
|
|
| sed -E 's/^\(ok 4 //; s/\)$//')
|
|
fi
|
|
|
|
local P F
|
|
P=$(echo "$LINE" | sed -E 's/^\(([0-9]+) ([0-9]+)\).*/\1/')
|
|
F=$(echo "$LINE" | sed -E 's/^\(([0-9]+) ([0-9]+)\).*/\2/')
|
|
P=${P:-0}
|
|
F=${F:-0}
|
|
echo "${P} ${F}"
|
|
}
|
|
|
|
declare -A SUITE_PASS
|
|
declare -A SUITE_FAIL
|
|
TOTAL_PASS=0
|
|
TOTAL_FAIL=0
|
|
|
|
echo "Running APL conformance suite..." >&2
|
|
for s in "${SUITES[@]}"; do
|
|
read -r p f < <(run_suite "$s")
|
|
SUITE_PASS[$s]=$p
|
|
SUITE_FAIL[$s]=$f
|
|
TOTAL_PASS=$((TOTAL_PASS + p))
|
|
TOTAL_FAIL=$((TOTAL_FAIL + f))
|
|
printf " %-12s %d/%d\n" "$s" "$p" "$((p+f))" >&2
|
|
done
|
|
|
|
# scoreboard.json
|
|
{
|
|
printf '{\n'
|
|
printf ' "suites": {\n'
|
|
first=1
|
|
for s in "${SUITES[@]}"; do
|
|
if [ $first -eq 0 ]; then printf ',\n'; fi
|
|
printf ' "%s": {"pass": %d, "fail": %d}' "$s" "${SUITE_PASS[$s]}" "${SUITE_FAIL[$s]}"
|
|
first=0
|
|
done
|
|
printf '\n },\n'
|
|
printf ' "total_pass": %d,\n' "$TOTAL_PASS"
|
|
printf ' "total_fail": %d,\n' "$TOTAL_FAIL"
|
|
printf ' "total": %d\n' "$((TOTAL_PASS + TOTAL_FAIL))"
|
|
printf '}\n'
|
|
} > "$OUT_JSON"
|
|
|
|
# scoreboard.md
|
|
{
|
|
printf '# APL Conformance Scoreboard\n\n'
|
|
printf '_Generated by `lib/apl/conformance.sh`_\n\n'
|
|
printf '| Suite | Pass | Fail | Total |\n'
|
|
printf '|-------|-----:|-----:|------:|\n'
|
|
for s in "${SUITES[@]}"; do
|
|
p=${SUITE_PASS[$s]}
|
|
f=${SUITE_FAIL[$s]}
|
|
printf '| %s | %d | %d | %d |\n' "$s" "$p" "$f" "$((p+f))"
|
|
done
|
|
printf '| **Total** | **%d** | **%d** | **%d** |\n' "$TOTAL_PASS" "$TOTAL_FAIL" "$((TOTAL_PASS + TOTAL_FAIL))"
|
|
printf '\n'
|
|
printf '## Notes\n\n'
|
|
printf '%s\n' '- Suites use the standard `apl-test name got expected` framework loaded against `lib/apl/runtime.sx` + `lib/apl/transpile.sx`.'
|
|
printf '%s\n' '- `lib/apl/tests/parse.sx` and `lib/apl/tests/scalar.sx` use their own self-contained frameworks and are excluded from this scoreboard.'
|
|
} > "$OUT_MD"
|
|
|
|
echo "Wrote $OUT_JSON and $OUT_MD" >&2
|
|
echo "Total: $TOTAL_PASS pass, $TOTAL_FAIL fail" >&2
|
|
|
|
[ "$TOTAL_FAIL" -eq 0 ]
|