From ec26b61cbe1ec53755c1a316446c1f673005079c Mon Sep 17 00:00:00 2001 From: giles Date: Thu, 7 May 2026 03:37:58 +0000 Subject: [PATCH] =?UTF-8?q?apl:=20conformance.sh=20+=20scoreboard.{json,md?= =?UTF-8?q?}=20=E2=80=94=20Phase=205=20complete=20(269/269)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/apl/conformance.sh | 113 ++++++++++++++++++++++++++++++++++++++++ lib/apl/scoreboard.json | 12 +++++ lib/apl/scoreboard.md | 17 ++++++ plans/apl-on-sx.md | 3 +- 4 files changed, 144 insertions(+), 1 deletion(-) create mode 100755 lib/apl/conformance.sh create mode 100644 lib/apl/scoreboard.json create mode 100644 lib/apl/scoreboard.md diff --git a/lib/apl/conformance.sh b/lib/apl/conformance.sh new file mode 100755 index 00000000..467b883f --- /dev/null +++ b/lib/apl/conformance.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash +# lib/apl/conformance.sh — run APL test suites, emit scoreboard.json + scoreboard.md. + +set -uo pipefail +cd "$(git rev-parse --show-toplevel)" + +SX_SERVER="${SX_SERVER:-/root/rose-ash/hosts/ocaml/_build/default/bin/sx_server.exe}" +if [ ! -x "$SX_SERVER" ]; then + SX_SERVER="hosts/ocaml/_build/default/bin/sx_server.exe" +fi +if [ ! -x "$SX_SERVER" ]; then + echo "ERROR: sx_server.exe not found." >&2 + exit 1 +fi + +SUITES=(structural operators dfn tradfn valence) + +OUT_JSON="lib/apl/scoreboard.json" +OUT_MD="lib/apl/scoreboard.md" + +run_suite() { + local suite=$1 + local file="lib/apl/tests/${suite}.sx" + local TMP + TMP=$(mktemp) + cat > "$TMP" << EPOCHS +(epoch 1) +(load "spec/stdlib.sx") +(load "lib/apl/runtime.sx") +(load "lib/apl/transpile.sx") +(epoch 2) +(eval "(define apl-test-pass 0)") +(eval "(define apl-test-fail 0)") +(eval "(define apl-test (fn (name got expected) (if (= got expected) (set! apl-test-pass (+ apl-test-pass 1)) (set! apl-test-fail (+ apl-test-fail 1)))))") +(epoch 3) +(load "${file}") +(epoch 4) +(eval "(list apl-test-pass apl-test-fail)") +EPOCHS + + local OUTPUT + OUTPUT=$(timeout 60 "$SX_SERVER" < "$TMP" 2>/dev/null) + rm -f "$TMP" + + local LINE + LINE=$(echo "$OUTPUT" | awk '/^\(ok-len 4 / {getline; print; exit}') + if [ -z "$LINE" ]; then + LINE=$(echo "$OUTPUT" | grep -E '^\(ok 4 \([0-9]+ [0-9]+\)\)' | tail -1 \ + | sed -E 's/^\(ok 4 //; s/\)$//') + fi + + local P F + P=$(echo "$LINE" | sed -E 's/^\(([0-9]+) ([0-9]+)\).*/\1/') + F=$(echo "$LINE" | sed -E 's/^\(([0-9]+) ([0-9]+)\).*/\2/') + P=${P:-0} + F=${F:-0} + echo "${P} ${F}" +} + +declare -A SUITE_PASS +declare -A SUITE_FAIL +TOTAL_PASS=0 +TOTAL_FAIL=0 + +echo "Running APL conformance suite..." >&2 +for s in "${SUITES[@]}"; do + read -r p f < <(run_suite "$s") + SUITE_PASS[$s]=$p + SUITE_FAIL[$s]=$f + TOTAL_PASS=$((TOTAL_PASS + p)) + TOTAL_FAIL=$((TOTAL_FAIL + f)) + printf " %-12s %d/%d\n" "$s" "$p" "$((p+f))" >&2 +done + +# scoreboard.json +{ + printf '{\n' + printf ' "suites": {\n' + first=1 + for s in "${SUITES[@]}"; do + if [ $first -eq 0 ]; then printf ',\n'; fi + printf ' "%s": {"pass": %d, "fail": %d}' "$s" "${SUITE_PASS[$s]}" "${SUITE_FAIL[$s]}" + first=0 + done + printf '\n },\n' + printf ' "total_pass": %d,\n' "$TOTAL_PASS" + printf ' "total_fail": %d,\n' "$TOTAL_FAIL" + printf ' "total": %d\n' "$((TOTAL_PASS + TOTAL_FAIL))" + printf '}\n' +} > "$OUT_JSON" + +# scoreboard.md +{ + printf '# APL Conformance Scoreboard\n\n' + printf '_Generated by `lib/apl/conformance.sh`_\n\n' + printf '| Suite | Pass | Fail | Total |\n' + printf '|-------|-----:|-----:|------:|\n' + for s in "${SUITES[@]}"; do + p=${SUITE_PASS[$s]} + f=${SUITE_FAIL[$s]} + printf '| %s | %d | %d | %d |\n' "$s" "$p" "$f" "$((p+f))" + done + printf '| **Total** | **%d** | **%d** | **%d** |\n' "$TOTAL_PASS" "$TOTAL_FAIL" "$((TOTAL_PASS + TOTAL_FAIL))" + printf '\n' + printf '## Notes\n\n' + printf '%s\n' '- Suites use the standard `apl-test name got expected` framework loaded against `lib/apl/runtime.sx` + `lib/apl/transpile.sx`.' + printf '%s\n' '- `lib/apl/tests/parse.sx` and `lib/apl/tests/scalar.sx` use their own self-contained frameworks and are excluded from this scoreboard.' +} > "$OUT_MD" + +echo "Wrote $OUT_JSON and $OUT_MD" >&2 +echo "Total: $TOTAL_PASS pass, $TOTAL_FAIL fail" >&2 + +[ "$TOTAL_FAIL" -eq 0 ] diff --git a/lib/apl/scoreboard.json b/lib/apl/scoreboard.json new file mode 100644 index 00000000..98fd7834 --- /dev/null +++ b/lib/apl/scoreboard.json @@ -0,0 +1,12 @@ +{ + "suites": { + "structural": {"pass": 94, "fail": 0}, + "operators": {"pass": 117, "fail": 0}, + "dfn": {"pass": 24, "fail": 0}, + "tradfn": {"pass": 20, "fail": 0}, + "valence": {"pass": 14, "fail": 0} + }, + "total_pass": 269, + "total_fail": 0, + "total": 269 +} diff --git a/lib/apl/scoreboard.md b/lib/apl/scoreboard.md new file mode 100644 index 00000000..231fa535 --- /dev/null +++ b/lib/apl/scoreboard.md @@ -0,0 +1,17 @@ +# APL Conformance Scoreboard + +_Generated by `lib/apl/conformance.sh`_ + +| Suite | Pass | Fail | Total | +|-------|-----:|-----:|------:| +| structural | 94 | 0 | 94 | +| operators | 117 | 0 | 117 | +| dfn | 24 | 0 | 24 | +| tradfn | 20 | 0 | 20 | +| valence | 14 | 0 | 14 | +| **Total** | **269** | **0** | **269** | + +## Notes + +- Suites use the standard `apl-test name got expected` framework loaded against `lib/apl/runtime.sx` + `lib/apl/transpile.sx`. +- `lib/apl/tests/parse.sx` and `lib/apl/tests/scalar.sx` use their own self-contained frameworks and are excluded from this scoreboard. diff --git a/plans/apl-on-sx.md b/plans/apl-on-sx.md index 6f41492c..34ade48d 100644 --- a/plans/apl-on-sx.md +++ b/plans/apl-on-sx.md @@ -91,7 +91,7 @@ Core mapping: - [x] Tradfn `∇` header: `R←L F R;l1;l2`, statement-by-statement, branch via `→linenum` - [x] Dyalog control words: `:If/:Else/:EndIf`, `:While/:EndWhile`, `:For X :In V :EndFor`, `:Select/:Case/:EndSelect`, `:Trap`/`:EndTrap` _(Trap deferred — no exception machinery yet)_ - [x] Niladic / monadic / dyadic dispatch (function valence at definition time) -- [ ] `lib/apl/conformance.sh` + runner, `scoreboard.json` + `scoreboard.md` +- [x] `lib/apl/conformance.sh` + runner, `scoreboard.json` + `scoreboard.md` ### Phase 6 — classic programs + drive corpus - [ ] Classic programs in `lib/apl/tests/programs/`: @@ -118,6 +118,7 @@ data; format for string templating. _Newest first._ +- 2026-05-07: Phase 5 conformance.sh + scoreboard.{json,md} — per-suite runner; current snapshot 269/269; **Phase 5 complete** - 2026-05-07: Phase 5 valence dispatch — apl-dfn-valence (AST scan for ⍺/⍵), apl-tradfn-valence (slot check), apl-call unified entry; +14 tests; 269/269 tests - 2026-05-07: Phase 5 control words — :If/:Else, :While, :For/:In, :Select/:Case via apl-tradfn-eval-block/stmt threading env; :Trap deferred; +10 tests (sum loop, factorial, dispatch, nested); 255/255 tests - 2026-05-07: Phase 5 tradfn — apl-call-tradfn + apl-tradfn-loop; line-numbered stmts, :branch goto, →0 exits, locals; +10 tests including loop sum; 245/245 tests