diff --git a/lib/apl/conformance.conf b/lib/apl/conformance.conf new file mode 100644 index 00000000..23ec56e6 --- /dev/null +++ b/lib/apl/conformance.conf @@ -0,0 +1,63 @@ +# APL conformance config — sourced by lib/guest/conformance.sh. + +LANG_NAME=apl +MODE=counters +COUNTERS_PASS=apl-test-pass +COUNTERS_FAIL=apl-test-fail +TIMEOUT_PER_SUITE=300 + +PRELOADS=( + spec/stdlib.sx + lib/r7rs.sx + lib/apl/runtime.sx + lib/apl/tokenizer.sx + lib/apl/parser.sx + lib/apl/transpile.sx + lib/apl/test-harness.sx +) + +SUITES=( + "structural:lib/apl/tests/structural.sx" + "operators:lib/apl/tests/operators.sx" + "dfn:lib/apl/tests/dfn.sx" + "tradfn:lib/apl/tests/tradfn.sx" + "valence:lib/apl/tests/valence.sx" + "programs:lib/apl/tests/programs.sx" + "system:lib/apl/tests/system.sx" + "idioms:lib/apl/tests/idioms.sx" + "eval-ops:lib/apl/tests/eval-ops.sx" + "pipeline:lib/apl/tests/pipeline.sx" +) + +emit_scoreboard_json() { + local n=${#GC_NAMES[@]} i sep + printf '{\n' + printf ' "suites": {\n' + for ((i=0; i&2 - exit 1 -fi - -SUITES=(structural operators dfn tradfn valence programs system idioms eval-ops pipeline) - -OUT_JSON="lib/apl/scoreboard.json" -OUT_MD="lib/apl/scoreboard.md" - -run_suite() { - local suite=$1 - local file="lib/apl/tests/${suite}.sx" - local TMP - TMP=$(mktemp) - cat > "$TMP" << EPOCHS -(epoch 1) -(load "spec/stdlib.sx") -(load "lib/r7rs.sx") -(load "lib/apl/runtime.sx") -(load "lib/apl/tokenizer.sx") -(load "lib/apl/parser.sx") -(load "lib/apl/transpile.sx") -(epoch 2) -(eval "(define apl-test-pass 0)") -(eval "(define apl-test-fail 0)") -(eval "(define apl-test (fn (name got expected) (if (= got expected) (set! apl-test-pass (+ apl-test-pass 1)) (set! apl-test-fail (+ apl-test-fail 1)))))") -(epoch 3) -(load "${file}") -(epoch 4) -(eval "(list apl-test-pass apl-test-fail)") -EPOCHS - - local OUTPUT - OUTPUT=$(timeout 300 "$SX_SERVER" < "$TMP" 2>/dev/null) - rm -f "$TMP" - - local LINE - LINE=$(echo "$OUTPUT" | awk '/^\(ok-len 4 / {getline; print; exit}') - if [ -z "$LINE" ]; then - LINE=$(echo "$OUTPUT" | grep -E '^\(ok 4 \([0-9]+ [0-9]+\)\)' | tail -1 \ - | sed -E 's/^\(ok 4 //; s/\)$//') - fi - - local P F - P=$(echo "$LINE" | sed -E 's/^\(([0-9]+) ([0-9]+)\).*/\1/') - F=$(echo "$LINE" | sed -E 's/^\(([0-9]+) ([0-9]+)\).*/\2/') - P=${P:-0} - F=${F:-0} - echo "${P} ${F}" -} - -declare -A SUITE_PASS -declare -A SUITE_FAIL -TOTAL_PASS=0 -TOTAL_FAIL=0 - -echo "Running APL conformance suite..." >&2 -for s in "${SUITES[@]}"; do - read -r p f < <(run_suite "$s") - SUITE_PASS[$s]=$p - SUITE_FAIL[$s]=$f - TOTAL_PASS=$((TOTAL_PASS + p)) - TOTAL_FAIL=$((TOTAL_FAIL + f)) - printf " %-12s %d/%d\n" "$s" "$p" "$((p+f))" >&2 -done - -# scoreboard.json -{ - printf '{\n' - printf ' "suites": {\n' - first=1 - for s in "${SUITES[@]}"; do - if [ $first -eq 0 ]; then printf ',\n'; fi - printf ' "%s": {"pass": %d, "fail": %d}' "$s" "${SUITE_PASS[$s]}" "${SUITE_FAIL[$s]}" - first=0 - done - printf '\n },\n' - printf ' "total_pass": %d,\n' "$TOTAL_PASS" - printf ' "total_fail": %d,\n' "$TOTAL_FAIL" - printf ' "total": %d\n' "$((TOTAL_PASS + TOTAL_FAIL))" - printf '}\n' -} > "$OUT_JSON" - -# scoreboard.md -{ - printf '# APL Conformance Scoreboard\n\n' - printf '_Generated by `lib/apl/conformance.sh`_\n\n' - printf '| Suite | Pass | Fail | Total |\n' - printf '|-------|-----:|-----:|------:|\n' - for s in "${SUITES[@]}"; do - p=${SUITE_PASS[$s]} - f=${SUITE_FAIL[$s]} - printf '| %s | %d | %d | %d |\n' "$s" "$p" "$f" "$((p+f))" - done - printf '| **Total** | **%d** | **%d** | **%d** |\n' "$TOTAL_PASS" "$TOTAL_FAIL" "$((TOTAL_PASS + TOTAL_FAIL))" - printf '\n' - printf '## Notes\n\n' - printf '%s\n' '- Suites use the standard `apl-test name got expected` framework loaded against `lib/apl/runtime.sx` + `lib/apl/transpile.sx`.' - printf '%s\n' '- `lib/apl/tests/parse.sx` and `lib/apl/tests/scalar.sx` use their own self-contained frameworks and are excluded from this scoreboard.' -} > "$OUT_MD" - -echo "Wrote $OUT_JSON and $OUT_MD" >&2 -echo "Total: $TOTAL_PASS pass, $TOTAL_FAIL fail" >&2 - -[ "$TOTAL_FAIL" -eq 0 ] +# lib/apl/conformance.sh — APL conformance via the shared guest driver. +# Config lives in lib/apl/conformance.conf (MODE=counters). Override the binary +# with SX_SERVER=path/to/sx_server.exe bash lib/apl/conformance.sh +exec bash "$(dirname "$0")/../guest/conformance.sh" "$(dirname "$0")/conformance.conf" "$@" diff --git a/lib/apl/scoreboard.json b/lib/apl/scoreboard.json index 74c585d1..6c240937 100644 --- a/lib/apl/scoreboard.json +++ b/lib/apl/scoreboard.json @@ -9,9 +9,9 @@ "system": {"pass": 13, "fail": 0}, "idioms": {"pass": 64, "fail": 0}, "eval-ops": {"pass": 14, "fail": 0}, - "pipeline": {"pass": 40, "fail": 0} + "pipeline": {"pass": 152, "fail": 0} }, - "total_pass": 450, + "total_pass": 562, "total_fail": 0, - "total": 450 + "total": 562 } diff --git a/lib/apl/scoreboard.md b/lib/apl/scoreboard.md index 31af7af5..8eb87867 100644 --- a/lib/apl/scoreboard.md +++ b/lib/apl/scoreboard.md @@ -13,8 +13,8 @@ _Generated by `lib/apl/conformance.sh`_ | system | 13 | 0 | 13 | | idioms | 64 | 0 | 64 | | eval-ops | 14 | 0 | 14 | -| pipeline | 40 | 0 | 40 | -| **Total** | **450** | **0** | **450** | +| pipeline | 152 | 0 | 152 | +| **Total** | **562** | **0** | **562** | ## Notes diff --git a/lib/apl/test-harness.sx b/lib/apl/test-harness.sx new file mode 100644 index 00000000..61876415 --- /dev/null +++ b/lib/apl/test-harness.sx @@ -0,0 +1,15 @@ +; lib/apl/test-harness.sx — counters + assertion fn for the shared conformance +; driver (lib/guest/conformance.sh, MODE=counters). Loaded as a PRELOAD so each +; suite starts from a fresh 0/0; suites call (apl-test name got expected). + +(define apl-test-pass 0) +(define apl-test-fail 0) + +(define + apl-test + (fn + (name got expected) + (if + (= got expected) + (set! apl-test-pass (+ apl-test-pass 1)) + (set! apl-test-fail (+ apl-test-fail 1)))))