giles
9473911cf3
Some checks failed
Test, Build, and Deploy / test-build-deploy (push) Failing after 45s
lib/ocaml/conformance.sh runs the full test suite, classifies each result by description prefix into one of 14 suites (tokenize, parser, eval-core, phase2-refs/loops/function/exn, phase3-adt, phase4-modules, phase5-hm, phase6-stdlib, let-and, phase1-params, misc), and emits scoreboard.json + scoreboard.md. Per the briefing: "Once the scoreboard exists (Phase 5.1), it is your north star." Real OCaml testsuite vendoring deferred — needs more stdlib + ADT decls to make .ml files runnable.
117 lines
4.0 KiB
Bash
Executable File
117 lines
4.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# lib/ocaml/conformance.sh — run the OCaml-on-SX test suite and emit
|
|
# scoreboard.json + scoreboard.md broken into suites by epoch range.
|
|
#
|
|
# Suites are defined by epoch ranges in test.sh:
|
|
# 100-199 tokenize
|
|
# 200-329 parse-expr
|
|
# 270-329 parse-program (overlaps; assigned to parse-expr)
|
|
# 400-499 eval-core (atoms / arith / control / let / fn)
|
|
# 500-665 phase3-adt-match (incl ref + try/with)
|
|
# 700-754 phase4-modules
|
|
# 800-974 phase6-stdlib
|
|
# 850-852 let-and (small group)
|
|
# 900-913 phase5-hm
|
|
# 1000+ misc
|
|
|
|
set -uo pipefail
|
|
cd "$(git rev-parse --show-toplevel)"
|
|
|
|
SX_SERVER="${SX_SERVER:-hosts/ocaml/_build/default/bin/sx_server.exe}"
|
|
if [ ! -x "$SX_SERVER" ]; then
|
|
SX_SERVER="/root/rose-ash/hosts/ocaml/_build/default/bin/sx_server.exe"
|
|
fi
|
|
if [ ! -x "$SX_SERVER" ]; then
|
|
echo "ERROR: sx_server.exe not found." >&2
|
|
exit 1
|
|
fi
|
|
|
|
OUT_JSON="lib/ocaml/scoreboard.json"
|
|
OUT_MD="lib/ocaml/scoreboard.md"
|
|
|
|
# Run test.sh in verbose mode, capturing per-test pass/fail lines plus
|
|
# the trailing summary.
|
|
TMPLOG=$(mktemp)
|
|
trap "rm -f $TMPLOG" EXIT
|
|
bash lib/ocaml/test.sh -v > "$TMPLOG" 2>&1 || true
|
|
|
|
# Classification by epoch is non-trivial to recover from the human
|
|
# output, so we classify by the test-name prefix that test.sh emits.
|
|
declare -A SUITE_PASS
|
|
declare -A SUITE_FAIL
|
|
|
|
classify() {
|
|
local desc="$1"
|
|
case "$desc" in
|
|
*"tok"*|*"comment"*|*"keyword"*|*"primed"*|*"tyvar"*|*"underscored"*|*"hex"*|*"exponent"*|*"escape"*) echo "tokenize" ;;
|
|
*"parse"*|*"program"*|*"match"*|*"begin/end"*|*"::"*|*"|>"*|*"|"*) echo "parser" ;;
|
|
*"eval"*|*"truthy"*|*"closure"*|*"recur"*|*"fact"*|*"fib"*|*"sum"*|*"curried lambda"*) echo "eval-core" ;;
|
|
*"ref"*|*"deref"*|*"increment"*|*":="*) echo "phase2-refs" ;;
|
|
*"for"*|*"while"*|*"product"*) echo "phase2-loops" ;;
|
|
*"function "*|*"rec function"*) echo "phase2-function" ;;
|
|
*"try"*|*"raise"*|*"failwith"*|*"caught"*) echo "phase2-exn" ;;
|
|
*"None"*|*"Some"*|*"Pair"*|*"Ok"*|*"Error"*|*"ctor"*) echo "phase3-adt" ;;
|
|
*"module"*|*"functor"*|*"include"*|*"open"*|*"M.x"*|*"submodule"*|*"alias"*|*"Sphere"*|*"Identity"*|*"Outer.Inner"*) echo "phase4-modules" ;;
|
|
*"List."*|*"Option."*|*"Result."*|*"Char."*|*"Int."*|*"String."*) echo "phase6-stdlib" ;;
|
|
*"type "*|*"Int -> Int"*|*"poly"*|*"twice"*|*"Bool"*|*" -> "*) echo "phase5-hm" ;;
|
|
*"and y"*|*"mutual"*|*"odd"*|*"even"*) echo "let-and" ;;
|
|
*"unit "*|*"wildcard"*|*"top-level let f"*) echo "phase1-params" ;;
|
|
*) echo "misc" ;;
|
|
esac
|
|
}
|
|
|
|
while IFS= read -r line; do
|
|
if [[ "$line" =~ ^[[:space:]]*ok\ (.+)$ ]]; then
|
|
desc="${BASH_REMATCH[1]}"
|
|
suite=$(classify "$desc")
|
|
SUITE_PASS[$suite]=$(( ${SUITE_PASS[$suite]:-0} + 1 ))
|
|
elif [[ "$line" =~ ^[[:space:]]*FAIL\ (.+)\ \(epoch ]]; then
|
|
desc="${BASH_REMATCH[1]}"
|
|
suite=$(classify "$desc")
|
|
SUITE_FAIL[$suite]=$(( ${SUITE_FAIL[$suite]:-0} + 1 ))
|
|
fi
|
|
done < "$TMPLOG"
|
|
|
|
# Pull the final pass/total
|
|
TOTAL_PASS=0
|
|
TOTAL_FAIL=0
|
|
for s in "${!SUITE_PASS[@]}"; do
|
|
TOTAL_PASS=$(( TOTAL_PASS + ${SUITE_PASS[$s]:-0} ))
|
|
done
|
|
for s in "${!SUITE_FAIL[@]}"; do
|
|
TOTAL_FAIL=$(( TOTAL_FAIL + ${SUITE_FAIL[$s]:-0} ))
|
|
done
|
|
TOTAL=$((TOTAL_PASS + TOTAL_FAIL))
|
|
|
|
# Emit scoreboard.json (suites sorted)
|
|
{
|
|
printf '{\n "suites": {\n'
|
|
first=1
|
|
for s in $(printf '%s\n' "${!SUITE_PASS[@]}" "${!SUITE_FAIL[@]}" | sort -u); do
|
|
p=${SUITE_PASS[$s]:-0}
|
|
f=${SUITE_FAIL[$s]:-0}
|
|
if [ $first -eq 1 ]; then first=0; else printf ',\n'; fi
|
|
printf ' "%s": {"pass": %d, "fail": %d}' "$s" "$p" "$f"
|
|
done
|
|
printf '\n },\n'
|
|
printf ' "total_pass": %d,\n' "$TOTAL_PASS"
|
|
printf ' "total_fail": %d,\n' "$TOTAL_FAIL"
|
|
printf ' "total": %d\n' "$TOTAL"
|
|
printf '}\n'
|
|
} > "$OUT_JSON"
|
|
|
|
# Emit scoreboard.md
|
|
{
|
|
printf '# OCaml-on-SX scoreboard\n\n'
|
|
printf '%d / %d tests passing.\n\n' "$TOTAL_PASS" "$TOTAL"
|
|
printf '| Suite | Pass | Fail |\n'
|
|
printf '|---|---:|---:|\n'
|
|
for s in $(printf '%s\n' "${!SUITE_PASS[@]}" "${!SUITE_FAIL[@]}" | sort -u); do
|
|
p=${SUITE_PASS[$s]:-0}
|
|
f=${SUITE_FAIL[$s]:-0}
|
|
printf '| %s | %d | %d |\n' "$s" "$p" "$f"
|
|
done
|
|
} > "$OUT_MD"
|
|
|
|
cat "$OUT_MD"
|