giles
e115af86d8
Some checks failed
Test, Build, and Deploy / test-build-deploy (push) Failing after 38s
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
156 lines
4.7 KiB
Bash
Executable File
156 lines
4.7 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# lib/content/conformance.sh — run content-on-sx suites, emit scoreboard.
|
|
|
|
set -uo pipefail
|
|
cd "$(git rev-parse --show-toplevel)"
|
|
|
|
SX_SERVER="hosts/ocaml/_build/default/bin/sx_server.exe"
|
|
if [ ! -x "$SX_SERVER" ]; then
|
|
MAIN_ROOT=$(git worktree list | head -1 | awk '{print $1}')
|
|
if [ -x "$MAIN_ROOT/$SX_SERVER" ]; then
|
|
SX_SERVER="$MAIN_ROOT/$SX_SERVER"
|
|
else
|
|
echo "ERROR: sx_server.exe not found." >&2
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
SUITES=(block doc render api meta page page-full markdown text section compose tree-edit move clone query toc anchor outline flatten transform normalize find-replace stats table callout media data wire validate store snapshot crdt crdt-store sync md-import md-doc fed)
|
|
|
|
OUT_JSON="lib/content/scoreboard.json"
|
|
OUT_MD="lib/content/scoreboard.md"
|
|
|
|
run_suite() {
|
|
local suite=$1
|
|
local file="lib/content/tests/${suite}.sx"
|
|
local TMP
|
|
TMP=$(mktemp)
|
|
cat > "$TMP" << EPOCHS
|
|
(epoch 1)
|
|
(load "lib/smalltalk/tokenizer.sx")
|
|
(load "lib/smalltalk/parser.sx")
|
|
(load "lib/guest/reflective/class-chain.sx")
|
|
(load "lib/smalltalk/runtime.sx")
|
|
(load "lib/guest/reflective/env.sx")
|
|
(load "lib/smalltalk/eval.sx")
|
|
(load "lib/persist/event.sx")
|
|
(load "lib/persist/backend.sx")
|
|
(load "lib/persist/log.sx")
|
|
(load "lib/persist/kv.sx")
|
|
(load "lib/persist/api.sx")
|
|
(load "lib/content/block.sx")
|
|
(load "lib/content/doc.sx")
|
|
(load "lib/content/render.sx")
|
|
(load "lib/content/api.sx")
|
|
(load "lib/content/meta.sx")
|
|
(load "lib/content/text.sx")
|
|
(load "lib/content/section.sx")
|
|
(load "lib/content/compose.sx")
|
|
(load "lib/content/tree-edit.sx")
|
|
(load "lib/content/move.sx")
|
|
(load "lib/content/clone.sx")
|
|
(load "lib/content/query.sx")
|
|
(load "lib/content/toc.sx")
|
|
(load "lib/content/anchor.sx")
|
|
(load "lib/content/outline.sx")
|
|
(load "lib/content/flatten.sx")
|
|
(load "lib/content/transform.sx")
|
|
(load "lib/content/normalize.sx")
|
|
(load "lib/content/find-replace.sx")
|
|
(load "lib/content/stats.sx")
|
|
(load "lib/content/table.sx")
|
|
(load "lib/content/callout.sx")
|
|
(load "lib/content/media.sx")
|
|
(load "lib/content/data.sx")
|
|
(load "lib/content/wire.sx")
|
|
(load "lib/content/page.sx")
|
|
(load "lib/content/page-full.sx")
|
|
(load "lib/content/markdown.sx")
|
|
(load "lib/content/validate.sx")
|
|
(load "lib/content/store.sx")
|
|
(load "lib/content/snapshot.sx")
|
|
(load "lib/content/crdt.sx")
|
|
(load "lib/content/crdt-store.sx")
|
|
(load "lib/content/sync.sx")
|
|
(load "lib/content/md-import.sx")
|
|
(load "lib/content/md-doc.sx")
|
|
(load "lib/content/fed.sx")
|
|
(epoch 2)
|
|
(eval "(define content-test-pass 0)")
|
|
(eval "(define content-test-fail 0)")
|
|
(eval "(define content-test-fails (list))")
|
|
(eval "(define content-test (fn (name got expected) (if (= got expected) (set! content-test-pass (+ content-test-pass 1)) (begin (set! content-test-fail (+ content-test-fail 1)) (set! content-test-fails (cons name content-test-fails))))))")
|
|
(epoch 3)
|
|
(load "${file}")
|
|
(epoch 4)
|
|
(eval "(list content-test-pass content-test-fail)")
|
|
EPOCHS
|
|
|
|
local OUTPUT
|
|
OUTPUT=$(timeout 240 "$SX_SERVER" < "$TMP" 2>/dev/null)
|
|
rm -f "$TMP"
|
|
|
|
local LINE
|
|
LINE=$(echo "$OUTPUT" | awk '/^\(ok-len 4 / {getline; print; exit}')
|
|
if [ -z "$LINE" ]; then
|
|
LINE=$(echo "$OUTPUT" | grep -E '^\(ok 4 \([0-9]+ [0-9]+\)\)' | tail -1 \
|
|
| sed -E 's/^\(ok 4 //; s/\)$//')
|
|
fi
|
|
|
|
local P F
|
|
P=$(echo "$LINE" | sed -E 's/^\(([0-9]+) ([0-9]+)\).*/\1/')
|
|
F=$(echo "$LINE" | sed -E 's/^\(([0-9]+) ([0-9]+)\).*/\2/')
|
|
P=${P:-0}
|
|
F=${F:-0}
|
|
echo "${P} ${F}"
|
|
}
|
|
|
|
declare -A SUITE_PASS
|
|
declare -A SUITE_FAIL
|
|
TOTAL_PASS=0
|
|
TOTAL_FAIL=0
|
|
|
|
echo "Running content conformance suite..." >&2
|
|
for s in "${SUITES[@]}"; do
|
|
read -r p f < <(run_suite "$s")
|
|
SUITE_PASS[$s]=$p
|
|
SUITE_FAIL[$s]=$f
|
|
TOTAL_PASS=$((TOTAL_PASS + p))
|
|
TOTAL_FAIL=$((TOTAL_FAIL + f))
|
|
printf " %-12s %d/%d\n" "$s" "$p" "$((p+f))" >&2
|
|
done
|
|
|
|
{
|
|
printf '{\n'
|
|
printf ' "suites": {\n'
|
|
first=1
|
|
for s in "${SUITES[@]}"; do
|
|
if [ $first -eq 0 ]; then printf ',\n'; fi
|
|
printf ' "%s": {"pass": %d, "fail": %d}' "$s" "${SUITE_PASS[$s]}" "${SUITE_FAIL[$s]}"
|
|
first=0
|
|
done
|
|
printf '\n },\n'
|
|
printf ' "total_pass": %d,\n' "$TOTAL_PASS"
|
|
printf ' "total_fail": %d,\n' "$TOTAL_FAIL"
|
|
printf ' "total": %d\n' "$((TOTAL_PASS + TOTAL_FAIL))"
|
|
printf '}\n'
|
|
} > "$OUT_JSON"
|
|
|
|
{
|
|
printf '# content-on-sx Conformance Scoreboard\n\n'
|
|
printf '_Generated by `lib/content/conformance.sh`_\n\n'
|
|
printf '| Suite | Pass | Fail | Total |\n'
|
|
printf '|-------|-----:|-----:|------:|\n'
|
|
for s in "${SUITES[@]}"; do
|
|
p=${SUITE_PASS[$s]}
|
|
f=${SUITE_FAIL[$s]}
|
|
printf '| %s | %d | %d | %d |\n' "$s" "$p" "$f" "$((p+f))"
|
|
done
|
|
printf '| **Total** | **%d** | **%d** | **%d** |\n' "$TOTAL_PASS" "$TOTAL_FAIL" "$((TOTAL_PASS + TOTAL_FAIL))"
|
|
} > "$OUT_MD"
|
|
|
|
echo "Wrote $OUT_JSON and $OUT_MD" >&2
|
|
echo "Total: $TOTAL_PASS pass, $TOTAL_FAIL fail" >&2
|
|
|
|
[ "$TOTAL_FAIL" -eq 0 ]
|