rose-ash/lib/gitea/conformance.sh

#!/usr/bin/env bash
# lib/gitea/conformance.sh — run sx-gitea test suites, emit scoreboard.json + scoreboard.md.
#
# Usage:
#   bash lib/gitea/conformance.sh        # run all suites
#   bash lib/gitea/conformance.sh -v     # also print failure details

set -uo pipefail
cd "$(git rev-parse --show-toplevel)"

SX_SERVER="${SX_SERVER:-hosts/ocaml/_build/default/bin/sx_server.exe}"
if [ ! -x "$SX_SERVER" ]; then
  SX_SERVER="/root/rose-ash/hosts/ocaml/_build/default/bin/sx_server.exe"
fi
if [ ! -x "$SX_SERVER" ]; then
  echo "ERROR: sx_server.exe not found." >&2
  exit 1
fi

VERBOSE="${1:-}"

# suite name | pass counter | fail counter | failures list
SUITES=(
  "repo|gitea-repo-pass|gitea-repo-fail|gitea-repo-fails"
  "access|gitea-access-pass|gitea-access-fail|gitea-access-fails"
  "wire|gitea-wire-pass|gitea-wire-fail|gitea-wire-fails"
)

OUT_JSON="lib/gitea/scoreboard.json"
OUT_MD="lib/gitea/scoreboard.md"

# Library load order: kernel stdlib, persist, artdag canon, sx-git, dream
# (types/router/middleware/error/html/json/api), then the gitea modules.
MODULES=(
  "spec/stdlib.sx"
  "spec/parser.sx"
  "lib/r7rs.sx"
  "lib/persist/event.sx"
  "lib/persist/backend.sx"
  "lib/persist/log.sx"
  "lib/persist/kv.sx"
  "lib/artdag/dag.sx"
  "lib/git/object.sx"
  "lib/git/ref.sx"
  "lib/git/dag.sx"
  "lib/git/worktree.sx"
  "lib/git/diff.sx"
  "lib/git/merge.sx"
  "lib/git/porcelain.sx"
  "lib/datalog/tokenizer.sx"
  "lib/datalog/parser.sx"
  "lib/datalog/unify.sx"
  "lib/datalog/db.sx"
  "lib/datalog/builtins.sx"
  "lib/datalog/aggregates.sx"
  "lib/datalog/strata.sx"
  "lib/datalog/eval.sx"
  "lib/datalog/api.sx"
  "lib/datalog/magic.sx"
  "lib/acl/schema.sx"
  "lib/acl/facts.sx"
  "lib/acl/engine.sx"
  "lib/dream/types.sx"
  "lib/dream/router.sx"
  "lib/dream/middleware.sx"
  "lib/dream/error.sx"
  "lib/dream/html.sx"
  "lib/dream/json.sx"
  "lib/dream/auth.sx"
  "lib/dream/api.sx"
  "lib/gitea/repo.sx"
  "lib/gitea/access.sx"
  "lib/gitea/web.sx"
  "lib/gitea/wire.sx"
)

run_suite() {
  local suite=$1 passvar=$2 failvar=$3 failsvar=$4
  local file="lib/gitea/tests/${suite}.sx"
  local TMP
  TMP=$(mktemp)
  {
    echo "(epoch 1)"
    for M in "${MODULES[@]}"; do echo "(load \"$M\")"; done
    echo "(epoch 2)"
    echo "(load \"${file}\")"
    echo "(epoch 3)"
    echo "(eval \"(list ${passvar} ${failvar})\")"
    echo "(epoch 4)"
    echo "(eval \"(inspect ${failsvar})\")"
  } > "$TMP"

  local OUTPUT
  OUTPUT=$(timeout 600 "$SX_SERVER" < "$TMP" 2>/dev/null)
  rm -f "$TMP"

  local LINE
  LINE=$(echo "$OUTPUT" | awk '/^\(ok-len 3 / {getline; print; exit}')
  if [ -z "$LINE" ]; then
    LINE=$(echo "$OUTPUT" | grep -E '^\(ok 3 \([0-9]+ [0-9]+\)\)' | tail -1 \
            | sed -E 's/^\(ok 3 //; s/\)$//')
  fi

  local P F
  P=$(echo "$LINE" | sed -E 's/^\(([0-9]+) ([0-9]+)\).*/\1/')
  F=$(echo "$LINE" | sed -E 's/^\(([0-9]+) ([0-9]+)\).*/\2/')
  P=${P:-0}
  F=${F:-0}

  if [ -n "$VERBOSE" ] && [ "$F" != "0" ]; then
    echo "  --- ${suite} failures ---" >&2
    echo "$OUTPUT" | awk '/^\(ok(-len)? 4 /,0' | head -40 >&2
  fi

  echo "${P} ${F}"
}

declare -A SUITE_PASS
declare -A SUITE_FAIL
TOTAL_PASS=0
TOTAL_FAIL=0

echo "Running sx-gitea conformance suite..." >&2
for entry in "${SUITES[@]}"; do
  IFS='|' read -r s passvar failvar failsvar <<< "$entry"
  read -r p f < <(run_suite "$s" "$passvar" "$failvar" "$failsvar")
  SUITE_PASS[$s]=$p
  SUITE_FAIL[$s]=$f
  TOTAL_PASS=$((TOTAL_PASS + p))
  TOTAL_FAIL=$((TOTAL_FAIL + f))
  printf "  %-12s %d/%d\n" "$s" "$p" "$((p+f))" >&2
done

{
  printf '{\n'
  printf '  "suites": {\n'
  first=1
  for entry in "${SUITES[@]}"; do
    IFS='|' read -r s _ _ _ <<< "$entry"
    if [ $first -eq 0 ]; then printf ',\n'; fi
    printf '    "%s": {"pass": %d, "fail": %d}' "$s" "${SUITE_PASS[$s]}" "${SUITE_FAIL[$s]}"
    first=0
  done
  printf '\n  },\n'
  printf '  "total_pass": %d,\n' "$TOTAL_PASS"
  printf '  "total_fail": %d,\n' "$TOTAL_FAIL"
  printf '  "total": %d\n' "$((TOTAL_PASS + TOTAL_FAIL))"
  printf '}\n'
} > "$OUT_JSON"

{
  printf '# sx-gitea Conformance Scoreboard\n\n'
  printf '_Generated by `lib/gitea/conformance.sh`_\n\n'
  printf '| Suite | Pass | Fail | Total |\n'
  printf '|-------|-----:|-----:|------:|\n'
  for entry in "${SUITES[@]}"; do
    IFS='|' read -r s _ _ _ <<< "$entry"
    p=${SUITE_PASS[$s]}
    f=${SUITE_FAIL[$s]}
    printf '| %s | %d | %d | %d |\n' "$s" "$p" "$f" "$((p+f))"
  done
  printf '| **Total** | **%d** | **%d** | **%d** |\n' "$TOTAL_PASS" "$TOTAL_FAIL" "$((TOTAL_PASS + TOTAL_FAIL))"
} > "$OUT_MD"

echo "Wrote $OUT_JSON and $OUT_MD" >&2
echo "Total: $TOTAL_PASS pass, $TOTAL_FAIL fail" >&2

[ "$TOTAL_FAIL" -eq 0 ]