rose-ash/lib/commerce/conformance.sh

#!/usr/bin/env bash
# lib/commerce/conformance.sh — run commerce test suites in one sx_server
# process per suite, emit scoreboard.json + scoreboard.md.
#
# commerce-on-sx builds pricing/promotion as miniKanren relations, so every
# suite loads the miniKanren stack first, then the commerce modules.

set -uo pipefail
cd "$(git rev-parse --show-toplevel)"

SX_SERVER="${SX_SERVER:-/root/rose-ash/hosts/ocaml/_build/default/bin/sx_server.exe}"
if [ ! -x "$SX_SERVER" ]; then
  SX_SERVER="hosts/ocaml/_build/default/bin/sx_server.exe"
fi
if [ ! -x "$SX_SERVER" ]; then
  echo "ERROR: sx_server.exe not found." >&2
  exit 1
fi

SUITES=(catalog cart price api promo stack quote ledger order)

OUT_JSON="lib/commerce/scoreboard.json"
OUT_MD="lib/commerce/scoreboard.md"

run_suite() {
  local suite=$1
  local file="lib/commerce/tests/${suite}.sx"
  local TMP
  TMP=$(mktemp)
  cat > "$TMP" << EPOCHS
(epoch 1)
(load "spec/stdlib.sx")
(load "lib/r7rs.sx")
(load "lib/guest/match.sx")
(load "lib/minikanren/unify.sx")
(load "lib/minikanren/stream.sx")
(load "lib/minikanren/goals.sx")
(load "lib/minikanren/fresh.sx")
(load "lib/minikanren/conde.sx")
(load "lib/minikanren/run.sx")
(load "lib/minikanren/relations.sx")
(load "lib/minikanren/project.sx")
(load "lib/minikanren/intarith.sx")
(load "lib/minikanren/matche.sx")
(load "lib/minikanren/defrel.sx")
(load "lib/persist/event.sx")
(load "lib/persist/backend.sx")
(load "lib/persist/log.sx")
(load "lib/persist/kv.sx")
(load "lib/persist/idempotency.sx")
(load "lib/guest/lex.sx")
(load "lib/guest/reflective/env.sx")
(load "lib/guest/reflective/quoting.sx")
(load "lib/scheme/parser.sx")
(load "lib/scheme/eval.sx")
(load "lib/scheme/runtime.sx")
(load "lib/flow/spec.sx")
(load "lib/flow/store.sx")
(load "lib/flow/remote.sx")
(load "lib/flow/host.sx")
(load "lib/flow/api.sx")
(load "lib/commerce/catalog.sx")
(load "lib/commerce/cart.sx")
(load "lib/commerce/price.sx")
(load "lib/commerce/api.sx")
(load "lib/commerce/promo.sx")
(load "lib/commerce/stack.sx")
(load "lib/commerce/quote.sx")
(load "lib/commerce/ledger.sx")
(load "lib/commerce/order.sx")
(epoch 2)
(eval "(define ct-pass 0)")
(eval "(define ct-fail 0)")
(eval "(define ct-fails (list))")
(eval "(define commerce-test (fn (name got expected) (if (= got expected) (set! ct-pass (+ ct-pass 1)) (begin (set! ct-fail (+ ct-fail 1)) (append! ct-fails name)))))")
(epoch 3)
(load "${file}")
(epoch 4)
(eval "(list ct-pass ct-fail)")
(eval "ct-fails")
EPOCHS

  local OUTPUT
  OUTPUT=$(timeout 560 "$SX_SERVER" < "$TMP" 2>/dev/null)
  rm -f "$TMP"

  # The (list ct-pass ct-fail) result follows its (ok-len 2 N) ack line.
  local LINE
  LINE=$(echo "$OUTPUT" | grep -oE '^\([0-9]+ [0-9]+\)$' | tail -1)
  local P F
  P=$(echo "$LINE" | sed -E 's/^\(([0-9]+) ([0-9]+)\)$/\1/')
  F=$(echo "$LINE" | sed -E 's/^\(([0-9]+) ([0-9]+)\)$/\2/')
  P=${P:-0}
  F=${F:-0}
  echo "${P} ${F}"
}

declare -A SUITE_PASS
declare -A SUITE_FAIL
TOTAL_PASS=0
TOTAL_FAIL=0

echo "Running commerce conformance suite..." >&2
for s in "${SUITES[@]}"; do
  read -r p f < <(run_suite "$s")
  SUITE_PASS[$s]=$p
  SUITE_FAIL[$s]=$f
  TOTAL_PASS=$((TOTAL_PASS + p))
  TOTAL_FAIL=$((TOTAL_FAIL + f))
  printf "  %-12s %d/%d\n" "$s" "$p" "$((p+f))" >&2
done

{
  printf '{\n'
  printf '  "suites": {\n'
  first=1
  for s in "${SUITES[@]}"; do
    if [ $first -eq 0 ]; then printf ',\n'; fi
    printf '    "%s": {"pass": %d, "fail": %d}' "$s" "${SUITE_PASS[$s]}" "${SUITE_FAIL[$s]}"
    first=0
  done
  printf '\n  },\n'
  printf '  "total_pass": %d,\n' "$TOTAL_PASS"
  printf '  "total_fail": %d,\n' "$TOTAL_FAIL"
  printf '  "total": %d\n' "$((TOTAL_PASS + TOTAL_FAIL))"
  printf '}\n'
} > "$OUT_JSON"

{
  printf '# commerce Conformance Scoreboard\n\n'
  printf '_Generated by `lib/commerce/conformance.sh`_\n\n'
  printf '| Suite | Pass | Fail | Total |\n'
  printf '|-------|-----:|-----:|------:|\n'
  for s in "${SUITES[@]}"; do
    p=${SUITE_PASS[$s]}
    f=${SUITE_FAIL[$s]}
    printf '| %s | %d | %d | %d |\n' "$s" "$p" "$f" "$((p+f))"
  done
  printf '| **Total** | **%d** | **%d** | **%d** |\n' "$TOTAL_PASS" "$TOTAL_FAIL" "$((TOTAL_PASS + TOTAL_FAIL))"
} > "$OUT_MD"

echo "Wrote $OUT_JSON and $OUT_MD" >&2
echo "Total: $TOTAL_PASS pass, $TOTAL_FAIL fail" >&2

[ "$TOTAL_FAIL" -eq 0 ]