From 8292607e388f729017affa3a82cc74391801be8d Mon Sep 17 00:00:00 2001 From: giles Date: Sat, 6 Jun 2026 18:56:19 +0000 Subject: [PATCH] =?UTF-8?q?mod:=20Ext=2011=20=E2=80=94=20batch=20triage=20?= =?UTF-8?q?+=20corpus=20analytics,=20277/277?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit mod/decide-batch triages a queue; mod/action-histogram summarizes outcomes by action; mod/rule-coverage + mod/never-fired measure which rules fire across a corpus — the empirical complement to lint's static unreachable check (lint finds rules that can't fire; never-fired finds rules that didn't). +17 tests. Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/mod/batch.sx | 55 +++++++++++++++++++++ lib/mod/conformance.conf | 2 + lib/mod/scoreboard.json | 9 ++-- lib/mod/scoreboard.md | 3 +- lib/mod/tests/batch.sx | 101 +++++++++++++++++++++++++++++++++++++++ plans/mod-on-sx.md | 12 ++++- 6 files changed, 176 insertions(+), 6 deletions(-) create mode 100644 lib/mod/batch.sx create mode 100644 lib/mod/tests/batch.sx diff --git a/lib/mod/batch.sx b/lib/mod/batch.sx new file mode 100644 index 00000000..2ef4e618 --- /dev/null +++ b/lib/mod/batch.sx @@ -0,0 +1,55 @@ +;; lib/mod/batch.sx — batch triage + corpus analytics. +;; +;; Operational layer: decide a whole queue of reports at once, summarize the +;; outcomes by action, and measure which rules actually fire across a corpus. +;; mod/never-fired is the empirical complement to lint's static unreachable check +;; (Ext 5): lint finds rules that CAN'T fire by structure; never-fired finds rules +;; that DIDN'T fire on real data. + +(define + mod/decide-batch + (fn + (reports rules) + (map (fn (r) (mod/decide-report r reports rules)) reports))) + +(define + mod/count-action + (fn + (decisions action) + (reduce + (fn (acc d) (if (= (get d :action) action) (+ acc 1) acc)) + 0 + decisions))) + +(define mod/action-histogram (fn (decisions) {:keep (mod/count-action decisions "keep") :remove (mod/count-action decisions "remove") :escalate (mod/count-action decisions "escalate") :hide (mod/count-action decisions "hide") :ban (mod/count-action decisions "ban")})) + +(define + mod/rule-fire-count + (fn + (decisions rule-name) + (reduce + (fn (acc d) (if (= (get d :rule) rule-name) (+ acc 1) acc)) + 0 + decisions))) + +(define + mod/rule-coverage + (fn + (reports rules) + (let + ((decisions (mod/decide-batch reports rules))) + (map (fn (rule) {:rule (mod/rule-name rule) :fired (mod/rule-fire-count decisions (mod/rule-name rule))}) rules)))) + +(define + mod/never-fired + (fn + (reports rules) + (reduce + (fn + (acc c) + (if + (= (get c :fired) 0) + (append acc (list (get c :rule))) + acc)) + (list) + (mod/rule-coverage reports rules)))) diff --git a/lib/mod/conformance.conf b/lib/mod/conformance.conf index 517be3c0..154e11b2 100644 --- a/lib/mod/conformance.conf +++ b/lib/mod/conformance.conf @@ -19,6 +19,7 @@ PRELOADS=( lib/mod/quorum.sx lib/mod/trace.sx lib/mod/whatif.sx + lib/mod/batch.sx lib/mod/lifecycle.sx lib/mod/audit.sx lib/mod/api.sx @@ -40,4 +41,5 @@ SUITES=( "quorum:lib/mod/tests/quorum.sx:(mod-quorum-tests-run!)" "trace:lib/mod/tests/trace.sx:(mod-trace-tests-run!)" "whatif:lib/mod/tests/whatif.sx:(mod-whatif-tests-run!)" + "batch:lib/mod/tests/batch.sx:(mod-batch-tests-run!)" ) diff --git a/lib/mod/scoreboard.json b/lib/mod/scoreboard.json index 3c9acdc6..ebcd4326 100644 --- a/lib/mod/scoreboard.json +++ b/lib/mod/scoreboard.json @@ -1,8 +1,8 @@ { "lang": "mod", - "total_passed": 260, + "total_passed": 277, "total_failed": 0, - "total": 260, + "total": 277, "suites": [ {"name":"decide","passed":31,"failed":0,"total":31}, {"name":"audit","passed":29,"failed":0,"total":29}, @@ -15,7 +15,8 @@ {"name":"offenders","passed":19,"failed":0,"total":19}, {"name":"quorum","passed":9,"failed":0,"total":9}, {"name":"trace","passed":15,"failed":0,"total":15}, - {"name":"whatif","passed":13,"failed":0,"total":13} + {"name":"whatif","passed":13,"failed":0,"total":13}, + {"name":"batch","passed":17,"failed":0,"total":17} ], - "generated": "2026-06-06T18:51:15+00:00" + "generated": "2026-06-06T18:55:47+00:00" } diff --git a/lib/mod/scoreboard.md b/lib/mod/scoreboard.md index 365c2351..eb159e6f 100644 --- a/lib/mod/scoreboard.md +++ b/lib/mod/scoreboard.md @@ -1,6 +1,6 @@ # mod scoreboard -**260 / 260 passing** (0 failure(s)). +**277 / 277 passing** (0 failure(s)). | Suite | Passed | Total | Status | |-------|--------|-------|--------| @@ -16,3 +16,4 @@ | quorum | 9 | 9 | ok | | trace | 15 | 15 | ok | | whatif | 13 | 13 | ok | +| batch | 17 | 17 | ok | diff --git a/lib/mod/tests/batch.sx b/lib/mod/tests/batch.sx new file mode 100644 index 00000000..0835f291 --- /dev/null +++ b/lib/mod/tests/batch.sx @@ -0,0 +1,101 @@ +;; lib/mod/tests/batch.sx — Ext 11: batch triage + corpus analytics. + +(define mod-b-count 0) +(define mod-b-pass 0) +(define mod-b-fail 0) +(define mod-b-failures (list)) + +(define + mod-b-test! + (fn + (name got expected) + (begin + (set! mod-b-count (+ mod-b-count 1)) + (if + (= got expected) + (set! mod-b-pass (+ mod-b-pass 1)) + (begin + (set! mod-b-fail (+ mod-b-fail 1)) + (append! + mod-b-failures + (str name "\n expected: " expected "\n got: " got))))))) + +;; corpus: 2 spam, 1 abuse, 2 clean — distinct subjects so the count rule stays quiet +(define + mod-b-corpus + (list + (mod/mk-report "r1" "u" "s1" "this is spam") + (mod/mk-report "r2" "u" "s2" "buy now offer") + (mod/mk-report "r3" "u" "s3" "harassment here") + (mod/mk-report "r4" "u" "s4" "a fine post") + (mod/mk-report "r5" "u" "s5" "thanks for sharing"))) + +(define mod-b-decisions (mod/decide-batch mod-b-corpus mod/default-rules)) + +;; ── decide-batch ── + +(mod-b-test! "one decision per report" (len mod-b-decisions) 5) +(mod-b-test! + "first decision is hide" + (get (first mod-b-decisions) :action) + "hide") + +;; ── action histogram ── + +(define mod-b-hist (mod/action-histogram mod-b-decisions)) +(mod-b-test! "histogram hide count" (get mod-b-hist :hide) 2) +(mod-b-test! "histogram remove count" (get mod-b-hist :remove) 1) +(mod-b-test! "histogram keep count" (get mod-b-hist :keep) 2) +(mod-b-test! "histogram escalate count" (get mod-b-hist :escalate) 0) +(mod-b-test! "histogram ban count" (get mod-b-hist :ban) 0) +(mod-b-test! + "histogram totals match corpus" + (+ + (+ (get mod-b-hist :hide) (get mod-b-hist :remove)) + (+ + (get mod-b-hist :keep) + (+ (get mod-b-hist :escalate) (get mod-b-hist :ban)))) + 5) + +;; ── rule coverage (empirical) ── + +(define mod-b-cov (mod/rule-coverage mod-b-corpus mod/default-rules)) +(mod-b-test! "coverage has one row per rule" (len mod-b-cov) 6) +(mod-b-test! + "spam-hide fired twice" + (mod/rule-fire-count mod-b-decisions "spam-hide") + 2) +(mod-b-test! + "abuse-remove fired once" + (mod/rule-fire-count mod-b-decisions "abuse-remove") + 1) +(mod-b-test! + "default-keep fired twice" + (mod/rule-fire-count mod-b-decisions "default-keep") + 2) + +;; ── never-fired: rules not exercised by this corpus ── + +(define mod-b-never (mod/never-fired mod-b-corpus mod/default-rules)) +(mod-b-test! + "exonerated-keep never fired" + (mod/member? "exonerated-keep" mod-b-never) + true) +(mod-b-test! + "reviewer-remove never fired" + (mod/member? "reviewer-remove" mod-b-never) + true) +(mod-b-test! + "repeated-escalate never fired" + (mod/member? "repeated-escalate" mod-b-never) + true) +(mod-b-test! + "spam-hide DID fire (not in never-fired)" + (mod/member? "spam-hide" mod-b-never) + false) +(mod-b-test! + "three rules never fired on this corpus" + (len mod-b-never) + 3) + +(define mod-batch-tests-run! (fn () {:failures mod-b-failures :total mod-b-count :passed mod-b-pass :failed mod-b-fail})) diff --git a/plans/mod-on-sx.md b/plans/mod-on-sx.md index 5e074024..684e4eda 100644 --- a/plans/mod-on-sx.md +++ b/plans/mod-on-sx.md @@ -16,7 +16,7 @@ federation extension. ## Status (rolling) -`bash lib/mod/conformance.sh` → **260/260** (roadmap + 10 extensions complete) +`bash lib/mod/conformance.sh` → **277/277** (roadmap + 11 extensions complete) ## Ground rules @@ -147,6 +147,11 @@ lib/mod/fed.sx derivation goal-by-goal with `[proved]`/`[unproved]` marks and unification bindings. E.g. `Report rc: escalate (rule: repeated-escalate)` … `[proved] report(rc, B, S), report_count(S, N), N >= 3 {B=ann, N=3, S=dave}`. +- [x] **Ext 11 — batch triage + corpus analytics** (`lib/mod/batch.sx`, +17). + `mod/decide-batch` triages a queue; `mod/action-histogram` summarizes outcomes by + action; `mod/rule-coverage` / `mod/never-fired` measure which rules fire across a + corpus — the *empirical* complement to lint's static unreachable check (Ext 5): + lint finds rules that can't fire, never-fired finds rules that didn't. - [x] **Ext 10 — policy what-if / impact** (`lib/mod/whatif.sx`, +13). `mod/decision-diff` compares one report's action under two rule sets; `mod/policy-impact` runs a batch and returns only the reports whose decision @@ -191,6 +196,11 @@ lib/mod/fed.sx ## Progress log +- **Ext 11 — batch triage + corpus analytics, 277/277** (+17). Operational layer: + triage a queue, histogram the outcomes, and measure rule coverage over real + data. `never-fired` pairs with lint (Ext 5) — static "can't fire" vs empirical + "didn't fire" — giving policy authors both views of dead rules. Histogram avoids + dict mutation by counting over a fixed action vocabulary. - **Ext 10 — policy what-if / impact, 260/260** (+13). Decisions are now comparable across rule sets — diff one report, or batch a whole set and surface only the flips. Pure SX over `decide-report`, no engine change. Closes the