Compare commits
25 Commits
loops/feed
...
loops/rada
| Author | SHA1 | Date | |
|---|---|---|---|
| 8cb985a2f3 | |||
| 80a925018c | |||
| adad4f4436 | |||
| a752334cc0 | |||
| 2b77dc9537 | |||
| 453f244a97 | |||
| 05f3ef9104 | |||
| 4b9b15e7c8 | |||
| dbc2daf64d | |||
| b6c2995b19 | |||
| d05b49873b | |||
| 8f9b8d6f5d | |||
| 4ee15a7ddd | |||
| 3480100caa | |||
| 0bd0003550 | |||
| d9f18a635e | |||
| 3aac6aae98 | |||
| 0d06966808 | |||
| 98ef13ad2a | |||
| 20c4a48d3b | |||
| b3e1af96af | |||
| 919bd961d1 | |||
| 1902cce57f | |||
| ff537bfba2 | |||
| 1e4cf25015 |
63
lib/apl/conformance.conf
Normal file
63
lib/apl/conformance.conf
Normal file
@@ -0,0 +1,63 @@
|
|||||||
|
# APL conformance config — sourced by lib/guest/conformance.sh.
|
||||||
|
|
||||||
|
LANG_NAME=apl
|
||||||
|
MODE=counters
|
||||||
|
COUNTERS_PASS=apl-test-pass
|
||||||
|
COUNTERS_FAIL=apl-test-fail
|
||||||
|
TIMEOUT_PER_SUITE=300
|
||||||
|
|
||||||
|
PRELOADS=(
|
||||||
|
spec/stdlib.sx
|
||||||
|
lib/r7rs.sx
|
||||||
|
lib/apl/runtime.sx
|
||||||
|
lib/apl/tokenizer.sx
|
||||||
|
lib/apl/parser.sx
|
||||||
|
lib/apl/transpile.sx
|
||||||
|
lib/apl/test-harness.sx
|
||||||
|
)
|
||||||
|
|
||||||
|
SUITES=(
|
||||||
|
"structural:lib/apl/tests/structural.sx"
|
||||||
|
"operators:lib/apl/tests/operators.sx"
|
||||||
|
"dfn:lib/apl/tests/dfn.sx"
|
||||||
|
"tradfn:lib/apl/tests/tradfn.sx"
|
||||||
|
"valence:lib/apl/tests/valence.sx"
|
||||||
|
"programs:lib/apl/tests/programs.sx"
|
||||||
|
"system:lib/apl/tests/system.sx"
|
||||||
|
"idioms:lib/apl/tests/idioms.sx"
|
||||||
|
"eval-ops:lib/apl/tests/eval-ops.sx"
|
||||||
|
"pipeline:lib/apl/tests/pipeline.sx"
|
||||||
|
)
|
||||||
|
|
||||||
|
emit_scoreboard_json() {
|
||||||
|
local n=${#GC_NAMES[@]} i sep
|
||||||
|
printf '{\n'
|
||||||
|
printf ' "suites": {\n'
|
||||||
|
for ((i=0; i<n; i++)); do
|
||||||
|
sep=","; [ $i -eq $((n-1)) ] && sep=""
|
||||||
|
printf ' "%s": {"pass": %d, "fail": %d}%s\n' \
|
||||||
|
"${GC_NAMES[$i]}" "${GC_PASS[$i]}" "${GC_FAIL[$i]}" "$sep"
|
||||||
|
done
|
||||||
|
printf ' },\n'
|
||||||
|
printf ' "total_pass": %d,\n' "$GC_TOTAL_PASS"
|
||||||
|
printf ' "total_fail": %d,\n' "$GC_TOTAL_FAIL"
|
||||||
|
printf ' "total": %d\n' "$GC_TOTAL"
|
||||||
|
printf '}\n'
|
||||||
|
}
|
||||||
|
|
||||||
|
emit_scoreboard_md() {
|
||||||
|
local n=${#GC_NAMES[@]} i
|
||||||
|
printf '# APL Conformance Scoreboard\n\n'
|
||||||
|
printf '_Generated by `lib/apl/conformance.sh`_\n\n'
|
||||||
|
printf '| Suite | Pass | Fail | Total |\n'
|
||||||
|
printf '|-------|-----:|-----:|------:|\n'
|
||||||
|
for ((i=0; i<n; i++)); do
|
||||||
|
printf '| %s | %d | %d | %d |\n' \
|
||||||
|
"${GC_NAMES[$i]}" "${GC_PASS[$i]}" "${GC_FAIL[$i]}" "${GC_TOTAL_S[$i]}"
|
||||||
|
done
|
||||||
|
printf '| **Total** | **%d** | **%d** | **%d** |\n' "$GC_TOTAL_PASS" "$GC_TOTAL_FAIL" "$GC_TOTAL"
|
||||||
|
printf '\n'
|
||||||
|
printf '## Notes\n\n'
|
||||||
|
printf '%s\n' '- Suites use the standard `apl-test name got expected` framework loaded against `lib/apl/runtime.sx` + `lib/apl/transpile.sx`.'
|
||||||
|
printf '%s\n' '- `lib/apl/tests/parse.sx` and `lib/apl/tests/scalar.sx` use their own self-contained frameworks and are excluded from this scoreboard.'
|
||||||
|
}
|
||||||
@@ -1,116 +1,5 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
# lib/apl/conformance.sh — run APL test suites, emit scoreboard.json + scoreboard.md.
|
# lib/apl/conformance.sh — APL conformance via the shared guest driver.
|
||||||
|
# Config lives in lib/apl/conformance.conf (MODE=counters). Override the binary
|
||||||
set -uo pipefail
|
# with SX_SERVER=path/to/sx_server.exe bash lib/apl/conformance.sh
|
||||||
cd "$(git rev-parse --show-toplevel)"
|
exec bash "$(dirname "$0")/../guest/conformance.sh" "$(dirname "$0")/conformance.conf" "$@"
|
||||||
|
|
||||||
SX_SERVER="${SX_SERVER:-/root/rose-ash/hosts/ocaml/_build/default/bin/sx_server.exe}"
|
|
||||||
if [ ! -x "$SX_SERVER" ]; then
|
|
||||||
SX_SERVER="hosts/ocaml/_build/default/bin/sx_server.exe"
|
|
||||||
fi
|
|
||||||
if [ ! -x "$SX_SERVER" ]; then
|
|
||||||
echo "ERROR: sx_server.exe not found." >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
SUITES=(structural operators dfn tradfn valence programs system idioms eval-ops pipeline)
|
|
||||||
|
|
||||||
OUT_JSON="lib/apl/scoreboard.json"
|
|
||||||
OUT_MD="lib/apl/scoreboard.md"
|
|
||||||
|
|
||||||
run_suite() {
|
|
||||||
local suite=$1
|
|
||||||
local file="lib/apl/tests/${suite}.sx"
|
|
||||||
local TMP
|
|
||||||
TMP=$(mktemp)
|
|
||||||
cat > "$TMP" << EPOCHS
|
|
||||||
(epoch 1)
|
|
||||||
(load "spec/stdlib.sx")
|
|
||||||
(load "lib/r7rs.sx")
|
|
||||||
(load "lib/apl/runtime.sx")
|
|
||||||
(load "lib/apl/tokenizer.sx")
|
|
||||||
(load "lib/apl/parser.sx")
|
|
||||||
(load "lib/apl/transpile.sx")
|
|
||||||
(epoch 2)
|
|
||||||
(eval "(define apl-test-pass 0)")
|
|
||||||
(eval "(define apl-test-fail 0)")
|
|
||||||
(eval "(define apl-test (fn (name got expected) (if (= got expected) (set! apl-test-pass (+ apl-test-pass 1)) (set! apl-test-fail (+ apl-test-fail 1)))))")
|
|
||||||
(epoch 3)
|
|
||||||
(load "${file}")
|
|
||||||
(epoch 4)
|
|
||||||
(eval "(list apl-test-pass apl-test-fail)")
|
|
||||||
EPOCHS
|
|
||||||
|
|
||||||
local OUTPUT
|
|
||||||
OUTPUT=$(timeout 300 "$SX_SERVER" < "$TMP" 2>/dev/null)
|
|
||||||
rm -f "$TMP"
|
|
||||||
|
|
||||||
local LINE
|
|
||||||
LINE=$(echo "$OUTPUT" | awk '/^\(ok-len 4 / {getline; print; exit}')
|
|
||||||
if [ -z "$LINE" ]; then
|
|
||||||
LINE=$(echo "$OUTPUT" | grep -E '^\(ok 4 \([0-9]+ [0-9]+\)\)' | tail -1 \
|
|
||||||
| sed -E 's/^\(ok 4 //; s/\)$//')
|
|
||||||
fi
|
|
||||||
|
|
||||||
local P F
|
|
||||||
P=$(echo "$LINE" | sed -E 's/^\(([0-9]+) ([0-9]+)\).*/\1/')
|
|
||||||
F=$(echo "$LINE" | sed -E 's/^\(([0-9]+) ([0-9]+)\).*/\2/')
|
|
||||||
P=${P:-0}
|
|
||||||
F=${F:-0}
|
|
||||||
echo "${P} ${F}"
|
|
||||||
}
|
|
||||||
|
|
||||||
declare -A SUITE_PASS
|
|
||||||
declare -A SUITE_FAIL
|
|
||||||
TOTAL_PASS=0
|
|
||||||
TOTAL_FAIL=0
|
|
||||||
|
|
||||||
echo "Running APL conformance suite..." >&2
|
|
||||||
for s in "${SUITES[@]}"; do
|
|
||||||
read -r p f < <(run_suite "$s")
|
|
||||||
SUITE_PASS[$s]=$p
|
|
||||||
SUITE_FAIL[$s]=$f
|
|
||||||
TOTAL_PASS=$((TOTAL_PASS + p))
|
|
||||||
TOTAL_FAIL=$((TOTAL_FAIL + f))
|
|
||||||
printf " %-12s %d/%d\n" "$s" "$p" "$((p+f))" >&2
|
|
||||||
done
|
|
||||||
|
|
||||||
# scoreboard.json
|
|
||||||
{
|
|
||||||
printf '{\n'
|
|
||||||
printf ' "suites": {\n'
|
|
||||||
first=1
|
|
||||||
for s in "${SUITES[@]}"; do
|
|
||||||
if [ $first -eq 0 ]; then printf ',\n'; fi
|
|
||||||
printf ' "%s": {"pass": %d, "fail": %d}' "$s" "${SUITE_PASS[$s]}" "${SUITE_FAIL[$s]}"
|
|
||||||
first=0
|
|
||||||
done
|
|
||||||
printf '\n },\n'
|
|
||||||
printf ' "total_pass": %d,\n' "$TOTAL_PASS"
|
|
||||||
printf ' "total_fail": %d,\n' "$TOTAL_FAIL"
|
|
||||||
printf ' "total": %d\n' "$((TOTAL_PASS + TOTAL_FAIL))"
|
|
||||||
printf '}\n'
|
|
||||||
} > "$OUT_JSON"
|
|
||||||
|
|
||||||
# scoreboard.md
|
|
||||||
{
|
|
||||||
printf '# APL Conformance Scoreboard\n\n'
|
|
||||||
printf '_Generated by `lib/apl/conformance.sh`_\n\n'
|
|
||||||
printf '| Suite | Pass | Fail | Total |\n'
|
|
||||||
printf '|-------|-----:|-----:|------:|\n'
|
|
||||||
for s in "${SUITES[@]}"; do
|
|
||||||
p=${SUITE_PASS[$s]}
|
|
||||||
f=${SUITE_FAIL[$s]}
|
|
||||||
printf '| %s | %d | %d | %d |\n' "$s" "$p" "$f" "$((p+f))"
|
|
||||||
done
|
|
||||||
printf '| **Total** | **%d** | **%d** | **%d** |\n' "$TOTAL_PASS" "$TOTAL_FAIL" "$((TOTAL_PASS + TOTAL_FAIL))"
|
|
||||||
printf '\n'
|
|
||||||
printf '## Notes\n\n'
|
|
||||||
printf '%s\n' '- Suites use the standard `apl-test name got expected` framework loaded against `lib/apl/runtime.sx` + `lib/apl/transpile.sx`.'
|
|
||||||
printf '%s\n' '- `lib/apl/tests/parse.sx` and `lib/apl/tests/scalar.sx` use their own self-contained frameworks and are excluded from this scoreboard.'
|
|
||||||
} > "$OUT_MD"
|
|
||||||
|
|
||||||
echo "Wrote $OUT_JSON and $OUT_MD" >&2
|
|
||||||
echo "Total: $TOTAL_PASS pass, $TOTAL_FAIL fail" >&2
|
|
||||||
|
|
||||||
[ "$TOTAL_FAIL" -eq 0 ]
|
|
||||||
|
|||||||
@@ -9,9 +9,9 @@
|
|||||||
"system": {"pass": 13, "fail": 0},
|
"system": {"pass": 13, "fail": 0},
|
||||||
"idioms": {"pass": 64, "fail": 0},
|
"idioms": {"pass": 64, "fail": 0},
|
||||||
"eval-ops": {"pass": 14, "fail": 0},
|
"eval-ops": {"pass": 14, "fail": 0},
|
||||||
"pipeline": {"pass": 40, "fail": 0}
|
"pipeline": {"pass": 152, "fail": 0}
|
||||||
},
|
},
|
||||||
"total_pass": 450,
|
"total_pass": 562,
|
||||||
"total_fail": 0,
|
"total_fail": 0,
|
||||||
"total": 450
|
"total": 562
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -13,8 +13,8 @@ _Generated by `lib/apl/conformance.sh`_
|
|||||||
| system | 13 | 0 | 13 |
|
| system | 13 | 0 | 13 |
|
||||||
| idioms | 64 | 0 | 64 |
|
| idioms | 64 | 0 | 64 |
|
||||||
| eval-ops | 14 | 0 | 14 |
|
| eval-ops | 14 | 0 | 14 |
|
||||||
| pipeline | 40 | 0 | 40 |
|
| pipeline | 152 | 0 | 152 |
|
||||||
| **Total** | **450** | **0** | **450** |
|
| **Total** | **562** | **0** | **562** |
|
||||||
|
|
||||||
## Notes
|
## Notes
|
||||||
|
|
||||||
|
|||||||
15
lib/apl/test-harness.sx
Normal file
15
lib/apl/test-harness.sx
Normal file
@@ -0,0 +1,15 @@
|
|||||||
|
; lib/apl/test-harness.sx — counters + assertion fn for the shared conformance
|
||||||
|
; driver (lib/guest/conformance.sh, MODE=counters). Loaded as a PRELOAD so each
|
||||||
|
; suite starts from a fresh 0/0; suites call (apl-test name got expected).
|
||||||
|
|
||||||
|
(define apl-test-pass 0)
|
||||||
|
(define apl-test-fail 0)
|
||||||
|
|
||||||
|
(define
|
||||||
|
apl-test
|
||||||
|
(fn
|
||||||
|
(name got expected)
|
||||||
|
(if
|
||||||
|
(= got expected)
|
||||||
|
(set! apl-test-pass (+ apl-test-pass 1))
|
||||||
|
(set! apl-test-fail (+ apl-test-fail 1)))))
|
||||||
423
plans/abstractions.md
Normal file
423
plans/abstractions.md
Normal file
@@ -0,0 +1,423 @@
|
|||||||
|
# Abstraction Radar — backlog
|
||||||
|
|
||||||
|
Maintained by the read-only `radar` loop (see `plans/agent-briefings/radar-loop.md`).
|
||||||
|
Detection only — implementation is a separate, coordinated step owned by the
|
||||||
|
relevant subsystem loop, never by radar.
|
||||||
|
|
||||||
|
**AHA gate to reach _Proposed_:** ≥3 real consumers · all past Phase 2 & API-stable ·
|
||||||
|
structurally identical (file:line evidence) · a natural home (usually NOT lib/guest).
|
||||||
|
Anything short → _Watching_ (what's missing) or _Rejected_ (why).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Last scan
|
||||||
|
|
||||||
|
- **Date:** 2026-06-07 (radar loop, pass 19)
|
||||||
|
- **Pass 19 — honest empty pass.** Scanned 10 active subsystems. content/index.sx is a
|
||||||
|
blog index/tag-cloud listing (presentation, not full-text search — no search reinvention)
|
||||||
|
and content/multi-doc indexing adds no per-viewer filter. **W2 re-tested: still 2**
|
||||||
|
(feed, search) — acl's `permit?`-like matches are its own authZ *engine* (the home),
|
||||||
|
not a downstream read filter. No new candidate cleared any gate.
|
||||||
|
- **Date:** 2026-06-07 (radar loop, pass 18)
|
||||||
|
- **Pass 18 — W1 gate re-test.** events shipped Phase 4 federation (5th consumer): a 5th
|
||||||
|
divergent merge (sorted agenda + `:origin` provenance), trust-gate = runtime list
|
||||||
|
membership (shares mod's mechanism, not acl's). Reinforces W1's "theme not shape" — but
|
||||||
|
the **inject-fed-sx-transport seam is now 5/5**, strengthening "all are fed-sx
|
||||||
|
consumers-in-waiting." Trust sub-pattern refined: mod+events (runtime set) vs acl (rule).
|
||||||
|
- **Date:** 2026-06-07 (radar loop, pass 17)
|
||||||
|
- **Pass 17 — filename census declared EXHAUSTED** (see the Census-status table above).
|
||||||
|
Examined the last unswept ≥2 recurrences (schema/engine = acl⇄mod substrate twins;
|
||||||
|
catalog/batch = name collisions; store = divergent). No new candidate. Incremental churn
|
||||||
|
elsewhere (content 621/621, identity PAR, events reminders). Future passes pivot from
|
||||||
|
censusing to re-testing gates as consumers mature.
|
||||||
|
- **Date:** 2026-06-07 (radar loop, pass 16)
|
||||||
|
- **Pass 16:** events started Phase 3 — **durable notification delivery on `lib/flow`**
|
||||||
|
(new W8: at-least-once + idempotency exemplar; fed-sx/mod roll their own outbox). The two
|
||||||
|
`notify.sx` (feed vs events) are a name collision (read-side digest vs delivery), noted
|
||||||
|
in W8. Substrate-adoption story deepening: app domains now consume persist (content/
|
||||||
|
commerce/events), flow (events), commerce (events), acl-authZ (identity).
|
||||||
|
- **Date:** 2026-06-07 (radar loop, pass 15)
|
||||||
|
- **Pass 15:** added the **scanning-method note** above after `query.sx` again proved to
|
||||||
|
be merged-lib copies (lib/prolog + lib/persist in every worktree). Corrected census
|
||||||
|
surfaced `wire`×2 (content+mod) → Rejected (shared role, divergent structure: generic SX
|
||||||
|
serializer vs bespoke pipe-format under a Prolog-env string-prim constraint). events↔
|
||||||
|
commerce integration appeared (paid tickets); acl/mod/search quiescent ~7 passes (now
|
||||||
|
API-stable). No new gate-clearer.
|
||||||
|
- **Date:** 2026-06-07 (radar loop, pass 14)
|
||||||
|
- **Pass 14:** filename census flagged `snapshot`×?? — but the `*/lib/persist/snapshot.sx`
|
||||||
|
copies are just the merged `lib/persist` in each worktree, NOT consumers (same artifact
|
||||||
|
as `lib/feed/rank.sx` everywhere). The one distinct file, `content/snapshot.sx`,
|
||||||
|
reimplements persist's projection-checkpoint on raw KV instead of using `persist/snapshot`
|
||||||
|
→ new W7 (persist-adoption nudge). `audit`×3 = the W4 fakes (acl/mod/identity), known.
|
||||||
|
- **Date:** 2026-06-07 (radar loop, pass 13)
|
||||||
|
- **Pass 13 — honest re-test, no gate-clearer.** Re-tested the two longest-waiting gates
|
||||||
|
against the maturing app-domain loops: **W2** (per-viewer visibility) still 2 consumers
|
||||||
|
(feed, search) — commerce/content/events/identity add no per-viewer read filter; **W3**
|
||||||
|
(pagination) still 2 (feed, search) — `content/page.sx` is an HTML wrapper, not
|
||||||
|
pagination (filename collision, noted in W3). Incremental churn only elsewhere.
|
||||||
|
- **Date:** 2026-06-07 (radar loop, pass 12)
|
||||||
|
- **Pass 12:** `events` shipped **transactional booking on persist** (3rd live persist
|
||||||
|
consumer) using `persist/append-expect` (optimistic-concurrency CAS, lock-free capacity
|
||||||
|
safety). W4 ledger now shows a persist feature-ladder append → append-once → append-expect
|
||||||
|
that the hand-rolled fakes can't match. No new candidate; W4 reinforced.
|
||||||
|
- **Date:** 2026-06-07 (radar loop, pass 11)
|
||||||
|
- **Pass 11 — W4 sharpened with a consumer ledger.** commerce built an **order ledger on
|
||||||
|
persist** (2nd live exemplar; uses `persist/append-once` for webhook idempotency) and
|
||||||
|
identity a **grant audit ledger** (in-memory Erlang fake, gated on an Erlang↔persist
|
||||||
|
bridge). The append-only monotonic-seq event-log pattern is now validated across 4
|
||||||
|
domains, 2 live on persist + 3 fakes flagged for adoption. See W4 table.
|
||||||
|
- **Date:** 2026-06-07 (radar loop, pass 10)
|
||||||
|
- **Pass 10:** commerce/content/events/identity advancing (content 238/238). Probed a
|
||||||
|
shape outside the routing table — **guarded lifecycle state machines** (mod/lifecycle +
|
||||||
|
identity/membership) → new W6: shared *design principle*, divergent *structure*
|
||||||
|
(SX transition-table vs Erlang gen_server), NOT an extraction target. No gate-clearer.
|
||||||
|
- **Date:** 2026-06-07 (radar loop, pass 9)
|
||||||
|
- **Pass 9:** `commerce` + `content` reached Phase 2 (`content` 162/162). **Key find:
|
||||||
|
`content` built its op log directly on `persist/log`** (backend-injected, append+replay-
|
||||||
|
to-seq) — the live reference exemplar for W4 (see W4). `events` MONTHLY RRULE,
|
||||||
|
`identity` OAuth2 auth-code + PKCE, search boolean-filtered ranked. A1 still 6 adopters.
|
||||||
|
- **Date:** 2026-06-06 (radar loop, pass 8)
|
||||||
|
- **Pass 8 — fleet expanded by 4 app-domain loops** (the briefing's anticipated
|
||||||
|
`commerce`/`identity` arrivals, auto-picked up by dynamic discovery). All early-stage,
|
||||||
|
**pre-Phase-2 → moving targets, none count toward any gate yet**:
|
||||||
|
- `commerce` (Phase 1: `api/cart/catalog/price`). Its "per-line audit" is a cost
|
||||||
|
*breakdown view* (`api.sx:44`), **not** an append-only decision log → NOT a W4
|
||||||
|
consumer.
|
||||||
|
- `events` (Phase 1: `calendar.sx`, RRULE expansion).
|
||||||
|
- `identity` (early: `session/token`). Defers authZ to acl (`token.sx:15`) — reinforces
|
||||||
|
W2's "delegate `permit?` to acl-on-sx" routing; identity = authN, acl = authZ.
|
||||||
|
- `content` (just-started: `block.sx`).
|
||||||
|
These are the future consumers W2/W3 are waiting on — re-check their per-viewer filters
|
||||||
|
/ pagination once each clears Phase 2. No new gate-clearer this pass.
|
||||||
|
- **Pass 7:** **A1 jumped 4→6 adopters** — `acl` + `mod` migrated to the shared
|
||||||
|
conformance driver (first app-domain adopters; proves it generalizes past substrates).
|
||||||
|
`host-persist` closed its blob-adapter blocker (durable storage adapter now landing →
|
||||||
|
W4 migration path opening). search shipped proximity/NEAR; flow + persist quiescent.
|
||||||
|
- **Pass 6:** new worktree **`host-persist`** (active — building persist's durable host
|
||||||
|
adapter); `feed` went quiescent (left tmux). acl shipped hardening (+25), fed-sx-m1 at
|
||||||
|
Step 6c. **mod loop independently wrote a shared-plumbing note** (`mod-on-sx.md`,
|
||||||
|
538b8a53) corroborating W4/W5 — folded its claims + home disagreements into W1/W4/W5.
|
||||||
|
No new gate-clearer (audit log still 2 consumers), but consumers are now API-stable.
|
||||||
|
- **Pass 5:** search (+highlight/snippet) and fed-sx-m1 (+follower_graph) moved; rest
|
||||||
|
unchanged. Filename census: `api`×6, `fed`×3, then `schema/rank/query/page/explain/
|
||||||
|
engine/batch/audit`×2. Examined the ×6 `api.sx` → Rejected (shared name, divergent
|
||||||
|
structure incl. implicit-vs-explicit-state contract). rank/batch/engine all ≤2 +
|
||||||
|
substrate/domain-divergent → no new gate-clearer.
|
||||||
|
- **Pass 4:** no churn vs pass 3 (same worktrees/tmux/HEADs/adopters). Swept audit+explain
|
||||||
|
surfaces: acl/mod share an append-only-log shape (→ sharpened W4 with persist/log API
|
||||||
|
evidence) and a proof-explain shape (→ new W5, substrate-bound). No new gate-clearer.
|
||||||
|
- **Pass 3 (earlier today):** subsystem set + tmux + A1 adopters (4) all unchanged vs pass 2. Loops
|
||||||
|
advanced: acl shipped Phase 4 federation; search shipped Phase 4 + pagination; feed
|
||||||
|
shipped pagination/threading; mod at Ext 19 (capstone); persist did a worked acl-grants
|
||||||
|
migration (W4). New shape found: offset/limit pagination → folded into W3.
|
||||||
|
- **Subsystem set discovered:** loop worktrees `acl, erlang, fed-prims, fed-sx-m1,
|
||||||
|
feed, flow, go, kernel, mod, ocaml, persist, radar, ruby, search,
|
||||||
|
sx-vm-extensions`; main-repo `lib/*` incl. merged `feed` + substrates (`apl,
|
||||||
|
common-lisp, datalog, erlang, forth, go, haskell, hyperscript, js, lua, minikanren,
|
||||||
|
ocaml, prolog, scheme, smalltalk, tcl`) + `lib/guest`.
|
||||||
|
Actively looping (tmux): `acl, fed-sx-m1, feed, flow, mod, persist, search`
|
||||||
|
(+ radar).
|
||||||
|
- **New since pass 1:** worktrees `kernel` (empty/unset — not yet a repo) and `ocaml`
|
||||||
|
(`lib/ocaml/baseline` only). Both early-stage, pre–Phase 2 → out of proposal scope.
|
||||||
|
- Re-enumerate every pass; new loops (e.g. a future `commerce`/`identity`) auto-join.
|
||||||
|
|
||||||
|
**Census status (pass 17): EXHAUSTED.** Every own-namespace filename recurring ≥2× has
|
||||||
|
been examined and dispositioned — further filename-censusing is low-yield until new
|
||||||
|
subsystems/modules appear. Map:
|
||||||
|
| filename | owners | verdict |
|
||||||
|
|---|---|---|
|
||||||
|
| `api` ×10 | all | Rejected — shared role, divergent state contract |
|
||||||
|
| `fed`/`federation` | feed/search/mod/acl(+content) | W1 — theme not shape |
|
||||||
|
| `audit` ×3 | acl/mod/identity | W4 — append-only log → persist/log |
|
||||||
|
| `page` ×3 | feed/search (pagination) + content (HTML wrapper) | W3 + collision noted |
|
||||||
|
| `explain` ×2 | acl/mod | W5 — proof tree, substrate-bound |
|
||||||
|
| `snapshot` ×2 | persist(facet) + content(reinvents) | W7 |
|
||||||
|
| `wire` ×2 | content(SX serializer) / mod(pipe-format) | Rejected — divergent |
|
||||||
|
| `schema`,`engine` ×2 | acl/mod | substrate-twin parallels (Datalog vs Prolog); only audit (W4) is liftable |
|
||||||
|
| `catalog`,`batch` ×2 | commerce/persist, mod/persist | name collisions, unrelated |
|
||||||
|
| `store` ×2 | content(on persist) / flow(workflow records) | related concept, divergent |
|
||||||
|
| `rank` ×2 | feed/search | different domains (activities vs docs), ≤2 |
|
||||||
|
**acl⇄mod are structural twins** (decision engine over a logic substrate, Datalog vs
|
||||||
|
Prolog) — they parallel across engine/schema/explain/audit/fed, but only the *audit log*
|
||||||
|
is substrate-agnostic and liftable (→ W4); the rest are substrate-idiomatic. Next passes:
|
||||||
|
re-test gates (W2/W3/W8) as consumers mature, watch new modules — not re-census.
|
||||||
|
|
||||||
|
**Scanning-method note (learned the hard way, passes 5/12/14/15):** a filename census
|
||||||
|
for *cross-subsystem* recurrence MUST restrict to each subsystem's OWN namespace —
|
||||||
|
`X/lib/X/*.sx` — never `X/lib/*/`. The merged substrate libs (`lib/prolog`, `lib/persist`,
|
||||||
|
`lib/feed`, `lib/datalog`, …) are checked out inside *every* worktree, so a naive census
|
||||||
|
reports e.g. `query.sx`/`snapshot.sx`/`rank.sx` ×N as phantom recurrences that are really
|
||||||
|
one merged file copied N times. Correct one-liner:
|
||||||
|
`for w in <subsystems>; do for f in $w/lib/$w/*.sx; do basename $f .sx; done; done | sort | uniq -c | sort -rn`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Proposed (cleared the gate)
|
||||||
|
|
||||||
|
### A1 · Adopt the shared conformance driver across subsystems
|
||||||
|
- **Pattern:** every subsystem hand-rolls a near-identical `conformance.sh`
|
||||||
|
(epoch-load → eval → scoreboard emit) and an inline `<x>-test name got expected`
|
||||||
|
pass/fail counter.
|
||||||
|
- **Consumers (≥3, overwhelming):** 15 `lib/*/conformance.sh` — `apl, feed, datalog,
|
||||||
|
flow, mod, lua, erlang, forth, go, common-lisp, haskell, js, ocaml, prolog,
|
||||||
|
smalltalk, tcl`.
|
||||||
|
- **Home:** `lib/guest` — the one legitimate exception (the shared driver
|
||||||
|
`lib/guest/conformance.sh` + `lib/guest/conformance.sx` already exist; modes
|
||||||
|
`dict` and `counters`).
|
||||||
|
- **Status: IN PROGRESS — 6 adopters (pass 7).** `prolog` (dict), `haskell` (counters),
|
||||||
|
`apl` (dict), `datalog` (dict), and **`acl` (dict) + `mod` (dict), newly migrated this
|
||||||
|
pass** — all 3-line exec shims into `lib/guest/conformance.sh` with a `conformance.conf`.
|
||||||
|
**acl + mod are the first *app-domain* adopters** (not language substrates) — strong
|
||||||
|
evidence the driver generalizes beyond the substrate layer, which was the open question.
|
||||||
|
The `apl` migration earlier *surfaced a latent bug*: the old awk extractor
|
||||||
|
under-counted `pipeline` (40 vs the real 152 assertions); true apl total is **562**,
|
||||||
|
not 450 — evidence that adopting the driver also improves correctness.
|
||||||
|
- **Not a target (different harness shape):** `lua/conformance.sh` is a Python runner
|
||||||
|
(`lib/lua/conformance.py`) that walks real `*.lua` source files via `lua-eval-ast`
|
||||||
|
and classifies pass/fail/timeout — it does not run SX `deftest` suites with a
|
||||||
|
counter/dict scoreboard, so the shared driver does not fit. Excluded, not pending.
|
||||||
|
- **Remaining hand-rolled candidates (~120–220 lines each):** `common-lisp, erlang,
|
||||||
|
feed, forth, go, js, ocaml, smalltalk, tcl` — each its OWN loop's migration when
|
||||||
|
quiescent. (`search` + `lua` excluded: different harness shapes — search assembles a
|
||||||
|
Haskell source string, lua walks real `*.lua` files.)
|
||||||
|
- **Action:** each remaining subsystem's OWN loop migrates when quiescent — add a
|
||||||
|
`conformance.conf` (+ a `test-harness.sx` preload defining its counters) and
|
||||||
|
replace `conformance.sh` with the 1-line exec shim
|
||||||
|
(`exec bash …/guest/conformance.sh …/conformance.conf "$@"`). Recipe template:
|
||||||
|
`lib/haskell/conformance.conf` (counters) or `lib/prolog/conformance.conf` (dict).
|
||||||
|
Keep the `bash lib/X/conformance.sh` entry point so no loop is disrupted.
|
||||||
|
- **Priority: HIGH** (15 consumers, low risk, interface-preserving, additive).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Watching (real but not yet through the gate)
|
||||||
|
|
||||||
|
### W1 · Federation scaffold (merge / ingest / backfill / trust-gate)
|
||||||
|
- **FAILS the structural-identity gate (deep-dived 2026-06-06, all 4 read).** Consumer
|
||||||
|
count is met (4) but they are *superficially* similar, not structurally identical —
|
||||||
|
the federated unit and merge op differ fundamentally:
|
||||||
|
|
||||||
|
| Subsystem (file) | Federated unit | Merge op | Trust gate | Injected transport |
|
||||||
|
|---|---|---|---|---|
|
||||||
|
| feed (`fed.sx:14,18,40`) | activity streams | dedupe by `(actor verb object)` | none (visibility via `permit?` separately) | `send-fn`, `fetch-fn` |
|
||||||
|
| search (`fed.sx:8`) | inverted indices | relabel DocId `peer*1000+local` + union posting lists | none | none (pure merge fn) |
|
||||||
|
| mod (`fed.sx:11-14,99`) | moderation decisions | advisory-list vs applied-list; bind iff `mod/trusted?` | **yes — runtime list** `mod/trusted? peer scope` | mock outbox / `fed-send!` |
|
||||||
|
| acl (`federation.sx:43,56`) | Datalog delegate facts | pull facts, gate by `trust`/`level_covers` rule, re-saturate | **yes — Datalog rule** at query time | `transport` dict |
|
||||||
|
| events (`federation.sx`) | calendar agendas | fold trusted peers' agendas into one sorted agenda + `:origin` provenance | **yes — runtime list** `ev/trusts?` (peer-id ∈ trust-set) | injected behind `ev/peer-agenda` |
|
||||||
|
|
||||||
|
- **The ONLY real commonality is the injection seam** (now 5/5, pass 18), not extractable
|
||||||
|
code: every one says "the real transport is `fed-sx`'s job; inject `send-fn`/`fetch-fn`/
|
||||||
|
`transport`/`peer-agenda` and mock it in tests." That is an architectural *convention the
|
||||||
|
fleet already follows*. The merge op diverges 5 ways (dedupe / index-union / advisory /
|
||||||
|
fact-saturation / agenda-sort). The trust gate, where present, splits: **mod + events use
|
||||||
|
a runtime trust-set membership check; acl uses a declarative Datalog rule** — so even the
|
||||||
|
trust sub-pattern is 2-of-3, and the membership check is a trivial one-liner (below the
|
||||||
|
extraction threshold). No shared merge, no single shared trust mechanism.
|
||||||
|
- **Disposition:** do NOT extract a shared "federation lib." When `fed-sx` ships its
|
||||||
|
real transport, these 4 become its *consumers* (wiring `send-fn`/`fetch-fn`/`transport`
|
||||||
|
to it) — that work belongs to each subsystem's loop + the `fed-sx` loop, not a
|
||||||
|
cross-cutting extraction. Stop re-proposing on the shared name. Home: `fed-sx`.
|
||||||
|
- **Narrower sub-claim (mod note, pass 6; refined pass 18):** mod asserts the *fed
|
||||||
|
trust/outbox* shape shares between mod+acl. Radar evidence refines this: the trust gate
|
||||||
|
splits by mechanism, not by subsystem pair — **mod + events** both use a runtime
|
||||||
|
trust-set membership check (`mod/trusted?`, `ev/trusts?`), while **acl** uses a Datalog
|
||||||
|
rule. So a "trust-set membership" helper has 2 consumers (mod, events) — but it's a
|
||||||
|
one-line `member?` and the merge it gates diverges, so still not worth extracting.
|
||||||
|
Resolve at the architecture-merge point if a heavier shared trust-set surface emerges.
|
||||||
|
|
||||||
|
### W2 · Per-viewer visibility / permission filter
|
||||||
|
- **2 shipped consumers, same shape** — `filter <injected-permit> <ranked/candidate stream>`:
|
||||||
|
- `feed/lib/feed/acl.sx:27` `feed/visible = (feed/filter stream (fn (a) (permit? viewer a)))`,
|
||||||
|
capstone at `:34` (stream → ACL → rank → top-N). `permit?` injected, sig `(viewer activity)→bool`.
|
||||||
|
- `search/lib/search/fed.sx:16` `aclFilter permit docs = filter permit docs`;
|
||||||
|
`topNTfIdfAcl n permit ts idx = take n (aclFilter permit (rankTfIdf ts idx))`.
|
||||||
|
`permit` injected, sig `DocId→Bool` (viewer baked in by caller).
|
||||||
|
- **NOT a consumer:** `mod/lib/mod/policy.sx` is moderation policy (reviewer actions),
|
||||||
|
no per-viewer read filter. So mod won't be the 3rd.
|
||||||
|
- **Missing:** (a) only 2 consumers, need ≥3; (b) the two interfaces *diverge* —
|
||||||
|
feed passes `(viewer, item)`, search bakes the viewer in — so any shared form must
|
||||||
|
pick a convention; (c) both already **inject** the predicate, and the filter body is
|
||||||
|
literally one line (`filter permit xs`). Leaning toward: the predicate's home is
|
||||||
|
`acl-on-sx` (`permit?`), and the one-line filter is too thin to extract.
|
||||||
|
- **Home when ripe:** delegate `permit?` to `acl-on-sx`; do NOT extract the filter.
|
||||||
|
Re-check if a 3rd genuine per-viewer read filter ships (e.g. events/commerce).
|
||||||
|
|
||||||
|
### W3 · Collection helpers (group-by, dedupe-by-key, stable top-N, distinct-order, offset/limit page)
|
||||||
|
- feed built all of these on APL primitives. search/commerce/events will want
|
||||||
|
group-by / top-N.
|
||||||
|
- **NEW (2026-06-06): offset/limit pagination shipped in 2 subsystems, identical shape**
|
||||||
|
`take limit (drop offset xs)`:
|
||||||
|
- `feed/lib/feed/page.sx:9` `feed/page` (offset/limit window over a stream).
|
||||||
|
- `search/lib/search/page.sx:9` `paginate off lim docs = take lim (drop off docs)`.
|
||||||
|
- NOT a 3rd: `persist/lib/persist/query.sx:5` has a *since-cursor* for incremental log
|
||||||
|
consumption — resumable-stream semantics, not result windowing. Different shape.
|
||||||
|
- feed *also* has cursor-by-`:at` recency pagination (`page.sx:21-44`); search has no
|
||||||
|
cursor. So only the plain offset/limit window is shared, and it is a literal 1-liner.
|
||||||
|
- **Missing:** ≥3 stable consumers; AND every item here is collection math that belongs
|
||||||
|
in the **substrate** (APL/Haskell already expose grade/sort/unique/take/drop), not a
|
||||||
|
shared lib. A 1-line `take/drop` window is far below the extraction threshold. Watch;
|
||||||
|
revisit only if a non-substrate subsystem needs the same windowing without take/drop.
|
||||||
|
- **Filename-collision caution (pass 13):** `content/lib/content/page.sx` is an **HTML
|
||||||
|
page wrapper** (full HTML5 doc), NOT pagination — do not count it as a 3rd pagination
|
||||||
|
consumer. `page.sx` now means two unrelated things across the fleet. Re-tested pass 13:
|
||||||
|
pagination still only feed + search (2).
|
||||||
|
|
||||||
|
### W4 · In-memory store fakes → `persist-on-sx`
|
||||||
|
- Not an abstraction to extract — a migration target. Every subsystem fakes its
|
||||||
|
store with a mutable list (`feed/-log`, flow store, mod audit, …).
|
||||||
|
- **Owner:** `persist-on-sx` (in progress). Tracked there, listed here for visibility.
|
||||||
|
- **Concrete instance (file:line, found pass 4): the append-only decision/audit log.**
|
||||||
|
`acl/lib/acl/audit.sx` and `mod/lib/mod/audit.sx` are the SAME hand-rolled shape, and
|
||||||
|
`persist/lib/persist/log.sx` (the persist *log facet*) already implements it durably:
|
||||||
|
|
||||||
|
| role | acl/audit.sx | mod/audit.sx | persist/log.sx (target) |
|
||||||
|
|---|---|---|---|
|
||||||
|
| log var | `acl-audit-log` :9 | `mod/*audit-log*` :10 | backend stream |
|
||||||
|
| monotonic seq | `acl-audit-seq` :10 | `mod/*audit-seq*` :11 | per-stream high-water :1 |
|
||||||
|
| append (auto-seq) | `acl-audit-decide!` | commit :32 | `persist/append` :17 |
|
||||||
|
| count | `acl-audit-count` :51 | `mod/audit-count` :44 | `persist/count` :12 |
|
||||||
|
| read-all oldest-first | snapshot/tail :73 | `mod/audit-all` :43 | `persist/read` :29 |
|
||||||
|
| read seq≥from | — | by-seq | `persist/read-from` :31 |
|
||||||
|
|
||||||
|
Both deliberately use a monotonic seq with **no wall-clock** (deterministic/testable) —
|
||||||
|
identical to persist/log's design. Action when persist's host adapter lands: acl + mod
|
||||||
|
loops swap their in-memory log for `persist/log`. 2 consumers today; not a new lib —
|
||||||
|
the home already exists. Belongs to acl/mod loops × persist loop, not an extraction.
|
||||||
|
- **Cross-loop corroboration (pass 6):** the mod loop independently reached the same
|
||||||
|
conclusion — `mod/plans/mod-on-sx.md` (commit 538b8a53): *"mod-sx (Prolog) and acl-sx
|
||||||
|
(Datalog) converged on the same module shape … only the audit log + fed trust/outbox
|
||||||
|
shapes truly share; extract at the architecture-merge point, refactoring both consumers
|
||||||
|
atomically, not unilaterally from a loop branch."* Confirms the shape AND the
|
||||||
|
do-not-extract-unilaterally stance.
|
||||||
|
- **Home disagreement to resolve at merge:** mod's note proposes lifting the audit-log
|
||||||
|
primitives into **`lib/guest/`**. Radar routing disagrees: a durable append-only log is
|
||||||
|
a **`persist-on-sx`** concern (the log facet already exists), not language-impl plumbing.
|
||||||
|
Hold the line — `lib/guest` is lexer/parser/AST/HM/test-runner, not an event log.
|
||||||
|
- **Migration is becoming concrete:** new `host-persist` loop (worktree + tmux, pass 6)
|
||||||
|
is building the durable-storage host adapter persist was blocked on — once it lands,
|
||||||
|
acl/mod can actually swap to `persist/log`.
|
||||||
|
- **LIVE REFERENCE EXEMPLAR (pass 9): `content` already does it right.** `content`
|
||||||
|
(Phase 2 complete, 162/162) built its op log directly on `persist/log` instead of
|
||||||
|
faking it — `content/lib/content/store.sx`: backend injected via `(persist/open)`
|
||||||
|
("content knows nothing about which backend", :10); append op as event
|
||||||
|
`persist/append b (content/-stream doc-id) …` (:20); read `persist/read` (:36);
|
||||||
|
`persist/last-seq` (:47); **version = replay op stream up to a seq**
|
||||||
|
(filter `persist/event-seq ev <= seq`, :61). "The op log is the source of truth …
|
||||||
|
the materialised doc is a cache, never primary state."
|
||||||
|
This proves the W4 target is real, not hypothetical: acl + mod's hand-rolled
|
||||||
|
monotonic-seq logs should adopt exactly content's `persist/log` pattern.
|
||||||
|
- **Consumer ledger of the append-only monotonic-seq event log (pass 11):**
|
||||||
|
|
||||||
|
| consumer | what | backing | note |
|
||||||
|
|---|---|---|---|
|
||||||
|
| content (`store.sx`) | doc op log | **persist/log ✓ live** | plain append + replay-to-seq |
|
||||||
|
| commerce (`ledger.sx`) | order ledger | **persist/log ✓ live** | `persist/append-once` — idempotent, webhook-replay-safe :40,58 |
|
||||||
|
| events (`booking.sx`) | booking roster | **persist/log ✓ live** | `persist/append-expect` — optimistic-concurrency CAS, capacity-safe, lock-free |
|
||||||
|
| acl (`audit.sx`) | decision log | in-memory fake (SX) | migrate directly when host adapter lands |
|
||||||
|
| mod (`audit.sx`) | decision log | in-memory fake (SX) | migrate directly |
|
||||||
|
| identity (`audit.sx`) | grant ledger | in-memory fake (**Erlang**) | `{Seq,Subject,Action}`; needs an **Erlang↔persist bridge** first — author scoped it out until persist lands ("queryable semantics identical") |
|
||||||
|
|
||||||
|
- **Two takeaways:** (1) the pattern is **validated across domains** — CRDT doc ops,
|
||||||
|
financial orders, event bookings, rule decisions, OAuth grants all reduce to the same
|
||||||
|
append-only monotonic-seq stream; (2) migrating to `persist/log` is strictly *better*
|
||||||
|
than the fakes — persist exposes a **feature ladder the fakes don't have**:
|
||||||
|
`append` (content) → `append-once`/idempotency (commerce) → `append-expect`/optimistic-
|
||||||
|
concurrency (events). Every fake would have to reinvent a weaker version of these.
|
||||||
|
This is an **adoption** item (the home already exists), NOT a new extraction — owned by
|
||||||
|
persist/host-persist × each consumer loop. The SX fakes (acl, mod) migrate directly;
|
||||||
|
the Erlang fake (identity) is gated on an Erlang↔persist bridge.
|
||||||
|
|
||||||
|
### W5 · Proof-tree explanation over a logic-program derivation
|
||||||
|
- `acl/lib/acl/explain.sx` (reconstructs a canonical proof by goal-directed search over a
|
||||||
|
saturated Datalog db) and `mod/lib/mod/explain.sx` (renders a Prolog-style proof tree
|
||||||
|
goal-by-goal with proved/unproved marks + unification bindings) are the same *idea*.
|
||||||
|
- **Missing / disposition:** only 2 consumers, and they sit on **different substrates**
|
||||||
|
(acl→`lib/datalog`, mod→`lib/prolog`). Proof reconstruction/rendering is logic-engine
|
||||||
|
machinery → it belongs in each **substrate** (datalog/prolog), not a shared app lib.
|
||||||
|
Watch; revisit only if a 3rd logic-backed subsystem reimplements proof explanation.
|
||||||
|
- **Cross-loop note (pass 6):** mod's note calls `mod/proof-goals` (re-query-each-goal)
|
||||||
|
generic and proposes lifting it into **`lib/guest/`**. Radar caveat: proof-tree
|
||||||
|
reconstruction *is* engine-agnostic logic machinery, but `lib/guest` is for
|
||||||
|
lexer/parser/AST/HM/match/test-runner — a logic-engine proof helper is a poor fit there.
|
||||||
|
If genuinely shared by ≥3 engines, a `lib/logic`-style substrate helper is the better
|
||||||
|
home than `lib/guest`. Still 2 consumers → stays Watching either way.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### W8 · Durable outbound delivery (at-least-once + idempotency + retry)
|
||||||
|
- **Live exemplar on `lib/flow`:** `events/lib/events/notify.sx` — reminders/digests are
|
||||||
|
durable `flow`s: a flow `request`s delivery (suspend point), the **host** performs the
|
||||||
|
send via an injected `dispatch` transport, then resumes with the outcome; flow's
|
||||||
|
deterministic replay means a completed delivery never re-runs on recovery. At-least-once
|
||||||
|
with an idempotency key per message. This is "reliable delivery" done right on the flow
|
||||||
|
substrate.
|
||||||
|
- **Others roll their own:** `fed-sx` built its own outbox + `delivery_worker` + retry
|
||||||
|
bookkeeping (Steps 8a–d); `mod/fed.sx` has an in-memory outbox seam; `acl/federation`
|
||||||
|
propagates facts. Same *goal* (reliable outbound delivery, retry/idempotency) on
|
||||||
|
different machinery.
|
||||||
|
- **Disposition:** durable delivery is exactly what `lib/flow` is *for* (events proves
|
||||||
|
it). Watch whether fed-sx / mod converge their outbox onto flow, or stay bespoke for
|
||||||
|
perf/substrate reasons. 1 clean flow-based consumer today → Watching, not a proposal.
|
||||||
|
- **Name-collision caveat:** `notify.sx` means two unrelated things — `feed/notify.sx` is
|
||||||
|
a *read-side digest* (group inbox by verb+object), NOT delivery. Do not pair them.
|
||||||
|
|
||||||
|
### W7 · Snapshot/projection-checkpoint reimplemented vs `persist/snapshot` (delegate)
|
||||||
|
- `persist/lib/persist/snapshot.sx` already provides a **generic** projection checkpoint:
|
||||||
|
store `{:value :seq}` in the kv facet under a namespaced key; the headline property is
|
||||||
|
**snapshot + tail == full replay** (pure, clock-free).
|
||||||
|
- `content/lib/content/snapshot.sx` **reimplements that same pattern on raw persist KV**
|
||||||
|
rather than delegating: `persist/kv-put b (content/-snap-key doc-id) {:doc … :seq seq}`
|
||||||
|
(:20), `persist/kv-has?`/`kv-get` (:27-28), and its own tail-replay (:53-59). It never
|
||||||
|
calls `persist/snapshot-*`. content's doc-materialisation *is* a projection fold over
|
||||||
|
its op stream — exactly what `persist/snapshot` checkpoints generically.
|
||||||
|
- **Disposition:** persist-adoption nudge (like W4): content could delegate to
|
||||||
|
`persist/snapshot` (its projection = "fold ops → doc"), dropping the duplicated
|
||||||
|
KV+replay code. Home already exists → NOT an extraction; owned by content × persist
|
||||||
|
loops. Only 1 reinventor today; watch whether commerce/events/identity also hand-roll a
|
||||||
|
snapshot on raw KV instead of using the facet (would strengthen the nudge). NB timeline:
|
||||||
|
unclear if `persist/snapshot` predated content's — flag, don't blame.
|
||||||
|
|
||||||
|
### W6 · Guarded lifecycle state machine (illegal transition = explicit error)
|
||||||
|
- Recurs as a **design principle**, NOT a shared structure (found pass 10):
|
||||||
|
- `mod/lib/mod/lifecycle.sx` — pure SX: immutable case `{:state :error :history …}`,
|
||||||
|
explicit transition table `mod/lc-transitions` (:31), illegal transition returns the
|
||||||
|
case unchanged with `:error` set. States open→triaged→decided→appealed→final.
|
||||||
|
- `identity/lib/identity/membership.sx` — an **Erlang `gen_server`** fragment (identity
|
||||||
|
runs on erlang-on-sx): a `receive` loop with `case find(...) of … {error, St}` guards.
|
||||||
|
States none→pending→active→lapsed→revoked.
|
||||||
|
- **Both share the guideline** ("invalid transitions are explicit errors, never silent
|
||||||
|
no-ops") but **implement it substrate-idiomatically** — SX transition-table over
|
||||||
|
immutable values vs an Erlang process loop with per-message case guards. Same W1/`api.sx`
|
||||||
|
trap: shared *idea*, divergent *structure*.
|
||||||
|
- **Disposition:** not an extraction target — the FSM mechanism is ~10 substrate-specific
|
||||||
|
lines; the value is in each domain's state graph, not the plumbing. At most a **design
|
||||||
|
guideline** ("model lifecycle as a guarded FSM with explicit-error transitions"). Watch
|
||||||
|
whether commerce-checkout / events-booking add their own — if so it confirms the
|
||||||
|
*guideline*, still not a lib. Do not propose extracting a shared state-machine lib.
|
||||||
|
|
||||||
|
## Rejected (considered, declined — do not re-propose)
|
||||||
|
|
||||||
|
- **"Continuous auto-implementing abstractor loop."** Rejected at design time: an
|
||||||
|
agent writing across `lib/<x>/**` breaks the worktree isolation that makes the
|
||||||
|
fleet safe, and is rewarded for manufacturing premature/wrong abstractions. The
|
||||||
|
radar is read-only by design. (This file is the alternative.)
|
||||||
|
- **Shared `api.sx` "public boundary" module (×6).** Rejected pass 4-5: every subsystem
|
||||||
|
has an `api.sx` (acl, feed, flow, mod, persist, search — a 100% filename match), but it
|
||||||
|
is a naming *convention for the public entry point*, not a shared structure. They
|
||||||
|
disagree on the most basic contract: acl/feed use **implicit module state**
|
||||||
|
(`acl/api.sx` "implicit current db", `feed/api.sx` "single mutable log") while
|
||||||
|
`persist/api.sx` threads an **explicit backend as every call's first arg**; flow's api
|
||||||
|
*builds a Scheme env*, search's api *concatenates a Haskell source string*, mod's is a
|
||||||
|
*lifecycle state-machine façade* (17 defs vs persist's 1). Same role, no common shape —
|
||||||
|
the W1 coincidental-resemblance trap. Do not re-propose on the filename.
|
||||||
|
- **Shared `wire.sx` "serialization" module (×2).** Rejected pass 15: content + mod both
|
||||||
|
have a `wire.sx`, but `content/wire.sx` uses the **generic SX serializer**
|
||||||
|
(`serialize`/`parse`, full-fidelity round-trip) while `mod/wire.sx` is a **bespoke
|
||||||
|
versioned pipe-delimited line** (subset of fields, `split` hand-built over slice/len
|
||||||
|
because mod's Prolog-loaded env strips string prims). Shared role (wire format),
|
||||||
|
divergent structure + substrate constraint → not a candidate; the SX serializer is
|
||||||
|
already the shared tool for SX-substrate subsystems, and mod can't use it. (Same family
|
||||||
|
as the `api.sx` rejection above.)
|
||||||
|
- **Dumping app-domain plumbing into `lib/guest`.** Rejected: `lib/guest` is for
|
||||||
|
language-implementation plumbing. App patterns route to acl/fed-sx/persist/
|
||||||
|
substrate/host instead (see the routing rule in the briefing).
|
||||||
117
plans/agent-briefings/radar-loop.md
Normal file
117
plans/agent-briefings/radar-loop.md
Normal file
@@ -0,0 +1,117 @@
|
|||||||
|
# abstraction-radar loop agent (read-only scout)
|
||||||
|
|
||||||
|
Role: continuously scan **all** rose-ash subsystems for genuine abstraction /
|
||||||
|
deduplication opportunities and maintain a ranked, evidence-backed backlog at
|
||||||
|
`plans/abstractions.md`. You are a **scout, not an implementer** — you detect and
|
||||||
|
document; you never refactor across subsystems.
|
||||||
|
|
||||||
|
```
|
||||||
|
description: abstraction-radar (read-only scout)
|
||||||
|
subagent_type: general-purpose
|
||||||
|
run_in_background: true
|
||||||
|
isolation: worktree
|
||||||
|
```
|
||||||
|
|
||||||
|
## Prompt
|
||||||
|
|
||||||
|
You are the sole background agent on branch `loops/radar`, worktree
|
||||||
|
`/root/rose-ash-loops/radar`, forever. Self-paced. Your ONLY writes are to
|
||||||
|
`plans/abstractions.md` (and, rarely, refining this briefing). Push to
|
||||||
|
`origin/loops/radar` after each update. Never touch `main` or `architecture`.
|
||||||
|
|
||||||
|
## The one hard rule: you do NOT edit `lib/**` — ever
|
||||||
|
|
||||||
|
You read across every subsystem and write findings to `plans/abstractions.md`.
|
||||||
|
You do **not** implement abstractions, migrate code, or edit any `lib/<x>/**`
|
||||||
|
file in any worktree. Implementation is a separate, coordinated, human-triggered
|
||||||
|
step — proposing well is your whole job. An abstractor that writes across
|
||||||
|
subsystems would collide with the very isolation that keeps the other loops safe;
|
||||||
|
that is exactly why you are read-only.
|
||||||
|
|
||||||
|
## Dynamic discovery — re-enumerate every iteration, never hardcode
|
||||||
|
|
||||||
|
The set of subsystems grows as new loops are spawned. Each iteration, rebuild the
|
||||||
|
list from the filesystem + tmux so newly-added subsystems are automatically in
|
||||||
|
scope:
|
||||||
|
|
||||||
|
1. `ls -d /root/rose-ash-loops/*/` — every loop worktree. For a worktree named `X`,
|
||||||
|
its in-flight subsystem is `lib/X/` **inside that worktree**
|
||||||
|
(`/root/rose-ash-loops/X/lib/X/`) — that's the current, possibly-uncommitted
|
||||||
|
state. Read it there, not from your own worktree.
|
||||||
|
2. `ls -d /root/rose-ash/lib/*/` — subsystems merged into / dormant on the main repo
|
||||||
|
(e.g. `feed` once merged, the language substrates `apl`/`haskell`/`prolog`/…).
|
||||||
|
3. `tmux ls` — which subsystems are actively looping right now (affects whether a
|
||||||
|
candidate's consumers are "stable" — see the gate).
|
||||||
|
|
||||||
|
Treat the union as your scan surface. When a `commerce` or `identity` loop appears
|
||||||
|
later, step 1 picks it up with no change to you. Note in `abstractions.md` the
|
||||||
|
date and the subsystem set you scanned, so drift is visible.
|
||||||
|
|
||||||
|
## The AHA gate — before ANY candidate goes in the backlog as "proposed"
|
||||||
|
|
||||||
|
"Avoid Hasty Abstractions." A wrong shared abstraction is far costlier than the
|
||||||
|
duplication it replaces. A candidate may be listed as **proposed** only if ALL hold:
|
||||||
|
|
||||||
|
- **≥3 real consumers** (not 2 — three independent uses). Fewer → log it under
|
||||||
|
"Watching" with its consumer count, do not propose.
|
||||||
|
- **All consumers past Phase 2 and API-stable.** If a consumer's loop is mid-flight
|
||||||
|
and its interfaces are still moving (`tmux ls` shows it active + its plan has
|
||||||
|
unchecked early-phase boxes), the pattern is a moving target → "Watching."
|
||||||
|
- **Structurally identical, not superficially similar.** Show the shared shape with
|
||||||
|
file:line evidence from each consumer. Coincidental resemblance is the #1 trap.
|
||||||
|
- **It has a natural home.** And that home is usually **not** `lib/guest` — see the
|
||||||
|
routing rule below.
|
||||||
|
|
||||||
|
Anything failing a gate goes under **Watching** (with what's missing) or
|
||||||
|
**Rejected** (with why), never silently dropped — so it isn't re-proposed each pass.
|
||||||
|
|
||||||
|
## Routing rule — most patterns do NOT belong in lib/guest
|
||||||
|
|
||||||
|
`lib/guest` is for **language-implementation plumbing** (lexer/parser/AST/HM/match/
|
||||||
|
test-runner), and it has its own consumer-gated roadmap. App-subsystem patterns
|
||||||
|
almost always have a better home — route, don't dump:
|
||||||
|
|
||||||
|
| Pattern kind | Home (not lib/guest) |
|
||||||
|
|---|---|
|
||||||
|
| per-viewer visibility / permission filter | `acl-on-sx` (delegate to `permit?`) |
|
||||||
|
| federation scaffold (merge/ingest/backfill/trust) | `fed-sx` |
|
||||||
|
| durable store / event log / kv | `persist-on-sx` |
|
||||||
|
| collection math (group-by, dedupe, stable top-N) | the substrate (APL/Haskell/…) |
|
||||||
|
| HTTP/handler/middleware plumbing | `host-on-sx` |
|
||||||
|
| conformance/test harness | `lib/guest` (the one real exception — `test-runner.sx` + the shared driver live there) |
|
||||||
|
|
||||||
|
If a pattern's home is one of the subsystems, the recommended **action** is "adopt
|
||||||
|
/ delegate there," and the work belongs to that subsystem's own loop (in its
|
||||||
|
scope), not to a cross-cutting change.
|
||||||
|
|
||||||
|
## Each iteration
|
||||||
|
|
||||||
|
1. Re-discover the subsystem set (above). Record it + the date in `abstractions.md`.
|
||||||
|
2. Pick ONE thread: either deep-dive a "Watching" candidate to gather file:line
|
||||||
|
evidence and re-test its gates, or sweep for a new recurring shape across the
|
||||||
|
current set.
|
||||||
|
3. Update `plans/abstractions.md`: move items between Watching / Proposed /
|
||||||
|
In-progress (owned by a subsystem loop) / Done / Rejected, with evidence.
|
||||||
|
4. Keep it ranked by (consumers × effort-saved ÷ risk). Short, factual.
|
||||||
|
5. Commit (`radar: <one-line finding>`) and push to `origin/loops/radar`.
|
||||||
|
|
||||||
|
Do not invent work to look busy: if a pass finds nothing that clears the gate,
|
||||||
|
record "scanned N subsystems on <date>, no new candidates cleared the gate" and
|
||||||
|
stop until next iteration. Empty passes are a valid, honest result.
|
||||||
|
|
||||||
|
## Gotchas
|
||||||
|
|
||||||
|
- SX files: `sx-tree` MCP tools take `file:` not `path:`. But you mostly READ —
|
||||||
|
prefer `sx_find_across`, `sx_comp_usage`, `sx_comp_list`, `sx_summarise`, plus
|
||||||
|
`Grep`/`Glob`/`Bash` for cross-worktree scanning.
|
||||||
|
- `plans/abstractions.md` is a `.md` — edit it with normal Write/Edit, not sx-tree.
|
||||||
|
- Never run `sx_build`. You don't build anything; you read.
|
||||||
|
|
||||||
|
## Style
|
||||||
|
|
||||||
|
- Evidence over assertion: every claim cites file:line in ≥3 consumers.
|
||||||
|
- Honest empty passes. Rejected items stay rejected with a reason.
|
||||||
|
- One finding per commit. Update. Push. Next.
|
||||||
|
|
||||||
|
Go. Read `plans/abstractions.md` (seeded), re-discover the subsystem set, and
|
||||||
|
advance the highest-value thread.
|
||||||
82
plans/commerce-on-sx.md
Normal file
82
plans/commerce-on-sx.md
Normal file
@@ -0,0 +1,82 @@
|
|||||||
|
# commerce-on-sx: Catalog, cart, pricing & orders on miniKanren
|
||||||
|
|
||||||
|
> **DRAFT outline.** The revenue vertical. Depends on `persist-on-sx` (durable
|
||||||
|
> orders) and `flow-on-sx` (checkout as a durable flow). Don't start before
|
||||||
|
> persist-on-sx Phase 1 is green.
|
||||||
|
|
||||||
|
rose-ash's revenue engine — market (catalog), cart (checkout), orders (SumUp
|
||||||
|
payment, reconciliation) — has no SX subsystem. The hard part of commerce isn't
|
||||||
|
CRUD; it's **pricing**: discounts, bundles, tax, membership rates, promotions that
|
||||||
|
stack (or don't). These are relations, and a relational engine can run them in
|
||||||
|
multiple directions — forward ("what's the total?") and backward ("what promo code
|
||||||
|
yields this total?", "which line item triggered the discount?").
|
||||||
|
|
||||||
|
That's a miniKanren fit. Pricing/promotion rules are relational; cart and order
|
||||||
|
*lifecycle* (reserve → pay → fulfil → reconcile) is a durable `flow`; the order
|
||||||
|
ledger is a `persist` stream. Commerce is the first real **composition** subsystem.
|
||||||
|
|
||||||
|
End-state: a catalog model, a relational pricing/promotion engine, a cart with
|
||||||
|
deterministic totals, and an order lifecycle flow with payment-webhook
|
||||||
|
reconciliation — all auditable via the event log.
|
||||||
|
|
||||||
|
## Status (rolling)
|
||||||
|
|
||||||
|
`bash lib/commerce/conformance.sh` → **0/0** (not yet started)
|
||||||
|
|
||||||
|
## Ground rules
|
||||||
|
|
||||||
|
- **Scope:** only `lib/commerce/**` and `plans/commerce-on-sx.md`. May **import**
|
||||||
|
from `lib/minikanren/`, and (once they exist) `lib/persist/` + `lib/flow/`. Do not
|
||||||
|
edit substrates.
|
||||||
|
- **Architecture:** prices/promotions are miniKanren relations over catalog facts;
|
||||||
|
a cart total is a *deterministic* query result (first solution under a fixed rule
|
||||||
|
order). Order lifecycle is a `flow` that suspends at the payment IO boundary.
|
||||||
|
Money is integer minor units — never floats.
|
||||||
|
- **Determinism:** promotion stacking must have explicit, tested precedence;
|
||||||
|
totals must be reproducible from the cart + catalog snapshot.
|
||||||
|
- **Commits:** one feature per commit. Progress log + tick boxes.
|
||||||
|
|
||||||
|
## Architecture sketch
|
||||||
|
|
||||||
|
```
|
||||||
|
Catalog + cart Total / order
|
||||||
|
product(id,price,tags) {:subtotal :discounts :tax :total}
|
||||||
|
│ ▲
|
||||||
|
▼ │
|
||||||
|
lib/commerce/catalog.sx lib/commerce/price.sx
|
||||||
|
— product / variant / stock facts — miniKanren pricing relations
|
||||||
|
│ — promo stacking, membership rates
|
||||||
|
▼ ▲
|
||||||
|
lib/commerce/cart.sx lib/commerce/order.sx (flow + store)
|
||||||
|
— line items, quantities — reserve→pay→fulfil→reconcile
|
||||||
|
│ — SumUp webhook = flow resume
|
||||||
|
▼ │
|
||||||
|
lib/commerce/api.sx ── (commerce/add) (commerce/total) (commerce/checkout) ──┘
|
||||||
|
```
|
||||||
|
|
||||||
|
## Phase 1 — Catalog + cart + deterministic totals
|
||||||
|
- [ ] `catalog.sx` — product/variant/stock as facts
|
||||||
|
- [ ] `cart.sx` — line items, add/remove/qty
|
||||||
|
- [ ] `price.sx` — base pricing relation, subtotal; tax
|
||||||
|
- [ ] `api.sx` + tests + scoreboard + conformance.sh
|
||||||
|
|
||||||
|
## Phase 2 — Promotions (relational)
|
||||||
|
- [ ] promo rules: percentage, fixed, bundle, member rate
|
||||||
|
- [ ] explicit stacking precedence; "best price" backward query
|
||||||
|
- [ ] tests: stacking order, mutually-exclusive promos, member vs guest
|
||||||
|
|
||||||
|
## Phase 3 — Order lifecycle (flow + store)
|
||||||
|
- [ ] order flow: reserve stock → await payment → fulfil
|
||||||
|
- [ ] payment webhook resumes the suspended flow
|
||||||
|
- [ ] order ledger as a `persist` stream; idempotent reconciliation
|
||||||
|
|
||||||
|
## Phase 4 — Reconciliation + federation
|
||||||
|
- [ ] mismatch detection (paid≠ordered) as queries over the ledger
|
||||||
|
- [ ] cross-instance catalog (federated marketplace) — out-of-scope stub
|
||||||
|
- [ ] tests: webhook replay, partial refund, double-charge guard
|
||||||
|
|
||||||
|
## Progress log
|
||||||
|
(loop fills this in)
|
||||||
|
|
||||||
|
## Blockers
|
||||||
|
(loop fills this in)
|
||||||
82
plans/content-on-sx.md
Normal file
82
plans/content-on-sx.md
Normal file
@@ -0,0 +1,82 @@
|
|||||||
|
# content-on-sx: Documents, blocks & collaborative editing on Smalltalk
|
||||||
|
|
||||||
|
> **DRAFT outline.** The CMS vertical — blog, WYSIWYG editor, Ghost sync. Depends
|
||||||
|
> on `persist-on-sx` (document history as an event log). Ghost/CMS sync stays a thin
|
||||||
|
> external adapter (Python/FFI) until a native replacement exists.
|
||||||
|
|
||||||
|
rose-ash's `blog` domain is content management: a block-based WYSIWYG editor,
|
||||||
|
navigation, Ghost CMS sync. A document is a tree of live blocks; editing is a
|
||||||
|
stream of operations; collaboration needs conflict-free merge. That is an object
|
||||||
|
model — blocks are objects, edits are messages, and a document is the object graph
|
||||||
|
responding to them. Smalltalk's "everything is an object responding to messages"
|
||||||
|
maps directly to a block/WYSIWYG model, and a semilattice (CRDT) merge keeps
|
||||||
|
concurrent edits conflict-free.
|
||||||
|
|
||||||
|
End-state: a Smalltalk-on-SX document model (typed blocks, structural ops),
|
||||||
|
operation log + CRDT merge for collaborative editing, versioning/history via the
|
||||||
|
event store, and a render boundary to HTML/SX. External CMS (Ghost) sync is an
|
||||||
|
injected adapter, not core.
|
||||||
|
|
||||||
|
## Status (rolling)
|
||||||
|
|
||||||
|
`bash lib/content/conformance.sh` → **0/0** (not yet started)
|
||||||
|
|
||||||
|
## Ground rules
|
||||||
|
|
||||||
|
- **Scope:** only `lib/content/**` and `plans/content-on-sx.md`. May **import**
|
||||||
|
from `lib/smalltalk/`, and (once it exists) `lib/persist/`. Do not edit substrates.
|
||||||
|
- **Architecture:** a document is an ordered tree of blocks (objects); an edit is a
|
||||||
|
message (`insert`/`update`/`move`/`delete`); concurrent edits merge via a
|
||||||
|
commutative (CRDT/semilattice) operation so order doesn't matter. History is the
|
||||||
|
`persist` event stream; any version is a replay.
|
||||||
|
- **Determinism:** merge must be commutative + idempotent (test: apply ops in any
|
||||||
|
order / twice → same document).
|
||||||
|
- **Commits:** one feature per commit. Progress log + tick boxes.
|
||||||
|
|
||||||
|
## Architecture sketch
|
||||||
|
|
||||||
|
```
|
||||||
|
Edit op Rendered document
|
||||||
|
(insert block after id) ... HTML / SX tree
|
||||||
|
│ ▲
|
||||||
|
▼ │
|
||||||
|
lib/content/block.sx lib/content/render.sx
|
||||||
|
— typed blocks as objects — block tree → HTML/SX
|
||||||
|
— heading/text/image/embed — (reuses SX render boundary)
|
||||||
|
│ ▲
|
||||||
|
▼ │
|
||||||
|
lib/content/doc.sx lib/content/merge.sx
|
||||||
|
— ordered block tree — CRDT/semilattice op merge
|
||||||
|
— apply op, structural moves — concurrent-edit reconciliation
|
||||||
|
│ ▲
|
||||||
|
▼ │
|
||||||
|
lib/content/api.sx ── (content/edit) (content/render) (content/history) ──┐
|
||||||
|
│ │
|
||||||
|
├── op log + versions → persist │
|
||||||
|
└── Ghost/CMS sync → injected external adapter (thin, non-core) ──┘
|
||||||
|
```
|
||||||
|
|
||||||
|
## Phase 1 — Block document model
|
||||||
|
- [ ] `block.sx` — typed block objects
|
||||||
|
- [ ] `doc.sx` — ordered tree, apply edit op, structural moves
|
||||||
|
- [ ] `render.sx` — block tree → HTML/SX
|
||||||
|
- [ ] `api.sx` + tests + scoreboard + conformance.sh
|
||||||
|
|
||||||
|
## Phase 2 — Op log + versioning
|
||||||
|
- [ ] edit ops as `persist` events; replay to any version
|
||||||
|
- [ ] `(content/history doc)`, diff between versions
|
||||||
|
|
||||||
|
## Phase 3 — Collaborative merge (CRDT)
|
||||||
|
- [ ] commutative/idempotent op merge
|
||||||
|
- [ ] concurrent-edit tests (any order, double-apply → identical)
|
||||||
|
|
||||||
|
## Phase 4 — External sync + federation
|
||||||
|
- [ ] Ghost/CMS sync via injected adapter (import/export)
|
||||||
|
- [ ] federated documents (peer-authored blocks) — trust-gated stub
|
||||||
|
- [ ] tests: round-trip import/export, conflict on concurrent external edit
|
||||||
|
|
||||||
|
## Progress log
|
||||||
|
(loop fills this in)
|
||||||
|
|
||||||
|
## Blockers
|
||||||
|
(loop fills this in)
|
||||||
81
plans/events-on-sx.md
Normal file
81
plans/events-on-sx.md
Normal file
@@ -0,0 +1,81 @@
|
|||||||
|
# events-on-sx: Calendar, ticketing & notification delivery on Datalog
|
||||||
|
|
||||||
|
> **DRAFT outline.** The events vertical + the shared notification-delivery edge.
|
||||||
|
> Depends on `persist-on-sx` (bookings ledger) and `flow-on-sx` (reminders, retrying
|
||||||
|
> delivery). Pairs with `commerce-on-sx` for paid tickets.
|
||||||
|
|
||||||
|
rose-ash's `events` domain is calendar + ticketing: recurring events, availability,
|
||||||
|
capacity, bookings. Scheduling is constraint reasoning — "is this slot free given
|
||||||
|
recurrence, capacity, and the attendee's other bookings?" — which is rule
|
||||||
|
evaluation over facts. Datalog expresses availability, recurrence expansion, and
|
||||||
|
capacity as rules; a booking is a transaction; reminders and digests are durable
|
||||||
|
`flow`s. Notification *delivery* (email/push) — needed here and by `feed/notify` —
|
||||||
|
is folded in as an injected transport, extractable later.
|
||||||
|
|
||||||
|
End-state: a Datalog-on-SX events layer with recurrence expansion, availability +
|
||||||
|
capacity rules, transactional booking, and a flow-driven notification dispatcher
|
||||||
|
(reminders, digests, retries) over an injected transport.
|
||||||
|
|
||||||
|
## Status (rolling)
|
||||||
|
|
||||||
|
`bash lib/events/conformance.sh` → **0/0** (not yet started)
|
||||||
|
|
||||||
|
## Ground rules
|
||||||
|
|
||||||
|
- **Scope:** only `lib/events/**` and `plans/events-on-sx.md`. May **import** from
|
||||||
|
`lib/datalog/`, and (once they exist) `lib/persist/` + `lib/flow/`. Do not edit
|
||||||
|
substrates.
|
||||||
|
- **Architecture:** events/availability/capacity are Datalog facts + rules;
|
||||||
|
recurrence expands to occurrence facts within a window; a booking checks rules
|
||||||
|
then appends a `persist` event (idempotent, capacity-safe). Notifications are flows
|
||||||
|
that suspend on transport IO and retry on failure.
|
||||||
|
- **Determinism:** recurrence expansion + availability must be reproducible for a
|
||||||
|
fixed window + ruleset; capacity checks must be race-safe (no overbooking).
|
||||||
|
- **Commits:** one feature per commit. Progress log + tick boxes.
|
||||||
|
|
||||||
|
## Architecture sketch
|
||||||
|
|
||||||
|
```
|
||||||
|
Event + booking Result
|
||||||
|
event(id,start,rrule,capacity) {:booked | :full | :conflict} + reminders
|
||||||
|
│ ▲
|
||||||
|
▼ │
|
||||||
|
lib/events/calendar.sx lib/events/availability.sx
|
||||||
|
— event facts, recurrence (RRULE) — free/busy + capacity rules (Datalog)
|
||||||
|
— expand occurrences in window │
|
||||||
|
│ ▲
|
||||||
|
▼ │
|
||||||
|
lib/events/booking.sx lib/events/notify.sx (flow)
|
||||||
|
— transactional, capacity-safe — reminders / digests, retry on fail
|
||||||
|
— bookings → persist ledger — injected transport (email/push)
|
||||||
|
│ │
|
||||||
|
▼ ▼
|
||||||
|
lib/events/api.sx ── (events/schedule) (events/book) (events/agenda) ──────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
## Phase 1 — Calendar + recurrence
|
||||||
|
- [ ] `calendar.sx` — event facts, RRULE expansion in a window
|
||||||
|
- [ ] `availability.sx` — free/busy rules
|
||||||
|
- [ ] `api.sx` + tests + scoreboard + conformance.sh
|
||||||
|
|
||||||
|
## Phase 2 — Ticketing + booking
|
||||||
|
- [ ] capacity rules; transactional booking → `persist` (no overbooking)
|
||||||
|
- [ ] paid tickets compose with `commerce` order flow
|
||||||
|
- [ ] tests: capacity edge, double-book guard, conflict detection
|
||||||
|
|
||||||
|
## Phase 3 — Notification delivery (flow)
|
||||||
|
- [ ] `notify.sx` — reminder/digest flows over injected transport
|
||||||
|
- [ ] retry/backoff on transport failure (flow suspend/resume)
|
||||||
|
- [ ] tests: delivery success, retry path, idempotent re-send
|
||||||
|
- [ ] NOTE: shared with `feed/notify` — candidate for later extraction to a
|
||||||
|
`delivery-on-sx` once a second consumer is real
|
||||||
|
|
||||||
|
## Phase 4 — Federation
|
||||||
|
- [ ] cross-instance events (peer calendar) — trust-gated stub
|
||||||
|
- [ ] tests: federated agenda merge
|
||||||
|
|
||||||
|
## Progress log
|
||||||
|
(loop fills this in)
|
||||||
|
|
||||||
|
## Blockers
|
||||||
|
(loop fills this in)
|
||||||
100
plans/host-on-sx.md
Normal file
100
plans/host-on-sx.md
Normal file
@@ -0,0 +1,100 @@
|
|||||||
|
# host-on-sx: The SX web host — off Quart, onto the kernel (Dream-bound)
|
||||||
|
|
||||||
|
> **DRAFT outline.** The integration boundary that turns the subsystem libraries
|
||||||
|
> into running services, and the strangler path off Python/Quart. This is the
|
||||||
|
> dependency hub — it imports every subsystem. Decision recorded below: native
|
||||||
|
> server + SXTP **now**, `dream-on-sx` framework layer **next**, Python only at the
|
||||||
|
> external-integration edges.
|
||||||
|
|
||||||
|
The subsystems (`feed`, `search`, `acl`, `mod`, `flow`, `commerce`, `identity`,
|
||||||
|
`content`, `events`) are libraries. Something has to receive an HTTP request, route
|
||||||
|
it, call the right subsystem, and serialize the response. Today that's Python/Quart
|
||||||
|
— the one large non-SX component in the stack: separate runtime, deploy, and
|
||||||
|
failure mode. The goal is to move the web/host/domain layer onto the SX substrate
|
||||||
|
and retire Quart, **incrementally (strangler-fig), never big-bang.**
|
||||||
|
|
||||||
|
This is already underway: a native OCaml HTTP server is live in prod on
|
||||||
|
`sx.rose-ash.com` (~3ms cached, ~323 req/s, ~2MB RSS), `defhandler`/`defpage`
|
||||||
|
exist, and a partial **SXTP** protocol is specced. That is the unblocked near-term
|
||||||
|
host — no `ocaml-on-sx` dependency.
|
||||||
|
|
||||||
|
## Two layers, two timelines
|
||||||
|
|
||||||
|
1. **Now (unblocked): native server + SXTP adapter + SX handlers.** Route rose-ash
|
||||||
|
endpoints onto the SX host one at a time. Each migrated endpoint is an SX
|
||||||
|
handler dispatching to a subsystem; Quart proxies the rest until cut over.
|
||||||
|
2. **Next: `dream-on-sx` as the framework layer.** Dream gives Quart-grade
|
||||||
|
ergonomics — typed routing, middleware stacks, sessions, CSRF. It is gated on
|
||||||
|
`ocaml-on-sx` Phases 1–5 + minimal stdlib. **This plan is the concrete target
|
||||||
|
user that un-parks `dream-on-sx`** (see `plans/dream-on-sx.md`): "the subsystems
|
||||||
|
need an HTTP front door" is the real feature pulling Dream. Until then, do not
|
||||||
|
block migration on Dream — the native server is sufficient.
|
||||||
|
3. **Always: Python only at the edges.** External integrations — SumUp payments,
|
||||||
|
Ghost CMS, ActivityPub crypto, IPFS/Kubo — ride Python libraries today. They
|
||||||
|
stay as thin injected adapters (Python/FFI) behind subsystem interfaces until
|
||||||
|
native replacements exist. "Drop Quart" ≠ "drop every line of Python."
|
||||||
|
|
||||||
|
## Status (rolling)
|
||||||
|
|
||||||
|
`bash lib/host/conformance.sh` → **0/0** (not yet started)
|
||||||
|
|
||||||
|
## Ground rules
|
||||||
|
|
||||||
|
- **Scope:** `lib/host/**` and `plans/host-on-sx.md`. May **import** every subsystem
|
||||||
|
+ the kernel's server/SXTP surface. Do **not** edit `spec/`, `hosts/`, `shared/`,
|
||||||
|
or subsystem internals — wire to their public APIs only. Host-primitive / server
|
||||||
|
changes belong in `hosts/` (out of scope) → Blockers.
|
||||||
|
- **Architecture:** a route maps (method, path) → handler; a handler is an SX fn
|
||||||
|
`request -> response` that calls subsystem APIs; middleware is composed handlers
|
||||||
|
(auth via `identity`, permission via `acl`, mute via subsystem prefs). SXTP is the
|
||||||
|
wire format between host and subsystem-as-service.
|
||||||
|
- **Migration discipline:** each endpoint moved must be behavior-equivalent to its
|
||||||
|
Quart original (golden-response test before flip). Keep a migration ledger.
|
||||||
|
- **Commits:** one feature per commit. Progress log + tick boxes.
|
||||||
|
|
||||||
|
## Architecture sketch
|
||||||
|
|
||||||
|
```
|
||||||
|
HTTP request HTTP response
|
||||||
|
│ ▲
|
||||||
|
▼ │
|
||||||
|
native OCaml http server (prod) ──────► lib/host/router.sx
|
||||||
|
(hosts/ — out of scope) — (method,path) → handler
|
||||||
|
│ ▲
|
||||||
|
▼ │
|
||||||
|
lib/host/middleware.sx lib/host/handler.sx
|
||||||
|
— auth(identity) ∘ acl ∘ mute ∘ ... — request → subsystem call → response
|
||||||
|
│ ▲
|
||||||
|
▼ │
|
||||||
|
lib/host/sxtp.sx subsystem APIs (feed/search/commerce/…)
|
||||||
|
— wire format, host↔service — called via public interfaces
|
||||||
|
│
|
||||||
|
└── external edges: SumUp / Ghost / AP / IPFS → injected Python/FFI adapters
|
||||||
|
```
|
||||||
|
|
||||||
|
## Phase 1 — Router + handler + one real endpoint
|
||||||
|
- [ ] `router.sx` — route table, (method,path) match
|
||||||
|
- [ ] `handler.sx` — request/response model, subsystem dispatch
|
||||||
|
- [ ] migrate ONE read endpoint (e.g. a feed timeline) end-to-end, golden test
|
||||||
|
- [ ] `conformance.sh` + scoreboard
|
||||||
|
|
||||||
|
## Phase 2 — Middleware + SXTP
|
||||||
|
- [ ] `middleware.sx` — composable auth/acl/mute/error layers
|
||||||
|
- [ ] `sxtp.sx` — host↔subsystem wire format (align with existing spec)
|
||||||
|
- [ ] migrate a write endpoint (auth + permission + action)
|
||||||
|
|
||||||
|
## Phase 3 — Strangler migration ledger
|
||||||
|
- [ ] enumerate Quart endpoints; track migrated vs proxied
|
||||||
|
- [ ] golden-response harness vs the live Quart responses
|
||||||
|
- [ ] cut over a whole domain (smallest: `likes` or `relations`) as proof
|
||||||
|
|
||||||
|
## Phase 4 — Dream framework layer (gated)
|
||||||
|
- [ ] gate: `ocaml-on-sx` Phases 1–5 + minimal stdlib green
|
||||||
|
- [ ] adopt `dream-on-sx` routing/middleware/session ergonomics over the same handlers
|
||||||
|
- [ ] re-home external adapters as native where replacements land
|
||||||
|
|
||||||
|
## Progress log
|
||||||
|
(loop fills this in)
|
||||||
|
|
||||||
|
## Blockers
|
||||||
|
(loop fills this in)
|
||||||
84
plans/identity-on-sx.md
Normal file
84
plans/identity-on-sx.md
Normal file
@@ -0,0 +1,84 @@
|
|||||||
|
# identity-on-sx: OAuth2, sessions & membership on Erlang
|
||||||
|
|
||||||
|
> **DRAFT outline.** The identity core `acl-on-sx` assumes already exists. `acl`
|
||||||
|
> answers "may X do Y"; identity answers "who is X, and how did they prove it."
|
||||||
|
> Depends on `persist-on-sx` (grant/audit ledger). Pairs with `acl-on-sx`.
|
||||||
|
|
||||||
|
rose-ash's `account` domain is the OAuth2 authorization server every other app is
|
||||||
|
a client of: silent SSO, per-app first-party cookies, grant verification,
|
||||||
|
membership. Sessions and grants are **long-lived, concurrent, individually
|
||||||
|
addressable, and expire on their own** — that is the actor model. Erlang's
|
||||||
|
processes + mailboxes map cleanly: a session is a process, token issue/refresh/
|
||||||
|
revoke are messages, expiry is a process timeout, and SSO is one process answering
|
||||||
|
many apps.
|
||||||
|
|
||||||
|
End-state: an Erlang-on-SX layer with the OAuth2 authorization-code + silent
|
||||||
|
(`prompt=none`) flows as message protocols, a session/grant registry, token
|
||||||
|
lifecycle (issue/refresh/revoke/introspect), and membership state — all auditable
|
||||||
|
through the event log, all authorization questions delegated to `acl-on-sx`.
|
||||||
|
|
||||||
|
## Status (rolling)
|
||||||
|
|
||||||
|
`bash lib/identity/conformance.sh` → **0/0** (not yet started)
|
||||||
|
|
||||||
|
## Ground rules
|
||||||
|
|
||||||
|
- **Scope:** only `lib/identity/**` and `plans/identity-on-sx.md`. May **import**
|
||||||
|
from `lib/erlang/`, and (once they exist) `lib/persist/` + `lib/acl/`. Do not edit
|
||||||
|
substrates.
|
||||||
|
- **Architecture:** a session/grant is a process holding its own state; the
|
||||||
|
registry routes messages by subject/client id. Tokens are opaque + introspected,
|
||||||
|
not self-validating (revocation must be real). Authorization decisions are NOT
|
||||||
|
made here — `identity` proves identity, `acl` decides permission.
|
||||||
|
- **Security:** revocation is immediate (kill the process / tombstone the grant);
|
||||||
|
no decision relies on a token that outlived its grant. Negative answers are
|
||||||
|
explicit, never "absence of a yes."
|
||||||
|
- **Commits:** one feature per commit. Progress log + tick boxes.
|
||||||
|
|
||||||
|
## Architecture sketch
|
||||||
|
|
||||||
|
```
|
||||||
|
Auth request Token / session
|
||||||
|
(authorize client scope subject) {:access :refresh :expires :grant}
|
||||||
|
│ ▲
|
||||||
|
▼ │
|
||||||
|
lib/identity/oauth.sx lib/identity/token.sx
|
||||||
|
— authz-code + prompt=none flows — issue / refresh / revoke / introspect
|
||||||
|
— as Erlang message protocols — opaque tokens, grant-backed
|
||||||
|
│ ▲
|
||||||
|
▼ │
|
||||||
|
lib/identity/session.sx lib/identity/registry.sx
|
||||||
|
— session = process, expiry=timeout — route by subject/client; SSO fan-out
|
||||||
|
│ │
|
||||||
|
▼ ▼
|
||||||
|
lib/identity/api.sx ── (identity/login) (identity/grant?) (identity/revoke) ──┐
|
||||||
|
│ │
|
||||||
|
└──────── grant + audit events → persist ; permission? → acl ──────────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
## Phase 1 — Sessions + tokens
|
||||||
|
- [ ] `session.sx` — session process, create/lookup/expire
|
||||||
|
- [ ] `token.sx` — issue/introspect/revoke (opaque, grant-backed)
|
||||||
|
- [ ] `registry.sx` — route by subject/client
|
||||||
|
- [ ] `api.sx` + tests + scoreboard + conformance.sh
|
||||||
|
|
||||||
|
## Phase 2 — OAuth2 flows
|
||||||
|
- [ ] authorization-code flow as a message protocol
|
||||||
|
- [ ] refresh + rotation; revocation cascades to issued tokens
|
||||||
|
- [ ] tests: full code exchange, refresh, revoke-then-use (must fail)
|
||||||
|
|
||||||
|
## Phase 3 — Silent SSO + membership
|
||||||
|
- [ ] `prompt=none` cross-app login (one session, many clients)
|
||||||
|
- [ ] membership state + per-app grant projection
|
||||||
|
- [ ] grant verification delegated cache (mirror Redis-cache pattern)
|
||||||
|
|
||||||
|
## Phase 4 — Audit + federation
|
||||||
|
- [ ] every issue/refresh/revoke is a `persist` event; `(identity/audit subject)`
|
||||||
|
- [ ] federated identity (peer-asserted subject) — advisory, trust-gated stub
|
||||||
|
- [ ] tests: audit completeness, cross-instance subject mapping
|
||||||
|
|
||||||
|
## Progress log
|
||||||
|
(loop fills this in)
|
||||||
|
|
||||||
|
## Blockers
|
||||||
|
(loop fills this in)
|
||||||
119
plans/persist-on-sx.md
Normal file
119
plans/persist-on-sx.md
Normal file
@@ -0,0 +1,119 @@
|
|||||||
|
# persist-on-sx: Durable state on the SX kernel
|
||||||
|
|
||||||
|
> **DRAFT outline.** Foundation subsystem — the durable substrate the other five
|
||||||
|
> currently fake with in-memory mutable lists. Build this first.
|
||||||
|
>
|
||||||
|
> **"persist" = persistence / data store, NOT the shop.** The shop/commerce vertical
|
||||||
|
> is `commerce-on-sx`.
|
||||||
|
|
||||||
|
rose-ash needs durable state: every subsystem (feed log, flow store, mod audit,
|
||||||
|
search index, acl grants, sessions) today hand-rolls an in-memory structure that
|
||||||
|
vanishes on restart. `persist-on-sx` is the one durable substrate they share. It
|
||||||
|
lives directly on the SX kernel's IO-suspension primitives (`perform`/`cek-resume`
|
||||||
|
— the third CEK phase) so a read/write `perform`s and the kernel persists at the
|
||||||
|
boundary. Concrete storage backends are injected.
|
||||||
|
|
||||||
|
## Does it cover ALL persistence? No — and on purpose.
|
||||||
|
|
||||||
|
Event-sourcing-everything is a known trap (replay cost, event schema evolution,
|
||||||
|
awkward ad-hoc queries, 5MB images in a log). So persist owns the **durable
|
||||||
|
source-of-truth substrate**, exposed as **two facets over one backend protocol**,
|
||||||
|
with two things explicitly delegated out:
|
||||||
|
|
||||||
|
| Shape | Owner | Notes |
|
||||||
|
|-------|-------|-------|
|
||||||
|
| **Event streams** (append-only, history matters) | persist — **log facet** | feed activities, mod audit, order ledger, flow state, content edits |
|
||||||
|
| **Current-state values** (KV / document, no history) | persist — **kv facet** | profiles, stock counts, config, session blobs; also where projections materialize |
|
||||||
|
| **Snapshots / read models** (derived, queryable) | persist — projections → kv/log | rebuildable from the log; persisted so you don't replay to answer a query |
|
||||||
|
| **Blobs / large objects** (images, media) | **delegated** → content-addressed store (artdag/IPFS already) | persist stores the *reference/CID*, never the bytes |
|
||||||
|
| **Cache** (ephemeral, evictable) | **out of scope** | not persistence — different lifecycle (Redis-shaped) |
|
||||||
|
| **Ad-hoc relational query** | the subsystem, over a projected read model | the log is bad at "all orders by X in March"; project into a queryable kv/SQL backend |
|
||||||
|
|
||||||
|
So: persist is the **single durable substrate** for state that's either a stream of
|
||||||
|
changes or a current value — but it does **not** force everything into an event
|
||||||
|
log, it does **not** hold blobs (only their content-addressed refs), and it does
|
||||||
|
**not** do caching. Those boundaries are the whole point of calling it a substrate
|
||||||
|
rather than "the database."
|
||||||
|
|
||||||
|
End-state: `log` (append/read streams) + `kv` (get/put/delete by key) facets, an
|
||||||
|
injectable backend protocol (mem → file → Postgres → IPFS-ref), pure projections
|
||||||
|
with incremental snapshots, optimistic concurrency, and a subscription hook so
|
||||||
|
read models (feeds, indices, audit logs) update incrementally.
|
||||||
|
|
||||||
|
## Status (rolling)
|
||||||
|
|
||||||
|
`bash lib/persist/conformance.sh` → **0/0** (not yet started)
|
||||||
|
|
||||||
|
## Ground rules
|
||||||
|
|
||||||
|
- **Scope:** only `lib/persist/**` and `plans/persist-on-sx.md`. May **import** the
|
||||||
|
kernel's IO-suspension surface (`perform`, platform IO ops) — verify what's
|
||||||
|
exported first. Do not add host primitives; a missing durable IO op is a Blockers
|
||||||
|
entry (it belongs in `hosts/`, out of scope).
|
||||||
|
- **Architecture:** an event is `{:stream :seq :type :at :data}`; the log is an
|
||||||
|
ordered append-only vector; a projection is `(fold step seed events)`; a kv value
|
||||||
|
is `(get/put/delete key)`. Both facets sit on one injected backend
|
||||||
|
`{:append :read :kv-get :kv-put :snapshot-read :snapshot-write}`. The in-memory
|
||||||
|
backend is the test default; real backends wire in unchanged.
|
||||||
|
- **Determinism:** replay is pure — same log → same state, always. No clocks or
|
||||||
|
randomness inside projections; time lives on the event.
|
||||||
|
- **Blobs:** store the content-address/CID and metadata; never the bytes. The blob
|
||||||
|
backend is a separate injected dependency.
|
||||||
|
- **Commits:** one feature per commit. Progress log + tick boxes.
|
||||||
|
|
||||||
|
## Architecture sketch
|
||||||
|
|
||||||
|
```
|
||||||
|
Command / write Read model / value
|
||||||
|
(append stream type data) (project stream step seed)
|
||||||
|
(kv-put key value) (kv-get key)
|
||||||
|
│ ▲
|
||||||
|
▼ │
|
||||||
|
lib/persist/event.sx lib/persist/project.sx
|
||||||
|
— {:stream :seq :type :at :data} — fold step seed; incremental from snapshot
|
||||||
|
│ ▲
|
||||||
|
▼ │
|
||||||
|
lib/persist/log.sx lib/persist/kv.sx lib/persist/snapshot.sx
|
||||||
|
— append/read — get/put/delete — checkpoint; replay = snapshot + tail
|
||||||
|
— optimistic seq — current-state
|
||||||
|
│ │ ▲
|
||||||
|
└──────────────────┴── (perform → backend) ───┘
|
||||||
|
│
|
||||||
|
lib/persist/backend.sx lib/persist/api.sx
|
||||||
|
— injected protocol — (persist/append) (persist/project)
|
||||||
|
— mem | file | pg | ipfs-ref — (persist/kv-get/put) (persist/subscribe)
|
||||||
|
│
|
||||||
|
└── blobs → content-addressed store (artdag/IPFS), by reference only
|
||||||
|
```
|
||||||
|
|
||||||
|
## Phase 1 — Log + kv + in-memory backend
|
||||||
|
- [ ] `event.sx` — event record, stream/seq helpers
|
||||||
|
- [ ] `backend.sx` — injectable protocol + in-memory impl (log + kv)
|
||||||
|
- [ ] `log.sx` — `append` (optimistic seq), `read`, `read-from`
|
||||||
|
- [ ] `kv.sx` — `get`/`put`/`delete` current-state
|
||||||
|
- [ ] `api.sx` + tests + scoreboard + conformance.sh
|
||||||
|
|
||||||
|
## Phase 2 — Projections + subscriptions
|
||||||
|
- [ ] `project.sx` — `(project stream step seed)`, incremental fold
|
||||||
|
- [ ] subscription hook — projection / kv read model re-runs on append
|
||||||
|
- [ ] concurrency conflict surfaced as a real result, not a crash
|
||||||
|
|
||||||
|
## Phase 3 — Snapshots + replay
|
||||||
|
- [ ] `snapshot.sx` — checkpoint a projection; replay = snapshot + tail
|
||||||
|
- [ ] compaction policy; replay-determinism tests
|
||||||
|
|
||||||
|
## Phase 4 — Durable backends via kernel IO
|
||||||
|
- [ ] file/log backend driven through `perform` (IO-suspension boundary)
|
||||||
|
- [ ] blob backend interface (store ref/CID; bytes live in artdag/IPFS)
|
||||||
|
- [ ] crash/restart replay test (mock IO platform)
|
||||||
|
- [ ] migration notes for swapping mem → durable under a live subsystem
|
||||||
|
|
||||||
|
## Consumers (post-foundation, not in scope here)
|
||||||
|
feed/-log, flow store, mod/audit, search index, acl grants, identity sessions all
|
||||||
|
become `persist` log or kv. Track each migration in that subsystem's plan.
|
||||||
|
|
||||||
|
## Progress log
|
||||||
|
(loop fills this in)
|
||||||
|
|
||||||
|
## Blockers
|
||||||
|
(loop fills this in)
|
||||||
Reference in New Issue
Block a user