diff --git a/lib/persist/conformance.sh b/lib/persist/conformance.sh index d7ebd032..bb83f151 100755 --- a/lib/persist/conformance.sh +++ b/lib/persist/conformance.sh @@ -13,7 +13,7 @@ if [ ! -x "$SX_SERVER" ]; then exit 1 fi -SUITES=(event log kv project subscribe concurrency snapshot compaction durable blob view cas catalog query batch upcast recovery) +SUITES=(event log kv project subscribe concurrency snapshot compaction durable blob view cas catalog query batch upcast idempotency recovery) OUT_JSON="lib/persist/scoreboard.json" OUT_MD="lib/persist/scoreboard.md" @@ -42,6 +42,7 @@ run_suite() { (load "lib/persist/query.sx") (load "lib/persist/batch.sx") (load "lib/persist/upcast.sx") +(load "lib/persist/idempotency.sx") (load "lib/persist/subscribe.sx") (load "lib/persist/api.sx") (epoch 2) diff --git a/lib/persist/idempotency.sx b/lib/persist/idempotency.sx new file mode 100644 index 00000000..c689aa2e --- /dev/null +++ b/lib/persist/idempotency.sx @@ -0,0 +1,28 @@ +; persist/idempotency — exactly-once append under retries. A command retried +; after a network blip must not append its event twice. The caller supplies an +; idempotency key; the first append for that (stream, key) stores the event and +; remembers the key in the kv facet; a repeat returns the SAME event without +; appending. Because the marker lives in kv, idempotency holds across a restart +; too. Keyed per stream. Requires: lib/persist/log.sx, lib/persist/kv.sx. + +(define persist/idem-key (fn (stream key) (str "idem/" stream "/" key))) + +; true if an append-once has already been recorded for (stream, key) +(define + persist/seen? + (fn (b stream key) (persist/kv-has? b (persist/idem-key stream key)))) + +; append at most once per (stream, key). Returns the stored event either way — +; freshly appended on first use, the remembered one on a repeat. +(define + persist/append-once + (fn + (b stream key type at data) + (let + ((k (persist/idem-key stream key))) + (if + (persist/kv-has? b k) + (persist/kv-get b k) + (let + ((ev (persist/append b stream type at data))) + (begin (persist/kv-put b k ev) ev)))))) diff --git a/lib/persist/scoreboard.json b/lib/persist/scoreboard.json index d236f68c..b9888053 100644 --- a/lib/persist/scoreboard.json +++ b/lib/persist/scoreboard.json @@ -16,9 +16,10 @@ "query": {"pass": 9, "fail": 0}, "batch": {"pass": 10, "fail": 0}, "upcast": {"pass": 9, "fail": 0}, + "idempotency": {"pass": 9, "fail": 0}, "recovery": {"pass": 6, "fail": 0} }, - "total_pass": 171, + "total_pass": 180, "total_fail": 0, - "total": 171 + "total": 180 } diff --git a/lib/persist/scoreboard.md b/lib/persist/scoreboard.md index 88f3b77b..c8f05201 100644 --- a/lib/persist/scoreboard.md +++ b/lib/persist/scoreboard.md @@ -20,5 +20,6 @@ _Generated by `lib/persist/conformance.sh`_ | query | 9 | 0 | 9 | | batch | 10 | 0 | 10 | | upcast | 9 | 0 | 9 | +| idempotency | 9 | 0 | 9 | | recovery | 6 | 0 | 6 | -| **Total** | **171** | **0** | **171** | +| **Total** | **180** | **0** | **180** | diff --git a/lib/persist/tests/idempotency.sx b/lib/persist/tests/idempotency.sx new file mode 100644 index 00000000..a1ee8e0a --- /dev/null +++ b/lib/persist/tests/idempotency.sx @@ -0,0 +1,92 @@ +; Extension — exactly-once append under retries. + +(persist-test + "seen? false before first append" + (persist/seen? (persist/open) "orders" "cmd-1") + false) +(persist-test + "append-once appends on first use" + (let + ((b (persist/open))) + (begin + (persist/append-once b "orders" "cmd-1" "placed" 0 {}) + (persist/count b "orders"))) + 1) +(persist-test + "seen? true after first append" + (let + ((b (persist/open))) + (begin + (persist/append-once b "orders" "cmd-1" "placed" 0 {}) + (persist/seen? b "orders" "cmd-1"))) + true) +(persist-test + "repeat with same key does not append again" + (let + ((b (persist/open))) + (begin + (persist/append-once b "orders" "cmd-1" "placed" 0 {}) + (persist/append-once b "orders" "cmd-1" "placed" 0 {}) + (persist/append-once b "orders" "cmd-1" "placed" 0 {}) + (persist/count b "orders"))) + 1) +(persist-test + "repeat returns the same event (same seq)" + (let + ((b (persist/open))) + (let + ((e1 (persist/append-once b "orders" "cmd-1" "placed" 0 {}))) + (persist/event-seq + (persist/append-once b "orders" "cmd-1" "placed" 0 {})))) + 1) +(persist-test + "different keys append separately" + (let + ((b (persist/open))) + (begin + (persist/append-once b "orders" "cmd-1" "placed" 0 {}) + (persist/append-once b "orders" "cmd-2" "placed" 0 {}) + (persist/count b "orders"))) + 2) +(persist-test + "idempotency is per-stream" + (let + ((b (persist/open))) + (begin + (persist/append-once b "a" "cmd-1" "x" 0 {}) + (persist/append-once b "b" "cmd-1" "x" 0 {}) + (list (persist/count b "a") (persist/count b "b")))) + (list 1 1)) +(persist-test + "stored data is preserved on first append" + (let + ((b (persist/open))) + (get + (persist/event-data + (persist/append-once b "s" "k" "x" 0 {:n 9})) + :n)) + 9) +(persist-test + "idempotency survives restart on the durable backend" + (let + ((disk (persist/mem-backend))) + (begin + (persist/append-once + (persist/mock-durable disk) + "orders" + "cmd-1" + "placed" + 0 + {}) + (let + ((db2 (persist/mock-durable disk))) + (begin + (persist/append-once + db2 + "orders" + "cmd-1" + "placed" + 0 + {}) + (persist/count db2 "orders"))))) + 1) diff --git a/plans/persist-on-sx.md b/plans/persist-on-sx.md index 15d8087a..e4e5e852 100644 --- a/plans/persist-on-sx.md +++ b/plans/persist-on-sx.md @@ -42,7 +42,7 @@ read models (feeds, indices, audit logs) update incrementally. ## Status (rolling) -`bash lib/persist/conformance.sh` → **171/171** (Phases 1–4 complete + extensions) +`bash lib/persist/conformance.sh` → **180/180** (Phases 1–4 complete + extensions) ## Ground rules @@ -173,11 +173,19 @@ over an in-process disk (the mock-IO harness). `upcast-data` helper merges new `:data` fields. Addresses the schema-evolution trap without rewriting history. +- [x] `idempotency.sx` — exactly-once append under retries: `persist/append-once` + keyed by a caller idempotency key (per stream), returning the same event on a + repeat. Marker lives in kv, so idempotency holds across restart. `seen?` check. + ## Consumers (post-foundation, not in scope here) feed/-log, flow store, mod/audit, search index, acl grants, identity sessions all become `persist` log or kv. Track each migration in that subsystem's plan. ## Progress log +- **Ext: exactly-once append (180/180).** `idempotency.sx` — + `persist/append-once` appends at most once per (stream, idempotency key), + returning the same event on a repeat; the marker lives in kv so it survives + restart (verified on durable). `persist/seen?` check. 9 tests. - **Ext: event schema evolution (171/171).** `upcast.sx` — per-type pure `(event -> event)` upcasters in an immutable registry; `read-upcast`/ `project-upcast` lift legacy events to the current shape on read so