From b0874b1282e27188b10ba2cc3b7a433bf9164a96 Mon Sep 17 00:00:00 2001 From: giles Date: Sat, 6 Jun 2026 18:39:41 +0000 Subject: [PATCH] =?UTF-8?q?persist:=20snapshots=20=E2=80=94=20checkpoint?= =?UTF-8?q?=20+=20replay=20=3D=20snapshot=20+=20tail=20+=2011=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit snapshot.sx: snapshot is a projection state {:value :seq} stored in kv under snapshot/. persist/checkpoint replays and saves; persist/replay folds only the tail after the snapshot. Tests assert snapshot+tail == full replay both ways + determinism. 65/65. Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/persist/conformance.sh | 3 +- lib/persist/scoreboard.json | 7 ++- lib/persist/scoreboard.md | 3 +- lib/persist/snapshot.sx | 40 ++++++++++++ lib/persist/tests/snapshot.sx | 114 ++++++++++++++++++++++++++++++++++ plans/persist-on-sx.md | 7 ++- 6 files changed, 168 insertions(+), 6 deletions(-) create mode 100644 lib/persist/snapshot.sx create mode 100644 lib/persist/tests/snapshot.sx diff --git a/lib/persist/conformance.sh b/lib/persist/conformance.sh index d930b4e3..b03e1b02 100755 --- a/lib/persist/conformance.sh +++ b/lib/persist/conformance.sh @@ -13,7 +13,7 @@ if [ ! -x "$SX_SERVER" ]; then exit 1 fi -SUITES=(event log kv project subscribe concurrency) +SUITES=(event log kv project subscribe concurrency snapshot) OUT_JSON="lib/persist/scoreboard.json" OUT_MD="lib/persist/scoreboard.md" @@ -33,6 +33,7 @@ run_suite() { (load "lib/persist/kv.sx") (load "lib/persist/project.sx") (load "lib/persist/concurrency.sx") +(load "lib/persist/snapshot.sx") (load "lib/persist/subscribe.sx") (load "lib/persist/api.sx") (epoch 2) diff --git a/lib/persist/scoreboard.json b/lib/persist/scoreboard.json index 65674ec4..2befd965 100644 --- a/lib/persist/scoreboard.json +++ b/lib/persist/scoreboard.json @@ -5,9 +5,10 @@ "kv": {"pass": 13, "fail": 0}, "project": {"pass": 9, "fail": 0}, "subscribe": {"pass": 9, "fail": 0}, - "concurrency": {"pass": 8, "fail": 0} + "concurrency": {"pass": 8, "fail": 0}, + "snapshot": {"pass": 11, "fail": 0} }, - "total_pass": 54, + "total_pass": 65, "total_fail": 0, - "total": 54 + "total": 65 } diff --git a/lib/persist/scoreboard.md b/lib/persist/scoreboard.md index 6b922c07..4bb832d0 100644 --- a/lib/persist/scoreboard.md +++ b/lib/persist/scoreboard.md @@ -10,4 +10,5 @@ _Generated by `lib/persist/conformance.sh`_ | project | 9 | 0 | 9 | | subscribe | 9 | 0 | 9 | | concurrency | 8 | 0 | 8 | -| **Total** | **54** | **0** | **54** | +| snapshot | 11 | 0 | 11 | +| **Total** | **65** | **0** | **65** | diff --git a/lib/persist/snapshot.sx b/lib/persist/snapshot.sx new file mode 100644 index 00000000..1b138062 --- /dev/null +++ b/lib/persist/snapshot.sx @@ -0,0 +1,40 @@ +; persist/snapshot — checkpoint a projection so a read model rebuilds as +; snapshot + tail instead of a full replay. A snapshot is just a projection +; state {:value :seq} stored in the kv facet under a namespaced key. The +; headline property (tested both ways): snapshot + tail == full replay. Replay +; is pure — it depends only on the stored snapshot and the log tail, never a +; clock. Requires: lib/persist/project.sx, lib/persist/kv.sx. + +(define persist/snapshot-key (fn (name) (str "snapshot/" name))) + +; load the stored snapshot for name, or a fresh {:value seed :seq 0} if none +(define + persist/snapshot-load + (fn + (b name seed) + (persist/kv-get-or b (persist/snapshot-key name) {:value seed :seq 0}))) + +; store a projection state as the snapshot for name; returns the state +(define + persist/snapshot-save + (fn (b name state) (persist/kv-put b (persist/snapshot-key name) state))) + +(define + persist/snapshot-exists? + (fn (b name) (persist/kv-has? b (persist/snapshot-key name)))) + +; replay = snapshot + tail: load the snapshot then fold events after it +(define + persist/replay + (fn + (b stream name step seed) + (persist/project-resume b stream step (persist/snapshot-load b name seed)))) + +; replay then persist the new snapshot; returns the updated state +(define + persist/checkpoint + (fn + (b stream name step seed) + (let + ((state (persist/replay b stream name step seed))) + (begin (persist/snapshot-save b name state) state)))) diff --git a/lib/persist/tests/snapshot.sx b/lib/persist/tests/snapshot.sx new file mode 100644 index 00000000..d81fc170 --- /dev/null +++ b/lib/persist/tests/snapshot.sx @@ -0,0 +1,114 @@ +; Phase 3 — snapshots + replay. Headline: snapshot + tail == full replay. + +(define snap-count (fn (acc e) (+ acc 1))) + +(persist-test + "no snapshot loads fresh seed state" + (persist/snapshot-load (persist/open) "feed" 0) + {:value 0 :seq 0}) +(persist-test + "snapshot-exists? false initially" + (persist/snapshot-exists? (persist/open) "feed") + false) +(persist-test + "checkpoint stores a snapshot" + (let + ((b (persist/open))) + (begin + (persist/append b "s" "x" 0 {}) + (persist/checkpoint b "s" "snap" snap-count 0) + (persist/snapshot-exists? b "snap"))) + true) +(persist-test + "checkpoint value equals full projection" + (let + ((b (persist/open))) + (begin + (persist/append b "s" "x" 0 {}) + (persist/append b "s" "x" 0 {}) + (persist/append b "s" "x" 0 {}) + (persist/project-value + (persist/checkpoint b "s" "snap" snap-count 0)))) + 3) +(persist-test + "checkpoint records the last seq" + (let + ((b (persist/open))) + (begin + (persist/append b "s" "x" 0 {}) + (persist/append b "s" "x" 0 {}) + (persist/project-seq + (persist/checkpoint b "s" "snap" snap-count 0)))) + 2) +(persist-test + "replay after checkpoint only folds the tail" + (let + ((b (persist/open))) + (begin + (persist/append b "s" "x" 0 {}) + (persist/append b "s" "x" 0 {}) + (persist/checkpoint b "s" "snap" snap-count 0) + (persist/append b "s" "x" 0 {}) + (persist/project-value + (persist/replay b "s" "snap" snap-count 0)))) + 3) +(persist-test + "snapshot + tail == full replay (value)" + (let + ((b (persist/open))) + (begin + (persist/append b "s" "x" 0 {}) + (persist/append b "s" "x" 0 {}) + (persist/checkpoint b "s" "snap" snap-count 0) + (persist/append b "s" "x" 0 {}) + (persist/append b "s" "x" 0 {}) + (equal? + (persist/project-value + (persist/replay b "s" "snap" snap-count 0)) + (persist/project-fold b "s" snap-count 0)))) + true) +(persist-test + "snapshot + tail == full replay (whole state)" + (let + ((b (persist/open))) + (begin + (persist/append b "s" "x" 0 {}) + (persist/checkpoint b "s" "snap" snap-count 0) + (persist/append b "s" "x" 0 {}) + (persist/append b "s" "x" 0 {}) + (equal? + (persist/replay b "s" "snap" snap-count 0) + (persist/project b "s" snap-count 0)))) + true) +(persist-test + "replay determinism: two replays from same snapshot agree" + (let + ((b (persist/open))) + (begin + (persist/append b "s" "x" 0 {}) + (persist/checkpoint b "s" "snap" snap-count 0) + (persist/append b "s" "x" 0 {}) + (equal? + (persist/replay b "s" "snap" snap-count 0) + (persist/replay b "s" "snap" snap-count 0)))) + true) +(persist-test + "re-checkpoint advances the snapshot" + (let + ((b (persist/open))) + (begin + (persist/append b "s" "x" 0 {}) + (persist/checkpoint b "s" "snap" snap-count 0) + (persist/append b "s" "x" 0 {}) + (persist/checkpoint b "s" "snap" snap-count 0) + (persist/project-seq (persist/snapshot-load b "snap" 0)))) + 2) +(persist-test + "snapshots are keyed independently" + (let + ((b (persist/open))) + (begin + (persist/append b "s" "x" 0 {}) + (persist/checkpoint b "s" "a" snap-count 0) + (persist/snapshot-exists? b "b"))) + false) diff --git a/plans/persist-on-sx.md b/plans/persist-on-sx.md index b14fc32f..9917f527 100644 --- a/plans/persist-on-sx.md +++ b/plans/persist-on-sx.md @@ -99,7 +99,7 @@ lib/persist/backend.sx lib/persist/api.sx - [x] concurrency conflict surfaced as a real result, not a crash ## Phase 3 — Snapshots + replay -- [ ] `snapshot.sx` — checkpoint a projection; replay = snapshot + tail +- [x] `snapshot.sx` — checkpoint a projection; replay = snapshot + tail - [ ] compaction policy; replay-determinism tests ## Phase 4 — Durable backends via kernel IO @@ -113,6 +113,11 @@ feed/-log, flow store, mod/audit, search index, acl grants, identity sessions al become `persist` log or kv. Track each migration in that subsystem's plan. ## Progress log +- **Phase 3a (65/65).** `snapshot.sx` — a snapshot is a projection state + `{:value :seq}` stored in the kv facet under `snapshot/`. + `persist/checkpoint` replays + saves; `persist/replay` = snapshot + tail. + 11 tests assert the headline both ways: snapshot+tail == full replay (value + and whole state), plus replay determinism. - **Phase 2c (54/54) — Phase 2 complete.** `concurrency.sx` — optimistic concurrency: `persist/append-expect b stream expected ...` refuses the append if the stream advanced past `expected`, returning a conflict VALUE