From ef38b24110ce4e94fc6f44cffd2f2079024fd675 Mon Sep 17 00:00:00 2001 From: giles Date: Sun, 7 Jun 2026 01:19:15 +0000 Subject: [PATCH] content: durable CRDT replication (crdt-store) + 14 tests (277/277) Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/content/conformance.sh | 3 +- lib/content/crdt-store.sx | 71 ++++++++++++++++ lib/content/scoreboard.json | 5 +- lib/content/scoreboard.md | 3 +- lib/content/tests/crdt-store.sx | 139 ++++++++++++++++++++++++++++++++ plans/content-on-sx.md | 10 ++- 6 files changed, 226 insertions(+), 5 deletions(-) create mode 100644 lib/content/crdt-store.sx create mode 100644 lib/content/tests/crdt-store.sx diff --git a/lib/content/conformance.sh b/lib/content/conformance.sh index 5504dbbd..768d833c 100755 --- a/lib/content/conformance.sh +++ b/lib/content/conformance.sh @@ -15,7 +15,7 @@ if [ ! -x "$SX_SERVER" ]; then fi fi -SUITES=(block doc render api markdown store crdt sync fed) +SUITES=(block doc render api markdown store crdt crdt-store sync fed) OUT_JSON="lib/content/scoreboard.json" OUT_MD="lib/content/scoreboard.md" @@ -45,6 +45,7 @@ run_suite() { (load "lib/content/markdown.sx") (load "lib/content/store.sx") (load "lib/content/crdt.sx") +(load "lib/content/crdt-store.sx") (load "lib/content/sync.sx") (load "lib/content/fed.sx") (epoch 2) diff --git a/lib/content/crdt-store.sx b/lib/content/crdt-store.sx new file mode 100644 index 00000000..e9f8e57d --- /dev/null +++ b/lib/content/crdt-store.sx @@ -0,0 +1,71 @@ +;; content-on-sx — durable collaborative replication: CRDT ops on persist. +;; +;; Each replica appends its CRDT ops to its own persist stream +;; (crdt::). Any node reconstructs the converged document by +;; replaying every replica's log into a CvRDT state and merging them. Because +;; the merge is a join and crdt-apply is order/duplicate-insensitive, the +;; converged result is identical regardless of replica order or re-delivery — +;; the durable log + CRDT give offline-capable, eventually-consistent editing. +;; +;; Requires (loaded by harness): crdt.sx (+ deps) and persist +;; (event/backend/log/kv/api). Backend `b` injected via (persist/open). + +(define crdt/-stream (fn (doc-id replica) (str "crdt:" doc-id ":" replica))) + +;; ── commit ops to a replica's durable log ── +(define + crdt/commit! + (fn + (b doc-id replica op at) + (persist/append b (crdt/-stream doc-id replica) (get op :op) at op))) + +(define + crdt/commit-all! + (fn + (b doc-id replica ops at) + (if + (= (len ops) 0) + nil + (begin + (crdt/commit! b doc-id replica (first ops) at) + (crdt/commit-all! b doc-id replica (rest ops) at))))) + +;; ── read a replica's log ── +(define + crdt/log + (fn (b doc-id replica) (persist/read b (crdt/-stream doc-id replica)))) + +(define + crdt/replica-ops + (fn + (b doc-id replica) + (map (fn (ev) (persist/event-data ev)) (crdt/log b doc-id replica)))) + +(define + crdt/replica-version + (fn (b doc-id replica) (persist/last-seq b (crdt/-stream doc-id replica)))) + +;; ── replay one replica's log into a CvRDT state ── +(define + crdt/replay + (fn + (b doc-id replica) + (crdt-apply-all (crdt-empty) (crdt/replica-ops b doc-id replica)))) + +;; ── converge: merge every replica's replayed state ── +(define + crdt/converge + (fn + (b doc-id replicas) + (crdt-merge-all (map (fn (r) (crdt/replay b doc-id r)) replicas)))) + +;; ── converged, materialised document ── +(define + crdt/document + (fn + (b doc-id replicas) + (crdt-materialize doc-id (crdt/converge b doc-id replicas)))) + +(define + crdt/order + (fn (b doc-id replicas) (crdt-order (crdt/converge b doc-id replicas)))) diff --git a/lib/content/scoreboard.json b/lib/content/scoreboard.json index 7b982ffd..ba8f3941 100644 --- a/lib/content/scoreboard.json +++ b/lib/content/scoreboard.json @@ -7,10 +7,11 @@ "markdown": {"pass": 20, "fail": 0}, "store": {"pass": 29, "fail": 0}, "crdt": {"pass": 34, "fail": 0}, + "crdt-store": {"pass": 14, "fail": 0}, "sync": {"pass": 14, "fail": 0}, "fed": {"pass": 20, "fail": 0} }, - "total_pass": 263, + "total_pass": 277, "total_fail": 0, - "total": 263 + "total": 277 } diff --git a/lib/content/scoreboard.md b/lib/content/scoreboard.md index 6ae10bb5..5249a8bb 100644 --- a/lib/content/scoreboard.md +++ b/lib/content/scoreboard.md @@ -11,6 +11,7 @@ _Generated by `lib/content/conformance.sh`_ | markdown | 20 | 0 | 20 | | store | 29 | 0 | 29 | | crdt | 34 | 0 | 34 | +| crdt-store | 14 | 0 | 14 | | sync | 14 | 0 | 14 | | fed | 20 | 0 | 20 | -| **Total** | **263** | **0** | **263** | +| **Total** | **277** | **0** | **277** | diff --git a/lib/content/tests/crdt-store.sx b/lib/content/tests/crdt-store.sx new file mode 100644 index 00000000..7ad7d12a --- /dev/null +++ b/lib/content/tests/crdt-store.sx @@ -0,0 +1,139 @@ +;; Extension — durable collaborative replication (CRDT ops on persist). +;; Replicas log independently; converge merges the logs deterministically. + +(st-bootstrap-classes!) +(content-bootstrap-blocks!) +(content-bootstrap-doc!) +(content-bootstrap-render!) + +(define same? (fn (a b) (= (get a :elements) (get b :elements)))) +(define B (persist/open)) + +;; replica "a" (origin): inserts h, p +(crdt/commit! + B + "doc" + "a" + (crdt-op-insert + "h" + "heading" + (crdt-pos 1 0) + (list (list "level" 1) (list "text" "T")) + 1 + 1) + 1) +(crdt/commit! + B + "doc" + "a" + (crdt-op-insert + "p" + "text" + (crdt-pos 2 0) + (list (list "text" "Body")) + 1 + 1) + 1) + +;; replica "b" (concurrent): edits p, inserts x +(crdt/commit-all! + B + "doc" + "b" + (list + (crdt-op-update "p" "text" "Edited" 5 2) + (crdt-op-insert + "x" + "text" + (crdt-pos 3 0) + (list (list "text" "X")) + 6 + 2)) + 5) + +;; ── durability ── +(content-test + "replica a version" + (crdt/replica-version B "doc" "a") + 2) +(content-test + "replica b version" + (crdt/replica-version B "doc" "b") + 2) +(content-test + "replica a ops len" + (len (crdt/replica-ops B "doc" "a")) + 2) + +;; ── single-replica replay ── +(content-test + "replay a order" + (crdt-order (crdt/replay B "doc" "a")) + (list "h" "p")) +(content-test + "replay a == apply-all" + (same? + (crdt/replay B "doc" "a") + (crdt-apply-all (crdt-empty) (crdt/replica-ops B "doc" "a"))) + true) + +;; ── converge ── +(content-test + "converge order" + (crdt/order B "doc" (list "a" "b")) + (list "h" "p" "x")) +(content-test + "converge replica-order-independent" + (same? + (crdt/converge B "doc" (list "a" "b")) + (crdt/converge B "doc" (list "b" "a"))) + true) +(content-test + "converge LWW p edited" + (str + (blk-send (doc-find (crdt/document B "doc" (list "a" "b")) "p") "text")) + "Edited") +(content-test + "converged document render" + (asHTML (crdt/document B "doc" (list "a" "b"))) + "

T

Edited

X

") + +;; ── duplicate delivery is idempotent ── +(crdt/commit! + B + "doc" + "a" + (crdt-op-insert + "p" + "text" + (crdt-pos 2 0) + (list (list "text" "Body")) + 1 + 1) + 1) +(content-test + "duplicate op no effect on converge" + (crdt/order B "doc" (list "a" "b")) + (list "h" "p" "x")) +(content-test + "duplicate keeps LWW value" + (str + (blk-send (doc-find (crdt/document B "doc" (list "a" "b")) "p") "text")) + "Edited") + +;; ── new op on a replica is reflected after re-converge ── +(crdt/commit! B "doc" "b" (crdt-op-delete "h") 9) +(content-test + "delete reflected after reconverge" + (crdt/order B "doc" (list "a" "b")) + (list "p" "x")) + +;; ── isolation: unknown doc converges to empty ── +(content-test + "unknown doc empty" + (crdt/order B "other" (list "a" "b")) + (list)) +(content-test + "unknown replica empty ops" + (len (crdt/replica-ops B "doc" "zzz")) + 0) diff --git a/plans/content-on-sx.md b/plans/content-on-sx.md index 429771d7..8c526778 100644 --- a/plans/content-on-sx.md +++ b/plans/content-on-sx.md @@ -19,7 +19,7 @@ injected adapter, not core. ## Status (rolling) -`bash lib/content/conformance.sh` → **263/263** (Phases 1–4 COMPLETE + extensions: HTML/SX escaping, Markdown render) +`bash lib/content/conformance.sh` → **277/277** (Phases 1–4 COMPLETE + extensions: HTML/SX escaping, Markdown, durable CRDT replication) ## Ground rules @@ -79,9 +79,17 @@ lib/content/api.sx ── (content/edit) (content/render) (content/history) ─ - [x] HTML escaping at the render boundary (`String>>htmlEscaped`: & < > ") - [x] asSx wire string-escaping (`String>>sxEscaped`: \ and " in SX literals) - [x] Markdown render mode (`asMarkdown:` / `content/render doc "md"`) +- [x] durable CRDT replication (`crdt-store.sx`: ops on persist, replay + converge) ## Progress log +- 2026-06-07 — Extension: durable CRDT replication (`crdt-store.sx`), uniting + Phase 2 (persist) + Phase 3 (CvRDT). Each replica appends its CRDT ops to its + own stream (`crdt::`); `crdt/replay` folds one log into a state, + `crdt/converge` merges every replica's replayed state, `crdt/document` / + `crdt/order` materialise. Converged result is identical regardless of replica + order or duplicate delivery (join + idempotent apply) → offline-capable, + eventually-consistent editing. 14 tests; suite 277/277. - 2026-06-07 — Extension: Markdown render mode (`markdown.sx`). Third boundary format alongside asHTML/asSx via the same polymorphic dispatch; blocks answer `asMarkdown: nl` (boundary supplies the newline — this Smalltalk dialect has