From 6a246039b5c91569a05df4abf5b77552914c76ad Mon Sep 17 00:00:00 2001 From: giles Date: Sat, 6 Jun 2026 23:51:46 +0000 Subject: [PATCH 01/49] content: typed block objects on smalltalk + 38 tests Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/content/block.sx | 163 ++++++++++++++++++++++++++++++++++++ lib/content/conformance.sh | 114 +++++++++++++++++++++++++ lib/content/scoreboard.json | 8 ++ lib/content/scoreboard.md | 8 ++ lib/content/tests/block.sx | 75 +++++++++++++++++ plans/content-on-sx.md | 18 +++- 6 files changed, 382 insertions(+), 4 deletions(-) create mode 100644 lib/content/block.sx create mode 100755 lib/content/conformance.sh create mode 100644 lib/content/scoreboard.json create mode 100644 lib/content/scoreboard.md create mode 100644 lib/content/tests/block.sx diff --git a/lib/content/block.sx b/lib/content/block.sx new file mode 100644 index 00000000..d4aabf06 --- /dev/null +++ b/lib/content/block.sx @@ -0,0 +1,163 @@ +;; content-on-sx — typed block objects on Smalltalk-on-SX. +;; +;; A block is a Smalltalk instance. Behaviour (type tag, later render) is a +;; message, not a property switch. Fields are immutable: blk-set / mk-* build a +;; fresh instance via the functional st-iv-set!, so old versions are never +;; clobbered (history-safe for the persist op log and CRDT merge). +;; +;; Hierarchy: +;; CtBlock (id) +;; CtText (text) +;; CtHeading (level) +;; CtCode (language) +;; CtQuote (cite) +;; CtImage (src alt) +;; CtEmbed (url provider) +;; CtDivider +;; CtList (ordered items) + +(define + ct-def-method! + (fn (cls sel src) (st-class-add-method! cls sel (st-parse-method src)))) + +;; Register the block hierarchy in the Smalltalk class table. Call AFTER +;; st-bootstrap-classes! (which resets the table). Idempotent. +(define + content-bootstrap-blocks! + (fn + () + (begin + (st-class-define! "CtBlock" "Object" (list "id")) + (ct-def-method! "CtBlock" "id" "id ^ id") + (ct-def-method! "CtBlock" "type" "type ^ #block") + (ct-def-method! "CtBlock" "isBlock" "isBlock ^ true") + (st-class-define! "CtText" "CtBlock" (list "text")) + (ct-def-method! "CtText" "text" "text ^ text") + (ct-def-method! "CtText" "type" "type ^ #text") + (st-class-define! "CtHeading" "CtText" (list "level")) + (ct-def-method! "CtHeading" "level" "level ^ level") + (ct-def-method! "CtHeading" "type" "type ^ #heading") + (st-class-define! "CtCode" "CtText" (list "language")) + (ct-def-method! "CtCode" "language" "language ^ language") + (ct-def-method! "CtCode" "type" "type ^ #code") + (st-class-define! "CtQuote" "CtText" (list "cite")) + (ct-def-method! "CtQuote" "cite" "cite ^ cite") + (ct-def-method! "CtQuote" "type" "type ^ #quote") + (st-class-define! "CtImage" "CtBlock" (list "src" "alt")) + (ct-def-method! "CtImage" "src" "src ^ src") + (ct-def-method! "CtImage" "alt" "alt ^ alt") + (ct-def-method! "CtImage" "type" "type ^ #image") + (st-class-define! "CtEmbed" "CtBlock" (list "url" "provider")) + (ct-def-method! "CtEmbed" "url" "url ^ url") + (ct-def-method! "CtEmbed" "provider" "provider ^ provider") + (ct-def-method! "CtEmbed" "type" "type ^ #embed") + (st-class-define! "CtDivider" "CtBlock" (list)) + (ct-def-method! "CtDivider" "type" "type ^ #divider") + (st-class-define! "CtList" "CtBlock" (list "ordered" "items")) + (ct-def-method! "CtList" "ordered" "ordered ^ ordered") + (ct-def-method! "CtList" "items" "items ^ items") + (ct-def-method! "CtList" "type" "type ^ #list") + true))) + +;; Apply (name value) pairs functionally onto a fresh instance. +(define + ct-apply-fields + (fn + (inst pairs) + (if + (= (len pairs) 0) + inst + (ct-apply-fields + (st-iv-set! + inst + (first (first pairs)) + (first (rest (first pairs)))) + (rest pairs))))) + +(define + ct-class-for-type + (fn + (tag) + (cond + ((= tag "text") "CtText") + ((= tag "heading") "CtHeading") + ((= tag "code") "CtCode") + ((= tag "quote") "CtQuote") + ((= tag "image") "CtImage") + ((= tag "embed") "CtEmbed") + ((= tag "divider") "CtDivider") + ((= tag "list") "CtList") + (else (error (str "unknown block type: " tag)))))) + +;; Generic constructor — wire tag + id + (name value) field pairs. +(define + mk-block + (fn + (type-tag id fields) + (ct-apply-fields + (st-iv-set! (st-make-instance (ct-class-for-type type-tag)) "id" id) + fields))) + +(define + mk-text + (fn (id text) (mk-block "text" id (list (list "text" text))))) + +(define + mk-heading + (fn + (id level text) + (mk-block "heading" id (list (list "level" level) (list "text" text))))) + +(define + mk-code + (fn + (id language text) + (mk-block + "code" + id + (list (list "language" language) (list "text" text))))) + +(define + mk-quote + (fn + (id cite text) + (mk-block "quote" id (list (list "cite" cite) (list "text" text))))) + +(define + mk-image + (fn + (id src alt) + (mk-block "image" id (list (list "src" src) (list "alt" alt))))) + +(define + mk-embed + (fn + (id url provider) + (mk-block "embed" id (list (list "url" url) (list "provider" provider))))) + +(define mk-divider (fn (id) (mk-block "divider" id (list)))) + +(define + mk-list + (fn + (id ordered items) + (mk-block + "list" + id + (list (list "ordered" ordered) (list "items" items))))) + +;; Accessors. blk-type / blk-id go through message dispatch (polymorphic); +;; blk-get reads any ivar directly; blk-set is copy-on-write. +(define blk-id (fn (b) (st-send b "id" (list)))) +(define blk-type (fn (b) (str (st-send b "type" (list))))) +(define blk-send (fn (b sel) (st-send b sel (list)))) +(define blk-get (fn (b field) (st-iv-get b field))) +(define blk-set (fn (b field val) (st-iv-set! b field val))) + +(define + block? + (fn + (v) + (and + (st-instance? v) + (st-class-inherits-from? (get v :class) "CtBlock")))) diff --git a/lib/content/conformance.sh b/lib/content/conformance.sh new file mode 100755 index 00000000..87c5a412 --- /dev/null +++ b/lib/content/conformance.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +# lib/content/conformance.sh — run content-on-sx suites, emit scoreboard. + +set -uo pipefail +cd "$(git rev-parse --show-toplevel)" + +SX_SERVER="hosts/ocaml/_build/default/bin/sx_server.exe" +if [ ! -x "$SX_SERVER" ]; then + MAIN_ROOT=$(git worktree list | head -1 | awk '{print $1}') + if [ -x "$MAIN_ROOT/$SX_SERVER" ]; then + SX_SERVER="$MAIN_ROOT/$SX_SERVER" + else + echo "ERROR: sx_server.exe not found." >&2 + exit 1 + fi +fi + +SUITES=(block) + +OUT_JSON="lib/content/scoreboard.json" +OUT_MD="lib/content/scoreboard.md" + +run_suite() { + local suite=$1 + local file="lib/content/tests/${suite}.sx" + local TMP + TMP=$(mktemp) + cat > "$TMP" << EPOCHS +(epoch 1) +(load "lib/smalltalk/tokenizer.sx") +(load "lib/smalltalk/parser.sx") +(load "lib/guest/reflective/class-chain.sx") +(load "lib/smalltalk/runtime.sx") +(load "lib/guest/reflective/env.sx") +(load "lib/smalltalk/eval.sx") +(load "lib/content/block.sx") +(epoch 2) +(eval "(define content-test-pass 0)") +(eval "(define content-test-fail 0)") +(eval "(define content-test-fails (list))") +(eval "(define content-test (fn (name got expected) (if (= got expected) (set! content-test-pass (+ content-test-pass 1)) (begin (set! content-test-fail (+ content-test-fail 1)) (set! content-test-fails (cons name content-test-fails))))))") +(epoch 3) +(load "${file}") +(epoch 4) +(eval "(list content-test-pass content-test-fail)") +EPOCHS + + local OUTPUT + OUTPUT=$(timeout 240 "$SX_SERVER" < "$TMP" 2>/dev/null) + rm -f "$TMP" + + local LINE + LINE=$(echo "$OUTPUT" | awk '/^\(ok-len 4 / {getline; print; exit}') + if [ -z "$LINE" ]; then + LINE=$(echo "$OUTPUT" | grep -E '^\(ok 4 \([0-9]+ [0-9]+\)\)' | tail -1 \ + | sed -E 's/^\(ok 4 //; s/\)$//') + fi + + local P F + P=$(echo "$LINE" | sed -E 's/^\(([0-9]+) ([0-9]+)\).*/\1/') + F=$(echo "$LINE" | sed -E 's/^\(([0-9]+) ([0-9]+)\).*/\2/') + P=${P:-0} + F=${F:-0} + echo "${P} ${F}" +} + +declare -A SUITE_PASS +declare -A SUITE_FAIL +TOTAL_PASS=0 +TOTAL_FAIL=0 + +echo "Running content conformance suite..." >&2 +for s in "${SUITES[@]}"; do + read -r p f < <(run_suite "$s") + SUITE_PASS[$s]=$p + SUITE_FAIL[$s]=$f + TOTAL_PASS=$((TOTAL_PASS + p)) + TOTAL_FAIL=$((TOTAL_FAIL + f)) + printf " %-12s %d/%d\n" "$s" "$p" "$((p+f))" >&2 +done + +{ + printf '{\n' + printf ' "suites": {\n' + first=1 + for s in "${SUITES[@]}"; do + if [ $first -eq 0 ]; then printf ',\n'; fi + printf ' "%s": {"pass": %d, "fail": %d}' "$s" "${SUITE_PASS[$s]}" "${SUITE_FAIL[$s]}" + first=0 + done + printf '\n },\n' + printf ' "total_pass": %d,\n' "$TOTAL_PASS" + printf ' "total_fail": %d,\n' "$TOTAL_FAIL" + printf ' "total": %d\n' "$((TOTAL_PASS + TOTAL_FAIL))" + printf '}\n' +} > "$OUT_JSON" + +{ + printf '# content-on-sx Conformance Scoreboard\n\n' + printf '_Generated by `lib/content/conformance.sh`_\n\n' + printf '| Suite | Pass | Fail | Total |\n' + printf '|-------|-----:|-----:|------:|\n' + for s in "${SUITES[@]}"; do + p=${SUITE_PASS[$s]} + f=${SUITE_FAIL[$s]} + printf '| %s | %d | %d | %d |\n' "$s" "$p" "$f" "$((p+f))" + done + printf '| **Total** | **%d** | **%d** | **%d** |\n' "$TOTAL_PASS" "$TOTAL_FAIL" "$((TOTAL_PASS + TOTAL_FAIL))" +} > "$OUT_MD" + +echo "Wrote $OUT_JSON and $OUT_MD" >&2 +echo "Total: $TOTAL_PASS pass, $TOTAL_FAIL fail" >&2 + +[ "$TOTAL_FAIL" -eq 0 ] diff --git a/lib/content/scoreboard.json b/lib/content/scoreboard.json new file mode 100644 index 00000000..6ebb3777 --- /dev/null +++ b/lib/content/scoreboard.json @@ -0,0 +1,8 @@ +{ + "suites": { + "block": {"pass": 38, "fail": 0} + }, + "total_pass": 38, + "total_fail": 0, + "total": 38 +} diff --git a/lib/content/scoreboard.md b/lib/content/scoreboard.md new file mode 100644 index 00000000..13b4bd16 --- /dev/null +++ b/lib/content/scoreboard.md @@ -0,0 +1,8 @@ +# content-on-sx Conformance Scoreboard + +_Generated by `lib/content/conformance.sh`_ + +| Suite | Pass | Fail | Total | +|-------|-----:|-----:|------:| +| block | 38 | 0 | 38 | +| **Total** | **38** | **0** | **38** | diff --git a/lib/content/tests/block.sx b/lib/content/tests/block.sx new file mode 100644 index 00000000..c6fc49c8 --- /dev/null +++ b/lib/content/tests/block.sx @@ -0,0 +1,75 @@ +;; Phase 1 — typed block objects. Behaviour via message dispatch; fields +;; immutable (copy-on-write). + +(st-bootstrap-classes!) +(content-bootstrap-blocks!) + +;; ── construction + polymorphic type dispatch ── +(define h (mk-heading "b1" 2 "Title")) +(define t (mk-text "b2" "Body text")) +(define img (mk-image "b3" "/cat.png" "a cat")) +(define code (mk-code "b4" "sx" "(+ 1 2)")) +(define q (mk-quote "b5" "Ada" "to err")) +(define em (mk-embed "b6" "https://v/1" "vimeo")) +(define dv (mk-divider "b7")) +(define ls (mk-list "b8" true (list "one" "two"))) + +(content-test "heading type" (blk-type h) "heading") +(content-test "text type" (blk-type t) "text") +(content-test "image type" (blk-type img) "image") +(content-test "code type" (blk-type code) "code") +(content-test "quote type" (blk-type q) "quote") +(content-test "embed type" (blk-type em) "embed") +(content-test "divider type" (blk-type dv) "divider") +(content-test "list type" (blk-type ls) "list") + +;; ── id via message dispatch ── +(content-test "heading id" (blk-id h) "b1") +(content-test "image id" (blk-id img) "b3") +(content-test "divider id" (blk-id dv) "b7") + +;; ── field reads via messages (incl. inherited text) ── +(content-test "heading text inherited" (str (blk-send h "text")) "Title") +(content-test "heading level" (blk-send h "level") 2) +(content-test "text body" (str (blk-send t "text")) "Body text") +(content-test "image src" (str (blk-send img "src")) "/cat.png") +(content-test "image alt" (str (blk-send img "alt")) "a cat") +(content-test "code language" (str (blk-send code "language")) "sx") +(content-test "code text inherited" (str (blk-send code "text")) "(+ 1 2)") +(content-test "quote cite" (str (blk-send q "cite")) "Ada") +(content-test "embed url" (str (blk-send em "url")) "https://v/1") +(content-test "embed provider" (str (blk-send em "provider")) "vimeo") +(content-test "list ordered" (blk-send ls "ordered") true) +(content-test "list items" (blk-send ls "items") (list "one" "two")) + +;; ── blk-get reads ivars directly ── +(content-test "blk-get level" (blk-get h "level") 2) +(content-test "blk-get missing nil" (blk-get h "nope") nil) + +;; ── copy-on-write: blk-set returns a new block, original untouched ── +(define h2 (blk-set h "level" 1)) +(content-test "blk-set new value" (blk-send h2 "level") 1) +(content-test "blk-set original unchanged" (blk-send h "level") 2) +(content-test "blk-set keeps id" (blk-id h2) "b1") +(content-test "blk-set keeps text" (str (blk-send h2 "text")) "Title") + +;; ── predicate ── +(content-test "block? on heading" (block? h) true) +(content-test "block? on divider" (block? dv) true) +(content-test "block? on number" (block? 5) false) +(content-test "block? on string" (block? "x") false) + +;; ── isBlock message inherited by all ── +(content-test "isBlock heading" (blk-send h "isBlock") true) +(content-test "isBlock list" (blk-send ls "isBlock") true) + +;; ── generic mk-block via wire tag ── +(define + g + (mk-block + "heading" + "g1" + (list (list "level" 3) (list "text" "Gen")))) +(content-test "mk-block type" (blk-type g) "heading") +(content-test "mk-block level" (blk-send g "level") 3) +(content-test "mk-block text" (str (blk-send g "text")) "Gen") diff --git a/plans/content-on-sx.md b/plans/content-on-sx.md index 03f606fa..eab4a1f5 100644 --- a/plans/content-on-sx.md +++ b/plans/content-on-sx.md @@ -19,7 +19,7 @@ injected adapter, not core. ## Status (rolling) -`bash lib/content/conformance.sh` → **0/0** (not yet started) +`bash lib/content/conformance.sh` → **38/38** (Phase 1: typed blocks) ## Ground rules @@ -57,7 +57,7 @@ lib/content/api.sx ── (content/edit) (content/render) (content/history) ─ ``` ## Phase 1 — Block document model -- [ ] `block.sx` — typed block objects +- [x] `block.sx` — typed block objects - [ ] `doc.sx` — ordered tree, apply edit op, structural moves - [ ] `render.sx` — block tree → HTML/SX - [ ] `api.sx` + tests + scoreboard + conformance.sh @@ -76,7 +76,17 @@ lib/content/api.sx ── (content/edit) (content/render) (content/history) ─ - [ ] tests: round-trip import/export, conflict on concurrent external edit ## Progress log -(loop fills this in) + +- 2026-06-06 — Phase 1 `block.sx`: typed block objects as Smalltalk instances + (`CtBlock` hierarchy: text/heading/code/quote/image/embed/divider/list). + Type tag + accessors are message sends (polymorphic dispatch); fields are + immutable copy-on-write via functional `st-iv-set!` (history-safe). Added + `mk-*` constructors, `block?` predicate, `lib/content/conformance.sh` + + scoreboard. 38/38. ## Blockers -(loop fills this in) + +- Smalltalk-only load chain (tokenizer/parser/runtime/eval) does **not** load + `lib/r7rs.sx`/`spec/stdlib.sx`, so r7rs aliases (`car`/`cdr`/`null?`) are + absent. Use base SX primitives (`first`/`rest`/`(= (len x) 0)`) in + `lib/content/**`. Not a substrate bug — just the load surface. From 6e52ad51263f93f1b2828d52d3906972d47a4b57 Mon Sep 17 00:00:00 2001 From: giles Date: Sat, 6 Jun 2026 23:57:34 +0000 Subject: [PATCH 02/49] content: ordered block document + edit ops + 40 tests Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/content/conformance.sh | 3 +- lib/content/doc.sx | 194 ++++++++++++++++++++++++++++++++++++ lib/content/scoreboard.json | 7 +- lib/content/scoreboard.md | 3 +- lib/content/tests/doc.sx | 132 ++++++++++++++++++++++++ plans/content-on-sx.md | 10 +- 6 files changed, 342 insertions(+), 7 deletions(-) create mode 100644 lib/content/doc.sx create mode 100644 lib/content/tests/doc.sx diff --git a/lib/content/conformance.sh b/lib/content/conformance.sh index 87c5a412..84d92a01 100755 --- a/lib/content/conformance.sh +++ b/lib/content/conformance.sh @@ -15,7 +15,7 @@ if [ ! -x "$SX_SERVER" ]; then fi fi -SUITES=(block) +SUITES=(block doc) OUT_JSON="lib/content/scoreboard.json" OUT_MD="lib/content/scoreboard.md" @@ -34,6 +34,7 @@ run_suite() { (load "lib/guest/reflective/env.sx") (load "lib/smalltalk/eval.sx") (load "lib/content/block.sx") +(load "lib/content/doc.sx") (epoch 2) (eval "(define content-test-pass 0)") (eval "(define content-test-fail 0)") diff --git a/lib/content/doc.sx b/lib/content/doc.sx new file mode 100644 index 00000000..90e9bede --- /dev/null +++ b/lib/content/doc.sx @@ -0,0 +1,194 @@ +;; content-on-sx — ordered block document on Smalltalk-on-SX. +;; +;; A document (CtDoc) is a Smalltalk object holding an ordered sequence of block +;; objects. Editing is a stream of ops (data dicts); doc-apply interprets one op +;; and returns a NEW document — the input is never mutated, so any version is the +;; head of an op stream (replay-friendly for persist + CRDT merge). +;; +;; Op shapes (data, not objects — they are the persist event payload): +;; {:op "insert" :block :after } ; after nil = prepend +;; {:op "update" :id :field :value } +;; {:op "move" :id :index } +;; {:op "delete" :id } + +(define + content-bootstrap-doc! + (fn + () + (begin + (st-class-define! "CtDoc" "Object" (list "id" "blocks")) + (ct-def-method! "CtDoc" "id" "id ^ id") + (ct-def-method! "CtDoc" "blocks" "blocks ^ blocks") + (ct-def-method! "CtDoc" "type" "type ^ #document") + true))) + +;; ── construction ── +(define + doc-new + (fn + (id blocks) + (st-iv-set! + (st-iv-set! (st-make-instance "CtDoc") "id" id) + "blocks" + blocks))) + +(define doc-empty (fn (id) (doc-new id (list)))) + +;; ── accessors (message dispatch) ── +(define doc-id (fn (doc) (st-send doc "id" (list)))) +(define doc-type (fn (doc) (str (st-send doc "type" (list))))) +(define doc-blocks (fn (doc) (st-send doc "blocks" (list)))) +(define doc-count (fn (doc) (len (doc-blocks doc)))) +(define doc-block-at (fn (doc i) (nth (doc-blocks doc) i))) + +(define doc? (fn (v) (and (st-instance? v) (= (get v :class) "CtDoc")))) + +;; ── list helpers over block sequences ── +(define + ct-index-loop + (fn + (blocks id i) + (cond + ((= (len blocks) 0) -1) + ((= (blk-id (first blocks)) id) i) + (else (ct-index-loop (rest blocks) id (+ i 1)))))) + +(define ct-index-of (fn (blocks id) (ct-index-loop blocks id 0))) + +(define + ct-insert-at + (fn + (blocks i x) + (cond + ((= i 0) (cons x blocks)) + ((= (len blocks) 0) (list x)) + (else + (cons + (first blocks) + (ct-insert-at (rest blocks) (- i 1) x)))))) + +(define + ct-remove-id + (fn + (blocks id) + (filter (fn (b) (if (= (blk-id b) id) false true)) blocks))) + +(define + ct-replace-id + (fn + (blocks id f) + (map (fn (b) (if (= (blk-id b) id) (f b) b)) blocks))) + +;; ── query ── +(define doc-index-of (fn (doc id) (ct-index-of (doc-blocks doc) id))) + +(define + doc-find + (fn + (doc id) + (let + ((hits (filter (fn (b) (= (blk-id b) id)) (doc-blocks doc)))) + (if (= (len hits) 0) nil (first hits))))) + +(define + doc-has? + (fn (doc id) (if (= (doc-index-of doc id) -1) false true))) + +;; ── structural edits (each returns a new document) ── +(define doc-with-blocks (fn (doc blocks) (st-iv-set! doc "blocks" blocks))) + +(define + doc-append + (fn + (doc block) + (doc-with-blocks doc (append (doc-blocks doc) (list block))))) + +(define + doc-insert-at + (fn + (doc block i) + (doc-with-blocks doc (ct-insert-at (doc-blocks doc) i block)))) + +(define + doc-insert-after + (fn + (doc block after-id) + (let + ((blocks (doc-blocks doc))) + (if + (= after-id nil) + (doc-with-blocks doc (cons block blocks)) + (let + ((idx (ct-index-of blocks after-id))) + (if + (= idx -1) + (doc-with-blocks doc (append blocks (list block))) + (doc-with-blocks + doc + (ct-insert-at blocks (+ idx 1) block)))))))) + +(define + doc-update + (fn + (doc id field value) + (doc-with-blocks + doc + (ct-replace-id (doc-blocks doc) id (fn (b) (blk-set b field value)))))) + +(define + doc-delete + (fn (doc id) (doc-with-blocks doc (ct-remove-id (doc-blocks doc) id)))) + +(define + doc-move + (fn + (doc id i) + (let + ((blk (doc-find doc id))) + (if + (= blk nil) + doc + (doc-with-blocks + doc + (ct-insert-at (ct-remove-id (doc-blocks doc) id) i blk)))))) + +;; ── op constructors (data payload, reused by persist op log) ── +(define op-insert (fn (block after) {:after after :op "insert" :block block})) + +(define op-update (fn (id field value) {:field field :id id :op "update" :value value})) + +(define op-move (fn (id index) {:id id :op "move" :index index})) + +(define op-delete (fn (id) {:id id :op "delete"})) + +;; ── op interpreter ── +(define + doc-apply + (fn + (doc op) + (let + ((kind (get op :op))) + (cond + ((= kind "insert") + (doc-insert-after doc (get op :block) (get op :after))) + ((= kind "update") + (doc-update doc (get op :id) (get op :field) (get op :value))) + ((= kind "move") (doc-move doc (get op :id) (get op :index))) + ((= kind "delete") (doc-delete doc (get op :id))) + (else (error (str "unknown op: " kind))))))) + +(define + doc-apply-all + (fn + (doc ops) + (if + (= (len ops) 0) + doc + (doc-apply-all (doc-apply doc (first ops)) (rest ops))))) + +;; ── render-agnostic snapshot: list of (id . type) for assertions/debug ── +(define doc-ids (fn (doc) (map (fn (b) (blk-id b)) (doc-blocks doc)))) + +(define + doc-types + (fn (doc) (map (fn (b) (blk-type b)) (doc-blocks doc)))) diff --git a/lib/content/scoreboard.json b/lib/content/scoreboard.json index 6ebb3777..ae8afa03 100644 --- a/lib/content/scoreboard.json +++ b/lib/content/scoreboard.json @@ -1,8 +1,9 @@ { "suites": { - "block": {"pass": 38, "fail": 0} + "block": {"pass": 38, "fail": 0}, + "doc": {"pass": 40, "fail": 0} }, - "total_pass": 38, + "total_pass": 78, "total_fail": 0, - "total": 38 + "total": 78 } diff --git a/lib/content/scoreboard.md b/lib/content/scoreboard.md index 13b4bd16..3912f6bf 100644 --- a/lib/content/scoreboard.md +++ b/lib/content/scoreboard.md @@ -5,4 +5,5 @@ _Generated by `lib/content/conformance.sh`_ | Suite | Pass | Fail | Total | |-------|-----:|-----:|------:| | block | 38 | 0 | 38 | -| **Total** | **38** | **0** | **38** | +| doc | 40 | 0 | 40 | +| **Total** | **78** | **0** | **78** | diff --git a/lib/content/tests/doc.sx b/lib/content/tests/doc.sx new file mode 100644 index 00000000..12681403 --- /dev/null +++ b/lib/content/tests/doc.sx @@ -0,0 +1,132 @@ +;; Phase 1 — ordered block document: apply edit ops, structural moves. +;; Every op returns a NEW document; the input is never mutated. + +(st-bootstrap-classes!) +(content-bootstrap-blocks!) +(content-bootstrap-doc!) + +(define h (mk-heading "h" 1 "Title")) +(define p1 (mk-text "p1" "First")) +(define p2 (mk-text "p2" "Second")) +(define img (mk-image "img" "/c.png" "cat")) + +;; ── empty + construction ── +(define d0 (doc-empty "doc1")) +(content-test "empty id" (doc-id d0) "doc1") +(content-test "empty type" (doc-type d0) "document") +(content-test "empty count" (doc-count d0) 0) +(content-test "doc? on doc" (doc? d0) true) +(content-test "doc? on block" (doc? h) false) + +;; ── append + order ── +(define d1 (doc-append (doc-append (doc-append d0 h) p1) p2)) +(content-test "append count" (doc-count d1) 3) +(content-test "append order" (doc-ids d1) (list "h" "p1" "p2")) +(content-test "append types" (doc-types d1) (list "heading" "text" "text")) +(content-test "block-at 0" (blk-id (doc-block-at d1 0)) "h") + +;; ── append is immutable ── +(content-test "append leaves original" (doc-count d0) 0) + +;; ── find / index / has ── +(content-test "find p1" (blk-id (doc-find d1 "p1")) "p1") +(content-test "find missing" (doc-find d1 "nope") nil) +(content-test "index-of p2" (doc-index-of d1 "p2") 2) +(content-test "index-of missing" (doc-index-of d1 "nope") -1) +(content-test "has? yes" (doc-has? d1 "h") true) +(content-test "has? no" (doc-has? d1 "x") false) + +;; ── insert-after ── +(define d2 (doc-insert-after d1 img "h")) +(content-test "insert-after order" (doc-ids d2) (list "h" "img" "p1" "p2")) +(content-test + "insert-after prepend" + (doc-ids (doc-insert-after d1 img nil)) + (list "img" "h" "p1" "p2")) +(content-test + "insert-after missing appends" + (doc-ids (doc-insert-after d1 img "zzz")) + (list "h" "p1" "p2" "img")) +(content-test "insert-after immutable" (doc-ids d1) (list "h" "p1" "p2")) + +;; ── insert-at ── +(content-test + "insert-at 0" + (doc-ids (doc-insert-at d1 img 0)) + (list "img" "h" "p1" "p2")) +(content-test + "insert-at 1" + (doc-ids (doc-insert-at d1 img 1)) + (list "h" "img" "p1" "p2")) + +;; ── update (copy-on-write block) ── +(define d3 (doc-update d1 "p1" "text" "Edited")) +(content-test + "update value" + (str (blk-send (doc-find d3 "p1") "text")) + "Edited") +(content-test "update keeps order" (doc-ids d3) (list "h" "p1" "p2")) +(content-test + "update immutable" + (str (blk-send (doc-find d1 "p1") "text")) + "First") + +;; ── delete ── +(define d4 (doc-delete d1 "p1")) +(content-test "delete order" (doc-ids d4) (list "h" "p2")) +(content-test "delete count" (doc-count d4) 2) +(content-test "delete immutable" (doc-count d1) 3) +(content-test + "delete missing no-op" + (doc-ids (doc-delete d1 "x")) + (list "h" "p1" "p2")) + +;; ── move ── +(content-test + "move p2 to front" + (doc-ids (doc-move d1 "p2" 0)) + (list "p2" "h" "p1")) +(content-test + "move h to end" + (doc-ids (doc-move d1 "h" 2)) + (list "p1" "p2" "h")) +(content-test + "move missing no-op" + (doc-ids (doc-move d1 "x" 0)) + (list "h" "p1" "p2")) +(content-test "move immutable" (doc-ids d1) (list "h" "p1" "p2")) + +;; ── op constructors + interpreter ── +(content-test + "op-insert apply" + (doc-ids (doc-apply d1 (op-insert img "h"))) + (list "h" "img" "p1" "p2")) +(content-test + "op-delete apply" + (doc-ids (doc-apply d1 (op-delete "h"))) + (list "p1" "p2")) +(content-test + "op-move apply" + (doc-ids (doc-apply d1 (op-move "p2" 0))) + (list "p2" "h" "p1")) +(content-test + "op-update apply" + (str + (blk-send + (doc-find (doc-apply d1 (op-update "p1" "text" "X")) "p1") + "text")) + "X") + +;; ── apply-all: a stream of ops ── +(define + ops + (list (op-insert img "h") (op-delete "p1") (op-move "p2" 0))) +(content-test + "apply-all" + (doc-ids (doc-apply-all d1 ops)) + (list "p2" "h" "img")) +(content-test "apply-all immutable" (doc-ids d1) (list "h" "p1" "p2")) +(content-test + "apply-all empty" + (doc-ids (doc-apply-all d1 (list))) + (list "h" "p1" "p2")) diff --git a/plans/content-on-sx.md b/plans/content-on-sx.md index eab4a1f5..9bf7ebbe 100644 --- a/plans/content-on-sx.md +++ b/plans/content-on-sx.md @@ -19,7 +19,7 @@ injected adapter, not core. ## Status (rolling) -`bash lib/content/conformance.sh` → **38/38** (Phase 1: typed blocks) +`bash lib/content/conformance.sh` → **78/78** (Phase 1: blocks + doc) ## Ground rules @@ -58,7 +58,7 @@ lib/content/api.sx ── (content/edit) (content/render) (content/history) ─ ## Phase 1 — Block document model - [x] `block.sx` — typed block objects -- [ ] `doc.sx` — ordered tree, apply edit op, structural moves +- [x] `doc.sx` — ordered tree, apply edit op, structural moves - [ ] `render.sx` — block tree → HTML/SX - [ ] `api.sx` + tests + scoreboard + conformance.sh @@ -77,6 +77,12 @@ lib/content/api.sx ── (content/edit) (content/render) (content/history) ─ ## Progress log +- 2026-06-06 — Phase 1 `doc.sx`: ordered block document (`CtDoc`) as a + Smalltalk object holding an ordered block sequence. Edit ops are data dicts + (`insert`/`update`/`move`/`delete`) with `op-*` constructors; `doc-apply` / + `doc-apply-all` interpret an op stream, each returning a NEW document (input + never mutated → replay-safe). Structural moves, insert-after/at, find/index, + immutability all tested. 40 tests; suite 78/78. - 2026-06-06 — Phase 1 `block.sx`: typed block objects as Smalltalk instances (`CtBlock` hierarchy: text/heading/code/quote/image/embed/divider/list). Type tag + accessors are message sends (polymorphic dispatch); fields are From 0d93a9820fa09bf3d526c5182ea8fe064eb82d5d Mon Sep 17 00:00:00 2001 From: giles Date: Sun, 7 Jun 2026 00:03:05 +0000 Subject: [PATCH 03/49] content: render boundary (asHTML/asSx polymorphic) + 29 tests Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/content/conformance.sh | 3 +- lib/content/render.sx | 78 +++++++++++++++++++++++++++++++++++++ lib/content/scoreboard.json | 7 ++-- lib/content/scoreboard.md | 3 +- lib/content/tests/render.sx | 73 ++++++++++++++++++++++++++++++++++ plans/content-on-sx.md | 11 +++++- 6 files changed, 168 insertions(+), 7 deletions(-) create mode 100644 lib/content/render.sx create mode 100644 lib/content/tests/render.sx diff --git a/lib/content/conformance.sh b/lib/content/conformance.sh index 84d92a01..60931794 100755 --- a/lib/content/conformance.sh +++ b/lib/content/conformance.sh @@ -15,7 +15,7 @@ if [ ! -x "$SX_SERVER" ]; then fi fi -SUITES=(block doc) +SUITES=(block doc render) OUT_JSON="lib/content/scoreboard.json" OUT_MD="lib/content/scoreboard.md" @@ -35,6 +35,7 @@ run_suite() { (load "lib/smalltalk/eval.sx") (load "lib/content/block.sx") (load "lib/content/doc.sx") +(load "lib/content/render.sx") (epoch 2) (eval "(define content-test-pass 0)") (eval "(define content-test-fail 0)") diff --git a/lib/content/render.sx b/lib/content/render.sx new file mode 100644 index 00000000..e939ebb7 --- /dev/null +++ b/lib/content/render.sx @@ -0,0 +1,78 @@ +;; content-on-sx — render boundary. +;; +;; Rendering is a message, not a property switch: every block (and the document) +;; answers asHTML and asSx. The internal model carries no presentation — the +;; boundary format is chosen by which message you send. The document folds its +;; children's renderings, so (asHTML doc) / (asSx doc) are pure polymorphic +;; sends with no type dispatch in the SX layer. +;; +;; NOTE: no HTML escaping yet — text is emitted verbatim. Escaping is a boundary +;; concern to add before any untrusted content reaches render. + +(define + content-bootstrap-render! + (fn + () + (begin + (ct-def-method! + "CtHeading" + "asHTML" + "asHTML | t | t := level printString. ^ '' , text , ''") + (ct-def-method! "CtText" "asHTML" "asHTML ^ '

' , text , '

'") + (ct-def-method! + "CtCode" + "asHTML" + "asHTML ^ '
' , text , '
'") + (ct-def-method! + "CtQuote" + "asHTML" + "asHTML ^ '
' , text , '
'") + (ct-def-method! + "CtImage" + "asHTML" + "asHTML ^ '\"''") + (ct-def-method! + "CtEmbed" + "asHTML" + "asHTML ^ ''") + (ct-def-method! "CtDivider" "asHTML" "asHTML ^ '
'") + (ct-def-method! + "CtList" + "asHTML" + "asHTML | tag | tag := ordered ifTrue: ['ol'] ifFalse: ['ul']. ^ '<' , tag , '>' , (items inject: '' into: [:a :x | a , '
  • ' , x , '
  • ']) , ''") + (ct-def-method! + "CtDoc" + "asHTML" + "asHTML ^ blocks inject: '' into: [:a :b | a , (b asHTML)]") + (ct-def-method! + "CtHeading" + "asSx" + "asSx | t | t := level printString. ^ '(h' , t , ' \"' , text , '\")'") + (ct-def-method! "CtText" "asSx" "asSx ^ '(p \"' , text , '\")'") + (ct-def-method! "CtCode" "asSx" "asSx ^ '(pre (code \"' , text , '\"))'") + (ct-def-method! "CtQuote" "asSx" "asSx ^ '(blockquote \"' , text , '\")'") + (ct-def-method! + "CtImage" + "asSx" + "asSx ^ '(img :src \"' , src , '\" :alt \"' , alt , '\")'") + (ct-def-method! "CtEmbed" "asSx" "asSx ^ '(iframe :src \"' , url , '\")'") + (ct-def-method! "CtDivider" "asSx" "asSx ^ '(hr)'") + (ct-def-method! + "CtList" + "asSx" + "asSx | tag | tag := ordered ifTrue: ['ol'] ifFalse: ['ul']. ^ '(' , tag , ' ' , (items inject: '' into: [:a :x | a , '(li \"' , x , '\")']) , ')'") + (ct-def-method! + "CtDoc" + "asSx" + "asSx ^ '(article ' , (blocks inject: '' into: [:a :b | a , (b asSx)]) , ')'") + true))) + +;; ── SX boundary API — pure message sends ── +(define asHTML (fn (node) (str (st-send node "asHTML" (list))))) +(define asSx (fn (node) (str (st-send node "asSx" (list))))) + +;; readable aliases +(define render-html asHTML) +(define render-sx asSx) +(define block-html asHTML) +(define block-sx asSx) diff --git a/lib/content/scoreboard.json b/lib/content/scoreboard.json index ae8afa03..f50e783e 100644 --- a/lib/content/scoreboard.json +++ b/lib/content/scoreboard.json @@ -1,9 +1,10 @@ { "suites": { "block": {"pass": 38, "fail": 0}, - "doc": {"pass": 40, "fail": 0} + "doc": {"pass": 40, "fail": 0}, + "render": {"pass": 29, "fail": 0} }, - "total_pass": 78, + "total_pass": 107, "total_fail": 0, - "total": 78 + "total": 107 } diff --git a/lib/content/scoreboard.md b/lib/content/scoreboard.md index 3912f6bf..40e6e23a 100644 --- a/lib/content/scoreboard.md +++ b/lib/content/scoreboard.md @@ -6,4 +6,5 @@ _Generated by `lib/content/conformance.sh`_ |-------|-----:|-----:|------:| | block | 38 | 0 | 38 | | doc | 40 | 0 | 40 | -| **Total** | **78** | **0** | **78** | +| render | 29 | 0 | 29 | +| **Total** | **107** | **0** | **107** | diff --git a/lib/content/tests/render.sx b/lib/content/tests/render.sx new file mode 100644 index 00000000..f7ade85a --- /dev/null +++ b/lib/content/tests/render.sx @@ -0,0 +1,73 @@ +;; Phase 1 — render boundary. asHTML / asSx are polymorphic message sends on +;; blocks and the document. + +(st-bootstrap-classes!) +(content-bootstrap-blocks!) +(content-bootstrap-doc!) +(content-bootstrap-render!) + +(define h (mk-heading "h" 2 "Title")) +(define p (mk-text "p" "Hello")) +(define code (mk-code "c" "sx" "(+ 1 2)")) +(define q (mk-quote "q" "Ada" "to err")) +(define img (mk-image "i" "/c.png" "cat")) +(define em (mk-embed "e" "https://v/1" "vimeo")) +(define dv (mk-divider "d")) +(define ul (mk-list "u" false (list "a" "b"))) +(define ol (mk-list "o" true (list "x" "y"))) + +;; ── per-block asHTML ── +(content-test "heading html" (asHTML h) "

    Title

    ") +(content-test "text html" (asHTML p) "

    Hello

    ") +(content-test + "code html" + (asHTML code) + "
    (+ 1 2)
    ") +(content-test "quote html" (asHTML q) "
    to err
    ") +(content-test "image html" (asHTML img) "\"cat\"") +(content-test "embed html" (asHTML em) "") +(content-test "divider html" (asHTML dv) "
    ") +(content-test "ul html" (asHTML ul) "
    • a
    • b
    ") +(content-test "ol html" (asHTML ol) "
    1. x
    2. y
    ") + +;; ── per-block asSx ── +(content-test "heading sx" (asSx h) "(h2 \"Title\")") +(content-test "text sx" (asSx p) "(p \"Hello\")") +(content-test "code sx" (asSx code) "(pre (code \"(+ 1 2)\"))") +(content-test "quote sx" (asSx q) "(blockquote \"to err\")") +(content-test "image sx" (asSx img) "(img :src \"/c.png\" :alt \"cat\")") +(content-test "embed sx" (asSx em) "(iframe :src \"https://v/1\")") +(content-test "divider sx" (asSx dv) "(hr)") +(content-test "ul sx" (asSx ul) "(ul (li \"a\")(li \"b\"))") +(content-test "ol sx" (asSx ol) "(ol (li \"x\")(li \"y\"))") + +;; ── document folds children (pure message dispatch) ── +(define d (doc-append (doc-append (doc-append (doc-empty "doc") h) p) dv)) +(content-test "doc html" (asHTML d) "

    Title

    Hello


    ") +(content-test "doc sx" (asSx d) "(article (h2 \"Title\")(p \"Hello\")(hr))") +(content-test "empty doc html" (asHTML (doc-empty "e")) "") +(content-test "empty doc sx" (asSx (doc-empty "e")) "(article )") + +;; ── render-* / block-* aliases ── +(content-test "render-html alias" (render-html d) (asHTML d)) +(content-test "render-sx alias" (render-sx d) (asSx d)) +(content-test "block-html alias" (block-html h) "

    Title

    ") + +;; ── render reflects edits (immutability: each render is of a version) ── +(define d2 (doc-update d "p" "text" "Edited")) +(content-test + "render after update" + (asHTML d2) + "

    Title

    Edited


    ") +(content-test + "original render unchanged" + (asHTML d) + "

    Title

    Hello


    ") +(content-test + "render after move" + (asHTML (doc-move d "h" 2)) + "

    Hello


    Title

    ") +(content-test + "render after delete" + (asHTML (doc-delete d "p")) + "

    Title


    ") diff --git a/plans/content-on-sx.md b/plans/content-on-sx.md index 9bf7ebbe..3024cc52 100644 --- a/plans/content-on-sx.md +++ b/plans/content-on-sx.md @@ -19,7 +19,7 @@ injected adapter, not core. ## Status (rolling) -`bash lib/content/conformance.sh` → **78/78** (Phase 1: blocks + doc) +`bash lib/content/conformance.sh` → **107/107** (Phase 1: blocks + doc + render) ## Ground rules @@ -59,7 +59,7 @@ lib/content/api.sx ── (content/edit) (content/render) (content/history) ─ ## Phase 1 — Block document model - [x] `block.sx` — typed block objects - [x] `doc.sx` — ordered tree, apply edit op, structural moves -- [ ] `render.sx` — block tree → HTML/SX +- [x] `render.sx` — block tree → HTML/SX - [ ] `api.sx` + tests + scoreboard + conformance.sh ## Phase 2 — Op log + versioning @@ -77,6 +77,13 @@ lib/content/api.sx ── (content/edit) (content/render) (content/history) ─ ## Progress log +- 2026-06-07 — Phase 1 `render.sx`: render boundary as polymorphic message + dispatch. Every block and `CtDoc` answers `asHTML` / `asSx`; the document + folds children via Smalltalk `inject:into:` (works on raw SX lists), so + `(asHTML doc)` / `(asSx doc)` are pure sends with zero type-switching in SX. + Lists/headings render in Smalltalk source. No HTML escaping yet (noted in + render.sx — boundary concern before untrusted content). 29 tests; suite + 107/107. - 2026-06-06 — Phase 1 `doc.sx`: ordered block document (`CtDoc`) as a Smalltalk object holding an ordered block sequence. Edit ops are data dicts (`insert`/`update`/`move`/`delete`) with `op-*` constructors; `doc-apply` / From 8dc918764599be1d0d90a388593f7c9a6acd8747 Mon Sep 17 00:00:00 2001 From: giles Date: Sun, 7 Jun 2026 00:08:42 +0000 Subject: [PATCH 04/49] content: content/* API facade + 26 tests (Phase 1 complete, 133/133) Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/content/api.sx | 63 +++++++++++++++++++++++ lib/content/conformance.sh | 3 +- lib/content/scoreboard.json | 7 +-- lib/content/scoreboard.md | 3 +- lib/content/tests/api.sx | 99 +++++++++++++++++++++++++++++++++++++ plans/content-on-sx.md | 11 ++++- 6 files changed, 179 insertions(+), 7 deletions(-) create mode 100644 lib/content/api.sx create mode 100644 lib/content/tests/api.sx diff --git a/lib/content/api.sx b/lib/content/api.sx new file mode 100644 index 00000000..becac4ed --- /dev/null +++ b/lib/content/api.sx @@ -0,0 +1,63 @@ +;; content-on-sx — public API facade. +;; +;; The stable surface other code calls. Composes block + doc + render. Document +;; values are immutable; every edit returns a new document, so callers hold +;; explicit versions (the persist op log in Phase 2 becomes the source of truth). +;; +;; Requires (loaded by the harness): block.sx, doc.sx, render.sx and a base +;; Smalltalk class table (st-bootstrap-classes!). + +;; Register the content class hierarchy + render methods. Caller bootstraps the +;; base Smalltalk classes first; this only adds content classes (idempotent). +(define + content/bootstrap! + (fn + () + (begin + (content-bootstrap-blocks!) + (content-bootstrap-doc!) + (content-bootstrap-render!) + true))) + +;; ── documents ── +(define content/new doc-new) +(define content/empty doc-empty) +(define content/append doc-append) +(define content/blocks doc-blocks) +(define content/count doc-count) +(define content/find doc-find) +(define content/has? doc-has?) +(define content/ids doc-ids) +(define content/types doc-types) + +;; ── blocks ── +(define content/block mk-block) + +;; ── edit ops (data payload) ── +(define content/insert op-insert) +(define content/update op-update) +(define content/move op-move) +(define content/delete op-delete) + +(define content/op? (fn (x) (and (dict? x) (has-key? x :op)))) + +;; edit — apply one op or a stream of ops; returns a new document. +(define + content/edit + (fn + (doc ops) + (if (content/op? ops) (doc-apply doc ops) (doc-apply-all doc ops)))) + +;; ── render boundary ── +;; fmt is "html"/"sx" (or :html/:sx — keywords evaluate to their name). +(define + content/render + (fn + (doc fmt) + (cond + ((= fmt "html") (asHTML doc)) + ((= fmt "sx") (asSx doc)) + (else (error (str "unknown render format: " fmt)))))) + +(define content/html asHTML) +(define content/sx asSx) diff --git a/lib/content/conformance.sh b/lib/content/conformance.sh index 60931794..de8ba21a 100755 --- a/lib/content/conformance.sh +++ b/lib/content/conformance.sh @@ -15,7 +15,7 @@ if [ ! -x "$SX_SERVER" ]; then fi fi -SUITES=(block doc render) +SUITES=(block doc render api) OUT_JSON="lib/content/scoreboard.json" OUT_MD="lib/content/scoreboard.md" @@ -36,6 +36,7 @@ run_suite() { (load "lib/content/block.sx") (load "lib/content/doc.sx") (load "lib/content/render.sx") +(load "lib/content/api.sx") (epoch 2) (eval "(define content-test-pass 0)") (eval "(define content-test-fail 0)") diff --git a/lib/content/scoreboard.json b/lib/content/scoreboard.json index f50e783e..b9270ccf 100644 --- a/lib/content/scoreboard.json +++ b/lib/content/scoreboard.json @@ -2,9 +2,10 @@ "suites": { "block": {"pass": 38, "fail": 0}, "doc": {"pass": 40, "fail": 0}, - "render": {"pass": 29, "fail": 0} + "render": {"pass": 29, "fail": 0}, + "api": {"pass": 26, "fail": 0} }, - "total_pass": 107, + "total_pass": 133, "total_fail": 0, - "total": 107 + "total": 133 } diff --git a/lib/content/scoreboard.md b/lib/content/scoreboard.md index 40e6e23a..526ac685 100644 --- a/lib/content/scoreboard.md +++ b/lib/content/scoreboard.md @@ -7,4 +7,5 @@ _Generated by `lib/content/conformance.sh`_ | block | 38 | 0 | 38 | | doc | 40 | 0 | 40 | | render | 29 | 0 | 29 | -| **Total** | **107** | **0** | **107** | +| api | 26 | 0 | 26 | +| **Total** | **133** | **0** | **133** | diff --git a/lib/content/tests/api.sx b/lib/content/tests/api.sx new file mode 100644 index 00000000..4c3fa3ea --- /dev/null +++ b/lib/content/tests/api.sx @@ -0,0 +1,99 @@ +;; Phase 1 — public API facade. End-to-end through content/*. + +(st-bootstrap-classes!) +(content/bootstrap!) + +;; ── build a document via the facade ── +(define d0 (content/empty "post")) +(define + h + (content/block + "heading" + "h" + (list (list "level" 1) (list "text" "Hi")))) +(define p (content/block "text" "p" (list (list "text" "World")))) +(define d1 (content/append (content/append d0 h) p)) + +(content/op? (content/insert h nil)) +(content-test "count" (content/count d1) 2) +(content-test "ids" (content/ids d1) (list "h" "p")) +(content-test "types" (content/types d1) (list "heading" "text")) +(content-test "find" (blk-id (content/find d1 "p")) "p") +(content-test "has? yes" (content/has? d1 "h") true) +(content-test "has? no" (content/has? d1 "x") false) + +;; ── content/op? distinguishes a single op from a list / a block ── +(content-test "op? on insert" (content/op? (content/insert h nil)) true) +(content-test + "op? on update" + (content/op? (content/update "p" "text" "z")) + true) +(content-test "op? on list" (content/op? (list (content/delete "h"))) false) +(content-test "op? on block" (content/op? h) false) +(content-test "op? on doc" (content/op? d1) false) + +;; ── edit with a single op ── +(define + img + (content/block + "image" + "img" + (list (list "src" "/c.png") (list "alt" "cat")))) +(define d2 (content/edit d1 (content/insert img "h"))) +(content-test "edit single op order" (content/ids d2) (list "h" "img" "p")) +(content-test "edit single immutable" (content/ids d1) (list "h" "p")) +(content-test + "edit update" + (str + (blk-send + (content/find + (content/edit d1 (content/update "p" "text" "Edited")) + "p") + "text")) + "Edited") +(content-test + "edit delete" + (content/ids (content/edit d1 (content/delete "h"))) + (list "p")) +(content-test + "edit move" + (content/ids (content/edit d1 (content/move "p" 0))) + (list "p" "h")) + +;; ── edit with a stream of ops ── +(define ops (list (content/insert img "h") (content/delete "p"))) +(content-test + "edit op stream" + (content/ids (content/edit d1 ops)) + (list "h" "img")) +(content-test "edit op stream immutable" (content/ids d1) (list "h" "p")) + +;; ── render via facade ── +(content-test + "render html" + (content/render d1 "html") + "

    Hi

    World

    ") +(content-test + "render sx" + (content/render d1 "sx") + "(article (h1 \"Hi\")(p \"World\"))") +(content-test + "render html keyword" + (content/render d1 :html) + "

    Hi

    World

    ") +(content-test + "render sx keyword" + (content/render d1 :sx) + "(article (h1 \"Hi\")(p \"World\"))") +(content-test "content/html" (content/html d1) "

    Hi

    World

    ") +(content-test "content/sx" (content/sx d1) "(article (h1 \"Hi\")(p \"World\"))") + +;; ── render reflects each version ── +(content-test + "render edited version" + (content/render (content/edit d1 (content/update "h" "text" "Hey")) "html") + "

    Hey

    World

    ") +(content-test + "render original unchanged" + (content/render d1 "html") + "

    Hi

    World

    ") diff --git a/plans/content-on-sx.md b/plans/content-on-sx.md index 3024cc52..7917f442 100644 --- a/plans/content-on-sx.md +++ b/plans/content-on-sx.md @@ -19,7 +19,7 @@ injected adapter, not core. ## Status (rolling) -`bash lib/content/conformance.sh` → **107/107** (Phase 1: blocks + doc + render) +`bash lib/content/conformance.sh` → **133/133** (Phase 1 complete: blocks + doc + render + api) ## Ground rules @@ -60,7 +60,7 @@ lib/content/api.sx ── (content/edit) (content/render) (content/history) ─ - [x] `block.sx` — typed block objects - [x] `doc.sx` — ordered tree, apply edit op, structural moves - [x] `render.sx` — block tree → HTML/SX -- [ ] `api.sx` + tests + scoreboard + conformance.sh +- [x] `api.sx` + tests + scoreboard + conformance.sh ## Phase 2 — Op log + versioning - [ ] edit ops as `persist` events; replay to any version @@ -77,6 +77,13 @@ lib/content/api.sx ── (content/edit) (content/render) (content/history) ─ ## Progress log +- 2026-06-07 — Phase 1 `api.sx` (**Phase 1 complete**): `content/*` facade over + block + doc + render. `content/bootstrap!` registers the hierarchy; + `content/edit` applies one op or an op stream; `content/render` picks the + boundary format ("html"/"sx" or keyword). Re-exports `content/new`, + `content/append`, `content/insert|update|move|delete`, `content/find`, etc. + `content/op?` distinguishes a single op from a list/block. 26 tests; suite + 133/133. content/history deferred to Phase 2 (needs the persist op log). - 2026-06-07 — Phase 1 `render.sx`: render boundary as polymorphic message dispatch. Every block and `CtDoc` answers `asHTML` / `asSx`; the document folds children via Smalltalk `inject:into:` (works on raw SX lists), so From 18696f3251411761dc37d552d9cb1a0a973328ad Mon Sep 17 00:00:00 2001 From: giles Date: Sun, 7 Jun 2026 00:15:55 +0000 Subject: [PATCH 05/49] content: persist-backed op log + versioning + diff (Phase 2 complete, 162/162) Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/content/conformance.sh | 8 ++- lib/content/scoreboard.json | 7 ++- lib/content/scoreboard.md | 3 +- lib/content/store.sx | 101 ++++++++++++++++++++++++++++++ lib/content/tests/store.sx | 121 ++++++++++++++++++++++++++++++++++++ plans/content-on-sx.md | 15 ++++- 6 files changed, 247 insertions(+), 8 deletions(-) create mode 100644 lib/content/store.sx create mode 100644 lib/content/tests/store.sx diff --git a/lib/content/conformance.sh b/lib/content/conformance.sh index de8ba21a..8f43b843 100755 --- a/lib/content/conformance.sh +++ b/lib/content/conformance.sh @@ -15,7 +15,7 @@ if [ ! -x "$SX_SERVER" ]; then fi fi -SUITES=(block doc render api) +SUITES=(block doc render api store) OUT_JSON="lib/content/scoreboard.json" OUT_MD="lib/content/scoreboard.md" @@ -33,10 +33,16 @@ run_suite() { (load "lib/smalltalk/runtime.sx") (load "lib/guest/reflective/env.sx") (load "lib/smalltalk/eval.sx") +(load "lib/persist/event.sx") +(load "lib/persist/backend.sx") +(load "lib/persist/log.sx") +(load "lib/persist/kv.sx") +(load "lib/persist/api.sx") (load "lib/content/block.sx") (load "lib/content/doc.sx") (load "lib/content/render.sx") (load "lib/content/api.sx") +(load "lib/content/store.sx") (epoch 2) (eval "(define content-test-pass 0)") (eval "(define content-test-fail 0)") diff --git a/lib/content/scoreboard.json b/lib/content/scoreboard.json index b9270ccf..25263cb1 100644 --- a/lib/content/scoreboard.json +++ b/lib/content/scoreboard.json @@ -3,9 +3,10 @@ "block": {"pass": 38, "fail": 0}, "doc": {"pass": 40, "fail": 0}, "render": {"pass": 29, "fail": 0}, - "api": {"pass": 26, "fail": 0} + "api": {"pass": 26, "fail": 0}, + "store": {"pass": 29, "fail": 0} }, - "total_pass": 133, + "total_pass": 162, "total_fail": 0, - "total": 133 + "total": 162 } diff --git a/lib/content/scoreboard.md b/lib/content/scoreboard.md index 526ac685..6a922a4b 100644 --- a/lib/content/scoreboard.md +++ b/lib/content/scoreboard.md @@ -8,4 +8,5 @@ _Generated by `lib/content/conformance.sh`_ | doc | 40 | 0 | 40 | | render | 29 | 0 | 29 | | api | 26 | 0 | 26 | -| **Total** | **133** | **0** | **133** | +| store | 29 | 0 | 29 | +| **Total** | **162** | **0** | **162** | diff --git a/lib/content/store.sx b/lib/content/store.sx new file mode 100644 index 00000000..ef840c13 --- /dev/null +++ b/lib/content/store.sx @@ -0,0 +1,101 @@ +;; content-on-sx — op log + versioning over the persist event stream. +;; +;; The op log is the source of truth. Editing a document = appending the edit op +;; as a persist event to the document's stream. Any version of the document is a +;; replay of its op stream up to a sequence number; the materialised doc is a +;; cache, never primary state. +;; +;; Requires (loaded by the harness): block.sx, doc.sx, and persist +;; (event/backend/log/kv/api). The persist backend `b` is opened by the caller +;; via (persist/open) and injected — content knows nothing about which backend. + +(define content/-stream (fn (doc-id) (str "content:" doc-id))) + +;; ── commit: append an edit op as an event. `at` is a caller-supplied logical +;; timestamp (Date.now is unavailable in-kernel). Returns the stored event. ── +(define + content/commit! + (fn + (b doc-id op at) + (persist/append b (content/-stream doc-id) (get op :op) at op))) + +(define + content/commit-all! + (fn + (b doc-id ops at) + (if + (= (len ops) 0) + nil + (begin + (content/commit! b doc-id (first ops) at) + (content/commit-all! b doc-id (rest ops) at))))) + +;; ── read the raw log / op stream ── +(define + content/log + (fn (b doc-id) (persist/read b (content/-stream doc-id)))) + +(define + content/ops + (fn + (b doc-id) + (map (fn (ev) (persist/event-data ev)) (content/log b doc-id)))) + +;; logical version count (highest seq assigned, survives compaction) +(define + content/version-count + (fn (b doc-id) (persist/last-seq b (content/-stream doc-id)))) + +;; ── replay ── +;; head — materialise the latest document by folding all ops. +(define + content/head + (fn (b doc-id) (doc-apply-all (doc-empty doc-id) (content/ops b doc-id)))) + +;; at — materialise the document as of sequence `seq` (a version). +(define + content/at + (fn + (b doc-id seq) + (let + ((evs (filter (fn (ev) (<= (persist/event-seq ev) seq)) (content/log b doc-id)))) + (doc-apply-all + (doc-empty doc-id) + (map (fn (ev) (persist/event-data ev)) evs))))) + +;; ── history: per-version metadata, oldest-first ── +(define + content/history + (fn (b doc-id) (map (fn (ev) {:type (persist/event-type ev) :at (persist/event-at ev) :seq (persist/event-seq ev)}) (content/log b doc-id)))) + +;; ── diff between two materialised document versions ── +;; Returns {:added (ids) :removed (ids) :changed (ids)} where changed = ids +;; present in both whose block content differs. +(define + content/-missing? + (fn (doc id) (= (ct-index-of (doc-blocks doc) id) -1))) + +(define + content/-changed + (fn + (old new) + (filter + (fn + (id) + (let + ((bo (doc-find old id)) (bn (doc-find new id))) + (cond + ((= bo nil) false) + ((= bn nil) false) + ((= bo bn) false) + (else true)))) + (doc-ids old)))) + +(define content/diff (fn (old new) {:changed (content/-changed old new) :removed (filter (fn (id) (content/-missing? new id)) (doc-ids old)) :added (filter (fn (id) (content/-missing? old id)) (doc-ids new))})) + +;; convenience: diff two persisted versions by seq. +(define + content/diff-versions + (fn + (b doc-id seq-a seq-b) + (content/diff (content/at b doc-id seq-a) (content/at b doc-id seq-b)))) diff --git a/lib/content/tests/store.sx b/lib/content/tests/store.sx new file mode 100644 index 00000000..6065bc5e --- /dev/null +++ b/lib/content/tests/store.sx @@ -0,0 +1,121 @@ +;; Phase 2 — op log + versioning over persist. The log is the source of truth; +;; any version is a replay of the op stream up to a seq. + +(st-bootstrap-classes!) +(content-bootstrap-blocks!) +(content-bootstrap-doc!) + +(define B (persist/open)) +(define h (mk-heading "h" 1 "Title")) +(define p (mk-text "p" "Body")) +(define img (mk-image "img" "/c.png" "cat")) + +;; ── commit an op stream ── +(content/commit! B "post" (op-insert h nil) 10) +(content/commit! B "post" (op-insert p "h") 11) +(content/commit! B "post" (op-insert img "h") 12) +(content/commit! B "post" (op-update "p" "text" "Edited") 13) +(content/commit! B "post" (op-delete "img") 14) + +(content-test "version-count" (content/version-count B "post") 5) +(content-test "log length" (len (content/log B "post")) 5) + +;; ── head: latest materialised document ── +(content-test "head ids" (doc-ids (content/head B "post")) (list "h" "p")) +(content-test + "head p edited" + (str (blk-send (doc-find (content/head B "post") "p") "text")) + "Edited") + +;; ── replay to any version ── +(content-test + "at seq1" + (doc-ids (content/at B "post" 1)) + (list "h")) +(content-test + "at seq2" + (doc-ids (content/at B "post" 2)) + (list "h" "p")) +(content-test + "at seq3" + (doc-ids (content/at B "post" 3)) + (list "h" "img" "p")) +(content-test + "at seq3 p original" + (str (blk-send (doc-find (content/at B "post" 3) "p") "text")) + "Body") +(content-test + "at seq4 p edited" + (str (blk-send (doc-find (content/at B "post" 4) "p") "text")) + "Edited") +(content-test + "at seq5 img gone" + (doc-ids (content/at B "post" 5)) + (list "h" "p")) +(content-test + "at seq0 empty" + (doc-ids (content/at B "post" 0)) + (list)) + +;; ── ops accessor ── +(content-test + "ops kinds" + (map (fn (o) (get o :op)) (content/ops B "post")) + (list "insert" "insert" "insert" "update" "delete")) + +;; ── history metadata ── +(define hist (content/history B "post")) +(content-test "history length" (len hist) 5) +(content-test "history first seq" (get (first hist) :seq) 1) +(content-test "history first type" (get (first hist) :type) "insert") +(content-test "history first at" (get (first hist) :at) 10) +(content-test + "history fourth type" + (get (nth hist 3) :type) + "update") + +;; ── diff between versions ── +(define dvf (content/diff-versions B "post" 1 3)) +(content-test "diff added" (get dvf :added) (list "img" "p")) +(content-test "diff removed empty" (get dvf :removed) (list)) +(content-test "diff changed empty" (get dvf :changed) (list)) + +(define dvf2 (content/diff-versions B "post" 3 5)) +(content-test "diff2 removed" (get dvf2 :removed) (list "img")) +(content-test "diff2 changed" (get dvf2 :changed) (list "p")) +(content-test "diff2 added empty" (get dvf2 :added) (list)) + +;; ── direct diff of two materialised docs ── +(define da (content/at B "post" 2)) +(define db (content/at B "post" 5)) +(content-test + "direct diff changed" + (get (content/diff da db) :changed) + (list "p")) +(content-test + "direct diff no-op" + (get (content/diff da da) :changed) + (list)) + +;; ── commit-all batch ── +(define B2 (persist/open)) +(content/commit-all! + B2 + "doc2" + (list (op-insert h nil) (op-insert p "h")) + 1) +(content-test "commit-all count" (content/version-count B2 "doc2") 2) +(content-test + "commit-all head" + (doc-ids (content/head B2 "doc2")) + (list "h" "p")) + +;; ── stream isolation ── +(content-test + "separate stream empty" + (content/version-count B "doc2") + 0) +(content-test + "head of empty stream" + (doc-ids (content/head B "never")) + (list)) diff --git a/plans/content-on-sx.md b/plans/content-on-sx.md index 7917f442..8a9782d8 100644 --- a/plans/content-on-sx.md +++ b/plans/content-on-sx.md @@ -19,7 +19,7 @@ injected adapter, not core. ## Status (rolling) -`bash lib/content/conformance.sh` → **133/133** (Phase 1 complete: blocks + doc + render + api) +`bash lib/content/conformance.sh` → **162/162** (Phase 1 complete + Phase 2: persist op log) ## Ground rules @@ -63,8 +63,8 @@ lib/content/api.sx ── (content/edit) (content/render) (content/history) ─ - [x] `api.sx` + tests + scoreboard + conformance.sh ## Phase 2 — Op log + versioning -- [ ] edit ops as `persist` events; replay to any version -- [ ] `(content/history doc)`, diff between versions +- [x] edit ops as `persist` events; replay to any version +- [x] `(content/history doc)`, diff between versions ## Phase 3 — Collaborative merge (CRDT) - [ ] commutative/idempotent op merge @@ -77,6 +77,15 @@ lib/content/api.sx ── (content/edit) (content/render) (content/history) ─ ## Progress log +- 2026-06-07 — Phase 2 `store.sx` (**Phase 2 complete**): op log + versioning + over the persist event stream. `content/commit!` appends an edit op as a + persist event to the doc's stream (`content:`); the log is the source of + truth. `content/head` / `content/at b id seq` replay the op stream to the + latest / any version (materialised doc is a cache, never primary state). + `content/history` returns per-version metadata; `content/diff` / + `content/diff-versions` report added/removed/changed block ids. Backend is + injected via `(persist/open)` — content knows nothing about which backend. + Minimal persist load (event/backend/log/kv/api). 29 tests; suite 162/162. - 2026-06-07 — Phase 1 `api.sx` (**Phase 1 complete**): `content/*` facade over block + doc + render. `content/bootstrap!` registers the hierarchy; `content/edit` applies one op or an op stream; `content/render` picks the From edf0ab17554f4dcd8a58fa6dba240036bc1b1eed Mon Sep 17 00:00:00 2001 From: giles Date: Sun, 7 Jun 2026 00:29:38 +0000 Subject: [PATCH 06/49] content: CvRDT collaborative merge + 34 convergence tests (Phase 3 complete, 196/196) Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/content/conformance.sh | 3 +- lib/content/crdt.sx | 378 ++++++++++++++++++++++++++++++++++++ lib/content/scoreboard.json | 7 +- lib/content/scoreboard.md | 3 +- lib/content/tests/crdt.sx | 315 ++++++++++++++++++++++++++++++ plans/content-on-sx.md | 18 +- 6 files changed, 716 insertions(+), 8 deletions(-) create mode 100644 lib/content/crdt.sx create mode 100644 lib/content/tests/crdt.sx diff --git a/lib/content/conformance.sh b/lib/content/conformance.sh index 8f43b843..1a7e20e0 100755 --- a/lib/content/conformance.sh +++ b/lib/content/conformance.sh @@ -15,7 +15,7 @@ if [ ! -x "$SX_SERVER" ]; then fi fi -SUITES=(block doc render api store) +SUITES=(block doc render api store crdt) OUT_JSON="lib/content/scoreboard.json" OUT_MD="lib/content/scoreboard.md" @@ -43,6 +43,7 @@ run_suite() { (load "lib/content/render.sx") (load "lib/content/api.sx") (load "lib/content/store.sx") +(load "lib/content/crdt.sx") (epoch 2) (eval "(define content-test-pass 0)") (eval "(define content-test-fail 0)") diff --git a/lib/content/crdt.sx b/lib/content/crdt.sx new file mode 100644 index 00000000..c48cc16f --- /dev/null +++ b/lib/content/crdt.sx @@ -0,0 +1,378 @@ +;; content-on-sx — collaborative merge (state-based CvRDT). +;; +;; The merge is a join (least upper bound) on a semilattice, so it is +;; commutative, associative and idempotent BY CONSTRUCTION — applying ops in any +;; order, or merging replicas in any order / twice, converges to the same +;; document. This is NOT last-write-wins-as-cop-out: ordering uses unique dense +;; position keys (Logoot), presence uses OR-tombstones (remove-wins), and each +;; field is an LWW-Register keyed by a logical (ts, actor) clock — an explicit, +;; deterministic per-field conflict policy. +;; +;; Every op (insert/update/delete) contributes a PARTIAL element; the per-id +;; state is the join of all contributions. So update-before-insert and +;; delete-before-insert are not lost — they merge when the rest arrives. +;; +;; Shapes: +;; state = {:elements element>} +;; element = {:id :pos :type :deleted :fields register>} +;; register = {:value v :ts :actor } +;; position = list of cells; cell = (list digit actor); lexicographic order +;; +;; Requires (loaded by harness): block.sx, doc.sx. + +(define CRDT-BASE 65536) + +;; ── position order (Logoot) ── +(define + crdt-cell-cmp + (fn + (c1 c2) + (let + ((d1 (first c1)) (d2 (first c2))) + (cond + ((< d1 d2) -1) + ((> d1 d2) 1) + (else + (let + ((a1 (first (rest c1))) (a2 (first (rest c2)))) + (cond + ((< a1 a2) -1) + ((> a1 a2) 1) + (else 0)))))))) + +(define + crdt-pos-compare + (fn + (p1 p2) + (cond + ((and (= (len p1) 0) (= (len p2) 0)) 0) + ((= (len p1) 0) -1) + ((= (len p2) 0) 1) + (else + (let + ((c (crdt-cell-cmp (first p1) (first p2)))) + (if (= c 0) (crdt-pos-compare (rest p1) (rest p2)) c)))))) + +;; single-cell position constructor (handy for explicit tests) +(define crdt-pos (fn (digit actor) (list (list digit actor)))) + +;; allocate a position strictly between left and right (nil = unbounded) +(define + cr-alloc + (fn + (left right actor i acc) + (let + ((ld (if (< i (len left)) (first (nth left i)) 0)) + (rd (if (< i (len right)) (first (nth right i)) CRDT-BASE))) + (if + (> (- rd ld) 1) + (append + acc + (list + (list + (+ + ld + (+ + 1 + (floor (/ (- (- rd ld) 1) 2)))) + actor))) + (cr-alloc + left + right + actor + (+ i 1) + (append + acc + (list + (list + ld + (if (< i (len left)) (first (rest (nth left i))) actor))))))))) + +(define + crdt-pos-between + (fn + (left right actor) + (cr-alloc + (if (= left nil) (list) left) + (if (= right nil) (list) right) + actor + 0 + (list)))) + +;; ── register (LWW by logical (ts, actor)) ── +(define + crdt-reg-max + (fn + (r1 r2) + (cond + ((= r1 nil) r2) + ((= r2 nil) r1) + (else + (let + ((t1 (get r1 :ts)) (t2 (get r2 :ts))) + (cond + ((> t1 t2) r1) + ((< t1 t2) r2) + (else (if (>= (get r1 :actor) (get r2 :actor)) r1 r2)))))))) + +;; ── small set/dict helpers ── +(define + crdt-member? + (fn + (x xs) + (cond + ((= (len xs) 0) false) + ((= (first xs) x) true) + (else (crdt-member? x (rest xs)))))) + +(define + crdt-dedup-loop + (fn + (xs seen) + (if + (= (len xs) 0) + (reverse seen) + (if + (crdt-member? (first xs) seen) + (crdt-dedup-loop (rest xs) seen) + (crdt-dedup-loop (rest xs) (cons (first xs) seen)))))) + +(define crdt-dedup (fn (xs) (crdt-dedup-loop xs (list)))) + +(define + crdt-union-keys + (fn (d1 d2) (crdt-dedup (append (keys d1) (keys d2))))) + +;; ── element join ── +(define + crdt-merge-pos + (fn + (p1 p2) + (cond + ((= p1 nil) p2) + ((= p2 nil) p1) + ((<= (crdt-pos-compare p1 p2) 0) p1) + (else p2)))) + +(define crdt-merge-type (fn (t1 t2) (if (= t1 nil) t2 t1))) + +(define + crdt-merge-fields-loop + (fn + (names f1 f2 acc) + (if + (= (len names) 0) + acc + (let + ((nm (first names))) + (crdt-merge-fields-loop + (rest names) + f1 + f2 + (assoc acc nm (crdt-reg-max (get f1 nm) (get f2 nm)))))))) + +(define + crdt-merge-fields + (fn + (f1 f2) + (crdt-merge-fields-loop (crdt-union-keys f1 f2) f1 f2 {}))) + +(define crdt-merge-element (fn (e1 e2) {:fields (crdt-merge-fields (get e1 :fields) (get e2 :fields)) :id (get e1 :id) :type (crdt-merge-type (get e1 :type) (get e2 :type)) :deleted (or (= (get e1 :deleted) true) (= (get e2 :deleted) true)) :pos (crdt-merge-pos (get e1 :pos) (get e2 :pos))})) + +;; ── state ── +(define crdt-empty (fn () {:elements {}})) + +(define + crdt-add-element + (fn + (state elem) + (let + ((elems (get state :elements)) (id (get elem :id))) + (let + ((existing (get elems id))) + (assoc + state + :elements (assoc + elems + id + (if (= existing nil) elem (crdt-merge-element existing elem)))))))) + +(define + crdt-build-fields-loop + (fn + (pairs ts actor acc) + (if + (= (len pairs) 0) + acc + (crdt-build-fields-loop + (rest pairs) + ts + actor + (assoc acc (first (first pairs)) {:ts ts :actor actor :value (first (rest (first pairs)))}))))) + +(define + crdt-build-fields + (fn (pairs ts actor) (crdt-build-fields-loop pairs ts actor {}))) + +;; ── ops as partial-element contributions ── +(define + crdt-insert + (fn + (state id type pos fields ts actor) + (crdt-add-element state {:fields (crdt-build-fields fields ts actor) :id id :type type :deleted false :pos pos}))) + +(define + crdt-update + (fn (state id fname value ts actor) (crdt-add-element state {:fields (assoc {} fname {:ts ts :actor actor :value value}) :id id :type nil :deleted false :pos nil}))) + +(define crdt-delete (fn (state id) (crdt-add-element state {:fields {} :id id :type nil :deleted true :pos nil}))) + +;; ── state merge (join) ── +(define + crdt-merge-loop + (fn + (ids ea eb acc) + (if + (= (len ids) 0) + acc + (let + ((id (first ids))) + (let + ((x (get ea id)) (y (get eb id))) + (crdt-merge-loop + (rest ids) + ea + eb + (assoc + acc + id + (cond + ((= x nil) y) + ((= y nil) x) + (else (crdt-merge-element x y)))))))))) + +(define crdt-merge (fn (a b) {:elements (crdt-merge-loop (crdt-union-keys (get a :elements) (get b :elements)) (get a :elements) (get b :elements) {})})) + +(define + crdt-merge-all + (fn + (states) + (if + (= (len states) 0) + (crdt-empty) + (if + (= (len states) 1) + (first states) + (crdt-merge (first states) (crdt-merge-all (rest states))))))) + +;; ── op interpreter ── +(define crdt-op-insert (fn (id type pos fields ts actor) {:ts ts :fields fields :id id :type type :op "insert" :actor actor :pos pos})) + +(define crdt-op-update (fn (id field value ts actor) {:ts ts :field field :id id :op "update" :actor actor :value value})) + +(define crdt-op-delete (fn (id) {:id id :op "delete"})) + +(define + crdt-apply + (fn + (state op) + (let + ((k (get op :op))) + (cond + ((= k "insert") + (crdt-insert + state + (get op :id) + (get op :type) + (get op :pos) + (get op :fields) + (get op :ts) + (get op :actor))) + ((= k "update") + (crdt-update + state + (get op :id) + (get op :field) + (get op :value) + (get op :ts) + (get op :actor))) + ((= k "delete") (crdt-delete state (get op :id))) + (else (error (str "unknown crdt op: " k))))))) + +(define + crdt-apply-all + (fn + (state ops) + (if + (= (len ops) 0) + state + (crdt-apply-all (crdt-apply state (first ops)) (rest ops))))) + +;; ── materialise to a Phase-1 document ── +(define + crdt-elements-list + (fn + (state) + (map + (fn (id) (get (get state :elements) id)) + (keys (get state :elements))))) + +(define + crdt-live? + (fn + (e) + (and + (= (get e :deleted) false) + (if (= (get e :pos) nil) false true) + (if (= (get e :type) nil) false true)))) + +(define + crdt-live-elements + (fn (state) (filter crdt-live? (crdt-elements-list state)))) + +(define + crdt-insert-sorted + (fn + (e sorted) + (cond + ((= (len sorted) 0) (list e)) + ((< (crdt-pos-compare (get e :pos) (get (first sorted) :pos)) 0) + (cons e sorted)) + (else (cons (first sorted) (crdt-insert-sorted e (rest sorted))))))) + +(define + crdt-sort-by-pos + (fn + (elems) + (if + (= (len elems) 0) + (list) + (crdt-insert-sorted (first elems) (crdt-sort-by-pos (rest elems)))))) + +(define + crdt-field-pairs + (fn + (fields) + (map (fn (nm) (list nm (get (get fields nm) :value))) (keys fields)))) + +(define + crdt-element->block + (fn + (e) + (mk-block (get e :type) (get e :id) (crdt-field-pairs (get e :fields))))) + +(define + crdt-order + (fn + (state) + (map + (fn (e) (get e :id)) + (crdt-sort-by-pos (crdt-live-elements state))))) + +(define + crdt-materialize + (fn + (doc-id state) + (doc-new + doc-id + (map crdt-element->block (crdt-sort-by-pos (crdt-live-elements state)))))) diff --git a/lib/content/scoreboard.json b/lib/content/scoreboard.json index 25263cb1..490e10f4 100644 --- a/lib/content/scoreboard.json +++ b/lib/content/scoreboard.json @@ -4,9 +4,10 @@ "doc": {"pass": 40, "fail": 0}, "render": {"pass": 29, "fail": 0}, "api": {"pass": 26, "fail": 0}, - "store": {"pass": 29, "fail": 0} + "store": {"pass": 29, "fail": 0}, + "crdt": {"pass": 34, "fail": 0} }, - "total_pass": 162, + "total_pass": 196, "total_fail": 0, - "total": 162 + "total": 196 } diff --git a/lib/content/scoreboard.md b/lib/content/scoreboard.md index 6a922a4b..fc7fac98 100644 --- a/lib/content/scoreboard.md +++ b/lib/content/scoreboard.md @@ -9,4 +9,5 @@ _Generated by `lib/content/conformance.sh`_ | render | 29 | 0 | 29 | | api | 26 | 0 | 26 | | store | 29 | 0 | 29 | -| **Total** | **162** | **0** | **162** | +| crdt | 34 | 0 | 34 | +| **Total** | **196** | **0** | **196** | diff --git a/lib/content/tests/crdt.sx b/lib/content/tests/crdt.sx new file mode 100644 index 00000000..542554b4 --- /dev/null +++ b/lib/content/tests/crdt.sx @@ -0,0 +1,315 @@ +;; Phase 3 — collaborative merge (CvRDT). The merge is a join: commutative, +;; associative, idempotent. Tests apply ops in any order, twice, and merge +;; replicas both ways — all must converge to identical state. + +(st-bootstrap-classes!) +(content-bootstrap-blocks!) +(content-bootstrap-doc!) +(content-bootstrap-render!) + +(define same? (fn (a b) (= (get a :elements) (get b :elements)))) + +;; ── position order (Logoot) ── +(content-test + "pos lt" + (crdt-pos-compare + (crdt-pos 1 0) + (crdt-pos 2 0)) + -1) +(content-test + "pos gt" + (crdt-pos-compare + (crdt-pos 2 0) + (crdt-pos 1 0)) + 1) +(content-test + "pos eq" + (crdt-pos-compare + (crdt-pos 1 0) + (crdt-pos 1 0)) + 0) +(content-test + "pos actor tiebreak" + (crdt-pos-compare + (crdt-pos 1 1) + (crdt-pos 1 2)) + -1) +(content-test + "between > left" + (< + (crdt-pos-compare + (crdt-pos 1 0) + (crdt-pos-between + (crdt-pos 1 0) + (crdt-pos 2 0) + 9)) + 0) + true) +(content-test + "between < right" + (< + (crdt-pos-compare + (crdt-pos-between + (crdt-pos 1 0) + (crdt-pos 2 0) + 9) + (crdt-pos 2 0)) + 0) + true) +(content-test + "between start < right" + (< + (crdt-pos-compare + (crdt-pos-between nil (crdt-pos 5 0) 9) + (crdt-pos 5 0)) + 0) + true) +(content-test + "between end > left" + (< + (crdt-pos-compare + (crdt-pos 5 0) + (crdt-pos-between (crdt-pos 5 0) nil 9)) + 0) + true) + +;; ── build + materialise ── +(define + base + (crdt-insert + (crdt-insert + (crdt-empty) + "h" + "heading" + (crdt-pos 1 0) + (list (list "level" 1) (list "text" "Title")) + 1 + 0) + "p" + "text" + (crdt-pos 2 0) + (list (list "text" "Body")) + 1 + 0)) + +(content-test "order" (crdt-order base) (list "h" "p")) +(content-test + "materialize ids" + (doc-ids (crdt-materialize "d" base)) + (list "h" "p")) +(content-test + "materialize render" + (asHTML (crdt-materialize "d" base)) + "

    Title

    Body

    ") + +;; ── commutativity: ops in any order converge ── +(define + opA + (crdt-op-insert + "x" + "text" + (crdt-pos 3 0) + (list (list "text" "X")) + 2 + 1)) +(define opB (crdt-op-update "p" "text" "Edited" 5 1)) +(define opC (crdt-op-delete "h")) +(define s-abc (crdt-apply-all base (list opA opB opC))) +(define s-cba (crdt-apply-all base (list opC opB opA))) +(define s-bca (crdt-apply-all base (list opB opC opA))) +(content-test "commutative abc=cba" (same? s-abc s-cba) true) +(content-test "commutative abc=bca" (same? s-abc s-bca) true) +(content-test "commutative result order" (crdt-order s-abc) (list "p" "x")) + +;; ── idempotence: applying ops twice changes nothing ── +(content-test + "idempotent ops" + (same? s-abc (crdt-apply-all s-abc (list opA opB opC))) + true) + +;; ── update-before-insert is not lost ── +(define + ub + (crdt-apply-all + (crdt-empty) + (list + (crdt-op-update "z" "text" "late" 3 1) + (crdt-op-insert + "z" + "text" + (crdt-pos 1 0) + (list (list "text" "orig")) + 1 + 1)))) +(content-test + "update before insert kept" + (str (blk-send (doc-find (crdt-materialize "d" ub) "z") "text")) + "late") + +;; ── delete-before-insert: remove-wins ── +(define + db + (crdt-apply-all + (crdt-empty) + (list + (crdt-op-delete "k") + (crdt-op-insert + "k" + "text" + (crdt-pos 1 0) + (list (list "text" "x")) + 1 + 1)))) +(content-test "delete before insert removes" (crdt-order db) (list)) + +;; ── concurrent inserts converge + deterministic order ── +(define + rA + (crdt-insert + base + "a1" + "text" + (crdt-pos 5 1) + (list (list "text" "A")) + 2 + 1)) +(define + rB + (crdt-insert + base + "b1" + "text" + (crdt-pos 5 2) + (list (list "text" "B")) + 2 + 2)) +(content-test + "merge commutes" + (same? (crdt-merge rA rB) (crdt-merge rB rA)) + true) +(content-test + "merge order deterministic AB" + (crdt-order (crdt-merge rA rB)) + (list "h" "p" "a1" "b1")) +(content-test + "merge order deterministic BA" + (crdt-order (crdt-merge rB rA)) + (list "h" "p" "a1" "b1")) + +;; ── merge idempotence ── +(define mAB (crdt-merge rA rB)) +(content-test "merge idempotent self" (same? (crdt-merge mAB mAB) mAB) true) +(content-test + "merge idempotent remerge" + (same? (crdt-merge mAB rA) mAB) + true) + +;; ── concurrent same-field update: LWW by (ts, actor) ── +(define u1 (crdt-update base "p" "text" "v-ts5" 5 1)) +(define u2 (crdt-update base "p" "text" "v-ts7" 7 2)) +(content-test + "LWW higher ts wins" + (str + (blk-send + (doc-find (crdt-materialize "d" (crdt-merge u1 u2)) "p") + "text")) + "v-ts7") +(content-test + "LWW commutes" + (same? (crdt-merge u1 u2) (crdt-merge u2 u1)) + true) +(define t1 (crdt-update base "p" "text" "actor1" 9 1)) +(define t2 (crdt-update base "p" "text" "actor2" 9 2)) +(content-test + "LWW tie -> actor wins" + (str + (blk-send + (doc-find (crdt-materialize "d" (crdt-merge t1 t2)) "p") + "text")) + "actor2") + +;; ── concurrent disjoint-field updates both survive ── +(define f1 (crdt-update base "h" "text" "NewTitle" 5 1)) +(define f2 (crdt-update base "h" "level" 3 5 2)) +(define fm (crdt-merge f1 f2)) +(content-test + "disjoint field text" + (str (blk-send (doc-find (crdt-materialize "d" fm) "h") "text")) + "NewTitle") +(content-test + "disjoint field level" + (blk-send (doc-find (crdt-materialize "d" fm) "h") "level") + 3) +(content-test "disjoint commutes" (same? fm (crdt-merge f2 f1)) true) + +;; ── associativity ── +(define c1 (crdt-update base "p" "text" "c1" 4 1)) +(define + c2 + (crdt-insert + base + "n2" + "text" + (crdt-pos 6 0) + (list (list "text" "N")) + 2 + 2)) +(define c3 (crdt-delete base "h")) +(content-test + "associative" + (same? + (crdt-merge (crdt-merge c1 c2) c3) + (crdt-merge c1 (crdt-merge c2 c3))) + true) +(content-test + "merge-all = fold" + (same? + (crdt-merge-all (list c1 c2 c3)) + (crdt-merge c1 (crdt-merge c2 c3))) + true) + +;; ── full convergence: two replicas, divergent edits, merge both ways ── +(define + repl-1 + (crdt-apply-all + base + (list + (crdt-op-update "p" "text" "from-1" 5 1) + (crdt-op-insert + "img" + "image" + (crdt-pos-between + (crdt-pos 1 0) + (crdt-pos 2 0) + 1) + (list (list "src" "/a.png") (list "alt" "a")) + 6 + 1)))) +(define + repl-2 + (crdt-apply-all + base + (list + (crdt-op-delete "h") + (crdt-op-update "p" "text" "from-2" 7 2)))) +(content-test + "two-replica converges" + (same? (crdt-merge repl-1 repl-2) (crdt-merge repl-2 repl-1)) + true) +(content-test + "two-replica result order" + (crdt-order (crdt-merge repl-1 repl-2)) + (list "img" "p")) +(content-test + "two-replica LWW field" + (str + (blk-send + (doc-find (crdt-materialize "d" (crdt-merge repl-1 repl-2)) "p") + "text")) + "from-2") +(content-test + "two-replica idempotent" + (same? + (crdt-merge (crdt-merge repl-1 repl-2) repl-1) + (crdt-merge repl-1 repl-2)) + true) diff --git a/plans/content-on-sx.md b/plans/content-on-sx.md index 8a9782d8..2d8904cd 100644 --- a/plans/content-on-sx.md +++ b/plans/content-on-sx.md @@ -19,7 +19,7 @@ injected adapter, not core. ## Status (rolling) -`bash lib/content/conformance.sh` → **162/162** (Phase 1 complete + Phase 2: persist op log) +`bash lib/content/conformance.sh` → **196/196** (Phases 1–3 complete: blocks, doc, render, api, persist op log, CRDT merge) ## Ground rules @@ -67,8 +67,8 @@ lib/content/api.sx ── (content/edit) (content/render) (content/history) ─ - [x] `(content/history doc)`, diff between versions ## Phase 3 — Collaborative merge (CRDT) -- [ ] commutative/idempotent op merge -- [ ] concurrent-edit tests (any order, double-apply → identical) +- [x] commutative/idempotent op merge +- [x] concurrent-edit tests (any order, double-apply → identical) ## Phase 4 — External sync + federation - [ ] Ghost/CMS sync via injected adapter (import/export) @@ -77,6 +77,18 @@ lib/content/api.sx ── (content/edit) (content/render) (content/history) ─ ## Progress log +- 2026-06-07 — Phase 3 `crdt.sx` (**Phase 3 complete**): collaborative merge as + a state-based CvRDT. Merge is a join (lub) on a semilattice → commutative, + associative, idempotent by construction. Ordering = unique dense Logoot + position keys (cell = (digit actor), lexicographic); presence = OR-tombstones + (remove-wins); each field = an LWW-Register keyed by logical (ts, actor). Every + op contributes a PARTIAL element and per-id state is their join, so + update-/delete-before-insert are not lost. `crdt-materialize` bridges back to a + Phase-1 `CtDoc` (sort live elements by pos → blocks). Tests prove: ops in any + order converge, double-apply is a no-op, merge commutes/associates/is + idempotent, concurrent inserts order deterministically, same-field LWW by + (ts,actor), disjoint fields both survive, two divergent replicas converge both + ways. 34 tests; suite 196/196. - 2026-06-07 — Phase 2 `store.sx` (**Phase 2 complete**): op log + versioning over the persist event stream. `content/commit!` appends an edit op as a persist event to the doc's stream (`content:`); the log is the source of From ab48a3ba1f73e3102e257d6fdb5765835a9002f8 Mon Sep 17 00:00:00 2001 From: giles Date: Sun, 7 Jun 2026 00:37:12 +0000 Subject: [PATCH 07/49] content: Ghost/CMS sync via injected adapter + round-trip tests (210/210) Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/content/conformance.sh | 3 +- lib/content/scoreboard.json | 7 ++-- lib/content/scoreboard.md | 3 +- lib/content/sync.sx | 71 +++++++++++++++++++++++++++++++++++ lib/content/tests/sync.sx | 74 +++++++++++++++++++++++++++++++++++++ plans/content-on-sx.md | 14 +++++-- 6 files changed, 164 insertions(+), 8 deletions(-) create mode 100644 lib/content/sync.sx create mode 100644 lib/content/tests/sync.sx diff --git a/lib/content/conformance.sh b/lib/content/conformance.sh index 1a7e20e0..ac8399c9 100755 --- a/lib/content/conformance.sh +++ b/lib/content/conformance.sh @@ -15,7 +15,7 @@ if [ ! -x "$SX_SERVER" ]; then fi fi -SUITES=(block doc render api store crdt) +SUITES=(block doc render api store crdt sync) OUT_JSON="lib/content/scoreboard.json" OUT_MD="lib/content/scoreboard.md" @@ -44,6 +44,7 @@ run_suite() { (load "lib/content/api.sx") (load "lib/content/store.sx") (load "lib/content/crdt.sx") +(load "lib/content/sync.sx") (epoch 2) (eval "(define content-test-pass 0)") (eval "(define content-test-fail 0)") diff --git a/lib/content/scoreboard.json b/lib/content/scoreboard.json index 490e10f4..3c580915 100644 --- a/lib/content/scoreboard.json +++ b/lib/content/scoreboard.json @@ -5,9 +5,10 @@ "render": {"pass": 29, "fail": 0}, "api": {"pass": 26, "fail": 0}, "store": {"pass": 29, "fail": 0}, - "crdt": {"pass": 34, "fail": 0} + "crdt": {"pass": 34, "fail": 0}, + "sync": {"pass": 14, "fail": 0} }, - "total_pass": 196, + "total_pass": 210, "total_fail": 0, - "total": 196 + "total": 210 } diff --git a/lib/content/scoreboard.md b/lib/content/scoreboard.md index fc7fac98..7fb590aa 100644 --- a/lib/content/scoreboard.md +++ b/lib/content/scoreboard.md @@ -10,4 +10,5 @@ _Generated by `lib/content/conformance.sh`_ | api | 26 | 0 | 26 | | store | 29 | 0 | 29 | | crdt | 34 | 0 | 34 | -| **Total** | **196** | **0** | **196** | +| sync | 14 | 0 | 14 | +| **Total** | **210** | **0** | **210** | diff --git a/lib/content/sync.sx b/lib/content/sync.sx new file mode 100644 index 00000000..45b6ac60 --- /dev/null +++ b/lib/content/sync.sx @@ -0,0 +1,71 @@ +;; content-on-sx — external CMS sync via an injected adapter. +;; +;; Sync is a peripheral, not a feature. The core defines a SHAPE — an adapter is +;; a dict {:import (fn external doc-id -> doc) :export (fn doc -> external)} — and +;; delegates to it. The core knows nothing about Ghost's data model; all +;; translation lives in the adapter. Swap the adapter and the core is unchanged; +;; if Ghost goes away, nothing here does. +;; +;; Requires (loaded by harness): block.sx, doc.sx. + +;; ── generic boundary: pure delegation ── +(define + content/import + (fn (adapter external doc-id) ((get adapter :import) external doc-id))) + +(define content/export (fn (adapter doc) ((get adapter :export) doc))) + +;; round-trip a document through an adapter (export then import). +(define + content/round-trip + (fn + (adapter doc) + (content/import adapter (content/export adapter doc) (doc-id doc)))) + +;; ── a Ghost-flavoured adapter (the peripheral). Ghost knowledge is confined +;; here: a post is {:title :sections (list section)}; a section is a tagged dict +;; {:kind ...} that this adapter maps to/from content blocks. ── +(define + ghost-section->block + (fn + (sec) + (let + ((kind (get sec :kind)) (id (get sec :id))) + (cond + ((= kind "heading") + (mk-heading id (get sec :level) (get sec :text))) + ((= kind "paragraph") (mk-text id (get sec :text))) + ((= kind "image") (mk-image id (get sec :src) (get sec :alt))) + ((= kind "code") (mk-code id (get sec :language) (get sec :text))) + ((= kind "quote") (mk-quote id (get sec :cite) (get sec :text))) + ((= kind "hr") (mk-divider id)) + ((= kind "list") (mk-list id (get sec :ordered) (get sec :items))) + ((= kind "embed") (mk-embed id (get sec :url) (get sec :provider))) + (else (mk-text id (get sec :text))))))) + +(define + block->ghost-section + (fn + (b) + (let + ((t (blk-type b)) (id (blk-id b))) + (cond + ((= t "heading") {:id id :text (str (blk-send b "text")) :kind "heading" :level (blk-send b "level")}) + ((= t "text") {:id id :text (str (blk-send b "text")) :kind "paragraph"}) + ((= t "image") {:id id :src (str (blk-send b "src")) :alt (str (blk-send b "alt")) :kind "image"}) + ((= t "code") {:id id :text (str (blk-send b "text")) :kind "code" :language (str (blk-send b "language"))}) + ((= t "quote") {:cite (str (blk-send b "cite")) :id id :text (str (blk-send b "text")) :kind "quote"}) + ((= t "divider") {:id id :kind "hr"}) + ((= t "list") {:items (blk-send b "items") :id id :kind "list" :ordered (blk-send b "ordered")}) + ((= t "embed") {:id id :provider (str (blk-send b "provider")) :kind "embed" :url (str (blk-send b "url"))}) + (else {:id id :text "" :kind "paragraph"}))))) + +(define + ghost-import + (fn + (post doc-id) + (doc-new doc-id (map ghost-section->block (get post :sections))))) + +(define ghost-export (fn (doc) {:sections (map block->ghost-section (doc-blocks doc))})) + +(define ghost-adapter {:export ghost-export :import ghost-import}) diff --git a/lib/content/tests/sync.sx b/lib/content/tests/sync.sx new file mode 100644 index 00000000..fa87cb19 --- /dev/null +++ b/lib/content/tests/sync.sx @@ -0,0 +1,74 @@ +;; Phase 4 — external CMS sync via injected adapter. Import/export round-trip. + +(st-bootstrap-classes!) +(content-bootstrap-blocks!) +(content-bootstrap-doc!) +(content-bootstrap-render!) + +;; ── a Ghost post (external shape) ── +(define post {:sections (list {:id "h" :text "Hello" :kind "heading" :level 1} {:id "p" :text "World" :kind "paragraph"} {:id "i" :src "/c.png" :alt "cat" :kind "image"} {:id "d" :kind "hr"} {:items (list "a" "b") :id "l" :kind "list" :ordered true}) :title "Hello"}) + +;; ── import (delegates to adapter) ── +(define doc (content/import ghost-adapter post "post")) +(content-test "import doc-id" (doc-id doc) "post") +(content-test "import ids" (doc-ids doc) (list "h" "p" "i" "d" "l")) +(content-test + "import types" + (doc-types doc) + (list "heading" "text" "image" "divider" "list")) +(content-test + "import renders" + (content/render doc "html") + "

    Hello

    World

    \"cat\"
    1. a
    2. b
    ") +(content-test + "import preserves heading level" + (blk-send (doc-find doc "h") "level") + 1) +(content-test + "import preserves list items" + (blk-send (doc-find doc "l") "items") + (list "a" "b")) + +;; ── export (delegates to adapter) ── +(define out (content/export ghost-adapter doc)) +(content-test + "export sections round-trip" + (get out :sections) + (get post :sections)) + +;; ── round-trip: export then import yields the same document ── +(define doc2 (content/round-trip ghost-adapter doc)) +(content-test "round-trip ids" (doc-ids doc2) (doc-ids doc)) +(content-test + "round-trip render" + (content/render doc2 "html") + (content/render doc "html")) + +;; ── round-trip the external form: import . export . import == import ── +(content-test + "external round-trip sections" + (get + (content/export ghost-adapter (content/import ghost-adapter post "post")) + :sections) + (get post :sections)) + +;; ── core knows nothing about Ghost: a different (stub) adapter works the same ── +(define raw-adapter {:export (fn (d) (str (blk-send (doc-find d "only") "text"))) :import (fn (ext doc-id) (doc-new doc-id (list (mk-text "only" ext))))}) +(define rdoc (content/import raw-adapter "just text" "r")) +(content-test "alt adapter import" (doc-ids rdoc) (list "only")) +(content-test + "alt adapter export" + (content/export raw-adapter rdoc) + "just text") + +;; ── code / quote / embed kinds round-trip ── +(define post2 {:sections (list {:id "c" :text "(+ 1 2)" :kind "code" :language "sx"} {:cite "Ada" :id "q" :text "to err" :kind "quote"} {:id "e" :provider "vimeo" :kind "embed" :url "https://v/1"})}) +(define d3 (content/import ghost-adapter post2 "p2")) +(content-test + "code/quote/embed types" + (doc-types d3) + (list "code" "quote" "embed")) +(content-test + "code/quote/embed round-trip" + (get (content/export ghost-adapter d3) :sections) + (get post2 :sections)) diff --git a/plans/content-on-sx.md b/plans/content-on-sx.md index 2d8904cd..d203bbc4 100644 --- a/plans/content-on-sx.md +++ b/plans/content-on-sx.md @@ -19,7 +19,7 @@ injected adapter, not core. ## Status (rolling) -`bash lib/content/conformance.sh` → **196/196** (Phases 1–3 complete: blocks, doc, render, api, persist op log, CRDT merge) +`bash lib/content/conformance.sh` → **210/210** (Phases 1–3 complete + Phase 4 Ghost adapter) ## Ground rules @@ -71,12 +71,20 @@ lib/content/api.sx ── (content/edit) (content/render) (content/history) ─ - [x] concurrent-edit tests (any order, double-apply → identical) ## Phase 4 — External sync + federation -- [ ] Ghost/CMS sync via injected adapter (import/export) +- [x] Ghost/CMS sync via injected adapter (import/export) - [ ] federated documents (peer-authored blocks) — trust-gated stub -- [ ] tests: round-trip import/export, conflict on concurrent external edit +- [~] tests: round-trip import/export (done), conflict on concurrent external edit (pending) ## Progress log +- 2026-06-07 — Phase 4 `sync.sx` (cb1): external CMS sync via an injected + adapter. Core defines the shape — `{:import :export}` — and delegates; + `content/import` / `content/export` / `content/round-trip` know nothing about + Ghost. A Ghost-flavoured adapter confines all format translation (post + `:sections` ↔ content blocks, all 8 kinds). Swapping in a stub `raw-adapter` + works identically. Round-trip (export∘import and import∘export) preserves ids, + types, fields, order. 14 tests; suite 210/210. Next: trust-gated federation + + concurrent-external-edit conflict (via CRDT). - 2026-06-07 — Phase 3 `crdt.sx` (**Phase 3 complete**): collaborative merge as a state-based CvRDT. Merge is a join (lub) on a semilattice → commutative, associative, idempotent by construction. Ordering = unique dense Logoot From 9722e97e0a9580c7dc07d7aa2979bcef4f19f452 Mon Sep 17 00:00:00 2001 From: giles Date: Sun, 7 Jun 2026 00:42:49 +0000 Subject: [PATCH 08/49] content: trust-gated federation + conflict tests (Phase 4 complete, roadmap done, 230/230) Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/content/conformance.sh | 3 +- lib/content/fed.sx | 68 +++++++++++++++++ lib/content/scoreboard.json | 7 +- lib/content/scoreboard.md | 3 +- lib/content/tests/fed.sx | 148 ++++++++++++++++++++++++++++++++++++ plans/content-on-sx.md | 14 +++- 6 files changed, 235 insertions(+), 8 deletions(-) create mode 100644 lib/content/fed.sx create mode 100644 lib/content/tests/fed.sx diff --git a/lib/content/conformance.sh b/lib/content/conformance.sh index ac8399c9..249955da 100755 --- a/lib/content/conformance.sh +++ b/lib/content/conformance.sh @@ -15,7 +15,7 @@ if [ ! -x "$SX_SERVER" ]; then fi fi -SUITES=(block doc render api store crdt sync) +SUITES=(block doc render api store crdt sync fed) OUT_JSON="lib/content/scoreboard.json" OUT_MD="lib/content/scoreboard.md" @@ -45,6 +45,7 @@ run_suite() { (load "lib/content/store.sx") (load "lib/content/crdt.sx") (load "lib/content/sync.sx") +(load "lib/content/fed.sx") (epoch 2) (eval "(define content-test-pass 0)") (eval "(define content-test-fail 0)") diff --git a/lib/content/fed.sx b/lib/content/fed.sx new file mode 100644 index 00000000..29df1ed9 --- /dev/null +++ b/lib/content/fed.sx @@ -0,0 +1,68 @@ +;; content-on-sx — federated documents: trust-gated peer-authored ops. +;; +;; A peer-authored op carries provenance (:author, and a :sig stub). We never +;; auto-accept: a peer op is applied only if it passes a trust gate. The gate is +;; a predicate (fn op -> bool) so acl-on-sx can inject real trust facts later; +;; the convenience form takes an explicit trusted-actor list (the stub). +;; +;; Accepted ops flow through the CvRDT merge (Phase 3), so concurrent local and +;; external edits reconcile deterministically (same-field LWW, order-independent). +;; +;; Requires (loaded by harness): crdt.sx (and its deps). + +;; tag an op with provenance +(define content/authored (fn (op author) (assoc op :author author))) + +(define + content/signed + (fn (op author sig) (assoc (assoc op :author author) :sig sig))) + +;; explicit trust stub: membership in a trusted-actor list +(define content/trusted? (fn (trust author) (crdt-member? author trust))) + +;; general form: accept? is a predicate (fn op -> bool). Applies accepted ops +;; through the CRDT; quarantines the rest. Returns +;; {:state :accepted (ops) :rejected (ops)}. +(define + content/-merge-peer-loop + (fn + (state accept? ops accepted rejected) + (if + (= (len ops) 0) + {:state state :accepted (reverse accepted) :rejected (reverse rejected)} + (let + ((op (first ops))) + (if + (accept? op) + (content/-merge-peer-loop + (crdt-apply state op) + accept? + (rest ops) + (cons op accepted) + rejected) + (content/-merge-peer-loop + state + accept? + (rest ops) + accepted + (cons op rejected))))))) + +(define + content/merge-peer-with + (fn + (state accept? ops) + (content/-merge-peer-loop state accept? ops (list) (list)))) + +;; convenience: trust = list of trusted actor ids +(define + content/merge-peer + (fn + (state trust ops) + (content/merge-peer-with + state + (fn (op) (content/trusted? trust (get op :author))) + ops))) + +(define content/accepted (fn (res) (get res :accepted))) +(define content/rejected (fn (res) (get res :rejected))) +(define content/peer-state (fn (res) (get res :state))) diff --git a/lib/content/scoreboard.json b/lib/content/scoreboard.json index 3c580915..fcd1eda0 100644 --- a/lib/content/scoreboard.json +++ b/lib/content/scoreboard.json @@ -6,9 +6,10 @@ "api": {"pass": 26, "fail": 0}, "store": {"pass": 29, "fail": 0}, "crdt": {"pass": 34, "fail": 0}, - "sync": {"pass": 14, "fail": 0} + "sync": {"pass": 14, "fail": 0}, + "fed": {"pass": 20, "fail": 0} }, - "total_pass": 210, + "total_pass": 230, "total_fail": 0, - "total": 210 + "total": 230 } diff --git a/lib/content/scoreboard.md b/lib/content/scoreboard.md index 7fb590aa..a84b6913 100644 --- a/lib/content/scoreboard.md +++ b/lib/content/scoreboard.md @@ -11,4 +11,5 @@ _Generated by `lib/content/conformance.sh`_ | store | 29 | 0 | 29 | | crdt | 34 | 0 | 34 | | sync | 14 | 0 | 14 | -| **Total** | **210** | **0** | **210** | +| fed | 20 | 0 | 20 | +| **Total** | **230** | **0** | **230** | diff --git a/lib/content/tests/fed.sx b/lib/content/tests/fed.sx new file mode 100644 index 00000000..6f651528 --- /dev/null +++ b/lib/content/tests/fed.sx @@ -0,0 +1,148 @@ +;; Phase 4 — federated documents: trust-gated peer ops + concurrent-external- +;; edit conflict resolution via the CRDT. + +(st-bootstrap-classes!) +(content-bootstrap-blocks!) +(content-bootstrap-doc!) +(content-bootstrap-render!) + +(define same? (fn (a b) (= (get a :elements) (get b :elements)))) + +;; base shared document, then a local edit +(define + base + (crdt-insert + (crdt-insert + (crdt-empty) + "h" + "heading" + (crdt-pos 1 0) + (list (list "level" 1) (list "text" "T")) + 1 + 0) + "p" + "text" + (crdt-pos 2 0) + (list (list "text" "Body")) + 1 + 0)) +(define local (crdt-update base "p" "text" "local" 5 1)) + +;; ── provenance ── +(content-test + "authored tags author" + (get (content/authored (crdt-op-delete "h") "ed") :author) + "ed") +(content-test + "signed tags sig" + (get (content/signed (crdt-op-delete "h") "ed" "sig1") :sig) + "sig1") +(content-test "trusted? yes" (content/trusted? (list "ed" "al") "ed") true) +(content-test "trusted? no" (content/trusted? (list "ed") "mal") false) + +;; peer ops: ed is trusted, mal is not +(define + peer-ops + (list + (content/authored + (crdt-op-update "p" "text" "peer-ed" 7 2) + "ed") + (content/authored + (crdt-op-insert + "x" + "text" + (crdt-pos 3 0) + (list (list "text" "X")) + 8 + 2) + "ed") + (content/authored (crdt-op-delete "h") "mal"))) + +(define res (content/merge-peer local (list "ed") peer-ops)) + +;; ── trust gate: only ed's ops applied ── +(content-test "accepted count" (len (content/accepted res)) 2) +(content-test "rejected count" (len (content/rejected res)) 1) +(content-test + "rejected is mal's" + (get (first (content/rejected res)) :author) + "mal") + +;; ── resulting document ── +(define rdoc (crdt-materialize "d" (content/peer-state res))) +(content-test "untrusted delete blocked: h survives" (doc-has? rdoc "h") true) +(content-test "trusted insert applied: x present" (doc-has? rdoc "x") true) +(content-test "result order" (doc-ids rdoc) (list "h" "p" "x")) +(content-test + "trusted edit wins (ts7 > ts5)" + (str (blk-send (doc-find rdoc "p") "text")) + "peer-ed") + +;; ── order-independence of accepted peer ops ── +(define res-rev (content/merge-peer local (list "ed") (reverse peer-ops))) +(content-test + "peer merge order-independent" + (same? (content/peer-state res) (content/peer-state res-rev)) + true) + +;; ── trust = nobody → nothing applied, state unchanged ── +(define res0 (content/merge-peer local (list) peer-ops)) +(content-test + "no trust accepts none" + (len (content/accepted res0)) + 0) +(content-test + "no trust rejects all" + (len (content/rejected res0)) + 3) +(content-test + "no trust state unchanged" + (same? (content/peer-state res0) local) + true) + +;; ── pluggable predicate gate (acl-on-sx hook) ── +(define + res-pred + (content/merge-peer-with + local + (fn (op) (= (get op :author) "ed")) + peer-ops)) +(content-test + "predicate gate == list gate" + (same? (content/peer-state res-pred) (content/peer-state res)) + true) + +;; ── conflict on concurrent external edit: local vs external, same field ── +;; external (peer) state edits p concurrently with a later ts; CRDT reconciles. +(define + external + (crdt-update base "p" "text" "external" 9 2)) +(content-test + "conflict LWW deterministic" + (str + (blk-send + (doc-find (crdt-materialize "d" (crdt-merge local external)) "p") + "text")) + "external") +(content-test + "conflict merge commutes" + (same? (crdt-merge local external) (crdt-merge external local)) + true) +(content-test + "conflict merge idempotent" + (same? + (crdt-merge (crdt-merge local external) external) + (crdt-merge local external)) + true) + +;; concurrent external edit with LOWER ts loses to local +(define + external-old + (crdt-update base "p" "text" "stale" 3 2)) +(content-test + "older external loses to local" + (str + (blk-send + (doc-find (crdt-materialize "d" (crdt-merge local external-old)) "p") + "text")) + "local") diff --git a/plans/content-on-sx.md b/plans/content-on-sx.md index d203bbc4..5be0b222 100644 --- a/plans/content-on-sx.md +++ b/plans/content-on-sx.md @@ -19,7 +19,7 @@ injected adapter, not core. ## Status (rolling) -`bash lib/content/conformance.sh` → **210/210** (Phases 1–3 complete + Phase 4 Ghost adapter) +`bash lib/content/conformance.sh` → **230/230** (Phases 1–4 COMPLETE: blocks, doc, render, api, persist op log, CRDT merge, Ghost sync, federation) ## Ground rules @@ -72,11 +72,19 @@ lib/content/api.sx ── (content/edit) (content/render) (content/history) ─ ## Phase 4 — External sync + federation - [x] Ghost/CMS sync via injected adapter (import/export) -- [ ] federated documents (peer-authored blocks) — trust-gated stub -- [~] tests: round-trip import/export (done), conflict on concurrent external edit (pending) +- [x] federated documents (peer-authored blocks) — trust-gated stub +- [x] tests: round-trip import/export, conflict on concurrent external edit ## Progress log +- 2026-06-07 — Phase 4 `fed.sx` (**Phase 4 COMPLETE — roadmap done**): + trust-gated federation. Peer ops carry provenance (`:author`, `:sig` stub); + none are auto-accepted. The trust gate is a pluggable predicate (acl-on-sx + hook) with a trusted-actor-list convenience stub. `content/merge-peer[-with]` + applies only accepted ops through the CvRDT and quarantines the rest + (`{:state :accepted :rejected}`). Concurrent local/external edits reconcile + deterministically: same-field LWW by (ts,actor), commutative, idempotent; + untrusted ops never touch state. 20 tests; suite 230/230. - 2026-06-07 — Phase 4 `sync.sx` (cb1): external CMS sync via an injected adapter. Core defines the shape — `{:import :export}` — and delegates; `content/import` / `content/export` / `content/round-trip` know nothing about From 2c1d8c8064ee87cbe12b62dd5f63f5723998d3a2 Mon Sep 17 00:00:00 2001 From: giles Date: Sun, 7 Jun 2026 00:53:06 +0000 Subject: [PATCH 09/49] content: HTML escaping at render boundary (String>>htmlEscaped) + 8 tests (238/238) Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/content/render.sx | 26 +++++++++++++++--------- lib/content/scoreboard.json | 6 +++--- lib/content/scoreboard.md | 4 ++-- lib/content/tests/render.sx | 40 ++++++++++++++++++++++++++++++++++++- plans/content-on-sx.md | 14 ++++++++++++- 5 files changed, 74 insertions(+), 16 deletions(-) diff --git a/lib/content/render.sx b/lib/content/render.sx index e939ebb7..124aeffb 100644 --- a/lib/content/render.sx +++ b/lib/content/render.sx @@ -6,40 +6,48 @@ ;; children's renderings, so (asHTML doc) / (asSx doc) are pure polymorphic ;; sends with no type dispatch in the SX layer. ;; -;; NOTE: no HTML escaping yet — text is emitted verbatim. Escaping is a boundary -;; concern to add before any untrusted content reaches render. +;; HTML escaping happens HERE, at the boundary: text and attribute values are +;; passed through String>>htmlEscaped (& < > "), so untrusted content cannot +;; break out of its element. asSx wire output is not yet string-escaped (next). (define content-bootstrap-render! (fn () (begin + (ct-def-method! + "String" + "htmlEscaped" + "htmlEscaped | out i n c | out := ''. n := self size. i := 1. [i <= n] whileTrue: [c := self at: i. (c = $&) ifTrue: [out := out , '&'] ifFalse: [(c = $<) ifTrue: [out := out , '<'] ifFalse: [(c = $>) ifTrue: [out := out , '>'] ifFalse: [(c = $\") ifTrue: [out := out , '"'] ifFalse: [out := out , c asString]]]]. i := i + 1]. ^ out") (ct-def-method! "CtHeading" "asHTML" - "asHTML | t | t := level printString. ^ '' , text , ''") - (ct-def-method! "CtText" "asHTML" "asHTML ^ '

    ' , text , '

    '") + "asHTML | t | t := level printString. ^ '' , text htmlEscaped , ''") + (ct-def-method! + "CtText" + "asHTML" + "asHTML ^ '

    ' , text htmlEscaped , '

    '") (ct-def-method! "CtCode" "asHTML" - "asHTML ^ '
    ' , text , '
    '") + "asHTML ^ '
    ' , text htmlEscaped , '
    '") (ct-def-method! "CtQuote" "asHTML" - "asHTML ^ '
    ' , text , '
    '") + "asHTML ^ '
    ' , text htmlEscaped , '
    '") (ct-def-method! "CtImage" "asHTML" - "asHTML ^ '\"''") + "asHTML ^ '\"''") (ct-def-method! "CtEmbed" "asHTML" - "asHTML ^ ''") + "asHTML ^ ''") (ct-def-method! "CtDivider" "asHTML" "asHTML ^ '
    '") (ct-def-method! "CtList" "asHTML" - "asHTML | tag | tag := ordered ifTrue: ['ol'] ifFalse: ['ul']. ^ '<' , tag , '>' , (items inject: '' into: [:a :x | a , '
  • ' , x , '
  • ']) , ''") + "asHTML | tag | tag := ordered ifTrue: ['ol'] ifFalse: ['ul']. ^ '<' , tag , '>' , (items inject: '' into: [:a :x | a , '
  • ' , x htmlEscaped , '
  • ']) , ''") (ct-def-method! "CtDoc" "asHTML" diff --git a/lib/content/scoreboard.json b/lib/content/scoreboard.json index fcd1eda0..5f5b647a 100644 --- a/lib/content/scoreboard.json +++ b/lib/content/scoreboard.json @@ -2,14 +2,14 @@ "suites": { "block": {"pass": 38, "fail": 0}, "doc": {"pass": 40, "fail": 0}, - "render": {"pass": 29, "fail": 0}, + "render": {"pass": 37, "fail": 0}, "api": {"pass": 26, "fail": 0}, "store": {"pass": 29, "fail": 0}, "crdt": {"pass": 34, "fail": 0}, "sync": {"pass": 14, "fail": 0}, "fed": {"pass": 20, "fail": 0} }, - "total_pass": 230, + "total_pass": 238, "total_fail": 0, - "total": 230 + "total": 238 } diff --git a/lib/content/scoreboard.md b/lib/content/scoreboard.md index a84b6913..8b66364d 100644 --- a/lib/content/scoreboard.md +++ b/lib/content/scoreboard.md @@ -6,10 +6,10 @@ _Generated by `lib/content/conformance.sh`_ |-------|-----:|-----:|------:| | block | 38 | 0 | 38 | | doc | 40 | 0 | 40 | -| render | 29 | 0 | 29 | +| render | 37 | 0 | 37 | | api | 26 | 0 | 26 | | store | 29 | 0 | 29 | | crdt | 34 | 0 | 34 | | sync | 14 | 0 | 14 | | fed | 20 | 0 | 20 | -| **Total** | **230** | **0** | **230** | +| **Total** | **238** | **0** | **238** | diff --git a/lib/content/tests/render.sx b/lib/content/tests/render.sx index f7ade85a..b81d0a72 100644 --- a/lib/content/tests/render.sx +++ b/lib/content/tests/render.sx @@ -1,5 +1,5 @@ ;; Phase 1 — render boundary. asHTML / asSx are polymorphic message sends on -;; blocks and the document. +;; blocks and the document. HTML escaping happens at the boundary. (st-bootstrap-classes!) (content-bootstrap-blocks!) @@ -71,3 +71,41 @@ "render after delete" (asHTML (doc-delete d "p")) "

    Title


    ") + +;; ── HTML escaping at the boundary ── +(define xh (mk-heading "xh" 2 "A < B & \"C\"")) +(define xp (mk-text "xp" "")) +(define xi (mk-image "xi" "/a.png?x=1&y=2" "tag ")) +(define xl (mk-list "xl" false (list "a<1" "b&2"))) +(content-test + "escape heading text" + (asHTML xh) + "

    A < B & "C"

    ") +(content-test + "escape paragraph" + (asHTML xp) + "

    <script>alert(1)</script>

    ") +(content-test + "escape image attrs" + (asHTML xi) + "\"tag") +(content-test + "escape list items" + (asHTML xl) + "
    • a<1
    • b&2
    ") +(content-test + "escape ampersand once" + (asHTML (mk-text "amp" "a & b")) + "

    a & b

    ") +(content-test + "escape in document" + (asHTML (doc-append (doc-empty "e") xp)) + "

    <script>alert(1)</script>

    ") +(content-test + "no over-escape plain" + (asHTML (mk-text "plain" "hello world")) + "

    hello world

    ") +(content-test + "escape code body" + (asHTML (mk-code "xc" "html" "
    &
    ")) + "
    <div> & </div>
    ") diff --git a/plans/content-on-sx.md b/plans/content-on-sx.md index 5be0b222..9413fb09 100644 --- a/plans/content-on-sx.md +++ b/plans/content-on-sx.md @@ -19,7 +19,7 @@ injected adapter, not core. ## Status (rolling) -`bash lib/content/conformance.sh` → **230/230** (Phases 1–4 COMPLETE: blocks, doc, render, api, persist op log, CRDT merge, Ghost sync, federation) +`bash lib/content/conformance.sh` → **238/238** (Phases 1–4 COMPLETE + extensions: HTML escaping) ## Ground rules @@ -75,8 +75,20 @@ lib/content/api.sx ── (content/edit) (content/render) (content/history) ─ - [x] federated documents (peer-authored blocks) — trust-gated stub - [x] tests: round-trip import/export, conflict on concurrent external edit +## Extensions (post-roadmap) +- [x] HTML escaping at the render boundary (`String>>htmlEscaped`: & < > ") +- [ ] asSx wire string-escaping (" and \ in SX string literals) + ## Progress log +- 2026-06-07 — Extension: HTML escaping at the render boundary. Added + `String>>htmlEscaped` (recursive char walk escaping & < > ", order-safe so & + isn't double-escaped) and routed every `asHTML` text/attr through it — heading, + text, code body + language, quote, image src/alt, embed url, list items. + Render stays fully polymorphic in Smalltalk; escaping lives at the boundary. + +8 render tests (incl. `