diff --git a/docker-compose.dev-sx-host.yml b/docker-compose.dev-sx-host.yml index 8a63156e..5f01c24a 100644 --- a/docker-compose.dev-sx-host.yml +++ b/docker-compose.dev-sx-host.yml @@ -20,6 +20,8 @@ services: HOST_PORT: "8000" # Bind all interfaces so Caddy (on externalnet) can reach it. SX_HTTP_HOST: "0.0.0.0" + # Durable persist store root — on a named volume so data survives restarts. + SX_PERSIST_DIR: /data/persist OCAMLRUNPARAM: "b" volumes: # SX source (hot-reload on container restart) @@ -27,6 +29,10 @@ services: - ./lib:/app/lib:ro # OCaml server binary — this worktree's build (has the SX_HTTP_HOST bind fix) - ./hosts/ocaml/_build/default/bin/sx_server.exe:/app/bin/sx_server:ro + # Durable persist store (the SX op-log/kv on disk) — survives restarts. + # Host dir, chowned to the image's appuser (uid 10001) so the non-root + # server can write: sudo mkdir -p /root/sx-host-persist && sudo chown 10001:10001 /root/sx-host-persist + - /root/sx-host-persist:/data/persist networks: - externalnet - default diff --git a/lib/host/blog.sx b/lib/host/blog.sx index 049dec76..7d0ad3ce 100644 --- a/lib/host/blog.sx +++ b/lib/host/blog.sx @@ -1,34 +1,72 @@ -;; lib/host/blog.sx — Blog domain on the host. Serves published posts as HTML at -;; GET // — the original strangler target (Quart: blog/bp/post/routes.py, -;; handler post_detail). Published posts are world-visible, so this endpoint is -;; ANONYMOUS — no auth, visibility is trivially "visible". +;; lib/host/blog.sx — Blog domain on the host. Posts are content-on-sx documents +;; whose SOURCE OF TRUTH is the durable SX store (persist op-log on disk): a post +;; is published by appending insert ops to its stream. Serving GET // renders +;; the post to HTML via content/html. The original strangler target (Quart blog +;; post_detail); published posts are world-visible, so this endpoint is ANONYMOUS. ;; -;; A post is a content-on-sx document (CtDoc) rendered to HTML via the content -;; facade (content/html). Posts live in an in-memory registry keyed by slug: this -;; is the "prove the machinery" step — swap host/blog-lookup for a persist-backed -;; content stream later without touching the handler or the route. -;; Depends on lib/content/* (+ the Smalltalk + persist preloads its classes need) -;; + lib/dream/* + lib/host/handler.sx. +;; READ PATH — materialised view, not per-request IO. The durable backend reads +;; via `perform` (kernel IO suspension), which is serviceable on the main thread +;; (boot) but NOT inside an http-listen request handler thread. So posts are +;; materialised from the store into an in-memory view at boot (and on publish), +;; and request handlers read that view — fast, perform-free. The store stays the +;; source of truth; the view is a cache rebuilt from it on startup. +;; Depends on lib/content/* (+ Smalltalk + persist preloads) + lib/dream/* + +;; lib/host/handler.sx. -;; Register the content class table + render methods (idempotent). Must run before -;; any CtDoc is built/rendered; called at module load below. +;; Register content classes + render methods (idempotent); called at load below. (define host/blog-bootstrap! (fn () (begin (st-bootstrap-classes!) (content/bootstrap!)))) -;; ── in-memory post registry (slug -> CtDoc) ───────────────────────── -(define host/blog-posts {}) -(define host/blog-register! - (fn (slug doc) (set! host/blog-posts (assoc host/blog-posts slug doc)))) -(define host/blog-lookup (fn (slug) (get host/blog-posts slug))) -(define host/blog-reset! (fn () (set! host/blog-posts {}))) +;; ── store (durable source of truth) + view (in-memory serving cache) ─ +(define host/blog-store (persist/open)) +(define host/blog-view {}) +(define host/blog-use-store! + (fn (b) (begin (set! host/blog-store b) (set! host/blog-view {})))) -;; Build a simple post doc (title heading + body paragraph). Convenience for -;; seeding and tests; real posts arrive from the content store. -(define host/blog-make - (fn (slug title body) - (doc-append - (doc-append (doc-empty slug) (mk-heading (str slug "-h") 1 title)) - (mk-text (str slug "-body") body)))) +;; content streams are keyed "content:"; recover the slug. +(define host/blog--stream-slug + (fn (stream) + (if (starts-with? stream "content:") (substr stream 8) nil))) + +;; ── publish + lookup ──────────────────────────────────────────────── +;; Publish a simple post (title heading + body paragraph): append its insert ops +;; to the durable store, then refresh the in-memory view. `at` is a logical ts. +(define host/blog-publish! + (fn (slug title body at) + (let ((hid (str slug "-h")) (tid (str slug "-body"))) + (content/commit-all! host/blog-store slug + (list + (op-insert (mk-heading hid 1 title) nil) + (op-insert (mk-text tid body) hid)) + at) + (set! host/blog-view + (assoc host/blog-view slug (content/head host/blog-store slug)))))) + +;; Materialise every persisted post from the store into the view. Run at boot on +;; the main thread (content/head performs IO, fine here, not in a request). +(define host/blog-load-all! + (fn () + (for-each + (fn (stream) + (let ((slug (host/blog--stream-slug stream))) + (when slug + (let ((doc (content/head host/blog-store slug))) + (when (> (content/count doc) 0) + (set! host/blog-view (assoc host/blog-view slug doc))))))) + (persist/backend-streams host/blog-store)))) + +;; Idempotent seed: if the slug isn't already materialised, recover it from the +;; store (prior run) or publish it fresh. No duplicate ops on restart. +(define host/blog-seed! + (fn (slug title body at) + (when (nil? (get host/blog-view slug)) + (let ((existing (content/head host/blog-store slug))) + (if (> (content/count existing) 0) + (set! host/blog-view (assoc host/blog-view slug existing)) + (host/blog-publish! slug title body at)))))) + +;; Lookup is pure in-memory (no perform) — safe inside a request handler. +(define host/blog-lookup (fn (slug) (get host/blog-view slug))) ;; ── handler: GET // -> rendered HTML (200) or 404 ───────────── (define host/blog-post diff --git a/lib/host/conformance.sh b/lib/host/conformance.sh index 05e8dd30..7326cd52 100755 --- a/lib/host/conformance.sh +++ b/lib/host/conformance.sh @@ -70,6 +70,8 @@ MODULES=( "lib/content/doc.sx" "lib/content/render.sx" "lib/content/api.sx" + "lib/content/store.sx" + "lib/persist/durable.sx" "lib/dream/types.sx" "lib/dream/json.sx" "lib/dream/auth.sx" diff --git a/lib/host/serve.sh b/lib/host/serve.sh index 87ea91ef..6bde5d8c 100755 --- a/lib/host/serve.sh +++ b/lib/host/serve.sh @@ -75,6 +75,8 @@ MODULES=( "lib/content/doc.sx" "lib/content/render.sx" "lib/content/api.sx" + "lib/content/store.sx" + "lib/persist/durable.sx" "lib/dream/types.sx" "lib/dream/json.sx" "lib/dream/auth.sx" @@ -95,10 +97,20 @@ EPOCH=1 for M in "${MODULES[@]}"; do echo "(epoch $EPOCH)"; echo "(load \"$M\")"; EPOCH=$((EPOCH+1)) done - # Seed a welcome post so blog.rose-ash.com/welcome/ renders live (until posts - # arrive from a persist-backed content store). + # Point the blog at the DURABLE file backend (persists under $SX_PERSIST_DIR), + # then idempotently seed a welcome post — re-seeding is a no-op if it already + # exists on disk, so restarts don't duplicate blocks. echo "(epoch $EPOCH)" - echo "(eval \"(host/blog-register! \\\"welcome\\\" (host/blog-make \\\"welcome\\\" \\\"Welcome to the SX host\\\" \\\"This page is rendered by lib/host on the SX runtime — no Quart.\\\"))\")" + echo "(eval \"(host/blog-use-store! (persist/durable-backend))\")" + EPOCH=$((EPOCH+1)) + echo "(epoch $EPOCH)" + # Materialise any persisted posts into the in-memory view, then ensure the + # welcome post exists (idempotent). Both run on the main thread (IO is fine + # here; request handlers only read the view). + echo "(eval \"(host/blog-load-all!)\")" + EPOCH=$((EPOCH+1)) + echo "(epoch $EPOCH)" + echo "(eval \"(host/blog-seed! \\\"welcome\\\" \\\"Welcome to the SX host\\\" \\\"This page is rendered by lib/host on the SX runtime, persisted in the SX store — no Quart.\\\" 1)\")" EPOCH=$((EPOCH+1)) echo "(epoch $EPOCH)" # Anonymous read endpoints: feed timeline + relations container reads + blog diff --git a/lib/host/tests/blog.sx b/lib/host/tests/blog.sx index 02899e73..1e1af943 100644 --- a/lib/host/tests/blog.sx +++ b/lib/host/tests/blog.sx @@ -22,9 +22,9 @@ (define host-bl-app (host/make-app (list host/feed-routes host/blog-routes))) -;; ── render a registered post ──────────────────────────────────────── -(host/blog-reset!) -(host/blog-register! "welcome" (host/blog-make "welcome" "Hello SX" "Served by lib/host.")) +;; ── publish a post to a fresh in-memory store (hermetic) ──────────── +(host/blog-use-store! (persist/open)) +(host/blog-publish! "welcome" "Hello SX" "Served by lib/host." 1) (host-bl-test "post 200" @@ -48,11 +48,17 @@ (dream-status (host-bl-app (host-bl-req "/welcome"))) 200) -;; golden: endpoint body == content facade render of the same doc +;; golden: endpoint body == the exact rendered HTML of the published post (host-bl-test - "golden = content/html" + "golden render" (dream-resp-body (host-bl-app (host-bl-req "/welcome/"))) - (content/html (host/blog-make "welcome" "Hello SX" "Served by lib/host."))) + "

Hello SX

Served by lib/host.

") + +;; persistence: a post materialises from the op-log (replay), and re-seeding is +;; idempotent (no duplicate blocks appended). +(host-bl-test "lookup materialises 2 blocks" (content/count (host/blog-lookup "welcome")) 2) +(host/blog-seed! "welcome" "Hello SX" "Served by lib/host." 2) +(host-bl-test "re-seed is idempotent" (content/count (host/blog-lookup "welcome")) 2) ;; ── unknown slug -> 404 ───────────────────────────────────────────── (host-bl-test diff --git a/plans/host-on-sx.md b/plans/host-on-sx.md index 1ea233cd..5d536328 100644 --- a/plans/host-on-sx.md +++ b/plans/host-on-sx.md @@ -36,8 +36,18 @@ host — no `ocaml-on-sx` dependency. ## Status (rolling) -`bash lib/host/conformance.sh` → **156/156** (9 suites: handler, middleware, sxtp, -router, feed, relations, blog, server, ledger). Phases 1 & 2 DONE; Phase 3 cut-over +`bash lib/host/conformance.sh` → **158/158** (9 suites: handler, middleware, sxtp, +router, feed, relations, blog, server, ledger). Blog posts now persist in the +durable SX store (`persist/durable-backend`, on-disk under `$SX_PERSIST_DIR`), +materialised into an in-memory view at boot and served from there. + +> **Per-request IO gap (kernel):** `http-listen` handlers run via +> `Sx_runtime.sx_call`, which does NOT drive `perform`/IO to completion the way +> the main eval loop does (`Sx_types._cek_io_resolver` is global but the handler +> path returns the suspension). Proven: a handler doing a durable `persist/read` +> returns an empty/broken response. Workaround for blog reads = materialise at +> boot. REAL FIX (next): make the http-listen handler invocation resolve IO like +> the main loop — then handlers can read/write the store per request. Phases 1 & 2 DONE; Phase 3 cut-over landed (50% off Quart). **The host now serves live HTTP** — `lib/host/server.sx` bridges the native `http-listen` server to the Dream app and `lib/host/serve.sh` boots it (verified: GET /health, /feed, /feed?actor=, relations get-children/