From 7c11d4edaa086078ccf289c96501995717da2151 Mon Sep 17 00:00:00 2001 From: giles Date: Fri, 19 Jun 2026 18:59:36 +0000 Subject: [PATCH] host: per-request IO kernel fix + fully-dynamic blog (no cache), 159/159 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit KERNEL (sx_server.ml): route http-listen handlers through cek_run_with_io instead of bare Sx_runtime.sx_call, so handlers resolve per-request IO (durable persist reads/writes) via the same IO-driving runner the REPL uses. Verified: per-request read+write, 10 concurrent writes (15 on disk, no corruption), handler errors don't crash the server, http contract 6/6. BLOG: fully dynamic — host/blog-post reads the post from the durable store (content/head) AND renders (content/html) per request, no in-memory view, no cached output. Possible because of the IO fix. Honest ~2s due to interpreted Smalltalk render. Render speed is NOT solved here: the JIT (precompiler) isn't installed in the serving mode and currently miscompiles the Smalltalk evaluator's nested ASTs (enabling it breaks ~60% of tests). Fixing the JIT is a separate, high-payoff effort. Documented in the plan. Co-Authored-By: Claude Opus 4.8 --- hosts/ocaml/bin/sx_server.ml | 10 +++++- lib/host/blog.sx | 70 +++++++++++++----------------------- lib/host/serve.sh | 8 ++--- lib/host/tests/blog.sx | 9 ++--- plans/host-on-sx.md | 29 +++++++++++---- 5 files changed, 63 insertions(+), 63 deletions(-) diff --git a/hosts/ocaml/bin/sx_server.ml b/hosts/ocaml/bin/sx_server.ml index 006ab145..eaf8f93b 100644 --- a/hosts/ocaml/bin/sx_server.ml +++ b/hosts/ocaml/bin/sx_server.ml @@ -814,7 +814,15 @@ let setup_evaluator_bridge env = Hashtbl.replace req "body" (String body); Mutex.lock mtx; let resp = - (try Sx_runtime.sx_call handler [Dict req] + (* Run the handler through the IO-aware CEK runner (not bare + sx_call) so request handlers can perform per-request IO — + durable store reads/writes resolve via cek_run_with_io's + suspension loop instead of returning an unresolved suspension. *) + (try + let st = Sx_ref.continue_with_call handler + (List [Dict req]) (Env (Sx_types.make_env ())) + (List [Dict req]) (List []) in + cek_run_with_io st with e -> Mutex.unlock mtx; raise e) in Mutex.unlock mtx; let getk k = match resp with diff --git a/lib/host/blog.sx b/lib/host/blog.sx index 7d0ad3ce..b6397409 100644 --- a/lib/host/blog.sx +++ b/lib/host/blog.sx @@ -1,15 +1,16 @@ ;; lib/host/blog.sx — Blog domain on the host. Posts are content-on-sx documents -;; whose SOURCE OF TRUTH is the durable SX store (persist op-log on disk): a post -;; is published by appending insert ops to its stream. Serving GET // renders -;; the post to HTML via content/html. The original strangler target (Quart blog +;; whose source of truth is the durable SX store (persist op-log on disk). Serving +;; GET // is FULLY DYNAMIC: the handler reads the post from the store and +;; renders it to HTML, per request — no in-memory view, no cached output. This is +;; possible because http-listen handlers now resolve per-request IO (the +;; cek_run_with_io kernel fix). The original strangler target (Quart blog ;; post_detail); published posts are world-visible, so this endpoint is ANONYMOUS. ;; -;; READ PATH — materialised view, not per-request IO. The durable backend reads -;; via `perform` (kernel IO suspension), which is serviceable on the main thread -;; (boot) but NOT inside an http-listen request handler thread. So posts are -;; materialised from the store into an in-memory view at boot (and on publish), -;; and request handlers read that view — fast, perform-free. The store stays the -;; source of truth; the view is a cache rebuilt from it on startup. +;; NOTE ON SPEED: content/html runs the interpreted Smalltalk-on-SX dispatch +;; (~2s for a tiny doc) because the JIT is not installed in this serving mode AND +;; currently miscompiles the Smalltalk evaluator's nested ASTs. Making the render +;; fast is a JIT-compiler fix (or a Smalltalk-interpreter optimisation), tracked +;; separately — it is NOT solved by caching the output. ;; Depends on lib/content/* (+ Smalltalk + persist preloads) + lib/dream/* + ;; lib/host/handler.sx. @@ -17,20 +18,13 @@ (define host/blog-bootstrap! (fn () (begin (st-bootstrap-classes!) (content/bootstrap!)))) -;; ── store (durable source of truth) + view (in-memory serving cache) ─ +;; ── store (durable source of truth, injectable) ───────────────────── (define host/blog-store (persist/open)) -(define host/blog-view {}) -(define host/blog-use-store! - (fn (b) (begin (set! host/blog-store b) (set! host/blog-view {})))) +(define host/blog-use-store! (fn (b) (set! host/blog-store b))) -;; content streams are keyed "content:"; recover the slug. -(define host/blog--stream-slug - (fn (stream) - (if (starts-with? stream "content:") (substr stream 8) nil))) - -;; ── publish + lookup ──────────────────────────────────────────────── +;; ── publish + lookup (per-request, against the store) ─────────────── ;; Publish a simple post (title heading + body paragraph): append its insert ops -;; to the durable store, then refresh the in-memory view. `at` is a logical ts. +;; to the durable store. `at` is a caller-supplied logical timestamp. (define host/blog-publish! (fn (slug title body at) (let ((hid (str slug "-h")) (tid (str slug "-body"))) @@ -38,35 +32,21 @@ (list (op-insert (mk-heading hid 1 title) nil) (op-insert (mk-text tid body) hid)) - at) - (set! host/blog-view - (assoc host/blog-view slug (content/head host/blog-store slug)))))) + at)))) -;; Materialise every persisted post from the store into the view. Run at boot on -;; the main thread (content/head performs IO, fine here, not in a request). -(define host/blog-load-all! - (fn () - (for-each - (fn (stream) - (let ((slug (host/blog--stream-slug stream))) - (when slug - (let ((doc (content/head host/blog-store slug))) - (when (> (content/count doc) 0) - (set! host/blog-view (assoc host/blog-view slug doc))))))) - (persist/backend-streams host/blog-store)))) - -;; Idempotent seed: if the slug isn't already materialised, recover it from the -;; store (prior run) or publish it fresh. No duplicate ops on restart. +;; Idempotent seed: publish only if the slug has no content yet (so a restart +;; replaying serve.sh doesn't append duplicate blocks to a persisted post). (define host/blog-seed! (fn (slug title body at) - (when (nil? (get host/blog-view slug)) - (let ((existing (content/head host/blog-store slug))) - (if (> (content/count existing) 0) - (set! host/blog-view (assoc host/blog-view slug existing)) - (host/blog-publish! slug title body at)))))) + (when (= (content/count (content/head host/blog-store slug)) 0) + (host/blog-publish! slug title body at)))) -;; Lookup is pure in-memory (no perform) — safe inside a request handler. -(define host/blog-lookup (fn (slug) (get host/blog-view slug))) +;; Materialise the post from the store by replaying its op-log; nil if no content. +;; Reads the durable store via per-request IO (works inside the handler thread). +(define host/blog-lookup + (fn (slug) + (let ((doc (content/head host/blog-store slug))) + (if (> (content/count doc) 0) doc nil)))) ;; ── handler: GET // -> rendered HTML (200) or 404 ───────────── (define host/blog-post diff --git a/lib/host/serve.sh b/lib/host/serve.sh index 6bde5d8c..e2298ecb 100755 --- a/lib/host/serve.sh +++ b/lib/host/serve.sh @@ -104,12 +104,8 @@ EPOCH=1 echo "(eval \"(host/blog-use-store! (persist/durable-backend))\")" EPOCH=$((EPOCH+1)) echo "(epoch $EPOCH)" - # Materialise any persisted posts into the in-memory view, then ensure the - # welcome post exists (idempotent). Both run on the main thread (IO is fine - # here; request handlers only read the view). - echo "(eval \"(host/blog-load-all!)\")" - EPOCH=$((EPOCH+1)) - echo "(epoch $EPOCH)" + # Idempotently seed the welcome post into the durable store (no-op if present). + # Handlers read + render from the store per request (per-request IO). echo "(eval \"(host/blog-seed! \\\"welcome\\\" \\\"Welcome to the SX host\\\" \\\"This page is rendered by lib/host on the SX runtime, persisted in the SX store — no Quart.\\\" 1)\")" EPOCH=$((EPOCH+1)) echo "(epoch $EPOCH)" diff --git a/lib/host/tests/blog.sx b/lib/host/tests/blog.sx index 1e1af943..7c934d77 100644 --- a/lib/host/tests/blog.sx +++ b/lib/host/tests/blog.sx @@ -54,11 +54,12 @@ (dream-resp-body (host-bl-app (host-bl-req "/welcome/"))) "

Hello SX

Served by lib/host.

") -;; persistence: a post materialises from the op-log (replay), and re-seeding is -;; idempotent (no duplicate blocks appended). -(host-bl-test "lookup materialises 2 blocks" (content/count (host/blog-lookup "welcome")) 2) +;; persistence: the store holds 2 blocks (op-log replay), lookup materialises the +;; doc from the store per call, and re-seeding is idempotent (no duplicate blocks). +(host-bl-test "store has 2 blocks" (content/count (content/head host/blog-store "welcome")) 2) +(host-bl-test "lookup materialises the doc" (content/count (host/blog-lookup "welcome")) 2) (host/blog-seed! "welcome" "Hello SX" "Served by lib/host." 2) -(host-bl-test "re-seed is idempotent" (content/count (host/blog-lookup "welcome")) 2) +(host-bl-test "re-seed is idempotent" (content/count (content/head host/blog-store "welcome")) 2) ;; ── unknown slug -> 404 ───────────────────────────────────────────── (host-bl-test diff --git a/plans/host-on-sx.md b/plans/host-on-sx.md index 5d536328..25ccbc5d 100644 --- a/plans/host-on-sx.md +++ b/plans/host-on-sx.md @@ -41,13 +41,28 @@ router, feed, relations, blog, server, ledger). Blog posts now persist in the durable SX store (`persist/durable-backend`, on-disk under `$SX_PERSIST_DIR`), materialised into an in-memory view at boot and served from there. -> **Per-request IO gap (kernel):** `http-listen` handlers run via -> `Sx_runtime.sx_call`, which does NOT drive `perform`/IO to completion the way -> the main eval loop does (`Sx_types._cek_io_resolver` is global but the handler -> path returns the suspension). Proven: a handler doing a durable `persist/read` -> returns an empty/broken response. Workaround for blog reads = materialise at -> boot. REAL FIX (next): make the http-listen handler invocation resolve IO like -> the main loop — then handlers can read/write the store per request. Phases 1 & 2 DONE; Phase 3 cut-over +> **Per-request IO (kernel) — FIXED.** `http-listen` handlers used to run via +> `Sx_runtime.sx_call` (bare CEK, no IO resolution), so a handler doing a durable +> `persist/read` returned an unresolved suspension. Fixed in `sx_server.ml`: the +> handler now runs through `cek_run_with_io` (`Sx_ref.continue_with_call` → +> `cek_run_with_io`), the same IO-driving runner the REPL uses — it resolves +> persist ops via `Sx_persist_store.handle_op` between CEK steps. Verified: +> handlers do per-request durable reads + writes (incl. 10 concurrent, 15 events +> on disk, no corruption); handler errors don't crash the server. NOTE: this is +> the per-request *IO* fix; it does NOT speed up the interpreted Smalltalk render +> (`/welcome/` still ~2s) — that's a separate concern, addressed by caching the +> rendered HTML at boot. (Pre-existing: an erroring handler closes the connection +> with no response instead of a 500 — worth improving later.) +> +> **Render speed (separate from IO) — NOT precompiled.** `/welcome/` is ~2s because +> the interpreted Smalltalk-on-SX render runs on the tree-walking CEK: the JIT hook +> (`register_jit_hook`) is installed only in `--http` page mode, not the epoch/ +> http-listen serving mode (`make_server_env`), so zero `[jit]` activity. Enabling +> it in that mode breaks correctness (router 3/6, feed 4/11, … — the known JIT- +> bytecode bug on complex nested ASTs, which the Smalltalk evaluator is). So the +> render is slow until the JIT compiler is fixed (big win, broad payoff — its own +> loop) or the Smalltalk interpreter is optimised. Blog is FULLY DYNAMIC (reads +> store + renders per request, no cache) — slowness is honest, not hidden. Phases 1 & 2 DONE; Phase 3 cut-over landed (50% off Quart). **The host now serves live HTTP** — `lib/host/server.sx` bridges the native `http-listen` server to the Dream app and `lib/host/serve.sh` boots it (verified: GET /health, /feed, /feed?actor=, relations get-children/