persist: crash/restart recovery integration + migration notes — Phase 4 complete

recovery.sx: 6-test end-to-end crash/restart of an order ledger (log +
subscription kv read model + snapshot + compaction + invoice blob ref) on the
durable backend; everything survives a restart over the same disk + content
store, seq continues, two restarts converge. Migration notes (mem → durable
under a live subsystem) added to the plan. Roadmap done, 111/111.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

lib/persist/conformance.sh

@@ -13,7 +13,7 @@ if [ ! -x "$SX_SERVER" ]; then
   exit 1
 fi
 
-SUITES=(event log kv project subscribe concurrency snapshot compaction durable blob)
+SUITES=(event log kv project subscribe concurrency snapshot compaction durable blob recovery)
 
 OUT_JSON="lib/persist/scoreboard.json"
 OUT_MD="lib/persist/scoreboard.md"

lib/persist/scoreboard.json

@@ -9,9 +9,10 @@
     "snapshot": {"pass": 11, "fail": 0},
     "compaction": {"pass": 11, "fail": 0},
     "durable": {"pass": 15, "fail": 0},
-    "blob": {"pass": 14, "fail": 0}
+    "blob": {"pass": 14, "fail": 0},
+    "recovery": {"pass": 6, "fail": 0}
   },
-  "total_pass": 105,
+  "total_pass": 111,
   "total_fail": 0,
-  "total": 105
+  "total": 111
 }

lib/persist/scoreboard.md

@@ -14,4 +14,5 @@ _Generated by `lib/persist/conformance.sh`_
 | compaction | 11 | 0 | 11 |
 | durable | 15 | 0 | 15 |
 | blob | 14 | 0 | 14 |
-| **Total** | **105** | **0** | **105** |
+| recovery | 6 | 0 | 6 |
+| **Total** | **111** | **0** | **111** |

lib/persist/tests/recovery.sx

@@ -0,0 +1,126 @@
+; Phase 4 — crash/restart integration. A whole subsystem (an order ledger:
+; event log + a kv read model kept by a subscription + a periodic snapshot + an
+; invoice blob ref) on the durable backend must survive a restart. "Crash" =
+; drop every in-process object (backend, hub, projections); "restart" = rebuild
+; them over the SAME disk + blob store. Nothing but the disk and content store
+; carries across, exactly as a real process restart.
+
+(define rec-count (fn (acc e) (+ acc 1)))
+
+(persist-test
+  "log survives restart and seq continues"
+  (let
+    ((disk (persist/mem-backend)))
+    (begin
+      (let
+        ((db (persist/mock-durable disk)))
+        (begin
+          (persist/append db "orders" "placed" 0 {:id "a"})
+          (persist/append db "orders" "placed" 1 {:id "b"})))
+      (let
+        ((db2 (persist/mock-durable disk)))
+        (list
+          (persist/project-fold db2 "orders" rec-count 0)
+          (persist/event-seq
+            (persist/append db2 "orders" "placed" 2 {:id "c"}))))))
+  (list 2 3))
+(persist-test
+  "subscription-driven kv read model survives restart"
+  (let
+    ((disk (persist/mem-backend)))
+    (begin
+      (let
+        ((h (persist/hub (persist/mock-durable disk))))
+        (begin
+          (persist/subscribe
+            h
+            "orders"
+            (fn
+              (bk s e)
+              (persist/kv-update
+                bk
+                "order-count"
+                0
+                (fn (n) (+ n 1)))))
+          (persist/publish h "orders" "placed" 0 {})
+          (persist/publish h "orders" "placed" 1 {})))
+      (let
+        ((db2 (persist/mock-durable disk)))
+        (persist/kv-get db2 "order-count"))))
+  2)
+(persist-test
+  "snapshot taken before crash drives replay after restart"
+  (let
+    ((disk (persist/mem-backend)))
+    (begin
+      (let
+        ((db (persist/mock-durable disk)))
+        (begin
+          (persist/append db "orders" "placed" 0 {})
+          (persist/append db "orders" "placed" 1 {})
+          (persist/checkpoint db "orders" "count" rec-count 0)
+          (persist/append db "orders" "placed" 2 {})))
+      (let
+        ((db2 (persist/mock-durable disk)))
+        (equal?
+          (persist/project-value
+            (persist/replay db2 "orders" "count" rec-count 0))
+          (persist/project-fold db2 "orders" rec-count 0)))))
+  true)
+(persist-test
+  "compacted log still replays correctly after restart"
+  (let
+    ((disk (persist/mem-backend)))
+    (begin
+      (let
+        ((db (persist/mock-durable disk)))
+        (begin
+          (persist/append db "orders" "placed" 0 {})
+          (persist/append db "orders" "placed" 1 {})
+          (persist/append db "orders" "placed" 2 {})
+          (persist/compact db "orders" "count" rec-count 0)
+          (persist/append db "orders" "placed" 3 {})))
+      (let
+        ((db2 (persist/mock-durable disk)))
+        (persist/project-value
+          (persist/replay db2 "orders" "count" rec-count 0)))))
+  4)
+(persist-test
+  "invoice blob ref survives restart, bytes fetched from content store"
+  (let
+    ((disk (persist/mem-backend)) (store (persist/mem-backend)))
+    (begin
+      (let
+        ((db (persist/mock-durable disk)) (blob (persist/mock-blob store)))
+        (persist/kv-put
+          db
+          "invoice"
+          (persist/blob-store blob "INVOICEPDF" "application/pdf")))
+      (let
+        ((db2 (persist/mock-durable disk))
+          (blob2 (persist/mock-blob store)))
+        (persist/blob-fetch blob2 (persist/kv-get db2 "invoice")))))
+  "INVOICEPDF")
+(persist-test
+  "two independent restarts converge to the same state (determinism)"
+  (let
+    ((disk (persist/mem-backend)))
+    (begin
+      (let
+        ((db (persist/mock-durable disk)))
+        (begin
+          (persist/append db "orders" "placed" 0 {})
+          (persist/append db "orders" "placed" 1 {})
+          (persist/append db "orders" "placed" 2 {})))
+      (equal?
+        (persist/project-fold
+          (persist/mock-durable disk)
+          "orders"
+          rec-count
+          0)
+        (persist/project-fold
+          (persist/mock-durable disk)
+          "orders"
+          rec-count
+          0))))
+  true)