From 91ffba9975dc363ee0ffe66e873eaa3c555a8d2b Mon Sep 17 00:00:00 2001 From: giles Date: Sat, 6 Jun 2026 16:22:22 +0000 Subject: [PATCH 01/20] =?UTF-8?q?flow:=20Phase=201=20declarative=20DAG=20?= =?UTF-8?q?=E2=80=94=20sequence/parallel/defflow=20combinators=20+=2018=20?= =?UTF-8?q?tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flow combinators as a Scheme prelude loaded onto scheme-standard-env; a flow is a Scheme procedure input->output, run inside the interpreter (sets up Phase 3 call/cc suspend). flow/start entry point, conformance runner, scoreboard. Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/flow/api.sx | 30 ++++++++++ lib/flow/conformance.sh | 90 +++++++++++++++++++++++++++++ lib/flow/scoreboard.json | 11 ++++ lib/flow/scoreboard.md | 41 +++++++++++++ lib/flow/spec.sx | 29 ++++++++++ lib/flow/tests/basic.sx | 121 +++++++++++++++++++++++++++++++++++++++ plans/flow-on-sx.md | 33 +++++++---- 7 files changed, 343 insertions(+), 12 deletions(-) create mode 100644 lib/flow/api.sx create mode 100755 lib/flow/conformance.sh create mode 100644 lib/flow/scoreboard.json create mode 100644 lib/flow/scoreboard.md create mode 100644 lib/flow/spec.sx create mode 100644 lib/flow/tests/basic.sx diff --git a/lib/flow/api.sx b/lib/flow/api.sx new file mode 100644 index 00000000..dc23514e --- /dev/null +++ b/lib/flow/api.sx @@ -0,0 +1,30 @@ +;; lib/flow/api.sx — flow runtime entry points. +;; +;; Builds a Scheme env preloaded with the flow combinators (lib/flow/spec.sx) +;; plus the public flow API, and provides SX helpers to run flow programs. +;; +;; Scheme-level API (available inside flow programs): +;; (flow/start flow input) — run a flow with the given input, return result +;; +;; SX-level helpers (for hosts and tests): +;; (flow-make-env) — fresh standard env + combinators + api +;; (flow-run src) — eval a Scheme program string in a fresh flow env +;; (flow-run-in env src) — eval a Scheme program string in a given env + +(define flow-api-src "(define (flow/start flow input) (flow input))") + +(define + flow-make-env + (fn + () + (let + ((env (scheme-standard-env))) + (flow-load-combinators! env) + (scheme-eval-program (scheme-parse-all flow-api-src) env) + env))) + +(define + flow-run-in + (fn (env src) (scheme-eval-program (scheme-parse-all src) env))) + +(define flow-run (fn (src) (flow-run-in (flow-make-env) src))) diff --git a/lib/flow/conformance.sh b/lib/flow/conformance.sh new file mode 100755 index 00000000..b0fbab42 --- /dev/null +++ b/lib/flow/conformance.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# flow-on-sx conformance runner — runs all flow test suites in one sx_server process. +# +# Usage: +# bash lib/flow/conformance.sh # run all suites +# bash lib/flow/conformance.sh -v # verbose (list each suite) + +set -uo pipefail +cd "$(git rev-parse --show-toplevel)" + +SX_SERVER="${SX_SERVER:-hosts/ocaml/_build/default/bin/sx_server.exe}" +if [ ! -x "$SX_SERVER" ]; then + SX_SERVER="/root/rose-ash/hosts/ocaml/_build/default/bin/sx_server.exe" +fi +if [ ! -x "$SX_SERVER" ]; then + echo "ERROR: sx_server.exe not found." >&2 + exit 1 +fi + +VERBOSE="${1:-}" + +# Suites: NAME RUNNER-FN PATH +SUITES=( + "basic flow-basic-tests-run! lib/flow/tests/basic.sx" +) + +TMPFILE=$(mktemp); trap "rm -f $TMPFILE" EXIT +EPOCH=1 + +emit_load () { echo "(epoch $EPOCH)"; echo "(load \"$1\")"; EPOCH=$((EPOCH+1)); } +emit_eval () { echo "(epoch $EPOCH)"; echo "(eval \"$1\")"; EPOCH=$((EPOCH+1)); } + +{ + emit_load "lib/guest/lex.sx" + emit_load "lib/guest/reflective/env.sx" + emit_load "lib/guest/reflective/quoting.sx" + emit_load "lib/scheme/parser.sx" + emit_load "lib/scheme/eval.sx" + emit_load "lib/scheme/runtime.sx" + emit_load "lib/flow/spec.sx" + emit_load "lib/flow/api.sx" + for SUITE in "${SUITES[@]}"; do + read -r _NAME _RUNNER FILE <<< "$SUITE" + emit_load "$FILE" + emit_eval "($_RUNNER)" + done +} > "$TMPFILE" + +OUTPUT=$(timeout 180 "$SX_SERVER" < "$TMPFILE" 2>&1 || true) + +TOTAL_PASS=0 +TOTAL_FAIL=0 +FAILED_SUITES=() + +LAST_DICT_LINES=$(echo "$OUTPUT" | grep -E '^\{:' || true) + +I=0 +while read -r LINE; do + [ -z "$LINE" ] && continue + P=$(echo "$LINE" | grep -oE ':passed [0-9]+' | awk '{print $2}') + F=$(echo "$LINE" | grep -oE ':failed [0-9]+' | awk '{print $2}') + [ -z "$P" ] && P=0 + [ -z "$F" ] && F=0 + SUITE_INFO="${SUITES[$I]}" + SUITE_NAME=$(echo "$SUITE_INFO" | awk '{print $1}') + TOTAL_PASS=$((TOTAL_PASS + P)) + TOTAL_FAIL=$((TOTAL_FAIL + F)) + if [ "$F" -gt 0 ]; then + FAILED_SUITES+=("$SUITE_NAME: $P/$((P+F))") + printf 'X %-12s %d/%d\n' "$SUITE_NAME" "$P" "$((P+F))" + echo "$LINE" | grep -oE ':name "[^"]*"' | sed 's/:name / fail: /' + elif [ "$VERBOSE" = "-v" ]; then + printf 'ok %-12s %d passed\n' "$SUITE_NAME" "$P" + fi + I=$((I+1)) +done <<< "$LAST_DICT_LINES" + +TOTAL=$((TOTAL_PASS + TOTAL_FAIL)) +if [ "$TOTAL" -eq 0 ]; then + echo "ERROR: no suite results parsed. Raw output:" >&2 + echo "$OUTPUT" >&2 + exit 1 +fi +if [ $TOTAL_FAIL -eq 0 ]; then + echo "ok $TOTAL_PASS/$TOTAL flow-on-sx tests passed (${#SUITES[@]} suites)" +else + echo "FAIL $TOTAL_PASS/$TOTAL passed, $TOTAL_FAIL failed:" + for S in "${FAILED_SUITES[@]}"; do echo " $S"; done + exit 1 +fi diff --git a/lib/flow/scoreboard.json b/lib/flow/scoreboard.json new file mode 100644 index 00000000..7fef1c7f --- /dev/null +++ b/lib/flow/scoreboard.json @@ -0,0 +1,11 @@ +{ + "total": 18, + "passed": 18, + "failed": 0, + "suites": { + "basic": { "passed": 18, "total": 18 } + }, + "phases": { + "phase1": "in-progress" + } +} diff --git a/lib/flow/scoreboard.md b/lib/flow/scoreboard.md new file mode 100644 index 00000000..6bf2efeb --- /dev/null +++ b/lib/flow/scoreboard.md @@ -0,0 +1,41 @@ +# flow-on-sx Scoreboard + +**All tests pass: 18 / 18 across 1 suite.** + +`bash lib/flow/conformance.sh` + +## Per-suite breakdown + +| Suite | Passing | Covers | +|-------|--------:|--------| +| basic | 18 | Phase 1: single nodes, linear sequence, data-flow threading, defflow, parallel fan/join, nested composition, publish-shaped flow | + +## Architecture + +Flow combinators are a **Scheme prelude** (`lib/flow/spec.sx`) loaded onto +`scheme-standard-env`. A flow is a Scheme procedure `input -> output`. The whole +flow executes inside the Scheme interpreter, so Phase 3's `suspend` (call/cc) will +capture the flow continuation directly. + +- `lib/flow/spec.sx` — combinators: `flow-node`, `flow-id`, `flow-const`, + `sequence`, `parallel`, `defflow`; `flow-load-combinators!`. +- `lib/flow/api.sx` — `flow/start` (Scheme); `flow-make-env`, `flow-run`, + `flow-run-in` (SX helpers). +- `lib/flow/tests/basic.sx` — 18 cases. +- `lib/flow/conformance.sh` — loads substrate + flow layer, runs suites. + +## Semantics notes + +- **node** = 1-arg Scheme procedure; the upstream value is the argument. A node + ignoring its argument is effectively a thunk. +- **sequence** threads left-to-right; empty sequence = identity. +- **parallel** fans the same input to every branch and joins results into a list. + Evaluation is **sequential** for now; true concurrency arrives in Phase 3. + +## Phases + +- [~] Phase 1 — Declarative DAG + sequential execution (combinators + 18 tests done; + `flow/start` done) +- [ ] Phase 2 — Control flow + error handling +- [ ] Phase 3 — Suspend / resume (the showcase) +- [ ] Phase 4 — Distributed nodes via fed-sx diff --git a/lib/flow/spec.sx b/lib/flow/spec.sx new file mode 100644 index 00000000..35e75150 --- /dev/null +++ b/lib/flow/spec.sx @@ -0,0 +1,29 @@ +;; lib/flow/spec.sx — flow combinators as a Scheme prelude. +;; +;; A flow is a Scheme procedure of one argument: the upstream value. +;; node : input -> output +;; A leaf node ignoring its argument is effectively a thunk. Combinators +;; build composite nodes out of child nodes. The whole flow runs INSIDE the +;; Scheme interpreter so that Phase 3's `suspend` (call/cc) can capture the +;; flow continuation directly. +;; +;; Phase 1 combinators: +;; (flow-node f) — wrap a 1-arg procedure as a node (identity) +;; (flow-id input) — pass the upstream value through unchanged +;; (flow-const v) — node that ignores input and yields v +;; (sequence n ...) — thread input left-to-right through children +;; (parallel n ...) — fan input to every child, join results into a list +;; (SEQUENTIAL evaluation; true concurrency is Phase 3) +;; (defflow name body)— bind a named flow + +(define + flow-combinators-src + "(define (flow-node f) f)\n (define (flow-id input) input)\n (define (flow-const v) (lambda (input) v))\n (define (flow-seq-step ns v)\n (if (null? ns) v (flow-seq-step (cdr ns) ((car ns) v))))\n (define sequence (lambda ns (lambda (input) (flow-seq-step ns input))))\n (define parallel (lambda ns (lambda (input) (map (lambda (n) (n input)) ns))))\n (define-syntax defflow\n (syntax-rules ()\n ((defflow nm body) (define nm body))))") + +(define + flow-load-combinators! + (fn + (env) + (begin + (scheme-eval-program (scheme-parse-all flow-combinators-src) env) + env))) diff --git a/lib/flow/tests/basic.sx b/lib/flow/tests/basic.sx new file mode 100644 index 00000000..da9d8972 --- /dev/null +++ b/lib/flow/tests/basic.sx @@ -0,0 +1,121 @@ +;; lib/flow/tests/basic.sx — Phase 1: declarative DAG + sequential execution. + +(define flow-basic-pass 0) +(define flow-basic-fail 0) +(define flow-basic-fails (list)) + +(define + flow-basic-test + (fn + (name actual expected) + (if + (= actual expected) + (set! flow-basic-pass (+ flow-basic-pass 1)) + (begin + (set! flow-basic-fail (+ flow-basic-fail 1)) + (append! flow-basic-fails {:name name :expected expected :actual actual}))))) + +;; Run a Scheme flow-program string and return its final value. +(define flow-b (fn (src) (flow-run src))) + +;; Scheme strings are boxed as {:scm-string "..."}; unwrap to a host string. +(define flow-bs (fn (src) (get (flow-run src) :scm-string))) + +;; ── single node ───────────────────────────────────────────────── +(flow-basic-test + "node: identity passes input through" + (flow-b "(flow/start flow-id 7)") + 7) +(flow-basic-test + "node: const ignores input" + (flow-b "(flow/start (flow-const 99) 1)") + 99) +(flow-basic-test + "node: bare lambda is a node" + (flow-b "(flow/start (lambda (x) (* x x)) 6)") + 36) + +;; ── linear sequence ───────────────────────────────────────────── +(flow-basic-test + "sequence: empty is identity" + (flow-b "(flow/start (sequence) 42)") + 42) +(flow-basic-test + "sequence: single child" + (flow-b "(flow/start (sequence (lambda (x) (+ x 1))) 41)") + 42) +(flow-basic-test + "sequence: two children thread" + (flow-b + "(flow/start (sequence (lambda (x) (+ x 1)) (lambda (x) (* x 10))) 4)") + 50) +(flow-basic-test + "sequence: three children thread" + (flow-b + "(flow/start (sequence (lambda (x) (+ x 1)) (lambda (x) (* x 2)) (lambda (x) (- x 3))) 5)") + 9) + +;; ── data flow between nodes ───────────────────────────────────── +(flow-basic-test + "data flow: string accumulation" + (flow-bs + "(flow/start (sequence (lambda (s) (string-append s \"-a\")) (lambda (s) (string-append s \"-b\"))) \"x\")") + "x-a-b") +(flow-basic-test + "data flow: list build" + (flow-b + "(flow/start (sequence (lambda (x) (cons x (list))) (lambda (xs) (cons 0 xs))) 7)") + (list 0 7)) + +;; ── defflow ───────────────────────────────────────────────────── +(flow-basic-test + "defflow: names a flow" + (flow-b + "(defflow inc2 (sequence (lambda (x) (+ x 1)) (lambda (x) (+ x 1)))) (flow/start inc2 40)") + 42) +(flow-basic-test + "defflow: reusable" + (flow-b + "(defflow dbl (lambda (x) (* x 2))) (+ (flow/start dbl 3) (flow/start dbl 10))") + 26) + +;; ── parallel (sequential semantics, join into list) ───────────── +(flow-basic-test + "parallel: fans input to all branches" + (flow-b + "(flow/start (parallel (lambda (x) (+ x 1)) (lambda (x) (* x 2)) (lambda (x) (- x 3))) 10)") + (list 11 20 7)) +(flow-basic-test + "parallel: empty joins to empty list" + (flow-b "(flow/start (parallel) 5)") + (list)) +(flow-basic-test + "parallel: single branch" + (flow-b "(flow/start (parallel (lambda (x) (* x x))) 9)") + (list 81)) + +;; ── nested composition ────────────────────────────────────────── +(flow-basic-test + "nested: sequence of sequences" + (flow-b + "(flow/start (sequence (sequence (lambda (x) (+ x 1)) (lambda (x) (+ x 1))) (sequence (lambda (x) (* x 3)))) 0)") + 6) +(flow-basic-test + "nested: parallel inside sequence, join then reduce" + (flow-b + "(flow/start (sequence (parallel (lambda (x) (+ x 1)) (lambda (x) (* x 2))) (lambda (xs) (apply + xs))) 10)") + 31) +(flow-basic-test + "nested: sequence inside parallel branch" + (flow-b + "(flow/start (parallel (sequence (lambda (x) (+ x 1)) (lambda (x) (* x 2))) (lambda (x) x)) 5)") + (list 12 5)) + +;; ── publish-shaped flow (the architecture sketch) ─────────────── +(flow-basic-test + "publish: write -> (review | spell) -> join lengths" + (flow-b + "(defflow publish (sequence (lambda (draft) (string-append draft \"!\")) (parallel (lambda (c) (string-length c)) (lambda (c) (string-length (string-append c \"?\")))))) (flow/start publish \"hi\")") + (list 3 4)) + +(define flow-basic-tests-run! (fn () {:total (+ flow-basic-pass flow-basic-fail) :passed flow-basic-pass :failed flow-basic-fail :fails flow-basic-fails})) diff --git a/plans/flow-on-sx.md b/plans/flow-on-sx.md index 46517942..d0dbf392 100644 --- a/plans/flow-on-sx.md +++ b/plans/flow-on-sx.md @@ -16,7 +16,7 @@ federation extension via fed-sx for remote-node execution. ## Status (rolling) -`bash lib/flow/conformance.sh` → **0/0** (not yet started) +`bash lib/flow/conformance.sh` → **18/18** (Phase 1 in progress) ## Ground rules @@ -62,15 +62,16 @@ lib/flow/spec.sx lib/flow/runtime.sx lib/flow/store.sx ## Phase 1 — Declarative DAG + sequential execution -- [ ] `lib/flow/spec.sx` — `defflow` macro, `sequence` combinator -- [ ] node = Scheme thunk; output threads to next node (data flow) -- [ ] `parallel` combinator (sequential semantics for now — TRUE parallelism in Phase 3) -- [ ] runtime executes a flow synchronously, returns final value -- [ ] `lib/flow/api.sx` — `(flow/start name args)` entry point -- [ ] `lib/flow/tests/basic.sx` — 15+ cases: linear sequence, nested sequences, - data flow between nodes, parallel-with-join -- [ ] `lib/flow/scoreboard.{json,md}` -- [ ] `lib/flow/conformance.sh` +- [x] `lib/flow/spec.sx` — `defflow` macro, `sequence` combinator +- [x] node = Scheme procedure of one arg (upstream value threaded in); output + threads to next node (data flow). A node ignoring its arg is a thunk. +- [x] `parallel` combinator (sequential semantics for now — TRUE parallelism in Phase 3) +- [x] runtime executes a flow synchronously, returns final value +- [x] `lib/flow/api.sx` — `(flow/start flow input)` entry point +- [x] `lib/flow/tests/basic.sx` — 18 cases: single nodes, linear/nested sequence, + data flow between nodes, parallel-with-join, publish-shaped flow +- [x] `lib/flow/scoreboard.{json,md}` +- [x] `lib/flow/conformance.sh` ## Phase 2 — Control flow + error handling @@ -101,8 +102,16 @@ lib/flow/spec.sx lib/flow/runtime.sx lib/flow/store.sx ## Progress log -(loop fills this in) +- **Phase 1 (combinators + sequential runtime).** Flow built as a Scheme prelude + loaded onto `scheme-standard-env`: a flow is a Scheme procedure `input -> output`, + so the whole flow runs inside the interpreter (sets up Phase 3 call/cc suspend). + Combinators `flow-node`/`flow-id`/`flow-const`/`sequence`/`parallel`/`defflow` in + `spec.sx`; `flow/start` + SX helpers (`flow-make-env`/`flow-run`) in `api.sx`. + 18/18 in `tests/basic.sx`. Substrate constraints found: dotted rest params + `(a . rest)` and named `let` are unsupported in `lib/scheme/eval.sx`, so + combinators use `(lambda args ...)` variadics + top-level recursion. Scheme + strings come back boxed as `{:scm-string "..."}` — unwrap with `(get s :scm-string)`. ## Blockers -(loop fills this in) +(none) From 65cbdb838757d2a60d5d4aa7faadbc85d02bcee6 Mon Sep 17 00:00:00 2001 From: giles Date: Sat, 6 Jun 2026 16:32:37 +0000 Subject: [PATCH 02/20] flow: branch combinator (conditional) + 6 tests Phase 2 control flow. (branch pred then else) selects then/else node by running pred on the threaded input; named 'branch' since 'cond' is a Scheme special form. Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/flow/conformance.sh | 1 + lib/flow/scoreboard.json | 10 +++++--- lib/flow/scoreboard.md | 8 +++--- lib/flow/spec.sx | 11 +++++++- lib/flow/tests/control.sx | 53 +++++++++++++++++++++++++++++++++++++++ plans/flow-on-sx.md | 5 ++-- 6 files changed, 77 insertions(+), 11 deletions(-) create mode 100644 lib/flow/tests/control.sx diff --git a/lib/flow/conformance.sh b/lib/flow/conformance.sh index b0fbab42..007e7bf2 100755 --- a/lib/flow/conformance.sh +++ b/lib/flow/conformance.sh @@ -22,6 +22,7 @@ VERBOSE="${1:-}" # Suites: NAME RUNNER-FN PATH SUITES=( "basic flow-basic-tests-run! lib/flow/tests/basic.sx" + "control flow-ctl-tests-run! lib/flow/tests/control.sx" ) TMPFILE=$(mktemp); trap "rm -f $TMPFILE" EXIT diff --git a/lib/flow/scoreboard.json b/lib/flow/scoreboard.json index 7fef1c7f..1b51b7e7 100644 --- a/lib/flow/scoreboard.json +++ b/lib/flow/scoreboard.json @@ -1,11 +1,13 @@ { - "total": 18, - "passed": 18, + "total": 24, + "passed": 24, "failed": 0, "suites": { - "basic": { "passed": 18, "total": 18 } + "basic": { "passed": 18, "total": 18 }, + "control": { "passed": 6, "total": 6 } }, "phases": { - "phase1": "in-progress" + "phase1": "done", + "phase2": "in-progress" } } diff --git a/lib/flow/scoreboard.md b/lib/flow/scoreboard.md index 6bf2efeb..19baeaa0 100644 --- a/lib/flow/scoreboard.md +++ b/lib/flow/scoreboard.md @@ -1,6 +1,6 @@ # flow-on-sx Scoreboard -**All tests pass: 18 / 18 across 1 suite.** +**All tests pass: 24 / 24 across 2 suites.** `bash lib/flow/conformance.sh` @@ -9,6 +9,7 @@ | Suite | Passing | Covers | |-------|--------:|--------| | basic | 18 | Phase 1: single nodes, linear sequence, data-flow threading, defflow, parallel fan/join, nested composition, publish-shaped flow | +| control | 6 | Phase 2: `branch` conditional — true/false select, threaded predicate, full-node branches, nested 3-way, approval gate | ## Architecture @@ -34,8 +35,7 @@ capture the flow continuation directly. ## Phases -- [~] Phase 1 — Declarative DAG + sequential execution (combinators + 18 tests done; - `flow/start` done) -- [ ] Phase 2 — Control flow + error handling +- [x] Phase 1 — Declarative DAG + sequential execution (combinators + 18 tests, `flow/start`) +- [~] Phase 2 — Control flow + error handling (`branch` done) - [ ] Phase 3 — Suspend / resume (the showcase) - [ ] Phase 4 — Distributed nodes via fed-sx diff --git a/lib/flow/spec.sx b/lib/flow/spec.sx index 35e75150..324ad4cf 100644 --- a/lib/flow/spec.sx +++ b/lib/flow/spec.sx @@ -7,7 +7,7 @@ ;; Scheme interpreter so that Phase 3's `suspend` (call/cc) can capture the ;; flow continuation directly. ;; -;; Phase 1 combinators: +;; Phase 1 combinators (flow-combinators-src): ;; (flow-node f) — wrap a 1-arg procedure as a node (identity) ;; (flow-id input) — pass the upstream value through unchanged ;; (flow-const v) — node that ignores input and yields v @@ -15,15 +15,24 @@ ;; (parallel n ...) — fan input to every child, join results into a list ;; (SEQUENTIAL evaluation; true concurrency is Phase 3) ;; (defflow name body)— bind a named flow +;; +;; Phase 2 combinators (flow-control-src): +;; (branch pred then else) — pred on input selects then/else node +;; (`cond` is a Scheme special form, so the combinator is named `branch`) (define flow-combinators-src "(define (flow-node f) f)\n (define (flow-id input) input)\n (define (flow-const v) (lambda (input) v))\n (define (flow-seq-step ns v)\n (if (null? ns) v (flow-seq-step (cdr ns) ((car ns) v))))\n (define sequence (lambda ns (lambda (input) (flow-seq-step ns input))))\n (define parallel (lambda ns (lambda (input) (map (lambda (n) (n input)) ns))))\n (define-syntax defflow\n (syntax-rules ()\n ((defflow nm body) (define nm body))))") +(define + flow-control-src + "(define (branch pred then else)\n (lambda (input) (if (pred input) (then input) (else input))))") + (define flow-load-combinators! (fn (env) (begin (scheme-eval-program (scheme-parse-all flow-combinators-src) env) + (scheme-eval-program (scheme-parse-all flow-control-src) env) env))) diff --git a/lib/flow/tests/control.sx b/lib/flow/tests/control.sx new file mode 100644 index 00000000..b4a7a046 --- /dev/null +++ b/lib/flow/tests/control.sx @@ -0,0 +1,53 @@ +;; lib/flow/tests/control.sx — Phase 2: control flow + error handling. + +(define flow-ctl-pass 0) +(define flow-ctl-fail 0) +(define flow-ctl-fails (list)) + +(define + flow-ctl-test + (fn + (name actual expected) + (if + (= actual expected) + (set! flow-ctl-pass (+ flow-ctl-pass 1)) + (begin + (set! flow-ctl-fail (+ flow-ctl-fail 1)) + (append! flow-ctl-fails {:name name :expected expected :actual actual}))))) + +(define flow-c (fn (src) (flow-run src))) +(define flow-cs (fn (src) (get (flow-run src) :scm-string))) + +;; ── branch ────────────────────────────────────────────────────── +(flow-ctl-test + "branch: true selects then" + (flow-c + "(flow/start (branch (lambda (x) (> x 0)) (lambda (x) (* x 100)) (lambda (x) (- 0 x))) 5)") + 500) +(flow-ctl-test + "branch: false selects else" + (flow-c + "(flow/start (branch (lambda (x) (> x 0)) (lambda (x) (* x 100)) (lambda (x) (- 0 x))) -3)") + 3) +(flow-ctl-test + "branch: predicate sees the threaded input" + (flow-c + "(flow/start (sequence (lambda (x) (+ x 1)) (branch (lambda (x) (> x 3)) (flow-const 100) (flow-const 0))) 3)") + 100) +(flow-ctl-test + "branch: branches are full nodes (sequence inside)" + (flow-c + "(flow/start (branch (lambda (x) (< x 10)) (sequence (lambda (x) (+ x 1)) (lambda (x) (* x 2))) (flow-const 0)) 4)") + 10) +(flow-ctl-test + "branch: nested branch (3-way sign)" + (flow-c + "(defflow sign (branch (lambda (x) (> x 0)) (flow-const 1) (branch (lambda (x) (< x 0)) (flow-const -1) (flow-const 0)))) (list (flow/start sign 7) (flow/start sign -7) (flow/start sign 0))") + (list 1 -1 0)) +(flow-ctl-test + "branch: publish-shaped approval gate" + (flow-cs + "(defflow publish (branch (lambda (post) (>= (string-length post) 3)) (lambda (post) (string-append post \" [published]\")) (lambda (post) (string-append post \" [rejected]\")))) (flow/start publish \"ok\")") + "ok [rejected]") + +(define flow-ctl-tests-run! (fn () {:total (+ flow-ctl-pass flow-ctl-fail) :passed flow-ctl-pass :failed flow-ctl-fail :fails flow-ctl-fails})) diff --git a/plans/flow-on-sx.md b/plans/flow-on-sx.md index d0dbf392..c2407f95 100644 --- a/plans/flow-on-sx.md +++ b/plans/flow-on-sx.md @@ -16,7 +16,7 @@ federation extension via fed-sx for remote-node execution. ## Status (rolling) -`bash lib/flow/conformance.sh` → **18/18** (Phase 1 in progress) +`bash lib/flow/conformance.sh` → **24/24** (Phase 1 done; Phase 2 in progress) ## Ground rules @@ -75,7 +75,8 @@ lib/flow/spec.sx lib/flow/runtime.sx lib/flow/store.sx ## Phase 2 — Control flow + error handling -- [ ] `cond` combinator — predicate selects branch +- [x] `cond` combinator — predicate selects branch (named `branch`; `cond` is a + Scheme special form). `(branch pred then else)` — 6 tests. - [ ] `retry n [backoff]` — re-runs node up to n times on exception - [ ] `timeout ms` — bounds node execution - [ ] `try-catch` — exception handler with reified error From 1731476dc61c90c569dacbfacb9f20c348c5f8fd Mon Sep 17 00:00:00 2001 From: giles Date: Sat, 6 Jun 2026 16:35:40 +0000 Subject: [PATCH 03/20] =?UTF-8?q?flow:=20error=20model=20=E2=80=94=20fail/?= =?UTF-8?q?failed=3F/fail-reason=20failure=20values=20+=206=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Explicit (fail reason) values flow downstream as data and are inspected with failed?/fail-reason — distinct from raised exceptions (retry/try-catch territory). Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/flow/scoreboard.json | 11 ++++------- lib/flow/scoreboard.md | 6 +++--- lib/flow/spec.sx | 5 ++++- lib/flow/tests/control.sx | 28 ++++++++++++++++++++++++++++ plans/flow-on-sx.md | 6 ++++-- 5 files changed, 43 insertions(+), 13 deletions(-) diff --git a/lib/flow/scoreboard.json b/lib/flow/scoreboard.json index 1b51b7e7..20bfa126 100644 --- a/lib/flow/scoreboard.json +++ b/lib/flow/scoreboard.json @@ -1,13 +1,10 @@ { - "total": 24, - "passed": 24, + "total": 30, + "passed": 30, "failed": 0, "suites": { "basic": { "passed": 18, "total": 18 }, - "control": { "passed": 6, "total": 6 } + "control": { "passed": 12, "total": 12 } }, - "phases": { - "phase1": "done", - "phase2": "in-progress" - } + "phases": { "phase1": "done", "phase2": "in-progress" } } diff --git a/lib/flow/scoreboard.md b/lib/flow/scoreboard.md index 19baeaa0..8382091a 100644 --- a/lib/flow/scoreboard.md +++ b/lib/flow/scoreboard.md @@ -1,6 +1,6 @@ # flow-on-sx Scoreboard -**All tests pass: 24 / 24 across 2 suites.** +**All tests pass: 30 / 30 across 2 suites.** `bash lib/flow/conformance.sh` @@ -9,7 +9,7 @@ | Suite | Passing | Covers | |-------|--------:|--------| | basic | 18 | Phase 1: single nodes, linear sequence, data-flow threading, defflow, parallel fan/join, nested composition, publish-shaped flow | -| control | 6 | Phase 2: `branch` conditional — true/false select, threaded predicate, full-node branches, nested 3-way, approval gate | +| control | 12 | Phase 2: `branch` conditional (6); error model `fail`/`failed?`/`fail-reason` failure values that flow as data (6) | ## Architecture @@ -36,6 +36,6 @@ capture the flow continuation directly. ## Phases - [x] Phase 1 — Declarative DAG + sequential execution (combinators + 18 tests, `flow/start`) -- [~] Phase 2 — Control flow + error handling (`branch` done) +- [~] Phase 2 — Control flow + error handling (`branch`, error model done) - [ ] Phase 3 — Suspend / resume (the showcase) - [ ] Phase 4 — Distributed nodes via fed-sx diff --git a/lib/flow/spec.sx b/lib/flow/spec.sx index 324ad4cf..874f0399 100644 --- a/lib/flow/spec.sx +++ b/lib/flow/spec.sx @@ -19,6 +19,9 @@ ;; Phase 2 combinators (flow-control-src): ;; (branch pred then else) — pred on input selects then/else node ;; (`cond` is a Scheme special form, so the combinator is named `branch`) +;; (fail reason) — make an explicit failure value (data, not an exception) +;; (failed? x) — is x a failure value? +;; (fail-reason x) — the reason carried by a failure value (define flow-combinators-src @@ -26,7 +29,7 @@ (define flow-control-src - "(define (branch pred then else)\n (lambda (input) (if (pred input) (then input) (else input))))") + "(define (branch pred then else)\n (lambda (input) (if (pred input) (then input) (else input))))\n (define (fail reason) (list (quote flow-fail) reason))\n (define (failed? x) (and (pair? x) (eq? (car x) (quote flow-fail))))\n (define (fail-reason x) (car (cdr x)))") (define flow-load-combinators! diff --git a/lib/flow/tests/control.sx b/lib/flow/tests/control.sx index b4a7a046..403241fa 100644 --- a/lib/flow/tests/control.sx +++ b/lib/flow/tests/control.sx @@ -50,4 +50,32 @@ "(defflow publish (branch (lambda (post) (>= (string-length post) 3)) (lambda (post) (string-append post \" [published]\")) (lambda (post) (string-append post \" [rejected]\")))) (flow/start publish \"ok\")") "ok [rejected]") +;; ── error model — explicit (fail reason) values ───────────────── +(flow-ctl-test + "fail: failed? is true for a failure value" + (flow-c "(failed? (fail 404))") + true) +(flow-ctl-test + "fail: fail-reason extracts the reason" + (flow-c "(fail-reason (fail 404))") + 404) +(flow-ctl-test + "fail: failed? is false for a plain value" + (flow-c "(failed? 7)") + false) +(flow-ctl-test + "fail: failed? is false for an ordinary list" + (flow-c "(failed? (list 1 2 3))") + false) +(flow-ctl-test + "fail: a node may emit a failure as data" + (flow-c + "(defflow validate (lambda (s) (if (>= (string-length s) 3) s (fail (quote too-short))))) (failed? (flow/start validate \"hi\"))") + true) +(flow-ctl-test + "fail: failure flows downstream, branch recovers" + (flow-c + "(defflow guarded (sequence (lambda (s) (if (>= (string-length s) 3) (string-length s) (fail (quote too-short)))) (branch failed? (lambda (f) (list (quote recovered) (fail-reason f))) (lambda (n) (list (quote ok) n))))) (flow/start guarded \"hi\")") + (list "recovered" "too-short")) + (define flow-ctl-tests-run! (fn () {:total (+ flow-ctl-pass flow-ctl-fail) :passed flow-ctl-pass :failed flow-ctl-fail :fails flow-ctl-fails})) diff --git a/plans/flow-on-sx.md b/plans/flow-on-sx.md index c2407f95..da3ae225 100644 --- a/plans/flow-on-sx.md +++ b/plans/flow-on-sx.md @@ -16,7 +16,7 @@ federation extension via fed-sx for remote-node execution. ## Status (rolling) -`bash lib/flow/conformance.sh` → **24/24** (Phase 1 done; Phase 2 in progress) +`bash lib/flow/conformance.sh` → **30/30** (Phase 1 done; Phase 2 in progress) ## Ground rules @@ -80,7 +80,9 @@ lib/flow/spec.sx lib/flow/runtime.sx lib/flow/store.sx - [ ] `retry n [backoff]` — re-runs node up to n times on exception - [ ] `timeout ms` — bounds node execution - [ ] `try-catch` — exception handler with reified error -- [ ] error model — exceptions vs explicit `(fail :reason ...)` results +- [x] error model — exceptions vs explicit `(fail reason)` results: `fail`/`failed?`/ + `fail-reason` produce/inspect failure values that flow downstream as data + (distinct from raised exceptions caught by retry/try-catch). 6 tests. - [ ] `lib/flow/tests/control.sx` — 25+ cases: each combinator + composition ## Phase 3 — Suspend / resume (the showcase) From f3da3b975a1076cc860032d8fea2678a6207c960 Mon Sep 17 00:00:00 2001 From: giles Date: Sat, 6 Jun 2026 16:37:26 +0000 Subject: [PATCH 04/20] =?UTF-8?q?flow:=20try-catch=20combinator=20?= =?UTF-8?q?=E2=80=94=20reify=20raised=20exceptions=20+=206=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (try-catch node handler) runs node; on a raised exception calls (handler error) with the reified error via Scheme guard, returns the handler value. Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/flow/scoreboard.json | 6 +++--- lib/flow/scoreboard.md | 6 +++--- lib/flow/spec.sx | 4 +++- lib/flow/tests/control.sx | 30 ++++++++++++++++++++++++++++++ plans/flow-on-sx.md | 5 +++-- 5 files changed, 42 insertions(+), 9 deletions(-) diff --git a/lib/flow/scoreboard.json b/lib/flow/scoreboard.json index 20bfa126..2d128269 100644 --- a/lib/flow/scoreboard.json +++ b/lib/flow/scoreboard.json @@ -1,10 +1,10 @@ { - "total": 30, - "passed": 30, + "total": 36, + "passed": 36, "failed": 0, "suites": { "basic": { "passed": 18, "total": 18 }, - "control": { "passed": 12, "total": 12 } + "control": { "passed": 18, "total": 18 } }, "phases": { "phase1": "done", "phase2": "in-progress" } } diff --git a/lib/flow/scoreboard.md b/lib/flow/scoreboard.md index 8382091a..e72e5f88 100644 --- a/lib/flow/scoreboard.md +++ b/lib/flow/scoreboard.md @@ -1,6 +1,6 @@ # flow-on-sx Scoreboard -**All tests pass: 30 / 30 across 2 suites.** +**All tests pass: 36 / 36 across 2 suites.** `bash lib/flow/conformance.sh` @@ -9,7 +9,7 @@ | Suite | Passing | Covers | |-------|--------:|--------| | basic | 18 | Phase 1: single nodes, linear sequence, data-flow threading, defflow, parallel fan/join, nested composition, publish-shaped flow | -| control | 12 | Phase 2: `branch` conditional (6); error model `fail`/`failed?`/`fail-reason` failure values that flow as data (6) | +| control | 18 | Phase 2: `branch` (6); error model `fail`/`failed?`/`fail-reason` (6); `try-catch` reifying raised exceptions (6) | ## Architecture @@ -36,6 +36,6 @@ capture the flow continuation directly. ## Phases - [x] Phase 1 — Declarative DAG + sequential execution (combinators + 18 tests, `flow/start`) -- [~] Phase 2 — Control flow + error handling (`branch`, error model done) +- [~] Phase 2 — Control flow + error handling (`branch`, error model, `try-catch` done) - [ ] Phase 3 — Suspend / resume (the showcase) - [ ] Phase 4 — Distributed nodes via fed-sx diff --git a/lib/flow/spec.sx b/lib/flow/spec.sx index 874f0399..5b5a473e 100644 --- a/lib/flow/spec.sx +++ b/lib/flow/spec.sx @@ -22,6 +22,8 @@ ;; (fail reason) — make an explicit failure value (data, not an exception) ;; (failed? x) — is x a failure value? ;; (fail-reason x) — the reason carried by a failure value +;; (try-catch node handler) — run node; if it raises, call (handler error) +;; with the reified error and return the handler's value (define flow-combinators-src @@ -29,7 +31,7 @@ (define flow-control-src - "(define (branch pred then else)\n (lambda (input) (if (pred input) (then input) (else input))))\n (define (fail reason) (list (quote flow-fail) reason))\n (define (failed? x) (and (pair? x) (eq? (car x) (quote flow-fail))))\n (define (fail-reason x) (car (cdr x)))") + "(define (branch pred then else)\n (lambda (input) (if (pred input) (then input) (else input))))\n (define (fail reason) (list (quote flow-fail) reason))\n (define (failed? x) (and (pair? x) (eq? (car x) (quote flow-fail))))\n (define (fail-reason x) (car (cdr x)))\n (define (try-catch node handler)\n (lambda (input) (guard (e (#t (handler e))) (node input))))") (define flow-load-combinators! diff --git a/lib/flow/tests/control.sx b/lib/flow/tests/control.sx index 403241fa..1d8df9a1 100644 --- a/lib/flow/tests/control.sx +++ b/lib/flow/tests/control.sx @@ -78,4 +78,34 @@ "(defflow guarded (sequence (lambda (s) (if (>= (string-length s) 3) (string-length s) (fail (quote too-short)))) (branch failed? (lambda (f) (list (quote recovered) (fail-reason f))) (lambda (n) (list (quote ok) n))))) (flow/start guarded \"hi\")") (list "recovered" "too-short")) +;; ── try-catch — reify raised exceptions ───────────────────────── +(flow-ctl-test + "try-catch: no exception returns node result" + (flow-c "(flow/start (try-catch (lambda (x) (* x 2)) (lambda (e) -1)) 5)") + 10) +(flow-ctl-test + "try-catch: handler runs on raise" + (flow-c + "(flow/start (try-catch (lambda (x) (raise (quote boom))) (flow-const 99)) 1)") + 99) +(flow-ctl-test + "try-catch: handler receives the reified error" + (flow-c "(flow/start (try-catch (lambda (x) (raise 42)) (lambda (e) e)) 0)") + 42) +(flow-ctl-test + "try-catch: catches exception from deep inside a sequence" + (flow-c + "(flow/start (try-catch (sequence (lambda (x) (+ x 1)) (lambda (x) (raise (quote deep)))) (flow-const -99)) 5)") + -99) +(flow-ctl-test + "try-catch: handler may convert to a failure value" + (flow-c + "(failed? (flow/start (try-catch (lambda (x) (raise (quote bad))) (lambda (e) (fail e))) 0))") + true) +(flow-ctl-test + "try-catch: composes — recover then continue" + (flow-c + "(flow/start (sequence (try-catch (lambda (x) (raise (quote x))) (flow-const 10)) (lambda (n) (* n 5))) 0)") + 50) + (define flow-ctl-tests-run! (fn () {:total (+ flow-ctl-pass flow-ctl-fail) :passed flow-ctl-pass :failed flow-ctl-fail :fails flow-ctl-fails})) diff --git a/plans/flow-on-sx.md b/plans/flow-on-sx.md index da3ae225..968dcbdf 100644 --- a/plans/flow-on-sx.md +++ b/plans/flow-on-sx.md @@ -16,7 +16,7 @@ federation extension via fed-sx for remote-node execution. ## Status (rolling) -`bash lib/flow/conformance.sh` → **30/30** (Phase 1 done; Phase 2 in progress) +`bash lib/flow/conformance.sh` → **36/36** (Phase 1 done; Phase 2 in progress) ## Ground rules @@ -79,7 +79,8 @@ lib/flow/spec.sx lib/flow/runtime.sx lib/flow/store.sx Scheme special form). `(branch pred then else)` — 6 tests. - [ ] `retry n [backoff]` — re-runs node up to n times on exception - [ ] `timeout ms` — bounds node execution -- [ ] `try-catch` — exception handler with reified error +- [x] `try-catch` — exception handler with reified error: `(try-catch node handler)` + runs node; on raise, calls `(handler error)` and returns its value. 6 tests. - [x] error model — exceptions vs explicit `(fail reason)` results: `fail`/`failed?`/ `fail-reason` produce/inspect failure values that flow downstream as data (distinct from raised exceptions caught by retry/try-catch). 6 tests. From 4674620d7e924ba18ca35e2b3ad479c43d0c1a58 Mon Sep 17 00:00:00 2001 From: giles Date: Sat, 6 Jun 2026 16:39:21 +0000 Subject: [PATCH 05/20] =?UTF-8?q?flow:=20retry=20combinator=20=E2=80=94=20?= =?UTF-8?q?re-run=20node=20on=20raised=20exceptions=20+=206=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (retry n node) re-runs up to n attempts on a raised exception; the last attempt's exception propagates. Explicit (fail ...) values are NOT retried — they pass through. Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/flow/scoreboard.json | 6 +++--- lib/flow/scoreboard.md | 6 +++--- lib/flow/spec.sx | 6 +++++- lib/flow/tests/control.sx | 32 ++++++++++++++++++++++++++++++++ plans/flow-on-sx.md | 6 ++++-- 5 files changed, 47 insertions(+), 9 deletions(-) diff --git a/lib/flow/scoreboard.json b/lib/flow/scoreboard.json index 2d128269..181eeb3a 100644 --- a/lib/flow/scoreboard.json +++ b/lib/flow/scoreboard.json @@ -1,10 +1,10 @@ { - "total": 36, - "passed": 36, + "total": 42, + "passed": 42, "failed": 0, "suites": { "basic": { "passed": 18, "total": 18 }, - "control": { "passed": 18, "total": 18 } + "control": { "passed": 24, "total": 24 } }, "phases": { "phase1": "done", "phase2": "in-progress" } } diff --git a/lib/flow/scoreboard.md b/lib/flow/scoreboard.md index e72e5f88..dbaa9fb4 100644 --- a/lib/flow/scoreboard.md +++ b/lib/flow/scoreboard.md @@ -1,6 +1,6 @@ # flow-on-sx Scoreboard -**All tests pass: 36 / 36 across 2 suites.** +**All tests pass: 42 / 42 across 2 suites.** `bash lib/flow/conformance.sh` @@ -9,7 +9,7 @@ | Suite | Passing | Covers | |-------|--------:|--------| | basic | 18 | Phase 1: single nodes, linear sequence, data-flow threading, defflow, parallel fan/join, nested composition, publish-shaped flow | -| control | 18 | Phase 2: `branch` (6); error model `fail`/`failed?`/`fail-reason` (6); `try-catch` reifying raised exceptions (6) | +| control | 24 | Phase 2: `branch` (6); error model `fail`/`failed?`/`fail-reason` (6); `try-catch` (6); `retry n` re-running on raised exceptions (6) | ## Architecture @@ -36,6 +36,6 @@ capture the flow continuation directly. ## Phases - [x] Phase 1 — Declarative DAG + sequential execution (combinators + 18 tests, `flow/start`) -- [~] Phase 2 — Control flow + error handling (`branch`, error model, `try-catch` done) +- [~] Phase 2 — Control flow + error handling (`branch`, error model, `try-catch`, `retry` done) - [ ] Phase 3 — Suspend / resume (the showcase) - [ ] Phase 4 — Distributed nodes via fed-sx diff --git a/lib/flow/spec.sx b/lib/flow/spec.sx index 5b5a473e..3ff6a542 100644 --- a/lib/flow/spec.sx +++ b/lib/flow/spec.sx @@ -24,6 +24,10 @@ ;; (fail-reason x) — the reason carried by a failure value ;; (try-catch node handler) — run node; if it raises, call (handler error) ;; with the reified error and return the handler's value +;; (retry n node) — run node, re-running up to n attempts total on a raised +;; exception; the last attempt's exception propagates. Only RAISED exceptions +;; are retried — explicit (fail ...) values pass through unchanged. (Once a +;; node has suspended in Phase 3, retry does not re-run it; resume continues.) (define flow-combinators-src @@ -31,7 +35,7 @@ (define flow-control-src - "(define (branch pred then else)\n (lambda (input) (if (pred input) (then input) (else input))))\n (define (fail reason) (list (quote flow-fail) reason))\n (define (failed? x) (and (pair? x) (eq? (car x) (quote flow-fail))))\n (define (fail-reason x) (car (cdr x)))\n (define (try-catch node handler)\n (lambda (input) (guard (e (#t (handler e))) (node input))))") + "(define (branch pred then else)\n (lambda (input) (if (pred input) (then input) (else input))))\n (define (fail reason) (list (quote flow-fail) reason))\n (define (failed? x) (and (pair? x) (eq? (car x) (quote flow-fail))))\n (define (fail-reason x) (car (cdr x)))\n (define (try-catch node handler)\n (lambda (input) (guard (e (#t (handler e))) (node input))))\n (define (flow-retry-step n node input)\n (guard (e (#t (if (<= n 1) (raise e) (flow-retry-step (- n 1) node input))))\n (node input)))\n (define (retry n node) (lambda (input) (flow-retry-step n node input)))") (define flow-load-combinators! diff --git a/lib/flow/tests/control.sx b/lib/flow/tests/control.sx index 1d8df9a1..0e28c3e9 100644 --- a/lib/flow/tests/control.sx +++ b/lib/flow/tests/control.sx @@ -108,4 +108,36 @@ "(flow/start (sequence (try-catch (lambda (x) (raise (quote x))) (flow-const 10)) (lambda (n) (* n 5))) 0)") 50) +;; ── retry — re-run on raised exceptions ───────────────────────── +(flow-ctl-test + "retry: succeeds after transient failures" + (flow-c + "(define ctr 0) (defflow flaky (lambda (x) (set! ctr (+ ctr 1)) (if (< ctr 3) (raise (quote nope)) (* x 10)))) (list (flow/start (retry 5 flaky) 7) ctr)") + (list 70 3)) +(flow-ctl-test + "retry: exhausted re-raises (caught by try-catch)" + (flow-c + "(flow/start (try-catch (retry 2 (lambda (x) (raise (quote always)))) (flow-const (quote gaveup))) 0)") + "gaveup") +(flow-ctl-test + "retry: n=1 means a single attempt" + (flow-c + "(define ctr 0) (flow/start (try-catch (retry 1 (lambda (x) (set! ctr (+ ctr 1)) (raise (quote bad)))) (lambda (e) ctr)) 0)") + 1) +(flow-ctl-test + "retry: success on first attempt does not re-run" + (flow-c + "(define ctr 0) (flow/start (sequence (retry 5 (lambda (x) (set! ctr (+ ctr 1)) (* x 2))) (lambda (n) ctr)) 21)") + 1) +(flow-ctl-test + "retry: does not retry explicit failure values" + (flow-c + "(define ctr 0) (failed? (flow/start (retry 5 (lambda (x) (set! ctr (+ ctr 1)) (fail (quote bad)))) 0))") + true) +(flow-ctl-test + "retry: failure-value path runs node exactly once" + (flow-c + "(define ctr 0) (flow/start (sequence (retry 5 (lambda (x) (set! ctr (+ ctr 1)) (fail (quote bad)))) (lambda (f) ctr)) 0)") + 1) + (define flow-ctl-tests-run! (fn () {:total (+ flow-ctl-pass flow-ctl-fail) :passed flow-ctl-pass :failed flow-ctl-fail :fails flow-ctl-fails})) diff --git a/plans/flow-on-sx.md b/plans/flow-on-sx.md index 968dcbdf..b913051f 100644 --- a/plans/flow-on-sx.md +++ b/plans/flow-on-sx.md @@ -16,7 +16,7 @@ federation extension via fed-sx for remote-node execution. ## Status (rolling) -`bash lib/flow/conformance.sh` → **36/36** (Phase 1 done; Phase 2 in progress) +`bash lib/flow/conformance.sh` → **42/42** (Phase 1 done; Phase 2 in progress) ## Ground rules @@ -77,7 +77,9 @@ lib/flow/spec.sx lib/flow/runtime.sx lib/flow/store.sx - [x] `cond` combinator — predicate selects branch (named `branch`; `cond` is a Scheme special form). `(branch pred then else)` — 6 tests. -- [ ] `retry n [backoff]` — re-runs node up to n times on exception +- [x] `retry n` — re-runs node up to n attempts on a raised exception; last + exception propagates. Only raised exceptions are retried — `(fail ...)` values + pass through. 6 tests. (Backoff deferred: no wall clock in pure SX.) - [ ] `timeout ms` — bounds node execution - [x] `try-catch` — exception handler with reified error: `(try-catch node handler)` runs node; on raise, calls `(handler error)` and returns its value. 6 tests. From e762cc2e327524ab3f03b2e045f6502a81bb4d79 Mon Sep 17 00:00:00 2001 From: giles Date: Sat, 6 Jun 2026 16:42:16 +0000 Subject: [PATCH 06/20] =?UTF-8?q?flow:=20timeout=20combinator=20=E2=80=94?= =?UTF-8?q?=20cooperative=20step=20budget=20+=207=20tests=20(Phase=202=20c?= =?UTF-8?q?omplete)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (timeout budget node) bounds a node deterministically: nodes opt in via (tick), budget ticks are allowed, the next raises flow-timeout. No scheduler/clock in pure SX so the budget is a step count, not wall-clock. Budgets nest and are per-run. Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/flow/scoreboard.json | 8 ++++---- lib/flow/scoreboard.md | 6 +++--- lib/flow/spec.sx | 9 ++++++++- lib/flow/tests/control.sx | 36 ++++++++++++++++++++++++++++++++++++ plans/flow-on-sx.md | 10 +++++++--- 5 files changed, 58 insertions(+), 11 deletions(-) diff --git a/lib/flow/scoreboard.json b/lib/flow/scoreboard.json index 181eeb3a..8ed50f27 100644 --- a/lib/flow/scoreboard.json +++ b/lib/flow/scoreboard.json @@ -1,10 +1,10 @@ { - "total": 42, - "passed": 42, + "total": 49, + "passed": 49, "failed": 0, "suites": { "basic": { "passed": 18, "total": 18 }, - "control": { "passed": 24, "total": 24 } + "control": { "passed": 31, "total": 31 } }, - "phases": { "phase1": "done", "phase2": "in-progress" } + "phases": { "phase1": "done", "phase2": "done", "phase3": "pending" } } diff --git a/lib/flow/scoreboard.md b/lib/flow/scoreboard.md index dbaa9fb4..4432b438 100644 --- a/lib/flow/scoreboard.md +++ b/lib/flow/scoreboard.md @@ -1,6 +1,6 @@ # flow-on-sx Scoreboard -**All tests pass: 42 / 42 across 2 suites.** +**All tests pass: 49 / 49 across 2 suites.** `bash lib/flow/conformance.sh` @@ -9,7 +9,7 @@ | Suite | Passing | Covers | |-------|--------:|--------| | basic | 18 | Phase 1: single nodes, linear sequence, data-flow threading, defflow, parallel fan/join, nested composition, publish-shaped flow | -| control | 24 | Phase 2: `branch` (6); error model `fail`/`failed?`/`fail-reason` (6); `try-catch` (6); `retry n` re-running on raised exceptions (6) | +| control | 31 | Phase 2: `branch` (6); error model `fail`/`failed?`/`fail-reason` (6); `try-catch` (6); `retry n` (6); `timeout` cooperative step budget (7) | ## Architecture @@ -36,6 +36,6 @@ capture the flow continuation directly. ## Phases - [x] Phase 1 — Declarative DAG + sequential execution (combinators + 18 tests, `flow/start`) -- [~] Phase 2 — Control flow + error handling (`branch`, error model, `try-catch`, `retry` done) +- [x] Phase 2 — Control flow + error handling (branch, error model, try-catch, retry, timeout) - [ ] Phase 3 — Suspend / resume (the showcase) - [ ] Phase 4 — Distributed nodes via fed-sx diff --git a/lib/flow/spec.sx b/lib/flow/spec.sx index 3ff6a542..72e78ebe 100644 --- a/lib/flow/spec.sx +++ b/lib/flow/spec.sx @@ -28,6 +28,13 @@ ;; exception; the last attempt's exception propagates. Only RAISED exceptions ;; are retried — explicit (fail ...) values pass through unchanged. (Once a ;; node has suspended in Phase 3, retry does not re-run it; resume continues.) +;; (timeout budget node) — bound node by a COOPERATIVE STEP BUDGET. There is no +;; scheduler or wall clock in pure SX, so timeout is deterministic: a node opts +;; in by calling (tick) at safe points. `budget` ticks are allowed; the next +;; tick raises (quote flow-timeout) (catchable by try-catch). A node that never +;; ticks is unbounded. Budgets nest (save/restore) and are isolated per flow +;; run (fresh env per flow-make-env). +;; (tick) — consume one unit of the active timeout budget (define flow-combinators-src @@ -35,7 +42,7 @@ (define flow-control-src - "(define (branch pred then else)\n (lambda (input) (if (pred input) (then input) (else input))))\n (define (fail reason) (list (quote flow-fail) reason))\n (define (failed? x) (and (pair? x) (eq? (car x) (quote flow-fail))))\n (define (fail-reason x) (car (cdr x)))\n (define (try-catch node handler)\n (lambda (input) (guard (e (#t (handler e))) (node input))))\n (define (flow-retry-step n node input)\n (guard (e (#t (if (<= n 1) (raise e) (flow-retry-step (- n 1) node input))))\n (node input)))\n (define (retry n node) (lambda (input) (flow-retry-step n node input)))") + "(define (branch pred then else)\n (lambda (input) (if (pred input) (then input) (else input))))\n (define (fail reason) (list (quote flow-fail) reason))\n (define (failed? x) (and (pair? x) (eq? (car x) (quote flow-fail))))\n (define (fail-reason x) (car (cdr x)))\n (define (try-catch node handler)\n (lambda (input) (guard (e (#t (handler e))) (node input))))\n (define (flow-retry-step n node input)\n (guard (e (#t (if (<= n 1) (raise e) (flow-retry-step (- n 1) node input))))\n (node input)))\n (define (retry n node) (lambda (input) (flow-retry-step n node input)))\n (define flow-timeout-budget -1)\n (define (tick)\n (if (< flow-timeout-budget 0)\n 0\n (begin\n (set! flow-timeout-budget (- flow-timeout-budget 1))\n (if (< flow-timeout-budget 0)\n (raise (quote flow-timeout))\n flow-timeout-budget))))\n (define (timeout budget node)\n (lambda (input)\n (let ((saved flow-timeout-budget))\n (set! flow-timeout-budget budget)\n (guard (e (#t (begin (set! flow-timeout-budget saved) (raise e))))\n (let ((result (node input)))\n (set! flow-timeout-budget saved)\n result)))))") (define flow-load-combinators! diff --git a/lib/flow/tests/control.sx b/lib/flow/tests/control.sx index 0e28c3e9..e4fc086a 100644 --- a/lib/flow/tests/control.sx +++ b/lib/flow/tests/control.sx @@ -140,4 +140,40 @@ "(define ctr 0) (flow/start (sequence (retry 5 (lambda (x) (set! ctr (+ ctr 1)) (fail (quote bad)))) (lambda (f) ctr)) 0)") 1) +;; ── timeout — cooperative step budget ─────────────────────────── +(flow-ctl-test + "timeout: work within budget completes" + (flow-c + "(define (cd n) (if (<= n 0) 99 (begin (tick) (cd (- n 1))))) (flow/start (try-catch (timeout 10 (lambda (x) (cd x))) (flow-const (quote timed-out))) 5)") + 99) +(flow-ctl-test + "timeout: work exceeding budget raises flow-timeout" + (flow-c + "(define (cd n) (if (<= n 0) 99 (begin (tick) (cd (- n 1))))) (flow/start (try-catch (timeout 10 (lambda (x) (cd x))) (flow-const (quote timed-out))) 20)") + "timed-out") +(flow-ctl-test + "timeout: exact budget boundary completes" + (flow-c + "(define (cd n) (if (<= n 0) 99 (begin (tick) (cd (- n 1))))) (flow/start (try-catch (timeout 5 (lambda (x) (cd x))) (flow-const (quote timed-out))) 5)") + 99) +(flow-ctl-test + "timeout: one tick over the budget raises" + (flow-c + "(define (cd n) (if (<= n 0) 99 (begin (tick) (cd (- n 1))))) (flow/start (try-catch (timeout 5 (lambda (x) (cd x))) (flow-const (quote timed-out))) 6)") + "timed-out") +(flow-ctl-test + "timeout: the raised error is identifiable" + (flow-c + "(define (cd n) (if (<= n 0) 99 (begin (tick) (cd (- n 1))))) (flow/start (try-catch (timeout 2 (lambda (x) (cd x))) (lambda (e) e)) 9)") + "flow-timeout") +(flow-ctl-test + "timeout: a node that never ticks is unbounded" + (flow-c "(flow/start (timeout 0 (lambda (x) (* x 2))) 5)") + 10) +(flow-ctl-test + "timeout: budget is restored across sequential timeouts" + (flow-c + "(define (cd n) (if (<= n 0) 1 (begin (tick) (cd (- n 1))))) (flow/start (sequence (timeout 4 (lambda (x) (cd x))) (timeout 4 (lambda (x) (cd 3))) (lambda (x) (begin (tick) (+ x 100)))) 3)") + 101) + (define flow-ctl-tests-run! (fn () {:total (+ flow-ctl-pass flow-ctl-fail) :passed flow-ctl-pass :failed flow-ctl-fail :fails flow-ctl-fails})) diff --git a/plans/flow-on-sx.md b/plans/flow-on-sx.md index b913051f..ba10374f 100644 --- a/plans/flow-on-sx.md +++ b/plans/flow-on-sx.md @@ -16,7 +16,7 @@ federation extension via fed-sx for remote-node execution. ## Status (rolling) -`bash lib/flow/conformance.sh` → **42/42** (Phase 1 done; Phase 2 in progress) +`bash lib/flow/conformance.sh` → **49/49** (Phases 1-2 done; Phase 3 next) ## Ground rules @@ -80,13 +80,17 @@ lib/flow/spec.sx lib/flow/runtime.sx lib/flow/store.sx - [x] `retry n` — re-runs node up to n attempts on a raised exception; last exception propagates. Only raised exceptions are retried — `(fail ...)` values pass through. 6 tests. (Backoff deferred: no wall clock in pure SX.) -- [ ] `timeout ms` — bounds node execution +- [x] `timeout budget` — bounds node execution via a **cooperative step budget** + (deterministic; no scheduler/clock in pure SX). Nodes opt in via `(tick)`; + `budget` ticks allowed, the next raises `flow-timeout`. Non-ticking nodes are + unbounded; budgets nest. 7 tests. - [x] `try-catch` — exception handler with reified error: `(try-catch node handler)` runs node; on raise, calls `(handler error)` and returns its value. 6 tests. - [x] error model — exceptions vs explicit `(fail reason)` results: `fail`/`failed?`/ `fail-reason` produce/inspect failure values that flow downstream as data (distinct from raised exceptions caught by retry/try-catch). 6 tests. -- [ ] `lib/flow/tests/control.sx` — 25+ cases: each combinator + composition +- [x] `lib/flow/tests/control.sx` — 31 cases: branch, error model, try-catch, + retry, timeout + compositions ## Phase 3 — Suspend / resume (the showcase) From e896deffc8dd9b8afc6b5f834669461c9e64f266 Mon Sep 17 00:00:00 2001 From: giles Date: Sat, 6 Jun 2026 17:20:09 +0000 Subject: [PATCH 07/20] flow: Phase 3 suspend/resume/cancel via deterministic replay + 17 tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Guest Scheme call/cc is escape-only (re-entry hangs), so durable resume uses deterministic replay: suspend escapes to the driver; resume re-runs the flow and replays resolved suspends from a (tag value) log. No live continuation is ever serialized — persisted state is plain data, survives restart. Adds flow/start (now state-returning, backward compatible), flow/resume, flow/cancel, store.sx. Harness reuses one env with a per-test reset (full env rebuild 66x was too slow). Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/flow/api.sx | 47 +++++++++++++--- lib/flow/conformance.sh | 2 + lib/flow/scoreboard.json | 9 +-- lib/flow/scoreboard.md | 4 +- lib/flow/spec.sx | 51 ++++++++--------- lib/flow/store.sx | 29 ++++++++++ lib/flow/tests/suspend.sx | 114 ++++++++++++++++++++++++++++++++++++++ plans/flow-on-sx.md | 22 +++++--- 8 files changed, 230 insertions(+), 48 deletions(-) create mode 100644 lib/flow/store.sx create mode 100644 lib/flow/tests/suspend.sx diff --git a/lib/flow/api.sx b/lib/flow/api.sx index dc23514e..b7810e60 100644 --- a/lib/flow/api.sx +++ b/lib/flow/api.sx @@ -1,17 +1,24 @@ ;; lib/flow/api.sx — flow runtime entry points. ;; -;; Builds a Scheme env preloaded with the flow combinators (lib/flow/spec.sx) -;; plus the public flow API, and provides SX helpers to run flow programs. +;; Builds a Scheme env preloaded with the flow combinators (lib/flow/spec.sx) and +;; the durable store + lifecycle (lib/flow/store.sx), and provides SX helpers to +;; run flow programs. ;; ;; Scheme-level API (available inside flow programs): -;; (flow/start flow input) — run a flow with the given input, return result +;; (flow/start flow input) — run a flow; raw result if it completes, else +;; (flow-suspended id tag). Defined in store.sx. +;; (flow/resume id value) — resume a suspended flow (store.sx) +;; (flow/cancel id) — cancel a flow (store.sx) +;; (suspend tag) — suspension point (spec.sx) ;; ;; SX-level helpers (for hosts and tests): -;; (flow-make-env) — fresh standard env + combinators + api -;; (flow-run src) — eval a Scheme program string in a fresh flow env +;; (flow-make-env) — fresh standard env + combinators + store +;; (flow-run src) — eval a Scheme program string in a reset shared env ;; (flow-run-in env src) — eval a Scheme program string in a given env - -(define flow-api-src "(define (flow/start flow input) (flow input))") +;; +;; flow-run reuses ONE env (building the full standard env is expensive) and +;; resets the mutable flow globals before each program, so tests stay isolated +;; without paying for a fresh standard env each time. (define flow-make-env @@ -20,11 +27,33 @@ (let ((env (scheme-standard-env))) (flow-load-combinators! env) - (scheme-eval-program (scheme-parse-all flow-api-src) env) + (flow-load-store! env) env))) (define flow-run-in (fn (env src) (scheme-eval-program (scheme-parse-all src) env))) -(define flow-run (fn (src) (flow-run-in (flow-make-env) src))) +(define + flow-reset-src + "(set! flow-store (list)) (set! flow-next-id 0) (set! flow-replay-log (list)) (set! flow-suspend-k #f) (set! flow-timeout-budget -1)") + +(define flow-env-cache false) + +(define + flow-shared-env + (fn + () + (begin + (if flow-env-cache nil (set! flow-env-cache (flow-make-env))) + flow-env-cache))) + +(define + flow-run + (fn + (src) + (let + ((env (flow-shared-env))) + (begin + (scheme-eval-program (scheme-parse-all flow-reset-src) env) + (scheme-eval-program (scheme-parse-all src) env))))) diff --git a/lib/flow/conformance.sh b/lib/flow/conformance.sh index 007e7bf2..f53f1fe7 100755 --- a/lib/flow/conformance.sh +++ b/lib/flow/conformance.sh @@ -23,6 +23,7 @@ VERBOSE="${1:-}" SUITES=( "basic flow-basic-tests-run! lib/flow/tests/basic.sx" "control flow-ctl-tests-run! lib/flow/tests/control.sx" + "suspend flow-sus-tests-run! lib/flow/tests/suspend.sx" ) TMPFILE=$(mktemp); trap "rm -f $TMPFILE" EXIT @@ -39,6 +40,7 @@ emit_eval () { echo "(epoch $EPOCH)"; echo "(eval \"$1\")"; EPOCH=$((EPOCH+1)); emit_load "lib/scheme/eval.sx" emit_load "lib/scheme/runtime.sx" emit_load "lib/flow/spec.sx" + emit_load "lib/flow/store.sx" emit_load "lib/flow/api.sx" for SUITE in "${SUITES[@]}"; do read -r _NAME _RUNNER FILE <<< "$SUITE" diff --git a/lib/flow/scoreboard.json b/lib/flow/scoreboard.json index 8ed50f27..333ae875 100644 --- a/lib/flow/scoreboard.json +++ b/lib/flow/scoreboard.json @@ -1,10 +1,11 @@ { - "total": 49, - "passed": 49, + "total": 66, + "passed": 66, "failed": 0, "suites": { "basic": { "passed": 18, "total": 18 }, - "control": { "passed": 31, "total": 31 } + "control": { "passed": 31, "total": 31 }, + "suspend": { "passed": 17, "total": 17 } }, - "phases": { "phase1": "done", "phase2": "done", "phase3": "pending" } + "phases": { "phase1": "done", "phase2": "done", "phase3": "in-progress" } } diff --git a/lib/flow/scoreboard.md b/lib/flow/scoreboard.md index 4432b438..c9acd6d6 100644 --- a/lib/flow/scoreboard.md +++ b/lib/flow/scoreboard.md @@ -1,6 +1,6 @@ # flow-on-sx Scoreboard -**All tests pass: 49 / 49 across 2 suites.** +**All tests pass: 66 / 66 across 3 suites.** `bash lib/flow/conformance.sh` @@ -10,6 +10,7 @@ |-------|--------:|--------| | basic | 18 | Phase 1: single nodes, linear sequence, data-flow threading, defflow, parallel fan/join, nested composition, publish-shaped flow | | control | 31 | Phase 2: `branch` (6); error model `fail`/`failed?`/`fail-reason` (6); `try-catch` (6); `retry n` (6); `timeout` cooperative step budget (7) | +| suspend | 17 | Phase 3: suspend/resume/cancel via deterministic replay; multi-step, replay determinism, lifecycle guards, suspend-in-branch | ## Architecture @@ -37,5 +38,6 @@ capture the flow continuation directly. - [x] Phase 1 — Declarative DAG + sequential execution (combinators + 18 tests, `flow/start`) - [x] Phase 2 — Control flow + error handling (branch, error model, try-catch, retry, timeout) +- [~] Phase 3 — Suspend/resume (suspend/resume/cancel done via deterministic replay; crash-recovery next) - [ ] Phase 3 — Suspend / resume (the showcase) - [ ] Phase 4 — Distributed nodes via fed-sx diff --git a/lib/flow/spec.sx b/lib/flow/spec.sx index 72e78ebe..72afeaed 100644 --- a/lib/flow/spec.sx +++ b/lib/flow/spec.sx @@ -4,37 +4,29 @@ ;; node : input -> output ;; A leaf node ignoring its argument is effectively a thunk. Combinators ;; build composite nodes out of child nodes. The whole flow runs INSIDE the -;; Scheme interpreter so that Phase 3's `suspend` (call/cc) can capture the -;; flow continuation directly. +;; Scheme interpreter. ;; ;; Phase 1 combinators (flow-combinators-src): -;; (flow-node f) — wrap a 1-arg procedure as a node (identity) -;; (flow-id input) — pass the upstream value through unchanged -;; (flow-const v) — node that ignores input and yields v -;; (sequence n ...) — thread input left-to-right through children -;; (parallel n ...) — fan input to every child, join results into a list -;; (SEQUENTIAL evaluation; true concurrency is Phase 3) -;; (defflow name body)— bind a named flow +;; flow-node / flow-id / flow-const / sequence / parallel / defflow ;; ;; Phase 2 combinators (flow-control-src): -;; (branch pred then else) — pred on input selects then/else node -;; (`cond` is a Scheme special form, so the combinator is named `branch`) -;; (fail reason) — make an explicit failure value (data, not an exception) -;; (failed? x) — is x a failure value? -;; (fail-reason x) — the reason carried by a failure value -;; (try-catch node handler) — run node; if it raises, call (handler error) -;; with the reified error and return the handler's value -;; (retry n node) — run node, re-running up to n attempts total on a raised -;; exception; the last attempt's exception propagates. Only RAISED exceptions -;; are retried — explicit (fail ...) values pass through unchanged. (Once a -;; node has suspended in Phase 3, retry does not re-run it; resume continues.) -;; (timeout budget node) — bound node by a COOPERATIVE STEP BUDGET. There is no -;; scheduler or wall clock in pure SX, so timeout is deterministic: a node opts -;; in by calling (tick) at safe points. `budget` ticks are allowed; the next -;; tick raises (quote flow-timeout) (catchable by try-catch). A node that never -;; ticks is unbounded. Budgets nest (save/restore) and are isolated per flow -;; run (fresh env per flow-make-env). -;; (tick) — consume one unit of the active timeout budget +;; branch / fail / failed? / fail-reason / try-catch / retry / timeout / tick +;; +;; Phase 3 suspend core (flow-suspend-src): +;; The guest Scheme's call/cc is ESCAPE-ONLY (re-invoking a captured k after it +;; returns hangs the runtime), so suspend/resume CANNOT re-enter a continuation. +;; Instead, durability uses DETERMINISTIC REPLAY: a flow re-runs from the start +;; on each resume; suspend points that have already been resolved replay their +;; logged value, and the first unresolved suspend escapes back to the driver. +;; The entire persisted state is the replay log (plain (tag value) data), which +;; survives process restart — no live continuation is ever serialized. +;; +;; (suspend tag) — if tag is in the replay log, return its value; else escape +;; to the driver as (flow-suspended tag). tags must be unique & deterministic +;; across replays. ALL effects/non-determinism must go through suspend so their +;; results are logged (otherwise they re-run on every replay). +;; (flow-drive flow input log) — run flow with the given replay log; returns +;; (flow-done result) or (flow-suspended tag). (define flow-combinators-src @@ -44,6 +36,10 @@ flow-control-src "(define (branch pred then else)\n (lambda (input) (if (pred input) (then input) (else input))))\n (define (fail reason) (list (quote flow-fail) reason))\n (define (failed? x) (and (pair? x) (eq? (car x) (quote flow-fail))))\n (define (fail-reason x) (car (cdr x)))\n (define (try-catch node handler)\n (lambda (input) (guard (e (#t (handler e))) (node input))))\n (define (flow-retry-step n node input)\n (guard (e (#t (if (<= n 1) (raise e) (flow-retry-step (- n 1) node input))))\n (node input)))\n (define (retry n node) (lambda (input) (flow-retry-step n node input)))\n (define flow-timeout-budget -1)\n (define (tick)\n (if (< flow-timeout-budget 0)\n 0\n (begin\n (set! flow-timeout-budget (- flow-timeout-budget 1))\n (if (< flow-timeout-budget 0)\n (raise (quote flow-timeout))\n flow-timeout-budget))))\n (define (timeout budget node)\n (lambda (input)\n (let ((saved flow-timeout-budget))\n (set! flow-timeout-budget budget)\n (guard (e (#t (begin (set! flow-timeout-budget saved) (raise e))))\n (let ((result (node input)))\n (set! flow-timeout-budget saved)\n result)))))") +(define + flow-suspend-src + "(define flow-replay-log (list))\n (define flow-suspend-k #f)\n (define (flow-log-lookup tag log)\n (if (null? log)\n (list #f #f)\n (if (eq? (car (car log)) tag)\n (list #t (car (cdr (car log))))\n (flow-log-lookup tag (cdr log)))))\n (define (suspend tag)\n (let ((hit (flow-log-lookup tag flow-replay-log)))\n (if (car hit)\n (car (cdr hit))\n (flow-suspend-k (list (quote flow-suspended) tag)))))\n (define (flow-drive flow input log)\n (set! flow-replay-log log)\n (call/cc\n (lambda (k)\n (set! flow-suspend-k k)\n (list (quote flow-done) (flow input)))))") + (define flow-load-combinators! (fn @@ -51,4 +47,5 @@ (begin (scheme-eval-program (scheme-parse-all flow-combinators-src) env) (scheme-eval-program (scheme-parse-all flow-control-src) env) + (scheme-eval-program (scheme-parse-all flow-suspend-src) env) env))) diff --git a/lib/flow/store.sx b/lib/flow/store.sx new file mode 100644 index 00000000..06a6251f --- /dev/null +++ b/lib/flow/store.sx @@ -0,0 +1,29 @@ +;; lib/flow/store.sx — durable flow store + lifecycle (Phase 3). +;; +;; The store maps flow-id -> record. A record is a plain list: +;; (flow input log status payload) +;; flow — the flow procedure (live; re-resolved by name on restart) +;; input — the original start input (plain data) +;; log — replay log: list of (tag value) entries (plain data) +;; status — done | suspended | cancelled +;; payload — result (done) | waiting-tag (suspended) | #f (cancelled) +;; +;; Lifecycle (all use deterministic replay via flow-drive — see spec.sx): +;; (flow/start flow input) — run from empty log. If it completes, return the raw +;; result (backward compatible with Phases 1-2). If it suspends, register the +;; record and return (flow-suspended id tag). +;; (flow/resume id value) — append (tag value) to the log and re-drive. Returns +;; the raw result on completion, (flow-suspended id tag) on a further suspend, +;; or (flow-error reason) if the id is unknown / not suspended. +;; (flow/cancel id) — mark cancelled; a later resume is rejected (the stale +;; replay can never wake a cancelled flow). + +(define + flow-store-src + "(define flow-store (list))\n (define flow-next-id 0)\n (define (flow-store-put! id rec) (set! flow-store (cons (list id rec) flow-store)))\n (define (flow-store-find id store)\n (if (null? store)\n (list)\n (if (= (car (car store)) id)\n (car (cdr (car store)))\n (flow-store-find id (cdr store)))))\n (define (flow-store-get id) (flow-store-find id flow-store))\n (define (flow-mk-rec flow input log status payload)\n (list flow input log status payload))\n (define (flow-rec-flow r) (car r))\n (define (flow-rec-input r) (car (cdr r)))\n (define (flow-rec-log r) (car (cdr (cdr r))))\n (define (flow-rec-status r) (car (cdr (cdr (cdr r)))))\n (define (flow-rec-payload r) (car (cdr (cdr (cdr (cdr r))))))\n (define (flow-outcome id flow input log outcome)\n (if (eq? (car outcome) (quote flow-done))\n (begin\n (flow-store-put! id (flow-mk-rec flow input log (quote done) (car (cdr outcome))))\n (car (cdr outcome)))\n (begin\n (flow-store-put! id (flow-mk-rec flow input log (quote suspended) (car (cdr outcome))))\n (list (quote flow-suspended) id (car (cdr outcome))))))\n (define (flow/start flow input)\n (set! flow-next-id (+ flow-next-id 1))\n (flow-outcome flow-next-id flow input (list) (flow-drive flow input (list))))\n (define (flow/resume id value)\n (let ((rec (flow-store-get id)))\n (if (null? rec)\n (list (quote flow-error) (quote no-such-flow))\n (if (eq? (flow-rec-status rec) (quote suspended))\n (let ((newlog (cons (list (flow-rec-payload rec) value) (flow-rec-log rec))))\n (flow-outcome id (flow-rec-flow rec) (flow-rec-input rec) newlog\n (flow-drive (flow-rec-flow rec) (flow-rec-input rec) newlog)))\n (list (quote flow-error) (quote not-suspended))))))\n (define (flow/cancel id)\n (let ((rec (flow-store-get id)))\n (if (null? rec)\n (list (quote flow-error) (quote no-such-flow))\n (begin\n (flow-store-put! id\n (flow-mk-rec (flow-rec-flow rec) (flow-rec-input rec) (flow-rec-log rec) (quote cancelled) #f))\n (list (quote flow-cancelled) id)))))") + +(define + flow-load-store! + (fn + (env) + (begin (scheme-eval-program (scheme-parse-all flow-store-src) env) env))) diff --git a/lib/flow/tests/suspend.sx b/lib/flow/tests/suspend.sx new file mode 100644 index 00000000..e9dd825a --- /dev/null +++ b/lib/flow/tests/suspend.sx @@ -0,0 +1,114 @@ +;; lib/flow/tests/suspend.sx — Phase 3: suspend / resume / cancel (deterministic replay). + +(define flow-sus-pass 0) +(define flow-sus-fail 0) +(define flow-sus-fails (list)) + +(define + flow-sus-test + (fn + (name actual expected) + (if + (= actual expected) + (set! flow-sus-pass (+ flow-sus-pass 1)) + (begin + (set! flow-sus-fail (+ flow-sus-fail 1)) + (append! flow-sus-fails {:name name :expected expected :actual actual}))))) + +(define flow-s (fn (src) (flow-run src))) + +;; ── flow/start ────────────────────────────────────────────────── +(flow-sus-test + "start: non-suspending flow returns the raw result" + (flow-s "(flow/start (lambda (x) (* x 2)) 5)") + 10) +(flow-sus-test + "start: a suspending flow returns a flow-suspended state" + (flow-s + "(defflow w (sequence (lambda (x) (+ x 1)) (lambda (g) (suspend (quote await))) (lambda (c) c))) (car (flow/start w 10))") + "flow-suspended") +(flow-sus-test + "start: suspended state carries a numeric id" + (flow-s + "(defflow w (lambda (x) (suspend (quote await)))) (car (cdr (flow/start w 10)))") + 1) +(flow-sus-test + "start: suspended state carries the suspend tag" + (flow-s + "(defflow w (lambda (x) (suspend (quote await)))) (car (cdr (cdr (flow/start w 10))))") + "await") + +;; ── flow/resume ───────────────────────────────────────────────── +(flow-sus-test + "resume: injects the value and completes" + (flow-s + "(defflow w (sequence (lambda (x) (+ x 1)) (lambda (g) (suspend (quote await))) (lambda (c) (list (quote done) c)))) (define s (flow/start w 10)) (flow/resume (car (cdr s)) 777)") + (list "done" 777)) +(flow-sus-test + "resume: injected value threads into the next node" + (flow-s + "(defflow w (sequence (lambda (x) (suspend (quote v))) (lambda (n) (* n 3)))) (define s (flow/start w 0)) (flow/resume (car (cdr s)) 14)") + 42) +(flow-sus-test + "resume: replays earlier suspends (recompute is deterministic)" + (flow-s + "(define runs 0) (defflow w (sequence (lambda (x) (begin (set! runs (+ runs 1)) (+ x 1))) (lambda (g) (suspend (quote await))) (lambda (c) c))) (define s (flow/start w 10)) (flow/resume (car (cdr s)) 99) runs") + 2) + +;; ── multi-step suspension ─────────────────────────────────────── +(flow-sus-test + "multi: first resume suspends at the next tag" + (flow-s + "(defflow two (sequence (lambda (x) (suspend (quote a))) (lambda (x) (suspend (quote b))) (lambda (x) (list (quote end) x)))) (define s (flow/start two 0)) (define s2 (flow/resume (car (cdr s)) 100)) (car (cdr (cdr s2)))") + "b") +(flow-sus-test + "multi: second resume completes with the latest value" + (flow-s + "(defflow two (sequence (lambda (x) (suspend (quote a))) (lambda (x) (suspend (quote b))) (lambda (x) (list (quote end) x)))) (define id (car (cdr (flow/start two 0)))) (flow/resume id 100) (flow/resume id 200)") + (list "end" 200)) + +;; ── error / lifecycle guards ──────────────────────────────────── +(flow-sus-test + "resume: completed flow cannot be resumed again" + (flow-s + "(defflow w (lambda (x) (suspend (quote q)))) (define id (car (cdr (flow/start w 0)))) (flow/resume id 1) (flow/resume id 2)") + (list "flow-error" "not-suspended")) +(flow-sus-test + "resume: unknown id errors" + (flow-s "(flow/resume 999 1)") + (list "flow-error" "no-such-flow")) + +;; ── flow/cancel ───────────────────────────────────────────────── +(flow-sus-test + "cancel: returns a flow-cancelled state" + (flow-s + "(defflow w (lambda (x) (suspend (quote q)))) (define id (car (cdr (flow/start w 0)))) (flow/cancel id)") + (list "flow-cancelled" 1)) +(flow-sus-test + "cancel: a cancelled flow cannot be resumed (stale resume rejected)" + (flow-s + "(defflow w (lambda (x) (suspend (quote q)))) (define id (car (cdr (flow/start w 0)))) (flow/cancel id) (flow/resume id 5)") + (list "flow-error" "not-suspended")) +(flow-sus-test + "cancel: unknown id errors" + (flow-s "(flow/cancel 999)") + (list "flow-error" "no-such-flow")) + +;; ── composition ───────────────────────────────────────────────── +(flow-sus-test + "suspend inside a branch arm" + (flow-s + "(defflow gate (branch (lambda (x) (> x 0)) (lambda (x) (suspend (quote approve))) (flow-const (quote rejected)))) (define s (flow/start gate 5)) (flow/resume (car (cdr s)) (quote approved))") + "approved") +(flow-sus-test + "two independent runs get independent ids" + (flow-s + "(defflow w (lambda (x) (suspend (quote q)))) (list (car (cdr (flow/start w 0))) (car (cdr (flow/start w 0))))") + (list 1 2)) +(flow-sus-test + "suspend reason may be a structured value" + (flow-s + "(defflow w (lambda (x) (suspend (list (quote needs) (quote approval))))) (car (cdr (cdr (flow/start w 0))))") + (list "needs" "approval")) + +(define flow-sus-tests-run! (fn () {:total (+ flow-sus-pass flow-sus-fail) :passed flow-sus-pass :failed flow-sus-fail :fails flow-sus-fails})) diff --git a/plans/flow-on-sx.md b/plans/flow-on-sx.md index ba10374f..6ee6b710 100644 --- a/plans/flow-on-sx.md +++ b/plans/flow-on-sx.md @@ -16,7 +16,7 @@ federation extension via fed-sx for remote-node execution. ## Status (rolling) -`bash lib/flow/conformance.sh` → **49/49** (Phases 1-2 done; Phase 3 next) +`bash lib/flow/conformance.sh` → **66/66** (Phases 1-2 done; Phase 3 suspend/resume/cancel done, crash-recovery next) ## Ground rules @@ -94,13 +94,21 @@ lib/flow/spec.sx lib/flow/runtime.sx lib/flow/store.sx ## Phase 3 — Suspend / resume (the showcase) -- [ ] `(suspend reason)` — `call/cc` captures continuation, returns flow-id to caller -- [ ] `lib/flow/store.sx` — serialize flow state (continuation + open vars) -- [ ] `(flow/resume id value)` — load continuation, inject value, re-enter -- [ ] `(flow/cancel id)` — explicit termination +- [x] `(suspend tag)` — guest call/cc is ESCAPE-ONLY (re-entry hangs), so resume + uses **deterministic replay**: suspend escapes to the driver as `(flow-suspended + tag)`; resume re-runs the flow, replaying resolved suspends from a `(tag value)` + log. No live continuation is ever serialized — the log is plain data. +- [x] `lib/flow/store.sx` — flow store: id→record `(flow input log status payload)`; + `flow-drive` runs a flow against a replay log. +- [x] `(flow/resume id value)` — append `(tag value)` to the log, re-drive; raw + result on completion, `(flow-suspended id tag)` on a further suspend. +- [x] `(flow/cancel id)` — mark cancelled; a later resume is rejected (stale replay + cannot wake a cancelled flow). - [ ] crash recovery — on restart, scan store for paused flows, mark resumable -- [ ] `lib/flow/tests/suspend.sx` — pause-resume scenarios, cancellation, "restart" - scenarios (simulated by re-loading store) +- [x] `lib/flow/tests/suspend.sx` — 17 cases: start/resume/cancel, multi-step, + replay determinism, lifecycle guards, suspend-in-branch +- Harness: `flow-run` now reuses one env with a per-test reset (building the full + standard env 66× was too slow) — see `api.sx`. ## Phase 4 — Distributed nodes via fed-sx From 97c7623743032f979c2a5f2246d9f367e8757efa Mon Sep 17 00:00:00 2001 From: giles Date: Sat, 6 Jun 2026 17:25:47 +0000 Subject: [PATCH 08/20] =?UTF-8?q?flow:=20crash=20recovery=20=E2=80=94=20st?= =?UTF-8?q?ore=20export/import=20+=20resumable=20scan=20+=208=20tests=20(P?= =?UTF-8?q?hase=203=20complete)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Records are name-keyed (defflow registers names); flow-store-export nulls live procs to plain data, flow-store-import! restores, flow-resumable-ids scans for paused flows. Resume re-resolves the proc by name, so a flow survives a wiped store (simulated restart). The whole durable model persists only plain data. Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/flow/conformance.sh | 1 + lib/flow/scoreboard.json | 9 ++--- lib/flow/scoreboard.md | 5 +-- lib/flow/spec.sx | 4 ++- lib/flow/store.sx | 32 ++++++++++------- lib/flow/tests/recovery.sx | 71 ++++++++++++++++++++++++++++++++++++++ plans/flow-on-sx.md | 8 +++-- 7 files changed, 108 insertions(+), 22 deletions(-) create mode 100644 lib/flow/tests/recovery.sx diff --git a/lib/flow/conformance.sh b/lib/flow/conformance.sh index f53f1fe7..237b288b 100755 --- a/lib/flow/conformance.sh +++ b/lib/flow/conformance.sh @@ -24,6 +24,7 @@ SUITES=( "basic flow-basic-tests-run! lib/flow/tests/basic.sx" "control flow-ctl-tests-run! lib/flow/tests/control.sx" "suspend flow-sus-tests-run! lib/flow/tests/suspend.sx" + "recovery flow-rec-tests-run! lib/flow/tests/recovery.sx" ) TMPFILE=$(mktemp); trap "rm -f $TMPFILE" EXIT diff --git a/lib/flow/scoreboard.json b/lib/flow/scoreboard.json index 333ae875..63dff0b8 100644 --- a/lib/flow/scoreboard.json +++ b/lib/flow/scoreboard.json @@ -1,11 +1,12 @@ { - "total": 66, - "passed": 66, + "total": 74, + "passed": 74, "failed": 0, "suites": { "basic": { "passed": 18, "total": 18 }, "control": { "passed": 31, "total": 31 }, - "suspend": { "passed": 17, "total": 17 } + "suspend": { "passed": 17, "total": 17 }, + "recovery": { "passed": 8, "total": 8 } }, - "phases": { "phase1": "done", "phase2": "done", "phase3": "in-progress" } + "phases": { "phase1": "done", "phase2": "done", "phase3": "done", "phase4": "pending" } } diff --git a/lib/flow/scoreboard.md b/lib/flow/scoreboard.md index c9acd6d6..e47a695a 100644 --- a/lib/flow/scoreboard.md +++ b/lib/flow/scoreboard.md @@ -1,6 +1,6 @@ # flow-on-sx Scoreboard -**All tests pass: 66 / 66 across 3 suites.** +**All tests pass: 74 / 74 across 4 suites.** `bash lib/flow/conformance.sh` @@ -11,6 +11,7 @@ | basic | 18 | Phase 1: single nodes, linear sequence, data-flow threading, defflow, parallel fan/join, nested composition, publish-shaped flow | | control | 31 | Phase 2: `branch` (6); error model `fail`/`failed?`/`fail-reason` (6); `try-catch` (6); `retry n` (6); `timeout` cooperative step budget (7) | | suspend | 17 | Phase 3: suspend/resume/cancel via deterministic replay; multi-step, replay determinism, lifecycle guards, suspend-in-branch | +| recovery | 8 | Phase 3: crash recovery — store export/import, resumable scan, restart-at-every-step, replay-log survival | ## Architecture @@ -38,6 +39,6 @@ capture the flow continuation directly. - [x] Phase 1 — Declarative DAG + sequential execution (combinators + 18 tests, `flow/start`) - [x] Phase 2 — Control flow + error handling (branch, error model, try-catch, retry, timeout) -- [~] Phase 3 — Suspend/resume (suspend/resume/cancel done via deterministic replay; crash-recovery next) +- [x] Phase 3 — Suspend/resume (suspend/resume/cancel + crash recovery via deterministic replay) - [ ] Phase 3 — Suspend / resume (the showcase) - [ ] Phase 4 — Distributed nodes via fed-sx diff --git a/lib/flow/spec.sx b/lib/flow/spec.sx index 72afeaed..8a610724 100644 --- a/lib/flow/spec.sx +++ b/lib/flow/spec.sx @@ -8,6 +8,8 @@ ;; ;; Phase 1 combinators (flow-combinators-src): ;; flow-node / flow-id / flow-const / sequence / parallel / defflow +;; defflow both binds the flow and registers it by name (flow-register!, in +;; store.sx) so it can be re-resolved after a process restart. ;; ;; Phase 2 combinators (flow-control-src): ;; branch / fail / failed? / fail-reason / try-catch / retry / timeout / tick @@ -30,7 +32,7 @@ (define flow-combinators-src - "(define (flow-node f) f)\n (define (flow-id input) input)\n (define (flow-const v) (lambda (input) v))\n (define (flow-seq-step ns v)\n (if (null? ns) v (flow-seq-step (cdr ns) ((car ns) v))))\n (define sequence (lambda ns (lambda (input) (flow-seq-step ns input))))\n (define parallel (lambda ns (lambda (input) (map (lambda (n) (n input)) ns))))\n (define-syntax defflow\n (syntax-rules ()\n ((defflow nm body) (define nm body))))") + "(define (flow-node f) f)\n (define (flow-id input) input)\n (define (flow-const v) (lambda (input) v))\n (define (flow-seq-step ns v)\n (if (null? ns) v (flow-seq-step (cdr ns) ((car ns) v))))\n (define sequence (lambda ns (lambda (input) (flow-seq-step ns input))))\n (define parallel (lambda ns (lambda (input) (map (lambda (n) (n input)) ns))))\n (define-syntax defflow\n (syntax-rules ()\n ((defflow nm body)\n (begin (define nm body) (flow-register! (quote nm) nm)))))") (define flow-control-src diff --git a/lib/flow/store.sx b/lib/flow/store.sx index 06a6251f..37284524 100644 --- a/lib/flow/store.sx +++ b/lib/flow/store.sx @@ -1,26 +1,32 @@ -;; lib/flow/store.sx — durable flow store + lifecycle (Phase 3). +;; lib/flow/store.sx — durable flow store + lifecycle + crash recovery (Phase 3). ;; -;; The store maps flow-id -> record. A record is a plain list: -;; (flow input log status payload) -;; flow — the flow procedure (live; re-resolved by name on restart) +;; The store maps flow-id -> record (one entry per id, newest wins). A record is: +;; (name proc input log status payload) +;; name — registered flow name (symbol) or #f for an anonymous flow +;; proc — the live flow procedure (#f after export; re-resolved by name) ;; input — the original start input (plain data) ;; log — replay log: list of (tag value) entries (plain data) ;; status — done | suspended | cancelled ;; payload — result (done) | waiting-tag (suspended) | #f (cancelled) ;; +;; A record is SERIALIZABLE once its proc is nulled (flow-store-export): name, +;; input, and log are plain data. On restart the flow definitions are reloaded +;; (defflow re-registers names), the store is reimported, and resume re-resolves +;; the proc by name — no live continuation is ever persisted. +;; ;; Lifecycle (all use deterministic replay via flow-drive — see spec.sx): -;; (flow/start flow input) — run from empty log. If it completes, return the raw -;; result (backward compatible with Phases 1-2). If it suspends, register the -;; record and return (flow-suspended id tag). -;; (flow/resume id value) — append (tag value) to the log and re-drive. Returns -;; the raw result on completion, (flow-suspended id tag) on a further suspend, -;; or (flow-error reason) if the id is unknown / not suspended. -;; (flow/cancel id) — mark cancelled; a later resume is rejected (the stale -;; replay can never wake a cancelled flow). +;; (flow/start flow input) — raw result if it completes (backward compatible), +;; else (flow-suspended id tag). +;; (flow/resume id value) — append (tag value) to the log and re-drive. +;; (flow/cancel id) — mark cancelled; a later resume is rejected. +;; Crash recovery: +;; (flow-store-export) — store as plain data (procs nulled) +;; (flow-store-import! d) — replace the store from exported data +;; (flow-resumable-ids) — ids of suspended (resumable) flows (define flow-store-src - "(define flow-store (list))\n (define flow-next-id 0)\n (define (flow-store-put! id rec) (set! flow-store (cons (list id rec) flow-store)))\n (define (flow-store-find id store)\n (if (null? store)\n (list)\n (if (= (car (car store)) id)\n (car (cdr (car store)))\n (flow-store-find id (cdr store)))))\n (define (flow-store-get id) (flow-store-find id flow-store))\n (define (flow-mk-rec flow input log status payload)\n (list flow input log status payload))\n (define (flow-rec-flow r) (car r))\n (define (flow-rec-input r) (car (cdr r)))\n (define (flow-rec-log r) (car (cdr (cdr r))))\n (define (flow-rec-status r) (car (cdr (cdr (cdr r)))))\n (define (flow-rec-payload r) (car (cdr (cdr (cdr (cdr r))))))\n (define (flow-outcome id flow input log outcome)\n (if (eq? (car outcome) (quote flow-done))\n (begin\n (flow-store-put! id (flow-mk-rec flow input log (quote done) (car (cdr outcome))))\n (car (cdr outcome)))\n (begin\n (flow-store-put! id (flow-mk-rec flow input log (quote suspended) (car (cdr outcome))))\n (list (quote flow-suspended) id (car (cdr outcome))))))\n (define (flow/start flow input)\n (set! flow-next-id (+ flow-next-id 1))\n (flow-outcome flow-next-id flow input (list) (flow-drive flow input (list))))\n (define (flow/resume id value)\n (let ((rec (flow-store-get id)))\n (if (null? rec)\n (list (quote flow-error) (quote no-such-flow))\n (if (eq? (flow-rec-status rec) (quote suspended))\n (let ((newlog (cons (list (flow-rec-payload rec) value) (flow-rec-log rec))))\n (flow-outcome id (flow-rec-flow rec) (flow-rec-input rec) newlog\n (flow-drive (flow-rec-flow rec) (flow-rec-input rec) newlog)))\n (list (quote flow-error) (quote not-suspended))))))\n (define (flow/cancel id)\n (let ((rec (flow-store-get id)))\n (if (null? rec)\n (list (quote flow-error) (quote no-such-flow))\n (begin\n (flow-store-put! id\n (flow-mk-rec (flow-rec-flow rec) (flow-rec-input rec) (flow-rec-log rec) (quote cancelled) #f))\n (list (quote flow-cancelled) id)))))") + "(define flow-registry (list))\n (define (flow-register! name proc) (set! flow-registry (cons (list name proc) flow-registry)))\n (define (flow-lookup-in name reg)\n (if (null? reg)\n #f\n (if (eq? (car (car reg)) name) (car (cdr (car reg))) (flow-lookup-in name (cdr reg)))))\n (define (flow-lookup name) (flow-lookup-in name flow-registry))\n (define (flow-name-of proc reg)\n (if (null? reg)\n #f\n (if (eq? (car (cdr (car reg))) proc) (car (car reg)) (flow-name-of proc (cdr reg)))))\n\n (define flow-store (list))\n (define flow-next-id 0)\n (define (flow-store-remove id store)\n (if (null? store)\n (list)\n (if (= (car (car store)) id)\n (flow-store-remove id (cdr store))\n (cons (car store) (flow-store-remove id (cdr store))))))\n (define (flow-store-put! id rec) (set! flow-store (cons (list id rec) (flow-store-remove id flow-store))))\n (define (flow-store-find id store)\n (if (null? store)\n (list)\n (if (= (car (car store)) id)\n (car (cdr (car store)))\n (flow-store-find id (cdr store)))))\n (define (flow-store-get id) (flow-store-find id flow-store))\n\n (define (flow-mk-rec name proc input log status payload)\n (list name proc input log status payload))\n (define (flow-rec-name r) (car r))\n (define (flow-rec-proc r) (car (cdr r)))\n (define (flow-rec-input r) (car (cdr (cdr r))))\n (define (flow-rec-log r) (car (cdr (cdr (cdr r)))))\n (define (flow-rec-status r) (car (cdr (cdr (cdr (cdr r))))))\n (define (flow-rec-payload r) (car (cdr (cdr (cdr (cdr (cdr r)))))))\n (define (flow-rec-resolve rec)\n (let ((byname (flow-lookup (flow-rec-name rec))))\n (if byname byname (flow-rec-proc rec))))\n\n (define (flow-outcome id name proc input log outcome)\n (if (eq? (car outcome) (quote flow-done))\n (begin\n (flow-store-put! id (flow-mk-rec name proc input log (quote done) (car (cdr outcome))))\n (car (cdr outcome)))\n (begin\n (flow-store-put! id (flow-mk-rec name proc input log (quote suspended) (car (cdr outcome))))\n (list (quote flow-suspended) id (car (cdr outcome))))))\n (define (flow/start flow input)\n (set! flow-next-id (+ flow-next-id 1))\n (flow-outcome flow-next-id (flow-name-of flow flow-registry) flow input (list)\n (flow-drive flow input (list))))\n (define (flow/resume id value)\n (let ((rec (flow-store-get id)))\n (if (null? rec)\n (list (quote flow-error) (quote no-such-flow))\n (if (eq? (flow-rec-status rec) (quote suspended))\n (let ((proc (flow-rec-resolve rec)))\n (let ((newlog (cons (list (flow-rec-payload rec) value) (flow-rec-log rec))))\n (flow-outcome id (flow-rec-name rec) proc (flow-rec-input rec) newlog\n (flow-drive proc (flow-rec-input rec) newlog))))\n (list (quote flow-error) (quote not-suspended))))))\n (define (flow/cancel id)\n (let ((rec (flow-store-get id)))\n (if (null? rec)\n (list (quote flow-error) (quote no-such-flow))\n (begin\n (flow-store-put! id\n (flow-mk-rec (flow-rec-name rec) (flow-rec-proc rec) (flow-rec-input rec)\n (flow-rec-log rec) (quote cancelled) #f))\n (list (quote flow-cancelled) id)))))\n\n (define (flow-export-entry entry)\n (let ((rec (car (cdr entry))))\n (list (car entry)\n (flow-mk-rec (flow-rec-name rec) #f (flow-rec-input rec)\n (flow-rec-log rec) (flow-rec-status rec) (flow-rec-payload rec)))))\n (define (flow-export-map store)\n (if (null? store) (list) (cons (flow-export-entry (car store)) (flow-export-map (cdr store)))))\n (define (flow-store-export) (flow-export-map flow-store))\n (define (flow-max-id store m)\n (if (null? store) m (flow-max-id (cdr store) (if (> (car (car store)) m) (car (car store)) m))))\n (define (flow-store-import! data)\n (begin (set! flow-store data) (set! flow-next-id (flow-max-id data 0))))\n (define (flow-collect-resumable store)\n (if (null? store)\n (list)\n (if (eq? (flow-rec-status (car (cdr (car store)))) (quote suspended))\n (cons (car (car store)) (flow-collect-resumable (cdr store)))\n (flow-collect-resumable (cdr store)))))\n (define (flow-resumable-ids) (flow-collect-resumable flow-store))") (define flow-load-store! diff --git a/lib/flow/tests/recovery.sx b/lib/flow/tests/recovery.sx new file mode 100644 index 00000000..cbe3b2f4 --- /dev/null +++ b/lib/flow/tests/recovery.sx @@ -0,0 +1,71 @@ +;; lib/flow/tests/recovery.sx — Phase 3: crash recovery (store export/import + restart). +;; +;; "restart" is simulated within one program: (set! flow-store (list)) wipes the +;; in-memory store (process death), while flow-registry persists as it would after +;; reloading flow definitions. Recovery = import the exported (plain-data) store and +;; resume; the flow proc is re-resolved by name. + +(define flow-rec-pass 0) +(define flow-rec-fail 0) +(define flow-rec-fails (list)) + +(define + flow-rec-test + (fn + (name actual expected) + (if + (= actual expected) + (set! flow-rec-pass (+ flow-rec-pass 1)) + (begin + (set! flow-rec-fail (+ flow-rec-fail 1)) + (append! flow-rec-fails {:name name :expected expected :actual actual}))))) + +(define flow-r (fn (src) (flow-run src))) + +;; ── export / wipe / import ────────────────────────────────────── +(flow-rec-test + "export nulls the live procedure" + (flow-r + "(defflow w (lambda (x) (suspend (quote await)))) (flow/start w 10) (car (cdr (car (cdr (car (flow-store-export))))))") + false) +(flow-rec-test + "a wiped store loses the flow (process death)" + (flow-r + "(defflow w (lambda (x) (suspend (quote await)))) (define id (car (cdr (flow/start w 10)))) (set! flow-store (list)) (flow/resume id 1)") + (list "flow-error" "no-such-flow")) +(flow-rec-test + "import restores a wiped store and resume completes" + (flow-r + "(defflow w (sequence (lambda (x) (suspend (quote await))) (lambda (c) (list (quote done) c)))) (define id (car (cdr (flow/start w 10)))) (define saved (flow-store-export)) (set! flow-store (list)) (flow-store-import! saved) (flow/resume id 777)") + (list "done" 777)) + +;; ── resumable scan ────────────────────────────────────────────── +(flow-rec-test + "resumable-ids lists the suspended flow after import" + (flow-r + "(defflow w (lambda (x) (suspend (quote await)))) (define id (car (cdr (flow/start w 10)))) (define saved (flow-store-export)) (set! flow-store (list)) (flow-store-import! saved) (flow-resumable-ids)") + (list 1)) +(flow-rec-test + "resumable-ids excludes completed flows" + (flow-r + "(defflow w (sequence (lambda (x) (suspend (quote await))) (lambda (c) c))) (define id (car (cdr (flow/start w 10)))) (flow/resume id 5) (flow-resumable-ids)") + (list)) +(flow-rec-test + "resumable-ids excludes cancelled flows after import" + (flow-r + "(defflow w (lambda (x) (suspend (quote await)))) (define id (car (cdr (flow/start w 10)))) (flow/cancel id) (define saved (flow-store-export)) (set! flow-store (list)) (flow-store-import! saved) (flow-resumable-ids)") + (list)) + +;; ── restart at every step ─────────────────────────────────────── +(flow-rec-test + "two suspends survive a restart between each step" + (flow-r + "(defflow two (sequence (lambda (x) (suspend (quote a))) (lambda (x) (suspend (quote b))) (lambda (x) (list (quote end) x)))) (define id (car (cdr (flow/start two 0)))) (define s1 (flow-store-export)) (set! flow-store (list)) (flow-store-import! s1) (flow/resume id 100) (define s2 (flow-store-export)) (set! flow-store (list)) (flow-store-import! s2) (flow/resume id 200)") + (list "end" 200)) +(flow-rec-test + "import preserves the replay log (earlier value survives restart)" + (flow-r + "(defflow two (sequence (lambda (x) (suspend (quote a))) (lambda (x) (suspend (quote b))) (lambda (x) (list x)))) (define id (car (cdr (flow/start two 0)))) (flow/resume id 11) (define saved (flow-store-export)) (set! flow-store (list)) (flow-store-import! saved) (flow/resume id 22)") + (list 22)) + +(define flow-rec-tests-run! (fn () {:total (+ flow-rec-pass flow-rec-fail) :passed flow-rec-pass :failed flow-rec-fail :fails flow-rec-fails})) diff --git a/plans/flow-on-sx.md b/plans/flow-on-sx.md index 6ee6b710..cef6d66c 100644 --- a/plans/flow-on-sx.md +++ b/plans/flow-on-sx.md @@ -16,7 +16,7 @@ federation extension via fed-sx for remote-node execution. ## Status (rolling) -`bash lib/flow/conformance.sh` → **66/66** (Phases 1-2 done; Phase 3 suspend/resume/cancel done, crash-recovery next) +`bash lib/flow/conformance.sh` → **74/74** (Phases 1-3 done; Phase 4 fed-sx next) ## Ground rules @@ -104,7 +104,11 @@ lib/flow/spec.sx lib/flow/runtime.sx lib/flow/store.sx result on completion, `(flow-suspended id tag)` on a further suspend. - [x] `(flow/cancel id)` — mark cancelled; a later resume is rejected (stale replay cannot wake a cancelled flow). -- [ ] crash recovery — on restart, scan store for paused flows, mark resumable +- [x] crash recovery — `flow-store-export` (procs nulled → plain data), + `flow-store-import!`, `flow-resumable-ids`. Records are name-keyed; resume + re-resolves the proc by name (defflow registers names), so a flow survives a + wiped store. `tests/recovery.sx`, 8 cases (export/wipe/import, resumable scan, + restart-at-every-step, replay-log survival). - [x] `lib/flow/tests/suspend.sx` — 17 cases: start/resume/cancel, multi-step, replay determinism, lifecycle guards, suspend-in-branch - Harness: `flow-run` now reuses one env with a per-test reset (building the full From e1f802cfff56f17f6b4489c6965ac7e6cf7bbf84 Mon Sep 17 00:00:00 2001 From: giles Date: Sat, 6 Jun 2026 17:40:25 +0000 Subject: [PATCH 09/20] flow: remote-node via mock fed-sx transport + 7 tests (Phase 4 begins) (remote-node addr fn) runs a node on a federation peer. Transport is the fed-sx boundary, mocked by a peer registry (flow-peer-register!); raises flow-remote-unreachable / flow-remote-no-fn. Composes with sequence/suspend/retry. Also fixes conformance.sh to load remote.sx before api.sx. Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/flow/api.sx | 15 +++++---- lib/flow/conformance.sh | 2 ++ lib/flow/remote.sx | 22 +++++++++++++ lib/flow/scoreboard.json | 9 ++--- lib/flow/scoreboard.md | 4 ++- lib/flow/tests/distributed.sx | 62 +++++++++++++++++++++++++++++++++++ plans/flow-on-sx.md | 7 ++-- 7 files changed, 108 insertions(+), 13 deletions(-) create mode 100644 lib/flow/remote.sx create mode 100644 lib/flow/tests/distributed.sx diff --git a/lib/flow/api.sx b/lib/flow/api.sx index b7810e60..757bf352 100644 --- a/lib/flow/api.sx +++ b/lib/flow/api.sx @@ -1,8 +1,8 @@ ;; lib/flow/api.sx — flow runtime entry points. ;; -;; Builds a Scheme env preloaded with the flow combinators (lib/flow/spec.sx) and -;; the durable store + lifecycle (lib/flow/store.sx), and provides SX helpers to -;; run flow programs. +;; Builds a Scheme env preloaded with the flow combinators (lib/flow/spec.sx), +;; the durable store + lifecycle (lib/flow/store.sx), and the fed-sx remote layer +;; (lib/flow/remote.sx), and provides SX helpers to run flow programs. ;; ;; Scheme-level API (available inside flow programs): ;; (flow/start flow input) — run a flow; raw result if it completes, else @@ -10,15 +10,17 @@ ;; (flow/resume id value) — resume a suspended flow (store.sx) ;; (flow/cancel id) — cancel a flow (store.sx) ;; (suspend tag) — suspension point (spec.sx) +;; (remote-node addr fn) — node executed on a federation peer (remote.sx) ;; ;; SX-level helpers (for hosts and tests): -;; (flow-make-env) — fresh standard env + combinators + store +;; (flow-make-env) — fresh standard env + combinators + store + remote ;; (flow-run src) — eval a Scheme program string in a reset shared env ;; (flow-run-in env src) — eval a Scheme program string in a given env ;; ;; flow-run reuses ONE env (building the full standard env is expensive) and ;; resets the mutable flow globals before each program, so tests stay isolated -;; without paying for a fresh standard env each time. +;; without paying for a fresh standard env each time. flow-registry persists (it +;; models reloaded flow definitions surviving a restart). (define flow-make-env @@ -28,6 +30,7 @@ ((env (scheme-standard-env))) (flow-load-combinators! env) (flow-load-store! env) + (flow-load-remote! env) env))) (define @@ -36,7 +39,7 @@ (define flow-reset-src - "(set! flow-store (list)) (set! flow-next-id 0) (set! flow-replay-log (list)) (set! flow-suspend-k #f) (set! flow-timeout-budget -1)") + "(set! flow-store (list)) (set! flow-next-id 0) (set! flow-replay-log (list)) (set! flow-suspend-k #f) (set! flow-timeout-budget -1) (set! flow-peers (list))") (define flow-env-cache false) diff --git a/lib/flow/conformance.sh b/lib/flow/conformance.sh index 237b288b..4fd45ee0 100755 --- a/lib/flow/conformance.sh +++ b/lib/flow/conformance.sh @@ -25,6 +25,7 @@ SUITES=( "control flow-ctl-tests-run! lib/flow/tests/control.sx" "suspend flow-sus-tests-run! lib/flow/tests/suspend.sx" "recovery flow-rec-tests-run! lib/flow/tests/recovery.sx" + "distributed flow-dist-tests-run! lib/flow/tests/distributed.sx" ) TMPFILE=$(mktemp); trap "rm -f $TMPFILE" EXIT @@ -42,6 +43,7 @@ emit_eval () { echo "(epoch $EPOCH)"; echo "(eval \"$1\")"; EPOCH=$((EPOCH+1)); emit_load "lib/scheme/runtime.sx" emit_load "lib/flow/spec.sx" emit_load "lib/flow/store.sx" + emit_load "lib/flow/remote.sx" emit_load "lib/flow/api.sx" for SUITE in "${SUITES[@]}"; do read -r _NAME _RUNNER FILE <<< "$SUITE" diff --git a/lib/flow/remote.sx b/lib/flow/remote.sx new file mode 100644 index 00000000..7b42ab44 --- /dev/null +++ b/lib/flow/remote.sx @@ -0,0 +1,22 @@ +;; lib/flow/remote.sx — distributed nodes via fed-sx (Phase 4). +;; +;; A node can execute on a federation peer. The transport is the fed-sx boundary; +;; it is MOCKED in tests by a peer registry mapping addr -> function table. In +;; production flow-transport would issue a fed-sx call; here it dispatches locally. +;; +;; (flow-peer-register! addr table) — register a mock peer. table is a list of +;; (fn-name proc) entries — the functions that peer exposes. +;; (flow-transport addr fn input) — invoke fn on the peer with input. Raises +;; (flow-remote-unreachable) if the addr is unknown, (flow-remote-no-fn) if the +;; peer does not expose fn. +;; (remote-node addr fn) — a node that runs fn on the peer at addr. + +(define + flow-remote-src + "(define flow-peers (list))\n (define (flow-assoc key alist)\n (if (null? alist)\n #f\n (if (eq? (car (car alist)) key) (car (cdr (car alist))) (flow-assoc key (cdr alist)))))\n (define (flow-peer-register! addr table) (set! flow-peers (cons (list addr table) flow-peers)))\n (define (flow-transport addr fn input)\n (let ((table (flow-assoc addr flow-peers)))\n (if table\n (let ((proc (flow-assoc fn table)))\n (if proc (proc input) (raise (quote flow-remote-no-fn))))\n (raise (quote flow-remote-unreachable)))))\n (define (remote-node addr fn) (lambda (input) (flow-transport addr fn input)))") + +(define + flow-load-remote! + (fn + (env) + (begin (scheme-eval-program (scheme-parse-all flow-remote-src) env) env))) diff --git a/lib/flow/scoreboard.json b/lib/flow/scoreboard.json index 63dff0b8..fd71294b 100644 --- a/lib/flow/scoreboard.json +++ b/lib/flow/scoreboard.json @@ -1,12 +1,13 @@ { - "total": 74, - "passed": 74, + "total": 81, + "passed": 81, "failed": 0, "suites": { "basic": { "passed": 18, "total": 18 }, "control": { "passed": 31, "total": 31 }, "suspend": { "passed": 17, "total": 17 }, - "recovery": { "passed": 8, "total": 8 } + "recovery": { "passed": 8, "total": 8 }, + "distributed": { "passed": 7, "total": 7 } }, - "phases": { "phase1": "done", "phase2": "done", "phase3": "done", "phase4": "pending" } + "phases": { "phase1": "done", "phase2": "done", "phase3": "done", "phase4": "in-progress" } } diff --git a/lib/flow/scoreboard.md b/lib/flow/scoreboard.md index e47a695a..45ffe7ea 100644 --- a/lib/flow/scoreboard.md +++ b/lib/flow/scoreboard.md @@ -1,6 +1,6 @@ # flow-on-sx Scoreboard -**All tests pass: 74 / 74 across 4 suites.** +**All tests pass: 81 / 81 across 5 suites.** `bash lib/flow/conformance.sh` @@ -12,6 +12,7 @@ | control | 31 | Phase 2: `branch` (6); error model `fail`/`failed?`/`fail-reason` (6); `try-catch` (6); `retry n` (6); `timeout` cooperative step budget (7) | | suspend | 17 | Phase 3: suspend/resume/cancel via deterministic replay; multi-step, replay determinism, lifecycle guards, suspend-in-branch | | recovery | 8 | Phase 3: crash recovery — store export/import, resumable scan, restart-at-every-step, replay-log survival | +| distributed | 7 | Phase 4: `remote-node` on mock fed-sx peers; compose/suspend/retry, unreachable + no-fn errors | ## Architecture @@ -40,5 +41,6 @@ capture the flow continuation directly. - [x] Phase 1 — Declarative DAG + sequential execution (combinators + 18 tests, `flow/start`) - [x] Phase 2 — Control flow + error handling (branch, error model, try-catch, retry, timeout) - [x] Phase 3 — Suspend/resume (suspend/resume/cancel + crash recovery via deterministic replay) +- [~] Phase 4 — Distributed nodes via fed-sx (remote-node done; failover + handoff next) - [ ] Phase 3 — Suspend / resume (the showcase) - [ ] Phase 4 — Distributed nodes via fed-sx diff --git a/lib/flow/tests/distributed.sx b/lib/flow/tests/distributed.sx new file mode 100644 index 00000000..15ab80d6 --- /dev/null +++ b/lib/flow/tests/distributed.sx @@ -0,0 +1,62 @@ +;; lib/flow/tests/distributed.sx — Phase 4: distributed nodes via fed-sx (mocked). + +(define flow-dist-pass 0) +(define flow-dist-fail 0) +(define flow-dist-fails (list)) + +(define + flow-dist-test + (fn + (name actual expected) + (if + (= actual expected) + (set! flow-dist-pass (+ flow-dist-pass 1)) + (begin + (set! flow-dist-fail (+ flow-dist-fail 1)) + (append! flow-dist-fails {:name name :expected expected :actual actual}))))) + +(define flow-d (fn (src) (flow-run src))) + +;; A mock peer "edge" exposing double/inc, registered at the top of each program. +(define + peer-setup + "(flow-peer-register! (quote edge) (list (list (quote double) (lambda (x) (* x 2))) (list (quote inc) (lambda (x) (+ x 1)))))") + +;; ── remote-node ───────────────────────────────────────────────── +(flow-dist-test + "remote: a node executes on a peer" + (flow-d + "(flow-peer-register! (quote edge) (list (list (quote double) (lambda (x) (* x 2))))) (flow/start (remote-node (quote edge) (quote double)) 21)") + 42) +(flow-dist-test + "remote: remote nodes compose in a sequence" + (flow-d + "(flow-peer-register! (quote edge) (list (list (quote inc) (lambda (x) (+ x 1))) (list (quote double) (lambda (x) (* x 2))))) (flow/start (sequence (remote-node (quote edge) (quote inc)) (remote-node (quote edge) (quote double))) 4)") + 10) +(flow-dist-test + "remote: a remote node mixes with local nodes" + (flow-d + "(flow-peer-register! (quote edge) (list (list (quote double) (lambda (x) (* x 2))))) (flow/start (sequence (lambda (x) (+ x 5)) (remote-node (quote edge) (quote double)) (lambda (x) (- x 1))) 10)") + 29) +(flow-dist-test + "remote: unreachable peer raises flow-remote-unreachable" + (flow-d + "(flow/start (try-catch (remote-node (quote ghost) (quote double)) (lambda (e) e)) 1)") + "flow-remote-unreachable") +(flow-dist-test + "remote: unknown function on a peer raises flow-remote-no-fn" + (flow-d + "(flow-peer-register! (quote edge) (list (list (quote double) (lambda (x) (* x 2))))) (flow/start (try-catch (remote-node (quote edge) (quote missing)) (lambda (e) e)) 1)") + "flow-remote-no-fn") +(flow-dist-test + "remote: a remote node can suspend the flow (peer returns control)" + (flow-d + "(flow-peer-register! (quote edge) (list (list (quote review) (lambda (x) x)))) (flow/start (sequence (remote-node (quote edge) (quote review)) (lambda (x) (suspend (quote human))) (lambda (v) (list (quote published) v))) 7)") + (list "flow-suspended" 1 "human")) +(flow-dist-test + "remote: a transient remote failure is recoverable with retry" + (flow-d + "(define hits 0) (flow-peer-register! (quote edge) (list (list (quote flaky) (lambda (x) (begin (set! hits (+ hits 1)) (if (< hits 2) (raise (quote down)) (* x 3))))))) (list (flow/start (retry 3 (remote-node (quote edge) (quote flaky))) 7) hits)") + (list 21 2)) + +(define flow-dist-tests-run! (fn () {:total (+ flow-dist-pass flow-dist-fail) :passed flow-dist-pass :failed flow-dist-fail :fails flow-dist-fails})) diff --git a/plans/flow-on-sx.md b/plans/flow-on-sx.md index cef6d66c..9e22921a 100644 --- a/plans/flow-on-sx.md +++ b/plans/flow-on-sx.md @@ -16,7 +16,7 @@ federation extension via fed-sx for remote-node execution. ## Status (rolling) -`bash lib/flow/conformance.sh` → **74/74** (Phases 1-3 done; Phase 4 fed-sx next) +`bash lib/flow/conformance.sh` → **81/81** (Phases 1-3 done; Phase 4 in progress: remote-node done) ## Ground rules @@ -116,7 +116,10 @@ lib/flow/spec.sx lib/flow/runtime.sx lib/flow/store.sx ## Phase 4 — Distributed nodes via fed-sx -- [ ] `(remote-node addr fn args)` — execute node on a federation peer +- [x] `(remote-node addr fn)` — execute a node on a federation peer. Transport is + the fed-sx boundary, MOCKED via a peer registry (`flow-peer-register!`); raises + `flow-remote-unreachable` / `flow-remote-no-fn`. Composes with sequence, suspend, + retry. `tests/distributed.sx`, 7 cases. - [ ] failure semantics — retry on different peer, fall through to local - [ ] persistence across instances — flow state replicates via fed-sx - [ ] handoff — flow started here can resume on a peer if the local instance is down From f8722b3b08bd42fa1d2cd330bf8f2a74d854df05 Mon Sep 17 00:00:00 2001 From: giles Date: Sat, 6 Jun 2026 17:44:04 +0000 Subject: [PATCH 10/20] =?UTF-8?q?flow:=20remote-failover=20=E2=80=94=20try?= =?UTF-8?q?=20peers=20in=20order,=20fall=20through=20to=20local=20+=206=20?= =?UTF-8?q?tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (remote-failover addrs fn local) tries fn on each peer in order, moves to the next on any raised error, and runs the local node if every peer fails. Threads input, composes in sequences. Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/flow/remote.sx | 5 ++++- lib/flow/scoreboard.json | 6 +++--- lib/flow/scoreboard.md | 4 ++-- lib/flow/tests/distributed.sx | 37 ++++++++++++++++++++++++++++++----- plans/flow-on-sx.md | 6 ++++-- 5 files changed, 45 insertions(+), 13 deletions(-) diff --git a/lib/flow/remote.sx b/lib/flow/remote.sx index 7b42ab44..8d998f49 100644 --- a/lib/flow/remote.sx +++ b/lib/flow/remote.sx @@ -10,10 +10,13 @@ ;; (flow-remote-unreachable) if the addr is unknown, (flow-remote-no-fn) if the ;; peer does not expose fn. ;; (remote-node addr fn) — a node that runs fn on the peer at addr. +;; (remote-failover addrs fn local) — try fn on each peer in addrs in order; on a +;; raised error move to the next peer; if every peer fails, run the `local` +;; node as a fallback. Threads the input through unchanged. (define flow-remote-src - "(define flow-peers (list))\n (define (flow-assoc key alist)\n (if (null? alist)\n #f\n (if (eq? (car (car alist)) key) (car (cdr (car alist))) (flow-assoc key (cdr alist)))))\n (define (flow-peer-register! addr table) (set! flow-peers (cons (list addr table) flow-peers)))\n (define (flow-transport addr fn input)\n (let ((table (flow-assoc addr flow-peers)))\n (if table\n (let ((proc (flow-assoc fn table)))\n (if proc (proc input) (raise (quote flow-remote-no-fn))))\n (raise (quote flow-remote-unreachable)))))\n (define (remote-node addr fn) (lambda (input) (flow-transport addr fn input)))") + "(define flow-peers (list))\n (define (flow-assoc key alist)\n (if (null? alist)\n #f\n (if (eq? (car (car alist)) key) (car (cdr (car alist))) (flow-assoc key (cdr alist)))))\n (define (flow-peer-register! addr table) (set! flow-peers (cons (list addr table) flow-peers)))\n (define (flow-transport addr fn input)\n (let ((table (flow-assoc addr flow-peers)))\n (if table\n (let ((proc (flow-assoc fn table)))\n (if proc (proc input) (raise (quote flow-remote-no-fn))))\n (raise (quote flow-remote-unreachable)))))\n (define (remote-node addr fn) (lambda (input) (flow-transport addr fn input)))\n (define (flow-failover-step addrs fn input local)\n (if (null? addrs)\n (local input)\n (guard (e (#t (flow-failover-step (cdr addrs) fn input local)))\n (flow-transport (car addrs) fn input))))\n (define (remote-failover addrs fn local)\n (lambda (input) (flow-failover-step addrs fn input local)))") (define flow-load-remote! diff --git a/lib/flow/scoreboard.json b/lib/flow/scoreboard.json index fd71294b..dcc71985 100644 --- a/lib/flow/scoreboard.json +++ b/lib/flow/scoreboard.json @@ -1,13 +1,13 @@ { - "total": 81, - "passed": 81, + "total": 87, + "passed": 87, "failed": 0, "suites": { "basic": { "passed": 18, "total": 18 }, "control": { "passed": 31, "total": 31 }, "suspend": { "passed": 17, "total": 17 }, "recovery": { "passed": 8, "total": 8 }, - "distributed": { "passed": 7, "total": 7 } + "distributed": { "passed": 13, "total": 13 } }, "phases": { "phase1": "done", "phase2": "done", "phase3": "done", "phase4": "in-progress" } } diff --git a/lib/flow/scoreboard.md b/lib/flow/scoreboard.md index 45ffe7ea..05749e04 100644 --- a/lib/flow/scoreboard.md +++ b/lib/flow/scoreboard.md @@ -1,6 +1,6 @@ # flow-on-sx Scoreboard -**All tests pass: 81 / 81 across 5 suites.** +**All tests pass: 87 / 87 across 5 suites.** `bash lib/flow/conformance.sh` @@ -12,7 +12,7 @@ | control | 31 | Phase 2: `branch` (6); error model `fail`/`failed?`/`fail-reason` (6); `try-catch` (6); `retry n` (6); `timeout` cooperative step budget (7) | | suspend | 17 | Phase 3: suspend/resume/cancel via deterministic replay; multi-step, replay determinism, lifecycle guards, suspend-in-branch | | recovery | 8 | Phase 3: crash recovery — store export/import, resumable scan, restart-at-every-step, replay-log survival | -| distributed | 7 | Phase 4: `remote-node` on mock fed-sx peers; compose/suspend/retry, unreachable + no-fn errors | +| distributed | 13 | Phase 4: `remote-node` on mock fed-sx peers (7); `remote-failover` across peers + local fallback (6) | ## Architecture diff --git a/lib/flow/tests/distributed.sx b/lib/flow/tests/distributed.sx index 15ab80d6..456f1c68 100644 --- a/lib/flow/tests/distributed.sx +++ b/lib/flow/tests/distributed.sx @@ -17,11 +17,6 @@ (define flow-d (fn (src) (flow-run src))) -;; A mock peer "edge" exposing double/inc, registered at the top of each program. -(define - peer-setup - "(flow-peer-register! (quote edge) (list (list (quote double) (lambda (x) (* x 2))) (list (quote inc) (lambda (x) (+ x 1)))))") - ;; ── remote-node ───────────────────────────────────────────────── (flow-dist-test "remote: a node executes on a peer" @@ -59,4 +54,36 @@ "(define hits 0) (flow-peer-register! (quote edge) (list (list (quote flaky) (lambda (x) (begin (set! hits (+ hits 1)) (if (< hits 2) (raise (quote down)) (* x 3))))))) (list (flow/start (retry 3 (remote-node (quote edge) (quote flaky))) 7) hits)") (list 21 2)) +;; ── failover (retry on a different peer, fall through to local) ── +(flow-dist-test + "failover: first reachable peer serves the request" + (flow-d + "(flow-peer-register! (quote p2) (list (list (quote f) (lambda (x) (+ x 100))))) (flow/start (remote-failover (list (quote p2) (quote down)) (quote f) (flow-const (quote local))) 5)") + 105) +(flow-dist-test + "failover: skips an unreachable peer to the next one" + (flow-d + "(flow-peer-register! (quote p2) (list (list (quote f) (lambda (x) (+ x 100))))) (flow/start (remote-failover (list (quote down) (quote p2)) (quote f) (flow-const (quote local))) 5)") + 105) +(flow-dist-test + "failover: skips a peer whose function raises" + (flow-d + "(flow-peer-register! (quote bad) (list (list (quote f) (lambda (x) (raise (quote boom)))))) (flow-peer-register! (quote good) (list (list (quote f) (lambda (x) (* x 10))))) (flow/start (remote-failover (list (quote bad) (quote good)) (quote f) (flow-const 0)) 4)") + 40) +(flow-dist-test + "failover: all peers fail, the local fallback runs" + (flow-d + "(flow/start (remote-failover (list (quote down1) (quote down2)) (quote f) (lambda (x) (* x -1))) 9)") + -9) +(flow-dist-test + "failover: threads the input through to the chosen peer" + (flow-d + "(flow-peer-register! (quote p) (list (list (quote f) (lambda (x) (list (quote got) x))))) (flow/start (sequence (lambda (x) (+ x 1)) (remote-failover (list (quote p)) (quote f) (flow-const 0))) 41)") + (list "got" 42)) +(flow-dist-test + "failover: composes inside a larger sequence" + (flow-d + "(flow-peer-register! (quote p) (list (list (quote f) (lambda (x) (* x 2))))) (flow/start (sequence (remote-failover (list (quote down) (quote p)) (quote f) (flow-const 1)) (lambda (x) (+ x 3))) 5)") + 13) + (define flow-dist-tests-run! (fn () {:total (+ flow-dist-pass flow-dist-fail) :passed flow-dist-pass :failed flow-dist-fail :fails flow-dist-fails})) diff --git a/plans/flow-on-sx.md b/plans/flow-on-sx.md index 9e22921a..6b41030c 100644 --- a/plans/flow-on-sx.md +++ b/plans/flow-on-sx.md @@ -16,7 +16,7 @@ federation extension via fed-sx for remote-node execution. ## Status (rolling) -`bash lib/flow/conformance.sh` → **81/81** (Phases 1-3 done; Phase 4 in progress: remote-node done) +`bash lib/flow/conformance.sh` → **87/87** (Phases 1-3 done; Phase 4 in progress: remote-node + failover done) ## Ground rules @@ -120,7 +120,9 @@ lib/flow/spec.sx lib/flow/runtime.sx lib/flow/store.sx the fed-sx boundary, MOCKED via a peer registry (`flow-peer-register!`); raises `flow-remote-unreachable` / `flow-remote-no-fn`. Composes with sequence, suspend, retry. `tests/distributed.sx`, 7 cases. -- [ ] failure semantics — retry on different peer, fall through to local +- [x] failure semantics — `(remote-failover addrs fn local)` tries each peer in + order, moves to the next on any raised error, and runs the `local` node if every + peer fails. 6 tests. - [ ] persistence across instances — flow state replicates via fed-sx - [ ] handoff — flow started here can resume on a peer if the local instance is down - [ ] `lib/flow/tests/distributed.sx` — federated flow scenarios (mock fed-sx in tests) From 16cb727406cca9217df339e7c24c4fbcfaf67011 Mon Sep 17 00:00:00 2001 From: giles Date: Sat, 6 Jun 2026 17:48:39 +0000 Subject: [PATCH 11/20] flow: replication + handoff across instances + 6 tests (Phase 4 complete) flow-replicate-to copies the plain-data store export to a peer's replica slot; flow-restore-from imports it. Handoff = replicate, local instance dies, peer restores and resumes by id. The replay log survives the move, so all resolved suspends carry over. Same durable-data mechanism as crash recovery, across instances. All four phases complete: 93/93. Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/flow/api.sx | 2 +- lib/flow/conformance.sh | 2 +- lib/flow/remote.sx | 13 +++++++++++-- lib/flow/scoreboard.json | 8 ++++---- lib/flow/scoreboard.md | 6 +++--- lib/flow/tests/distributed.sx | 31 +++++++++++++++++++++++++++++++ plans/flow-on-sx.md | 23 +++++++++++++++++++---- 7 files changed, 70 insertions(+), 15 deletions(-) diff --git a/lib/flow/api.sx b/lib/flow/api.sx index 757bf352..d3e8dce3 100644 --- a/lib/flow/api.sx +++ b/lib/flow/api.sx @@ -39,7 +39,7 @@ (define flow-reset-src - "(set! flow-store (list)) (set! flow-next-id 0) (set! flow-replay-log (list)) (set! flow-suspend-k #f) (set! flow-timeout-budget -1) (set! flow-peers (list))") + "(set! flow-store (list)) (set! flow-next-id 0) (set! flow-replay-log (list)) (set! flow-suspend-k #f) (set! flow-timeout-budget -1) (set! flow-peers (list)) (set! flow-replicas (list))") (define flow-env-cache false) diff --git a/lib/flow/conformance.sh b/lib/flow/conformance.sh index 4fd45ee0..f717029b 100755 --- a/lib/flow/conformance.sh +++ b/lib/flow/conformance.sh @@ -52,7 +52,7 @@ emit_eval () { echo "(epoch $EPOCH)"; echo "(eval \"$1\")"; EPOCH=$((EPOCH+1)); done } > "$TMPFILE" -OUTPUT=$(timeout 180 "$SX_SERVER" < "$TMPFILE" 2>&1 || true) +OUTPUT=$(timeout 300 "$SX_SERVER" < "$TMPFILE" 2>&1 || true) TOTAL_PASS=0 TOTAL_FAIL=0 diff --git a/lib/flow/remote.sx b/lib/flow/remote.sx index 8d998f49..2ddc6a1a 100644 --- a/lib/flow/remote.sx +++ b/lib/flow/remote.sx @@ -12,11 +12,20 @@ ;; (remote-node addr fn) — a node that runs fn on the peer at addr. ;; (remote-failover addrs fn local) — try fn on each peer in addrs in order; on a ;; raised error move to the next peer; if every peer fails, run the `local` -;; node as a fallback. Threads the input through unchanged. +;; node as a fallback. +;; +;; Persistence across instances + handoff. Each instance runs the same flow +;; definitions, so the only thing that needs to cross the wire is the (plain-data) +;; store — exactly flow-store-export from store.sx. Replication pushes that export +;; to a peer's replica slot; handoff = restore the replica on the peer and resume. +;; +;; (flow-replicate-to addr) — copy this instance's store to peer addr's replica +;; (flow-restore-from addr) — import the replica from peer addr (#t / #f) +;; (flow-replica-get addr) — the raw replicated store at addr (or #f) (define flow-remote-src - "(define flow-peers (list))\n (define (flow-assoc key alist)\n (if (null? alist)\n #f\n (if (eq? (car (car alist)) key) (car (cdr (car alist))) (flow-assoc key (cdr alist)))))\n (define (flow-peer-register! addr table) (set! flow-peers (cons (list addr table) flow-peers)))\n (define (flow-transport addr fn input)\n (let ((table (flow-assoc addr flow-peers)))\n (if table\n (let ((proc (flow-assoc fn table)))\n (if proc (proc input) (raise (quote flow-remote-no-fn))))\n (raise (quote flow-remote-unreachable)))))\n (define (remote-node addr fn) (lambda (input) (flow-transport addr fn input)))\n (define (flow-failover-step addrs fn input local)\n (if (null? addrs)\n (local input)\n (guard (e (#t (flow-failover-step (cdr addrs) fn input local)))\n (flow-transport (car addrs) fn input))))\n (define (remote-failover addrs fn local)\n (lambda (input) (flow-failover-step addrs fn input local)))") + "(define flow-peers (list))\n (define (flow-assoc key alist)\n (if (null? alist)\n #f\n (if (eq? (car (car alist)) key) (car (cdr (car alist))) (flow-assoc key (cdr alist)))))\n (define (flow-peer-register! addr table) (set! flow-peers (cons (list addr table) flow-peers)))\n (define (flow-transport addr fn input)\n (let ((table (flow-assoc addr flow-peers)))\n (if table\n (let ((proc (flow-assoc fn table)))\n (if proc (proc input) (raise (quote flow-remote-no-fn))))\n (raise (quote flow-remote-unreachable)))))\n (define (remote-node addr fn) (lambda (input) (flow-transport addr fn input)))\n (define (flow-failover-step addrs fn input local)\n (if (null? addrs)\n (local input)\n (guard (e (#t (flow-failover-step (cdr addrs) fn input local)))\n (flow-transport (car addrs) fn input))))\n (define (remote-failover addrs fn local)\n (lambda (input) (flow-failover-step addrs fn input local)))\n\n (define flow-replicas (list))\n (define (flow-replicas-remove addr reps)\n (if (null? reps)\n (list)\n (if (eq? (car (car reps)) addr)\n (flow-replicas-remove addr (cdr reps))\n (cons (car reps) (flow-replicas-remove addr (cdr reps))))))\n (define (flow-replicate-to addr)\n (set! flow-replicas (cons (list addr (flow-store-export)) (flow-replicas-remove addr flow-replicas))))\n (define (flow-replica-get addr) (flow-assoc addr flow-replicas))\n (define (flow-restore-from addr)\n (let ((data (flow-replica-get addr)))\n (if data (begin (flow-store-import! data) #t) #f)))") (define flow-load-remote! diff --git a/lib/flow/scoreboard.json b/lib/flow/scoreboard.json index dcc71985..a31a9169 100644 --- a/lib/flow/scoreboard.json +++ b/lib/flow/scoreboard.json @@ -1,13 +1,13 @@ { - "total": 87, - "passed": 87, + "total": 93, + "passed": 93, "failed": 0, "suites": { "basic": { "passed": 18, "total": 18 }, "control": { "passed": 31, "total": 31 }, "suspend": { "passed": 17, "total": 17 }, "recovery": { "passed": 8, "total": 8 }, - "distributed": { "passed": 13, "total": 13 } + "distributed": { "passed": 19, "total": 19 } }, - "phases": { "phase1": "done", "phase2": "done", "phase3": "done", "phase4": "in-progress" } + "phases": { "phase1": "done", "phase2": "done", "phase3": "done", "phase4": "done" } } diff --git a/lib/flow/scoreboard.md b/lib/flow/scoreboard.md index 05749e04..1d2e832d 100644 --- a/lib/flow/scoreboard.md +++ b/lib/flow/scoreboard.md @@ -1,6 +1,6 @@ # flow-on-sx Scoreboard -**All tests pass: 87 / 87 across 5 suites.** +**All tests pass: 93 / 93 across 5 suites. All four phases complete.** `bash lib/flow/conformance.sh` @@ -12,7 +12,7 @@ | control | 31 | Phase 2: `branch` (6); error model `fail`/`failed?`/`fail-reason` (6); `try-catch` (6); `retry n` (6); `timeout` cooperative step budget (7) | | suspend | 17 | Phase 3: suspend/resume/cancel via deterministic replay; multi-step, replay determinism, lifecycle guards, suspend-in-branch | | recovery | 8 | Phase 3: crash recovery — store export/import, resumable scan, restart-at-every-step, replay-log survival | -| distributed | 13 | Phase 4: `remote-node` on mock fed-sx peers (7); `remote-failover` across peers + local fallback (6) | +| distributed | 19 | Phase 4: `remote-node` (7); `remote-failover` (6); replication + handoff across instances (6) | ## Architecture @@ -41,6 +41,6 @@ capture the flow continuation directly. - [x] Phase 1 — Declarative DAG + sequential execution (combinators + 18 tests, `flow/start`) - [x] Phase 2 — Control flow + error handling (branch, error model, try-catch, retry, timeout) - [x] Phase 3 — Suspend/resume (suspend/resume/cancel + crash recovery via deterministic replay) -- [~] Phase 4 — Distributed nodes via fed-sx (remote-node done; failover + handoff next) +- [x] Phase 4 — Distributed nodes via fed-sx (remote-node, failover, replication + handoff) - [ ] Phase 3 — Suspend / resume (the showcase) - [ ] Phase 4 — Distributed nodes via fed-sx diff --git a/lib/flow/tests/distributed.sx b/lib/flow/tests/distributed.sx index 456f1c68..cd6bbb49 100644 --- a/lib/flow/tests/distributed.sx +++ b/lib/flow/tests/distributed.sx @@ -86,4 +86,35 @@ "(flow-peer-register! (quote p) (list (list (quote f) (lambda (x) (* x 2))))) (flow/start (sequence (remote-failover (list (quote down) (quote p)) (quote f) (flow-const 1)) (lambda (x) (+ x 3))) 5)") 13) +;; ── replication + handoff ─────────────────────────────────────── +(flow-dist-test + "replicate: a peer holds the exported store" + (flow-d + "(defflow w (lambda (x) (suspend (quote q)))) (flow/start w 10) (flow-replicate-to (quote peerB)) (if (flow-replica-get (quote peerB)) (quote replicated) (quote missing))") + "replicated") +(flow-dist-test + "handoff: a peer resumes a flow after the local instance dies" + (flow-d + "(defflow w (sequence (lambda (x) (suspend (quote q))) (lambda (v) (list (quote done) v)))) (define id (car (cdr (flow/start w 10)))) (flow-replicate-to (quote peerB)) (set! flow-store (list)) (flow-restore-from (quote peerB)) (flow/resume id 55)") + (list "done" 55)) +(flow-dist-test + "handoff: restored peer reports the flow as resumable" + (flow-d + "(defflow w (lambda (x) (suspend (quote q)))) (define id (car (cdr (flow/start w 10)))) (flow-replicate-to (quote peerB)) (set! flow-store (list)) (flow-restore-from (quote peerB)) (flow-resumable-ids)") + (list 1)) +(flow-dist-test + "handoff: without restore the dead instance has lost the flow" + (flow-d + "(defflow w (lambda (x) (suspend (quote q)))) (define id (car (cdr (flow/start w 10)))) (flow-replicate-to (quote peerB)) (set! flow-store (list)) (flow/resume id 1)") + (list "flow-error" "no-such-flow")) +(flow-dist-test + "restore: from an unknown peer yields false" + (flow-d "(flow-restore-from (quote nowhere))") + false) +(flow-dist-test + "handoff: replication preserves the replay log across the move" + (flow-d + "(defflow two (sequence (lambda (x) (suspend (quote a))) (lambda (x) (suspend (quote b))) (lambda (x) (list x)))) (define id (car (cdr (flow/start two 0)))) (flow/resume id 11) (flow-replicate-to (quote peerB)) (set! flow-store (list)) (flow-restore-from (quote peerB)) (flow/resume id 22)") + (list 22)) + (define flow-dist-tests-run! (fn () {:total (+ flow-dist-pass flow-dist-fail) :passed flow-dist-pass :failed flow-dist-fail :fails flow-dist-fails})) diff --git a/plans/flow-on-sx.md b/plans/flow-on-sx.md index 6b41030c..2a8f2e5d 100644 --- a/plans/flow-on-sx.md +++ b/plans/flow-on-sx.md @@ -16,7 +16,7 @@ federation extension via fed-sx for remote-node execution. ## Status (rolling) -`bash lib/flow/conformance.sh` → **87/87** (Phases 1-3 done; Phase 4 in progress: remote-node + failover done) +`bash lib/flow/conformance.sh` → **93/93** (all four phases complete) ## Ground rules @@ -123,9 +123,15 @@ lib/flow/spec.sx lib/flow/runtime.sx lib/flow/store.sx - [x] failure semantics — `(remote-failover addrs fn local)` tries each peer in order, moves to the next on any raised error, and runs the `local` node if every peer fails. 6 tests. -- [ ] persistence across instances — flow state replicates via fed-sx -- [ ] handoff — flow started here can resume on a peer if the local instance is down -- [ ] `lib/flow/tests/distributed.sx` — federated flow scenarios (mock fed-sx in tests) +- [x] persistence across instances — `(flow-replicate-to addr)` copies this + instance's store (the plain-data export) to a peer's replica slot; + `(flow-restore-from addr)` imports it. Same mechanism as crash recovery, across + instances. +- [x] handoff — a flow started here resumes on a peer after the local instance dies: + replicate → wipe local store → restore on peer → `flow/resume`. The replay log + (and thus all resolved suspends) survives the move. +- [x] `lib/flow/tests/distributed.sx` — 19 cases: remote-node, failover, + replication, handoff (including replay-log survival across the move) ## Progress log @@ -139,6 +145,15 @@ lib/flow/spec.sx lib/flow/runtime.sx lib/flow/store.sx combinators use `(lambda args ...)` variadics + top-level recursion. Scheme strings come back boxed as `{:scm-string "..."}` — unwrap with `(get s :scm-string)`. +- **Phases 2-4.** Control flow (branch/retry/timeout/try-catch + fail-value error + model), then the showcase: durable suspend/resume. Guest call/cc is escape-only + (re-entry hangs), so resume uses **deterministic replay** — re-run the flow, + replaying resolved suspends from a `(tag value)` log; only plain data persists, so + flows survive a wiped store (crash recovery) and a move to another instance + (replication + handoff). Phase 4 models the fed-sx boundary with a mock peer + registry. Timeout is a cooperative step budget (no wall clock in pure SX). Test + harness reuses one env with a per-test reset for speed. + ## Blockers (none) From c1d24eb9b3c4855c7c853a670fb24ccd6451a042 Mon Sep 17 00:00:00 2001 From: giles Date: Sat, 6 Jun 2026 17:53:23 +0000 Subject: [PATCH 12/20] =?UTF-8?q?flow:=20operational=20introspection=20API?= =?UTF-8?q?=20=E2=80=94=20flow/status,result,list,pending=20+=2012=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit flow/status id -> done|suspended|cancelled|unknown; flow/result id -> value or error; flow/list -> (id status) per flow; flow/pending -> (id waiting-tag) for suspended flows (operator view of what each awaits). Pure store introspection. Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/flow/conformance.sh | 1 + lib/flow/scoreboard.json | 9 +++-- lib/flow/scoreboard.md | 4 +- lib/flow/store.sx | 7 +++- lib/flow/tests/api.sx | 79 ++++++++++++++++++++++++++++++++++++++++ plans/flow-on-sx.md | 16 +++++++- 6 files changed, 109 insertions(+), 7 deletions(-) create mode 100644 lib/flow/tests/api.sx diff --git a/lib/flow/conformance.sh b/lib/flow/conformance.sh index f717029b..b3f5f143 100755 --- a/lib/flow/conformance.sh +++ b/lib/flow/conformance.sh @@ -26,6 +26,7 @@ SUITES=( "suspend flow-sus-tests-run! lib/flow/tests/suspend.sx" "recovery flow-rec-tests-run! lib/flow/tests/recovery.sx" "distributed flow-dist-tests-run! lib/flow/tests/distributed.sx" + "api flow-api-tests-run! lib/flow/tests/api.sx" ) TMPFILE=$(mktemp); trap "rm -f $TMPFILE" EXIT diff --git a/lib/flow/scoreboard.json b/lib/flow/scoreboard.json index a31a9169..b8ccffb4 100644 --- a/lib/flow/scoreboard.json +++ b/lib/flow/scoreboard.json @@ -1,13 +1,14 @@ { - "total": 93, - "passed": 93, + "total": 105, + "passed": 105, "failed": 0, "suites": { "basic": { "passed": 18, "total": 18 }, "control": { "passed": 31, "total": 31 }, "suspend": { "passed": 17, "total": 17 }, "recovery": { "passed": 8, "total": 8 }, - "distributed": { "passed": 19, "total": 19 } + "distributed": { "passed": 19, "total": 19 }, + "api": { "passed": 12, "total": 12 } }, - "phases": { "phase1": "done", "phase2": "done", "phase3": "done", "phase4": "done" } + "phases": { "phase1": "done", "phase2": "done", "phase3": "done", "phase4": "done", "phase5": "in-progress" } } diff --git a/lib/flow/scoreboard.md b/lib/flow/scoreboard.md index 1d2e832d..c1f0d720 100644 --- a/lib/flow/scoreboard.md +++ b/lib/flow/scoreboard.md @@ -1,6 +1,6 @@ # flow-on-sx Scoreboard -**All tests pass: 93 / 93 across 5 suites. All four phases complete.** +**All tests pass: 105 / 105 across 6 suites. Phases 1-4 complete; Phase 5 in progress.** `bash lib/flow/conformance.sh` @@ -13,6 +13,7 @@ | suspend | 17 | Phase 3: suspend/resume/cancel via deterministic replay; multi-step, replay determinism, lifecycle guards, suspend-in-branch | | recovery | 8 | Phase 3: crash recovery — store export/import, resumable scan, restart-at-every-step, replay-log survival | | distributed | 19 | Phase 4: `remote-node` (7); `remote-failover` (6); replication + handoff across instances (6) | +| api | 12 | Phase 5: introspection — `flow/status`, `flow/result`, `flow/list`, `flow/pending` | ## Architecture @@ -42,5 +43,6 @@ capture the flow continuation directly. - [x] Phase 2 — Control flow + error handling (branch, error model, try-catch, retry, timeout) - [x] Phase 3 — Suspend/resume (suspend/resume/cancel + crash recovery via deterministic replay) - [x] Phase 4 — Distributed nodes via fed-sx (remote-node, failover, replication + handoff) +- [~] Phase 5 — Operational API + combinators (introspection done; tap/recover/map-flow next) - [ ] Phase 3 — Suspend / resume (the showcase) - [ ] Phase 4 — Distributed nodes via fed-sx diff --git a/lib/flow/store.sx b/lib/flow/store.sx index 37284524..b88e6c5b 100644 --- a/lib/flow/store.sx +++ b/lib/flow/store.sx @@ -23,10 +23,15 @@ ;; (flow-store-export) — store as plain data (procs nulled) ;; (flow-store-import! d) — replace the store from exported data ;; (flow-resumable-ids) — ids of suspended (resumable) flows +;; Introspection (Phase 5): +;; (flow/status id) — done | suspended | cancelled | unknown +;; (flow/result id) — result if done, else (flow-error reason) +;; (flow/list) — list of (id status) for every flow +;; (flow/pending) — list of (id waiting-tag) for suspended flows (define flow-store-src - "(define flow-registry (list))\n (define (flow-register! name proc) (set! flow-registry (cons (list name proc) flow-registry)))\n (define (flow-lookup-in name reg)\n (if (null? reg)\n #f\n (if (eq? (car (car reg)) name) (car (cdr (car reg))) (flow-lookup-in name (cdr reg)))))\n (define (flow-lookup name) (flow-lookup-in name flow-registry))\n (define (flow-name-of proc reg)\n (if (null? reg)\n #f\n (if (eq? (car (cdr (car reg))) proc) (car (car reg)) (flow-name-of proc (cdr reg)))))\n\n (define flow-store (list))\n (define flow-next-id 0)\n (define (flow-store-remove id store)\n (if (null? store)\n (list)\n (if (= (car (car store)) id)\n (flow-store-remove id (cdr store))\n (cons (car store) (flow-store-remove id (cdr store))))))\n (define (flow-store-put! id rec) (set! flow-store (cons (list id rec) (flow-store-remove id flow-store))))\n (define (flow-store-find id store)\n (if (null? store)\n (list)\n (if (= (car (car store)) id)\n (car (cdr (car store)))\n (flow-store-find id (cdr store)))))\n (define (flow-store-get id) (flow-store-find id flow-store))\n\n (define (flow-mk-rec name proc input log status payload)\n (list name proc input log status payload))\n (define (flow-rec-name r) (car r))\n (define (flow-rec-proc r) (car (cdr r)))\n (define (flow-rec-input r) (car (cdr (cdr r))))\n (define (flow-rec-log r) (car (cdr (cdr (cdr r)))))\n (define (flow-rec-status r) (car (cdr (cdr (cdr (cdr r))))))\n (define (flow-rec-payload r) (car (cdr (cdr (cdr (cdr (cdr r)))))))\n (define (flow-rec-resolve rec)\n (let ((byname (flow-lookup (flow-rec-name rec))))\n (if byname byname (flow-rec-proc rec))))\n\n (define (flow-outcome id name proc input log outcome)\n (if (eq? (car outcome) (quote flow-done))\n (begin\n (flow-store-put! id (flow-mk-rec name proc input log (quote done) (car (cdr outcome))))\n (car (cdr outcome)))\n (begin\n (flow-store-put! id (flow-mk-rec name proc input log (quote suspended) (car (cdr outcome))))\n (list (quote flow-suspended) id (car (cdr outcome))))))\n (define (flow/start flow input)\n (set! flow-next-id (+ flow-next-id 1))\n (flow-outcome flow-next-id (flow-name-of flow flow-registry) flow input (list)\n (flow-drive flow input (list))))\n (define (flow/resume id value)\n (let ((rec (flow-store-get id)))\n (if (null? rec)\n (list (quote flow-error) (quote no-such-flow))\n (if (eq? (flow-rec-status rec) (quote suspended))\n (let ((proc (flow-rec-resolve rec)))\n (let ((newlog (cons (list (flow-rec-payload rec) value) (flow-rec-log rec))))\n (flow-outcome id (flow-rec-name rec) proc (flow-rec-input rec) newlog\n (flow-drive proc (flow-rec-input rec) newlog))))\n (list (quote flow-error) (quote not-suspended))))))\n (define (flow/cancel id)\n (let ((rec (flow-store-get id)))\n (if (null? rec)\n (list (quote flow-error) (quote no-such-flow))\n (begin\n (flow-store-put! id\n (flow-mk-rec (flow-rec-name rec) (flow-rec-proc rec) (flow-rec-input rec)\n (flow-rec-log rec) (quote cancelled) #f))\n (list (quote flow-cancelled) id)))))\n\n (define (flow-export-entry entry)\n (let ((rec (car (cdr entry))))\n (list (car entry)\n (flow-mk-rec (flow-rec-name rec) #f (flow-rec-input rec)\n (flow-rec-log rec) (flow-rec-status rec) (flow-rec-payload rec)))))\n (define (flow-export-map store)\n (if (null? store) (list) (cons (flow-export-entry (car store)) (flow-export-map (cdr store)))))\n (define (flow-store-export) (flow-export-map flow-store))\n (define (flow-max-id store m)\n (if (null? store) m (flow-max-id (cdr store) (if (> (car (car store)) m) (car (car store)) m))))\n (define (flow-store-import! data)\n (begin (set! flow-store data) (set! flow-next-id (flow-max-id data 0))))\n (define (flow-collect-resumable store)\n (if (null? store)\n (list)\n (if (eq? (flow-rec-status (car (cdr (car store)))) (quote suspended))\n (cons (car (car store)) (flow-collect-resumable (cdr store)))\n (flow-collect-resumable (cdr store)))))\n (define (flow-resumable-ids) (flow-collect-resumable flow-store))") + "(define flow-registry (list))\n (define (flow-register! name proc) (set! flow-registry (cons (list name proc) flow-registry)))\n (define (flow-lookup-in name reg)\n (if (null? reg)\n #f\n (if (eq? (car (car reg)) name) (car (cdr (car reg))) (flow-lookup-in name (cdr reg)))))\n (define (flow-lookup name) (flow-lookup-in name flow-registry))\n (define (flow-name-of proc reg)\n (if (null? reg)\n #f\n (if (eq? (car (cdr (car reg))) proc) (car (car reg)) (flow-name-of proc (cdr reg)))))\n\n (define flow-store (list))\n (define flow-next-id 0)\n (define (flow-store-remove id store)\n (if (null? store)\n (list)\n (if (= (car (car store)) id)\n (flow-store-remove id (cdr store))\n (cons (car store) (flow-store-remove id (cdr store))))))\n (define (flow-store-put! id rec) (set! flow-store (cons (list id rec) (flow-store-remove id flow-store))))\n (define (flow-store-find id store)\n (if (null? store)\n (list)\n (if (= (car (car store)) id)\n (car (cdr (car store)))\n (flow-store-find id (cdr store)))))\n (define (flow-store-get id) (flow-store-find id flow-store))\n\n (define (flow-mk-rec name proc input log status payload)\n (list name proc input log status payload))\n (define (flow-rec-name r) (car r))\n (define (flow-rec-proc r) (car (cdr r)))\n (define (flow-rec-input r) (car (cdr (cdr r))))\n (define (flow-rec-log r) (car (cdr (cdr (cdr r)))))\n (define (flow-rec-status r) (car (cdr (cdr (cdr (cdr r))))))\n (define (flow-rec-payload r) (car (cdr (cdr (cdr (cdr (cdr r)))))))\n (define (flow-rec-resolve rec)\n (let ((byname (flow-lookup (flow-rec-name rec))))\n (if byname byname (flow-rec-proc rec))))\n\n (define (flow-outcome id name proc input log outcome)\n (if (eq? (car outcome) (quote flow-done))\n (begin\n (flow-store-put! id (flow-mk-rec name proc input log (quote done) (car (cdr outcome))))\n (car (cdr outcome)))\n (begin\n (flow-store-put! id (flow-mk-rec name proc input log (quote suspended) (car (cdr outcome))))\n (list (quote flow-suspended) id (car (cdr outcome))))))\n (define (flow/start flow input)\n (set! flow-next-id (+ flow-next-id 1))\n (flow-outcome flow-next-id (flow-name-of flow flow-registry) flow input (list)\n (flow-drive flow input (list))))\n (define (flow/resume id value)\n (let ((rec (flow-store-get id)))\n (if (null? rec)\n (list (quote flow-error) (quote no-such-flow))\n (if (eq? (flow-rec-status rec) (quote suspended))\n (let ((proc (flow-rec-resolve rec)))\n (let ((newlog (cons (list (flow-rec-payload rec) value) (flow-rec-log rec))))\n (flow-outcome id (flow-rec-name rec) proc (flow-rec-input rec) newlog\n (flow-drive proc (flow-rec-input rec) newlog))))\n (list (quote flow-error) (quote not-suspended))))))\n (define (flow/cancel id)\n (let ((rec (flow-store-get id)))\n (if (null? rec)\n (list (quote flow-error) (quote no-such-flow))\n (begin\n (flow-store-put! id\n (flow-mk-rec (flow-rec-name rec) (flow-rec-proc rec) (flow-rec-input rec)\n (flow-rec-log rec) (quote cancelled) #f))\n (list (quote flow-cancelled) id)))))\n\n (define (flow-export-entry entry)\n (let ((rec (car (cdr entry))))\n (list (car entry)\n (flow-mk-rec (flow-rec-name rec) #f (flow-rec-input rec)\n (flow-rec-log rec) (flow-rec-status rec) (flow-rec-payload rec)))))\n (define (flow-export-map store)\n (if (null? store) (list) (cons (flow-export-entry (car store)) (flow-export-map (cdr store)))))\n (define (flow-store-export) (flow-export-map flow-store))\n (define (flow-max-id store m)\n (if (null? store) m (flow-max-id (cdr store) (if (> (car (car store)) m) (car (car store)) m))))\n (define (flow-store-import! data)\n (begin (set! flow-store data) (set! flow-next-id (flow-max-id data 0))))\n (define (flow-collect-resumable store)\n (if (null? store)\n (list)\n (if (eq? (flow-rec-status (car (cdr (car store)))) (quote suspended))\n (cons (car (car store)) (flow-collect-resumable (cdr store)))\n (flow-collect-resumable (cdr store)))))\n (define (flow-resumable-ids) (flow-collect-resumable flow-store))\n\n (define (flow/status id)\n (let ((rec (flow-store-get id)))\n (if (null? rec) (quote unknown) (flow-rec-status rec))))\n (define (flow/result id)\n (let ((rec (flow-store-get id)))\n (if (null? rec)\n (list (quote flow-error) (quote no-such-flow))\n (if (eq? (flow-rec-status rec) (quote done))\n (flow-rec-payload rec)\n (list (quote flow-error) (quote not-done))))))\n (define (flow-list-step store)\n (if (null? store)\n (list)\n (cons (list (car (car store)) (flow-rec-status (car (cdr (car store)))))\n (flow-list-step (cdr store)))))\n (define (flow/list) (flow-list-step flow-store))\n (define (flow-pending-step store)\n (if (null? store)\n (list)\n (if (eq? (flow-rec-status (car (cdr (car store)))) (quote suspended))\n (cons (list (car (car store)) (flow-rec-payload (car (cdr (car store)))))\n (flow-pending-step (cdr store)))\n (flow-pending-step (cdr store)))))\n (define (flow/pending) (flow-pending-step flow-store))") (define flow-load-store! diff --git a/lib/flow/tests/api.sx b/lib/flow/tests/api.sx new file mode 100644 index 00000000..6211b0f0 --- /dev/null +++ b/lib/flow/tests/api.sx @@ -0,0 +1,79 @@ +;; lib/flow/tests/api.sx — Phase 5: operational introspection API. + +(define flow-api-pass 0) +(define flow-api-fail 0) +(define flow-api-fails (list)) + +(define + flow-api-test + (fn + (name actual expected) + (if + (= actual expected) + (set! flow-api-pass (+ flow-api-pass 1)) + (begin + (set! flow-api-fail (+ flow-api-fail 1)) + (append! flow-api-fails {:name name :expected expected :actual actual}))))) + +(define flow-a (fn (src) (flow-run src))) + +;; ── flow/status ───────────────────────────────────────────────── +(flow-api-test "status: unknown id" (flow-a "(flow/status 999)") "unknown") +(flow-api-test + "status: suspended flow" + (flow-a + "(defflow w (lambda (x) (suspend (quote q)))) (define id (car (cdr (flow/start w 0)))) (flow/status id)") + "suspended") +(flow-api-test + "status: completed flow" + (flow-a + "(defflow w (sequence (lambda (x) (suspend (quote q))) (lambda (v) v))) (define id (car (cdr (flow/start w 0)))) (flow/resume id 5) (flow/status id)") + "done") +(flow-api-test + "status: cancelled flow" + (flow-a + "(defflow w (lambda (x) (suspend (quote q)))) (define id (car (cdr (flow/start w 0)))) (flow/cancel id) (flow/status id)") + "cancelled") + +;; ── flow/result ───────────────────────────────────────────────── +(flow-api-test + "result: returns the value of a completed flow" + (flow-a + "(defflow w (sequence (lambda (x) (suspend (quote q))) (lambda (v) (list (quote got) v)))) (define id (car (cdr (flow/start w 0)))) (flow/resume id 9) (flow/result id)") + (list "got" 9)) +(flow-api-test + "result: a still-suspended flow has no result" + (flow-a + "(defflow w (lambda (x) (suspend (quote q)))) (define id (car (cdr (flow/start w 0)))) (flow/result id)") + (list "flow-error" "not-done")) +(flow-api-test + "result: unknown id errors" + (flow-a "(flow/result 999)") + (list "flow-error" "no-such-flow")) + +;; ── flow/list ─────────────────────────────────────────────────── +(flow-api-test "list: empty store" (flow-a "(flow/list)") (list)) +(flow-api-test + "list: reports id + status for each flow (newest first)" + (flow-a + "(defflow w (lambda (x) (suspend (quote q)))) (flow/start w 0) (flow/start (lambda (x) (* x 2)) 5) (flow/list)") + (list (list 2 "done") (list 1 "suspended"))) + +;; ── flow/pending ──────────────────────────────────────────────── +(flow-api-test + "pending: lists suspended flows with their waiting tag" + (flow-a + "(defflow w (lambda (x) (suspend (quote review)))) (flow/start w 0) (flow/pending)") + (list (list 1 "review"))) +(flow-api-test + "pending: excludes completed and cancelled flows" + (flow-a + "(defflow w (lambda (x) (suspend (quote q)))) (defflow v (sequence (lambda (x) (suspend (quote r))) (lambda (y) y))) (define i1 (car (cdr (flow/start w 0)))) (define i2 (car (cdr (flow/start v 0)))) (define i3 (car (cdr (flow/start w 0)))) (flow/resume i2 1) (flow/cancel i3) (flow/pending)") + (list (list 1 "q"))) +(flow-api-test + "pending: operator can drain all pending flows" + (flow-a + "(defflow w (sequence (lambda (x) (suspend (quote q))) (lambda (v) (* v 10)))) (flow/start w 0) (flow/start w 0) (define ps (flow/pending)) (flow/resume (car (car ps)) 1) (flow/resume (car (car (cdr ps))) 2) (flow/list)") + (list (list 1 "done") (list 2 "done"))) + +(define flow-api-tests-run! (fn () {:total (+ flow-api-pass flow-api-fail) :passed flow-api-pass :failed flow-api-fail :fails flow-api-fails})) diff --git a/plans/flow-on-sx.md b/plans/flow-on-sx.md index 2a8f2e5d..4c9640b3 100644 --- a/plans/flow-on-sx.md +++ b/plans/flow-on-sx.md @@ -16,7 +16,7 @@ federation extension via fed-sx for remote-node execution. ## Status (rolling) -`bash lib/flow/conformance.sh` → **93/93** (all four phases complete) +`bash lib/flow/conformance.sh` → **105/105** (Phases 1-4 complete; Phase 5 in progress) ## Ground rules @@ -133,6 +133,20 @@ lib/flow/spec.sx lib/flow/runtime.sx lib/flow/store.sx - [x] `lib/flow/tests/distributed.sx` — 19 cases: remote-node, failover, replication, handoff (including replay-log survival across the move) +## Phase 5 — Operational API + combinator library + +The four roadmap phases are complete; this phase rounds out the engine into +something operators and authors actually use. Accumulation, not a rewrite. + +- [x] introspection API — `flow/status id`, `flow/result id`, `flow/list`, + `flow/pending` (operator view of what each suspended flow awaits). 12 tests in + `tests/api.sx`. +- [ ] `tap` — side-effecting pass-through node (logging/metrics) that returns input +- [ ] `recover` — complement to try-catch for the fail-VALUE channel: run node; if it + yields `(fail ...)`, run a recovery node on the reason +- [ ] `map-flow` — run a flow per item of a list, join results (sequential) +- [ ] `lib/flow/tests/api.sx` — introspection + new combinators + ## Progress log - **Phase 1 (combinators + sequential runtime).** Flow built as a Scheme prelude From 0e6ba556476b8c174cf1395fa6851d918e2ca065 Mon Sep 17 00:00:00 2001 From: giles Date: Sat, 6 Jun 2026 17:57:48 +0000 Subject: [PATCH 13/20] =?UTF-8?q?flow:=20combinator=20library=20=E2=80=94?= =?UTF-8?q?=20tap,=20recover,=20map-flow=20+=2011=20tests=20(Phase=205=20c?= =?UTF-8?q?omplete)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tap: side-effecting pass-through (returns input). recover: fail-VALUE counterpart of try-catch (run node; on (fail r) run handler on r). map-flow: run a node over each item of a list, join results sequentially. 116/116 across 7 suites. Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/flow/conformance.sh | 1 + lib/flow/scoreboard.json | 9 ++-- lib/flow/scoreboard.md | 5 ++- lib/flow/spec.sx | 7 +++- lib/flow/tests/combinators.sx | 77 +++++++++++++++++++++++++++++++++++ plans/flow-on-sx.md | 10 ++--- 6 files changed, 96 insertions(+), 13 deletions(-) create mode 100644 lib/flow/tests/combinators.sx diff --git a/lib/flow/conformance.sh b/lib/flow/conformance.sh index b3f5f143..53542736 100755 --- a/lib/flow/conformance.sh +++ b/lib/flow/conformance.sh @@ -27,6 +27,7 @@ SUITES=( "recovery flow-rec-tests-run! lib/flow/tests/recovery.sx" "distributed flow-dist-tests-run! lib/flow/tests/distributed.sx" "api flow-api-tests-run! lib/flow/tests/api.sx" + "combinators flow-cmb-tests-run! lib/flow/tests/combinators.sx" ) TMPFILE=$(mktemp); trap "rm -f $TMPFILE" EXIT diff --git a/lib/flow/scoreboard.json b/lib/flow/scoreboard.json index b8ccffb4..0b50b43b 100644 --- a/lib/flow/scoreboard.json +++ b/lib/flow/scoreboard.json @@ -1,6 +1,6 @@ { - "total": 105, - "passed": 105, + "total": 116, + "passed": 116, "failed": 0, "suites": { "basic": { "passed": 18, "total": 18 }, @@ -8,7 +8,8 @@ "suspend": { "passed": 17, "total": 17 }, "recovery": { "passed": 8, "total": 8 }, "distributed": { "passed": 19, "total": 19 }, - "api": { "passed": 12, "total": 12 } + "api": { "passed": 12, "total": 12 }, + "combinators": { "passed": 11, "total": 11 } }, - "phases": { "phase1": "done", "phase2": "done", "phase3": "done", "phase4": "done", "phase5": "in-progress" } + "phases": { "phase1": "done", "phase2": "done", "phase3": "done", "phase4": "done", "phase5": "done" } } diff --git a/lib/flow/scoreboard.md b/lib/flow/scoreboard.md index c1f0d720..a35e93f9 100644 --- a/lib/flow/scoreboard.md +++ b/lib/flow/scoreboard.md @@ -1,6 +1,6 @@ # flow-on-sx Scoreboard -**All tests pass: 105 / 105 across 6 suites. Phases 1-4 complete; Phase 5 in progress.** +**All tests pass: 116 / 116 across 7 suites. Phases 1-5 complete.** `bash lib/flow/conformance.sh` @@ -14,6 +14,7 @@ | recovery | 8 | Phase 3: crash recovery — store export/import, resumable scan, restart-at-every-step, replay-log survival | | distributed | 19 | Phase 4: `remote-node` (7); `remote-failover` (6); replication + handoff across instances (6) | | api | 12 | Phase 5: introspection — `flow/status`, `flow/result`, `flow/list`, `flow/pending` | +| combinators | 11 | Phase 5: `tap` side-effect, `recover` (fail-value), `map-flow` fan-over-list | ## Architecture @@ -43,6 +44,6 @@ capture the flow continuation directly. - [x] Phase 2 — Control flow + error handling (branch, error model, try-catch, retry, timeout) - [x] Phase 3 — Suspend/resume (suspend/resume/cancel + crash recovery via deterministic replay) - [x] Phase 4 — Distributed nodes via fed-sx (remote-node, failover, replication + handoff) -- [~] Phase 5 — Operational API + combinators (introspection done; tap/recover/map-flow next) +- [x] Phase 5 — Operational API + combinators (introspection, tap, recover, map-flow) - [ ] Phase 3 — Suspend / resume (the showcase) - [ ] Phase 4 — Distributed nodes via fed-sx diff --git a/lib/flow/spec.sx b/lib/flow/spec.sx index 8a610724..c15e9cb3 100644 --- a/lib/flow/spec.sx +++ b/lib/flow/spec.sx @@ -10,9 +10,12 @@ ;; flow-node / flow-id / flow-const / sequence / parallel / defflow ;; defflow both binds the flow and registers it by name (flow-register!, in ;; store.sx) so it can be re-resolved after a process restart. +;; map-flow (Phase 5): run a node over each item of a list input, join results. ;; ;; Phase 2 combinators (flow-control-src): ;; branch / fail / failed? / fail-reason / try-catch / retry / timeout / tick +;; tap (Phase 5): side-effecting pass-through (returns input unchanged). +;; recover (Phase 5): the fail-VALUE counterpart of try-catch. ;; ;; Phase 3 suspend core (flow-suspend-src): ;; The guest Scheme's call/cc is ESCAPE-ONLY (re-invoking a captured k after it @@ -32,11 +35,11 @@ (define flow-combinators-src - "(define (flow-node f) f)\n (define (flow-id input) input)\n (define (flow-const v) (lambda (input) v))\n (define (flow-seq-step ns v)\n (if (null? ns) v (flow-seq-step (cdr ns) ((car ns) v))))\n (define sequence (lambda ns (lambda (input) (flow-seq-step ns input))))\n (define parallel (lambda ns (lambda (input) (map (lambda (n) (n input)) ns))))\n (define-syntax defflow\n (syntax-rules ()\n ((defflow nm body)\n (begin (define nm body) (flow-register! (quote nm) nm)))))") + "(define (flow-node f) f)\n (define (flow-id input) input)\n (define (flow-const v) (lambda (input) v))\n (define (flow-seq-step ns v)\n (if (null? ns) v (flow-seq-step (cdr ns) ((car ns) v))))\n (define sequence (lambda ns (lambda (input) (flow-seq-step ns input))))\n (define parallel (lambda ns (lambda (input) (map (lambda (n) (n input)) ns))))\n (define (map-flow node) (lambda (items) (map node items)))\n (define-syntax defflow\n (syntax-rules ()\n ((defflow nm body)\n (begin (define nm body) (flow-register! (quote nm) nm)))))") (define flow-control-src - "(define (branch pred then else)\n (lambda (input) (if (pred input) (then input) (else input))))\n (define (fail reason) (list (quote flow-fail) reason))\n (define (failed? x) (and (pair? x) (eq? (car x) (quote flow-fail))))\n (define (fail-reason x) (car (cdr x)))\n (define (try-catch node handler)\n (lambda (input) (guard (e (#t (handler e))) (node input))))\n (define (flow-retry-step n node input)\n (guard (e (#t (if (<= n 1) (raise e) (flow-retry-step (- n 1) node input))))\n (node input)))\n (define (retry n node) (lambda (input) (flow-retry-step n node input)))\n (define flow-timeout-budget -1)\n (define (tick)\n (if (< flow-timeout-budget 0)\n 0\n (begin\n (set! flow-timeout-budget (- flow-timeout-budget 1))\n (if (< flow-timeout-budget 0)\n (raise (quote flow-timeout))\n flow-timeout-budget))))\n (define (timeout budget node)\n (lambda (input)\n (let ((saved flow-timeout-budget))\n (set! flow-timeout-budget budget)\n (guard (e (#t (begin (set! flow-timeout-budget saved) (raise e))))\n (let ((result (node input)))\n (set! flow-timeout-budget saved)\n result)))))") + "(define (branch pred then else)\n (lambda (input) (if (pred input) (then input) (else input))))\n (define (fail reason) (list (quote flow-fail) reason))\n (define (failed? x) (and (pair? x) (eq? (car x) (quote flow-fail))))\n (define (fail-reason x) (car (cdr x)))\n (define (recover node handler)\n (lambda (input)\n (let ((r (node input)))\n (if (failed? r) (handler (fail-reason r)) r))))\n (define (tap effect)\n (lambda (input) (begin (effect input) input)))\n (define (try-catch node handler)\n (lambda (input) (guard (e (#t (handler e))) (node input))))\n (define (flow-retry-step n node input)\n (guard (e (#t (if (<= n 1) (raise e) (flow-retry-step (- n 1) node input))))\n (node input)))\n (define (retry n node) (lambda (input) (flow-retry-step n node input)))\n (define flow-timeout-budget -1)\n (define (tick)\n (if (< flow-timeout-budget 0)\n 0\n (begin\n (set! flow-timeout-budget (- flow-timeout-budget 1))\n (if (< flow-timeout-budget 0)\n (raise (quote flow-timeout))\n flow-timeout-budget))))\n (define (timeout budget node)\n (lambda (input)\n (let ((saved flow-timeout-budget))\n (set! flow-timeout-budget budget)\n (guard (e (#t (begin (set! flow-timeout-budget saved) (raise e))))\n (let ((result (node input)))\n (set! flow-timeout-budget saved)\n result)))))") (define flow-suspend-src diff --git a/lib/flow/tests/combinators.sx b/lib/flow/tests/combinators.sx new file mode 100644 index 00000000..467e010c --- /dev/null +++ b/lib/flow/tests/combinators.sx @@ -0,0 +1,77 @@ +;; lib/flow/tests/combinators.sx — Phase 5: combinator library (tap, recover, map-flow). + +(define flow-cmb-pass 0) +(define flow-cmb-fail 0) +(define flow-cmb-fails (list)) + +(define + flow-cmb-test + (fn + (name actual expected) + (if + (= actual expected) + (set! flow-cmb-pass (+ flow-cmb-pass 1)) + (begin + (set! flow-cmb-fail (+ flow-cmb-fail 1)) + (append! flow-cmb-fails {:name name :expected expected :actual actual}))))) + +(define flow-m (fn (src) (flow-run src))) + +;; ── tap (side-effecting pass-through) ─────────────────────────── +(flow-cmb-test + "tap: returns input unchanged" + (flow-m "(flow/start (tap (lambda (x) (* x 999))) 7)") + 7) +(flow-cmb-test + "tap: runs the side effect" + (flow-m + "(define seen 0) (flow/start (tap (lambda (x) (set! seen x))) 42) seen") + 42) +(flow-cmb-test + "tap: value flows on while the effect observes it" + (flow-m + "(define log 0) (flow/start (sequence (lambda (x) (+ x 1)) (tap (lambda (x) (set! log x))) (lambda (x) (* x 2))) 10) (list log (flow/result 1))") + (list 11 22)) + +;; ── recover (fail-value counterpart of try-catch) ─────────────── +(flow-cmb-test + "recover: passes a non-fail value through" + (flow-m "(flow/start (recover (lambda (x) (* x 2)) (lambda (r) -1)) 5)") + 10) +(flow-cmb-test + "recover: handles a fail value via the reason" + (flow-m + "(flow/start (recover (lambda (x) (fail (quote too-small))) (lambda (r) (list (quote recovered) r))) 1)") + (list "recovered" "too-small")) +(flow-cmb-test + "recover: handler can supply a default value" + (flow-m + "(flow/start (sequence (recover (lambda (x) (if (> x 0) x (fail (quote neg))) ) (flow-const 0)) (lambda (x) (* x 10))) -3)") + 0) +(flow-cmb-test + "recover: does not catch raised exceptions (those are try-catch's job)" + (flow-m + "(flow/start (try-catch (recover (lambda (x) (raise (quote boom))) (flow-const 0)) (lambda (e) e)) 1)") + "boom") + +;; ── map-flow (run a node over a list, join) ───────────────────── +(flow-cmb-test + "map-flow: applies the node to each item" + (flow-m "(flow/start (map-flow (lambda (x) (* x x))) (list 1 2 3 4))") + (list 1 4 9 16)) +(flow-cmb-test + "map-flow: empty list joins to empty" + (flow-m "(flow/start (map-flow (lambda (x) (+ x 1))) (list))") + (list)) +(flow-cmb-test + "map-flow: each item runs an independent sub-flow" + (flow-m + "(flow/start (map-flow (sequence (lambda (x) (+ x 1)) (lambda (x) (* x 2)))) (list 0 4 9))") + (list 2 10 20)) +(flow-cmb-test + "map-flow: composes — fan over a list then reduce the join" + (flow-m + "(flow/start (sequence (map-flow (lambda (x) (* x 10))) (lambda (xs) (apply + xs))) (list 1 2 3))") + 60) + +(define flow-cmb-tests-run! (fn () {:total (+ flow-cmb-pass flow-cmb-fail) :passed flow-cmb-pass :failed flow-cmb-fail :fails flow-cmb-fails})) diff --git a/plans/flow-on-sx.md b/plans/flow-on-sx.md index 4c9640b3..0deaaac6 100644 --- a/plans/flow-on-sx.md +++ b/plans/flow-on-sx.md @@ -16,7 +16,7 @@ federation extension via fed-sx for remote-node execution. ## Status (rolling) -`bash lib/flow/conformance.sh` → **105/105** (Phases 1-4 complete; Phase 5 in progress) +`bash lib/flow/conformance.sh` → **116/116** (Phases 1-5 complete) ## Ground rules @@ -141,11 +141,11 @@ something operators and authors actually use. Accumulation, not a rewrite. - [x] introspection API — `flow/status id`, `flow/result id`, `flow/list`, `flow/pending` (operator view of what each suspended flow awaits). 12 tests in `tests/api.sx`. -- [ ] `tap` — side-effecting pass-through node (logging/metrics) that returns input -- [ ] `recover` — complement to try-catch for the fail-VALUE channel: run node; if it +- [x] `tap` — side-effecting pass-through node (logging/metrics) that returns input +- [x] `recover` — complement to try-catch for the fail-VALUE channel: run node; if it yields `(fail ...)`, run a recovery node on the reason -- [ ] `map-flow` — run a flow per item of a list, join results (sequential) -- [ ] `lib/flow/tests/api.sx` — introspection + new combinators +- [x] `map-flow` — run a flow per item of a list, join results (sequential) +- [x] `lib/flow/tests/api.sx` (12) + `lib/flow/tests/combinators.sx` (11) ## Progress log From 0a1b89c97589e11fe958a85c02debe6c59cb28f7 Mon Sep 17 00:00:00 2001 From: giles Date: Sat, 6 Jun 2026 18:02:59 +0000 Subject: [PATCH 14/20] flow: bounded iteration combinators flow-while/flow-until + 6 tests (flow-while pred body max) / (flow-until pred body max) re-run body threading the value while/until pred holds, capped at max steps for a deterministic bound (no unbounded loops in pure SX). 122/122 across 7 suites. Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/flow/scoreboard.json | 6 +++--- lib/flow/scoreboard.md | 4 ++-- lib/flow/spec.sx | 5 ++++- lib/flow/tests/combinators.sx | 33 ++++++++++++++++++++++++++++++++- plans/flow-on-sx.md | 6 ++++-- 5 files changed, 45 insertions(+), 9 deletions(-) diff --git a/lib/flow/scoreboard.json b/lib/flow/scoreboard.json index 0b50b43b..0c32983f 100644 --- a/lib/flow/scoreboard.json +++ b/lib/flow/scoreboard.json @@ -1,6 +1,6 @@ { - "total": 116, - "passed": 116, + "total": 122, + "passed": 122, "failed": 0, "suites": { "basic": { "passed": 18, "total": 18 }, @@ -9,7 +9,7 @@ "recovery": { "passed": 8, "total": 8 }, "distributed": { "passed": 19, "total": 19 }, "api": { "passed": 12, "total": 12 }, - "combinators": { "passed": 11, "total": 11 } + "combinators": { "passed": 17, "total": 17 } }, "phases": { "phase1": "done", "phase2": "done", "phase3": "done", "phase4": "done", "phase5": "done" } } diff --git a/lib/flow/scoreboard.md b/lib/flow/scoreboard.md index a35e93f9..988adb5f 100644 --- a/lib/flow/scoreboard.md +++ b/lib/flow/scoreboard.md @@ -1,6 +1,6 @@ # flow-on-sx Scoreboard -**All tests pass: 116 / 116 across 7 suites. Phases 1-5 complete.** +**All tests pass: 122 / 122 across 7 suites. Phases 1-5 complete.** `bash lib/flow/conformance.sh` @@ -14,7 +14,7 @@ | recovery | 8 | Phase 3: crash recovery — store export/import, resumable scan, restart-at-every-step, replay-log survival | | distributed | 19 | Phase 4: `remote-node` (7); `remote-failover` (6); replication + handoff across instances (6) | | api | 12 | Phase 5: introspection — `flow/status`, `flow/result`, `flow/list`, `flow/pending` | -| combinators | 11 | Phase 5: `tap` side-effect, `recover` (fail-value), `map-flow` fan-over-list | +| combinators | 17 | Phase 5: `tap`, `recover` (fail-value), `map-flow` fan-over-list, `flow-while`/`flow-until` bounded iteration | ## Architecture diff --git a/lib/flow/spec.sx b/lib/flow/spec.sx index c15e9cb3..933c571c 100644 --- a/lib/flow/spec.sx +++ b/lib/flow/spec.sx @@ -11,6 +11,9 @@ ;; defflow both binds the flow and registers it by name (flow-register!, in ;; store.sx) so it can be re-resolved after a process restart. ;; map-flow (Phase 5): run a node over each item of a list input, join results. +;; flow-while / flow-until (Phase 5): bounded iteration — re-run body, threading +;; the value, while/until pred holds, up to `max` steps (deterministic bound; no +;; unbounded loops in pure SX). ;; ;; Phase 2 combinators (flow-control-src): ;; branch / fail / failed? / fail-reason / try-catch / retry / timeout / tick @@ -35,7 +38,7 @@ (define flow-combinators-src - "(define (flow-node f) f)\n (define (flow-id input) input)\n (define (flow-const v) (lambda (input) v))\n (define (flow-seq-step ns v)\n (if (null? ns) v (flow-seq-step (cdr ns) ((car ns) v))))\n (define sequence (lambda ns (lambda (input) (flow-seq-step ns input))))\n (define parallel (lambda ns (lambda (input) (map (lambda (n) (n input)) ns))))\n (define (map-flow node) (lambda (items) (map node items)))\n (define-syntax defflow\n (syntax-rules ()\n ((defflow nm body)\n (begin (define nm body) (flow-register! (quote nm) nm)))))") + "(define (flow-node f) f)\n (define (flow-id input) input)\n (define (flow-const v) (lambda (input) v))\n (define (flow-seq-step ns v)\n (if (null? ns) v (flow-seq-step (cdr ns) ((car ns) v))))\n (define sequence (lambda ns (lambda (input) (flow-seq-step ns input))))\n (define parallel (lambda ns (lambda (input) (map (lambda (n) (n input)) ns))))\n (define (map-flow node) (lambda (items) (map node items)))\n (define (flow-while-step pred body input n)\n (if (<= n 0)\n input\n (if (pred input) (flow-while-step pred body (body input) (- n 1)) input)))\n (define (flow-while pred body max) (lambda (input) (flow-while-step pred body input max)))\n (define (flow-until-step pred body input n)\n (if (<= n 0)\n input\n (if (pred input) input (flow-until-step pred body (body input) (- n 1)))))\n (define (flow-until pred body max) (lambda (input) (flow-until-step pred body input max)))\n (define-syntax defflow\n (syntax-rules ()\n ((defflow nm body)\n (begin (define nm body) (flow-register! (quote nm) nm)))))") (define flow-control-src diff --git a/lib/flow/tests/combinators.sx b/lib/flow/tests/combinators.sx index 467e010c..7931bfeb 100644 --- a/lib/flow/tests/combinators.sx +++ b/lib/flow/tests/combinators.sx @@ -1,4 +1,4 @@ -;; lib/flow/tests/combinators.sx — Phase 5: combinator library (tap, recover, map-flow). +;; lib/flow/tests/combinators.sx — Phase 5: combinator library (tap, recover, map-flow, iteration). (define flow-cmb-pass 0) (define flow-cmb-fail 0) @@ -74,4 +74,35 @@ "(flow/start (sequence (map-flow (lambda (x) (* x 10))) (lambda (xs) (apply + xs))) (list 1 2 3))") 60) +;; ── flow-while / flow-until (bounded iteration) ───────────────── +(flow-cmb-test + "flow-while: iterates while the predicate holds" + (flow-m + "(flow/start (flow-while (lambda (x) (< x 10)) (lambda (x) (+ x 1)) 100) 0)") + 10) +(flow-cmb-test + "flow-while: a false predicate leaves input unchanged" + (flow-m + "(flow/start (flow-while (lambda (x) (< x 0)) (lambda (x) (+ x 1)) 100) 5)") + 5) +(flow-cmb-test + "flow-while: respects the max-iteration bound" + (flow-m "(flow/start (flow-while (lambda (x) #t) (lambda (x) (+ x 1)) 3) 0)") + 3) +(flow-cmb-test + "flow-while: doubles until past a threshold" + (flow-m + "(flow/start (flow-while (lambda (x) (< x 50)) (lambda (x) (* x 2)) 100) 3)") + 96) +(flow-cmb-test + "flow-until: iterates until the predicate becomes true" + (flow-m + "(flow/start (flow-until (lambda (x) (>= x 10)) (lambda (x) (+ x 3)) 100) 0)") + 12) +(flow-cmb-test + "flow-until: composes inside a sequence" + (flow-m + "(flow/start (sequence (flow-until (lambda (x) (> x 100)) (lambda (x) (* x 3)) 100) (lambda (x) (- x 100))) 5)") + 35) + (define flow-cmb-tests-run! (fn () {:total (+ flow-cmb-pass flow-cmb-fail) :passed flow-cmb-pass :failed flow-cmb-fail :fails flow-cmb-fails})) diff --git a/plans/flow-on-sx.md b/plans/flow-on-sx.md index 0deaaac6..86b9a957 100644 --- a/plans/flow-on-sx.md +++ b/plans/flow-on-sx.md @@ -16,7 +16,7 @@ federation extension via fed-sx for remote-node execution. ## Status (rolling) -`bash lib/flow/conformance.sh` → **116/116** (Phases 1-5 complete) +`bash lib/flow/conformance.sh` → **122/122** (Phases 1-5 complete) ## Ground rules @@ -145,7 +145,9 @@ something operators and authors actually use. Accumulation, not a rewrite. - [x] `recover` — complement to try-catch for the fail-VALUE channel: run node; if it yields `(fail ...)`, run a recovery node on the reason - [x] `map-flow` — run a flow per item of a list, join results (sequential) -- [x] `lib/flow/tests/api.sx` (12) + `lib/flow/tests/combinators.sx` (11) +- [x] `flow-while` / `flow-until` — bounded iteration: re-run body threading the + value while/until pred holds, capped at `max` steps (deterministic bound) +- [x] `lib/flow/tests/api.sx` (12) + `lib/flow/tests/combinators.sx` (17) ## Progress log From d9b9da3843423140b771d52142a884b49a10259c Mon Sep 17 00:00:00 2001 From: giles Date: Sat, 6 Jun 2026 18:09:21 +0000 Subject: [PATCH 15/20] =?UTF-8?q?flow:=20railway=20attempt=20combinator=20?= =?UTF-8?q?=E2=80=94=20fail-value=20short-circuit=20+=2010=20tests=20(Phas?= =?UTF-8?q?e=206)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (attempt n1 n2 ...) threads like sequence but stops at the first node returning a (fail ...) value, returning that failure. Makes the fail/recover error model compose into validation/ETL pipelines (railway-oriented). 132/132 across 8 suites. Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/flow/conformance.sh | 1 + lib/flow/scoreboard.json | 9 ++--- lib/flow/scoreboard.md | 3 +- lib/flow/spec.sx | 4 ++- lib/flow/tests/railway.sx | 73 +++++++++++++++++++++++++++++++++++++++ plans/flow-on-sx.md | 12 ++++++- 6 files changed, 95 insertions(+), 7 deletions(-) create mode 100644 lib/flow/tests/railway.sx diff --git a/lib/flow/conformance.sh b/lib/flow/conformance.sh index 53542736..bcdc52dd 100755 --- a/lib/flow/conformance.sh +++ b/lib/flow/conformance.sh @@ -28,6 +28,7 @@ SUITES=( "distributed flow-dist-tests-run! lib/flow/tests/distributed.sx" "api flow-api-tests-run! lib/flow/tests/api.sx" "combinators flow-cmb-tests-run! lib/flow/tests/combinators.sx" + "railway flow-rail-tests-run! lib/flow/tests/railway.sx" ) TMPFILE=$(mktemp); trap "rm -f $TMPFILE" EXIT diff --git a/lib/flow/scoreboard.json b/lib/flow/scoreboard.json index 0c32983f..cc1f1082 100644 --- a/lib/flow/scoreboard.json +++ b/lib/flow/scoreboard.json @@ -1,6 +1,6 @@ { - "total": 122, - "passed": 122, + "total": 132, + "passed": 132, "failed": 0, "suites": { "basic": { "passed": 18, "total": 18 }, @@ -9,7 +9,8 @@ "recovery": { "passed": 8, "total": 8 }, "distributed": { "passed": 19, "total": 19 }, "api": { "passed": 12, "total": 12 }, - "combinators": { "passed": 17, "total": 17 } + "combinators": { "passed": 17, "total": 17 }, + "railway": { "passed": 10, "total": 10 } }, - "phases": { "phase1": "done", "phase2": "done", "phase3": "done", "phase4": "done", "phase5": "done" } + "phases": { "phase1": "done", "phase2": "done", "phase3": "done", "phase4": "done", "phase5": "done", "phase6": "done" } } diff --git a/lib/flow/scoreboard.md b/lib/flow/scoreboard.md index 988adb5f..90e9f6d8 100644 --- a/lib/flow/scoreboard.md +++ b/lib/flow/scoreboard.md @@ -1,6 +1,6 @@ # flow-on-sx Scoreboard -**All tests pass: 122 / 122 across 7 suites. Phases 1-5 complete.** +**All tests pass: 132 / 132 across 8 suites. Phases 1-6 complete.** `bash lib/flow/conformance.sh` @@ -15,6 +15,7 @@ | distributed | 19 | Phase 4: `remote-node` (7); `remote-failover` (6); replication + handoff across instances (6) | | api | 12 | Phase 5: introspection — `flow/status`, `flow/result`, `flow/list`, `flow/pending` | | combinators | 17 | Phase 5: `tap`, `recover` (fail-value), `map-flow` fan-over-list, `flow-while`/`flow-until` bounded iteration | +| railway | 10 | Phase 6: `attempt` — fail-value short-circuiting sequence + recover rejoin | ## Architecture diff --git a/lib/flow/spec.sx b/lib/flow/spec.sx index 933c571c..2bfd5c56 100644 --- a/lib/flow/spec.sx +++ b/lib/flow/spec.sx @@ -19,6 +19,8 @@ ;; branch / fail / failed? / fail-reason / try-catch / retry / timeout / tick ;; tap (Phase 5): side-effecting pass-through (returns input unchanged). ;; recover (Phase 5): the fail-VALUE counterpart of try-catch. +;; attempt (Phase 6): railway sequence — thread nodes left-to-right but stop at +;; the first node that returns a (fail ...) value, returning that failure. ;; ;; Phase 3 suspend core (flow-suspend-src): ;; The guest Scheme's call/cc is ESCAPE-ONLY (re-invoking a captured k after it @@ -42,7 +44,7 @@ (define flow-control-src - "(define (branch pred then else)\n (lambda (input) (if (pred input) (then input) (else input))))\n (define (fail reason) (list (quote flow-fail) reason))\n (define (failed? x) (and (pair? x) (eq? (car x) (quote flow-fail))))\n (define (fail-reason x) (car (cdr x)))\n (define (recover node handler)\n (lambda (input)\n (let ((r (node input)))\n (if (failed? r) (handler (fail-reason r)) r))))\n (define (tap effect)\n (lambda (input) (begin (effect input) input)))\n (define (try-catch node handler)\n (lambda (input) (guard (e (#t (handler e))) (node input))))\n (define (flow-retry-step n node input)\n (guard (e (#t (if (<= n 1) (raise e) (flow-retry-step (- n 1) node input))))\n (node input)))\n (define (retry n node) (lambda (input) (flow-retry-step n node input)))\n (define flow-timeout-budget -1)\n (define (tick)\n (if (< flow-timeout-budget 0)\n 0\n (begin\n (set! flow-timeout-budget (- flow-timeout-budget 1))\n (if (< flow-timeout-budget 0)\n (raise (quote flow-timeout))\n flow-timeout-budget))))\n (define (timeout budget node)\n (lambda (input)\n (let ((saved flow-timeout-budget))\n (set! flow-timeout-budget budget)\n (guard (e (#t (begin (set! flow-timeout-budget saved) (raise e))))\n (let ((result (node input)))\n (set! flow-timeout-budget saved)\n result)))))") + "(define (branch pred then else)\n (lambda (input) (if (pred input) (then input) (else input))))\n (define (fail reason) (list (quote flow-fail) reason))\n (define (failed? x) (and (pair? x) (eq? (car x) (quote flow-fail))))\n (define (fail-reason x) (car (cdr x)))\n (define (recover node handler)\n (lambda (input)\n (let ((r (node input)))\n (if (failed? r) (handler (fail-reason r)) r))))\n (define (tap effect)\n (lambda (input) (begin (effect input) input)))\n (define (flow-attempt-step ns v)\n (if (failed? v)\n v\n (if (null? ns) v (flow-attempt-step (cdr ns) ((car ns) v)))))\n (define attempt (lambda ns (lambda (input) (flow-attempt-step ns input))))\n (define (try-catch node handler)\n (lambda (input) (guard (e (#t (handler e))) (node input))))\n (define (flow-retry-step n node input)\n (guard (e (#t (if (<= n 1) (raise e) (flow-retry-step (- n 1) node input))))\n (node input)))\n (define (retry n node) (lambda (input) (flow-retry-step n node input)))\n (define flow-timeout-budget -1)\n (define (tick)\n (if (< flow-timeout-budget 0)\n 0\n (begin\n (set! flow-timeout-budget (- flow-timeout-budget 1))\n (if (< flow-timeout-budget 0)\n (raise (quote flow-timeout))\n flow-timeout-budget))))\n (define (timeout budget node)\n (lambda (input)\n (let ((saved flow-timeout-budget))\n (set! flow-timeout-budget budget)\n (guard (e (#t (begin (set! flow-timeout-budget saved) (raise e))))\n (let ((result (node input)))\n (set! flow-timeout-budget saved)\n result)))))") (define flow-suspend-src diff --git a/lib/flow/tests/railway.sx b/lib/flow/tests/railway.sx new file mode 100644 index 00000000..8b90e918 --- /dev/null +++ b/lib/flow/tests/railway.sx @@ -0,0 +1,73 @@ +;; lib/flow/tests/railway.sx — Phase 6: railway-oriented composition (attempt). + +(define flow-rail-pass 0) +(define flow-rail-fail 0) +(define flow-rail-fails (list)) + +(define + flow-rail-test + (fn + (name actual expected) + (if + (= actual expected) + (set! flow-rail-pass (+ flow-rail-pass 1)) + (begin + (set! flow-rail-fail (+ flow-rail-fail 1)) + (append! flow-rail-fails {:name name :expected expected :actual actual}))))) + +(define flow-r (fn (src) (flow-run src))) + +;; ── attempt — short-circuit on the first (fail ...) ───────────── +(flow-rail-test + "attempt: threads like sequence when nothing fails" + (flow-r + "(flow/start (attempt (lambda (x) (+ x 1)) (lambda (x) (* x 10))) 4)") + 50) +(flow-rail-test + "attempt: empty is identity" + (flow-r "(flow/start (attempt) 7)") + 7) +(flow-rail-test + "attempt: returns the first failure" + (flow-r + "(failed? (flow/start (attempt (lambda (x) (fail (quote bad))) (lambda (x) (* x 10))) 4))") + true) +(flow-rail-test + "attempt: the failure carries its reason" + (flow-r + "(fail-reason (flow/start (attempt (lambda (x) x) (lambda (x) (fail (quote rejected)))) 4))") + "rejected") +(flow-rail-test + "attempt: nodes after a failure do not run" + (flow-r + "(define ran 0) (flow/start (attempt (lambda (x) (fail (quote stop))) (lambda (x) (begin (set! ran (+ ran 1)) x))) 0) ran") + 0) +(flow-rail-test + "attempt: a failed input short-circuits immediately" + (flow-r + "(define ran 0) (fail-reason (flow/start (attempt (lambda (x) (begin (set! ran (+ ran 1)) x))) (fail (quote pre))))") + "pre") +(flow-rail-test + "attempt: middle failure halts the chain" + (flow-r + "(define ran 0) (flow/start (attempt (lambda (x) (+ x 1)) (lambda (x) (fail (quote mid))) (lambda (x) (begin (set! ran (+ ran 1)) x))) 5) ran") + 0) + +;; ── attempt + recover (rejoin the happy track) ────────────────── +(flow-rail-test + "attempt + recover: recover turns a failure into a value" + (flow-r + "(flow/start (recover (attempt (lambda (x) (if (> x 0) x (fail (quote non-positive)))) (lambda (x) (* x 2))) (flow-const 0)) -5)") + 0) +(flow-rail-test + "attempt + recover: happy path passes recover through" + (flow-r + "(flow/start (recover (attempt (lambda (x) (if (> x 0) x (fail (quote non-positive)))) (lambda (x) (* x 2))) (flow-const 0)) 5)") + 10) +(flow-rail-test + "attempt: validation pipeline reports the failing stage" + (flow-r + "(defflow validate (attempt (lambda (s) (if (>= (string-length s) 3) s (fail (quote too-short)))) (lambda (s) (if (<= (string-length s) 8) s (fail (quote too-long)))) (lambda (s) (list (quote ok) (string-length s))))) (list (fail-reason (flow/start validate \"hi\")) (flow/start validate \"hello\"))") + (list "too-short" (list "ok" 5))) + +(define flow-rail-tests-run! (fn () {:total (+ flow-rail-pass flow-rail-fail) :passed flow-rail-pass :failed flow-rail-fail :fails flow-rail-fails})) diff --git a/plans/flow-on-sx.md b/plans/flow-on-sx.md index 86b9a957..278937ad 100644 --- a/plans/flow-on-sx.md +++ b/plans/flow-on-sx.md @@ -16,7 +16,7 @@ federation extension via fed-sx for remote-node execution. ## Status (rolling) -`bash lib/flow/conformance.sh` → **122/122** (Phases 1-5 complete) +`bash lib/flow/conformance.sh` → **132/132** (Phases 1-6 complete) ## Ground rules @@ -149,6 +149,16 @@ something operators and authors actually use. Accumulation, not a rewrite. value while/until pred holds, capped at `max` steps (deterministic bound) - [x] `lib/flow/tests/api.sx` (12) + `lib/flow/tests/combinators.sx` (17) +## Phase 6 — Railway-oriented composition + +Make the `(fail reason)` value channel compose into real validation/ETL pipelines. + +- [x] `attempt` — like `sequence`, but short-circuits at the first node that returns + a `(fail ...)` value, returning that failure (the railway track). Pairs with + `recover` for the rejoin. +- [x] `lib/flow/tests/railway.sx` — 10 cases: fail short-circuiting, no-run-after- + failure, recover rejoin, validation pipeline reporting the failing stage + ## Progress log - **Phase 1 (combinators + sequential runtime).** Flow built as a Scheme prelude From 2b47b2925c0c5e9b2616c976927917eb70332e7b Mon Sep 17 00:00:00 2001 From: giles Date: Sat, 6 Jun 2026 18:17:40 +0000 Subject: [PATCH 16/20] flow: end-to-end integration suite + 10 tests (Phase 7) Realistic flows composing every phase: an order pipeline (validate via attempt -> payment suspend -> branch -> ledger federation via remote-node) and an onboarding flow, each run through the full lifecycle including a simulated crash (export/wipe/ import) and a peer handoff mid-flow, with flow/pending|status|result introspection. 142/142 across 9 suites. Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/flow/conformance.sh | 1 + lib/flow/scoreboard.json | 9 +-- lib/flow/scoreboard.md | 3 +- lib/flow/tests/integration.sx | 115 ++++++++++++++++++++++++++++++++++ plans/flow-on-sx.md | 12 +++- 5 files changed, 134 insertions(+), 6 deletions(-) create mode 100644 lib/flow/tests/integration.sx diff --git a/lib/flow/conformance.sh b/lib/flow/conformance.sh index bcdc52dd..5d0e7b6b 100755 --- a/lib/flow/conformance.sh +++ b/lib/flow/conformance.sh @@ -29,6 +29,7 @@ SUITES=( "api flow-api-tests-run! lib/flow/tests/api.sx" "combinators flow-cmb-tests-run! lib/flow/tests/combinators.sx" "railway flow-rail-tests-run! lib/flow/tests/railway.sx" + "integration flow-int-tests-run! lib/flow/tests/integration.sx" ) TMPFILE=$(mktemp); trap "rm -f $TMPFILE" EXIT diff --git a/lib/flow/scoreboard.json b/lib/flow/scoreboard.json index cc1f1082..b8cff6b9 100644 --- a/lib/flow/scoreboard.json +++ b/lib/flow/scoreboard.json @@ -1,6 +1,6 @@ { - "total": 132, - "passed": 132, + "total": 142, + "passed": 142, "failed": 0, "suites": { "basic": { "passed": 18, "total": 18 }, @@ -10,7 +10,8 @@ "distributed": { "passed": 19, "total": 19 }, "api": { "passed": 12, "total": 12 }, "combinators": { "passed": 17, "total": 17 }, - "railway": { "passed": 10, "total": 10 } + "railway": { "passed": 10, "total": 10 }, + "integration": { "passed": 10, "total": 10 } }, - "phases": { "phase1": "done", "phase2": "done", "phase3": "done", "phase4": "done", "phase5": "done", "phase6": "done" } + "phases": { "phase1": "done", "phase2": "done", "phase3": "done", "phase4": "done", "phase5": "done", "phase6": "done", "phase7": "done" } } diff --git a/lib/flow/scoreboard.md b/lib/flow/scoreboard.md index 90e9f6d8..0f539a36 100644 --- a/lib/flow/scoreboard.md +++ b/lib/flow/scoreboard.md @@ -1,6 +1,6 @@ # flow-on-sx Scoreboard -**All tests pass: 132 / 132 across 8 suites. Phases 1-6 complete.** +**All tests pass: 142 / 142 across 9 suites. Phases 1-7 complete.** `bash lib/flow/conformance.sh` @@ -16,6 +16,7 @@ | api | 12 | Phase 5: introspection — `flow/status`, `flow/result`, `flow/list`, `flow/pending` | | combinators | 17 | Phase 5: `tap`, `recover` (fail-value), `map-flow` fan-over-list, `flow-while`/`flow-until` bounded iteration | | railway | 10 | Phase 6: `attempt` — fail-value short-circuiting sequence + recover rejoin | +| integration | 10 | Phase 7: end-to-end order + onboarding flows composing every phase (suspend, branch, federation, crash recovery, handoff, introspection) | ## Architecture diff --git a/lib/flow/tests/integration.sx b/lib/flow/tests/integration.sx new file mode 100644 index 00000000..46352682 --- /dev/null +++ b/lib/flow/tests/integration.sx @@ -0,0 +1,115 @@ +;; lib/flow/tests/integration.sx — Phase 7: end-to-end flows composing every phase. + +(define flow-int-pass 0) +(define flow-int-fail 0) +(define flow-int-fails (list)) + +(define + flow-int-test + (fn + (name actual expected) + (if + (= actual expected) + (set! flow-int-pass (+ flow-int-pass 1)) + (begin + (set! flow-int-fail (+ flow-int-fail 1)) + (append! flow-int-fails {:name name :expected expected :actual actual}))))) + +(define flow-i (fn (src) (flow-run src))) + +;; The order-processing flow, defined once per program via this prelude string: +;; validate amount (attempt: fail if <= 0) +;; -> suspend for payment confirmation (resume value = confirmed amount) +;; -> branch: confirmed>0 ? record on the ledger peer : declined failure +(define + order-prelude + "(flow-peer-register! (quote ledger) (list (list (quote record) (lambda (amt) (list (quote recorded) amt)))))\n (defflow order\n (attempt\n (lambda (amt) (if (> amt 0) amt (fail (quote invalid-amount))))\n (lambda (amt) (suspend (quote await-payment)))\n (branch (lambda (amt) (> amt 0))\n (remote-node (quote ledger) (quote record))\n (flow-const (fail (quote declined))))))") + +;; ── happy path through every phase ────────────────────────────── +(flow-int-test + "order: validate -> suspend -> resume -> branch -> federate" + (flow-i + (str + order-prelude + "(define id (car (cdr (flow/start order 100)))) (flow/resume id 250)")) + (list "recorded" 250)) +(flow-int-test + "order: starting suspends awaiting payment" + (flow-i + (str + order-prelude + "(define s (flow/start order 100)) (list (car s) (car (cdr (cdr s))))")) + (list "flow-suspended" "await-payment")) +(flow-int-test + "order: invalid amount fails up front and never suspends" + (flow-i + (str + order-prelude + "(define r (flow/start order -5)) (list (failed? r) (fail-reason r))")) + (list true "invalid-amount")) +(flow-int-test + "order: a declined payment yields a failure value" + (flow-i + (str + order-prelude + "(define id (car (cdr (flow/start order 100)))) (failed? (flow/resume id 0))")) + true) + +;; ── crash recovery mid-flow ───────────────────────────────────── +(flow-int-test + "order: survives a simulated crash between suspend and resume" + (flow-i + (str + order-prelude + "(define id (car (cdr (flow/start order 100)))) (define saved (flow-store-export)) (set! flow-store (list)) (flow-store-import! saved) (flow/resume id 250)")) + (list "recorded" 250)) + +;; ── handoff to a peer mid-flow ────────────────────────────────── +(flow-int-test + "order: hands off to a peer that resumes and completes" + (flow-i + (str + order-prelude + "(define id (car (cdr (flow/start order 100)))) (flow-replicate-to (quote nodeB)) (set! flow-store (list)) (flow-restore-from (quote nodeB)) (flow/resume id 250)")) + (list "recorded" 250)) + +;; ── introspection during the flow's life ──────────────────────── +(flow-int-test + "order: pending shows what the flow awaits, then result after resume" + (flow-i + (str + order-prelude + "(define id (car (cdr (flow/start order 100)))) (define p (flow/pending)) (flow/resume id 250) (list p (flow/status id) (flow/result id))")) + (list + (list (list 1 "await-payment")) + "done" + (list "recorded" 250))) + +;; ── onboarding: two human steps + cancellation ────────────────── +(define + onboard-prelude + "(defflow onboard\n (sequence\n (lambda (user) (+ user 1))\n (lambda (x) (suspend (quote confirm-email)))\n (lambda (x) (suspend (quote complete-profile)))\n (lambda (x) (list (quote onboarded) x))))") + +(flow-int-test + "onboard: two suspends resume in order to completion" + (flow-i + (str + onboard-prelude + "(define id (car (cdr (flow/start onboard 0)))) (flow/resume id 7) (flow/resume id 9)")) + (list "onboarded" 9)) +(flow-int-test + "onboard: the second pending tag appears after the first resume" + (flow-i + (str + onboard-prelude + "(define id (car (cdr (flow/start onboard 0)))) (flow/resume id 7) (car (cdr (car (flow/pending))))")) + "complete-profile") +(flow-int-test + "onboard: cancelling abandons the flow" + (flow-i + (str + onboard-prelude + "(define id (car (cdr (flow/start onboard 0)))) (flow/cancel id) (list (flow/status id) (car (flow/resume id 7)))")) + (list "cancelled" "flow-error")) + +(define flow-int-tests-run! (fn () {:total (+ flow-int-pass flow-int-fail) :passed flow-int-pass :failed flow-int-fail :fails flow-int-fails})) diff --git a/plans/flow-on-sx.md b/plans/flow-on-sx.md index 278937ad..95a8a57b 100644 --- a/plans/flow-on-sx.md +++ b/plans/flow-on-sx.md @@ -16,7 +16,7 @@ federation extension via fed-sx for remote-node execution. ## Status (rolling) -`bash lib/flow/conformance.sh` → **132/132** (Phases 1-6 complete) +`bash lib/flow/conformance.sh` → **142/142** (Phases 1-7 complete) ## Ground rules @@ -159,6 +159,16 @@ Make the `(fail reason)` value channel compose into real validation/ETL pipeline - [x] `lib/flow/tests/railway.sx` — 10 cases: fail short-circuiting, no-run-after- failure, recover rejoin, validation pipeline reporting the failing stage +## Phase 7 — End-to-end integration + +Prove the phases compose: realistic flows exercising attempt + suspend + branch + +remote-node + crash-recovery + handoff + introspection together. + +- [x] `lib/flow/tests/integration.sx` — 10 cases: an order-processing flow (validate + → payment suspend → branch → ledger federation) and an onboarding flow, run through + the full lifecycle including a simulated crash and a peer handoff mid-flow, plus + introspection (`flow/pending`/`status`/`result`) during the flow's life + ## Progress log - **Phase 1 (combinators + sequential runtime).** Flow built as a Scheme prelude From aabb95025605d30a9f8bfe03b290f0865ac75054 Mon Sep 17 00:00:00 2001 From: giles Date: Sat, 6 Jun 2026 18:34:53 +0000 Subject: [PATCH 17/20] flow: store hygiene flow/gc + flow/forget + 9 tests flow/gc drops terminal (done/cancelled) records, keeps live suspended flows, returns count removed; flow/forget id drops one terminal record and refuses live flows. Bounds unbounded store growth (retention/GC). Bumped conformance sx_server timeout to 540s for the 10-suite run under CPU contention. 151/151 across 10 suites. Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/flow/conformance.sh | 3 +- lib/flow/scoreboard.json | 7 ++-- lib/flow/scoreboard.md | 3 +- lib/flow/store.sx | 7 +++- lib/flow/tests/hygiene.sx | 67 +++++++++++++++++++++++++++++++++++++++ plans/flow-on-sx.md | 5 ++- 6 files changed, 85 insertions(+), 7 deletions(-) create mode 100644 lib/flow/tests/hygiene.sx diff --git a/lib/flow/conformance.sh b/lib/flow/conformance.sh index 5d0e7b6b..787c9c9e 100755 --- a/lib/flow/conformance.sh +++ b/lib/flow/conformance.sh @@ -30,6 +30,7 @@ SUITES=( "combinators flow-cmb-tests-run! lib/flow/tests/combinators.sx" "railway flow-rail-tests-run! lib/flow/tests/railway.sx" "integration flow-int-tests-run! lib/flow/tests/integration.sx" + "hygiene flow-hyg-tests-run! lib/flow/tests/hygiene.sx" ) TMPFILE=$(mktemp); trap "rm -f $TMPFILE" EXIT @@ -56,7 +57,7 @@ emit_eval () { echo "(epoch $EPOCH)"; echo "(eval \"$1\")"; EPOCH=$((EPOCH+1)); done } > "$TMPFILE" -OUTPUT=$(timeout 300 "$SX_SERVER" < "$TMPFILE" 2>&1 || true) +OUTPUT=$(timeout 540 "$SX_SERVER" < "$TMPFILE" 2>&1 || true) TOTAL_PASS=0 TOTAL_FAIL=0 diff --git a/lib/flow/scoreboard.json b/lib/flow/scoreboard.json index b8cff6b9..012c8bfb 100644 --- a/lib/flow/scoreboard.json +++ b/lib/flow/scoreboard.json @@ -1,6 +1,6 @@ { - "total": 142, - "passed": 142, + "total": 151, + "passed": 151, "failed": 0, "suites": { "basic": { "passed": 18, "total": 18 }, @@ -11,7 +11,8 @@ "api": { "passed": 12, "total": 12 }, "combinators": { "passed": 17, "total": 17 }, "railway": { "passed": 10, "total": 10 }, - "integration": { "passed": 10, "total": 10 } + "integration": { "passed": 10, "total": 10 }, + "hygiene": { "passed": 9, "total": 9 } }, "phases": { "phase1": "done", "phase2": "done", "phase3": "done", "phase4": "done", "phase5": "done", "phase6": "done", "phase7": "done" } } diff --git a/lib/flow/scoreboard.md b/lib/flow/scoreboard.md index 0f539a36..41eaf45d 100644 --- a/lib/flow/scoreboard.md +++ b/lib/flow/scoreboard.md @@ -1,6 +1,6 @@ # flow-on-sx Scoreboard -**All tests pass: 142 / 142 across 9 suites. Phases 1-7 complete.** +**All tests pass: 151 / 151 across 10 suites. Phases 1-7 complete.** `bash lib/flow/conformance.sh` @@ -17,6 +17,7 @@ | combinators | 17 | Phase 5: `tap`, `recover` (fail-value), `map-flow` fan-over-list, `flow-while`/`flow-until` bounded iteration | | railway | 10 | Phase 6: `attempt` — fail-value short-circuiting sequence + recover rejoin | | integration | 10 | Phase 7: end-to-end order + onboarding flows composing every phase (suspend, branch, federation, crash recovery, handoff, introspection) | +| hygiene | 9 | Phase 5: `flow/gc` (prune terminal flows), `flow/forget` (drop one terminal record) | ## Architecture diff --git a/lib/flow/store.sx b/lib/flow/store.sx index b88e6c5b..abd75360 100644 --- a/lib/flow/store.sx +++ b/lib/flow/store.sx @@ -28,10 +28,15 @@ ;; (flow/result id) — result if done, else (flow-error reason) ;; (flow/list) — list of (id status) for every flow ;; (flow/pending) — list of (id waiting-tag) for suspended flows +;; Store hygiene (Phase 5): +;; (flow/gc) — drop all terminal (done/cancelled) records, keeping +;; suspended (live) flows; returns the count removed +;; (flow/forget id) — drop one TERMINAL record (#t); refuses to forget a +;; still-suspended flow or an unknown id (#f) (define flow-store-src - "(define flow-registry (list))\n (define (flow-register! name proc) (set! flow-registry (cons (list name proc) flow-registry)))\n (define (flow-lookup-in name reg)\n (if (null? reg)\n #f\n (if (eq? (car (car reg)) name) (car (cdr (car reg))) (flow-lookup-in name (cdr reg)))))\n (define (flow-lookup name) (flow-lookup-in name flow-registry))\n (define (flow-name-of proc reg)\n (if (null? reg)\n #f\n (if (eq? (car (cdr (car reg))) proc) (car (car reg)) (flow-name-of proc (cdr reg)))))\n\n (define flow-store (list))\n (define flow-next-id 0)\n (define (flow-store-remove id store)\n (if (null? store)\n (list)\n (if (= (car (car store)) id)\n (flow-store-remove id (cdr store))\n (cons (car store) (flow-store-remove id (cdr store))))))\n (define (flow-store-put! id rec) (set! flow-store (cons (list id rec) (flow-store-remove id flow-store))))\n (define (flow-store-find id store)\n (if (null? store)\n (list)\n (if (= (car (car store)) id)\n (car (cdr (car store)))\n (flow-store-find id (cdr store)))))\n (define (flow-store-get id) (flow-store-find id flow-store))\n\n (define (flow-mk-rec name proc input log status payload)\n (list name proc input log status payload))\n (define (flow-rec-name r) (car r))\n (define (flow-rec-proc r) (car (cdr r)))\n (define (flow-rec-input r) (car (cdr (cdr r))))\n (define (flow-rec-log r) (car (cdr (cdr (cdr r)))))\n (define (flow-rec-status r) (car (cdr (cdr (cdr (cdr r))))))\n (define (flow-rec-payload r) (car (cdr (cdr (cdr (cdr (cdr r)))))))\n (define (flow-rec-resolve rec)\n (let ((byname (flow-lookup (flow-rec-name rec))))\n (if byname byname (flow-rec-proc rec))))\n\n (define (flow-outcome id name proc input log outcome)\n (if (eq? (car outcome) (quote flow-done))\n (begin\n (flow-store-put! id (flow-mk-rec name proc input log (quote done) (car (cdr outcome))))\n (car (cdr outcome)))\n (begin\n (flow-store-put! id (flow-mk-rec name proc input log (quote suspended) (car (cdr outcome))))\n (list (quote flow-suspended) id (car (cdr outcome))))))\n (define (flow/start flow input)\n (set! flow-next-id (+ flow-next-id 1))\n (flow-outcome flow-next-id (flow-name-of flow flow-registry) flow input (list)\n (flow-drive flow input (list))))\n (define (flow/resume id value)\n (let ((rec (flow-store-get id)))\n (if (null? rec)\n (list (quote flow-error) (quote no-such-flow))\n (if (eq? (flow-rec-status rec) (quote suspended))\n (let ((proc (flow-rec-resolve rec)))\n (let ((newlog (cons (list (flow-rec-payload rec) value) (flow-rec-log rec))))\n (flow-outcome id (flow-rec-name rec) proc (flow-rec-input rec) newlog\n (flow-drive proc (flow-rec-input rec) newlog))))\n (list (quote flow-error) (quote not-suspended))))))\n (define (flow/cancel id)\n (let ((rec (flow-store-get id)))\n (if (null? rec)\n (list (quote flow-error) (quote no-such-flow))\n (begin\n (flow-store-put! id\n (flow-mk-rec (flow-rec-name rec) (flow-rec-proc rec) (flow-rec-input rec)\n (flow-rec-log rec) (quote cancelled) #f))\n (list (quote flow-cancelled) id)))))\n\n (define (flow-export-entry entry)\n (let ((rec (car (cdr entry))))\n (list (car entry)\n (flow-mk-rec (flow-rec-name rec) #f (flow-rec-input rec)\n (flow-rec-log rec) (flow-rec-status rec) (flow-rec-payload rec)))))\n (define (flow-export-map store)\n (if (null? store) (list) (cons (flow-export-entry (car store)) (flow-export-map (cdr store)))))\n (define (flow-store-export) (flow-export-map flow-store))\n (define (flow-max-id store m)\n (if (null? store) m (flow-max-id (cdr store) (if (> (car (car store)) m) (car (car store)) m))))\n (define (flow-store-import! data)\n (begin (set! flow-store data) (set! flow-next-id (flow-max-id data 0))))\n (define (flow-collect-resumable store)\n (if (null? store)\n (list)\n (if (eq? (flow-rec-status (car (cdr (car store)))) (quote suspended))\n (cons (car (car store)) (flow-collect-resumable (cdr store)))\n (flow-collect-resumable (cdr store)))))\n (define (flow-resumable-ids) (flow-collect-resumable flow-store))\n\n (define (flow/status id)\n (let ((rec (flow-store-get id)))\n (if (null? rec) (quote unknown) (flow-rec-status rec))))\n (define (flow/result id)\n (let ((rec (flow-store-get id)))\n (if (null? rec)\n (list (quote flow-error) (quote no-such-flow))\n (if (eq? (flow-rec-status rec) (quote done))\n (flow-rec-payload rec)\n (list (quote flow-error) (quote not-done))))))\n (define (flow-list-step store)\n (if (null? store)\n (list)\n (cons (list (car (car store)) (flow-rec-status (car (cdr (car store)))))\n (flow-list-step (cdr store)))))\n (define (flow/list) (flow-list-step flow-store))\n (define (flow-pending-step store)\n (if (null? store)\n (list)\n (if (eq? (flow-rec-status (car (cdr (car store)))) (quote suspended))\n (cons (list (car (car store)) (flow-rec-payload (car (cdr (car store)))))\n (flow-pending-step (cdr store)))\n (flow-pending-step (cdr store)))))\n (define (flow/pending) (flow-pending-step flow-store))") + "(define flow-registry (list))\n (define (flow-register! name proc) (set! flow-registry (cons (list name proc) flow-registry)))\n (define (flow-lookup-in name reg)\n (if (null? reg)\n #f\n (if (eq? (car (car reg)) name) (car (cdr (car reg))) (flow-lookup-in name (cdr reg)))))\n (define (flow-lookup name) (flow-lookup-in name flow-registry))\n (define (flow-name-of proc reg)\n (if (null? reg)\n #f\n (if (eq? (car (cdr (car reg))) proc) (car (car reg)) (flow-name-of proc (cdr reg)))))\n\n (define flow-store (list))\n (define flow-next-id 0)\n (define (flow-store-remove id store)\n (if (null? store)\n (list)\n (if (= (car (car store)) id)\n (flow-store-remove id (cdr store))\n (cons (car store) (flow-store-remove id (cdr store))))))\n (define (flow-store-put! id rec) (set! flow-store (cons (list id rec) (flow-store-remove id flow-store))))\n (define (flow-store-find id store)\n (if (null? store)\n (list)\n (if (= (car (car store)) id)\n (car (cdr (car store)))\n (flow-store-find id (cdr store)))))\n (define (flow-store-get id) (flow-store-find id flow-store))\n\n (define (flow-mk-rec name proc input log status payload)\n (list name proc input log status payload))\n (define (flow-rec-name r) (car r))\n (define (flow-rec-proc r) (car (cdr r)))\n (define (flow-rec-input r) (car (cdr (cdr r))))\n (define (flow-rec-log r) (car (cdr (cdr (cdr r)))))\n (define (flow-rec-status r) (car (cdr (cdr (cdr (cdr r))))))\n (define (flow-rec-payload r) (car (cdr (cdr (cdr (cdr (cdr r)))))))\n (define (flow-rec-resolve rec)\n (let ((byname (flow-lookup (flow-rec-name rec))))\n (if byname byname (flow-rec-proc rec))))\n\n (define (flow-outcome id name proc input log outcome)\n (if (eq? (car outcome) (quote flow-done))\n (begin\n (flow-store-put! id (flow-mk-rec name proc input log (quote done) (car (cdr outcome))))\n (car (cdr outcome)))\n (begin\n (flow-store-put! id (flow-mk-rec name proc input log (quote suspended) (car (cdr outcome))))\n (list (quote flow-suspended) id (car (cdr outcome))))))\n (define (flow/start flow input)\n (set! flow-next-id (+ flow-next-id 1))\n (flow-outcome flow-next-id (flow-name-of flow flow-registry) flow input (list)\n (flow-drive flow input (list))))\n (define (flow/resume id value)\n (let ((rec (flow-store-get id)))\n (if (null? rec)\n (list (quote flow-error) (quote no-such-flow))\n (if (eq? (flow-rec-status rec) (quote suspended))\n (let ((proc (flow-rec-resolve rec)))\n (let ((newlog (cons (list (flow-rec-payload rec) value) (flow-rec-log rec))))\n (flow-outcome id (flow-rec-name rec) proc (flow-rec-input rec) newlog\n (flow-drive proc (flow-rec-input rec) newlog))))\n (list (quote flow-error) (quote not-suspended))))))\n (define (flow/cancel id)\n (let ((rec (flow-store-get id)))\n (if (null? rec)\n (list (quote flow-error) (quote no-such-flow))\n (begin\n (flow-store-put! id\n (flow-mk-rec (flow-rec-name rec) (flow-rec-proc rec) (flow-rec-input rec)\n (flow-rec-log rec) (quote cancelled) #f))\n (list (quote flow-cancelled) id)))))\n\n (define (flow-export-entry entry)\n (let ((rec (car (cdr entry))))\n (list (car entry)\n (flow-mk-rec (flow-rec-name rec) #f (flow-rec-input rec)\n (flow-rec-log rec) (flow-rec-status rec) (flow-rec-payload rec)))))\n (define (flow-export-map store)\n (if (null? store) (list) (cons (flow-export-entry (car store)) (flow-export-map (cdr store)))))\n (define (flow-store-export) (flow-export-map flow-store))\n (define (flow-max-id store m)\n (if (null? store) m (flow-max-id (cdr store) (if (> (car (car store)) m) (car (car store)) m))))\n (define (flow-store-import! data)\n (begin (set! flow-store data) (set! flow-next-id (flow-max-id data 0))))\n (define (flow-collect-resumable store)\n (if (null? store)\n (list)\n (if (eq? (flow-rec-status (car (cdr (car store)))) (quote suspended))\n (cons (car (car store)) (flow-collect-resumable (cdr store)))\n (flow-collect-resumable (cdr store)))))\n (define (flow-resumable-ids) (flow-collect-resumable flow-store))\n\n (define (flow/status id)\n (let ((rec (flow-store-get id)))\n (if (null? rec) (quote unknown) (flow-rec-status rec))))\n (define (flow/result id)\n (let ((rec (flow-store-get id)))\n (if (null? rec)\n (list (quote flow-error) (quote no-such-flow))\n (if (eq? (flow-rec-status rec) (quote done))\n (flow-rec-payload rec)\n (list (quote flow-error) (quote not-done))))))\n (define (flow-list-step store)\n (if (null? store)\n (list)\n (cons (list (car (car store)) (flow-rec-status (car (cdr (car store)))))\n (flow-list-step (cdr store)))))\n (define (flow/list) (flow-list-step flow-store))\n (define (flow-pending-step store)\n (if (null? store)\n (list)\n (if (eq? (flow-rec-status (car (cdr (car store)))) (quote suspended))\n (cons (list (car (car store)) (flow-rec-payload (car (cdr (car store)))))\n (flow-pending-step (cdr store)))\n (flow-pending-step (cdr store)))))\n (define (flow/pending) (flow-pending-step flow-store))\n\n (define (flow-store-count store) (if (null? store) 0 (+ 1 (flow-store-count (cdr store)))))\n (define (flow-gc-keep store)\n (if (null? store)\n (list)\n (if (eq? (flow-rec-status (car (cdr (car store)))) (quote suspended))\n (cons (car store) (flow-gc-keep (cdr store)))\n (flow-gc-keep (cdr store)))))\n (define (flow/gc)\n (let ((before (flow-store-count flow-store)))\n (set! flow-store (flow-gc-keep flow-store))\n (- before (flow-store-count flow-store))))\n (define (flow/forget id)\n (let ((rec (flow-store-get id)))\n (if (null? rec)\n #f\n (if (eq? (flow-rec-status rec) (quote suspended))\n #f\n (begin (set! flow-store (flow-store-remove id flow-store)) #t)))))") (define flow-load-store! diff --git a/lib/flow/tests/hygiene.sx b/lib/flow/tests/hygiene.sx new file mode 100644 index 00000000..a53122f8 --- /dev/null +++ b/lib/flow/tests/hygiene.sx @@ -0,0 +1,67 @@ +;; lib/flow/tests/hygiene.sx — Phase 5: store hygiene (flow/gc, flow/forget). + +(define flow-hyg-pass 0) +(define flow-hyg-fail 0) +(define flow-hyg-fails (list)) + +(define + flow-hyg-test + (fn + (name actual expected) + (if + (= actual expected) + (set! flow-hyg-pass (+ flow-hyg-pass 1)) + (begin + (set! flow-hyg-fail (+ flow-hyg-fail 1)) + (append! flow-hyg-fails {:name name :expected expected :actual actual}))))) + +(define flow-h (fn (src) (flow-run src))) + +;; ── flow/gc ───────────────────────────────────────────────────── +(flow-hyg-test + "gc: empty store removes nothing" + (flow-h "(flow/gc)") + 0) +(flow-hyg-test + "gc: removes a done flow, keeps a suspended one" + (flow-h + "(defflow w (lambda (x) (suspend (quote q)))) (flow/start w 0) (flow/start (lambda (x) x) 5) (define removed (flow/gc)) (list removed (flow/list))") + (list 1 (list (list 1 "suspended")))) +(flow-hyg-test + "gc: removes a cancelled flow" + (flow-h + "(defflow w (lambda (x) (suspend (quote q)))) (define id (car (cdr (flow/start w 0)))) (flow/cancel id) (flow/gc)") + 1) +(flow-hyg-test + "gc: a kept suspended flow is still resumable" + (flow-h + "(defflow w (sequence (lambda (x) (suspend (quote q))) (lambda (v) (* v 2)))) (define id (car (cdr (flow/start w 0)))) (flow/start (lambda (x) x) 1) (flow/gc) (flow/resume id 21)") + 42) +(flow-hyg-test + "gc: counts every terminal flow it drops" + (flow-h + "(flow/start (lambda (x) x) 1) (flow/start (lambda (x) x) 2) (defflow w (lambda (x) (suspend (quote q)))) (flow/start w 0) (flow/gc)") + 2) + +;; ── flow/forget ───────────────────────────────────────────────── +(flow-hyg-test + "forget: drops a completed flow" + (flow-h + "(defflow w (sequence (lambda (x) (suspend (quote q))) (lambda (v) v))) (define id (car (cdr (flow/start w 0)))) (flow/resume id 7) (list (flow/forget id) (flow/status id))") + (list true "unknown")) +(flow-hyg-test + "forget: refuses to drop a live (suspended) flow" + (flow-h + "(defflow w (lambda (x) (suspend (quote q)))) (define id (car (cdr (flow/start w 0)))) (list (flow/forget id) (flow/status id))") + (list false "suspended")) +(flow-hyg-test + "forget: drops a cancelled flow" + (flow-h + "(defflow w (lambda (x) (suspend (quote q)))) (define id (car (cdr (flow/start w 0)))) (flow/cancel id) (list (flow/forget id) (flow/status id))") + (list true "unknown")) +(flow-hyg-test + "forget: unknown id yields false" + (flow-h "(flow/forget 999)") + false) + +(define flow-hyg-tests-run! (fn () {:total (+ flow-hyg-pass flow-hyg-fail) :passed flow-hyg-pass :failed flow-hyg-fail :fails flow-hyg-fails})) diff --git a/plans/flow-on-sx.md b/plans/flow-on-sx.md index 95a8a57b..108e74f9 100644 --- a/plans/flow-on-sx.md +++ b/plans/flow-on-sx.md @@ -16,7 +16,7 @@ federation extension via fed-sx for remote-node execution. ## Status (rolling) -`bash lib/flow/conformance.sh` → **142/142** (Phases 1-7 complete) +`bash lib/flow/conformance.sh` → **151/151** (Phases 1-7 complete; +store hygiene) ## Ground rules @@ -141,6 +141,9 @@ something operators and authors actually use. Accumulation, not a rewrite. - [x] introspection API — `flow/status id`, `flow/result id`, `flow/list`, `flow/pending` (operator view of what each suspended flow awaits). 12 tests in `tests/api.sx`. +- [x] store hygiene — `flow/gc` drops terminal (done/cancelled) records keeping + live suspended flows (returns count); `flow/forget id` drops one terminal record + and refuses live flows. Bounds unbounded store growth. 9 tests in `tests/hygiene.sx`. - [x] `tap` — side-effecting pass-through node (logging/metrics) that returns input - [x] `recover` — complement to try-catch for the fail-VALUE channel: run node; if it yields `(fail ...)`, run a recovery node on the reason From c2d628e9c394d6a17641838c8bbf54d0cf8a4fde Mon Sep 17 00:00:00 2001 From: giles Date: Sat, 6 Jun 2026 18:37:10 +0000 Subject: [PATCH 18/20] =?UTF-8?q?flow:=20README=20=E2=80=94=20API=20refere?= =?UTF-8?q?nce=20+=20deterministic-replay=20contract?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit User-facing docs for the flow engine: the node model, every combinator, the suspend/resume durability contract (escape-only call/cc -> deterministic replay), lifecycle/introspection/hygiene API, fed-sx distribution, and substrate notes. Doc-only; 151/151 unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/flow/README.md | 141 ++++++++++++++++++++++++++++++++++++++++++++ plans/flow-on-sx.md | 6 ++ 2 files changed, 147 insertions(+) create mode 100644 lib/flow/README.md diff --git a/lib/flow/README.md b/lib/flow/README.md new file mode 100644 index 00000000..9c2dee4c --- /dev/null +++ b/lib/flow/README.md @@ -0,0 +1,141 @@ +# flow — durable DAG workflows on Scheme + +`flow` is a workflow engine for rose-ash: content pipelines (write → review → +publish → federate), scheduled jobs, and multi-step user flows (signup, confirm, +onboard) that **survive process restarts**. It is a thin Scheme prelude over the +Scheme-on-SX guest (`lib/scheme/`); a flow runs *inside* the interpreter. + +Run the suite: `bash lib/flow/conformance.sh` → **151/151 across 10 suites**. + +## Model + +A **flow** is just a Scheme procedure of one argument — the upstream value: + +``` +node : input -> output +``` + +Combinators build composite nodes out of child nodes. A node that ignores its +argument is effectively a thunk. There is no separate "graph" object: composition +*is* function composition, so flows are values you can name, pass, and nest. + +```scheme +(defflow publish + (sequence + (lambda (draft) (string-append draft "!")) + (branch (lambda (post) (>= (string-length post) 3)) + (remote-node 'fed 'publish) + (flow-const 'rejected)))) + +(flow/start publish "hello") ; => federated, or a (flow-suspended id tag) state +``` + +## Building blocks (`spec.sx`) + +| Combinator | Meaning | +|---|---| +| `(flow-node f)` / `(flow-id x)` / `(flow-const v)` | leaf nodes | +| `(sequence n ...)` | thread input left-to-right | +| `(parallel n ...)` | fan input to every child, join results into a list (sequential eval) | +| `(map-flow node)` | run `node` over each item of a list input, join results | +| `(flow-while pred body max)` / `(flow-until ...)` | bounded iteration (cap `max` steps) | +| `(defflow name body)` | bind + register a named flow (so it survives restart) | + +## Control flow + errors (`spec.sx`) + +| Combinator | Meaning | +|---|---| +| `(branch pred then else)` | `pred` on input selects `then`/`else` (`cond` is a Scheme special form) | +| `(retry n node)` | re-run on a *raised exception*, up to `n` attempts | +| `(timeout budget node)` | cooperative **step budget**: nodes call `(tick)`; the `(budget+1)`-th tick raises `flow-timeout` | +| `(try-catch node handler)` | catch a raised exception → `(handler error)` | +| `(fail reason)` / `(failed? x)` / `(fail-reason x)` | explicit failure *values* (flow downstream as data) | +| `(recover node handler)` | the fail-VALUE counterpart of try-catch | +| `(attempt n ...)` | railway sequence: stop at the first node returning a `(fail ...)` | +| `(tap effect)` | run a side effect, return input unchanged | + +**Two error channels, on purpose.** Raised exceptions are for *bugs/transients* +(caught by `retry`/`try-catch`). `(fail reason)` values are for *expected business +outcomes* (validation rejected, declined) and compose via `attempt`/`recover`. + +## Suspend / resume — the durable core (`spec.sx`, `store.sx`) + +The guest Scheme's `call/cc` is **escape-only** — re-invoking a captured +continuation after it returns *hangs* the runtime. So flow does **not** serialize +continuations. Instead it uses **deterministic replay**: + +- `(suspend tag)` — if `tag` is already in the replay log, return its logged value; + otherwise escape to the driver as `(flow-suspended tag)`. +- `resume` appends `(tag value)` to the log and **re-runs the flow from the start**. + Already-resolved suspends replay their values; the first unresolved one escapes + again (or the flow completes). + +The entire persisted state is the replay log — plain data. No live continuation is +ever stored, so flows survive process restarts and even moves between instances. + +> **Author contract:** suspend `tag`s must be unique and deterministic across +> replays, and **all** non-determinism / side effects must go through suspend +> points (so their results are logged) — otherwise they re-run on every replay. + +### Lifecycle (`store.sx`) + +```scheme +(flow/start flow input) ; raw result if it completes, else (flow-suspended id tag) +(flow/resume id value) ; inject value at the waiting tag, continue +(flow/cancel id) ; terminate; a later resume is rejected +``` + +### Introspection & hygiene + +```scheme +(flow/status id) ; done | suspended | cancelled | unknown +(flow/result id) ; result if done, else (flow-error reason) +(flow/list) ; ((id status) ...) +(flow/pending) ; ((id waiting-tag) ...) — what each suspended flow awaits +(flow/gc) ; drop terminal records, keep live ones; returns count removed +(flow/forget id) ; drop one terminal record (refuses live flows) +``` + +### Crash recovery + +```scheme +(flow-store-export) ; the store as plain data (live procs nulled) +(flow-store-import! d) ; restore the store from exported data +(flow-resumable-ids) ; ids of suspended flows to wake on restart +``` + +On restart the flow definitions are reloaded (`defflow` re-registers names) and the +exported store reimported; `resume` re-resolves each flow's procedure **by name**. + +## Distribution via fed-sx (`remote.sx`) + +```scheme +(flow-peer-register! addr table) ; mock a peer's exposed functions (fed-sx boundary) +(remote-node addr fn) ; run a node on a peer +(remote-failover addrs fn local) ; try peers in order, fall through to a local node +(flow-replicate-to addr) ; copy this store to a peer's replica slot +(flow-restore-from addr) ; import a peer's replica (handoff) +``` + +**Handoff** is crash recovery across instances: replicate → local instance dies → +peer restores the (plain-data) store and resumes. The replay log carries over, so +all resolved suspends survive the move. + +## Files + +| File | Contents | +|---|---| +| `spec.sx` | combinators (flow-combinators-src / flow-control-src / flow-suspend-src) | +| `store.sx` | durable store, lifecycle, crash recovery, introspection, hygiene | +| `remote.sx` | fed-sx transport (mock peer registry), failover, replication | +| `api.sx` | `flow-make-env` / `flow-run` SX helpers (one cached env, per-test reset) | +| `tests/*.sx` | 10 suites, 151 cases | +| `conformance.sh` | loads substrate + flow layer, runs every suite | + +## Notes on the substrate + +The guest Scheme (`lib/scheme/`, imported read-only) lacks dotted-rest params +`(a . rest)` and named `let`; combinators use `(lambda args ...)` variadics + top- +level recursion. `cons` is list-only (no dotted pairs), so log/assoc entries are +2-element lists. Strings box as `{:scm-string "..."}`. Timeout is a step budget +because there is no wall clock; `parallel` is sequential for the same reason. diff --git a/plans/flow-on-sx.md b/plans/flow-on-sx.md index 108e74f9..c14d8631 100644 --- a/plans/flow-on-sx.md +++ b/plans/flow-on-sx.md @@ -193,6 +193,12 @@ remote-node + crash-recovery + handoff + introspection together. registry. Timeout is a cooperative step budget (no wall clock in pure SX). Test harness reuses one env with a per-test reset for speed. +- **Phases 5-7 + docs.** Operational API (introspection, hygiene), combinator + library (tap/recover/map-flow/while/until), railway `attempt`, end-to-end + integration suite, and `lib/flow/README.md` (full API reference + replay-semantics + contract). **151/151 across 10 suites.** Conformance sx_server timeout raised to + 540s for the 10-suite run under shared-machine CPU contention. + ## Blockers (none) From 3cbf33d2d2db71622e75b9b7a09fd3b442a84dd3 Mon Sep 17 00:00:00 2001 From: giles Date: Sat, 6 Jun 2026 19:24:16 +0000 Subject: [PATCH 19/20] flow: host integration ABI (request/await/host-queue) + 11 tests (Phase 8) The seam for hooking flow to art-dag and human-in-the-loop later. (request kind payload) suspends with a typed (flow-request kind payload) envelope and returns the host's resume value; await-human/await-render sugar. (flow-host-requests) is the host work queue: (id kind payload) for every suspended flow awaiting a host effect; request?/request-kind/request-payload parse a tag. Tests include the art-dag-shaped driver loop (render -> human-review -> publish). Host owns IO+persistence; flow only requests (replay-safe). 162/162 across 11 suites. Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/flow/api.sx | 9 +++-- lib/flow/conformance.sh | 2 + lib/flow/host.sx | 35 ++++++++++++++++ lib/flow/scoreboard.json | 9 +++-- lib/flow/scoreboard.md | 3 +- lib/flow/tests/host.sx | 87 ++++++++++++++++++++++++++++++++++++++++ plans/flow-on-sx.md | 23 ++++++++++- 7 files changed, 159 insertions(+), 9 deletions(-) create mode 100644 lib/flow/host.sx create mode 100644 lib/flow/tests/host.sx diff --git a/lib/flow/api.sx b/lib/flow/api.sx index d3e8dce3..b6feca69 100644 --- a/lib/flow/api.sx +++ b/lib/flow/api.sx @@ -1,8 +1,9 @@ ;; lib/flow/api.sx — flow runtime entry points. ;; ;; Builds a Scheme env preloaded with the flow combinators (lib/flow/spec.sx), -;; the durable store + lifecycle (lib/flow/store.sx), and the fed-sx remote layer -;; (lib/flow/remote.sx), and provides SX helpers to run flow programs. +;; the durable store + lifecycle (lib/flow/store.sx), the fed-sx remote layer +;; (lib/flow/remote.sx), and the host integration ABI (lib/flow/host.sx), and +;; provides SX helpers to run flow programs. ;; ;; Scheme-level API (available inside flow programs): ;; (flow/start flow input) — run a flow; raw result if it completes, else @@ -10,10 +11,11 @@ ;; (flow/resume id value) — resume a suspended flow (store.sx) ;; (flow/cancel id) — cancel a flow (store.sx) ;; (suspend tag) — suspension point (spec.sx) +;; (request kind payload) — host request envelope over suspend (host.sx) ;; (remote-node addr fn) — node executed on a federation peer (remote.sx) ;; ;; SX-level helpers (for hosts and tests): -;; (flow-make-env) — fresh standard env + combinators + store + remote +;; (flow-make-env) — fresh standard env + combinators + store + remote + host ;; (flow-run src) — eval a Scheme program string in a reset shared env ;; (flow-run-in env src) — eval a Scheme program string in a given env ;; @@ -31,6 +33,7 @@ (flow-load-combinators! env) (flow-load-store! env) (flow-load-remote! env) + (flow-load-host! env) env))) (define diff --git a/lib/flow/conformance.sh b/lib/flow/conformance.sh index 787c9c9e..31a1d860 100755 --- a/lib/flow/conformance.sh +++ b/lib/flow/conformance.sh @@ -31,6 +31,7 @@ SUITES=( "railway flow-rail-tests-run! lib/flow/tests/railway.sx" "integration flow-int-tests-run! lib/flow/tests/integration.sx" "hygiene flow-hyg-tests-run! lib/flow/tests/hygiene.sx" + "host flow-hst-tests-run! lib/flow/tests/host.sx" ) TMPFILE=$(mktemp); trap "rm -f $TMPFILE" EXIT @@ -49,6 +50,7 @@ emit_eval () { echo "(epoch $EPOCH)"; echo "(eval \"$1\")"; EPOCH=$((EPOCH+1)); emit_load "lib/flow/spec.sx" emit_load "lib/flow/store.sx" emit_load "lib/flow/remote.sx" + emit_load "lib/flow/host.sx" emit_load "lib/flow/api.sx" for SUITE in "${SUITES[@]}"; do read -r _NAME _RUNNER FILE <<< "$SUITE" diff --git a/lib/flow/host.sx b/lib/flow/host.sx new file mode 100644 index 00000000..28e3b556 --- /dev/null +++ b/lib/flow/host.sx @@ -0,0 +1,35 @@ +;; lib/flow/host.sx — the host integration ABI (Phase 8). +;; +;; `suspend` is flow's seam to the outside world, but a bare (suspend tag) is just a +;; signal — every author would invent their own tag shape. This layer defines a +;; stable request/response contract so a host (e.g. an art-dag driver, or a human +;; review UI) can hook in WITHOUT reverse-engineering ad-hoc tags. +;; +;; A flow asks the host to do something and waits for the answer: +;; (request kind payload) — suspend with a typed envelope (flow-request kind +;; payload); evaluates to the host's resume value. +;; (await-human prompt) — request kind=human (a decision point) +;; (await-render recipe) — request kind=render (e.g. an art-dag job) +;; (await-effect kind p) — request of an arbitrary kind +;; +;; The host drives flows by polling its work queue and resuming: +;; (flow-host-requests) — ((id kind payload) ...) for every SUSPENDED flow whose +;; waiting tag is a host request. The host dispatches by kind (render -> submit a +;; Celery job; human -> show UI), then calls (flow/resume id answer). +;; (request? tag) / (request-kind tag) / (request-payload tag) — parse one tag. +;; +;; Contract: the host owns IO and persistence. flow stays deterministic — a flow +;; never performs IO itself, it only `request`s; the host performs the effect and +;; feeds the result back via resume (which the replay log records, so the effect is +;; not re-run on recovery). Persist with flow-store-export after each transition and +;; flow-store-import! on boot. + +(define + flow-host-src + "(define (request kind payload) (suspend (list (quote flow-request) kind payload)))\n (define (request? tag) (and (pair? tag) (eq? (car tag) (quote flow-request))))\n (define (request-kind tag) (car (cdr tag)))\n (define (request-payload tag) (car (cdr (cdr tag))))\n (define (await-human prompt) (request (quote human) prompt))\n (define (await-render recipe) (request (quote render) recipe))\n (define (await-effect kind payload) (request kind payload))\n (define (flow-host-req-step pend)\n (if (null? pend)\n (list)\n (let ((id (car (car pend))) (tag (car (cdr (car pend)))))\n (if (request? tag)\n (cons (list id (request-kind tag) (request-payload tag))\n (flow-host-req-step (cdr pend)))\n (flow-host-req-step (cdr pend))))))\n (define (flow-host-requests) (flow-host-req-step (flow/pending)))") + +(define + flow-load-host! + (fn + (env) + (begin (scheme-eval-program (scheme-parse-all flow-host-src) env) env))) diff --git a/lib/flow/scoreboard.json b/lib/flow/scoreboard.json index 012c8bfb..8cedc169 100644 --- a/lib/flow/scoreboard.json +++ b/lib/flow/scoreboard.json @@ -1,6 +1,6 @@ { - "total": 151, - "passed": 151, + "total": 162, + "passed": 162, "failed": 0, "suites": { "basic": { "passed": 18, "total": 18 }, @@ -12,7 +12,8 @@ "combinators": { "passed": 17, "total": 17 }, "railway": { "passed": 10, "total": 10 }, "integration": { "passed": 10, "total": 10 }, - "hygiene": { "passed": 9, "total": 9 } + "hygiene": { "passed": 9, "total": 9 }, + "host": { "passed": 11, "total": 11 } }, - "phases": { "phase1": "done", "phase2": "done", "phase3": "done", "phase4": "done", "phase5": "done", "phase6": "done", "phase7": "done" } + "phases": { "phase1": "done", "phase2": "done", "phase3": "done", "phase4": "done", "phase5": "done", "phase6": "done", "phase7": "done", "phase8": "done" } } diff --git a/lib/flow/scoreboard.md b/lib/flow/scoreboard.md index 41eaf45d..7ee0da8d 100644 --- a/lib/flow/scoreboard.md +++ b/lib/flow/scoreboard.md @@ -1,6 +1,6 @@ # flow-on-sx Scoreboard -**All tests pass: 151 / 151 across 10 suites. Phases 1-7 complete.** +**All tests pass: 162 / 162 across 11 suites. Phases 1-8 complete.** `bash lib/flow/conformance.sh` @@ -18,6 +18,7 @@ | railway | 10 | Phase 6: `attempt` — fail-value short-circuiting sequence + recover rejoin | | integration | 10 | Phase 7: end-to-end order + onboarding flows composing every phase (suspend, branch, federation, crash recovery, handoff, introspection) | | hygiene | 9 | Phase 5: `flow/gc` (prune terminal flows), `flow/forget` (drop one terminal record) | +| host | 11 | Phase 8: host ABI — `request`/`await-human`/`await-render`, `flow-host-requests` work queue; art-dag-shaped driver loop | ## Architecture diff --git a/lib/flow/tests/host.sx b/lib/flow/tests/host.sx new file mode 100644 index 00000000..74ec3b5d --- /dev/null +++ b/lib/flow/tests/host.sx @@ -0,0 +1,87 @@ +;; lib/flow/tests/host.sx — Phase 8: host integration ABI (request/await/host-queue). + +(define flow-hst-pass 0) +(define flow-hst-fail 0) +(define flow-hst-fails (list)) + +(define + flow-hst-test + (fn + (name actual expected) + (if + (= actual expected) + (set! flow-hst-pass (+ flow-hst-pass 1)) + (begin + (set! flow-hst-fail (+ flow-hst-fail 1)) + (append! flow-hst-fails {:name name :expected expected :actual actual}))))) + +(define flow-hst (fn (src) (flow-run src))) + +;; ── request envelope ──────────────────────────────────────────── +(flow-hst-test + "request: suspends with a typed envelope" + (flow-hst + "(car (cdr (cdr (flow/start (lambda (x) (request (quote render) x)) 5))))") + (list "flow-request" "render" 5)) +(flow-hst-test + "request?: recognizes an envelope" + (flow-hst "(request? (list (quote flow-request) (quote human) 1))") + true) +(flow-hst-test + "request?: a plain tag is not a request" + (flow-hst "(request? (list (quote review) 1))") + false) +(flow-hst-test + "request-kind / request-payload: parse the envelope" + (flow-hst + "(define t (list (quote flow-request) (quote render) (list (quote recipe) 7))) (list (request-kind t) (request-payload t))") + (list "render" (list "recipe" 7))) + +;; ── named decision points ─────────────────────────────────────── +(flow-hst-test + "await-human: is a request of kind human" + (flow-hst + "(car (cdr (cdr (flow/start (lambda (x) (await-human x)) (quote approve?)))))") + (list "flow-request" "human" "approve?")) +(flow-hst-test + "await-render: is a request of kind render" + (flow-hst + "(car (cdr (cdr (flow/start (lambda (x) (await-render x)) (quote recipe)))))") + (list "flow-request" "render" "recipe")) +(flow-hst-test + "request: the host's resume value flows back into the flow" + (flow-hst + "(defflow f (sequence (lambda (x) (await-render x)) (lambda (art) (list (quote got) art)))) (define id (car (cdr (flow/start f 1)))) (flow/resume id (quote the-artifact))") + (list "got" "the-artifact")) + +;; ── host work queue ───────────────────────────────────────────── +(flow-hst-test + "flow-host-requests: lists (id kind payload) for pending requests" + (flow-hst + "(flow/start (lambda (x) (await-render x)) 99) (flow-host-requests)") + (list (list 1 "render" 99))) +(flow-hst-test + "flow-host-requests: excludes bare (non-request) suspends" + (flow-hst + "(defflow a (lambda (x) (await-render x))) (defflow b (lambda (x) (suspend (quote plain)))) (flow/start a 1) (flow/start b 2) (flow-host-requests)") + (list (list 1 "render" 1))) + +;; ── the art-dag-shaped host driver loop ───────────────────────── +;; A host: poll requests, dispatch by kind (render -> compute; human -> decide), +;; resume with the result. Drives a render -> human-review -> publish pipeline. +(flow-hst-test + "host driver: render then human-review then publish" + (flow-hst + "(defflow pipeline (sequence (lambda (recipe) (await-render recipe)) (lambda (art) (await-human (list (quote review) art))) (branch (lambda (d) (eq? d (quote approve))) (flow-const (quote published)) (flow-const (fail (quote rejected)))))) (define id (car (cdr (flow/start pipeline 99)))) (define r1 (flow-host-requests)) (flow/resume id (list (quote art) 99)) (define r2 (flow-host-requests)) (flow/resume id (quote approve)) (list r1 r2 (flow/status id) (flow/result id))") + (list + (list (list 1 "render" 99)) + (list (list 1 "human" (list "review" (list "art" 99)))) + "done" + "published")) +(flow-hst-test + "host driver: rejection at the human gate yields a failure" + (flow-hst + "(defflow pipeline (sequence (lambda (recipe) (await-render recipe)) (lambda (art) (await-human (list (quote review) art))) (branch (lambda (d) (eq? d (quote approve))) (flow-const (quote published)) (flow-const (fail (quote rejected)))))) (define id (car (cdr (flow/start pipeline 1)))) (flow/resume id (quote artifact)) (failed? (flow/resume id (quote reject)))") + true) + +(define flow-hst-tests-run! (fn () {:total (+ flow-hst-pass flow-hst-fail) :passed flow-hst-pass :failed flow-hst-fail :fails flow-hst-fails})) diff --git a/plans/flow-on-sx.md b/plans/flow-on-sx.md index c14d8631..2d336da6 100644 --- a/plans/flow-on-sx.md +++ b/plans/flow-on-sx.md @@ -16,7 +16,7 @@ federation extension via fed-sx for remote-node execution. ## Status (rolling) -`bash lib/flow/conformance.sh` → **151/151** (Phases 1-7 complete; +store hygiene) +`bash lib/flow/conformance.sh` → **162/162** (Phases 1-8 complete; host ABI for art-dag) ## Ground rules @@ -162,6 +162,27 @@ Make the `(fail reason)` value channel compose into real validation/ETL pipeline - [x] `lib/flow/tests/railway.sx` — 10 cases: fail short-circuiting, no-run-after- failure, recover rejoin, validation pipeline reporting the failing stage +## Phase 8 — Host integration ABI (art-dag / human-in-the-loop) + +`suspend` is the seam to the outside world, but a bare tag is an ad-hoc convention. +This phase defines a stable request/response contract a host (an art-dag driver, a +review UI) codes against — so flow can orchestrate art-dag with human decision +points later without reverse-engineering tag shapes. `lib/flow/host.sx`. + +- [x] `(request kind payload)` — suspend with a typed `(flow-request kind payload)` + envelope; evaluates to the host's resume value. `await-human`/`await-render`/ + `await-effect` sugar. +- [x] `(flow-host-requests)` — the host work queue: `(id kind payload)` for every + suspended flow waiting on a host request; `request?`/`request-kind`/ + `request-payload` parse a tag. +- [x] `lib/flow/tests/host.sx` — 11 cases incl. the art-dag-shaped driver loop + (render → human-review → publish, driven by polling the queue + resume). +- Contract (documented in `host.sx` + README): the host owns IO + persistence; a + flow never does IO, it only `request`s; the host performs the effect and feeds the + result back via resume (logged, so not re-run on recovery). NOT done here (host + side, out of `lib/flow` scope): the real Celery/IPFS bridge and a persistent store + backend — those live in the art-dag integration, coding against this ABI. + ## Phase 7 — End-to-end integration Prove the phases compose: realistic flows exercising attempt + suspend + branch + From 9cfca1d00886e426ca85e2a9f9ef9a06c786e61a Mon Sep 17 00:00:00 2001 From: giles Date: Sat, 6 Jun 2026 19:33:04 +0000 Subject: [PATCH 20/20] flow: reference host driver flow-drive-host/flow-run-host + 4 tests Completes the host ABI from work-queue to driver loop: the host supplies only a (kind payload) -> answer dispatch fn; flow-drive-host services one tick of pending requests, flow-run-host ticks until quiescent (bounded). Tested via the art-dag render -> human-review -> publish pipeline driven entirely by flow-run-host. The art-dag integration is now: define dispatch, call flow-run-host. 166/166, 11 suites. Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/flow/host.sx | 9 ++++++++- lib/flow/scoreboard.json | 6 +++--- lib/flow/scoreboard.md | 4 ++-- lib/flow/tests/host.sx | 27 +++++++++++++++++++++++---- plans/flow-on-sx.md | 9 ++++++--- 5 files changed, 42 insertions(+), 13 deletions(-) diff --git a/lib/flow/host.sx b/lib/flow/host.sx index 28e3b556..da92ac7a 100644 --- a/lib/flow/host.sx +++ b/lib/flow/host.sx @@ -18,6 +18,13 @@ ;; Celery job; human -> show UI), then calls (flow/resume id answer). ;; (request? tag) / (request-kind tag) / (request-payload tag) — parse one tag. ;; +;; Reference driver — the host only supplies `dispatch`, a (kind payload) -> answer: +;; (flow-drive-host dispatch) — one tick: service every CURRENTLY pending +;; request (snapshot), resuming each with (dispatch kind payload); returns the +;; count serviced. Resumes may create new requests — serviced on the next tick. +;; (flow-run-host dispatch maxticks) — tick until quiescent (no pending requests) +;; or maxticks reached; returns total requests serviced. Bounded for determinism. +;; ;; Contract: the host owns IO and persistence. flow stays deterministic — a flow ;; never performs IO itself, it only `request`s; the host performs the effect and ;; feeds the result back via resume (which the replay log records, so the effect is @@ -26,7 +33,7 @@ (define flow-host-src - "(define (request kind payload) (suspend (list (quote flow-request) kind payload)))\n (define (request? tag) (and (pair? tag) (eq? (car tag) (quote flow-request))))\n (define (request-kind tag) (car (cdr tag)))\n (define (request-payload tag) (car (cdr (cdr tag))))\n (define (await-human prompt) (request (quote human) prompt))\n (define (await-render recipe) (request (quote render) recipe))\n (define (await-effect kind payload) (request kind payload))\n (define (flow-host-req-step pend)\n (if (null? pend)\n (list)\n (let ((id (car (car pend))) (tag (car (cdr (car pend)))))\n (if (request? tag)\n (cons (list id (request-kind tag) (request-payload tag))\n (flow-host-req-step (cdr pend)))\n (flow-host-req-step (cdr pend))))))\n (define (flow-host-requests) (flow-host-req-step (flow/pending)))") + "(define (request kind payload) (suspend (list (quote flow-request) kind payload)))\n (define (request? tag) (and (pair? tag) (eq? (car tag) (quote flow-request))))\n (define (request-kind tag) (car (cdr tag)))\n (define (request-payload tag) (car (cdr (cdr tag))))\n (define (await-human prompt) (request (quote human) prompt))\n (define (await-render recipe) (request (quote render) recipe))\n (define (await-effect kind payload) (request kind payload))\n (define (flow-host-req-step pend)\n (if (null? pend)\n (list)\n (let ((id (car (car pend))) (tag (car (cdr (car pend)))))\n (if (request? tag)\n (cons (list id (request-kind tag) (request-payload tag))\n (flow-host-req-step (cdr pend)))\n (flow-host-req-step (cdr pend))))))\n (define (flow-host-requests) (flow-host-req-step (flow/pending)))\n (define (flow-drive-host-step reqs dispatch)\n (if (null? reqs)\n 0\n (begin\n (flow/resume (car (car reqs)) (dispatch (car (cdr (car reqs))) (car (cdr (cdr (car reqs))))))\n (+ 1 (flow-drive-host-step (cdr reqs) dispatch)))))\n (define (flow-drive-host dispatch) (flow-drive-host-step (flow-host-requests) dispatch))\n (define (flow-run-host dispatch maxticks)\n (if (<= maxticks 0)\n 0\n (let ((n (flow-drive-host dispatch)))\n (if (= n 0) 0 (+ n (flow-run-host dispatch (- maxticks 1)))))))") (define flow-load-host! diff --git a/lib/flow/scoreboard.json b/lib/flow/scoreboard.json index 8cedc169..5229b185 100644 --- a/lib/flow/scoreboard.json +++ b/lib/flow/scoreboard.json @@ -1,6 +1,6 @@ { - "total": 162, - "passed": 162, + "total": 166, + "passed": 166, "failed": 0, "suites": { "basic": { "passed": 18, "total": 18 }, @@ -13,7 +13,7 @@ "railway": { "passed": 10, "total": 10 }, "integration": { "passed": 10, "total": 10 }, "hygiene": { "passed": 9, "total": 9 }, - "host": { "passed": 11, "total": 11 } + "host": { "passed": 15, "total": 15 } }, "phases": { "phase1": "done", "phase2": "done", "phase3": "done", "phase4": "done", "phase5": "done", "phase6": "done", "phase7": "done", "phase8": "done" } } diff --git a/lib/flow/scoreboard.md b/lib/flow/scoreboard.md index 7ee0da8d..70afaeee 100644 --- a/lib/flow/scoreboard.md +++ b/lib/flow/scoreboard.md @@ -1,6 +1,6 @@ # flow-on-sx Scoreboard -**All tests pass: 162 / 162 across 11 suites. Phases 1-8 complete.** +**All tests pass: 166 / 166 across 11 suites. Phases 1-8 complete.** `bash lib/flow/conformance.sh` @@ -18,7 +18,7 @@ | railway | 10 | Phase 6: `attempt` — fail-value short-circuiting sequence + recover rejoin | | integration | 10 | Phase 7: end-to-end order + onboarding flows composing every phase (suspend, branch, federation, crash recovery, handoff, introspection) | | hygiene | 9 | Phase 5: `flow/gc` (prune terminal flows), `flow/forget` (drop one terminal record) | -| host | 11 | Phase 8: host ABI — `request`/`await-human`/`await-render`, `flow-host-requests` work queue; art-dag-shaped driver loop | +| host | 15 | Phase 8: host ABI — `request`/`await-human`/`await-render`, `flow-host-requests` queue, `flow-run-host` reference driver; art-dag-shaped render→review→publish loop | ## Architecture diff --git a/lib/flow/tests/host.sx b/lib/flow/tests/host.sx index 74ec3b5d..d0e8335a 100644 --- a/lib/flow/tests/host.sx +++ b/lib/flow/tests/host.sx @@ -1,4 +1,4 @@ -;; lib/flow/tests/host.sx — Phase 8: host integration ABI (request/await/host-queue). +;; lib/flow/tests/host.sx — Phase 8: host integration ABI (request/await/host-queue/driver). (define flow-hst-pass 0) (define flow-hst-fail 0) @@ -66,9 +66,7 @@ "(defflow a (lambda (x) (await-render x))) (defflow b (lambda (x) (suspend (quote plain)))) (flow/start a 1) (flow/start b 2) (flow-host-requests)") (list (list 1 "render" 1))) -;; ── the art-dag-shaped host driver loop ───────────────────────── -;; A host: poll requests, dispatch by kind (render -> compute; human -> decide), -;; resume with the result. Drives a render -> human-review -> publish pipeline. +;; ── the art-dag-shaped host driver loop (manual resumes) ──────── (flow-hst-test "host driver: render then human-review then publish" (flow-hst @@ -84,4 +82,25 @@ "(defflow pipeline (sequence (lambda (recipe) (await-render recipe)) (lambda (art) (await-human (list (quote review) art))) (branch (lambda (d) (eq? d (quote approve))) (flow-const (quote published)) (flow-const (fail (quote rejected)))))) (define id (car (cdr (flow/start pipeline 1)))) (flow/resume id (quote artifact)) (failed? (flow/resume id (quote reject)))") true) +;; ── reference driver: host supplies only a dispatch fn ────────── +(flow-hst-test + "flow-drive-host: one tick services every pending request" + (flow-hst + "(flow/start (lambda (x) (await-render x)) 5) (define n (flow-drive-host (lambda (k p) (list (quote done) p)))) (list n (flow/status 1) (flow/result 1))") + (list 1 "done" (list "done" 5))) +(flow-hst-test + "flow-run-host: drives a render -> human pipeline to completion" + (flow-hst + "(defflow pipeline (sequence (lambda (recipe) (await-render recipe)) (lambda (art) (await-human (list (quote review) art))) (branch (lambda (d) (eq? d (quote approve))) (flow-const (quote published)) (flow-const (fail (quote rejected)))))) (define id (car (cdr (flow/start pipeline 99)))) (define serviced (flow-run-host (lambda (kind payload) (if (eq? kind (quote render)) (list (quote art) payload) (quote approve))) 10)) (list serviced (flow/status id) (flow/result id))") + (list 2 "done" "published")) +(flow-hst-test + "flow-run-host: returns 0 when nothing is pending" + (flow-hst "(flow-run-host (lambda (k p) p) 5)") + 0) +(flow-hst-test + "flow-run-host: respects the maxticks bound" + (flow-hst + "(defflow pipe2 (sequence (lambda (r) (await-render r)) (lambda (a) (await-human a)) (lambda (d) d))) (define id (car (cdr (flow/start pipe2 1)))) (define serviced (flow-run-host (lambda (k p) p) 1)) (list serviced (flow/status id))") + (list 1 "suspended")) + (define flow-hst-tests-run! (fn () {:total (+ flow-hst-pass flow-hst-fail) :passed flow-hst-pass :failed flow-hst-fail :fails flow-hst-fails})) diff --git a/plans/flow-on-sx.md b/plans/flow-on-sx.md index 2d336da6..1310998c 100644 --- a/plans/flow-on-sx.md +++ b/plans/flow-on-sx.md @@ -16,7 +16,7 @@ federation extension via fed-sx for remote-node execution. ## Status (rolling) -`bash lib/flow/conformance.sh` → **162/162** (Phases 1-8 complete; host ABI for art-dag) +`bash lib/flow/conformance.sh` → **166/166** (Phases 1-8 complete; host ABI + reference driver) ## Ground rules @@ -175,8 +175,11 @@ points later without reverse-engineering tag shapes. `lib/flow/host.sx`. - [x] `(flow-host-requests)` — the host work queue: `(id kind payload)` for every suspended flow waiting on a host request; `request?`/`request-kind`/ `request-payload` parse a tag. -- [x] `lib/flow/tests/host.sx` — 11 cases incl. the art-dag-shaped driver loop - (render → human-review → publish, driven by polling the queue + resume). +- [x] `(flow-drive-host dispatch)` / `(flow-run-host dispatch maxticks)` — reference + host driver: the host supplies only a `(kind payload) -> answer` dispatch fn; the + loop drains pending requests and resumes until quiescent (bounded). +- [x] `lib/flow/tests/host.sx` — 15 cases incl. the art-dag-shaped driver loop + (render → human-review → publish) run both manually and via `flow-run-host`. - Contract (documented in `host.sx` + README): the host owns IO + persistence; a flow never does IO, it only `request`s; the host performs the effect and feeds the result back via resume (logged, so not re-run on recovery). NOT done here (host