flow: remote-failover — try peers in order, fall through to local + 6 tests

(remote-failover addrs fn local) tries fn on each peer in order, moves to the next
on any raised error, and runs the local node if every peer fails. Threads input,
composes in sequences.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

lib/flow/tests/distributed.sx

@@ -17,11 +17,6 @@
 
 (define flow-d (fn (src) (flow-run src)))
 
-;; A mock peer "edge" exposing double/inc, registered at the top of each program.
-(define
-  peer-setup
-  "(flow-peer-register! (quote edge) (list (list (quote double) (lambda (x) (* x 2))) (list (quote inc) (lambda (x) (+ x 1)))))")
-
 ;; ── remote-node ─────────────────────────────────────────────────
 (flow-dist-test
   "remote: a node executes on a peer"
@@ -59,4 +54,36 @@
     "(define hits 0) (flow-peer-register! (quote edge) (list (list (quote flaky) (lambda (x) (begin (set! hits (+ hits 1)) (if (< hits 2) (raise (quote down)) (* x 3))))))) (list (flow/start (retry 3 (remote-node (quote edge) (quote flaky))) 7) hits)")
   (list 21 2))
 
+;; ── failover (retry on a different peer, fall through to local) ──
+(flow-dist-test
+  "failover: first reachable peer serves the request"
+  (flow-d
+    "(flow-peer-register! (quote p2) (list (list (quote f) (lambda (x) (+ x 100))))) (flow/start (remote-failover (list (quote p2) (quote down)) (quote f) (flow-const (quote local))) 5)")
+  105)
+(flow-dist-test
+  "failover: skips an unreachable peer to the next one"
+  (flow-d
+    "(flow-peer-register! (quote p2) (list (list (quote f) (lambda (x) (+ x 100))))) (flow/start (remote-failover (list (quote down) (quote p2)) (quote f) (flow-const (quote local))) 5)")
+  105)
+(flow-dist-test
+  "failover: skips a peer whose function raises"
+  (flow-d
+    "(flow-peer-register! (quote bad) (list (list (quote f) (lambda (x) (raise (quote boom)))))) (flow-peer-register! (quote good) (list (list (quote f) (lambda (x) (* x 10))))) (flow/start (remote-failover (list (quote bad) (quote good)) (quote f) (flow-const 0)) 4)")
+  40)
+(flow-dist-test
+  "failover: all peers fail, the local fallback runs"
+  (flow-d
+    "(flow/start (remote-failover (list (quote down1) (quote down2)) (quote f) (lambda (x) (* x -1))) 9)")
+  -9)
+(flow-dist-test
+  "failover: threads the input through to the chosen peer"
+  (flow-d
+    "(flow-peer-register! (quote p) (list (list (quote f) (lambda (x) (list (quote got) x))))) (flow/start (sequence (lambda (x) (+ x 1)) (remote-failover (list (quote p)) (quote f) (flow-const 0))) 41)")
+  (list "got" 42))
+(flow-dist-test
+  "failover: composes inside a larger sequence"
+  (flow-d
+    "(flow-peer-register! (quote p) (list (list (quote f) (lambda (x) (* x 2))))) (flow/start (sequence (remote-failover (list (quote down) (quote p)) (quote f) (flow-const 1)) (lambda (x) (+ x 3))) 5)")
+  13)
+
 (define flow-dist-tests-run! (fn () {:total (+ flow-dist-pass flow-dist-fail) :passed flow-dist-pass :failed flow-dist-fail :fails flow-dist-fails}))