feed: Phase 4 visibility + federation — per-viewer ACL, fanout partition, inbound/backfill/ingest, e2e feed/timeline + 22 tests

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

lib/feed/acl.sx

@@ -0,0 +1,38 @@
+; feed/acl — per-viewer visibility filtering. The same candidate stream yields
+; different timelines for different viewers, so ACL is applied per request and
+; pre-ACL timelines are never cached.
+;
+; permit? is injected: (permit? viewer activity) -> bool. Wire a real acl-sx
+; predicate here; feed/permit-acl? is a self-contained default that reads an
+; optional :visible-to allowlist on the activity.
+;
+; Requires: lib/feed/normalize.sx, lib/feed/stream.sx, lib/feed/fanout.sx
+; (feed/-elem?), lib/feed/rank.sx (feed/top).
+
+; default permit: actor always sees own activity; absent/nil :visible-to is
+; public; otherwise viewer must be in the allowlist.
+(define
+  feed/permit-acl?
+  (fn
+    (viewer a)
+    (or
+      (equal? viewer (get a :actor))
+      (let
+        ((allowed (get a :visible-to nil)))
+        (if (= allowed nil) true (feed/-elem? viewer allowed))))))
+
+(define feed/permit-public? (fn (viewer a) true))
+
+; filter a stream to what viewer may read
+(define
+  feed/visible
+  (fn
+    (stream viewer permit?)
+    (feed/filter stream (fn (a) (permit? viewer a)))))
+
+; the capstone: candidate stream -> ACL for viewer -> rank -> top-N
+(define
+  feed/timeline
+  (fn
+    (stream viewer permit? score-fn n)
+    (feed/top (feed/visible stream viewer permit?) score-fn n)))

lib/feed/conformance.sh

@@ -13,7 +13,7 @@ if [ ! -x "$SX_SERVER" ]; then
   exit 1
 fi
 
-SUITES=(basic fanout rank)
+SUITES=(basic fanout rank integration)
 
 OUT_JSON="lib/feed/scoreboard.json"
 OUT_MD="lib/feed/scoreboard.md"
@@ -35,6 +35,8 @@ run_suite() {
 (load "lib/feed/dedupe.sx")
 (load "lib/feed/aggregate.sx")
 (load "lib/feed/rank.sx")
+(load "lib/feed/acl.sx")
+(load "lib/feed/fed.sx")
 (epoch 2)
 (eval "(define feed-test-pass 0)")
 (eval "(define feed-test-fail 0)")

lib/feed/fed.sx

@@ -0,0 +1,60 @@
+; feed/fed — federation. Outbound: a local post fans out, then splits into local
+; vs remote inboxes; remote events are handed to an injected send-fn. Inbound:
+; peer activities merge into the local stream, deduped. Backfill: pull peer
+; history via an injected fetch-fn and merge.
+;
+; remote? / send-fn / fetch-fn are injected so real fed-sx transport wires in here
+; without feed depending on it.
+;
+; Requires: lib/feed/normalize.sx, lib/feed/stream.sx, lib/feed/fanout.sx,
+; lib/feed/dedupe.sx.
+
+; --- merge / ingest ---------------------------------------------------------
+
+(define
+  feed/merge
+  (fn (s1 s2) (feed/stream (append (feed/items s1) (feed/items s2)))))
+
+; merge a peer stream into local, dropping (actor verb object) duplicates
+(define
+  feed/ingest
+  (fn (local peer) (feed/dedupe-activities (feed/merge local peer))))
+
+; --- inbound ----------------------------------------------------------------
+
+; peer pushes raw activities to the local inbox; normalize + ingest
+(define
+  feed/inbound
+  (fn
+    (local raw-activities)
+    (feed/ingest local (feed/stream (map feed/normalize raw-activities)))))
+
+; backfill on subscribe: pull peer history via fetch-fn, normalize, ingest
+(define
+  feed/backfill
+  (fn (local fetch-fn peer-id) (feed/inbound local (fetch-fn peer-id))))
+
+; --- outbound ---------------------------------------------------------------
+
+; split an inbox into local vs remote deliveries by viewer-id predicate
+(define feed/partition-inbox (fn (inbox remote?) {:local (feed/filter inbox (fn (ev) (not (remote? (get ev :to))))) :remote (feed/filter inbox (fn (ev) (remote? (get ev :to))))}))
+
+; fan a stream out over the graph, then partition by locality
+(define
+  feed/federate
+  (fn
+    (stream graph remote?)
+    (feed/partition-inbox (feed/fanout stream graph) remote?)))
+
+; deliver: hand each remote event to send-fn, return the local inbox to enqueue
+(define
+  feed/deliver
+  (fn
+    (stream graph remote? send-fn)
+    (let
+      ((parts (feed/federate stream graph remote?)))
+      (begin
+        (for-each
+          (fn (ev) (send-fn (get ev :to) (get ev :activity)))
+          (feed/items (get parts :remote)))
+        (get parts :local)))))

lib/feed/normalize.sx

@@ -1,6 +1,8 @@
 ; feed/normalize — coerce arbitrary input into the canonical activity record.
 ; An activity is a small dict {:actor :verb :object :at :tags}; a stream is an
-; APL vector of such dicts (see stream.sx).
+; APL vector of such dicts (see stream.sx). Extra keys on the raw input survive
+; (e.g. :visible-to for ACL, peer metadata for federation) — :tags is the
+; flexible bag but the record is not closed.
 
 (define feed/activity-keys (list :actor :verb :object :at :tags))
 
@@ -8,7 +10,9 @@
   feed/normalize
   (fn
     (raw)
-    (let ((d (if (= (type-of raw) "dict") raw {}))) {:actor (get d :actor "") :object (get d :object nil) :at (get d :at 0) :tags (let ((t (get d :tags (list)))) (if (list? t) t (list t))) :verb (get d :verb "post")})))
+    (let
+      ((d (if (= (type-of raw) "dict") raw {})))
+      (merge d {:actor (get d :actor "") :object (get d :object nil) :at (get d :at 0) :tags (let ((t (get d :tags (list)))) (if (list? t) t (list t))) :verb (get d :verb "post")}))))
 
 (define
   feed/activity

lib/feed/scoreboard.json

@@ -2,9 +2,10 @@
   "suites": {
     "basic": {"pass": 30, "fail": 0},
     "fanout": {"pass": 29, "fail": 0},
-    "rank": {"pass": 24, "fail": 0}
+    "rank": {"pass": 24, "fail": 0},
+    "integration": {"pass": 22, "fail": 0}
   },
-  "total_pass": 83,
+  "total_pass": 105,
   "total_fail": 0,
-  "total": 83
+  "total": 105
 }

lib/feed/scoreboard.md

@@ -7,4 +7,5 @@ _Generated by `lib/feed/conformance.sh`_
 | basic | 30 | 0 | 30 |
 | fanout | 29 | 0 | 29 |
 | rank | 24 | 0 | 24 |
-| **Total** | **83** | **0** | **83** |
+| integration | 22 | 0 | 22 |
+| **Total** | **105** | **0** | **105** |

lib/feed/tests/integration.sx

@@ -0,0 +1,155 @@
+; Phase 4 — visibility (ACL) + federation, and the end-to-end timeline.
+; (feed-test name got expected)
+
+; ---------- ACL visibility ----------
+; pub: public. sec: bob, allows carol. dm: frank, allows dave.
+
+(define
+  C
+  (feed/stream
+    (list
+      (feed/normalize {:actor "alice" :object "pub" :at 10})
+      (feed/normalize {:actor "bob" :object "sec" :visible-to (list "carol") :at 20})
+      (feed/normalize {:actor "frank" :object "dm" :visible-to (list "dave") :at 30}))))
+
+(feed-test
+  "public visible to anyone"
+  (feed/count (feed/visible C "zoe" feed/permit-acl?))
+  1)
+(feed-test
+  "carol sees allowlisted + public"
+  (feed/count (feed/visible C "carol" feed/permit-acl?))
+  2)
+(feed-test
+  "dave sees dm + public"
+  (feed/count (feed/visible C "dave" feed/permit-acl?))
+  2)
+(feed-test
+  "author always sees own private"
+  (feed/count (feed/visible C "frank" feed/permit-acl?))
+  2)
+(feed-test
+  "permit-public? lets all through"
+  (feed/count (feed/visible C "zoe" feed/permit-public?))
+  3)
+(feed-test
+  "visible objects for dave"
+  (map
+    (fn (a) (get a :object))
+    (feed/items (feed/visible C "dave" feed/permit-acl?)))
+  (list "pub" "dm"))
+
+; per-viewer: same stream, different timelines
+(feed-test
+  "zoe timeline differs from carol"
+  (not
+    (=
+      (feed/count (feed/visible C "zoe" feed/permit-acl?))
+      (feed/count (feed/visible C "carol" feed/permit-acl?))))
+  true)
+
+; ---------- federation: merge / ingest ----------
+
+(define
+  L
+  (feed/stream
+    (list
+      (feed/activity "alice" "post" "p1" 10 (list))
+      (feed/activity "alice" "post" "p2" 20 (list)))))
+(define
+  P
+  (feed/stream
+    (list
+      (feed/activity "alice" "post" "p2" 20 (list))
+      (feed/activity "peer" "post" "p9" 25 (list)))))
+
+(feed-test "merge concatenates" (feed/count (feed/merge L P)) 4)
+(feed-test
+  "ingest dedupes overlap"
+  (feed/count (feed/ingest L P))
+  3)
+
+(feed-test
+  "inbound normalizes + ingests"
+  (feed/count (feed/inbound L (list {:actor "peer" :object "p9" :at 25} {:actor "alice" :object "p1" :at 10})))
+  3)
+
+; backfill via injected fetch-fn
+(define peer-history (fn (peer-id) (list {:actor peer-id :object "h1" :at 1} {:actor peer-id :object "h2" :at 2})))
+(feed-test
+  "backfill merges peer history"
+  (feed/count (feed/backfill L peer-history "remote"))
+  4)
+(feed-test
+  "backfill objects present"
+  (map
+    (fn (a) (get a :object))
+    (feed/items
+      (feed/by-actor (feed/backfill L peer-history "remote") "remote")))
+  (list "h1" "h2"))
+
+; ---------- federation: outbound partition ----------
+
+; bob (local), alice@remote + carol@remote (remote) follow star
+(define
+  Gf
+  (feed/follow-graph
+    (list
+      (list "bob" "star")
+      (list "alice@remote" "star")
+      (list "carol@remote" "star"))))
+(define
+  Sf
+  (feed/stream (list (feed/activity "star" "post" "s1" 1 (list)))))
+(define
+  remote?
+  (fn (id) (feed/-elem? id (list "alice@remote" "carol@remote"))))
+(define parts (feed/federate Sf Gf remote?))
+
+(feed-test "local deliveries" (feed/count (get parts :local)) 1)
+(feed-test "remote deliveries" (feed/count (get parts :remote)) 2)
+(feed-test
+  "local recipient is bob"
+  (feed/recipients (get parts :local))
+  (list "bob"))
+
+; deliver: send-fn receives each remote event, local inbox returned
+(define sent (list))
+(define send-fn (fn (to act) (set! sent (append sent (list to)))))
+(define local-inbox (feed/deliver Sf Gf remote? send-fn))
+(feed-test "deliver returns local inbox" (feed/count local-inbox) 1)
+(feed-test "deliver sent to both remotes" (len sent) 2)
+(feed-test "deliver remote targets" sent (list "alice@remote" "carol@remote"))
+
+; ---------- end-to-end: federated, ACL-filtered, ranked timeline ----------
+
+(define
+  base
+  (feed/stream
+    (list
+      (feed/normalize {:actor "alice" :object "a1" :at 100})
+      (feed/normalize {:actor "bob" :object "b1" :visible-to (list "carol") :at 90})
+      (feed/normalize {:actor "eve" :object "e1" :visible-to (list "dave") :at 80}))))
+(define federated (feed/inbound base (list {:actor "peer" :object "x1" :at 110})))
+(define rec (feed/recency 120 10))
+(define
+  carol-tl
+  (feed/timeline federated "carol" feed/permit-acl? rec 3))
+
+; eve's :visible-to excludes carol -> filtered out; peer/alice public, bob allows carol
+(feed-test "carol federated timeline count" (feed/count carol-tl) 3)
+(feed-test
+  "carol timeline order (recency)"
+  (map (fn (a) (get a :object)) (feed/items carol-tl))
+  (list "x1" "a1" "b1"))
+(feed-test
+  "eve dm excluded from carol"
+  (feed/-elem? "e1" (map (fn (a) (get a :object)) (feed/items carol-tl)))
+  false)
+(feed-test
+  "dave sees eve dm not bob"
+  (map
+    (fn (a) (get a :object))
+    (feed/items
+      (feed/timeline federated "dave" feed/permit-acl? rec 5)))
+  (list "x1" "a1" "e1"))

plans/feed-on-sx.md

@@ -14,7 +14,7 @@ APL, ACL visibility filtering via `lib/acl/`, federation via fed-sx.
 
 ## Status (rolling)
 
-`bash lib/feed/conformance.sh` → **83/83** (Phases 1–3 complete)
+`bash lib/feed/conformance.sh` → **105/105** (Phases 1–4 complete)
 
 ## Ground rules
 
@@ -92,12 +92,19 @@ lib/feed/api.sx           lib/feed/fed.sx
 
 ## Phase 4 — Visibility filter + federation
 
-- [ ] ACL filter — each candidate activity passed through `(acl/permit? viewer :read
-  activity)`
-- [ ] fed-sx outbound — local `feed/post` fans out to remote followers' inboxes
-- [ ] fed-sx inbound — peer activities arrive at local inbox
-- [ ] backfill on subscribe — request peer history, merge into local stream
-- [ ] `lib/feed/tests/integration.sx` — federated timeline with ACL applied
+`lib/acl/` and fed-sx don't exist yet and are out of scope (import `lib/apl/`
+only), so ACL/transport are injected: `permit?`, `remote?`, `send-fn`, `fetch-fn`
+are function parameters. Real acl-sx / fed-sx wire in at the call site unchanged.
+
+- [x] ACL filter — `feed/visible stream viewer permit?`; default `feed/permit-acl?`
+  reads `:visible-to` allowlist (+ author-sees-own); per-viewer, never cached
+- [x] fed-sx outbound — `feed/federate`/`feed/deliver` fan out then partition
+  local vs remote inboxes; remote events handed to injected `send-fn`
+- [x] fed-sx inbound — `feed/inbound` normalizes + `feed/ingest` dedupes peer
+  activities into the local stream
+- [x] backfill on subscribe — `feed/backfill local fetch-fn peer-id`
+- [x] `lib/feed/tests/integration.sx` — 22 cases incl. end-to-end
+  `feed/timeline` (federated → ACL for viewer → recency rank → top-N)
 
 ## Progress log
 
@@ -120,6 +127,19 @@ lib/feed/api.sx           lib/feed/fed.sx
   single-arg ascending only — no comparator — so ranking uses a stable two-pass
   `apl-grade-down` (by :at desc, then by score desc) for deterministic tie-breaks.
   Dict keys must be strings, so composite group keys are string-joined ("actor#day").
+- **Phase 4 done (105/105 total).** `acl.sx` (per-viewer `feed/visible`,
+  `feed/timeline` capstone) + `fed.sx` (merge/ingest/inbound/backfill/federate/
+  deliver). ACL/transport are dependency-injected (permit?/remote?/send-fn/fetch-fn)
+  since lib/acl + fed-sx don't exist. `feed/normalize` now MERGEs defaults over the
+  raw dict (was projecting to 5 keys) so extra metadata (:visible-to, peer fields)
+  survives — matches the "flexible bag" principle.
+
+## Roadmap is complete (all 4 phases). Possible follow-ups:
+
+- Wire real acl-sx once `lib/acl/` exists (swap injected `permit?`).
+- Wire real fed-sx transport (swap `send-fn`/`fetch-fn`).
+- TF-IDF over `:tags` for content ranking (sketch mentions it; not yet built).
+- Notification feed (verb-filtered, per-recipient) as a thin layer over fanout.
 
 (none)