From 37226cf6ebcc8c09f6c476a1c4c6713ec9c21c10 Mon Sep 17 00:00:00 2001 From: giles Date: Sat, 6 Jun 2026 16:48:27 +0000 Subject: [PATCH] =?UTF-8?q?feed:=20Phase=204=20visibility=20+=20federation?= =?UTF-8?q?=20=E2=80=94=20per-viewer=20ACL,=20fanout=20partition,=20inboun?= =?UTF-8?q?d/backfill/ingest,=20e2e=20feed/timeline=20+=2022=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/feed/acl.sx | 38 +++++++++ lib/feed/conformance.sh | 4 +- lib/feed/fed.sx | 60 +++++++++++++ lib/feed/normalize.sx | 8 +- lib/feed/scoreboard.json | 7 +- lib/feed/scoreboard.md | 3 +- lib/feed/tests/integration.sx | 155 ++++++++++++++++++++++++++++++++++ plans/feed-on-sx.md | 34 ++++++-- 8 files changed, 295 insertions(+), 14 deletions(-) create mode 100644 lib/feed/acl.sx create mode 100644 lib/feed/fed.sx create mode 100644 lib/feed/tests/integration.sx diff --git a/lib/feed/acl.sx b/lib/feed/acl.sx new file mode 100644 index 00000000..5ecfc150 --- /dev/null +++ b/lib/feed/acl.sx @@ -0,0 +1,38 @@ +; feed/acl — per-viewer visibility filtering. The same candidate stream yields +; different timelines for different viewers, so ACL is applied per request and +; pre-ACL timelines are never cached. +; +; permit? is injected: (permit? viewer activity) -> bool. Wire a real acl-sx +; predicate here; feed/permit-acl? is a self-contained default that reads an +; optional :visible-to allowlist on the activity. +; +; Requires: lib/feed/normalize.sx, lib/feed/stream.sx, lib/feed/fanout.sx +; (feed/-elem?), lib/feed/rank.sx (feed/top). + +; default permit: actor always sees own activity; absent/nil :visible-to is +; public; otherwise viewer must be in the allowlist. +(define + feed/permit-acl? + (fn + (viewer a) + (or + (equal? viewer (get a :actor)) + (let + ((allowed (get a :visible-to nil))) + (if (= allowed nil) true (feed/-elem? viewer allowed)))))) + +(define feed/permit-public? (fn (viewer a) true)) + +; filter a stream to what viewer may read +(define + feed/visible + (fn + (stream viewer permit?) + (feed/filter stream (fn (a) (permit? viewer a))))) + +; the capstone: candidate stream -> ACL for viewer -> rank -> top-N +(define + feed/timeline + (fn + (stream viewer permit? score-fn n) + (feed/top (feed/visible stream viewer permit?) score-fn n))) diff --git a/lib/feed/conformance.sh b/lib/feed/conformance.sh index eb51dc88..d4a37cd5 100755 --- a/lib/feed/conformance.sh +++ b/lib/feed/conformance.sh @@ -13,7 +13,7 @@ if [ ! -x "$SX_SERVER" ]; then exit 1 fi -SUITES=(basic fanout rank) +SUITES=(basic fanout rank integration) OUT_JSON="lib/feed/scoreboard.json" OUT_MD="lib/feed/scoreboard.md" @@ -35,6 +35,8 @@ run_suite() { (load "lib/feed/dedupe.sx") (load "lib/feed/aggregate.sx") (load "lib/feed/rank.sx") +(load "lib/feed/acl.sx") +(load "lib/feed/fed.sx") (epoch 2) (eval "(define feed-test-pass 0)") (eval "(define feed-test-fail 0)") diff --git a/lib/feed/fed.sx b/lib/feed/fed.sx new file mode 100644 index 00000000..f5ff7543 --- /dev/null +++ b/lib/feed/fed.sx @@ -0,0 +1,60 @@ +; feed/fed — federation. Outbound: a local post fans out, then splits into local +; vs remote inboxes; remote events are handed to an injected send-fn. Inbound: +; peer activities merge into the local stream, deduped. Backfill: pull peer +; history via an injected fetch-fn and merge. +; +; remote? / send-fn / fetch-fn are injected so real fed-sx transport wires in here +; without feed depending on it. +; +; Requires: lib/feed/normalize.sx, lib/feed/stream.sx, lib/feed/fanout.sx, +; lib/feed/dedupe.sx. + +; --- merge / ingest --------------------------------------------------------- + +(define + feed/merge + (fn (s1 s2) (feed/stream (append (feed/items s1) (feed/items s2))))) + +; merge a peer stream into local, dropping (actor verb object) duplicates +(define + feed/ingest + (fn (local peer) (feed/dedupe-activities (feed/merge local peer)))) + +; --- inbound ---------------------------------------------------------------- + +; peer pushes raw activities to the local inbox; normalize + ingest +(define + feed/inbound + (fn + (local raw-activities) + (feed/ingest local (feed/stream (map feed/normalize raw-activities))))) + +; backfill on subscribe: pull peer history via fetch-fn, normalize, ingest +(define + feed/backfill + (fn (local fetch-fn peer-id) (feed/inbound local (fetch-fn peer-id)))) + +; --- outbound --------------------------------------------------------------- + +; split an inbox into local vs remote deliveries by viewer-id predicate +(define feed/partition-inbox (fn (inbox remote?) {:local (feed/filter inbox (fn (ev) (not (remote? (get ev :to))))) :remote (feed/filter inbox (fn (ev) (remote? (get ev :to))))})) + +; fan a stream out over the graph, then partition by locality +(define + feed/federate + (fn + (stream graph remote?) + (feed/partition-inbox (feed/fanout stream graph) remote?))) + +; deliver: hand each remote event to send-fn, return the local inbox to enqueue +(define + feed/deliver + (fn + (stream graph remote? send-fn) + (let + ((parts (feed/federate stream graph remote?))) + (begin + (for-each + (fn (ev) (send-fn (get ev :to) (get ev :activity))) + (feed/items (get parts :remote))) + (get parts :local))))) diff --git a/lib/feed/normalize.sx b/lib/feed/normalize.sx index d121963a..b2e3abc2 100644 --- a/lib/feed/normalize.sx +++ b/lib/feed/normalize.sx @@ -1,6 +1,8 @@ ; feed/normalize — coerce arbitrary input into the canonical activity record. ; An activity is a small dict {:actor :verb :object :at :tags}; a stream is an -; APL vector of such dicts (see stream.sx). +; APL vector of such dicts (see stream.sx). Extra keys on the raw input survive +; (e.g. :visible-to for ACL, peer metadata for federation) — :tags is the +; flexible bag but the record is not closed. (define feed/activity-keys (list :actor :verb :object :at :tags)) @@ -8,7 +10,9 @@ feed/normalize (fn (raw) - (let ((d (if (= (type-of raw) "dict") raw {}))) {:actor (get d :actor "") :object (get d :object nil) :at (get d :at 0) :tags (let ((t (get d :tags (list)))) (if (list? t) t (list t))) :verb (get d :verb "post")}))) + (let + ((d (if (= (type-of raw) "dict") raw {}))) + (merge d {:actor (get d :actor "") :object (get d :object nil) :at (get d :at 0) :tags (let ((t (get d :tags (list)))) (if (list? t) t (list t))) :verb (get d :verb "post")})))) (define feed/activity diff --git a/lib/feed/scoreboard.json b/lib/feed/scoreboard.json index df5bd7e4..ac5682db 100644 --- a/lib/feed/scoreboard.json +++ b/lib/feed/scoreboard.json @@ -2,9 +2,10 @@ "suites": { "basic": {"pass": 30, "fail": 0}, "fanout": {"pass": 29, "fail": 0}, - "rank": {"pass": 24, "fail": 0} + "rank": {"pass": 24, "fail": 0}, + "integration": {"pass": 22, "fail": 0} }, - "total_pass": 83, + "total_pass": 105, "total_fail": 0, - "total": 83 + "total": 105 } diff --git a/lib/feed/scoreboard.md b/lib/feed/scoreboard.md index feb827a8..dba890ec 100644 --- a/lib/feed/scoreboard.md +++ b/lib/feed/scoreboard.md @@ -7,4 +7,5 @@ _Generated by `lib/feed/conformance.sh`_ | basic | 30 | 0 | 30 | | fanout | 29 | 0 | 29 | | rank | 24 | 0 | 24 | -| **Total** | **83** | **0** | **83** | +| integration | 22 | 0 | 22 | +| **Total** | **105** | **0** | **105** | diff --git a/lib/feed/tests/integration.sx b/lib/feed/tests/integration.sx new file mode 100644 index 00000000..08ead747 --- /dev/null +++ b/lib/feed/tests/integration.sx @@ -0,0 +1,155 @@ +; Phase 4 — visibility (ACL) + federation, and the end-to-end timeline. +; (feed-test name got expected) + +; ---------- ACL visibility ---------- +; pub: public. sec: bob, allows carol. dm: frank, allows dave. + +(define + C + (feed/stream + (list + (feed/normalize {:actor "alice" :object "pub" :at 10}) + (feed/normalize {:actor "bob" :object "sec" :visible-to (list "carol") :at 20}) + (feed/normalize {:actor "frank" :object "dm" :visible-to (list "dave") :at 30})))) + +(feed-test + "public visible to anyone" + (feed/count (feed/visible C "zoe" feed/permit-acl?)) + 1) +(feed-test + "carol sees allowlisted + public" + (feed/count (feed/visible C "carol" feed/permit-acl?)) + 2) +(feed-test + "dave sees dm + public" + (feed/count (feed/visible C "dave" feed/permit-acl?)) + 2) +(feed-test + "author always sees own private" + (feed/count (feed/visible C "frank" feed/permit-acl?)) + 2) +(feed-test + "permit-public? lets all through" + (feed/count (feed/visible C "zoe" feed/permit-public?)) + 3) +(feed-test + "visible objects for dave" + (map + (fn (a) (get a :object)) + (feed/items (feed/visible C "dave" feed/permit-acl?))) + (list "pub" "dm")) + +; per-viewer: same stream, different timelines +(feed-test + "zoe timeline differs from carol" + (not + (= + (feed/count (feed/visible C "zoe" feed/permit-acl?)) + (feed/count (feed/visible C "carol" feed/permit-acl?)))) + true) + +; ---------- federation: merge / ingest ---------- + +(define + L + (feed/stream + (list + (feed/activity "alice" "post" "p1" 10 (list)) + (feed/activity "alice" "post" "p2" 20 (list))))) +(define + P + (feed/stream + (list + (feed/activity "alice" "post" "p2" 20 (list)) + (feed/activity "peer" "post" "p9" 25 (list))))) + +(feed-test "merge concatenates" (feed/count (feed/merge L P)) 4) +(feed-test + "ingest dedupes overlap" + (feed/count (feed/ingest L P)) + 3) + +(feed-test + "inbound normalizes + ingests" + (feed/count (feed/inbound L (list {:actor "peer" :object "p9" :at 25} {:actor "alice" :object "p1" :at 10}))) + 3) + +; backfill via injected fetch-fn +(define peer-history (fn (peer-id) (list {:actor peer-id :object "h1" :at 1} {:actor peer-id :object "h2" :at 2}))) +(feed-test + "backfill merges peer history" + (feed/count (feed/backfill L peer-history "remote")) + 4) +(feed-test + "backfill objects present" + (map + (fn (a) (get a :object)) + (feed/items + (feed/by-actor (feed/backfill L peer-history "remote") "remote"))) + (list "h1" "h2")) + +; ---------- federation: outbound partition ---------- + +; bob (local), alice@remote + carol@remote (remote) follow star +(define + Gf + (feed/follow-graph + (list + (list "bob" "star") + (list "alice@remote" "star") + (list "carol@remote" "star")))) +(define + Sf + (feed/stream (list (feed/activity "star" "post" "s1" 1 (list))))) +(define + remote? + (fn (id) (feed/-elem? id (list "alice@remote" "carol@remote")))) +(define parts (feed/federate Sf Gf remote?)) + +(feed-test "local deliveries" (feed/count (get parts :local)) 1) +(feed-test "remote deliveries" (feed/count (get parts :remote)) 2) +(feed-test + "local recipient is bob" + (feed/recipients (get parts :local)) + (list "bob")) + +; deliver: send-fn receives each remote event, local inbox returned +(define sent (list)) +(define send-fn (fn (to act) (set! sent (append sent (list to))))) +(define local-inbox (feed/deliver Sf Gf remote? send-fn)) +(feed-test "deliver returns local inbox" (feed/count local-inbox) 1) +(feed-test "deliver sent to both remotes" (len sent) 2) +(feed-test "deliver remote targets" sent (list "alice@remote" "carol@remote")) + +; ---------- end-to-end: federated, ACL-filtered, ranked timeline ---------- + +(define + base + (feed/stream + (list + (feed/normalize {:actor "alice" :object "a1" :at 100}) + (feed/normalize {:actor "bob" :object "b1" :visible-to (list "carol") :at 90}) + (feed/normalize {:actor "eve" :object "e1" :visible-to (list "dave") :at 80})))) +(define federated (feed/inbound base (list {:actor "peer" :object "x1" :at 110}))) +(define rec (feed/recency 120 10)) +(define + carol-tl + (feed/timeline federated "carol" feed/permit-acl? rec 3)) + +; eve's :visible-to excludes carol -> filtered out; peer/alice public, bob allows carol +(feed-test "carol federated timeline count" (feed/count carol-tl) 3) +(feed-test + "carol timeline order (recency)" + (map (fn (a) (get a :object)) (feed/items carol-tl)) + (list "x1" "a1" "b1")) +(feed-test + "eve dm excluded from carol" + (feed/-elem? "e1" (map (fn (a) (get a :object)) (feed/items carol-tl))) + false) +(feed-test + "dave sees eve dm not bob" + (map + (fn (a) (get a :object)) + (feed/items + (feed/timeline federated "dave" feed/permit-acl? rec 5))) + (list "x1" "a1" "e1")) diff --git a/plans/feed-on-sx.md b/plans/feed-on-sx.md index 8dc48552..860c2427 100644 --- a/plans/feed-on-sx.md +++ b/plans/feed-on-sx.md @@ -14,7 +14,7 @@ APL, ACL visibility filtering via `lib/acl/`, federation via fed-sx. ## Status (rolling) -`bash lib/feed/conformance.sh` → **83/83** (Phases 1–3 complete) +`bash lib/feed/conformance.sh` → **105/105** (Phases 1–4 complete) ## Ground rules @@ -92,12 +92,19 @@ lib/feed/api.sx lib/feed/fed.sx ## Phase 4 — Visibility filter + federation -- [ ] ACL filter — each candidate activity passed through `(acl/permit? viewer :read - activity)` -- [ ] fed-sx outbound — local `feed/post` fans out to remote followers' inboxes -- [ ] fed-sx inbound — peer activities arrive at local inbox -- [ ] backfill on subscribe — request peer history, merge into local stream -- [ ] `lib/feed/tests/integration.sx` — federated timeline with ACL applied +`lib/acl/` and fed-sx don't exist yet and are out of scope (import `lib/apl/` +only), so ACL/transport are injected: `permit?`, `remote?`, `send-fn`, `fetch-fn` +are function parameters. Real acl-sx / fed-sx wire in at the call site unchanged. + +- [x] ACL filter — `feed/visible stream viewer permit?`; default `feed/permit-acl?` + reads `:visible-to` allowlist (+ author-sees-own); per-viewer, never cached +- [x] fed-sx outbound — `feed/federate`/`feed/deliver` fan out then partition + local vs remote inboxes; remote events handed to injected `send-fn` +- [x] fed-sx inbound — `feed/inbound` normalizes + `feed/ingest` dedupes peer + activities into the local stream +- [x] backfill on subscribe — `feed/backfill local fetch-fn peer-id` +- [x] `lib/feed/tests/integration.sx` — 22 cases incl. end-to-end + `feed/timeline` (federated → ACL for viewer → recency rank → top-N) ## Progress log @@ -120,6 +127,19 @@ lib/feed/api.sx lib/feed/fed.sx single-arg ascending only — no comparator — so ranking uses a stable two-pass `apl-grade-down` (by :at desc, then by score desc) for deterministic tie-breaks. Dict keys must be strings, so composite group keys are string-joined ("actor#day"). +- **Phase 4 done (105/105 total).** `acl.sx` (per-viewer `feed/visible`, + `feed/timeline` capstone) + `fed.sx` (merge/ingest/inbound/backfill/federate/ + deliver). ACL/transport are dependency-injected (permit?/remote?/send-fn/fetch-fn) + since lib/acl + fed-sx don't exist. `feed/normalize` now MERGEs defaults over the + raw dict (was projecting to 5 keys) so extra metadata (:visible-to, peer fields) + survives — matches the "flexible bag" principle. + +## Roadmap is complete (all 4 phases). Possible follow-ups: + +- Wire real acl-sx once `lib/acl/` exists (swap injected `permit?`). +- Wire real fed-sx transport (swap `send-fn`/`fetch-fn`). +- TF-IDF over `:tags` for content ranking (sketch mentions it; not yet built). +- Notification feed (verb-filtered, per-recipient) as a thin layer over fanout. (none)