diff --git a/lib/feed/acl.sx b/lib/feed/acl.sx new file mode 100644 index 00000000..5ecfc150 --- /dev/null +++ b/lib/feed/acl.sx @@ -0,0 +1,38 @@ +; feed/acl — per-viewer visibility filtering. The same candidate stream yields +; different timelines for different viewers, so ACL is applied per request and +; pre-ACL timelines are never cached. +; +; permit? is injected: (permit? viewer activity) -> bool. Wire a real acl-sx +; predicate here; feed/permit-acl? is a self-contained default that reads an +; optional :visible-to allowlist on the activity. +; +; Requires: lib/feed/normalize.sx, lib/feed/stream.sx, lib/feed/fanout.sx +; (feed/-elem?), lib/feed/rank.sx (feed/top). + +; default permit: actor always sees own activity; absent/nil :visible-to is +; public; otherwise viewer must be in the allowlist. +(define + feed/permit-acl? + (fn + (viewer a) + (or + (equal? viewer (get a :actor)) + (let + ((allowed (get a :visible-to nil))) + (if (= allowed nil) true (feed/-elem? viewer allowed)))))) + +(define feed/permit-public? (fn (viewer a) true)) + +; filter a stream to what viewer may read +(define + feed/visible + (fn + (stream viewer permit?) + (feed/filter stream (fn (a) (permit? viewer a))))) + +; the capstone: candidate stream -> ACL for viewer -> rank -> top-N +(define + feed/timeline + (fn + (stream viewer permit? score-fn n) + (feed/top (feed/visible stream viewer permit?) score-fn n))) diff --git a/lib/feed/aggregate.sx b/lib/feed/aggregate.sx new file mode 100644 index 00000000..cc146cd3 --- /dev/null +++ b/lib/feed/aggregate.sx @@ -0,0 +1,62 @@ +; feed/aggregate — group-by / counting via key-reduce. Keys must be strings +; (dict keys), so composite keys (actor, day) are joined into one string. +; +; Requires: lib/feed/normalize.sx, lib/feed/stream.sx. + +; group activities into a dict: key-string -> (list of activities), order-preserving +(define + feed/group-by + (fn + (stream key-fn) + (reduce + (fn + (g a) + (let + ((k (key-fn a))) + (assoc g k (append (get g k (list)) (list a))))) + {} + (feed/items stream)))) + +; key-string -> count +(define + feed/group-count + (fn + (stream key-fn) + (reduce + (fn + (g a) + (let + ((k (key-fn a))) + (assoc g k (+ (get g k 0) 1)))) + {} + (feed/items stream)))) + +; --- composite keys --------------------------------------------------------- + +(define feed/day (fn (at window) (floor (/ at window)))) + +; (actor, day-bucket) -> "actor#day" +(define + feed/actor-day-key + (fn + (window) + (fn + (a) + (string-append + (get a :actor) + "#" + (number->string (feed/day (get a :at) window)))))) + +(define + feed/by-actor-day + (fn (stream window) (feed/group-count stream (feed/actor-day-key window)))) + +; per-actor activity counts +(define + feed/actor-counts + (fn (stream) (feed/group-count stream feed/actor))) + +; per-object activity counts (engagement) +(define + feed/object-counts + (fn (stream) (feed/group-count stream feed/object))) diff --git a/lib/feed/api.sx b/lib/feed/api.sx new file mode 100644 index 00000000..0233e6dd --- /dev/null +++ b/lib/feed/api.sx @@ -0,0 +1,24 @@ +; feed/api — ergonomic API over the stream layer for non-APL callers. +; A single mutable activity log; post appends, all returns it as a stream. +; +; Requires: lib/feed/normalize.sx, lib/feed/stream.sx (loaded by harness). + +(define feed/-log (list)) + +; post — normalize then append. Returns the stored activity. +(define + feed/post + (fn + (raw) + (let + ((a (feed/normalize raw))) + (begin (set! feed/-log (append feed/-log (list a))) a)))) + +; all — the whole log as a stream (insertion order) +(define feed/all (fn () (feed/stream feed/-log))) + +; reset! — clear the log (test hygiene) +(define feed/reset! (fn () (begin (set! feed/-log (list)) nil))) + +; size — number of posted activities +(define feed/size (fn () (len feed/-log))) diff --git a/lib/feed/conformance.sh b/lib/feed/conformance.sh new file mode 100755 index 00000000..75f1e32e --- /dev/null +++ b/lib/feed/conformance.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash +# lib/feed/conformance.sh — run feed test suites, emit scoreboard.json + scoreboard.md. + +set -uo pipefail +cd "$(git rev-parse --show-toplevel)" + +SX_SERVER="${SX_SERVER:-/root/rose-ash/hosts/ocaml/_build/default/bin/sx_server.exe}" +if [ ! -x "$SX_SERVER" ]; then + SX_SERVER="hosts/ocaml/_build/default/bin/sx_server.exe" +fi +if [ ! -x "$SX_SERVER" ]; then + echo "ERROR: sx_server.exe not found." >&2 + exit 1 +fi + +SUITES=(basic fanout rank integration content notify home dedupe trending mute page thread) + +OUT_JSON="lib/feed/scoreboard.json" +OUT_MD="lib/feed/scoreboard.md" + +run_suite() { + local suite=$1 + local file="lib/feed/tests/${suite}.sx" + local TMP + TMP=$(mktemp) + cat > "$TMP" << EPOCHS +(epoch 1) +(load "spec/stdlib.sx") +(load "lib/r7rs.sx") +(load "lib/apl/runtime.sx") +(load "lib/feed/normalize.sx") +(load "lib/feed/stream.sx") +(load "lib/feed/api.sx") +(load "lib/feed/fanout.sx") +(load "lib/feed/dedupe.sx") +(load "lib/feed/aggregate.sx") +(load "lib/feed/rank.sx") +(load "lib/feed/acl.sx") +(load "lib/feed/fed.sx") +(load "lib/feed/content.sx") +(load "lib/feed/notify.sx") +(load "lib/feed/home.sx") +(load "lib/feed/trending.sx") +(load "lib/feed/mute.sx") +(load "lib/feed/page.sx") +(load "lib/feed/thread.sx") +(epoch 2) +(eval "(define feed-test-pass 0)") +(eval "(define feed-test-fail 0)") +(eval "(define feed-test (fn (name got expected) (if (= got expected) (set! feed-test-pass (+ feed-test-pass 1)) (set! feed-test-fail (+ feed-test-fail 1)))))") +(epoch 3) +(load "${file}") +(epoch 4) +(eval "(list feed-test-pass feed-test-fail)") +EPOCHS + + local OUTPUT + OUTPUT=$(timeout 300 "$SX_SERVER" < "$TMP" 2>/dev/null) + rm -f "$TMP" + + local LINE + LINE=$(echo "$OUTPUT" | awk '/^\(ok-len 4 / {getline; print; exit}') + if [ -z "$LINE" ]; then + LINE=$(echo "$OUTPUT" | grep -E '^\(ok 4 \([0-9]+ [0-9]+\)\)' | tail -1 \ + | sed -E 's/^\(ok 4 //; s/\)$//') + fi + + local P F + P=$(echo "$LINE" | sed -E 's/^\(([0-9]+) ([0-9]+)\).*/\1/') + F=$(echo "$LINE" | sed -E 's/^\(([0-9]+) ([0-9]+)\).*/\2/') + P=${P:-0} + F=${F:-0} + echo "${P} ${F}" +} + +declare -A SUITE_PASS +declare -A SUITE_FAIL +TOTAL_PASS=0 +TOTAL_FAIL=0 + +echo "Running feed conformance suite..." >&2 +for s in "${SUITES[@]}"; do + read -r p f < <(run_suite "$s") + SUITE_PASS[$s]=$p + SUITE_FAIL[$s]=$f + TOTAL_PASS=$((TOTAL_PASS + p)) + TOTAL_FAIL=$((TOTAL_FAIL + f)) + printf " %-12s %d/%d\n" "$s" "$p" "$((p+f))" >&2 +done + +# scoreboard.json +{ + printf '{\n' + printf ' "suites": {\n' + first=1 + for s in "${SUITES[@]}"; do + if [ $first -eq 0 ]; then printf ',\n'; fi + printf ' "%s": {"pass": %d, "fail": %d}' "$s" "${SUITE_PASS[$s]}" "${SUITE_FAIL[$s]}" + first=0 + done + printf '\n },\n' + printf ' "total_pass": %d,\n' "$TOTAL_PASS" + printf ' "total_fail": %d,\n' "$TOTAL_FAIL" + printf ' "total": %d\n' "$((TOTAL_PASS + TOTAL_FAIL))" + printf '}\n' +} > "$OUT_JSON" + +# scoreboard.md +{ + printf '# feed Conformance Scoreboard\n\n' + printf '_Generated by `lib/feed/conformance.sh`_\n\n' + printf '| Suite | Pass | Fail | Total |\n' + printf '|-------|-----:|-----:|------:|\n' + for s in "${SUITES[@]}"; do + p=${SUITE_PASS[$s]} + f=${SUITE_FAIL[$s]} + printf '| %s | %d | %d | %d |\n' "$s" "$p" "$f" "$((p+f))" + done + printf '| **Total** | **%d** | **%d** | **%d** |\n' "$TOTAL_PASS" "$TOTAL_FAIL" "$((TOTAL_PASS + TOTAL_FAIL))" +} > "$OUT_MD" + +echo "Wrote $OUT_JSON and $OUT_MD" >&2 +echo "Total: $TOTAL_PASS pass, $TOTAL_FAIL fail" >&2 + +[ "$TOTAL_FAIL" -eq 0 ] diff --git a/lib/feed/content.sx b/lib/feed/content.sx new file mode 100644 index 00000000..f15e8434 --- /dev/null +++ b/lib/feed/content.sx @@ -0,0 +1,68 @@ +; feed/content — TF-IDF relevance over activity :tags. Rare tags carry more +; signal, so an activity matching an uncommon tag ranks above one matching a +; common tag. Composes with rank.sx: feed/tfidf-score is just another scorer. +; +; Requires: lib/feed/normalize.sx, lib/feed/stream.sx, lib/feed/fanout.sx +; (feed/-distinct), lib/feed/rank.sx (feed/rank). + +; document frequency: tag -> number of activities whose :tags contain it +; (a tag repeated within one activity counts once toward df) +(define + feed/tag-df + (fn + (stream) + (reduce + (fn + (df a) + (reduce + (fn (d t) (assoc d t (+ (get d t 0) 1))) + df + (feed/-distinct (get a :tags)))) + {} + (feed/items stream)))) + +; inverse document frequency: tag -> log(N / df) +(define + feed/tag-idf + (fn + (stream) + (let + ((n (feed/count stream)) (df (feed/tag-df stream))) + (reduce + (fn (idf t) (assoc idf t (log (/ n (get df t))))) + {} + (keys df))))) + +; term frequency within one activity: tag -> occurrence count +(define + feed/-tf + (fn + (a) + (reduce + (fn (tf t) (assoc tf t (+ (get tf t 0) 1))) + {} + (get a :tags)))) + +; relevance of an activity to a query (list of tags) given precomputed idf: +; sum over query tags of tf(tag in activity) * idf(tag in corpus) +(define + feed/tfidf-score + (fn + (idf query) + (fn + (a) + (let + ((tf (feed/-tf a))) + (reduce + (fn + (acc t) + (+ acc (* (get tf t 0) (get idf t 0)))) + 0 + query))))) + +; rank a stream by relevance to query tags (idf computed over the stream itself) +(define + feed/by-relevance + (fn + (stream query) + (feed/rank stream (feed/tfidf-score (feed/tag-idf stream) query)))) diff --git a/lib/feed/dedupe.sx b/lib/feed/dedupe.sx new file mode 100644 index 00000000..12dd9a36 --- /dev/null +++ b/lib/feed/dedupe.sx @@ -0,0 +1,76 @@ +; feed/dedupe — collapse duplicate items, keeping first occurrence per key. +; Each verb may want its own key (see briefing): "alice posted X" keys on +; (actor verb object) — distinct per actor; "alice liked X / bob liked X" +; collapse on (verb object) so the cross-actor likes fold into one. +; +; Requires: lib/feed/normalize.sx, lib/feed/stream.sx, lib/feed/fanout.sx +; (feed/-elem? lives in fanout.sx). + +; generic: dedupe a stream by key-fn, first occurrence wins (stable) +(define + feed/-dedup-by + (fn + (items key-fn) + (get + (reduce + (fn + (st x) + (let + ((k (key-fn x))) + (if (feed/-elem? k (get st :seen)) st {:seen (append (get st :seen) (list k)) :out (append (get st :out) (list x))}))) + {:seen (list) :out (list)} + items) + :out))) + +(define + feed/dedupe + (fn + (stream key-fn) + (feed/stream (feed/-dedup-by (feed/items stream) key-fn)))) + +; --- keys ------------------------------------------------------------------- + +(define + feed/activity-key + (fn (a) (list (get a :actor) (get a :verb) (get a :object)))) + +; collapse cross-actor duplicates of the same verb+object (e.g. likes) +(define feed/collapse-key (fn (a) (list (get a :verb) (get a :object)))) + +; per-receiver inbox key — one inbox event per (receiver, actor, verb, object) +(define + feed/event-key + (fn + (ev) + (let + ((a (get ev :activity))) + (list (get ev :to) (get a :actor) (get a :verb) (get a :object))))) + +; verbs whose duplicates collapse across actors (reactions, not authorship). +; rebindable: callers can (set! feed/collapse-verbs ...) to tune the policy. +(define + feed/collapse-verbs + (list "like" "favourite" "follow" "boost" "repost")) + +; per-verb key: collapse-verbs fold on (verb object); the rest key on +; (actor verb object). +(define + feed/smart-key + (fn + (a) + (if + (feed/-elem? (get a :verb) feed/collapse-verbs) + (feed/collapse-key a) + (feed/activity-key a)))) + +; --- ready-made dedupers ---------------------------------------------------- + +(define feed/dedupe-activities (fn (s) (feed/dedupe s feed/activity-key))) + +(define feed/dedupe-collapse (fn (s) (feed/dedupe s feed/collapse-key))) + +; verb-aware: reactions collapse cross-actor, posts stay distinct per actor +(define feed/dedupe-smart (fn (s) (feed/dedupe s feed/smart-key))) + +; dedupe an inbox: at most one event per receiver per (actor verb object) +(define feed/dedupe-inbox (fn (inbox) (feed/dedupe inbox feed/event-key))) diff --git a/lib/feed/fanout.sx b/lib/feed/fanout.sx new file mode 100644 index 00000000..4d816e51 --- /dev/null +++ b/lib/feed/fanout.sx @@ -0,0 +1,114 @@ +; feed/fanout — THE SHOWCASE. Fan activities out to followers via the APL outer +; product (∘.×). activities ∘.× audience → an (activity × follower) matrix of +; inbox events; flatten to a vector; guard-keep only real follow edges. +; +; Requires: lib/apl/runtime.sx, lib/feed/normalize.sx, lib/feed/stream.sx. +; +; NOTE: apl-outer's combiner result is run through (if (scalar? r) (disclose r) r). +; A bare dict counts as a scalar (shape ()) and disclose nils it — so the combiner +; must (enclose ...) its event dict; apl-outer then discloses it back intact. + +; --- graph: {followee -> (list of followers)} ------------------------------- + +(define feed/followers (fn (graph user) (get graph user (list)))) + +; build a graph from (follower followee) edges: "follower follows followee" +(define + feed/follow-graph + (fn + (edges) + (reduce + (fn + (g e) + (let + ((follower (first e)) (followee (nth e 1))) + (assoc + g + followee + (append (feed/followers g followee) (list follower))))) + {} + edges))) + +; --- helpers ---------------------------------------------------------------- + +; unwrap an apl-scalar (has :ravel) back to its value; pass activities through +(define + feed/-val + (fn + (x) + (if (and (= (type-of x) "dict") (has-key? x :ravel)) (disclose x) x))) + +(define feed/-elem? (fn (x lst) (some (fn (y) (equal? x y)) lst))) + +(define + feed/-distinct + (fn + (lst) + (if + (= (len lst) 0) + (list) + (get (apl-unique (make-array (list (len lst)) lst)) :ravel)))) + +; rank-2 matrix -> rank-1 stream of its ravel +(define feed/-flatten (fn (arr) (feed/stream (get arr :ravel)))) + +; distinct receivers across the whole graph, sorted for determinism +; (dict key order is unspecified, so sort to pin audience/recipient ordering) +(define + feed/audience + (fn + (graph) + (sort + (feed/-distinct + (reduce + (fn (acc k) (append acc (feed/followers graph k))) + (list) + (keys graph)))))) + +; --- the outer product ------------------------------------------------------ + +; one (activity, follower) inbox event, enclosed so apl-outer keeps the dict +(define feed/-mk-event (fn (a f) (enclose {:activity (feed/-val a) :to (feed/-val f)}))) + +; keep events where :to actually follows the activity's actor +(define + feed/-edge? + (fn + (graph) + (fn + (ev) + (feed/-elem? + (get ev :to) + (feed/followers graph (get (get ev :activity) :actor)))))) + +; fanout — activities ∘.× audience, flatten, guard-keep real edges +(define + feed/fanout + (fn + (stream graph) + (let + ((matrix (apl-outer feed/-mk-event stream (feed/stream (feed/audience graph))))) + (feed/filter (feed/-flatten matrix) (feed/-edge? graph))))) + +; --- inbox queries ---------------------------------------------------------- + +(define + feed/inbox-for + (fn + (inbox user) + (feed/filter inbox (fn (ev) (equal? (get ev :to) user))))) + +(define + feed/recipients + (fn + (inbox) + (feed/-distinct (map (fn (ev) (get ev :to)) (feed/items inbox))))) + +; the activities (unwrapped) destined for a user +(define + feed/inbox-activities + (fn + (inbox user) + (map + (fn (ev) (get ev :activity)) + (feed/items (feed/inbox-for inbox user))))) diff --git a/lib/feed/fed.sx b/lib/feed/fed.sx new file mode 100644 index 00000000..f5ff7543 --- /dev/null +++ b/lib/feed/fed.sx @@ -0,0 +1,60 @@ +; feed/fed — federation. Outbound: a local post fans out, then splits into local +; vs remote inboxes; remote events are handed to an injected send-fn. Inbound: +; peer activities merge into the local stream, deduped. Backfill: pull peer +; history via an injected fetch-fn and merge. +; +; remote? / send-fn / fetch-fn are injected so real fed-sx transport wires in here +; without feed depending on it. +; +; Requires: lib/feed/normalize.sx, lib/feed/stream.sx, lib/feed/fanout.sx, +; lib/feed/dedupe.sx. + +; --- merge / ingest --------------------------------------------------------- + +(define + feed/merge + (fn (s1 s2) (feed/stream (append (feed/items s1) (feed/items s2))))) + +; merge a peer stream into local, dropping (actor verb object) duplicates +(define + feed/ingest + (fn (local peer) (feed/dedupe-activities (feed/merge local peer)))) + +; --- inbound ---------------------------------------------------------------- + +; peer pushes raw activities to the local inbox; normalize + ingest +(define + feed/inbound + (fn + (local raw-activities) + (feed/ingest local (feed/stream (map feed/normalize raw-activities))))) + +; backfill on subscribe: pull peer history via fetch-fn, normalize, ingest +(define + feed/backfill + (fn (local fetch-fn peer-id) (feed/inbound local (fetch-fn peer-id)))) + +; --- outbound --------------------------------------------------------------- + +; split an inbox into local vs remote deliveries by viewer-id predicate +(define feed/partition-inbox (fn (inbox remote?) {:local (feed/filter inbox (fn (ev) (not (remote? (get ev :to))))) :remote (feed/filter inbox (fn (ev) (remote? (get ev :to))))})) + +; fan a stream out over the graph, then partition by locality +(define + feed/federate + (fn + (stream graph remote?) + (feed/partition-inbox (feed/fanout stream graph) remote?))) + +; deliver: hand each remote event to send-fn, return the local inbox to enqueue +(define + feed/deliver + (fn + (stream graph remote? send-fn) + (let + ((parts (feed/federate stream graph remote?))) + (begin + (for-each + (fn (ev) (send-fn (get ev :to) (get ev :activity))) + (feed/items (get parts :remote))) + (get parts :local))))) diff --git a/lib/feed/home.sx b/lib/feed/home.sx new file mode 100644 index 00000000..a4b3eb3f --- /dev/null +++ b/lib/feed/home.sx @@ -0,0 +1,23 @@ +; feed/home — the capstone. A user's home timeline is the whole pipeline as one +; line: fan all activities out over the follow graph, take the events landing in +; the viewer's inbox, dedupe cross-posts, apply the viewer's ACL, rank, take N. +; +; Requires: fanout.sx, dedupe.sx, acl.sx (feed/timeline), rank.sx, stream.sx. + +; the activities in a user's inbox, as a stream +(define + feed/inbox-stream + (fn (inbox user) (feed/stream (feed/inbox-activities inbox user)))) + +; fanout ∘ inbox ∘ dedupe ∘ ACL ∘ rank ∘ take +(define + feed/home + (fn + (stream graph viewer permit? score-fn n) + (feed/timeline + (feed/dedupe-activities + (feed/inbox-stream (feed/fanout stream graph) viewer)) + viewer + permit? + score-fn + n))) diff --git a/lib/feed/mute.sx b/lib/feed/mute.sx new file mode 100644 index 00000000..1a8c5f59 --- /dev/null +++ b/lib/feed/mute.sx @@ -0,0 +1,44 @@ +; feed/mute — viewer-controlled filtering. ACL (acl.sx) is author-controlled +; visibility; mute is the reader's own preference: hide muted actors or tags. +; Like ACL it is per-viewer and applied per request, never cached. +; +; Requires: lib/feed/normalize.sx, lib/feed/stream.sx, lib/feed/fanout.sx +; (feed/-elem?). + +; drop activities authored by a muted actor +(define + feed/mute-actors + (fn + (stream actors) + (feed/filter + stream + (fn (a) (not (feed/-elem? (get a :actor) actors)))))) + +; drop activities carrying any muted tag +(define + feed/mute-tags + (fn + (stream tags) + (feed/filter + stream + (fn (a) (not (some (fn (t) (feed/-elem? t tags)) (get a :tags))))))) + +; drop activities about a muted object (thread mute) +(define + feed/mute-objects + (fn + (stream objects) + (feed/filter + stream + (fn (a) (not (feed/-elem? (get a :object) objects)))))) + +; apply a viewer preference bag: {:mute-actors (...) :mute-tags (...) :mute-objects (...)} +(define + feed/apply-prefs + (fn + (stream prefs) + (feed/mute-objects + (feed/mute-tags + (feed/mute-actors stream (get prefs :mute-actors (list))) + (get prefs :mute-tags (list))) + (get prefs :mute-objects (list))))) diff --git a/lib/feed/normalize.sx b/lib/feed/normalize.sx new file mode 100644 index 00000000..b2e3abc2 --- /dev/null +++ b/lib/feed/normalize.sx @@ -0,0 +1,31 @@ +; feed/normalize — coerce arbitrary input into the canonical activity record. +; An activity is a small dict {:actor :verb :object :at :tags}; a stream is an +; APL vector of such dicts (see stream.sx). Extra keys on the raw input survive +; (e.g. :visible-to for ACL, peer metadata for federation) — :tags is the +; flexible bag but the record is not closed. + +(define feed/activity-keys (list :actor :verb :object :at :tags)) + +(define + feed/normalize + (fn + (raw) + (let + ((d (if (= (type-of raw) "dict") raw {}))) + (merge d {:actor (get d :actor "") :object (get d :object nil) :at (get d :at 0) :tags (let ((t (get d :tags (list)))) (if (list? t) t (list t))) :verb (get d :verb "post")})))) + +(define + feed/activity + (fn (actor verb object at tags) (feed/normalize {:actor actor :object object :at at :tags tags :verb verb}))) + +(define feed/actor (fn (a) (get a :actor))) +(define feed/verb (fn (a) (get a :verb))) +(define feed/object (fn (a) (get a :object))) +(define feed/at (fn (a) (get a :at))) +(define feed/tags (fn (a) (get a :tags))) + +(define + feed/activity? + (fn + (a) + (and (= (type-of a) "dict") (has-key? a :actor) (has-key? a :verb)))) diff --git a/lib/feed/notify.sx b/lib/feed/notify.sx new file mode 100644 index 00000000..7c499af7 --- /dev/null +++ b/lib/feed/notify.sx @@ -0,0 +1,45 @@ +; feed/notify — a notification feed is a thin layer over a recipient's inbox: +; the events directed at a user, optionally verb-filtered, and a digest that +; collapses "alice, bob and 1 other liked X" by (verb, object). +; +; Requires: lib/feed/normalize.sx, lib/feed/stream.sx, lib/feed/fanout.sx +; (feed/inbox-for, feed/-elem?). + +; all inbox events for a user (their raw notifications) +(define feed/notifications (fn (inbox user) (feed/inbox-for inbox user))) + +; restrict to notification-worthy verbs (e.g. (list "like" "reply" "follow")) +(define + feed/notify-verbs + (fn + (inbox user verbs) + (feed/filter + (feed/inbox-for inbox user) + (fn (ev) (feed/-elem? (get (get ev :activity) :verb) verbs))))) + +; group key "verb|object" — deterministic, sortable +(define + feed/-notify-key + (fn + (ev) + (let + ((a (get ev :activity))) + (string-append (get a :verb) "|" (get a :object))))) + +; digest: one entry per (verb, object) with the distinct actors and a count, +; ordered by key for determinism. +(define + feed/notify-digest + (fn + (inbox user) + (let + ((events (feed/items (feed/inbox-for inbox user)))) + (let + ((groups (reduce (fn (g ev) (let ((a (get ev :activity)) (k (feed/-notify-key ev))) (let ((cur (get g k {:object (get a :object) :actors (list) :verb (get a :verb)}))) (assoc g k (assoc cur :actors (append (get cur :actors) (list (get a :actor)))))))) {} events))) + (map + (fn + (k) + (let + ((grp (get groups k))) + (assoc grp :count (len (get grp :actors))))) + (sort (keys groups))))))) diff --git a/lib/feed/page.sx b/lib/feed/page.sx new file mode 100644 index 00000000..de255e7b --- /dev/null +++ b/lib/feed/page.sx @@ -0,0 +1,50 @@ +; feed/page — pagination. Offset/limit for indexed access, and cursor-based +; (by :at) for recency feeds, which is stable under inserts: a cursor is the +; :at of the last item seen, and the next page is the newest items older than it. +; +; Requires: lib/feed/stream.sx (feed/recent, feed/take, feed/filter). + +; --- offset / limit --------------------------------------------------------- + +(define + feed/page + (fn + (stream offset limit) + (feed/stream (take (drop (feed/items stream) offset) limit)))) + +(define + feed/page-count + (fn (stream limit) (ceil (/ (feed/count stream) limit)))) + +; --- cursor (recency feeds) ------------------------------------------------- + +; activities strictly older than cursor (scroll down / load older) +(define + feed/before + (fn + (stream cursor) + (feed/filter stream (fn (a) (< (get a :at) cursor))))) + +; activities strictly newer than cursor (load newer / "N new posts") +(define + feed/after + (fn + (stream cursor) + (feed/filter stream (fn (a) (> (get a :at) cursor))))) + +; one page: the `limit` newest activities older than cursor, newest first +(define + feed/page-before + (fn + (stream cursor limit) + (feed/take (feed/recent (feed/before stream cursor)) limit))) + +; cursor to fetch the next (older) page: :at of the last item of a page, +; or nil when the page is empty (end of feed) +(define + feed/next-cursor + (fn + (page) + (let + ((items (feed/items page))) + (if (= (len items) 0) nil (get (last items) :at))))) diff --git a/lib/feed/rank.sx b/lib/feed/rank.sx new file mode 100644 index 00000000..36e30411 --- /dev/null +++ b/lib/feed/rank.sx @@ -0,0 +1,92 @@ +; feed/rank — scoring + ranking. Scorers are (activity -> number). Ranking is a +; stable two-pass grade-down: first by :at descending (the tiebreak), then by +; score descending — so ties resolve by recency, then by input order. Fully +; deterministic on ties. +; +; Requires: lib/apl/runtime.sx, lib/feed/normalize.sx, lib/feed/stream.sx. + +; --- scorers ---------------------------------------------------------------- + +; recency: half-life decay. score = 0.5 ^ (age / half-life). at==now -> 1.0. +(define + feed/recency + (fn + (now half-life) + (fn (a) (expt 0.5 (/ (- now (get a :at)) half-life))))) + +; velocity: how many of this actor's activities fall in (at-window, at] — +; a burst of recent activity scores higher. +(define + feed/velocity + (fn + (stream window) + (fn + (a) + (len + (filter + (fn + (b) + (and + (equal? (get b :actor) (get a :actor)) + (<= (get b :at) (get a :at)) + (> (get b :at) (- (get a :at) window)))) + (feed/items stream)))))) + +; engagement: how many activities in the stream touch this activity's :object +(define + feed/engagement + (fn + (stream) + (fn + (a) + (len + (filter + (fn (b) (equal? (get b :object) (get a :object))) + (feed/items stream)))))) + +; composite: weighted sum. parts = (list (list weight scorer) ...) +(define + feed/composite + (fn + (parts) + (fn + (a) + (reduce + (fn (acc p) (+ acc (* (first p) ((nth p 1) a)))) + 0 + parts)))) + +; --- ranking ---------------------------------------------------------------- + +; stable reorder of items by key-fn, descending (grade-down is stable) +(define + feed/-desc-by + (fn + (items key-fn) + (let + ((keys (make-array (list (len items)) (map key-fn items)))) + (let + ((order (get (apl-grade-down keys) :ravel))) + (map (fn (i) (nth items (- i 1))) order))))) + +; rank by score descending; ties -> :at descending -> input order +(define + feed/rank + (fn + (stream score-fn) + (let + ((by-at (feed/-desc-by (feed/items stream) feed/at))) + (feed/stream (feed/-desc-by by-at score-fn))))) + +; attach a :score to each activity (for inspection / debugging) +(define + feed/with-scores + (fn + (stream score-fn) + (feed/stream + (map (fn (a) (assoc a :score (score-fn a))) (feed/items stream))))) + +; top-N ranked timeline +(define + feed/top + (fn (stream score-fn n) (feed/take (feed/rank stream score-fn) n))) diff --git a/lib/feed/scoreboard.json b/lib/feed/scoreboard.json new file mode 100644 index 00000000..18a55a13 --- /dev/null +++ b/lib/feed/scoreboard.json @@ -0,0 +1,19 @@ +{ + "suites": { + "basic": {"pass": 30, "fail": 0}, + "fanout": {"pass": 29, "fail": 0}, + "rank": {"pass": 24, "fail": 0}, + "integration": {"pass": 22, "fail": 0}, + "content": {"pass": 15, "fail": 0}, + "notify": {"pass": 8, "fail": 0}, + "home": {"pass": 6, "fail": 0}, + "dedupe": {"pass": 9, "fail": 0}, + "trending": {"pass": 11, "fail": 0}, + "mute": {"pass": 9, "fail": 0}, + "page": {"pass": 14, "fail": 0}, + "thread": {"pass": 12, "fail": 0} + }, + "total_pass": 189, + "total_fail": 0, + "total": 189 +} diff --git a/lib/feed/scoreboard.md b/lib/feed/scoreboard.md new file mode 100644 index 00000000..8a2c7b55 --- /dev/null +++ b/lib/feed/scoreboard.md @@ -0,0 +1,19 @@ +# feed Conformance Scoreboard + +_Generated by `lib/feed/conformance.sh`_ + +| Suite | Pass | Fail | Total | +|-------|-----:|-----:|------:| +| basic | 30 | 0 | 30 | +| fanout | 29 | 0 | 29 | +| rank | 24 | 0 | 24 | +| integration | 22 | 0 | 22 | +| content | 15 | 0 | 15 | +| notify | 8 | 0 | 8 | +| home | 6 | 0 | 6 | +| dedupe | 9 | 0 | 9 | +| trending | 11 | 0 | 11 | +| mute | 9 | 0 | 9 | +| page | 14 | 0 | 14 | +| thread | 12 | 0 | 12 | +| **Total** | **189** | **0** | **189** | diff --git a/lib/feed/stream.sx b/lib/feed/stream.sx new file mode 100644 index 00000000..06cbf4de --- /dev/null +++ b/lib/feed/stream.sx @@ -0,0 +1,75 @@ +; feed/stream — a stream is an APL vector (rank-1 array) whose ravel holds +; activity dicts. Operations lift APL primitives onto this shape: filter via +; compress (/), sort via grade (⍋), take via ↑, reverse via ⌽. +; +; Requires: lib/apl/runtime.sx, lib/feed/normalize.sx (loaded by harness). + +(define feed/stream (fn (acts) (make-array (list (len acts)) acts))) + +(define feed/items (fn (s) (get s :ravel))) + +(define feed/count (fn (s) (len (get s :ravel)))) + +(define feed/empty (feed/stream (list))) + +(define feed/empty? (fn (s) (= (feed/count s) 0))) + +; filter — bool mask ∘ compress. pred : activity -> truthy +(define + feed/filter + (fn + (s pred) + (let + ((items (get s :ravel))) + (let + ((mask (make-array (list (len items)) (map (fn (a) (if (pred a) 1 0)) items)))) + (apl-compress mask s))))) + +; sort-by — ascending, stable on ties (grade-up is stable). key-fn : activity -> number +(define + feed/sort-by + (fn + (s key-fn) + (let + ((items (get s :ravel))) + (let + ((keys (make-array (list (len items)) (map key-fn items)))) + (let + ((order (get (apl-grade-up keys) :ravel))) + (feed/stream (map (fn (i) (nth items (- i 1))) order))))))) + +(define feed/sort-by-at (fn (s) (feed/sort-by s feed/at))) + +; newest-first: ascending sort then reverse (⌽) +(define feed/recent (fn (s) (apl-reverse (feed/sort-by-at s)))) + +; take N (↑), clamped to stream length so it never over-takes/pads +(define + feed/take + (fn + (s n) + (let + ((c (feed/count s))) + (if (>= n c) s (apl-take (apl-scalar n) s))))) + +(define feed/reverse (fn (s) (apl-reverse s))) + +; common predicates +(define + feed/by-actor + (fn (s actor) (feed/filter s (fn (a) (equal? (get a :actor) actor))))) + +(define + feed/by-verb + (fn (s verb) (feed/filter s (fn (a) (equal? (get a :verb) verb))))) + +(define + feed/by-object + (fn + (s object) + (feed/filter s (fn (a) (equal? (get a :object) object))))) + +; activities at or after timestamp t +(define + feed/since + (fn (s t) (feed/filter s (fn (a) (>= (get a :at) t))))) diff --git a/lib/feed/tests/basic.sx b/lib/feed/tests/basic.sx new file mode 100644 index 00000000..b8b60c5a --- /dev/null +++ b/lib/feed/tests/basic.sx @@ -0,0 +1,118 @@ +; Phase 1 — normalize, stream ops, api. Uses the feed-test harness +; (feed-test name got expected) provided by conformance.sh. + +; ---------- normalize ---------- + +(feed-test + "normalize default actor" + (feed/actor (feed/normalize {})) + "") +(feed-test + "normalize default verb" + (feed/verb (feed/normalize {})) + "post") +(feed-test + "normalize default at" + (feed/at (feed/normalize {})) + 0) +(feed-test + "normalize default object" + (feed/object (feed/normalize {})) + nil) +(feed-test + "normalize default tags" + (feed/tags (feed/normalize {})) + (list)) +(feed-test + "normalize keeps actor" + (feed/actor (feed/normalize {:actor "alice"})) + "alice") +(feed-test + "normalize keeps verb" + (feed/verb (feed/normalize {:verb "like"})) + "like") +(feed-test + "normalize scalar tag -> list" + (feed/tags (feed/normalize {:tags "x"})) + (list "x")) +(feed-test + "normalize list tags kept" + (feed/tags (feed/normalize {:tags (list "a" "b")})) + (list "a" "b")) +(feed-test + "activity constructor at" + (feed/at (feed/activity "a" "post" "o" 5 (list))) + 5) +(feed-test + "activity? on activity" + (feed/activity? (feed/normalize {:actor "a"})) + true) +(feed-test "activity? on number" (feed/activity? 5) false) +(feed-test "activity? on bare dict" (feed/activity? {:foo 1}) false) + +; ---------- stream ---------- + +(define + S + (feed/stream + (list + (feed/activity "alice" "post" "p1" 30 (list)) + (feed/activity "bob" "like" "p1" 10 (list)) + (feed/activity "alice" "post" "p2" 20 (list))))) + +(feed-test "stream count" (feed/count S) 3) +(feed-test "stream items len" (len (feed/items S)) 3) +(feed-test + "sort-by-at actors asc" + (map feed/actor (feed/items (feed/sort-by-at S))) + (list "bob" "alice" "alice")) +(feed-test + "recent newest first" + (map feed/at (feed/items (feed/recent S))) + (list 30 20 10)) +(feed-test + "take 2 of recent" + (feed/count (feed/take (feed/recent S) 2)) + 2) +(feed-test + "take clamps past end" + (feed/count (feed/take S 10)) + 3) +(feed-test + "by-actor alice count" + (feed/count (feed/by-actor S "alice")) + 2) +(feed-test + "by-verb like actor" + (map feed/actor (feed/items (feed/by-verb S "like"))) + (list "bob")) +(feed-test + "by-object p1 count" + (feed/count (feed/by-object S "p1")) + 2) +(feed-test + "since 20 count" + (feed/count (feed/since S 20)) + 2) +(feed-test + "reverse ats" + (map feed/at (feed/items (feed/reverse S))) + (list 20 10 30)) +(feed-test "empty? on empty" (feed/empty? feed/empty) true) +(feed-test + "empty? on filtered-out" + (feed/empty? (feed/by-actor S "zzz")) + true) + +; ---------- api ---------- + +(feed/reset!) +(feed/post {:actor "x" :at 1 :verb "post"}) +(feed/post {:actor "y" :at 2 :verb "like"}) +(feed-test "api size after posts" (feed/size) 2) +(feed-test "api all count" (feed/count (feed/all)) 2) +(feed-test + "post returns normalized verb" + (feed/verb (feed/post {:actor "z"})) + "post") +(feed-test "api size after third post" (feed/size) 3) diff --git a/lib/feed/tests/content.sx b/lib/feed/tests/content.sx new file mode 100644 index 00000000..dd742adf --- /dev/null +++ b/lib/feed/tests/content.sx @@ -0,0 +1,85 @@ +; Follow-up — TF-IDF content ranking over :tags. (feed-test name got expected) + +(define + corpus + (feed/stream + (list + (feed/normalize {:actor "u" :object "o1" :at 10 :tags (list "cats" "funny")}) + (feed/normalize {:actor "u" :object "o2" :at 20 :tags (list "cats" "news")}) + (feed/normalize {:actor "u" :object "o3" :at 30 :tags (list "politics" "news")}) + (feed/normalize {:actor "u" :object "o4" :at 40 :tags (list "cats")})))) + +; ---------- document frequency ---------- + +(feed-test "df cats" (get (feed/tag-df corpus) "cats") 3) +(feed-test "df news" (get (feed/tag-df corpus) "news") 2) +(feed-test "df funny" (get (feed/tag-df corpus) "funny") 1) +(feed-test "df politics" (get (feed/tag-df corpus) "politics") 1) +(feed-test "df full" (feed/tag-df corpus) {:news 2 :funny 1 :politics 1 :cats 3}) + +; ---------- inverse document frequency ---------- + +(feed-test + "idf news = log(4/2)" + (get (feed/tag-idf corpus) "news") + (log 2)) +(feed-test + "idf funny = log(4/1)" + (get (feed/tag-idf corpus) "funny") + (log 4)) +(feed-test + "rarer tag has higher idf" + (> + (get (feed/tag-idf corpus) "funny") + (get (feed/tag-idf corpus) "cats")) + true) + +; ---------- tf-idf scoring ---------- + +(define idf (feed/tag-idf corpus)) + +(feed-test + "score query funny on o1" + ((feed/tfidf-score idf (list "funny")) (feed/normalize {:actor "u" :object "x" :tags (list "cats" "funny")})) + (log 4)) +(feed-test + "score query funny on non-match" + ((feed/tfidf-score idf (list "funny")) (feed/normalize {:actor "u" :object "x" :tags (list "cats")})) + 0) +(feed-test + "unknown query tag scores 0" + ((feed/tfidf-score idf (list "zzz")) (feed/normalize {:actor "u" :object "x" :tags (list "cats")})) + 0) + +; ---------- ranking by relevance ---------- + +; query news: o2,o3 match (score log2), o1,o4 don't (0); ties break by :at desc +(feed-test + "by-relevance news order" + (map + (fn (a) (get a :object)) + (feed/items (feed/by-relevance corpus (list "news")))) + (list "o3" "o2" "o4" "o1")) + +; query funny: only o1 matches -> ranks first +(feed-test + "by-relevance funny first" + (get + (nth (feed/items (feed/by-relevance corpus (list "funny"))) 0) + :object) + "o1") + +; query (cats news): o2 carries both tags -> highest combined tf-idf +(feed-test + "by-relevance cats+news top" + (get + (nth + (feed/items (feed/by-relevance corpus (list "cats" "news"))) + 0) + :object) + "o2") + +(feed-test + "by-relevance preserves count" + (feed/count (feed/by-relevance corpus (list "cats"))) + 4) diff --git a/lib/feed/tests/dedupe.sx b/lib/feed/tests/dedupe.sx new file mode 100644 index 00000000..69f129af --- /dev/null +++ b/lib/feed/tests/dedupe.sx @@ -0,0 +1,56 @@ +; Follow-up — verb-aware (smart) dedupe. (feed-test name got expected) + +; reactions (like/follow) collapse cross-actor; posts stay distinct per actor +(define + M + (feed/stream + (list + (feed/activity "alice" "like" "X" 1 (list)) + (feed/activity "bob" "like" "X" 2 (list)) + (feed/activity "alice" "post" "P" 3 (list)) + (feed/activity "bob" "post" "P" 4 (list)) + (feed/activity "alice" "follow" "C" 5 (list)) + (feed/activity "bob" "follow" "C" 6 (list))))) ; collapses + +(feed-test + "smart dedupe total" + (feed/count (feed/dedupe-smart M)) + 4) +(feed-test + "smart keeps both posts" + (feed/count (feed/by-verb (feed/dedupe-smart M) "post")) + 2) +(feed-test + "smart collapses likes to one" + (feed/count (feed/by-verb (feed/dedupe-smart M) "like")) + 1) +(feed-test + "smart collapses follows to one" + (feed/count (feed/by-verb (feed/dedupe-smart M) "follow")) + 1) +(feed-test + "collapsed like keeps first actor" + (map feed/actor (feed/items (feed/by-verb (feed/dedupe-smart M) "like"))) + (list "alice")) + +; contrast: plain activity dedupe keeps cross-actor likes distinct +(feed-test + "activity dedupe keeps both likes" + (feed/count (feed/by-verb (feed/dedupe-activities M) "like")) + 2) + +; contrast: blanket collapse folds the two posts (same verb+object) too +(feed-test + "collapse dedupe folds posts" + (feed/count (feed/by-verb (feed/dedupe-collapse M) "post")) + 1) + +; smart-key dispatch +(feed-test + "smart-key reaction -> (verb object)" + (feed/smart-key (feed/activity "alice" "like" "X" 0 (list))) + (list "like" "X")) +(feed-test + "smart-key post -> (actor verb object)" + (feed/smart-key (feed/activity "alice" "post" "P" 0 (list))) + (list "alice" "post" "P")) diff --git a/lib/feed/tests/fanout.sx b/lib/feed/tests/fanout.sx new file mode 100644 index 00000000..84bd7988 --- /dev/null +++ b/lib/feed/tests/fanout.sx @@ -0,0 +1,187 @@ +; Phase 2 — fanout via outer product + dedupe. (feed-test name got expected) + +; ---------- graph ---------- + +; edges: (follower followee). bob,carol follow alice; carol,dave follow bob. +(define + G + (feed/follow-graph + (list + (list "bob" "alice") + (list "carol" "alice") + (list "carol" "bob") + (list "dave" "bob")))) + +(feed-test "followers alice" (feed/followers G "alice") (list "bob" "carol")) +(feed-test "followers bob" (feed/followers G "bob") (list "carol" "dave")) +(feed-test "followers unknown" (feed/followers G "zzz") (list)) +(feed-test "audience distinct" (feed/audience G) (list "bob" "carol" "dave")) + +; ---------- fanout ---------- + +(define + S + (feed/stream + (list + (feed/activity "alice" "post" "p1" 10 (list)) + (feed/activity "alice" "post" "p2" 20 (list)) + (feed/activity "bob" "like" "p1" 30 (list))))) + +(define IB (feed/fanout S G)) + +(feed-test "fanout total edges" (feed/count IB) 6) +(feed-test + "inbox bob count" + (feed/count (feed/inbox-for IB "bob")) + 2) +(feed-test + "inbox carol count" + (feed/count (feed/inbox-for IB "carol")) + 3) +(feed-test + "inbox dave count" + (feed/count (feed/inbox-for IB "dave")) + 1) +(feed-test + "inbox alice (follows none)" + (feed/count (feed/inbox-for IB "alice")) + 0) +(feed-test + "recipients order" + (feed/recipients IB) + (list "bob" "carol" "dave")) +(feed-test + "bob inbox objects" + (map (fn (a) (get a :object)) (feed/inbox-activities IB "bob")) + (list "p1" "p2")) +(feed-test + "dave inbox objects" + (map (fn (a) (get a :object)) (feed/inbox-activities IB "dave")) + (list "p1")) +(feed-test + "dave inbox verb" + (map (fn (a) (get a :verb)) (feed/inbox-activities IB "dave")) + (list "like")) + +; empty graph → no audience → no edges +(feed-test + "empty graph fanout" + (feed/count (feed/fanout S {})) + 0) + +; actor nobody follows produces no edges +(define + Sghost + (feed/stream (list (feed/activity "ghost" "post" "g1" 5 (list))))) +(feed-test + "unfollowed actor fanout" + (feed/count (feed/fanout Sghost G)) + 0) + +; ---------- high fanout (popular actor) ---------- + +(define + Gstar + (feed/follow-graph + (list + (list "u1" "star") + (list "u2" "star") + (list "u3" "star") + (list "u4" "star") + (list "u5" "star")))) +(define + Sstar + (feed/stream (list (feed/activity "star" "post" "s1" 1 (list))))) +(feed-test + "star fanout count" + (feed/count (feed/fanout Sstar Gstar)) + 5) +(feed-test "star audience size" (len (feed/audience Gstar)) 5) + +; ---------- mutual follow ---------- + +(define Gmut (feed/follow-graph (list (list "a" "b") (list "b" "a")))) +(define + Smut + (feed/stream + (list + (feed/activity "a" "post" "pa" 1 (list)) + (feed/activity "b" "post" "pb" 2 (list))))) +(define IBmut (feed/fanout Smut Gmut)) +(feed-test "mutual total" (feed/count IBmut) 2) +(feed-test + "mutual a gets pb" + (map (fn (x) (get x :object)) (feed/inbox-activities IBmut "a")) + (list "pb")) +(feed-test + "mutual b gets pa" + (map (fn (x) (get x :object)) (feed/inbox-activities IBmut "b")) + (list "pa")) + +; ---------- dedupe ---------- + +(define + Sdup2 + (feed/stream + (list + (feed/activity "alice" "post" "p1" 1 (list)) + (feed/activity "alice" "post" "p1" 9 (list)) + (feed/activity "alice" "post" "p2" 2 (list))))) +(feed-test + "dedupe-activities collapses dup" + (feed/count (feed/dedupe-activities Sdup2)) + 2) +(feed-test + "dedupe-activities keeps distinct" + (map + (fn (a) (get a :object)) + (feed/items (feed/dedupe-activities Sdup2))) + (list "p1" "p2")) + +(define + Slikes + (feed/stream + (list + (feed/activity "alice" "like" "X" 1 (list)) + (feed/activity "bob" "like" "X" 2 (list)) + (feed/activity "carol" "like" "Y" 3 (list))))) +(feed-test + "collapse cross-actor likes" + (feed/count (feed/dedupe-collapse Slikes)) + 2) +(feed-test + "collapse keeps distinct objects" + (map + (fn (a) (get a :object)) + (feed/items (feed/dedupe-collapse Slikes))) + (list "X" "Y")) + +(feed-test + "activity-key shape" + (feed/activity-key (feed/activity "a" "post" "o" 0 (list))) + (list "a" "post" "o")) +(feed-test + "collapse-key shape" + (feed/collapse-key (feed/activity "a" "like" "o" 0 (list))) + (list "like" "o")) + +; cross-post: alice posts p1 twice → bob's inbox has it twice → dedupe-inbox → once +(define + Scross + (feed/stream + (list + (feed/activity "alice" "post" "p1" 1 (list)) + (feed/activity "alice" "post" "p1" 5 (list))))) +(define IBcross (feed/fanout Scross G)) +(feed-test + "cross-post raw bob count" + (feed/count (feed/inbox-for IBcross "bob")) + 2) +(feed-test + "cross-post deduped bob count" + (feed/count (feed/inbox-for (feed/dedupe-inbox IBcross) "bob")) + 1) +(feed-test + "dedupe-inbox keeps distinct receivers" + (feed/count (feed/dedupe-inbox IBcross)) + 2) diff --git a/lib/feed/tests/home.sx b/lib/feed/tests/home.sx new file mode 100644 index 00000000..0a52c2b8 --- /dev/null +++ b/lib/feed/tests/home.sx @@ -0,0 +1,73 @@ +; Follow-up — feed/home capstone pipeline. (feed-test name got expected) + +; alice follows star and bob (edges: follower followee) +(define + G + (feed/follow-graph (list (list "alice" "star") (list "alice" "bob")))) + +; star posts s1 then s2; bob posts b1; star re-posts s1 (cross-post dup); +; zoe posts z1 (alice does NOT follow zoe) +(define + S + (feed/stream + (list + (feed/activity "star" "post" "s1" 10 (list)) + (feed/activity "star" "post" "s2" 20 (list)) + (feed/activity "bob" "post" "b1" 15 (list)) + (feed/activity "star" "post" "s1" 5 (list)) + (feed/activity "zoe" "post" "z1" 30 (list))))) + +(define rec (feed/recency 100 10)) + +(feed-test + "home count (deduped, followed only)" + (feed/count (feed/home S G "alice" feed/permit-public? rec 10)) + 3) + +(feed-test + "home order by recency" + (map + (fn (a) (get a :object)) + (feed/items (feed/home S G "alice" feed/permit-public? rec 10))) + (list "s2" "b1" "s1")) + +(feed-test + "home excludes unfollowed zoe" + (feed/-elem? + "z1" + (map + (fn (a) (get a :object)) + (feed/items (feed/home S G "alice" feed/permit-public? rec 10)))) + false) + +(feed-test + "home top-2" + (map + (fn (a) (get a :object)) + (feed/items (feed/home S G "alice" feed/permit-public? rec 2))) + (list "s2" "b1")) + +(feed-test + "home dedupes cross-post (one s1)" + (len + (filter + (fn (o) (equal? o "s1")) + (map + (fn (a) (get a :object)) + (feed/items + (feed/home S G "alice" feed/permit-public? rec 10))))) + 1) + +; ACL applied per-viewer in the home pipeline +(define + Sacl + (feed/stream + (list (feed/normalize {:actor "star" :object "pub" :at 20}) (feed/normalize {:actor "star" :object "sec" :visible-to (list "carol") :at 25})))) +(define Gacl (feed/follow-graph (list (list "alice" "star")))) + +(feed-test + "home hides activity alice not permitted" + (map + (fn (a) (get a :object)) + (feed/items (feed/home Sacl Gacl "alice" feed/permit-acl? rec 10))) + (list "pub")) diff --git a/lib/feed/tests/integration.sx b/lib/feed/tests/integration.sx new file mode 100644 index 00000000..08ead747 --- /dev/null +++ b/lib/feed/tests/integration.sx @@ -0,0 +1,155 @@ +; Phase 4 — visibility (ACL) + federation, and the end-to-end timeline. +; (feed-test name got expected) + +; ---------- ACL visibility ---------- +; pub: public. sec: bob, allows carol. dm: frank, allows dave. + +(define + C + (feed/stream + (list + (feed/normalize {:actor "alice" :object "pub" :at 10}) + (feed/normalize {:actor "bob" :object "sec" :visible-to (list "carol") :at 20}) + (feed/normalize {:actor "frank" :object "dm" :visible-to (list "dave") :at 30})))) + +(feed-test + "public visible to anyone" + (feed/count (feed/visible C "zoe" feed/permit-acl?)) + 1) +(feed-test + "carol sees allowlisted + public" + (feed/count (feed/visible C "carol" feed/permit-acl?)) + 2) +(feed-test + "dave sees dm + public" + (feed/count (feed/visible C "dave" feed/permit-acl?)) + 2) +(feed-test + "author always sees own private" + (feed/count (feed/visible C "frank" feed/permit-acl?)) + 2) +(feed-test + "permit-public? lets all through" + (feed/count (feed/visible C "zoe" feed/permit-public?)) + 3) +(feed-test + "visible objects for dave" + (map + (fn (a) (get a :object)) + (feed/items (feed/visible C "dave" feed/permit-acl?))) + (list "pub" "dm")) + +; per-viewer: same stream, different timelines +(feed-test + "zoe timeline differs from carol" + (not + (= + (feed/count (feed/visible C "zoe" feed/permit-acl?)) + (feed/count (feed/visible C "carol" feed/permit-acl?)))) + true) + +; ---------- federation: merge / ingest ---------- + +(define + L + (feed/stream + (list + (feed/activity "alice" "post" "p1" 10 (list)) + (feed/activity "alice" "post" "p2" 20 (list))))) +(define + P + (feed/stream + (list + (feed/activity "alice" "post" "p2" 20 (list)) + (feed/activity "peer" "post" "p9" 25 (list))))) + +(feed-test "merge concatenates" (feed/count (feed/merge L P)) 4) +(feed-test + "ingest dedupes overlap" + (feed/count (feed/ingest L P)) + 3) + +(feed-test + "inbound normalizes + ingests" + (feed/count (feed/inbound L (list {:actor "peer" :object "p9" :at 25} {:actor "alice" :object "p1" :at 10}))) + 3) + +; backfill via injected fetch-fn +(define peer-history (fn (peer-id) (list {:actor peer-id :object "h1" :at 1} {:actor peer-id :object "h2" :at 2}))) +(feed-test + "backfill merges peer history" + (feed/count (feed/backfill L peer-history "remote")) + 4) +(feed-test + "backfill objects present" + (map + (fn (a) (get a :object)) + (feed/items + (feed/by-actor (feed/backfill L peer-history "remote") "remote"))) + (list "h1" "h2")) + +; ---------- federation: outbound partition ---------- + +; bob (local), alice@remote + carol@remote (remote) follow star +(define + Gf + (feed/follow-graph + (list + (list "bob" "star") + (list "alice@remote" "star") + (list "carol@remote" "star")))) +(define + Sf + (feed/stream (list (feed/activity "star" "post" "s1" 1 (list))))) +(define + remote? + (fn (id) (feed/-elem? id (list "alice@remote" "carol@remote")))) +(define parts (feed/federate Sf Gf remote?)) + +(feed-test "local deliveries" (feed/count (get parts :local)) 1) +(feed-test "remote deliveries" (feed/count (get parts :remote)) 2) +(feed-test + "local recipient is bob" + (feed/recipients (get parts :local)) + (list "bob")) + +; deliver: send-fn receives each remote event, local inbox returned +(define sent (list)) +(define send-fn (fn (to act) (set! sent (append sent (list to))))) +(define local-inbox (feed/deliver Sf Gf remote? send-fn)) +(feed-test "deliver returns local inbox" (feed/count local-inbox) 1) +(feed-test "deliver sent to both remotes" (len sent) 2) +(feed-test "deliver remote targets" sent (list "alice@remote" "carol@remote")) + +; ---------- end-to-end: federated, ACL-filtered, ranked timeline ---------- + +(define + base + (feed/stream + (list + (feed/normalize {:actor "alice" :object "a1" :at 100}) + (feed/normalize {:actor "bob" :object "b1" :visible-to (list "carol") :at 90}) + (feed/normalize {:actor "eve" :object "e1" :visible-to (list "dave") :at 80})))) +(define federated (feed/inbound base (list {:actor "peer" :object "x1" :at 110}))) +(define rec (feed/recency 120 10)) +(define + carol-tl + (feed/timeline federated "carol" feed/permit-acl? rec 3)) + +; eve's :visible-to excludes carol -> filtered out; peer/alice public, bob allows carol +(feed-test "carol federated timeline count" (feed/count carol-tl) 3) +(feed-test + "carol timeline order (recency)" + (map (fn (a) (get a :object)) (feed/items carol-tl)) + (list "x1" "a1" "b1")) +(feed-test + "eve dm excluded from carol" + (feed/-elem? "e1" (map (fn (a) (get a :object)) (feed/items carol-tl))) + false) +(feed-test + "dave sees eve dm not bob" + (map + (fn (a) (get a :object)) + (feed/items + (feed/timeline federated "dave" feed/permit-acl? rec 5))) + (list "x1" "a1" "e1")) diff --git a/lib/feed/tests/mute.sx b/lib/feed/tests/mute.sx new file mode 100644 index 00000000..81ee1e69 --- /dev/null +++ b/lib/feed/tests/mute.sx @@ -0,0 +1,68 @@ +; Follow-up — viewer mute/block filtering. (feed-test name got expected) + +(define + S + (feed/stream + (list + (feed/normalize {:actor "alice" :object "P1" :at 1 :tags (list "news")}) + (feed/normalize {:actor "bob" :object "P2" :at 2 :tags (list "spam")}) + (feed/normalize {:actor "alice" :object "P3" :at 3 :tags (list "cats")}) + (feed/normalize {:actor "carol" :object "P4" :at 4 :tags (list "news" "spam")})))) + +; ---------- mute actors ---------- + +(feed-test + "mute bob drops his post" + (map + (fn (a) (get a :object)) + (feed/items (feed/mute-actors S (list "bob")))) + (list "P1" "P3" "P4")) +(feed-test + "mute alice drops two" + (feed/count (feed/mute-actors S (list "alice"))) + 2) +(feed-test + "mute nobody keeps all" + (feed/count (feed/mute-actors S (list))) + 4) + +; ---------- mute tags ---------- + +(feed-test + "mute spam tag drops two" + (map + (fn (a) (get a :object)) + (feed/items (feed/mute-tags S (list "spam")))) + (list "P1" "P3")) +(feed-test + "mute news+cats leaves spam-only" + (map + (fn (a) (get a :object)) + (feed/items (feed/mute-tags S (list "news" "cats")))) + (list "P2")) + +; ---------- mute objects ---------- + +(feed-test + "mute object P3 (thread mute)" + (feed/count (feed/mute-objects S (list "P3"))) + 3) + +; ---------- combined prefs ---------- + +(feed-test + "apply-prefs actors + tags" + (map + (fn (a) (get a :object)) + (feed/items (feed/apply-prefs S {:mute-actors (list "bob") :mute-tags (list "cats")}))) + (list "P1" "P4")) +(feed-test + "apply-prefs empty keeps all" + (feed/count (feed/apply-prefs S {})) + 4) +(feed-test + "apply-prefs all three filters" + (map + (fn (a) (get a :object)) + (feed/items (feed/apply-prefs S {:mute-objects (list "P3") :mute-actors (list "carol") :mute-tags (list "spam")}))) + (list "P1")) diff --git a/lib/feed/tests/notify.sx b/lib/feed/tests/notify.sx new file mode 100644 index 00000000..d7212912 --- /dev/null +++ b/lib/feed/tests/notify.sx @@ -0,0 +1,69 @@ +; Follow-up — notification feed over an inbox. (feed-test name got expected) + +; an inbox is a stream of {:to receiver :activity act} events +(define mk-ev (fn (to act) {:activity act :to to})) + +(define + IB + (feed/stream + (list + (mk-ev "alice" (feed/activity "bob" "like" "P" 10 (list))) + (mk-ev "alice" (feed/activity "carol" "like" "P" 20 (list))) + (mk-ev "alice" (feed/activity "dave" "reply" "Q" 30 (list))) + (mk-ev "bob" (feed/activity "eve" "like" "R" 40 (list)))))) + +; ---------- raw notifications ---------- + +(feed-test + "alice notification count" + (feed/count (feed/notifications IB "alice")) + 3) +(feed-test + "bob notification count" + (feed/count (feed/notifications IB "bob")) + 1) +(feed-test + "zoe no notifications" + (feed/count (feed/notifications IB "zoe")) + 0) + +; ---------- verb filtering ---------- + +(feed-test + "alice likes only" + (feed/count (feed/notify-verbs IB "alice" (list "like"))) + 2) +(feed-test + "alice replies only" + (feed/count (feed/notify-verbs IB "alice" (list "reply"))) + 1) +(feed-test + "alice like+reply" + (feed/count (feed/notify-verbs IB "alice" (list "like" "reply"))) + 3) +(feed-test + "alice follow (none)" + (feed/count (feed/notify-verbs IB "alice" (list "follow"))) + 0) + +; ---------- digest ---------- + +(define dig (feed/notify-digest IB "alice")) + +(feed-test "digest group count" (len dig) 2) +(feed-test + "digest sorted by key (like|P before reply|Q)" + (map (fn (g) (get g :object)) dig) + (list "P" "Q")) +(feed-test + "like group actors" + (get (nth dig 0) :actors) + (list "bob" "carol")) +(feed-test "like group count" (get (nth dig 0) :count) 2) +(feed-test "like group verb" (get (nth dig 0) :verb) "like") +(feed-test "reply group count" (get (nth dig 1) :count) 1) +(feed-test + "reply group actors" + (get (nth dig 1) :actors) + (list "dave")) +(feed-test "empty digest for zoe" (feed/notify-digest IB "zoe") (list)) diff --git a/lib/feed/tests/page.sx b/lib/feed/tests/page.sx new file mode 100644 index 00000000..e1a1af87 --- /dev/null +++ b/lib/feed/tests/page.sx @@ -0,0 +1,86 @@ +; Follow-up — pagination (offset + cursor). (feed-test name got expected) + +; ---------- offset / limit ---------- + +(define + O + (feed/stream + (list + (feed/activity "u" "post" "o1" 1 (list)) + (feed/activity "u" "post" "o2" 2 (list)) + (feed/activity "u" "post" "o3" 3 (list)) + (feed/activity "u" "post" "o4" 4 (list)) + (feed/activity "u" "post" "o5" 5 (list))))) + +(feed-test + "page 1" + (map + (fn (a) (get a :object)) + (feed/items (feed/page O 0 2))) + (list "o1" "o2")) +(feed-test + "page 2" + (map + (fn (a) (get a :object)) + (feed/items (feed/page O 2 2))) + (list "o3" "o4")) +(feed-test + "page 3 (partial)" + (map + (fn (a) (get a :object)) + (feed/items (feed/page O 4 2))) + (list "o5")) +(feed-test + "page past end empty" + (feed/count (feed/page O 10 2)) + 0) +(feed-test "page-count 5/2 = 3" (feed/page-count O 2) 3) +(feed-test "page-count 5/5 = 1" (feed/page-count O 5) 1) + +; ---------- cursor (recency) ---------- + +(define + R + (feed/stream + (list + (feed/activity "u" "post" "a" 50 (list)) + (feed/activity "u" "post" "b" 40 (list)) + (feed/activity "u" "post" "c" 30 (list)) + (feed/activity "u" "post" "d" 20 (list)) + (feed/activity "u" "post" "e" 10 (list))))) + +(define p1 (feed/page-before R 100 2)) +(feed-test + "cursor page 1 newest first" + (map (fn (a) (get a :object)) (feed/items p1)) + (list "a" "b")) +(feed-test "next cursor after page 1" (feed/next-cursor p1) 40) + +(define p2 (feed/page-before R (feed/next-cursor p1) 2)) +(feed-test + "cursor page 2" + (map (fn (a) (get a :object)) (feed/items p2)) + (list "c" "d")) +(feed-test "next cursor after page 2" (feed/next-cursor p2) 20) + +(define p3 (feed/page-before R (feed/next-cursor p2) 2)) +(feed-test + "cursor page 3 (partial)" + (map (fn (a) (get a :object)) (feed/items p3)) + (list "e")) + +(feed-test + "empty page nil cursor" + (feed/next-cursor (feed/page-before R 5 2)) + nil) + +(feed-test + "after cursor loads newer" + (map + (fn (a) (get a :object)) + (feed/items (feed/recent (feed/after R 30)))) + (list "a" "b")) +(feed-test + "before cursor count" + (feed/count (feed/before R 30)) + 2) diff --git a/lib/feed/tests/rank.sx b/lib/feed/tests/rank.sx new file mode 100644 index 00000000..920bb99e --- /dev/null +++ b/lib/feed/tests/rank.sx @@ -0,0 +1,160 @@ +; Phase 3 — aggregation + ranking. (feed-test name got expected) + +; ---------- aggregation ---------- + +(define + A + (feed/stream + (list + (feed/activity "alice" "post" "p1" 5 (list)) + (feed/activity "alice" "post" "p2" 15 (list)) + (feed/activity "bob" "post" "p3" 25 (list)) + (feed/activity "alice" "like" "p1" 35 (list))))) + +(feed-test "actor-counts" (feed/actor-counts A) {:alice 3 :bob 1}) +(feed-test "object-counts" (feed/object-counts A) {:p2 1 :p3 1 :p1 2}) +(feed-test + "group-by actor alice len" + (len (get (feed/group-by A feed/actor) "alice")) + 3) +(feed-test + "group-count empty" + (feed/group-count feed/empty feed/actor) + {}) + +; day bucketing +(define + D + (feed/stream + (list + (feed/activity "alice" "post" "p1" 5 (list)) + (feed/activity "alice" "post" "p2" 8 (list)) + (feed/activity "alice" "post" "p3" 12 (list))))) + +(feed-test "feed/day floor" (feed/day 12 10) 1) +(feed-test "feed/day same bucket" (feed/day 8 10) 0) +(feed-test "by-actor-day" (feed/by-actor-day D 10) {:alice#0 2 :alice#1 1}) + +; ---------- recency ---------- + +(define rec (feed/recency 100 10)) +(feed-test + "recency at=now -> 1" + (rec (feed/activity "x" "post" "o" 100 (list))) + 1) +(feed-test + "recency age=hl -> .5" + (rec (feed/activity "x" "post" "o" 90 (list))) + 0.5) +(feed-test + "recency age=2hl -> .25" + (rec (feed/activity "x" "post" "o" 80 (list))) + 0.25) + +; ---------- velocity ---------- + +(define vel (feed/velocity D 10)) +(feed-test + "velocity burst (at=12)" + (vel (feed/activity "alice" "post" "z" 12 (list))) + 3) +(feed-test + "velocity mid (at=8)" + (vel (feed/activity "alice" "post" "z" 8 (list))) + 2) +(feed-test + "velocity first (at=5)" + (vel (feed/activity "alice" "post" "z" 5 (list))) + 1) +(feed-test + "velocity other actor" + (vel (feed/activity "bob" "post" "z" 12 (list))) + 0) + +; ---------- engagement ---------- + +(define eng (feed/engagement A)) +(feed-test + "engagement p1" + (eng (feed/activity "x" "post" "p1" 0 (list))) + 2) +(feed-test + "engagement p2" + (eng (feed/activity "x" "post" "p2" 0 (list))) + 1) + +; ---------- composite ---------- + +(define + cmp1 + (feed/composite (list (list 2 (fn (a) (get a :at)))))) +(feed-test + "composite single part" + (cmp1 (feed/activity "x" "post" "o" 5 (list))) + 10) +(define + cmp2 + (feed/composite + (list + (list 2 (fn (a) (get a :at))) + (list 3 (fn (a) 1))))) +(feed-test + "composite two parts" + (cmp2 (feed/activity "x" "post" "o" 5 (list))) + 13) + +; ---------- ranking ---------- + +(define + R + (feed/stream + (list + (feed/activity "u" "post" "oC" 80 (list)) + (feed/activity "u" "post" "oA" 100 (list)) + (feed/activity "u" "post" "oB" 90 (list))))) + +(feed-test + "rank by recency objects" + (map (fn (a) (get a :object)) (feed/items (feed/rank R rec))) + (list "oA" "oB" "oC")) +(feed-test + "top-2 by recency" + (map (fn (a) (get a :object)) (feed/items (feed/top R rec 2))) + (list "oA" "oB")) +(feed-test "top-2 count" (feed/count (feed/top R rec 2)) 2) + +; constant score -> tiebreak by :at descending +(define + T + (feed/stream + (list + (feed/activity "u" "post" "f" 10 (list)) + (feed/activity "u" "post" "g" 30 (list)) + (feed/activity "u" "post" "h" 20 (list))))) +(feed-test + "tiebreak at-desc" + (map + (fn (a) (get a :object)) + (feed/items (feed/rank T (fn (a) 0)))) + (list "g" "h" "f")) + +; equal score AND equal :at -> stable input order +(define + E + (feed/stream + (list + (feed/activity "u" "post" "first" 50 (list)) + (feed/activity "u" "post" "second" 50 (list))))) +(feed-test + "stable equal-key input order" + (map + (fn (a) (get a :object)) + (feed/items (feed/rank E (fn (a) 0)))) + (list "first" "second")) + +(feed-test + "with-scores attaches score" + (get (nth (feed/items (feed/with-scores R rec)) 1) :score) + 1) + +(feed-test "rank preserves count" (feed/count (feed/rank A rec)) 4) diff --git a/lib/feed/tests/thread.sx b/lib/feed/tests/thread.sx new file mode 100644 index 00000000..3153fa5c --- /dev/null +++ b/lib/feed/tests/thread.sx @@ -0,0 +1,49 @@ +; Follow-up — conversation threading via :reply-to closure. (feed-test name got expected) + +(define + S + (feed/stream + (list + (feed/normalize {:actor "a" :object "root" :at 1}) + (feed/normalize {:actor "b" :object "r1" :at 2 :verb "reply" :reply-to "root"}) + (feed/normalize {:actor "c" :object "r2" :at 3 :verb "reply" :reply-to "root"}) + (feed/normalize {:actor "d" :object "r3" :at 4 :verb "reply" :reply-to "r1"}) + (feed/normalize {:actor "e" :object "x" :at 5})))) + +; ---------- direct replies ---------- + +(feed-test "direct replies to root" (feed/reply-count S "root") 2) +(feed-test "direct replies to r1" (feed/reply-count S "r1") 1) +(feed-test "no replies to r3" (feed/reply-count S "r3") 0) +(feed-test + "replies objects to root" + (map (fn (a) (get a :object)) (feed/items (feed/replies S "root"))) + (list "r1" "r2")) + +; ---------- thread closure ---------- + +(feed-test + "thread objects root (transitive)" + (feed/thread-objects S "root") + (list "root" "r1" "r2" "r3")) +(feed-test + "thread root chronological" + (map (fn (a) (get a :object)) (feed/items (feed/thread S "root"))) + (list "root" "r1" "r2" "r3")) +(feed-test "thread size root" (feed/thread-size S "root") 4) +(feed-test + "thread excludes unrelated x" + (feed/-elem? + "x" + (map (fn (a) (get a :object)) (feed/items (feed/thread S "root")))) + false) + +; ---------- sub-thread ---------- + +(feed-test + "thread from r1 (sub-tree)" + (map (fn (a) (get a :object)) (feed/items (feed/thread S "r1"))) + (list "r1" "r3")) +(feed-test "thread size r1" (feed/thread-size S "r1") 2) +(feed-test "leaf thread is itself" (feed/thread-size S "r3") 1) +(feed-test "unrelated thread is itself" (feed/thread-size S "x") 1) diff --git a/lib/feed/tests/trending.sx b/lib/feed/tests/trending.sx new file mode 100644 index 00000000..63c54c09 --- /dev/null +++ b/lib/feed/tests/trending.sx @@ -0,0 +1,82 @@ +; Follow-up — trending objects/actors by recent activity. (feed-test name got expected) + +; window (50,100]: X@60,X@70 (a), Y@80 (b), Z@90 (c); W@40 is too old +(define + S + (feed/stream + (list + (feed/activity "a" "post" "X" 60 (list)) + (feed/activity "a" "post" "X" 70 (list)) + (feed/activity "b" "post" "Y" 80 (list)) + (feed/activity "c" "post" "Z" 90 (list)) + (feed/activity "d" "post" "W" 40 (list))))) + +; ---------- trending objects ---------- + +(feed-test + "trending count (3 in window)" + (len (feed/trending S 100 50 10)) + 3) +(feed-test + "trending top object" + (get + (nth (feed/trending S 100 50 10) 0) + :object) + "X") +(feed-test + "trending top count" + (get + (nth (feed/trending S 100 50 10) 0) + :count) + 2) +(feed-test + "trending order (count desc, key asc tiebreak)" + (map + (fn (e) (get e :object)) + (feed/trending S 100 50 10)) + (list "X" "Y" "Z")) +(feed-test + "trending top-2" + (map + (fn (e) (get e :object)) + (feed/trending S 100 50 2)) + (list "X" "Y")) +(feed-test + "old object W excluded" + (feed/-elem? + "W" + (map + (fn (e) (get e :object)) + (feed/trending S 100 50 10))) + false) +(feed-test + "narrow window keeps only newest" + (map + (fn (e) (get e :object)) + (feed/trending S 100 15 10)) + (list "Z")) +(feed-test + "empty window -> nothing" + (feed/trending S 100 5 10) + (list)) + +; ---------- trending actors ---------- + +(feed-test + "trending actor top" + (get + (nth (feed/trending-actors S 100 50 10) 0) + :actor) + "a") +(feed-test + "trending actor count" + (get + (nth (feed/trending-actors S 100 50 10) 0) + :count) + 2) +(feed-test + "trending actors order" + (map + (fn (e) (get e :actor)) + (feed/trending-actors S 100 50 10)) + (list "a" "b" "c")) diff --git a/lib/feed/thread.sx b/lib/feed/thread.sx new file mode 100644 index 00000000..a9522814 --- /dev/null +++ b/lib/feed/thread.sx @@ -0,0 +1,59 @@ +; feed/thread — conversation threading. A reply carries :reply-to +; (normalize preserves it). A thread is the transitive closure over :reply-to from +; a root object: root + replies + replies-to-replies, gathered chronologically. +; +; Requires: lib/feed/normalize.sx, lib/feed/stream.sx, lib/feed/fanout.sx +; (feed/-elem?, feed/-distinct). + +; direct replies to an object +(define + feed/replies + (fn + (stream object) + (feed/filter stream (fn (a) (equal? (get a :reply-to) object))))) + +(define + feed/reply-count + (fn (stream object) (feed/count (feed/replies stream object)))) + +; iterate f from x until the result stops growing (set-closure fixpoint) +(define + feed/-fixpoint + (fn + (f x) + (let + ((nx (f x))) + (if (= (len nx) (len x)) x (feed/-fixpoint f nx))))) + +; the set of object-ids in the thread rooted at `root` +(define + feed/thread-objects + (fn + (stream root) + (let + ((all (feed/items stream))) + (feed/-fixpoint + (fn + (acc) + (feed/-distinct + (append + acc + (map + (fn (a) (get a :object)) + (filter (fn (a) (feed/-elem? (get a :reply-to) acc)) all))))) + (list root))))) + +; the full thread as a chronological stream (root + all descendants) +(define + feed/thread + (fn + (stream root) + (let + ((objs (feed/thread-objects stream root))) + (feed/sort-by-at + (feed/filter stream (fn (a) (feed/-elem? (get a :object) objs))))))) + +; how many activities are in the thread (root counts as 1) +(define + feed/thread-size + (fn (stream root) (feed/count (feed/thread stream root)))) diff --git a/lib/feed/trending.sx b/lib/feed/trending.sx new file mode 100644 index 00000000..c0a48b1a --- /dev/null +++ b/lib/feed/trending.sx @@ -0,0 +1,42 @@ +; feed/trending — what's hot right now: objects (or actors) ranked by activity +; count within a recency window. Deterministic: count descending, ties broken by +; key ascending (entries are pre-sorted by key, then stable grade-down by count). +; +; Requires: lib/feed/stream.sx, lib/feed/aggregate.sx (object/actor-counts), +; lib/feed/rank.sx (feed/-desc-by). + +; activities within (now-window, now] +(define + feed/-recent + (fn + (stream now window) + (feed/filter + stream + (fn (a) (and (<= (get a :at) now) (> (get a :at) (- now window))))))) + +; counts dict -> top-N entries {label key, :count n}, count desc, key asc +(define + feed/-top-counts + (fn + (counts label n) + (let + ((entries (map (fn (k) (assoc {:count (get counts k)} label k)) (sort (keys counts))))) + (take (feed/-desc-by entries (fn (e) (get e :count))) n)))) + +; top-N trending objects in the window +(define + feed/trending + (fn + (stream now window n) + (feed/-top-counts + (feed/object-counts (feed/-recent stream now window)) + :object n))) + +; top-N most active actors in the window +(define + feed/trending-actors + (fn + (stream now window n) + (feed/-top-counts + (feed/actor-counts (feed/-recent stream now window)) + :actor n))) diff --git a/plans/feed-on-sx.md b/plans/feed-on-sx.md index 53acd1fd..4c0c29fb 100644 --- a/plans/feed-on-sx.md +++ b/plans/feed-on-sx.md @@ -14,7 +14,7 @@ APL, ACL visibility filtering via `lib/acl/`, federation via fed-sx. ## Status (rolling) -`bash lib/feed/conformance.sh` → **0/0** (not yet started) +`bash lib/feed/conformance.sh` → **189/189** (Phases 1–4 + TF-IDF, notifications, home, smart-dedupe, trending, mute, pagination, threading) ## Ground rules @@ -59,47 +59,118 @@ lib/feed/api.sx lib/feed/fed.sx ## Phase 1 — Stream model + basic ops -- [ ] `lib/feed/normalize.sx` — activity record schema; coerce arbitrary inputs -- [ ] `lib/feed/stream.sx` — APL vector representation; filter by predicate; sort by +- [x] `lib/feed/normalize.sx` — activity record schema; coerce arbitrary inputs +- [x] `lib/feed/stream.sx` — APL vector representation; filter by predicate; sort by `:at`; take N (`↑`); reverse (`⌽`) -- [ ] `lib/feed/api.sx` — `(feed/post activity)`, `(feed/all)` -- [ ] `lib/feed/tests/basic.sx` — 15+ cases: post, query, filter, sort -- [ ] `lib/feed/scoreboard.{json,md}` -- [ ] `lib/feed/conformance.sh` +- [x] `lib/feed/api.sx` — `(feed/post activity)`, `(feed/all)` +- [x] `lib/feed/tests/basic.sx` — 30 cases: normalize defaults, filter, sort, take, api +- [x] `lib/feed/scoreboard.{json,md}` +- [x] `lib/feed/conformance.sh` ## Phase 2 — Fanout via outer product -- [ ] follower graph: `followers user → vector of user ids` -- [ ] fanout: activities `∘.×` followers → matrix `(activity, follower)` pairs -- [ ] flatten to inbox events vector -- [ ] dedupe — group by `(actor, verb, object)` collapse to one inbox event per - receiver -- [ ] `lib/feed/tests/fanout.sx` — 20+ cases: small graph, mutual follow, popular - actor (high-fanout), cross-post dedupe +- [x] follower graph: `followers user → vector of user ids` (`feed/follow-graph`, + `feed/followers`; graph = `{followee -> (followers)}` dict) +- [x] fanout: activities `∘.×` audience → matrix via `apl-outer feed/-mk-event` +- [x] flatten to inbox events vector (`feed/-flatten` rank-2 → rank-1) +- [x] dedupe — `feed/dedupe-inbox` by `(to, actor, verb, object)`; also + `feed/dedupe-activities` `(actor verb object)` and `feed/dedupe-collapse` + `(verb object)` for cross-actor likes +- [x] `lib/feed/tests/fanout.sx` — 29 cases: small graph, mutual follow, star + (high-fanout), empty graph, unfollowed actor, cross-post dedupe ## Phase 3 — Aggregation + ranking -- [ ] group-by — `(actor, day) → count` via key-reduce -- [ ] velocity score — recent activity count over window -- [ ] recency score — decay by age -- [ ] composite rank — weighted sum of components -- [ ] top-N per timeline -- [ ] `lib/feed/tests/rank.sx` — 20+ cases: ranking stable on tie, decay shape, - per-user weighting +- [x] group-by — `feed/group-by`/`feed/group-count` key-reduce; `feed/by-actor-day` + buckets `(actor, day)` via `feed/day` (string-joined keys) +- [x] velocity score — `feed/velocity` counts actor's activities in `(at-window, at]` +- [x] recency score — `feed/recency` half-life decay `0.5^(age/hl)` +- [x] composite rank — `feed/composite` weighted sum of `(weight scorer)` parts +- [x] top-N per timeline — `feed/top` = rank then take +- [x] `lib/feed/tests/rank.sx` — 24 cases: decay shape, velocity burst, stable + tie-break, top-N, composite ## Phase 4 — Visibility filter + federation -- [ ] ACL filter — each candidate activity passed through `(acl/permit? viewer :read - activity)` -- [ ] fed-sx outbound — local `feed/post` fans out to remote followers' inboxes -- [ ] fed-sx inbound — peer activities arrive at local inbox -- [ ] backfill on subscribe — request peer history, merge into local stream -- [ ] `lib/feed/tests/integration.sx` — federated timeline with ACL applied +`lib/acl/` and fed-sx don't exist yet and are out of scope (import `lib/apl/` +only), so ACL/transport are injected: `permit?`, `remote?`, `send-fn`, `fetch-fn` +are function parameters. Real acl-sx / fed-sx wire in at the call site unchanged. + +- [x] ACL filter — `feed/visible stream viewer permit?`; default `feed/permit-acl?` + reads `:visible-to` allowlist (+ author-sees-own); per-viewer, never cached +- [x] fed-sx outbound — `feed/federate`/`feed/deliver` fan out then partition + local vs remote inboxes; remote events handed to injected `send-fn` +- [x] fed-sx inbound — `feed/inbound` normalizes + `feed/ingest` dedupes peer + activities into the local stream +- [x] backfill on subscribe — `feed/backfill local fetch-fn peer-id` +- [x] `lib/feed/tests/integration.sx` — 22 cases incl. end-to-end + `feed/timeline` (federated → ACL for viewer → recency rank → top-N) ## Progress log -(loop fills this in) +- **Phase 1 done (30/30).** Stream = APL rank-1 array whose ravel holds activity + dicts. `normalize.sx` (record schema + accessors), `stream.sx` (filter via `/` + compress, sort via `⍋` grade-up [stable], take via `↑`, reverse via `⌽`, + by-actor/verb/object/since predicates), `api.sx` (mutable log: post/all/reset!/size). + Substrate: `apl-compress`, `apl-grade-up`, `apl-take`, `apl-reverse`, `make-array`. + Grade-up returns 1-based indices (⎕IO=1), is stable on ties → deterministic sort. +- **Phase 2 done (59/59 total).** `fanout.sx` (graph + `apl-outer` showcase), + `dedupe.sx` (per-key dedupe, first-wins stable). Key APL gotcha: `scalar?` is + true for ANY dict and `disclose` nils a non-array dict, so an apl-outer combiner + MUST `enclose` its event dict — apl-outer discloses it back intact. `apl-unique` + preserves first-occurrence order; dict `keys` order is NOT stable, so + `feed/audience` sorts (else recipient ordering flakes). `apl-compress` needs a + rank-1 array, so the (activity×follower) matrix is flattened to its ravel before + the edge-guard filter. +- **Phase 3 done (83/83 total).** `aggregate.sx` (group-by/count, day buckets) + + `rank.sx` (recency/velocity/engagement scorers, composite, top-N). `sort` is + single-arg ascending only — no comparator — so ranking uses a stable two-pass + `apl-grade-down` (by :at desc, then by score desc) for deterministic tie-breaks. + Dict keys must be strings, so composite group keys are string-joined ("actor#day"). +- **Phase 4 done (105/105 total).** `acl.sx` (per-viewer `feed/visible`, + `feed/timeline` capstone) + `fed.sx` (merge/ingest/inbound/backfill/federate/ + deliver). ACL/transport are dependency-injected (permit?/remote?/send-fn/fetch-fn) + since lib/acl + fed-sx don't exist. `feed/normalize` now MERGEs defaults over the + raw dict (was projecting to 5 keys) so extra metadata (:visible-to, peer fields) + survives — matches the "flexible bag" principle. -## Blockers +## Roadmap is complete (all 4 phases). Possible follow-ups: -(loop fills this in) +- Wire real acl-sx once `lib/acl/` exists (swap injected `permit?`). +- Wire real fed-sx transport (swap `send-fn`/`fetch-fn`). +- [x] TF-IDF over `:tags` for content ranking — `content.sx`: `feed/tag-df`, + `feed/tag-idf` (log N/df), `feed/tfidf-score`, `feed/by-relevance`; 15 tests. + Composes as a scorer with rank.sx. (120/120 total.) +- [x] Notification feed (verb-filtered, per-recipient) — `notify.sx`: + `feed/notifications`, `feed/notify-verbs`, `feed/notify-digest` (collapses + "X, Y liked Z" by (verb,object), sorted-deterministic); 8 tests. (128/128 total.) +- [x] **Capstone** `feed/home` — the whole pipeline as one line: fanout ∘ inbox ∘ + dedupe ∘ ACL ∘ rank ∘ take (`home.sx`); 6 tests incl. per-viewer ACL + cross-post + dedupe. (134/134 total.) +- [x] Per-verb dedupe rules (briefing gotcha #3) — `feed/dedupe-smart` / + `feed/smart-key`: reactions (like/follow/boost/...) collapse cross-actor on + (verb,object); posts stay distinct per actor. `feed/collapse-verbs` is + rebindable policy; 9 tests. (143/143 total.) +- [x] Trending — `feed/trending` / `feed/trending-actors`: objects/actors ranked + by activity count in a recency window, count-desc with key-asc tiebreak + (`trending.sx`); 11 tests. (154/154 total.) +- [x] Mute/block — `feed/mute-actors` / `feed/mute-tags` / `feed/mute-objects` / + `feed/apply-prefs`: viewer-controlled per-request filtering (complements ACL's + author-controlled visibility) (`mute.sx`); 9 tests. (163/163 total.) +- [x] Pagination — `feed/page`/`feed/page-count` (offset) + `feed/before`/ + `feed/after`/`feed/page-before`/`feed/next-cursor` (cursor by :at, stable under + inserts) (`page.sx`); 14 tests. (177/177 total.) +- [x] Threading — `feed/replies`/`feed/reply-count`/`feed/thread`/ + `feed/thread-objects`/`feed/thread-size`: conversation closure over `:reply-to` + (transitive fixpoint), chronological (`thread.sx`); 12 tests. (189/189 total.) + +(none) + +## Notes for next iteration + +- sx-tree MCP tools take `file:` NOT `path:` (CLAUDE.md is stale). Wrong key → + `Yojson Type_error("Expected string, got null")`. Looks like a broken binary, isn't. +- sx_server binary lives in main repo: `/root/rose-ash/hosts/ocaml/_build/default/bin/sx_server.exe` + (worktree has no `_build`). conformance.sh already points there with relative fallback. +- Phase 2 substrate verified available: `apl-outer` (∘.×), `apl-member` (∊), + `apl-unique`, `apl-iota` (1-based).