diff --git a/lib/feed/aggregate.sx b/lib/feed/aggregate.sx new file mode 100644 index 00000000..cc146cd3 --- /dev/null +++ b/lib/feed/aggregate.sx @@ -0,0 +1,62 @@ +; feed/aggregate — group-by / counting via key-reduce. Keys must be strings +; (dict keys), so composite keys (actor, day) are joined into one string. +; +; Requires: lib/feed/normalize.sx, lib/feed/stream.sx. + +; group activities into a dict: key-string -> (list of activities), order-preserving +(define + feed/group-by + (fn + (stream key-fn) + (reduce + (fn + (g a) + (let + ((k (key-fn a))) + (assoc g k (append (get g k (list)) (list a))))) + {} + (feed/items stream)))) + +; key-string -> count +(define + feed/group-count + (fn + (stream key-fn) + (reduce + (fn + (g a) + (let + ((k (key-fn a))) + (assoc g k (+ (get g k 0) 1)))) + {} + (feed/items stream)))) + +; --- composite keys --------------------------------------------------------- + +(define feed/day (fn (at window) (floor (/ at window)))) + +; (actor, day-bucket) -> "actor#day" +(define + feed/actor-day-key + (fn + (window) + (fn + (a) + (string-append + (get a :actor) + "#" + (number->string (feed/day (get a :at) window)))))) + +(define + feed/by-actor-day + (fn (stream window) (feed/group-count stream (feed/actor-day-key window)))) + +; per-actor activity counts +(define + feed/actor-counts + (fn (stream) (feed/group-count stream feed/actor))) + +; per-object activity counts (engagement) +(define + feed/object-counts + (fn (stream) (feed/group-count stream feed/object))) diff --git a/lib/feed/conformance.sh b/lib/feed/conformance.sh index 4560ec45..eb51dc88 100755 --- a/lib/feed/conformance.sh +++ b/lib/feed/conformance.sh @@ -13,7 +13,7 @@ if [ ! -x "$SX_SERVER" ]; then exit 1 fi -SUITES=(basic fanout) +SUITES=(basic fanout rank) OUT_JSON="lib/feed/scoreboard.json" OUT_MD="lib/feed/scoreboard.md" @@ -33,6 +33,8 @@ run_suite() { (load "lib/feed/api.sx") (load "lib/feed/fanout.sx") (load "lib/feed/dedupe.sx") +(load "lib/feed/aggregate.sx") +(load "lib/feed/rank.sx") (epoch 2) (eval "(define feed-test-pass 0)") (eval "(define feed-test-fail 0)") diff --git a/lib/feed/rank.sx b/lib/feed/rank.sx new file mode 100644 index 00000000..36e30411 --- /dev/null +++ b/lib/feed/rank.sx @@ -0,0 +1,92 @@ +; feed/rank — scoring + ranking. Scorers are (activity -> number). Ranking is a +; stable two-pass grade-down: first by :at descending (the tiebreak), then by +; score descending — so ties resolve by recency, then by input order. Fully +; deterministic on ties. +; +; Requires: lib/apl/runtime.sx, lib/feed/normalize.sx, lib/feed/stream.sx. + +; --- scorers ---------------------------------------------------------------- + +; recency: half-life decay. score = 0.5 ^ (age / half-life). at==now -> 1.0. +(define + feed/recency + (fn + (now half-life) + (fn (a) (expt 0.5 (/ (- now (get a :at)) half-life))))) + +; velocity: how many of this actor's activities fall in (at-window, at] — +; a burst of recent activity scores higher. +(define + feed/velocity + (fn + (stream window) + (fn + (a) + (len + (filter + (fn + (b) + (and + (equal? (get b :actor) (get a :actor)) + (<= (get b :at) (get a :at)) + (> (get b :at) (- (get a :at) window)))) + (feed/items stream)))))) + +; engagement: how many activities in the stream touch this activity's :object +(define + feed/engagement + (fn + (stream) + (fn + (a) + (len + (filter + (fn (b) (equal? (get b :object) (get a :object))) + (feed/items stream)))))) + +; composite: weighted sum. parts = (list (list weight scorer) ...) +(define + feed/composite + (fn + (parts) + (fn + (a) + (reduce + (fn (acc p) (+ acc (* (first p) ((nth p 1) a)))) + 0 + parts)))) + +; --- ranking ---------------------------------------------------------------- + +; stable reorder of items by key-fn, descending (grade-down is stable) +(define + feed/-desc-by + (fn + (items key-fn) + (let + ((keys (make-array (list (len items)) (map key-fn items)))) + (let + ((order (get (apl-grade-down keys) :ravel))) + (map (fn (i) (nth items (- i 1))) order))))) + +; rank by score descending; ties -> :at descending -> input order +(define + feed/rank + (fn + (stream score-fn) + (let + ((by-at (feed/-desc-by (feed/items stream) feed/at))) + (feed/stream (feed/-desc-by by-at score-fn))))) + +; attach a :score to each activity (for inspection / debugging) +(define + feed/with-scores + (fn + (stream score-fn) + (feed/stream + (map (fn (a) (assoc a :score (score-fn a))) (feed/items stream))))) + +; top-N ranked timeline +(define + feed/top + (fn (stream score-fn n) (feed/take (feed/rank stream score-fn) n))) diff --git a/lib/feed/scoreboard.json b/lib/feed/scoreboard.json index d72fe5d9..df5bd7e4 100644 --- a/lib/feed/scoreboard.json +++ b/lib/feed/scoreboard.json @@ -1,9 +1,10 @@ { "suites": { "basic": {"pass": 30, "fail": 0}, - "fanout": {"pass": 29, "fail": 0} + "fanout": {"pass": 29, "fail": 0}, + "rank": {"pass": 24, "fail": 0} }, - "total_pass": 59, + "total_pass": 83, "total_fail": 0, - "total": 59 + "total": 83 } diff --git a/lib/feed/scoreboard.md b/lib/feed/scoreboard.md index c264255c..feb827a8 100644 --- a/lib/feed/scoreboard.md +++ b/lib/feed/scoreboard.md @@ -6,4 +6,5 @@ _Generated by `lib/feed/conformance.sh`_ |-------|-----:|-----:|------:| | basic | 30 | 0 | 30 | | fanout | 29 | 0 | 29 | -| **Total** | **59** | **0** | **59** | +| rank | 24 | 0 | 24 | +| **Total** | **83** | **0** | **83** | diff --git a/lib/feed/tests/rank.sx b/lib/feed/tests/rank.sx new file mode 100644 index 00000000..920bb99e --- /dev/null +++ b/lib/feed/tests/rank.sx @@ -0,0 +1,160 @@ +; Phase 3 — aggregation + ranking. (feed-test name got expected) + +; ---------- aggregation ---------- + +(define + A + (feed/stream + (list + (feed/activity "alice" "post" "p1" 5 (list)) + (feed/activity "alice" "post" "p2" 15 (list)) + (feed/activity "bob" "post" "p3" 25 (list)) + (feed/activity "alice" "like" "p1" 35 (list))))) + +(feed-test "actor-counts" (feed/actor-counts A) {:alice 3 :bob 1}) +(feed-test "object-counts" (feed/object-counts A) {:p2 1 :p3 1 :p1 2}) +(feed-test + "group-by actor alice len" + (len (get (feed/group-by A feed/actor) "alice")) + 3) +(feed-test + "group-count empty" + (feed/group-count feed/empty feed/actor) + {}) + +; day bucketing +(define + D + (feed/stream + (list + (feed/activity "alice" "post" "p1" 5 (list)) + (feed/activity "alice" "post" "p2" 8 (list)) + (feed/activity "alice" "post" "p3" 12 (list))))) + +(feed-test "feed/day floor" (feed/day 12 10) 1) +(feed-test "feed/day same bucket" (feed/day 8 10) 0) +(feed-test "by-actor-day" (feed/by-actor-day D 10) {:alice#0 2 :alice#1 1}) + +; ---------- recency ---------- + +(define rec (feed/recency 100 10)) +(feed-test + "recency at=now -> 1" + (rec (feed/activity "x" "post" "o" 100 (list))) + 1) +(feed-test + "recency age=hl -> .5" + (rec (feed/activity "x" "post" "o" 90 (list))) + 0.5) +(feed-test + "recency age=2hl -> .25" + (rec (feed/activity "x" "post" "o" 80 (list))) + 0.25) + +; ---------- velocity ---------- + +(define vel (feed/velocity D 10)) +(feed-test + "velocity burst (at=12)" + (vel (feed/activity "alice" "post" "z" 12 (list))) + 3) +(feed-test + "velocity mid (at=8)" + (vel (feed/activity "alice" "post" "z" 8 (list))) + 2) +(feed-test + "velocity first (at=5)" + (vel (feed/activity "alice" "post" "z" 5 (list))) + 1) +(feed-test + "velocity other actor" + (vel (feed/activity "bob" "post" "z" 12 (list))) + 0) + +; ---------- engagement ---------- + +(define eng (feed/engagement A)) +(feed-test + "engagement p1" + (eng (feed/activity "x" "post" "p1" 0 (list))) + 2) +(feed-test + "engagement p2" + (eng (feed/activity "x" "post" "p2" 0 (list))) + 1) + +; ---------- composite ---------- + +(define + cmp1 + (feed/composite (list (list 2 (fn (a) (get a :at)))))) +(feed-test + "composite single part" + (cmp1 (feed/activity "x" "post" "o" 5 (list))) + 10) +(define + cmp2 + (feed/composite + (list + (list 2 (fn (a) (get a :at))) + (list 3 (fn (a) 1))))) +(feed-test + "composite two parts" + (cmp2 (feed/activity "x" "post" "o" 5 (list))) + 13) + +; ---------- ranking ---------- + +(define + R + (feed/stream + (list + (feed/activity "u" "post" "oC" 80 (list)) + (feed/activity "u" "post" "oA" 100 (list)) + (feed/activity "u" "post" "oB" 90 (list))))) + +(feed-test + "rank by recency objects" + (map (fn (a) (get a :object)) (feed/items (feed/rank R rec))) + (list "oA" "oB" "oC")) +(feed-test + "top-2 by recency" + (map (fn (a) (get a :object)) (feed/items (feed/top R rec 2))) + (list "oA" "oB")) +(feed-test "top-2 count" (feed/count (feed/top R rec 2)) 2) + +; constant score -> tiebreak by :at descending +(define + T + (feed/stream + (list + (feed/activity "u" "post" "f" 10 (list)) + (feed/activity "u" "post" "g" 30 (list)) + (feed/activity "u" "post" "h" 20 (list))))) +(feed-test + "tiebreak at-desc" + (map + (fn (a) (get a :object)) + (feed/items (feed/rank T (fn (a) 0)))) + (list "g" "h" "f")) + +; equal score AND equal :at -> stable input order +(define + E + (feed/stream + (list + (feed/activity "u" "post" "first" 50 (list)) + (feed/activity "u" "post" "second" 50 (list))))) +(feed-test + "stable equal-key input order" + (map + (fn (a) (get a :object)) + (feed/items (feed/rank E (fn (a) 0)))) + (list "first" "second")) + +(feed-test + "with-scores attaches score" + (get (nth (feed/items (feed/with-scores R rec)) 1) :score) + 1) + +(feed-test "rank preserves count" (feed/count (feed/rank A rec)) 4) diff --git a/plans/feed-on-sx.md b/plans/feed-on-sx.md index 665f70a1..8dc48552 100644 --- a/plans/feed-on-sx.md +++ b/plans/feed-on-sx.md @@ -14,7 +14,7 @@ APL, ACL visibility filtering via `lib/acl/`, federation via fed-sx. ## Status (rolling) -`bash lib/feed/conformance.sh` → **59/59** (Phases 1–2 complete) +`bash lib/feed/conformance.sh` → **83/83** (Phases 1–3 complete) ## Ground rules @@ -81,13 +81,14 @@ lib/feed/api.sx lib/feed/fed.sx ## Phase 3 — Aggregation + ranking -- [ ] group-by — `(actor, day) → count` via key-reduce -- [ ] velocity score — recent activity count over window -- [ ] recency score — decay by age -- [ ] composite rank — weighted sum of components -- [ ] top-N per timeline -- [ ] `lib/feed/tests/rank.sx` — 20+ cases: ranking stable on tie, decay shape, - per-user weighting +- [x] group-by — `feed/group-by`/`feed/group-count` key-reduce; `feed/by-actor-day` + buckets `(actor, day)` via `feed/day` (string-joined keys) +- [x] velocity score — `feed/velocity` counts actor's activities in `(at-window, at]` +- [x] recency score — `feed/recency` half-life decay `0.5^(age/hl)` +- [x] composite rank — `feed/composite` weighted sum of `(weight scorer)` parts +- [x] top-N per timeline — `feed/top` = rank then take +- [x] `lib/feed/tests/rank.sx` — 24 cases: decay shape, velocity burst, stable + tie-break, top-N, composite ## Phase 4 — Visibility filter + federation @@ -114,6 +115,11 @@ lib/feed/api.sx lib/feed/fed.sx `feed/audience` sorts (else recipient ordering flakes). `apl-compress` needs a rank-1 array, so the (activity×follower) matrix is flattened to its ravel before the edge-guard filter. +- **Phase 3 done (83/83 total).** `aggregate.sx` (group-by/count, day buckets) + + `rank.sx` (recency/velocity/engagement scorers, composite, top-N). `sort` is + single-arg ascending only — no comparator — so ranking uses a stable two-pass + `apl-grade-down` (by :at desc, then by score desc) for deterministic tie-breaks. + Dict keys must be strings, so composite group keys are string-joined ("actor#day"). (none)