feed: Phase 1 stream model — normalize, APL-backed filter/sort/take/reverse, post/all api + 30 tests

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

lib/feed/api.sx

@@ -0,0 +1,24 @@
+; feed/api — ergonomic API over the stream layer for non-APL callers.
+; A single mutable activity log; post appends, all returns it as a stream.
+;
+; Requires: lib/feed/normalize.sx, lib/feed/stream.sx (loaded by harness).
+
+(define feed/-log (list))
+
+; post — normalize then append. Returns the stored activity.
+(define
+  feed/post
+  (fn
+    (raw)
+    (let
+      ((a (feed/normalize raw)))
+      (begin (set! feed/-log (append feed/-log (list a))) a))))
+
+; all — the whole log as a stream (insertion order)
+(define feed/all (fn () (feed/stream feed/-log)))
+
+; reset! — clear the log (test hygiene)
+(define feed/reset! (fn () (begin (set! feed/-log (list)) nil)))
+
+; size — number of posted activities
+(define feed/size (fn () (len feed/-log)))

lib/feed/conformance.sh

@@ -0,0 +1,112 @@
+#!/usr/bin/env bash
+# lib/feed/conformance.sh — run feed test suites, emit scoreboard.json + scoreboard.md.
+
+set -uo pipefail
+cd "$(git rev-parse --show-toplevel)"
+
+SX_SERVER="${SX_SERVER:-/root/rose-ash/hosts/ocaml/_build/default/bin/sx_server.exe}"
+if [ ! -x "$SX_SERVER" ]; then
+  SX_SERVER="hosts/ocaml/_build/default/bin/sx_server.exe"
+fi
+if [ ! -x "$SX_SERVER" ]; then
+  echo "ERROR: sx_server.exe not found." >&2
+  exit 1
+fi
+
+SUITES=(basic)
+
+OUT_JSON="lib/feed/scoreboard.json"
+OUT_MD="lib/feed/scoreboard.md"
+
+run_suite() {
+  local suite=$1
+  local file="lib/feed/tests/${suite}.sx"
+  local TMP
+  TMP=$(mktemp)
+  cat > "$TMP" << EPOCHS
+(epoch 1)
+(load "spec/stdlib.sx")
+(load "lib/r7rs.sx")
+(load "lib/apl/runtime.sx")
+(load "lib/feed/normalize.sx")
+(load "lib/feed/stream.sx")
+(load "lib/feed/api.sx")
+(epoch 2)
+(eval "(define feed-test-pass 0)")
+(eval "(define feed-test-fail 0)")
+(eval "(define feed-test (fn (name got expected) (if (= got expected) (set! feed-test-pass (+ feed-test-pass 1)) (set! feed-test-fail (+ feed-test-fail 1)))))")
+(epoch 3)
+(load "${file}")
+(epoch 4)
+(eval "(list feed-test-pass feed-test-fail)")
+EPOCHS
+
+  local OUTPUT
+  OUTPUT=$(timeout 300 "$SX_SERVER" < "$TMP" 2>/dev/null)
+  rm -f "$TMP"
+
+  local LINE
+  LINE=$(echo "$OUTPUT" | awk '/^\(ok-len 4 / {getline; print; exit}')
+  if [ -z "$LINE" ]; then
+    LINE=$(echo "$OUTPUT" | grep -E '^\(ok 4 \([0-9]+ [0-9]+\)\)' | tail -1 \
+            | sed -E 's/^\(ok 4 //; s/\)$//')
+  fi
+
+  local P F
+  P=$(echo "$LINE" | sed -E 's/^\(([0-9]+) ([0-9]+)\).*/\1/')
+  F=$(echo "$LINE" | sed -E 's/^\(([0-9]+) ([0-9]+)\).*/\2/')
+  P=${P:-0}
+  F=${F:-0}
+  echo "${P} ${F}"
+}
+
+declare -A SUITE_PASS
+declare -A SUITE_FAIL
+TOTAL_PASS=0
+TOTAL_FAIL=0
+
+echo "Running feed conformance suite..." >&2
+for s in "${SUITES[@]}"; do
+  read -r p f < <(run_suite "$s")
+  SUITE_PASS[$s]=$p
+  SUITE_FAIL[$s]=$f
+  TOTAL_PASS=$((TOTAL_PASS + p))
+  TOTAL_FAIL=$((TOTAL_FAIL + f))
+  printf "  %-12s %d/%d\n" "$s" "$p" "$((p+f))" >&2
+done
+
+# scoreboard.json
+{
+  printf '{\n'
+  printf '  "suites": {\n'
+  first=1
+  for s in "${SUITES[@]}"; do
+    if [ $first -eq 0 ]; then printf ',\n'; fi
+    printf '    "%s": {"pass": %d, "fail": %d}' "$s" "${SUITE_PASS[$s]}" "${SUITE_FAIL[$s]}"
+    first=0
+  done
+  printf '\n  },\n'
+  printf '  "total_pass": %d,\n' "$TOTAL_PASS"
+  printf '  "total_fail": %d,\n' "$TOTAL_FAIL"
+  printf '  "total": %d\n' "$((TOTAL_PASS + TOTAL_FAIL))"
+  printf '}\n'
+} > "$OUT_JSON"
+
+# scoreboard.md
+{
+  printf '# feed Conformance Scoreboard\n\n'
+  printf '_Generated by `lib/feed/conformance.sh`_\n\n'
+  printf '| Suite | Pass | Fail | Total |\n'
+  printf '|-------|-----:|-----:|------:|\n'
+  for s in "${SUITES[@]}"; do
+    p=${SUITE_PASS[$s]}
+    f=${SUITE_FAIL[$s]}
+    printf '| %s | %d | %d | %d |\n' "$s" "$p" "$f" "$((p+f))"
+  done
+  printf '| **Total** | **%d** | **%d** | **%d** |\n' "$TOTAL_PASS" "$TOTAL_FAIL" "$((TOTAL_PASS + TOTAL_FAIL))"
+} > "$OUT_MD"
+
+echo "Wrote $OUT_JSON and $OUT_MD" >&2
+echo "Total: $TOTAL_PASS pass, $TOTAL_FAIL fail" >&2
+
+[ "$TOTAL_FAIL" -eq 0 ]

lib/feed/normalize.sx

@@ -0,0 +1,27 @@
+; feed/normalize — coerce arbitrary input into the canonical activity record.
+; An activity is a small dict {:actor :verb :object :at :tags}; a stream is an
+; APL vector of such dicts (see stream.sx).
+
+(define feed/activity-keys (list :actor :verb :object :at :tags))
+
+(define
+  feed/normalize
+  (fn
+    (raw)
+    (let ((d (if (= (type-of raw) "dict") raw {}))) {:actor (get d :actor "") :object (get d :object nil) :at (get d :at 0) :tags (let ((t (get d :tags (list)))) (if (list? t) t (list t))) :verb (get d :verb "post")})))
+
+(define
+  feed/activity
+  (fn (actor verb object at tags) (feed/normalize {:actor actor :object object :at at :tags tags :verb verb})))
+
+(define feed/actor (fn (a) (get a :actor)))
+(define feed/verb (fn (a) (get a :verb)))
+(define feed/object (fn (a) (get a :object)))
+(define feed/at (fn (a) (get a :at)))
+(define feed/tags (fn (a) (get a :tags)))
+
+(define
+  feed/activity?
+  (fn
+    (a)
+    (and (= (type-of a) "dict") (has-key? a :actor) (has-key? a :verb))))

lib/feed/scoreboard.json

@@ -0,0 +1,8 @@
+{
+  "suites": {
+    "basic": {"pass": 30, "fail": 0}
+  },
+  "total_pass": 30,
+  "total_fail": 0,
+  "total": 30
+}

lib/feed/scoreboard.md

@@ -0,0 +1,8 @@
+# feed Conformance Scoreboard
+
+_Generated by `lib/feed/conformance.sh`_
+
+| Suite | Pass | Fail | Total |
+|-------|-----:|-----:|------:|
+| basic | 30 | 0 | 30 |
+| **Total** | **30** | **0** | **30** |

lib/feed/stream.sx

@@ -0,0 +1,75 @@
+; feed/stream — a stream is an APL vector (rank-1 array) whose ravel holds
+; activity dicts. Operations lift APL primitives onto this shape: filter via
+; compress (/), sort via grade (⍋), take via ↑, reverse via ⌽.
+;
+; Requires: lib/apl/runtime.sx, lib/feed/normalize.sx (loaded by harness).
+
+(define feed/stream (fn (acts) (make-array (list (len acts)) acts)))
+
+(define feed/items (fn (s) (get s :ravel)))
+
+(define feed/count (fn (s) (len (get s :ravel))))
+
+(define feed/empty (feed/stream (list)))
+
+(define feed/empty? (fn (s) (= (feed/count s) 0)))
+
+; filter — bool mask ∘ compress. pred : activity -> truthy
+(define
+  feed/filter
+  (fn
+    (s pred)
+    (let
+      ((items (get s :ravel)))
+      (let
+        ((mask (make-array (list (len items)) (map (fn (a) (if (pred a) 1 0)) items))))
+        (apl-compress mask s)))))
+
+; sort-by — ascending, stable on ties (grade-up is stable). key-fn : activity -> number
+(define
+  feed/sort-by
+  (fn
+    (s key-fn)
+    (let
+      ((items (get s :ravel)))
+      (let
+        ((keys (make-array (list (len items)) (map key-fn items))))
+        (let
+          ((order (get (apl-grade-up keys) :ravel)))
+          (feed/stream (map (fn (i) (nth items (- i 1))) order)))))))
+
+(define feed/sort-by-at (fn (s) (feed/sort-by s feed/at)))
+
+; newest-first: ascending sort then reverse (⌽)
+(define feed/recent (fn (s) (apl-reverse (feed/sort-by-at s))))
+
+; take N (↑), clamped to stream length so it never over-takes/pads
+(define
+  feed/take
+  (fn
+    (s n)
+    (let
+      ((c (feed/count s)))
+      (if (>= n c) s (apl-take (apl-scalar n) s)))))
+
+(define feed/reverse (fn (s) (apl-reverse s)))
+
+; common predicates
+(define
+  feed/by-actor
+  (fn (s actor) (feed/filter s (fn (a) (equal? (get a :actor) actor)))))
+
+(define
+  feed/by-verb
+  (fn (s verb) (feed/filter s (fn (a) (equal? (get a :verb) verb)))))
+
+(define
+  feed/by-object
+  (fn
+    (s object)
+    (feed/filter s (fn (a) (equal? (get a :object) object)))))
+
+; activities at or after timestamp t
+(define
+  feed/since
+  (fn (s t) (feed/filter s (fn (a) (>= (get a :at) t)))))

lib/feed/tests/basic.sx

@@ -0,0 +1,118 @@
+; Phase 1 — normalize, stream ops, api. Uses the feed-test harness
+; (feed-test name got expected) provided by conformance.sh.
+
+; ---------- normalize ----------
+
+(feed-test
+  "normalize default actor"
+  (feed/actor (feed/normalize {}))
+  "")
+(feed-test
+  "normalize default verb"
+  (feed/verb (feed/normalize {}))
+  "post")
+(feed-test
+  "normalize default at"
+  (feed/at (feed/normalize {}))
+  0)
+(feed-test
+  "normalize default object"
+  (feed/object (feed/normalize {}))
+  nil)
+(feed-test
+  "normalize default tags"
+  (feed/tags (feed/normalize {}))
+  (list))
+(feed-test
+  "normalize keeps actor"
+  (feed/actor (feed/normalize {:actor "alice"}))
+  "alice")
+(feed-test
+  "normalize keeps verb"
+  (feed/verb (feed/normalize {:verb "like"}))
+  "like")
+(feed-test
+  "normalize scalar tag -> list"
+  (feed/tags (feed/normalize {:tags "x"}))
+  (list "x"))
+(feed-test
+  "normalize list tags kept"
+  (feed/tags (feed/normalize {:tags (list "a" "b")}))
+  (list "a" "b"))
+(feed-test
+  "activity constructor at"
+  (feed/at (feed/activity "a" "post" "o" 5 (list)))
+  5)
+(feed-test
+  "activity? on activity"
+  (feed/activity? (feed/normalize {:actor "a"}))
+  true)
+(feed-test "activity? on number" (feed/activity? 5) false)
+(feed-test "activity? on bare dict" (feed/activity? {:foo 1}) false)
+
+; ---------- stream ----------
+
+(define
+  S
+  (feed/stream
+    (list
+      (feed/activity "alice" "post" "p1" 30 (list))
+      (feed/activity "bob" "like" "p1" 10 (list))
+      (feed/activity "alice" "post" "p2" 20 (list)))))
+
+(feed-test "stream count" (feed/count S) 3)
+(feed-test "stream items len" (len (feed/items S)) 3)
+(feed-test
+  "sort-by-at actors asc"
+  (map feed/actor (feed/items (feed/sort-by-at S)))
+  (list "bob" "alice" "alice"))
+(feed-test
+  "recent newest first"
+  (map feed/at (feed/items (feed/recent S)))
+  (list 30 20 10))
+(feed-test
+  "take 2 of recent"
+  (feed/count (feed/take (feed/recent S) 2))
+  2)
+(feed-test
+  "take clamps past end"
+  (feed/count (feed/take S 10))
+  3)
+(feed-test
+  "by-actor alice count"
+  (feed/count (feed/by-actor S "alice"))
+  2)
+(feed-test
+  "by-verb like actor"
+  (map feed/actor (feed/items (feed/by-verb S "like")))
+  (list "bob"))
+(feed-test
+  "by-object p1 count"
+  (feed/count (feed/by-object S "p1"))
+  2)
+(feed-test
+  "since 20 count"
+  (feed/count (feed/since S 20))
+  2)
+(feed-test
+  "reverse ats"
+  (map feed/at (feed/items (feed/reverse S)))
+  (list 20 10 30))
+(feed-test "empty? on empty" (feed/empty? feed/empty) true)
+(feed-test
+  "empty? on filtered-out"
+  (feed/empty? (feed/by-actor S "zzz"))
+  true)
+
+; ---------- api ----------
+
+(feed/reset!)
+(feed/post {:actor "x" :at 1 :verb "post"})
+(feed/post {:actor "y" :at 2 :verb "like"})
+(feed-test "api size after posts" (feed/size) 2)
+(feed-test "api all count" (feed/count (feed/all)) 2)
+(feed-test
+  "post returns normalized verb"
+  (feed/verb (feed/post {:actor "z"}))
+  "post")
+(feed-test "api size after third post" (feed/size) 3)

plans/feed-on-sx.md

@@ -14,7 +14,7 @@ APL, ACL visibility filtering via `lib/acl/`, federation via fed-sx.
 
 ## Status (rolling)
 
-`bash lib/feed/conformance.sh` → **0/0** (not yet started)
+`bash lib/feed/conformance.sh` → **30/30** (Phase 1 complete)
 
 ## Ground rules
 
@@ -59,13 +59,13 @@ lib/feed/api.sx           lib/feed/fed.sx
 
 ## Phase 1 — Stream model + basic ops
 
-- [ ] `lib/feed/normalize.sx` — activity record schema; coerce arbitrary inputs
+- [x] `lib/feed/normalize.sx` — activity record schema; coerce arbitrary inputs
-- [ ] `lib/feed/stream.sx` — APL vector representation; filter by predicate; sort by
+- [x] `lib/feed/stream.sx` — APL vector representation; filter by predicate; sort by
   `:at`; take N (`↑`); reverse (`⌽`)
-- [ ] `lib/feed/api.sx` — `(feed/post activity)`, `(feed/all)`
+- [x] `lib/feed/api.sx` — `(feed/post activity)`, `(feed/all)`
-- [ ] `lib/feed/tests/basic.sx` — 15+ cases: post, query, filter, sort
+- [x] `lib/feed/tests/basic.sx` — 30 cases: normalize defaults, filter, sort, take, api
-- [ ] `lib/feed/scoreboard.{json,md}`
+- [x] `lib/feed/scoreboard.{json,md}`
-- [ ] `lib/feed/conformance.sh`
+- [x] `lib/feed/conformance.sh`
 
 ## Phase 2 — Fanout via outer product
 
@@ -98,8 +98,22 @@ lib/feed/api.sx           lib/feed/fed.sx
 
 ## Progress log
 
-(loop fills this in)
+- **Phase 1 done (30/30).** Stream = APL rank-1 array whose ravel holds activity
+  dicts. `normalize.sx` (record schema + accessors), `stream.sx` (filter via `/`
+  compress, sort via `⍋` grade-up [stable], take via `↑`, reverse via `⌽`,
+  by-actor/verb/object/since predicates), `api.sx` (mutable log: post/all/reset!/size).
+  Substrate: `apl-compress`, `apl-grade-up`, `apl-take`, `apl-reverse`, `make-array`.
+  Grade-up returns 1-based indices (⎕IO=1), is stable on ties → deterministic sort.
 
 ## Blockers
 
-(loop fills this in)
+(none)
+
+## Notes for next iteration
+
+- sx-tree MCP tools take `file:` NOT `path:` (CLAUDE.md is stale). Wrong key →
+  `Yojson Type_error("Expected string, got null")`. Looks like a broken binary, isn't.
+- sx_server binary lives in main repo: `/root/rose-ash/hosts/ocaml/_build/default/bin/sx_server.exe`
+  (worktree has no `_build`). conformance.sh already points there with relative fallback.
+- Phase 2 substrate verified available: `apl-outer` (∘.×), `apl-member` (∊),
+  `apl-unique`, `apl-iota` (1-based).