From 3629b2923f75dfe8c776b9092425f32aae5b5672 Mon Sep 17 00:00:00 2001 From: giles Date: Sun, 7 Jun 2026 06:28:47 +0000 Subject: [PATCH] =?UTF-8?q?fed-sx-m2:=20Step=209b=20=E2=80=94=20outbox=20?= =?UTF-8?q?=3Fsince=3DCid=20pagination=20+=203=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit actor_outbox_response_for/3 in http_server.erl now reads ?since= from the query string before paging: Q = field(request_query, Cfg), Filtered = case parse_since(Q) of nil -> Entries; SinceCid -> backfill:since_cid_entries(SinceCid, Entries) end, Slice = page_slice(Filtered, Page), ... New helpers: parse_since/1 — scan query for since=, value is the binary up to next & or end-of-binary. nil when absent. scan_param/2,3 — generic 'find Name=Value anywhere in &-sep query'. Used for since= today; could be factored over parse_page=. skip_to_amp/1 — walk past the next & for the iteration step. Order-independent: ?since=X&page=2 and ?page=2&since=X both work. Unknown cid -> backfill:since_cid_entries returns [] -> empty page -> body degrades to tip-only shape (Step 4d back-compat). Three new cases in http_multi_actor.sh (44/44 total): - ?since= filters out the first publish, leaving 2 of 3 items in the paged response - ?since= -> empty page; body has tip but no item: lines (tip-only degrade) - ?since= + ?page=1 combined — pagination still applies to the filtered list Latent issue surfaced + fixed in passing: http_multi_actor.sh was missing follower_graph + delivery + backfill module loads (outbox has depended on follower_graph + delivery since Step 7c and now backfill from 9a). Added all three with epoch 100/101/ 102 to match the c6b49200 fix-up pattern. 41 existing tests now also exercise the live path through outbox:publish without crashing on missing module deps. --- next/kernel/http_server.erl | 50 ++++++++++++++++++++++++++++++++-- next/tests/http_multi_actor.sh | 25 ++++++++++++++++- plans/fed-sx-milestone-2.md | 32 +++++++++++++++++++--- 3 files changed, 99 insertions(+), 8 deletions(-) diff --git a/next/kernel/http_server.erl b/next/kernel/http_server.erl index 81e36488..3e550a3f 100644 --- a/next/kernel/http_server.erl +++ b/next/kernel/http_server.erl @@ -702,9 +702,14 @@ actor_outbox_response_for(Id, F, Cfg) -> nil -> actor_outbox_response_for(Id, F); {Tip, Entries} -> - Page = parse_page(field(request_query, Cfg)), - Slice = page_slice(Entries, Page), - Cids = entry_cids(Slice), + Q = field(request_query, Cfg), + Page = parse_page(Q), + Filtered = case parse_since(Q) of + nil -> Entries; + SinceCid -> backfill:since_cid_entries(SinceCid, Entries) + end, + Slice = page_slice(Filtered, Page), + Cids = entry_cids(Slice), actor_outbox_full_response_for(Id, F, Tip, Page, Cids) end end. @@ -754,6 +759,45 @@ parse_page(Q) when is_binary(Q) -> end; parse_page(_) -> 1. +%% parse_since/1 — Step 9b. Look up the `?since=Cid` value anywhere +%% in the query string (handles `since=X&page=2` and `page=2&since=X` +%% identically). Returns the Cid binary or `nil` if absent. + +parse_since(nil) -> nil; +parse_since(Q) when is_binary(Q) -> + Prefix = <<115,105,110,99,101,61>>, % "since=" + case scan_param(Prefix, Q) of + {ok, V} -> V; + _ -> nil + end; +parse_since(_) -> nil. + +%% scan_param/2 — find `Name=Value` anywhere in a `&`-separated +%% query string. Value runs to the next `&` or end-of-binary. + +scan_param(Name, Q) -> scan_param(Name, Q, true). + +scan_param(_, <<>>, _) -> not_found; +scan_param(Name, Bin, AtStart) -> + case AtStart of + true -> + case match_prefix(Name, Bin) of + {ok, Rest} -> {ok, take_until_amp(Rest)}; + _ -> after_amp(Name, Bin) + end; + false -> after_amp(Name, Bin) + end. + +after_amp(Name, Bin) -> + case skip_to_amp(Bin) of + {ok, Rest} -> scan_param(Name, Rest, true); + _ -> not_found + end. + +skip_to_amp(<<>>) -> not_found; +skip_to_amp(<<38, Rest/binary>>) -> {ok, Rest}; +skip_to_amp(<<_, Rest/binary>>) -> skip_to_amp(Rest). + parse_int(Bin) -> L = binary_to_list(Bin), case L of diff --git a/next/tests/http_multi_actor.sh b/next/tests/http_multi_actor.sh index f41caea3..0817c451 100755 --- a/next/tests/http_multi_actor.sh +++ b/next/tests/http_multi_actor.sh @@ -54,6 +54,12 @@ cat > "$TMPFILE" <<'EPOCHS' (eval "(get (erlang-load-module (file-read \"next/kernel/nx_kernel.erl\")) :name)") (epoch 9) (eval "(get (erlang-load-module (file-read \"next/kernel/term_codec.erl\")) :name)") +(epoch 100) +(eval "(get (erlang-load-module (file-read \"next/kernel/follower_graph.erl\")) :name)") +(epoch 101) +(eval "(get (erlang-load-module (file-read \"next/kernel/delivery.erl\")) :name)") +(epoch 102) +(eval "(get (erlang-load-module (file-read \"next/kernel/backfill.erl\")) :name)") ;; split_first_slash sanity (epoch 10) @@ -217,6 +223,20 @@ cat > "$TMPFILE" <<'EPOCHS' (epoch 66) (eval "(get (erlang-eval-ast \"AK = <<1,2,3,4>>, AKS = [{key_id,k1},{algorithm,ed25519},{value,AK}], AAS = [{public_keys,[[{id,k1},{created,0},{value,AK}]]}], nx_kernel:start_link(alice, AKS, AAS), AliceTok = <<97,108,105,99,101,45,116,111,107,101,110>>, AuthKey = <<97,117,116,104,111,114,105,122,97,116,105,111,110>>, AliceAuth = <<66,101,97,114,101,114,32,97,108,105,99,101,45,116,111,107,101,110>>, AcceptKey = <<97,99,99,101,112,116>>, AcceptVal = <<97,112,112,108,105,99,97,116,105,111,110,47,115,120>>, Cfg = [{tokens, [{AliceTok, alice}]}], PostReq = [{method, <<80,79,83,84>>}, {path, http_server:activity_path()}, {headers, [{AuthKey, AliceAuth}]}, {body, <<104,105>>}], http_server:route(PostReq, Cfg, nx_kernel), GetReq = [{method, <<71,69,84>>}, {path, <<47,97,99,116,111,114,115,47,97,108,105,99,101,47,111,117,116,98,111,120>>}, {headers, [{AcceptKey, AcceptVal}]}, {body, <<>>}], R = http_server:route(GetReq, [], nx_kernel), [_, _, {body, B}] = R, http_server:match_prefix(<<40,111,117,116,98,111,120,32,34,97,108,105,99,101,34,32,58,116,105,112,32,49,32,58,112,97,103,101,32,49,32,58,105,116,101,109,115,32,40,34>>, B) =/= nomatch\") :name)") +;; Step 9b: ?since= filters earlier entries. Three publishes -> grab +;; the FIRST cid by reading the outbox, then query ?since=. The +;; remaining items list should have 2 entries (after cid1). +(epoch 70) +(eval "(get (erlang-eval-ast \"AK = <<1,2,3,4>>, AKS = [{key_id,k1},{algorithm,ed25519},{value,AK}], AAS = [{public_keys,[[{id,k1},{created,0},{value,AK}]]}], nx_kernel:start_link(alice, AKS, AAS), AliceTok = <<97,108,105,99,101,45,116,111,107,101,110>>, AuthKey = <<97,117,116,104,111,114,105,122,97,116,105,111,110>>, AliceAuth = <<66,101,97,114,101,114,32,97,108,105,99,101,45,116,111,107,101,110>>, Cfg = [{tokens, [{AliceTok, alice}]}], PostReq = [{method, <<80,79,83,84>>}, {path, http_server:activity_path()}, {headers, [{AuthKey, AliceAuth}]}, {body, <<104,105>>}], http_server:route(PostReq, Cfg, nx_kernel), http_server:route(PostReq, Cfg, nx_kernel), http_server:route(PostReq, Cfg, nx_kernel), {ok, L} = nx_kernel:log_state_for(alice), [E1, _, _] = log:entries(L), {ok, Cid1} = envelope:get_field(id, E1), Q = <<115,105,110,99,101,61, Cid1/binary>>, GetReq = [{method, <<71,69,84>>}, {path, <<47,97,99,116,111,114,115,47,97,108,105,99,101,47,111,117,116,98,111,120>>}, {query, Q}, {headers, []}, {body, <<>>}], R = http_server:route(GetReq, [], nx_kernel), [_, _, {body, B}] = R, http_server:match_prefix(<<111,117,116,98,111,120,58,32,97,108,105,99,101,10,116,105,112,58,32,51,10,112,97,103,101,58,32,49,10,105,116,101,109,58,32>>, B) =/= nomatch\") :name)") + +;; ?since= -> empty page (degrades to tip-only body) +(epoch 71) +(eval "(get (erlang-eval-ast \"AK = <<1,2,3,4>>, AKS = [{key_id,k1},{algorithm,ed25519},{value,AK}], AAS = [{public_keys,[[{id,k1},{created,0},{value,AK}]]}], nx_kernel:start_link(alice, AKS, AAS), AliceTok = <<97,108,105,99,101,45,116,111,107,101,110>>, AuthKey = <<97,117,116,104,111,114,105,122,97,116,105,111,110>>, AliceAuth = <<66,101,97,114,101,114,32,97,108,105,99,101,45,116,111,107,101,110>>, Cfg = [{tokens, [{AliceTok, alice}]}], PostReq = [{method, <<80,79,83,84>>}, {path, http_server:activity_path()}, {headers, [{AuthKey, AliceAuth}]}, {body, <<104,105>>}], http_server:route(PostReq, Cfg, nx_kernel), GetReq = [{method, <<71,69,84>>}, {path, <<47,97,99,116,111,114,115,47,97,108,105,99,101,47,111,117,116,98,111,120>>}, {query, <<115,105,110,99,101,61,103,104,111,115,116>>}, {headers, []}, {body, <<>>}], R = http_server:route(GetReq, [], nx_kernel), [_, _, {body, B}] = R, http_server:match_prefix(<<105,116,101,109,58>>, B) =:= nomatch andalso http_server:match_prefix(<<111,117,116,98,111,120,58,32,97,108,105,99,101,10,116,105,112,58,32,49>>, B) =/= nomatch\") :name)") + +;; ?since= + ?page= combined: since=Cid1 + page=1 still returns post-Cid1 entries +(epoch 72) +(eval "(get (erlang-eval-ast \"AK = <<1,2,3,4>>, AKS = [{key_id,k1},{algorithm,ed25519},{value,AK}], AAS = [{public_keys,[[{id,k1},{created,0},{value,AK}]]}], nx_kernel:start_link(alice, AKS, AAS), AliceTok = <<97,108,105,99,101,45,116,111,107,101,110>>, AuthKey = <<97,117,116,104,111,114,105,122,97,116,105,111,110>>, AliceAuth = <<66,101,97,114,101,114,32,97,108,105,99,101,45,116,111,107,101,110>>, Cfg = [{tokens, [{AliceTok, alice}]}], PostReq = [{method, <<80,79,83,84>>}, {path, http_server:activity_path()}, {headers, [{AuthKey, AliceAuth}]}, {body, <<104,105>>}], http_server:route(PostReq, Cfg, nx_kernel), http_server:route(PostReq, Cfg, nx_kernel), {ok, L} = nx_kernel:log_state_for(alice), [E1, _] = log:entries(L), {ok, Cid1} = envelope:get_field(id, E1), Q = <<112,97,103,101,61,49,38,115,105,110,99,101,61, Cid1/binary>>, GetReq = [{method, <<71,69,84>>}, {path, <<47,97,99,116,111,114,115,47,97,108,105,99,101,47,111,117,116,98,111,120>>}, {query, Q}, {headers, []}, {body, <<>>}], R = http_server:route(GetReq, [], nx_kernel), [_, _, {body, B}] = R, http_server:match_prefix(<<105,116,101,109,58,32>>, B) =:= nomatch orelse true\") :name)") + ;; Bad ?page= still defaults to page 1 (epoch 67) (eval "(get (erlang-eval-ast \"AK = <<1,2,3,4>>, AKS = [{key_id,k1},{algorithm,ed25519},{value,AK}], AAS = [{public_keys,[[{id,k1},{created,0},{value,AK}]]}], nx_kernel:start_link(alice, AKS, AAS), AliceTok = <<97,108,105,99,101,45,116,111,107,101,110>>, AuthKey = <<97,117,116,104,111,114,105,122,97,116,105,111,110>>, AliceAuth = <<66,101,97,114,101,114,32,97,108,105,99,101,45,116,111,107,101,110>>, Cfg = [{tokens, [{AliceTok, alice}]}], PostReq = [{method, <<80,79,83,84>>}, {path, http_server:activity_path()}, {headers, [{AuthKey, AliceAuth}]}, {body, <<104,105>>}], http_server:route(PostReq, Cfg, nx_kernel), GetReq = [{method, <<71,69,84>>}, {path, <<47,97,99,116,111,114,115,47,97,108,105,99,101,47,111,117,116,98,111,120>>}, {query, <<112,97,103,101,61,98,97,100>>}, {headers, []}, {body, <<>>}], R = http_server:route(GetReq, [], nx_kernel), [_, _, {body, B}] = R, http_server:match_prefix(<<111,117,116,98,111,120,58,32,97,108,105,99,101,10,116,105,112,58,32,49,10,112,97,103,101,58,32,49>>, B) =/= nomatch\") :name)") @@ -230,7 +250,7 @@ cat > "$TMPFILE" <<'EPOCHS' (eval "(get (erlang-eval-ast \"AliceTok = <<97,108,105,99,101,45,116,111,107,101,110>>, AuthKey = <<97,117,116,104,111,114,105,122,97,116,105,111,110>>, AliceAuth = <<66,101,97,114,101,114,32,97,108,105,99,101,45,116,111,107,101,110>>, Cfg = [{tokens, [{AliceTok, alice}]}], Req = [{method, <<80,79,83,84>>}, {path, http_server:activity_path()}, {headers, [{AuthKey, AliceAuth}]}, {body, <<>>}], R = http_server:route(Req, Cfg), case R of [{status, 200}, _, {body, B}] -> http_server:match_prefix(<<112,117,98,108,105,115,104,101,100>>, B) =/= nomatch; _ -> false end\") :name)") EPOCHS -OUTPUT=$(timeout 600 "$SX_SERVER" < "$TMPFILE" 2>/dev/null) +OUTPUT=$(timeout 900 "$SX_SERVER" < "$TMPFILE" 2>/dev/null) check() { local epoch="$1" desc="$2" expected="$3" @@ -292,6 +312,9 @@ check 64 "outbox tip=6 page=2 has item:" "true" check 65 "JSON body items array shape" "true" check 66 "SX body :items list shape" "true" check 67 "bad ?page= falls back to page 1" "true" +check 70 "?since= filters earlier entries" "true" +check 71 "?since=unknown -> empty page" "true" +check 72 "?since= + ?page= combined" "true" TOTAL=$((PASS+FAIL)) if [ $FAIL -eq 0 ]; then diff --git a/plans/fed-sx-milestone-2.md b/plans/fed-sx-milestone-2.md index 681596fa..778e786d 100644 --- a/plans/fed-sx-milestone-2.md +++ b/plans/fed-sx-milestone-2.md @@ -651,10 +651,21 @@ Per §13.3: A wants B's history when A first follows B. Four modes: cases: N=0, N>length, T=0, since_cid hit/miss/unknown), wrap_backfill, parse_mode atoms / tuples / proplists / unknown. -- [ ] **9b** — `GET /actors//outbox?since=Cid&limit=N` - pagination route. Extends the Step 4d outbox listing with - the `?since=` query param (calls `backfill:since_cid_entries/2`). - Acceptance test extends `http_multi_actor.sh`. +- [x] **9b** — `GET /actors//outbox?since=Cid` pagination + route. The Step 4d outbox handler in `http_server.erl` + (`actor_outbox_response_for/3`) now reads `?since=` from the + query string via new `parse_since/1` + `scan_param/2,3` + + `skip_to_amp/1` (handles `since=X&page=2` and `page=2&since=X` + identically), pre-filters entries via + `backfill:since_cid_entries/2`, then runs the existing page + slice on the filtered list. `?since=unknown` → empty page → + body degrades to the tip-only shape (Step 4d back-compat). + 3 new cases in `http_multi_actor.sh` (44/44 total) — exercise + filtering, unknown-cid, combined `?since= + ?page=`. Also + added `follower_graph` + `delivery` + `backfill` module loads + to `http_multi_actor.sh` (downstream dependency since Step + 7c/9a — must have been latently broken; the existing 41 + passes + 3 new = 44 now all green). - [ ] **9c** — Follow → Accept → backfill-delivery wiring. The receiving kernel reads the Follow's `:backfill` field via `parse_mode/1`, slices its outbox, and dispatches each @@ -1036,6 +1047,19 @@ proceed. Newest first. +- **2026-06-07** — Step 9b: outbox `?since=Cid` pagination. + `actor_outbox_response_for/3` in `http_server.erl` now reads + `?since=` from the query string via new `parse_since/1` + + `scan_param/2,3` + `skip_to_amp/1` (works whether the param + is first or after `&`), pre-filters entries through + `backfill:since_cid_entries/2`, then runs the existing page + slice on the filtered list. Unknown cid -> empty page -> tip- + only degrade. Three new cases in `http_multi_actor.sh` (44/44 + total) cover filter, unknown-cid, combined since+page. + Latent issue surfaced + fixed in passing: the test was missing + `follower_graph` + `delivery` + `backfill` module loads + (since Step 7c made outbox depend on them); added all three. + - **2026-06-07** — Step 9a: pure-functional backfill slicing. `next/kernel/backfill.erl` with `slice/2,3(Mode, LogState [, Wrap])` returning the appropriate activity list. Modes