#!/usr/bin/env bash # next/tests/delivery_retry.sh — m2 Step 8b-pure test. # # Pure-functional retry-time bookkeeping for the delivery worker. # record_failure bumps the attempt counter and computes the next # retry time per backoff_for. record_success clears state for a # cid. next_due returns cids whose retry time has passed. # # Real timer wiring (erlang:send_after self-cast) is Step 8b-timer # once substrate support lands. set -uo pipefail cd "$(git rev-parse --show-toplevel)" SX_SERVER="${SX_SERVER:-hosts/ocaml/_build/default/bin/sx_server.exe}" if [ ! -x "$SX_SERVER" ]; then SX_SERVER="/root/rose-ash/hosts/ocaml/_build/default/bin/sx_server.exe" fi if [ ! -x "$SX_SERVER" ]; then echo "ERROR: sx_server.exe not found." >&2 exit 1 fi VERBOSE="${1:-}" PASS=0; FAIL=0; ERRORS="" TMPFILE=$(mktemp); trap "rm -f $TMPFILE" EXIT SETUP='Act1 = [{id, <<1>>}, {type, note}, {actor, alice}], Act2 = [{id, <<2>>}, {type, note}, {actor, alice}],' cat > "$TMPFILE" <>, S), delivery_worker:next_retry_at(<<1>>, S), delivery_worker:dead_letter_list(S)} =:= {0, undefined, []}\") :name)") ;; record_failure bumps the attempt counter (epoch 11) (eval "(get (erlang-eval-ast \"${SETUP} S0 = delivery_worker:enqueue_pure(bob, Act1, delivery_worker:new(bob)), S1 = delivery_worker:record_failure_pure(<<1>>, 1000, S0), delivery_worker:attempts_for(<<1>>, S1) =:= 1\") :name)") ;; record_failure sets next_retry_at = Now + backoff(1) = Now + 30 (epoch 12) (eval "(get (erlang-eval-ast \"${SETUP} S0 = delivery_worker:enqueue_pure(bob, Act1, delivery_worker:new(bob)), S1 = delivery_worker:record_failure_pure(<<1>>, 1000, S0), delivery_worker:next_retry_at(<<1>>, S1) =:= 1030\") :name)") ;; Second failure -> attempts=2, NextRetryAt = Now+300 (epoch 13) (eval "(get (erlang-eval-ast \"${SETUP} S0 = delivery_worker:enqueue_pure(bob, Act1, delivery_worker:new(bob)), S1 = delivery_worker:record_failure_pure(<<1>>, 1000, S0), S2 = delivery_worker:record_failure_pure(<<1>>, 2000, S1), {delivery_worker:attempts_for(<<1>>, S2), delivery_worker:next_retry_at(<<1>>, S2)} =:= {2, 2300}\") :name)") ;; record_success clears attempts + next_retry for the cid (epoch 14) (eval "(get (erlang-eval-ast \"${SETUP} S0 = delivery_worker:enqueue_pure(bob, Act1, delivery_worker:new(bob)), S1 = delivery_worker:record_failure_pure(<<1>>, 1000, S0), S2 = delivery_worker:record_success_pure(<<1>>, S1), {delivery_worker:attempts_for(<<1>>, S2), delivery_worker:next_retry_at(<<1>>, S2)} =:= {0, undefined}\") :name)") ;; next_due returns Cids whose retry time has passed (epoch 15) (eval "(get (erlang-eval-ast \"${SETUP} S0 = delivery_worker:enqueue_pure(bob, Act1, delivery_worker:new(bob)), S1 = delivery_worker:record_failure_pure(<<1>>, 1000, S0), delivery_worker:next_due_pure(1030, S1) =:= [<<1>>]\") :name)") ;; next_due returns [] before retry time (epoch 16) (eval "(get (erlang-eval-ast \"${SETUP} S0 = delivery_worker:enqueue_pure(bob, Act1, delivery_worker:new(bob)), S1 = delivery_worker:record_failure_pure(<<1>>, 1000, S0), delivery_worker:next_due_pure(1020, S1) =:= []\") :name)") ;; 6th failure -> dead_letter; activity moves out of :pending (epoch 17) (eval "(get (erlang-eval-ast \"${SETUP} F = fun(S) -> delivery_worker:record_failure_pure(<<1>>, 1000, S) end, S0 = delivery_worker:enqueue_pure(bob, Act1, delivery_worker:new(bob)), S6 = F(F(F(F(F(F(S0)))))), {delivery_worker:dead_letter_list(S6), delivery_worker:pending(S6)} =:= {[Act1], []}\") :name)") ;; Dead-lettered cid is no longer in next_retry (epoch 18) (eval "(get (erlang-eval-ast \"${SETUP} F = fun(S) -> delivery_worker:record_failure_pure(<<1>>, 1000, S) end, S0 = delivery_worker:enqueue_pure(bob, Act1, delivery_worker:new(bob)), S6 = F(F(F(F(F(F(S0)))))), delivery_worker:next_retry_at(<<1>>, S6) =:= undefined\") :name)") ;; Two cids: success on one doesn't disturb the other's retry state (epoch 19) (eval "(get (erlang-eval-ast \"${SETUP} S0 = delivery_worker:enqueue_pure(bob, Act1, delivery_worker:enqueue_pure(bob, Act2, delivery_worker:new(bob))), S1 = delivery_worker:record_failure_pure(<<1>>, 1000, S0), S2 = delivery_worker:record_failure_pure(<<2>>, 1000, S1), S3 = delivery_worker:record_success_pure(<<1>>, S2), delivery_worker:next_retry_at(<<2>>, S3) =:= 1030\") :name)") EPOCHS OUTPUT=$(timeout 240 "$SX_SERVER" < "$TMPFILE" 2>/dev/null) check() { local epoch="$1" desc="$2" expected="$3" local actual actual=$(echo "$OUTPUT" | awk -v e="$epoch" ' $0 ~ "^\\(ok-len " e " " { getline; print; exit } $0 ~ "^\\(ok " e " " { print; exit } $0 ~ "^\\(error " e " " { print; exit } ') [ -z "$actual" ] && actual="" if echo "$actual" | grep -qF -- "$expected"; then PASS=$((PASS+1)) [ "$VERBOSE" = "-v" ] && echo " ok $desc" else FAIL=$((FAIL+1)) ERRORS+=" FAIL [$desc] (epoch $epoch) expected: $expected | actual: $actual " fi } check 3 "module loaded" "delivery_worker" check 10 "fresh state empty" "true" check 11 "record_failure bumps attempts" "true" check 12 "record_failure sets next_retry_at" "true" check 13 "second failure: slot 2 = +300" "true" check 14 "record_success clears state" "true" check 15 "next_due returns due cids" "true" check 16 "next_due empty before due" "true" check 17 "6th failure -> dead_letter" "true" check 18 "dead-lettered cid out of retry" "true" check 19 "success on one preserves other" "true" TOTAL=$((PASS+FAIL)) if [ $FAIL -eq 0 ]; then echo "ok $PASS/$TOTAL next/tests/delivery_retry.sh passed" else echo "FAIL $PASS/$TOTAL passed, $FAIL failed:" echo "$ERRORS" fi [ $FAIL -eq 0 ]