coop/rose-ash
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge loops/fed-sx-m2 into architecture: federation milestone 2

Browse Source 
m2 lands multi-actor + cross-instance federation on the fed-sx
substrate. Feature-complete except 8b-timer (retry-loop wiring,
gated on erlang:send_after substrate primitive in loops/erlang).

Highlights:
- Multi-actor gen_server kernel (one nx_kernel handles N actors)
- Per-actor HTTP routes /actors/<id>/{inbox,outbox} + actor-doc
- Inbound signature verify + peer-AS cache + auto-Accept publish
- Outbound delivery_set with audience expansion + delivery_worker
- Native httpc:request/4 BIF wrapper + live HTTP dispatch
- Discovery: peer-actor fetch + cache on demand
- Backfill on Follow accept (in-process + paginated outbox)
- Two-instance smoke test passes 6/6 (real cross-host HTTP flow)

Substrate fixes carried in this merge (textually identical to
upstream-arrived copies, will conflict on scoreboard files only):
- Blockers #1: er-bif-http-listen marshaller bridge rewrite
- Blockers #4: er-sched-step-alive! :pending-args extension
  (lets receive in a kernel-aware route suspend+resume cleanly)

Conformance 761/761 still green on m2 tip.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

# Conflicts:
#	lib/erlang/runtime.sx
This commit is contained in:
giles
2026-06-28 16:57:55 +00:00
parent 25276dc70d 29e4234b14
commit 2bafb4f7d2
151 changed files with 20041 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

336
lib/erlang/runtime.sx
View File

@@ -731,7 +731,10 @@
0
(if
(= prev-k nil)
(er-apply-fun (er-proc-field pid :initial-fun) (list))
(er-apply-fun
(er-proc-field pid :initial-fun)
(let ((args (er-proc-field pid :pending-args)))
(cond (= args nil) (list) :else args)))
(do (er-proc-set! pid :continuation nil) (prev-k nil)))))
(let
((r (nth result-ref 0)))
@@ -956,8 +959,118 @@
(= ty "nil") (er-mk-nil)
:else v))))
;; ── HTTP request/response marshaling (Step 8b-start) ────────────
;; The native `http-listen` primitive hands the handler an SX dict
;; {:method :path :query :headers :body}
;; and expects an SX dict back
;; {:status :headers :body}
;; This layer converts so Erlang handlers see proper proplists:
;; [{method, <<"GET">>}, {path, <<"/foo">>}, {query, <<>>},
;; {headers, [{<<"content-type">>, <<"text/plain">>}, ...]},
;; {body, <<...>>}]
;; Headers ride as a nested proplist with binary keys — header names
;; are arbitrary user input, so they stay out of the atom table. The
;; outer request keys (method/path/query/headers/body) are fixed and
;; small, so they become atoms (cheap to pattern-match against).
(define er-of-sx-deep
(fn (v)
(cond
(= (type-of v) "dict") (er-dict-to-header-proplist v)
:else (er-of-sx v))))
(define er-dict-to-header-proplist
(fn (d)
(let ((ks (keys d)) (out (er-mk-nil)))
(for-each
(fn (i)
(let ((idx (- (- (len ks) 1) i)))
(let ((k (nth ks idx)))
(let ((v (get d k)))
(set!
out
(er-mk-cons
(er-mk-tuple
(list
(er-mk-binary (map char->integer (string->list k)))
(er-of-sx-deep v)))
out))))))
(range 0 (len ks)))
out)))
(define er-request-dict-to-proplist
(fn (d)
(cond
(not (= (type-of d) "dict")) (er-of-sx d)
:else
(let ((ks (keys d)) (out (er-mk-nil)))
(for-each
(fn (i)
(let ((idx (- (- (len ks) 1) i)))
(let ((k (nth ks idx)))
(let ((v (get d k)))
(set!
out
(er-mk-cons
(er-mk-tuple
(list (er-mk-atom k) (er-of-sx-deep v)))
out))))))
(range 0 (len ks)))
out))))
;; Inverse: handler's proplist response -> SX dict for native send.
;; Value rules:
;; Erlang binary -> SX string (bytes joined)
;; Erlang integer -> SX number passthrough
;; Erlang cons of 2-tuples -> nested SX dict (e.g. headers)
;; Erlang cons (other shapes) -> SX list via er-to-sx
;; anything else -> er-to-sx passthrough
(define er-proplist-2tuple?
(fn (v)
(cond
(er-nil? v) true
(er-cons? v)
(let ((h (get v :head)))
(cond
(and (er-tuple? h) (= (len (get h :elements)) 2))
(er-proplist-2tuple? (get v :tail))
:else false))
:else false)))
(define er-to-sx-deep
(fn (v)
(cond
(er-binary? v) (list->string (map integer->char (get v :bytes)))
(and (er-cons? v) (er-proplist-2tuple? v)) (er-proplist-to-dict v)
:else (er-to-sx v))))
(define er-proplist-to-dict
(fn (pl)
(let ((d (dict)))
(er-proplist-fill! pl d)
d)))
(define er-proplist-fill!
(fn (pl d)
(cond
(er-nil? pl) nil
(er-cons? pl)
(let ((head (get pl :head)) (tail (get pl :tail)))
(cond
(and (er-tuple? head) (= (len (get head :elements)) 2))
(let ((kv (get head :elements)))
(let ((k (nth kv 0)) (v (nth kv 1)))
(let ((key-str
(cond
(er-atom? k) (get k :name)
(er-binary? k)
(list->string (map integer->char (get k :bytes)))
:else (str k))))
(dict-set! d key-str (er-to-sx-deep v))
(er-proplist-fill! tail d))))
:else (er-proplist-fill! tail d)))
:else nil)))
;; Load an Erlang module declaration. Source must start with
;; `-module(Name).` and contain function definitions. Functions
@@ -1468,9 +1581,141 @@
;; entry is keyed by "Module/Name/Arity"; multi-arity BIFs register
;; once per arity. Called eagerly at the end of runtime.sx so the
;; registry is ready before any erlang-eval-ast call.
(define er-register-builtin-bifs!
(fn ()
;; erlang module — type predicates (all pure)
(define
er-bif-http-listen
(fn
(vs)
(let
((port (nth vs 0)) (handler (nth vs 1)))
(cond
(not (= (type-of port) "number"))
(raise (er-mk-error-marker (er-mk-atom "badarg")))
(not (er-fun? handler))
(raise (er-mk-error-marker (er-mk-atom "badarg")))
:else (let
;; Bridge between native http-listen and Erlang handler.
;;
;; Inbound: native passes Req as SX Dict
;; {:method :path :query :headers :body}
;; converted to Erlang request proplist via the live
;; er-request-dict-to-proplist marshaller — that's the
;; same shape http_server:route/2 consumes (binaries
;; for path/method/body, dict-like proplist for headers).
;;
;; Outbound: Erlang handler returns
;; [{status, Int}, {headers, [{Bin, Bin}, ...]}, {body, Bin}]
;; converted back to SX Dict via er-proplist-to-dict —
;; binary values become SX strings, the headers cons
;; flattens to a nested SX dict (via er-to-sx-deep's
;; proplist-2tuple detection). Matches what native
;; http-listen serialises to the wire.
;;
;; (Step 8b-bridge originally shipped parallel
;; er-http-req-of-sx / er-http-resp-to-sx helpers; commit
;; 78eae9ef deleted them as dead because the BIF body
;; still referenced them — Blockers #1. This rewrite
;; threads through the live marshallers instead.)
;; Run the handler as a SCHEDULED er-process so any
;; `receive` (e.g. gen_server:call inside a kernel-aware
;; route) suspends and resumes inside the SX scheduler.
;; Without this, native http-listen invokes the handler
;; closure on a fresh OCaml thread that has no scheduler
;; frame, so the receive's er-suspend-marker propagates
;; out and the connection writes nothing — the Blockers
;; #4 deadlock the m2 loop observed.
;;
;; er-spawn-fun requires an er-fun (Erlang-AST-shaped
;; dict); handler IS one (created by user `fun (Req) ->
;; route(Req, Cfg) end`). To feed req-pl as the call
;; argument we stash it on the process record's
;; :pending-args field — er-sched-step-alive! reads it
;; on first step (the alternative was a host-closure-to-
;; er-fun wrapper, which needs AST construction).
((sx-handler
(fn (req-dict)
(let ((req-pl (er-request-dict-to-proplist req-dict)))
(let ((proc (er-proc-new! (er-env-new))))
(dict-set! proc :initial-fun handler)
(dict-set! proc :pending-args (list req-pl))
(er-sched-run-all!)
(let ((resp-pl (er-proc-field (get proc :pid) :exit-result)))
(er-proplist-to-dict resp-pl)))))))
(http-listen port sx-handler))))))
;; httpc:request/4(Url, Method, Headers, Body) - BRIEFING-EXCEPTION:
;; the m2 briefing's one allowed scope exception for Step 8e, mirroring
;; M1 Step 8a's http:listen wrapper on the client side.
;;
;; Url is an Erlang binary (must start with http://).
;; Method is an Erlang atom or binary; passed through to the native
;; verbatim, so callers should supply 'get / 'post or <<"GET">> as
;; appropriate (the native compares uppercase).
;; Headers is an Erlang proplist [{Name, Value}, ...]; names and
;; values are binaries or atoms (er-proplist-to-dict handles both).
;; Body is an Erlang binary (use <<>> for empty).
;;
;; Returns a 4-tuple {ok, StatusInt, HeadersProplist, BodyBinary}.
;; The native primitive raises Eval_error on DNS / connect / bad URL;
;; we catch the host exception here and re-raise as an Erlang error
;; marker so callers can use try/catch error:{network, _} -> _ end.
(define
er-bif-httpc-request
(fn
(vs)
(let
((url (nth vs 0))
(method (nth vs 1))
(headers (nth vs 2))
(body (nth vs 3)))
(let
((url-str
(cond
(er-binary? url) (list->string (map integer->char (get url :bytes)))
:else (raise (er-mk-error-marker (er-mk-atom "badarg")))))
(method-str
(cond
;; Erlang convention is lowercase atoms (get/post/put/...);
;; the HTTP wire wants uppercase. Binaries pass through so
;; callers can override with mixed-case verbs if needed.
(er-atom? method) (upcase (get method :name))
(er-binary? method) (list->string (map integer->char (get method :bytes)))
:else (raise (er-mk-error-marker (er-mk-atom "badarg")))))
(headers-dict
(cond
(er-nil? headers) (dict)
(er-cons? headers) (er-proplist-to-dict headers)
:else (raise (er-mk-error-marker (er-mk-atom "badarg")))))
(body-str
(cond
(er-binary? body) (list->string (map integer->char (get body :bytes)))
(er-nil? body) ""
:else (raise (er-mk-error-marker (er-mk-atom "badarg"))))))
(let ((resp-ref (list nil)) (err-ref (list nil)))
(guard (c (:else (set-nth! err-ref 0 c)))
(set-nth! resp-ref 0
(http-request method-str url-str headers-dict body-str)))
(cond
(not (= (nth err-ref 0) nil))
;; Host error -> Erlang error:{network, ReasonBinary}
(raise (er-mk-error-marker
(er-mk-tuple (list
(er-mk-atom "network")
(er-mk-binary (map char->integer
(string->list (str (nth err-ref 0)))))))))
:else
(let ((resp (nth resp-ref 0)))
(er-mk-tuple
(list
(er-mk-atom "ok")
(get resp :status)
(er-of-sx-deep (get resp :headers))
(er-mk-binary (map char->integer (string->list (get resp :body)))))))))))))
;; Register everything at load time.
(define
er-register-builtin-bifs!
(fn
()
(er-register-pure-bif! "erlang" "is_integer" 1 er-bif-is-integer)
(er-register-pure-bif! "erlang" "is_atom" 1 er-bif-is-atom)
(er-register-pure-bif! "erlang" "is_list" 1 er-bif-is-list)
@@ -1479,27 +1724,61 @@
(er-register-pure-bif! "erlang" "is_float" 1 er-bif-is-float)
(er-register-pure-bif! "erlang" "is_boolean" 1 er-bif-is-boolean)
(er-register-pure-bif! "erlang" "is_pid" 1 er-bif-is-pid)
(er-register-pure-bif! "erlang" "is_reference" 1 er-bif-is-reference)
(er-register-pure-bif!
"erlang"
"is_reference"
1
er-bif-is-reference)
(er-register-pure-bif! "erlang" "is_binary" 1 er-bif-is-binary)
(er-register-pure-bif! "erlang" "is_function" 1 er-bif-is-function)
(er-register-pure-bif! "erlang" "is_function" 2 er-bif-is-function)
;; erlang module — pure data ops
(er-register-pure-bif!
"erlang"
"is_function"
1
er-bif-is-function)
(er-register-pure-bif!
"erlang"
"is_function"
2
er-bif-is-function)
(er-register-pure-bif! "erlang" "length" 1 er-bif-length)
(er-register-pure-bif! "erlang" "hd" 1 er-bif-hd)
(er-register-pure-bif! "erlang" "tl" 1 er-bif-tl)
(er-register-pure-bif! "erlang" "element" 2 er-bif-element)
(er-register-pure-bif! "erlang" "tuple_size" 1 er-bif-tuple-size)
(er-register-pure-bif! "erlang" "byte_size" 1 er-bif-byte-size)
(er-register-pure-bif! "erlang" "atom_to_list" 1 er-bif-atom-to-list)
(er-register-pure-bif! "erlang" "list_to_atom" 1 er-bif-list-to-atom)
(er-register-pure-bif!
"erlang"
"atom_to_list"
1
er-bif-atom-to-list)
(er-register-pure-bif!
"erlang"
"list_to_atom"
1
er-bif-list-to-atom)
(er-register-pure-bif! "erlang" "abs" 1 er-bif-abs)
(er-register-pure-bif! "erlang" "min" 2 er-bif-min)
(er-register-pure-bif! "erlang" "max" 2 er-bif-max)
(er-register-pure-bif! "erlang" "tuple_to_list" 1 er-bif-tuple-to-list)
(er-register-pure-bif! "erlang" "list_to_tuple" 1 er-bif-list-to-tuple)
(er-register-pure-bif! "erlang" "integer_to_list" 1 er-bif-integer-to-list)
(er-register-pure-bif! "erlang" "list_to_integer" 1 er-bif-list-to-integer)
;; erlang module — process / runtime (side-effecting)
(er-register-pure-bif!
"erlang"
"tuple_to_list"
1
er-bif-tuple-to-list)
(er-register-pure-bif!
"erlang"
"list_to_tuple"
1
er-bif-list-to-tuple)
(er-register-pure-bif!
"erlang"
"integer_to_list"
1
er-bif-integer-to-list)
(er-register-pure-bif!
"erlang"
"list_to_integer"
1
er-bif-list-to-integer)
(er-register-bif! "erlang" "self" 0 er-bif-self)
(er-register-bif! "erlang" "spawn" 1 er-bif-spawn)
(er-register-bif! "erlang" "spawn" 3 er-bif-spawn)
@@ -1515,12 +1794,16 @@
(er-register-bif! "erlang" "unregister" 1 er-bif-unregister)
(er-register-bif! "erlang" "whereis" 1 er-bif-whereis)
(er-register-bif! "erlang" "registered" 0 er-bif-registered)
;; erlang module — exception raising (modelled as side-effecting)
(er-register-bif! "erlang" "throw" 1
(er-register-bif!
"erlang"
"throw"
1
(fn (vs) (raise (er-mk-throw-marker (er-bif-arg1 vs "throw")))))
(er-register-bif! "erlang" "error" 1
(er-register-bif!
"erlang"
"error"
1
(fn (vs) (raise (er-mk-error-marker (er-bif-arg1 vs "error")))))
;; lists module — all pure
(er-register-pure-bif! "lists" "reverse" 1 er-bif-lists-reverse)
(er-register-pure-bif! "lists" "map" 2 er-bif-lists-map)
(er-register-pure-bif! "lists" "foldl" 3 er-bif-lists-foldl)
@@ -1534,11 +1817,13 @@
(er-register-pure-bif! "lists" "filter" 2 er-bif-lists-filter)
(er-register-pure-bif! "lists" "any" 2 er-bif-lists-any)
(er-register-pure-bif! "lists" "all" 2 er-bif-lists-all)
(er-register-pure-bif! "lists" "duplicate" 2 er-bif-lists-duplicate)
;; io module — side-effecting (writes to io buffer)
(er-register-pure-bif!
"lists"
"duplicate"
2
er-bif-lists-duplicate)
(er-register-bif! "io" "format" 1 er-bif-io-format)
(er-register-bif! "io" "format" 2 er-bif-io-format)
;; ets module — side-effecting (mutates table state)
(er-register-bif! "ets" "new" 2 er-bif-ets-new)
(er-register-bif! "ets" "insert" 2 er-bif-ets-insert)
(er-register-bif! "ets" "lookup" 2 er-bif-ets-lookup)
@@ -1546,18 +1831,15 @@
(er-register-bif! "ets" "delete" 2 er-bif-ets-delete)
(er-register-bif! "ets" "tab2list" 1 er-bif-ets-tab2list)
(er-register-bif! "ets" "info" 2 er-bif-ets-info)
;; code module — side-effecting (mutates module registry, kills procs)
(er-register-bif! "code" "load_binary" 3 er-bif-code-load-binary)
(er-register-bif! "code" "purge" 1 er-bif-code-purge)
(er-register-bif! "code" "soft_purge" 1 er-bif-code-soft-purge)
(er-register-bif! "code" "which" 1 er-bif-code-which)
(er-register-bif! "code" "is_loaded" 1 er-bif-code-is-loaded)
(er-register-bif! "code" "all_loaded" 0 er-bif-code-all-loaded)
;; file module
(er-register-bif! "file" "read_file" 1 er-bif-file-read-file)
(er-register-bif! "file" "write_file" 2 er-bif-file-write-file)
(er-register-bif! "file" "delete" 1 er-bif-file-delete)
;; Phase 8 FFI — host-primitive BIFs (loops/fed-prims)
(er-register-pure-bif! "crypto" "hash" 2 er-bif-crypto-hash)
(er-register-pure-bif! "cid" "from_bytes" 1 er-bif-cid-from-bytes)
(er-register-pure-bif! "cid" "to_string" 1 er-bif-cid-to-string)
@@ -1623,6 +1905,10 @@
(er-register-pure-bif! "erlang" "list_to_binary" 1 er-bif-list-to-binary)
(er-mk-atom "ok")))
;; ── m2 federation BIFs (top-level registration; defs above) ─────
(er-register-bif! "http" "listen" 2 er-bif-http-listen)
(er-register-bif! "httpc" "request" 4 er-bif-httpc-request)
;; Register everything at load time.
(jit-exclude! "er-*" "erlang-*")