From 24e3bf53b041871f1837adbbb381b2d4e301a79d Mon Sep 17 00:00:00 2001 From: giles Date: Thu, 4 Jun 2026 22:44:02 +0000 Subject: [PATCH] =?UTF-8?q?fed-sx-m1:=20Step=203b=20substrate=20fix=20?= =?UTF-8?q?=E2=80=94=20binary=5Fto=5Flist/1=20+=20list=5Fto=5Fbinary/1=20B?= =?UTF-8?q?IFs=20(+9=20ffi,=20738/738)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/erlang/runtime.sx | 59 +++++++++++++++++++++++++++++++++++++ lib/erlang/scoreboard.json | 6 ++-- lib/erlang/scoreboard.md | 4 +-- lib/erlang/tests/ffi.sx | 45 ++++++++++++++++++++++++++++ next/README.md | 15 ++++++---- plans/fed-sx-milestone-1.md | 3 +- 6 files changed, 121 insertions(+), 11 deletions(-) diff --git a/lib/erlang/runtime.sx b/lib/erlang/runtime.sx index 17a5ad99..36745b87 100644 --- a/lib/erlang/runtime.sx +++ b/lib/erlang/runtime.sx @@ -1615,7 +1615,66 @@ (er-register-pure-bif! "crypto" "hash" 2 er-bif-crypto-hash) (er-register-pure-bif! "cid" "from_bytes" 1 er-bif-cid-from-bytes) (er-register-pure-bif! "cid" "to_string" 1 er-bif-cid-to-string) + +;; ── binary_to_list / list_to_binary (Step 3b — term codec) ────── +;; Standard Erlang semantics: +;; binary_to_list(<>) -> [B1, B2, ...] (Erlang cons of ints) +;; list_to_binary(IoList) -> <<...>> (flattens nested +;; iolists; elements are byte ints 0-255 or binaries) +;; Bad arg / out-of-range byte / non-iolist element -> error:badarg. + +(define er-bif-binary-to-list + (fn (vs) + (let ((v (nth vs 0))) + (cond + (not (er-binary? v)) + (raise (er-mk-error-marker (er-mk-atom "badarg"))) + :else + (let ((bs (get v :bytes)) (out (er-mk-nil))) + (for-each + (fn (i) + (set! out (er-mk-cons (nth bs (- (- (len bs) 1) i)) out))) + (range 0 (len bs))) + out))))) + +;; Walk an Erlang iolist, appending bytes to `acc` (a mutable SX list). +;; Accepts: nil, cons-of-X, binary, integer in 0..255. Anything else +;; signals failure by setting (nth fail 0) to true. +(define er-iolist-walk! + (fn (v acc fail) + (cond + (nth fail 0) nil + (er-nil? v) nil + (er-cons? v) + (do (er-iolist-walk! (get v :head) acc fail) + (er-iolist-walk! (get v :tail) acc fail)) + (er-binary? v) + (for-each + (fn (i) (append! acc (nth (get v :bytes) i))) + (range 0 (len (get v :bytes)))) + (= (type-of v) "number") + (cond + (and (>= v 0) (<= v 255)) (append! acc v) + :else (set-nth! fail 0 true)) + :else (set-nth! fail 0 true)))) + +(define er-bif-list-to-binary + (fn (vs) + (let ((v (nth vs 0)) (acc (list)) (fail (list false))) + (cond + (not (or (er-nil? v) (er-cons? v) (er-binary? v))) + (raise (er-mk-error-marker (er-mk-atom "badarg"))) + :else + (do + (er-iolist-walk! v acc fail) + (cond + (nth fail 0) + (raise (er-mk-error-marker (er-mk-atom "badarg"))) + :else (er-mk-binary acc))))))) + (er-register-bif! "file" "list_dir" 1 er-bif-file-list-dir) + (er-register-pure-bif! "erlang" "binary_to_list" 1 er-bif-binary-to-list) + (er-register-pure-bif! "erlang" "list_to_binary" 1 er-bif-list-to-binary) (er-mk-atom "ok"))) (er-register-bif! "http" "listen" 2 er-bif-http-listen) diff --git a/lib/erlang/scoreboard.json b/lib/erlang/scoreboard.json index f5b6e981..fac8aff3 100644 --- a/lib/erlang/scoreboard.json +++ b/lib/erlang/scoreboard.json @@ -1,7 +1,7 @@ { "language": "erlang", - "total_pass": 729, - "total": 729, + "total_pass": 738, + "total": 738, "suites": [ {"name":"tokenize","pass":62,"total":62,"status":"ok"}, {"name":"parse","pass":52,"total":52,"status":"ok"}, @@ -12,7 +12,7 @@ {"name":"bank","pass":8,"total":8,"status":"ok"}, {"name":"echo","pass":7,"total":7,"status":"ok"}, {"name":"fib","pass":8,"total":8,"status":"ok"}, - {"name":"ffi","pass":28,"total":28,"status":"ok"}, + {"name":"ffi","pass":37,"total":37,"status":"ok"}, {"name":"vm","pass":78,"total":78,"status":"ok"} ] } diff --git a/lib/erlang/scoreboard.md b/lib/erlang/scoreboard.md index 75f3fe39..75cac040 100644 --- a/lib/erlang/scoreboard.md +++ b/lib/erlang/scoreboard.md @@ -1,6 +1,6 @@ # Erlang-on-SX Scoreboard -**Total: 729 / 729 tests passing** +**Total: 738 / 738 tests passing** | | Suite | Pass | Total | |---|---|---|---| @@ -13,7 +13,7 @@ | ✅ | bank | 8 | 8 | | ✅ | echo | 7 | 7 | | ✅ | fib | 8 | 8 | -| ✅ | ffi | 28 | 28 | +| ✅ | ffi | 37 | 37 | | ✅ | vm | 78 | 78 | diff --git a/lib/erlang/tests/ffi.sx b/lib/erlang/tests/ffi.sx index e08a31bf..29af1c9e 100644 --- a/lib/erlang/tests/ffi.sx +++ b/lib/erlang/tests/ffi.sx @@ -160,6 +160,51 @@ (ffi-nm (ffi-ev "element(2, file:list_dir(\"/no/such/dir/xyz\"))")) "enoent") +(er-ffi-test + "binary_to_list <<1,2,3>> length" + (ffi-ev "length(binary_to_list(<<1,2,3,4,5>>))") + 5) + +(er-ffi-test + "binary_to_list hd byte" + (ffi-ev "hd(binary_to_list(<<7,8,9>>))") + 7) + +(er-ffi-test + "binary_to_list empty -> []" + (ffi-nm (ffi-ev "case binary_to_list(<<>>) of [] -> empty end")) + "empty") + +(er-ffi-test + "list_to_binary flat list bytes" + (ffi-ev "byte_size(list_to_binary([1,2,3]))") + 3) + +(er-ffi-test + "list_to_binary nested iolist" + (ffi-ev "byte_size(list_to_binary([1, <<2,3>>, [4, [5]]]))") + 5) + +(er-ffi-test + "list_to_binary round-trip via binary_to_list" + (ffi-nm (ffi-ev "list_to_binary(binary_to_list(<<10,20,30>>)) =:= <<10,20,30>>")) + "true") + +(er-ffi-test + "binary_to_list non-binary -> error:badarg" + (ffi-nm (ffi-ev "try binary_to_list(42) catch error:badarg -> ok end")) + "ok") + +(er-ffi-test + "list_to_binary out-of-range byte -> error:badarg" + (ffi-nm (ffi-ev "try list_to_binary([300]) catch error:badarg -> ok end")) + "ok") + +(er-ffi-test + "list_to_binary non-iolist -> error:badarg" + (ffi-nm (ffi-ev "try list_to_binary(42) catch error:badarg -> ok end")) + "ok") + ;; ── Still deferred (no host primitive): httpc (HTTP client, v2), ;; sqlite-* (v2 indexes). Assert NOT registered so a future iteration ;; that wires them without updating this suite fails fast. diff --git a/next/README.md b/next/README.md index e4a360ef..73c20535 100644 --- a/next/README.md +++ b/next/README.md @@ -103,11 +103,16 @@ The kernel calls into these host primitives: `crypto:hash/2`, These three gaps block the remaining unchecked deliverables: -1. **Term codec** (`3b`/`3c`) — `atom_to_list`/`integer_to_list` return - SX-strings (an opaque OCaml-string type), not Erlang charlists; - `binary_to_list`/`list_to_binary` are unregistered; `$X` char literals - decode to `nil` in `parse-number`. Net effect: no in-Erlang term ↔ binary - round-trip path. Blocks on-disk log persistence. +1. **Term codec** (`3b`/`3c`) — **byte-level path resolved 2026-06-04:** + `erlang:binary_to_list/1` and `erlang:list_to_binary/1` are now registered + in `lib/erlang/runtime.sx` (738/738 conformance, +9 ffi tests). `list_to_binary` + is iolist-aware (`[1, <<2,3>>, [4, [5]]]` → `<<1,2,3,4,5>>`); round-trip + `list_to_binary(binary_to_list(B)) =:= B` holds. Step 3b on-disk segment + writer is unblocked if it uses byte ints directly. Still parked: + `atom_to_list`/`integer_to_list` return SX-strings (an opaque OCaml-string + type), not Erlang charlists; `$X` char literals decode to `nil` in + `parse-number`. Both still block code that wants Erlang-idiomatic + `[$h,$i | T]` patterns on atom/integer names. 2. **SX-source eval bridge** — There's no BIF that lets Erlang call into the SX evaluator on a parsed source string. Blocks evaluating the `:schema` / diff --git a/plans/fed-sx-milestone-1.md b/plans/fed-sx-milestone-1.md index 7a35a1b0..ac9e3920 100644 --- a/plans/fed-sx-milestone-1.md +++ b/plans/fed-sx-milestone-1.md @@ -200,7 +200,7 @@ verify_signature(Activity, ActorState) -> - [ ] **3b** — *Parked behind substrate gap (see Blockers below).* Term codec + on-disk persistence: serializer/parser writing each activity as a JSONL-style line; restart-resumes-tip from the segment file. - [ ] **3c** — Segment rotation at size threshold + gen_server-mediated concurrent appends. -**Blockers (Step 3b):** The Erlang port returns SX strings (an opaque OCaml-string type) from `atom_to_list/1` and `integer_to_list/1`, rejects them from `++`/list pattern matching, and does not register `binary_to_list`/`list_to_binary`. `$X` character literals decode to `nil` in `parse-number`. Net effect: there is no in-Erlang path from an arbitrary term to a byte sequence (or back) that doesn't go through a temp-file round-trip through the filesystem. Workaround paths: (a) add a `term_to_binary`/`binary_to_term` BIF in a separate substrate loop, (b) accept a filesystem-mediated SX-string→binary helper and live with the O(N) IO cost, (c) restrict the on-disk format to a binary-only encoding with a per-instance atom-id table for atoms (introduces an extra durability dependency). Decision to defer; revisit once a downstream Step (5–8) forces the issue or a substrate BIF arrives. In-memory log from 3a is sufficient to unblock Step 5+ which consume the API surface. +**Blockers (Step 3b) — byte-level path resolved 2026-06-04:** `binary_to_list/1` and `list_to_binary/1` are now registered Erlang BIFs in `lib/erlang/runtime.sx` (Step 3b substrate fix, +9 ffi tests, 738/738 conformance). `list_to_binary` is iolist-aware: accepts nested cons of integer bytes (0-255) and/or binaries; `binary_to_list` returns a proper Erlang charlist of integers. Round-trip verified: `list_to_binary(binary_to_list(B)) =:= B`. On-disk segment writer (3b) can now build segment bytes from `[Header, IoListPayload]` and reconstruct on read — option (c) of the original workaround menu is now cheap. Still parked: `atom_to_list/1`/`integer_to_list/1` return SX strings rather than Erlang charlists, and `$X` char literals decode to `nil` in `parse-number`. Neither blocks the on-disk format if the encoding uses byte ints directly (no string→list coercion); both still block code that wants to write Erlang-idiomatic `[$h,$i | T]` patterns. 3b on-disk implementation is unblocked; revisit the remaining two gaps if a downstream Step requires charlist arithmetic on atom/integer names. **Deliverables:** @@ -1003,6 +1003,7 @@ A few things still under-specified; resolve as work begins. Newest first. One line per sub-deliverable commit. Erlang conformance gate (`bash lib/erlang/conformance.sh`) must remain 729/729 on every entry. +- **2026-06-04** — Step 3b substrate fix: registered `erlang:binary_to_list/1` and `erlang:list_to_binary/1` in `lib/erlang/runtime.sx` — the byte-level half of the term-codec gap. `binary_to_list` returns a proper Erlang charlist (`er-mk-cons` chain of byte ints). `list_to_binary` is iolist-aware via a recursive `er-iolist-walk!` that accepts nil / cons / binary / integer 0-255 and flattens nested iolists (e.g. `[1, <<2,3>>, [4, [5]]]` → `<<1,2,3,4,5>>`); out-of-range bytes or non-iolist elements raise `error:badarg`. Round-trip verified: `list_to_binary(binary_to_list(B)) =:= B`. +9 ffi tests (length, hd, empty→[], flat byte_size, nested-iolist, round-trip, 3 badarg paths). On-disk segment writer (3b) now has a complete `[Header | IoListPayload] → Binary` path; the remaining two substrate gaps (`atom_to_list`/`integer_to_list` as Erlang charlists, `$X` char-literal decoding) are still parked but no longer block 3b implementation if the encoding uses byte ints directly. Erlang conformance **738/738** (ffi 28→37). Plan Blockers note for Step 3b updated to reflect the partial resolution. - **2026-05-28** — Step 4f-consolidate: `bootstrap:start/3(ActorId, KeySpec, ActorState)` brings up the full kernel substrate in one call — starts the registry gen_server, populates it from the canonical genesis bundle (31 entries across 7 kinds), then starts nx_kernel. Returns the kernel Pid (gen_server convention in this port returns raw Pid not `{ok, Pid}`). Tests verify whereis(nx_kernel), per-kind counts (3/10/7/3/3/2/3), registry lookup of a known entry (`create`), publish + log_tip advance. `next/tests/bootstrap_start.sh` 10/10. Erlang conformance 729/729. - **2026-05-28** — Step 7d-pure: `next/kernel/sandbox.erl` — `eval_pure/2(Fun, Arg)` and `eval_pure/3(Fun, Activity, State)`. try/catch envelope returns `{ok, Result}` on success and `{error, {Class, Reason}}` for each of the three exception classes (throw, error, exit). The 3-arity variant matches the projection-fold shape so the scheduler can wrap fold bodies. Port note: this Erlang implementation catches by explicit class names rather than the open `Class:Reason` pattern — wrappers enumerate `throw:Reason / error:Reason / exit:Reason` explicitly. Real gas budget + IO denial + env-stripping lands with SX-source eval; the wrapper API doesn't change. `next/tests/sandbox_eval.sh` 13/13. Erlang conformance 729/729. - **2026-05-28** — Step 9b-pure: **reactive application extensibility, proven end-to-end.** Mirrors §Step 9b structurally without TCP/curl/JSON. A trigger projection (Erlang-fun fold over `{Captured, Count}` state) matches Note activities whose `:object :tags` contains `smoketest`, constructs a derived `TestEcho` activity with `:object :echoes` pointing at the Note's `:id`, and captures it into projection state. Order-independent; non-Note + non-smoketest + Note-without-tags + sig-failed publishes all suppressed correctly. Multi-tag (e.g. `[smoketest, foo, bar]`) still matches. Cascade publish (the trigger actually publishing the derived activity back through outbox) is deferred — the gen_server reentrancy that introduces is a v2 concern; the projection-state capture is sufficient proof of the match-then-derive mechanism. `next/tests/smoke_app_pure.sh` 12/12. Erlang conformance 729/729.