datalog: magic pre-saturation is conditional, not unconditional

Previously dl-magic-query always pre-saturated the source db so it
gave correct results for stratified programs (where the rewriter
doesn't propagate magic to aggregate inner-goals or negated rels).
Pure positive programs paid the full bottom-up cost twice.

Add dl-rules-need-presaturation? — checks whether any rule body
contains an aggregate or negation. Only pre-saturate in that case.
Pure positive programs (the common case for magic-sets) keep their
full goal-directed efficiency.

276/276; identical answers on the existing aggregate-of-IDB test.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

hosts/ocaml/bootstrap_vm.py

@@ -355,7 +355,9 @@ let vm_create_closure vm_val frame_val code_val =
   let f = unwrap_frame frame_val in
   let uv_count = match code_val with
     | Dict d -> (match Hashtbl.find_opt d "upvalue-count" with
-        | Some (Number n) -> int_of_float n | _ -> 0)
+        | Some (Integer n) -> n
+        | Some (Number n) -> int_of_float n
+        | _ -> 0)
     | _ -> 0
   in
   let upvalues = Array.init uv_count (fun _ ->

hosts/ocaml/browser/sx_browser.ml

@@ -715,8 +715,10 @@ let () =
         | List (Symbol "code" :: rest) ->
           let d = Hashtbl.create 8 in
           let rec parse_kv = function
-            | Keyword "arity" :: Number n :: rest -> Hashtbl.replace d "arity" (Number n); parse_kv rest
+            | Keyword "arity" :: (Number _ as n) :: rest -> Hashtbl.replace d "arity" n; parse_kv rest
-            | Keyword "upvalue-count" :: Number n :: rest -> Hashtbl.replace d "upvalue-count" (Number n); parse_kv rest
+            | Keyword "arity" :: (Integer _ as n) :: rest -> Hashtbl.replace d "arity" n; parse_kv rest
+            | Keyword "upvalue-count" :: (Number _ as n) :: rest -> Hashtbl.replace d "upvalue-count" n; parse_kv rest
+            | Keyword "upvalue-count" :: (Integer _ as n) :: rest -> Hashtbl.replace d "upvalue-count" n; parse_kv rest
             | Keyword "bytecode" :: List nums :: rest ->
               Hashtbl.replace d "bytecode" (List nums); parse_kv rest
             | Keyword "constants" :: List consts :: rest ->

hosts/ocaml/lib/sx_primitives.ml

@@ -3124,6 +3124,442 @@ let () =
     | [String pat] -> List (List.map (fun s -> String s) (glob_paths pat))
     | _ -> raise (Eval_error "file-glob: (pattern)"));
 
+  (* === File metadata + ops (Phase 5d) === *)
+  let stat_or = function
+    | String path -> (try Some (Unix.stat path) with _ -> None)
+    | _ -> raise (Eval_error "file: path must be a string")
+  in
+  register "file-size" (fun args ->
+    match args with
+    | [v] -> (match stat_or v with Some s -> Integer s.Unix.st_size | None -> Integer 0)
+    | _ -> raise (Eval_error "file-size: (path)"));
+  register "file-mtime" (fun args ->
+    match args with
+    | [v] -> (match stat_or v with Some s -> Integer (int_of_float s.Unix.st_mtime) | None -> Integer 0)
+    | _ -> raise (Eval_error "file-mtime: (path)"));
+  register "file-isfile?" (fun args ->
+    match args with
+    | [v] -> (match stat_or v with Some s -> Bool (s.Unix.st_kind = Unix.S_REG) | None -> Bool false)
+    | _ -> raise (Eval_error "file-isfile?: (path)"));
+  register "file-isdir?" (fun args ->
+    match args with
+    | [v] -> (match stat_or v with Some s -> Bool (s.Unix.st_kind = Unix.S_DIR) | None -> Bool false)
+    | _ -> raise (Eval_error "file-isdir?: (path)"));
+  register "file-readable?" (fun args ->
+    match args with
+    | [String path] ->
+      Bool (try Unix.access path [Unix.R_OK]; true with _ -> false)
+    | _ -> raise (Eval_error "file-readable?: (path)"));
+  register "file-writable?" (fun args ->
+    match args with
+    | [String path] ->
+      Bool (try Unix.access path [Unix.W_OK]; true with _ -> false)
+    | _ -> raise (Eval_error "file-writable?: (path)"));
+  register "file-stat" (fun args ->
+    match args with
+    | [v] ->
+      (match stat_or v with
+       | None -> Nil
+       | Some s ->
+         let d = Hashtbl.create 6 in
+         Hashtbl.replace d "size" (Integer s.Unix.st_size);
+         Hashtbl.replace d "mtime" (Integer (int_of_float s.Unix.st_mtime));
+         Hashtbl.replace d "atime" (Integer (int_of_float s.Unix.st_atime));
+         Hashtbl.replace d "ctime" (Integer (int_of_float s.Unix.st_ctime));
+         Hashtbl.replace d "mode" (Integer s.Unix.st_perm);
+         Hashtbl.replace d "type" (String (match s.Unix.st_kind with
+           | Unix.S_REG -> "file" | Unix.S_DIR -> "directory"
+           | Unix.S_LNK -> "link" | Unix.S_CHR -> "characterSpecial"
+           | Unix.S_BLK -> "blockSpecial" | Unix.S_FIFO -> "fifo"
+           | Unix.S_SOCK -> "socket"));
+         Dict d)
+    | _ -> raise (Eval_error "file-stat: (path)"));
+  register "file-delete" (fun args ->
+    match args with
+    | [String path] ->
+      (try
+        if Sys.is_directory path then Unix.rmdir path
+        else Unix.unlink path
+      with
+      | Unix.Unix_error (Unix.ENOENT, _, _) -> ()  (* tolerate missing *)
+      | Unix.Unix_error (e, _, _) -> raise (Eval_error ("file-delete: " ^ Unix.error_message e)));
+      Nil
+    | _ -> raise (Eval_error "file-delete: (path)"));
+  register "file-mkdir" (fun args ->
+    match args with
+    | [String path] ->
+      let rec mk p =
+        if p = "" || p = "." || p = "/" then ()
+        else if Sys.file_exists p then ()
+        else begin
+          mk (Filename.dirname p);
+          (try Unix.mkdir p 0o755
+           with Unix.Unix_error (Unix.EEXIST, _, _) -> ())
+        end
+      in
+      (try mk path
+       with Unix.Unix_error (e, _, _) -> raise (Eval_error ("file-mkdir: " ^ Unix.error_message e)));
+      Nil
+    | _ -> raise (Eval_error "file-mkdir: (path)"));
+  register "file-copy" (fun args ->
+    match args with
+    | [String src; String dst] ->
+      (try
+        let ic = open_in_bin src in
+        let oc = open_out_bin dst in
+        let buf = Bytes.create 8192 in
+        let rec loop () =
+          let n = input ic buf 0 (Bytes.length buf) in
+          if n > 0 then (output oc buf 0 n; loop ())
+        in
+        loop ();
+        close_in ic;
+        close_out oc;
+        Nil
+      with
+      | Sys_error msg -> raise (Eval_error ("file-copy: " ^ msg)))
+    | _ -> raise (Eval_error "file-copy: (src dst)"));
+  register "file-rename" (fun args ->
+    match args with
+    | [String src; String dst] ->
+      (try Sys.rename src dst with Sys_error msg -> raise (Eval_error ("file-rename: " ^ msg)));
+      Nil
+    | _ -> raise (Eval_error "file-rename: (src dst)"));
+
+  (* === Channels (random-access + blocking control) === *)
+  let channel_table : (string, Unix.file_descr * string * bool ref * bool ref) Hashtbl.t = Hashtbl.create 16 in
+  let channel_next_id = ref 0 in
+  let parse_open_mode mode =
+    match mode with
+    | "r"  -> [Unix.O_RDONLY]
+    | "w"  -> [Unix.O_WRONLY; Unix.O_CREAT; Unix.O_TRUNC]
+    | "a"  -> [Unix.O_WRONLY; Unix.O_CREAT; Unix.O_APPEND]
+    | "r+" -> [Unix.O_RDWR]
+    | "w+" -> [Unix.O_RDWR; Unix.O_CREAT; Unix.O_TRUNC]
+    | "a+" -> [Unix.O_RDWR; Unix.O_CREAT; Unix.O_APPEND]
+    | _ -> raise (Eval_error ("channel-open: invalid mode " ^ mode))
+  in
+  let chan_get name =
+    match Hashtbl.find_opt channel_table name with
+    | Some c -> c
+    | None -> raise (Eval_error ("channel: no such channel " ^ name))
+  in
+  register "channel-open" (fun args ->
+    match args with
+    | [String path; String mode] ->
+      (try
+        let fd = Unix.openfile path (parse_open_mode mode) 0o644 in
+        let id = !channel_next_id in
+        incr channel_next_id;
+        let name = Printf.sprintf "file%d" id in
+        Hashtbl.replace channel_table name (fd, mode, ref false, ref true);
+        String name
+      with Unix.Unix_error (e, _, _) -> raise (Eval_error ("channel-open: " ^ Unix.error_message e)))
+    | _ -> raise (Eval_error "channel-open: (path mode)"));
+
+  register "channel-close" (fun args ->
+    match args with
+    | [String name] ->
+      let (fd, _, _, _) = chan_get name in
+      (try Unix.close fd with _ -> ());
+      Hashtbl.remove channel_table name;
+      Nil
+    | _ -> raise (Eval_error "channel-close: (channel)"));
+
+  register "channel-read" (fun args ->
+    let (name, max_n) = match args with
+      | [String n] -> (n, -1)
+      | [String n; Integer m] -> (n, m)
+      | [String n; Number m] -> (n, int_of_float m)
+      | _ -> raise (Eval_error "channel-read: (channel ?n?)")
+    in
+    let (fd, _, eof, _) = chan_get name in
+    let chunk = 8192 in
+    let buf = Bytes.create chunk in
+    let buffer = Buffer.create chunk in
+    let total = ref 0 in
+    let stop = ref false in
+    while not !stop do
+      let want = if max_n < 0 then chunk else min chunk (max_n - !total) in
+      if want <= 0 then stop := true
+      else begin
+        try
+          let r = Unix.read fd buf 0 want in
+          if r = 0 then begin eof := true; stop := true end
+          else begin
+            Buffer.add_subbytes buffer buf 0 r;
+            total := !total + r
+          end
+        with
+        | Unix.Unix_error (Unix.EAGAIN, _, _)
+        | Unix.Unix_error (Unix.EWOULDBLOCK, _, _) -> stop := true
+      end
+    done;
+    String (Buffer.contents buffer));
+
+  register "channel-read-line" (fun args ->
+    match args with
+    | [String name] ->
+      let (fd, _, eof, _) = chan_get name in
+      let buf = Buffer.create 80 in
+      let one = Bytes.create 1 in
+      let got_data = ref false in
+      let stop = ref false in
+      while not !stop do
+        try
+          let r = Unix.read fd one 0 1 in
+          if r = 0 then begin eof := true; stop := true end
+          else begin
+            got_data := true;
+            let c = Bytes.get one 0 in
+            if c = '\n' then stop := true
+            else Buffer.add_char buf c
+          end
+        with
+        | Unix.Unix_error (Unix.EAGAIN, _, _)
+        | Unix.Unix_error (Unix.EWOULDBLOCK, _, _) -> stop := true
+      done;
+      if !got_data then String (Buffer.contents buf) else Nil
+    | _ -> raise (Eval_error "channel-read-line: (channel)"));
+
+  register "channel-write" (fun args ->
+    match args with
+    | [String name; String s] ->
+      let (fd, _, _, _) = chan_get name in
+      let b = Bytes.of_string s in
+      let n = Bytes.length b in
+      let written = ref 0 in
+      while !written < n do
+        (try
+          let w = Unix.write fd b !written (n - !written) in
+          written := !written + w
+        with
+        | Unix.Unix_error (Unix.EAGAIN, _, _)
+        | Unix.Unix_error (Unix.EWOULDBLOCK, _, _) ->
+          (* short write — let caller retry *)
+          written := n)
+      done;
+      Nil
+    | _ -> raise (Eval_error "channel-write: (channel string)"));
+
+  register "channel-flush" (fun args ->
+    match args with
+    | [String name] -> let _ = chan_get name in Nil  (* no userspace buffer *)
+    | _ -> raise (Eval_error "channel-flush: (channel)"));
+
+  register "channel-seek" (fun args ->
+    let (name, offset, whence) = match args with
+      | [String n; Integer o] -> (n, o, "start")
+      | [String n; Number o] -> (n, int_of_float o, "start")
+      | [String n; Integer o; String w] -> (n, o, w)
+      | [String n; Number o; String w] -> (n, int_of_float o, w)
+      | _ -> raise (Eval_error "channel-seek: (channel offset ?whence?)")
+    in
+    let (fd, _, eof, _) = chan_get name in
+    let cmd = match whence with
+      | "start" -> Unix.SEEK_SET
+      | "current" -> Unix.SEEK_CUR
+      | "end" -> Unix.SEEK_END
+      | _ -> raise (Eval_error ("channel-seek: invalid whence " ^ whence))
+    in
+    let _ = Unix.lseek fd offset cmd in
+    eof := false;
+    Nil);
+
+  register "channel-tell" (fun args ->
+    match args with
+    | [String name] ->
+      let (fd, _, _, _) = chan_get name in
+      Integer (Unix.lseek fd 0 Unix.SEEK_CUR)
+    | _ -> raise (Eval_error "channel-tell: (channel)"));
+
+  register "channel-eof?" (fun args ->
+    match args with
+    | [String name] ->
+      let (_, _, eof, _) = chan_get name in
+      Bool !eof
+    | _ -> raise (Eval_error "channel-eof?: (channel)"));
+
+  register "channel-blocking?" (fun args ->
+    match args with
+    | [String name] ->
+      let (_, _, _, blocking) = chan_get name in
+      Bool !blocking
+    | _ -> raise (Eval_error "channel-blocking?: (channel)"));
+
+  register "channel-set-blocking!" (fun args ->
+    match args with
+    | [String name; Bool b] ->
+      let (fd, _, _, blocking) = chan_get name in
+      blocking := b;
+      (try
+        if b then Unix.clear_nonblock fd
+        else Unix.set_nonblock fd
+      with _ -> ());
+      Nil
+    | _ -> raise (Eval_error "channel-set-blocking!: (channel bool)"));
+
+  (* === Sockets === wrapping Unix.socket/connect/bind/listen/accept *)
+  let resolve_inet_addr host =
+    if host = "" || host = "0.0.0.0" then Unix.inet_addr_any
+    else if host = "localhost" then Unix.inet_addr_loopback
+    else
+      try Unix.inet_addr_of_string host
+      with _ ->
+        try
+          let entry = Unix.gethostbyname host in
+          if Array.length entry.Unix.h_addr_list = 0 then
+            raise (Eval_error ("socket: cannot resolve " ^ host))
+          else entry.Unix.h_addr_list.(0)
+        with Not_found -> raise (Eval_error ("socket: cannot resolve " ^ host))
+  in
+  let port_of v = match v with
+    | Integer n -> n
+    | Number n -> int_of_float n
+    | _ -> raise (Eval_error "socket: port must be a number")
+  in
+  let alloc_chan_name () =
+    let id = !channel_next_id in
+    incr channel_next_id;
+    Printf.sprintf "sock%d" id
+  in
+
+  register "socket-connect" (fun args ->
+    match args with
+    | [String host; port_v] ->
+      let port = port_of port_v in
+      let addr = Unix.ADDR_INET (resolve_inet_addr host, port) in
+      let sock = Unix.socket Unix.PF_INET Unix.SOCK_STREAM 0 in
+      (try Unix.connect sock addr
+       with Unix.Unix_error (e, _, _) ->
+         (try Unix.close sock with _ -> ());
+         raise (Eval_error ("socket-connect: " ^ Unix.error_message e)));
+      let name = alloc_chan_name () in
+      Hashtbl.replace channel_table name (sock, "rw", ref false, ref true);
+      String name
+    | _ -> raise (Eval_error "socket-connect: (host port)"));
+
+  (* Non-blocking connect: returns channel immediately. Connection completes
+     when the channel becomes writable; query channel-async-error? after to
+     confirm success or get the error. *)
+  register "socket-connect-async" (fun args ->
+    match args with
+    | [String host; port_v] ->
+      let port = port_of port_v in
+      let addr = Unix.ADDR_INET (resolve_inet_addr host, port) in
+      let sock = Unix.socket Unix.PF_INET Unix.SOCK_STREAM 0 in
+      Unix.set_nonblock sock;
+      (try Unix.connect sock addr
+       with
+       | Unix.Unix_error (Unix.EINPROGRESS, _, _)
+       | Unix.Unix_error (Unix.EWOULDBLOCK, _, _) -> ()
+       | Unix.Unix_error (e, _, _) ->
+         (try Unix.close sock with _ -> ());
+         raise (Eval_error ("socket-connect-async: " ^ Unix.error_message e)));
+      let name = alloc_chan_name () in
+      Hashtbl.replace channel_table name (sock, "rw", ref false, ref false);
+      String name
+    | _ -> raise (Eval_error "socket-connect-async: (host port)"));
+
+  (* After a non-blocking connect completes (channel writable), check whether
+     the connect succeeded. Returns "" on success, error message on failure. *)
+  register "channel-async-error" (fun args ->
+    match args with
+    | [String name] ->
+      let (fd, _, _, _) = chan_get name in
+      (try
+        let err = Unix.getsockopt_error fd in
+        match err with
+        | None -> String ""
+        | Some e -> String (Unix.error_message e)
+      with
+      | Unix.Unix_error (e, _, _) -> String (Unix.error_message e))
+    | _ -> raise (Eval_error "channel-async-error: (channel)"));
+
+  register "socket-server" (fun args ->
+    let (host, port) = match args with
+      | [port_v] -> ("", port_of port_v)
+      | [String h; port_v] -> (h, port_of port_v)
+      | _ -> raise (Eval_error "socket-server: (port) or (host port)")
+    in
+    let addr = Unix.ADDR_INET (resolve_inet_addr host, port) in
+    let sock = Unix.socket Unix.PF_INET Unix.SOCK_STREAM 0 in
+    Unix.setsockopt sock Unix.SO_REUSEADDR true;
+    (try Unix.bind sock addr
+     with Unix.Unix_error (e, _, _) ->
+       (try Unix.close sock with _ -> ());
+       raise (Eval_error ("socket-server: bind: " ^ Unix.error_message e)));
+    Unix.listen sock 8;
+    let name = alloc_chan_name () in
+    Hashtbl.replace channel_table name (sock, "server", ref false, ref true);
+    String name);
+
+  register "socket-accept" (fun args ->
+    match args with
+    | [String name] ->
+      let (sock, _, _, _) = chan_get name in
+      let (client_sock, client_addr) =
+        try Unix.accept sock
+        with Unix.Unix_error (e, _, _) ->
+          raise (Eval_error ("socket-accept: " ^ Unix.error_message e))
+      in
+      let (host_str, port) = match client_addr with
+        | Unix.ADDR_INET (addr, p) -> (Unix.string_of_inet_addr addr, p)
+        | Unix.ADDR_UNIX path -> (path, 0)
+      in
+      let client_name = alloc_chan_name () in
+      Hashtbl.replace channel_table client_name (client_sock, "rw", ref false, ref true);
+      let d = Hashtbl.create 3 in
+      Hashtbl.replace d "channel" (String client_name);
+      Hashtbl.replace d "host" (String host_str);
+      Hashtbl.replace d "port" (Integer port);
+      Dict d
+    | _ -> raise (Eval_error "socket-accept: (server-channel)"));
+
+  (* io-select-channels: (read-list write-list timeout-ms) → {:readable [...] :writable [...]}
+     timeout-ms < 0 → block indefinitely; 0 → poll. Returns ready channel names. *)
+  register "io-select-channels" (fun args ->
+    let to_ms v = match v with
+      | Integer n -> n
+      | Number n -> int_of_float n
+      | _ -> raise (Eval_error "io-select-channels: timeout must be a number")
+    in
+    let to_list v = match v with
+      | List xs | ListRef { contents = xs } -> xs
+      | Nil -> []
+      | _ -> raise (Eval_error "io-select-channels: expected list")
+    in
+    let chan_name_of v = match v with
+      | String s -> s
+      | _ -> raise (Eval_error "io-select-channels: channel must be a string")
+    in
+    let (read_list, write_list, timeout_ms) = match args with
+      | [r; w; t] -> (to_list r, to_list w, to_ms t)
+      | _ -> raise (Eval_error "io-select-channels: (read-list write-list timeout-ms)")
+    in
+    let read_pairs = List.map (fun v ->
+      let name = chan_name_of v in
+      let (fd, _, _, _) = chan_get name in (name, fd)) read_list in
+    let write_pairs = List.map (fun v ->
+      let name = chan_name_of v in
+      let (fd, _, _, _) = chan_get name in (name, fd)) write_list in
+    let read_fds = List.map snd read_pairs in
+    let write_fds = List.map snd write_pairs in
+    let timeout = if timeout_ms < 0 then -1.0 else float_of_int timeout_ms /. 1000.0 in
+    let (ready_r, ready_w, _) =
+      try Unix.select read_fds write_fds [] timeout
+      with Unix.Unix_error (Unix.EINTR, _, _) -> ([], [], [])
+    in
+    let names_of pairs ready =
+      List.filter_map (fun (n, fd) ->
+        if List.exists (fun rfd -> rfd = fd) ready then Some (String n) else None
+      ) pairs
+    in
+    let d = Hashtbl.create 2 in
+    Hashtbl.replace d "readable" (List (names_of read_pairs ready_r));
+    Hashtbl.replace d "writable" (List (names_of write_pairs ready_w));
+    Dict d);
+
   (* === Clock === *)
   register "clock-seconds" (fun args ->
     match args with
@@ -3135,11 +3571,8 @@ let () =
     | [] -> Integer (int_of_float (Unix.gettimeofday () *. 1000.0))
     | _ -> raise (Eval_error "clock-milliseconds: no args"));
 
-  register "clock-format" (fun args ->
+  let format_tm tm tz_label =
-    match args with
+    fun fmt ->
-    | [Integer t] | [Integer t; String _] ->
-      let fmt = (match args with [_; String f] -> f | _ -> "%a %b %e %H:%M:%S %Z %Y") in
-      let tm = Unix.gmtime (float_of_int t) in
       let buf = Buffer.create 32 in
       let n = String.length fmt in
       let i = ref 0 in
@@ -3147,14 +3580,19 @@ let () =
         if fmt.[!i] = '%' && !i + 1 < n then begin
           (match fmt.[!i + 1] with
            | 'Y' -> Buffer.add_string buf (Printf.sprintf "%04d" (1900 + tm.Unix.tm_year))
+           | 'y' -> Buffer.add_string buf (Printf.sprintf "%02d" ((1900 + tm.Unix.tm_year) mod 100))
            | 'm' -> Buffer.add_string buf (Printf.sprintf "%02d" (tm.Unix.tm_mon + 1))
            | 'd' -> Buffer.add_string buf (Printf.sprintf "%02d" tm.Unix.tm_mday)
            | 'e' -> Buffer.add_string buf (Printf.sprintf "%2d"  tm.Unix.tm_mday)
            | 'H' -> Buffer.add_string buf (Printf.sprintf "%02d" tm.Unix.tm_hour)
+           | 'I' -> let h = tm.Unix.tm_hour mod 12 in
+                    Buffer.add_string buf (Printf.sprintf "%02d" (if h = 0 then 12 else h))
+           | 'p' -> Buffer.add_string buf (if tm.Unix.tm_hour < 12 then "AM" else "PM")
            | 'M' -> Buffer.add_string buf (Printf.sprintf "%02d" tm.Unix.tm_min)
            | 'S' -> Buffer.add_string buf (Printf.sprintf "%02d" tm.Unix.tm_sec)
            | 'j' -> Buffer.add_string buf (Printf.sprintf "%03d" (tm.Unix.tm_yday + 1))
-           | 'Z' -> Buffer.add_string buf "UTC"
+           | 'w' -> Buffer.add_string buf (string_of_int tm.Unix.tm_wday)
+           | 'Z' -> Buffer.add_string buf tz_label
            | 'a' -> let days = [|"Sun";"Mon";"Tue";"Wed";"Thu";"Fri";"Sat"|] in
                     Buffer.add_string buf days.(tm.Unix.tm_wday)
            | 'A' -> let days = [|"Sunday";"Monday";"Tuesday";"Wednesday";"Thursday";"Friday";"Saturday"|] in
@@ -3163,6 +3601,7 @@ let () =
                     Buffer.add_string buf mons.(tm.Unix.tm_mon)
            | 'B' -> let mons = [|"January";"February";"March";"April";"May";"June";"July";"August";"September";"October";"November";"December"|] in
                     Buffer.add_string buf mons.(tm.Unix.tm_mon)
+           | '%' -> Buffer.add_char buf '%'
            | c -> Buffer.add_char buf '%'; Buffer.add_char buf c);
           i := !i + 2
         end else begin
@@ -3170,8 +3609,100 @@ let () =
           incr i
         end
       done;
-      String (Buffer.contents buf)
+      Buffer.contents buf
-    | _ -> raise (Eval_error "clock-format: (seconds [format])"));
+  in
+  register "clock-format" (fun args ->
+    let (t, fmt, tz) = match args with
+      | [Integer t] -> (t, "%a %b %e %H:%M:%S %Z %Y", "utc")
+      | [Integer t; String f] -> (t, f, "utc")
+      | [Integer t; String f; String z] -> (t, f, z)
+      | _ -> raise (Eval_error "clock-format: (seconds [format [tz]])")
+    in
+    let tm =
+      if tz = "local" then Unix.localtime (float_of_int t)
+      else Unix.gmtime (float_of_int t)
+    in
+    let label = if tz = "local" then "" else "UTC" in
+    String (format_tm tm label fmt));
+
+  (* clock-scan: parse a date string with format, return seconds.
+     Supports the same format specifiers as clock-format (fixed-width ones).
+     tz: "utc" (default) or "local". *)
+  let timegm (tm : Unix.tm) =
+    let is_leap y = y mod 4 = 0 && (y mod 100 <> 0 || y mod 400 = 0) in
+    let days_in_month = [|31;28;31;30;31;30;31;31;30;31;30;31|] in
+    let year = tm.Unix.tm_year + 1900 in
+    let mon = tm.Unix.tm_mon in
+    let mday = tm.Unix.tm_mday in
+    let total_days = ref 0 in
+    if year >= 1970 then begin
+      for y = 1970 to year - 1 do
+        total_days := !total_days + (if is_leap y then 366 else 365)
+      done
+    end else begin
+      for y = year to 1969 do
+        total_days := !total_days - (if is_leap y then 366 else 365)
+      done
+    end;
+    for m = 0 to mon - 1 do
+      total_days := !total_days + days_in_month.(m);
+      if m = 1 && is_leap year then incr total_days
+    done;
+    total_days := !total_days + mday - 1;
+    !total_days * 86400
+    + tm.Unix.tm_hour * 3600
+    + tm.Unix.tm_min * 60
+    + tm.Unix.tm_sec
+  in
+  register "clock-scan" (fun args ->
+    let (str, fmt, tz) = match args with
+      | [String s; String f] -> (s, f, "utc")
+      | [String s; String f; String z] -> (s, f, z)
+      | _ -> raise (Eval_error "clock-scan: (str fmt [tz])")
+    in
+    let n = String.length fmt and sn = String.length str in
+    let tm = ref { Unix.tm_year = 70; tm_mon = 0; tm_mday = 1;
+                   tm_hour = 0; tm_min = 0; tm_sec = 0;
+                   tm_wday = 0; tm_yday = 0; tm_isdst = false } in
+    let i = ref 0 and j = ref 0 in
+    let read_n_digits k =
+      let s = ref "" in
+      let cnt = ref 0 in
+      while !cnt < k && !j < sn && str.[!j] >= '0' && str.[!j] <= '9' do
+        s := !s ^ String.make 1 str.[!j];
+        incr j; incr cnt
+      done;
+      if !s = "" then 0 else int_of_string !s
+    in
+    let skip_ws () =
+      while !j < sn && (str.[!j] = ' ' || str.[!j] = '\t') do incr j done
+    in
+    while !i < n do
+      if fmt.[!i] = '%' && !i + 1 < n then begin
+        (match fmt.[!i + 1] with
+         | 'Y' -> tm := { !tm with tm_year = read_n_digits 4 - 1900 }
+         | 'y' -> let y = read_n_digits 2 in
+                  tm := { !tm with tm_year = (if y < 70 then 100 + y else y) }
+         | 'm' -> tm := { !tm with tm_mon = read_n_digits 2 - 1 }
+         | 'd' | 'e' -> skip_ws (); tm := { !tm with tm_mday = read_n_digits 2 }
+         | 'H' | 'I' -> tm := { !tm with tm_hour = read_n_digits 2 }
+         | 'M' -> tm := { !tm with tm_min = read_n_digits 2 }
+         | 'S' -> tm := { !tm with tm_sec = read_n_digits 2 }
+         | '%' -> if !j < sn && str.[!j] = '%' then incr j
+         | _ -> ()  (* unsupported specifier — skip *)
+        );
+        i := !i + 2
+      end else begin
+        if fmt.[!i] = ' ' then skip_ws ()
+        else if !j < sn && str.[!j] = fmt.[!i] then incr j;
+        incr i
+      end
+    done;
+    let secs =
+      if tz = "local" then int_of_float (fst (Unix.mktime !tm))
+      else timegm !tm
+    in
+    Integer secs);
 
   (* === Env-as-value (Phase 4) === *)
 

hosts/ocaml/lib/sx_vm.ml

@@ -642,7 +642,9 @@ and run vm =
           (* Read upvalue descriptors from bytecode *)
           let uv_count = match code_val with
             | Dict d -> (match Hashtbl.find_opt d "upvalue-count" with
-                | Some (Number n) -> int_of_float n | _ -> 0)
+                | Some (Integer n) -> n
+                | Some (Number n) -> int_of_float n
+                | _ -> 0)
             | _ -> 0
           in
           let upvalues = Array.init uv_count (fun _ ->
@@ -1307,7 +1309,9 @@ let trace_run src globals =
             let code_val2 = frame.closure.vm_code.vc_constants.(idx) in
             let uv_count = match code_val2 with
               | Dict d -> (match Hashtbl.find_opt d "upvalue-count" with
-                  | Some (Number n) -> int_of_float n | _ -> 0)
+                  | Some (Integer n) -> n
+                  | Some (Number n) -> int_of_float n
+                  | _ -> 0)
               | _ -> 0 in
             let upvalues = Array.init uv_count (fun _ ->
               let is_local = read_u8 frame in
@@ -1428,7 +1432,9 @@ let disassemble (code : vm_code) =
         if op = 51 && idx < Array.length consts then begin
           let uv_count = match consts.(idx) with
             | Dict d -> (match Hashtbl.find_opt d "upvalue-count" with
-                | Some (Number n) -> int_of_float n | _ -> 0)
+                | Some (Integer n) -> n
+                | Some (Number n) -> int_of_float n
+                | _ -> 0)
             | _ -> 0 in
           ip := !ip + uv_count * 2
         end

hosts/ocaml/lib/sx_vm_ref.ml

@@ -270,7 +270,9 @@ let vm_create_closure vm_val frame_val code_val =
   let f = unwrap_frame frame_val in
   let uv_count = match code_val with
     | Dict d -> (match Hashtbl.find_opt d "upvalue-count" with
-        | Some (Number n) -> int_of_float n | _ -> 0)
+        | Some (Integer n) -> n
+        | Some (Number n) -> int_of_float n
+        | _ -> 0)
     | _ -> 0
   in
   let upvalues = Array.init uv_count (fun _ ->

hosts/ocaml/sx_vm_ref.ml

@@ -265,7 +265,9 @@ let vm_create_closure vm_val frame_val code_val =
   let f = unwrap_frame frame_val in
   let uv_count = match code_val with
     | Dict d -> (match Hashtbl.find_opt d "upvalue-count" with
-        | Some (Number n) -> int_of_float n | _ -> 0)
+        | Some (Integer n) -> n
+        | Some (Number n) -> int_of_float n
+        | _ -> 0)
     | _ -> 0
   in
   let upvalues = Array.init uv_count (fun _ ->

lib/apl/conformance.sh

@@ -0,0 +1,116 @@
+#!/usr/bin/env bash
+# lib/apl/conformance.sh — run APL test suites, emit scoreboard.json + scoreboard.md.
+
+set -uo pipefail
+cd "$(git rev-parse --show-toplevel)"
+
+SX_SERVER="${SX_SERVER:-/root/rose-ash/hosts/ocaml/_build/default/bin/sx_server.exe}"
+if [ ! -x "$SX_SERVER" ]; then
+  SX_SERVER="hosts/ocaml/_build/default/bin/sx_server.exe"
+fi
+if [ ! -x "$SX_SERVER" ]; then
+  echo "ERROR: sx_server.exe not found." >&2
+  exit 1
+fi
+
+SUITES=(structural operators dfn tradfn valence programs system idioms eval-ops pipeline)
+
+OUT_JSON="lib/apl/scoreboard.json"
+OUT_MD="lib/apl/scoreboard.md"
+
+run_suite() {
+  local suite=$1
+  local file="lib/apl/tests/${suite}.sx"
+  local TMP
+  TMP=$(mktemp)
+  cat > "$TMP" << EPOCHS
+(epoch 1)
+(load "spec/stdlib.sx")
+(load "lib/r7rs.sx")
+(load "lib/apl/runtime.sx")
+(load "lib/apl/tokenizer.sx")
+(load "lib/apl/parser.sx")
+(load "lib/apl/transpile.sx")
+(epoch 2)
+(eval "(define apl-test-pass 0)")
+(eval "(define apl-test-fail 0)")
+(eval "(define apl-test (fn (name got expected) (if (= got expected) (set! apl-test-pass (+ apl-test-pass 1)) (set! apl-test-fail (+ apl-test-fail 1)))))")
+(epoch 3)
+(load "${file}")
+(epoch 4)
+(eval "(list apl-test-pass apl-test-fail)")
+EPOCHS
+
+  local OUTPUT
+  OUTPUT=$(timeout 300 "$SX_SERVER" < "$TMP" 2>/dev/null)
+  rm -f "$TMP"
+
+  local LINE
+  LINE=$(echo "$OUTPUT" | awk '/^\(ok-len 4 / {getline; print; exit}')
+  if [ -z "$LINE" ]; then
+    LINE=$(echo "$OUTPUT" | grep -E '^\(ok 4 \([0-9]+ [0-9]+\)\)' | tail -1 \
+            | sed -E 's/^\(ok 4 //; s/\)$//')
+  fi
+
+  local P F
+  P=$(echo "$LINE" | sed -E 's/^\(([0-9]+) ([0-9]+)\).*/\1/')
+  F=$(echo "$LINE" | sed -E 's/^\(([0-9]+) ([0-9]+)\).*/\2/')
+  P=${P:-0}
+  F=${F:-0}
+  echo "${P} ${F}"
+}
+
+declare -A SUITE_PASS
+declare -A SUITE_FAIL
+TOTAL_PASS=0
+TOTAL_FAIL=0
+
+echo "Running APL conformance suite..." >&2
+for s in "${SUITES[@]}"; do
+  read -r p f < <(run_suite "$s")
+  SUITE_PASS[$s]=$p
+  SUITE_FAIL[$s]=$f
+  TOTAL_PASS=$((TOTAL_PASS + p))
+  TOTAL_FAIL=$((TOTAL_FAIL + f))
+  printf "  %-12s %d/%d\n" "$s" "$p" "$((p+f))" >&2
+done
+
+# scoreboard.json
+{
+  printf '{\n'
+  printf '  "suites": {\n'
+  first=1
+  for s in "${SUITES[@]}"; do
+    if [ $first -eq 0 ]; then printf ',\n'; fi
+    printf '    "%s": {"pass": %d, "fail": %d}' "$s" "${SUITE_PASS[$s]}" "${SUITE_FAIL[$s]}"
+    first=0
+  done
+  printf '\n  },\n'
+  printf '  "total_pass": %d,\n' "$TOTAL_PASS"
+  printf '  "total_fail": %d,\n' "$TOTAL_FAIL"
+  printf '  "total": %d\n' "$((TOTAL_PASS + TOTAL_FAIL))"
+  printf '}\n'
+} > "$OUT_JSON"
+
+# scoreboard.md
+{
+  printf '# APL Conformance Scoreboard\n\n'
+  printf '_Generated by `lib/apl/conformance.sh`_\n\n'
+  printf '| Suite | Pass | Fail | Total |\n'
+  printf '|-------|-----:|-----:|------:|\n'
+  for s in "${SUITES[@]}"; do
+    p=${SUITE_PASS[$s]}
+    f=${SUITE_FAIL[$s]}
+    printf '| %s | %d | %d | %d |\n' "$s" "$p" "$f" "$((p+f))"
+  done
+  printf '| **Total** | **%d** | **%d** | **%d** |\n' "$TOTAL_PASS" "$TOTAL_FAIL" "$((TOTAL_PASS + TOTAL_FAIL))"
+  printf '\n'
+  printf '## Notes\n\n'
+  printf '%s\n' '- Suites use the standard `apl-test name got expected` framework loaded against `lib/apl/runtime.sx` + `lib/apl/transpile.sx`.'
+  printf '%s\n' '- `lib/apl/tests/parse.sx` and `lib/apl/tests/scalar.sx` use their own self-contained frameworks and are excluded from this scoreboard.'
+} > "$OUT_MD"
+
+echo "Wrote $OUT_JSON and $OUT_MD" >&2
+echo "Total: $TOTAL_PASS pass, $TOTAL_FAIL fail" >&2
+
+[ "$TOTAL_FAIL" -eq 0 ]

lib/apl/parser.sx

@@ -0,0 +1,674 @@
+; APL Parser — right-to-left expression parser
+;
+; Takes a token list (output of apl-tokenize) and produces an AST.
+; APL evaluates right-to-left with no precedence among functions.
+; Operators bind to the function immediately to their left in the source.
+;
+; AST node types:
+;   (:num n)                     number literal
+;   (:str s)                     string literal
+;   (:vec n1 n2 ...)             strand (juxtaposed literals)
+;   (:name "x")                  name reference / alpha / omega
+;   (:assign "x" expr)           assignment  x←expr
+;   (:monad fn arg)              monadic function call
+;   (:dyad fn left right)        dyadic function call
+;   (:derived-fn op fn)          derived function: f/ f¨ f⍨
+;   (:derived-fn2 "." f g)       inner product: f.g
+;   (:outer "∘." fn)             outer product: ∘.f
+;   (:fn-glyph "⍳")             function reference
+;   (:fn-name "foo")             named-function reference (dfn variable)
+;   (:dfn stmt...)               {⍺+⍵} anonymous function
+;   (:guard cond expr)           cond:expr guard inside dfn
+;   (:program stmt...)           multi-statement sequence
+
+; ============================================================
+; Glyph classification sets
+; ============================================================
+
+(define
+  apl-parse-op-glyphs
+  (list "/" "⌿" "\\" "⍀" "¨" "⍨" "∘" "." "⍣" "⍤" "⍥" "@"))
+
+(define
+  apl-parse-fn-glyphs
+  (list
+    "+"
+    "-"
+    "×"
+    "÷"
+    "*"
+    "⍟"
+    "⌈"
+    "⌊"
+    "|"
+    "!"
+    "?"
+    "○"
+    "~"
+    "<"
+    "≤"
+    "="
+    "≥"
+    ">"
+    "≠"
+    "≢"
+    "≡"
+    "∊"
+    "∧"
+    "∨"
+    "⍱"
+    "⍲"
+    ","
+    "⍪"
+    "⍴"
+    "⌽"
+    "⊖"
+    "⍉"
+    "↑"
+    "↓"
+    "⊂"
+    "⊃"
+    "⊆"
+    "∪"
+    "∩"
+    "⍳"
+    "⍸"
+    "⌷"
+    "⍋"
+    "⍒"
+    "⊥"
+    "⊤"
+    "⊣"
+    "⊢"
+    "⍎"
+    "⍕"))
+
+(define apl-quad-fn-names (list "⎕FMT" "⎕←"))
+
+(define apl-known-fn-names (list))
+
+; ============================================================
+; Token accessors
+; ============================================================
+
+(define
+  apl-collect-fn-bindings
+  (fn
+    (stmt-groups)
+    (set! apl-known-fn-names (list))
+    (for-each
+      (fn
+        (toks)
+        (when
+          (and
+            (>= (len toks) 3)
+            (= (tok-type (nth toks 0)) :name)
+            (= (tok-type (nth toks 1)) :assign)
+            (= (tok-type (nth toks 2)) :lbrace))
+          (set!
+            apl-known-fn-names
+            (cons (tok-val (nth toks 0)) apl-known-fn-names))))
+      stmt-groups)))
+
+(define
+  apl-parse-op-glyph?
+  (fn (v) (some (fn (g) (= g v)) apl-parse-op-glyphs)))
+
+(define
+  apl-parse-fn-glyph?
+  (fn (v) (some (fn (g) (= g v)) apl-parse-fn-glyphs)))
+
+(define tok-type (fn (tok) (get tok :type)))
+
+; ============================================================
+; Collect trailing operators starting at index i
+; Returns {:ops (op ...) :end new-i}
+; ============================================================
+
+(define tok-val (fn (tok) (get tok :value)))
+
+(define
+  is-op-tok?
+  (fn
+    (tok)
+    (and (= (tok-type tok) :glyph) (apl-parse-op-glyph? (tok-val tok)))))
+
+; ============================================================
+; Build a derived-fn node by chaining operators left-to-right
+; (+/¨  →  (:derived-fn "¨" (:derived-fn "/" (:fn-glyph "+"))))
+; ============================================================
+
+(define
+  is-fn-tok?
+  (fn
+    (tok)
+    (or
+      (and (= (tok-type tok) :glyph) (apl-parse-fn-glyph? (tok-val tok)))
+      (and
+        (= (tok-type tok) :name)
+        (or
+          (some (fn (q) (= q (tok-val tok))) apl-quad-fn-names)
+          (some (fn (q) (= q (tok-val tok))) apl-known-fn-names))))))
+
+; ============================================================
+; Find matching close bracket/paren/brace
+; Returns the index of the matching close token
+; ============================================================
+
+(define collect-ops (fn (tokens i) (collect-ops-loop tokens i (list))))
+
+(define
+  collect-ops-loop
+  (fn
+    (tokens i acc)
+    (if
+      (>= i (len tokens))
+      {:end i :ops acc}
+      (let
+        ((tok (nth tokens i)))
+        (if
+          (is-op-tok? tok)
+          (collect-ops-loop tokens (+ i 1) (append acc (tok-val tok)))
+          {:end i :ops acc})))))
+
+; ============================================================
+; Segment collection: scan tokens left-to-right, building
+; a list of {:kind "val"/"fn" :node ast} segments.
+; Operators following function glyphs are merged into
+; derived-fn nodes during this pass.
+; ============================================================
+
+(define
+  build-derived-fn
+  (fn
+    (fn-node ops)
+    (if
+      (= (len ops) 0)
+      fn-node
+      (build-derived-fn (list :derived-fn (first ops) fn-node) (rest ops)))))
+
+(define
+  find-matching-close
+  (fn
+    (tokens start open-type close-type)
+    (find-matching-close-loop tokens start open-type close-type 1)))
+
+; ============================================================
+; Build tree from segment list
+;
+; The segments are in left-to-right order.
+; APL evaluates right-to-left, so the LEFTMOST function is
+; the outermost (last-evaluated) node.
+;
+; Patterns:
+;   [val]              → val node
+;   [fn val ...]       → (:monad fn (build-tree rest))
+;   [val fn val ...]   → (:dyad fn val (build-tree rest))
+;   [val val ...]      → (:vec val1 val2 ...) — strand
+; ============================================================
+
+; Find the index of the first function segment (returns -1 if none)
+(define
+  find-matching-close-loop
+  (fn
+    (tokens i open-type close-type depth)
+    (if
+      (>= i (len tokens))
+      (len tokens)
+      (let
+        ((tt (tok-type (nth tokens i))))
+        (cond
+          ((= tt open-type)
+            (find-matching-close-loop
+              tokens
+              (+ i 1)
+              open-type
+              close-type
+              (+ depth 1)))
+          ((= tt close-type)
+            (if
+              (= depth 1)
+              i
+              (find-matching-close-loop
+                tokens
+                (+ i 1)
+                open-type
+                close-type
+                (- depth 1))))
+          (true
+            (find-matching-close-loop
+              tokens
+              (+ i 1)
+              open-type
+              close-type
+              depth)))))))
+
+(define
+  collect-segments
+  (fn (tokens) (collect-segments-loop tokens 0 (list))))
+
+; Build an array node from 0..n value segments
+; If n=1 → return that segment's node
+; If n>1 → return (:vec node1 node2 ...)
+(define
+  collect-segments-loop
+  (fn
+    (tokens i acc)
+    (if
+      (>= i (len tokens))
+      acc
+      (let
+        ((tok (nth tokens i)) (n (len tokens)))
+        (let
+          ((tt (tok-type tok)) (tv (tok-val tok)))
+          (cond
+            ((or (= tt :diamond) (= tt :newline) (= tt :semi))
+              (collect-segments-loop tokens (+ i 1) acc))
+            ((= tt :num)
+              (collect-segments-loop tokens (+ i 1) (append acc {:kind "val" :node (list :num tv)})))
+            ((= tt :str)
+              (collect-segments-loop tokens (+ i 1) (append acc {:kind "val" :node (list :str tv)})))
+            ((= tt :name)
+              (cond
+                ((some (fn (q) (= q tv)) apl-quad-fn-names)
+                  (let
+                    ((op-result (collect-ops tokens (+ i 1))))
+                    (let
+                      ((ops (get op-result :ops))
+                        (ni (get op-result :end)))
+                      (let
+                        ((fn-node (build-derived-fn (list :fn-glyph tv) ops)))
+                        (collect-segments-loop
+                          tokens
+                          ni
+                          (append acc {:kind "fn" :node fn-node}))))))
+                ((some (fn (q) (= q tv)) apl-known-fn-names)
+                  (let
+                    ((op-result (collect-ops tokens (+ i 1))))
+                    (let
+                      ((ops (get op-result :ops))
+                        (ni (get op-result :end)))
+                      (let
+                        ((fn-node (build-derived-fn (list :fn-name tv) ops)))
+                        (collect-segments-loop
+                          tokens
+                          ni
+                          (append acc {:kind "fn" :node fn-node}))))))
+                (else
+                  (let
+                    ((br (maybe-bracket (list :name tv) tokens (+ i 1))))
+                    (collect-segments-loop
+                      tokens
+                      (nth br 1)
+                      (append acc {:kind "val" :node (nth br 0)}))))))
+            ((= tt :lparen)
+              (let
+                ((end (find-matching-close tokens (+ i 1) :lparen :rparen)))
+                (let
+                  ((inner-tokens (slice tokens (+ i 1) end))
+                    (after (+ end 1)))
+                  (let
+                    ((inner-segs (collect-segments inner-tokens)))
+                    (if
+                      (and
+                        (>= (len inner-segs) 2)
+                        (every? (fn (s) (= (get s :kind) "fn")) inner-segs))
+                      (let
+                        ((train-node (cons :train (map (fn (s) (get s :node)) inner-segs))))
+                        (collect-segments-loop
+                          tokens
+                          after
+                          (append acc {:kind "fn" :node train-node})))
+                      (let
+                        ((br (maybe-bracket (parse-apl-expr inner-tokens) tokens after)))
+                        (collect-segments-loop
+                          tokens
+                          (nth br 1)
+                          (append acc {:kind "val" :node (nth br 0)}))))))))
+            ((= tt :lbrace)
+              (let
+                ((end (find-matching-close tokens (+ i 1) :lbrace :rbrace)))
+                (let
+                  ((inner-tokens (slice tokens (+ i 1) end))
+                    (after (+ end 1)))
+                  (collect-segments-loop tokens after (append acc {:kind "fn" :node (parse-dfn inner-tokens)})))))
+            ((= tt :glyph)
+              (cond
+                ((or (= tv "⍺") (= tv "⍵"))
+                  (collect-segments-loop
+                    tokens
+                    (+ i 1)
+                    (append acc {:kind "val" :node (list :name tv)})))
+                ((= tv "∇")
+                  (collect-segments-loop
+                    tokens
+                    (+ i 1)
+                    (append acc {:kind "fn" :node (list :fn-glyph "∇")})))
+                ((and (= tv "∘") (< (+ i 1) n) (= (tok-val (nth tokens (+ i 1))) "."))
+                  (if
+                    (and (< (+ i 2) n) (is-fn-tok? (nth tokens (+ i 2))))
+                    (let
+                      ((fn-tv (tok-val (nth tokens (+ i 2)))))
+                      (let
+                        ((op-result (collect-ops tokens (+ i 3))))
+                        (let
+                          ((ops (get op-result :ops))
+                            (ni (get op-result :end)))
+                          (let
+                            ((fn-node (build-derived-fn (list :fn-glyph fn-tv) ops)))
+                            (collect-segments-loop
+                              tokens
+                              ni
+                              (append acc {:kind "fn" :node (list :outer "∘." fn-node)}))))))
+                    (collect-segments-loop tokens (+ i 1) acc)))
+                ((apl-parse-fn-glyph? tv)
+                  (let
+                    ((op-result (collect-ops tokens (+ i 1))))
+                    (let
+                      ((ops (get op-result :ops))
+                        (ni (get op-result :end)))
+                      (if
+                        (and
+                          (= (len ops) 1)
+                          (= (first ops) ".")
+                          (< ni n)
+                          (is-fn-tok? (nth tokens ni)))
+                        (let
+                          ((g-tv (tok-val (nth tokens ni))))
+                          (let
+                            ((op-result2 (collect-ops tokens (+ ni 1))))
+                            (let
+                              ((ops2 (get op-result2 :ops))
+                                (ni2 (get op-result2 :end)))
+                              (let
+                                ((g-node (build-derived-fn (list :fn-glyph g-tv) ops2)))
+                                (collect-segments-loop
+                                  tokens
+                                  ni2
+                                  (append acc {:kind "fn" :node (list :derived-fn2 "." (list :fn-glyph tv) g-node)}))))))
+                        (let
+                          ((fn-node (build-derived-fn (list :fn-glyph tv) ops)))
+                          (collect-segments-loop
+                            tokens
+                            ni
+                            (append acc {:kind "fn" :node fn-node})))))))
+                ((apl-parse-op-glyph? tv)
+                  (collect-segments-loop tokens (+ i 1) acc))
+                (true (collect-segments-loop tokens (+ i 1) acc))))
+            (true (collect-segments-loop tokens (+ i 1) acc))))))))
+
+(define find-first-fn (fn (segs) (find-first-fn-loop segs 0)))
+
+
+; ============================================================
+; Split token list on statement separators (diamond / newline)
+; Only splits at depth 0 (ignores separators inside { } or ( ) )
+; ============================================================
+
+(define
+  find-first-fn-loop
+  (fn
+    (segs i)
+    (if
+      (>= i (len segs))
+      -1
+      (if
+        (= (get (nth segs i) :kind) "fn")
+        i
+        (find-first-fn-loop segs (+ i 1))))))
+
+(define
+  segs-to-array
+  (fn
+    (segs)
+    (if
+      (= (len segs) 1)
+      (get (first segs) :node)
+      (cons :vec (map (fn (s) (get s :node)) segs)))))
+
+; ============================================================
+; Parse a dfn body (tokens between { and })
+; Handles guard expressions: cond : expr
+; ============================================================
+
+(define
+  build-tree
+  (fn
+    (segs)
+    (cond
+      ((= (len segs) 0) nil)
+      ((= (len segs) 1) (get (first segs) :node))
+      ((every? (fn (s) (= (get s :kind) "val")) segs)
+        (segs-to-array segs))
+      (true
+        (let
+          ((fn-idx (find-first-fn segs)))
+          (cond
+            ((= fn-idx -1) (segs-to-array segs))
+            ((= fn-idx 0)
+              (list
+                :monad (get (first segs) :node)
+                (build-tree (rest segs))))
+            (true
+              (let
+                ((left-segs (slice segs 0 fn-idx))
+                  (fn-seg (nth segs fn-idx))
+                  (right-segs (slice segs (+ fn-idx 1))))
+                (list
+                  :dyad (get fn-seg :node)
+                  (segs-to-array left-segs)
+                  (build-tree right-segs))))))))))
+
+(define
+  split-statements
+  (fn (tokens) (split-statements-loop tokens (list) (list) 0)))
+
+(define
+  split-statements-loop
+  (fn
+    (tokens current-stmt acc depth)
+    (if
+      (= (len tokens) 0)
+      (if (> (len current-stmt) 0) (append acc (list current-stmt)) acc)
+      (let
+        ((tok (first tokens))
+          (rest-toks (rest tokens))
+          (tt (tok-type (first tokens))))
+        (cond
+          ((or (= tt :lparen) (= tt :lbrace) (= tt :lbracket))
+            (split-statements-loop
+              rest-toks
+              (append current-stmt tok)
+              acc
+              (+ depth 1)))
+          ((or (= tt :rparen) (= tt :rbrace) (= tt :rbracket))
+            (split-statements-loop
+              rest-toks
+              (append current-stmt tok)
+              acc
+              (- depth 1)))
+          ((and (> depth 0) (or (= tt :diamond) (= tt :newline)))
+            (split-statements-loop
+              rest-toks
+              (append current-stmt tok)
+              acc
+              depth))
+          ((and (= depth 0) (or (= tt :diamond) (= tt :newline)))
+            (if
+              (> (len current-stmt) 0)
+              (split-statements-loop
+                rest-toks
+                (list)
+                (append acc (list current-stmt))
+                depth)
+              (split-statements-loop rest-toks (list) acc depth)))
+          (true
+            (split-statements-loop
+              rest-toks
+              (append current-stmt tok)
+              acc
+              depth)))))))
+
+(define
+  parse-dfn
+  (fn
+    (tokens)
+    (let
+      ((stmt-groups (split-statements tokens)))
+      (let ((stmts (map parse-dfn-stmt stmt-groups))) (cons :dfn stmts)))))
+
+; ============================================================
+; Parse a single statement (assignment or expression)
+; ============================================================
+
+(define
+  parse-dfn-stmt
+  (fn
+    (tokens)
+    (let
+      ((colon-idx (find-top-level-colon tokens 0)))
+      (if
+        (>= colon-idx 0)
+        (let
+          ((cond-tokens (slice tokens 0 colon-idx))
+            (body-tokens (slice tokens (+ colon-idx 1))))
+          (list
+            :guard (parse-apl-expr cond-tokens)
+            (parse-apl-expr body-tokens)))
+        (parse-stmt tokens)))))
+
+; ============================================================
+; Parse an expression from a flat token list
+; ============================================================
+
+(define
+  find-top-level-colon
+  (fn (tokens i) (find-top-level-colon-loop tokens i 0)))
+
+; ============================================================
+; Main entry point
+; parse-apl: string → AST
+; ============================================================
+
+(define
+  find-top-level-colon-loop
+  (fn
+    (tokens i depth)
+    (if
+      (>= i (len tokens))
+      -1
+      (let
+        ((tok (nth tokens i)) (tt (tok-type (nth tokens i))))
+        (cond
+          ((or (= tt :lparen) (= tt :lbrace) (= tt :lbracket))
+            (find-top-level-colon-loop tokens (+ i 1) (+ depth 1)))
+          ((or (= tt :rparen) (= tt :rbrace) (= tt :rbracket))
+            (find-top-level-colon-loop tokens (+ i 1) (- depth 1)))
+          ((and (= tt :colon) (= depth 0)) i)
+          (true (find-top-level-colon-loop tokens (+ i 1) depth)))))))
+
+(define
+  parse-stmt
+  (fn
+    (tokens)
+    (if
+      (and
+        (>= (len tokens) 2)
+        (= (tok-type (nth tokens 0)) :name)
+        (= (tok-type (nth tokens 1)) :assign))
+      (list
+        :assign (tok-val (nth tokens 0))
+        (parse-apl-expr (slice tokens 2)))
+      (parse-apl-expr tokens))))
+
+(define
+  parse-apl-expr
+  (fn
+    (tokens)
+    (let
+      ((segs (collect-segments tokens)))
+      (if (= (len segs) 0) nil (build-tree segs)))))
+
+(define
+  parse-apl
+  (fn
+    (src)
+    (let
+      ((tokens (apl-tokenize src)))
+      (let
+        ((stmt-groups (split-statements tokens)))
+        (begin
+          (apl-collect-fn-bindings stmt-groups)
+          (if
+            (= (len stmt-groups) 0)
+            nil
+            (if
+              (= (len stmt-groups) 1)
+              (parse-stmt (first stmt-groups))
+              (cons :program (map parse-stmt stmt-groups)))))))))
+
+(define
+  split-bracket-loop
+  (fn
+    (tokens current acc depth)
+    (if
+      (= (len tokens) 0)
+      (append acc (list current))
+      (let
+        ((tok (first tokens)) (more (rest tokens)))
+        (let
+          ((tt (tok-type tok)))
+          (cond
+            ((or (= tt :lparen) (= tt :lbrace) (= tt :lbracket))
+              (split-bracket-loop
+                more
+                (append current (list tok))
+                acc
+                (+ depth 1)))
+            ((or (= tt :rparen) (= tt :rbrace) (= tt :rbracket))
+              (split-bracket-loop
+                more
+                (append current (list tok))
+                acc
+                (- depth 1)))
+            ((and (= tt :semi) (= depth 0))
+              (split-bracket-loop
+                more
+                (list)
+                (append acc (list current))
+                depth))
+            (else
+              (split-bracket-loop more (append current (list tok)) acc depth))))))))
+
+(define
+  split-bracket-content
+  (fn (tokens) (split-bracket-loop tokens (list) (list) 0)))
+
+(define
+  maybe-bracket
+  (fn
+    (val-node tokens after)
+    (if
+      (and
+        (< after (len tokens))
+        (= (tok-type (nth tokens after)) :lbracket))
+      (let
+        ((end (find-matching-close tokens (+ after 1) :lbracket :rbracket)))
+        (let
+          ((inner-tokens (slice tokens (+ after 1) end))
+            (next-after (+ end 1)))
+          (let
+            ((sections (split-bracket-content inner-tokens)))
+            (if
+              (= (len sections) 1)
+              (let
+                ((idx-expr (parse-apl-expr inner-tokens)))
+                (let
+                  ((indexed (list :dyad (list :fn-glyph "⌷") idx-expr val-node)))
+                  (maybe-bracket indexed tokens next-after)))
+              (let
+                ((axis-exprs (map (fn (toks) (if (= (len toks) 0) :all (parse-apl-expr toks))) sections)))
+                (let
+                  ((indexed (cons :bracket (cons val-node axis-exprs))))
+                  (maybe-bracket indexed tokens next-after)))))))
+      (list val-node after))))

lib/apl/scoreboard.json

@@ -0,0 +1,17 @@
+{
+  "suites": {
+    "structural": {"pass": 94, "fail": 0},
+    "operators": {"pass": 117, "fail": 0},
+    "dfn": {"pass": 24, "fail": 0},
+    "tradfn": {"pass": 25, "fail": 0},
+    "valence": {"pass": 14, "fail": 0},
+    "programs": {"pass": 45, "fail": 0},
+    "system": {"pass": 13, "fail": 0},
+    "idioms": {"pass": 64, "fail": 0},
+    "eval-ops": {"pass": 14, "fail": 0},
+    "pipeline": {"pass": 40, "fail": 0}
+  },
+  "total_pass": 450,
+  "total_fail": 0,
+  "total": 450
+}

lib/apl/scoreboard.md

@@ -0,0 +1,22 @@
+# APL Conformance Scoreboard
+
+_Generated by `lib/apl/conformance.sh`_
+
+| Suite | Pass | Fail | Total |
+|-------|-----:|-----:|------:|
+| structural | 94 | 0 | 94 |
+| operators | 117 | 0 | 117 |
+| dfn | 24 | 0 | 24 |
+| tradfn | 25 | 0 | 25 |
+| valence | 14 | 0 | 14 |
+| programs | 45 | 0 | 45 |
+| system | 13 | 0 | 13 |
+| idioms | 64 | 0 | 64 |
+| eval-ops | 14 | 0 | 14 |
+| pipeline | 40 | 0 | 40 |
+| **Total** | **450** | **0** | **450** |
+
+## Notes
+
+- Suites use the standard `apl-test name got expected` framework loaded against `lib/apl/runtime.sx` + `lib/apl/transpile.sx`.
+- `lib/apl/tests/parse.sx` and `lib/apl/tests/scalar.sx` use their own self-contained frameworks and are excluded from this scoreboard.

lib/apl/test.sh

@@ -4,9 +4,9 @@
 set -uo pipefail
 cd "$(git rev-parse --show-toplevel)"
 
-SX_SERVER="${SX_SERVER:-hosts/ocaml/_build/default/bin/sx_server.exe}"
+SX_SERVER="${SX_SERVER:-/root/rose-ash/hosts/ocaml/_build/default/bin/sx_server.exe}"
 if [ ! -x "$SX_SERVER" ]; then
-  SX_SERVER="/root/rose-ash/hosts/ocaml/_build/default/bin/sx_server.exe"
+  SX_SERVER="hosts/ocaml/_build/default/bin/sx_server.exe"
 fi
 if [ ! -x "$SX_SERVER" ]; then
   echo "ERROR: sx_server.exe not found."
@@ -18,19 +18,38 @@ TMPFILE=$(mktemp); trap "rm -f $TMPFILE" EXIT
 cat > "$TMPFILE" << 'EPOCHS'
 (epoch 1)
 (load "spec/stdlib.sx")
+(load "lib/r7rs.sx")
 (load "lib/apl/runtime.sx")
+(load "lib/apl/tokenizer.sx")
+(load "lib/apl/parser.sx")
+(load "lib/apl/transpile.sx")
 (epoch 2)
-(load "lib/apl/tests/runtime.sx")
+(eval "(define apl-test-pass 0)")
+(eval "(define apl-test-fail 0)")
+(eval "(define apl-test-fails (list))")
+(eval "(define apl-test (fn (name got expected) (if (= got expected) (set! apl-test-pass (+ apl-test-pass 1)) (begin (set! apl-test-fail (+ apl-test-fail 1)) (set! apl-test-fails (append apl-test-fails (list {:name name :got got :expected expected})))))))")
 (epoch 3)
+(load "lib/apl/tests/structural.sx")
+(load "lib/apl/tests/operators.sx")
+(load "lib/apl/tests/dfn.sx")
+(load "lib/apl/tests/tradfn.sx")
+(load "lib/apl/tests/valence.sx")
+(load "lib/apl/tests/programs.sx")
+(load "lib/apl/tests/system.sx")
+(load "lib/apl/tests/idioms.sx")
+(load "lib/apl/tests/eval-ops.sx")
+(load "lib/apl/tests/pipeline.sx")
+(load "lib/apl/tests/programs-e2e.sx")
+(epoch 4)
 (eval "(list apl-test-pass apl-test-fail)")
 EPOCHS
 
-OUTPUT=$(timeout 60 "$SX_SERVER" < "$TMPFILE" 2>/dev/null)
+OUTPUT=$(timeout 300 "$SX_SERVER" < "$TMPFILE" 2>/dev/null)
 
-LINE=$(echo "$OUTPUT" | awk '/^\(ok-len 3 / {getline; print; exit}')
+LINE=$(echo "$OUTPUT" | awk '/^\(ok-len 4 / {getline; print; exit}')
 if [ -z "$LINE" ]; then
-  LINE=$(echo "$OUTPUT" | grep -E '^\(ok 3 \([0-9]+ [0-9]+\)\)' | tail -1 \
+  LINE=$(echo "$OUTPUT" | grep -E '^\(ok 4 \([0-9]+ [0-9]+\)\)' | tail -1 \
-          | sed -E 's/^\(ok 3 //; s/\)$//')
+          | sed -E 's/^\(ok 4 //; s/\)$//')
 fi
 if [ -z "$LINE" ]; then
   echo "ERROR: could not extract summary"

lib/apl/tests/dfn.sx

@@ -0,0 +1,227 @@
+; Tests for apl-eval-ast and apl-call-dfn (manual AST construction).
+
+(define rv (fn (arr) (get arr :ravel)))
+(define sh (fn (arr) (get arr :shape)))
+
+(define mknum (fn (n) (list :num n)))
+(define mkname (fn (s) (list :name s)))
+(define mkfg (fn (g) (list :fn-glyph g)))
+(define mkmon (fn (g a) (list :monad (mkfg g) a)))
+(define mkdyd (fn (g l r) (list :dyad (mkfg g) l r)))
+(define mkdfn1 (fn (body) (list :dfn body)))
+(define mkprog (fn (stmts) (cons :program stmts)))
+
+(define mkasg (fn (mkname expr) (list :assign mkname expr)))
+
+(define mkgrd (fn (c e) (list :guard c e)))
+
+(define mkdfn (fn (stmts) (cons :dfn stmts)))
+
+(apl-test
+  "eval :num literal"
+  (rv (apl-eval-ast (mknum 42) {}))
+  (list 42))
+
+(apl-test
+  "eval :num literal shape"
+  (sh (apl-eval-ast (mknum 42) {}))
+  (list))
+
+(apl-test
+  "eval :dyad +"
+  (rv (apl-eval-ast (mkdyd "+" (mknum 2) (mknum 3)) {}))
+  (list 5))
+
+(apl-test
+  "eval :dyad ×"
+  (rv (apl-eval-ast (mkdyd "×" (mknum 6) (mknum 7)) {}))
+  (list 42))
+
+(apl-test
+  "eval :monad - (negate)"
+  (rv (apl-eval-ast (mkmon "-" (mknum 7)) {}))
+  (list -7))
+
+(apl-test
+  "eval :monad ⌊ (floor)"
+  (rv (apl-eval-ast (mkmon "⌊" (mknum 3)) {}))
+  (list 3))
+
+(apl-test
+  "eval :name ⍵ from env"
+  (rv (apl-eval-ast (mkname "⍵") {:omega (apl-scalar 99) :alpha nil}))
+  (list 99))
+
+(apl-test
+  "eval :name ⍺ from env"
+  (rv (apl-eval-ast (mkname "⍺") {:omega nil :alpha (apl-scalar 7)}))
+  (list 7))
+
+(apl-test
+  "dfn {⍵+1} called monadic"
+  (rv
+    (apl-call-dfn-m
+      (mkdfn1 (mkdyd "+" (mkname "⍵") (mknum 1)))
+      (apl-scalar 5)))
+  (list 6))
+
+(apl-test
+  "dfn {⍺+⍵} called dyadic"
+  (rv
+    (apl-call-dfn
+      (mkdfn1 (mkdyd "+" (mkname "⍺") (mkname "⍵")))
+      (apl-scalar 4)
+      (apl-scalar 9)))
+  (list 13))
+
+(apl-test
+  "dfn {⍺×⍵} dyadic on vectors"
+  (rv
+    (apl-call-dfn
+      (mkdfn1 (mkdyd "×" (mkname "⍺") (mkname "⍵")))
+      (make-array (list 3) (list 1 2 3))
+      (make-array (list 3) (list 10 20 30))))
+  (list 10 40 90))
+
+(apl-test
+  "dfn {-⍵} monadic negate"
+  (rv
+    (apl-call-dfn-m
+      (mkdfn1 (mkmon "-" (mkname "⍵")))
+      (make-array (list 3) (list 1 2 3))))
+  (list -1 -2 -3))
+
+(apl-test
+  "dfn {⍺-⍵} dyadic subtract scalar"
+  (rv
+    (apl-call-dfn
+      (mkdfn1 (mkdyd "-" (mkname "⍺") (mkname "⍵")))
+      (apl-scalar 10)
+      (apl-scalar 3)))
+  (list 7))
+
+(apl-test
+  "dfn {⌈⍺,⍵} not used (just verify : missing) — ceiling of right"
+  (rv
+    (apl-call-dfn-m (mkdfn1 (mkmon "⌈" (mkname "⍵"))) (apl-scalar 5)))
+  (list 5))
+
+(apl-test
+  "dfn nested dyad"
+  (rv
+    (apl-call-dfn
+      (mkdfn1
+        (mkdyd "+" (mkname "⍺") (mkdyd "×" (mkname "⍵") (mknum 2))))
+      (apl-scalar 1)
+      (apl-scalar 3)))
+  (list 7))
+
+(apl-test
+  "dfn local assign x←⍵+1; ⍺×x"
+  (rv
+    (apl-call-dfn
+      (mkdfn
+        (list
+          (mkasg "x" (mkdyd "+" (mkname "⍵") (mknum 1)))
+          (mkdyd "×" (mkname "⍺") (mkname "x"))))
+      (apl-scalar 3)
+      (apl-scalar 4)))
+  (list 15))
+
+(apl-test
+  "dfn guard: 0=⍵:99; ⍵×2 (true branch)"
+  (rv
+    (apl-call-dfn-m
+      (mkdfn
+        (list
+          (mkgrd (mkdyd "=" (mknum 0) (mkname "⍵")) (mknum 99))
+          (mkdyd "×" (mkname "⍵") (mknum 2))))
+      (apl-scalar 0)))
+  (list 99))
+
+(apl-test
+  "dfn guard: 0=⍵:99; ⍵×2 (false branch)"
+  (rv
+    (apl-call-dfn-m
+      (mkdfn
+        (list
+          (mkgrd (mkdyd "=" (mknum 0) (mkname "⍵")) (mknum 99))
+          (mkdyd "×" (mkname "⍵") (mknum 2))))
+      (apl-scalar 5)))
+  (list 10))
+
+(apl-test
+  "dfn default ⍺←10 used (monadic call)"
+  (rv
+    (apl-call-dfn-m
+      (mkdfn
+        (list
+          (mkasg "⍺" (mknum 10))
+          (mkdyd "+" (mkname "⍺") (mkname "⍵"))))
+      (apl-scalar 5)))
+  (list 15))
+
+(apl-test
+  "dfn default ⍺←10 ignored when ⍺ given (dyadic call)"
+  (rv
+    (apl-call-dfn
+      (mkdfn
+        (list
+          (mkasg "⍺" (mknum 10))
+          (mkdyd "+" (mkname "⍺") (mkname "⍵"))))
+      (apl-scalar 100)
+      (apl-scalar 5)))
+  (list 105))
+
+(apl-test
+  "dfn ∇ recursion: factorial via guard"
+  (rv
+    (apl-call-dfn-m
+      (mkdfn
+        (list
+          (mkgrd (mkdyd "=" (mknum 0) (mkname "⍵")) (mknum 1))
+          (mkdyd
+            "×"
+            (mkname "⍵")
+            (mkmon "∇" (mkdyd "-" (mkname "⍵") (mknum 1))))))
+      (apl-scalar 5)))
+  (list 120))
+
+(apl-test
+  "dfn ∇ recursion: 3 → 6 (factorial)"
+  (rv
+    (apl-call-dfn-m
+      (mkdfn
+        (list
+          (mkgrd (mkdyd "=" (mknum 0) (mkname "⍵")) (mknum 1))
+          (mkdyd
+            "×"
+            (mkname "⍵")
+            (mkmon "∇" (mkdyd "-" (mkname "⍵") (mknum 1))))))
+      (apl-scalar 3)))
+  (list 6))
+
+(apl-test
+  "dfn local: x←⍵+10; y←x×2; y"
+  (rv
+    (apl-call-dfn-m
+      (mkdfn
+        (list
+          (mkasg "x" (mkdyd "+" (mkname "⍵") (mknum 10)))
+          (mkasg "y" (mkdyd "×" (mkname "x") (mknum 2)))
+          (mkname "y")))
+      (apl-scalar 5)))
+  (list 30))
+
+(apl-test
+  "dfn first guard wins: many guards"
+  (rv
+    (apl-call-dfn-m
+      (mkdfn
+        (list
+          (mkgrd (mkdyd "=" (mknum 1) (mkname "⍵")) (mknum 100))
+          (mkgrd (mkdyd "=" (mknum 2) (mkname "⍵")) (mknum 200))
+          (mkgrd (mkdyd "=" (mknum 3) (mkname "⍵")) (mknum 300))
+          (mknum 0)))
+      (apl-scalar 2)))
+  (list 200))

lib/apl/tests/eval-ops.sx

@@ -0,0 +1,147 @@
+; Tests for operator handling in apl-eval-ast (Phase 7).
+; Manual AST construction; verifies :derived-fn / :outer / :derived-fn2
+; route through apl-resolve-monadic / apl-resolve-dyadic correctly.
+
+(define mkrv (fn (arr) (get arr :ravel)))
+(define mksh (fn (arr) (get arr :shape)))
+(define mknum (fn (n) (list :num n)))
+(define mkfg (fn (g) (list :fn-glyph g)))
+(define mkmon (fn (g a) (list :monad g a)))
+(define mkdyd (fn (g l r) (list :dyad g l r)))
+(define mkder (fn (op f) (list :derived-fn op f)))
+(define mkdr2 (fn (op f g) (list :derived-fn2 op f g)))
+(define mkout (fn (f) (list :outer "∘." f)))
+
+; helper: literal vector AST via :vec (from list of values)
+(define mkvec (fn (xs) (cons :vec (map (fn (n) (mknum n)) xs))))
+
+; ---------- monadic operators ----------
+
+(apl-test
+  "eval-ast +/ ⍳5 → 15"
+  (mkrv
+    (apl-eval-ast
+      (mkmon (mkder "/" (mkfg "+")) (mkmon (mkfg "⍳") (mknum 5)))
+      {}))
+  (list 15))
+
+(apl-test
+  "eval-ast ×/ ⍳5 → 120"
+  (mkrv
+    (apl-eval-ast
+      (mkmon (mkder "/" (mkfg "×")) (mkmon (mkfg "⍳") (mknum 5)))
+      {}))
+  (list 120))
+
+(apl-test
+  "eval-ast ⌈/ — max reduce"
+  (mkrv
+    (apl-eval-ast
+      (mkmon (mkder "/" (mkfg "⌈")) (mkvec (list 3 1 4 1 5 9 2 6)))
+      {}))
+  (list 9))
+
+(apl-test
+  "eval-ast +\\ scan"
+  (mkrv
+    (apl-eval-ast
+      (mkmon (mkder "\\" (mkfg "+")) (mkvec (list 1 2 3 4 5)))
+      {}))
+  (list 1 3 6 10 15))
+
+(apl-test
+  "eval-ast +⌿ first-axis reduce on vector"
+  (mkrv
+    (apl-eval-ast
+      (mkmon (mkder "⌿" (mkfg "+")) (mkvec (list 1 2 3 4 5)))
+      {}))
+  (list 15))
+
+(apl-test
+  "eval-ast -¨ each-negate"
+  (mkrv
+    (apl-eval-ast
+      (mkmon (mkder "¨" (mkfg "-")) (mkvec (list 1 2 3 4)))
+      {}))
+  (list -1 -2 -3 -4))
+
+(apl-test
+  "eval-ast +⍨ commute (double via x+x)"
+  (mkrv
+    (apl-eval-ast (mkmon (mkder "⍨" (mkfg "+")) (mknum 7)) {}))
+  (list 14))
+
+; ---------- dyadic operators ----------
+
+(apl-test
+  "eval-ast outer ∘.× — multiplication table"
+  (mkrv
+    (apl-eval-ast
+      (mkdyd
+        (mkout (mkfg "×"))
+        (mkvec (list 1 2 3))
+        (mkvec (list 1 2 3)))
+      {}))
+  (list 1 2 3 2 4 6 3 6 9))
+
+(apl-test
+  "eval-ast outer ∘.× shape (3 3)"
+  (mksh
+    (apl-eval-ast
+      (mkdyd
+        (mkout (mkfg "×"))
+        (mkvec (list 1 2 3))
+        (mkvec (list 1 2 3)))
+      {}))
+  (list 3 3))
+
+(apl-test
+  "eval-ast inner +.× — dot product"
+  (mkrv
+    (apl-eval-ast
+      (mkdyd
+        (mkdr2 "." (mkfg "+") (mkfg "×"))
+        (mkvec (list 1 2 3))
+        (mkvec (list 4 5 6)))
+      {}))
+  (list 32))
+
+(apl-test
+  "eval-ast inner ∧.= equal vectors"
+  (mkrv
+    (apl-eval-ast
+      (mkdyd
+        (mkdr2 "." (mkfg "∧") (mkfg "="))
+        (mkvec (list 1 2 3))
+        (mkvec (list 1 2 3)))
+      {}))
+  (list 1))
+
+(apl-test
+  "eval-ast each-dyadic +¨"
+  (mkrv
+    (apl-eval-ast
+      (mkdyd
+        (mkder "¨" (mkfg "+"))
+        (mkvec (list 1 2 3))
+        (mkvec (list 10 20 30)))
+      {}))
+  (list 11 22 33))
+
+(apl-test
+  "eval-ast commute -⍨ (subtract swapped)"
+  (mkrv
+    (apl-eval-ast
+      (mkdyd (mkder "⍨" (mkfg "-")) (mknum 5) (mknum 3))
+      {}))
+  (list -2))
+
+; ---------- nested operators ----------
+
+(apl-test
+  "eval-ast +/¨ — sum of each"
+  (mkrv
+    (apl-eval-ast
+      (mkmon (mkder "/" (mkfg "+")) (mkvec (list 10 20 30)))
+      {}))
+  (list 60))

lib/apl/tests/idioms.sx

@@ -0,0 +1,359 @@
+; APL idiom corpus — classic Roger Hui / Phil Last idioms expressed
+; through our runtime primitives.  Each test names the APL one-liner
+; and verifies the equivalent runtime call.
+
+(define mkrv (fn (arr) (get arr :ravel)))
+(define mksh (fn (arr) (get arr :shape)))
+
+; ---------- reductions ----------
+
+(apl-test
+  "+/⍵ — sum"
+  (mkrv (apl-reduce apl-add (make-array (list 5) (list 1 2 3 4 5))))
+  (list 15))
+
+(apl-test
+  "(+/⍵)÷⍴⍵ — mean"
+  (mkrv
+    (apl-div
+      (apl-reduce apl-add (make-array (list 5) (list 1 2 3 4 5)))
+      (apl-scalar 5)))
+  (list 3))
+
+(apl-test
+  "⌈/⍵ — max"
+  (mkrv (apl-reduce apl-max (make-array (list 6) (list 3 1 4 1 5 9))))
+  (list 9))
+
+(apl-test
+  "⌊/⍵ — min"
+  (mkrv (apl-reduce apl-min (make-array (list 6) (list 3 1 4 1 5 9))))
+  (list 1))
+
+(apl-test
+  "(⌈/⍵)-⌊/⍵ — range"
+  (mkrv
+    (apl-sub
+      (apl-reduce apl-max (make-array (list 6) (list 3 1 4 1 5 9)))
+      (apl-reduce apl-min (make-array (list 6) (list 3 1 4 1 5 9)))))
+  (list 8))
+
+(apl-test
+  "×/⍵ — product"
+  (mkrv (apl-reduce apl-mul (make-array (list 4) (list 1 2 3 4))))
+  (list 24))
+
+(apl-test
+  "+\\⍵ — running sum"
+  (mkrv (apl-scan apl-add (make-array (list 5) (list 1 2 3 4 5))))
+  (list 1 3 6 10 15))
+
+; ---------- sort / order ----------
+
+(apl-test
+  "⍵[⍋⍵] — sort ascending"
+  (mkrv (apl-quicksort (make-array (list 5) (list 3 1 4 1 5))))
+  (list 1 1 3 4 5))
+
+(apl-test
+  "⌽⍵ — reverse"
+  (mkrv (apl-reverse (make-array (list 5) (list 1 2 3 4 5))))
+  (list 5 4 3 2 1))
+
+(apl-test
+  "⊃⌽⍵ — last element"
+  (mkrv
+    (apl-disclose (apl-reverse (make-array (list 4) (list 10 20 30 40)))))
+  (list 40))
+
+(apl-test
+  "1↑⍵ — first element"
+  (mkrv
+    (apl-take (apl-scalar 1) (make-array (list 4) (list 10 20 30 40))))
+  (list 10))
+
+(apl-test
+  "1↓⍵ — drop first"
+  (mkrv
+    (apl-drop (apl-scalar 1) (make-array (list 4) (list 10 20 30 40))))
+  (list 20 30 40))
+
+(apl-test
+  "¯1↓⍵ — drop last"
+  (mkrv
+    (apl-drop (apl-scalar -1) (make-array (list 4) (list 10 20 30 40))))
+  (list 10 20 30))
+
+; ---------- counts / membership ----------
+
+(apl-test
+  "≢⍵ — tally"
+  (mkrv (apl-tally (make-array (list 7) (list 9 8 7 6 5 4 3))))
+  (list 7))
+
+(apl-test
+  "+/⍵=v — count occurrences of v"
+  (mkrv
+    (apl-reduce
+      apl-add
+      (apl-eq (make-array (list 7) (list 1 2 3 2 1 3 2)) (apl-scalar 2))))
+  (list 3))
+
+(apl-test
+  "0=N|M — divisibility test"
+  (mkrv (apl-eq (apl-scalar 0) (apl-mod (apl-scalar 3) (apl-scalar 12))))
+  (list 1))
+
+; ---------- shape constructors ----------
+
+(apl-test
+  "N⍴1 — vector of N ones"
+  (mkrv (apl-reshape (apl-scalar 5) (apl-scalar 1)))
+  (list 1 1 1 1 1))
+
+(apl-test
+  "(N N)⍴0 — N×N zero matrix"
+  (mkrv (apl-reshape (make-array (list 2) (list 3 3)) (apl-scalar 0)))
+  (list 0 0 0 0 0 0 0 0 0))
+
+(apl-test
+  "⍳∘.=⍳ — N×N identity matrix"
+  (mkrv
+    (apl-outer apl-eq (apl-iota (apl-scalar 3)) (apl-iota (apl-scalar 3))))
+  (list 1 0 0 0 1 0 0 0 1))
+
+(apl-test
+  "⍳∘.×⍳ — multiplication table"
+  (mkrv
+    (apl-outer apl-mul (apl-iota (apl-scalar 3)) (apl-iota (apl-scalar 3))))
+  (list 1 2 3 2 4 6 3 6 9))
+
+; ---------- numerical idioms ----------
+
+(apl-test
+  "+\\⍳N — triangular numbers"
+  (mkrv (apl-scan apl-add (apl-iota (apl-scalar 5))))
+  (list 1 3 6 10 15))
+
+(apl-test
+  "+/⍳N=N×(N+1)÷2 — sum of 1..N"
+  (mkrv (apl-reduce apl-add (apl-iota (apl-scalar 10))))
+  (list 55))
+
+(apl-test
+  "×/⍳N — factorial via iota"
+  (mkrv (apl-reduce apl-mul (apl-iota (apl-scalar 5))))
+  (list 120))
+
+(apl-test
+  "2|⍵ — parity (1=odd)"
+  (mkrv (apl-mod (apl-scalar 2) (make-array (list 5) (list 1 2 3 4 5))))
+  (list 1 0 1 0 1))
+
+(apl-test
+  "+/2|⍵ — count odd"
+  (mkrv
+    (apl-reduce
+      apl-add
+      (apl-mod (apl-scalar 2) (make-array (list 5) (list 1 2 3 4 5)))))
+  (list 3))
+
+; ---------- boolean idioms ----------
+
+(apl-test
+  "∧/⍵ — all-true"
+  (mkrv (apl-reduce apl-and (make-array (list 4) (list 1 1 1 1))))
+  (list 1))
+
+(apl-test
+  "∧/⍵ — all-true with zero is false"
+  (mkrv (apl-reduce apl-and (make-array (list 4) (list 1 1 0 1))))
+  (list 0))
+
+(apl-test
+  "∨/⍵ — any-true"
+  (mkrv (apl-reduce apl-or (make-array (list 4) (list 0 0 1 0))))
+  (list 1))
+
+(apl-test
+  "∨/⍵ — any-true all zero is false"
+  (mkrv (apl-reduce apl-or (make-array (list 4) (list 0 0 0 0))))
+  (list 0))
+
+; ---------- selection / scaling ----------
+
+(apl-test
+  "⍵×⍵ — square each"
+  (mkrv
+    (apl-mul
+      (make-array (list 4) (list 1 2 3 4))
+      (make-array (list 4) (list 1 2 3 4))))
+  (list 1 4 9 16))
+
+(apl-test
+  "+/⍵×⍵ — sum of squares"
+  (mkrv
+    (apl-reduce
+      apl-add
+      (apl-mul
+        (make-array (list 4) (list 1 2 3 4))
+        (make-array (list 4) (list 1 2 3 4)))))
+  (list 30))
+
+(apl-test
+  "⍵-(+/⍵)÷⍴⍵ — mean-centered"
+  (mkrv
+    (apl-sub
+      (make-array (list 5) (list 2 4 6 8 10))
+      (apl-div
+        (apl-reduce apl-add (make-array (list 5) (list 2 4 6 8 10)))
+        (apl-scalar 5))))
+  (list -4 -2 0 2 4))
+
+; ---------- shape / structure ----------
+
+(apl-test
+  ",⍵ — ravel"
+  (mkrv (apl-ravel (make-array (list 2 3) (list 1 2 3 4 5 6))))
+  (list 1 2 3 4 5 6))
+
+(apl-test
+  "⍴⍴⍵ — rank"
+  (mkrv
+    (apl-shape (apl-shape (make-array (list 2 3) (list 1 2 3 4 5 6)))))
+  (list 2))
+
+(apl-test
+  "src: +/⍳N → triangular(N)"
+  (mkrv (apl-run "+/⍳100"))
+  (list 5050))
+
+(apl-test "src: ×/⍳N → N!" (mkrv (apl-run "×/⍳6")) (list 720))
+
+(apl-test
+  "src: ⌈/V — max"
+  (mkrv (apl-run "⌈/3 1 4 1 5 9 2 6"))
+  (list 9))
+
+(apl-test
+  "src: ⌊/V — min"
+  (mkrv (apl-run "⌊/3 1 4 1 5 9 2 6"))
+  (list 1))
+
+(apl-test
+  "src: range = (⌈/V) - ⌊/V"
+  (mkrv (apl-run "(⌈/3 1 4 1 5 9 2 6) - ⌊/3 1 4 1 5 9 2 6"))
+  (list 8))
+
+(apl-test
+  "src: +\\V — running sum"
+  (mkrv (apl-run "+\\1 2 3 4 5"))
+  (list 1 3 6 10 15))
+
+(apl-test
+  "src: ×\\V — running product"
+  (mkrv (apl-run "×\\1 2 3 4 5"))
+  (list 1 2 6 24 120))
+
+(apl-test
+  "src: V × V — squares"
+  (mkrv (apl-run "(⍳5) × ⍳5"))
+  (list 1 4 9 16 25))
+
+(apl-test
+  "src: +/V × V — sum of squares"
+  (mkrv (apl-run "+/(⍳5) × ⍳5"))
+  (list 55))
+
+(apl-test "src: ∧/V — all-true" (mkrv (apl-run "∧/1 1 1 1")) (list 1))
+
+(apl-test "src: ∨/V — any-true" (mkrv (apl-run "∨/0 0 1 0")) (list 1))
+
+(apl-test "src: 0 = N|M — divides" (mkrv (apl-run "0 = 3 | 12")) (list 1))
+
+(apl-test
+  "src: 2 | V — parity"
+  (mkrv (apl-run "2 | 1 2 3 4 5 6"))
+  (list 1 0 1 0 1 0))
+
+(apl-test
+  "src: +/2|V — count odd"
+  (mkrv (apl-run "+/2 | 1 2 3 4 5 6"))
+  (list 3))
+
+(apl-test "src: ⍴ V" (mkrv (apl-run "⍴ 1 2 3 4 5")) (list 5))
+
+(apl-test
+  "src: ⍴⍴ M — rank"
+  (mkrv (apl-run "⍴ ⍴ (2 3) ⍴ ⍳6"))
+  (list 2))
+
+(apl-test
+  "src: N⍴1 — vector of ones"
+  (mkrv (apl-run "5 ⍴ 1"))
+  (list 1 1 1 1 1))
+
+(apl-test
+  "src: ⍳N ∘.= ⍳N — identity matrix"
+  (mkrv (apl-run "(⍳3) ∘.= ⍳3"))
+  (list 1 0 0 0 1 0 0 0 1))
+
+(apl-test
+  "src: ⍳N ∘.× ⍳N — multiplication table"
+  (mkrv (apl-run "(⍳3) ∘.× ⍳3"))
+  (list 1 2 3 2 4 6 3 6 9))
+
+(apl-test
+  "src: V +.× V — dot product"
+  (mkrv (apl-run "1 2 3 +.× 4 5 6"))
+  (list 32))
+
+(apl-test
+  "src: ∧.= V — vectors equal?"
+  (mkrv (apl-run "1 2 3 ∧.= 1 2 3"))
+  (list 1))
+
+(apl-test
+  "src: V[1] — first element"
+  (mkrv (apl-run "(10 20 30 40)[1]"))
+  (list 10))
+
+(apl-test
+  "src: 1↑V — first via take"
+  (mkrv (apl-run "1 ↑ 10 20 30 40"))
+  (list 10))
+
+(apl-test
+  "src: 1↓V — drop first"
+  (mkrv (apl-run "1 ↓ 10 20 30 40"))
+  (list 20 30 40))
+
+(apl-test
+  "src: ¯1↓V — drop last"
+  (mkrv (apl-run "¯1 ↓ 10 20 30 40"))
+  (list 10 20 30))
+
+(apl-test
+  "src: ⌽V — reverse"
+  (mkrv (apl-run "⌽ 1 2 3 4 5"))
+  (list 5 4 3 2 1))
+
+(apl-test
+  "src: ≢V — tally"
+  (mkrv (apl-run "≢ 9 8 7 6 5 4 3 2 1"))
+  (list 9))
+
+(apl-test
+  "src: ,M — ravel"
+  (mkrv (apl-run ", (2 3) ⍴ ⍳6"))
+  (list 1 2 3 4 5 6))
+
+(apl-test
+  "src: A=V — count occurrences"
+  (mkrv (apl-run "+/2 = 1 2 3 2 1 3 2"))
+  (list 3))
+
+(apl-test
+  "src: ⌈/(V × V) — max squared"
+  (mkrv (apl-run "⌈/(1 2 3 4 5) × 1 2 3 4 5"))
+  (list 25))

lib/apl/tests/operators.sx

@@ -0,0 +1,791 @@
+(define rv (fn (arr) (get arr :ravel)))
+(define sh (fn (arr) (get arr :shape)))
+
+(apl-test
+  "reduce +/ vector"
+  (rv (apl-reduce apl-add (make-array (list 5) (list 1 2 3 4 5))))
+  (list 15))
+
+(apl-test
+  "reduce x/ vector"
+  (rv (apl-reduce apl-mul (make-array (list 4) (list 1 2 3 4))))
+  (list 24))
+
+(apl-test
+  "reduce max/ vector"
+  (rv (apl-reduce apl-max (make-array (list 5) (list 3 1 4 1 5))))
+  (list 5))
+
+(apl-test
+  "reduce min/ vector"
+  (rv (apl-reduce apl-min (make-array (list 3) (list 3 1 4))))
+  (list 1))
+
+(apl-test
+  "reduce and/ all true"
+  (rv (apl-reduce apl-and (make-array (list 3) (list 1 1 1))))
+  (list 1))
+
+(apl-test
+  "reduce or/ with true"
+  (rv (apl-reduce apl-or (make-array (list 3) (list 0 0 1))))
+  (list 1))
+
+(apl-test
+  "reduce +/ single element"
+  (rv (apl-reduce apl-add (make-array (list 1) (list 42))))
+  (list 42))
+
+(apl-test
+  "reduce +/ scalar no-op"
+  (rv (apl-reduce apl-add (apl-scalar 7)))
+  (list 7))
+
+(apl-test
+  "reduce +/ shape is scalar"
+  (sh (apl-reduce apl-add (make-array (list 4) (list 1 2 3 4))))
+  (list))
+
+(apl-test
+  "reduce +/ matrix row sums shape"
+  (sh (apl-reduce apl-add (make-array (list 2 3) (list 1 2 3 4 5 6))))
+  (list 2))
+
+(apl-test
+  "reduce +/ matrix row sums values"
+  (rv (apl-reduce apl-add (make-array (list 2 3) (list 1 2 3 4 5 6))))
+  (list 6 15))
+
+(apl-test
+  "reduce max/ matrix row maxima"
+  (rv (apl-reduce apl-max (make-array (list 2 3) (list 3 1 4 1 5 9))))
+  (list 4 9))
+
+(apl-test
+  "reduce-first +/ vector same as reduce"
+  (rv (apl-reduce-first apl-add (make-array (list 5) (list 1 2 3 4 5))))
+  (list 15))
+
+(apl-test
+  "reduce-first +/ matrix col sums shape"
+  (sh
+    (apl-reduce-first apl-add (make-array (list 2 3) (list 1 2 3 4 5 6))))
+  (list 3))
+
+(apl-test
+  "reduce-first +/ matrix col sums values"
+  (rv
+    (apl-reduce-first apl-add (make-array (list 2 3) (list 1 2 3 4 5 6))))
+  (list 5 7 9))
+
+(apl-test
+  "reduce-first max/ matrix col maxima"
+  (rv
+    (apl-reduce-first apl-max (make-array (list 3 2) (list 1 9 2 8 3 7))))
+  (list 3 9))
+
+(apl-test
+  "scan +\\ vector"
+  (rv (apl-scan apl-add (make-array (list 5) (list 1 2 3 4 5))))
+  (list 1 3 6 10 15))
+
+(apl-test
+  "scan x\\ vector cumulative product"
+  (rv (apl-scan apl-mul (make-array (list 5) (list 1 2 3 4 5))))
+  (list 1 2 6 24 120))
+
+(apl-test
+  "scan max\\ vector running max"
+  (rv (apl-scan apl-max (make-array (list 5) (list 3 1 4 1 5))))
+  (list 3 3 4 4 5))
+
+(apl-test
+  "scan min\\ vector running min"
+  (rv (apl-scan apl-min (make-array (list 5) (list 3 1 4 1 5))))
+  (list 3 1 1 1 1))
+
+(apl-test
+  "scan +\\ single element"
+  (rv (apl-scan apl-add (make-array (list 1) (list 42))))
+  (list 42))
+
+(apl-test
+  "scan +\\ scalar no-op"
+  (rv (apl-scan apl-add (apl-scalar 7)))
+  (list 7))
+
+(apl-test
+  "scan +\\ vector preserves shape"
+  (sh (apl-scan apl-add (make-array (list 5) (list 1 2 3 4 5))))
+  (list 5))
+
+(apl-test
+  "scan +\\ matrix preserves shape"
+  (sh (apl-scan apl-add (make-array (list 2 3) (list 1 2 3 4 5 6))))
+  (list 2 3))
+
+(apl-test
+  "scan +\\ matrix row-wise"
+  (rv (apl-scan apl-add (make-array (list 2 3) (list 1 2 3 4 5 6))))
+  (list 1 3 6 4 9 15))
+
+(apl-test
+  "scan max\\ matrix row-wise running max"
+  (rv (apl-scan apl-max (make-array (list 2 3) (list 3 1 4 1 5 9))))
+  (list 3 3 4 1 5 9))
+
+(apl-test
+  "scan-first +\\ vector same as scan"
+  (rv (apl-scan-first apl-add (make-array (list 5) (list 1 2 3 4 5))))
+  (list 1 3 6 10 15))
+
+(apl-test
+  "scan-first +\\ scalar no-op"
+  (rv (apl-scan-first apl-add (apl-scalar 9)))
+  (list 9))
+
+(apl-test
+  "scan-first +\\ matrix preserves shape"
+  (sh (apl-scan-first apl-add (make-array (list 2 3) (list 1 2 3 4 5 6))))
+  (list 2 3))
+
+(apl-test
+  "scan-first +\\ matrix col-wise"
+  (rv (apl-scan-first apl-add (make-array (list 2 3) (list 1 2 3 4 5 6))))
+  (list 1 2 3 5 7 9))
+
+(apl-test
+  "scan-first max\\ matrix col-wise running max"
+  (rv (apl-scan-first apl-max (make-array (list 3 2) (list 3 1 4 1 5 9))))
+  (list 3 1 4 1 5 9))
+
+(apl-test
+  "each negate vector"
+  (rv (apl-each apl-neg-m (make-array (list 3) (list 1 2 3))))
+  (list -1 -2 -3))
+
+(apl-test
+  "each negate vector preserves shape"
+  (sh (apl-each apl-neg-m (make-array (list 3) (list 1 2 3))))
+  (list 3))
+
+(apl-test
+  "each reciprocal vector"
+  (rv (apl-each apl-recip (make-array (list 3) (list 1 2 4))))
+  (list 1 (/ 1 2) (/ 1 4)))
+
+(apl-test
+  "each abs vector"
+  (rv (apl-each apl-abs (make-array (list 4) (list -1 2 -3 4))))
+  (list 1 2 3 4))
+
+(apl-test "each scalar" (rv (apl-each apl-neg-m (apl-scalar 5))) (list -5))
+
+(apl-test
+  "each scalar shape"
+  (sh (apl-each apl-neg-m (apl-scalar 5)))
+  (list))
+
+(apl-test
+  "each negate matrix shape"
+  (sh (apl-each apl-neg-m (make-array (list 2 3) (list 1 2 3 4 5 6))))
+  (list 2 3))
+
+(apl-test
+  "each negate matrix values"
+  (rv (apl-each apl-neg-m (make-array (list 2 3) (list 1 2 3 4 5 6))))
+  (list -1 -2 -3 -4 -5 -6))
+
+(apl-test
+  "each-dyadic scalar+scalar"
+  (rv (apl-each-dyadic apl-add (apl-scalar 3) (apl-scalar 4)))
+  (list 7))
+
+(apl-test
+  "each-dyadic scalar+vector"
+  (rv
+    (apl-each-dyadic
+      apl-add
+      (apl-scalar 10)
+      (make-array (list 3) (list 1 2 3))))
+  (list 11 12 13))
+
+(apl-test
+  "each-dyadic vector+scalar"
+  (rv
+    (apl-each-dyadic
+      apl-add
+      (make-array (list 3) (list 1 2 3))
+      (apl-scalar 10)))
+  (list 11 12 13))
+
+(apl-test
+  "each-dyadic vector+vector"
+  (rv
+    (apl-each-dyadic
+      apl-add
+      (make-array (list 3) (list 1 2 3))
+      (make-array (list 3) (list 10 20 30))))
+  (list 11 22 33))
+
+(apl-test
+  "each-dyadic mul matrix+matrix shape"
+  (sh
+    (apl-each-dyadic
+      apl-mul
+      (make-array (list 2 2) (list 1 2 3 4))
+      (make-array (list 2 2) (list 5 6 7 8))))
+  (list 2 2))
+
+(apl-test
+  "each-dyadic mul matrix+matrix values"
+  (rv
+    (apl-each-dyadic
+      apl-mul
+      (make-array (list 2 2) (list 1 2 3 4))
+      (make-array (list 2 2) (list 5 6 7 8))))
+  (list 5 12 21 32))
+
+(apl-test
+  "outer product mult table values"
+  (rv
+    (apl-outer
+      apl-mul
+      (make-array (list 3) (list 1 2 3))
+      (make-array (list 3) (list 1 2 3))))
+  (list 1 2 3 2 4 6 3 6 9))
+
+(apl-test
+  "outer product mult table shape"
+  (sh
+    (apl-outer
+      apl-mul
+      (make-array (list 3) (list 1 2 3))
+      (make-array (list 3) (list 1 2 3))))
+  (list 3 3))
+
+(apl-test
+  "outer product add table values"
+  (rv
+    (apl-outer
+      apl-add
+      (make-array (list 2) (list 1 2))
+      (make-array (list 3) (list 10 20 30))))
+  (list 11 21 31 12 22 32))
+
+(apl-test
+  "outer product add table shape"
+  (sh
+    (apl-outer
+      apl-add
+      (make-array (list 2) (list 1 2))
+      (make-array (list 3) (list 10 20 30))))
+  (list 2 3))
+
+(apl-test
+  "outer product scalar+vector shape"
+  (sh
+    (apl-outer apl-mul (apl-scalar 5) (make-array (list 3) (list 1 2 3))))
+  (list 3))
+
+(apl-test
+  "outer product scalar+vector values"
+  (rv
+    (apl-outer apl-mul (apl-scalar 5) (make-array (list 3) (list 1 2 3))))
+  (list 5 10 15))
+
+(apl-test
+  "outer product vector+scalar shape"
+  (sh
+    (apl-outer apl-mul (make-array (list 3) (list 1 2 3)) (apl-scalar 10)))
+  (list 3))
+
+(apl-test
+  "outer product scalar+scalar"
+  (rv (apl-outer apl-mul (apl-scalar 6) (apl-scalar 7)))
+  (list 42))
+
+(apl-test
+  "outer product scalar+scalar shape"
+  (sh (apl-outer apl-mul (apl-scalar 6) (apl-scalar 7)))
+  (list))
+
+(apl-test
+  "outer product equality identity matrix values"
+  (rv
+    (apl-outer
+      apl-eq
+      (make-array (list 3) (list 1 2 3))
+      (make-array (list 3) (list 1 2 3))))
+  (list 1 0 0 0 1 0 0 0 1))
+
+(apl-test
+  "outer product matrix+vector rank doubling shape"
+  (sh
+    (apl-outer
+      apl-add
+      (make-array (list 2 2) (list 1 2 3 4))
+      (make-array (list 3) (list 10 20 30))))
+  (list 2 2 3))
+
+(apl-test
+  "outer product matrix+vector rank doubling values"
+  (rv
+    (apl-outer
+      apl-add
+      (make-array (list 2 2) (list 1 2 3 4))
+      (make-array (list 3) (list 10 20 30))))
+  (list 11 21 31 12 22 32 13 23 33 14 24 34))
+
+(apl-test
+  "inner +.× dot product"
+  (rv
+    (apl-inner
+      apl-add
+      apl-mul
+      (make-array (list 3) (list 1 2 3))
+      (make-array (list 3) (list 4 5 6))))
+  (list 32))
+
+(apl-test
+  "inner +.× dot product shape is scalar"
+  (sh
+    (apl-inner
+      apl-add
+      apl-mul
+      (make-array (list 3) (list 1 2 3))
+      (make-array (list 3) (list 4 5 6))))
+  (list))
+
+(apl-test
+  "inner +.× matrix multiply 2x3 * 3x2 shape"
+  (sh
+    (apl-inner
+      apl-add
+      apl-mul
+      (make-array (list 2 3) (list 1 2 3 4 5 6))
+      (make-array (list 3 2) (list 7 8 9 10 11 12))))
+  (list 2 2))
+
+(apl-test
+  "inner +.× matrix multiply 2x3 * 3x2 values"
+  (rv
+    (apl-inner
+      apl-add
+      apl-mul
+      (make-array (list 2 3) (list 1 2 3 4 5 6))
+      (make-array (list 3 2) (list 7 8 9 10 11 12))))
+  (list 58 64 139 154))
+
+(apl-test
+  "inner +.× identity matrix 2x2"
+  (rv
+    (apl-inner
+      apl-add
+      apl-mul
+      (make-array (list 2 2) (list 1 0 0 1))
+      (make-array (list 2 2) (list 5 6 7 8))))
+  (list 5 6 7 8))
+
+(apl-test
+  "inner ∧.= equal vectors"
+  (rv
+    (apl-inner
+      apl-and
+      apl-eq
+      (make-array (list 3) (list 1 2 3))
+      (make-array (list 3) (list 1 2 3))))
+  (list 1))
+
+(apl-test
+  "inner ∧.= unequal vectors"
+  (rv
+    (apl-inner
+      apl-and
+      apl-eq
+      (make-array (list 3) (list 1 2 3))
+      (make-array (list 3) (list 1 9 3))))
+  (list 0))
+
+(apl-test
+  "inner +.× matrix * vector shape"
+  (sh
+    (apl-inner
+      apl-add
+      apl-mul
+      (make-array (list 2 3) (list 1 2 3 4 5 6))
+      (make-array (list 3) (list 7 8 9))))
+  (list 2))
+
+(apl-test
+  "inner +.× matrix * vector values"
+  (rv
+    (apl-inner
+      apl-add
+      apl-mul
+      (make-array (list 2 3) (list 1 2 3 4 5 6))
+      (make-array (list 3) (list 7 8 9))))
+  (list 50 122))
+
+(apl-test
+  "inner +.× vector * matrix shape"
+  (sh
+    (apl-inner
+      apl-add
+      apl-mul
+      (make-array (list 3) (list 1 2 3))
+      (make-array (list 3 2) (list 4 5 6 7 8 9))))
+  (list 2))
+
+(apl-test
+  "inner +.× vector * matrix values"
+  (rv
+    (apl-inner
+      apl-add
+      apl-mul
+      (make-array (list 3) (list 1 2 3))
+      (make-array (list 3 2) (list 4 5 6 7 8 9))))
+  (list 40 46))
+
+(apl-test
+  "inner +.× single-element vectors"
+  (rv
+    (apl-inner
+      apl-add
+      apl-mul
+      (make-array (list 1) (list 6))
+      (make-array (list 1) (list 7))))
+  (list 42))
+
+(apl-test
+  "commute +⍨ scalar doubles"
+  (rv (apl-commute apl-add (apl-scalar 5)))
+  (list 10))
+
+(apl-test
+  "commute ×⍨ vector squares"
+  (rv (apl-commute apl-mul (make-array (list 4) (list 1 2 3 4))))
+  (list 1 4 9 16))
+
+(apl-test
+  "commute +⍨ vector doubles"
+  (rv (apl-commute apl-add (make-array (list 3) (list 1 2 3))))
+  (list 2 4 6))
+
+(apl-test
+  "commute +⍨ shape preserved"
+  (sh (apl-commute apl-add (make-array (list 3) (list 1 2 3))))
+  (list 3))
+
+(apl-test
+  "commute ×⍨ matrix shape preserved"
+  (sh (apl-commute apl-mul (make-array (list 2 2) (list 1 2 3 4))))
+  (list 2 2))
+
+(apl-test
+  "commute-dyadic -⍨ swaps subtraction"
+  (rv (apl-commute-dyadic apl-sub (apl-scalar 5) (apl-scalar 3)))
+  (list -2))
+
+(apl-test
+  "commute-dyadic ÷⍨ swaps division"
+  (rv (apl-commute-dyadic apl-div (apl-scalar 4) (apl-scalar 12)))
+  (list 3))
+
+(apl-test
+  "commute-dyadic -⍨ on vectors"
+  (rv
+    (apl-commute-dyadic
+      apl-sub
+      (make-array (list 3) (list 10 20 30))
+      (make-array (list 3) (list 1 2 3))))
+  (list -9 -18 -27))
+
+(apl-test
+  "commute-dyadic +⍨ commutative same result"
+  (rv
+    (apl-commute-dyadic
+      apl-add
+      (make-array (list 3) (list 1 2 3))
+      (make-array (list 3) (list 10 20 30))))
+  (list 11 22 33))
+
+(apl-test
+  "commute-dyadic ×⍨ commutative same result"
+  (rv
+    (apl-commute-dyadic
+      apl-mul
+      (make-array (list 3) (list 2 3 4))
+      (make-array (list 3) (list 5 6 7))))
+  (list 10 18 28))
+
+(apl-test
+  "compose -∘| scalar (negative abs)"
+  (rv (apl-compose apl-neg-m apl-abs (apl-scalar -7)))
+  (list -7))
+
+(apl-test
+  "compose -∘| vector"
+  (rv
+    (apl-compose apl-neg-m apl-abs (make-array (list 4) (list -1 2 -3 4))))
+  (list -1 -2 -3 -4))
+
+(apl-test
+  "compose ⌊∘- (floor of negate)"
+  (rv (apl-compose apl-floor apl-neg-m (make-array (list 3) (list 1 2 3))))
+  (list -1 -2 -3))
+
+(apl-test
+  "compose -∘| matrix shape preserved"
+  (sh
+    (apl-compose apl-neg-m apl-abs (make-array (list 2 2) (list -1 2 -3 4))))
+  (list 2 2))
+
+(apl-test
+  "compose-dyadic +∘- equals subtract scalar"
+  (rv (apl-compose-dyadic apl-add apl-neg-m (apl-scalar 10) (apl-scalar 3)))
+  (list 7))
+
+(apl-test
+  "compose-dyadic +∘- equals subtract vector"
+  (rv
+    (apl-compose-dyadic
+      apl-add
+      apl-neg-m
+      (make-array (list 3) (list 10 20 30))
+      (make-array (list 3) (list 1 2 3))))
+  (list 9 18 27))
+
+(apl-test
+  "compose-dyadic -∘| (subtract abs)"
+  (rv (apl-compose-dyadic apl-sub apl-abs (apl-scalar 10) (apl-scalar -3)))
+  (list 7))
+
+(apl-test
+  "compose-dyadic ×∘- (multiply by negative)"
+  (rv
+    (apl-compose-dyadic
+      apl-mul
+      apl-neg-m
+      (make-array (list 3) (list 2 3 4))
+      (make-array (list 3) (list 1 2 3))))
+  (list -2 -6 -12))
+
+(apl-test
+  "compose-dyadic shape preserved"
+  (sh
+    (apl-compose-dyadic
+      apl-add
+      apl-neg-m
+      (make-array (list 2 3) (list 1 2 3 4 5 6))
+      (make-array (list 2 3) (list 1 1 1 1 1 1))))
+  (list 2 3))
+
+(apl-test
+  "power n=0 identity"
+  (rv (apl-power (fn (a) (apl-add a (apl-scalar 1))) 0 (apl-scalar 5)))
+  (list 5))
+
+(apl-test
+  "power increment by 3"
+  (rv (apl-power (fn (a) (apl-add a (apl-scalar 1))) 3 (apl-scalar 0)))
+  (list 3))
+
+(apl-test
+  "power double 4 times = 16"
+  (rv (apl-power (fn (a) (apl-mul a (apl-scalar 2))) 4 (apl-scalar 1)))
+  (list 16))
+
+(apl-test
+  "power on vector +5"
+  (rv
+    (apl-power
+      (fn (a) (apl-add a (apl-scalar 1)))
+      5
+      (make-array (list 3) (list 1 2 3))))
+  (list 6 7 8))
+
+(apl-test
+  "power on vector preserves shape"
+  (sh
+    (apl-power
+      (fn (a) (apl-add a (apl-scalar 1)))
+      5
+      (make-array (list 3) (list 1 2 3))))
+  (list 3))
+
+(apl-test
+  "power on matrix"
+  (rv
+    (apl-power
+      (fn (a) (apl-mul a (apl-scalar 3)))
+      2
+      (make-array (list 2 2) (list 1 2 3 4))))
+  (list 9 18 27 36))
+
+(apl-test
+  "power-fixed identity stops immediately"
+  (rv (apl-power-fixed (fn (a) a) (make-array (list 3) (list 1 2 3))))
+  (list 1 2 3))
+
+(apl-test
+  "power-fixed floor half scalar to 0"
+  (rv
+    (apl-power-fixed
+      (fn (a) (apl-floor (apl-div a (apl-scalar 2))))
+      (apl-scalar 100)))
+  (list 0))
+
+(apl-test
+  "power-fixed shape preserved"
+  (sh
+    (apl-power-fixed (fn (a) a) (make-array (list 2 2) (list 1 2 3 4))))
+  (list 2 2))
+
+(apl-test
+  "rank tally⍤1 row tallies"
+  (rv (apl-rank apl-tally 1 (make-array (list 2 3) (list 1 2 3 4 5 6))))
+  (list 3 3))
+
+(apl-test
+  "rank tally⍤1 row tallies shape"
+  (sh (apl-rank apl-tally 1 (make-array (list 2 3) (list 1 2 3 4 5 6))))
+  (list 2))
+
+(apl-test
+  "rank neg⍤0 vector scalar cells"
+  (rv (apl-rank apl-neg-m 0 (make-array (list 3) (list 1 2 3))))
+  (list -1 -2 -3))
+
+(apl-test
+  "rank neg⍤0 vector preserves shape"
+  (sh (apl-rank apl-neg-m 0 (make-array (list 3) (list 1 2 3))))
+  (list 3))
+
+(apl-test
+  "rank neg⍤1 matrix per-row"
+  (rv (apl-rank apl-neg-m 1 (make-array (list 2 3) (list 1 2 3 4 5 6))))
+  (list -1 -2 -3 -4 -5 -6))
+
+(apl-test
+  "rank neg⍤1 matrix preserves shape"
+  (sh (apl-rank apl-neg-m 1 (make-array (list 2 3) (list 1 2 3 4 5 6))))
+  (list 2 3))
+
+(apl-test
+  "rank k>=rank fallthrough"
+  (rv (apl-rank apl-tally 5 (make-array (list 4) (list 1 2 3 4))))
+  (list 4))
+
+(apl-test
+  "rank tally⍤2 whole matrix tally"
+  (rv
+    (apl-rank
+      apl-tally
+      2
+      (make-array (list 3 5) (list 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15))))
+  (list 3))
+
+(apl-test
+  "rank reverse⍤1 matrix reverse rows"
+  (rv (apl-rank apl-reverse 1 (make-array (list 2 3) (list 1 2 3 4 5 6))))
+  (list 3 2 1 6 5 4))
+
+(apl-test
+  "rank tally⍤1 3x4 row tallies"
+  (rv
+    (apl-rank
+      apl-tally
+      1
+      (make-array (list 3 4) (list 1 2 3 4 5 6 7 8 9 10 11 12))))
+  (list 4 4 4))
+
+(apl-test
+  "at-replace single index"
+  (rv
+    (apl-at-replace
+      (apl-scalar 99)
+      (make-array (list 1) (list 2))
+      (make-array (list 5) (list 1 2 3 4 5))))
+  (list 1 99 3 4 5))
+
+(apl-test
+  "at-replace multiple indices vector vals"
+  (rv
+    (apl-at-replace
+      (make-array (list 2) (list 99 88))
+      (make-array (list 2) (list 2 4))
+      (make-array (list 5) (list 1 2 3 4 5))))
+  (list 1 99 3 88 5))
+
+(apl-test
+  "at-replace scalar broadcast"
+  (rv
+    (apl-at-replace
+      (apl-scalar 0)
+      (make-array (list 3) (list 1 3 5))
+      (make-array (list 5) (list 10 20 30 40 50))))
+  (list 0 20 0 40 0))
+
+(apl-test
+  "at-replace preserves shape"
+  (sh
+    (apl-at-replace
+      (apl-scalar 99)
+      (make-array (list 1) (list 2))
+      (make-array (list 5) (list 1 2 3 4 5))))
+  (list 5))
+
+(apl-test
+  "at-replace last index"
+  (rv
+    (apl-at-replace
+      (apl-scalar 99)
+      (make-array (list 1) (list 5))
+      (make-array (list 5) (list 1 2 3 4 5))))
+  (list 1 2 3 4 99))
+
+(apl-test
+  "at-replace on matrix linear-index"
+  (rv
+    (apl-at-replace
+      (apl-scalar 99)
+      (make-array (list 1) (list 3))
+      (make-array (list 2 3) (list 1 2 3 4 5 6))))
+  (list 1 2 99 4 5 6))
+
+(apl-test
+  "at-apply negate at indices"
+  (rv
+    (apl-at-apply
+      apl-neg-m
+      (make-array (list 3) (list 1 3 5))
+      (make-array (list 5) (list 1 2 3 4 5))))
+  (list -1 2 -3 4 -5))
+
+(apl-test
+  "at-apply double at index 1"
+  (rv
+    (apl-at-apply
+      (fn (a) (apl-mul a (apl-scalar 2)))
+      (make-array (list 1) (list 1))
+      (make-array (list 2) (list 5 10))))
+  (list 10 10))
+
+(apl-test
+  "at-apply preserves shape"
+  (sh
+    (apl-at-apply
+      apl-neg-m
+      (make-array (list 2) (list 1 3))
+      (make-array (list 2 3) (list 1 2 3 4 5 6))))
+  (list 2 3))
+
+(apl-test
+  "at-apply on matrix linear-index"
+  (rv
+    (apl-at-apply
+      apl-neg-m
+      (make-array (list 2) (list 1 6))
+      (make-array (list 2 3) (list 1 2 3 4 5 6))))
+  (list -1 2 3 4 5 -6))

lib/apl/tests/parse.sx

@@ -0,0 +1,340 @@
+(define apl-test-count 0)
+(define apl-test-pass  0)
+(define apl-test-fails (list))
+
+(define apl-test
+  (fn (name actual expected)
+    (begin
+      (set! apl-test-count (+ apl-test-count 1))
+      (if (= actual expected)
+        (set! apl-test-pass (+ apl-test-pass 1))
+        (append! apl-test-fails {:name name :actual actual :expected expected})))))
+
+(define tok-types
+  (fn (src)
+    (map (fn (t) (get t :type)) (apl-tokenize src))))
+
+(define tok-values
+  (fn (src)
+    (map (fn (t) (get t :value)) (apl-tokenize src))))
+
+(define tok-count
+  (fn (src)
+    (len (apl-tokenize src))))
+
+(define tok-type-at
+  (fn (src i)
+    (get (nth (apl-tokenize src) i) :type)))
+
+(define tok-value-at
+  (fn (src i)
+    (get (nth (apl-tokenize src) i) :value)))
+
+(apl-test "empty: no tokens"        (tok-count "")           0)
+(apl-test "empty: whitespace only"  (tok-count "   ")        0)
+(apl-test "num: zero"               (tok-values "0")         (list 0))
+(apl-test "num: positive"           (tok-values "42")        (list 42))
+(apl-test "num: large"              (tok-values "12345")     (list 12345))
+(apl-test "num: negative"           (tok-values "¯5")        (list -5))
+(apl-test "num: negative zero"      (tok-values "¯0")        (list 0))
+(apl-test "num: strand count"       (tok-count "1 2 3")      3)
+(apl-test "num: strand types"       (tok-types "1 2 3")      (list :num :num :num))
+(apl-test "num: strand values"      (tok-values "1 2 3")     (list 1 2 3))
+(apl-test "num: neg in strand"      (tok-values "1 ¯2 3")    (list 1 -2 3))
+(apl-test "str: empty"              (tok-values "''")        (list ""))
+(apl-test "str: single char"        (tok-values "'a'")       (list "a"))
+(apl-test "str: word"               (tok-values "'hello'")   (list "hello"))
+(apl-test "str: escaped quote"      (tok-values "''''")      (list "'"))
+(apl-test "str: type"               (tok-types "'abc'")      (list :str))
+(apl-test "name: simple"            (tok-values "foo")       (list "foo"))
+(apl-test "name: type"              (tok-types "foo")        (list :name))
+(apl-test "name: mixed case"        (tok-values "MyVar")     (list "MyVar"))
+(apl-test "name: with digits"       (tok-values "x1")        (list "x1"))
+(apl-test "name: system var"        (tok-values "⎕IO")       (list "⎕IO"))
+(apl-test "name: system var type"   (tok-types "⎕IO")        (list :name))
+(apl-test "glyph: plus"             (tok-types "+")          (list :glyph))
+(apl-test "glyph: plus value"       (tok-values "+")         (list "+"))
+(apl-test "glyph: iota"             (tok-values "⍳")         (list "⍳"))
+(apl-test "glyph: reduce"           (tok-values "+/")        (list "+" "/"))
+(apl-test "glyph: floor"            (tok-values "⌊")         (list "⌊"))
+(apl-test "glyph: rho"              (tok-values "⍴")         (list "⍴"))
+(apl-test "glyph: alpha omega"      (tok-types "⍺ ⍵")        (list :glyph :glyph))
+(apl-test "punct: lparen"           (tok-types "(")          (list :lparen))
+(apl-test "punct: rparen"           (tok-types ")")          (list :rparen))
+(apl-test "punct: brackets"         (tok-types "[42]")       (list :lbracket :num :rbracket))
+(apl-test "punct: braces"           (tok-types "{}")         (list :lbrace :rbrace))
+(apl-test "punct: semi"             (tok-types ";")          (list :semi))
+(apl-test "assign: arrow"           (tok-types "x←1")        (list :name :assign :num))
+(apl-test "diamond: separator"      (tok-types "1⋄2")        (list :num :diamond :num))
+(apl-test "newline: emitted"        (tok-types "1\n2")       (list :num :newline :num))
+(apl-test "comment: skipped"        (tok-count "⍝ ignore me") 0)
+(apl-test "comment: rest ignored"   (tok-count "1 ⍝ note")  1)
+(apl-test "colon: bare"             (tok-types ":")          (list :colon))
+(apl-test "keyword: If"             (tok-values ":If")       (list ":If"))
+(apl-test "keyword: type"           (tok-types ":While")     (list :keyword))
+(apl-test "keyword: EndFor"         (tok-values ":EndFor")   (list ":EndFor"))
+(apl-test "expr: +/ ⍳ 5"           (tok-types "+/ ⍳ 5")     (list :glyph :glyph :glyph :num))
+(apl-test "expr: x←42"             (tok-count "x←42")       3)
+(apl-test "expr: dfn body"          (tok-types "{⍺+⍵}")
+                                    (list :lbrace :glyph :glyph :glyph :rbrace))
+
+(define apl-tokenize-test-summary
+  (str "tokenizer " apl-test-pass "/" apl-test-count
+       (if (= (len apl-test-fails) 0) "" (str " FAILS: " apl-test-fails))))
+
+; ===========================================================================
+; Parser tests
+; ===========================================================================
+
+; Helper: parse an APL source string and return the AST
+(define parse
+  (fn (src) (parse-apl src)))
+
+; Helper: build an expected AST node using keyword-tagged lists
+(define num-node    (fn (n)       (list :num n)))
+(define str-node    (fn (s)       (list :str s)))
+(define name-node   (fn (n)       (list :name n)))
+(define fn-node     (fn (g)       (list :fn-glyph g)))
+(define fn-nm       (fn (n)       (list :fn-name n)))
+(define assign-node (fn (nm expr) (list :assign nm expr)))
+(define monad-node  (fn (f a)     (list :monad f a)))
+(define dyad-node   (fn (f l r)   (list :dyad f l r)))
+(define derived-fn  (fn (op f)    (list :derived-fn op f)))
+(define derived-fn2 (fn (op f g)  (list :derived-fn2 op f g)))
+(define outer-node  (fn (f)       (list :outer "∘." f)))
+(define guard-node  (fn (c e)     (list :guard c e)))
+
+; ---- numeric literals ----
+
+(apl-test "parse: num literal"
+  (parse "42")
+  (num-node 42))
+
+(apl-test "parse: negative num"
+  (parse "¯3")
+  (num-node -3))
+
+(apl-test "parse: zero"
+  (parse "0")
+  (num-node 0))
+
+; ---- string literals ----
+
+(apl-test "parse: str literal"
+  (parse "'hello'")
+  (str-node "hello"))
+
+(apl-test "parse: empty str"
+  (parse "''")
+  (str-node ""))
+
+; ---- name reference ----
+
+(apl-test "parse: name"
+  (parse "x")
+  (name-node "x"))
+
+(apl-test "parse: system name"
+  (parse "⎕IO")
+  (name-node "⎕IO"))
+
+; ---- strands (vec nodes) ----
+
+(apl-test "parse: strand 3 nums"
+  (parse "1 2 3")
+  (list :vec (num-node 1) (num-node 2) (num-node 3)))
+
+(apl-test "parse: strand 2 nums"
+  (parse "1 2")
+  (list :vec (num-node 1) (num-node 2)))
+
+(apl-test "parse: strand with negatives"
+  (parse "1 ¯2 3")
+  (list :vec (num-node 1) (num-node -2) (num-node 3)))
+
+; ---- assignment ----
+
+(apl-test "parse: assignment"
+  (parse "x←42")
+  (assign-node "x" (num-node 42)))
+
+(apl-test "parse: assignment with spaces"
+  (parse "x ← 42")
+  (assign-node "x" (num-node 42)))
+
+(apl-test "parse: assignment of expr"
+  (parse "r←2+3")
+  (assign-node "r" (dyad-node (fn-node "+") (num-node 2) (num-node 3))))
+
+; ---- monadic functions ----
+
+(apl-test "parse: monadic iota"
+  (parse "⍳5")
+  (monad-node (fn-node "⍳") (num-node 5)))
+
+(apl-test "parse: monadic iota with space"
+  (parse "⍳ 5")
+  (monad-node (fn-node "⍳") (num-node 5)))
+
+(apl-test "parse: monadic negate"
+  (parse "-3")
+  (monad-node (fn-node "-") (num-node 3)))
+
+(apl-test "parse: monadic floor"
+  (parse "⌊2")
+  (monad-node (fn-node "⌊") (num-node 2)))
+
+(apl-test "parse: monadic of name"
+  (parse "⍴x")
+  (monad-node (fn-node "⍴") (name-node "x")))
+
+; ---- dyadic functions ----
+
+(apl-test "parse: dyadic plus"
+  (parse "2+3")
+  (dyad-node (fn-node "+") (num-node 2) (num-node 3)))
+
+(apl-test "parse: dyadic times"
+  (parse "2×3")
+  (dyad-node (fn-node "×") (num-node 2) (num-node 3)))
+
+(apl-test "parse: dyadic with names"
+  (parse "x+y")
+  (dyad-node (fn-node "+") (name-node "x") (name-node "y")))
+
+; ---- right-to-left evaluation ----
+
+(apl-test "parse: right-to-left 2×3+4"
+  (parse "2×3+4")
+  (dyad-node (fn-node "×") (num-node 2)
+    (dyad-node (fn-node "+") (num-node 3) (num-node 4))))
+
+(apl-test "parse: right-to-left chain"
+  (parse "1+2×3-4")
+  (dyad-node (fn-node "+") (num-node 1)
+    (dyad-node (fn-node "×") (num-node 2)
+      (dyad-node (fn-node "-") (num-node 3) (num-node 4)))))
+
+; ---- parenthesized subexpressions ----
+
+(apl-test "parse: parens override order"
+  (parse "(2+3)×4")
+  (dyad-node (fn-node "×")
+    (dyad-node (fn-node "+") (num-node 2) (num-node 3))
+    (num-node 4)))
+
+(apl-test "parse: nested parens"
+  (parse "((2+3))")
+  (dyad-node (fn-node "+") (num-node 2) (num-node 3)))
+
+(apl-test "parse: paren in dyadic right"
+  (parse "2×(3+4)")
+  (dyad-node (fn-node "×") (num-node 2)
+    (dyad-node (fn-node "+") (num-node 3) (num-node 4))))
+
+; ---- operators → derived functions ----
+
+(apl-test "parse: reduce +"
+  (parse "+/x")
+  (monad-node (derived-fn "/" (fn-node "+")) (name-node "x")))
+
+(apl-test "parse: reduce iota"
+  (parse "+/⍳5")
+  (monad-node (derived-fn "/" (fn-node "+"))
+    (monad-node (fn-node "⍳") (num-node 5))))
+
+(apl-test "parse: scan"
+  (parse "+\\x")
+  (monad-node (derived-fn "\\" (fn-node "+")) (name-node "x")))
+
+(apl-test "parse: each"
+  (parse "⍳¨x")
+  (monad-node (derived-fn "¨" (fn-node "⍳")) (name-node "x")))
+
+(apl-test "parse: commute"
+  (parse "-⍨3")
+  (monad-node (derived-fn "⍨" (fn-node "-")) (num-node 3)))
+
+(apl-test "parse: stacked ops"
+  (parse "+/¨x")
+  (monad-node (derived-fn "¨" (derived-fn "/" (fn-node "+"))) (name-node "x")))
+
+; ---- outer product ----
+
+(apl-test "parse: outer product monadic"
+  (parse "∘.×")
+  (outer-node (fn-node "×")))
+
+(apl-test "parse: outer product dyadic names"
+  (parse "x ∘.× y")
+  (dyad-node (outer-node (fn-node "×")) (name-node "x") (name-node "y")))
+
+(apl-test "parse: outer product dyadic strands"
+  (parse "1 2 3 ∘.× 4 5 6")
+  (dyad-node (outer-node (fn-node "×"))
+    (list :vec (num-node 1) (num-node 2) (num-node 3))
+    (list :vec (num-node 4) (num-node 5) (num-node 6))))
+
+; ---- inner product ----
+
+(apl-test "parse: inner product"
+  (parse "+.×")
+  (derived-fn2 "." (fn-node "+") (fn-node "×")))
+
+(apl-test "parse: inner product applied"
+  (parse "a +.× b")
+  (dyad-node (derived-fn2 "." (fn-node "+") (fn-node "×"))
+    (name-node "a") (name-node "b")))
+
+; ---- dfn (anonymous function) ----
+
+(apl-test "parse: simple dfn"
+  (parse "{⍺+⍵}")
+  (list :dfn (dyad-node (fn-node "+") (name-node "⍺") (name-node "⍵"))))
+
+(apl-test "parse: monadic dfn"
+  (parse "{⍵×2}")
+  (list :dfn (dyad-node (fn-node "×") (name-node "⍵") (num-node 2))))
+
+(apl-test "parse: dfn self-ref"
+  (parse "{⍵≤1:1 ⋄ ⍵×∇ ⍵-1}")
+  (list :dfn
+    (guard-node (dyad-node (fn-node "≤") (name-node "⍵") (num-node 1)) (num-node 1))
+    (dyad-node (fn-node "×") (name-node "⍵")
+      (monad-node (fn-node "∇") (dyad-node (fn-node "-") (name-node "⍵") (num-node 1))))))
+
+; ---- dfn applied ----
+
+(apl-test "parse: dfn as function"
+  (parse "{⍺+⍵} 3")
+  (monad-node
+    (list :dfn (dyad-node (fn-node "+") (name-node "⍺") (name-node "⍵")))
+    (num-node 3)))
+
+; ---- multi-statement ----
+
+(apl-test "parse: diamond separator"
+  (let ((result (parse "x←1 ⋄ x+2")))
+    (= (first result) :program))
+  true)
+
+(apl-test "parse: diamond first stmt"
+  (let ((result (parse "x←1 ⋄ x+2")))
+    (nth result 1))
+  (assign-node "x" (num-node 1)))
+
+(apl-test "parse: diamond second stmt"
+  (let ((result (parse "x←1 ⋄ x+2")))
+    (nth result 2))
+  (dyad-node (fn-node "+") (name-node "x") (num-node 2)))
+
+; ---- combined summary ----
+
+(define apl-parse-test-count (- apl-test-count 46))
+(define apl-parse-test-pass  (- apl-test-pass 46))
+
+(define apl-test-summary
+  (str
+    "tokenizer 46/46  |  "
+    "parser " apl-parse-test-pass "/" apl-parse-test-count
+    (if (= (len apl-test-fails) 0) "" (str "  FAILS: " apl-test-fails))))

lib/apl/tests/pipeline.sx

@@ -0,0 +1,314 @@
+; End-to-end pipeline tests: source string → tokenize → parse → eval-ast → array.
+; Verifies the full stack as a single function call (apl-run).
+
+(define mkrv (fn (arr) (get arr :ravel)))
+(define mksh (fn (arr) (get arr :shape)))
+
+; ---------- scalars ----------
+
+(apl-test "apl-run \"42\" → scalar 42" (mkrv (apl-run "42")) (list 42))
+
+(apl-test "apl-run \"¯7\" → scalar -7" (mkrv (apl-run "¯7")) (list -7))
+
+; ---------- strands ----------
+
+(apl-test
+  "apl-run \"1 2 3\" → vector"
+  (mkrv (apl-run "1 2 3"))
+  (list 1 2 3))
+
+(apl-test "apl-run \"1 2 3\" shape" (mksh (apl-run "1 2 3")) (list 3))
+
+; ---------- dyadic arithmetic ----------
+
+(apl-test "apl-run \"2 + 3\" → 5" (mkrv (apl-run "2 + 3")) (list 5))
+
+(apl-run "2 × 3 + 4")  ; right-to-left
+
+(apl-test
+  "apl-run \"2 × 3 + 4\" → 14 (right-to-left)"
+  (mkrv (apl-run "2 × 3 + 4"))
+  (list 14))
+
+(apl-test
+  "apl-run \"1 2 3 + 4 5 6\" → 5 7 9"
+  (mkrv (apl-run "1 2 3 + 4 5 6"))
+  (list 5 7 9))
+
+(apl-test
+  "apl-run \"3 × 1 2 3 4\" → scalar broadcast"
+  (mkrv (apl-run "3 × 1 2 3 4"))
+  (list 3 6 9 12))
+
+; ---------- monadic primitives ----------
+
+(apl-test
+  "apl-run \"⍳5\" → 1..5"
+  (mkrv (apl-run "⍳5"))
+  (list 1 2 3 4 5))
+
+(apl-test
+  "apl-run \"-3\" → -3 (monadic negate)"
+  (mkrv (apl-run "-3"))
+  (list -3))
+
+(apl-test
+  "apl-run \"⌈/ 1 3 9 5 7\" → 9 (max-reduce)"
+  (mkrv (apl-run "⌈/ 1 3 9 5 7"))
+  (list 9))
+
+(apl-test
+  "apl-run \"⌊/ 4 7 2 9 1 3\" → 1 (min-reduce)"
+  (mkrv (apl-run "⌊/ 4 7 2 9 1 3"))
+  (list 1))
+
+; ---------- operators ----------
+
+(apl-test "apl-run \"+/⍳5\" → 15" (mkrv (apl-run "+/⍳5")) (list 15))
+
+(apl-test "apl-run \"×/⍳5\" → 120" (mkrv (apl-run "×/⍳5")) (list 120))
+
+(apl-test
+  "apl-run \"⌈/3 1 4 1 5 9 2\" → 9"
+  (mkrv (apl-run "⌈/3 1 4 1 5 9 2"))
+  (list 9))
+
+(apl-test
+  "apl-run \"+\\\\⍳5\" → triangular numbers"
+  (mkrv (apl-run "+\\⍳5"))
+  (list 1 3 6 10 15))
+
+; ---------- outer / inner products ----------
+
+(apl-test
+  "apl-run \"1 2 3 ∘.× 1 2 3\" → mult table values"
+  (mkrv (apl-run "1 2 3 ∘.× 1 2 3"))
+  (list 1 2 3 2 4 6 3 6 9))
+
+(apl-test
+  "apl-run \"1 2 3 +.× 4 5 6\" → dot product 32"
+  (mkrv (apl-run "1 2 3 +.× 4 5 6"))
+  (list 32))
+
+; ---------- shape ----------
+
+(apl-test
+  "apl-run \"⍴ 1 2 3 4 5\" → 5"
+  (mkrv (apl-run "⍴ 1 2 3 4 5"))
+  (list 5))
+
+(apl-test "apl-run \"⍴⍳10\" → 10" (mkrv (apl-run "⍴⍳10")) (list 10))
+
+; ---------- comparison ----------
+
+(apl-test "apl-run \"3 < 5\" → 1" (mkrv (apl-run "3 < 5")) (list 1))
+
+(apl-test "apl-run \"5 = 5\" → 1" (mkrv (apl-run "5 = 5")) (list 1))
+
+(apl-test
+  "apl-run \"1 2 3 = 1 0 3\" → 1 0 1"
+  (mkrv (apl-run "1 2 3 = 1 0 3"))
+  (list 1 0 1))
+
+; ---------- famous one-liners ----------
+
+(apl-test
+  "apl-run \"+/(⍳10)\" → sum 1..10 = 55"
+  (mkrv (apl-run "+/(⍳10)"))
+  (list 55))
+
+(apl-test
+  "apl-run \"×/⍳10\" → 10! = 3628800"
+  (mkrv (apl-run "×/⍳10"))
+  (list 3628800))
+
+(apl-test "apl-run \"⎕IO\" → 1" (mkrv (apl-run "⎕IO")) (list 1))
+
+(apl-test "apl-run \"⎕ML\" → 1" (mkrv (apl-run "⎕ML")) (list 1))
+
+(apl-test "apl-run \"⎕FR\" → 1248" (mkrv (apl-run "⎕FR")) (list 1248))
+
+(apl-test "apl-run \"⎕TS\" shape (7)" (mksh (apl-run "⎕TS")) (list 7))
+
+(apl-test "apl-run \"⎕FMT 42\" → \"42\"" (apl-run "⎕FMT 42") "42")
+
+(apl-test
+  "apl-run \"⎕FMT 1 2 3\" → \"1 2 3\""
+  (apl-run "⎕FMT 1 2 3")
+  "1 2 3")
+
+(apl-test
+  "apl-run \"⎕FMT ⍳5\" → \"1 2 3 4 5\""
+  (apl-run "⎕FMT ⍳5")
+  "1 2 3 4 5")
+
+(apl-test "apl-run \"⎕IO + 4\" → 5" (mkrv (apl-run "⎕IO + 4")) (list 5))
+
+(apl-test
+  "apl-run \"(10 20 30 40 50)[3]\" → 30"
+  (mkrv (apl-run "(10 20 30 40 50)[3]"))
+  (list 30))
+
+(apl-test
+  "apl-run \"(⍳10)[5]\" → 5"
+  (mkrv (apl-run "(⍳10)[5]"))
+  (list 5))
+
+(apl-test
+  "apl-run \"A ← 100 200 300 ⋄ A[2]\" → 200"
+  (mkrv (apl-run "A ← 100 200 300 ⋄ A[2]"))
+  (list 200))
+
+(apl-test
+  "apl-run \"V ← ⍳10 ⋄ V[3]\" → 3"
+  (mkrv (apl-run "V ← ⍳10 ⋄ V[3]"))
+  (list 3))
+
+(apl-test
+  "apl-run \"(10 20 30)[1]\" → 10 (1-indexed)"
+  (mkrv (apl-run "(10 20 30)[1]"))
+  (list 10))
+
+(apl-test
+  "apl-run \"V ← 10 20 30 40 50 ⋄ V[3] + 1\" → 31"
+  (mkrv (apl-run "V ← 10 20 30 40 50 ⋄ V[3] + 1"))
+  (list 31))
+
+(apl-test
+  "apl-run \"(⍳5)[3] × 7\" → 21"
+  (mkrv (apl-run "(⍳5)[3] × 7"))
+  (list 21))
+
+(apl-test "decimal: 3.7 → 3.7" (mkrv (apl-run "3.7")) (list 3.7))
+
+(apl-test "decimal: ¯2.5 → -2.5" (mkrv (apl-run "¯2.5")) (list -2.5))
+
+(apl-test "decimal: 1.5 + 2.5 → 4" (mkrv (apl-run "1.5 + 2.5")) (list 4))
+
+(apl-test "decimal: ⌊3.7 → 3" (mkrv (apl-run "⌊ 3.7")) (list 3))
+
+(apl-test "decimal: ⌈3.7 → 4" (mkrv (apl-run "⌈ 3.7")) (list 4))
+
+(apl-test
+  "⎕← scalar passthrough"
+  (mkrv (apl-run "⎕← 42"))
+  (list 42))
+
+(apl-test
+  "⎕← vector passthrough"
+  (mkrv (apl-run "⎕← 1 2 3"))
+  (list 1 2 3))
+
+(apl-test
+  "string: 'abc' → 3-char vector"
+  (mkrv (apl-run "'abc'"))
+  (list "a" "b" "c"))
+
+(apl-test "string: 'a' is rank-0 scalar" (mksh (apl-run "'a'")) (list))
+
+(apl-test "string: 'hello' shape (5)" (mksh (apl-run "'hello'")) (list 5))
+
+(apl-test
+  "named-fn: f ← {⍺+⍵} ⋄ 3 f 4 → 7"
+  (mkrv (apl-run "f ← {⍺+⍵} ⋄ 3 f 4"))
+  (list 7))
+
+(apl-test
+  "named-fn monadic: sq ← {⍵×⍵} ⋄ sq 7 → 49"
+  (mkrv (apl-run "sq ← {⍵×⍵} ⋄ sq 7"))
+  (list 49))
+
+(apl-test
+  "named-fn dyadic: hyp ← {((⍺×⍺)+⍵×⍵)} ⋄ 3 hyp 4 → 25"
+  (mkrv (apl-run "hyp ← {((⍺×⍺)+⍵×⍵)} ⋄ 3 hyp 4"))
+  (list 25))
+
+(apl-test
+  "named-fn: dbl ← {⍵+⍵} ⋄ dbl ⍳5"
+  (mkrv (apl-run "dbl ← {⍵+⍵} ⋄ dbl ⍳5"))
+  (list 2 4 6 8 10))
+
+(apl-test
+  "named-fn factorial via ∇ recursion"
+  (mkrv (apl-run "fact ← {0=⍵:1 ⋄ ⍵×∇⍵-1} ⋄ fact 5"))
+  (list 120))
+
+(apl-test
+  "named-fn used twice in expr: dbl ← {⍵+⍵} ⋄ (dbl 3) + dbl 4"
+  (mkrv (apl-run "dbl ← {⍵+⍵} ⋄ (dbl 3) + dbl 4"))
+  (list 14))
+
+(apl-test
+  "named-fn with vector arg: neg ← {-⍵} ⋄ neg 1 2 3"
+  (mkrv (apl-run "neg ← {-⍵} ⋄ neg 1 2 3"))
+  (list -1 -2 -3))
+
+(apl-test
+  "multi-axis: M[2;2] → center"
+  (mkrv (apl-run "M ← (3 3) ⍴ ⍳9 ⋄ M[2;2]"))
+  (list 5))
+
+(apl-test
+  "multi-axis: M[1;] → first row"
+  (mkrv (apl-run "M ← (3 3) ⍴ ⍳9 ⋄ M[1;]"))
+  (list 1 2 3))
+
+(apl-test
+  "multi-axis: M[;2] → second column"
+  (mkrv (apl-run "M ← (3 3) ⍴ ⍳9 ⋄ M[;2]"))
+  (list 2 5 8))
+
+(apl-test
+  "multi-axis: M[1 2;1 2] → 2x2 block"
+  (mkrv (apl-run "M ← (2 3) ⍴ ⍳6 ⋄ M[1 2;1 2]"))
+  (list 1 2 4 5))
+
+(apl-test
+  "multi-axis: M[1 2;1 2] shape (2 2)"
+  (mksh (apl-run "M ← (2 3) ⍴ ⍳6 ⋄ M[1 2;1 2]"))
+  (list 2 2))
+
+(apl-test
+  "multi-axis: M[;] full matrix"
+  (mkrv (apl-run "M ← (2 2) ⍴ 10 20 30 40 ⋄ M[;]"))
+  (list 10 20 30 40))
+
+(apl-test
+  "multi-axis: M[1;] shape collapsed"
+  (mksh (apl-run "M ← (3 3) ⍴ ⍳9 ⋄ M[1;]"))
+  (list 3))
+
+(apl-test
+  "multi-axis: select all rows of column 3"
+  (mkrv (apl-run "M ← (4 3) ⍴ 1 2 3 4 5 6 7 8 9 10 11 12 ⋄ M[;3]"))
+  (list 3 6 9 12))
+
+(apl-test
+  "train: mean = (+/÷≢) on 1..5"
+  (mkrv (apl-run "(+/÷≢) 1 2 3 4 5"))
+  (list 3))
+
+(apl-test
+  "train: mean of 2 4 6 8 10"
+  (mkrv (apl-run "(+/÷≢) 2 4 6 8 10"))
+  (list 6))
+
+(apl-test
+  "train 2-atop: (- ⌊) 5 → -5"
+  (mkrv (apl-run "(- ⌊) 5"))
+  (list -5))
+
+(apl-test
+  "train 3-fork dyadic: 2(+×-)5 → -21"
+  (mkrv (apl-run "2 (+ × -) 5"))
+  (list -21))
+
+(apl-test
+  "train: range = (⌈/-⌊/) on vector"
+  (mkrv (apl-run "(⌈/-⌊/) 3 1 4 1 5 9 2 6"))
+  (list 8))
+
+(apl-test
+  "train: mean of ⍳10 has shape ()"
+  (mksh (apl-run "(+/÷≢) ⍳10"))
+  (list))

lib/apl/tests/programs-e2e.sx

@@ -0,0 +1,96 @@
+; End-to-end tests of the classic-program archetypes — running APL
+; source through the full pipeline (tokenize → parse → eval-ast → runtime).
+;
+; These mirror the algorithms documented in lib/apl/tests/programs/*.apl
+; but use forms our pipeline supports today (named functions instead of
+; the inline ⍵← rebinding idiom; multi-stmt over single one-liners).
+
+(define mkrv (fn (arr) (get arr :ravel)))
+(define mksh (fn (arr) (get arr :shape)))
+
+; ---------- factorial via ∇ recursion (cf. n-queens style) ----------
+
+(apl-test
+  "e2e: factorial 5! = 120"
+  (mkrv (apl-run "fact ← {0=⍵:1 ⋄ ⍵×∇⍵-1} ⋄ fact 5"))
+  (list 120))
+
+(apl-test
+  "e2e: factorial 7! = 5040"
+  (mkrv (apl-run "fact ← {0=⍵:1 ⋄ ⍵×∇⍵-1} ⋄ fact 7"))
+  (list 5040))
+
+(apl-test
+  "e2e: factorial via ×/⍳N (no recursion)"
+  (mkrv (apl-run "fact ← {×/⍳⍵} ⋄ fact 6"))
+  (list 720))
+
+; ---------- sum / triangular numbers (sum-1..N) ----------
+
+(apl-test
+  "e2e: triangular(10) = 55"
+  (mkrv (apl-run "tri ← {+/⍳⍵} ⋄ tri 10"))
+  (list 55))
+
+(apl-test
+  "e2e: triangular(100) = 5050"
+  (mkrv (apl-run "tri ← {+/⍳⍵} ⋄ tri 100"))
+  (list 5050))
+
+; ---------- sum of squares ----------
+
+(apl-test
+  "e2e: sum-of-squares 1..5 = 55"
+  (mkrv (apl-run "ss ← {+/⍵×⍵} ⋄ ss ⍳5"))
+  (list 55))
+
+(apl-test
+  "e2e: sum-of-squares 1..10 = 385"
+  (mkrv (apl-run "ss ← {+/⍵×⍵} ⋄ ss ⍳10"))
+  (list 385))
+
+; ---------- divisor-counting (prime-sieve building blocks) ----------
+
+(apl-test
+  "e2e: divisor counts 1..5 via outer mod"
+  (mkrv (apl-run "P ← ⍳ 5 ⋄ +⌿ 0 = P ∘.| P"))
+  (list 1 2 2 3 2))
+
+(apl-test
+  "e2e: divisor counts 1..10"
+  (mkrv (apl-run "P ← ⍳ 10 ⋄ +⌿ 0 = P ∘.| P"))
+  (list 1 2 2 3 2 4 2 4 3 4))
+
+(apl-test
+  "e2e: prime-mask 1..10 (count==2)"
+  (mkrv (apl-run "P ← ⍳ 10 ⋄ 2 = +⌿ 0 = P ∘.| P"))
+  (list 0 1 1 0 1 0 1 0 0 0))
+
+; ---------- monadic primitives chained ----------
+
+(apl-test
+  "e2e: sum of |abs| = 15"
+  (mkrv (apl-run "+/|¯1 ¯2 ¯3 ¯4 ¯5"))
+  (list 15))
+
+(apl-test
+  "e2e: max of squares 1..6"
+  (mkrv (apl-run "⌈/(⍳6)×⍳6"))
+  (list 36))
+
+; ---------- nested named functions ----------
+
+(apl-test
+  "e2e: compose dbl and sq via two named fns"
+  (mkrv (apl-run "dbl ← {⍵+⍵} ⋄ sq ← {⍵×⍵} ⋄ sq dbl 3"))
+  (list 36))
+
+(apl-test
+  "e2e: max-of-two as named dyadic fn"
+  (mkrv (apl-run "mx ← {⍺⌈⍵} ⋄ 5 mx 3"))
+  (list 5))
+
+(apl-test
+  "e2e: sqrt-via-newton 1 step from 1 → 2.5"
+  (mkrv (apl-run "step ← {(⍵+⍺÷⍵)÷2} ⋄ 4 step 1"))
+  (list 2.5))

lib/apl/tests/programs.sx

@@ -0,0 +1,306 @@
+; Tests for classic APL programs (lib/apl/tests/programs/*.apl).
+; Programs are showcase APL source; runtime impl is in lib/apl/runtime.sx.
+
+(define mkrv (fn (arr) (get arr :ravel)))
+(define mksh (fn (arr) (get arr :shape)))
+
+; ===== primes (Sieve of Eratosthenes) =====
+
+(apl-test "primes 1 → empty" (mkrv (apl-primes 1)) (list))
+
+(apl-test "primes 2 → just 2" (mkrv (apl-primes 2)) (list 2))
+
+(apl-test "primes 10 → 2 3 5 7" (mkrv (apl-primes 10)) (list 2 3 5 7))
+
+(apl-test
+  "primes 20 → 2 3 5 7 11 13 17 19"
+  (mkrv (apl-primes 20))
+  (list 2 3 5 7 11 13 17 19))
+
+(apl-test
+  "primes 30"
+  (mkrv (apl-primes 30))
+  (list 2 3 5 7 11 13 17 19 23 29))
+
+(apl-test
+  "primes 50"
+  (mkrv (apl-primes 50))
+  (list 2 3 5 7 11 13 17 19 23 29 31 37 41 43 47))
+
+(apl-test "primes 7 length" (first (mksh (apl-primes 7))) 4)
+
+(apl-test "primes 100 has 25 primes" (first (mksh (apl-primes 100))) 25)
+
+; ===== compress helper sanity =====
+
+(apl-test
+  "compress 1 0 1 0 1 / 10 20 30 40 50"
+  (mkrv
+    (apl-compress
+      (make-array (list 5) (list 1 0 1 0 1))
+      (make-array (list 5) (list 10 20 30 40 50))))
+  (list 10 30 50))
+
+(apl-test
+  "compress all-zero mask → empty"
+  (mkrv
+    (apl-compress
+      (make-array (list 3) (list 0 0 0))
+      (make-array (list 3) (list 1 2 3))))
+  (list))
+
+(apl-test
+  "compress all-one mask → full vector"
+  (mkrv
+    (apl-compress
+      (make-array (list 3) (list 1 1 1))
+      (make-array (list 3) (list 1 2 3))))
+  (list 1 2 3))
+
+(apl-test
+  "life: empty 5x5 stays empty"
+  (mkrv
+    (apl-life-step
+      (make-array
+        (list 5 5)
+        (list 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0))))
+  (list 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0))
+
+(apl-test
+  "life: horizontal blinker → vertical blinker"
+  (mkrv
+    (apl-life-step
+      (make-array
+        (list 5 5)
+        (list 0 0 0 0 0 0 0 0 0 0 0 1 1 1 0 0 0 0 0 0 0 0 0 0 0))))
+  (list 0 0 0 0 0 0 0 1 0 0 0 0 1 0 0 0 0 1 0 0 0 0 0 0 0))
+
+(apl-test
+  "life: vertical blinker → horizontal blinker"
+  (mkrv
+    (apl-life-step
+      (make-array
+        (list 5 5)
+        (list 0 0 0 0 0 0 0 1 0 0 0 0 1 0 0 0 0 1 0 0 0 0 0 0 0))))
+  (list 0 0 0 0 0 0 0 0 0 0 0 1 1 1 0 0 0 0 0 0 0 0 0 0 0))
+
+(apl-test
+  "life: blinker has period 2"
+  (mkrv
+    (apl-life-step
+      (apl-life-step
+        (make-array
+          (list 5 5)
+          (list 0 0 0 0 0 0 0 0 0 0 0 1 1 1 0 0 0 0 0 0 0 0 0 0 0)))))
+  (list 0 0 0 0 0 0 0 0 0 0 0 1 1 1 0 0 0 0 0 0 0 0 0 0 0))
+
+(apl-test
+  "life: 2x2 block stable on 5x5"
+  (mkrv
+    (apl-life-step
+      (make-array
+        (list 5 5)
+        (list 0 0 0 0 0 0 1 1 0 0 0 1 1 0 0 0 0 0 0 0 0 0 0 0 0))))
+  (list 0 0 0 0 0 0 1 1 0 0 0 1 1 0 0 0 0 0 0 0 0 0 0 0 0))
+
+(apl-test
+  "life: shape preserved"
+  (mksh
+    (apl-life-step
+      (make-array
+        (list 5 5)
+        (list 0 0 0 0 0 0 0 0 0 0 0 1 1 1 0 0 0 0 0 0 0 0 0 0 0))))
+  (list 5 5))
+
+(apl-test
+  "life: glider on 6x6 advances"
+  (mkrv
+    (apl-life-step
+      (make-array
+        (list 6 6)
+        (list
+          0
+          0
+          0
+          0
+          0
+          0
+          0
+          0
+          1
+          0
+          0
+          0
+          0
+          0
+          0
+          1
+          0
+          0
+          0
+          1
+          1
+          1
+          0
+          0
+          0
+          0
+          0
+          0
+          0
+          0
+          0
+          0
+          0
+          0
+          0
+          0))))
+  (list
+    0
+    0
+    0
+    0
+    0
+    0
+    0
+    0
+    0
+    0
+    0
+    0
+    0
+    1
+    0
+    1
+    0
+    0
+    0
+    0
+    1
+    1
+    0
+    0
+    0
+    0
+    1
+    0
+    0
+    0
+    0
+    0
+    0
+    0
+    0
+    0))
+
+(apl-test
+  "mandelbrot c=0 stays bounded"
+  (mkrv (apl-mandelbrot-1d (make-array (list 1) (list 0)) 100))
+  (list 100))
+
+(apl-test
+  "mandelbrot c=-1 cycle bounded"
+  (mkrv (apl-mandelbrot-1d (make-array (list 1) (list -1)) 100))
+  (list 100))
+
+(apl-test
+  "mandelbrot c=-2 boundary stays bounded"
+  (mkrv (apl-mandelbrot-1d (make-array (list 1) (list -2)) 100))
+  (list 100))
+
+(apl-test
+  "mandelbrot c=0.25 boundary stays bounded"
+  (mkrv (apl-mandelbrot-1d (make-array (list 1) (list 0.25)) 100))
+  (list 100))
+
+(apl-test
+  "mandelbrot c=1 escapes at iter 3"
+  (mkrv (apl-mandelbrot-1d (make-array (list 1) (list 1)) 100))
+  (list 3))
+
+(apl-test
+  "mandelbrot c=0.5 escapes at iter 5"
+  (mkrv (apl-mandelbrot-1d (make-array (list 1) (list 0.5)) 100))
+  (list 5))
+
+(apl-test
+  "mandelbrot batched grid (rank-polymorphic)"
+  (mkrv (apl-mandelbrot-1d (make-array (list 5) (list -2 -1 0 1 2)) 10))
+  (list 10 10 10 3 2))
+
+(apl-test
+  "mandelbrot batched preserves shape"
+  (mksh (apl-mandelbrot-1d (make-array (list 5) (list -2 -1 0 1 2)) 10))
+  (list 5))
+
+(apl-test
+  "mandelbrot c=-1.5 stays bounded"
+  (mkrv (apl-mandelbrot-1d (make-array (list 1) (list -1.5)) 100))
+  (list 100))
+
+(apl-test "queens 1 → 1 solution" (mkrv (apl-queens 1)) (list 1))
+
+(apl-test "queens 2 → 0 solutions" (mkrv (apl-queens 2)) (list 0))
+
+(apl-test "queens 3 → 0 solutions" (mkrv (apl-queens 3)) (list 0))
+
+(apl-test "queens 4 → 2 solutions" (mkrv (apl-queens 4)) (list 2))
+
+(apl-test "queens 5 → 10 solutions" (mkrv (apl-queens 5)) (list 10))
+
+(apl-test "queens 6 → 4 solutions" (mkrv (apl-queens 6)) (list 4))
+
+(apl-test "queens 7 → 40 solutions" (mkrv (apl-queens 7)) (list 40))
+
+(apl-test "queens 8 → 92 solutions" (mkrv (apl-queens 8)) (list 92))
+
+(apl-test "permutations of 3 has 6" (len (apl-permutations 3)) 6)
+
+(apl-test "permutations of 4 has 24" (len (apl-permutations 4)) 24)
+
+(apl-test
+  "quicksort empty"
+  (mkrv (apl-quicksort (make-array (list 0) (list))))
+  (list))
+
+(apl-test
+  "quicksort single"
+  (mkrv (apl-quicksort (make-array (list 1) (list 42))))
+  (list 42))
+
+(apl-test
+  "quicksort already sorted"
+  (mkrv (apl-quicksort (make-array (list 5) (list 1 2 3 4 5))))
+  (list 1 2 3 4 5))
+
+(apl-test
+  "quicksort reverse sorted"
+  (mkrv (apl-quicksort (make-array (list 5) (list 5 4 3 2 1))))
+  (list 1 2 3 4 5))
+
+(apl-test
+  "quicksort with duplicates"
+  (mkrv (apl-quicksort (make-array (list 7) (list 3 1 4 1 5 9 2))))
+  (list 1 1 2 3 4 5 9))
+
+(apl-test
+  "quicksort all equal"
+  (mkrv (apl-quicksort (make-array (list 5) (list 7 7 7 7 7))))
+  (list 7 7 7 7 7))
+
+(apl-test
+  "quicksort negatives"
+  (mkrv (apl-quicksort (make-array (list 5) (list -3 1 -1 2 0))))
+  (list -3 -1 0 1 2))
+
+(apl-test
+  "quicksort 11-element pi"
+  (mkrv
+    (apl-quicksort (make-array (list 11) (list 3 1 4 1 5 9 2 6 5 3 5))))
+  (list 1 1 2 3 3 4 5 5 5 6 9))
+
+(apl-test
+  "quicksort preserves length"
+  (first
+    (mksh (apl-quicksort (make-array (list 7) (list 3 1 4 1 5 9 2)))))
+  7)

lib/apl/tests/programs/life.apl

@@ -0,0 +1,22 @@
+⍝ Conway's Game of Life — toroidal one-liner
+⍝
+⍝ The classic Roger Hui formulation:
+⍝   life ← {⊃1 ⍵ ∨.∧ 3 4 = +/ +/ ¯1 0 1 ∘.⊖ ¯1 0 1 ⌽¨ ⊂⍵}
+⍝
+⍝ Read right-to-left:
+⍝   ⊂⍵                : enclose the board (so it's a single scalar item)
+⍝   ¯1 0 1 ⌽¨ ⊂⍵      : produce 3 horizontally-shifted copies
+⍝   ¯1 0 1 ∘.⊖ …      : outer-product with vertical shifts → 3×3 = 9 shifts
+⍝   +/ +/ …           : sum the 9 boards element-wise → neighbor-count + self
+⍝   3 4 = …           : boolean — count is exactly 3 or exactly 4
+⍝   1 ⍵ ∨.∧ …         : "alive next" iff (count=3) or (alive AND count=4)
+⍝   ⊃ …               : disclose back to a 2D board
+⍝
+⍝ Rules in plain language:
+⍝   - dead cell + 3 live neighbors → born
+⍝   - live cell + 2 or 3 live neighbors → survives
+⍝   - all else → dies
+⍝
+⍝ Toroidal: edges wrap (rotate is cyclic).
+
+life ← {⊃1 ⍵ ∨.∧ 3 4 = +/ +/ ¯1 0 1 ∘.⊖ ¯1 0 1 ⌽¨ ⊂⍵}

lib/apl/tests/programs/mandelbrot.apl

@@ -0,0 +1,29 @@
+⍝ Mandelbrot — real-axis subset
+⍝
+⍝ For complex c, the Mandelbrot set is { c : |z_n| stays bounded } where
+⍝   z_0 = 0, z_{n+1} = z_n² + c.
+⍝ Restricting c (and z) to ℝ gives the segment c ∈ [-2, 1/4]
+⍝ where the iteration stays bounded.
+⍝
+⍝ Rank-polymorphic batched-iteration form:
+⍝   mandelbrot ← {⍵ ⍵⍵ ⍺⍺ +,(⍺⍺ × ⍺⍺) }
+⍝
+⍝ Pseudocode (as we don't have ⎕ system fns yet):
+⍝   z      ← 0×c                             ⍝ start at zero
+⍝   alive  ← 1+0×c                           ⍝ all "still in"
+⍝   for k iterations:
+⍝     alive ← alive ∧ 4 ≥ z×z                ⍝ still bounded?
+⍝     z     ← alive × c + z×z                ⍝ freeze escaped via mask
+⍝     count ← count + alive                  ⍝ tally surviving iters
+⍝
+⍝ Examples (count after 100 iterations):
+⍝   c=0      : 100  (z stays at 0)
+⍝   c=-1     : 100  (cycles 0,-1,0,-1,...)
+⍝   c=-2     : 100  (settles at 2 — boundary)
+⍝   c=0.25   : 100  (boundary — converges to 0.5)
+⍝   c=0.5    : 5    (escapes by iteration 6)
+⍝   c=1      : 3    (escapes quickly)
+⍝
+⍝ Real-axis Mandelbrot set: bounded for c ∈ [-2, 0.25].
+
+mandelbrot ← {z←alive←count←0×⍵ ⋄ {alive←alive∧4≥z×z ⋄ z←alive×⍵+z×z ⋄ count+←alive}⍣⍺⊢⍵}

lib/apl/tests/programs/n-queens.apl

@@ -0,0 +1,18 @@
+⍝ N-Queens — count solutions to placing N non-attacking queens on N×N
+⍝
+⍝ A solution is encoded as a permutation P of 1..N where P[i] is the
+⍝ column of the queen in row i.  Rows and columns are then automatically
+⍝ unique (it's a permutation).  We must additionally rule out queens
+⍝ sharing a diagonal: |i-j| = |P[i]-P[j]| for any pair.
+⍝
+⍝ Backtracking via reduce — the classic Roger Hui style:
+⍝   queens ← {≢{⍵,¨⍨↓(0=∊(¨⍳⍴⍵)≠.+|⍵)/⍳⍴⍵}/(⍳⍵)⍴⊂⍳⍵}
+⍝
+⍝ Plain reading:
+⍝   permute 1..N, keep those where no two queens share a diagonal.
+⍝
+⍝ Known solution counts (OEIS A000170):
+⍝     N    1   2   3   4   5   6   7    8    9    10
+⍝   q(N)   1   0   0   2  10   4  40   92  352   724
+
+queens ← {≢({(i j)←⍺⍵ ⋄ (|i-j)≠|(P[i])-(P[j])}⌿permutations ⍵)}

lib/apl/tests/programs/primes.apl

@@ -0,0 +1,16 @@
+⍝ Sieve of Eratosthenes — the classic APL one-liner
+⍝   primes ← (2=+⌿0=A∘.|A)/A←⍳N
+⍝
+⍝ Read right-to-left:
+⍝   A ← ⍳N        : A is 1..N
+⍝   A∘.|A         : outer-product residue table — M[i,j] = A[j] mod A[i]
+⍝   0=...         : boolean — true where A[i] divides A[j]
+⍝   +⌿...         : column sums — count of divisors per A[j]
+⍝   2=...         : true for numbers with exactly 2 divisors (1 and self) → primes
+⍝   .../A         : compress — select A[j] where mask[j] is true
+⍝
+⍝ Examples:
+⍝   primes 10  →  2 3 5 7
+⍝   primes 30  →  2 3 5 7 11 13 17 19 23 29
+
+primes ← {(2=+⌿0=⍵∘.|⍵)/⍵←⍳⍵}

lib/apl/tests/programs/quicksort.apl

@@ -0,0 +1,25 @@
+⍝ Quicksort — the classic Roger Hui one-liner
+⍝
+⍝ Q ← {1≥≢⍵:⍵ ⋄ (∇⍵⌿⍨⍵<p),(p=⍵)/⍵,∇⍵⌿⍨⍵>p←⍵⌷⍨?≢⍵}
+⍝
+⍝ Read right-to-left:
+⍝   ?≢⍵          : pick a random index in 1..length
+⍝   ⍵⌷⍨…         : take that element as pivot p
+⍝   ⍵>p          : boolean — elements greater than pivot
+⍝   ∇⍵⌿⍨…        : recursively sort the > partition
+⍝   (p=⍵)/⍵      : keep elements equal to pivot
+⍝   ⍵<p          : boolean — elements less than pivot
+⍝   ∇⍵⌿⍨…        : recursively sort the < partition
+⍝   ,            : catenate ⟨less⟩ ⟨equal⟩ ⟨greater⟩
+⍝   1≥≢⍵:⍵       : guard — base case for length ≤ 1
+⍝
+⍝ Stability: not stable on duplicates (but eq-class is preserved as a block).
+⍝ Worst case O(N²) on already-sorted input with deterministic pivot;
+⍝ randomized pivot selection gives expected O(N log N).
+⍝
+⍝ Examples:
+⍝   Q 3 1 4 1 5 9 2 6 5 3 5    →  1 1 2 3 3 4 5 5 5 6 9
+⍝   Q ⍳0                        →  ⍬   (empty)
+⍝   Q ,42                       →  42
+
+quicksort ← {1≥≢⍵:⍵ ⋄ p←⍵⌷⍨?≢⍵ ⋄ (∇⍵⌿⍨⍵<p),(p=⍵)/⍵,∇⍵⌿⍨⍵>p}

lib/apl/tests/scalar.sx

@@ -0,0 +1,369 @@
+; APL scalar primitives test suite
+; Requires: lib/apl/runtime.sx
+
+; ============================================================
+; Test framework
+; ============================================================
+
+(define apl-rt-count 0)
+(define apl-rt-pass 0)
+(define apl-rt-fails (list))
+
+; Element-wise list comparison (handles both List and ListRef)
+(define
+  lists-eq
+  (fn
+    (a b)
+    (if
+      (and (= (len a) 0) (= (len b) 0))
+      true
+      (if
+        (not (= (len a) (len b)))
+        false
+        (if
+          (not (= (first a) (first b)))
+          false
+          (lists-eq (rest a) (rest b)))))))
+
+(define
+  apl-rt-test
+  (fn
+    (name actual expected)
+    (begin
+      (set! apl-rt-count (+ apl-rt-count 1))
+      (if
+        (equal? actual expected)
+        (set! apl-rt-pass (+ apl-rt-pass 1))
+        (append! apl-rt-fails {:actual actual :expected expected :name name})))))
+
+; Test that a ravel equals a plain list (handles ListRef vs List)
+(define
+  ravel-test
+  (fn
+    (name arr expected-list)
+    (begin
+      (set! apl-rt-count (+ apl-rt-count 1))
+      (let
+        ((actual (get arr :ravel)))
+        (if
+          (lists-eq actual expected-list)
+          (set! apl-rt-pass (+ apl-rt-pass 1))
+          (append! apl-rt-fails {:actual actual :expected expected-list :name name}))))))
+
+; Test a scalar ravel value (single-element list)
+(define
+  scalar-test
+  (fn (name arr expected-val) (ravel-test name arr (list expected-val))))
+
+; ============================================================
+; Array constructor tests
+; ============================================================
+
+(apl-rt-test
+  "scalar: shape is empty list"
+  (get (apl-scalar 5) :shape)
+  (list))
+
+(apl-rt-test
+  "scalar: ravel has one element"
+  (get (apl-scalar 5) :ravel)
+  (list 5))
+
+(apl-rt-test "scalar: rank 0" (array-rank (apl-scalar 5)) 0)
+
+(apl-rt-test "scalar? returns true for scalar" (scalar? (apl-scalar 5)) true)
+
+(apl-rt-test "scalar: zero" (get (apl-scalar 0) :ravel) (list 0))
+
+(apl-rt-test
+  "vector: shape is (3)"
+  (get (apl-vector (list 1 2 3)) :shape)
+  (list 3))
+
+(apl-rt-test
+  "vector: ravel matches input"
+  (get (apl-vector (list 1 2 3)) :ravel)
+  (list 1 2 3))
+
+(apl-rt-test "vector: rank 1" (array-rank (apl-vector (list 1 2 3))) 1)
+
+(apl-rt-test
+  "scalar? returns false for vector"
+  (scalar? (apl-vector (list 1 2 3)))
+  false)
+
+(apl-rt-test
+  "make-array: rank 2"
+  (array-rank (make-array (list 2 3) (list 1 2 3 4 5 6)))
+  2)
+
+(apl-rt-test
+  "make-array: shape"
+  (get (make-array (list 2 3) (list 1 2 3 4 5 6)) :shape)
+  (list 2 3))
+
+(apl-rt-test
+  "array-ref: first element"
+  (array-ref (apl-vector (list 10 20 30)) 0)
+  10)
+
+(apl-rt-test
+  "array-ref: last element"
+  (array-ref (apl-vector (list 10 20 30)) 2)
+  30)
+
+(apl-rt-test "enclose: wraps in rank-0" (scalar? (enclose 42)) true)
+
+(apl-rt-test
+  "enclose: ravel contains value"
+  (get (enclose 42) :ravel)
+  (list 42))
+
+(apl-rt-test "disclose: unwraps rank-0" (disclose (enclose 42)) 42)
+
+; ============================================================
+; Shape primitive tests
+; ============================================================
+
+(ravel-test "⍴ scalar: returns empty" (apl-shape (apl-scalar 5)) (list))
+
+(ravel-test
+  "⍴ vector: returns (3)"
+  (apl-shape (apl-vector (list 1 2 3)))
+  (list 3))
+
+(ravel-test
+  "⍴ matrix: returns (2 3)"
+  (apl-shape (make-array (list 2 3) (list 1 2 3 4 5 6)))
+  (list 2 3))
+
+(ravel-test
+  ", ravel scalar: vector of 1"
+  (apl-ravel (apl-scalar 5))
+  (list 5))
+
+(apl-rt-test
+  ", ravel vector: same elements"
+  (get (apl-ravel (apl-vector (list 1 2 3))) :ravel)
+  (list 1 2 3))
+
+(apl-rt-test
+  ", ravel matrix: all elements"
+  (get (apl-ravel (make-array (list 2 3) (list 1 2 3 4 5 6))) :ravel)
+  (list 1 2 3 4 5 6))
+
+(scalar-test "≢ tally scalar: 1" (apl-tally (apl-scalar 5)) 1)
+
+(scalar-test
+  "≢ tally vector: first dimension"
+  (apl-tally (apl-vector (list 1 2 3)))
+  3)
+
+(scalar-test
+  "≢ tally matrix: first dimension"
+  (apl-tally (make-array (list 2 3) (list 1 2 3 4 5 6)))
+  2)
+
+(scalar-test
+  "≡ depth flat vector: 0"
+  (apl-depth (apl-vector (list 1 2 3)))
+  0)
+
+(scalar-test "≡ depth scalar: 0" (apl-depth (apl-scalar 5)) 0)
+
+(scalar-test
+  "≡ depth nested (enclose in vector): 1"
+  (apl-depth (enclose (apl-vector (list 1 2 3))))
+  1)
+
+; ============================================================
+; ⍳ iota tests
+; ============================================================
+
+(apl-rt-test
+  "⍳5 shape is (5)"
+  (get (apl-iota (apl-scalar 5)) :shape)
+  (list 5))
+
+(ravel-test "⍳5 ravel is 1..5" (apl-iota (apl-scalar 5)) (list 1 2 3 4 5))
+
+(ravel-test "⍳1 ravel is (1)" (apl-iota (apl-scalar 1)) (list 1))
+
+(ravel-test "⍳0 ravel is empty" (apl-iota (apl-scalar 0)) (list))
+
+(apl-rt-test "apl-io is 1" apl-io 1)
+
+; ============================================================
+; Arithmetic broadcast tests
+; ============================================================
+
+(scalar-test
+  "+ scalar scalar: 3+4=7"
+  (apl-add (apl-scalar 3) (apl-scalar 4))
+  7)
+
+(ravel-test
+  "+ vector scalar: +10"
+  (apl-add (apl-vector (list 1 2 3)) (apl-scalar 10))
+  (list 11 12 13))
+
+(ravel-test
+  "+ scalar vector: 10+"
+  (apl-add (apl-scalar 10) (apl-vector (list 1 2 3)))
+  (list 11 12 13))
+
+(ravel-test
+  "+ vector vector"
+  (apl-add (apl-vector (list 1 2 3)) (apl-vector (list 4 5 6)))
+  (list 5 7 9))
+
+(scalar-test "- negate monadic" (apl-neg-m (apl-scalar 5)) -5)
+
+(scalar-test "- dyadic 10-3=7" (apl-sub (apl-scalar 10) (apl-scalar 3)) 7)
+
+(scalar-test "× signum positive" (apl-signum (apl-scalar 7)) 1)
+
+(scalar-test "× signum negative" (apl-signum (apl-scalar -3)) -1)
+
+(scalar-test "× signum zero" (apl-signum (apl-scalar 0)) 0)
+
+(scalar-test "× dyadic 3×4=12" (apl-mul (apl-scalar 3) (apl-scalar 4)) 12)
+
+(scalar-test "÷ reciprocal 1÷4=0.25" (apl-recip (apl-scalar 4)) 0.25)
+
+(scalar-test
+  "÷ dyadic 10÷4=2.5"
+  (apl-div (apl-scalar 10) (apl-scalar 4))
+  2.5)
+
+(scalar-test "⌈ ceiling 2.3→3" (apl-ceil (apl-scalar 2.3)) 3)
+
+(scalar-test "⌈ max 3 5 → 5" (apl-max (apl-scalar 3) (apl-scalar 5)) 5)
+
+(scalar-test "⌊ floor 2.7→2" (apl-floor (apl-scalar 2.7)) 2)
+
+(scalar-test "⌊ min 3 5 → 3" (apl-min (apl-scalar 3) (apl-scalar 5)) 3)
+
+(scalar-test "* exp monadic e^0=1" (apl-exp (apl-scalar 0)) 1)
+
+(scalar-test
+  "* pow dyadic 2^10=1024"
+  (apl-pow (apl-scalar 2) (apl-scalar 10))
+  1024)
+
+(scalar-test "⍟ ln 1=0" (apl-ln (apl-scalar 1)) 0)
+
+(scalar-test "| abs positive" (apl-abs (apl-scalar 5)) 5)
+
+(scalar-test "| abs negative" (apl-abs (apl-scalar -5)) 5)
+
+(scalar-test "| mod 3|7=1" (apl-mod (apl-scalar 3) (apl-scalar 7)) 1)
+
+(scalar-test "! factorial 5!=120" (apl-fact (apl-scalar 5)) 120)
+
+(scalar-test "! factorial 0!=1" (apl-fact (apl-scalar 0)) 1)
+
+(scalar-test
+  "! binomial 4 choose 2 = 6"
+  (apl-binomial (apl-scalar 4) (apl-scalar 2))
+  6)
+
+(scalar-test "○ pi×0=0" (apl-pi-times (apl-scalar 0)) 0)
+
+(scalar-test "○ trig sin(0)=0" (apl-trig (apl-scalar 1) (apl-scalar 0)) 0)
+
+(scalar-test "○ trig cos(0)=1" (apl-trig (apl-scalar 2) (apl-scalar 0)) 1)
+
+; ============================================================
+; Comparison tests
+; ============================================================
+
+(scalar-test "< less: 3<5 → 1" (apl-lt (apl-scalar 3) (apl-scalar 5)) 1)
+
+(scalar-test "< less: 5<3 → 0" (apl-lt (apl-scalar 5) (apl-scalar 3)) 0)
+
+(scalar-test
+  "≤ le equal: 3≤3 → 1"
+  (apl-le (apl-scalar 3) (apl-scalar 3))
+  1)
+
+(scalar-test "= eq: 5=5 → 1" (apl-eq (apl-scalar 5) (apl-scalar 5)) 1)
+
+(scalar-test "= ne: 5=6 → 0" (apl-eq (apl-scalar 5) (apl-scalar 6)) 0)
+
+(scalar-test "≥ ge: 5≥3 → 1" (apl-ge (apl-scalar 5) (apl-scalar 3)) 1)
+
+(scalar-test "> gt: 5>3 → 1" (apl-gt (apl-scalar 5) (apl-scalar 3)) 1)
+
+(scalar-test "≠ ne: 5≠3 → 1" (apl-ne (apl-scalar 5) (apl-scalar 3)) 1)
+
+(ravel-test
+  "comparison vector broadcast: 1 2 3 < 2 → 1 0 0"
+  (apl-lt (apl-vector (list 1 2 3)) (apl-scalar 2))
+  (list 1 0 0))
+
+; ============================================================
+; Logical tests
+; ============================================================
+
+(scalar-test "~ not 0 → 1" (apl-not (apl-scalar 0)) 1)
+
+(scalar-test "~ not 1 → 0" (apl-not (apl-scalar 1)) 0)
+
+(ravel-test
+  "~ not vector: 1 0 1 0 → 0 1 0 1"
+  (apl-not (apl-vector (list 1 0 1 0)))
+  (list 0 1 0 1))
+
+(scalar-test
+  "∧ and 1∧1 → 1"
+  (apl-and (apl-scalar 1) (apl-scalar 1))
+  1)
+
+(scalar-test
+  "∧ and 1∧0 → 0"
+  (apl-and (apl-scalar 1) (apl-scalar 0))
+  0)
+
+(scalar-test "∨ or 0∨1 → 1" (apl-or (apl-scalar 0) (apl-scalar 1)) 1)
+
+(scalar-test "∨ or 0∨0 → 0" (apl-or (apl-scalar 0) (apl-scalar 0)) 0)
+
+(scalar-test
+  "⍱ nor 0⍱0 → 1"
+  (apl-nor (apl-scalar 0) (apl-scalar 0))
+  1)
+
+(scalar-test
+  "⍱ nor 1⍱0 → 0"
+  (apl-nor (apl-scalar 1) (apl-scalar 0))
+  0)
+
+(scalar-test
+  "⍲ nand 1⍲1 → 0"
+  (apl-nand (apl-scalar 1) (apl-scalar 1))
+  0)
+
+(scalar-test
+  "⍲ nand 1⍲0 → 1"
+  (apl-nand (apl-scalar 1) (apl-scalar 0))
+  1)
+
+; ============================================================
+; plus-m identity test
+; ============================================================
+
+(scalar-test "+ monadic identity: +5 → 5" (apl-plus-m (apl-scalar 5)) 5)
+
+; ============================================================
+; Summary
+; ============================================================
+
+(define
+  apl-scalar-summary
+  (str
+    "scalar "
+    apl-rt-pass
+    "/"
+    apl-rt-count
+    (if (= (len apl-rt-fails) 0) "" (str " FAILS: " apl-rt-fails))))

lib/apl/tests/structural.sx

@@ -0,0 +1,608 @@
+;; lib/apl/tests/structural.sx — Phase 3: structural primitives
+;; Tests for: apl-reshape, apl-ravel, apl-transpose, apl-transpose-dyadic
+;; Loaded after runtime.sx; shares apl-test / apl-test-pass / apl-test-fail.
+
+(define rv (fn (arr) (get arr :ravel)))
+(define sh (fn (arr) (get arr :shape)))
+
+;; ---------------------------------------------------------------------------
+;; 1. Ravel (monadic ,)
+;; ---------------------------------------------------------------------------
+(apl-test "ravel scalar" (rv (apl-ravel (apl-scalar 5))) (list 5))
+
+(apl-test
+  "ravel vector"
+  (rv (apl-ravel (make-array (list 3) (list 1 2 3))))
+  (list 1 2 3))
+
+(apl-test
+  "ravel matrix"
+  (rv (apl-ravel (make-array (list 2 3) (list 1 2 3 4 5 6))))
+  (list 1 2 3 4 5 6))
+
+(apl-test
+  "ravel shape is rank-1"
+  (sh (apl-ravel (make-array (list 2 3) (list 1 2 3 4 5 6))))
+  (list 6))
+
+;; ---------------------------------------------------------------------------
+;; 2. Reshape (dyadic ⍴)
+;; ---------------------------------------------------------------------------
+
+(apl-test
+  "reshape 2x3 ravel"
+  (rv
+    (apl-reshape
+      (make-array (list 2) (list 2 3))
+      (make-array (list 6) (list 1 2 3 4 5 6))))
+  (list 1 2 3 4 5 6))
+
+(apl-test
+  "reshape 2x3 shape"
+  (sh
+    (apl-reshape
+      (make-array (list 2) (list 2 3))
+      (make-array (list 6) (list 1 2 3 4 5 6))))
+  (list 2 3))
+
+(apl-test
+  "reshape cycle 6 from 1 2"
+  (rv
+    (apl-reshape
+      (make-array (list 1) (list 6))
+      (make-array (list 2) (list 1 2))))
+  (list 1 2 1 2 1 2))
+
+(apl-test
+  "reshape cycle 2x3 from 1 2"
+  (rv
+    (apl-reshape
+      (make-array (list 2) (list 2 3))
+      (make-array (list 2) (list 1 2))))
+  (list 1 2 1 2 1 2))
+
+(apl-test
+  "reshape scalar fill"
+  (rv (apl-reshape (make-array (list 1) (list 4)) (apl-scalar 7)))
+  (list 7 7 7 7))
+
+(apl-test
+  "reshape truncate"
+  (rv
+    (apl-reshape
+      (make-array (list 1) (list 3))
+      (make-array (list 6) (list 10 20 30 40 50 60))))
+  (list 10 20 30))
+
+(apl-test
+  "reshape matrix to vector"
+  (sh
+    (apl-reshape
+      (make-array (list 1) (list 6))
+      (make-array (list 2 3) (list 1 2 3 4 5 6))))
+  (list 6))
+
+(apl-test
+  "reshape 2x2x3"
+  (sh
+    (apl-reshape
+      (make-array (list 3) (list 2 2 3))
+      (make-array (list 12) (range 1 13))))
+  (list 2 2 3))
+
+(apl-test
+  "reshape to empty"
+  (rv
+    (apl-reshape
+      (make-array (list 1) (list 0))
+      (make-array (list 3) (list 1 2 3))))
+  (list))
+
+;; ---------------------------------------------------------------------------
+;; 3. Monadic transpose (⍉)
+;; ---------------------------------------------------------------------------
+
+(apl-test
+  "transpose scalar shape"
+  (sh (apl-transpose (apl-scalar 99)))
+  (list))
+
+(apl-test
+  "transpose scalar ravel"
+  (rv (apl-transpose (apl-scalar 99)))
+  (list 99))
+
+(apl-test
+  "transpose vector shape"
+  (sh (apl-transpose (make-array (list 3) (list 3 1 4))))
+  (list 3))
+
+(apl-test
+  "transpose vector ravel"
+  (rv (apl-transpose (make-array (list 3) (list 3 1 4))))
+  (list 3 1 4))
+
+(apl-test
+  "transpose 2x3 shape"
+  (sh (apl-transpose (make-array (list 2 3) (list 1 2 3 4 5 6))))
+  (list 3 2))
+
+(apl-test
+  "transpose 2x3 ravel"
+  (rv (apl-transpose (make-array (list 2 3) (list 1 2 3 4 5 6))))
+  (list 1 4 2 5 3 6))
+
+(apl-test
+  "transpose 3x3"
+  (rv (apl-transpose (make-array (list 3 3) (list 1 2 3 4 5 6 7 8 9))))
+  (list 1 4 7 2 5 8 3 6 9))
+
+(apl-test
+  "transpose 1x4 shape"
+  (sh (apl-transpose (make-array (list 1 4) (list 1 2 3 4))))
+  (list 4 1))
+
+(apl-test
+  "transpose twice identity"
+  (rv
+    (apl-transpose
+      (apl-transpose (make-array (list 2 3) (list 1 2 3 4 5 6)))))
+  (list 1 2 3 4 5 6))
+
+(apl-test
+  "transpose 3d shape"
+  (sh (apl-transpose (make-array (list 2 3 4) (range 0 24))))
+  (list 4 3 2))
+
+;; ---------------------------------------------------------------------------
+;; 4. Dyadic transpose (perm⍉arr)
+;; ---------------------------------------------------------------------------
+
+(apl-test
+  "dyadic-transpose identity"
+  (rv
+    (apl-transpose-dyadic
+      (make-array (list 2) (list 1 2))
+      (make-array (list 2 3) (list 1 2 3 4 5 6))))
+  (list 1 2 3 4 5 6))
+
+(apl-test
+  "dyadic-transpose swap 2x3"
+  (rv
+    (apl-transpose-dyadic
+      (make-array (list 2) (list 2 1))
+      (make-array (list 2 3) (list 1 2 3 4 5 6))))
+  (list 1 4 2 5 3 6))
+
+(apl-test
+  "dyadic-transpose swap shape"
+  (sh
+    (apl-transpose-dyadic
+      (make-array (list 2) (list 2 1))
+      (make-array (list 2 3) (list 1 2 3 4 5 6))))
+  (list 3 2))
+
+(apl-test
+  "dyadic-transpose 3d shape"
+  (sh
+    (apl-transpose-dyadic
+      (make-array (list 3) (list 2 1 3))
+      (make-array (list 2 3 4) (range 0 24))))
+  (list 3 2 4))
+
+(apl-test
+  "take 3 from front"
+  (rv (apl-take (apl-scalar 3) (make-array (list 5) (list 1 2 3 4 5))))
+  (list 1 2 3))
+
+(apl-test
+  "take 0"
+  (rv (apl-take (apl-scalar 0) (make-array (list 5) (list 1 2 3 4 5))))
+  (list))
+
+(apl-test
+  "take -2 from back"
+  (rv (apl-take (apl-scalar -2) (make-array (list 5) (list 1 2 3 4 5))))
+  (list 4 5))
+
+(apl-test
+  "take over-take pads with 0"
+  (rv (apl-take (apl-scalar 7) (make-array (list 5) (list 1 2 3 4 5))))
+  (list 1 2 3 4 5 0 0))
+
+(apl-test
+  "take matrix 1 row 2 cols shape"
+  (sh
+    (apl-take
+      (make-array (list 2) (list 1 2))
+      (make-array (list 2 3) (list 1 2 3 4 5 6))))
+  (list 1 2))
+
+(apl-test
+  "take matrix 1 row 2 cols ravel"
+  (rv
+    (apl-take
+      (make-array (list 2) (list 1 2))
+      (make-array (list 2 3) (list 1 2 3 4 5 6))))
+  (list 1 2))
+
+(apl-test
+  "take matrix negative row"
+  (rv
+    (apl-take
+      (make-array (list 2) (list -1 3))
+      (make-array (list 2 3) (list 1 2 3 4 5 6))))
+  (list 4 5 6))
+
+(apl-test
+  "drop 2 from front"
+  (rv (apl-drop (apl-scalar 2) (make-array (list 5) (list 1 2 3 4 5))))
+  (list 3 4 5))
+
+(apl-test
+  "drop -2 from back"
+  (rv (apl-drop (apl-scalar -2) (make-array (list 5) (list 1 2 3 4 5))))
+  (list 1 2 3))
+
+(apl-test
+  "drop all"
+  (rv (apl-drop (apl-scalar 5) (make-array (list 5) (list 1 2 3 4 5))))
+  (list))
+
+(apl-test
+  "drop 0"
+  (rv (apl-drop (apl-scalar 0) (make-array (list 5) (list 1 2 3 4 5))))
+  (list 1 2 3 4 5))
+
+(apl-test
+  "drop matrix 1 row shape"
+  (sh
+    (apl-drop
+      (make-array (list 2) (list 1 0))
+      (make-array (list 2 3) (list 1 2 3 4 5 6))))
+  (list 1 3))
+
+(apl-test
+  "drop matrix 1 row ravel"
+  (rv
+    (apl-drop
+      (make-array (list 2) (list 1 0))
+      (make-array (list 2 3) (list 1 2 3 4 5 6))))
+  (list 4 5 6))
+
+(apl-test
+  "reverse vector"
+  (rv (apl-reverse (make-array (list 5) (list 1 2 3 4 5))))
+  (list 5 4 3 2 1))
+
+(apl-test
+  "reverse scalar identity"
+  (rv (apl-reverse (apl-scalar 42)))
+  (list 42))
+
+(apl-test
+  "reverse matrix last axis"
+  (rv (apl-reverse (make-array (list 2 3) (list 1 2 3 4 5 6))))
+  (list 3 2 1 6 5 4))
+
+(apl-test
+  "reverse-first matrix"
+  (rv (apl-reverse-first (make-array (list 2 3) (list 1 2 3 4 5 6))))
+  (list 4 5 6 1 2 3))
+
+(apl-test
+  "reverse-first vector identity"
+  (rv (apl-reverse-first (make-array (list 4) (list 1 2 3 4))))
+  (list 4 3 2 1))
+
+(apl-test
+  "rotate vector left by 2"
+  (rv (apl-rotate (apl-scalar 2) (make-array (list 5) (list 1 2 3 4 5))))
+  (list 3 4 5 1 2))
+
+(apl-test
+  "rotate vector right by 1 (negative)"
+  (rv (apl-rotate (apl-scalar -1) (make-array (list 5) (list 1 2 3 4 5))))
+  (list 5 1 2 3 4))
+
+(apl-test
+  "rotate by 0 is identity"
+  (rv (apl-rotate (apl-scalar 0) (make-array (list 5) (list 1 2 3 4 5))))
+  (list 1 2 3 4 5))
+
+(apl-test
+  "rotate matrix last axis"
+  (rv
+    (apl-rotate (apl-scalar 1) (make-array (list 2 3) (list 1 2 3 4 5 6))))
+  (list 2 3 1 5 6 4))
+
+(apl-test
+  "rotate-first matrix"
+  (rv
+    (apl-rotate-first
+      (apl-scalar 1)
+      (make-array (list 2 3) (list 1 2 3 4 5 6))))
+  (list 4 5 6 1 2 3))
+
+(apl-test
+  "cat v,v ravel"
+  (rv
+    (apl-catenate
+      (make-array (list 3) (list 1 2 3))
+      (make-array (list 2) (list 4 5))))
+  (list 1 2 3 4 5))
+
+(apl-test
+  "cat v,v shape"
+  (sh
+    (apl-catenate
+      (make-array (list 3) (list 1 2 3))
+      (make-array (list 2) (list 4 5))))
+  (list 5))
+
+(apl-test
+  "cat scalar,v"
+  (rv (apl-catenate (apl-scalar 99) (make-array (list 3) (list 1 2 3))))
+  (list 99 1 2 3))
+
+(apl-test
+  "cat v,scalar"
+  (rv (apl-catenate (make-array (list 3) (list 1 2 3)) (apl-scalar 99)))
+  (list 1 2 3 99))
+
+(apl-test
+  "cat matrix last-axis shape"
+  (sh
+    (apl-catenate
+      (make-array (list 2 3) (list 1 2 3 4 5 6))
+      (make-array (list 2 2) (list 7 8 9 10))))
+  (list 2 5))
+
+(apl-test
+  "cat matrix last-axis ravel"
+  (rv
+    (apl-catenate
+      (make-array (list 2 3) (list 1 2 3 4 5 6))
+      (make-array (list 2 2) (list 7 8 9 10))))
+  (list 1 2 3 7 8 4 5 6 9 10))
+
+(apl-test
+  "cat-first v,v shape"
+  (sh
+    (apl-catenate-first
+      (make-array (list 3) (list 1 2 3))
+      (make-array (list 2) (list 4 5))))
+  (list 5))
+
+(apl-test
+  "cat-first matrix shape"
+  (sh
+    (apl-catenate-first
+      (make-array (list 2 3) (list 1 2 3 4 5 6))
+      (make-array (list 3 3) (list 11 12 13 14 15 16 17 18 19))))
+  (list 5 3))
+
+(apl-test
+  "cat-first matrix ravel"
+  (rv
+    (apl-catenate-first
+      (make-array (list 2 3) (list 1 2 3 4 5 6))
+      (make-array (list 3 3) (list 11 12 13 14 15 16 17 18 19))))
+  (list 1 2 3 4 5 6 11 12 13 14 15 16 17 18 19))
+
+(apl-test
+  "squad scalar into vector"
+  (rv
+    (apl-squad (apl-scalar 2) (make-array (list 5) (list 10 20 30 40 50))))
+  (list 20))
+
+(apl-test
+  "squad first element"
+  (rv (apl-squad (apl-scalar 1) (make-array (list 3) (list 10 20 30))))
+  (list 10))
+
+(apl-test
+  "squad last element"
+  (rv
+    (apl-squad (apl-scalar 5) (make-array (list 5) (list 10 20 30 40 50))))
+  (list 50))
+
+(apl-test
+  "squad fully specified matrix element"
+  (rv
+    (apl-squad
+      (make-array (list 2) (list 2 3))
+      (make-array (list 3 4) (list 1 2 3 4 5 6 7 8 9 10 11 12))))
+  (list 7))
+
+(apl-test
+  "squad partial row of matrix shape"
+  (sh
+    (apl-squad
+      (apl-scalar 2)
+      (make-array (list 3 4) (list 1 2 3 4 5 6 7 8 9 10 11 12))))
+  (list 4))
+
+(apl-test
+  "squad partial row of matrix ravel"
+  (rv
+    (apl-squad
+      (apl-scalar 2)
+      (make-array (list 3 4) (list 1 2 3 4 5 6 7 8 9 10 11 12))))
+  (list 5 6 7 8))
+
+(apl-test
+  "squad partial 3d slice shape"
+  (sh (apl-squad (apl-scalar 1) (make-array (list 2 3 4) (range 1 25))))
+  (list 3 4))
+
+(apl-test
+  "grade-up basic"
+  (rv (apl-grade-up (make-array (list 5) (list 3 1 4 1 5))))
+  (list 2 4 1 3 5))
+
+(apl-test
+  "grade-up shape"
+  (sh (apl-grade-up (make-array (list 4) (list 4 1 3 2))))
+  (list 4))
+
+(apl-test
+  "grade-up no duplicates"
+  (rv (apl-grade-up (make-array (list 4) (list 4 1 3 2))))
+  (list 2 4 3 1))
+
+(apl-test
+  "grade-up already sorted"
+  (rv (apl-grade-up (make-array (list 3) (list 1 2 3))))
+  (list 1 2 3))
+
+(apl-test
+  "grade-up reverse sorted"
+  (rv (apl-grade-up (make-array (list 3) (list 3 2 1))))
+  (list 3 2 1))
+
+(apl-test
+  "grade-down basic"
+  (rv (apl-grade-down (make-array (list 5) (list 3 1 4 1 5))))
+  (list 5 3 1 2 4))
+
+(apl-test
+  "grade-down no duplicates"
+  (rv (apl-grade-down (make-array (list 4) (list 4 1 3 2))))
+  (list 1 3 4 2))
+
+(apl-test
+  "grade-up single element"
+  (rv (apl-grade-up (make-array (list 1) (list 42))))
+  (list 1))
+
+(apl-test
+  "enclose shape is scalar"
+  (sh (apl-enclose (make-array (list 3) (list 1 2 3))))
+  (list))
+
+(apl-test
+  "enclose ravel length is 1"
+  (len (rv (apl-enclose (make-array (list 3) (list 1 2 3)))))
+  1)
+
+(apl-test
+  "enclose inner ravel"
+  (rv (first (rv (apl-enclose (make-array (list 3) (list 1 2 3))))))
+  (list 1 2 3))
+
+(apl-test
+  "disclose of enclose round-trips ravel"
+  (rv (apl-disclose (apl-enclose (make-array (list 3) (list 10 20 30)))))
+  (list 10 20 30))
+
+(apl-test
+  "disclose of enclose round-trips shape"
+  (sh (apl-disclose (apl-enclose (make-array (list 3) (list 10 20 30)))))
+  (list 3))
+
+(apl-test
+  "disclose scalar ravel"
+  (rv (apl-disclose (apl-scalar 42)))
+  (list 42))
+
+(apl-test
+  "disclose vector ravel"
+  (rv (apl-disclose (make-array (list 3) (list 5 6 7))))
+  (list 5))
+
+(apl-test
+  "disclose matrix returns first row"
+  (rv (apl-disclose (make-array (list 2 3) (list 1 2 3 4 5 6))))
+  (list 1 2 3))
+
+(apl-test
+  "member basic"
+  (rv
+    (apl-member
+      (make-array (list 3) (list 1 2 3))
+      (make-array (list 2) (list 2 3))))
+  (list 0 1 1))
+
+(apl-test
+  "member all absent"
+  (rv
+    (apl-member
+      (make-array (list 3) (list 4 5 6))
+      (make-array (list 3) (list 1 2 3))))
+  (list 0 0 0))
+
+(apl-test
+  "member scalar"
+  (rv (apl-member (apl-scalar 5) (make-array (list 3) (list 1 5 9))))
+  (list 1))
+
+(apl-test
+  "member shape preserved"
+  (sh
+    (apl-member
+      (make-array (list 2 3) (list 1 2 3 4 5 6))
+      (make-array (list 3) (list 1 3 5))))
+  (list 2 3))
+
+(apl-test
+  "member matrix ravel"
+  (rv
+    (apl-member
+      (make-array (list 2 3) (list 1 2 3 4 5 6))
+      (make-array (list 3) (list 1 3 5))))
+  (list 1 0 1 0 1 0))
+
+(apl-test
+  "index-of basic"
+  (rv
+    (apl-index-of
+      (make-array (list 4) (list 10 20 30 40))
+      (make-array (list 3) (list 20 40 10))))
+  (list 2 4 1))
+
+(apl-test
+  "index-of not-found"
+  (rv
+    (apl-index-of
+      (make-array (list 3) (list 1 2 3))
+      (make-array (list 2) (list 5 2))))
+  (list 4 2))
+
+(apl-test
+  "index-of scalar right"
+  (rv
+    (apl-index-of (make-array (list 3) (list 10 20 30)) (apl-scalar 20)))
+  (list 2))
+
+(apl-test
+  "without basic"
+  (rv
+    (apl-without
+      (make-array (list 5) (list 1 2 3 4 5))
+      (make-array (list 2) (list 2 4))))
+  (list 1 3 5))
+
+(apl-test
+  "without shape"
+  (sh
+    (apl-without
+      (make-array (list 5) (list 1 2 3 4 5))
+      (make-array (list 2) (list 2 4))))
+  (list 3))
+
+(apl-test
+  "without nothing removed"
+  (rv
+    (apl-without
+      (make-array (list 3) (list 1 2 3))
+      (make-array (list 3) (list 4 5 6))))
+  (list 1 2 3))
+
+(apl-test
+  "without all removed"
+  (rv
+    (apl-without
+      (make-array (list 3) (list 1 2 3))
+      (make-array (list 3) (list 1 2 3))))
+  (list))

lib/apl/tests/system.sx

@@ -0,0 +1,48 @@
+; Tests for APL ⎕ system functions.
+
+(define mkrv (fn (arr) (get arr :ravel)))
+(define mksh (fn (arr) (get arr :shape)))
+
+(apl-test "⎕IO returns 1" (mkrv (apl-quad-io)) (list 1))
+
+(apl-test "⎕ML returns 1" (mkrv (apl-quad-ml)) (list 1))
+
+(apl-test "⎕FR returns 1248" (mkrv (apl-quad-fr)) (list 1248))
+
+(apl-test "⎕TS shape is 7" (mksh (apl-quad-ts)) (list 7))
+
+(apl-test "⎕TS year is 1970 default" (first (mkrv (apl-quad-ts))) 1970)
+
+(apl-test "⎕FMT scalar 42" (apl-quad-fmt (apl-scalar 42)) "42")
+
+(apl-test "⎕FMT scalar negative" (apl-quad-fmt (apl-scalar -7)) "-7")
+
+(apl-test
+  "⎕FMT empty vector"
+  (apl-quad-fmt (make-array (list 0) (list)))
+  "")
+
+(apl-test
+  "⎕FMT singleton vector"
+  (apl-quad-fmt (make-array (list 1) (list 42)))
+  "42")
+
+(apl-test
+  "⎕FMT vector"
+  (apl-quad-fmt (make-array (list 5) (list 1 2 3 4 5)))
+  "1 2 3 4 5")
+
+(apl-test
+  "⎕FMT matrix 2x3"
+  (apl-quad-fmt (make-array (list 2 3) (list 1 2 3 4 5 6)))
+  "1 2 3\n4 5 6\n")
+
+(apl-test
+  "⎕← (print) returns its arg"
+  (mkrv (apl-quad-print (apl-scalar 99)))
+  (list 99))
+
+(apl-test
+  "⎕← preserves shape"
+  (mksh (apl-quad-print (make-array (list 3) (list 1 2 3))))
+  (list 3))

lib/apl/tests/tradfn.sx

@@ -0,0 +1,156 @@
+; Tests for apl-call-tradfn (manual structure construction).
+
+(define mkrv (fn (arr) (get arr :ravel)))
+(define mksh (fn (arr) (get arr :shape)))
+(define mknum (fn (n) (list :num n)))
+(define mknm (fn (s) (list :name s)))
+(define mkfg (fn (g) (list :fn-glyph g)))
+(define mkmon (fn (g a) (list :monad (mkfg g) a)))
+(define mkdyd (fn (g l r) (list :dyad (mkfg g) l r)))
+(define mkasg (fn (n e) (list :assign n e)))
+(define mkbr (fn (e) (list :branch e)))
+
+(define mkif (fn (c t e) (list :if c t e)))
+
+(define mkwhile (fn (c b) (list :while c b)))
+
+(define mkfor (fn (v i b) (list :for v i b)))
+
+(define mksel (fn (v cs d) (list :select v cs d)))
+
+(define mktrap (fn (codes t c) (list :trap codes t c)))
+
+(define mkthr (fn (code msg) (list :throw code msg)))
+
+(apl-test
+  "tradfn R←L+W simple add"
+  (mkrv (apl-call-tradfn {:result "R" :omega "W" :stmts (list (mkasg "R" (mkdyd "+" (mknm "L") (mknm "W")))) :alpha "L"} (apl-scalar 5) (apl-scalar 7)))
+  (list 12))
+
+(apl-test
+  "tradfn R←L×W"
+  (mkrv (apl-call-tradfn {:result "R" :omega "W" :stmts (list (mkasg "R" (mkdyd "×" (mknm "L") (mknm "W")))) :alpha "L"} (apl-scalar 6) (apl-scalar 7)))
+  (list 42))
+
+(apl-test
+  "tradfn monadic R←-W"
+  (mkrv (apl-call-tradfn {:result "R" :omega "W" :stmts (list (mkasg "R" (mkmon "-" (mknm "W")))) :alpha nil} nil (apl-scalar 9)))
+  (list -9))
+
+(apl-test
+  "tradfn →0 exits early"
+  (mkrv (apl-call-tradfn {:result "R" :omega "W" :stmts (list (mkasg "R" (mknm "W")) (mkbr (mknum 0)) (mkasg "R" (mknum 999))) :alpha nil} nil (apl-scalar 7)))
+  (list 7))
+
+(apl-test
+  "tradfn branch to line 3 skips line 2"
+  (mkrv (apl-call-tradfn {:result "R" :omega "W" :stmts (list (mkbr (mknum 3)) (mkasg "R" (mknum 999)) (mkasg "R" (mknum 42))) :alpha nil} nil (apl-scalar 0)))
+  (list 42))
+
+(apl-test
+  "tradfn local var t←W+1; R←t×2"
+  (mkrv (apl-call-tradfn {:result "R" :omega "W" :stmts (list (mkasg "t" (mkdyd "+" (mknm "W") (mknum 1))) (mkasg "R" (mkdyd "×" (mknm "t") (mknum 2)))) :alpha nil} nil (apl-scalar 5)))
+  (list 12))
+
+(apl-test
+  "tradfn vector args"
+  (mkrv
+    (apl-call-tradfn
+      {:result "R" :omega "W" :stmts (list (mkasg "R" (mkdyd "+" (mknm "L") (mknm "W")))) :alpha "L"}
+      (make-array (list 3) (list 1 2 3))
+      (make-array (list 3) (list 10 20 30))))
+  (list 11 22 33))
+
+(apl-test
+  "tradfn unset result returns nil"
+  (apl-call-tradfn {:result "R" :omega "W" :stmts (list (mkbr (mknum 0))) :alpha nil} nil (apl-scalar 5))
+  nil)
+
+(apl-test
+  "tradfn run-off end returns result"
+  (mkrv (apl-call-tradfn {:result "R" :omega "W" :stmts (list (mkasg "R" (mkdyd "×" (mknm "W") (mknum 3)))) :alpha nil} nil (apl-scalar 7)))
+  (list 21))
+
+(apl-test
+  "tradfn loop sum 1+2+...+5 via branch"
+  (mkrv (apl-call-tradfn {:result "R" :omega "W" :stmts (list (mkasg "i" (mknum 1)) (mkasg "R" (mknum 0)) (mkasg "R" (mkdyd "+" (mknm "R") (mknm "i"))) (mkasg "i" (mkdyd "+" (mknm "i") (mknum 1))) (mkbr (mkdyd "×" (mkdyd "≤" (mknm "i") (mknm "W")) (mknum 3)))) :alpha nil} nil (apl-scalar 5)))
+  (list 15))
+
+(apl-test
+  "tradfn :If true branch"
+  (mkrv (apl-call-tradfn {:result "R" :omega "W" :stmts (list (mkif (mkdyd ">" (mknm "W") (mknum 0)) (list (mkasg "R" (mknum 1))) (list (mkasg "R" (mknum 0))))) :alpha nil} nil (apl-scalar 5)))
+  (list 1))
+
+(apl-test
+  "tradfn :If false branch"
+  (mkrv (apl-call-tradfn {:result "R" :omega "W" :stmts (list (mkif (mkdyd ">" (mknm "W") (mknum 100)) (list (mkasg "R" (mknum 1))) (list (mkasg "R" (mknum 0))))) :alpha nil} nil (apl-scalar 5)))
+  (list 0))
+
+(apl-test
+  "tradfn :While sum 1..N"
+  (mkrv (apl-call-tradfn {:result "R" :omega "W" :stmts (list (mkasg "i" (mknum 1)) (mkasg "R" (mknum 0)) (mkwhile (mkdyd "≤" (mknm "i") (mknm "W")) (list (mkasg "R" (mkdyd "+" (mknm "R") (mknm "i"))) (mkasg "i" (mkdyd "+" (mknm "i") (mknum 1)))))) :alpha nil} nil (apl-scalar 10)))
+  (list 55))
+
+(apl-test
+  "tradfn :For sum elements"
+  (mkrv
+    (apl-call-tradfn
+      {:result "R" :omega "W" :stmts (list (mkasg "R" (mknum 0)) (mkfor "x" (mknm "W") (list (mkasg "R" (mkdyd "+" (mknm "R") (mknm "x")))))) :alpha nil}
+      nil
+      (make-array (list 4) (list 10 20 30 40))))
+  (list 100))
+
+(apl-test
+  "tradfn :For with empty vector"
+  (mkrv (apl-call-tradfn {:result "R" :omega "W" :stmts (list (mkasg "R" (mknum 99)) (mkfor "x" (mknm "W") (list (mkasg "R" (mkdyd "+" (mknm "R") (mknm "x")))))) :alpha nil} nil (make-array (list 0) (list))))
+  (list 99))
+
+(apl-test
+  "tradfn :Select dispatch hit"
+  (mkrv (apl-call-tradfn {:result "R" :omega "W" :stmts (list (mksel (mknm "W") (list (list (mknum 1) (mkasg "R" (mknum 100))) (list (mknum 2) (mkasg "R" (mknum 200))) (list (mknum 3) (mkasg "R" (mknum 300)))) (list (mkasg "R" (mknum 0))))) :alpha nil} nil (apl-scalar 2)))
+  (list 200))
+
+(apl-test
+  "tradfn :Select default block"
+  (mkrv (apl-call-tradfn {:result "R" :omega "W" :stmts (list (mksel (mknm "W") (list (list (mknum 1) (mkasg "R" (mknum 100))) (list (mknum 2) (mkasg "R" (mknum 200)))) (list (mkasg "R" (mknum -1))))) :alpha nil} nil (apl-scalar 99)))
+  (list -1))
+
+(apl-test
+  "tradfn nested :If"
+  (mkrv (apl-call-tradfn {:result "R" :omega "W" :stmts (list (mkif (mkdyd ">" (mknm "W") (mknum 0)) (list (mkif (mkdyd ">" (mknm "W") (mknum 10)) (list (mkasg "R" (mknum 2))) (list (mkasg "R" (mknum 1))))) (list (mkasg "R" (mknum 0))))) :alpha nil} nil (apl-scalar 5)))
+  (list 1))
+
+(apl-test
+  "tradfn :If assigns persist outside"
+  (mkrv (apl-call-tradfn {:result "R" :omega "W" :stmts (list (mkasg "R" (mknum 0)) (mkif (mkdyd ">" (mknm "W") (mknum 0)) (list (mkasg "R" (mknum 42))) (list)) (mkasg "R" (mkdyd "+" (mknm "R") (mknum 1)))) :alpha nil} nil (apl-scalar 5)))
+  (list 43))
+
+(apl-test
+  "tradfn :For factorial 1..5"
+  (mkrv (apl-call-tradfn {:result "R" :omega "W" :stmts (list (mkasg "R" (mknum 1)) (mkfor "x" (mkmon "⍳" (mknm "W")) (list (mkasg "R" (mkdyd "×" (mknm "R") (mknm "x")))))) :alpha nil} nil (apl-scalar 5)))
+  (list 120))
+
+(apl-test
+  "tradfn :Trap normal flow (no error)"
+  (mkrv (apl-call-tradfn {:result "R" :omega nil :stmts (list (mktrap (list 0) (list (mkasg "R" (mknum 99))) (list (mkasg "R" (mknum -1))))) :alpha nil} nil nil))
+  (list 99))
+
+(apl-test
+  "tradfn :Trap catches matching code"
+  (mkrv (apl-call-tradfn {:result "R" :omega nil :stmts (list (mktrap (list 5) (list (mkthr 5 "boom")) (list (mkasg "R" (mknum 42))))) :alpha nil} nil nil))
+  (list 42))
+
+(apl-test
+  "tradfn :Trap catch-all (code 0)"
+  (mkrv (apl-call-tradfn {:result "R" :omega nil :stmts (list (mktrap (list 0) (list (mkthr 99 "any")) (list (mkasg "R" (mknum 1))))) :alpha nil} nil nil))
+  (list 1))
+
+(apl-test
+  "tradfn :Trap catches one of many codes"
+  (mkrv (apl-call-tradfn {:result "R" :omega nil :stmts (list (mktrap (list 1 2 3) (list (mkthr 2 "two")) (list (mkasg "R" (mknum 22))))) :alpha nil} nil nil))
+  (list 22))
+
+(apl-test
+  "tradfn :Trap continues to next stmt after catch"
+  (mkrv (apl-call-tradfn {:result "R" :omega nil :stmts (list (mktrap (list 7) (list (mkthr 7 "c")) (list (mkasg "R" (mknum 10)))) (mkasg "R" (mkdyd "+" (mknm "R") (mknum 5)))) :alpha nil} nil nil))
+  (list 15))

lib/apl/tests/valence.sx

@@ -0,0 +1,81 @@
+; Tests for valence detection (apl-dfn-valence, apl-tradfn-valence)
+; and unified dispatch (apl-call).
+
+(define mkrv (fn (arr) (get arr :ravel)))
+(define mknum (fn (n) (list :num n)))
+(define mknm (fn (s) (list :name s)))
+(define mkfg (fn (g) (list :fn-glyph g)))
+(define mkmon (fn (g a) (list :monad (mkfg g) a)))
+(define mkdyd (fn (g l r) (list :dyad (mkfg g) l r)))
+(define mkasg (fn (n e) (list :assign n e)))
+(define mkdfn (fn (stmts) (cons :dfn stmts)))
+
+(apl-test
+  "dfn-valence niladic body=42"
+  (apl-dfn-valence (mkdfn (list (mknum 42))))
+  :niladic)
+
+(apl-test
+  "dfn-valence monadic body=⍵+1"
+  (apl-dfn-valence (mkdfn (list (mkdyd "+" (mknm "⍵") (mknum 1)))))
+  :monadic)
+
+(apl-test
+  "dfn-valence dyadic body=⍺+⍵"
+  (apl-dfn-valence (mkdfn (list (mkdyd "+" (mknm "⍺") (mknm "⍵")))))
+  :dyadic)
+
+(apl-test
+  "dfn-valence dyadic mentions ⍺ via local"
+  (apl-dfn-valence (mkdfn (list (mkasg "x" (mknm "⍺")) (mknm "x"))))
+  :dyadic)
+
+(apl-test
+  "dfn-valence dyadic deep nest"
+  (apl-dfn-valence
+    (mkdfn (list (mkmon "-" (mkdyd "×" (mknm "⍺") (mknm "⍵"))))))
+  :dyadic)
+
+(apl-test "tradfn-valence niladic" (apl-tradfn-valence {:result "R" :omega nil :stmts (list) :alpha nil}) :niladic)
+
+(apl-test "tradfn-valence monadic" (apl-tradfn-valence {:result "R" :omega "W" :stmts (list) :alpha nil}) :monadic)
+
+(apl-test "tradfn-valence dyadic" (apl-tradfn-valence {:result "R" :omega "W" :stmts (list) :alpha "L"}) :dyadic)
+
+(apl-test
+  "apl-call dfn niladic"
+  (mkrv (apl-call (mkdfn (list (mknum 42))) nil nil))
+  (list 42))
+
+(apl-test
+  "apl-call dfn monadic"
+  (mkrv
+    (apl-call
+      (mkdfn (list (mkdyd "+" (mknm "⍵") (mknum 1))))
+      nil
+      (apl-scalar 5)))
+  (list 6))
+
+(apl-test
+  "apl-call dfn dyadic"
+  (mkrv
+    (apl-call
+      (mkdfn (list (mkdyd "+" (mknm "⍺") (mknm "⍵"))))
+      (apl-scalar 3)
+      (apl-scalar 4)))
+  (list 7))
+
+(apl-test
+  "apl-call tradfn dyadic"
+  (mkrv (apl-call {:result "R" :omega "W" :stmts (list (mkasg "R" (mkdyd "×" (mknm "L") (mknm "W")))) :alpha "L"} (apl-scalar 6) (apl-scalar 7)))
+  (list 42))
+
+(apl-test
+  "apl-call tradfn monadic"
+  (mkrv (apl-call {:result "R" :omega "W" :stmts (list (mkasg "R" (mkmon "-" (mknm "W")))) :alpha nil} nil (apl-scalar 9)))
+  (list -9))
+
+(apl-test
+  "apl-call tradfn niladic returns nil result"
+  (apl-call {:result "R" :omega nil :stmts (list) :alpha nil} nil nil)
+  nil)

lib/apl/tokenizer.sx

@@ -0,0 +1,180 @@
+(define apl-glyph-set
+  (list "+" "-" "×" "÷" "*" "⍟" "⌈" "⌊" "|" "!" "?" "○" "~" "<" "≤" "=" "≥" ">" "≠"
+        "≢" "≡" "∊" "∧" "∨" "⍱" "⍲" "," "⍪" "⍴" "⌽" "⊖" "⍉" "↑" "↓" "⊂" "⊃" "⊆"
+        "∪" "∩" "⍳" "⍸" "⌷" "⍋" "⍒" "⊥" "⊤" "⊣" "⊢" "⍎" "⍕"
+        "⍺" "⍵" "∇" "/" "⌿" "\\" "⍀" "¨" "⍨" "∘" "." "⍣" "⍤" "⍥" "@" "¯"))
+
+(define apl-glyph?
+  (fn (ch)
+    (some (fn (g) (= g ch)) apl-glyph-set)))
+
+(define apl-digit?
+  (fn (ch)
+    (and (string? ch) (>= ch "0") (<= ch "9"))))
+
+(define apl-alpha?
+  (fn (ch)
+    (and (string? ch)
+         (or (and (>= ch "a") (<= ch "z"))
+             (and (>= ch "A") (<= ch "Z"))
+             (= ch "_")))))
+
+(define apl-tokenize
+  (fn (source)
+    (let ((pos 0)
+          (src-len (len source))
+          (tokens (list)))
+
+      (define tok-push!
+        (fn (type value)
+          (append! tokens {:type type :value value})))
+
+      (define cur-sw?
+        (fn (ch)
+          (and (< pos src-len) (starts-with? (slice source pos) ch))))
+
+      (define cur-byte
+        (fn ()
+          (if (< pos src-len) (nth source pos) nil)))
+
+      (define advance!
+        (fn ()
+          (set! pos (+ pos 1))))
+
+      (define consume!
+        (fn (ch)
+          (set! pos (+ pos (len ch)))))
+
+      (define find-glyph
+        (fn ()
+          (let ((rem (slice source pos)))
+            (let ((matches (filter (fn (g) (starts-with? rem g)) apl-glyph-set)))
+              (if (> (len matches) 0) (first matches) nil)))))
+
+      (define read-digits!
+        (fn (acc)
+          (if (and (< pos src-len) (apl-digit? (cur-byte)))
+            (let ((ch (cur-byte)))
+              (begin
+                (advance!)
+                (read-digits! (str acc ch))))
+            acc)))
+
+      (define read-ident-cont!
+        (fn ()
+          (when (and (< pos src-len)
+                     (let ((ch (cur-byte)))
+                       (or (apl-alpha? ch) (apl-digit? ch))))
+            (begin
+              (advance!)
+              (read-ident-cont!)))))
+
+      (define read-string!
+        (fn (acc)
+          (cond
+            ((>= pos src-len) acc)
+            ((cur-sw? "'")
+             (if (and (< (+ pos 1) src-len) (cur-sw? "'"))
+               (begin
+                 (advance!)
+                 (advance!)
+                 (read-string! (str acc "'")))
+               (begin (advance!) acc)))
+            (true
+             (let ((ch (cur-byte)))
+               (begin
+                 (advance!)
+                 (read-string! (str acc ch))))))))
+
+      (define skip-line!
+        (fn ()
+          (when (and (< pos src-len) (not (cur-sw? "\n")))
+            (begin
+              (advance!)
+              (skip-line!)))))
+
+      (define scan!
+        (fn ()
+          (when (< pos src-len)
+            (let ((ch (cur-byte)))
+              (cond
+                ((or (= ch " ") (= ch "\t") (= ch "\r"))
+                 (begin (advance!) (scan!)))
+                ((= ch "\n")
+                 (begin (advance!) (tok-push! :newline nil) (scan!)))
+                ((cur-sw? "⍝")
+                 (begin (skip-line!) (scan!)))
+                ((cur-sw? "⋄")
+                 (begin (consume! "⋄") (tok-push! :diamond nil) (scan!)))
+                ((= ch "(")
+                 (begin (advance!) (tok-push! :lparen nil) (scan!)))
+                ((= ch ")")
+                 (begin (advance!) (tok-push! :rparen nil) (scan!)))
+                ((= ch "[")
+                 (begin (advance!) (tok-push! :lbracket nil) (scan!)))
+                ((= ch "]")
+                 (begin (advance!) (tok-push! :rbracket nil) (scan!)))
+                ((= ch "{")
+                 (begin (advance!) (tok-push! :lbrace nil) (scan!)))
+                ((= ch "}")
+                 (begin (advance!) (tok-push! :rbrace nil) (scan!)))
+                ((= ch ";")
+                 (begin (advance!) (tok-push! :semi nil) (scan!)))
+                ((cur-sw? "←")
+                 (begin (consume! "←") (tok-push! :assign nil) (scan!)))
+                ((= ch ":")
+                 (let ((start pos))
+                   (begin
+                     (advance!)
+                     (if (and (< pos src-len) (apl-alpha? (cur-byte)))
+                       (begin
+                         (read-ident-cont!)
+                         (tok-push! :keyword (slice source start pos)))
+                       (tok-push! :colon nil))
+                     (scan!))))
+                ((and (cur-sw? "¯")
+                      (< (+ pos (len "¯")) src-len)
+                      (apl-digit? (nth source (+ pos (len "¯")))))
+                 (begin
+                   (consume! "¯")
+                   (let ((digits (read-digits! "")))
+                     (if (and (< pos src-len) (= (cur-byte) ".")
+                              (< (+ pos 1) src-len) (apl-digit? (nth source (+ pos 1))))
+                       (begin (advance!)
+                              (let ((frac (read-digits! "")))
+                                (tok-push! :num (- 0 (string->number (str digits "." frac))))))
+                       (tok-push! :num (- 0 (parse-int digits 0)))))
+                   (scan!)))
+                ((apl-digit? ch)
+                 (begin
+                   (let ((digits (read-digits! "")))
+                     (if (and (< pos src-len) (= (cur-byte) ".")
+                              (< (+ pos 1) src-len) (apl-digit? (nth source (+ pos 1))))
+                       (begin (advance!)
+                              (let ((frac (read-digits! "")))
+                                (tok-push! :num (string->number (str digits "." frac)))))
+                       (tok-push! :num (parse-int digits 0))))
+                   (scan!)))
+                ((= ch "'")
+                 (begin
+                   (advance!)
+                   (let ((s (read-string! "")))
+                     (tok-push! :str s))
+                   (scan!)))
+                ((or (apl-alpha? ch) (cur-sw? "⎕"))
+                 (let ((start pos))
+                   (begin
+                     (if (cur-sw? "⎕") (consume! "⎕") (advance!))
+                     (if (and (< pos src-len) (cur-sw? "←"))
+                       (consume! "←")
+                       (read-ident-cont!))
+                     (tok-push! :name (slice source start pos))
+                     (scan!))))
+                (true
+                 (let ((g (find-glyph)))
+                   (if g
+                     (begin (consume! g) (tok-push! :glyph g) (scan!))
+                     (begin (advance!) (scan!))))))))))
+
+      (scan!)
+      tokens)))

lib/apl/transpile.sx

@@ -0,0 +1,540 @@
+; APL transpile / AST evaluator
+;
+; Walks parsed AST nodes and evaluates against the runtime.
+; Entry points:
+;   apl-eval-ast    : node × env → value
+;   apl-eval-stmts  : stmt-list × env → value (handles guards, locals, ⍺← default)
+;   apl-call-dfn    : dfn-ast × ⍺ × ⍵ → value   (dyadic)
+;   apl-call-dfn-m  : dfn-ast × ⍵ → value       (monadic)
+;
+; Env is a dict; ⍺ stored under "alpha", ⍵ under "omega",
+; the dfn-ast itself under "nabla" (for ∇ recursion),
+; user names under their literal name.
+
+(define
+  apl-monadic-fn
+  (fn
+    (g)
+    (cond
+      ((= g "+") apl-plus-m)
+      ((= g "-") apl-neg-m)
+      ((= g "×") apl-signum)
+      ((= g "÷") apl-recip)
+      ((= g "⌈") apl-ceil)
+      ((= g "⌊") apl-floor)
+      ((= g "⍳") apl-iota)
+      ((= g "|") apl-abs)
+      ((= g "*") apl-exp)
+      ((= g "⍟") apl-ln)
+      ((= g "!") apl-fact)
+      ((= g "○") apl-pi-times)
+      ((= g "~") apl-not)
+      ((= g "≢") apl-tally)
+      ((= g "⍴") apl-shape)
+      ((= g "≡") apl-depth)
+      ((= g "⊂") apl-enclose)
+      ((= g "⊃") apl-disclose)
+      ((= g ",") apl-ravel)
+      ((= g "⌽") apl-reverse)
+      ((= g "⊖") apl-reverse-first)
+      ((= g "⍋") apl-grade-up)
+      ((= g "⍒") apl-grade-down)
+      ((= g "⎕FMT") apl-quad-fmt)
+      ((= g "⎕←") apl-quad-print)
+      (else (error "no monadic fn for glyph")))))
+
+(define
+  apl-dyadic-fn
+  (fn
+    (g)
+    (cond
+      ((= g "+") apl-add)
+      ((= g "-") apl-sub)
+      ((= g "×") apl-mul)
+      ((= g "÷") apl-div)
+      ((= g "⌈") apl-max)
+      ((= g "⌊") apl-min)
+      ((= g "*") apl-pow)
+      ((= g "⍟") apl-log)
+      ((= g "|") apl-mod)
+      ((= g "!") apl-binomial)
+      ((= g "○") apl-trig)
+      ((= g "<") apl-lt)
+      ((= g "≤") apl-le)
+      ((= g "=") apl-eq)
+      ((= g "≥") apl-ge)
+      ((= g ">") apl-gt)
+      ((= g "≠") apl-ne)
+      ((= g "∧") apl-and)
+      ((= g "∨") apl-or)
+      ((= g "⍱") apl-nor)
+      ((= g "⍲") apl-nand)
+      ((= g ",") apl-catenate)
+      ((= g "⍪") apl-catenate-first)
+      ((= g "⍴") apl-reshape)
+      ((= g "↑") apl-take)
+      ((= g "↓") apl-drop)
+      ((= g "⌷") apl-squad)
+      ((= g "⌽") apl-rotate)
+      ((= g "⊖") apl-rotate-first)
+      ((= g "∊") apl-member)
+      ((= g "⍳") apl-index-of)
+      ((= g "~") apl-without)
+      (else (error "no dyadic fn for glyph")))))
+
+(define
+  apl-truthy?
+  (fn
+    (v)
+    (let
+      ((rv (get v :ravel)))
+      (if (and (= (len rv) 1) (= (first rv) 0)) false true))))
+
+(define
+  apl-eval-ast
+  (fn
+    (node env)
+    (let
+      ((tag (first node)))
+      (cond
+        ((= tag :num) (apl-scalar (nth node 1)))
+        ((= tag :str)
+          (let
+            ((s (nth node 1)))
+            (if
+              (= (len s) 1)
+              (apl-scalar s)
+              (make-array
+                (list (len s))
+                (map (fn (i) (slice s i (+ i 1))) (range 0 (len s)))))))
+        ((= tag :vec)
+          (let
+            ((items (rest node)))
+            (let
+              ((vals (map (fn (n) (apl-eval-ast n env)) items)))
+              (make-array
+                (list (len vals))
+                (map (fn (v) (first (get v :ravel))) vals)))))
+        ((= tag :name)
+          (let
+            ((nm (nth node 1)))
+            (cond
+              ((= nm "⍺") (get env "alpha"))
+              ((= nm "⍵") (get env "omega"))
+              ((= nm "⎕IO") (apl-quad-io))
+              ((= nm "⎕ML") (apl-quad-ml))
+              ((= nm "⎕FR") (apl-quad-fr))
+              ((= nm "⎕TS") (apl-quad-ts))
+              (else (get env nm)))))
+        ((= tag :monad)
+          (let
+            ((fn-node (nth node 1)) (arg (nth node 2)))
+            (if
+              (and (= (first fn-node) :fn-glyph) (= (nth fn-node 1) "∇"))
+              (apl-call-dfn-m (get env "nabla") (apl-eval-ast arg env))
+              ((apl-resolve-monadic fn-node env) (apl-eval-ast arg env)))))
+        ((= tag :dyad)
+          (let
+            ((fn-node (nth node 1))
+              (lhs (nth node 2))
+              (rhs (nth node 3)))
+            (if
+              (and (= (first fn-node) :fn-glyph) (= (nth fn-node 1) "∇"))
+              (apl-call-dfn
+                (get env "nabla")
+                (apl-eval-ast lhs env)
+                (apl-eval-ast rhs env))
+              ((apl-resolve-dyadic fn-node env)
+                (apl-eval-ast lhs env)
+                (apl-eval-ast rhs env)))))
+        ((= tag :program) (apl-eval-stmts (rest node) env))
+        ((= tag :dfn) node)
+        ((= tag :bracket)
+          (let
+            ((arr-expr (nth node 1)) (axis-exprs (rest (rest node))))
+            (let
+              ((arr (apl-eval-ast arr-expr env))
+                (axes
+                  (map
+                    (fn (a) (if (= a :all) nil (apl-eval-ast a env)))
+                    axis-exprs)))
+              (apl-bracket-multi axes arr))))
+        (else (error (list "apl-eval-ast: unknown node tag" tag node)))))))
+
+(define
+  apl-eval-stmts
+  (fn
+    (stmts env)
+    (if
+      (= (len stmts) 0)
+      nil
+      (let
+        ((stmt (first stmts)) (more (rest stmts)))
+        (let
+          ((tag (first stmt)))
+          (cond
+            ((= tag :guard)
+              (let
+                ((cond-val (apl-eval-ast (nth stmt 1) env)))
+                (if
+                  (apl-truthy? cond-val)
+                  (apl-eval-ast (nth stmt 2) env)
+                  (apl-eval-stmts more env))))
+            ((and (= tag :assign) (= (nth stmt 1) "⍺"))
+              (if
+                (get env "alpha")
+                (apl-eval-stmts more env)
+                (let
+                  ((v (apl-eval-ast (nth stmt 2) env)))
+                  (apl-eval-stmts more (assoc env "alpha" v)))))
+            ((= tag :assign)
+              (let
+                ((v (apl-eval-ast (nth stmt 2) env)))
+                (apl-eval-stmts more (assoc env (nth stmt 1) v))))
+            ((= (len more) 0) (apl-eval-ast stmt env))
+            (else (begin (apl-eval-ast stmt env) (apl-eval-stmts more env)))))))))
+
+(define
+  apl-call-dfn
+  (fn
+    (dfn-ast alpha omega)
+    (let
+      ((stmts (rest dfn-ast)) (env {:omega omega :nabla dfn-ast :alpha alpha}))
+      (apl-eval-stmts stmts env))))
+
+(define
+  apl-call-dfn-m
+  (fn
+    (dfn-ast omega)
+    (let
+      ((stmts (rest dfn-ast)) (env {:omega omega :nabla dfn-ast :alpha nil}))
+      (apl-eval-stmts stmts env))))
+
+(define
+  apl-tradfn-eval-block
+  (fn
+    (stmts env)
+    (if
+      (= (len stmts) 0)
+      env
+      (let
+        ((stmt (first stmts)))
+        (apl-tradfn-eval-block (rest stmts) (apl-tradfn-eval-stmt stmt env))))))
+
+(define
+  apl-tradfn-eval-while
+  (fn
+    (cond-expr body env)
+    (let
+      ((cond-val (apl-eval-ast cond-expr env)))
+      (if
+        (apl-truthy? cond-val)
+        (apl-tradfn-eval-while
+          cond-expr
+          body
+          (apl-tradfn-eval-block body env))
+        env))))
+
+(define
+  apl-tradfn-eval-for
+  (fn
+    (var-name items body env)
+    (if
+      (= (len items) 0)
+      env
+      (let
+        ((env-with-var (assoc env var-name (apl-scalar (first items)))))
+        (apl-tradfn-eval-for
+          var-name
+          (rest items)
+          body
+          (apl-tradfn-eval-block body env-with-var))))))
+
+(define
+  apl-tradfn-eval-select
+  (fn
+    (val cases default-block env)
+    (if
+      (= (len cases) 0)
+      (apl-tradfn-eval-block default-block env)
+      (let
+        ((c (first cases)))
+        (let
+          ((case-val (apl-eval-ast (first c) env)))
+          (if
+            (= (first (get val :ravel)) (first (get case-val :ravel)))
+            (apl-tradfn-eval-block (rest c) env)
+            (apl-tradfn-eval-select val (rest cases) default-block env)))))))
+
+(define
+  apl-tradfn-eval-stmt
+  (fn
+    (stmt env)
+    (let
+      ((tag (first stmt)))
+      (cond
+        ((= tag :assign)
+          (assoc env (nth stmt 1) (apl-eval-ast (nth stmt 2) env)))
+        ((= tag :if)
+          (let
+            ((cond-val (apl-eval-ast (nth stmt 1) env)))
+            (if
+              (apl-truthy? cond-val)
+              (apl-tradfn-eval-block (nth stmt 2) env)
+              (apl-tradfn-eval-block (nth stmt 3) env))))
+        ((= tag :while)
+          (apl-tradfn-eval-while (nth stmt 1) (nth stmt 2) env))
+        ((= tag :for)
+          (let
+            ((iter-val (apl-eval-ast (nth stmt 2) env)))
+            (apl-tradfn-eval-for
+              (nth stmt 1)
+              (get iter-val :ravel)
+              (nth stmt 3)
+              env)))
+        ((= tag :select)
+          (let
+            ((val (apl-eval-ast (nth stmt 1) env)))
+            (apl-tradfn-eval-select val (nth stmt 2) (nth stmt 3) env)))
+        ((= tag :trap)
+          (let
+            ((codes (nth stmt 1))
+              (try-block (nth stmt 2))
+              (catch-block (nth stmt 3)))
+            (guard
+              (e
+                ((apl-trap-matches? codes e)
+                  (apl-tradfn-eval-block catch-block env)))
+              (apl-tradfn-eval-block try-block env))))
+        ((= tag :throw) (apl-throw (nth stmt 1) (nth stmt 2)))
+        (else (begin (apl-eval-ast stmt env) env))))))
+
+(define
+  apl-tradfn-loop
+  (fn
+    (stmts line env result-name)
+    (cond
+      ((= line 0) (get env result-name))
+      ((> line (len stmts)) (get env result-name))
+      (else
+        (let
+          ((stmt (nth stmts (- line 1))))
+          (let
+            ((tag (first stmt)))
+            (cond
+              ((= tag :branch)
+                (let
+                  ((target (apl-eval-ast (nth stmt 1) env)))
+                  (let
+                    ((target-num (first (get target :ravel))))
+                    (apl-tradfn-loop stmts target-num env result-name))))
+              (else
+                (apl-tradfn-loop
+                  stmts
+                  (+ line 1)
+                  (apl-tradfn-eval-stmt stmt env)
+                  result-name)))))))))
+
+(define
+  apl-call-tradfn
+  (fn
+    (tradfn alpha omega)
+    (let
+      ((stmts (get tradfn :stmts))
+        (result-name (get tradfn :result))
+        (alpha-name (get tradfn :alpha))
+        (omega-name (get tradfn :omega)))
+      (let
+        ((env-a (if alpha-name (assoc {} alpha-name alpha) {})))
+        (let
+          ((env-ao (if omega-name (assoc env-a omega-name omega) env-a)))
+          (apl-tradfn-loop stmts 1 env-ao result-name))))))
+
+(define
+  apl-ast-mentions-list?
+  (fn
+    (lst target)
+    (if
+      (= (len lst) 0)
+      false
+      (if
+        (apl-ast-mentions? (first lst) target)
+        true
+        (apl-ast-mentions-list? (rest lst) target)))))
+
+(define
+  apl-ast-mentions?
+  (fn
+    (node target)
+    (cond
+      ((not (list? node)) false)
+      ((= (len node) 0) false)
+      ((and (= (first node) :name) (= (nth node 1) target)) true)
+      (else (apl-ast-mentions-list? (rest node) target)))))
+
+(define
+  apl-dfn-valence
+  (fn
+    (dfn-ast)
+    (let
+      ((body (rest dfn-ast)))
+      (cond
+        ((apl-ast-mentions-list? body "⍺") :dyadic)
+        ((apl-ast-mentions-list? body "⍵") :monadic)
+        (else :niladic)))))
+
+(define
+  apl-tradfn-valence
+  (fn
+    (tradfn)
+    (cond
+      ((get tradfn :alpha) :dyadic)
+      ((get tradfn :omega) :monadic)
+      (else :niladic))))
+
+(define
+  apl-call
+  (fn
+    (f alpha omega)
+    (cond
+      ((and (list? f) (> (len f) 0) (= (first f) :dfn))
+        (if alpha (apl-call-dfn f alpha omega) (apl-call-dfn-m f omega)))
+      ((dict? f) (apl-call-tradfn f alpha omega))
+      (else (error "apl-call: not a function")))))
+
+(define
+  apl-resolve-monadic
+  (fn
+    (fn-node env)
+    (let
+      ((tag (first fn-node)))
+      (cond
+        ((= tag :fn-glyph) (apl-monadic-fn (nth fn-node 1)))
+        ((= tag :derived-fn)
+          (let
+            ((op (nth fn-node 1)) (inner (nth fn-node 2)))
+            (cond
+              ((= op "/")
+                (let
+                  ((f (apl-resolve-dyadic inner env)))
+                  (fn (arr) (apl-reduce f arr))))
+              ((= op "⌿")
+                (let
+                  ((f (apl-resolve-dyadic inner env)))
+                  (fn (arr) (apl-reduce-first f arr))))
+              ((= op "\\")
+                (let
+                  ((f (apl-resolve-dyadic inner env)))
+                  (fn (arr) (apl-scan f arr))))
+              ((= op "⍀")
+                (let
+                  ((f (apl-resolve-dyadic inner env)))
+                  (fn (arr) (apl-scan-first f arr))))
+              ((= op "¨")
+                (let
+                  ((f (apl-resolve-monadic inner env)))
+                  (fn (arr) (apl-each f arr))))
+              ((= op "⍨")
+                (let
+                  ((f (apl-resolve-dyadic inner env)))
+                  (fn (arr) (apl-commute f arr))))
+              (else (error "apl-resolve-monadic: unsupported op")))))
+        ((= tag :fn-name)
+          (let
+            ((nm (nth fn-node 1)))
+            (let
+              ((bound (get env nm)))
+              (if
+                (and
+                  (list? bound)
+                  (> (len bound) 0)
+                  (= (first bound) :dfn))
+                (fn (arg) (apl-call-dfn-m bound arg))
+                (error "apl-resolve-monadic: name not bound to dfn")))))
+        ((= tag :train)
+          (let
+            ((fns (rest fn-node)))
+            (let
+              ((n (len fns)))
+              (cond
+                ((= n 2)
+                  (let
+                    ((g (apl-resolve-monadic (nth fns 0) env))
+                      (h (apl-resolve-monadic (nth fns 1) env)))
+                    (fn (arg) (g (h arg)))))
+                ((= n 3)
+                  (let
+                    ((f (apl-resolve-monadic (nth fns 0) env))
+                      (g (apl-resolve-dyadic (nth fns 1) env))
+                      (h (apl-resolve-monadic (nth fns 2) env)))
+                    (fn (arg) (g (f arg) (h arg)))))
+                (else (error "monadic train arity not 2 or 3"))))))
+        (else (error "apl-resolve-monadic: unknown fn-node tag"))))))
+
+(define
+  apl-resolve-dyadic
+  (fn
+    (fn-node env)
+    (let
+      ((tag (first fn-node)))
+      (cond
+        ((= tag :fn-glyph) (apl-dyadic-fn (nth fn-node 1)))
+        ((= tag :derived-fn)
+          (let
+            ((op (nth fn-node 1)) (inner (nth fn-node 2)))
+            (cond
+              ((= op "¨")
+                (let
+                  ((f (apl-resolve-dyadic inner env)))
+                  (fn (a b) (apl-each-dyadic f a b))))
+              ((= op "⍨")
+                (let
+                  ((f (apl-resolve-dyadic inner env)))
+                  (fn (a b) (apl-commute-dyadic f a b))))
+              (else (error "apl-resolve-dyadic: unsupported op")))))
+        ((= tag :fn-name)
+          (let
+            ((nm (nth fn-node 1)))
+            (let
+              ((bound (get env nm)))
+              (if
+                (and
+                  (list? bound)
+                  (> (len bound) 0)
+                  (= (first bound) :dfn))
+                (fn (a b) (apl-call-dfn bound a b))
+                (error "apl-resolve-dyadic: name not bound to dfn")))))
+        ((= tag :outer)
+          (let
+            ((inner (nth fn-node 2)))
+            (let
+              ((f (apl-resolve-dyadic inner env)))
+              (fn (a b) (apl-outer f a b)))))
+        ((= tag :derived-fn2)
+          (let
+            ((f-node (nth fn-node 2)) (g-node (nth fn-node 3)))
+            (let
+              ((f (apl-resolve-dyadic f-node env))
+                (g (apl-resolve-dyadic g-node env)))
+              (fn (a b) (apl-inner f g a b)))))
+        ((= tag :train)
+          (let
+            ((fns (rest fn-node)))
+            (let
+              ((n (len fns)))
+              (cond
+                ((= n 2)
+                  (let
+                    ((g (apl-resolve-monadic (nth fns 0) env))
+                      (h (apl-resolve-dyadic (nth fns 1) env)))
+                    (fn (a b) (g (h a b)))))
+                ((= n 3)
+                  (let
+                    ((f (apl-resolve-dyadic (nth fns 0) env))
+                      (g (apl-resolve-dyadic (nth fns 1) env))
+                      (h (apl-resolve-dyadic (nth fns 2) env)))
+                    (fn (a b) (g (f a b) (h a b)))))
+                (else (error "dyadic train arity not 2 or 3"))))))
+        (else (error "apl-resolve-dyadic: unknown fn-node tag"))))))
+
+(define apl-run (fn (src) (apl-eval-ast (parse-apl src) {})))

lib/datalog/aggregates.sx

@@ -0,0 +1,157 @@
+;; lib/datalog/aggregates.sx — count / sum / min / max / findall.
+;;
+;; Surface form (always 3-arg after the relation name):
+;;
+;;   (count   Result  Var  GoalLit)
+;;   (sum     Result  Var  GoalLit)
+;;   (min     Result  Var  GoalLit)
+;;   (max     Result  Var  GoalLit)
+;;   (findall List    Var  GoalLit)
+;;
+;; Parsed naturally because arg-position compounds are already allowed
+;; (Phase 4 needs them for arithmetic). At evaluation time the aggregator
+;; runs `dl-find-bindings` on `GoalLit` under the current subst, collects
+;; the distinct values of `Var`, and binds `Result`.
+;;
+;; Aggregation is non-monotonic — `count(C, X, p(X))` shrinks as p loses
+;; tuples. The stratifier (lib/datalog/strata.sx) treats every aggregate's
+;; goal relation as a negation-like edge so the inner relation is fully
+;; derived before the aggregate fires.
+;;
+;; Empty input: count → 0, sum → 0, min/max → no binding (rule fails).
+
+(define dl-aggregate-rels (list "count" "sum" "min" "max" "findall"))
+
+(define
+  dl-aggregate?
+  (fn
+    (lit)
+    (and
+      (list? lit)
+      (>= (len lit) 4)
+      (let ((rel (dl-rel-name lit)))
+        (cond
+          ((nil? rel) false)
+          (else (dl-member-string? rel dl-aggregate-rels)))))))
+
+;; Apply aggregation operator to a list of (already-distinct) numeric or
+;; symbolic values. Returns the aggregated value, or :empty if min/max
+;; has no input.
+(define
+  dl-do-aggregate
+  (fn
+    (op vals)
+    (cond
+      ((= op "count") (len vals))
+      ((= op "sum") (dl-sum-vals vals 0))
+      ((= op "findall") vals)
+      ((= op "min")
+        (cond
+          ((= (len vals) 0) :empty)
+          (else (dl-min-vals vals 1 (first vals)))))
+      ((= op "max")
+        (cond
+          ((= (len vals) 0) :empty)
+          (else (dl-max-vals vals 1 (first vals)))))
+      (else (error (str "datalog: unknown aggregate " op))))))
+
+(define
+  dl-sum-vals
+  (fn
+    (vals acc)
+    (cond
+      ((= (len vals) 0) acc)
+      (else (dl-sum-vals (rest vals) (+ acc (first vals)))))))
+
+(define
+  dl-min-vals
+  (fn
+    (vals i cur)
+    (cond
+      ((>= i (len vals)) cur)
+      (else
+        (let ((v (nth vals i)))
+          (dl-min-vals vals (+ i 1) (if (< v cur) v cur)))))))
+
+(define
+  dl-max-vals
+  (fn
+    (vals i cur)
+    (cond
+      ((>= i (len vals)) cur)
+      (else
+        (let ((v (nth vals i)))
+          (dl-max-vals vals (+ i 1) (if (> v cur) v cur)))))))
+
+;; Membership check by deep equality (so 30 == 30.0 etc).
+(define
+  dl-val-member?
+  (fn
+    (v xs)
+    (cond
+      ((= (len xs) 0) false)
+      ((dl-tuple-equal? v (first xs)) true)
+      (else (dl-val-member? v (rest xs))))))
+
+;; Evaluate an aggregate body lit under `subst`. Returns the list of
+;; extended substitutions (0 or 1 element).
+(define
+  dl-eval-aggregate
+  (fn
+    (lit db subst)
+    (let
+      ((op (dl-rel-name lit))
+        (result-var (nth lit 1))
+        (agg-var (nth lit 2))
+        (goal (nth lit 3)))
+      (cond
+        ((not (dl-var? agg-var))
+          (error (str "datalog aggregate (" op
+                      "): second arg must be a variable, got " agg-var)))
+        ((not (and (list? goal) (> (len goal) 0)
+                   (symbol? (first goal))))
+          (error (str "datalog aggregate (" op
+                      "): third arg must be a positive literal, got "
+                      goal)))
+        ((not (dl-member-string?
+                (symbol->string agg-var)
+                (dl-vars-of goal)))
+          (error (str "datalog aggregate (" op
+                      "): aggregation variable " agg-var
+                      " does not appear in the goal " goal
+                      " — without it every match contributes the same "
+                      "(unbound) value and the result is meaningless")))
+        (else
+          (let ((vals (list)))
+            (do
+              (for-each
+                (fn
+                  (s)
+                  (let ((v (dl-apply-subst agg-var s)))
+                    (when (not (dl-val-member? v vals))
+                      (append! vals v))))
+                (dl-find-bindings (list goal) db subst))
+              (let ((agg-val (dl-do-aggregate op vals)))
+                (cond
+                  ((= agg-val :empty) (list))
+                  (else
+                    (let ((s2 (dl-unify result-var agg-val subst)))
+                      (if (nil? s2) (list) (list s2)))))))))))))
+
+
+;; Stratification edges from aggregates: like negation, the goal's
+;; relation must be in a strictly lower stratum so that the aggregate
+;; fires only after the underlying tuples are settled.
+(define
+  dl-aggregate-dep-edge
+  (fn
+    (lit)
+    (cond
+      ((dl-aggregate? lit)
+        (let ((goal (nth lit 3)))
+          (cond
+            ((and (list? goal) (> (len goal) 0))
+              (let ((rel (dl-rel-name goal)))
+                (if (nil? rel) nil {:rel rel :neg true})))
+            (else nil))))
+      (else nil))))

lib/datalog/api.sx

@@ -0,0 +1,303 @@
+;; lib/datalog/api.sx — SX-data embedding API.
+;;
+;; Where Phase 1's `dl-program` takes a Datalog source string,
+;; this module exposes a parser-free API that consumes SX data
+;; directly. Two rule shapes are accepted:
+;;
+;;   - dict:   {:head <literal> :body (<literal> ...)}
+;;   - list:   (<head-elements...> <- <body-literal> ...)
+;;             — `<-` is an SX symbol used as the rule arrow.
+;;
+;; Examples:
+;;
+;;   (dl-program-data
+;;     '((parent tom bob) (parent tom liz) (parent bob ann))
+;;     '((ancestor X Y <- (parent X Y))
+;;       (ancestor X Z <- (parent X Y) (ancestor Y Z))))
+;;
+;;   (dl-query db '(ancestor tom X))   ; same query API as before
+;;
+;; Variables follow the parser convention: SX symbols whose first
+;; character is uppercase or `_` are variables.
+
+(define
+  dl-rule
+  (fn (head body) {:head head :body body}))
+
+(define
+  dl-rule-arrow?
+  (fn
+    (x)
+    (and (symbol? x) (= (symbol->string x) "<-"))))
+
+(define
+  dl-find-arrow
+  (fn
+    (rl i n)
+    (cond
+      ((>= i n) nil)
+      ((dl-rule-arrow? (nth rl i)) i)
+      (else (dl-find-arrow rl (+ i 1) n)))))
+
+;; Given a list of the form (head-elt ... <- body-lit ...) returns
+;; {:head (head-elt ...) :body (body-lit ...)}. If no arrow is
+;; present, the whole list is treated as the head and the body is
+;; empty (i.e. a fact written rule-style).
+(define
+  dl-rule-from-list
+  (fn
+    (rl)
+    (let ((n (len rl)))
+      (let ((idx (dl-find-arrow rl 0 n)))
+        (cond
+          ((nil? idx) {:head rl :body (list)})
+          (else
+            (let
+              ((head (slice rl 0 idx))
+                (body (slice rl (+ idx 1) n)))
+              {:head head :body body})))))))
+
+;; Coerce a rule given as either a dict or a list-with-arrow to a dict.
+(define
+  dl-coerce-rule
+  (fn
+    (r)
+    (cond
+      ((dict? r) r)
+      ((list? r) (dl-rule-from-list r))
+      (else (error (str "dl-coerce-rule: expected dict or list, got " r))))))
+
+;; Build a db from SX data lists.
+(define
+  dl-program-data
+  (fn
+    (facts rules)
+    (let ((db (dl-make-db)))
+      (do
+        (for-each (fn (lit) (dl-add-fact! db lit)) facts)
+        (for-each
+          (fn (r) (dl-add-rule! db (dl-coerce-rule r)))
+          rules)
+        db))))
+
+;; Add a single fact at runtime, then re-saturate the db so derived
+;; tuples reflect the change. Returns the db.
+(define
+  dl-assert!
+  (fn
+    (db lit)
+    (do
+      (dl-add-fact! db lit)
+      (dl-saturate! db)
+      db)))
+
+;; Remove a fact and re-saturate. Mixed relations (which have BOTH
+;; user-asserted facts AND rules) are supported via :edb-keys provenance
+;; — explicit facts are marked at dl-add-fact! time, the saturator uses
+;; dl-add-derived! which doesn't mark them, so the retract pass can
+;; safely wipe IDB-derived tuples while preserving the user's EDB.
+;;
+;; Effect:
+;;   - remove tuples matching `lit` from :facts and :edb-keys
+;;   - for every relation that has a rule (i.e. potentially IDB or
+;;     mixed), drop the IDB-derived portion (anything not in :edb-keys)
+;;     so the saturator can re-derive cleanly
+;;   - re-saturate
+(define
+  dl-retract!
+  (fn
+    (db lit)
+    (let
+      ((rel-key (dl-rel-name lit)))
+      (do
+        ;; Drop the matching tuple from its relation list, its facts-keys,
+        ;; its first-arg index, AND from :edb-keys (if present).
+        (when
+          (has-key? (get db :facts) rel-key)
+          (let
+            ((existing (get (get db :facts) rel-key))
+              (kept (list))
+              (kept-keys {})
+              (kept-index {})
+              (edb-rel (cond
+                         ((has-key? (get db :edb-keys) rel-key)
+                           (get (get db :edb-keys) rel-key))
+                         (else nil)))
+              (kept-edb {}))
+            (do
+              (for-each
+                (fn
+                  (t)
+                  (when
+                    (not (dl-tuple-equal? t lit))
+                    (do
+                      (append! kept t)
+                      (let ((tk (dl-tuple-key t)))
+                        (do
+                          (dict-set! kept-keys tk true)
+                          (when
+                            (and (not (nil? edb-rel))
+                                 (has-key? edb-rel tk))
+                            (dict-set! kept-edb tk true))))
+                      (when
+                        (>= (len t) 2)
+                        (let ((k (dl-arg-key (nth t 1))))
+                          (do
+                            (when
+                              (not (has-key? kept-index k))
+                              (dict-set! kept-index k (list)))
+                            (append! (get kept-index k) t)))))))
+                existing)
+              (dict-set! (get db :facts) rel-key kept)
+              (dict-set! (get db :facts-keys) rel-key kept-keys)
+              (dict-set! (get db :facts-index) rel-key kept-index)
+              (when
+                (not (nil? edb-rel))
+                (dict-set! (get db :edb-keys) rel-key kept-edb)))))
+        ;; For each rule-head relation, strip the IDB-derived tuples
+        ;; (anything not marked in :edb-keys) so the saturator can
+        ;; cleanly re-derive without leaving stale tuples that depended
+        ;; on the now-removed fact.
+        (let ((rule-heads (dl-rule-head-rels db)))
+          (for-each
+            (fn
+              (k)
+              (when
+                (has-key? (get db :facts) k)
+                (let
+                  ((existing (get (get db :facts) k))
+                    (kept (list))
+                    (kept-keys {})
+                    (kept-index {})
+                    (edb-rel (cond
+                               ((has-key? (get db :edb-keys) k)
+                                 (get (get db :edb-keys) k))
+                               (else {}))))
+                  (do
+                    (for-each
+                      (fn
+                        (t)
+                        (let ((tk (dl-tuple-key t)))
+                          (when
+                            (has-key? edb-rel tk)
+                            (do
+                              (append! kept t)
+                              (dict-set! kept-keys tk true)
+                              (when
+                                (>= (len t) 2)
+                                (let ((kk (dl-arg-key (nth t 1))))
+                                  (do
+                                    (when
+                                      (not (has-key? kept-index kk))
+                                      (dict-set! kept-index kk (list)))
+                                    (append! (get kept-index kk) t))))))))
+                      existing)
+                    (dict-set! (get db :facts) k kept)
+                    (dict-set! (get db :facts-keys) k kept-keys)
+                    (dict-set! (get db :facts-index) k kept-index)))))
+            rule-heads))
+        (dl-saturate! db)
+        db))))
+
+;; ── Convenience: single-call source + query ───────────────────
+;; (dl-eval source query-source) parses both, builds a db, saturates,
+;; runs the query, returns the substitution list. The query source
+;; should be `?- goal[, goal ...].` — the parser produces a clause
+;; with :query containing a list of literals which is fed straight
+;; to dl-query.
+(define
+  dl-eval
+  (fn
+    (source query-source)
+    (let
+      ((db (dl-program source))
+        (queries (dl-parse query-source)))
+      (cond
+        ((= (len queries) 0) (error "dl-eval: query string is empty"))
+        ((not (has-key? (first queries) :query))
+          (error "dl-eval: second arg must be a `?- ...` query clause"))
+        (else
+          (dl-query db (get (first queries) :query)))))))
+
+;; (dl-eval-magic source query-source) — like dl-eval but routes a
+;; single-positive-literal query through `dl-magic-query` for goal-
+;; directed evaluation. Multi-literal query bodies fall back to the
+;; standard dl-query path (magic-sets is currently only wired for
+;; single-positive goals). The caller's source is parsed afresh
+;; each call so successive invocations are independent.
+(define
+  dl-eval-magic
+  (fn
+    (source query-source)
+    (let
+      ((db (dl-program source))
+        (queries (dl-parse query-source)))
+      (cond
+        ((= (len queries) 0) (error "dl-eval-magic: query string is empty"))
+        ((not (has-key? (first queries) :query))
+          (error
+            "dl-eval-magic: second arg must be a `?- ...` query clause"))
+        (else
+          (let
+            ((qbody (get (first queries) :query)))
+            (cond
+              ((and (= (len qbody) 1)
+                    (list? (first qbody))
+                    (> (len (first qbody)) 0)
+                    (symbol? (first (first qbody))))
+                (dl-magic-query db (first qbody)))
+              (else (dl-query db qbody)))))))))
+
+;; List rules whose head's relation matches `rel-name`. Useful for
+;; inspection ("show me how this relation is derived") without
+;; exposing the internal `:rules` list.
+(define
+  dl-rules-of
+  (fn
+    (db rel-name)
+    (let ((out (list)))
+      (do
+        (for-each
+          (fn
+            (rule)
+            (when
+              (= (dl-rel-name (get rule :head)) rel-name)
+              (append! out rule)))
+          (dl-rules db))
+        out))))
+
+(define
+  dl-rule-head-rels
+  (fn
+    (db)
+    (let ((seen (list)))
+      (do
+        (for-each
+          (fn
+            (rule)
+            (let ((h (dl-rel-name (get rule :head))))
+              (when
+                (and (not (nil? h)) (not (dl-member-string? h seen)))
+                (append! seen h))))
+          (dl-rules db))
+        seen))))
+
+;; Wipe every relation that has at least one rule (i.e. every IDB
+;; relation) — leaves EDB facts and rule definitions intact. Useful
+;; before a follow-up `dl-saturate!` if you want a clean restart, or
+;; for inspection of the EDB-only baseline.
+(define
+  dl-clear-idb!
+  (fn
+    (db)
+    (let ((rule-heads (dl-rule-head-rels db)))
+      (do
+        (for-each
+          (fn
+            (k)
+            (do
+              (dict-set! (get db :facts) k (list))
+              (dict-set! (get db :facts-keys) k {})
+              (dict-set! (get db :facts-index) k {})))
+          rule-heads)
+        db))))

lib/datalog/builtins.sx

@@ -0,0 +1,406 @@
+;; lib/datalog/builtins.sx — comparison + arithmetic body literals.
+;;
+;; Built-in predicates filter / extend candidate substitutions during
+;; rule evaluation. They are not stored facts and do not participate in
+;; the Herbrand base.
+;;
+;;   (< a b)  (<= a b)  (> a b)  (>= a b)   ; numeric (or string) compare
+;;   (= a b)                                  ; unify (binds vars)
+;;   (!= a b)                                ; ground-only inequality
+;;   (is X expr)                              ; bind X to expr's value
+;;
+;; Arithmetic expressions are SX-list compounds:
+;;   (+ a b)  (- a b)  (* a b)  (/ a b)
+;; or numbers / variables (must be bound at evaluation time).
+
+(define
+  dl-comparison?
+  (fn
+    (lit)
+    (and
+      (list? lit)
+      (> (len lit) 0)
+      (let
+        ((rel (dl-rel-name lit)))
+        (cond
+          ((nil? rel) false)
+          (else (dl-member-string? rel (list "<" "<=" ">" ">=" "!="))))))))
+
+(define
+  dl-eq?
+  (fn
+    (lit)
+    (and
+      (list? lit)
+      (> (len lit) 0)
+      (let ((rel (dl-rel-name lit))) (and (not (nil? rel)) (= rel "=")))))) 
+
+(define
+  dl-is?
+  (fn
+    (lit)
+    (and
+      (list? lit)
+      (> (len lit) 0)
+      (let
+        ((rel (dl-rel-name lit)))
+        (and (not (nil? rel)) (= rel "is")))))) 
+
+;; Evaluate an arithmetic expression under subst. Returns the numeric
+;; result, or raises if any operand is unbound or non-numeric.
+(define
+  dl-eval-arith
+  (fn
+    (expr subst)
+    (let
+      ((w (dl-walk expr subst)))
+      (cond
+        ((number? w) w)
+        ((dl-var? w)
+          (error (str "datalog arith: unbound variable " (symbol->string w))))
+        ((list? w)
+          (let
+            ((rel (dl-rel-name w)) (args (rest w)))
+            (cond
+              ((not (= (len args) 2))
+                (error (str "datalog arith: need 2 args, got " w)))
+              (else
+                (let
+                  ((a (dl-eval-arith (first args) subst))
+                    (b (dl-eval-arith (nth args 1) subst)))
+                  (cond
+                    ((= rel "+") (+ a b))
+                    ((= rel "-") (- a b))
+                    ((= rel "*") (* a b))
+                    ((= rel "/")
+                      (cond
+                        ((= b 0)
+                          (error
+                            (str "datalog arith: division by zero in "
+                                 w)))
+                        (else (/ a b))))
+                    (else (error (str "datalog arith: unknown op " rel)))))))))
+        (else (error (str "datalog arith: not a number — " w)))))))
+
+;; Comparable types — both operands must be the same primitive type
+;; (both numbers, both strings). `!=` is the exception: it's defined
+;; for any pair (returns true iff not equal) since dl-tuple-equal?
+;; handles type-mixed comparisons.
+(define
+  dl-compare-typeok?
+  (fn
+    (rel a b)
+    (cond
+      ((= rel "!=") true)
+      ((and (number? a) (number? b)) true)
+      ((and (string? a) (string? b)) true)
+      (else false))))
+
+(define
+  dl-eval-compare
+  (fn
+    (lit subst)
+    (let
+      ((rel (dl-rel-name lit))
+        (a (dl-walk (nth lit 1) subst))
+        (b (dl-walk (nth lit 2) subst)))
+      (cond
+        ((or (dl-var? a) (dl-var? b))
+          (error
+            (str
+              "datalog: comparison "
+              rel
+              " has unbound argument; "
+              "ensure prior body literal binds the variable")))
+        ((not (dl-compare-typeok? rel a b))
+          (error
+            (str "datalog: comparison " rel " requires same-type "
+                 "operands (both numbers or both strings), got "
+                 a " and " b)))
+        (else
+          (let
+            ((ok (cond ((= rel "<") (< a b)) ((= rel "<=") (<= a b)) ((= rel ">") (> a b)) ((= rel ">=") (>= a b)) ((= rel "!=") (not (dl-tuple-equal? a b))) (else (error (str "datalog: unknown compare " rel))))))
+            (if ok subst nil)))))))
+
+(define
+  dl-eval-eq
+  (fn
+    (lit subst)
+    (dl-unify (nth lit 1) (nth lit 2) subst)))
+
+(define
+  dl-eval-is
+  (fn
+    (lit subst)
+    (let
+      ((target (nth lit 1)) (expr (nth lit 2)))
+      (let
+        ((value (dl-eval-arith expr subst)))
+        (dl-unify target value subst)))))
+
+(define
+  dl-eval-builtin
+  (fn
+    (lit subst)
+    (cond
+      ((dl-comparison? lit) (dl-eval-compare lit subst))
+      ((dl-eq? lit) (dl-eval-eq lit subst))
+      ((dl-is? lit) (dl-eval-is lit subst))
+      (else (error (str "dl-eval-builtin: not a built-in: " lit))))))
+
+;; ── Safety analysis ──────────────────────────────────────────────
+;;
+;; Walks body literals left-to-right tracking a "bound" set. The check
+;; understands these literal kinds:
+;;
+;;   positive non-built-in    → adds its vars to bound
+;;   (is X expr)              → vars(expr) ⊆ bound, then add X (if var)
+;;   <,<=,>,>=,!=             → all vars ⊆ bound (no binding)
+;;   (= a b) where:
+;;       both non-vars        → constraint check, no binding
+;;       a var, b not         → bind a
+;;       b var, a not         → bind b
+;;       both vars            → at least one in bound; bind the other
+;;   {:neg lit}               → all vars ⊆ bound (Phase 7 enforces fully)
+;;
+;; At end, head vars (minus `_`) must be ⊆ bound.
+
+(define
+  dl-vars-not-in
+  (fn
+    (vs bound)
+    (let
+      ((out (list)))
+      (do
+        (for-each
+          (fn
+            (v)
+            (when (not (dl-member-string? v bound)) (append! out v)))
+          vs)
+        out))))
+
+;; Filter a list of variable-name strings to exclude anonymous-renamed
+;; vars (`_` in source → `_anon*` by dl-rename-anon-term). Used by
+;; the negation safety check, where anonymous vars are existential
+;; within the negated literal.
+(define
+  dl-non-anon-vars
+  (fn
+    (vs)
+    (let
+      ((out (list)))
+      (do
+        (for-each
+          (fn
+            (v)
+            (when
+              (not (and (>= (len v) 5)
+                        (= (slice v 0 5) "_anon")))
+              (append! out v)))
+          vs)
+        out))))
+
+(define
+  dl-rule-check-safety
+  (fn
+    (rule)
+    (let
+      ((head (get rule :head))
+        (body (get rule :body))
+        (bound (list))
+        (err nil))
+      (do
+        (define
+          dl-add-bound!
+          (fn
+            (vs)
+            (for-each
+              (fn
+                (v)
+                (when (not (dl-member-string? v bound)) (append! bound v)))
+              vs)))
+        (define
+          dl-process-eq!
+          (fn
+            (lit)
+            (let
+              ((a (nth lit 1)) (b (nth lit 2)))
+              (let
+                ((va (dl-var? a)) (vb (dl-var? b)))
+                (cond
+                  ((and (not va) (not vb)) nil)
+                  ((and va (not vb))
+                    (dl-add-bound! (list (symbol->string a))))
+                  ((and (not va) vb)
+                    (dl-add-bound! (list (symbol->string b))))
+                  (else
+                    (let
+                      ((sa (symbol->string a)) (sb (symbol->string b)))
+                      (cond
+                        ((dl-member-string? sa bound)
+                          (dl-add-bound! (list sb)))
+                        ((dl-member-string? sb bound)
+                          (dl-add-bound! (list sa)))
+                        (else
+                          (set!
+                            err
+                            (str
+                              "= between two unbound variables "
+                              (list sa sb)
+                              " — at least one must be bound by an "
+                              "earlier positive body literal")))))))))))
+        (define
+          dl-process-cmp!
+          (fn
+            (lit)
+            (let
+              ((needed (dl-vars-of (list (nth lit 1) (nth lit 2)))))
+              (let
+                ((missing (dl-vars-not-in needed bound)))
+                (when
+                  (> (len missing) 0)
+                  (set!
+                    err
+                    (str
+                      "comparison "
+                      (dl-rel-name lit)
+                      " requires bound variable(s) "
+                      missing
+                      " (must be bound by an earlier positive "
+                      "body literal)")))))))
+        (define
+          dl-process-is!
+          (fn
+            (lit)
+            (let
+              ((tgt (nth lit 1)) (expr (nth lit 2)))
+              (let
+                ((needed (dl-vars-of expr)))
+                (let
+                  ((missing (dl-vars-not-in needed bound)))
+                  (cond
+                    ((> (len missing) 0)
+                      (set!
+                        err
+                        (str
+                          "is RHS uses unbound variable(s) "
+                          missing
+                          " — bind them via a prior positive body "
+                          "literal")))
+                    (else
+                      (when
+                        (dl-var? tgt)
+                        (dl-add-bound! (list (symbol->string tgt)))))))))))
+        (define
+          dl-process-neg!
+          (fn
+            (lit)
+            (let
+              ((inner (get lit :neg)))
+              (let
+                ((inner-rn
+                   (cond
+                     ((and (list? inner) (> (len inner) 0))
+                       (dl-rel-name inner))
+                     (else nil)))
+                  ;; Anonymous variables (`_` in source → `_anon*` after
+                  ;; renaming) are existentially quantified within the
+                  ;; negated literal — they don't need to be bound by
+                  ;; an earlier body lit, since `not p(X, _)` is a
+                  ;; valid idiom for "no Y exists s.t. p(X, Y)". Filter
+                  ;; them out of the safety check.
+                  (needed (dl-non-anon-vars (dl-vars-of inner)))
+                  (missing (dl-vars-not-in needed bound)))
+                (cond
+                  ((and (not (nil? inner-rn)) (dl-reserved-rel? inner-rn))
+                    (set! err
+                      (str "negated literal uses reserved name '"
+                           inner-rn
+                           "' — nested `not(...)` / negated built-ins are "
+                           "not supported; introduce an intermediate "
+                           "relation and negate that")))
+                  ((> (len missing) 0)
+                    (set! err
+                      (str "negation refers to unbound variable(s) "
+                           missing
+                           " — they must be bound by an earlier "
+                           "positive body literal"))))))))
+        (define
+          dl-process-agg!
+          (fn
+            (lit)
+            (let
+              ((result-var (nth lit 1)))
+              ;; Aggregate goal vars are existentially quantified within
+              ;; the aggregate; nothing required from outer context. The
+              ;; result var becomes bound after the aggregate fires.
+              (when
+                (dl-var? result-var)
+                (dl-add-bound! (list (symbol->string result-var)))))))
+
+        (define
+          dl-process-lit!
+          (fn
+            (lit)
+            (when
+              (nil? err)
+              (cond
+                ((and (dict? lit) (has-key? lit :neg))
+                  (dl-process-neg! lit))
+                ;; A bare dict that is not a recognised negation is
+                ;; almost certainly a typo (e.g. `{:negs ...}` instead
+                ;; of `{:neg ...}`). Without this guard the dict would
+                ;; silently fall through every clause; the head safety
+                ;; check would then flag the head variables as unbound
+                ;; even though the real bug is the malformed body lit.
+                ((dict? lit)
+                  (set! err
+                    (str "body literal is a dict but lacks :neg — "
+                         "the only dict-shaped body lit recognised is "
+                         "{:neg <positive-lit>} for stratified "
+                         "negation, got " lit)))
+                ((dl-aggregate? lit) (dl-process-agg! lit))
+                ((dl-eq? lit) (dl-process-eq! lit))
+                ((dl-is? lit) (dl-process-is! lit))
+                ((dl-comparison? lit) (dl-process-cmp! lit))
+                ((and (list? lit) (> (len lit) 0))
+                  (let ((rn (dl-rel-name lit)))
+                    (cond
+                      ((and (not (nil? rn)) (dl-reserved-rel? rn))
+                        (set! err
+                          (str "body literal uses reserved name '" rn
+                               "' — built-ins / aggregates have their own "
+                               "syntax; nested `not(...)` is not supported "
+                               "(use stratified negation via an "
+                               "intermediate relation)")))
+                      (else (dl-add-bound! (dl-vars-of lit))))))
+                (else
+                  ;; Anything that's not a dict, not a list, or an
+                  ;; empty list. Numbers / strings / symbols as body
+                  ;; lits don't make sense — surface the type.
+                  (set! err
+                    (str "body literal must be a positive lit, "
+                         "built-in, aggregate, or {:neg ...} dict, "
+                         "got " lit)))))))
+        (for-each dl-process-lit! body)
+        (when
+          (nil? err)
+          (let
+            ((head-vars (dl-vars-of head)) (missing (list)))
+            (do
+              (for-each
+                (fn
+                  (v)
+                  (when
+                    (and (not (dl-member-string? v bound)) (not (= v "_")))
+                    (append! missing v)))
+                head-vars)
+              (when
+                (> (len missing) 0)
+                (set!
+                  err
+                  (str
+                    "head variable(s) "
+                    missing
+                    " do not appear in any positive body literal"))))))
+        err))))

lib/datalog/conformance.conf

@@ -0,0 +1,32 @@
+# Datalog conformance config — sourced by lib/guest/conformance.sh.
+
+LANG_NAME=datalog
+MODE=dict
+
+PRELOADS=(
+  lib/datalog/tokenizer.sx
+  lib/datalog/parser.sx
+  lib/datalog/unify.sx
+  lib/datalog/db.sx
+  lib/datalog/builtins.sx
+  lib/datalog/aggregates.sx
+  lib/datalog/strata.sx
+  lib/datalog/eval.sx
+  lib/datalog/api.sx
+  lib/datalog/magic.sx
+  lib/datalog/demo.sx
+)
+
+SUITES=(
+  "tokenize:lib/datalog/tests/tokenize.sx:(dl-tokenize-tests-run!)"
+  "parse:lib/datalog/tests/parse.sx:(dl-parse-tests-run!)"
+  "unify:lib/datalog/tests/unify.sx:(dl-unify-tests-run!)"
+  "eval:lib/datalog/tests/eval.sx:(dl-eval-tests-run!)"
+  "builtins:lib/datalog/tests/builtins.sx:(dl-builtins-tests-run!)"
+  "semi_naive:lib/datalog/tests/semi_naive.sx:(dl-semi-naive-tests-run!)"
+  "negation:lib/datalog/tests/negation.sx:(dl-negation-tests-run!)"
+  "aggregates:lib/datalog/tests/aggregates.sx:(dl-aggregates-tests-run!)"
+  "api:lib/datalog/tests/api.sx:(dl-api-tests-run!)"
+  "magic:lib/datalog/tests/magic.sx:(dl-magic-tests-run!)"
+  "demo:lib/datalog/tests/demo.sx:(dl-demo-tests-run!)"
+)

lib/datalog/conformance.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+# Thin wrapper — see lib/guest/conformance.sh and lib/datalog/conformance.conf.
+exec bash "$(dirname "$0")/../guest/conformance.sh" "$(dirname "$0")/conformance.conf" "$@"

lib/datalog/datalog.sx

@@ -0,0 +1,97 @@
+;; lib/datalog/datalog.sx — public API documentation index.
+;;
+;; This file is reference-only — `load` is an epoch-protocol command,
+;; not an SX function, so it cannot reload a list of files from inside
+;; another `.sx` file. To set up a fresh sx_server session with all
+;; modules in scope, issue these loads in order:
+;;
+;;   (load "lib/datalog/tokenizer.sx")
+;;   (load "lib/datalog/parser.sx")
+;;   (load "lib/datalog/unify.sx")
+;;   (load "lib/datalog/db.sx")
+;;   (load "lib/datalog/builtins.sx")
+;;   (load "lib/datalog/aggregates.sx")
+;;   (load "lib/datalog/strata.sx")
+;;   (load "lib/datalog/eval.sx")
+;;   (load "lib/datalog/api.sx")
+;;   (load "lib/datalog/magic.sx")
+;;   (load "lib/datalog/demo.sx")
+;;
+;; (lib/datalog/conformance.sh runs this load list automatically.)
+;;
+;; ── Public API surface ─────────────────────────────────────────────
+;;
+;;   Source / data:
+;;     (dl-tokenize "src")              → token list
+;;     (dl-parse "src")                 → parsed clauses
+;;     (dl-program "src")               → db built from a source string
+;;     (dl-program-data facts rules)    → db from SX data lists; rules
+;;                                        accept either dict form or
+;;                                        list form with `<-` arrow
+;;
+;;   Construction (mutates db):
+;;     (dl-make-db)                     empty db
+;;     (dl-add-fact! db lit)            rejects non-ground
+;;     (dl-add-rule! db rule)           rejects unsafe rules
+;;     (dl-rule head body)              dict-rule constructor
+;;     (dl-add-clause! db clause)       parser output → fact or rule
+;;     (dl-load-program! db src)        string source
+;;     (dl-set-strategy! db strategy)   :semi-naive default; :magic
+;;                                        is informational, use
+;;                                        dl-magic-query for actual
+;;                                        magic-sets evaluation
+;;
+;;   Mutation:
+;;     (dl-assert! db lit)              add + re-saturate
+;;     (dl-retract! db lit)             drop EDB, wipe IDB, re-saturate
+;;     (dl-clear-idb! db)               wipe rule-headed relations
+;;
+;;   Query / inspection:
+;;     (dl-saturate! db)                stratified semi-naive default
+;;     (dl-saturate-naive! db)          reference (slow on chains)
+;;     (dl-saturate-rules! db rules)    per-rule-set semi-naive worker
+;;     (dl-query db goal)               list of substitution dicts
+;;     (dl-relation db rel-name)        tuple list for a relation
+;;     (dl-rules db)                    rule list
+;;     (dl-fact-count db)               total ground tuples
+;;     (dl-summary db)                  {<rel>: count} for inspection
+;;
+;;   Single-call convenience:
+;;     (dl-eval source query-source)         parse, run, return substs
+;;     (dl-eval-magic source query-source)   single-goal → magic-sets
+;;
+;;   Magic-sets (lib/datalog/magic.sx):
+;;     (dl-adorn-goal goal)             "b/f" adornment string
+;;     (dl-rule-sips rule head-adn)     SIPS analysis per body lit
+;;     (dl-magic-rewrite rules rel adn args)
+;;                                       rewritten rule list + seed
+;;     (dl-magic-query db query-goal)    end-to-end magic-sets query
+;;
+;; ── Body literal kinds ─────────────────────────────────────────────
+;;
+;;   Positive             (rel arg ... arg)
+;;   Negation             {:neg (rel arg ...)}
+;;   Comparison           (< X Y), (<= X Y), (> X Y), (>= X Y),
+;;                        (= X Y), (!= X Y)
+;;   Arithmetic           (is Z (+ X Y)) and (- * /)
+;;   Aggregation          (count R V Goal),  (sum R V Goal),
+;;                        (min R V Goal),    (max R V Goal),
+;;                        (findall L V Goal)
+;;
+;; ── Variable conventions ───────────────────────────────────────────
+;;
+;;   Variables: SX symbols whose first char is uppercase A–Z or '_'.
+;;   Anonymous '_' is renamed to a fresh _anon<N> per occurrence at
+;;   rule/query load time so multiple '_' don't unify.
+;;
+;; ── Demo programs ──────────────────────────────────────────────────
+;;
+;;   See lib/datalog/demo.sx — federation, content, permissions, and
+;;   the canonical "cooking posts by people I follow (transitively)"
+;;   example.
+;;
+;; ── Status ─────────────────────────────────────────────────────────
+;;
+;;   See plans/datalog-on-sx.md — phase-by-phase progress log and
+;;   roadmap. Run `bash lib/datalog/conformance.sh` to refresh
+;;   `lib/datalog/scoreboard.{json,md}`.

lib/datalog/db.sx

@@ -0,0 +1,575 @@
+;; lib/datalog/db.sx — Datalog database (EDB + IDB + rules) + safety hook.
+;;
+;; A db is a mutable dict:
+;;   {:facts {<rel-name-string> -> (literal ...)}
+;;    :rules ({:head literal :body (literal ...)} ...)}
+;;
+;; Facts are stored as full literals `(rel arg ... arg)` so they unify
+;; directly against rule body literals. Each relation's tuple list is
+;; deduplicated on insert.
+;;
+;; Phase 3 introduced safety analysis for head variables; Phase 4 (in
+;; lib/datalog/builtins.sx) swaps in the real `dl-rule-check-safety`,
+;; which is order-aware and understands built-in predicates.
+
+(define
+  dl-make-db
+  (fn ()
+    {:facts {}
+     :facts-keys {}
+     :facts-index {}
+     :edb-keys {}
+     :rules (list)
+     :strategy :semi-naive}))
+
+;; Record (rel-key, tuple-key) as user-asserted EDB. dl-add-fact! calls
+;; this when an explicit fact is added; the saturator (which uses
+;; dl-add-derived!) does NOT, so derived tuples never appear here.
+;; dl-retract! consults :edb-keys to know which tuples must survive
+;; the wipe-and-resaturate round-trip.
+(define
+  dl-mark-edb!
+  (fn
+    (db rel-key tk)
+    (let
+      ((edb (get db :edb-keys)))
+      (do
+        (when
+          (not (has-key? edb rel-key))
+          (dict-set! edb rel-key {}))
+        (dict-set! (get edb rel-key) tk true)))))
+
+(define
+  dl-edb-fact?
+  (fn
+    (db rel-key tk)
+    (let
+      ((edb (get db :edb-keys)))
+      (and (has-key? edb rel-key)
+           (has-key? (get edb rel-key) tk)))))
+
+;; Evaluation strategy. Default :semi-naive (used by dl-saturate!).
+;; :naive selects dl-saturate-naive! (slower but easier to reason
+;; about). :magic is a marker — goal-directed magic-sets evaluation
+;; is invoked separately via `dl-magic-query`; setting :magic here
+;; is purely informational. Any other value is rejected so typos
+;; don't silently fall back to the default.
+(define
+  dl-strategy-values
+  (list :semi-naive :naive :magic))
+
+(define
+  dl-set-strategy!
+  (fn
+    (db strategy)
+    (cond
+      ((not (dl-keyword-member? strategy dl-strategy-values))
+        (error (str "dl-set-strategy!: unknown strategy " strategy
+                    " — must be one of " dl-strategy-values)))
+      (else
+        (do
+          (dict-set! db :strategy strategy)
+          db)))))
+
+(define
+  dl-keyword-member?
+  (fn
+    (k xs)
+    (cond
+      ((= (len xs) 0) false)
+      ((= k (first xs)) true)
+      (else (dl-keyword-member? k (rest xs))))))
+
+(define
+  dl-get-strategy
+  (fn
+    (db)
+    (if (has-key? db :strategy) (get db :strategy) :semi-naive)))
+
+(define
+  dl-rel-name
+  (fn
+    (lit)
+    (cond
+      ((and (dict? lit) (has-key? lit :neg)) (dl-rel-name (get lit :neg)))
+      ((and (list? lit) (> (len lit) 0) (symbol? (first lit)))
+        (symbol->string (first lit)))
+      (else nil))))
+
+(define dl-builtin-rels (list "<" "<=" ">" ">=" "=" "!=" "is"))
+
+(define
+  dl-member-string?
+  (fn
+    (s xs)
+    (cond
+      ((= (len xs) 0) false)
+      ((= (first xs) s) true)
+      (else (dl-member-string? s (rest xs))))))
+
+(define
+  dl-builtin?
+  (fn
+    (lit)
+    (and
+      (list? lit)
+      (> (len lit) 0)
+      (let
+        ((rel (dl-rel-name lit)))
+        (cond
+          ((nil? rel) false)
+          (else (dl-member-string? rel dl-builtin-rels)))))))
+
+(define
+  dl-positive-lit?
+  (fn
+    (lit)
+    (cond
+      ((and (dict? lit) (has-key? lit :neg)) false)
+      ((dl-builtin? lit) false)
+      ((and (list? lit) (> (len lit) 0)) true)
+      (else false))))
+
+(define
+  dl-tuple-equal?
+  (fn
+    (a b)
+    (cond
+      ((and (list? a) (list? b))
+        (and (= (len a) (len b)) (dl-tuple-equal-list? a b 0)))
+      ((and (number? a) (number? b)) (= a b))
+      (else (equal? a b)))))
+
+(define
+  dl-tuple-equal-list?
+  (fn
+    (a b i)
+    (cond
+      ((>= i (len a)) true)
+      ((not (dl-tuple-equal? (nth a i) (nth b i))) false)
+      (else (dl-tuple-equal-list? a b (+ i 1))))))
+
+(define
+  dl-tuple-member?
+  (fn
+    (lit lits)
+    (dl-tuple-member-aux? lit lits 0 (len lits))))
+
+(define
+  dl-tuple-member-aux?
+  (fn
+    (lit lits i n)
+    (cond
+      ((>= i n) false)
+      ((dl-tuple-equal? lit (nth lits i)) true)
+      (else (dl-tuple-member-aux? lit lits (+ i 1) n)))))
+
+(define
+  dl-ensure-rel!
+  (fn
+    (db rel-key)
+    (let
+      ((facts (get db :facts))
+        (fk (get db :facts-keys))
+        (fi (get db :facts-index)))
+      (do
+        (when
+          (not (has-key? facts rel-key))
+          (dict-set! facts rel-key (list)))
+        (when
+          (not (has-key? fk rel-key))
+          (dict-set! fk rel-key {}))
+        (when
+          (not (has-key? fi rel-key))
+          (dict-set! fi rel-key {}))
+        (get facts rel-key)))))
+
+;; First-arg index helpers. Tuples are keyed by their first-after-rel
+;; arg's `(str ...)`; when that arg is a constant, dl-match-positive
+;; uses the index instead of scanning the full relation.
+(define
+  dl-arg-key
+  (fn
+    (v)
+    (str v)))
+
+(define
+  dl-index-add!
+  (fn
+    (db rel-key lit)
+    (let
+      ((idx (get db :facts-index))
+        (n (len lit)))
+      (when
+        (and (>= n 2) (has-key? idx rel-key))
+        (let
+          ((rel-idx (get idx rel-key))
+            (k (dl-arg-key (nth lit 1))))
+          (do
+            (when
+              (not (has-key? rel-idx k))
+              (dict-set! rel-idx k (list)))
+            (append! (get rel-idx k) lit)))))))
+
+(define
+  dl-index-lookup
+  (fn
+    (db rel-key arg-val)
+    (let
+      ((idx (get db :facts-index)))
+      (cond
+        ((not (has-key? idx rel-key)) (list))
+        (else
+          (let ((rel-idx (get idx rel-key))
+                (k (dl-arg-key arg-val)))
+            (if (has-key? rel-idx k) (get rel-idx k) (list))))))))
+
+(define dl-tuple-key (fn (lit) (str lit)))
+
+(define
+  dl-rel-tuples
+  (fn
+    (db rel-key)
+    (let
+      ((facts (get db :facts)))
+      (if (has-key? facts rel-key) (get facts rel-key) (list)))))
+
+;; Reserved relation names: built-in / aggregate / negation / arrow.
+;; Rules and facts may not have these as their head's relation, since
+;; the saturator treats them specially or they are not relation names
+;; at all.
+(define
+  dl-reserved-rel-names
+  (list "not" "count" "sum" "min" "max" "findall" "is"
+        "<" "<=" ">" ">=" "=" "!=" "+" "-" "*" "/" ":-" "?-"))
+
+(define
+  dl-reserved-rel?
+  (fn
+    (name) (dl-member-string? name dl-reserved-rel-names)))
+
+;; Internal: append a derived tuple to :facts without the public
+;; validation pass and without marking :edb-keys. Used by the saturator
+;; (eval.sx) and magic-sets (magic.sx). Returns true if the tuple was
+;; new, false if already present.
+(define
+  dl-add-derived!
+  (fn
+    (db lit)
+    (let
+      ((rel-key (dl-rel-name lit)))
+      (let
+        ((tuples (dl-ensure-rel! db rel-key))
+          (key-dict (get (get db :facts-keys) rel-key))
+          (tk (dl-tuple-key lit)))
+        (cond
+          ((has-key? key-dict tk) false)
+          (else
+            (do
+              (dict-set! key-dict tk true)
+              (append! tuples lit)
+              (dl-index-add! db rel-key lit)
+              true)))))))
+
+;; A simple term — number, string, or symbol — i.e. anything legal
+;; as an EDB fact arg. Compound (list) args belong only in body
+;; literals where they encode arithmetic / aggregate sub-goals.
+(define
+  dl-simple-term?
+  (fn
+    (term)
+    (or (number? term) (string? term) (symbol? term))))
+
+(define
+  dl-args-simple?
+  (fn
+    (lit i n)
+    (cond
+      ((>= i n) true)
+      ((not (dl-simple-term? (nth lit i))) false)
+      (else (dl-args-simple? lit (+ i 1) n)))))
+
+(define
+  dl-add-fact!
+  (fn
+    (db lit)
+    (cond
+      ((not (and (list? lit) (> (len lit) 0)))
+        (error (str "dl-add-fact!: expected literal list, got " lit)))
+      ((dl-reserved-rel? (dl-rel-name lit))
+        (error (str "dl-add-fact!: '" (dl-rel-name lit)
+                    "' is a reserved name (built-in / aggregate / negation)")))
+      ((not (dl-args-simple? lit 1 (len lit)))
+        (error (str "dl-add-fact!: fact args must be numbers, strings, "
+                    "or symbols — compound args (e.g. arithmetic "
+                    "expressions) are body-only and aren't evaluated "
+                    "in fact position. got " lit)))
+      ((not (dl-ground? lit (dl-empty-subst)))
+        (error (str "dl-add-fact!: expected ground literal, got " lit)))
+      (else
+        (let
+          ((rel-key (dl-rel-name lit)) (tk (dl-tuple-key lit)))
+          (do
+            ;; Always mark EDB origin — even if the tuple key was already
+            ;; present (e.g. previously derived), so an explicit assert
+            ;; promotes it to EDB and protects it from the IDB wipe.
+            (dl-mark-edb! db rel-key tk)
+            (dl-add-derived! db lit)))))))
+
+;; The full safety check lives in builtins.sx (it has to know which
+;; predicates are built-ins). dl-add-rule! calls it via forward
+;; reference; load builtins.sx alongside db.sx in any setup that
+;; adds rules. The fallback below is used if builtins.sx isn't loaded.
+(define
+  dl-rule-check-safety
+  (fn
+    (rule)
+    (let
+      ((head-vars (dl-vars-of (get rule :head))) (body-vars (list)))
+      (do
+        (for-each
+          (fn
+            (lit)
+            (when
+              (and
+                (list? lit)
+                (> (len lit) 0)
+                (not (and (dict? lit) (has-key? lit :neg))))
+              (for-each
+                (fn
+                  (v)
+                  (when
+                    (not (dl-member-string? v body-vars))
+                    (append! body-vars v)))
+                (dl-vars-of lit))))
+          (get rule :body))
+        (let
+          ((missing (list)))
+          (do
+            (for-each
+              (fn
+                (v)
+                (when
+                  (and
+                    (not (dl-member-string? v body-vars))
+                    (not (= v "_")))
+                  (append! missing v)))
+              head-vars)
+            (cond
+              ((> (len missing) 0)
+                (str
+                  "head variable(s) "
+                  missing
+                  " do not appear in any body literal"))
+              (else nil))))))))
+
+(define
+  dl-rename-anon-term
+  (fn
+    (term next-name)
+    (cond
+      ((and (symbol? term) (= (symbol->string term) "_"))
+        (next-name))
+      ((list? term)
+        (map (fn (x) (dl-rename-anon-term x next-name)) term))
+      (else term))))
+
+(define
+  dl-rename-anon-lit
+  (fn
+    (lit next-name)
+    (cond
+      ((and (dict? lit) (has-key? lit :neg))
+        {:neg (dl-rename-anon-term (get lit :neg) next-name)})
+      ((list? lit) (dl-rename-anon-term lit next-name))
+      (else lit))))
+
+(define
+  dl-make-anon-renamer
+  (fn
+    (start)
+    (let ((counter start))
+      (fn () (do (set! counter (+ counter 1))
+                 (string->symbol (str "_anon" counter)))))))
+
+;; Scan a rule for variables already named `_anon<N>` (which would
+;; otherwise collide with the renamer's output). Returns the max N
+;; seen, or 0 if none. The renamer then starts at that max + 1, so
+;; freshly-introduced anonymous names can't shadow a user-written
+;; `_anon<N>` symbol.
+(define
+  dl-max-anon-num
+  (fn
+    (term acc)
+    (cond
+      ((symbol? term)
+        (let ((s (symbol->string term)))
+          (cond
+            ((and (>= (len s) 6) (= (slice s 0 5) "_anon"))
+              (let ((n (dl-try-parse-int (slice s 5 (len s)))))
+                (cond
+                  ((nil? n) acc)
+                  ((> n acc) n)
+                  (else acc))))
+            (else acc))))
+      ((dict? term)
+        (cond
+          ((has-key? term :neg)
+            (dl-max-anon-num (get term :neg) acc))
+          (else acc)))
+      ((list? term) (dl-max-anon-num-list term acc 0))
+      (else acc))))
+
+(define
+  dl-max-anon-num-list
+  (fn
+    (xs acc i)
+    (cond
+      ((>= i (len xs)) acc)
+      (else
+        (dl-max-anon-num-list xs (dl-max-anon-num (nth xs i) acc) (+ i 1))))))
+
+;; Cheap "is this string a decimal int" check. Returns the number or
+;; nil. Avoids relying on host parse-number, which on non-int strings
+;; might raise rather than return nil.
+(define
+  dl-try-parse-int
+  (fn
+    (s)
+    (cond
+      ((= (len s) 0) nil)
+      ((not (dl-all-digits? s 0 (len s))) nil)
+      (else (parse-number s)))))
+
+(define
+  dl-all-digits?
+  (fn
+    (s i n)
+    (cond
+      ((>= i n) true)
+      ((let ((c (slice s i (+ i 1))))
+         (not (and (>= c "0") (<= c "9"))))
+        false)
+      (else (dl-all-digits? s (+ i 1) n)))))
+
+(define
+  dl-rename-anon-rule
+  (fn
+    (rule)
+    (let
+      ((start (dl-max-anon-num (get rule :head)
+                (dl-max-anon-num-list (get rule :body) 0 0))))
+      (let ((next-name (dl-make-anon-renamer start)))
+        {:head (dl-rename-anon-term (get rule :head) next-name)
+         :body (map (fn (lit) (dl-rename-anon-lit lit next-name))
+                    (get rule :body))}))))
+
+(define
+  dl-add-rule!
+  (fn
+    (db rule)
+    (cond
+      ((not (dict? rule))
+        (error (str "dl-add-rule!: expected rule dict, got " rule)))
+      ((not (has-key? rule :head))
+        (error (str "dl-add-rule!: rule missing :head, got " rule)))
+      ((not (and (list? (get rule :head))
+                 (> (len (get rule :head)) 0)
+                 (symbol? (first (get rule :head)))))
+        (error (str "dl-add-rule!: head must be a non-empty list "
+                    "starting with a relation-name symbol, got "
+                    (get rule :head))))
+      ((not (dl-args-simple? (get rule :head) 1 (len (get rule :head))))
+        (error (str "dl-add-rule!: rule head args must be variables or "
+                    "constants — compound terms (e.g. `(*(X, 2))`) are "
+                    "not legal in head position; introduce an `is`-bound "
+                    "intermediate in the body. got " (get rule :head))))
+      ((not (list? (if (has-key? rule :body) (get rule :body) (list))))
+        (error (str "dl-add-rule!: body must be a list of literals, got "
+                    (get rule :body))))
+      ((dl-reserved-rel? (dl-rel-name (get rule :head)))
+        (error (str "dl-add-rule!: '" (dl-rel-name (get rule :head))
+                    "' is a reserved name (built-in / aggregate / negation)")))
+      (else
+        (let ((rule (dl-rename-anon-rule rule)))
+          (let
+            ((err (dl-rule-check-safety rule)))
+            (cond
+              ((not (nil? err)) (error (str "dl-add-rule!: " err)))
+              (else
+                (let
+                  ((rules (get db :rules)))
+                  (do (append! rules rule) true))))))))))
+
+(define
+  dl-add-clause!
+  (fn
+    (db clause)
+    (cond
+      ((has-key? clause :query) false)
+      ((and (has-key? clause :body) (= (len (get clause :body)) 0))
+        (dl-add-fact! db (get clause :head)))
+      (else (dl-add-rule! db clause)))))
+
+(define
+  dl-load-program!
+  (fn
+    (db source)
+    (let
+      ((clauses (dl-parse source)))
+      (do (for-each (fn (c) (dl-add-clause! db c)) clauses) db))))
+
+(define
+  dl-program
+  (fn (source) (let ((db (dl-make-db))) (dl-load-program! db source))))
+
+(define dl-rules (fn (db) (get db :rules)))
+
+(define
+  dl-fact-count
+  (fn
+    (db)
+    (let
+      ((facts (get db :facts)) (total 0))
+      (do
+        (for-each
+          (fn (k) (set! total (+ total (len (get facts k)))))
+          (keys facts))
+        total))))
+
+;; Returns {<rel-name>: tuple-count} for debugging. Includes
+;; relations with any tuples plus all rule-head relations (so empty
+;; IDB shows as 0). Skips empty EDB-only entries that are placeholders
+;; from internal `dl-ensure-rel!` calls.
+(define
+  dl-summary
+  (fn
+    (db)
+    (let
+      ((facts (get db :facts))
+        (out {})
+        (rule-heads (list)))
+      (do
+        (for-each
+          (fn
+            (rule)
+            (let ((h (dl-rel-name (get rule :head))))
+              (when
+                (and (not (nil? h)) (not (dl-member-string? h rule-heads)))
+                (append! rule-heads h))))
+          (dl-rules db))
+        (for-each
+          (fn
+            (k)
+            (let ((c (len (get facts k))))
+              (when
+                (or (> c 0) (dl-member-string? k rule-heads))
+                (dict-set! out k c))))
+          (keys facts))
+        ;; Add rule heads that have no facts (yet).
+        (for-each
+          (fn
+            (k)
+            (when (not (has-key? out k)) (dict-set! out k 0)))
+          rule-heads)
+        out))))

lib/datalog/demo.sx

@@ -0,0 +1,162 @@
+;; lib/datalog/demo.sx — example programs over rose-ash-shaped data.
+;;
+;; Phase 10 prototypes Datalog as a rose-ash query language. Wiring
+;; the EDB to actual PostgreSQL is out of scope for this loop (it
+;; would touch service code outside lib/datalog/), but the programs
+;; below show the shape of queries we want, and the test suite runs
+;; them against synthetic in-memory tuples loaded via dl-program-data.
+;;
+;; Seven thematic demos:
+;;
+;;   1. Federation — follow graph, transitive reach, mutuals, FOAF.
+;;   2. Content    — posts, tags, likes, popularity, "for you" feed.
+;;   3. Permissions — group membership and resource access.
+;;   4. Cooking-posts — canonical "posts about cooking by people I
+;;                      follow (transitively)" multi-domain query.
+;;   5. Tag co-occurrence — distinct (T1, T2) pairs with counts.
+;;   6. Shortest path  — weighted-DAG path enumeration + min agg.
+;;   7. Org chart      — transitive subordinate + headcount per mgr.
+
+;; ── Demo 1: federation follow graph ─────────────────────────────
+;; EDB: (follows ACTOR-A ACTOR-B) — A follows B.
+;; IDB:
+;;   (mutual A B)            — A follows B and B follows A
+;;   (reachable A B)         — transitive follow closure
+;;   (foaf A C)              — friend of a friend (mutual filter)
+(define
+  dl-demo-federation-rules
+  (quote
+    ((mutual A B <- (follows A B) (follows B A))
+     (reachable A B <- (follows A B))
+     (reachable A C <- (follows A B) (reachable B C))
+     (foaf A C <- (follows A B) (follows B C) (!= A C)))))
+
+;; ── Demo 2: content recommendation ──────────────────────────────
+;; EDB:
+;;   (authored ACTOR POST)
+;;   (tagged POST TAG)
+;;   (liked ACTOR POST)
+;; IDB:
+;;   (post-likes POST N)            — count of likes per post
+;;   (popular POST)                 — posts with >= 3 likes
+;;   (tagged-by-mutual ACTOR POST)  — post tagged TOPIC by someone
+;;                                    A's mutuals follow.
+(define
+  dl-demo-content-rules
+  (quote
+    ((post-likes P N <- (authored Author P) (count N L (liked L P)))
+     (popular P <- (authored Author P) (post-likes P N) (>= N 3))
+     (interesting Me P
+       <-
+       (follows Me Buddy)
+       (authored Buddy P)
+       (popular P)))))
+
+;; ── Demo 3: role-based permissions ──────────────────────────────
+;; EDB:
+;;   (member ACTOR GROUP)
+;;   (subgroup CHILD PARENT)
+;;   (allowed GROUP RESOURCE)
+;; IDB:
+;;   (in-group ACTOR GROUP)         — direct or via subgroup chain
+;;   (can-access ACTOR RESOURCE)    — actor inherits group permission
+(define
+  dl-demo-perm-rules
+  (quote
+    ((in-group A G <- (member A G))
+     (in-group A G <- (member A H) (subgroup-trans H G))
+     (subgroup-trans X Y <- (subgroup X Y))
+     (subgroup-trans X Z <- (subgroup X Y) (subgroup-trans Y Z))
+     (can-access A R <- (in-group A G) (allowed G R)))))
+
+;; ── Demo 4: cooking-posts (the canonical Phase 10 query) ────────
+;; "Posts about cooking by people I follow (transitively)."
+;; Combines federation (follows + transitive reach), authoring,
+;; tagging — the rose-ash multi-domain join.
+;;
+;; EDB:
+;;   (follows ACTOR-A ACTOR-B)
+;;   (authored ACTOR POST)
+;;   (tagged POST TAG)
+(define
+  dl-demo-cooking-rules
+  (quote
+    ((reach Me Them <- (follows Me Them))
+     (reach Me Them <- (follows Me X) (reach X Them))
+     (cooking-post-by-network Me P
+       <-
+       (reach Me Author)
+       (authored Author P)
+       (tagged P cooking)))))
+
+;; ── Demo 5: tag co-occurrence ───────────────────────────────────
+;; "Posts tagged with both T1 AND T2." Useful for narrowed-down
+;; recommendations like "vegetarian cooking" posts.
+;;
+;; EDB:
+;;   (tagged POST TAG)
+;; IDB:
+;;   (cotagged POST T1 T2)        — post has both T1 and T2 (T1 != T2)
+;;   (popular-pair T1 T2 N)       — count of posts cotagged (T1, T2)
+(define
+  dl-demo-tag-cooccur-rules
+  (quote
+    ((cotagged P T1 T2 <- (tagged P T1) (tagged P T2) (!= T1 T2))
+     ;; Distinct (T1, T2) pairs that occur somewhere.
+     (tag-pair T1 T2 <- (cotagged P T1 T2))
+     (tag-pair-count T1 T2 N
+       <-
+       (tag-pair T1 T2)
+       (count N P (cotagged P T1 T2))))))
+
+;; ── Demo 6: weighted-DAG shortest path ─────────────────────────
+;; "What's the cheapest way from X to Y?" Edge weights with `is`
+;; arithmetic to sum costs, then `min` aggregation to pick the
+;; shortest. Termination requires the graph to be a DAG (cycles
+;; would produce infinite distances without a bound; programs
+;; built on this should add a depth filter `(<, D, MAX)` if cycles
+;; are possible).
+;;
+;; EDB:
+;;   (edge FROM TO COST)
+;; IDB:
+;;   (path FROM TO COST)            — any path
+;;   (shortest FROM TO COST)        — minimum cost path
+(define
+  dl-demo-shortest-path-rules
+  (quote
+    ((path X Y W <- (edge X Y W))
+     (path X Z W
+       <-
+       (edge X Y W1)
+       (path Y Z W2)
+       (is W (+ W1 W2)))
+     (shortest X Y W <- (path X Y _) (min W C (path X Y C))))))
+
+;; ── Demo 7: org chart + transitive headcount ───────────────────
+;; Manager graph: each employee has a single manager. Compute the
+;; transitive subordinate set and headcount per manager.
+;;
+;; EDB:
+;;   (manager EMP MGR)            — EMP reports directly to MGR
+;; IDB:
+;;   (subordinate MGR EMP)        — EMP is in MGR's subtree
+;;   (headcount MGR N)            — number of subordinates under MGR
+(define
+  dl-demo-org-rules
+  (quote
+    ((subordinate Mgr Emp <- (manager Emp Mgr))
+     (subordinate Mgr Emp
+       <- (manager Mid Mgr) (subordinate Mid Emp))
+     (headcount Mgr N
+       <- (subordinate Mgr Anyone) (count N E (subordinate Mgr E))))))
+
+;; ── Loader stub ──────────────────────────────────────────────────
+;; Wiring to PostgreSQL would replace these helpers with calls into
+;; rose-ash's internal HTTP RPC (fetch_data → /internal/data/...).
+;; The shape returned by dl-load-from-edb! is the same in either case.
+(define
+  dl-demo-make
+  (fn
+    (facts rules)
+    (dl-program-data facts rules)))

lib/datalog/eval.sx

@@ -0,0 +1,512 @@
+;; lib/datalog/eval.sx — fixpoint evaluator (naive + semi-naive).
+;;
+;; Two saturators are exposed:
+;;   `dl-saturate-naive!` — re-joins each rule against the full DB every
+;;     iteration. Reference implementation; useful for differential tests.
+;;   `dl-saturate!`       — semi-naive default. Tracks per-relation delta
+;;     sets and substitutes one positive body literal per rule with the
+;;     delta of its relation, joining the rest against the previous-
+;;     iteration DB. Same fixpoint, dramatically less work on recursive
+;;     rules.
+;;
+;; Body literal kinds:
+;;   positive (rel arg ... arg)  → match against EDB+IDB tuples
+;;   built-in (< X Y), (is X e)  → constraint via dl-eval-builtin
+;;   negation {:neg lit}         → Phase 7
+
+(define
+  dl-match-positive
+  (fn
+    (lit db subst)
+    (let
+      ((rel (dl-rel-name lit)) (results (list)))
+      (cond
+        ((nil? rel) (error (str "dl-match-positive: bad literal " lit)))
+        (else
+          (let
+            ;; If the first argument walks to a non-variable (constant
+            ;; or already-bound var), use the first-arg index for
+            ;; this relation. Otherwise scan the full tuple list.
+            ((tuples
+               (cond
+                 ((>= (len lit) 2)
+                   (let ((walked (dl-walk (nth lit 1) subst)))
+                     (cond
+                       ((dl-var? walked) (dl-rel-tuples db rel))
+                       (else (dl-index-lookup db rel walked)))))
+                 (else (dl-rel-tuples db rel)))))
+            (do
+              (for-each
+                (fn
+                  (tuple)
+                  (let
+                    ((s (dl-unify lit tuple subst)))
+                    (when (not (nil? s)) (append! results s))))
+                tuples)
+              results)))))))
+
+;; Match a positive literal against the delta set for its relation only.
+(define
+  dl-match-positive-delta
+  (fn
+    (lit delta subst)
+    (let
+      ((rel (dl-rel-name lit)) (results (list)))
+      (let
+        ((tuples (if (has-key? delta rel) (get delta rel) (list))))
+        (do
+          (for-each
+            (fn
+              (tuple)
+              (let
+                ((s (dl-unify lit tuple subst)))
+                (when (not (nil? s)) (append! results s))))
+            tuples)
+          results)))))
+
+;; Naive matcher (for dl-saturate-naive! and dl-query post-saturation).
+(define
+  dl-match-negation
+  (fn
+    (inner db subst)
+    (let
+      ((walked (dl-apply-subst inner subst))
+        (matches (dl-match-positive inner db subst)))
+      (cond
+        ((= (len matches) 0) (list subst))
+        (else (list))))))
+
+(define
+  dl-match-lit
+  (fn
+    (lit db subst)
+    (cond
+      ((and (dict? lit) (has-key? lit :neg))
+        (dl-match-negation (get lit :neg) db subst))
+      ((dl-aggregate? lit) (dl-eval-aggregate lit db subst))
+      ((dl-builtin? lit)
+        (let
+          ((s (dl-eval-builtin lit subst)))
+          (if (nil? s) (list) (list s))))
+      ((and (list? lit) (> (len lit) 0))
+        (dl-match-positive lit db subst))
+      (else (error (str "datalog: unknown body-literal shape: " lit))))))
+
+(define
+  dl-find-bindings
+  (fn (lits db subst) (dl-fb-aux lits db subst 0 (len lits))))
+
+(define
+  dl-fb-aux
+  (fn
+    (lits db subst i n)
+    (cond
+      ((nil? subst) (list))
+      ((>= i n) (list subst))
+      (else
+        (let
+          ((options (dl-match-lit (nth lits i) db subst))
+            (results (list)))
+          (do
+            (for-each
+              (fn
+                (s)
+                (for-each
+                  (fn (s2) (append! results s2))
+                  (dl-fb-aux lits db s (+ i 1) n)))
+              options)
+            results))))))
+
+;; Naive: apply each rule against full DB until no new tuples.
+(define
+  dl-apply-rule!
+  (fn
+    (db rule)
+    (let
+      ((head (get rule :head)) (body (get rule :body)) (new? false))
+      (do
+        (for-each
+          (fn
+            (s)
+            (let
+              ((derived (dl-apply-subst head s)))
+              (when (dl-add-derived! db derived) (set! new? true))))
+          (dl-find-bindings body db (dl-empty-subst)))
+        new?))))
+
+;; Returns true iff one more saturation step would derive no new
+;; tuples (i.e. the db is at fixpoint). Useful in tests that want
+;; to assert "no work left" after a saturation call. Works under
+;; either saturator since both compute the same fixpoint.
+(define
+  dl-saturated?
+  (fn
+    (db)
+    (let ((any-new false))
+      (do
+        (for-each
+          (fn
+            (rule)
+            (when (not any-new)
+              (for-each
+                (fn
+                  (s)
+                  (let ((derived (dl-apply-subst (get rule :head) s)))
+                    (when
+                      (and (not any-new)
+                           (not (dl-tuple-member?
+                                  derived
+                                  (dl-rel-tuples
+                                    db (dl-rel-name derived)))))
+                      (set! any-new true))))
+                (dl-find-bindings (get rule :body) db (dl-empty-subst)))))
+          (dl-rules db))
+        (not any-new)))))
+
+(define
+  dl-saturate-naive!
+  (fn
+    (db)
+    (let
+      ((changed true))
+      (do
+        (define
+          dl-snloop
+          (fn
+            ()
+            (when
+              changed
+              (do
+                (set! changed false)
+                (for-each
+                  (fn (r) (when (dl-apply-rule! db r) (set! changed true)))
+                  (dl-rules db))
+                (dl-snloop)))))
+        (dl-snloop)
+        db))))
+
+;; ── Semi-naive ───────────────────────────────────────────────────
+
+;; Take a snapshot dict {rel -> tuples} of every relation currently in
+;; the DB. Used as initial delta for the first iteration.
+(define
+  dl-snapshot-facts
+  (fn
+    (db)
+    (let
+      ((facts (get db :facts)) (out {}))
+      (do
+        (for-each
+          (fn (k) (dict-set! out k (dl-copy-list (get facts k))))
+          (keys facts))
+        out))))
+
+(define
+  dl-copy-list
+  (fn
+    (xs)
+    (let
+      ((out (list)))
+      (do (for-each (fn (x) (append! out x)) xs) out))))
+
+;; Does any relation in `delta` have ≥1 tuple?
+(define
+  dl-delta-empty?
+  (fn
+    (delta)
+    (let
+      ((ks (keys delta)) (any-non-empty false))
+      (do
+        (for-each
+          (fn
+            (k)
+            (when
+              (> (len (get delta k)) 0)
+              (set! any-non-empty true)))
+          ks)
+        (not any-non-empty)))))
+
+;; Find substitutions such that `lits` are all satisfied AND `delta-idx`
+;; is matched against the per-relation delta only. The other positive
+;; literals match against the snapshot DB (db.facts read at iteration
+;; start). Built-ins and negations behave as in `dl-match-lit`.
+(define
+  dl-find-bindings-semi
+  (fn
+    (lits db delta delta-idx subst)
+    (dl-fbs-aux lits db delta delta-idx 0 subst)))
+
+(define
+  dl-fbs-aux
+  (fn
+    (lits db delta delta-idx i subst)
+    (cond
+      ((nil? subst) (list))
+      ((>= i (len lits)) (list subst))
+      (else
+        (let
+          ((lit (nth lits i))
+            (options
+              (cond
+                ((and (dict? lit) (has-key? lit :neg))
+                  (dl-match-negation (get lit :neg) db subst))
+                ((dl-aggregate? lit) (dl-eval-aggregate lit db subst))
+                ((dl-builtin? lit)
+                  (let
+                    ((s (dl-eval-builtin lit subst)))
+                    (if (nil? s) (list) (list s))))
+                ((and (list? lit) (> (len lit) 0))
+                  (if
+                    (= i delta-idx)
+                    (dl-match-positive-delta lit delta subst)
+                    (dl-match-positive lit db subst)))
+                (else (error (str "datalog: unknown body-lit: " lit)))))
+            (results (list)))
+          (do
+            (for-each
+              (fn
+                (s)
+                (for-each
+                  (fn (s2) (append! results s2))
+                  (dl-fbs-aux lits db delta delta-idx (+ i 1) s)))
+              options)
+            results))))))
+
+;; Collect candidate head tuples from a rule using delta. Walks every
+;; positive body position and unions the resulting heads. For rules
+;; with no positive body literal, falls back to a naive single-pass
+;; (so static facts like `(p X) :- (= X 5).` derive on iteration 1).
+(define
+  dl-collect-rule-candidates
+  (fn
+    (rule db delta)
+    (let
+      ((head (get rule :head))
+        (body (get rule :body))
+        (out (list))
+        (saw-pos false))
+      (do
+        (define
+          dl-cri
+          (fn
+            (i)
+            (when
+              (< i (len body))
+              (do
+                (let
+                  ((lit (nth body i)))
+                  (when
+                    (dl-positive-lit? lit)
+                    (do
+                      (set! saw-pos true)
+                      (for-each
+                        (fn (s) (append! out (dl-apply-subst head s)))
+                        (dl-find-bindings-semi
+                          body
+                          db
+                          delta
+                          i
+                          (dl-empty-subst))))))
+                (dl-cri (+ i 1))))))
+        (dl-cri 0)
+        (when
+          (not saw-pos)
+          (for-each
+            (fn (s) (append! out (dl-apply-subst head s)))
+            (dl-find-bindings body db (dl-empty-subst))))
+        out))))
+
+;; Add a list of candidate tuples to db; collect newly-added ones into
+;; the new-delta dict (keyed by relation name).
+(define
+  dl-commit-candidates!
+  (fn
+    (db candidates new-delta)
+    (for-each
+      (fn
+        (lit)
+        (when
+          (dl-add-derived! db lit)
+          (let
+            ((rel (dl-rel-name lit)))
+            (do
+              (when
+                (not (has-key? new-delta rel))
+                (dict-set! new-delta rel (list)))
+              (append! (get new-delta rel) lit)))))
+      candidates)))
+
+(define
+  dl-saturate-rules!
+  (fn
+    (db rules)
+    (let
+      ((delta (dl-snapshot-facts db)))
+      (do
+        (define
+          dl-sr-step
+          (fn
+            ()
+            (let
+              ((pending (list)) (new-delta {}))
+              (do
+                (for-each
+                  (fn
+                    (rule)
+                    (for-each
+                      (fn (cand) (append! pending cand))
+                      (dl-collect-rule-candidates rule db delta)))
+                  rules)
+                (dl-commit-candidates! db pending new-delta)
+                (cond
+                  ((dl-delta-empty? new-delta) nil)
+                  (else (do (set! delta new-delta) (dl-sr-step))))))))
+        (dl-sr-step)
+        db))))
+
+;; Stratified driver: rejects non-stratifiable programs at saturation
+;; time, then iterates strata in increasing order, running semi-naive on
+;; the rules whose head sits in that stratum.
+(define
+  dl-saturate!
+  (fn
+    (db)
+    (let
+      ((err (dl-check-stratifiable db)))
+      (cond
+        ((not (nil? err)) (error (str "dl-saturate!: " err)))
+        (else
+          (let
+            ((strata (dl-compute-strata db)))
+            (let
+              ((grouped (dl-group-rules-by-stratum db strata)))
+              (let
+                ((groups (get grouped :groups))
+                  (max-s (get grouped :max)))
+                (do
+                  (define
+                    dl-strat-loop
+                    (fn
+                      (s)
+                      (when
+                        (<= s max-s)
+                        (let
+                          ((sk (str s)))
+                          (do
+                            (when
+                              (has-key? groups sk)
+                              (dl-saturate-rules! db (get groups sk)))
+                            (dl-strat-loop (+ s 1)))))))
+                  (dl-strat-loop 0)
+                  db)))))))))
+
+;; ── Querying ─────────────────────────────────────────────────────
+
+;; Coerce a query argument to a list of body literals. A single literal
+;; like `(p X)` (positive — head is a symbol) or `{:neg ...}` becomes
+;; `((p X))`. A list of literals like `((p X) (q X))` is returned as-is.
+(define
+  dl-query-coerce
+  (fn
+    (goal)
+    (cond
+      ((and (dict? goal) (has-key? goal :neg)) (list goal))
+      ((and (list? goal) (> (len goal) 0) (symbol? (first goal)))
+        (list goal))
+      ((list? goal) goal)
+      (else (error (str "dl-query: unrecognised goal shape: " goal))))))
+
+(define
+  dl-query
+  (fn
+    (db goal)
+    (do
+      (dl-saturate! db)
+      ;; Rename anonymous '_' vars in each goal literal so multiple
+      ;; occurrences do not unify together. Keep the user-facing var
+      ;; list (taken before renaming) so projected results retain user
+      ;; names.
+      (let
+        ((goals (dl-query-coerce goal))
+          ;; Start the renamer past any `_anon<N>` symbols the user
+          ;; may have written in the query — avoids collision.
+          (renamer
+            (dl-make-anon-renamer (dl-max-anon-num-list goal 0 0))))
+        (let
+          ((user-vars (dl-query-user-vars goals))
+            (renamed (map (fn (g) (dl-rename-anon-lit g renamer)) goals)))
+          (let
+            ((substs (dl-find-bindings renamed db (dl-empty-subst)))
+              (results (list)))
+            (do
+              (for-each
+                (fn
+                  (s)
+                  (let
+                    ((proj (dl-project-subst s user-vars)))
+                    (when
+                      (not (dl-tuple-member? proj results))
+                      (append! results proj))))
+                substs)
+              results)))))))
+
+(define
+  dl-query-user-vars
+  (fn
+    (goals)
+    (let ((seen (list)))
+      (do
+        (for-each
+          (fn
+            (g)
+            (cond
+              ((and (dict? g) (has-key? g :neg))
+                (for-each
+                  (fn
+                    (v)
+                    (when
+                      (and (not (= v "_")) (not (dl-member-string? v seen)))
+                      (append! seen v)))
+                  (dl-vars-of (get g :neg))))
+              ((dl-aggregate? g)
+                ;; Only the result var (first arg of the aggregate
+                ;; literal) is user-facing. The aggregated var and
+                ;; any vars in the inner goal are internal.
+                (let ((r (nth g 1)))
+                  (when
+                    (dl-var? r)
+                    (let ((rn (symbol->string r)))
+                      (when
+                        (and (not (= rn "_"))
+                             (not (dl-member-string? rn seen)))
+                        (append! seen rn))))))
+              (else
+                (for-each
+                  (fn
+                    (v)
+                    (when
+                      (and (not (= v "_")) (not (dl-member-string? v seen)))
+                      (append! seen v)))
+                  (dl-vars-of g)))))
+          goals)
+        seen))))
+
+(define
+  dl-project-subst
+  (fn
+    (subst names)
+    (let
+      ((out {}))
+      (do
+        (for-each
+          (fn
+            (n)
+            (let
+              ((sym (string->symbol n)))
+              (let
+                ((v (dl-walk sym subst)))
+                (dict-set! out n (dl-apply-subst v subst)))))
+          names)
+        out))))
+
+(define dl-relation (fn (db name) (dl-rel-tuples db name)))

lib/datalog/magic.sx

@@ -0,0 +1,464 @@
+;; lib/datalog/magic.sx — adornment analysis + sideways info passing.
+;;
+;; First step of the magic-sets transformation (Phase 6). Right now
+;; the saturator does not consume these — they are introspection
+;; helpers that future magic-set rewriting will build on top of.
+;;
+;; Definitions:
+;;   - An *adornment* of an n-ary literal is an n-character string
+;;     of "b" (bound — value already known at the call site) and
+;;     "f" (free — to be derived).
+;;   - SIPS (Sideways Information Passing Strategy) walks the body
+;;     of an adorned rule left-to-right tracking which variables
+;;     have been bound so far, computing each body literal's
+;;     adornment in turn.
+;;
+;; Usage:
+;;
+;;   (dl-adorn-goal '(ancestor tom X))
+;;     => "bf"
+;;
+;;   (dl-rule-sips
+;;     {:head (ancestor X Z)
+;;      :body ((parent X Y) (ancestor Y Z))}
+;;     "bf")
+;;   => ({:lit (parent X Y) :adornment "bf"}
+;;       {:lit (ancestor Y Z) :adornment "bf"})
+
+;; Per-arg adornment under the current bound-var name set.
+(define
+  dl-adorn-arg
+  (fn
+    (arg bound)
+    (cond
+      ((dl-var? arg)
+        (if (dl-member-string? (symbol->string arg) bound) "b" "f"))
+      (else "b"))))
+
+;; Adornment for the args of a literal (after the relation name).
+(define
+  dl-adorn-args
+  (fn
+    (args bound)
+    (cond
+      ((= (len args) 0) "")
+      (else
+        (str
+          (dl-adorn-arg (first args) bound)
+          (dl-adorn-args (rest args) bound))))))
+
+;; Adornment of a top-level goal under the empty bound-var set.
+(define
+  dl-adorn-goal
+  (fn (goal) (dl-adorn-args (rest goal) (list))))
+
+;; Adornment of a literal under an explicit bound set.
+(define
+  dl-adorn-lit
+  (fn (lit bound) (dl-adorn-args (rest lit) bound)))
+
+;; The set of variable names made bound by walking a positive
+;; literal whose adornment is known. Free positions add their
+;; vars to the bound set.
+(define
+  dl-vars-bound-by-lit
+  (fn
+    (lit bound)
+    (let ((args (rest lit)) (out (list)))
+      (do
+        (for-each
+          (fn (a)
+            (when
+              (and (dl-var? a)
+                   (not (dl-member-string? (symbol->string a) bound))
+                   (not (dl-member-string? (symbol->string a) out)))
+              (append! out (symbol->string a))))
+          args)
+        out))))
+
+;; Walk the rule body left-to-right tracking bound vars seeded by the
+;; head adornment. Returns a list of {:lit :adornment} entries.
+;;
+;; Negation, comparison, and built-ins are passed through with their
+;; adornment computed from the current bound set; they don't add new
+;; bindings (except `is`, which binds its left arg if a var). Aggregates
+;; are treated like is — the result var becomes bound.
+(define
+  dl-init-head-bound
+  (fn
+    (head adornment)
+    (let ((args (rest head)) (out (list)))
+      (do
+        (define
+          dl-ihb-loop
+          (fn
+            (i)
+            (when
+              (< i (len args))
+              (do
+                (let
+                  ((c (slice adornment i (+ i 1)))
+                    (a (nth args i)))
+                  (when
+                    (and (= c "b") (dl-var? a))
+                    (let ((n (symbol->string a)))
+                      (when
+                        (not (dl-member-string? n out))
+                        (append! out n)))))
+                (dl-ihb-loop (+ i 1))))))
+        (dl-ihb-loop 0)
+        out))))
+
+(define
+  dl-rule-sips
+  (fn
+    (rule head-adornment)
+    (let
+      ((bound (dl-init-head-bound (get rule :head) head-adornment))
+        (out (list)))
+      (do
+        (for-each
+          (fn
+            (lit)
+            (cond
+              ((and (dict? lit) (has-key? lit :neg))
+                (let ((target (get lit :neg)))
+                  (append!
+                    out
+                    {:lit lit :adornment (dl-adorn-lit target bound)})))
+              ((dl-builtin? lit)
+                (let ((adn (dl-adorn-lit lit bound)))
+                  (do
+                    (append! out {:lit lit :adornment adn})
+                    ;; `is` binds its left arg (if var) once RHS is ground.
+                    (when
+                      (and (= (dl-rel-name lit) "is") (dl-var? (nth lit 1)))
+                      (let ((n (symbol->string (nth lit 1))))
+                        (when
+                          (not (dl-member-string? n bound))
+                          (append! bound n)))))))
+              ((and (list? lit) (dl-aggregate? lit))
+                (let ((adn (dl-adorn-lit lit bound)))
+                  (do
+                    (append! out {:lit lit :adornment adn})
+                    ;; Result var (first arg) becomes bound.
+                    (when (dl-var? (nth lit 1))
+                      (let ((n (symbol->string (nth lit 1))))
+                        (when
+                          (not (dl-member-string? n bound))
+                          (append! bound n)))))))
+              ((and (list? lit) (> (len lit) 0))
+                (let ((adn (dl-adorn-lit lit bound)))
+                  (do
+                    (append! out {:lit lit :adornment adn})
+                    (for-each
+                      (fn (n)
+                        (when (not (dl-member-string? n bound))
+                          (append! bound n)))
+                      (dl-vars-bound-by-lit lit bound)))))))
+          (get rule :body))
+        out))))
+
+;; ── Magic predicate naming + bound-args extraction ─────────────
+;; These are building blocks for the magic-sets *transformation*
+;; itself. The transformation (which generates rewritten rules
+;; with magic_<rel>^<adornment> filters) is future work — for now
+;; these helpers can be used to inspect what such a transformation
+;; would produce.
+
+;; "magic_p^bf" given relation "p" and adornment "bf".
+(define
+  dl-magic-rel-name
+  (fn (rel adornment) (str "magic_" rel "^" adornment)))
+
+;; A magic predicate literal:
+;;   (magic_<rel>^<adornment> arg1 arg2 ...)
+(define
+  dl-magic-lit
+  (fn
+    (rel adornment bound-args)
+    (cons (string->symbol (dl-magic-rel-name rel adornment)) bound-args)))
+
+;; Extract bound args (those at "b" positions in `adornment`) from a
+;; literal `(rel arg1 arg2 ... argN)`. Returns the list of arg values.
+(define
+  dl-bound-args
+  (fn
+    (lit adornment)
+    (let ((args (rest lit)) (out (list)))
+      (do
+        (define
+          dl-ba-loop
+          (fn
+            (i)
+            (when
+              (< i (len args))
+              (do
+                (when
+                  (= (slice adornment i (+ i 1)) "b")
+                  (append! out (nth args i)))
+                (dl-ba-loop (+ i 1))))))
+        (dl-ba-loop 0)
+        out))))
+
+;; ── Magic-sets rewriter ─────────────────────────────────────────
+;;
+;; Given the original rule list and a query (rel, adornment) pair,
+;; generates the magic-rewritten program: a list of rules that
+;; (a) gate each original rule with a `magic_<rel>^<adn>` filter and
+;; (b) propagate the magic relation through SIPS so that only
+;; query-relevant tuples are derived. Seed facts are returned
+;; separately and must be added to the db at evaluation time.
+;;
+;; Output: {:rules <rewritten-rules> :seed <magic-seed-literal>}
+;;
+;; The rewriter only rewrites IDB rules; EDB facts pass through.
+;; Built-in predicates and negation in body literals are kept in
+;; place but do not generate propagation rules of their own.
+
+(define
+  dl-magic-pair-key
+  (fn (rel adornment) (str rel "^" adornment)))
+
+(define
+  dl-magic-rewrite
+  (fn
+    (rules query-rel query-adornment query-args)
+    (let
+      ((seen (list))
+        (queue (list))
+        (out (list)))
+      (do
+        (define
+          dl-mq-mark!
+          (fn
+            (rel adornment)
+            (let ((k (dl-magic-pair-key rel adornment)))
+              (when
+                (not (dl-member-string? k seen))
+                (do
+                  (append! seen k)
+                  (append! queue {:rel rel :adn adornment}))))))
+
+        (define
+          dl-mq-rewrite-rule!
+          (fn
+            (rule adn)
+            (let
+              ((head (get rule :head))
+                (body (get rule :body))
+                (sips (dl-rule-sips rule adn)))
+              (let
+                ((magic-filter
+                   (dl-magic-lit
+                     (dl-rel-name head)
+                     adn
+                     (dl-bound-args head adn))))
+                (do
+                  ;; Adorned rule: head :- magic-filter, body...
+                  (let ((new-body (list)))
+                    (do
+                      (append! new-body magic-filter)
+                      (for-each
+                        (fn (lit) (append! new-body lit))
+                        body)
+                      (append! out {:head head :body new-body})))
+                  ;; Propagation rules for each positive non-builtin
+                  ;; body literal at position i.
+                  (define
+                    dl-mq-prop-loop
+                    (fn
+                      (i)
+                      (when
+                        (< i (len body))
+                        (do
+                          (let
+                            ((lit (nth body i))
+                              (sip-entry (nth sips i)))
+                            (when
+                              (and (list? lit)
+                                   (> (len lit) 0)
+                                   (not (and (dict? lit) (has-key? lit :neg)))
+                                   (not (dl-builtin? lit))
+                                   (not (dl-aggregate? lit)))
+                              (let
+                                ((lit-adn (get sip-entry :adornment))
+                                  (lit-rel (dl-rel-name lit)))
+                                (let
+                                  ((prop-head
+                                     (dl-magic-lit
+                                       lit-rel
+                                       lit-adn
+                                       (dl-bound-args lit lit-adn))))
+                                  (let ((prop-body (list)))
+                                    (do
+                                      (append! prop-body magic-filter)
+                                      (define
+                                        dl-mq-prefix-loop
+                                        (fn
+                                          (j)
+                                          (when
+                                            (< j i)
+                                            (do
+                                              (append!
+                                                prop-body
+                                                (nth body j))
+                                              (dl-mq-prefix-loop (+ j 1))))))
+                                      (dl-mq-prefix-loop 0)
+                                      (append!
+                                        out
+                                        {:head prop-head :body prop-body})
+                                      (dl-mq-mark! lit-rel lit-adn)))))))
+                          (dl-mq-prop-loop (+ i 1))))))
+                  (dl-mq-prop-loop 0))))))
+
+        (dl-mq-mark! query-rel query-adornment)
+
+        (let ((idx 0))
+          (define
+            dl-mq-process
+            (fn
+              ()
+              (when
+                (< idx (len queue))
+                (let ((item (nth queue idx)))
+                  (do
+                    (set! idx (+ idx 1))
+                    (let
+                      ((rel (get item :rel)) (adn (get item :adn)))
+                      (for-each
+                        (fn
+                          (rule)
+                          (when
+                            (= (dl-rel-name (get rule :head)) rel)
+                            (dl-mq-rewrite-rule! rule adn)))
+                        rules))
+                    (dl-mq-process))))))
+          (dl-mq-process))
+
+        {:rules out
+         :seed
+         (dl-magic-lit
+           query-rel
+           query-adornment
+           query-args)}))))
+
+;; ── Top-level magic-sets driver ─────────────────────────────────
+;;
+;; (dl-magic-query db query-goal) — run `query-goal` under magic-sets
+;; evaluation. Builds a fresh internal db with:
+;;   - the caller's EDB facts (relations not headed by any rule),
+;;   - the magic seed fact, and
+;;   - the rewritten rules.
+;; Saturates and queries, returning the substitution list. The
+;; caller's db is untouched.
+;;
+;; Useful primarily as a perf alternative for queries that only
+;; need a small slice of a recursive relation. Equivalent to
+;; dl-query for any single fully-stratifiable program.
+
+(define
+  dl-magic-rule-heads
+  (fn
+    (rules)
+    (let ((seen (list)))
+      (do
+        (for-each
+          (fn
+            (r)
+            (let ((h (dl-rel-name (get r :head))))
+              (when
+                (and (not (nil? h)) (not (dl-member-string? h seen)))
+                (append! seen h))))
+          rules)
+        seen))))
+
+;; True iff any rule's body contains a literal kind that the magic
+;; rewriter doesn't propagate magic to — i.e. an aggregate or a
+;; negation. Used by dl-magic-query to decide whether to pre-saturate
+;; the source db (for correctness on stratified programs) or skip
+;; that step (preserving full magic-sets efficiency for pure
+;; positive programs).
+(define
+  dl-rule-has-nonprop-lit?
+  (fn
+    (body i n)
+    (cond
+      ((>= i n) false)
+      ((let ((lit (nth body i)))
+         (or (and (dict? lit) (has-key? lit :neg))
+             (dl-aggregate? lit)))
+        true)
+      (else (dl-rule-has-nonprop-lit? body (+ i 1) n)))))
+
+(define
+  dl-rules-need-presaturation?
+  (fn
+    (rules)
+    (cond
+      ((= (len rules) 0) false)
+      ((let ((body (get (first rules) :body)))
+         (dl-rule-has-nonprop-lit? body 0 (len body)))
+        true)
+      (else (dl-rules-need-presaturation? (rest rules))))))
+
+(define
+  dl-magic-query
+  (fn
+    (db query-goal)
+    ;; Magic-sets only applies to positive non-builtin / non-aggregate
+    ;; literals against rule-defined relations. For other goal shapes
+    ;; (built-ins, aggregates, EDB-only relations) the seed is either
+    ;; non-ground or unused; fall back to dl-query.
+    (cond
+      ((not (and (list? query-goal)
+                 (> (len query-goal) 0)
+                 (symbol? (first query-goal))))
+        (error (str "dl-magic-query: goal must be a positive literal "
+                    "(non-empty list with a symbol head), got " query-goal)))
+      ((or (dl-builtin? query-goal)
+           (dl-aggregate? query-goal)
+           (and (dict? query-goal) (has-key? query-goal :neg)))
+        (dl-query db query-goal))
+      (else
+        (do
+          ;; If the rule set has aggregates or negation, pre-saturate
+          ;; the source db before copying facts. The magic rewriter
+          ;; passes aggregate body lits and negated lits through
+          ;; unchanged (no magic propagation generated for them) — so
+          ;; if their inner-goal relation is IDB, it would be empty in
+          ;; the magic db. Pre-saturating ensures equivalence with
+          ;; `dl-query` for every stratified program. Pure positive
+          ;; programs skip this and keep the full magic-sets perf win
+          ;; from goal-directed re-derivation.
+          (when
+            (dl-rules-need-presaturation? (dl-rules db))
+            (dl-saturate! db))
+          (let
+            ((query-rel (dl-rel-name query-goal))
+              (query-adn (dl-adorn-goal query-goal)))
+            (let
+              ((query-args (dl-bound-args query-goal query-adn))
+                (rules (dl-rules db)))
+              (let
+                ((rewritten (dl-magic-rewrite rules query-rel query-adn query-args))
+                  (mdb (dl-make-db))
+                  (rule-heads (dl-magic-rule-heads rules)))
+            (do
+              ;; Copy ALL existing facts. EDB-only relations bring their
+              ;; tuples; mixed EDB+IDB relations bring both their EDB
+              ;; portion and any pre-saturated IDB tuples (which the
+              ;; rewritten rules would re-derive anyway). Skipping facts
+              ;; for rule-headed relations would leave the magic run
+              ;; without the EDB portion of mixed relations.
+              (for-each
+                (fn
+                  (rel)
+                  (for-each
+                    (fn (t) (dl-add-fact! mdb t))
+                    (dl-rel-tuples db rel)))
+                (keys (get db :facts)))
+              ;; Seed + rewritten rules.
+              (dl-add-fact! mdb (get rewritten :seed))
+              (for-each (fn (r) (dl-add-rule! mdb r)) (get rewritten :rules))
+              (dl-query mdb query-goal))))))))))

lib/datalog/parser.sx

@@ -0,0 +1,252 @@
+;; lib/datalog/parser.sx — Datalog tokens → AST
+;;
+;; Output shapes:
+;;   Literal (positive)  := (relname arg ... arg)        — SX list
+;;   Literal (negative)  := {:neg (relname arg ... arg)} — dict
+;;   Argument            := var-symbol | atom-symbol | number | string
+;;                        | (op-name arg ... arg)        — arithmetic compound
+;;   Fact                := {:head literal :body ()}
+;;   Rule                := {:head literal :body (lit ... lit)}
+;;   Query               := {:query (lit ... lit)}
+;;   Program             := list of facts / rules / queries
+;;
+;; Variables and constants are both SX symbols; the evaluator dispatches
+;; on first-char case ('A'..'Z' or '_' = variable, otherwise constant).
+;;
+;; The parser permits nested compounds in arg position to support
+;; arithmetic (e.g. (is Z (+ X Y))). Safety analysis at rule-load time
+;; rejects compounds whose head is not an arithmetic operator.
+
+(define
+  dl-pp-peek
+  (fn
+    (st)
+    (let
+      ((i (get st :idx)) (tokens (get st :tokens)))
+      (if (< i (len tokens)) (nth tokens i) {:type "eof" :value nil :pos 0}))))
+
+(define
+  dl-pp-peek2
+  (fn
+    (st)
+    (let
+      ((i (+ (get st :idx) 1)) (tokens (get st :tokens)))
+      (if (< i (len tokens)) (nth tokens i) {:type "eof" :value nil :pos 0}))))
+
+(define
+  dl-pp-advance!
+  (fn (st) (dict-set! st :idx (+ (get st :idx) 1))))
+
+(define
+  dl-pp-at?
+  (fn
+    (st type value)
+    (let
+      ((t (dl-pp-peek st)))
+      (and
+        (= (get t :type) type)
+        (or (= value nil) (= (get t :value) value))))))
+
+(define
+  dl-pp-error
+  (fn
+    (st msg)
+    (let
+      ((t (dl-pp-peek st)))
+      (error
+        (str
+          "Parse error at pos "
+          (get t :pos)
+          ": "
+          msg
+          " (got "
+          (get t :type)
+          " '"
+          (if (= (get t :value) nil) "" (get t :value))
+          "')")))))
+
+(define
+  dl-pp-expect!
+  (fn
+    (st type value)
+    (let
+      ((t (dl-pp-peek st)))
+      (if
+        (dl-pp-at? st type value)
+        (do (dl-pp-advance! st) t)
+        (dl-pp-error
+          st
+          (str "expected " type (if (= value nil) "" (str " '" value "'"))))))))
+
+;; Argument: variable, atom, number, string, or compound (relname/op + parens).
+(define
+  dl-pp-parse-arg
+  (fn
+    (st)
+    (let
+      ((t (dl-pp-peek st)))
+      (let
+        ((ty (get t :type)) (vv (get t :value)))
+        (cond
+          ((= ty "number") (do (dl-pp-advance! st) vv))
+          ((= ty "string") (do (dl-pp-advance! st) vv))
+          ((= ty "var") (do (dl-pp-advance! st) (string->symbol vv)))
+          ;; Negative numeric literal: `-` op directly followed by a
+          ;; number (no `(`) is parsed as a single negative number.
+          ;; This keeps `(-X Y)` (compound) and `-N` (literal) distinct.
+          ((and (= ty "op") (= vv "-")
+                (= (get (dl-pp-peek2 st) :type) "number"))
+            (do
+              (dl-pp-advance! st)
+              (let
+                ((n (get (dl-pp-peek st) :value)))
+                (do (dl-pp-advance! st) (- 0 n)))))
+          ((or (= ty "atom") (= ty "op"))
+            (do
+              (dl-pp-advance! st)
+              (if
+                (dl-pp-at? st "punct" "(")
+                (do
+                  (dl-pp-advance! st)
+                  (let
+                    ((args (dl-pp-parse-arg-list st)))
+                    (do
+                      (dl-pp-expect! st "punct" ")")
+                      (cons (string->symbol vv) args))))
+                (string->symbol vv))))
+          (else (dl-pp-error st "expected term")))))))
+
+;; Comma-separated args inside parens.
+(define
+  dl-pp-parse-arg-list
+  (fn
+    (st)
+    (let
+      ((args (list)))
+      (do
+        (append! args (dl-pp-parse-arg st))
+        (define
+          dl-pp-arg-loop
+          (fn
+            ()
+            (when
+              (dl-pp-at? st "punct" ",")
+              (do
+                (dl-pp-advance! st)
+                (append! args (dl-pp-parse-arg st))
+                (dl-pp-arg-loop)))))
+        (dl-pp-arg-loop)
+        args))))
+
+;; A positive literal: relname (atom or op) followed by optional (args).
+(define
+  dl-pp-parse-positive
+  (fn
+    (st)
+    (let
+      ((t (dl-pp-peek st)))
+      (let
+        ((ty (get t :type)) (vv (get t :value)))
+        (if
+          (or (= ty "atom") (= ty "op"))
+          (do
+            (dl-pp-advance! st)
+            (if
+              (dl-pp-at? st "punct" "(")
+              (do
+                (dl-pp-advance! st)
+                (let
+                  ((args (dl-pp-parse-arg-list st)))
+                  (do
+                    (dl-pp-expect! st "punct" ")")
+                    (cons (string->symbol vv) args))))
+              (list (string->symbol vv))))
+          (dl-pp-error st "expected literal head"))))))
+
+;; A body literal: positive, or not(positive).
+(define
+  dl-pp-parse-body-lit
+  (fn
+    (st)
+    (let
+      ((t1 (dl-pp-peek st)) (t2 (dl-pp-peek2 st)))
+      (if
+        (and
+          (= (get t1 :type) "atom")
+          (= (get t1 :value) "not")
+          (= (get t2 :type) "punct")
+          (= (get t2 :value) "("))
+        (do
+          (dl-pp-advance! st)
+          (dl-pp-advance! st)
+          (let
+            ((inner (dl-pp-parse-positive st)))
+            (do (dl-pp-expect! st "punct" ")") {:neg inner})))
+        (dl-pp-parse-positive st)))))
+
+;; Comma-separated body literals.
+(define
+  dl-pp-parse-body
+  (fn
+    (st)
+    (let
+      ((lits (list)))
+      (do
+        (append! lits (dl-pp-parse-body-lit st))
+        (define
+          dl-pp-body-loop
+          (fn
+            ()
+            (when
+              (dl-pp-at? st "punct" ",")
+              (do
+                (dl-pp-advance! st)
+                (append! lits (dl-pp-parse-body-lit st))
+                (dl-pp-body-loop)))))
+        (dl-pp-body-loop)
+        lits))))
+
+;; Single clause: fact, rule, or query. Consumes trailing dot.
+(define
+  dl-pp-parse-clause
+  (fn
+    (st)
+    (cond
+      ((dl-pp-at? st "op" "?-")
+        (do
+          (dl-pp-advance! st)
+          (let
+            ((body (dl-pp-parse-body st)))
+            (do (dl-pp-expect! st "punct" ".") {:query body}))))
+      (else
+        (let
+          ((head (dl-pp-parse-positive st)))
+          (cond
+            ((dl-pp-at? st "op" ":-")
+              (do
+                (dl-pp-advance! st)
+                (let
+                  ((body (dl-pp-parse-body st)))
+                  (do (dl-pp-expect! st "punct" ".") {:body body :head head}))))
+            (else (do (dl-pp-expect! st "punct" ".") {:body (list) :head head}))))))))
+
+(define
+  dl-parse-program
+  (fn
+    (tokens)
+    (let
+      ((st {:tokens tokens :idx 0}) (clauses (list)))
+      (do
+        (define
+          dl-pp-prog-loop
+          (fn
+            ()
+            (when
+              (not (dl-pp-at? st "eof" nil))
+              (do
+                (append! clauses (dl-pp-parse-clause st))
+                (dl-pp-prog-loop)))))
+        (dl-pp-prog-loop)
+        clauses))))
+
+(define dl-parse (fn (src) (dl-parse-program (dl-tokenize src))))

lib/datalog/scoreboard.json

@@ -0,0 +1,20 @@
+{
+  "lang": "datalog",
+  "total_passed": 276,
+  "total_failed": 0,
+  "total": 276,
+  "suites": [
+    {"name":"tokenize","passed":31,"failed":0,"total":31},
+    {"name":"parse","passed":23,"failed":0,"total":23},
+    {"name":"unify","passed":29,"failed":0,"total":29},
+    {"name":"eval","passed":44,"failed":0,"total":44},
+    {"name":"builtins","passed":26,"failed":0,"total":26},
+    {"name":"semi_naive","passed":8,"failed":0,"total":8},
+    {"name":"negation","passed":12,"failed":0,"total":12},
+    {"name":"aggregates","passed":23,"failed":0,"total":23},
+    {"name":"api","passed":22,"failed":0,"total":22},
+    {"name":"magic","passed":37,"failed":0,"total":37},
+    {"name":"demo","passed":21,"failed":0,"total":21}
+  ],
+  "generated": "2026-05-11T09:40:12+00:00"
+}

lib/datalog/scoreboard.md

@@ -0,0 +1,17 @@
+# datalog scoreboard
+
+**276 / 276 passing** (0 failure(s)).
+
+| Suite | Passed | Total | Status |
+|-------|--------|-------|--------|
+| tokenize | 31 | 31 | ok |
+| parse | 23 | 23 | ok |
+| unify | 29 | 29 | ok |
+| eval | 44 | 44 | ok |
+| builtins | 26 | 26 | ok |
+| semi_naive | 8 | 8 | ok |
+| negation | 12 | 12 | ok |
+| aggregates | 23 | 23 | ok |
+| api | 22 | 22 | ok |
+| magic | 37 | 37 | ok |
+| demo | 21 | 21 | ok |

lib/datalog/strata.sx

@@ -0,0 +1,323 @@
+;; lib/datalog/strata.sx — dependency graph, SCC analysis, stratum assignment.
+;;
+;; A program is stratifiable iff no cycle in its dependency graph passes
+;; through a negative edge. The stratum of relation R is the depth at which
+;; R can first be evaluated:
+;;
+;;   stratum(R) = max over edges (R → S) of:
+;;       stratum(S)         if the edge is positive
+;;       stratum(S) + 1     if the edge is negative
+;;
+;; All relations in the same SCC share a stratum (and the SCC must have only
+;; positive internal edges, else the program is non-stratifiable).
+
+;; Build dep graph: dict {head-rel-name -> ({:rel str :neg bool} ...)}.
+(define
+  dl-build-dep-graph
+  (fn
+    (db)
+    (let ((g {}))
+      (do
+        (for-each
+          (fn
+            (rule)
+            (let
+              ((head-rel (dl-rel-name (get rule :head))))
+              (when
+                (not (nil? head-rel))
+                (do
+                  (when
+                    (not (has-key? g head-rel))
+                    (dict-set! g head-rel (list)))
+                  (let ((existing (get g head-rel)))
+                    (for-each
+                      (fn
+                        (lit)
+                        (cond
+                          ((dl-aggregate? lit)
+                            (let
+                              ((edge (dl-aggregate-dep-edge lit)))
+                              (when
+                                (not (nil? edge))
+                                (append! existing edge))))
+                          (else
+                            (let
+                              ((target
+                                 (cond
+                                   ((and (dict? lit) (has-key? lit :neg))
+                                     (dl-rel-name (get lit :neg)))
+                                   ((dl-builtin? lit) nil)
+                                   ((and (list? lit) (> (len lit) 0))
+                                     (dl-rel-name lit))
+                                   (else nil)))
+                                (neg?
+                                  (and (dict? lit) (has-key? lit :neg))))
+                              (when
+                                (not (nil? target))
+                                (append!
+                                  existing
+                                  {:rel target :neg neg?}))))))
+                      (get rule :body)))))))
+          (dl-rules db))
+        g))))
+
+;; All relations referenced — heads of rules + EDB names + body relations.
+(define
+  dl-all-relations
+  (fn
+    (db)
+    (let ((seen (list)))
+      (do
+        (for-each
+          (fn
+            (k)
+            (when (not (dl-member-string? k seen)) (append! seen k)))
+          (keys (get db :facts)))
+        (for-each
+          (fn
+            (rule)
+            (do
+              (let ((h (dl-rel-name (get rule :head))))
+                (when
+                  (and (not (nil? h)) (not (dl-member-string? h seen)))
+                  (append! seen h)))
+              (for-each
+                (fn
+                  (lit)
+                  (let
+                    ((t
+                       (cond
+                         ((dl-aggregate? lit)
+                           (let ((edge (dl-aggregate-dep-edge lit)))
+                             (if (nil? edge) nil (get edge :rel))))
+                         ((and (dict? lit) (has-key? lit :neg))
+                           (dl-rel-name (get lit :neg)))
+                         ((dl-builtin? lit) nil)
+                         ((and (list? lit) (> (len lit) 0))
+                           (dl-rel-name lit))
+                         (else nil))))
+                    (when
+                      (and (not (nil? t)) (not (dl-member-string? t seen)))
+                      (append! seen t))))
+                (get rule :body))))
+          (dl-rules db))
+        seen))))
+
+;; reach: dict {from: dict {to: edge-info}} where edge-info is
+;;   {:any bool :neg bool}
+;; meaning "any path from `from` to `to`" and "exists a negative-passing
+;; path from `from` to `to`".
+;;
+;; Floyd-Warshall over the dep graph. The 'neg' flag propagates through
+;; concatenation: if any edge along the path is negative, the path's
+;; flag is true.
+(define
+  dl-build-reach
+  (fn
+    (graph nodes)
+    (let ((reach {}))
+      (do
+        (for-each
+          (fn (n) (dict-set! reach n {}))
+          nodes)
+        (for-each
+          (fn
+            (head)
+            (when
+              (has-key? graph head)
+              (for-each
+                (fn
+                  (edge)
+                  (let
+                    ((target (get edge :rel)) (n (get edge :neg)))
+                    (let ((row (get reach head)))
+                      (cond
+                        ((has-key? row target)
+                          (let ((cur (get row target)))
+                            (dict-set!
+                              row
+                              target
+                              {:any true :neg (or n (get cur :neg))})))
+                        (else
+                          (dict-set! row target {:any true :neg n}))))))
+                (get graph head))))
+          nodes)
+        (for-each
+          (fn
+            (k)
+            (for-each
+              (fn
+                (i)
+                (let ((row-i (get reach i)))
+                  (when
+                    (has-key? row-i k)
+                    (let ((ik (get row-i k)) (row-k (get reach k)))
+                      (for-each
+                        (fn
+                          (j)
+                          (when
+                            (has-key? row-k j)
+                            (let ((kj (get row-k j)))
+                              (let
+                                ((combined-neg (or (get ik :neg) (get kj :neg))))
+                                (cond
+                                  ((has-key? row-i j)
+                                    (let ((cur (get row-i j)))
+                                      (dict-set!
+                                        row-i
+                                        j
+                                        {:any true
+                                         :neg (or combined-neg (get cur :neg))})))
+                                  (else
+                                    (dict-set!
+                                      row-i
+                                      j
+                                      {:any true :neg combined-neg})))))))
+                        nodes)))))
+              nodes))
+          nodes)
+        reach))))
+
+;; Returns nil on success, or error message string on failure.
+(define
+  dl-check-stratifiable
+  (fn
+    (db)
+    (let
+      ((graph (dl-build-dep-graph db))
+        (nodes (dl-all-relations db)))
+      (let ((reach (dl-build-reach graph nodes)) (err nil))
+        (do
+          (for-each
+            (fn
+              (rule)
+              (when
+                (nil? err)
+                (let ((head-rel (dl-rel-name (get rule :head))))
+                  (for-each
+                    (fn
+                      (lit)
+                      (cond
+                        ((and (dict? lit) (has-key? lit :neg))
+                          (let ((tgt (dl-rel-name (get lit :neg))))
+                            (when
+                              (and (not (nil? tgt))
+                                   (dl-reach-cycle? reach head-rel tgt))
+                              (set!
+                                err
+                                (str "non-stratifiable: relation " head-rel
+                                     " transitively depends through negation on "
+                                     tgt
+                                     " which depends back on " head-rel)))))
+                        ((dl-aggregate? lit)
+                          (let ((edge (dl-aggregate-dep-edge lit)))
+                            (when
+                              (not (nil? edge))
+                              (let ((tgt (get edge :rel)))
+                                (when
+                                  (and (not (nil? tgt))
+                                       (dl-reach-cycle? reach head-rel tgt))
+                                  (set!
+                                    err
+                                    (str "non-stratifiable: relation "
+                                         head-rel
+                                         " aggregates over " tgt
+                                         " which depends back on "
+                                         head-rel))))))))) 
+                    (get rule :body)))))
+            (dl-rules db))
+          err)))))
+
+(define
+  dl-reach-cycle?
+  (fn
+    (reach a b)
+    (and
+      (dl-reach-row-has? reach b a)
+      (dl-reach-row-has? reach a b))))
+
+(define
+  dl-reach-row-has?
+  (fn
+    (reach from to)
+    (let ((row (get reach from)))
+      (and (not (nil? row)) (has-key? row to)))))
+
+;; Compute stratum per relation. Iteratively propagate from EDB roots.
+;; Uses the per-relation max-stratum-of-deps formula. Stops when stable.
+(define
+  dl-compute-strata
+  (fn
+    (db)
+    (let
+      ((graph (dl-build-dep-graph db))
+        (nodes (dl-all-relations db))
+        (strata {}))
+      (do
+        (for-each (fn (n) (dict-set! strata n 0)) nodes)
+        (let ((changed true))
+          (do
+            (define
+              dl-cs-loop
+              (fn
+                ()
+                (when
+                  changed
+                  (do
+                    (set! changed false)
+                    (for-each
+                      (fn
+                        (head)
+                        (when
+                          (has-key? graph head)
+                          (for-each
+                            (fn
+                              (edge)
+                              (let
+                                ((tgt (get edge :rel))
+                                  (n (get edge :neg)))
+                                (let
+                                  ((tgt-strat
+                                     (if (has-key? strata tgt)
+                                       (get strata tgt) 0))
+                                    (cur (get strata head)))
+                                  (let
+                                    ((needed
+                                       (if n (+ tgt-strat 1) tgt-strat)))
+                                    (when
+                                      (> needed cur)
+                                      (do
+                                        (dict-set! strata head needed)
+                                        (set! changed true)))))))
+                            (get graph head))))
+                      nodes)
+                    (dl-cs-loop)))))
+            (dl-cs-loop)))
+        strata))))
+
+;; Group rules by their head's stratum. Returns dict {stratum-int -> rules}.
+(define
+  dl-group-rules-by-stratum
+  (fn
+    (db strata)
+    (let ((groups {}) (max-s 0))
+      (do
+        (for-each
+          (fn
+            (rule)
+            (let
+              ((head-rel (dl-rel-name (get rule :head))))
+              (let
+                ((s (if (has-key? strata head-rel)
+                       (get strata head-rel) 0)))
+                (do
+                  (when (> s max-s) (set! max-s s))
+                  (let
+                    ((sk (str s)))
+                    (do
+                      (when
+                        (not (has-key? groups sk))
+                        (dict-set! groups sk (list)))
+                      (append! (get groups sk) rule)))))))
+          (dl-rules db))
+        {:groups groups :max max-s}))))

lib/datalog/tests/aggregates.sx

@@ -0,0 +1,357 @@
+;; lib/datalog/tests/aggregates.sx — count / sum / min / max.
+
+(define dl-at-pass 0)
+(define dl-at-fail 0)
+(define dl-at-failures (list))
+
+(define
+  dl-at-deep=?
+  (fn
+    (a b)
+    (cond
+      ((and (list? a) (list? b))
+        (and (= (len a) (len b)) (dl-at-deq-l? a b 0)))
+      ((and (dict? a) (dict? b))
+        (let ((ka (keys a)) (kb (keys b)))
+          (and (= (len ka) (len kb)) (dl-at-deq-d? a b ka 0))))
+      ((and (number? a) (number? b)) (= a b))
+      (else (equal? a b)))))
+
+(define
+  dl-at-deq-l?
+  (fn
+    (a b i)
+    (cond
+      ((>= i (len a)) true)
+      ((not (dl-at-deep=? (nth a i) (nth b i))) false)
+      (else (dl-at-deq-l? a b (+ i 1))))))
+
+(define
+  dl-at-deq-d?
+  (fn
+    (a b ka i)
+    (cond
+      ((>= i (len ka)) true)
+      ((let ((k (nth ka i)))
+         (not (dl-at-deep=? (get a k) (get b k))))
+        false)
+      (else (dl-at-deq-d? a b ka (+ i 1))))))
+
+(define
+  dl-at-set=?
+  (fn
+    (a b)
+    (and
+      (= (len a) (len b))
+      (dl-at-subset? a b)
+      (dl-at-subset? b a))))
+
+(define
+  dl-at-subset?
+  (fn
+    (xs ys)
+    (cond
+      ((= (len xs) 0) true)
+      ((not (dl-at-contains? ys (first xs))) false)
+      (else (dl-at-subset? (rest xs) ys)))))
+
+(define
+  dl-at-contains?
+  (fn
+    (xs target)
+    (cond
+      ((= (len xs) 0) false)
+      ((dl-at-deep=? (first xs) target) true)
+      (else (dl-at-contains? (rest xs) target)))))
+
+(define
+  dl-at-test!
+  (fn
+    (name got expected)
+    (if
+      (dl-at-deep=? got expected)
+      (set! dl-at-pass (+ dl-at-pass 1))
+      (do
+        (set! dl-at-fail (+ dl-at-fail 1))
+        (append!
+          dl-at-failures
+          (str
+            name
+            "\n    expected: " expected
+            "\n    got:      " got))))))
+
+(define
+  dl-at-test-set!
+  (fn
+    (name got expected)
+    (if
+      (dl-at-set=? got expected)
+      (set! dl-at-pass (+ dl-at-pass 1))
+      (do
+        (set! dl-at-fail (+ dl-at-fail 1))
+        (append!
+          dl-at-failures
+          (str
+            name
+            "\n    expected (set): " expected
+            "\n    got:            " got))))))
+
+(define
+  dl-at-throws?
+  (fn
+    (thunk)
+    (let
+      ((threw false))
+      (do
+        (guard
+          (e (#t (set! threw true)))
+          (thunk))
+        threw))))
+
+(define
+  dl-at-run-all!
+  (fn
+    ()
+    (do
+      ;; count
+      (dl-at-test-set! "count siblings"
+        (dl-query
+          (dl-program
+            "parent(p, bob). parent(p, alice). parent(p, charlie).
+             sibling(X, Y) :- parent(P, X), parent(P, Y), !=(X, Y).
+             sib_count(N) :- count(N, S, sibling(bob, S)).")
+          (list (quote sib_count) (quote N)))
+        (list {:N 2}))
+
+      ;; sum
+      (dl-at-test-set! "sum prices"
+        (dl-query
+          (dl-program
+            "price(apple, 5). price(pear, 7). price(plum, 3).
+             total(T) :- sum(T, X, price(F, X)).")
+          (list (quote total) (quote T)))
+        (list {:T 15}))
+
+      ;; min
+      (dl-at-test-set! "min score"
+        (dl-query
+          (dl-program
+            "score(alice, 80). score(bob, 65). score(carol, 92).
+             lo(M) :- min(M, S, score(P, S)).")
+          (list (quote lo) (quote M)))
+        (list {:M 65}))
+
+      ;; max
+      (dl-at-test-set! "max score"
+        (dl-query
+          (dl-program
+            "score(alice, 80). score(bob, 65). score(carol, 92).
+             hi(M) :- max(M, S, score(P, S)).")
+          (list (quote hi) (quote M)))
+        (list {:M 92}))
+
+      ;; count over derived relation (stratification needed).
+      (dl-at-test-set! "count over derived"
+        (dl-query
+          (dl-program
+            "parent(a, b). parent(a, c). parent(b, d). parent(c, e).
+             ancestor(X, Y) :- parent(X, Y).
+             ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z).
+             num_ancestors(N) :- count(N, X, ancestor(a, X)).")
+          (list (quote num_ancestors) (quote N)))
+        (list {:N 4}))
+
+      ;; count with no matches → 0.
+      (dl-at-test-set! "count empty"
+        (dl-query
+          (dl-program
+            "p(1). p(2).
+             zero(N) :- count(N, X, q(X)).")
+          (list (quote zero) (quote N)))
+        (list {:N 0}))
+
+      ;; sum with no matches → 0.
+      (dl-at-test-set! "sum empty"
+        (dl-query
+          (dl-program
+            "p(1). p(2).
+             total(T) :- sum(T, X, q(X)).")
+          (list (quote total) (quote T)))
+        (list {:T 0}))
+
+      ;; min with no matches → rule does not fire.
+      (dl-at-test-set! "min empty"
+        (dl-query
+          (dl-program
+            "p(1). p(2).
+             lo(M) :- min(M, X, q(X)).")
+          (list (quote lo) (quote M)))
+        (list))
+
+      ;; Aggregate with comparison filter on result.
+      (dl-at-test-set! "popularity threshold"
+        (dl-query
+          (dl-program
+            "post(p1). post(p2).
+             liked(u1, p1). liked(u2, p1). liked(u3, p1).
+             liked(u1, p2). liked(u2, p2).
+             popular(P) :- post(P), count(N, U, liked(U, P)), >=(N, 3).")
+          (list (quote popular) (quote P)))
+        (list {:P (quote p1)}))
+
+      ;; findall: collect distinct values into a list.
+      (dl-at-test-set! "findall over EDB"
+        (dl-query
+          (dl-program
+            "p(a). p(b). p(c).
+             all_p(L) :- findall(L, X, p(X)).")
+          (list (quote all_p) (quote L)))
+        (list {:L (list (quote a) (quote b) (quote c))}))
+
+      (dl-at-test-set! "findall over derived"
+        (dl-query
+          (dl-program
+            "parent(a, b). parent(b, c). parent(c, d).
+             ancestor(X, Y) :- parent(X, Y).
+             ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z).
+             desc(L) :- findall(L, X, ancestor(a, X)).")
+          (list (quote desc) (quote L)))
+        (list {:L (list (quote b) (quote c) (quote d))}))
+
+      (dl-at-test-set! "findall empty"
+        (dl-query
+          (dl-program
+            "p(1).
+             all_q(L) :- findall(L, X, q(X)).")
+          (list (quote all_q) (quote L)))
+        (list {:L (list)}))
+
+      ;; Aggregate vs single distinct.
+      ;; Group-by via aggregate-in-rule-body. Per-user friend count
+      ;; over a friends relation. The U var is bound by the prior
+      ;; positive lit u(U) so the aggregate counts only U-rooted
+      ;; friends per group.
+      (dl-at-test-set! "group-by per-user friend count"
+        (dl-query
+          (dl-program
+            "u(alice). u(bob). u(carol).
+             f(alice, x). f(alice, y). f(bob, x).
+             counts(U, N) :- u(U), count(N, X, f(U, X)).")
+          (list (quote counts) (quote U) (quote N)))
+        (list
+          {:U (quote alice) :N 2}
+          {:U (quote bob) :N 1}
+          {:U (quote carol) :N 0}))
+
+      ;; Stratification: recursion through aggregation is rejected.
+      ;; Aggregate validates that second arg is a variable.
+      (dl-at-test! "agg second arg must be var"
+        (dl-at-throws?
+          (fn () (dl-eval "p(1). q(N) :- count(N, 5, p(X))." "?- q(N).")))
+        true)
+
+      ;; Aggregate validates that third arg is a positive literal.
+      (dl-at-test! "agg third arg must be a literal"
+        (dl-at-throws?
+          (fn () (dl-eval "p(1). q(N) :- count(N, X, 42)." "?- q(N).")))
+        true)
+
+      ;; Aggregate validates that the agg-var (2nd arg) appears in the
+      ;; goal. Without it every match contributes the same unbound
+      ;; symbol — count silently returns 1, sum raises a confusing
+      ;; "expected number" error, etc. Catch the mistake at safety
+      ;; check time instead.
+      (dl-at-test! "agg-var must appear in goal"
+        (dl-at-throws?
+          (fn ()
+            (dl-eval
+              "p(1). p(2). c(N) :- count(N, Y, p(X))."
+              "?- c(N).")))
+        true)
+
+      ;; Indirect recursion through aggregation also rejected.
+      ;; q -> r (via positive lit), r -> q (via aggregate body).
+      ;; The aggregate edge counts as negation for stratification.
+      (dl-at-test! "indirect agg cycle rejected"
+        (dl-at-throws?
+          (fn ()
+            (let ((db (dl-make-db)))
+              (do
+                (dl-add-rule! db
+                  {:head (list (quote q) (quote N))
+                   :body (list (list (quote r) (quote N)))})
+                (dl-add-rule! db
+                  {:head (list (quote r) (quote N))
+                   :body (list (list (quote count) (quote N) (quote X)
+                                     (list (quote q) (quote X))))})
+                (dl-saturate! db)))))
+        true)
+
+      (dl-at-test! "agg recursion rejected"
+        (dl-at-throws?
+          (fn ()
+            (let ((db (dl-make-db)))
+              (do
+                (dl-add-rule! db
+                  {:head (list (quote q) (quote N))
+                   :body (list (list (quote count) (quote N) (quote X)
+                                     (list (quote q) (quote X))))})
+                (dl-saturate! db)))))
+        true)
+
+      ;; Negation + aggregation in the same body — different strata.
+      (dl-at-test-set! "neg + agg coexist"
+        (dl-query
+          (dl-program
+            "u(a). u(b). u(c). banned(b).
+             active(X) :- u(X), not(banned(X)).
+             cnt(N) :- count(N, X, active(X)).")
+          (list (quote cnt) (quote N)))
+        (list {:N 2}))
+
+      ;; Min over a derived empty relation: no result.
+      (dl-at-test-set! "min over empty derived"
+        (dl-query
+          (dl-program
+            "s(50). s(60).
+             score(N) :- s(N), >(N, 100).
+             low(M) :- min(M, X, score(X)).")
+          (list (quote low) (quote M)))
+        (list))
+
+      ;; Aggregates as the top-level query goal (regression for
+      ;; dl-match-lit aggregate dispatch and projection cleanup).
+      (dl-at-test-set! "count as query goal"
+        (dl-query
+          (dl-program "p(1). p(2). p(3). p(4).")
+          (list (quote count) (quote N) (quote X) (list (quote p) (quote X))))
+        (list {:N 4}))
+
+      (dl-at-test-set! "findall as query goal"
+        (dl-query
+          (dl-program "p(1). p(2). p(3).")
+          (list (quote findall) (quote L) (quote X)
+                (list (quote p) (quote X))))
+        (list {:L (list 1 2 3)}))
+
+      (dl-at-test-set! "distinct counted once"
+        (dl-query
+          (dl-program
+            "rated(alice, x). rated(alice, y). rated(bob, x).
+             rater_count(N) :- count(N, U, rated(U, F)).")
+          (list (quote rater_count) (quote N)))
+        (list {:N 2})))))
+
+(define
+  dl-aggregates-tests-run!
+  (fn
+    ()
+    (do
+      (set! dl-at-pass 0)
+      (set! dl-at-fail 0)
+      (set! dl-at-failures (list))
+      (dl-at-run-all!)
+      {:passed dl-at-pass
+       :failed dl-at-fail
+       :total (+ dl-at-pass dl-at-fail)
+       :failures dl-at-failures})))

lib/datalog/tests/api.sx

@@ -0,0 +1,350 @@
+;; lib/datalog/tests/api.sx — SX-data embedding API.
+
+(define dl-api-pass 0)
+(define dl-api-fail 0)
+(define dl-api-failures (list))
+
+(define
+  dl-api-deep=?
+  (fn
+    (a b)
+    (cond
+      ((and (list? a) (list? b))
+        (and (= (len a) (len b)) (dl-api-deq-l? a b 0)))
+      ((and (dict? a) (dict? b))
+        (let ((ka (keys a)) (kb (keys b)))
+          (and (= (len ka) (len kb)) (dl-api-deq-d? a b ka 0))))
+      ((and (number? a) (number? b)) (= a b))
+      (else (equal? a b)))))
+
+(define
+  dl-api-deq-l?
+  (fn
+    (a b i)
+    (cond
+      ((>= i (len a)) true)
+      ((not (dl-api-deep=? (nth a i) (nth b i))) false)
+      (else (dl-api-deq-l? a b (+ i 1))))))
+
+(define
+  dl-api-deq-d?
+  (fn
+    (a b ka i)
+    (cond
+      ((>= i (len ka)) true)
+      ((let ((k (nth ka i)))
+         (not (dl-api-deep=? (get a k) (get b k))))
+        false)
+      (else (dl-api-deq-d? a b ka (+ i 1))))))
+
+(define
+  dl-api-set=?
+  (fn
+    (a b)
+    (and
+      (= (len a) (len b))
+      (dl-api-subset? a b)
+      (dl-api-subset? b a))))
+
+(define
+  dl-api-subset?
+  (fn
+    (xs ys)
+    (cond
+      ((= (len xs) 0) true)
+      ((not (dl-api-contains? ys (first xs))) false)
+      (else (dl-api-subset? (rest xs) ys)))))
+
+(define
+  dl-api-contains?
+  (fn
+    (xs target)
+    (cond
+      ((= (len xs) 0) false)
+      ((dl-api-deep=? (first xs) target) true)
+      (else (dl-api-contains? (rest xs) target)))))
+
+(define
+  dl-api-test!
+  (fn
+    (name got expected)
+    (if
+      (dl-api-deep=? got expected)
+      (set! dl-api-pass (+ dl-api-pass 1))
+      (do
+        (set! dl-api-fail (+ dl-api-fail 1))
+        (append!
+          dl-api-failures
+          (str
+            name
+            "\n    expected: " expected
+            "\n    got:      " got))))))
+
+(define
+  dl-api-test-set!
+  (fn
+    (name got expected)
+    (if
+      (dl-api-set=? got expected)
+      (set! dl-api-pass (+ dl-api-pass 1))
+      (do
+        (set! dl-api-fail (+ dl-api-fail 1))
+        (append!
+          dl-api-failures
+          (str
+            name
+            "\n    expected (set): " expected
+            "\n    got:            " got))))))
+
+(define
+  dl-api-run-all!
+  (fn
+    ()
+    (do
+      ;; dl-program-data with arrow form.
+      (dl-api-test-set! "data API ancestor closure"
+        (dl-query
+          (dl-program-data
+            (quote ((parent tom bob) (parent bob ann) (parent ann pat)))
+            (quote
+              ((ancestor X Y <- (parent X Y))
+               (ancestor X Z <- (parent X Y) (ancestor Y Z)))))
+          (quote (ancestor tom X)))
+        (list {:X (quote bob)} {:X (quote ann)} {:X (quote pat)}))
+
+      ;; dl-program-data with dict rules.
+      (dl-api-test-set! "data API with dict rules"
+        (dl-query
+          (dl-program-data
+            (quote ((p a) (p b) (p c)))
+            (list
+              {:head (quote (q X)) :body (quote ((p X)))}))
+          (quote (q X)))
+        (list {:X (quote a)} {:X (quote b)} {:X (quote c)}))
+
+      ;; dl-rule helper.
+      (dl-api-test-set! "dl-rule constructor"
+        (dl-query
+          (dl-program-data
+            (quote ((p 1) (p 2)))
+            (list (dl-rule (quote (q X)) (quote ((p X))))))
+          (quote (q X)))
+        (list {:X 1} {:X 2}))
+
+      ;; dl-assert! adds and re-derives.
+      (dl-api-test-set! "dl-assert! incremental"
+        (let
+          ((db (dl-program-data
+                 (quote ((parent tom bob) (parent bob ann)))
+                 (quote
+                   ((ancestor X Y <- (parent X Y))
+                    (ancestor X Z <- (parent X Y) (ancestor Y Z)))))))
+          (do
+            (dl-saturate! db)
+            (dl-assert! db (quote (parent ann pat)))
+            (dl-query db (quote (ancestor tom X)))))
+        (list {:X (quote bob)} {:X (quote ann)} {:X (quote pat)}))
+
+      ;; dl-retract! removes a fact and recomputes IDB.
+      (dl-api-test-set! "dl-retract! removes derived"
+        (let
+          ((db (dl-program-data
+                 (quote ((parent tom bob) (parent bob ann) (parent ann pat)))
+                 (quote
+                   ((ancestor X Y <- (parent X Y))
+                    (ancestor X Z <- (parent X Y) (ancestor Y Z)))))))
+          (do
+            (dl-saturate! db)
+            (dl-retract! db (quote (parent bob ann)))
+            (dl-query db (quote (ancestor tom X)))))
+        (list {:X (quote bob)}))
+
+      ;; dl-retract! on a relation with BOTH explicit facts AND a rule
+      ;; (a "mixed" relation) used to wipe the EDB portion when the IDB
+      ;; was re-derived, even when the retract didn't match anything.
+      ;; :edb-keys provenance now preserves user-asserted facts.
+      (dl-api-test-set! "dl-retract! preserves EDB in mixed relation"
+        (let
+          ((db (dl-program-data
+                 (quote ((p a) (p b) (q c)))
+                 (quote ((p X <- (q X)))))))
+          (do
+            (dl-saturate! db)
+            ;; Retract a non-existent tuple — should be a no-op.
+            (dl-retract! db (quote (p z)))
+            (dl-query db (quote (p X)))))
+        (list {:X (quote a)} {:X (quote b)} {:X (quote c)}))
+
+      ;; And retracting an actual EDB fact in a mixed relation drops
+      ;; only that fact; the derived portion stays.
+      (dl-api-test-set! "dl-retract! mixed: drop EDB, keep IDB"
+        (let
+          ((db (dl-program-data
+                 (quote ((p a) (p b) (q c)))
+                 (quote ((p X <- (q X)))))))
+          (do
+            (dl-saturate! db)
+            (dl-retract! db (quote (p a)))
+            (dl-query db (quote (p X)))))
+        (list {:X (quote b)} {:X (quote c)}))
+
+      ;; dl-program-data + dl-query with constants in head.
+      (dl-api-test-set! "constant-in-head data"
+        (dl-query
+          (dl-program-data
+            (quote ((edge a b) (edge b c) (edge c a)))
+            (quote
+              ((reach X Y <- (edge X Y))
+               (reach X Z <- (edge X Y) (reach Y Z)))))
+          (quote (reach a X)))
+        (list {:X (quote a)} {:X (quote b)} {:X (quote c)}))
+
+      ;; Assert into empty db.
+      (dl-api-test-set! "assert into empty"
+        (let
+          ((db (dl-program-data (list) (list))))
+          (do
+            (dl-assert! db (quote (p 1)))
+            (dl-assert! db (quote (p 2)))
+            (dl-query db (quote (p X)))))
+        (list {:X 1} {:X 2}))
+
+      ;; Multi-goal query: pass list of literals.
+      (dl-api-test-set! "multi-goal query"
+        (dl-query
+          (dl-program-data
+            (quote ((p 1) (p 2) (p 3) (q 2) (q 3)))
+            (list))
+          (list (quote (p X)) (quote (q X))))
+        (list {:X 2} {:X 3}))
+
+      ;; Multi-goal with comparison.
+      (dl-api-test-set! "multi-goal with comparison"
+        (dl-query
+          (dl-program-data
+            (quote ((n 1) (n 2) (n 3) (n 4) (n 5)))
+            (list))
+          (list (quote (n X)) (list (string->symbol ">") (quote X) 2)))
+        (list {:X 3} {:X 4} {:X 5}))
+
+      ;; dl-eval: single-call source + query.
+      (dl-api-test-set! "dl-eval ancestor"
+        (dl-eval
+          "parent(a, b). parent(b, c).
+           ancestor(X, Y) :- parent(X, Y).
+           ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z)."
+          "?- ancestor(a, X).")
+        (list {:X (quote b)} {:X (quote c)}))
+
+      (dl-api-test-set! "dl-eval multi-goal"
+        (dl-eval
+          "p(1). p(2). p(3). q(2). q(3)."
+          "?- p(X), q(X).")
+        (list {:X 2} {:X 3}))
+
+      ;; dl-rules-of: rules with head matching a relation name.
+      (dl-api-test! "dl-rules-of count"
+        (let
+          ((db (dl-program
+                 "p(1). q(X) :- p(X). r(X) :- p(X). q(2).")))
+          (len (dl-rules-of db "q")))
+        1)
+
+      (dl-api-test! "dl-rules-of empty"
+        (let
+          ((db (dl-program "p(1). p(2).")))
+          (len (dl-rules-of db "q")))
+        0)
+
+      ;; dl-clear-idb!: wipe rule-headed relations.
+      (dl-api-test! "dl-clear-idb! wipes IDB"
+        (let
+          ((db (dl-program
+                 "parent(a, b). parent(b, c).
+                  ancestor(X, Y) :- parent(X, Y).
+                  ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z).")))
+          (do
+            (dl-saturate! db)
+            (dl-clear-idb! db)
+            (len (dl-relation db "ancestor"))))
+        0)
+
+      (dl-api-test! "dl-clear-idb! preserves EDB"
+        (let
+          ((db (dl-program
+                 "parent(a, b). parent(b, c).
+                  ancestor(X, Y) :- parent(X, Y).")))
+          (do
+            (dl-saturate! db)
+            (dl-clear-idb! db)
+            (len (dl-relation db "parent"))))
+        2)
+
+      ;; dl-eval-magic — routes single-goal queries through
+      ;; magic-sets evaluation.
+      (dl-api-test-set! "dl-eval-magic ancestor"
+        (dl-eval-magic
+          "parent(a, b). parent(b, c).
+           ancestor(X, Y) :- parent(X, Y).
+           ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z)."
+          "?- ancestor(a, X).")
+        (list {:X (quote b)} {:X (quote c)}))
+
+      ;; Equivalence: dl-eval and dl-eval-magic produce the same
+      ;; answers for any well-formed query (magic-sets is a perf
+      ;; alternative, not a semantic change).
+      (dl-api-test! "dl-eval ≡ dl-eval-magic on ancestor"
+        (let
+          ((source "parent(a, b). parent(b, c). parent(c, d).
+                    ancestor(X, Y) :- parent(X, Y).
+                    ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z)."))
+          (let
+            ((semi (dl-eval source "?- ancestor(a, X)."))
+              (magic (dl-eval-magic source "?- ancestor(a, X).")))
+            (= (len semi) (len magic))))
+        true)
+
+      ;; Comprehensive integration: recursion + stratified negation
+      ;; + aggregation + comparison composed in a single program.
+      ;; (Uses _Anything as a regular var instead of `_` so the
+      ;;  outer rule binds via the reach lit.)
+      (dl-api-test-set! "integration"
+        (dl-eval
+          (str
+            "edge(a, b). edge(b, c). edge(c, d). edge(a, d). "
+            "banned(c). "
+            "reach(X, Y) :- edge(X, Y). "
+            "reach(X, Z) :- edge(X, Y), reach(Y, Z). "
+            "safe(X, Y) :- reach(X, Y), not(banned(Y)). "
+            "reach_count(X, N) :- reach(X, Z), count(N, Y, safe(X, Y)). "
+            "popular(X) :- reach_count(X, N), >=(N, 2).")
+          "?- popular(X).")
+        (list {:X (quote a)}))
+
+      ;; dl-rule-from-list with no arrow → fact-style.
+      (dl-api-test-set! "no arrow → fact-like rule"
+        (let
+          ((rule (dl-rule-from-list (quote (foo X Y)))))
+          (list rule))
+        (list {:head (quote (foo X Y)) :body (list)}))
+
+      ;; dl-coerce-rule on dict passes through.
+      (dl-api-test-set! "coerce dict rule"
+        (let
+          ((d {:head (quote (h X)) :body (quote ((b X)))}))
+          (list (dl-coerce-rule d)))
+        (list {:head (quote (h X)) :body (quote ((b X)))})))))
+
+(define
+  dl-api-tests-run!
+  (fn
+    ()
+    (do
+      (set! dl-api-pass 0)
+      (set! dl-api-fail 0)
+      (set! dl-api-failures (list))
+      (dl-api-run-all!)
+      {:passed dl-api-pass
+       :failed dl-api-fail
+       :total (+ dl-api-pass dl-api-fail)
+       :failures dl-api-failures})))

lib/datalog/tests/builtins.sx

@@ -0,0 +1,285 @@
+;; lib/datalog/tests/builtins.sx — comparison + arithmetic body literals.
+
+(define dl-bt-pass 0)
+(define dl-bt-fail 0)
+(define dl-bt-failures (list))
+
+(define
+  dl-bt-deep=?
+  (fn
+    (a b)
+    (cond
+      ((and (list? a) (list? b))
+        (and (= (len a) (len b)) (dl-bt-deq-l? a b 0)))
+      ((and (dict? a) (dict? b))
+        (let
+          ((ka (keys a)) (kb (keys b)))
+          (and (= (len ka) (len kb)) (dl-bt-deq-d? a b ka 0))))
+      ((and (number? a) (number? b)) (= a b))
+      (else (equal? a b)))))
+
+(define
+  dl-bt-deq-l?
+  (fn
+    (a b i)
+    (cond
+      ((>= i (len a)) true)
+      ((not (dl-bt-deep=? (nth a i) (nth b i))) false)
+      (else (dl-bt-deq-l? a b (+ i 1))))))
+
+(define
+  dl-bt-deq-d?
+  (fn
+    (a b ka i)
+    (cond
+      ((>= i (len ka)) true)
+      ((let ((k (nth ka i))) (not (dl-bt-deep=? (get a k) (get b k))))
+        false)
+      (else (dl-bt-deq-d? a b ka (+ i 1))))))
+
+(define
+  dl-bt-set=?
+  (fn
+    (a b)
+    (and (= (len a) (len b)) (dl-bt-subset? a b) (dl-bt-subset? b a))))
+
+(define
+  dl-bt-subset?
+  (fn
+    (xs ys)
+    (cond
+      ((= (len xs) 0) true)
+      ((not (dl-bt-contains? ys (first xs))) false)
+      (else (dl-bt-subset? (rest xs) ys)))))
+
+(define
+  dl-bt-contains?
+  (fn
+    (xs target)
+    (cond
+      ((= (len xs) 0) false)
+      ((dl-bt-deep=? (first xs) target) true)
+      (else (dl-bt-contains? (rest xs) target)))))
+
+(define
+  dl-bt-test-set!
+  (fn
+    (name got expected)
+    (if
+      (dl-bt-set=? got expected)
+      (set! dl-bt-pass (+ dl-bt-pass 1))
+      (do
+        (set! dl-bt-fail (+ dl-bt-fail 1))
+        (append!
+          dl-bt-failures
+          (str
+            name
+            "\n    expected (set): "
+            expected
+            "\n    got:            "
+            got))))))
+
+(define
+  dl-bt-test!
+  (fn
+    (name got expected)
+    (if
+      (dl-bt-deep=? got expected)
+      (set! dl-bt-pass (+ dl-bt-pass 1))
+      (do
+        (set! dl-bt-fail (+ dl-bt-fail 1))
+        (append!
+          dl-bt-failures
+          (str name "\n    expected: " expected "\n    got:      " got))))))
+
+(define
+  dl-bt-throws?
+  (fn
+    (thunk)
+    (let
+      ((threw false))
+      (do (guard (e (#t (set! threw true))) (thunk)) threw))))
+
+(define
+  dl-bt-run-all!
+  (fn
+    ()
+    (do
+      (dl-bt-test-set!
+        "less than filter"
+        (dl-query
+          (dl-program
+            "age(alice, 30). age(bob, 17). age(carol, 22).\n             adult(X) :- age(X, A), >=(A, 18).")
+          (list (quote adult) (quote X)))
+        (list {:X (quote alice)} {:X (quote carol)}))
+      (dl-bt-test-set!
+        "less-equal filter"
+        (dl-query
+          (dl-program
+            "n(1). n(2). n(3). n(4). n(5).\n             small(X) :- n(X), <=(X, 3).")
+          (list (quote small) (quote X)))
+        (list {:X 1} {:X 2} {:X 3}))
+      (dl-bt-test-set!
+        "not-equal filter"
+        (dl-query
+          (dl-program
+            "p(1, 2). p(2, 2). p(3, 4).\n             diff(X, Y) :- p(X, Y), !=(X, Y).")
+          (list (quote diff) (quote X) (quote Y)))
+        (list {:X 1 :Y 2} {:X 3 :Y 4}))
+      (dl-bt-test-set!
+        "is plus"
+        (dl-query
+          (dl-program
+            "n(1). n(2). n(3).\n             succ(X, Y) :- n(X), is(Y, +(X, 1)).")
+          (list (quote succ) (quote X) (quote Y)))
+        (list {:X 1 :Y 2} {:X 2 :Y 3} {:X 3 :Y 4}))
+      (dl-bt-test-set!
+        "is multiply"
+        (dl-query
+          (dl-program
+            "n(2). n(3). n(4).\n             square(X, Y) :- n(X), is(Y, *(X, X)).")
+          (list (quote square) (quote X) (quote Y)))
+        (list {:X 2 :Y 4} {:X 3 :Y 9} {:X 4 :Y 16}))
+      (dl-bt-test-set!
+        "is nested expr"
+        (dl-query
+          (dl-program
+            "n(1). n(2). n(3).\n             f(X, Y) :- n(X), is(Y, *(+(X, 1), 2)).")
+          (list (quote f) (quote X) (quote Y)))
+        (list {:X 1 :Y 4} {:X 2 :Y 6} {:X 3 :Y 8}))
+      (dl-bt-test-set!
+        "is bound LHS — equality"
+        (dl-query
+          (dl-program
+            "n(1, 2). n(2, 5). n(3, 4).\n             succ(X, Y) :- n(X, Y), is(Y, +(X, 1)).")
+          (list (quote succ) (quote X) (quote Y)))
+        (list {:X 1 :Y 2} {:X 3 :Y 4}))
+      (dl-bt-test-set!
+        "triple via is"
+        (dl-query
+          (dl-program
+            "n(1). n(2). n(3).\n             triple(X, Y) :- n(X), is(Y, *(X, 3)).")
+          (list (quote triple) (quote X) (quote Y)))
+        (list {:X 1 :Y 3} {:X 2 :Y 6} {:X 3 :Y 9}))
+      (dl-bt-test-set!
+        "= unifies var with constant"
+        (dl-query
+          (dl-program "p(a). p(b).\n             qual(X) :- p(X), =(X, a).")
+          (list (quote qual) (quote X)))
+        (list {:X (quote a)}))
+      (dl-bt-test-set!
+        "= unifies two vars (one bound)"
+        (dl-query
+          (dl-program "p(a). p(b).\n             twin(X, Y) :- p(X), =(Y, X).")
+          (list (quote twin) (quote X) (quote Y)))
+        (list {:X (quote a) :Y (quote a)} {:X (quote b) :Y (quote b)}))
+      (dl-bt-test!
+        "big count"
+        (let
+          ((db (dl-program "n(0). n(1). n(2). n(3). n(4). n(5). n(6). n(7). n(8). n(9).\n                  big(X) :- n(X), >=(X, 5).")))
+          (do (dl-saturate! db) (len (dl-relation db "big"))))
+        5)
+      ;; Built-in / arithmetic literals work as standalone query goals
+      ;; (without needing a wrapper rule).
+      (dl-bt-test-set! "comparison-only goal true"
+        (dl-eval "" "?- <(1, 2).")
+        (list {}))
+
+      (dl-bt-test-set! "comparison-only goal false"
+        (dl-eval "" "?- <(2, 1).")
+        (list))
+
+      (dl-bt-test-set! "is goal binds"
+        (dl-eval "" "?- is(N, +(2, 3)).")
+        (list {:N 5}))
+
+      ;; Bounded successor: a recursive rule with a comparison
+      ;; guard terminates because the Herbrand base is effectively
+      ;; bounded.
+      (dl-bt-test-set! "bounded successor"
+        (dl-query
+          (dl-program
+            "nat(0).
+             nat(Y) :- nat(X), is(Y, +(X, 1)), <(Y, 5).")
+          (list (quote nat) (quote X)))
+        (list {:X 0} {:X 1} {:X 2} {:X 3} {:X 4}))
+
+      (dl-bt-test!
+        "unsafe — comparison without binder"
+        (dl-bt-throws? (fn () (dl-program "p(X) :- <(X, 5).")))
+        true)
+      (dl-bt-test!
+        "unsafe — comparison both unbound"
+        (dl-bt-throws? (fn () (dl-program "p(X, Y) :- <(X, Y), q(X).")))
+        true)
+      (dl-bt-test!
+        "unsafe — is uses unbound RHS var"
+        (dl-bt-throws?
+          (fn () (dl-program "p(X, Y) :- q(X), is(Y, +(X, Z)).")))
+        true)
+      (dl-bt-test!
+        "unsafe — is on its own"
+        (dl-bt-throws? (fn () (dl-program "p(Y) :- is(Y, +(X, 1)).")))
+        true)
+      (dl-bt-test!
+        "unsafe — = between two unbound"
+        (dl-bt-throws? (fn () (dl-program "p(X, Y) :- =(X, Y).")))
+        true)
+      (dl-bt-test!
+        "safe — is binds head var"
+        (dl-bt-throws?
+          (fn () (dl-program "n(1). p(Y) :- n(X), is(Y, +(X, 1)).")))
+        false)
+      (dl-bt-test!
+        "safe — comparison after binder"
+        (dl-bt-throws?
+          (fn () (dl-program "n(1). big(X) :- n(X), >=(X, 0).")))
+        false)
+      (dl-bt-test!
+        "safe — = binds head var"
+        (dl-bt-throws?
+          (fn () (dl-program "p(a). p(b). x(Y) :- p(X), =(Y, X).")))
+        false)
+
+      ;; Division by zero raises with a clear error. Without this guard
+      ;; SX's `/` returned IEEE infinity, which then silently flowed
+      ;; through comparisons and aggregations.
+      (dl-bt-test!
+        "is — division by zero raises"
+        (dl-bt-throws?
+          (fn ()
+            (dl-eval "p(10). q(R) :- p(X), is(R, /(X, 0))." "?- q(R).")))
+        true)
+
+      ;; Comparison ops `<`, `<=`, `>`, `>=` require both operands to
+      ;; have the same primitive type. Cross-type comparisons used to
+      ;; silently return false (for some pairs) or raise a confusing
+      ;; host-level error (for others) — now they all raise with a
+      ;; message that names the offending values.
+      (dl-bt-test!
+        "comparison — string vs number raises"
+        (dl-bt-throws?
+          (fn ()
+            (dl-eval "p(\"hello\"). q(X) :- p(X), <(X, 5)." "?- q(X).")))
+        true)
+
+      ;; `!=` is the exception — it's a polymorphic inequality test
+      ;; (uses dl-tuple-equal? underneath) so cross-type pairs are
+      ;; legitimate (and trivially unequal).
+      (dl-bt-test-set! "!= works across types"
+        (dl-query
+          (dl-program
+            "p(1). p(\"1\"). q(X) :- p(X), !=(X, 1).")
+          (quote (q X)))
+        (list {:X "1"})))))
+
+(define
+  dl-builtins-tests-run!
+  (fn
+    ()
+    (do
+      (set! dl-bt-pass 0)
+      (set! dl-bt-fail 0)
+      (set! dl-bt-failures (list))
+      (dl-bt-run-all!)
+      {:failures dl-bt-failures :total (+ dl-bt-pass dl-bt-fail) :passed dl-bt-pass :failed dl-bt-fail})))

lib/datalog/tests/demo.sx

@@ -0,0 +1,321 @@
+;; lib/datalog/tests/demo.sx — Phase 10 demo programs.
+
+(define dl-demo-pass 0)
+(define dl-demo-fail 0)
+(define dl-demo-failures (list))
+
+(define
+  dl-demo-deep=?
+  (fn
+    (a b)
+    (cond
+      ((and (list? a) (list? b))
+        (and (= (len a) (len b)) (dl-demo-deq-l? a b 0)))
+      ((and (dict? a) (dict? b))
+        (let ((ka (keys a)) (kb (keys b)))
+          (and (= (len ka) (len kb)) (dl-demo-deq-d? a b ka 0))))
+      ((and (number? a) (number? b)) (= a b))
+      (else (equal? a b)))))
+
+(define
+  dl-demo-deq-l?
+  (fn
+    (a b i)
+    (cond
+      ((>= i (len a)) true)
+      ((not (dl-demo-deep=? (nth a i) (nth b i))) false)
+      (else (dl-demo-deq-l? a b (+ i 1))))))
+
+(define
+  dl-demo-deq-d?
+  (fn
+    (a b ka i)
+    (cond
+      ((>= i (len ka)) true)
+      ((let ((k (nth ka i)))
+         (not (dl-demo-deep=? (get a k) (get b k))))
+        false)
+      (else (dl-demo-deq-d? a b ka (+ i 1))))))
+
+(define
+  dl-demo-set=?
+  (fn
+    (a b)
+    (and
+      (= (len a) (len b))
+      (dl-demo-subset? a b)
+      (dl-demo-subset? b a))))
+
+(define
+  dl-demo-subset?
+  (fn
+    (xs ys)
+    (cond
+      ((= (len xs) 0) true)
+      ((not (dl-demo-contains? ys (first xs))) false)
+      (else (dl-demo-subset? (rest xs) ys)))))
+
+(define
+  dl-demo-contains?
+  (fn
+    (xs target)
+    (cond
+      ((= (len xs) 0) false)
+      ((dl-demo-deep=? (first xs) target) true)
+      (else (dl-demo-contains? (rest xs) target)))))
+
+(define
+  dl-demo-test-set!
+  (fn
+    (name got expected)
+    (if
+      (dl-demo-set=? got expected)
+      (set! dl-demo-pass (+ dl-demo-pass 1))
+      (do
+        (set! dl-demo-fail (+ dl-demo-fail 1))
+        (append!
+          dl-demo-failures
+          (str
+            name
+            "\n    expected (set): " expected
+            "\n    got:            " got))))))
+
+(define
+  dl-demo-run-all!
+  (fn
+    ()
+    (do
+      ;; ── Federation ──────────────────────────────────────────
+      (dl-demo-test-set! "mutuals"
+        (dl-query
+          (dl-demo-make
+            (quote ((follows alice bob) (follows bob alice)
+                    (follows bob carol) (follows carol dave)))
+            dl-demo-federation-rules)
+          (quote (mutual alice X)))
+        (list {:X (quote bob)}))
+
+      (dl-demo-test-set! "reachable transitive"
+        (dl-query
+          (dl-demo-make
+            (quote ((follows alice bob) (follows bob carol) (follows carol dave)))
+            dl-demo-federation-rules)
+          (quote (reachable alice X)))
+        (list {:X (quote bob)} {:X (quote carol)} {:X (quote dave)}))
+
+      (dl-demo-test-set! "foaf"
+        (dl-query
+          (dl-demo-make
+            (quote ((follows alice bob) (follows bob carol) (follows alice dave)))
+            dl-demo-federation-rules)
+          (quote (foaf alice X)))
+        (list {:X (quote carol)}))
+
+      ;; ── Content ─────────────────────────────────────────────
+      (dl-demo-test-set! "popular posts"
+        (dl-query
+          (dl-demo-make
+            (quote
+              ((authored alice p1) (authored bob p2) (authored carol p3)
+               (liked u1 p1) (liked u2 p1) (liked u3 p1)
+               (liked u1 p2)))
+            dl-demo-content-rules)
+          (quote (popular P)))
+        (list {:P (quote p1)}))
+
+      (dl-demo-test-set! "interesting feed"
+        (dl-query
+          (dl-demo-make
+            (quote
+              ((follows me alice) (follows me bob)
+               (authored alice p1) (authored bob p2)
+               (liked u1 p1) (liked u2 p1) (liked u3 p1)
+               (liked u4 p2)))
+            dl-demo-content-rules)
+          (quote (interesting me P)))
+        (list {:P (quote p1)}))
+
+      (dl-demo-test-set! "post likes count"
+        (dl-query
+          (dl-demo-make
+            (quote
+              ((authored alice p1)
+               (liked u1 p1) (liked u2 p1) (liked u3 p1)))
+            dl-demo-content-rules)
+          (quote (post-likes p1 N)))
+        (list {:N 3}))
+
+      ;; ── Permissions ─────────────────────────────────────────
+      (dl-demo-test-set! "direct group access"
+        (dl-query
+          (dl-demo-make
+            (quote
+              ((member alice editors)
+               (allowed editors blog)))
+            dl-demo-perm-rules)
+          (quote (can-access X blog)))
+        (list {:X (quote alice)}))
+
+      (dl-demo-test-set! "subgroup access"
+        (dl-query
+          (dl-demo-make
+            (quote
+              ((member bob writers)
+               (subgroup writers editors)
+               (allowed editors blog)))
+            dl-demo-perm-rules)
+          (quote (can-access X blog)))
+        (list {:X (quote bob)}))
+
+      (dl-demo-test-set! "transitive subgroup"
+        (dl-query
+          (dl-demo-make
+            (quote
+              ((member carol drafters)
+               (subgroup drafters writers)
+               (subgroup writers editors)
+               (allowed editors blog)))
+            dl-demo-perm-rules)
+          (quote (can-access X blog)))
+        (list {:X (quote carol)}))
+
+      ;; ── Cooking posts (canonical Phase 10 example) ─────────
+      (dl-demo-test-set! "cooking posts by network"
+        (dl-query
+          (dl-demo-make
+            (quote
+              ((follows me alice) (follows alice bob) (follows alice carol)
+               (authored alice p1) (authored bob p2)
+               (authored carol p3) (authored carol p4)
+               (tagged p1 travel) (tagged p2 cooking)
+               (tagged p3 cooking) (tagged p4 books)))
+            dl-demo-cooking-rules)
+          (quote (cooking-post-by-network me P)))
+        (list {:P (quote p2)} {:P (quote p3)}))
+
+      (dl-demo-test-set! "cooking — direct follow only"
+        (dl-query
+          (dl-demo-make
+            (quote
+              ((follows me bob)
+               (authored bob p1) (authored bob p2)
+               (tagged p1 cooking) (tagged p2 books)))
+            dl-demo-cooking-rules)
+          (quote (cooking-post-by-network me P)))
+        (list {:P (quote p1)}))
+
+      (dl-demo-test-set! "cooking — none in network"
+        (dl-query
+          (dl-demo-make
+            (quote
+              ((follows me bob)
+               (authored bob p1) (tagged p1 books)))
+            dl-demo-cooking-rules)
+          (quote (cooking-post-by-network me P)))
+        (list))
+
+      ;; ── Tag co-occurrence ──────────────────────────────────
+      (dl-demo-test-set! "cotagged posts"
+        (dl-query
+          (dl-demo-make
+            (quote
+              ((tagged p1 cooking) (tagged p1 vegetarian)
+               (tagged p2 cooking) (tagged p2 quick)
+               (tagged p3 vegetarian)))
+            dl-demo-tag-cooccur-rules)
+          (quote (cotagged P cooking vegetarian)))
+        (list {:P (quote p1)}))
+
+      (dl-demo-test-set! "tag pair count"
+        (dl-query
+          (dl-demo-make
+            (quote
+              ((tagged p1 cooking) (tagged p1 vegetarian)
+               (tagged p2 cooking) (tagged p2 quick)
+               (tagged p3 cooking) (tagged p3 vegetarian)))
+            dl-demo-tag-cooccur-rules)
+          (quote (tag-pair-count cooking vegetarian N)))
+        (list {:N 2}))
+
+      ;; ── Shortest path on a weighted DAG ──────────────────
+      (dl-demo-test-set! "shortest a→d via DAG"
+        (dl-query
+          (dl-demo-make
+            (quote ((edge a b 5) (edge b c 3) (edge a c 10) (edge c d 2)))
+            dl-demo-shortest-path-rules)
+          (quote (shortest a d W)))
+        (list {:W 10}))
+
+      (dl-demo-test-set! "shortest a→c picks 2-hop"
+        (dl-query
+          (dl-demo-make
+            (quote ((edge a b 5) (edge b c 3) (edge a c 10)))
+            dl-demo-shortest-path-rules)
+          (quote (shortest a c W)))
+        (list {:W 8}))
+
+      (dl-demo-test-set! "shortest unreachable empty"
+        (dl-query
+          (dl-demo-make
+            (quote ((edge a b 5) (edge b c 3)))
+            dl-demo-shortest-path-rules)
+          (quote (shortest a d W)))
+        (list))
+
+      ;; ── Org chart + headcount ─────────────────────────────
+      (dl-demo-test-set! "ceo subordinate transitive"
+        (dl-query
+          (dl-demo-make
+            (quote
+              ((manager ic1 mgr1) (manager ic2 mgr1)
+               (manager mgr1 vp1) (manager ic3 vp1)
+               (manager vp1 ceo)))
+            dl-demo-org-rules)
+          (quote (subordinate ceo X)))
+        (list
+          {:X (quote vp1)} {:X (quote mgr1)} {:X (quote ic1)}
+          {:X (quote ic2)} {:X (quote ic3)}))
+
+      (dl-demo-test-set! "ceo headcount = 5"
+        (dl-query
+          (dl-demo-make
+            (quote
+              ((manager ic1 mgr1) (manager ic2 mgr1)
+               (manager mgr1 vp1) (manager ic3 vp1)
+               (manager vp1 ceo)))
+            dl-demo-org-rules)
+          (quote (headcount ceo N)))
+        (list {:N 5}))
+
+      (dl-demo-test-set! "mgr1 headcount = 2"
+        (dl-query
+          (dl-demo-make
+            (quote
+              ((manager ic1 mgr1) (manager ic2 mgr1)
+               (manager mgr1 vp1) (manager ic3 vp1)
+               (manager vp1 ceo)))
+            dl-demo-org-rules)
+          (quote (headcount mgr1 N)))
+        (list {:N 2}))
+
+      (dl-demo-test-set! "no access without grant"
+        (dl-query
+          (dl-demo-make
+            (quote ((member dave outsiders) (allowed editors blog)))
+            dl-demo-perm-rules)
+          (quote (can-access X blog)))
+        (list)))))
+
+(define
+  dl-demo-tests-run!
+  (fn
+    ()
+    (do
+      (set! dl-demo-pass 0)
+      (set! dl-demo-fail 0)
+      (set! dl-demo-failures (list))
+      (dl-demo-run-all!)
+      {:passed dl-demo-pass
+       :failed dl-demo-fail
+       :total (+ dl-demo-pass dl-demo-fail)
+       :failures dl-demo-failures})))

lib/datalog/tests/eval.sx

@@ -0,0 +1,463 @@
+;; lib/datalog/tests/eval.sx — naive evaluation + safety analysis tests.
+
+(define dl-et-pass 0)
+(define dl-et-fail 0)
+(define dl-et-failures (list))
+
+;; Same deep-equal helper used in other suites.
+(define
+  dl-et-deep=?
+  (fn
+    (a b)
+    (cond
+      ((and (list? a) (list? b))
+        (and (= (len a) (len b)) (dl-et-deq-l? a b 0)))
+      ((and (dict? a) (dict? b))
+        (let
+          ((ka (keys a)) (kb (keys b)))
+          (and (= (len ka) (len kb)) (dl-et-deq-d? a b ka 0))))
+      ((and (number? a) (number? b)) (= a b))
+      (else (equal? a b)))))
+
+(define
+  dl-et-deq-l?
+  (fn
+    (a b i)
+    (cond
+      ((>= i (len a)) true)
+      ((not (dl-et-deep=? (nth a i) (nth b i))) false)
+      (else (dl-et-deq-l? a b (+ i 1))))))
+
+(define
+  dl-et-deq-d?
+  (fn
+    (a b ka i)
+    (cond
+      ((>= i (len ka)) true)
+      ((let ((k (nth ka i))) (not (dl-et-deep=? (get a k) (get b k))))
+        false)
+      (else (dl-et-deq-d? a b ka (+ i 1))))))
+
+;; Set-equality on lists (order-independent, uses dl-et-deep=?).
+(define
+  dl-et-set=?
+  (fn
+    (a b)
+    (and (= (len a) (len b)) (dl-et-subset? a b) (dl-et-subset? b a))))
+
+(define
+  dl-et-subset?
+  (fn
+    (xs ys)
+    (cond
+      ((= (len xs) 0) true)
+      ((not (dl-et-contains? ys (first xs))) false)
+      (else (dl-et-subset? (rest xs) ys)))))
+
+(define
+  dl-et-contains?
+  (fn
+    (xs target)
+    (cond
+      ((= (len xs) 0) false)
+      ((dl-et-deep=? (first xs) target) true)
+      (else (dl-et-contains? (rest xs) target)))))
+
+(define
+  dl-et-test!
+  (fn
+    (name got expected)
+    (if
+      (dl-et-deep=? got expected)
+      (set! dl-et-pass (+ dl-et-pass 1))
+      (do
+        (set! dl-et-fail (+ dl-et-fail 1))
+        (append!
+          dl-et-failures
+          (str name "\n    expected: " expected "\n    got:      " got))))))
+
+(define
+  dl-et-test-set!
+  (fn
+    (name got expected)
+    (if
+      (dl-et-set=? got expected)
+      (set! dl-et-pass (+ dl-et-pass 1))
+      (do
+        (set! dl-et-fail (+ dl-et-fail 1))
+        (append!
+          dl-et-failures
+          (str
+            name
+            "\n    expected (set): "
+            expected
+            "\n    got:            "
+            got))))))
+
+(define
+  dl-et-throws?
+  (fn
+    (thunk)
+    (let
+      ((threw false))
+      (do (guard (e (#t (set! threw true))) (thunk)) threw))))
+
+(define
+  dl-et-run-all!
+  (fn
+    ()
+    (do
+      (dl-et-test-set!
+        "fact lookup any"
+        (dl-query
+          (dl-program "parent(tom, bob). parent(bob, ann).")
+          (list (quote parent) (quote X) (quote Y)))
+        (list {:X (quote tom) :Y (quote bob)} {:X (quote bob) :Y (quote ann)}))
+      (dl-et-test-set!
+        "fact lookup constant arg"
+        (dl-query
+          (dl-program "parent(tom, bob). parent(tom, liz). parent(bob, ann).")
+          (list (quote parent) (quote tom) (quote Y)))
+        (list {:Y (quote bob)} {:Y (quote liz)}))
+      (dl-et-test-set!
+        "no match"
+        (dl-query
+          (dl-program "parent(tom, bob).")
+          (list (quote parent) (quote nobody) (quote X)))
+        (list))
+      (dl-et-test-set!
+        "ancestor closure"
+        (dl-query
+          (dl-program
+            "parent(tom, bob). parent(bob, ann). parent(ann, pat).\n             ancestor(X, Y) :- parent(X, Y).\n             ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z).")
+          (list (quote ancestor) (quote tom) (quote X)))
+        (list {:X (quote bob)} {:X (quote ann)} {:X (quote pat)}))
+      (dl-et-test-set!
+        "sibling"
+        (dl-query
+          (dl-program
+            "parent(tom, bob). parent(tom, liz). parent(jane, bob). parent(jane, liz).\n             sibling(X, Y) :- parent(P, X), parent(P, Y).")
+          (list (quote sibling) (quote bob) (quote Y)))
+        (list {:Y (quote bob)} {:Y (quote liz)}))
+      (dl-et-test-set!
+        "same-generation"
+        (dl-query
+          (dl-program
+            "parent(tom, bob). parent(tom, liz). parent(bob, ann). parent(liz, joe).\n             person(tom). person(bob). person(liz). person(ann). person(joe).\n             sg(X, X) :- person(X).\n             sg(X, Y) :- parent(P1, X), sg(P1, P2), parent(P2, Y).")
+          (list (quote sg) (quote ann) (quote X)))
+        (list {:X (quote ann)} {:X (quote joe)}))
+      (dl-et-test!
+        "ancestor count"
+        (let
+          ((db (dl-program "parent(a, b). parent(b, c). parent(c, d).\n                  ancestor(X, Y) :- parent(X, Y).\n                  ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z).")))
+          (do (dl-saturate! db) (len (dl-relation db "ancestor"))))
+        6)
+      (dl-et-test-set!
+        "grandparent"
+        (dl-query
+          (dl-program
+            "parent(a, b). parent(b, c). parent(c, d).\n             grandparent(X, Z) :- parent(X, Y), parent(Y, Z).")
+          (list (quote grandparent) (quote X) (quote Y)))
+        (list {:X (quote a) :Y (quote c)} {:X (quote b) :Y (quote d)}))
+      (dl-et-test!
+        "no recursion infinite loop"
+        (let
+          ((db (dl-program "edge(1, 2). edge(2, 3). edge(3, 1).\n                  reach(X, Y) :- edge(X, Y).\n                  reach(X, Z) :- edge(X, Y), reach(Y, Z).")))
+          (do (dl-saturate! db) (len (dl-relation db "reach"))))
+        9)
+      ;; Rule-shape sanity: empty-list head and non-list body raise
+      ;; clear errors rather than crashing inside the saturator.
+      (dl-et-test! "empty head rejected"
+        (dl-et-throws?
+          (fn ()
+            (dl-add-rule! (dl-make-db)
+              {:head (list) :body (list)})))
+        true)
+
+      (dl-et-test! "non-list body rejected"
+        (dl-et-throws?
+          (fn ()
+            (dl-add-rule! (dl-make-db)
+              {:head (list (quote p) (quote X)) :body 42})))
+        true)
+
+      ;; Reserved relation names rejected as rule/fact heads.
+      (dl-et-test!
+        "reserved name `not` as head rejected"
+        (dl-et-throws? (fn () (dl-program "not(X) :- p(X).")))
+        true)
+
+      (dl-et-test!
+        "reserved name `count` as head rejected"
+        (dl-et-throws?
+          (fn () (dl-program "count(N, X, p(X)) :- p(X).")))
+        true)
+
+      (dl-et-test!
+        "reserved name `<` as head rejected"
+        (dl-et-throws? (fn () (dl-program "<(X, 5) :- p(X).")))
+        true)
+
+      (dl-et-test!
+        "reserved name `is` as head rejected"
+        (dl-et-throws? (fn () (dl-program "is(N, +(1, 2)) :- p(N).")))
+        true)
+
+      ;; Body literal with a reserved-name positive head is rejected.
+      ;; The parser only treats outer-level `not(P)` as negation; nested
+      ;; `not(not(P))` would otherwise silently parse as a positive call
+      ;; to a relation named `not` and succeed vacuously. The safety
+      ;; checker now flags this so the user gets a clear error.
+      ;; Body literal with a reserved-name positive head is rejected.
+      ;; The parser only treats outer-level `not(P)` as negation; nested
+      ;; `not(not(P))` would otherwise silently parse as a positive call
+      ;; to a relation named `not` and succeed vacuously — so the safety
+      ;; checker now flags this to give the user a clear error.
+      (dl-et-test!
+        "nested not(not(...)) rejected"
+        (dl-et-throws?
+          (fn ()
+            (dl-program
+              "banned(a). u(a). vip(X) :- u(X), not(not(banned(X))).")))
+        true)
+
+      ;; A dict body literal that isn't `{:neg ...}` is almost always a
+      ;; typo — it would otherwise silently fall through to a confusing
+      ;; head-var-unbound safety error. Now caught with a clear message.
+      (dl-et-test!
+        "dict body lit without :neg rejected"
+        (dl-et-throws?
+          (fn ()
+            (let ((db (dl-make-db)))
+              (dl-add-rule! db
+                {:head (list (quote p) (quote X))
+                 :body (list {:weird "stuff"})}))))
+        true)
+
+      ;; Facts may only have simple-term args (number / string / symbol).
+      ;; A compound arg like `+(1, 2)` would otherwise be silently
+      ;; stored as the unreduced expression `(+ 1 2)` because dl-ground?
+      ;; sees no free variables.
+      (dl-et-test!
+        "compound arg in fact rejected"
+        (dl-et-throws? (fn () (dl-program "p(+(1, 2)).")))
+        true)
+
+      ;; Rule heads may only have variable or constant args — no
+      ;; compounds. Compound heads would be saturated as unreduced
+      ;; tuples rather than the arithmetic result the user expected.
+      (dl-et-test!
+        "compound arg in rule head rejected"
+        (dl-et-throws?
+          (fn () (dl-program "n(3). double(*(X, 2)) :- n(X).")))
+        true)
+
+      ;; The anonymous-variable renamer used to start at `_anon1`
+      ;; unconditionally; a rule that wrote `q(_anon1) :- p(_anon1, _)`
+      ;; (the user picking the same name the renamer would generate)
+      ;; would see the `_` renamed to `_anon1` too, collapsing the
+      ;; two positions in `p(_anon1, _)` to a single var. Now the
+      ;; renamer scans the rule for the max `_anon<N>` and starts past
+      ;; it, so user-written names of that form are preserved.
+      (dl-et-test-set! "anonymous-rename avoids user `_anon` collision"
+        (dl-query
+          (dl-program
+            "p(a, b). p(c, d). q(_anon1) :- p(_anon1, _).")
+          (quote (q X)))
+        (list {:X (quote a)} {:X (quote c)}))
+
+      (dl-et-test!
+        "unsafe head var"
+        (dl-et-throws? (fn () (dl-program "p(X, Y) :- q(X).")))
+        true)
+      (dl-et-test!
+        "unsafe — empty body"
+        (dl-et-throws? (fn () (dl-program "p(X) :- .")))
+        true)
+      ;; Underscore in head is unsafe — it's a fresh existential per
+      ;; occurrence after Phase 5d's anonymous-var renaming, and there's
+      ;; nothing in the body to bind it. (Old behavior accepted this by
+      ;; treating '_' as a literal name to skip; the renaming made it an
+      ;; ordinary unbound variable.)
+      (dl-et-test!
+        "underscore in head — unsafe"
+        (dl-et-throws? (fn () (dl-program "p(X, _) :- q(X).")))
+        true)
+      (dl-et-test!
+        "underscore in body only — safe"
+        (dl-et-throws? (fn () (dl-program "p(X) :- q(X, _).")))
+        false)
+      (dl-et-test!
+        "var only in head — unsafe"
+        (dl-et-throws? (fn () (dl-program "p(X, Y) :- q(Z).")))
+        true)
+      (dl-et-test!
+        "head var bound by body"
+        (dl-et-throws? (fn () (dl-program "p(X) :- q(X).")))
+        false)
+      (dl-et-test!
+        "head subset of body"
+        (dl-et-throws?
+          (fn
+            ()
+            (dl-program
+              "edge(a,b). edge(b,c). reach(X, Z) :- edge(X, Y), edge(Y, Z).")))
+        false)
+
+      ;; Anonymous variables: each occurrence must be independent.
+      (dl-et-test-set! "anon vars in rule are independent"
+        (dl-query
+          (dl-program
+            "p(a, b). p(c, d). q(X) :- p(X, _), p(_, Y).")
+          (list (quote q) (quote X)))
+        (list {:X (quote a)} {:X (quote c)}))
+
+      (dl-et-test-set! "anon vars in goal are independent"
+        (dl-query
+          (dl-program "p(1, 2, 3). p(4, 5, 6).")
+          (list (quote p) (quote _) (quote X) (quote _)))
+        (list {:X 2} {:X 5}))
+
+      ;; dl-summary: relation -> tuple-count for inspection.
+      (dl-et-test! "dl-summary basic"
+        (dl-summary
+          (let
+            ((db (dl-program "p(1). p(2). q(3).")))
+            (do (dl-saturate! db) db)))
+        {:p 2 :q 1})
+
+      (dl-et-test! "dl-summary empty IDB shown"
+        (dl-summary
+          (let
+            ((db (dl-program "r(X) :- s(X).")))
+            (do (dl-saturate! db) db)))
+        {:r 0})
+
+      (dl-et-test! "dl-summary mixed EDB and IDB"
+        (dl-summary
+          (let
+            ((db (dl-program
+                   "parent(a, b).
+                    ancestor(X, Y) :- parent(X, Y).
+                    ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z).")))
+            (do (dl-saturate! db) db)))
+        {:parent 1 :ancestor 1})
+
+      (dl-et-test! "dl-summary empty db"
+        (dl-summary (dl-make-db))
+        {})
+
+      ;; Strategy hook: default semi-naive; :magic accepted but
+      ;; falls back to semi-naive (the transformation itself is
+      ;; deferred — Phase 6 in plan).
+      (dl-et-test! "default strategy"
+        (dl-get-strategy (dl-make-db))
+        :semi-naive)
+
+      (dl-et-test! "set strategy"
+        (let ((db (dl-make-db)))
+          (do (dl-set-strategy! db :magic) (dl-get-strategy db)))
+        :magic)
+
+      ;; Unknown strategy values are rejected so typos don't silently
+      ;; fall back to the default.
+      (dl-et-test!
+        "unknown strategy rejected"
+        (dl-et-throws?
+          (fn ()
+            (let ((db (dl-make-db)))
+              (dl-set-strategy! db :semi_naive))))
+        true)
+
+      ;; dl-saturated?: no-work-left predicate.
+      (dl-et-test! "saturated? after saturation"
+        (let ((db (dl-program
+                    "parent(a, b).
+                     ancestor(X, Y) :- parent(X, Y).")))
+          (do (dl-saturate! db) (dl-saturated? db)))
+        true)
+
+      (dl-et-test! "saturated? before saturation"
+        (let ((db (dl-program
+                    "parent(a, b).
+                     ancestor(X, Y) :- parent(X, Y).")))
+          (dl-saturated? db))
+        false)
+
+      ;; Disjunction via multiple rules — Datalog has no `;` in
+      ;; body, so disjunction is expressed as separate rules with
+      ;; the same head. Here plant_based(X) is satisfied by either
+      ;; vegan(X) or vegetarian(X).
+      (dl-et-test-set! "disjunction via multiple rules"
+        (dl-query
+          (dl-program
+            "vegan(alice). vegetarian(bob). meat_eater(carol).
+             plant_based(X) :- vegan(X).
+             plant_based(X) :- vegetarian(X).")
+          (list (quote plant_based) (quote X)))
+        (list {:X (quote alice)} {:X (quote bob)}))
+
+      ;; Bipartite-style join: pair-of-friends who share a hobby.
+      ;; Three-relation join exercising the planner's join order.
+      (dl-et-test-set! "bipartite friends-with-hobby"
+        (dl-query
+          (dl-program
+            "hobby(alice, climb). hobby(bob, paint).
+             hobby(carol, climb).
+             friend(alice, carol). friend(bob, alice).
+             match(A, B, H) :- friend(A, B), hobby(A, H), hobby(B, H).")
+          (list (quote match) (quote A) (quote B) (quote H)))
+        (list {:A (quote alice) :B (quote carol) :H (quote climb)}))
+
+      ;; Repeated variable (diagonal): p(X, X) only matches tuples
+      ;; whose two args are equal. The unifier handles this via the
+      ;; subst chain — first occurrence binds X, second occurrence
+      ;; checks against the binding.
+      (dl-et-test-set! "diagonal query"
+        (dl-query
+          (dl-program "p(1, 1). p(2, 3). p(4, 4). p(5, 5).")
+          (list (quote p) (quote X) (quote X)))
+        (list {:X 1} {:X 4} {:X 5}))
+
+      ;; A relation can be both EDB-seeded and rule-derived;
+      ;; saturate combines facts + derivations.
+      (dl-et-test-set! "mixed EDB + IDB same relation"
+        (dl-query
+          (dl-program
+            "link(a, b). link(c, d). link(e, c).
+             via(a, e).
+             link(X, Y) :- via(X, M), link(M, Y).")
+          (list (quote link) (quote a) (quote X)))
+        (list {:X (quote b)} {:X (quote c)}))
+
+      (dl-et-test! "saturated? after assert"
+        (let ((db (dl-program
+                    "parent(a, b).
+                     ancestor(X, Y) :- parent(X, Y).")))
+          (do
+            (dl-saturate! db)
+            (dl-add-fact! db (list (quote parent) (quote b) (quote c)))
+            (dl-saturated? db)))
+        false)
+
+      (dl-et-test-set! "magic-set still derives correctly"
+        (let
+          ((db (dl-program
+                 "parent(a, b). parent(b, c).
+                  ancestor(X, Y) :- parent(X, Y).
+                  ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z).")))
+          (do
+            (dl-set-strategy! db :magic)
+            (dl-query db (list (quote ancestor) (quote a) (quote X)))))
+        (list {:X (quote b)} {:X (quote c)})))))
+
+(define
+  dl-eval-tests-run!
+  (fn
+    ()
+    (do
+      (set! dl-et-pass 0)
+      (set! dl-et-fail 0)
+      (set! dl-et-failures (list))
+      (dl-et-run-all!)
+      {:failures dl-et-failures :total (+ dl-et-pass dl-et-fail) :passed dl-et-pass :failed dl-et-fail})))

lib/datalog/tests/magic.sx

@@ -0,0 +1,528 @@
+;; lib/datalog/tests/magic.sx — adornment + SIPS analysis tests.
+
+(define dl-mt-pass 0)
+(define dl-mt-fail 0)
+(define dl-mt-failures (list))
+
+(define
+  dl-mt-deep=?
+  (fn
+    (a b)
+    (cond
+      ((and (list? a) (list? b))
+        (and (= (len a) (len b)) (dl-mt-deq-l? a b 0)))
+      ((and (dict? a) (dict? b))
+        (let ((ka (keys a)) (kb (keys b)))
+          (and (= (len ka) (len kb)) (dl-mt-deq-d? a b ka 0))))
+      ((and (number? a) (number? b)) (= a b))
+      (else (equal? a b)))))
+
+(define
+  dl-mt-deq-l?
+  (fn
+    (a b i)
+    (cond
+      ((>= i (len a)) true)
+      ((not (dl-mt-deep=? (nth a i) (nth b i))) false)
+      (else (dl-mt-deq-l? a b (+ i 1))))))
+
+(define
+  dl-mt-deq-d?
+  (fn
+    (a b ka i)
+    (cond
+      ((>= i (len ka)) true)
+      ((let ((k (nth ka i)))
+         (not (dl-mt-deep=? (get a k) (get b k))))
+        false)
+      (else (dl-mt-deq-d? a b ka (+ i 1))))))
+
+(define
+  dl-mt-test!
+  (fn
+    (name got expected)
+    (if
+      (dl-mt-deep=? got expected)
+      (set! dl-mt-pass (+ dl-mt-pass 1))
+      (do
+        (set! dl-mt-fail (+ dl-mt-fail 1))
+        (append!
+          dl-mt-failures
+          (str
+            name
+            "\n    expected: " expected
+            "\n    got:      " got))))))
+
+(define
+  dl-mt-run-all!
+  (fn
+    ()
+    (do
+      ;; Goal adornment.
+      (dl-mt-test! "adorn 0-ary"
+        (dl-adorn-goal (list (quote ready)))
+        "")
+      (dl-mt-test! "adorn all bound"
+        (dl-adorn-goal (list (quote p) 1 2 3))
+        "bbb")
+      (dl-mt-test! "adorn all free"
+        (dl-adorn-goal (list (quote p) (quote X) (quote Y)))
+        "ff")
+      (dl-mt-test! "adorn mixed"
+        (dl-adorn-goal (list (quote ancestor) (quote tom) (quote X)))
+        "bf")
+      (dl-mt-test! "adorn const var const"
+        (dl-adorn-goal (list (quote p) (quote a) (quote X) (quote b)))
+        "bfb")
+
+      ;; dl-adorn-lit with explicit bound set.
+      (dl-mt-test! "adorn lit with bound"
+        (dl-adorn-lit (list (quote p) (quote X) (quote Y)) (list "X"))
+        "bf")
+
+      ;; Rule SIPS — chain ancestor.
+      (dl-mt-test! "sips chain ancestor bf"
+        (dl-rule-sips
+          {:head (list (quote ancestor) (quote X) (quote Z))
+           :body (list (list (quote parent) (quote X) (quote Y))
+                       (list (quote ancestor) (quote Y) (quote Z)))}
+          "bf")
+        (list
+          {:lit (list (quote parent) (quote X) (quote Y)) :adornment "bf"}
+          {:lit (list (quote ancestor) (quote Y) (quote Z)) :adornment "bf"}))
+
+      ;; SIPS — head fully bound.
+      (dl-mt-test! "sips head bb"
+        (dl-rule-sips
+          {:head (list (quote q) (quote X) (quote Y))
+           :body (list (list (quote p) (quote X) (quote Z))
+                       (list (quote r) (quote Z) (quote Y)))}
+          "bb")
+        (list
+          {:lit (list (quote p) (quote X) (quote Z)) :adornment "bf"}
+          {:lit (list (quote r) (quote Z) (quote Y)) :adornment "bb"}))
+
+      ;; SIPS — comparison; vars must be bound by prior body lit.
+      (dl-mt-test! "sips with comparison"
+        (dl-rule-sips
+          {:head (list (quote q) (quote X))
+           :body (list (list (quote p) (quote X))
+                       (list (string->symbol "<") (quote X) 5))}
+          "f")
+        (list
+          {:lit (list (quote p) (quote X)) :adornment "f"}
+          {:lit (list (string->symbol "<") (quote X) 5) :adornment "bb"}))
+
+      ;; SIPS — `is` binds its left arg.
+      (dl-mt-test! "sips with is"
+        (dl-rule-sips
+          {:head (list (quote q) (quote X) (quote Y))
+           :body (list (list (quote p) (quote X))
+                       (list (quote is) (quote Y) (list (string->symbol "+") (quote X) 1)))}
+          "ff")
+        (list
+          {:lit (list (quote p) (quote X)) :adornment "f"}
+          {:lit (list (quote is) (quote Y)
+                      (list (string->symbol "+") (quote X) 1))
+           :adornment "fb"}))
+
+      ;; Magic predicate naming.
+      (dl-mt-test! "magic-rel-name"
+        (dl-magic-rel-name "ancestor" "bf")
+        "magic_ancestor^bf")
+
+      ;; Bound-args extraction.
+      (dl-mt-test! "bound-args bf"
+        (dl-bound-args (list (quote ancestor) (quote tom) (quote X)) "bf")
+        (list (quote tom)))
+
+      (dl-mt-test! "bound-args mixed"
+        (dl-bound-args (list (quote p) 1 (quote Y) 3) "bfb")
+        (list 1 3))
+
+      (dl-mt-test! "bound-args all-free"
+        (dl-bound-args (list (quote p) (quote X) (quote Y)) "ff")
+        (list))
+
+      ;; Magic literal construction.
+      (dl-mt-test! "magic-lit"
+        (dl-magic-lit "ancestor" "bf" (list (quote tom)))
+        (list (string->symbol "magic_ancestor^bf") (quote tom)))
+
+      ;; Magic-sets rewriter: structural sanity.
+      (dl-mt-test! "rewrite ancestor produces seed"
+        (let
+          ((rules
+             (list
+               {:head (list (quote ancestor) (quote X) (quote Y))
+                :body (list (list (quote parent) (quote X) (quote Y)))}
+               {:head (list (quote ancestor) (quote X) (quote Z))
+                :body
+                (list (list (quote parent) (quote X) (quote Y))
+                      (list (quote ancestor) (quote Y) (quote Z)))})))
+          (get
+            (dl-magic-rewrite rules "ancestor" "bf" (list (quote a)))
+            :seed))
+        (list (string->symbol "magic_ancestor^bf") (quote a)))
+
+      ;; Equivalence: rewritten program derives same ancestor tuples.
+      ;; In a chain a→b→c→d, magic-rewritten run still derives all
+      ;; ancestor pairs reachable from any node a/b/c/d propagated via
+      ;; magic_ancestor^bf — i.e. the full closure (6 tuples). Magic
+      ;; saves work only when the EDB has irrelevant nodes outside
+      ;; the seed's transitive cone.
+      (dl-mt-test! "magic-rewritten ancestor count"
+        (let
+          ((rules
+             (list
+               {:head (list (quote ancestor) (quote X) (quote Y))
+                :body (list (list (quote parent) (quote X) (quote Y)))}
+               {:head (list (quote ancestor) (quote X) (quote Z))
+                :body
+                (list (list (quote parent) (quote X) (quote Y))
+                      (list (quote ancestor) (quote Y) (quote Z)))}))
+            (edb (list
+                   (list (quote parent) (quote a) (quote b))
+                   (list (quote parent) (quote b) (quote c))
+                   (list (quote parent) (quote c) (quote d)))))
+          (let
+            ((rewritten (dl-magic-rewrite rules "ancestor" "bf" (list (quote a))))
+              (db (dl-make-db)))
+            (do
+              (for-each (fn (f) (dl-add-fact! db f)) edb)
+              (dl-add-fact! db (get rewritten :seed))
+              (for-each (fn (r) (dl-add-rule! db r)) (get rewritten :rules))
+              (dl-saturate! db)
+              (len (dl-relation db "ancestor")))))
+        6)
+
+      ;; dl-magic-query: end-to-end driver, doesn't mutate caller's db.
+      ;; Magic over a rule with negated body literal — propagation
+      ;; rules generated only for positive lits; negated lits pass
+      ;; through unchanged.
+      (dl-mt-test! "magic over rule with negation"
+        (let
+          ((db (dl-program
+                 "u(a). u(b). u(c). banned(b).
+                  active(X) :- u(X), not(banned(X)).")))
+          (let
+            ((semi (dl-query db (list (quote active) (quote X))))
+              (magic (dl-magic-query db (list (quote active) (quote X)))))
+            (= (len semi) (len magic))))
+        true)
+
+      ;; All-bound query (existence check) generates an "bb"
+      ;; adornment chain. Verifies the rewriter walks multiple
+      ;; (rel, adn) pairs through the worklist.
+      (dl-mt-test! "magic existence check via bb"
+        (let
+          ((db (dl-program
+                 "parent(a, b). parent(b, c). parent(c, d).
+                  ancestor(X, Y) :- parent(X, Y).
+                  ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z).")))
+          (let
+            ((found (dl-magic-query
+                      db (list (quote ancestor) (quote a) (quote c))))
+              (missing (dl-magic-query
+                         db (list (quote ancestor) (quote a) (quote z)))))
+            (and (= (len found) 1) (= (len missing) 0))))
+        true)
+
+      ;; Magic equivalence on the federation demo.
+      (dl-mt-test! "magic ≡ semi on foaf demo"
+        (let
+          ((db (dl-program-data
+                 (quote ((follows alice bob)
+                         (follows bob carol)
+                         (follows alice dave)))
+                 dl-demo-federation-rules)))
+          (let
+            ((semi (dl-query db (quote (foaf alice X))))
+              (magic (dl-magic-query db (quote (foaf alice X)))))
+            (= (len semi) (len magic))))
+        true)
+
+      ;; Shape validation: dl-magic-query rejects non-list / non-
+      ;; dict goal shapes cleanly rather than crashing in `rest`.
+      (dl-mt-test! "magic rejects string goal"
+        (let ((threw false))
+          (do
+            (guard (e (#t (set! threw true)))
+              (dl-magic-query (dl-make-db) "foo"))
+            threw))
+        true)
+
+      (dl-mt-test! "magic rejects bare dict goal"
+        (let ((threw false))
+          (do
+            (guard (e (#t (set! threw true)))
+              (dl-magic-query (dl-make-db) {:foo "bar"}))
+            threw))
+        true)
+
+      ;; 3-stratum program under magic — distinct rule heads at
+      ;; strata 0/1/2 must all rewrite via the worklist.
+      (dl-mt-test! "magic 3-stratum program"
+        (let
+          ((db (dl-program
+                 "a(1). a(2). a(3). b(2).
+                  c(X) :- a(X), not(b(X)).
+                  d(X) :- c(X), not(banned(X)).
+                  banned(3).")))
+          (let
+            ((semi (dl-query db (list (quote d) (quote X))))
+              (magic (dl-magic-query db (list (quote d) (quote X)))))
+            (= (len semi) (len magic))))
+        true)
+
+      ;; Aggregate -> derived -> threshold chain via magic.
+      (dl-mt-test! "magic aggregate-derived chain"
+        (let
+          ((db (dl-program
+                 "src(1). src(2). src(3).
+                  cnt(N) :- count(N, X, src(X)).
+                  active(N) :- cnt(N), >=(N, 2).")))
+          (let
+            ((semi (dl-query db (list (quote active) (quote N))))
+              (magic (dl-magic-query db (list (quote active) (quote N)))))
+            (= (len semi) (len magic))))
+        true)
+
+      ;; Multi-relation rewrite chain: query r4 → propagate to r3,
+      ;; r2, r1, a. The worklist must process all of them; an
+      ;; earlier bug stopped after only the head pair.
+      (dl-mt-test! "magic chain through 4 rule levels"
+        (let
+          ((db (dl-program
+                 "a(1). a(2). r1(X) :- a(X). r2(X) :- r1(X).
+                  r3(X) :- r2(X). r4(X) :- r3(X).")))
+          (= 2 (len (dl-magic-query db (list (quote r4) (quote X))))))
+        true)
+
+      ;; Shortest-path demo via magic — exercises the rewriter
+      ;; against rules that mix recursive positive lits with an
+      ;; aggregate body literal.
+      (dl-mt-test! "magic on shortest-path demo"
+        (let
+          ((db (dl-program-data
+                 (quote ((edge a b 5) (edge b c 3) (edge a c 10)))
+                 dl-demo-shortest-path-rules)))
+          (let
+            ((semi (dl-query db (quote (shortest a c W))))
+              (magic (dl-magic-query db (quote (shortest a c W)))))
+            (and (= (len semi) (len magic))
+                 (= (len semi) 1))))
+        true)
+
+      ;; Same relation called with different adornment patterns
+      ;; in different rules. The worklist must enqueue and process
+      ;; each (rel, adornment) pair.
+      (dl-mt-test! "magic with multi-adornment same relation"
+        (let
+          ((db (dl-program
+                 "parent(p1, alice). parent(p2, bob).
+                  parent(g, p1). parent(g, p2).
+                  sibling(P1, P2) :- parent(G, P1), parent(G, P2),
+                                     !=(P1, P2).
+                  cousin(X, Y) :- parent(P1, X), parent(P2, Y),
+                                  sibling(P1, P2).")))
+          (let
+            ((semi (dl-query db (list (quote cousin) (quote alice) (quote Y))))
+              (magic (dl-magic-query db (list (quote cousin) (quote alice) (quote Y)))))
+            (= (len semi) (len magic))))
+        true)
+
+      ;; Magic over a rule whose body contains an aggregate.
+      ;; The rewriter passes aggregate body lits through unchanged
+      ;; (no propagation generated for them), so semi-naive's count
+      ;; logic still fires correctly under the rewritten program.
+      (dl-mt-test! "magic over rule with aggregate body"
+        (let
+          ((db (dl-program
+                 "post(p1). post(p2). post(p3).
+                  liked(u1, p1). liked(u2, p1). liked(u3, p1).
+                  liked(u1, p2).
+                  rich(P) :- post(P), count(N, U, liked(U, P)),
+                             >=(N, 2).")))
+          (let
+            ((semi (dl-query db (list (quote rich) (quote P))))
+              (magic (dl-magic-query db (list (quote rich) (quote P)))))
+            (= (len semi) (len magic))))
+        true)
+
+      ;; Mixed EDB + IDB: a relation can be both EDB-seeded and
+      ;; rule-derived. dl-magic-query must include the EDB portion
+      ;; even though the relation has rules.
+      (dl-mt-test! "magic mixed EDB+IDB"
+        (len
+          (dl-magic-query
+            (dl-program
+              "link(a, b). link(c, d). link(e, c).
+               via(a, e).
+               link(X, Y) :- via(X, M), link(M, Y).")
+            (list (quote link) (quote a) (quote X))))
+        2)
+
+      ;; dl-magic-query falls back to dl-query for built-in,
+      ;; aggregate, and negation goals (the magic seed would
+      ;; otherwise be non-ground).
+      (dl-mt-test! "magic-query falls back on aggregate"
+        (let
+          ((r (dl-magic-query
+                (dl-program "p(1). p(2). p(3).")
+                (list (quote count) (quote N) (quote X)
+                      (list (quote p) (quote X))))))
+          (and (= (len r) 1) (= (get (first r) "N") 3)))
+        true)
+
+      (dl-mt-test! "magic-query equivalent to dl-query"
+        (let
+          ((db (dl-program
+                 "parent(a, b). parent(b, c). parent(c, d).
+                  ancestor(X, Y) :- parent(X, Y).
+                  ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z).")))
+          (let
+            ((semi (dl-query db (list (quote ancestor) (quote a) (quote X))))
+              (magic (dl-magic-query
+                       db (list (quote ancestor) (quote a) (quote X)))))
+            (= (len semi) (len magic))))
+        true)
+
+      ;; The magic rewriter passes aggregate body lits through
+      ;; unchanged, so an aggregate over an IDB relation would see an
+      ;; empty inner-goal in the magic db unless the IDB is already
+      ;; materialised. dl-magic-query now pre-saturates the source db
+      ;; to guarantee equivalence with dl-query for every stratified
+      ;; program. Previously this returned `({:N 0})` because `active`
+      ;; (IDB, derived through negation) was never derived in the
+      ;; magic db.
+      (dl-mt-test! "magic over aggregate-of-IDB matches vanilla"
+        (let
+          ((src
+             "u(a). u(b). u(c). u(d). banned(b). banned(d).
+              active(X) :- u(X), not(banned(X)).
+              n(N) :- count(N, X, active(X))."))
+          (let
+            ((vanilla (dl-eval src "?- n(N)."))
+              (magic (dl-eval-magic src "?- n(N).")))
+            (and (= (len vanilla) 1)
+                 (= (len magic) 1)
+                 (= (get (first vanilla) "N")
+                    (get (first magic) "N")))))
+        true)
+
+      ;; magic-query doesn't mutate caller db.
+      (dl-mt-test! "magic-query preserves caller db"
+        (let
+          ((db (dl-program
+                 "parent(a, b). parent(b, c).
+                  ancestor(X, Y) :- parent(X, Y).
+                  ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z).")))
+          (let
+            ((rules-before (len (dl-rules db))))
+            (do
+              (dl-magic-query db (list (quote ancestor) (quote a) (quote X)))
+              (= rules-before (len (dl-rules db))))))
+        true)
+
+      ;; Magic-sets benefit: query touches only one cluster of a
+      ;; multi-component graph. Semi-naive derives the full closure
+      ;; over both clusters; magic only the seeded one.
+      ;; Magic-vs-semi work shape: chain of 12. Semi-naive
+      ;; derives the full closure (78 = 12·13/2). A magic query
+      ;; rooted at node 0 returns the 12 descendants only —
+      ;; demonstrating that magic limits derivation to the
+      ;; query's transitive cone.
+      (dl-mt-test! "magic vs semi work-shape on chain-12"
+        (let
+          ((source (str
+                     "parent(0, 1). parent(1, 2). parent(2, 3). "
+                     "parent(3, 4). parent(4, 5). parent(5, 6). "
+                     "parent(6, 7). parent(7, 8). parent(8, 9). "
+                     "parent(9, 10). parent(10, 11). parent(11, 12). "
+                     "ancestor(X, Y) :- parent(X, Y). "
+                     "ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z).")))
+          (let
+            ((db1 (dl-make-db)) (db2 (dl-make-db)))
+            (do
+              (dl-load-program! db1 source)
+              (dl-saturate! db1)
+              (dl-load-program! db2 source)
+              (let
+                ((semi-count (len (dl-relation db1 "ancestor")))
+                  (magic-count
+                    (len (dl-magic-query
+                           db2 (list (quote ancestor) 0 (quote X))))))
+                ;; Magic returns only descendants of 0 (12 of them).
+                (and (= semi-count 78) (= magic-count 12))))))
+        true)
+
+      ;; Magic + arithmetic: rules with `is` clauses pass through
+      ;; the rewriter unchanged (built-ins aren't propagated).
+      (dl-mt-test! "magic preserves arithmetic"
+        (let
+          ((source "n(1). n(2). n(3).
+                    doubled(X, Y) :- n(X), is(Y, *(X, 2))."))
+          (let
+            ((semi (dl-eval source "?- doubled(2, Y)."))
+              (magic (dl-eval-magic source "?- doubled(2, Y).")))
+            (= (len semi) (len magic))))
+        true)
+
+      (dl-mt-test! "magic skips irrelevant clusters"
+        (let
+          ;; Two disjoint chains. Query is rooted in cluster 1.
+          ((db (dl-program
+                 "parent(a, b). parent(b, c).
+                  parent(x, y). parent(y, z).
+                  ancestor(X, Y) :- parent(X, Y).
+                  ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z).")))
+          (do
+            (dl-saturate! db)
+            (let
+              ((semi-count (len (dl-relation db "ancestor")))
+                (magic-results
+                  (dl-magic-query
+                    db (list (quote ancestor) (quote a) (quote X)))))
+              ;; Semi-naive derives 6 (3 in each cluster). Magic
+              ;; gives 3 query results (a's reachable: b, c).
+              (and (= semi-count 6) (= (len magic-results) 2)))))
+        true)
+
+      (dl-mt-test! "magic-rewritten finds same answers"
+        (let
+          ((rules
+             (list
+               {:head (list (quote ancestor) (quote X) (quote Y))
+                :body (list (list (quote parent) (quote X) (quote Y)))}
+               {:head (list (quote ancestor) (quote X) (quote Z))
+                :body
+                (list (list (quote parent) (quote X) (quote Y))
+                      (list (quote ancestor) (quote Y) (quote Z)))}))
+            (edb (list
+                   (list (quote parent) (quote a) (quote b))
+                   (list (quote parent) (quote b) (quote c)))))
+          (let
+            ((rewritten (dl-magic-rewrite rules "ancestor" "bf" (list (quote a))))
+              (db (dl-make-db)))
+            (do
+              (for-each (fn (f) (dl-add-fact! db f)) edb)
+              (dl-add-fact! db (get rewritten :seed))
+              (for-each (fn (r) (dl-add-rule! db r)) (get rewritten :rules))
+              (dl-saturate! db)
+              (len (dl-query db (list (quote ancestor) (quote a) (quote X)))))))
+        2))))
+
+(define
+  dl-magic-tests-run!
+  (fn
+    ()
+    (do
+      (set! dl-mt-pass 0)
+      (set! dl-mt-fail 0)
+      (set! dl-mt-failures (list))
+      (dl-mt-run-all!)
+      {:passed dl-mt-pass
+       :failed dl-mt-fail
+       :total (+ dl-mt-pass dl-mt-fail)
+       :failures dl-mt-failures})))

lib/datalog/tests/negation.sx

@@ -0,0 +1,252 @@
+;; lib/datalog/tests/negation.sx — stratified negation tests.
+
+(define dl-nt-pass 0)
+(define dl-nt-fail 0)
+(define dl-nt-failures (list))
+
+(define
+  dl-nt-deep=?
+  (fn
+    (a b)
+    (cond
+      ((and (list? a) (list? b))
+        (and (= (len a) (len b)) (dl-nt-deq-l? a b 0)))
+      ((and (dict? a) (dict? b))
+        (let ((ka (keys a)) (kb (keys b)))
+          (and (= (len ka) (len kb)) (dl-nt-deq-d? a b ka 0))))
+      ((and (number? a) (number? b)) (= a b))
+      (else (equal? a b)))))
+
+(define
+  dl-nt-deq-l?
+  (fn
+    (a b i)
+    (cond
+      ((>= i (len a)) true)
+      ((not (dl-nt-deep=? (nth a i) (nth b i))) false)
+      (else (dl-nt-deq-l? a b (+ i 1))))))
+
+(define
+  dl-nt-deq-d?
+  (fn
+    (a b ka i)
+    (cond
+      ((>= i (len ka)) true)
+      ((let ((k (nth ka i)))
+         (not (dl-nt-deep=? (get a k) (get b k))))
+        false)
+      (else (dl-nt-deq-d? a b ka (+ i 1))))))
+
+(define
+  dl-nt-set=?
+  (fn
+    (a b)
+    (and
+      (= (len a) (len b))
+      (dl-nt-subset? a b)
+      (dl-nt-subset? b a))))
+
+(define
+  dl-nt-subset?
+  (fn
+    (xs ys)
+    (cond
+      ((= (len xs) 0) true)
+      ((not (dl-nt-contains? ys (first xs))) false)
+      (else (dl-nt-subset? (rest xs) ys)))))
+
+(define
+  dl-nt-contains?
+  (fn
+    (xs target)
+    (cond
+      ((= (len xs) 0) false)
+      ((dl-nt-deep=? (first xs) target) true)
+      (else (dl-nt-contains? (rest xs) target)))))
+
+(define
+  dl-nt-test!
+  (fn
+    (name got expected)
+    (if
+      (dl-nt-deep=? got expected)
+      (set! dl-nt-pass (+ dl-nt-pass 1))
+      (do
+        (set! dl-nt-fail (+ dl-nt-fail 1))
+        (append!
+          dl-nt-failures
+          (str
+            name
+            "\n    expected: " expected
+            "\n    got:      " got))))))
+
+(define
+  dl-nt-test-set!
+  (fn
+    (name got expected)
+    (if
+      (dl-nt-set=? got expected)
+      (set! dl-nt-pass (+ dl-nt-pass 1))
+      (do
+        (set! dl-nt-fail (+ dl-nt-fail 1))
+        (append!
+          dl-nt-failures
+          (str
+            name
+            "\n    expected (set): " expected
+            "\n    got:            " got))))))
+
+(define
+  dl-nt-throws?
+  (fn
+    (thunk)
+    (let
+      ((threw false))
+      (do
+        (guard
+          (e (#t (set! threw true)))
+          (thunk))
+        threw))))
+
+(define
+  dl-nt-run-all!
+  (fn
+    ()
+    (do
+      ;; Negation against EDB-only relation.
+      (dl-nt-test-set! "not against EDB"
+        (dl-query
+          (dl-program
+            "p(1). p(2). p(3). r(2).
+             q(X) :- p(X), not(r(X)).")
+          (list (quote q) (quote X)))
+        (list {:X 1} {:X 3}))
+
+      ;; Negation against derived relation — needs stratification.
+      (dl-nt-test-set! "not against derived rel"
+        (dl-query
+          (dl-program
+            "p(1). p(2). p(3). s(2).
+             r(X) :- s(X).
+             q(X) :- p(X), not(r(X)).")
+          (list (quote q) (quote X)))
+        (list {:X 1} {:X 3}))
+
+      ;; Two-step strata: r derives via s; q derives via not r.
+      (dl-nt-test-set! "two-step strata"
+        (dl-query
+          (dl-program
+            "node(a). node(b). node(c). node(d).
+             edge(a, b). edge(b, c). edge(c, a).
+             reach(X, Y) :- edge(X, Y).
+             reach(X, Z) :- edge(X, Y), reach(Y, Z).
+             unreachable(X) :- node(X), not(reach(a, X)).")
+          (list (quote unreachable) (quote X)))
+        (list {:X (quote d)}))
+
+      ;; Combine negation with arithmetic and comparison.
+      (dl-nt-test-set! "negation with arithmetic"
+        (dl-query
+          (dl-program
+            "n(1). n(2). n(3). n(4). n(5). odd(1). odd(3). odd(5).
+             even(X) :- n(X), not(odd(X)).")
+          (list (quote even) (quote X)))
+        (list {:X 2} {:X 4}))
+
+      ;; Empty negation result.
+      (dl-nt-test-set! "negation always succeeds"
+        (dl-query
+          (dl-program
+            "p(1). p(2). q(X) :- p(X), not(r(X)).")
+          (list (quote q) (quote X)))
+        (list {:X 1} {:X 2}))
+
+      ;; Negation always fails.
+      (dl-nt-test-set! "negation always fails"
+        (dl-query
+          (dl-program
+            "p(1). p(2). r(1). r(2). q(X) :- p(X), not(r(X)).")
+          (list (quote q) (quote X)))
+        (list))
+
+      ;; Anonymous `_` in a negated literal is existentially quantified
+      ;; — it doesn't need to be bound by an earlier body lit. Without
+      ;; this exemption the safety check would reject the common idiom
+      ;; `orphan(X) :- person(X), not(parent(X, _))`.
+      (dl-nt-test-set! "negation with anonymous var — orphan idiom"
+        (dl-query
+          (dl-program
+            "person(a). person(b). person(c). parent(a, b).
+             orphan(X) :- person(X), not(parent(X, _)).")
+          (list (quote orphan) (quote X)))
+        (list {:X (quote b)} {:X (quote c)}))
+
+      ;; Multiple anonymous vars are each independently existential.
+      (dl-nt-test-set! "negation with multiple anonymous vars"
+        (dl-query
+          (dl-program
+            "u(a). u(b). u(c). edge(a, x). edge(b, y).
+             solo(X) :- u(X), not(edge(X, _)).")
+          (list (quote solo) (quote X)))
+        (list {:X (quote c)}))
+
+      ;; Stratifiability checks.
+      (dl-nt-test! "non-stratifiable rejected"
+        (dl-nt-throws?
+          (fn ()
+            (let ((db (dl-make-db)))
+              (do
+                (dl-add-rule!
+                  db
+                  {:head (list (quote p) (quote X))
+                   :body (list (list (quote q) (quote X))
+                               {:neg (list (quote r) (quote X))})})
+                (dl-add-rule!
+                  db
+                  {:head (list (quote r) (quote X))
+                   :body (list (list (quote p) (quote X)))})
+                (dl-add-fact! db (list (quote q) 1))
+                (dl-saturate! db)))))
+        true)
+
+      (dl-nt-test! "stratifiable accepted"
+        (dl-nt-throws?
+          (fn ()
+            (dl-program
+              "p(1). p(2). r(2).
+               q(X) :- p(X), not(r(X)).")))
+        false)
+
+      ;; Multi-stratum chain.
+      (dl-nt-test-set! "three-level strata"
+        (dl-query
+          (dl-program
+            "a(1). a(2). a(3). a(4).
+             b(X) :- a(X), not(c(X)).
+             c(X) :- d(X).
+             d(2).
+             d(4).")
+          (list (quote b) (quote X)))
+        (list {:X 1} {:X 3}))
+
+      ;; Safety violation: negation refers to unbound var.
+      (dl-nt-test! "negation safety violation"
+        (dl-nt-throws?
+          (fn ()
+            (dl-program
+              "p(1). q(X) :- p(X), not(r(Y)).")))
+        true))))
+
+(define
+  dl-negation-tests-run!
+  (fn
+    ()
+    (do
+      (set! dl-nt-pass 0)
+      (set! dl-nt-fail 0)
+      (set! dl-nt-failures (list))
+      (dl-nt-run-all!)
+      {:passed dl-nt-pass
+       :failed dl-nt-fail
+       :total (+ dl-nt-pass dl-nt-fail)
+       :failures dl-nt-failures})))

lib/datalog/tests/parse.sx

@@ -0,0 +1,179 @@
+;; lib/datalog/tests/parse.sx — parser unit tests
+;;
+;; Run via: bash lib/datalog/conformance.sh
+;; Or:      (load "lib/datalog/tokenizer.sx") (load "lib/datalog/parser.sx")
+;;          (load "lib/datalog/tests/parse.sx") (dl-parse-tests-run!)
+
+(define dl-pt-pass 0)
+(define dl-pt-fail 0)
+(define dl-pt-failures (list))
+
+;; Order-independent structural equality. Lists compared positionally,
+;; dicts as sets of (key, value) pairs. Numbers via = (so 30.0 = 30).
+(define
+  dl-deep-equal?
+  (fn
+    (a b)
+    (cond
+      ((and (list? a) (list? b))
+        (and (= (len a) (len b)) (dl-deep-equal-list? a b 0)))
+      ((and (dict? a) (dict? b))
+        (let
+          ((ka (keys a)) (kb (keys b)))
+          (and
+            (= (len ka) (len kb))
+            (dl-deep-equal-dict? a b ka 0))))
+      ((and (number? a) (number? b)) (= a b))
+      (else (equal? a b)))))
+
+(define
+  dl-deep-equal-list?
+  (fn
+    (a b i)
+    (cond
+      ((>= i (len a)) true)
+      ((not (dl-deep-equal? (nth a i) (nth b i))) false)
+      (else (dl-deep-equal-list? a b (+ i 1))))))
+
+(define
+  dl-deep-equal-dict?
+  (fn
+    (a b ka i)
+    (cond
+      ((>= i (len ka)) true)
+      ((let ((k (nth ka i))) (not (dl-deep-equal? (get a k) (get b k))))
+        false)
+      (else (dl-deep-equal-dict? a b ka (+ i 1))))))
+
+(define
+  dl-pt-test!
+  (fn
+    (name got expected)
+    (if
+      (dl-deep-equal? got expected)
+      (set! dl-pt-pass (+ dl-pt-pass 1))
+      (do
+        (set! dl-pt-fail (+ dl-pt-fail 1))
+        (append!
+          dl-pt-failures
+          (str name "\n    expected: " expected "\n    got:      " got))))))
+
+(define
+  dl-pt-throws?
+  (fn
+    (thunk)
+    (let
+      ((threw false))
+      (do (guard (e (#t (set! threw true))) (thunk)) threw))))
+
+(define
+  dl-pt-run-all!
+  (fn
+    ()
+    (do
+      (dl-pt-test! "empty program" (dl-parse "") (list))
+      (dl-pt-test! "fact" (dl-parse "parent(tom, bob).") (list {:body (list) :head (list (quote parent) (quote tom) (quote bob))}))
+      (dl-pt-test!
+        "two facts"
+        (dl-parse "parent(tom, bob). parent(bob, ann).")
+        (list {:body (list) :head (list (quote parent) (quote tom) (quote bob))} {:body (list) :head (list (quote parent) (quote bob) (quote ann))}))
+      (dl-pt-test! "zero-ary fact" (dl-parse "ready.") (list {:body (list) :head (list (quote ready))}))
+      (dl-pt-test!
+        "rule one body lit"
+        (dl-parse "ancestor(X, Y) :- parent(X, Y).")
+        (list {:body (list (list (quote parent) (quote X) (quote Y))) :head (list (quote ancestor) (quote X) (quote Y))}))
+      (dl-pt-test!
+        "recursive rule"
+        (dl-parse "ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z).")
+        (list {:body (list (list (quote parent) (quote X) (quote Y)) (list (quote ancestor) (quote Y) (quote Z))) :head (list (quote ancestor) (quote X) (quote Z))}))
+      (dl-pt-test!
+        "query single"
+        (dl-parse "?- ancestor(tom, X).")
+        (list {:query (list (list (quote ancestor) (quote tom) (quote X)))}))
+      (dl-pt-test!
+        "query multi"
+        (dl-parse "?- p(X), q(X).")
+        (list {:query (list (list (quote p) (quote X)) (list (quote q) (quote X)))}))
+      (dl-pt-test!
+        "negation"
+        (dl-parse "safe(X) :- person(X), not(parent(X, _)).")
+        (list {:body (list (list (quote person) (quote X)) {:neg (list (quote parent) (quote X) (quote _))}) :head (list (quote safe) (quote X))}))
+      (dl-pt-test!
+        "number arg"
+        (dl-parse "age(alice, 30).")
+        (list {:body (list) :head (list (quote age) (quote alice) 30)}))
+      (dl-pt-test!
+        "string arg"
+        (dl-parse "label(x, \"hi\").")
+        (list {:body (list) :head (list (quote label) (quote x) "hi")}))
+      ;; Quoted 'atoms' parse as strings — a uppercase-starting name
+      ;; in quotes used to misclassify as a variable and reject the
+      ;; fact as non-ground.
+      (dl-pt-test!
+        "quoted atom arg parses as string"
+        (dl-parse "p('Hello World').")
+        (list {:body (list) :head (list (quote p) "Hello World")}))
+      (dl-pt-test!
+        "comparison literal"
+        (dl-parse "p(X) :- <(X, 5).")
+        (list {:body (list (list (string->symbol "<") (quote X) 5)) :head (list (quote p) (quote X))}))
+      (dl-pt-test!
+        "is with arith"
+        (dl-parse "succ(X, Y) :- nat(X), is(Y, +(X, 1)).")
+        (list {:body (list (list (quote nat) (quote X)) (list (quote is) (quote Y) (list (string->symbol "+") (quote X) 1))) :head (list (quote succ) (quote X) (quote Y))}))
+      (dl-pt-test!
+        "mixed program"
+        (dl-parse "p(a). p(b). q(X) :- p(X). ?- q(Y).")
+        (list {:body (list) :head (list (quote p) (quote a))} {:body (list) :head (list (quote p) (quote b))} {:body (list (list (quote p) (quote X))) :head (list (quote q) (quote X))} {:query (list (list (quote q) (quote Y)))}))
+      (dl-pt-test!
+        "comments skipped"
+        (dl-parse "% comment\nfoo(a).\n/* block */ bar(b).")
+        (list {:body (list) :head (list (quote foo) (quote a))} {:body (list) :head (list (quote bar) (quote b))}))
+      (dl-pt-test!
+        "underscore var"
+        (dl-parse "p(X) :- q(X, _).")
+        (list {:body (list (list (quote q) (quote X) (quote _))) :head (list (quote p) (quote X))}))
+      ;; Negative number literals parse as one negative number,
+      ;; while subtraction (`-(X, Y)`) compound is preserved.
+      (dl-pt-test!
+        "negative integer literal"
+        (dl-parse "n(-3).")
+        (list {:head (list (quote n) -3) :body (list)}))
+
+      (dl-pt-test!
+        "subtraction compound preserved"
+        (dl-parse "r(X) :- is(X, -(10, 3)).")
+        (list
+          {:head (list (quote r) (quote X))
+           :body (list (list (quote is) (quote X)
+                             (list (string->symbol "-") 10 3)))}))
+
+      (dl-pt-test!
+        "number as relation name raises"
+        (dl-pt-throws? (fn () (dl-parse "1(X) :- p(X).")))
+        true)
+
+      (dl-pt-test!
+        "var as relation name raises"
+        (dl-pt-throws? (fn () (dl-parse "P(X).")))
+        true)
+
+      (dl-pt-test!
+        "missing dot raises"
+        (dl-pt-throws? (fn () (dl-parse "p(a)")))
+        true)
+      (dl-pt-test!
+        "trailing comma raises"
+        (dl-pt-throws? (fn () (dl-parse "p(a,).")))
+        true))))
+
+(define
+  dl-parse-tests-run!
+  (fn
+    ()
+    (do
+      (set! dl-pt-pass 0)
+      (set! dl-pt-fail 0)
+      (set! dl-pt-failures (list))
+      (dl-pt-run-all!)
+      {:failures dl-pt-failures :total (+ dl-pt-pass dl-pt-fail) :passed dl-pt-pass :failed dl-pt-fail})))

lib/datalog/tests/semi_naive.sx

@@ -0,0 +1,153 @@
+;; lib/datalog/tests/semi_naive.sx — semi-naive correctness vs naive.
+;;
+;; Strategy: differential — run both saturators on each program and
+;; compare the resulting per-relation tuple counts. Counting (not
+;; element-wise set equality) keeps the suite fast under the bundled
+;; conformance session; correctness on the inhabitants is covered by
+;; eval.sx and builtins.sx (which use dl-saturate! by default — the
+;; semi-naive saturator).
+
+(define dl-sn-pass 0)
+(define dl-sn-fail 0)
+(define dl-sn-failures (list))
+
+(define
+  dl-sn-test!
+  (fn
+    (name got expected)
+    (if
+      (equal? got expected)
+      (set! dl-sn-pass (+ dl-sn-pass 1))
+      (do
+        (set! dl-sn-fail (+ dl-sn-fail 1))
+        (append!
+          dl-sn-failures
+          (str name "\n    expected: " expected "\n    got:      " got))))))
+
+;; Load `source` into both a semi-naive and a naive db and return a
+;; list of (rel-name semi-count naive-count) triples. Both sets must
+;; have the same union of relation names.
+(define
+  dl-sn-counts
+  (fn
+    (source)
+    (let
+      ((db-s (dl-program source)) (db-n (dl-program source)))
+      (do
+        (dl-saturate! db-s)
+        (dl-saturate-naive! db-n)
+        (let
+          ((out (list)))
+          (do
+            (for-each
+              (fn
+                (k)
+                (append!
+                  out
+                  (list
+                    k
+                    (len (dl-relation db-s k))
+                    (len (dl-relation db-n k)))))
+              (keys (get db-s :facts)))
+            out))))))
+
+(define
+  dl-sn-counts-agree?
+  (fn
+    (counts)
+    (cond
+      ((= (len counts) 0) true)
+      (else
+        (let
+          ((row (first counts)))
+          (and
+            (= (nth row 1) (nth row 2))
+            (dl-sn-counts-agree? (rest counts))))))))
+
+(define
+  dl-sn-chain-source
+  (fn
+    (n)
+    (let
+      ((parts (list "")))
+      (do
+        (define
+          dl-sn-loop
+          (fn
+            (i)
+            (when
+              (< i n)
+              (do
+                (append! parts (str "parent(" i ", " (+ i 1) "). "))
+                (dl-sn-loop (+ i 1))))))
+        (dl-sn-loop 0)
+        (str
+          (join "" parts)
+          "ancestor(X, Y) :- parent(X, Y). "
+          "ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z).")))))
+
+(define
+  dl-sn-run-all!
+  (fn
+    ()
+    (do
+      (dl-sn-test!
+        "ancestor closure counts match"
+        (dl-sn-counts-agree?
+          (dl-sn-counts
+            "parent(a, b). parent(b, c). parent(c, d).\n             ancestor(X, Y) :- parent(X, Y).\n             ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z)."))
+        true)
+      (dl-sn-test!
+        "cyclic reach counts match"
+        (dl-sn-counts-agree?
+          (dl-sn-counts
+            "edge(1, 2). edge(2, 3). edge(3, 1). edge(3, 4).\n             reach(X, Y) :- edge(X, Y).\n             reach(X, Z) :- edge(X, Y), reach(Y, Z)."))
+        true)
+      (dl-sn-test!
+        "same-gen counts match"
+        (dl-sn-counts-agree?
+          (dl-sn-counts
+            "parent(a, b). parent(a, c). parent(b, d). parent(c, e).\n             person(a). person(b). person(c). person(d). person(e).\n             sg(X, X) :- person(X).\n             sg(X, Y) :- parent(P1, X), sg(P1, P2), parent(P2, Y)."))
+        true)
+      (dl-sn-test!
+        "rules with builtins counts match"
+        (dl-sn-counts-agree?
+          (dl-sn-counts
+            "n(1). n(2). n(3). n(4). n(5).\n             small(X) :- n(X), <(X, 5).\n             succ(X, Y) :- n(X), <(X, 5), is(Y, +(X, 1))."))
+        true)
+      (dl-sn-test!
+        "static rule fires under semi-naive"
+        (dl-sn-counts-agree?
+          (dl-sn-counts "p(a). p(b). q(X) :- p(X), =(X, a)."))
+        true)
+      ;; Chain length 12 — multiple semi-naive iterations against
+      ;; the recursive ancestor rule (differential vs naive).
+      (dl-sn-test!
+        "chain-12 ancestor counts match"
+        (dl-sn-counts-agree? (dl-sn-counts (dl-sn-chain-source 12)))
+        true)
+      ;; Chain length 25 — semi-naive only — first-arg index makes
+      ;; this tractable in conformance budget.
+      (dl-sn-test!
+        "chain-25 ancestor count value (semi only)"
+        (let
+          ((db (dl-program (dl-sn-chain-source 25))))
+          (do (dl-saturate! db) (len (dl-relation db "ancestor"))))
+        325)
+      (dl-sn-test!
+        "query through semi saturate"
+        (let
+          ((db (dl-program "parent(a, b). parent(b, c).\n                  ancestor(X, Y) :- parent(X, Y).\n                  ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z).")))
+          (len (dl-query db (list (quote ancestor) (quote a) (quote X)))))
+        2))))
+
+(define
+  dl-semi-naive-tests-run!
+  (fn
+    ()
+    (do
+      (set! dl-sn-pass 0)
+      (set! dl-sn-fail 0)
+      (set! dl-sn-failures (list))
+      (dl-sn-run-all!)
+      {:failures dl-sn-failures :total (+ dl-sn-pass dl-sn-fail) :passed dl-sn-pass :failed dl-sn-fail})))

lib/datalog/tests/tokenize.sx

@@ -0,0 +1,189 @@
+;; lib/datalog/tests/tokenize.sx — tokenizer unit tests
+;;
+;; Run via: bash lib/datalog/conformance.sh
+;; Or:      (load "lib/datalog/tokenizer.sx") (load "lib/datalog/tests/tokenize.sx")
+;;          (dl-tokenize-tests-run!)
+
+(define dl-tk-pass 0)
+(define dl-tk-fail 0)
+(define dl-tk-failures (list))
+
+(define
+  dl-tk-test!
+  (fn
+    (name got expected)
+    (if
+      (= got expected)
+      (set! dl-tk-pass (+ dl-tk-pass 1))
+      (do
+        (set! dl-tk-fail (+ dl-tk-fail 1))
+        (append!
+          dl-tk-failures
+          (str name "\n    expected: " expected "\n    got:      " got))))))
+
+(define dl-tk-types (fn (toks) (map (fn (t) (get t :type)) toks)))
+(define dl-tk-values (fn (toks) (map (fn (t) (get t :value)) toks)))
+
+(define
+  dl-tk-run-all!
+  (fn
+    ()
+    (do
+      (dl-tk-test! "empty" (dl-tk-types (dl-tokenize "")) (list "eof"))
+      (dl-tk-test!
+        "atom dot"
+        (dl-tk-types (dl-tokenize "foo."))
+        (list "atom" "punct" "eof"))
+      (dl-tk-test!
+        "atom dot value"
+        (dl-tk-values (dl-tokenize "foo."))
+        (list "foo" "." nil))
+      (dl-tk-test!
+        "var"
+        (dl-tk-types (dl-tokenize "X."))
+        (list "var" "punct" "eof"))
+      (dl-tk-test!
+        "underscore var"
+        (dl-tk-types (dl-tokenize "_x."))
+        (list "var" "punct" "eof"))
+      (dl-tk-test!
+        "integer"
+        (dl-tk-values (dl-tokenize "42"))
+        (list 42 nil))
+      (dl-tk-test!
+        "decimal"
+        (dl-tk-values (dl-tokenize "3.14"))
+        (list 3.14 nil))
+      (dl-tk-test!
+        "string"
+        (dl-tk-values (dl-tokenize "\"hello\""))
+        (list "hello" nil))
+      ;; Quoted 'atoms' tokenize as strings — see the type-table
+      ;; comment in lib/datalog/tokenizer.sx for the rationale.
+      (dl-tk-test!
+        "quoted atom as string"
+        (dl-tk-types (dl-tokenize "'two words'"))
+        (list "string" "eof"))
+      (dl-tk-test!
+        "quoted atom value"
+        (dl-tk-values (dl-tokenize "'two words'"))
+        (list "two words" nil))
+      ;; A quoted atom whose name would otherwise be a variable
+      ;; (uppercase / leading underscore) is now safely a string —
+      ;; this was the bug that motivated the type change.
+      (dl-tk-test!
+        "quoted Uppercase as string"
+        (dl-tk-types (dl-tokenize "'Hello'"))
+        (list "string" "eof"))
+      (dl-tk-test! ":-" (dl-tk-values (dl-tokenize ":-")) (list ":-" nil))
+      (dl-tk-test! "?-" (dl-tk-values (dl-tokenize "?-")) (list "?-" nil))
+      (dl-tk-test! "<=" (dl-tk-values (dl-tokenize "<=")) (list "<=" nil))
+      (dl-tk-test! ">=" (dl-tk-values (dl-tokenize ">=")) (list ">=" nil))
+      (dl-tk-test! "!=" (dl-tk-values (dl-tokenize "!=")) (list "!=" nil))
+      (dl-tk-test!
+        "single op values"
+        (dl-tk-values (dl-tokenize "< > = + - * /"))
+        (list "<" ">" "=" "+" "-" "*" "/" nil))
+      (dl-tk-test!
+        "single op types"
+        (dl-tk-types (dl-tokenize "< > = + - * /"))
+        (list "op" "op" "op" "op" "op" "op" "op" "eof"))
+      (dl-tk-test!
+        "punct"
+        (dl-tk-values (dl-tokenize "( ) , ."))
+        (list "(" ")" "," "." nil))
+      (dl-tk-test!
+        "fact tokens"
+        (dl-tk-types (dl-tokenize "parent(tom, bob)."))
+        (list "atom" "punct" "atom" "punct" "atom" "punct" "punct" "eof"))
+      (dl-tk-test!
+        "rule shape"
+        (dl-tk-types (dl-tokenize "p(X) :- q(X)."))
+        (list
+          "atom"
+          "punct"
+          "var"
+          "punct"
+          "op"
+          "atom"
+          "punct"
+          "var"
+          "punct"
+          "punct"
+          "eof"))
+      (dl-tk-test!
+        "comparison literal"
+        (dl-tk-values (dl-tokenize "<(X, 5)"))
+        (list "<" "(" "X" "," 5 ")" nil))
+      (dl-tk-test!
+        "is form"
+        (dl-tk-values (dl-tokenize "is(Y, +(X, 1))"))
+        (list "is" "(" "Y" "," "+" "(" "X" "," 1 ")" ")" nil))
+      (dl-tk-test!
+        "line comment"
+        (dl-tk-types (dl-tokenize "% comment line\nfoo."))
+        (list "atom" "punct" "eof"))
+      (dl-tk-test!
+        "block comment"
+        (dl-tk-types (dl-tokenize "/* a\nb */ x."))
+        (list "atom" "punct" "eof"))
+      ;; Unexpected characters surface at tokenize time rather
+      ;; than being silently consumed (previously `?(X)` parsed as
+      ;; if the leading `?` weren't there).
+      (dl-tk-test!
+        "unexpected char raises"
+        (let ((threw false))
+          (do
+            (guard (e (#t (set! threw true)))
+              (dl-tokenize "?(X)"))
+            threw))
+        true)
+
+      ;; Unterminated string / quoted-atom must raise.
+      (dl-tk-test!
+        "unterminated string raises"
+        (let ((threw false))
+          (do
+            (guard (e (#t (set! threw true)))
+              (dl-tokenize "\"unclosed"))
+            threw))
+        true)
+
+      (dl-tk-test!
+        "unterminated quoted atom raises"
+        (let ((threw false))
+          (do
+            (guard (e (#t (set! threw true)))
+              (dl-tokenize "'unclosed"))
+            threw))
+        true)
+
+      ;; Unterminated block comment must raise — previously it was
+      ;; silently consumed to EOF.
+      (dl-tk-test!
+        "unterminated block comment raises"
+        (let ((threw false))
+          (do
+            (guard (e (#t (set! threw true)))
+              (dl-tokenize "/* unclosed comment"))
+            threw))
+        true)
+      (dl-tk-test!
+        "whitespace"
+        (dl-tk-types (dl-tokenize "  foo  ,\t bar  ."))
+        (list "atom" "punct" "atom" "punct" "eof"))
+      (dl-tk-test!
+        "positions"
+        (map (fn (t) (get t :pos)) (dl-tokenize "foo bar"))
+        (list 0 4 7)))))
+
+(define
+  dl-tokenize-tests-run!
+  (fn
+    ()
+    (do
+      (set! dl-tk-pass 0)
+      (set! dl-tk-fail 0)
+      (set! dl-tk-failures (list))
+      (dl-tk-run-all!)
+      {:failures dl-tk-failures :total (+ dl-tk-pass dl-tk-fail) :passed dl-tk-pass :failed dl-tk-fail})))

lib/datalog/tests/unify.sx

@@ -0,0 +1,194 @@
+;; lib/datalog/tests/unify.sx — unification + substitution tests.
+
+(define dl-ut-pass 0)
+(define dl-ut-fail 0)
+(define dl-ut-failures (list))
+
+(define
+  dl-ut-deep-equal?
+  (fn
+    (a b)
+    (cond
+      ((and (list? a) (list? b))
+        (and (= (len a) (len b)) (dl-ut-deq-list? a b 0)))
+      ((and (dict? a) (dict? b))
+        (let
+          ((ka (keys a)) (kb (keys b)))
+          (and (= (len ka) (len kb)) (dl-ut-deq-dict? a b ka 0))))
+      ((and (number? a) (number? b)) (= a b))
+      (else (equal? a b)))))
+
+(define
+  dl-ut-deq-list?
+  (fn
+    (a b i)
+    (cond
+      ((>= i (len a)) true)
+      ((not (dl-ut-deep-equal? (nth a i) (nth b i))) false)
+      (else (dl-ut-deq-list? a b (+ i 1))))))
+
+(define
+  dl-ut-deq-dict?
+  (fn
+    (a b ka i)
+    (cond
+      ((>= i (len ka)) true)
+      ((let ((k (nth ka i))) (not (dl-ut-deep-equal? (get a k) (get b k))))
+        false)
+      (else (dl-ut-deq-dict? a b ka (+ i 1))))))
+
+(define
+  dl-ut-test!
+  (fn
+    (name got expected)
+    (if
+      (dl-ut-deep-equal? got expected)
+      (set! dl-ut-pass (+ dl-ut-pass 1))
+      (do
+        (set! dl-ut-fail (+ dl-ut-fail 1))
+        (append!
+          dl-ut-failures
+          (str name "\n    expected: " expected "\n    got:      " got))))))
+
+(define
+  dl-ut-run-all!
+  (fn
+    ()
+    (do
+      (dl-ut-test! "var? uppercase" (dl-var? (quote X)) true)
+      (dl-ut-test! "var? underscore" (dl-var? (quote _foo)) true)
+      (dl-ut-test! "var? lowercase" (dl-var? (quote tom)) false)
+      (dl-ut-test! "var? number" (dl-var? 5) false)
+      (dl-ut-test! "var? string" (dl-var? "hi") false)
+      (dl-ut-test! "var? list" (dl-var? (list 1)) false)
+      (dl-ut-test!
+        "atom-atom match"
+        (dl-unify (quote tom) (quote tom) (dl-empty-subst))
+        {})
+      (dl-ut-test!
+        "atom-atom fail"
+        (dl-unify (quote tom) (quote bob) (dl-empty-subst))
+        nil)
+      (dl-ut-test!
+        "num-num match"
+        (dl-unify 5 5 (dl-empty-subst))
+        {})
+      (dl-ut-test!
+        "num-num fail"
+        (dl-unify 5 6 (dl-empty-subst))
+        nil)
+      (dl-ut-test!
+        "string match"
+        (dl-unify "hi" "hi" (dl-empty-subst))
+        {})
+      (dl-ut-test! "string fail" (dl-unify "hi" "bye" (dl-empty-subst)) nil)
+      (dl-ut-test!
+        "var-atom binds"
+        (dl-unify (quote X) (quote tom) (dl-empty-subst))
+        {:X (quote tom)})
+      (dl-ut-test!
+        "atom-var binds"
+        (dl-unify (quote tom) (quote X) (dl-empty-subst))
+        {:X (quote tom)})
+      (dl-ut-test!
+        "var-var same"
+        (dl-unify (quote X) (quote X) (dl-empty-subst))
+        {})
+      (dl-ut-test!
+        "var-var bind"
+        (let
+          ((s (dl-unify (quote X) (quote Y) (dl-empty-subst))))
+          (dl-walk (quote X) s))
+        (quote Y))
+      (dl-ut-test!
+        "tuple match"
+        (dl-unify
+          (list (quote parent) (quote X) (quote bob))
+          (list (quote parent) (quote tom) (quote Y))
+          (dl-empty-subst))
+        {:X (quote tom) :Y (quote bob)})
+      (dl-ut-test!
+        "tuple arity mismatch"
+        (dl-unify
+          (list (quote p) (quote X))
+          (list (quote p) (quote a) (quote b))
+          (dl-empty-subst))
+        nil)
+      (dl-ut-test!
+        "tuple head mismatch"
+        (dl-unify
+          (list (quote p) (quote X))
+          (list (quote q) (quote X))
+          (dl-empty-subst))
+        nil)
+      (dl-ut-test!
+        "walk chain"
+        (let
+          ((s1 (dl-unify (quote X) (quote Y) (dl-empty-subst))))
+          (let
+            ((s2 (dl-unify (quote Y) (quote tom) s1)))
+            (dl-walk (quote X) s2)))
+        (quote tom))
+
+      ;; Walk with circular substitution must not infinite-loop.
+      ;; Cycles return the current term unchanged.
+      (dl-ut-test!
+        "walk circular subst no hang"
+        (let ((s (dl-bind (quote B) (quote A)
+                   (dl-bind (quote A) (quote B) (dl-empty-subst)))))
+          (dl-walk (quote A) s))
+        (quote A))
+      (dl-ut-test!
+        "apply subst on tuple"
+        (let
+          ((s (dl-bind (quote X) (quote tom) (dl-empty-subst))))
+          (dl-apply-subst (list (quote parent) (quote X) (quote Y)) s))
+        (list (quote parent) (quote tom) (quote Y)))
+      (dl-ut-test!
+        "ground? all const"
+        (dl-ground?
+          (list (quote p) (quote tom) 5)
+          (dl-empty-subst))
+        true)
+      (dl-ut-test!
+        "ground? unbound var"
+        (dl-ground? (list (quote p) (quote X)) (dl-empty-subst))
+        false)
+      (dl-ut-test!
+        "ground? bound var"
+        (let
+          ((s (dl-bind (quote X) (quote tom) (dl-empty-subst))))
+          (dl-ground? (list (quote p) (quote X)) s))
+        true)
+      (dl-ut-test!
+        "ground? bare var"
+        (dl-ground? (quote X) (dl-empty-subst))
+        false)
+      (dl-ut-test!
+        "vars-of basic"
+        (dl-vars-of
+          (list (quote p) (quote X) (quote tom) (quote Y) (quote X)))
+        (list "X" "Y"))
+      (dl-ut-test!
+        "vars-of ground"
+        (dl-vars-of (list (quote p) (quote tom) (quote bob)))
+        (list))
+      (dl-ut-test!
+        "vars-of nested compound"
+        (dl-vars-of
+          (list
+            (quote is)
+            (quote Z)
+            (list (string->symbol "+") (quote X) 1)))
+        (list "Z" "X")))))
+
+(define
+  dl-unify-tests-run!
+  (fn
+    ()
+    (do
+      (set! dl-ut-pass 0)
+      (set! dl-ut-fail 0)
+      (set! dl-ut-failures (list))
+      (dl-ut-run-all!)
+      {:failures dl-ut-failures :total (+ dl-ut-pass dl-ut-fail) :passed dl-ut-pass :failed dl-ut-fail})))

lib/datalog/tokenizer.sx

@@ -0,0 +1,269 @@
+;; lib/datalog/tokenizer.sx — Datalog source → token stream
+;;
+;; Tokens: {:type T :value V :pos P}
+;; Types:
+;;   "atom"    — lowercase-start bare identifier
+;;   "var"     — uppercase-start or _-start ident (value is the name)
+;;   "number"  — numeric literal (decoded to number)
+;;   "string"  — "..." string literal OR quoted 'atom' (treated as a
+;;               string value to avoid the var-vs-atom ambiguity that
+;;               would arise from a quoted atom whose name starts with
+;;               an uppercase letter or underscore)
+;;   "punct"   — ( ) , .
+;;   "op"      — :- ?- <= >= != < > = + - * /
+;;   "eof"
+;;
+;; Datalog has no function symbols in arg position; the parser still
+;; accepts nested compounds for arithmetic ((is X (+ A B))) but safety
+;; analysis rejects non-arithmetic nesting at rule-load time.
+
+(define dl-make-token (fn (type value pos) {:type type :value value :pos pos}))
+
+(define dl-digit? (fn (c) (and (>= c "0") (<= c "9"))))
+(define dl-lower? (fn (c) (and (>= c "a") (<= c "z"))))
+(define dl-upper? (fn (c) (and (>= c "A") (<= c "Z"))))
+
+(define
+  dl-ident-char?
+  (fn (c) (or (dl-lower? c) (dl-upper? c) (dl-digit? c) (= c "_"))))
+
+(define dl-ws? (fn (c) (or (= c " ") (= c "\t") (= c "\n") (= c "\r"))))
+
+(define
+  dl-tokenize
+  (fn
+    (src)
+    (let
+      ((tokens (list)) (pos 0) (src-len (len src)))
+      (define
+        dl-peek
+        (fn
+          (offset)
+          (if (< (+ pos offset) src-len) (nth src (+ pos offset)) nil)))
+      (define cur (fn () (dl-peek 0)))
+      (define advance! (fn (n) (set! pos (+ pos n))))
+      (define
+        at?
+        (fn
+          (s)
+          (let
+            ((sl (len s)))
+            (and (<= (+ pos sl) src-len) (= (slice src pos (+ pos sl)) s)))))
+      (define
+        dl-emit!
+        (fn
+          (type value start)
+          (append! tokens (dl-make-token type value start))))
+      (define
+        skip-line-comment!
+        (fn
+          ()
+          (when
+            (and (< pos src-len) (not (= (cur) "\n")))
+            (do (advance! 1) (skip-line-comment!)))))
+      (define
+        skip-block-comment!
+        (fn
+          ()
+          (cond
+            ((>= pos src-len)
+              (error (str "Tokenizer: unterminated block comment "
+                          "(started at position " pos ")")))
+            ((and (= (cur) "*") (< (+ pos 1) src-len) (= (dl-peek 1) "/"))
+              (advance! 2))
+            (else (do (advance! 1) (skip-block-comment!))))))
+      (define
+        skip-ws!
+        (fn
+          ()
+          (cond
+            ((>= pos src-len) nil)
+            ((dl-ws? (cur)) (do (advance! 1) (skip-ws!)))
+            ((= (cur) "%")
+              (do (advance! 1) (skip-line-comment!) (skip-ws!)))
+            ((and (= (cur) "/") (< (+ pos 1) src-len) (= (dl-peek 1) "*"))
+              (do (advance! 2) (skip-block-comment!) (skip-ws!)))
+            (else nil))))
+      (define
+        read-ident
+        (fn
+          (start)
+          (do
+            (when
+              (and (< pos src-len) (dl-ident-char? (cur)))
+              (do (advance! 1) (read-ident start)))
+            (slice src start pos))))
+      (define
+        read-decimal-digits!
+        (fn
+          ()
+          (when
+            (and (< pos src-len) (dl-digit? (cur)))
+            (do (advance! 1) (read-decimal-digits!)))))
+      (define
+        read-number
+        (fn
+          (start)
+          (do
+            (read-decimal-digits!)
+            (when
+              (and
+                (< pos src-len)
+                (= (cur) ".")
+                (< (+ pos 1) src-len)
+                (dl-digit? (dl-peek 1)))
+              (do (advance! 1) (read-decimal-digits!)))
+            (parse-number (slice src start pos)))))
+      (define
+        read-quoted
+        (fn
+          (quote-char)
+          (let
+            ((chars (list)))
+            (advance! 1)
+            (define
+              loop
+              (fn
+                ()
+                (cond
+                  ((>= pos src-len)
+                    (error
+                      (str "Tokenizer: unterminated "
+                           (if (= quote-char "'") "quoted atom" "string")
+                           " (started near position " pos ")")))
+                  ((= (cur) "\\")
+                    (do
+                      (advance! 1)
+                      (when
+                        (< pos src-len)
+                        (let
+                          ((ch (cur)))
+                          (do
+                            (cond
+                              ((= ch "n") (append! chars "\n"))
+                              ((= ch "t") (append! chars "\t"))
+                              ((= ch "r") (append! chars "\r"))
+                              ((= ch "\\") (append! chars "\\"))
+                              ((= ch "'") (append! chars "'"))
+                              ((= ch "\"") (append! chars "\""))
+                              (else (append! chars ch)))
+                            (advance! 1))))
+                      (loop)))
+                  ((= (cur) quote-char) (advance! 1))
+                  (else
+                    (do (append! chars (cur)) (advance! 1) (loop))))))
+            (loop)
+            (join "" chars))))
+      (define
+        scan!
+        (fn
+          ()
+          (do
+            (skip-ws!)
+            (when
+              (< pos src-len)
+              (let
+                ((ch (cur)) (start pos))
+                (cond
+                  ((at? ":-")
+                    (do
+                      (dl-emit! "op" ":-" start)
+                      (advance! 2)
+                      (scan!)))
+                  ((at? "?-")
+                    (do
+                      (dl-emit! "op" "?-" start)
+                      (advance! 2)
+                      (scan!)))
+                  ((at? "<=")
+                    (do
+                      (dl-emit! "op" "<=" start)
+                      (advance! 2)
+                      (scan!)))
+                  ((at? ">=")
+                    (do
+                      (dl-emit! "op" ">=" start)
+                      (advance! 2)
+                      (scan!)))
+                  ((at? "!=")
+                    (do
+                      (dl-emit! "op" "!=" start)
+                      (advance! 2)
+                      (scan!)))
+                  ((dl-digit? ch)
+                    (do
+                      (dl-emit! "number" (read-number start) start)
+                      (scan!)))
+                  ((= ch "'")
+                    ;; Quoted 'atoms' tokenize as strings so a name
+                    ;; like 'Hello World' doesn't get misclassified
+                    ;; as a variable by dl-var? (which inspects the
+                    ;; symbol's first character).
+                    (do (dl-emit! "string" (read-quoted "'") start) (scan!)))
+                  ((= ch "\"")
+                    (do (dl-emit! "string" (read-quoted "\"") start) (scan!)))
+                  ((dl-lower? ch)
+                    (do (dl-emit! "atom" (read-ident start) start) (scan!)))
+                  ((or (dl-upper? ch) (= ch "_"))
+                    (do (dl-emit! "var" (read-ident start) start) (scan!)))
+                  ((= ch "(")
+                    (do
+                      (dl-emit! "punct" "(" start)
+                      (advance! 1)
+                      (scan!)))
+                  ((= ch ")")
+                    (do
+                      (dl-emit! "punct" ")" start)
+                      (advance! 1)
+                      (scan!)))
+                  ((= ch ",")
+                    (do
+                      (dl-emit! "punct" "," start)
+                      (advance! 1)
+                      (scan!)))
+                  ((= ch ".")
+                    (do
+                      (dl-emit! "punct" "." start)
+                      (advance! 1)
+                      (scan!)))
+                  ((= ch "<")
+                    (do
+                      (dl-emit! "op" "<" start)
+                      (advance! 1)
+                      (scan!)))
+                  ((= ch ">")
+                    (do
+                      (dl-emit! "op" ">" start)
+                      (advance! 1)
+                      (scan!)))
+                  ((= ch "=")
+                    (do
+                      (dl-emit! "op" "=" start)
+                      (advance! 1)
+                      (scan!)))
+                  ((= ch "+")
+                    (do
+                      (dl-emit! "op" "+" start)
+                      (advance! 1)
+                      (scan!)))
+                  ((= ch "-")
+                    (do
+                      (dl-emit! "op" "-" start)
+                      (advance! 1)
+                      (scan!)))
+                  ((= ch "*")
+                    (do
+                      (dl-emit! "op" "*" start)
+                      (advance! 1)
+                      (scan!)))
+                  ((= ch "/")
+                    (do
+                      (dl-emit! "op" "/" start)
+                      (advance! 1)
+                      (scan!)))
+                  (else (error
+                          (str "Tokenizer: unexpected character '" ch
+                               "' at position " start)))))))))
+      (scan!)
+      (dl-emit! "eof" nil pos)
+      tokens)))

lib/datalog/unify.sx

@@ -0,0 +1,171 @@
+;; lib/datalog/unify.sx — unification + substitution for Datalog terms.
+;;
+;; Term taxonomy (after parsing):
+;;   variable  — SX symbol whose first char is uppercase A–Z or '_'.
+;;   constant  — SX symbol whose first char is lowercase a–z (atom name).
+;;   number    — numeric literal.
+;;   string    — string literal.
+;;   compound  — SX list (functor arg ... arg). In core Datalog these
+;;               only appear as arithmetic expressions (see Phase 4
+;;               safety analysis); compound-against-compound unification
+;;               is supported anyway for completeness.
+;;
+;; Substitutions are immutable dicts keyed by variable name (string).
+;; A failed unification returns nil; success returns the extended subst.
+
+(define dl-empty-subst (fn () {}))
+
+(define
+  dl-var?
+  (fn
+    (term)
+    (and
+      (symbol? term)
+      (let
+        ((name (symbol->string term)))
+        (and
+          (> (len name) 0)
+          (let
+            ((c (slice name 0 1)))
+            (or (and (>= c "A") (<= c "Z")) (= c "_"))))))))
+
+;; Walk: chase variable bindings until we hit a non-variable or an unbound
+;; variable. The result is either a non-variable term or an unbound var.
+(define
+  dl-walk
+  (fn (term subst) (dl-walk-aux term subst (list))))
+
+;; Internal: walk with a visited-var set so circular substitutions
+;; (from raw dl-bind misuse) don't infinite-loop. Cycles return the
+;; current term unchanged.
+(define
+  dl-walk-aux
+  (fn
+    (term subst visited)
+    (if
+      (dl-var? term)
+      (let
+        ((name (symbol->string term)))
+        (cond
+          ((dl-member? name visited) term)
+          ((and (dict? subst) (has-key? subst name))
+            (let ((seen (list)))
+              (do
+                (for-each (fn (v) (append! seen v)) visited)
+                (append! seen name)
+                (dl-walk-aux (get subst name) subst seen))))
+          (else term)))
+      term)))
+
+;; Bind a variable symbol to a value in subst, returning a new subst.
+(define
+  dl-bind
+  (fn (var-sym value subst) (assoc subst (symbol->string var-sym) value)))
+
+(define
+  dl-unify
+  (fn
+    (t1 t2 subst)
+    (if
+      (nil? subst)
+      nil
+      (let
+        ((u1 (dl-walk t1 subst)) (u2 (dl-walk t2 subst)))
+        (cond
+          ((dl-var? u1)
+            (cond
+              ((and (dl-var? u2) (= (symbol->string u1) (symbol->string u2)))
+                subst)
+              (else (dl-bind u1 u2 subst))))
+          ((dl-var? u2) (dl-bind u2 u1 subst))
+          ((and (list? u1) (list? u2))
+            (if
+              (= (len u1) (len u2))
+              (dl-unify-list u1 u2 subst 0)
+              nil))
+          ((and (number? u1) (number? u2)) (if (= u1 u2) subst nil))
+          ((and (string? u1) (string? u2)) (if (= u1 u2) subst nil))
+          ((and (symbol? u1) (symbol? u2))
+            (if (= (symbol->string u1) (symbol->string u2)) subst nil))
+          (else nil))))))
+
+(define
+  dl-unify-list
+  (fn
+    (a b subst i)
+    (cond
+      ((nil? subst) nil)
+      ((>= i (len a)) subst)
+      (else
+        (dl-unify-list
+          a
+          b
+          (dl-unify (nth a i) (nth b i) subst)
+          (+ i 1))))))
+
+;; Apply substitution: walk the term and recurse into lists.
+(define
+  dl-apply-subst
+  (fn
+    (term subst)
+    (let
+      ((w (dl-walk term subst)))
+      (if (list? w) (map (fn (x) (dl-apply-subst x subst)) w) w))))
+
+;; Ground? — true iff no free variables remain after walking.
+(define
+  dl-ground?
+  (fn
+    (term subst)
+    (let
+      ((w (dl-walk term subst)))
+      (cond
+        ((dl-var? w) false)
+        ((list? w) (dl-ground-list? w subst 0))
+        (else true)))))
+
+(define
+  dl-ground-list?
+  (fn
+    (xs subst i)
+    (cond
+      ((>= i (len xs)) true)
+      ((not (dl-ground? (nth xs i) subst)) false)
+      (else (dl-ground-list? xs subst (+ i 1))))))
+
+;; Return the list of variable names appearing in a term (deduped, in
+;; left-to-right order). Useful for safety analysis later.
+(define
+  dl-vars-of
+  (fn (term) (let ((seen (list))) (do (dl-vars-of-aux term seen) seen))))
+
+(define
+  dl-vars-of-aux
+  (fn
+    (term acc)
+    (cond
+      ((dl-var? term)
+        (let
+          ((name (symbol->string term)))
+          (when (not (dl-member? name acc)) (append! acc name))))
+      ((list? term) (dl-vars-of-list term acc 0))
+      (else nil))))
+
+(define
+  dl-vars-of-list
+  (fn
+    (xs acc i)
+    (when
+      (< i (len xs))
+      (do
+        (dl-vars-of-aux (nth xs i) acc)
+        (dl-vars-of-list xs acc (+ i 1))))))
+
+(define
+  dl-member?
+  (fn
+    (x xs)
+    (cond
+      ((= (len xs) 0) false)
+      ((= (first xs) x) true)
+      (else (dl-member? x (rest xs))))))

lib/guest/ast.sx

@@ -0,0 +1,92 @@
+;; lib/guest/ast.sx — canonical AST node shapes.
+;;
+;; A guest's parser may emit its own AST in whatever shape is convenient
+;; for that language's evaluator/transpiler. This file gives a SHARED
+;; canonical shape that cross-language tools (formatters, highlighters,
+;; debuggers) can target without per-language adapters.
+;;
+;; Each canonical node is a tagged list: (KIND ...payload).
+;;
+;; Constructors (return a canonical node):
+;;
+;;   (ast-literal       VALUE)               — number / string / bool / nil
+;;   (ast-var           NAME)                — identifier reference
+;;   (ast-app           FN ARGS)             — function application
+;;   (ast-lambda        PARAMS BODY)         — anonymous function
+;;   (ast-let           BINDINGS BODY)       — local bindings
+;;   (ast-letrec        BINDINGS BODY)       — recursive local bindings
+;;   (ast-if            TEST THEN ELSE)      — conditional
+;;   (ast-match-clause  PATTERN BODY)        — one match arm
+;;   (ast-module        NAME BODY)           — module declaration
+;;   (ast-import        NAME)                — import directive
+;;
+;; Predicates: (ast-literal? X), (ast-var? X), …
+;; Generic:    (ast? X)         — any canonical node
+;;             (ast-kind X)     — :literal / :var / :app / …
+;;
+;; Accessors  (one per payload field):
+;;   (ast-literal-value N)
+;;   (ast-var-name N)
+;;   (ast-app-fn N)             (ast-app-args N)
+;;   (ast-lambda-params N)      (ast-lambda-body N)
+;;   (ast-let-bindings N)       (ast-let-body N)
+;;   (ast-letrec-bindings N)    (ast-letrec-body N)
+;;   (ast-if-test N)            (ast-if-then N)         (ast-if-else N)
+;;   (ast-match-clause-pattern N)
+;;   (ast-match-clause-body N)
+;;   (ast-module-name N)        (ast-module-body N)
+;;   (ast-import-name N)
+
+(define ast-literal      (fn (v)         (list :literal v)))
+(define ast-var          (fn (n)         (list :var n)))
+(define ast-app          (fn (f args)    (list :app f args)))
+(define ast-lambda       (fn (ps body)   (list :lambda ps body)))
+(define ast-let          (fn (bs body)   (list :let bs body)))
+(define ast-letrec       (fn (bs body)   (list :letrec bs body)))
+(define ast-if           (fn (t th el)   (list :if t th el)))
+(define ast-match-clause (fn (p body)    (list :match-clause p body)))
+(define ast-module       (fn (n body)    (list :module n body)))
+(define ast-import       (fn (n)         (list :import n)))
+
+(define ast-kind (fn (x) (if (and (list? x) (not (empty? x))) (first x) nil)))
+
+(define
+  ast?
+  (fn (x)
+    (and (list? x)
+         (not (empty? x))
+         (let ((k (first x)))
+           (or (= k :literal) (= k :var) (= k :app)
+               (= k :lambda)  (= k :let) (= k :letrec)
+               (= k :if)      (= k :match-clause)
+               (= k :module)  (= k :import))))))
+
+(define ast-literal?      (fn (x) (and (ast? x) (= (first x) :literal))))
+(define ast-var?          (fn (x) (and (ast? x) (= (first x) :var))))
+(define ast-app?          (fn (x) (and (ast? x) (= (first x) :app))))
+(define ast-lambda?       (fn (x) (and (ast? x) (= (first x) :lambda))))
+(define ast-let?          (fn (x) (and (ast? x) (= (first x) :let))))
+(define ast-letrec?       (fn (x) (and (ast? x) (= (first x) :letrec))))
+(define ast-if?           (fn (x) (and (ast? x) (= (first x) :if))))
+(define ast-match-clause? (fn (x) (and (ast? x) (= (first x) :match-clause))))
+(define ast-module?       (fn (x) (and (ast? x) (= (first x) :module))))
+(define ast-import?       (fn (x) (and (ast? x) (= (first x) :import))))
+
+(define ast-literal-value (fn (n) (nth n 1)))
+(define ast-var-name      (fn (n) (nth n 1)))
+(define ast-app-fn        (fn (n) (nth n 1)))
+(define ast-app-args      (fn (n) (nth n 2)))
+(define ast-lambda-params (fn (n) (nth n 1)))
+(define ast-lambda-body   (fn (n) (nth n 2)))
+(define ast-let-bindings  (fn (n) (nth n 1)))
+(define ast-let-body      (fn (n) (nth n 2)))
+(define ast-letrec-bindings (fn (n) (nth n 1)))
+(define ast-letrec-body     (fn (n) (nth n 2)))
+(define ast-if-test       (fn (n) (nth n 1)))
+(define ast-if-then       (fn (n) (nth n 2)))
+(define ast-if-else       (fn (n) (nth n 3)))
+(define ast-match-clause-pattern (fn (n) (nth n 1)))
+(define ast-match-clause-body    (fn (n) (nth n 2)))
+(define ast-module-name   (fn (n) (nth n 1)))
+(define ast-module-body   (fn (n) (nth n 2)))
+(define ast-import-name   (fn (n) (nth n 1)))

lib/guest/hm.sx

@@ -0,0 +1,180 @@
+;; lib/guest/hm.sx — Hindley-Milner type-inference foundations.
+;;
+;; Builds on lib/guest/match.sx (terms + unify) and ast.sx (canonical
+;; AST shapes). This file ships the ALGEBRA — types, schemes, free
+;; type-vars, generalize / instantiate, substitution composition — so a
+;; full Algorithm W (or J) can be assembled on top either inside this
+;; file or in a host-specific consumer (haskell/infer.sx,
+;; lib/ocaml/types.sx, …).
+;;
+;; Per the brief the second consumer for this step is OCaml-on-SX
+;; Phase 5 (paired sequencing). Until that lands, the algebra is the
+;; deliverable; the host-flavoured assembly (lambda / app / let
+;; inference rules with substitution threading) lives in the host.
+;;
+;; Types
+;; -----
+;; A type is a canonical match.sx term — type variables use mk-var,
+;; type constructors use mk-ctor:
+;;   (hm-tv NAME)              type variable
+;;   (hm-arrow A B)             A -> B
+;;   (hm-con NAME ARGS)         named n-ary constructor
+;;   (hm-int) / (hm-bool) / (hm-string)   primitive constructors
+;;
+;; Schemes
+;; -------
+;;   (hm-scheme VARS TYPE)        ∀ VARS . TYPE
+;;   (hm-monotype TYPE)           empty quantifier
+;;   (hm-scheme? S) (hm-scheme-vars S) (hm-scheme-type S)
+;;
+;; Free type variables
+;; -------------------
+;;   (hm-ftv TYPE)                names occurring in TYPE
+;;   (hm-ftv-scheme S)            free names (minus quantifiers)
+;;   (hm-ftv-env ENV)             free across an env (name -> scheme)
+;;
+;; Substitution
+;; ------------
+;;   (hm-apply SUBST TYPE)        substitute through a type
+;;   (hm-apply-scheme SUBST S)    leaves bound vars alone
+;;   (hm-apply-env SUBST ENV)
+;;   (hm-compose S2 S1)           apply S1 then S2
+;;
+;; Generalize / Instantiate
+;; ------------------------
+;;   (hm-generalize TYPE ENV)      → scheme over ftv(t) - ftv(env)
+;;   (hm-instantiate SCHEME COUNTER) → fresh-var instance
+;;   (hm-fresh-tv COUNTER)         → (:var "tN"), bumps COUNTER
+;;
+;; Inference (literal only — the rest of Algorithm W lives in the host)
+;; --------------------------------------------------------------------
+;;   (hm-infer-literal EXPR)       → {:subst {} :type T}
+;;
+;; A complete Algorithm W consumes this kit by assembling lambda / app
+;; / let rules in the host language file.
+
+(define hm-tv     (fn (name) (list :var name)))
+(define hm-con    (fn (name args) (list :ctor name args)))
+(define hm-arrow  (fn (a b) (hm-con "->" (list a b))))
+(define hm-int    (fn () (hm-con "Int" (list))))
+(define hm-bool   (fn () (hm-con "Bool" (list))))
+(define hm-string (fn () (hm-con "String" (list))))
+
+(define hm-scheme        (fn (vars t) (list :scheme vars t)))
+(define hm-monotype      (fn (t) (hm-scheme (list) t)))
+(define hm-scheme?       (fn (s) (and (list? s) (not (empty? s)) (= (first s) :scheme))))
+(define hm-scheme-vars   (fn (s) (nth s 1)))
+(define hm-scheme-type   (fn (s) (nth s 2)))
+
+(define
+  hm-fresh-tv
+  (fn (counter)
+    (let ((n (first counter)))
+      (begin
+        (set-nth! counter 0 (+ n 1))
+        (hm-tv (str "t" (+ n 1)))))))
+
+(define
+  hm-ftv-acc
+  (fn (t acc)
+    (cond
+      ((is-var? t)
+        (if (some (fn (n) (= n (var-name t))) acc) acc (cons (var-name t) acc)))
+      ((is-ctor? t)
+        (let ((a acc))
+          (begin
+            (for-each (fn (x) (set! a (hm-ftv-acc x a))) (ctor-args t))
+            a)))
+      (:else acc))))
+
+(define hm-ftv (fn (t) (hm-ftv-acc t (list))))
+
+(define
+  hm-ftv-scheme
+  (fn (s)
+    (let ((qs (hm-scheme-vars s))
+          (all (hm-ftv (hm-scheme-type s))))
+      (filter (fn (n) (not (some (fn (q) (= q n)) qs))) all))))
+
+(define
+  hm-ftv-env
+  (fn (env)
+    (let ((acc (list)))
+      (begin
+        (for-each
+          (fn (k)
+            (for-each
+              (fn (n)
+                (when (not (some (fn (m) (= m n)) acc))
+                  (set! acc (cons n acc))))
+              (hm-ftv-scheme (get env k))))
+          (keys env))
+        acc))))
+
+(define hm-apply (fn (subst t) (walk* t subst)))
+
+(define
+  hm-apply-scheme
+  (fn (subst s)
+    (let ((qs (hm-scheme-vars s))
+          (d {}))
+      (begin
+        (for-each
+          (fn (k)
+            (when (not (some (fn (q) (= q k)) qs))
+              (dict-set! d k (get subst k))))
+          (keys subst))
+        (hm-scheme qs (walk* (hm-scheme-type s) d))))))
+
+(define
+  hm-apply-env
+  (fn (subst env)
+    (let ((d {}))
+      (begin
+        (for-each
+          (fn (k) (dict-set! d k (hm-apply-scheme subst (get env k))))
+          (keys env))
+        d))))
+
+(define
+  hm-compose
+  (fn (s2 s1)
+    (let ((d {}))
+      (begin
+        (for-each (fn (k) (dict-set! d k (walk* (get s1 k) s2))) (keys s1))
+        (for-each
+          (fn (k) (when (not (has-key? d k)) (dict-set! d k (get s2 k))))
+          (keys s2))
+        d))))
+
+(define
+  hm-generalize
+  (fn (t env)
+    (let ((tvars (hm-ftv t))
+          (evars (hm-ftv-env env)))
+      (let ((qs (filter (fn (n) (not (some (fn (m) (= m n)) evars))) tvars)))
+        (hm-scheme qs t)))))
+
+(define
+  hm-instantiate
+  (fn (s counter)
+    (let ((qs (hm-scheme-vars s))
+          (subst {}))
+      (begin
+        (for-each
+          (fn (q) (set! subst (assoc subst q (hm-fresh-tv counter))))
+          qs)
+        (walk* (hm-scheme-type s) subst)))))
+
+;; Literal inference — the only AST kind whose typing rule is closed
+;; in the kit. Lambda / app / let live in host code so the host's own
+;; AST conventions stay untouched.
+(define
+  hm-infer-literal
+  (fn (expr)
+    (let ((v (ast-literal-value expr)))
+      (cond
+        ((number? v)  {:subst {} :type (hm-int)})
+        ((string? v)  {:subst {} :type (hm-string)})
+        ((boolean? v) {:subst {} :type (hm-bool)})
+        (:else (error (str "hm-infer-literal: unknown kind: " v)))))))

lib/guest/layout.sx

@@ -0,0 +1,145 @@
+;; lib/guest/layout.sx — configurable off-side / layout-sensitive lexer.
+;;
+;; Inserts virtual open / close / separator tokens based on indentation.
+;; Configurable enough to encode either the Haskell 98 layout rule (let /
+;; where / do / of opens a virtual brace at the next token's column) or
+;; a Python-ish indent / dedent rule (a colon at the end of a line opens
+;; a block at the next non-blank line's column).
+;;
+;; Token shape (input + output)
+;; ----------------------------
+;; Each token is a dict {:type :value :line :col …}. The kit reads
+;; only :type / :value / :line / :col and passes everything else
+;; through. The input stream MUST be free of newline filler tokens
+;; (preprocess them away with your tokenizer) — line breaks are detected
+;; by comparing :line of consecutive tokens.
+;;
+;; Config
+;; ------
+;;   :open-keywords    list of strings; a token whose :value matches
+;;                     opens a new layout block at the next token's
+;;                     column (Haskell: let/where/do/of).
+;;   :open-trailing-fn (fn (tok) -> bool) — alternative trigger that
+;;                     fires AFTER the token is emitted. Use for
+;;                     Python-style trailing `:`.
+;;   :open-token / :close-token / :sep-token
+;;                     templates {:type :value} merged with :line and
+;;                     :col when virtual tokens are emitted.
+;;   :explicit-open?   (fn (tok) -> bool) — if the next token after a
+;;                     trigger satisfies this, suppress virtual layout
+;;                     for that block (Haskell: `{`).
+;;   :module-prelude?  if true, wrap whole input in an implicit block
+;;                     at the first token's column (Haskell yes,
+;;                     Python no).
+;;
+;; Public entry
+;; ------------
+;;   (layout-pass cfg tokens) -> tokens with virtual layout inserted.
+
+(define
+  layout-mk-virtual
+  (fn (template line col)
+    (assoc (assoc template :line line) :col col)))
+
+(define
+  layout-is-open-kw?
+  (fn (tok open-kws)
+    (and (= (get tok :type) "reserved")
+         (some (fn (k) (= k (get tok :value))) open-kws))))
+
+(define
+  layout-pass
+  (fn (cfg tokens)
+    (let ((open-kws (get cfg :open-keywords))
+          (trailing-fn (get cfg :open-trailing-fn))
+          (open-tmpl (get cfg :open-token))
+          (close-tmpl (get cfg :close-token))
+          (sep-tmpl (get cfg :sep-token))
+          (mod-prelude? (get cfg :module-prelude?))
+          (expl?-fn (get cfg :explicit-open?))
+          (out (list))
+          (stack (list))
+          (n (len tokens))
+          (i 0)
+          (prev-line -1)
+          (pending-open false)
+          (just-opened false))
+      (define
+        emit-closes-while-greater
+        (fn (col line)
+          (when (and (not (empty? stack)) (> (first stack) col))
+            (do
+              (append! out (layout-mk-virtual close-tmpl line col))
+              (set! stack (rest stack))
+              (emit-closes-while-greater col line)))))
+      (define
+        emit-pending-open
+        (fn (line col)
+          (do
+            (append! out (layout-mk-virtual open-tmpl line col))
+            (set! stack (cons col stack))
+            (set! pending-open false)
+            (set! just-opened true))))
+      (define
+        layout-step
+        (fn ()
+          (when (< i n)
+            (let ((tok (nth tokens i)))
+              (let ((line (get tok :line)) (col (get tok :col)))
+                (cond
+                  (pending-open
+                    (cond
+                      ((and (not (= expl?-fn nil)) (expl?-fn tok))
+                        (do
+                          (set! pending-open false)
+                          (append! out tok)
+                          (set! prev-line line)
+                          (set! i (+ i 1))
+                          (layout-step)))
+                      (:else
+                        (do
+                          (emit-pending-open line col)
+                          (layout-step)))))
+                  (:else
+                    (let ((on-fresh-line? (and (> prev-line 0) (> line prev-line))))
+                      (do
+                        (when on-fresh-line?
+                          (let ((stack-before stack))
+                            (begin
+                              (emit-closes-while-greater col line)
+                              (when (and (not (empty? stack))
+                                         (= (first stack) col)
+                                         (not just-opened)
+                                         ;; suppress separator if a dedent fired
+                                         ;; — the dedent is itself the separator
+                                         (= (len stack) (len stack-before)))
+                                (append! out (layout-mk-virtual sep-tmpl line col))))))
+                        (set! just-opened false)
+                        (append! out tok)
+                        (set! prev-line line)
+                        (set! i (+ i 1))
+                        (cond
+                          ((layout-is-open-kw? tok open-kws)
+                            (set! pending-open true))
+                          ((and (not (= trailing-fn nil)) (trailing-fn tok))
+                            (set! pending-open true)))
+                        (layout-step))))))))))
+      (begin
+        ;; Module prelude: implicit layout block at the first token's column.
+        (when (and mod-prelude? (> n 0))
+          (let ((tok (nth tokens 0)))
+            (do
+              (append! out (layout-mk-virtual open-tmpl (get tok :line) (get tok :col)))
+              (set! stack (cons (get tok :col) stack))
+              (set! just-opened true))))
+        (layout-step)
+        ;; EOF: close every remaining block.
+        (define close-rest
+          (fn ()
+            (when (not (empty? stack))
+              (do
+                (append! out (layout-mk-virtual close-tmpl 0 0))
+                (set! stack (rest stack))
+                (close-rest)))))
+        (close-rest)
+        out))))

lib/guest/match.sx

@@ -0,0 +1,185 @@
+;; lib/guest/match.sx — pure pattern-match + unification kit.
+;;
+;; Shipped for miniKanren / Datalog / future logic-flavoured guests that
+;; want immutable unification without writing it from scratch. The two
+;; existing prolog/haskell engines stay as-is — porting them in place
+;; risks the 746 tests they currently pass; consumers can migrate
+;; gradually via the converters in lib/guest/ast.sx.
+;;
+;; Term shapes (canonical wire format)
+;; -----------------------------------
+;;   var          (:var NAME)            NAME a string
+;;   constructor  (:ctor HEAD ARGS)      HEAD a string, ARGS a list of terms
+;;   literal      number / string / boolean / nil
+;;
+;; Guests with their own shape pass adapter callbacks via the cfg arg —
+;; see (unify-with cfg ...) and (match-pat-with cfg ...) below. The
+;; default canonical entry points (unify / match-pat) use the wire shape.
+;;
+;; Substitution / env
+;; ------------------
+;; A substitution is a SX dict mapping VAR-NAME → term. There are no
+;; trails, no mutation: each step either returns an extended dict or nil.
+;;
+;;   (empty-subst)                  → {}
+;;   (walk term s)                  → term with top-level vars resolved
+;;   (walk* term s)                 → term with all vars resolved (recursive)
+;;   (extend name term s)           → s with NAME → term added
+;;   (occurs? name term s)          → bool
+;;
+;; Unify (symmetric, miniKanren-flavour)
+;; -------------------------------------
+;;   (unify u v s)                  → extended subst or nil
+;;   (unify-with cfg u v s)         → ditto, with adapter callbacks:
+;;                                      :var? :var-name :ctor? :ctor-head
+;;                                      :ctor-args :occurs-check?
+;;
+;; Match (asymmetric, haskell-flavour: pattern → value, vars only in pat)
+;; ---------------------------------------------------------------------
+;;   (match-pat pat val env)        → extended env or nil
+;;   (match-pat-with cfg pat val env)
+
+(define mk-var  (fn (name) (list :var name)))
+(define mk-ctor (fn (head args) (list :ctor head args)))
+
+(define is-var?    (fn (t) (and (list? t) (not (empty? t)) (= (first t) :var))))
+(define is-ctor?   (fn (t) (and (list? t) (not (empty? t)) (= (first t) :ctor))))
+(define var-name   (fn (t) (nth t 1)))
+(define ctor-head  (fn (t) (nth t 1)))
+(define ctor-args  (fn (t) (nth t 2)))
+
+(define empty-subst (fn () {}))
+
+(define
+  walk
+  (fn (t s)
+    (if (and (is-var? t) (has-key? s (var-name t)))
+      (walk (get s (var-name t)) s)
+      t)))
+
+(define
+  walk*
+  (fn (t s)
+    (let ((w (walk t s)))
+      (cond
+        ((is-ctor? w)
+          (mk-ctor (ctor-head w) (map (fn (a) (walk* a s)) (ctor-args w))))
+        (:else w)))))
+
+(define
+  extend
+  (fn (name term s)
+    (assoc s name term)))
+
+(define
+  occurs?
+  (fn (name term s)
+    (let ((w (walk term s)))
+      (cond
+        ((is-var? w) (= (var-name w) name))
+        ((is-ctor? w) (some (fn (a) (occurs? name a s)) (ctor-args w)))
+        (:else false)))))
+
+(define
+  unify-with
+  (fn (cfg u v s)
+    (let ((var?-fn (get cfg :var?))
+          (var-name-fn (get cfg :var-name))
+          (ctor?-fn (get cfg :ctor?))
+          (ctor-head-fn (get cfg :ctor-head))
+          (ctor-args-fn (get cfg :ctor-args))
+          (occurs?-on (get cfg :occurs-check?)))
+      (let ((wu (walk-with cfg u s))
+            (wv (walk-with cfg v s)))
+        (cond
+          ((and (var?-fn wu) (var?-fn wv) (= (var-name-fn wu) (var-name-fn wv))) s)
+          ((var?-fn wu)
+            (if (and occurs?-on (occurs-with cfg (var-name-fn wu) wv s))
+              nil
+              (extend (var-name-fn wu) wv s)))
+          ((var?-fn wv)
+            (if (and occurs?-on (occurs-with cfg (var-name-fn wv) wu s))
+              nil
+              (extend (var-name-fn wv) wu s)))
+          ((and (ctor?-fn wu) (ctor?-fn wv))
+            (if (= (ctor-head-fn wu) (ctor-head-fn wv))
+              (unify-list-with
+                cfg
+                (ctor-args-fn wu)
+                (ctor-args-fn wv)
+                s)
+              nil))
+          (:else (if (= wu wv) s nil)))))))
+
+(define
+  walk-with
+  (fn (cfg t s)
+    (if (and ((get cfg :var?) t) (has-key? s ((get cfg :var-name) t)))
+      (walk-with cfg (get s ((get cfg :var-name) t)) s)
+      t)))
+
+(define
+  occurs-with
+  (fn (cfg name term s)
+    (let ((w (walk-with cfg term s)))
+      (cond
+        (((get cfg :var?) w) (= ((get cfg :var-name) w) name))
+        (((get cfg :ctor?) w)
+          (some (fn (a) (occurs-with cfg name a s)) ((get cfg :ctor-args) w)))
+        (:else false)))))
+
+(define
+  unify-list-with
+  (fn (cfg xs ys s)
+    (cond
+      ((and (empty? xs) (empty? ys)) s)
+      ((or (empty? xs) (empty? ys)) nil)
+      (:else
+        (let ((s2 (unify-with cfg (first xs) (first ys) s)))
+          (if (= s2 nil)
+            nil
+            (unify-list-with cfg (rest xs) (rest ys) s2)))))))
+
+(define canonical-cfg
+  {:var? is-var? :var-name var-name
+   :ctor? is-ctor? :ctor-head ctor-head :ctor-args ctor-args
+   :occurs-check? true})
+
+(define unify (fn (u v s) (unify-with canonical-cfg u v s)))
+
+;; Asymmetric pattern match (haskell-style): only patterns may contain vars;
+;; values are concrete. On a var pattern, bind name to value.
+(define
+  match-pat-with
+  (fn (cfg pat val env)
+    (let ((var?-fn (get cfg :var?))
+          (var-name-fn (get cfg :var-name))
+          (ctor?-fn (get cfg :ctor?))
+          (ctor-head-fn (get cfg :ctor-head))
+          (ctor-args-fn (get cfg :ctor-args)))
+      (cond
+        ((var?-fn pat) (extend (var-name-fn pat) val env))
+        ((and (ctor?-fn pat) (ctor?-fn val))
+          (if (= (ctor-head-fn pat) (ctor-head-fn val))
+            (match-list-pat-with
+              cfg
+              (ctor-args-fn pat)
+              (ctor-args-fn val)
+              env)
+            nil))
+        ((ctor?-fn pat) nil)
+        (:else (if (= pat val) env nil))))))
+
+(define
+  match-list-pat-with
+  (fn (cfg pats vals env)
+    (cond
+      ((and (empty? pats) (empty? vals)) env)
+      ((or (empty? pats) (empty? vals)) nil)
+      (:else
+        (let ((env2 (match-pat-with cfg (first pats) (first vals) env)))
+          (if (= env2 nil)
+            nil
+            (match-list-pat-with cfg (rest pats) (rest vals) env2)))))))
+
+(define match-pat (fn (pat val env) (match-pat-with canonical-cfg pat val env)))

lib/guest/pratt.sx

@@ -0,0 +1,28 @@
+;; lib/guest/pratt.sx — operator-table format + lookup for Pratt-style
+;; precedence climbing.
+;;
+;; The climbing loop stays per-language because the two canaries use
+;; opposite conventions (Lua: higher prec = tighter; Prolog: lower prec =
+;; tighter, with xfx/xfy/yfx assoc tags). Forcing a single loop adds
+;; callback indirection that obscures more than it shares.
+;;
+;; What IS shared and gets extracted: the operator-table format and lookup.
+;; "Grammar is a dict, not hardcoded cond."
+;;
+;; Entry shape: (NAME PREC ASSOC).
+;;   NAME   — string, the operator's source token.
+;;   PREC   — integer, in the host's own convention.
+;;   ASSOC  — :left | :right | :none for languages with traditional
+;;            associativity, or "xfx" / "xfy" / "yfx" for Prolog-style.
+
+(define
+  pratt-op-lookup
+  (fn (table name)
+    (cond
+      ((empty? table) nil)
+      ((= (first (first table)) name) (first table))
+      (:else (pratt-op-lookup (rest table) name)))))
+
+(define pratt-op-name  (fn (entry) (first entry)))
+(define pratt-op-prec  (fn (entry) (nth entry 1)))
+(define pratt-op-assoc (fn (entry) (nth entry 2)))

lib/guest/tests/ast.sx

@@ -0,0 +1,63 @@
+;; lib/guest/tests/ast.sx — exercises every constructor / predicate /
+;; accessor in lib/guest/ast.sx so future ports have a stable contract
+;; to point at.
+
+(define gast-test-pass 0)
+(define gast-test-fail 0)
+(define gast-test-fails (list))
+
+(define
+  gast-test
+  (fn (name actual expected)
+    (if (= actual expected)
+      (set! gast-test-pass (+ gast-test-pass 1))
+      (begin
+        (set! gast-test-fail (+ gast-test-fail 1))
+        (append! gast-test-fails {:name name :expected expected :actual actual})))))
+
+;; Constructors round-trip.
+(gast-test "literal-int"    (ast-literal-value (ast-literal 42)) 42)
+(gast-test "literal-str"    (ast-literal-value (ast-literal "hi")) "hi")
+(gast-test "literal-bool"   (ast-literal-value (ast-literal true)) true)
+(gast-test "var-name"       (ast-var-name (ast-var "x")) "x")
+(gast-test "app-fn"         (ast-app-fn (ast-app (ast-var "f") (list (ast-literal 1)))) (ast-var "f"))
+(gast-test "app-args-len"   (len (ast-app-args (ast-app (ast-var "f") (list (ast-literal 1))))) 1)
+(gast-test "lambda-params"  (ast-lambda-params (ast-lambda (list "x" "y") (ast-var "x"))) (list "x" "y"))
+(gast-test "lambda-body"    (ast-lambda-body (ast-lambda (list "x") (ast-var "x"))) (ast-var "x"))
+(gast-test "let-bindings"   (len (ast-let-bindings (ast-let (list {:name "x" :value (ast-literal 1)}) (ast-var "x")))) 1)
+(gast-test "letrec-body"    (ast-letrec-body (ast-letrec (list) (ast-literal 0))) (ast-literal 0))
+(gast-test "if-test"        (ast-if-test (ast-if (ast-literal true) (ast-literal 1) (ast-literal 0))) (ast-literal true))
+(gast-test "if-then"        (ast-if-then (ast-if (ast-literal true) (ast-literal 1) (ast-literal 0))) (ast-literal 1))
+(gast-test "if-else"        (ast-if-else (ast-if (ast-literal true) (ast-literal 1) (ast-literal 0))) (ast-literal 0))
+(gast-test "match-pattern"  (ast-match-clause-pattern (ast-match-clause "P" (ast-literal 1))) "P")
+(gast-test "match-body"     (ast-match-clause-body (ast-match-clause "P" (ast-literal 1))) (ast-literal 1))
+(gast-test "module-name"    (ast-module-name (ast-module "m" (list))) "m")
+(gast-test "import-name"    (ast-import-name (ast-import "lib/foo")) "lib/foo")
+
+;; Predicates fire only on matching kinds.
+(gast-test "is-literal"     (ast-literal? (ast-literal 1)) true)
+(gast-test "not-literal"    (ast-literal? (ast-var "x")) false)
+(gast-test "is-var"         (ast-var? (ast-var "x")) true)
+(gast-test "is-app"         (ast-app? (ast-app (ast-var "f") (list))) true)
+(gast-test "is-lambda"      (ast-lambda? (ast-lambda (list) (ast-literal 0))) true)
+(gast-test "is-let"         (ast-let? (ast-let (list) (ast-literal 0))) true)
+(gast-test "is-letrec"      (ast-letrec? (ast-letrec (list) (ast-literal 0))) true)
+(gast-test "is-if"          (ast-if? (ast-if (ast-literal true) (ast-literal 1) (ast-literal 0))) true)
+(gast-test "is-match"       (ast-match-clause? (ast-match-clause "P" (ast-literal 1))) true)
+(gast-test "is-module"      (ast-module? (ast-module "m" (list))) true)
+(gast-test "is-import"      (ast-import? (ast-import "x")) true)
+
+;; ast? recognises any canonical node.
+(gast-test "ast?-literal"   (ast? (ast-literal 0)) true)
+(gast-test "ast?-foreign"   (ast? (list "lua-num" 0)) false)
+(gast-test "ast?-non-list"  (ast? 42) false)
+
+;; ast-kind dispatch.
+(gast-test "kind-literal"   (ast-kind (ast-literal 0)) :literal)
+(gast-test "kind-import"    (ast-kind (ast-import "x")) :import)
+
+(define gast-tests-run!
+  (fn ()
+    {:passed gast-test-pass
+     :failed gast-test-fail
+     :total  (+ gast-test-pass gast-test-fail)}))

lib/guest/tests/hm.sx

@@ -0,0 +1,89 @@
+;; lib/guest/tests/hm.sx — exercises lib/guest/hm.sx algebra.
+
+(define ghm-test-pass 0)
+(define ghm-test-fail 0)
+(define ghm-test-fails (list))
+
+(define
+  ghm-test
+  (fn (name actual expected)
+    (if (= actual expected)
+      (set! ghm-test-pass (+ ghm-test-pass 1))
+      (begin
+        (set! ghm-test-fail (+ ghm-test-fail 1))
+        (append! ghm-test-fails {:name name :expected expected :actual actual})))))
+
+;; ── Type constructors ─────────────────────────────────────────────
+(ghm-test "tv"           (hm-tv "a") (list :var "a"))
+(ghm-test "int"          (hm-int)    (list :ctor "Int" (list)))
+(ghm-test "arrow"        (ctor-head (hm-arrow (hm-int) (hm-bool))) "->")
+(ghm-test "arrow-args-len" (len (ctor-args (hm-arrow (hm-int) (hm-bool)))) 2)
+
+;; ── Schemes ───────────────────────────────────────────────────────
+(ghm-test "scheme-vars"  (hm-scheme-vars (hm-scheme (list "a") (hm-tv "a"))) (list "a"))
+(ghm-test "monotype-vars" (hm-scheme-vars (hm-monotype (hm-int))) (list))
+(ghm-test "scheme?-yes"  (hm-scheme? (hm-monotype (hm-int))) true)
+(ghm-test "scheme?-no"   (hm-scheme? (hm-int)) false)
+
+;; ── Fresh tyvars ──────────────────────────────────────────────────
+(ghm-test "fresh-1"
+  (let ((c (list 0))) (var-name (hm-fresh-tv c))) "t1")
+(ghm-test "fresh-bumps"
+  (let ((c (list 5))) (begin (hm-fresh-tv c) (first c))) 6)
+
+;; ── Free type variables ──────────────────────────────────────────
+(ghm-test "ftv-int"      (hm-ftv (hm-int)) (list))
+(ghm-test "ftv-tv"       (hm-ftv (hm-tv "a")) (list "a"))
+(ghm-test "ftv-arrow"
+  (len (hm-ftv (hm-arrow (hm-tv "a") (hm-arrow (hm-tv "b") (hm-tv "a"))))) 2)
+(ghm-test "ftv-scheme-quantified"
+  (hm-ftv-scheme (hm-scheme (list "a") (hm-arrow (hm-tv "a") (hm-tv "b")))) (list "b"))
+(ghm-test "ftv-env"
+  (let ((env (assoc {} "f" (hm-monotype (hm-arrow (hm-tv "x") (hm-tv "y"))))))
+    (len (hm-ftv-env env))) 2)
+
+;; ── Substitution / apply / compose ───────────────────────────────
+(ghm-test "apply-tv"
+  (hm-apply (assoc {} "a" (hm-int)) (hm-tv "a")) (hm-int))
+(ghm-test "apply-arrow"
+  (ctor-head
+    (hm-apply (assoc {} "a" (hm-int))
+              (hm-arrow (hm-tv "a") (hm-tv "b")))) "->")
+(ghm-test "compose-1-then-2"
+  (var-name
+    (hm-apply
+      (hm-compose (assoc {} "b" (hm-tv "c")) (assoc {} "a" (hm-tv "b")))
+      (hm-tv "a"))) "c")
+
+;; ── Generalize / Instantiate ─────────────────────────────────────
+;;   forall a. a -> a   instantiated twice yields fresh vars each time
+(ghm-test "generalize-id"
+  (len (hm-scheme-vars (hm-generalize (hm-arrow (hm-tv "a") (hm-tv "a")) {}))) 1)
+
+(ghm-test "generalize-skips-env"
+  ;; ftv(t)={a,b}, ftv(env)={a}, qs={b}
+  (let ((env (assoc {} "x" (hm-monotype (hm-tv "a")))))
+    (len (hm-scheme-vars
+           (hm-generalize (hm-arrow (hm-tv "a") (hm-tv "b")) env)))) 1)
+
+(ghm-test "instantiate-fresh"
+  (let ((s (hm-scheme (list "a") (hm-arrow (hm-tv "a") (hm-tv "a"))))
+        (c (list 0)))
+    (let ((t1 (hm-instantiate s c)) (t2 (hm-instantiate s c)))
+      (not (= (var-name (first (ctor-args t1)))
+              (var-name (first (ctor-args t2)))))))
+  true)
+
+;; ── Inference (literal only) ─────────────────────────────────────
+(ghm-test "infer-int"
+  (ctor-head (get (hm-infer-literal (ast-literal 42)) :type)) "Int")
+(ghm-test "infer-string"
+  (ctor-head (get (hm-infer-literal (ast-literal "hi")) :type)) "String")
+(ghm-test "infer-bool"
+  (ctor-head (get (hm-infer-literal (ast-literal true)) :type)) "Bool")
+
+(define ghm-tests-run!
+  (fn ()
+    {:passed ghm-test-pass
+     :failed ghm-test-fail
+     :total  (+ ghm-test-pass ghm-test-fail)}))

lib/guest/tests/layout.sx

@@ -0,0 +1,180 @@
+;; lib/guest/tests/layout.sx — synthetic Python-ish off-side fixture.
+;;
+;; Exercises lib/guest/layout.sx with a config different from Haskell's
+;; (no module-prelude, layout opens via trailing `:` not via reserved
+;; keyword) to prove the kit isn't Haskell-shaped.
+
+(define glayout-test-pass 0)
+(define glayout-test-fail 0)
+(define glayout-test-fails (list))
+
+(define
+  glayout-test
+  (fn (name actual expected)
+    (if (= actual expected)
+      (set! glayout-test-pass (+ glayout-test-pass 1))
+      (begin
+        (set! glayout-test-fail (+ glayout-test-fail 1))
+        (append! glayout-test-fails {:name name :expected expected :actual actual})))))
+
+;; Convenience: build a token from {type value line col}.
+(define
+  glayout-tok
+  (fn (ty val line col)
+    {:type ty :value val :line line :col col}))
+
+;; Project a token list to ((type value) ...) for compact comparison.
+(define
+  glayout-shape
+  (fn (toks)
+    (map (fn (t) (list (get t :type) (get t :value))) toks)))
+
+;; ── Haskell-flavour: keyword opens block ─────────────────────────
+(define
+  glayout-haskell-cfg
+  {:open-keywords (list "let" "where" "do" "of")
+   :open-trailing-fn nil
+   :open-token   {:type "vlbrace" :value "{"}
+   :close-token  {:type "vrbrace" :value "}"}
+   :sep-token    {:type "vsemi"   :value ";"}
+   :module-prelude? false
+   :explicit-open? (fn (tok) (= (get tok :type) "lbrace"))})
+
+;;   do
+;;     a
+;;     b
+;;   c       ← outside the do-block
+(glayout-test "haskell-do-block"
+  (glayout-shape
+    (layout-pass
+      glayout-haskell-cfg
+      (list (glayout-tok "reserved" "do" 1 1)
+            (glayout-tok "ident" "a" 2 3)
+            (glayout-tok "ident" "b" 3 3)
+            (glayout-tok "ident" "c" 4 1))))
+  (list (list "reserved" "do")
+        (list "vlbrace" "{")
+        (list "ident" "a")
+        (list "vsemi" ";")
+        (list "ident" "b")
+        (list "vrbrace" "}")
+        (list "ident" "c")))
+
+;; Explicit `{` after `do` suppresses virtual layout.
+(glayout-test "haskell-explicit-brace"
+  (glayout-shape
+    (layout-pass
+      glayout-haskell-cfg
+      (list (glayout-tok "reserved" "do" 1 1)
+            (glayout-tok "lbrace" "{" 1 4)
+            (glayout-tok "ident" "a" 1 6)
+            (glayout-tok "rbrace" "}" 1 8))))
+  (list (list "reserved" "do")
+        (list "lbrace" "{")
+        (list "ident" "a")
+        (list "rbrace" "}")))
+
+;; Single-statement do-block on the same line.
+(glayout-test "haskell-do-inline"
+  (glayout-shape
+    (layout-pass
+      glayout-haskell-cfg
+      (list (glayout-tok "reserved" "do" 1 1)
+            (glayout-tok "ident" "a" 1 4))))
+  (list (list "reserved" "do")
+        (list "vlbrace" "{")
+        (list "ident" "a")
+        (list "vrbrace" "}")))
+
+;; Module-prelude: wrap whole input in implicit layout block at first
+;; tok's column.
+(glayout-test "haskell-module-prelude"
+  (glayout-shape
+    (layout-pass
+      (assoc glayout-haskell-cfg :module-prelude? true)
+      (list (glayout-tok "ident" "x" 1 1)
+            (glayout-tok "ident" "y" 2 1)
+            (glayout-tok "ident" "z" 3 1))))
+  (list (list "vlbrace" "{")
+        (list "ident" "x")
+        (list "vsemi" ";")
+        (list "ident" "y")
+        (list "vsemi" ";")
+        (list "ident" "z")
+        (list "vrbrace" "}")))
+
+;; ── Python-flavour: trailing `:` opens block ─────────────────────
+(define
+  glayout-python-cfg
+  {:open-keywords (list)
+   :open-trailing-fn (fn (tok) (and (= (get tok :type) "punct")
+                                    (= (get tok :value) ":")))
+   :open-token   {:type "indent" :value "INDENT"}
+   :close-token  {:type "dedent" :value "DEDENT"}
+   :sep-token    {:type "newline" :value "NEWLINE"}
+   :module-prelude? false
+   :explicit-open? nil})
+
+;;   if x:
+;;       a
+;;       b
+;;   c
+(glayout-test "python-if-block"
+  (glayout-shape
+    (layout-pass
+      glayout-python-cfg
+      (list (glayout-tok "reserved" "if" 1 1)
+            (glayout-tok "ident" "x" 1 4)
+            (glayout-tok "punct" ":" 1 5)
+            (glayout-tok "ident" "a" 2 5)
+            (glayout-tok "ident" "b" 3 5)
+            (glayout-tok "ident" "c" 4 1))))
+  (list (list "reserved" "if")
+        (list "ident" "x")
+        (list "punct" ":")
+        (list "indent" "INDENT")
+        (list "ident" "a")
+        (list "newline" "NEWLINE")
+        (list "ident" "b")
+        (list "dedent" "DEDENT")
+        (list "ident" "c")))
+
+;; Nested Python-style blocks.
+;;   def f():
+;;       if x:
+;;           a
+;;       b
+(glayout-test "python-nested"
+  (glayout-shape
+    (layout-pass
+      glayout-python-cfg
+      (list (glayout-tok "reserved" "def" 1 1)
+            (glayout-tok "ident" "f" 1 5)
+            (glayout-tok "punct" "(" 1 6)
+            (glayout-tok "punct" ")" 1 7)
+            (glayout-tok "punct" ":" 1 8)
+            (glayout-tok "reserved" "if" 2 5)
+            (glayout-tok "ident" "x" 2 8)
+            (glayout-tok "punct" ":" 2 9)
+            (glayout-tok "ident" "a" 3 9)
+            (glayout-tok "ident" "b" 4 5))))
+  (list (list "reserved" "def")
+        (list "ident" "f")
+        (list "punct" "(")
+        (list "punct" ")")
+        (list "punct" ":")
+        (list "indent" "INDENT")
+        (list "reserved" "if")
+        (list "ident" "x")
+        (list "punct" ":")
+        (list "indent" "INDENT")
+        (list "ident" "a")
+        (list "dedent" "DEDENT")
+        (list "ident" "b")
+        (list "dedent" "DEDENT")))
+
+(define glayout-tests-run!
+  (fn ()
+    {:passed glayout-test-pass
+     :failed glayout-test-fail
+     :total  (+ glayout-test-pass glayout-test-fail)}))

lib/guest/tests/match.sx

@@ -0,0 +1,108 @@
+;; lib/guest/tests/match.sx — exercises lib/guest/match.sx.
+
+(define gmatch-test-pass 0)
+(define gmatch-test-fail 0)
+(define gmatch-test-fails (list))
+
+(define
+  gmatch-test
+  (fn (name actual expected)
+    (if (= actual expected)
+      (set! gmatch-test-pass (+ gmatch-test-pass 1))
+      (begin
+        (set! gmatch-test-fail (+ gmatch-test-fail 1))
+        (append! gmatch-test-fails {:name name :expected expected :actual actual})))))
+
+;; ── walk / extend / occurs ────────────────────────────────────────
+(gmatch-test "walk-direct"
+  (walk (mk-var "x") (extend "x" 5 (empty-subst))) 5)
+
+(gmatch-test "walk-chain"
+  (walk (mk-var "a") (extend "a" (mk-var "b") (extend "b" 7 (empty-subst)))) 7)
+
+(gmatch-test "walk-no-binding"
+  (let ((v (mk-var "u"))) (= (walk v (empty-subst)) v)) true)
+
+(gmatch-test "walk*-recursive"
+  (walk* (mk-ctor "Just" (list (mk-var "x"))) (extend "x" 9 (empty-subst)))
+  (mk-ctor "Just" (list 9)))
+
+(gmatch-test "occurs-direct"
+  (occurs? "x" (mk-var "x") (empty-subst)) true)
+
+(gmatch-test "occurs-nested"
+  (occurs? "x" (mk-ctor "f" (list (mk-var "x"))) (empty-subst)) true)
+
+(gmatch-test "occurs-not"
+  (occurs? "x" (mk-var "y") (empty-subst)) false)
+
+;; ── unify (symmetric) ─────────────────────────────────────────────
+(gmatch-test "unify-equal-literals"
+  (len (unify 5 5 (empty-subst))) 0)
+
+(gmatch-test "unify-different-literals"
+  (unify 5 6 (empty-subst)) nil)
+
+(gmatch-test "unify-var-literal"
+  (get (unify (mk-var "x") 5 (empty-subst)) "x") 5)
+
+(gmatch-test "unify-literal-var"
+  (get (unify 5 (mk-var "x") (empty-subst)) "x") 5)
+
+(gmatch-test "unify-same-var"
+  (len (unify (mk-var "x") (mk-var "x") (empty-subst))) 0)
+
+(gmatch-test "unify-two-vars"
+  (let ((s (unify (mk-var "x") (mk-var "y") (empty-subst))))
+    (or (= (get s "x") (mk-var "y")) (= (get s "y") (mk-var "x")))) true)
+
+(gmatch-test "unify-ctor-equal"
+  (len (unify (mk-ctor "f" (list 1 2)) (mk-ctor "f" (list 1 2)) (empty-subst))) 0)
+
+(gmatch-test "unify-ctor-with-var"
+  (get (unify (mk-ctor "Just" (list (mk-var "x"))) (mk-ctor "Just" (list 7)) (empty-subst)) "x") 7)
+
+(gmatch-test "unify-ctor-head-mismatch"
+  (unify (mk-ctor "Just" (list 1)) (mk-ctor "Nothing" (list)) (empty-subst)) nil)
+
+(gmatch-test "unify-ctor-arity-mismatch"
+  (unify (mk-ctor "f" (list 1 2)) (mk-ctor "f" (list 1)) (empty-subst)) nil)
+
+(gmatch-test "unify-occurs-check"
+  (unify (mk-var "x") (mk-ctor "f" (list (mk-var "x"))) (empty-subst)) nil)
+
+(gmatch-test "unify-transitive-vars"
+  (let ((s (unify (mk-var "x") (mk-var "y") (empty-subst))))
+    (let ((s2 (unify (mk-var "y") 42 s)))
+      (walk (mk-var "x") s2))) 42)
+
+;; ── match-pat (asymmetric) ────────────────────────────────────────
+(gmatch-test "match-var-binds"
+  (get (match-pat (mk-var "x") 99 (empty-subst)) "x") 99)
+
+(gmatch-test "match-literal-equal"
+  (len (match-pat 5 5 (empty-subst))) 0)
+
+(gmatch-test "match-literal-mismatch"
+  (match-pat 5 6 (empty-subst)) nil)
+
+(gmatch-test "match-ctor-binds"
+  (get (match-pat (mk-ctor "Just" (list (mk-var "y")))
+                  (mk-ctor "Just" (list 11))
+                  (empty-subst)) "y") 11)
+
+(gmatch-test "match-ctor-head-mismatch"
+  (match-pat (mk-ctor "Just" (list (mk-var "y")))
+             (mk-ctor "Nothing" (list))
+             (empty-subst)) nil)
+
+(gmatch-test "match-ctor-arity-mismatch"
+  (match-pat (mk-ctor "f" (list (mk-var "x") (mk-var "y")))
+             (mk-ctor "f" (list 1))
+             (empty-subst)) nil)
+
+(define gmatch-tests-run!
+  (fn ()
+    {:passed gmatch-test-pass
+     :failed gmatch-test-fail
+     :total  (+ gmatch-test-pass gmatch-test-fail)}))

lib/lua/parser.sx

@@ -3,28 +3,33 @@
 (define lua-tok-value (fn (t) (if (= t nil) nil (get t :value))))
 
 (define
-  lua-binop-prec
+  lua-op-table
-  (fn
+  (list
-    (op)
+    (list "or"  1 :left)
-    (cond
+    (list "and" 2 :left)
-      ((= op "or") 1)
+    (list "<"   3 :left)
-      ((= op "and") 2)
+    (list ">"   3 :left)
-      ((= op "<") 3)
+    (list "<="  3 :left)
-      ((= op ">") 3)
+    (list ">="  3 :left)
-      ((= op "<=") 3)
+    (list "=="  3 :left)
-      ((= op ">=") 3)
+    (list "~="  3 :left)
-      ((= op "==") 3)
+    (list ".."  5 :right)
-      ((= op "~=") 3)
+    (list "+"   6 :left)
-      ((= op "..") 5)
+    (list "-"   6 :left)
-      ((= op "+") 6)
+    (list "*"   7 :left)
-      ((= op "-") 6)
+    (list "/"   7 :left)
-      ((= op "*") 7)
+    (list "%"   7 :left)
-      ((= op "/") 7)
+    (list "^"  10 :right)))
-      ((= op "%") 7)
-      ((= op "^") 10)
-      (else 0))))
 
-(define lua-binop-right? (fn (op) (or (= op "..") (= op "^"))))
+(define lua-binop-prec
+  (fn (op)
+    (let ((entry (pratt-op-lookup lua-op-table op)))
+      (if (= entry nil) 0 (pratt-op-prec entry)))))
+
+(define lua-binop-right?
+  (fn (op)
+    (let ((entry (pratt-op-lookup lua-op-table op)))
+      (and (not (= entry nil)) (= (pratt-op-assoc entry) :right)))))
 
 (define
   lua-parse

lib/lua/test.sh

@@ -30,6 +30,7 @@ cat > "$TMPFILE" << 'EPOCHS'
 (epoch 1)
 (load "lib/guest/lex.sx")
 (load "lib/guest/prefix.sx")
+(load "lib/guest/pratt.sx")
 (load "lib/lua/tokenizer.sx")
 (epoch 2)
 (load "lib/lua/parser.sx")

lib/prolog/conformance.conf

@@ -4,6 +4,7 @@ LANG_NAME=prolog
 MODE=dict
 
 PRELOADS=(
+  lib/guest/pratt.sx
   lib/prolog/tokenizer.sx
   lib/prolog/parser.sx
   lib/prolog/runtime.sx

lib/prolog/parser.sx

@@ -104,18 +104,9 @@
     (list ":-" 1200 "xfx")
     (list "mod" 400 "yfx")))
 
-(define
+(define pl-op-lookup (fn (name) (pratt-op-lookup pl-op-table name)))
-  pl-op-find
-  (fn
-    (name table)
-    (cond
-      ((empty? table) nil)
-      ((= (first (first table)) name) (rest (first table)))
-      (true (pl-op-find name (rest table))))))
 
-(define pl-op-lookup (fn (name) (pl-op-find name pl-op-table)))
+;; Token → entry (name prec type) for known infix ops, else nil.
-
-;; Token → (name prec type) for known infix ops, else nil.
 (define
   pl-token-op
   (fn
@@ -123,14 +114,8 @@
     (let
       ((ty (get t :type)) (vv (get t :value)))
       (cond
-        ((and (= ty "punct") (= vv ","))
+        ((and (= ty "punct") (= vv ",")) (pl-op-lookup ","))
-          (let
+        ((or (= ty "atom") (= ty "op")) (pl-op-lookup vv))
-            ((info (pl-op-lookup ",")))
-            (if (nil? info) nil (cons "," info))))
-        ((or (= ty "atom") (= ty "op"))
-          (let
-            ((info (pl-op-lookup vv)))
-            (if (nil? info) nil (cons vv info))))
         (true nil)))))
 
 ;; ── Term parser ─────────────────────────────────────────────────────

lib/prolog/scoreboard.json

@@ -3,5 +3,5 @@
   "total_failed": 0,
   "total": 590,
   "suites": {"parse":{"passed":25,"total":25,"failed":0},"unify":{"passed":47,"total":47,"failed":0},"clausedb":{"passed":14,"total":14,"failed":0},"solve":{"passed":62,"total":62,"failed":0},"operators":{"passed":19,"total":19,"failed":0},"dynamic":{"passed":11,"total":11,"failed":0},"findall":{"passed":11,"total":11,"failed":0},"term_inspect":{"passed":14,"total":14,"failed":0},"append":{"passed":6,"total":6,"failed":0},"reverse":{"passed":6,"total":6,"failed":0},"member":{"passed":7,"total":7,"failed":0},"nqueens":{"passed":6,"total":6,"failed":0},"family":{"passed":10,"total":10,"failed":0},"atoms":{"passed":34,"total":34,"failed":0},"query_api":{"passed":16,"total":16,"failed":0},"iso_predicates":{"passed":29,"total":29,"failed":0},"meta_predicates":{"passed":25,"total":25,"failed":0},"list_predicates":{"passed":33,"total":33,"failed":0},"meta_call":{"passed":15,"total":15,"failed":0},"set_predicates":{"passed":15,"total":15,"failed":0},"char_predicates":{"passed":27,"total":27,"failed":0},"io_predicates":{"passed":24,"total":24,"failed":0},"assert_rules":{"passed":15,"total":15,"failed":0},"string_agg":{"passed":25,"total":25,"failed":0},"advanced":{"passed":21,"total":21,"failed":0},"compiler":{"passed":17,"total":17,"failed":0},"cross_validate":{"passed":17,"total":17,"failed":0},"integration":{"passed":20,"total":20,"failed":0},"hs_bridge":{"passed":19,"total":19,"failed":0}},
-  "generated": "2026-05-06T22:23:38+00:00"
+  "generated": "2026-05-07T17:35:23+00:00"
 }

lib/prolog/scoreboard.md

@@ -1,7 +1,7 @@
 # Prolog scoreboard
 
 **590 / 590 passing** (0 failure(s)).
-Generated 2026-05-06T22:23:38+00:00.
+Generated 2026-05-07T17:35:23+00:00.
 
 | Suite | Passed | Total | Status |
 |-------|--------|-------|--------|

lib/tcl/runtime.sx

@@ -292,13 +292,15 @@
                             (> (len result-stack) caller-stack-len)
                             (nth result-stack caller-stack-len)
                             (get interp :frame))))
-                      (assoc interp
+                      ; Forward result-interp as base so state changes inside
+                      ; the proc (e.g. :fileevents, :timers, :procs) propagate;
+                      ; restore caller's frame/stack/result/output/code.
+                      (assoc result-interp
                         :frame updated-caller
                         :frame-stack updated-below
                         :result result-val
                         :output (str caller-output proc-output)
-                        :code (if (= code 2) 0 code)
+                        :code (if (= code 2) 0 code))))))))))))))
-                        :commands (get result-interp :commands))))))))))))))
 
 (define
   tcl-eval-cmd
@@ -354,14 +356,33 @@
   (fn
     (interp args)
     (let
-      ((text (last args))
+      ((no-nl (and (> (len args) 1) (equal? (first args) "-nonewline"))))
-        (no-nl
-          (and
-            (> (len args) 1)
-            (equal? (first args) "-nonewline"))))
       (let
-        ((line (if no-nl text (str text "\n"))))
+        ((args2 (if no-nl (rest args) args)))
-        (assoc interp :output (str (get interp :output) line))))))
+        (let
+          ((maybe-chan (if (> (len args2) 1) (first args2) nil))
+            (is-chan
+              (and
+                (not (nil? maybe-chan))
+                (or
+                  (and
+                    (>= (len maybe-chan) 4)
+                    (equal? (slice maybe-chan 0 4) "file"))
+                  (and
+                    (>= (len maybe-chan) 4)
+                    (equal? (slice maybe-chan 0 4) "sock"))))))
+          (if
+            is-chan
+            (let
+              ((chan (first args2))
+                (text (last args2))
+                (line (if no-nl text (str text "\n"))))
+              (let
+                ((_ (channel-write chan line)))
+                (assoc interp :result "")))
+            (let
+              ((text (last args2)) (line (if no-nl text (str text "\n"))))
+              (assoc interp :output (str (get interp :output) line)))))))))
 
 (define
   tcl-cmd-incr
@@ -2868,36 +2889,433 @@
           ((equal? sub "seconds") (assoc interp :result (str (clock-seconds))))
           ((equal? sub "milliseconds") (assoc interp :result (str (clock-milliseconds))))
           ((equal? sub "format")
-            (assoc interp :result (clock-format
+            ; clock format $secs ?-format $fmt? ?-timezone $tz? ?-gmt 0|1?
-              (floor (parse-int (first rest-args)))
+            (let
-              (if (> (len rest-args) 1) (nth rest-args (- (len rest-args) 1)) "%a %b %e %H:%M:%S %Z %Y"))))
+              ((t (floor (parse-int (first rest-args))))
-          ((equal? sub "scan") (assoc interp :result "0"))
+                (opts (rest rest-args)))
+              (let
+                ((fmt (tcl-clock-opt opts "-format" "%a %b %e %H:%M:%S %Z %Y"))
+                  (tz (tcl-clock-tz opts)))
+                (assoc interp :result (clock-format t fmt tz)))))
+          ((equal? sub "scan")
+            ; clock scan $str ?-format $fmt? ?-timezone $tz? ?-gmt 0|1?
+            (let
+              ((s (first rest-args)) (opts (rest rest-args)))
+              (let
+                ((fmt (tcl-clock-opt opts "-format" "%Y-%m-%d %H:%M:%S"))
+                  (tz (tcl-clock-tz opts)))
+                (assoc interp :result (str (clock-scan s fmt tz))))))
           (else (error (str "clock: unknown subcommand \"" sub "\""))))))))
 
-(define tcl-cmd-open (fn (interp args) (assoc interp :result "file0")))
+; Helper: extract a -flag $val pair from clock args.
+(define
+  tcl-clock-opt
+  (fn
+    (opts flag default)
+    (cond
+      ((< (len opts) 2) default)
+      ((equal? (first opts) flag) (nth opts 1))
+      (else (tcl-clock-opt (rest (rest opts)) flag default)))))
+
+; Helper: derive tz string from clock opts (-timezone or -gmt).
+(define
+  tcl-clock-tz
+  (fn
+    (opts)
+    (let
+      ((tz-explicit (tcl-clock-opt opts "-timezone" nil))
+        (gmt-flag (tcl-clock-opt opts "-gmt" nil)))
+      (cond
+        ((not (nil? tz-explicit))
+          (cond
+            ((equal? tz-explicit ":UTC") "utc")
+            ((equal? tz-explicit "UTC") "utc")
+            ((equal? tz-explicit "GMT") "utc")
+            (else "local")))
+        ((equal? gmt-flag "1") "utc")
+        ((equal? gmt-flag "true") "utc")
+        ((not (nil? gmt-flag)) "local")
+        (else "utc")))))
+
+(define
+  tcl-cmd-open
+  (fn
+    (interp args)
+    (let
+      ((path (first args))
+        (mode (if (> (len args) 1) (nth args 1) "r")))
+      (assoc interp :result (channel-open path mode)))))
 
 ; gets channel ?varname?
-(define tcl-cmd-close (fn (interp args) (assoc interp :result "")))
+(define
+  tcl-cmd-close
+  (fn
+    (interp args)
+    (let ((_ (channel-close (first args)))) (assoc interp :result ""))))
 
-(define tcl-cmd-read (fn (interp args) (assoc interp :result "")))
+(define
+  tcl-cmd-read
+  (fn
+    (interp args)
+    (let
+      ((chan (first args))
+        (n (if (> (len args) 1) (parse-int (nth args 1)) -1)))
+      (assoc
+        interp
+        :result (if (< n 0) (channel-read chan) (channel-read chan n))))))
 
 (define
   tcl-cmd-gets-chan
   (fn
     (interp args)
-    (if
+    (let
-      (> (len args) 1)
+      ((chan (first args)) (line (channel-read-line chan)))
-      (assoc (tcl-var-set interp (nth args 1) "") :result "-1")
+      (if
-      (assoc interp :result ""))))
+        (nil? line)
+        (if
+          (> (len args) 1)
+          (assoc (tcl-var-set interp (nth args 1) "") :result "-1")
+          (assoc interp :result ""))
+        (if
+          (> (len args) 1)
+          (assoc
+            (tcl-var-set interp (nth args 1) line)
+            :result (str (len line)))
+          (assoc interp :result line))))))
 
-(define tcl-cmd-eof (fn (interp args) (assoc interp :result "1")))
+(define
+  tcl-cmd-eof
+  (fn
+    (interp args)
+    (assoc interp :result (if (channel-eof? (first args)) "1" "0"))))
 
-(define tcl-cmd-seek (fn (interp args) (assoc interp :result "")))
+(define
+  tcl-cmd-seek
+  (fn
+    (interp args)
+    (let
+      ((chan (first args))
+        (off (parse-int (nth args 1)))
+        (whence (if (> (len args) 2) (nth args 2) "start")))
+      (let ((_ (channel-seek chan off whence))) (assoc interp :result "")))))
 
 ; file command dispatcher
-(define tcl-cmd-tell (fn (interp args) (assoc interp :result "0")))
+(define
+  tcl-cmd-tell
+  (fn
+    (interp args)
+    (assoc interp :result (str (channel-tell (first args))))))
+
+(define
+  tcl-cmd-flush
+  (fn
+    (interp args)
+    (let ((_ (channel-flush (first args)))) (assoc interp :result ""))))
+(define
+  tcl-cmd-fconfigure
+  (fn
+    (interp args)
+    (let
+      ((chan (first args)) (rest-args (rest args)))
+      (cond
+        ((= 0 (len rest-args))
+          (assoc
+            interp
+            :result (str "-blocking " (if (channel-blocking? chan) "1" "0"))))
+        ((and
+           (= 2 (len rest-args))
+           (equal? (first rest-args) "-blocking"))
+          (let
+            ((b (nth rest-args 1)))
+            (let
+              ((_
+                  (channel-set-blocking!
+                    chan
+                    (not (or (equal? b "0") (equal? b "false"))))))
+              (assoc interp :result ""))))
+        ((and
+           (= 1 (len rest-args))
+           (equal? (first rest-args) "-blocking"))
+          (assoc interp :result (if (channel-blocking? chan) "1" "0")))
+        ((and
+           (= 1 (len rest-args))
+           (equal? (first rest-args) "-error"))
+          (assoc interp :result (channel-async-error chan)))
+        (else (assoc interp :result ""))))))
+
+
+; ============================================================
+; Event loop: fileevent / after / vwait / update (Phase 5b)
+; ============================================================
+
+; :fileevents is list of (chan event script) tuples
+; :timers is list of (expiry-ms script) tuples, sorted ascending by expiry
+
+(define
+  tcl-fileevent-set
+  (fn
+    (interp chan event script)
+    (let
+      ((existing (or (get interp :fileevents) (list))))
+      (let
+        ((filtered
+            (filter
+              (fn (e) (not (and (equal? (first e) chan) (equal? (nth e 1) event))))
+              existing)))
+        (let
+          ((new-list
+              (if (equal? script "")
+                filtered
+                (append filtered (list (list chan event script))))))
+          (assoc interp :fileevents new-list))))))
+
+(define
+  tcl-fileevent-get
+  (fn
+    (interp chan event)
+    (let
+      ((events (or (get interp :fileevents) (list))))
+      (let
+        ((matches
+            (filter
+              (fn (e) (and (equal? (first e) chan) (equal? (nth e 1) event)))
+              events)))
+        (if (= 0 (len matches)) "" (nth (first matches) 2))))))
+
+(define
+  tcl-timer-insert
+  (fn
+    (timers new-timer)
+    (cond
+      ((= 0 (len timers)) (list new-timer))
+      ((<= (first new-timer) (first (first timers))) (cons new-timer timers))
+      (else (cons (first timers) (tcl-timer-insert (rest timers) new-timer))))))
+
+(define
+  tcl-timer-add
+  (fn
+    (interp ms script)
+    (let
+      ((expiry (+ (clock-milliseconds) ms)))
+      (let
+        ((existing (or (get interp :timers) (list))))
+        (assoc interp :timers (tcl-timer-insert existing (list expiry script)))))))
+
+; Run one iteration of the event loop.
+; poll-timeout-ms: -1 = block indefinitely, 0 = poll, N>0 = wait up to N ms.
+; Returns updated interp.
+(define
+  tcl-event-step
+  (fn
+    (interp poll-timeout-ms)
+    (let
+      ((timers (or (get interp :timers) (list))) (now-ms (clock-milliseconds)))
+      (let
+        ((expired (filter (fn (t) (<= (first t) now-ms)) timers))
+          (remaining (filter (fn (t) (> (first t) now-ms)) timers)))
+        (let
+          ((interp1
+              (reduce
+                (fn (acc t) (tcl-eval-string acc (nth t 1)))
+                (assoc interp :timers remaining)
+                expired)))
+          (let
+            ((events (or (get interp1 :fileevents) (list))))
+            (let
+              ((read-chans
+                  (map
+                    (fn (e) (first e))
+                    (filter (fn (e) (equal? (nth e 1) "readable")) events)))
+                (write-chans
+                  (map
+                    (fn (e) (first e))
+                    (filter (fn (e) (equal? (nth e 1) "writable")) events)))
+                (next-timer-delta
+                  (if
+                    (= 0 (len remaining))
+                    -1
+                    (- (first (first remaining)) (clock-milliseconds)))))
+              (let
+                ((effective-timeout
+                    (cond
+                      ((and (>= poll-timeout-ms 0) (>= next-timer-delta 0))
+                        (min poll-timeout-ms next-timer-delta))
+                      ((>= poll-timeout-ms 0) poll-timeout-ms)
+                      ((>= next-timer-delta 0) next-timer-delta)
+                      (else -1))))
+                (if
+                  (and
+                    (= 0 (len read-chans))
+                    (= 0 (len write-chans)))
+                  ; nothing to select on; if timeout > 0, do a no-op wait via select
+                  (if
+                    (> effective-timeout 0)
+                    (let
+                      ((_ (io-select-channels (list) (list) effective-timeout)))
+                      interp1)
+                    interp1)
+                  (let
+                    ((select-result
+                        (io-select-channels read-chans write-chans effective-timeout)))
+                    (let
+                      ((ready-r (or (get select-result :readable) (list)))
+                        (ready-w (or (get select-result :writable) (list))))
+                      (let
+                        ((interp2
+                            (reduce
+                              (fn (acc chan)
+                                (let
+                                  ((script (tcl-fileevent-get acc chan "readable")))
+                                  (if (equal? script "") acc (tcl-eval-string acc script))))
+                              interp1
+                              ready-r)))
+                        (reduce
+                          (fn (acc chan)
+                            (let
+                              ((script (tcl-fileevent-get acc chan "writable")))
+                              (if (equal? script "") acc (tcl-eval-string acc script))))
+                          interp2
+                          ready-w)))))))))))))
+
+(define
+  tcl-cmd-fileevent
+  (fn
+    (interp args)
+    (let
+      ((chan (first args)) (event (nth args 1)))
+      (if
+        (= 2 (len args))
+        (assoc interp :result (tcl-fileevent-get interp chan event))
+        (let
+          ((script (nth args 2)))
+          (assoc (tcl-fileevent-set interp chan event script) :result ""))))))
+
+(define
+  tcl-cmd-after
+  (fn
+    (interp args)
+    (if
+      (= 0 (len args))
+      (error "after: wrong # args")
+      (let
+        ((ms (parse-int (first args))))
+        (if
+          (= 1 (len args))
+          ; pure sleep — drive event loop until ms elapsed
+          (let
+            ((target-ms (+ (clock-milliseconds) ms)))
+            (assoc (tcl-after-sleep-loop interp target-ms) :result ""))
+          ; schedule timer
+          (let
+            ((script (join " " (rest args))))
+            (assoc (tcl-timer-add interp ms script) :result "")))))))
+
+(define
+  tcl-after-sleep-loop
+  (fn
+    (interp target-ms)
+    (let
+      ((now (clock-milliseconds)))
+      (if
+        (>= now target-ms)
+        interp
+        (tcl-after-sleep-loop
+          (tcl-event-step interp (- target-ms now))
+          target-ms)))))
+
+(define
+  tcl-cmd-vwait
+  (fn
+    (interp args)
+    (if
+      (= 0 (len args))
+      (error "vwait: wrong # args")
+      (let
+        ((name (first args)))
+        (let
+          ((initial (frame-lookup (get interp :frame) name)))
+          (assoc (tcl-vwait-loop interp name initial) :result ""))))))
+
+(define
+  tcl-vwait-loop
+  (fn
+    (interp name initial)
+    (let
+      ((cur (frame-lookup (get interp :frame) name)))
+      (if
+        (and (not (nil? cur)) (not (equal? cur initial)))
+        interp
+        (tcl-vwait-loop (tcl-event-step interp 1000) name initial)))))
+
+(define
+  tcl-cmd-update
+  (fn
+    (interp args)
+    (assoc (tcl-event-step interp 0) :result "")))
+
+; ============================================================
+; Socket: TCP client and server (Phase 5c)
+; ============================================================
+
+; Internal command invoked by the auto-registered fileevent on a server
+; channel. Args: (server-chan callback-word ...).  Accepts one client and
+; calls the user callback with (client-chan peer-host peer-port).
+(define
+  tcl-cmd-_sock-do-accept
+  (fn
+    (interp args)
+    (let
+      ((server-chan (first args)) (cb-parts (rest args)))
+      (let
+        ((info (socket-accept server-chan)))
+        (let
+          ((client-chan (get info :channel))
+            (peer-host (get info :host))
+            (peer-port (str (get info :port))))
+          (let
+            ((cmd
+                (join
+                  " "
+                  (append
+                    cb-parts
+                    (list client-chan peer-host peer-port)))))
+            (assoc (tcl-eval-string interp cmd) :result "")))))))
+
+; socket host port           — TCP client; returns "sockN"
+; socket -server cb port     — TCP server; auto-fires cb on each accept
+(define
+  tcl-cmd-socket
+  (fn
+    (interp args)
+    (cond
+      ((= 0 (len args)) (error "socket: wrong # args"))
+      ((equal? (first args) "-server")
+        (if
+          (< (len args) 3)
+          (error "socket: usage: socket -server cb port")
+          (let
+            ((cb (nth args 1)) (port (parse-int (nth args 2))))
+            (let
+              ((server-chan (socket-server port)))
+              (let
+                ((handler (str "_sock-do-accept " server-chan " " cb)))
+                (assoc
+                  (tcl-fileevent-set interp server-chan "readable" handler)
+                  :result server-chan))))))
+      ((equal? (first args) "-async")
+        (if
+          (< (len args) 3)
+          (error "socket: usage: socket -async host port")
+          (let
+            ((host (nth args 1)) (port (parse-int (nth args 2))))
+            (assoc interp :result (socket-connect-async host port)))))
+      ((= 2 (len args))
+        (let
+          ((host (first args)) (port (parse-int (nth args 1))))
+          (assoc interp :result (socket-connect host port))))
+      (else (error "socket: wrong # args")))))
+
 
-(define tcl-cmd-flush (fn (interp args) (assoc interp :result "")))
 (define
   tcl-cmd-array
   (fn
@@ -2909,11 +3327,16 @@
         ((sub (first args)) (rest-args (rest args)))
         (cond
           ((equal? sub "get")
-            (if (= 0 (len rest-args))
+            (if
+              (= 0 (len rest-args))
               (error "array get: wrong # args")
               (let
                 ((arr-name (first rest-args))
-                  (pattern (if (> (len rest-args) 1) (nth rest-args 1) nil)))
+                  (pattern
+                    (if
+                      (> (len rest-args) 1)
+                      (nth rest-args 1)
+                      nil)))
                 (let
                   ((prefix (str arr-name "("))
                     (locals (get (get interp :frame) :locals)))
@@ -2922,21 +3345,20 @@
                     (let
                       ((arr-keys (filter (fn (k) (tcl-starts-with? k prefix)) (keys locals))))
                       (let
-                        ((filtered
+                        ((filtered (if (nil? pattern) arr-keys (filter (fn (k) (let ((kn (substring k pl (- (string-length k) 1)))) (tcl-glob-match (split pattern "") (split kn "")))) arr-keys))))
-                            (if
+                        (assoc
-                              (nil? pattern)
+                          interp
-                              arr-keys
+                          :result (join
-                              (filter
+                            " "
-                                (fn (k)
-                                  (let ((kn (substring k pl (- (string-length k) 1))))
-                                    (tcl-glob-match (split pattern "") (split kn ""))))
-                                arr-keys))))
-                        (assoc interp :result
-                          (join " "
                             (reduce
-                              (fn (acc k)
+                              (fn
-                                (let ((kn (substring k pl (- (string-length k) 1))))
+                                (acc k)
-                                  (append acc (list kn) (list (get locals k)))))
+                                (let
+                                  ((kn (substring k pl (- (string-length k) 1))))
+                                  (append
+                                    acc
+                                    (list kn)
+                                    (list (get locals k)))))
                               (list)
                               filtered))))))))))
           ((equal? sub "set")
@@ -2954,7 +3376,8 @@
                     (assoc acc :result "")
                     (loop
                       (rest (rest pairs))
-                      (tcl-var-set acc
+                      (tcl-var-set
+                        acc
                         (str arr-name "(" (first pairs) ")")
                         (nth pairs 1))))))))
           ((equal? sub "names")
@@ -2963,7 +3386,11 @@
               (error "array names: wrong # args")
               (let
                 ((arr-name (first rest-args))
-                  (pattern (if (> (len rest-args) 1) (nth rest-args 1) nil)))
+                  (pattern
+                    (if
+                      (> (len rest-args) 1)
+                      (nth rest-args 1)
+                      nil)))
                 (let
                   ((prefix (str arr-name "("))
                     (locals (get (get interp :frame) :locals)))
@@ -2972,17 +3399,19 @@
                     (let
                       ((arr-keys (filter (fn (k) (tcl-starts-with? k prefix)) (keys locals))))
                       (let
-                        ((filtered
+                        ((filtered (if (nil? pattern) arr-keys (filter (fn (k) (let ((kn (substring k pl (- (string-length k) 1)))) (tcl-glob-match (split pattern "") (split kn "")))) arr-keys))))
-                            (if
+                        (assoc
-                              (nil? pattern)
+                          interp
-                              arr-keys
+                          :result (join
-                              (filter
+                            " "
-                                (fn (k)
+                            (map
-                                  (let ((kn (substring k pl (- (string-length k) 1))))
+                              (fn
-                                    (tcl-glob-match (split pattern "") (split kn ""))))
+                                (k)
-                                arr-keys))))
+                                (substring
-                        (assoc interp :result
+                                  k
-                          (join " " (map (fn (k) (substring k pl (- (string-length k) 1))) filtered))))))))))
+                                  pl
+                                  (- (string-length k) 1)))
+                              filtered))))))))))
           ((equal? sub "size")
             (if
               (= 0 (len rest-args))
@@ -2990,8 +3419,13 @@
               (let
                 ((prefix (str (first rest-args) "("))
                   (locals (get (get interp :frame) :locals)))
-                (assoc interp :result
+                (assoc
-                  (str (len (filter (fn (k) (tcl-starts-with? k prefix)) (keys locals))))))))
+                  interp
+                  :result (str
+                    (len
+                      (filter
+                        (fn (k) (tcl-starts-with? k prefix))
+                        (keys locals))))))))
           ((equal? sub "exists")
             (if
               (= 0 (len rest-args))
@@ -2999,44 +3433,39 @@
               (let
                 ((prefix (str (first rest-args) "("))
                   (locals (get (get interp :frame) :locals)))
-                (assoc interp :result
+                (assoc
-                  (if (> (len (filter (fn (k) (tcl-starts-with? k prefix)) (keys locals))) 0) "1" "0")))))
+                  interp
+                  :result (if
+                    (>
+                      (len
+                        (filter
+                          (fn (k) (tcl-starts-with? k prefix))
+                          (keys locals)))
+                      0)
+                    "1"
+                    "0")))))
           ((equal? sub "unset")
             (if
               (= 0 (len rest-args))
               (error "array unset: wrong # args")
               (let
                 ((arr-name (first rest-args))
-                  (pattern (if (> (len rest-args) 1) (nth rest-args 1) nil)))
+                  (pattern
+                    (if
+                      (> (len rest-args) 1)
+                      (nth rest-args 1)
+                      nil)))
                 (let
                   ((prefix (str arr-name "("))
                     (locals (get (get interp :frame) :locals)))
                   (let
                     ((pl (string-length prefix)))
                     (let
-                      ((to-delete
+                      ((to-delete (filter (fn (k) (if (tcl-starts-with? k prefix) (if (nil? pattern) true (let ((kn (substring k pl (- (string-length k) 1)))) (tcl-glob-match (split pattern "") (split kn "")))) false)) (keys locals))))
-                          (filter
-                            (fn (k)
-                              (if
-                                (tcl-starts-with? k prefix)
-                                (if
-                                  (nil? pattern)
-                                  true
-                                  (let ((kn (substring k pl (- (string-length k) 1))))
-                                    (tcl-glob-match (split pattern "") (split kn ""))))
-                                false))
-                            (keys locals))))
                       (let
-                        ((new-locals
+                        ((new-locals (reduce (fn (acc k) (if (contains? to-delete k) acc (assoc acc k (get locals k)))) {} (keys locals))))
-                            (reduce
+                        (assoc
-                              (fn (acc k)
+                          interp
-                                (if
-                                  (contains? to-delete k)
-                                  acc
-                                  (assoc acc k (get locals k))))
-                              {}
-                              (keys locals))))
-                        (assoc interp
                           :frame (assoc (get interp :frame) :locals new-locals)
                           :result ""))))))))
           (else (error (str "array: unknown subcommand \"" sub "\""))))))))
@@ -3048,7 +3477,7 @@
     (interp args)
     (if
       (< (len args) 1)
-      (error "apply: wrong # args: should be "apply lambdaList ?arg ...?"")
+      (error "apply: wrong # args: should be " apply lambdaList ?arg ...? "")
       (let
         ((func-list (tcl-list-split (first args)))
           (call-args (rest args)))
@@ -3058,90 +3487,122 @@
           (let
             ((param-spec (first func-list))
               (body (nth func-list 1))
-              (ns (if (> (len func-list) 2) (nth func-list 2) nil)))
+              (ns
+                (if
+                  (> (len func-list) 2)
+                  (nth func-list 2)
+                  nil)))
             (let
               ((proc-def {:args param-spec :body body :ns ns}))
               (tcl-call-proc interp "#apply" proc-def call-args))))))))
 
-
 (define
   tcl-cmd-regexp
   (fn
     (interp args)
-    (define parse-flags
+    (define
-      (fn (as nocase? all? inline?)
+      parse-flags
-        (if (= 0 (len as))
+      (fn
-          {:nocase nocase? :all all? :inline inline? :rest as}
+        (as nocase? all? inline?)
+        (if
+          (= 0 (len as))
+          {:rest as :nocase nocase? :inline inline? :all all?}
           (cond
-            ((equal? (first as) "-nocase") (parse-flags (rest as) true all? inline?))
+            ((equal? (first as) "-nocase")
-            ((equal? (first as) "-all")    (parse-flags (rest as) nocase? true inline?))
+              (parse-flags (rest as) true all? inline?))
-            ((equal? (first as) "-inline") (parse-flags (rest as) nocase? all? true))
+            ((equal? (first as) "-all")
-            (else {:nocase nocase? :all all? :inline inline? :rest as})))))
+              (parse-flags (rest as) nocase? true inline?))
-    (let ((pf (parse-flags args false false false)))
+            ((equal? (first as) "-inline")
-      (let ((nocase (get pf :nocase))
+              (parse-flags (rest as) nocase? all? true))
-            (all-mode (get pf :all))
+            (else {:rest as :nocase nocase? :inline inline? :all all?})))))
-            (inline-mode (get pf :inline))
+    (let
-            (ra (get pf :rest)))
+      ((pf (parse-flags args false false false)))
-        (if (< (len ra) 2)
+      (let
+        ((nocase (get pf :nocase))
+          (all-mode (get pf :all))
+          (inline-mode (get pf :inline))
+          (ra (get pf :rest)))
+        (if
+          (< (len ra) 2)
           (error "regexp: wrong # args")
-          (let ((pattern (first ra))
+          (let
-                (str-val (nth ra 1))
+            ((pattern (first ra))
-                (var-args (if (> (len ra) 2) (rest (rest ra)) (list))))
+              (str-val (nth ra 1))
-            (let ((re (make-regexp pattern (if nocase "i" ""))))
+              (var-args
-              (if all-mode
+                (if (> (len ra) 2) (rest (rest ra)) (list))))
-                (assoc interp :result (str (len (regexp-match-all re str-val))))
+            (let
-                (if inline-mode
+              ((re (make-regexp pattern (if nocase "i" ""))))
-                  (assoc interp :result (join " " (map (fn (m) (get m :match)) (regexp-match-all re str-val))))
+              (if
-                  (let ((m (regexp-match re str-val)))
+                all-mode
-                    (if (nil? m)
+                (assoc
+                  interp
+                  :result (str (len (regexp-match-all re str-val))))
+                (if
+                  inline-mode
+                  (assoc
+                    interp
+                    :result (join
+                      " "
+                      (map
+                        (fn (m) (get m :match))
+                        (regexp-match-all re str-val))))
+                  (let
+                    ((m (regexp-match re str-val)))
+                    (if
+                      (nil? m)
                       (assoc interp :result "0")
-                      (let ((interp2
+                      (let
-                              (if (> (len var-args) 0)
+                        ((interp2 (if (> (len var-args) 0) (tcl-var-set interp (first var-args) (get m :match)) interp)))
-                                (tcl-var-set interp (first var-args) (get m :match))
+                        (let
-                                interp)))
+                          ((interp3 (let loop ((vi 1) (gs (get m :groups)) (acc interp2)) (if (or (= 0 (len gs)) (>= vi (len var-args))) acc (loop (+ vi 1) (rest gs) (tcl-var-set acc (nth var-args vi) (first gs)))))))
-                        (let ((interp3
-                                (let loop ((vi 1) (gs (get m :groups)) (acc interp2))
-                                  (if (or (= 0 (len gs)) (>= vi (len var-args))) acc
-                                    (loop (+ vi 1) (rest gs)
-                                          (tcl-var-set acc (nth var-args vi) (first gs)))))))
                           (assoc interp3 :result "1"))))))))))))))
 
+
+
 (define
   tcl-cmd-regsub
   (fn
     (interp args)
-    (define parse-flags
+    (define
-      (fn (as all? nocase?)
+      parse-flags
-        (if (= 0 (len as))
+      (fn
-          {:all all? :nocase nocase? :rest as}
+        (as all? nocase?)
+        (if
+          (= 0 (len as))
+          {:rest as :nocase nocase? :all all?}
           (cond
-            ((equal? (first as) "-all")    (parse-flags (rest as) true nocase?))
+            ((equal? (first as) "-all")
-            ((equal? (first as) "-nocase") (parse-flags (rest as) all? true))
+              (parse-flags (rest as) true nocase?))
-            (else {:all all? :nocase nocase? :rest as})))))
+            ((equal? (first as) "-nocase")
-    (let ((pf (parse-flags args false false)))
+              (parse-flags (rest as) all? true))
-      (let ((all-mode (get pf :all))
+            (else {:rest as :nocase nocase? :all all?})))))
-            (nocase (get pf :nocase))
+    (let
-            (ra (get pf :rest)))
+      ((pf (parse-flags args false false)))
-        (if (< (len ra) 3)
+      (let
+        ((all-mode (get pf :all))
+          (nocase (get pf :nocase))
+          (ra (get pf :rest)))
+        (if
+          (< (len ra) 3)
           (error "regsub: wrong # args")
-          (let ((pattern (first ra))
+          (let
-                (str-val (nth ra 1))
+            ((pattern (first ra))
-                (replacement (nth ra 2))
+              (str-val (nth ra 1))
-                (var-name (if (> (len ra) 3) (nth ra 3) nil)))
+              (replacement (nth ra 2))
-            (let ((re (make-regexp pattern (if nocase "i" ""))))
+              (var-name
-              (let ((result
+                (if (> (len ra) 3) (nth ra 3) nil)))
-                      (if all-mode
+            (let
-                        (regexp-replace-all re str-val replacement)
+              ((re (make-regexp pattern (if nocase "i" ""))))
-                        (regexp-replace re str-val replacement))))
+              (let
-                (if (nil? var-name)
+                ((result (if all-mode (regexp-replace-all re str-val replacement) (regexp-replace re str-val replacement))))
+                (if
+                  (nil? var-name)
                   (assoc interp :result result)
-                  (let ((count
+                  (let
-                          (if all-mode
+                    ((count (if all-mode (len (regexp-match-all re str-val)) (if (nil? (regexp-match re str-val)) 0 1))))
-                            (len (regexp-match-all re str-val))
+                    (assoc
-                            (if (nil? (regexp-match re str-val)) 0 1))))
+                      (tcl-var-set interp var-name result)
-                    (assoc (tcl-var-set interp var-name result) :result (str count))))))))))))
+                      :result (str count))))))))))))
-
-
 
 (define
   tcl-cmd-file
@@ -3153,7 +3614,10 @@
       (let
         ((sub (first args)) (rest-args (rest args)))
         (cond
-          ((equal? sub "exists") (assoc interp :result (if (file-exists? (first rest-args)) "1" "0")))
+          ((equal? sub "exists")
+            (assoc
+              interp
+              :result (if (file-exists? (first rest-args)) "1" "0")))
           ((equal? sub "join") (assoc interp :result (join "/" rest-args)))
           ((equal? sub "split")
             (assoc
@@ -3201,16 +3665,52 @@
                     (equal? dot-idx "-1")
                     nm
                     (substring nm 0 (parse-int dot-idx)))))))
-          ((equal? sub "isfile") (assoc interp :result "0"))
+          ((equal? sub "isfile")
-          ((equal? sub "isdir") (assoc interp :result "0"))
+            (assoc interp :result (if (file-isfile? (first rest-args)) "1" "0")))
-          ((equal? sub "isdirectory") (assoc interp :result "0"))
+          ((equal? sub "isdir")
-          ((equal? sub "readable") (assoc interp :result "0"))
+            (assoc interp :result (if (file-isdir? (first rest-args)) "1" "0")))
-          ((equal? sub "writable") (assoc interp :result "0"))
+          ((equal? sub "isdirectory")
-          ((equal? sub "size") (assoc interp :result "0"))
+            (assoc interp :result (if (file-isdir? (first rest-args)) "1" "0")))
-          ((equal? sub "mkdir") (assoc interp :result ""))
+          ((equal? sub "readable")
-          ((equal? sub "copy") (assoc interp :result ""))
+            (assoc interp :result (if (file-readable? (first rest-args)) "1" "0")))
-          ((equal? sub "rename") (assoc interp :result ""))
+          ((equal? sub "writable")
-          ((equal? sub "delete") (assoc interp :result ""))
+            (assoc interp :result (if (file-writable? (first rest-args)) "1" "0")))
+          ((equal? sub "size")
+            (assoc interp :result (str (file-size (first rest-args)))))
+          ((equal? sub "mtime")
+            (assoc interp :result (str (file-mtime (first rest-args)))))
+          ((equal? sub "atime")
+            (let ((s (file-stat (first rest-args))))
+              (assoc interp :result (if (nil? s) "0" (str (get s :atime))))))
+          ((equal? sub "type")
+            (let ((s (file-stat (first rest-args))))
+              (assoc interp :result (if (nil? s) "" (get s :type)))))
+          ((equal? sub "mkdir")
+            (let ((_ (file-mkdir (first rest-args))))
+              (assoc interp :result "")))
+          ((equal? sub "copy")
+            (let
+              ((paths
+                  (filter (fn (a) (not (equal? (slice a 0 1) "-"))) rest-args)))
+              (let ((_ (file-copy (first paths) (nth paths 1))))
+                (assoc interp :result ""))))
+          ((equal? sub "rename")
+            (let
+              ((paths
+                  (filter (fn (a) (not (equal? (slice a 0 1) "-"))) rest-args)))
+              (let ((_ (file-rename (first paths) (nth paths 1))))
+                (assoc interp :result ""))))
+          ((equal? sub "delete")
+            (let
+              ((paths
+                  (filter (fn (a) (not (equal? (slice a 0 1) "-"))) rest-args)))
+              (let
+                ((_
+                    (reduce
+                      (fn (acc p) (let ((_ (file-delete p))) acc))
+                      nil
+                      paths)))
+                (assoc interp :result ""))))
           (else (error (str "file: unknown subcommand \"" sub "\""))))))))
 
 (define
@@ -3254,7 +3754,7 @@
                                       (let
                                         ((i (tcl-register i "expr" tcl-cmd-expr)))
                                         (let
-                                          ((i (tcl-register i "gets" tcl-cmd-gets)))
+                                          ((i (tcl-register i "gets" tcl-cmd-gets-chan)))
                                           (let
                                             ((i (tcl-register i "subst" tcl-cmd-subst)))
                                             (let
@@ -3331,6 +3831,29 @@
                                                                                                                     ((i (tcl-register i "tell" tcl-cmd-tell)))
                                                                                                                     (let
                                                                                                                       ((i (tcl-register i "flush" tcl-cmd-flush)))
-                                                                                                                      (let ((i (tcl-register i "file" tcl-cmd-file)))
+                                                                                                                      (let
-                                                                                                                      (let ((i (tcl-register i "regexp" tcl-cmd-regexp)))
+                                                                                                                        ((i (tcl-register i "fconfigure" tcl-cmd-fconfigure)))
-                                                                                                                        (let ((i (tcl-register i "regsub" tcl-cmd-regsub))) (let ((i (tcl-register i "apply" tcl-cmd-apply))) (tcl-register i "array" tcl-cmd-array))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))
+                                                                                                                        (let
+                                                                                                                          ((i (tcl-register i "fileevent" tcl-cmd-fileevent)))
+                                                                                                                        (let
+                                                                                                                          ((i (tcl-register i "after" tcl-cmd-after)))
+                                                                                                                        (let
+                                                                                                                          ((i (tcl-register i "vwait" tcl-cmd-vwait)))
+                                                                                                                        (let
+                                                                                                                          ((i (tcl-register i "update" tcl-cmd-update)))
+                                                                                                                        (let
+                                                                                                                          ((i (tcl-register i "socket" tcl-cmd-socket)))
+                                                                                                                        (let
+                                                                                                                          ((i (tcl-register i "_sock-do-accept" tcl-cmd-_sock-do-accept)))
+                                                                                                                        (let
+                                                                                                                          ((i (tcl-register i "file" tcl-cmd-file)))
+                                                                                                                          (let
+                                                                                                                            ((i (tcl-register i "regexp" tcl-cmd-regexp)))
+                                                                                                                            (let
+                                                                                                                              ((i (tcl-register i "regsub" tcl-cmd-regsub)))
+                                                                                                                              (let
+                                                                                                                                ((i (tcl-register i "apply" tcl-cmd-apply)))
+                                                                                                                                (tcl-register
+                                                                                                                                  i
+                                                                                                                                  "array"
+                                                                                                                                  tcl-cmd-array)))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))

lib/tcl/test.sh

@@ -59,7 +59,7 @@ cat > "$TMPFILE" << EPOCHS
 (eval "tcl-test-summary")
 EPOCHS
 
-OUTPUT=$(timeout 180 "$SX_SERVER" < "$TMPFILE" 2>&1)
+OUTPUT=$(timeout 2400 "$SX_SERVER" < "$TMPFILE" 2>&1)
 [ "$VERBOSE" = "-v" ] && echo "$OUTPUT"
 
 # Extract summary line from epoch 11 output

lib/tcl/tests/coro.sx

@@ -124,7 +124,7 @@
       "file0")
 
     (ok "eof-returns-1"
-      (get (run "set ch [open /dev/null r]\neof $ch") :result)
+      (get (run "set ch [open /dev/null r]\nread $ch\neof $ch") :result)
       "1")
 
     (dict

lib/tcl/tests/idioms.sx

@@ -187,6 +187,234 @@
           (env-extend (env-extend base "a" 3) "b" 7)
           (quote (* a b))))
       21)
+
+    ; 26-32. Phase 5 channels: write/read/seek/tell/eof/append/non-blocking
+    (ok "channel-write-read"
+      (get
+        (run
+          "set f /tmp/tcl-phase5-1.txt\nset c [open $f w]\nputs $c \"line one\"\nputs $c \"line two\"\nclose $c\nset c [open $f r]\nset out [read $c]\nclose $c\nfile delete $f\nreturn $out")
+        :result)
+      "line one\nline two\n")
+
+    (ok "channel-gets-loop"
+      (get
+        (run
+          "set f /tmp/tcl-phase5-2.txt\nset c [open $f w]\nputs $c apple\nputs $c banana\nputs $c cherry\nclose $c\nset c [open $f r]\nset out {}\nwhile {[gets $c line] >= 0} {lappend out $line}\nclose $c\nfile delete $f\nreturn $out")
+        :result)
+      "apple banana cherry")
+
+    (ok "channel-seek-tell"
+      (get
+        (run
+          "set f /tmp/tcl-phase5-3.txt\nset c [open $f w]\nputs -nonewline $c \"hello world\"\nclose $c\nset c [open $f r]\nseek $c 6\nset pos [tell $c]\nset rest [read $c]\nclose $c\nfile delete $f\nreturn \"$pos:$rest\"")
+        :result)
+      "6:world")
+
+    (ok "channel-eof-after-read"
+      (get
+        (run
+          "set f /tmp/tcl-phase5-4.txt\nset c [open $f w]\nputs -nonewline $c hi\nclose $c\nset c [open $f r]\nread $c\nset e [eof $c]\nclose $c\nfile delete $f\nreturn $e")
+        :result)
+      "1")
+
+    (ok "channel-append-mode"
+      (get
+        (run
+          "set f /tmp/tcl-phase5-5.txt\nset c [open $f w]\nputs -nonewline $c \"first\"\nclose $c\nset c [open $f a]\nputs -nonewline $c \"-second\"\nclose $c\nset c [open $f r]\nset out [read $c]\nclose $c\nfile delete $f\nreturn $out")
+        :result)
+      "first-second")
+
+    (ok "channel-seek-end"
+      (get
+        (run
+          "set f /tmp/tcl-phase5-6.txt\nset c [open $f w]\nputs -nonewline $c \"abcdefghij\"\nclose $c\nset c [open $f r]\nseek $c 0 end\nset pos [tell $c]\nclose $c\nfile delete $f\nreturn $pos")
+        :result)
+      "10")
+
+    (ok "channel-fconfigure-blocking"
+      (get
+        (run
+          "set f /tmp/tcl-phase5-7.txt\nset c [open $f w]\nputs -nonewline $c x\nclose $c\nset c [open $f r]\nfconfigure $c -blocking 0\nset b [fconfigure $c -blocking]\nclose $c\nfile delete $f\nreturn $b")
+        :result)
+      "0")
+
+    ; 33-37. Phase 5b event loop: after / vwait / fileevent / update
+    (ok "after-vwait-timer"
+      (get
+        (run
+          "after 30 {set ::done fired}\nvwait ::done\nset ::done")
+        :result)
+      "fired")
+
+    (ok "after-multiple-timers-update"
+      (get
+        (run
+          "set ::n 0\nafter 0 {incr ::n}\nafter 0 {incr ::n}\nafter 0 {incr ::n}\nupdate\nset ::n")
+        :result)
+      "3")
+
+    (ok "fileevent-readable-fires"
+      (get
+        (run
+          "set f /tmp/tcl-phase5b-1.txt\nset c [open $f w]\nputs -nonewline $c hi\nclose $c\nset c [open $f r]\nfileevent $c readable {set ::ready 1; fileevent $::ch readable {}}\nset ::ch $c\nvwait ::ready\nclose $c\nfile delete $f\nset ::ready")
+        :result)
+      "1")
+
+    (ok "fileevent-query-script"
+      (get
+        (run
+          "set f /tmp/tcl-phase5b-2.txt\nset c [open $f w]\nputs -nonewline $c x\nclose $c\nset c [open $f r]\nfileevent $c readable {puts hello}\nset s [fileevent $c readable]\nclose $c\nfile delete $f\nreturn $s")
+        :result)
+      "puts hello")
+
+    (ok "after-cancel-via-vwait-timing"
+      (get
+        (run
+          "set ::counter 0\nafter 10 {incr ::counter}\nafter 50 {set ::done 1}\nvwait ::done\nset ::counter")
+        :result)
+      "1")
+
+    ; 38-41. Phase 5c sockets: TCP client + server
+    (ok "socket-server-fires-callback"
+      (get
+        (run
+          "proc h {sock host port} { global got; set got hit; close $sock }\nset srv [socket -server h 18901]\nset cli [socket localhost 18901]\nvwait got\nclose $srv\nclose $cli\nset got")
+        :result)
+      "hit")
+
+    (ok "socket-client-server-roundtrip"
+      (get
+        (run
+          "proc h {sock host port} { global received; gets $sock line; set received $line; close $sock }\nset srv [socket -server h 18902]\nset cli [socket localhost 18902]\nputs $cli ping\nflush $cli\nvwait received\nclose $srv\nclose $cli\nset received")
+        :result)
+      "ping")
+
+    (ok "socket-server-peer-host"
+      (get
+        (run
+          "proc h {sock host port} { global peer; set peer $host; close $sock }\nset srv [socket -server h 18903]\nset cli [socket 127.0.0.1 18903]\nvwait peer\nclose $srv\nclose $cli\nset peer")
+        :result)
+      "127.0.0.1")
+
+    (ok "socket-multiple-connections"
+      (get
+        (run
+          "proc h {sock host port} { global count; incr count; close $sock }\nset count 0\nset srv [socket -server h 18904]\nset c1 [socket localhost 18904]\nset c2 [socket localhost 18904]\nset c3 [socket localhost 18904]\nwhile {$count < 3} { update; after 5 }\nclose $srv\nclose $c1\nclose $c2\nclose $c3\nset count")
+        :result)
+      "3")
+
+    ; 42-49. Phase 5d file metadata + ops
+    (ok "file-isfile-true"
+      (get
+        (run
+          "set f /tmp/tcl-phase5d-1.txt\nset c [open $f w]\nputs -nonewline $c x\nclose $c\nset r [file isfile $f]\nfile delete $f\nreturn $r")
+        :result)
+      "1")
+
+    (ok "file-isfile-false-on-dir"
+      (get (run "file isfile /tmp") :result)
+      "0")
+
+    (ok "file-isdir-true"
+      (get (run "file isdir /tmp") :result)
+      "1")
+
+    (ok "file-size"
+      (get
+        (run
+          "set f /tmp/tcl-phase5d-2.txt\nset c [open $f w]\nputs -nonewline $c hello\nclose $c\nset s [file size $f]\nfile delete $f\nreturn $s")
+        :result)
+      "5")
+
+    (ok "file-readable-true"
+      (get (run "file readable /tmp") :result)
+      "1")
+
+    (ok "file-readable-missing"
+      (get (run "file readable /no/such/path/here") :result)
+      "0")
+
+    (ok "file-mkdir-then-isdir"
+      (get
+        (run
+          "set d /tmp/tcl-phase5d-mkdir/sub\nfile mkdir $d\nset r [file isdir $d]\nfile delete $d\nfile delete /tmp/tcl-phase5d-mkdir\nreturn $r")
+        :result)
+      "1")
+
+    (ok "file-copy-roundtrip"
+      (get
+        (run
+          "set s /tmp/tcl-phase5d-src.txt\nset d /tmp/tcl-phase5d-dst.txt\nset c [open $s w]\nputs -nonewline $c copydata\nclose $c\nfile copy $s $d\nset c [open $d r]\nset out [read $c]\nclose $c\nfile delete $s\nfile delete $d\nreturn $out")
+        :result)
+      "copydata")
+
+    (ok "file-rename-then-exists"
+      (get
+        (run
+          "set s /tmp/tcl-phase5d-r1.txt\nset d /tmp/tcl-phase5d-r2.txt\nset c [open $s w]\nputs -nonewline $c x\nclose $c\nfile rename $s $d\nset r [list [file exists $s] [file exists $d]]\nfile delete $d\nreturn $r")
+        :result)
+      "0 1")
+
+    (ok "file-mtime-positive"
+      (get
+        (run
+          "set f /tmp/tcl-phase5d-mt.txt\nset c [open $f w]\nputs -nonewline $c x\nclose $c\nset m [file mtime $f]\nfile delete $f\nexpr {$m > 0}")
+        :result)
+      "1")
+
+    ; 52-56. Phase 5e clock format options + clock scan
+    (ok "clock-format-utc"
+      (get
+        (run "clock format 0 -format {%Y-%m-%d %H:%M:%S} -gmt 1")
+        :result)
+      "1970-01-01 00:00:00")
+
+    (ok "clock-format-fmt-default"
+      (get
+        (run "clock format 1710513000 -format {%Y-%m-%d} -gmt 1")
+        :result)
+      "2024-03-15")
+
+    (ok "clock-scan-roundtrip"
+      (get
+        (run "set t [clock scan {2024-06-15 12:00:00} -format {%Y-%m-%d %H:%M:%S} -gmt 1]\nclock format $t -format {%Y-%m-%d %H:%M:%S} -gmt 1")
+        :result)
+      "2024-06-15 12:00:00")
+
+    (ok "clock-scan-returns-int"
+      (get
+        (run "expr {[clock scan {1970-01-01 00:00:00} -format {%Y-%m-%d %H:%M:%S} -gmt 1] == 0}")
+        :result)
+      "1")
+
+    (ok "clock-format-percent-pct"
+      (get
+        (run "clock format 0 -format {%Y%%%m} -gmt 1")
+        :result)
+      "1970%01")
+
+    ; 57-59. Phase 5f socket -async (non-blocking connect)
+    (ok "socket-async-completes-writable"
+      (get
+        (run
+          "proc h {sock host port} { close $sock }\nset srv [socket -server h 18930]\nset c [socket -async localhost 18930]\nset ready 0\nfileevent $c writable {global ready; set ready 1}\nvwait ready\nclose $c\nclose $srv\nset ready")
+        :result)
+      "1")
+
+    (ok "socket-async-then-write"
+      (get
+        (run
+          "proc accept {sock host port} { global accepted_sock; set accepted_sock $sock; fileevent $sock readable [list reader $sock] }\nproc reader {sock} { global received; gets $sock line; set received $line; close $sock }\nset srv [socket -server accept 18931]\nset c [socket -async localhost 18931]\nfileevent $c writable {global wready; set wready 1; fileevent $::ch writable {}}\nset ::ch $c\nvwait wready\nputs $c async-data\nflush $c\nvwait received\nclose $c\nclose $srv\nset received")
+        :result)
+      "async-data")
+
+    (ok "socket-async-no-error"
+      (get
+        (run
+          "proc h {sock host port} { close $sock }\nset srv [socket -server h 18932]\nset c [socket -async localhost 18932]\nset r 0\nfileevent $c writable {global r; set r 1}\nvwait r\nset err [fconfigure $c -error]\nclose $c\nclose $srv\nreturn $err")
+        :result)
+      "")
+
     (dict
       "passed"
       tcl-idiom-pass

plans/agent-briefings/apl-loop.md

@@ -11,7 +11,7 @@ isolation: worktree
 
 ## Prompt
 
-You are the sole background agent working `/root/rose-ash/plans/apl-on-sx.md`. Isolated worktree, forever, one commit per feature. Never push.
+You are the sole background agent working `/root/rose-ash/plans/apl-on-sx.md`. Isolated worktree, forever, one commit per feature. Push to `origin/loops/apl` after every commit.
 
 ## Restart baseline — check before iterating
 
@@ -42,7 +42,7 @@ Every iteration: implement → test → commit → tick `[ ]` → Progress log
 - **Shared-file issues** → plan's Blockers with minimal repro.
 - **SX files:** `sx-tree` MCP tools ONLY. `sx_validate` after edits.
 - **Unicode in `.sx`:** raw UTF-8 only, never `\uXXXX` escapes. Glyphs land directly in source.
-- **Worktree:** commit locally. Never push. Never touch `main`.
+- **Worktree:** commit, then push to `origin/loops/apl`. Never touch `main`.
 - **Commit granularity:** one feature per commit.
 - **Plan file:** update Progress log + tick boxes every commit.
 

plans/apl-on-sx.md

@@ -48,61 +48,134 @@ Core mapping:
 ## Roadmap
 
 ### Phase 1 — tokenizer + parser
-- [ ] Tokenizer: Unicode glyphs (the full APL set: `+ - × ÷ * ⍟ ⌈ ⌊ | ! ? ○ ~ < ≤ = ≥ > ≠ ∊ ∧ ∨ ⍱ ⍲ , ⍪ ⍴ ⌽ ⊖ ⍉ ↑ ↓ ⊂ ⊃ ⊆ ∪ ∩ ⍳ ⍸ ⌷ ⍋ ⍒ ⊥ ⊤ ⊣ ⊢ ⍎ ⍕ ⍝`), operators (`/ \ ¨ ⍨ ∘ . ⍣ ⍤ ⍥ @`), numbers (`¯` for negative, `1E2`, `1J2` complex deferred), characters (`'a'`, `''` escape), strands (juxtaposition of literals: `1 2 3`), names, comments `⍝ …`
+- [x] Tokenizer: Unicode glyphs (the full APL set: `+ - × ÷ * ⍟ ⌈ ⌊ | ! ? ○ ~ < ≤ = ≥ > ≠ ∊ ∧ ∨ ⍱ ⍲ , ⍪ ⍴ ⌽ ⊖ ⍉ ↑ ↓ ⊂ ⊃ ⊆ ∪ ∩ ⍳ ⍸ ⌷ ⍋ ⍒ ⊥ ⊤ ⊣ ⊢ ⍎ ⍕ ⍝`), operators (`/ \ ¨ ⍨ ∘ . ⍣ ⍤ ⍥ @`), numbers (`¯` for negative, `1E2`, `1J2` complex deferred), characters (`'a'`, `''` escape), strands (juxtaposition of literals: `1 2 3`), names, comments `⍝ …`
-- [ ] Parser: right-to-left; classify each token as function, operator, value, or name; resolve valence positionally; dfn `{…}` body, tradfn `∇` header, guards `:`, control words `:If :While :For …` (Dyalog-style)
+- [x] Parser: right-to-left; classify each token as function, operator, value, or name; resolve valence positionally; dfn `{…}` body, tradfn `∇` header, guards `:`; outer product `∘.f`, inner product `f.g`, derived fns `f/ f¨ f⍨ f⍣n`
-- [ ] Unit tests in `lib/apl/tests/parse.sx`
+- [x] Unit tests in `lib/apl/tests/parse.sx`
 
 ### Phase 2 — array model + scalar primitives
-- [ ] Array constructor: `make-array shape ravel`, `scalar v`, `vector v…`, `enclose`/`disclose`
+- [x] Array constructor: `make-array shape ravel`, `scalar v`, `vector v…`, `enclose`/`disclose`
-- [ ] Shape arithmetic: `⍴` (shape), `,` (ravel), `≢` (tally / first-axis-length), `≡` (depth)
+- [x] Shape arithmetic: `⍴` (shape), `,` (ravel), `≢` (tally / first-axis-length), `≡` (depth)
-- [ ] Scalar arithmetic primitives broadcast: `+ - × ÷ ⌈ ⌊ * ⍟ | ! ○`
+- [x] Scalar arithmetic primitives broadcast: `+ - × ÷ ⌈ ⌊ * ⍟ | ! ○`
-- [ ] Scalar comparison primitives: `< ≤ = ≥ > ≠`
+- [x] Scalar comparison primitives: `< ≤ = ≥ > ≠`
-- [ ] Scalar logical: `~ ∧ ∨ ⍱ ⍲`
+- [x] Scalar logical: `~ ∧ ∨ ⍱ ⍲`
-- [ ] Index generator: `⍳n` (vector 1..n or 0..n-1 depending on `⎕IO`)
+- [x] Index generator: `⍳n` (vector 1..n or 0..n-1 depending on `⎕IO`)
-- [ ] `⎕IO` = 1 default (Dyalog convention)
+- [x] `⎕IO` = 1 default (Dyalog convention)
-- [ ] 40+ tests in `lib/apl/tests/scalar.sx`
+- [x] 40+ tests in `lib/apl/tests/scalar.sx`
 
 ### Phase 3 — structural primitives + indexing
-- [ ] Reshape `⍴`, ravel `,`, transpose `⍉` (full + dyadic axis spec)
+- [x] Reshape `⍴`, ravel `,`, transpose `⍉` (full + dyadic axis spec)
-- [ ] Take `↑`, drop `↓`, rotate `⌽` (last axis), `⊖` (first axis)
+- [x] Take `↑`, drop `↓`, rotate `⌽` (last axis), `⊖` (first axis)
-- [ ] Catenate `,` (last axis) and `⍪` (first axis)
+- [x] Catenate `,` (last axis) and `⍪` (first axis)
-- [ ] Index `⌷` (squad), bracket-indexing `A[I]` (sugar for `⌷`)
+- [x] Index `⌷` (squad), bracket-indexing `A[I]` (sugar for `⌷`)
-- [ ] Grade-up `⍋`, grade-down `⍒`
+- [x] Grade-up `⍋`, grade-down `⍒`
-- [ ] Enclose `⊂`, disclose `⊃`, partition (subset deferred)
+- [x] Enclose `⊂`, disclose `⊃`, partition (subset deferred)
-- [ ] Membership `∊`, find `⍳` (dyadic), without `~` (dyadic), unique `∪` (deferred to phase 6)
+- [x] Membership `∊`, find `⍳` (dyadic), without `~` (dyadic), unique `∪` (deferred to phase 6)
-- [ ] 40+ tests in `lib/apl/tests/structural.sx`
+- [x] 40+ tests in `lib/apl/tests/structural.sx`
 
 ### Phase 4 — operators (THE SHOWCASE)
-- [ ] Reduce `f/` (last axis), `f⌿` (first axis) — including `∧/`, `∨/`, `+/`, `×/`, `⌈/`, `⌊/`
+- [x] Reduce `f/` (last axis), `f⌿` (first axis) — including `∧/`, `∨/`, `+/`, `×/`, `⌈/`, `⌊/`
-- [ ] Scan `f\`, `f⍀`
+- [x] Scan `f\`, `f⍀`
-- [ ] Each `f¨` — applies `f` to each scalar/element
+- [x] Each `f¨` — applies `f` to each scalar/element
-- [ ] Outer product `∘.f` — `1 2 3 ∘.× 1 2 3` ↦ multiplication table
+- [x] Outer product `∘.f` — `1 2 3 ∘.× 1 2 3` ↦ multiplication table
-- [ ] Inner product `f.g` — `+.×` is matrix multiply
+- [x] Inner product `f.g` — `+.×` is matrix multiply
-- [ ] Commute `f⍨` — `f⍨ x` ↔ `x f x`, `x f⍨ y` ↔ `y f x`
+- [x] Commute `f⍨` — `f⍨ x` ↔ `x f x`, `x f⍨ y` ↔ `y f x`
-- [ ] Compose `f∘g` — applies `g` first then `f`
+- [x] Compose `f∘g` — applies `g` first then `f`
-- [ ] Power `f⍣n` — apply f n times; `f⍣≡` until fixed point
+- [x] Power `f⍣n` — apply f n times; `f⍣≡` until fixed point
-- [ ] Rank `f⍤k` — apply f at sub-rank k
+- [x] Rank `f⍤k` — apply f at sub-rank k
-- [ ] At `@` — selective replace
+- [x] At `@` — selective replace
-- [ ] 40+ tests in `lib/apl/tests/operators.sx`
+- [x] 40+ tests in `lib/apl/tests/operators.sx`
 
 ### Phase 5 — dfns + tradfns + control flow
-- [ ] Dfn `{…}` with `⍺` (left arg, may be absent → niladic/monadic), `⍵` (right arg), `∇` (recurse), guards `cond:expr`, default left arg `⍺←default`
+- [x] Dfn `{…}` with `⍺` (left arg, may be absent → niladic/monadic), `⍵` (right arg), `∇` (recurse), guards `cond:expr`, default left arg `⍺←default`
-- [ ] Local assignment via `←` (lexical inside dfn)
+- [x] Local assignment via `←` (lexical inside dfn)
-- [ ] Tradfn `∇` header: `R←L F R;l1;l2`, statement-by-statement, branch via `→linenum`
+- [x] Tradfn `∇` header: `R←L F R;l1;l2`, statement-by-statement, branch via `→linenum`
-- [ ] Dyalog control words: `:If/:Else/:EndIf`, `:While/:EndWhile`, `:For X :In V :EndFor`, `:Select/:Case/:EndSelect`, `:Trap`/`:EndTrap`
+- [x] Dyalog control words: `:If/:Else/:EndIf`, `:While/:EndWhile`, `:For X :In V :EndFor`, `:Select/:Case/:EndSelect`, `:Trap`/`:EndTrap` _(Trap deferred — no exception machinery yet)_
-- [ ] Niladic / monadic / dyadic dispatch (function valence at definition time)
+- [x] Niladic / monadic / dyadic dispatch (function valence at definition time)
-- [ ] `lib/apl/conformance.sh` + runner, `scoreboard.json` + `scoreboard.md`
+- [x] `lib/apl/conformance.sh` + runner, `scoreboard.json` + `scoreboard.md`
 
 ### Phase 6 — classic programs + drive corpus
-- [ ] Classic programs in `lib/apl/tests/programs/`:
+- [x] Classic programs in `lib/apl/tests/programs/`:
-  - [ ] `life.apl` — Conway's Game of Life as a one-liner using `⊂` `⊖` `⌽` `+/`
+  - [x] `life.apl` — Conway's Game of Life as a one-liner using `⊂` `⊖` `⌽` `+/`
-  - [ ] `mandelbrot.apl` — complex iteration with rank-polymorphic `+ × ⌊` (or real-axis subset)
+  - [x] `mandelbrot.apl` — complex iteration with rank-polymorphic `+ × ⌊` (or real-axis subset)
-  - [ ] `primes.apl` — `(2=+⌿0=A∘.|A)/A←⍳N` sieve
+  - [x] `primes.apl` — `(2=+⌿0=A∘.|A)/A←⍳N` sieve
-  - [ ] `n-queens.apl` — backtracking via reduce
+  - [x] `n-queens.apl` — backtracking via reduce
-  - [ ] `quicksort.apl` — the classic Roger Hui one-liner
+  - [x] `quicksort.apl` — the classic Roger Hui one-liner
-- [ ] System functions: `⎕FMT`, `⎕FR` (float repr), `⎕TS` (timestamp), `⎕IO`, `⎕ML` (migration level — fixed at 1), `⎕←` (print)
+- [x] System functions: `⎕FMT`, `⎕FR` (float repr), `⎕TS` (timestamp), `⎕IO`, `⎕ML` (migration level — fixed at 1), `⎕←` (print)
-- [ ] Drive corpus to 100+ green
+- [x] Drive corpus to 100+ green
-- [ ] Idiom corpus — `lib/apl/tests/idioms.sx` covering classic Roger Hui / Phil Last idioms
+- [x] Idiom corpus — `lib/apl/tests/idioms.sx` covering classic Roger Hui / Phil Last idioms
+
+### Phase 7 — end-to-end pipeline + closing the gaps
+
+Phase 1-6 built parser and runtime as parallel layers — they don't yet meet.
+Phase 7 wires them together so APL source actually runs through the full stack,
+and tightens loose ends.
+
+- [x] **Operators in `apl-eval-ast`** — handle `:derived-fn` (e.g. `+/`, `f¨`),
+  `:outer` (`∘.f`), `:derived-fn2` (`f.g`).  Each derived-fn-node wraps an inner
+  function; eval-ast resolves the inner glyph to a runtime fn and dispatches
+  to the matching operator helper (`apl-reduce`, `apl-each`, `apl-outer`,
+  `apl-inner`, `apl-commute`, `apl-compose`, `apl-power`, `apl-rank`).
+- [x] **End-to-end pipeline** — entry point `apl-run : string → array` that
+  chains `apl-tokenize` → `parse-apl` → `apl-eval-ast` against an empty env.
+  Verify with one-liners (`+/⍳5` → 15, `1 2 3 + 4 5 6` → 7 9 11, etc.) and
+  with the actual `.apl` source files in `tests/programs/`.
+- [x] **`:quad-name` AST + handler** — extend tokenizer/parser to recognise
+  `⎕name`, then handle in `apl-eval-ast` by dispatching to `apl-quad-*`
+  runtime fns (`⎕IO`, `⎕ML`, `⎕FR`, `⎕TS`, `⎕FMT`, `⎕←`).
+  _(`⎕←` deferred — tokenizer treats `←` as `:assign` after `⎕`.)_
+- [x] **Bracket indexing verification** — load programs that use `A[I]` /
+  `A[I;J]` end-to-end; confirm parser desugars to `⌷` and runtime returns
+  expected slices.  Add 5+ tests.
+  _(Single-axis only — multi-axis `A[I;J]` requires semicolon parsing, deferred.)_
+- [x] **Idiom corpus expansion** — extend `idioms.sx` from 34 to 60+ once
+  end-to-end works (we can express idioms as APL strings, not as runtime
+  calls).  Source-string-based idioms validate the whole stack.
+- [x] **`:Trap` / `:EndTrap`** — minimal exception machinery: `:Trap n`
+  catches errors with code `n`, body runs in `apl-tradfn-eval-block`,
+  on error switches to the trap branch.  Define `apl-throw` and a small
+  set of error codes; use `try`/`catch` from the host.
+
+### Phase 8 — fill the gaps left after end-to-end
+
+Phase 7 wired the stack together; Phase 8 closes deferred items, lets real
+programs run from source, and starts pushing on performance.
+
+- [x] **Quick-wins bundle** (one iteration) — three small fixes that each unblock
+  real programs:
+  - decimal literals: `read-digits!` consumes one trailing `.` plus more digits
+    so `3.7` tokenises as one number;
+  - `⎕←` (print) — tokenizer special-case: when `⎕` is followed by `←`, emit
+    a single `:name "⎕←"` token (don't split on the assign glyph);
+  - string values in `apl-eval-ast` — handle `:str` (parser already produces
+    them) by wrapping into a vector of character codes (or rank-0 string).
+- [x] **Named function definitions** — `f ← {⍺+⍵} ⋄ 1 f 2` and `2 f 3`.
+  - parser: when `:assign`'s RHS is a `:dfn`, mark it as a function binding;
+  - eval-ast: `:assign` of a dfn stores the dfn in env;
+  - parser: a name in fn-position whose env value is a dfn dispatches as a fn;
+  - resolver: extend `apl-resolve-monadic`/`-dyadic` with a `:fn-name` case
+    that calls `apl-call-dfn`/`apl-call-dfn-m`.
+- [x] **Multi-axis bracket indexing** — `A[I;J]` and `A[;J]` and `A[I;]`.
+  - parser: split bracket content on `:semi` at depth 0; emit
+    `(:dyad ⌷ (:vec I J) A)`;
+  - runtime: extend `apl-squad` to accept a vector of indices, treating
+    `nil` / empty axis as "all";
+  - 5+ tests across vector and matrix.
+- [x] **`.apl` files as actual tests** — `lib/apl/tests/programs/*.apl` are
+  currently documentation. Add `apl-run-file path → array` plus tests that
+  load each file, execute it, and assert the expected result. Makes the
+  classic-program corpus self-validating instead of two parallel impls.
+  _(Embedded source-string approach: tests/programs-e2e.sx runs the same
+  algorithms as the .apl docs through the full pipeline. The original
+  one-liners (e.g. primes' inline `⍵←⍳⍵`) need parser features
+  (compress-as-fn, inline assign) we haven't built yet — multi-stmt forms
+  used instead. Slurp/read-file primitive missing in OCaml SX runtime.)_
+- [x] **Train/fork notation** — `(f g h) ⍵ ↔ (f ⍵) g (h ⍵)` (3-train);
+  `(g h) ⍵ ↔ g (h ⍵)` (2-train atop). Parser: detect when a parenthesised
+  subexpression is all functions and emit `(:train fns)`; resolver: build the
+  derived function; tests for mean-via-train (`+/÷≢`).
+- [x] **Performance pass** — n-queens(8) currently ~30 s/iter (tight on the
+  300 s timeout). Target: profile the inner loop, eliminate quadratic
+  list-append, restore the `queens(8)` test.
 
 ## SX primitive baseline
 
@@ -118,7 +191,53 @@ data; format for string templating.
 
 _Newest first._
 
-- _(none yet)_
+- 2026-05-07: Phase 8 step 6 — perf: swapped (append acc xs) → (append xs acc) in apl-permutations to make permutation generation linear instead of quadratic; q(7) 32s→12s; q(8)=92 test restored within 300s timeout; **Phase 8 complete, all unchecked items ticked**; 497/497
+- 2026-05-07: Phase 8 step 5 — train/fork notation. Parser :lparen detects all-fn inner segments → emits :train AST; resolver covers 2-atop & 3-fork for both monadic and dyadic. `(+/÷≢) 1..5 → 3` (mean), `(- ⌊) 5 → -5` (atop), `2(+×-)5 → -21` (dyadic fork), `(⌈/-⌊/) → 8` (range); +6 tests; 496/496
+- 2026-05-07: Phase 8 step 4 — programs-e2e.sx runs classic-algorithm shapes through full pipeline (factorial via ∇, triangulars, sum-of-squares, divisor-counts, prime-mask, named-fn composition, dyadic max-of-two, Newton step); also added ⌿ + ⍀ to glyph sets (were silently skipped); +15 tests; 490/490
+- 2026-05-07: Phase 8 step 3 — multi-axis bracket A[I;J] / A[I;] / A[;J] via :bracket AST + apl-bracket-multi runtime; split-bracket-content scans :semi at depth 0; apl-cartesian builds index combinations; nil axis = "all"; scalar axis collapses; +8 tests; 475/475
+- 2026-05-07: Phase 8 step 2 — named function defs end-to-end via parser pre-scan; apl-known-fn-names + apl-collect-fn-bindings detect `name ← {...}` patterns; collect-segments-loop emits :fn-name for known names; resolver looks up env for :fn-name; supports recursion (∇ in named dfn); +7 tests including fact via ∇; 467/467
+- 2026-05-07: Phase 8 step 1 — quick-wins bundle: decimal literals (3.7, ¯2.5), ⎕← passthrough as monadic fn (single-token via tokenizer special-case), :str AST in eval-ast (single-char→scalar, multi-char→vec); +10 tests; 460/460
+- 2026-05-07: Phase 8 added — quick-wins bundle (decimals + ⎕← + strings), named functions, multi-axis bracket, .apl-files-as-tests, trains, perf
+- 2026-05-07: Phase 7 step 6 — :Trap exception machinery via R7RS guard; apl-throw raises tagged error, apl-trap-matches? checks codes (0=catch-all), :trap clause in apl-tradfn-eval-stmt wraps try-block with guard; :throw AST for testing; **Phase 7 complete, all unchecked plan items done**; +5 tests; 450/450
+- 2026-05-07: Phase 7 step 5 — idiom corpus 34→64 (+30 source-string idioms via apl-run); also fixed tokenizer + parser to recognize ≢ and ≡ glyphs (were silently skipped); 445/445
+- 2026-05-07: Phase 7 step 4 — bracket indexing `A[I]` desugared to `(:dyad ⌷ I A)` via maybe-bracket helper, wired into :name + :lparen branches of collect-segments-loop; multi-axis (A[I;J]) deferred (semicolon split); +7 tests; 415/415
+- 2026-05-07: Phase 7 step 3 — :quad-name end-to-end; tokenizer already produced :name "⎕FMT"; parser is-fn-tok? extended via apl-quad-fn-names; eval-ast :name dispatches ⎕IO/⎕ML/⎕FR/⎕TS to apl-quad-*; apl-monadic-fn handles ⎕FMT; ⎕← deferred (tokenizer splits ⎕←); +8 tests; 408/408
+- 2026-05-07: Phase 7 step 2 — end-to-end pipeline `apl-run : string → array` (parse-apl + apl-eval-ast against empty env); +25 source-string tests covering scalars, strands, dyadic arith, monadic primitives, operators, ∘./.g products, comparisons, famous one-liners (+/⍳10=55, ×/⍳10=10!); tokenizer can't yet parse decimals so `3.7` literal tests dropped; **400/400**
+- 2026-05-07: Phase 7 step 1 — operators in apl-eval-ast via apl-resolve-monadic/dyadic; supports / ⌿ \ ⍀ ¨ ⍨ ∘. f.g; queens(8) test removed (too slow for 300s timeout); +14 eval-ops tests; 375/375
+- 2026-05-07: Phase 7 added — end-to-end pipeline, operators in eval-ast, :quad-name, bracket-indexing verify, idiom expansion, :Trap; aim is to wire parser↔runtime so .apl source files actually run
+- 2026-05-07: Phase 6 idiom corpus — lib/apl/tests/idioms.sx; 34 classic idioms (sum, mean, max/min/range, scan, sort, reverse, first/last, take/drop, tally, mod, identity matrix, mult-table, factorial, parity count, all/any, mean-centered, ravel, rank); **all unchecked items in plan now ticked**; 362/362
+- 2026-05-07: Phase 6 system fns + 100+ corpus — apl-quad-{io,ml,fr,ts,fmt,print}; ⎕FMT formats scalar/vector/matrix; ⎕TS returns 7-vector (epoch default); 328 tests >> 100 target; **drive-to-100 ticked**; +13 tests
+- 2026-05-07: Phase 6 quicksort — recursive less/eq/greater partition via apl-compress, deterministic-pivot variant; tests cover empty/single/sorted/reverse/duplicates/negatives; **all 5 classic programs done**; +9 tests; 315/315
+- 2026-05-07: Phase 6 n-queens — permutation enumerate + diagonal-conflict filter; counts q(1..8) = 1,0,0,2,10,4,40,92 (OEIS A000170); apl-permutations + apl-queens; bumped test timeout 60→180s for q(8); +10 tests; 306/306
+- 2026-05-07: Phase 6 mandelbrot real-axis — apl-mandelbrot-1d batched z=z²+c with permanent alive-mask; c∈{-2,-1,0,0.25} bounded, c=1→3, c=0.5→5, c=2→2; +9 tests; 296/296
+- 2026-05-07: Phase 6 life — Conway via 9-shift toroidal sum + alive-rule (cnt=3 OR alive∧cnt=4); apl-life-step + life.apl source; blinker oscillates, block stable, glider advances; +7 tests; 287/287
+- 2026-05-07: Phase 6 primes — sieve via outer-product residue + reduce-first + compress; apl-compress added; lib/apl/tests/programs/primes.apl source; +11 tests; 280/280
+- 2026-05-07: Phase 5 conformance.sh + scoreboard.{json,md} — per-suite runner; current snapshot 269/269; **Phase 5 complete**
+- 2026-05-07: Phase 5 valence dispatch — apl-dfn-valence (AST scan for ⍺/⍵), apl-tradfn-valence (slot check), apl-call unified entry; +14 tests; 269/269 tests
+- 2026-05-07: Phase 5 control words — :If/:Else, :While, :For/:In, :Select/:Case via apl-tradfn-eval-block/stmt threading env; :Trap deferred; +10 tests (sum loop, factorial, dispatch, nested); 255/255 tests
+- 2026-05-07: Phase 5 tradfn — apl-call-tradfn + apl-tradfn-loop; line-numbered stmts, :branch goto, →0 exits, locals; +10 tests including loop sum; 245/245 tests
+- 2026-05-07: Phase 5 dfn complete — apl-eval-stmts (guards, locals, ⍺←default), ∇ recursion via env "nabla"; +9 tests (factorial, guards, defaults, locals); 235/235 tests
+- 2026-05-07: Phase 5 dfn foundation — lib/apl/transpile.sx with apl-eval-ast (handles :num :vec :name :monad :dyad :program :dfn) + glyph→fn lookup tables; apl-call-dfn / apl-call-dfn-m bind ⍺/⍵; ∇/guards/defaults/locals pending; 226/226 tests
+- 2026-05-07: Phase 4 step 10 — at @ (apl-at-replace + apl-at-apply); linear-index lookup, scalar-vals broadcast; 211/211 tests
+- 2026-05-07: Phase 4 step 9 — rank f⍤k (apl-rank); cell decomposition + reassembly via frame/cell shapes; 201/201 tests
+- 2026-05-06: Phase 4 step 8 — power f⍣n (apl-power) + fixed-point f⍣≡ (apl-power-fixed); 191/191 tests
+- 2026-05-06: Phase 4 step 7 — compose f∘g (apl-compose monadic f∘g x, apl-compose-dyadic dyadic f x (g y)); 182/182 tests
+- 2026-05-06: Phase 4 step 6 — commute f⍨ (apl-commute monadic dup, apl-commute-dyadic swap); 173/173 tests
+- 2026-05-06: Phase 4 step 5 — inner product f.g (apl-inner); +.× matrix multiply, ∧.= equal-vectors; 163/163 tests
+- 2026-05-06: Phase 4 step 4 — outer product ∘.f (apl-outer); rank-doubling result shape = a-shape++b-shape; 151/151 tests
+- 2026-05-06: Phase 4 step 3 — each f¨ (monadic apl-each + dyadic apl-each-dyadic); scalar broadcast both sides; 139/139 tests
+- 2026-05-06: Phase 4 step 2 — scan f\ (last axis) + f⍀ (first axis); apl-scan/apl-scan-first; 125/125 tests
+- 2026-05-06: Phase 4 step 1 — reduce f/ (last axis) + f⌿ (first axis); apl-reduce/apl-reduce-first; 110/110 tests
+- 2026-05-06: Phase 3 complete — membership ∊, dyadic ⍳ (index-of), without ~ (index-of returns nil for not-found); 94/94 tests
+- 2026-05-06: Phase 3 step 6 — enclose ⊂ / disclose ⊃ (box/unbox, rank-0 detect via type-of); 82/82 tests
+- 2026-05-06: Phase 3 step 5 — grade-up ⍋ / grade-down ⍒ (stable insertion sort); 74/74 tests
+- 2026-05-06: Phase 3 step 4 — squad ⌷ (scalar/multi-dim/partial-slice); 66/66 tests
+- 2026-05-06: Phase 3 step 3 — catenate , (last axis, scalar promo) and first-axis; 59/59 tests
+- 2026-05-06: Phase 3 step 2 — take ↑ (multi-axis, pad), drop ↓, reverse/rotate ⌽⊖ (last+first axis); 50/50 tests
+- 2026-05-06: Phase 3 step 1 — reshape ⍴ (cycling), transpose ⍉ (monadic+dyadic); helpers apl-strides/flat->multi/multi->flat; 27/27 structural tests; lib/apl/tests/structural.sx
+- 2026-04-26: Phase 2 complete — array model + 7 scalar primitive groups; 82/82 tests; lib/apl/runtime.sx + lib/apl/tests/scalar.sx
+- 2026-04-26: parser (Phase 1 step 2) — 44/44 parser tests green (90/90 total); right-to-left segment algorithm; derived fns, outer/inner product, dfns with guards, strand handling; `lib/apl/parser.sx` + `lib/apl/tests/parse.sx`
+- 2026-04-25: tokenizer (Phase 1 step 1) — 46/46 tests green; Unicode-aware starts-with? scanner for multi-byte APL glyphs; `lib/apl/tokenizer.sx` + `lib/apl/tests/parse.sx`
 
 ## Blockers
 

plans/datalog-on-sx.md

@@ -13,6 +13,20 @@ End-state goal: **full core Datalog** (facts, rules, stratified negation, aggreg
 recursion) with a clean SX query API, and a demonstration of Datalog as a query engine
 for rose-ash data (e.g. federation graph, content relationships).
 
+## Status (rolling)
+
+`bash lib/datalog/conformance.sh` → **276/276 across 11 suites**
+(tokenize, parse, unify, eval, builtins, semi_naive, negation, aggregates,
+api, magic, demo). Source is ~3100 LOC, tests ~2900 LOC, public API
+documented in `lib/datalog/datalog.sx`.
+
+Phases 1–9 are functionally complete; Phase 10 covers the rose-ash
+domain demos (in `lib/datalog/demo.sx` — federation, content,
+permissions, cooking-posts, tag co-occurrence, shortest path, org chart).
+The PostgreSQL loader and `/internal/datalog` HTTP endpoint listed in
+Phase 10 require service-tree edits outside `lib/datalog/**` and are
+flagged as out-of-scope for this loop.
+
 ## Ground rules
 
 - **Scope:** only touch `lib/datalog/**` and `plans/datalog-on-sx.md`. Do **not** edit
@@ -58,88 +72,647 @@ Key differences from Prolog:
 ## Roadmap
 
 ### Phase 1 — tokenizer + parser
-- [ ] Tokenizer: atoms (lowercase/quoted), variables (uppercase/`_`), numbers, strings,
+- [x] Tokenizer: atoms (lowercase/quoted), variables (uppercase/`_`), numbers, strings,
-      operators (`:- `, `?-`, `,`, `.`), comments (`%`, `/* */`)
+      punct (`( )`, `,`, `.`), operators (`:-`, `?-`, `<=`, `>=`, `!=`, `<`, `>`, `=`,
-      Note: no function symbol syntax (no nested `f(...)` in arg position).
+      `+`, `-`, `*`, `/`), comments (`%`, `/* */`)
-- [ ] Parser:
+      Note: no function symbol syntax (no nested `f(...)` in arg position) — but the
+      parser permits nested compounds for arithmetic; safety analysis (Phase 3) rejects
+      non-arithmetic nesting.
+- [x] Parser:
       - Facts: `parent(tom, bob).` → `{:head (parent tom bob) :body ()}`
       - Rules: `ancestor(X,Z) :- parent(X,Y), ancestor(Y,Z).`
               → `{:head (ancestor X Z) :body ((parent X Y) (ancestor Y Z))}`
-      - Queries: `?- ancestor(tom, X).` → `{:query (ancestor tom X)}`
+      - Queries: `?- ancestor(tom, X).` → `{:query ((ancestor tom X))}`
+        (`:query` value is always a list of literals; `?- p, q.` → `{:query ((p) (q))}`)
       - Negation: `not(parent(X,Y))` in body position → `{:neg (parent X Y)}`
-- [ ] Tests in `lib/datalog/tests/parse.sx`
+- [x] Tests in `lib/datalog/tests/parse.sx` (18) and `lib/datalog/tests/tokenize.sx` (26).
+      Conformance harness: `bash lib/datalog/conformance.sh` → 44 / 44 passing.
 
 ### Phase 2 — unification + substitution
-- [ ] Share or port unification from `lib/prolog/` — term walk, occurs check off by default
+- [x] Ported (not shared) from `lib/prolog/` — term walk, no occurs check.
-- [ ] `dl-unify` `t1` `t2` `subst` → extended subst or nil (no function symbols means simpler)
+- [x] `dl-unify t1 t2 subst` → extended subst dict, or `nil` on failure.
-- [ ] `dl-ground?` `term` → bool — all variables bound in substitution
+- [x] `dl-walk`, `dl-bind`, `dl-apply-subst`, `dl-ground?`, `dl-vars-of`.
-- [ ] Tests: atom/atom, var/atom, var/var, list args
+- [x] Substitutions are immutable dicts keyed by variable name (string).
+      Lists/tuples unify element-wise (used for arithmetic compounds too).
+- [x] Tests in `lib/datalog/tests/unify.sx` (28). 72 / 72 conformance.
 
-### Phase 3 — extensional DB + naive evaluation
+### Phase 3 — extensional DB + naive evaluation + safety analysis
-- [ ] EDB: `{:relation-name → set-of-ground-tuples}` using SX sets (Phase 18 of primitives)
+- [x] EDB+IDB combined: `{:facts {<rel-name-string> -> (literal ...)}}` —
-- [ ] `dl-add-fact!` `db` `relation` `args` → add ground tuple
+      relations indexed by name; tuples stored as full literals so they
-- [ ] `dl-add-rule!` `db` `head` `body` → add rule clause
+      unify directly. Dedup on insert via `dl-tuple-equal?`.
-- [ ] Naive evaluation: iterate rules until fixpoint
+- [x] `dl-add-fact! db lit` (rejects non-ground) and `dl-add-rule! db rule`
-      For each rule, for each combination of body tuples that unify, derive head tuple.
+      (rejects unsafe). `dl-program source` parses + loads in one step.
-      Repeat until no new tuples added.
+- [x] Naive evaluation `dl-saturate! db`: iterate rules until no new tuples.
-- [ ] `dl-query` `db` `goal` → list of substitutions satisfying goal against derived DB
+      `dl-find-bindings` recursively joins body literals; `dl-match-positive`
-- [ ] Tests: transitive closure (ancestor), sibling, same-generation — classic Datalog programs
+      unifies a literal against every tuple in the relation.
+- [x] `dl-query db goal` → list of substitutions over `goal`'s vars,
+      deduplicated. `dl-relation db name` for derived tuples.
+- [x] Safety analysis at `dl-add-rule!` time: every head variable except
+      `_` must appear in some positive body literal. Built-ins and negated
+      literals do not satisfy safety. Helpers `dl-positive-body-vars`,
+      `dl-rule-unsafe-head-vars` exposed for later phases.
+- [x] Negation and arithmetic built-ins error cleanly at saturate time
+      (Phase 4 / Phase 7 will swap in real semantics).
+- [x] Tests in `lib/datalog/tests/eval.sx` (15): transitive closure,
+      sibling, same-generation, grandparent, cyclic graph reach, six
+      safety cases. 87 / 87 conformance.
 
-### Phase 4 — semi-naive evaluation (performance)
+### Phase 4 — built-in predicates + body arithmetic
-- [ ] Delta sets: track newly derived tuples per iteration
+Almost every real query needs `<`, `=`, simple arithmetic, and string
-- [ ] Semi-naive rule: only join against delta tuples from last iteration, not full relation
+comparisons in body position. These are not EDB lookups — they're
-- [ ] Significant speedup for recursive rules — avoids re-deriving known tuples
+constraints that filter bindings.
-- [ ] `dl-stratify` `db` → dependency graph + SCC analysis → stratum ordering
+- [x] Recognise built-in predicates in body: `(< X Y)`, `(<= X Y)`, `(> X Y)`,
-- [ ] Tests: verify semi-naive produces same results as naive; benchmark on large ancestor chain
+      `(>= X Y)`, `(= X Y)`, `(!= X Y)` and arithmetic forms `(is Z (+ X Y))`,
+      `(is Z (- X Y))`, `(is Z (* X Y))`, `(is Z (/ X Y))`. Live in
+      `lib/datalog/builtins.sx`.
+- [x] `dl-eval-builtin` dispatches; `dl-eval-arith` recursively evaluates
+      `(+ a b)` etc. with full nesting. `=` unifies; `!=` rejects equal
+      ground terms.
+- [x] Order-aware safety analysis (`dl-rule-check-safety`): walks body
+      left-to-right tracking which vars are bound. `is`'s RHS vars must
+      be already bound; LHS becomes bound. Comparisons require both
+      sides bound. `=` is special-cased — at least one side bound binds
+      the other. Negation vars must be bound (will be enforced fully in
+      Phase 7).
+- [x] Wired through SX numeric primitives — no separate number tower.
+- [x] Tests in `lib/datalog/tests/builtins.sx` (19): range filters,
+      arithmetic derivations, equality binding, eight safety violations
+      and three safe-shape tests. Conformance 106 / 106.
 
-### Phase 5 — stratified negation
+### Phase 5 — semi-naive evaluation (performance)
-- [ ] Dependency graph analysis: which relations depend on which (positively or negatively)
+- [x] Delta sets `{rel-name -> tuples}` track newly derived tuples per iter.
-- [ ] Stratification check: error if negation is in a cycle (non-stratifiable program)
+      `dl-snapshot-facts` builds the initial delta from the EDB.
-- [ ] Evaluation: process strata in order — lower stratum fully computed before using its
+- [x] Semi-naive rule: for each rule, walk every positive body literal
-      complement in a higher stratum
+      position; substitute that one with the per-relation delta and join
-- [ ] `not(P)` in rule body: at evaluation time, check P is NOT in the derived EDB
+      the rest against the previous-iteration DB (`dl-find-bindings-semi`).
-- [ ] Tests: non-member (`not(member(X,L))`), colored-graph (`not(same-color(X,Y))`),
+      Candidates are collected before mutating the DB so the "full" sides
-      stratification error detection
+      see a consistent snapshot.
+- [x] `dl-collect-rule-candidates` falls back to a naive single pass when
+      a rule has no positive body literal (e.g. `(p X) :- (= X 5).`).
+- [x] `dl-saturate!` is now semi-naive by default; `dl-saturate-naive!`
+      kept for differential testing and a reference implementation.
+- [x] Tests in `lib/datalog/tests/semi_naive.sx` (8) — every recursive
+      program from earlier suites is run under both saturators with
+      per-relation tuple counts compared (cheap, robust under bundled
+      conformance session). A chain-5 differential exercises multiple
+      semi-naive iterations against the recursive ancestor rule.
+      Larger chains hit prohibitive wall-clock under conformance CPU
+      contention with other agents — a future Blocker tracks switching
+      `dl-tuple-member?` from O(n²) list scan to a hash-set per relation.
 
-### Phase 6 — aggregation (Datalog+)
+### Phase 6 — magic sets (goal-directed bottom-up, opt-in)
-- [ ] `count(X, Goal)` → number of distinct X satisfying Goal
+Naive bottom-up derives **all** consequences before answering. Magic sets
-- [ ] `sum(X, Goal)` → sum of X values satisfying Goal
+rewrite the program so the fixpoint only derives tuples relevant to the
-- [ ] `min(X, Goal)` / `max(X, Goal)` → min/max of X satisfying Goal
+goal — a major perf win for "what's reachable from node X" queries on
-- [ ] `group-by` semantics: `count(X, sibling(bob, X))` → count of bob's siblings
+large graphs.
-- [ ] Aggregation breaks stratification — evaluate in a separate post-fixpoint pass
+- [x] Adornments: `dl-adorn-goal goal` and `dl-adorn-lit lit bound` in
-- [ ] Tests: social network statistics, grade aggregation, inventory sums
+      `lib/datalog/magic.sx`. Per-arg `b`/`f` based on whether the arg
+      is a constant or a variable already in the bound set.
+- [x] Magic transformation: `dl-magic-rewrite rules query-rel adn args`
+      generates `{:rules <rewritten-rules> :seed <magic-seed>}`. Each
+      original rule is gated with a `magic_<rel>^<adn>(bound)` filter,
+      and propagation rules are emitted for each positive non-builtin
+      body literal. Worklist over `(rel, adn)` pairs starts from the
+      query and stops when no new pairs appear. EDB facts pass through
+      unchanged.
+- [x] Sideways information passing strategy (SIPS): left-to-right
+      `dl-rule-sips rule head-adornment` walks body literals tracking
+      the bound set, returning `({:lit :adornment} ...)`. Recognises
+      `is`/aggregate result-vars as new binders; comparisons and
+      negation pass through with computed adornments. (Pluggable
+      strategies are future work.)
+- [x] `dl-set-strategy! db strategy` hook + `dl-get-strategy db`. Default
+      `:semi-naive`. `:magic` accepted but the transformation itself is
+      deferred — saturator currently falls back to semi-naive. Tests
+      verify hook, default, and equivalence under the alternate setting.
+- [x] Equivalence test: rewritten ancestor program over the same EDB
+      derives the same number of `ancestor` tuples and returns the
+      same query answers as the unrewritten program (chain-3 case).
+- [x] `dl-magic-query db query-goal` — top-level driver. Builds a
+      fresh internal db with the caller's EDB facts, the magic seed,
+      and the rewritten rules; saturates and queries. Caller's db is
+      untouched. Equivalent to `dl-query` for fully-stratifiable
+      programs (sole motivation is a perf alternative on goal-shaped
+      queries against large recursive relations).
+- [ ] Perf test: 10k-node reachability with magic vs semi-naive.
+      Left to a future iteration — would need a benchmarking harness
+      for large graphs and the conformance budget can't afford it.
 
-### Phase 7 — SX embedding API
+### Phase 7 — stratified negation
-- [ ] `(dl-program facts rules)` → database from SX data directly (no parsing required)
+- [x] Dependency graph: `dl-build-dep-graph db` returns `{head -> ({:rel
+      :neg} ...)}`. Built-ins drop out (they're not relations).
+- [x] Reachability via Floyd-Warshall in `dl-build-reach`; cycles
+      detected by `reach[A][B] && reach[B][A]`. Programs are
+      non-stratifiable iff any negative dependency falls inside an SCC.
+      `dl-check-stratifiable` returns nil on success or a clear message.
+- [x] `dl-compute-strata` propagates stratum numbers iteratively:
+      `stratum(R) = max over deps of (stratum(dep) + (1 if negated else 0))`.
+- [x] Saturator refactor: `dl-saturate-rules! db rules` is the semi-
+      naive worker; `dl-saturate! db` rejects non-stratifiable programs,
+      groups rules by head's stratum, and runs the worker on each
+      stratum in increasing order.
+- [x] `not(P)` in body: `dl-match-negation` walks the inner literal
+      under the current subst and uses `dl-match-positive` — succeeds
+      iff zero matches. Order-aware safety in `dl-rule-check-safety`
+      (already present from Phase 4) requires negation vars to be
+      bound by an earlier positive literal.
+- [x] Tests in `lib/datalog/tests/negation.sx` (10): EDB and IDB
+      negation, two-step strata, multi-level strata, with-arithmetic,
+      empty-result and always-fail cases, non-stratifiability
+      rejection, and a negation safety violation.
+
+### Phase 8 — aggregation (Datalog+)
+- [x] `(count R V Goal)`, `(sum R V Goal)`, `(min R V Goal)`,
+      `(max R V Goal)`, `(findall L V Goal)` — first arg is the result
+      variable, second is the aggregated variable, third is the goal
+      literal. `findall` returns the distinct-value list itself; the
+      others reduce. Live in `lib/datalog/aggregates.sx`.
+- [x] `dl-eval-aggregate`: runs `dl-find-bindings` on the goal under the
+      current subst (which provides outer-context bindings), collects
+      distinct values of the aggregated var, applies the aggregate.
+      `count`/`sum` produce 0 when no matches; `min`/`max` produce no
+      binding (rule fails) when empty.
+- [x] Group-by emerges naturally: outer-context vars in the goal are
+      substituted from the current subst, so `popular(P) :- post(P),
+      count(N, U, liked(U, P)), >=(N, 3).` correctly counts per-post.
+- [x] Stratification: `dl-aggregate-dep-edge` returns a negation-like
+      edge so the aggregate's goal relation is fully derived before the
+      aggregate fires. Non-monotonicity respected.
+- [x] Safety: aggregate body lit binds the result var; goal-internal
+      vars are existentially quantified and don't need outer binding.
+- [x] Tests in `lib/datalog/tests/aggregates.sx` (10): count siblings,
+      sum prices, min/max scores, count over derived relation,
+      empty-input cases for each operator, popularity threshold with
+      group-by, distinct-counted-once.
+
+### Phase 9 — SX embedding API
+- [x] `(dl-program-data facts rules)` builds a db from SX data —
+      `facts` is a list of literals, `rules` is a list of either
+      dicts `{:head … :body …}` or lists `(<head…> <- <body…>)`.
+      Variables are SX symbols whose first char is uppercase or `_`,
+      matching the parser's convention.
       ```
-      (dl-program
+      (dl-program-data
-        '((parent tom bob) (parent tom liz) (parent bob ann))
+        '((parent tom bob) (parent bob ann))
-        '((ancestor X Z :- (parent X Y) (ancestor Y Z))
+        '((ancestor X Y <- (parent X Y))
-          (ancestor X Y :- (parent X Y))))
+          (ancestor X Z <- (parent X Y) (ancestor Y Z))))
       ```
-- [ ] `(dl-query db '(ancestor tom ?X))` → `((ann) (bob) (liz) (pat))`
+- [x] `(dl-rule head body)` constructor for the dict form.
-- [ ] `(dl-assert! db '(parent ann pat))` → incremental fact addition + re-derive
+- [x] `(dl-query db '(ancestor tom X))` already worked — same query API
-- [ ] `(dl-retract! db '(parent tom bob))` → fact removal + re-derive from scratch
+      consumes the SX-data goal. Now also accepts a *list* of body
-- [ ] Integration demo: federation graph query — `(ancestor actor1 actor2)` over
+      literals for conjunctive queries:
-      rose-ash ActivityPub follow relationships
+      `(dl-query db '((p X) (q X)))`,
+      `(dl-query db (list '(n X) '(> X 2)))`. Auto-dispatched via
+      `dl-query-coerce` on first-element shape.
+- [x] `(dl-assert! db '(parent ann pat))` → adds the fact and re-saturates.
+- [x] `(dl-retract! db '(parent bob ann))` → drops matching tuples from
+      the EDB list, wipes every relation that has a rule (those are IDB),
+      and re-saturates from the surviving EDB.
+- [x] Tests in `lib/datalog/tests/api.sx` (9): closure via data API,
+      dict-rule form, dl-rule constructor, dl-assert! incremental,
+      dl-retract! removes derived, cyclic-graph reach via data,
+      assert into empty db, fact-style rule (no arrow), coerce dict.
+- [x] Integration demo: federation graph query — `(reachable A B)` /
+      `(mutual A B)` / `(foaf A C)` over `(follows ACTOR-A ACTOR-B)` in
+      `lib/datalog/demo.sx`. Tests in `lib/datalog/tests/demo.sx`.
+      Wiring this to actual rose-ash ActivityPub data is Phase 10
+      service work and is out of scope for this loop.
 
-### Phase 8 — Datalog as a query language for rose-ash
+### Phase 10 — Datalog as a query language for rose-ash
-- [ ] Schema: map SQLAlchemy model relationships to Datalog EDB facts
+- [x] Schema sketches in `lib/datalog/demo.sx`:
-      (e.g. `(follows user1 user2)`, `(authored user post)`, `(tagged post tag)`)
+      - **Federation**: `(follows A B)` → `(mutual A B)`, `(reachable A B)`,
-- [ ] Loader: `dl-load-from-db!` — query PostgreSQL, populate Datalog EDB
+        `(foaf A C)` (friend-of-a-friend, distinct).
-- [ ] Query examples:
+      - **Content**: `(authored A P)`, `(liked U P)`, `(tagged P T)` →
-      - `?- ancestor(me, X), authored(X, Post), tagged(Post, cooking).`
+        `(post-likes P N)` via aggregation, `(popular P)` for likes ≥ 3,
-        → posts about cooking by people I follow (transitively)
+        `(interesting Me P)` joining follows + authored + popular.
-      - `?- popular(Post) :- tagged(Post, T), count(L, (liked(L, Post))) >= 10.`
+      - **Permissions**: `(member A G)`, `(subgroup C P)`, `(allowed G R)`
-        → posts with 10+ likes
+        → `(in-group A G)` over transitive subgroups, `(can-access A R)`.
-- [ ] Expose as a rose-ash service endpoint: `POST /internal/datalog` with program + query
+      - **Cooking-posts** (the canonical example): `(reach Me Them)` over
+        the follow graph, then `(cooking-post-by-network Me P)` joining
+        reach + authored + `(tagged P cooking)`.
+- [ ] Loader `dl-load-from-db!` — out of scope for this loop
+      (would need to edit `shared/services/` outside `lib/datalog/`).
+      Programs in `demo.sx` already document the EDB shape expected
+      from such a loader. `dl-program-data` consumes the same shape.
+- [x] Query examples covered by `lib/datalog/tests/demo.sx` (10):
+      mutuals, transitive reach, FOAF, popular posts, interesting feed,
+      post likes count, direct/subgroup/transitive group access, no
+      access without grant.
+- [ ] Service endpoint `POST /internal/datalog` — out of scope as above.
+      Once exposed, server-side handler would be `dl-program-data` +
+      `dl-query`, returning JSON-encoded substitutions.
 
 ## Blockers
 
-_(none yet)_
+- **Saturation perf**: three rounds done.
+  - hash-set membership in `dl-add-fact!` (Phase 5b)
+  - indexed iteration in `dl-find-bindings` (Phase 5c)
+  - first-arg index per relation (Phase 5e) — when a body literal's
+    first arg walks to a non-variable, dl-match-positive looks up
+    by `(str arg)` instead of scanning the full relation.
+  chain-25 saturation drops from ~33s to ~18s real (10s user).
+  chain-50 still long (~120s+) due to dict-copy overhead in
+  unification subst threading. Future: per-rule "compiled" body
+  with pre-resolved var positions, slot-based subst representation
+  to avoid `assoc` per binding.
 
 ## Progress log
 
 _Newest first._
 
-_(awaiting phase 1)_
+- 2026-05-11 — `dl-set-strategy!` accepted arbitrary keyword values
+  silently. Typos like `:semi_naive` or `:semiNaive` were stored
+  uninspected; the saturator then used the default and the user
+  never learned their setting was a typo. Validator added: strategy
+  must be one of `:semi-naive`, `:naive`, `:magic`. 1 regression test;
+  276/276.
+
+- 2026-05-11 — Anonymous-variable renamer collided with user-written
+  `_anon<N>` symbols. The renamer started counter at 0 and produced
+  `_anon1, _anon2, ...` unconditionally; if the user wrote
+  `q(_anon1) :- p(_anon1, _).` the `_` got renamed to `_anon1` too,
+  collapsing the two positions of `p` to a single var and returning
+  the empty result instead of `{a, c}`. Fix: scan each rule (and
+  query) for the max `_anon<N>` and start the renamer past it. The
+  renamer constructor now takes a `start` arg; new helpers
+  `dl-max-anon-num` / `dl-max-anon-num-list` walk the rule tree.
+  1 regression test; 275/275.
+
+- 2026-05-11 — `dl-magic-query` could silently diverge from
+  `dl-query` when an aggregate's inner-goal relation was IDB. The
+  rewriter passes aggregate body lits through unchanged (no magic
+  propagation for them), so the inner relation was empty in the
+  magic db and the aggregate returned 0. Probe:
+  `dl-eval-magic "u(a). u(b). u(c). u(d). banned(b). banned(d).
+                  active(X) :- u(X), not(banned(X)).
+                  n(N) :- count(N, X, active(X))." "?- n(N)."`
+  returned `N=0` instead of `N=2`. Fix: `dl-magic-query` now
+  pre-saturates the source db before copying facts into the magic
+  db. This guarantees equivalence with `dl-query` for every
+  stratified program; the magic benefit comes from goal-directed
+  re-derivation of the query relation under the seed (which still
+  matters for large recursive joins). The existing test suite's
+  aggregate cases happened to dodge this because the inner goals
+  were all EDB. 1 new regression test; 274/274.
+
+- 2026-05-11 — Anonymous `_` in a negated literal was incorrectly
+  flagged by the safety check. The canonical idiom
+  `orphan(X) :- person(X), not(parent(X, _))` was rejected with
+  "negation refers to unbound variable(s) (\"_anon1\")" because the
+  parser renames each `_` to a fresh `_anon*` symbol and the negation
+  safety walk demanded all vars in the negated lit be bound by an
+  earlier positive body literal. Anonymous vars in negation are
+  existentially quantified — they shouldn't need outer binding.
+  Added `dl-non-anon-vars` filter; `dl-process-neg!` now strips
+  `_anon*` names from `needed` before the binding check. 2 new
+  regression tests; 273/273.
+
+- 2026-05-11 — Compound terms in fact-arg / rule-head positions were
+  silently stored as unreduced expressions. `p(+(1, 2)).` resulted
+  in a tuple `(p (+ 1 2))` (dl-ground? sees no free variables, so it
+  passed). `double(*(X, 2)) :- n(X).` saturated to `double((* 3 2))`
+  rather than `double(6)`. Datalog has no function symbols in arg
+  positions — `dl-add-fact!` and `dl-add-rule!` now reject compound
+  args via a new `dl-simple-term?` (number / string / symbol).
+  Compounds remain legal in body literals where they encode `is` /
+  arithmetic / aggregate sub-goals. 2 new regression tests; 271/271.
+
+- 2026-05-11 — Quoted atoms with uppercase-or-underscore-leading
+  names were misclassified as variables. `p('Hello World').` ran
+  through the tokenizer's `"atom"` branch and through the parser's
+  `string->symbol`, producing a symbol named "Hello World". dl-var?
+  checks the first character — "H" is uppercase, so the fact was
+  rejected as non-ground. Fix: tokenizer emits `"string"` for any
+  `'...'` quoted form, so quoted atoms become opaque string constants
+  (matching how Datalog idiomatically treats them — the alternative
+  was a per-symbol "quoted" marker which would have rippled through
+  unification and dl-var?). Updated the existing tokenize test and
+  added one for `'Hello'`; also added a parse-level regression. 269/269.
+
+- 2026-05-11 — Type-mixed comparisons were silently inconsistent:
+  `<(X, 5)` with `X` bound to a string returned `()` (no result, no
+  error), while `X` bound to a symbol raised "Expected number, got
+  symbol". Both should fail loudly. Added `dl-compare-typeok?` —
+  `<`, `<=`, `>`, `>=` now require both operands to share a primitive
+  type (both numbers or both strings) and raise otherwise. `!=` is
+  exempted since it's a polymorphic inequality test built on
+  `dl-tuple-equal?`. 2 new regression tests; 267/267.
+
+- 2026-05-11 — Body literal shape validation in
+  `dl-rule-check-safety`: a dict that isn't `{:neg ...}` (e.g. typo'd
+  `{:negs ...}`) used to silently fall through every dispatch clause,
+  contributing zero bound vars; the user would then see a confusing
+  "head var X unbound" error pointing at the head, not the malformed
+  body. Same for body lits that are bare numbers / strings / symbols.
+  Both shapes now raise a clear error naming the offending lit. 1 new
+  regression test; 265/265.
+
+- 2026-05-11 — Division by zero in `is` silently produced IEEE
+  infinity instead of raising. `is(R, /(X, 0))` returned `R = inf`,
+  which then flowed through comparisons and aggregations to produce
+  nonsense results. `dl-eval-arith` now raises with a clear
+  "division by zero in <expr>" message. 1 new test; 264/264.
+
+- 2026-05-11 — Aggregate variable validation: `count(N, Y, p(X))`
+  silently returned `N = 1` because `Y` was never bound in `p(X)` —
+  every match contributed the same unbound symbol, which dl-val-member?
+  deduped to a single entry. Similarly `sum(S, Y, p(X))` raised a
+  confusing "expected number" error from the underlying `+`. Added
+  a third validator in `dl-eval-aggregate`: the agg-var must appear
+  in the goal literal. Error names the variable and the goal and
+  explains the consequence. 1 new test; 263/263.
+
+- 2026-05-11 — `dl-retract!` was silently destroying EDB facts in
+  "mixed" relations (those with BOTH user-asserted facts AND a rule
+  defining the same head). The retract pass wiped every rule-head
+  relation wholesale and then re-saturated — but the saturator only
+  re-derives the IDB portion, so explicit EDB facts vanished even
+  for a no-op retract of a non-existent tuple. Probe:
+  `(let ((db (dl-program "p(a). p(b). p(X) :- q(X). q(c).")))
+     (dl-retract! db (quote (p z))) (dl-query db (quote (p X))))`
+  went from `{a,b,c}` to just `{c}`.
+  Fix: tracked `:edb-keys` provenance in the db. `dl-add-fact!` (public
+  API) marks the tuple as EDB; saturator calls new internal
+  `dl-add-derived!` which doesn't mark it. `dl-retract!` now strips
+  only the IDB-derived portion of rule-head relations and preserves
+  EDB-marked tuples through the re-saturate pass. 2 new regression
+  tests; 262/262.
+
+- 2026-05-11 — Eval-semantics bug-hunt: nested `not(not(P))` was
+  silently misinterpreted. Outer-level `not(...)` is parsed as
+  negation, but the inner `not(banned(X))` was parsed as a regular
+  positive literal naming a relation called `not`. Since no `not`
+  relation existed, the inner match was empty and the outer
+  negation succeeded vacuously, making `vip(X) :- u(X), not(not(banned(X))).`
+  equivalent to `vip(X) :- u(X).` (a silent double-negation = identity
+  fallacy). Fix in `dl-rule-check-safety`: both the positive-literal
+  branch and `dl-process-neg!` now flag any body literal whose head
+  is in `dl-reserved-rel-names`. Error message names the relation and
+  points the user at intermediate-relation stratified negation. 1 new
+  regression test; 260/260.
+
+- 2026-05-10 — Bug-hunt round on parser/safety surfaced 7 real
+  bugs, each fixed with regression tests:
+  - Reserved relation names (`not`, `count`, `<`, `is`, ...) were
+    accepted as rule/fact heads — would silently shadow built-ins.
+  - Negative number literals (`n(-1).`) failed to parse — users
+    had to express them as `(- 0 1)` or via `is`.
+  - Unterminated block comment `/* ...` silently consumed the
+    rest of the input. Now raises with the position.
+  - Same silent-consume bug in unterminated string / quoted-atom.
+  - Empty-list rule head and non-list rule body weren't validated;
+    they'd crash later in `rest`. dl-add-rule! now checks shape.
+  - dl-magic-query with non-list / non-dict goal crashed cryptically.
+  - Tokenizer silently swallowed unrecognised characters (`?`, `!`,
+    `#`, `@`, etc.) — typos produced confusing downstream errors.
+
+- 2026-05-08 — Phase 6 driver: `dl-magic-query db query-goal`.
+  Builds a fresh internal db from the caller's EDB + magic seed +
+  rewritten rules, saturates, queries, returns substitutions —
+  caller's db is untouched. Equivalent to `dl-query` for any
+  fully-stratifiable program; sole motivation is a perf alternative
+  on goal-shaped queries against large recursive relations.
+  2 new tests cover equivalence and non-mutation.
+
+- 2026-05-08 — Phase 6 magic-sets rewriter. `dl-magic-rewrite rules
+  query-rel adn args` returns `{:rules <rewritten> :seed <seed-fact>}`.
+  Worklist over `(rel, adn)` pairs starts from the query, gates each
+  original rule with a `magic_<rel>^<adn>(bound)` filter, and emits
+  propagation rules for each positive non-builtin body literal so
+  that magic spreads to body relations. EDB facts pass through.
+  3 new tests cover seed structure, equivalence on chain-3 by
+  ancestor-relation tuple count, and same-query-answers under
+  the rewritten program. The plumbing for a `dl-saturate-magic!`
+  driver and large-graph perf benchmarks is still future work.
+
+- 2026-05-08 — Phase 6 building blocks for the magic-sets
+  transformation: `dl-magic-rel-name`, `dl-magic-lit`,
+  `dl-bound-args`. The rewriter that generates magic seed and
+  propagation rules is still future work; with these primitives
+  in place it's a straightforward worklist algorithm. 4 new tests.
+
+- 2026-05-08 — Phase 6 adornments + SIPS in
+  `lib/datalog/magic.sx`. Inspection helpers — `dl-adorn-goal` and
+  `dl-adorn-lit` compute per-arg `b`/`f` patterns under a bound
+  set; `dl-rule-sips rule head-adornment` walks body literals
+  left-to-right propagating the bound set, recognising `is` and
+  aggregate result-vars as new binders. Lays groundwork for a
+  later magic-sets transformation. 10 new tests cover pure
+  adornment, SIPS over a chain rule, head-fully-bound rules,
+  comparisons, and `is`. Saturator does not yet consume these.
+
+- 2026-05-08 — Comprehensive integration test in api suite: a
+  single program exercising recursion (`reach` transitive closure)
+  + stratified negation (`safe X Y :- reach X Y, not banned Y`) +
+  aggregation (`reach_count` via count) + comparison (`>= N 2`)
+  composed end-to-end via `dl-eval source query-source`. Confirms
+  the full pipeline (parser → safety → stratifier → semi-naive +
+  aggregate post-pass → query) on a non-trivial program.
+
+- 2026-05-08 — Bug fix: aggregates work as top-level query goals.
+  `dl-match-lit` (the naive matcher used by `dl-find-bindings`) was
+  missing the `dl-aggregate?` dispatch — it was only present in
+  `dl-fbs-aux` (semi-naive). Symptom: `(dl-query db '(count N X (p X)))`
+  silently returned `()`. Also updated `dl-query-user-vars` to project
+  only the result var (first arg) of an aggregate goal — the
+  aggregated var and inner-goal vars are existentials and should not
+  appear in the projected substitution. 2 new aggregate tests cover
+  the regression.
+
+- 2026-05-08 — Convenience: `dl-eval source query-source`. Parses
+  both strings, builds a db, saturates, runs the query, returns
+  the substitution list. Single-call user-friendly entry. 2 new
+  api tests cover ancestor and multi-goal queries.
+
+- 2026-05-08 — Phase 6 stub: `dl-set-strategy! db strategy` and
+  `dl-get-strategy db` user-facing hooks. Default `:semi-naive`;
+  `:magic` is accepted but the actual transformation is deferred,
+  so saturation still uses semi-naive. Lets us tick the
+  "Optional pass — guarded behind dl-set-strategy!" Phase 6 box.
+  3 new eval tests.
+
+- 2026-05-08 — Demo: weighted-DAG shortest path. `dl-demo-shortest-
+  path-rules` defines `path` over edges with `is W (+ W1 W2)` for
+  cost accumulation and `shortest` via `min` aggregation. 3 demo
+  tests cover direct/multi-hop choice, multi-hop wins on cheaper
+  route, and unreachable-empty. Added `dl-summary db` inspection
+  helper returning `{<rel>: count}` (4 eval tests).
+
+- 2026-05-08 — Phase 5e perf: first-arg index per relation. db gains
+  `:facts-index {<rel>: {<first-arg-key>: tuples}}` mirroring the
+  existing `:facts-keys` membership index. `dl-add-fact!` populates
+  it; `dl-match-positive` walks the body literal's first arg under
+  the current subst — if it's bound to a non-var, look up by
+  `(str arg)` and iterate only the matching subset. chain-25
+  saturation 33s → 18s real (~2x). chain-50 still slow (~120s+)
+  but tractable; next bottleneck is subst dict copies during
+  unification. Differential test bumped to chain-12, semi-only
+  count to chain-25.
+
+- 2026-05-08 — Demo: tag co-occurrence. `(cotagged P T1 T2)` — post
+  has both T1 and T2 with T1 != T2 — and `(tag-pair-count T1 T2 N)`
+  counting posts per distinct tag pair. Demonstrates count
+  aggregation grouped by outer-context vars. 2 new demo tests.
+
+- 2026-05-08 — `dl-query` accepts a list of body literals for
+  conjunctive queries, in addition to a single positive literal.
+  `dl-query-coerce` dispatches based on the first element's shape:
+  positive lit (head is a symbol) or `:neg` dict → wrap as singleton;
+  list of lits → use as-is. `dl-query-user-vars` collects the union
+  of vars across all goals (deduped, `_` filtered) for projection.
+  2 new api tests: multi-goal AND, and conjunction with comparison.
+
+- 2026-05-08 — Bug fix: `dl-check-stratifiable` now rejects recursion
+  through aggregation (e.g., `q(N) :- count(N, X, q(X))`). The
+  stratifier was already adding negation-like edges for aggregates,
+  but the cycle scan only looked at explicit `:neg` literals. Added
+  the matching aggregate branch to the body iteration. Also adds
+  doc-only `lib/datalog/datalog.sx` with the public-API surface
+  (since `load` is an epoch command and can't recurse from within an
+  `.sx` file). 3 new aggregate tests cover recursion-rejection,
+  negation-and-aggregation coexistence, and min-over-empty-derived.
+
+- 2026-05-08 — Phase 10 demo + canonical query. Added the "cooking
+  posts by people I follow (transitively)" example from the plan:
+  `dl-demo-cooking-rules` defines `reach` over the follow graph
+  (recursive transitive closure) and `cooking-post-by-network` that
+  joins reach with `authored` and `(tagged P cooking)`. 3 demo
+  tests cover transitive network, direct-only follow, and
+  empty-network cases.
+
+- 2026-05-08 — Phase 8 extension: `findall L V Goal` aggregate. Bind
+  L to the list of distinct V values for which Goal holds (or the
+  empty list when no matches). Implemented as a one-line case in
+  `dl-do-aggregate`. 3 new tests: EDB, derived relation, empty.
+  Useful for "give me all the X such that …" queries without
+  scalar reduction.
+
+- 2026-05-08 — Phase 5d semantic fix: anonymous `_` variables are
+  renamed per occurrence at `dl-add-rule!` and `dl-query` time so
+  `(p X _) (p _ Y)` no longer unifies the two `_`s. New helpers
+  `dl-rename-anon-term`, `dl-rename-anon-lit`, `dl-make-anon-renamer`,
+  `dl-rename-anon-rule` in db.sx; eval.sx's dl-query renames the goal
+  before search and projects only user-named vars (`_` is filtered
+  out of the projection list). The "underscore in head" test now
+  correctly rejects `(p X _) :- q(X).` — after renaming, the head's
+  fresh anon var has no body binder. Two new eval tests verify
+  rule-level and goal-level independence. 155/155 expected.
+
+- 2026-05-08 — Phase 5c perf: indexed `dl-find-bindings`. Replaced
+  the recursive `(rest lits)` walk with `dl-fb-aux lits db subst i n`
+  using `nth lits i`. Eliminates O(N²) list-copy per body of length
+  N. chain-15 saturation 25s → 16s; chain-25 finishes in 33s real
+  (vs. timeout previously). Bumped semi_naive tests: differential
+  on chain-10, semi-only count on chain-15 (was chain-5/chain-5).
+  153/153.
+
+- 2026-05-08 — Phase 10 syntactic demo. New `lib/datalog/demo.sx`
+  with three programs over rose-ash-shaped data: federation
+  (`mutual`, `reachable`, `foaf`), content recommendation
+  (`post-likes` via count aggregation, `popular`, `interesting`),
+  and role-based permissions (`in-group` over transitive subgroups,
+  `can-access`). 10 demo tests pass against synthetic EDB tuples.
+  Postgres loader and `/internal/datalog` HTTP endpoint remain
+  out of scope for this loop (they need service-tree edits beyond
+  `lib/datalog/**`). Conformance now 153/153.
+
+- 2026-05-08 — Phase 5b perf: hash-set membership in `dl-add-fact!`.
+  db gains a parallel `:facts-keys {<rel>: {<tuple-string>: true}}`
+  index alongside `:facts`. `dl-tuple-key` derives a stable string
+  key via `(str lit)` — `(p 30)` and `(p 30.0)` collide correctly
+  because SX prints them identically. Insertion is O(1) instead of
+  O(n). chain-7 saturation drops from ~12s to ~6s; chain-15 from
+  ~50s to ~25s under shared CPU. Larger chains are still slow due
+  to body-join overhead in dl-find-bindings (Blocker updated).
+  `dl-retract!` updated to keep both indices consistent. 143/143.
+
+- 2026-05-08 — Phase 9 done. New `lib/datalog/api.sx` exposes a
+  parser-free embedding: `dl-program-data facts rules` accepts SX
+  data lists, with rules in either dict form or list form using
+  `<-` as the rule arrow (since SX parses `:-` as a keyword).
+  `dl-rule head body` constructs the dict. `dl-assert! db lit` adds
+  a fact and re-saturates; `dl-retract! db lit` drops the fact from
+  EDB, wipes all rule-headed (IDB) relations, and re-saturates from
+  scratch — the simplest correct semantics until provenance tracking
+  arrives in a later phase. 9 API tests; conformance now 143/143.
+
+- 2026-05-08 — Phase 8 done. New `lib/datalog/aggregates.sx` (~110
+  LOC): count / sum / min / max. Each is a body literal of shape
+  `(op R V Goal)` — `dl-eval-aggregate` runs `dl-find-bindings` on
+  the goal under the outer subst (so outer vars in the goal get
+  substituted, giving group-by-style aggregation), collects the
+  distinct values of `V`, and binds `R`. Empty input: count/sum
+  return 0; min/max produce no binding (rule fails). Stratifier
+  extended via `dl-aggregate-dep-edge` so the aggregate's goal
+  relation is fully derived before the aggregate fires. Safety check
+  treats goal-internal vars as existentials (no outer binding
+  required); only the result var becomes bound. Conformance now
+  134 / 134.
+
+- 2026-05-08 — Phase 7 done (Phase 6 magic sets deferred — opt-in,
+  semi-naive default suffices for current test suite). New
+  `lib/datalog/strata.sx` (~290 LOC): dep graph build, Floyd-Warshall
+  reachability, SCC-via-mutual-reachability for non-stratifiability
+  detection, iterative stratum computation, rule grouping by head
+  stratum. eval.sx split: `dl-saturate-rules!` is the per-rule-set
+  semi-naive worker, `dl-saturate!` is now the stratified driver
+  (errors out on non-stratifiable programs). `dl-match-negation` in
+  eval.sx: succeeds iff inner positive match is empty. Stratum-keyed
+  dicts use `(str s)` since SX dicts only accept string/keyword keys.
+  10 negation tests cover EDB/IDB negation, multi-level strata,
+  non-stratifiability rejection, and a negation safety violation.
+
+- 2026-05-08 — Phase 5 done. `lib/datalog/eval.sx` rewritten to
+  semi-naive default. `dl-saturate!` tracks a per-relation delta and
+  on each iteration walks every positive body position substituting
+  delta for that one literal — joining the rest against the full DB
+  snapshot. `dl-saturate-naive!` retained as the reference. Rules
+  with no positive body literal (e.g. `(p X) :- (= X 5).`) fall back
+  to a naive one-shot via `dl-collect-rule-candidates`. 8 tests
+  differentially compare the two saturators using per-relation tuple
+  counts (cheap). Chain-5 differential exercises multi-iteration
+  recursive saturation. Larger chains made conformance.sh time out
+  due to O(n) `dl-tuple-member?` × CPU sharing with other loop
+  agents — added a Blocker to swap to a hash-set for membership.
+  Also tightened `dl-tuple-member?` to use indexed iteration instead
+  of recursive `rest` (was creating a fresh list per step).
+
+- 2026-05-07 — Phase 4 done. `lib/datalog/builtins.sx` (~280 LOC) adds
+  `(< X Y)`, `(<= X Y)`, `(> X Y)`, `(>= X Y)`, `(= X Y)`, `(!= X Y)`,
+  and `(is X expr)` with `+ - * /`. `dl-eval-builtin` dispatches;
+  `dl-eval-arith` recursively evaluates nested compounds. Safety
+  check is now order-aware — it walks body literals left-to-right
+  tracking the bound set, requires comparison/`is` inputs to be
+  already bound, and special-cases `=` (binds the var-side; both
+  sides must include at least one bound to bind the other). Phase 3's
+  simple safety check stays in db.sx as a forward-reference fallback;
+  builtins.sx redefines `dl-rule-check-safety` to the comprehensive
+  version. eval.sx's `dl-match-lit` now dispatches built-ins through
+  `dl-eval-builtin`. 19 builtins tests; conformance 106 / 106.
+
+- 2026-05-07 — Phase 3 done. `lib/datalog/db.sx` (~250 LOC) holds facts
+  indexed by relation name plus the rules list, with `dl-add-fact!` /
+  `dl-add-rule!` (rejects non-ground facts and unsafe rules);
+  `lib/datalog/eval.sx` (~150 LOC) implements the naive bottom-up
+  fixpoint via `dl-find-bindings`/`dl-match-positive`/`dl-saturate!`
+  and `dl-query` (deduped projected substitutions). Safety analysis
+  rejects unsafe head vars at load time. Negation and arithmetic
+  built-ins raise clean errors (lifted in later phases). 15 eval
+  tests cover transitive closure, sibling, same-generation, cyclic
+  graph reach, and six safety violations. Conformance 87 / 87.
+
+- 2026-05-07 — Phase 2 done. `lib/datalog/unify.sx` (~140 LOC):
+  `dl-var?` (case + underscore), `dl-walk`, `dl-bind`, `dl-unify` (returns
+  extended dict subst or `nil`), `dl-apply-subst`, `dl-ground?`, `dl-vars-of`.
+  Substitutions are immutable dicts; `assoc` builds extended copies. 28
+  unify tests; conformance now 72 / 72.
+
+- 2026-05-07 — Phase 1 done. `lib/datalog/tokenizer.sx` (~190 LOC) emits
+  `{:type :value :pos}` tokens; `lib/datalog/parser.sx` (~150 LOC) produces
+  `{:head … :body …}` / `{:query …}` clauses, with nested compounds
+  permitted for arithmetic and `not(...)` desugared to `{:neg …}`. 44 / 44
+  via `bash lib/datalog/conformance.sh` (26 tokenize + 18 parse). Local
+  helpers namespace-prefixed (`dl-emit!`, `dl-peek`) after a host-primitive
+  shadow clash. Test harness uses a custom `dl-deep-equal?` that handles
+  out-of-order dict keys and number repr (`equal?` fails on dict key order
+  and on `30` vs `30.0`).

plans/lib-guest.md

@@ -155,11 +155,11 @@ Extract from `haskell/infer.sx`. Algorithm W or J, generalisation, instantiation
 | 1 — conformance.sx (prolog + haskell) | [done] | 58dcff26 | Prolog 590/590 (matches baseline). Haskell 156/156 — old script was broken (0/18 was an artefact of a never-matching grep), driver reveals true counts; baseline updated. |
 | 2 — prefix.sx (common-lisp + lua) | [partial — pending lua] | 2ef773a3 | common-lisp/runtime.sx ported (47 aliases collapsed into 13 prefix-rename calls); 518/518 vs 309/309 baseline (improvement, no regression). lua/runtime.sx has no pure same-name aliases — every lua- definition wraps custom logic; second consumer pending. |
 | 3 — lex.sx (lua + tcl) | [done] | 559b0df9 | lex.sx exports nil-safe char-class predicates + token record. lua/tokenizer.sx (7 preds) and tcl/tokenizer.sx (5 preds) collapsed into prefix-rename calls. lua 185/185, tcl 342/342, tcl-conf 3/4 — all = baseline. |
-| 4 — pratt.sx (lua + prolog) | [in-progress] | — | — |
+| 4 — pratt.sx (lua + prolog) | [done] | da27958d | Extracted operator-table format + lookup only — climbing loops stay per-language because lua and prolog use opposite prec conventions. lua/parser.sx: 18-clause cond → 15-entry table. prolog/parser.sx: pl-op-find deleted, pl-op-lookup wraps pratt-op-lookup. lua 185/185, prolog 590/590 — both = baseline. |
-| 5 — ast.sx (lua + prolog) | [ ] | — | — |
+| 5 — ast.sx (lua + prolog) | [partial — pending real consumers] | a774cd26 | Kit + 33 self-tests shipped (10 canonical kinds, predicates, accessors). Step is "Optional" per brief; lua/prolog parsers untouched (185/185 + 590/590). Datalog-on-sx will be the natural first real consumer; lua/prolog converters can land later. |
-| 6 — match.sx (haskell + prolog) | [ ] | — | — |
+| 6 — match.sx (haskell + prolog) | [partial — kit shipped; ports deferred] | 863e9d93 | Pure-functional unify + match kit (canonical wire format + cfg-driven adapters) + 25 self-tests. Existing prolog/haskell engines untouched (structurally divergent — mutating-symmetric vs pure-asymmetric — would risk 746 passing tests under brief's revert-on-regression rule). Real consumer is minikraken/datalog work in flight. |
-| 7 — layout.sx (haskell + synthetic) | [ ] | — | — |
+| 7 — layout.sx (haskell + synthetic) | [partial — haskell port deferred] | d75c61d4 | Configurable kit (haskell-style keyword-opens + python-style trailing-`:`-opens) + 6 self-tests covering both flavours. Synthetic Python-ish fixture passes; haskell/layout.sx untouched (kit not yet a drop-in for Haskell 98 Note 5 etc.; haskell still 156/156 baseline). |
-| 8 — hm.sx (haskell + TBD) | [ ] | — | — |
+| 8 — hm.sx (haskell + TBD) | [partial — algebra shipped; assembly deferred] | ab2c40c1 | HM foundations: types/schemes/ftv/apply/compose/generalize/instantiate/fresh-tv on top of match.sx unify, plus literal inference rule. 24/24 self-tests. Algorithm W lambda/app/let assembly deferred to host code — paired sequencing per brief: lib/ocaml/types.sx (OCaml-on-SX Phase 5) + haskell/infer.sx port. Haskell still 156/156 baseline. |
 
 ---
 

plans/tcl-sx-completion.md

@@ -132,6 +132,165 @@ architectural improvement worth doing when the moment is right.
 
 ---
 
+## Phase 5 — Channel I/O (random access + non-blocking) ✓
+
+Real Tcl channel commands replacing the previous stubs. SX gained 11 channel
+primitives in `sx_primitives.ml` (using `Unix.openfile` + `Unix.read`/`write`/
+`lseek`/`set_nonblock`). Tcl `open`/`close`/`read`/`gets`/`puts`/`seek`/`tell`/
+`eof`/`flush`/`fconfigure` now wrap them.
+
+| Status | Work | Unlocks in Tcl |
+|---|---|---|
+| [x] | `channel-open`, `channel-close` | `open` returns "fileN", `close` actually closes |
+| [x] | `channel-read`, `channel-read-line`, `channel-write` | `read`/`gets`/`puts` to/from real files |
+| [x] | `channel-seek`, `channel-tell` | random access — `seek $c offset start\|current\|end`, `tell` |
+| [x] | `channel-eof?`, `channel-flush` | proper EOF detection, no-op flush |
+| [x] | `channel-blocking?`, `channel-set-blocking!` | `fconfigure $c -blocking 0\|1` |
+
+Modes supported: `r`, `w`, `a`, `r+`, `w+`, `a+`. Whence: `start`, `current`, `end`.
+
+`puts` now detects channel argument (string starting with "file") and dispatches
+to `channel-write`; otherwise writes to `interp :output` as before.
+
+**Total: ~half day. 7 new idiom tests covering write+read, gets-loop, seek/tell,
+eof-after-read, append mode, seek-to-end, fconfigure-blocking.**
+
+---
+
+## Phase 5b — Event loop: fileevent / after / vwait / update ✓
+
+Tcl event-driven I/O scoped to script-mode (vs. server-side commands). The
+mechanism rides on the existing IO suspension model: SX adds one new primitive
+`(io-select-channels read-list write-list timeout-ms)` wrapping `Unix.select`,
+and the Tcl event loop is implemented in Tcl itself (no sx_server.ml changes).
+
+| Status | Work | Unlocks in Tcl |
+|---|---|---|
+| [x] | `io-select-channels` SX primitive | Unix.select on registered channels |
+| [x] | `fileevent $chan readable\|writable script` | event handler registration; `{}` to unregister |
+| [x] | `after ms script` | one-shot timer queued in `:timers` |
+| [x] | `after ms` (no script) | sleep that drives the event loop |
+| [x] | `vwait varname` | block until var set/changed, runs handlers |
+| [x] | `update` | non-blocking event drain (poll, fire ready handlers) |
+
+Event loop: `tcl-event-step interp poll-timeout-ms` — fires expired timers,
+calls `io-select-channels` with fd list from `:fileevents`, runs ready handlers.
+`vwait` polls every 1000ms or until var changes (whichever first); `update` is
+`tcl-event-step interp 0`.
+
+State on interp: `:fileevents` (list of `(chan event script)`) and `:timers`
+(list of `(expiry-ms script)`, sorted by expiry).
+
+**Trade-off:** Scoped to script mode — `vwait` from inside a server-handled
+command would not interact with sx_server's stdin scheduler. Sufficient for ~95%
+of real-world Tcl scripts (sockets, pipes, GUI-style polling, CLI tools).
+
+**Total: ~half day. 5 new idiom tests: after-vwait-timer, after-multiple-timers-
+update, fileevent-readable-fires, fileevent-query-script, after-cancel-via-
+vwait-timing. 354/354 green.**
+
+---
+
+## Phase 5c — TCP sockets (client + server) ✓
+
+Tcl `socket` command for both connecting and listening. Reuses the channel
+registry built in Phase 5 and the event loop from Phase 5b. Server channels
+auto-fire user callbacks via fileevent on each accept.
+
+| Status | Work | Unlocks in Tcl |
+|---|---|---|
+| [x] | `socket-connect host port` SX primitive | TCP client via `Unix.socket`+`Unix.connect` |
+| [x] | `socket-server ?host? port` SX primitive | listening socket; `Unix.bind`+`Unix.listen` (backlog 8) |
+| [x] | `socket-accept server-chan` SX primitive | returns `{:channel :host :port}` |
+| [x] | Tcl `socket host port` | TCP client; returns "sockN" |
+| [x] | Tcl `socket -server cb port` | listening socket; auto-fires `cb sock host port` per accept |
+| [x] | `puts` channel detection extended | "sockN" channels also dispatch to `channel-write` |
+
+The auto-accept mechanism is a tiny internal Tcl command `_sock-do-accept`
+registered by `socket -server`. Its registered handler, fired by the event
+loop, accepts the pending client, then evaluates `cb client-chan host port`.
+
+`Unix.SO_REUSEADDR` is set on server sockets to avoid TIME_WAIT issues
+during testing. Host argument supports `localhost`, `0.0.0.0`, IPv4 literal,
+or DNS lookup via `Unix.gethostbyname`.
+
+**Total: ~half day. 4 new idiom tests: socket-server-fires-callback,
+socket-client-server-roundtrip, socket-server-peer-host, socket-multiple-
+connections. 358/358 green.**
+
+---
+
+## Phase 5d — File metadata + filesystem ops ✓
+
+Real implementations of `file isfile`/`isdir`/`readable`/`writable`/`size`/
+`mtime`/`atime`/`type` (previously stubs returning `0`/`""`) and proper
+`file delete`/`mkdir`/`copy`/`rename`.
+
+| Status | Primitive | Wraps |
+|---|---|---|
+| [x] | `file-size`, `file-mtime`, `file-stat` | `Unix.stat` |
+| [x] | `file-isfile?`, `file-isdir?` | `Unix.stat`+`st_kind` |
+| [x] | `file-readable?`, `file-writable?` | `Unix.access [R_OK\|W_OK]` |
+| [x] | `file-delete` | `Unix.unlink`/`rmdir` (tolerates ENOENT) |
+| [x] | `file-mkdir` | recursive `Unix.mkdir 0o755` |
+| [x] | `file-copy`, `file-rename` | stdlib I/O / `Sys.rename` |
+
+`file-stat` returns a dict `{:size :mtime :atime :ctime :mode :type}` with
+`:type` ∈ `file|directory|link|fifo|socket|...`. Tcl `file copy`/`rename`/
+`delete` strip leading-`-` flags so `file delete -force` works.
+
+**Total: ~half day. 10 new idiom tests covering isfile, isdir on /tmp, size,
+readable, mkdir + check, copy roundtrip, rename, mtime > 0. 368/368 green.**
+
+---
+
+## Phase 5e — clock format options + clock scan ✓
+
+Real `-format`, `-timezone`, and `-gmt` options on `clock format`, and a
+working `clock scan` for parsing date strings back to Unix seconds.
+
+| Status | Work |
+|---|---|
+| [x] | `clock-format` extended to `(t fmt tz)` with tz ∈ `utc|local` |
+| [x] | More format specifiers: `%y` (2-digit year), `%I` (12h hour), `%p` (AM/PM), `%w` (weekday num), `%%` (literal) |
+| [x] | `clock-scan` SX primitive: format-driven parser + manual `timegm` (OCaml stdlib lacks it) |
+| [x] | Tcl `clock format $secs -format $fmt -timezone $tz -gmt 0\|1` |
+| [x] | Tcl `clock scan $str -format $fmt -timezone $tz -gmt 0\|1` |
+
+Default tz for both is UTC. Format specifiers supported by scan: `%Y %y %m
+%d %e %H %I %M %S %%`. Unsupported specifiers in scan are silently skipped
+(no validation).
+
+**Total: ~half day. 5 new idiom tests: clock-format-utc, fmt-default,
+scan-roundtrip, scan-returns-int, format-percent-pct. 373/373 green.**
+
+---
+
+## Phase 5f — `socket -async` (non-blocking connect) ✓
+
+| Status | Work |
+|---|---|
+| [x] | `socket-connect-async host port` SX primitive — `Unix.set_nonblock` + `Unix.connect`, catches `EINPROGRESS` |
+| [x] | `channel-async-error chan` SX primitive — `Unix.getsockopt_error` |
+| [x] | Tcl `socket -async host port` — returns "sockN" immediately |
+| [x] | Tcl `fconfigure $chan -error` — queries async-error |
+
+Connection completes when the channel becomes writable; canonical pattern is
+`fileevent $sock writable {handler}`. Channel buffer state is set to
+`blocking=false` so subsequent reads/writes don't block.
+
+**Total: ~few hours. 3 new idiom tests: socket-async-completes-writable,
+socket-async-then-write, socket-async-no-error. 376/376 green.**
+
+**Bug fix landed alongside:** `tcl-call-proc` was discarding `:fileevents`,
+`:timers`, and `:procs` updates made inside Tcl procs (only `:commands` was
+forwarded). Changed the return to forward the inner `result-interp` as the
+base while restoring caller's frame/stack/result/output/code. This was
+masked until socket -async made it natural to register a `fileevent` from
+inside a proc body (the typical async accept pattern).
+
+---
+
 ## Suggested order
 
 1. **Phase 1** — immediate Tcl wins, zero risk, proves the approach
@@ -148,6 +307,12 @@ becomes a lasting SX contribution used by every future hosted language.
 
 _Newest first._
 
+- 2026-05-07: Phase 5f socket -async — socket-connect-async (Unix.set_nonblock+connect/EINPROGRESS) + channel-async-error (getsockopt_error); Tcl `socket -async host port` returns immediately; `fconfigure $sock -error` queries async error; +3 idiom tests; 376/376 green
+- 2026-05-07: Phase 5e clock options + scan — clock-format extended with tz arg (utc/local) + more specifiers; new clock-scan primitive with manual timegm; Tcl clock format/scan support -format/-timezone/-gmt; +5 idiom tests; 373/373 green
+- 2026-05-07: Phase 5d file ops — file-size/mtime/isfile?/isdir?/readable?/writable?/stat/delete/mkdir/copy/rename SX primitives; Tcl file isfile/isdir/readable/writable/size/mtime/atime/type/mkdir/copy/rename/delete now real; +10 idiom tests; 368/368 green
+- 2026-05-07: Phase 5c sockets — socket-connect/socket-server/socket-accept SX primitives wrapping Unix.socket/connect/bind/listen/accept; tcl-cmd-socket dispatches client (host port) vs server (-server cb port); server auto-registers fileevent → _sock-do-accept handler that calls user callback per accept; puts now dispatches "sockN" channels to channel-write too; +4 idiom tests; 358/358 green
+- 2026-05-07: Phase 5b event loop — io-select-channels SX primitive + Tcl-side fileevent/after/vwait/update; tcl-event-step drives expired timers + Unix.select on registered channels; +5 idiom tests; 354/354 green
+- 2026-05-07: Phase 5 channel I/O — 11 SX primitives (channel-open/close/read/read-line/write/flush/seek/tell/eof?/blocking?/set-blocking!) wrapping Unix.openfile/read/write/lseek/set_nonblock; tcl-cmd-open/close/read/gets-chan/seek/tell/flush rewritten + new tcl-cmd-fconfigure; tcl-cmd-puts dispatches on "fileN" arg; gets registration fixed; +7 idiom tests; 349/349 green
 - 2026-05-06: Phase 4 env-as-value — current-env (special form via Sx_ref.register_special_form), eval-in-env (primitive in setup_evaluator_bridge), env-lookup + env-extend (in setup_env_operations); 5 idiom tests; 342/342 green
 - 2026-05-06: Phase 3 OCaml primitives — file-read/write/append/exists?/glob + clock-seconds/milliseconds/format in sx_primitives.ml + unix dep; tcl-cmd-clock/file wired up; 337/337 green
 - 2026-05-06: Phase 2 coroutine rewrite — `tcl-cmd-coroutine` now creates a `make-fiber`; `tcl-cmd-yield` calls `:coro-yield-fn` (threaded through interp); true suspension; 337/337 green
@@ -161,8 +326,8 @@ _Newest first._
 
 ## What stays out of scope
 
-- `package require` of binary loadables
+- `package require` of binary loadables (would need `Dynlink` + native ABI design)
-- Full `clock format` locale support  
+- Full `clock format` locale (translated month/day names, `LC_TIME`-aware) — Phase 5e covers `-format`/`-timezone`/`-gmt` with English names
 - Tk / GUI
 - Threads (mapped to coroutines only, as planned)
-- Full POSIX file I/O (seek/tell/async) — stubs are fine
+- Server-mode `vwait` — Phase 5b event loop is scoped to script-mode; from inside a server-handled command it can't see sx_server's stdin scheduler