datalog: magic pre-saturation is conditional, not unconditional

Previously dl-magic-query always pre-saturated the source db so it
gave correct results for stratified programs (where the rewriter
doesn't propagate magic to aggregate inner-goals or negated rels).
Pure positive programs paid the full bottom-up cost twice.

Add dl-rules-need-presaturation? — checks whether any rule body
contains an aggregate or negation. Only pre-saturate in that case.
Pure positive programs (the common case for magic-sets) keep their
full goal-directed efficiency.

276/276; identical answers on the existing aggregate-of-IDB test.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

hosts/ocaml/lib/sx_primitives.ml

@@ -3124,6 +3124,442 @@ let () =
     | [String pat] -> List (List.map (fun s -> String s) (glob_paths pat))
     | _ -> raise (Eval_error "file-glob: (pattern)"));
 
+  (* === File metadata + ops (Phase 5d) === *)
+  let stat_or = function
+    | String path -> (try Some (Unix.stat path) with _ -> None)
+    | _ -> raise (Eval_error "file: path must be a string")
+  in
+  register "file-size" (fun args ->
+    match args with
+    | [v] -> (match stat_or v with Some s -> Integer s.Unix.st_size | None -> Integer 0)
+    | _ -> raise (Eval_error "file-size: (path)"));
+  register "file-mtime" (fun args ->
+    match args with
+    | [v] -> (match stat_or v with Some s -> Integer (int_of_float s.Unix.st_mtime) | None -> Integer 0)
+    | _ -> raise (Eval_error "file-mtime: (path)"));
+  register "file-isfile?" (fun args ->
+    match args with
+    | [v] -> (match stat_or v with Some s -> Bool (s.Unix.st_kind = Unix.S_REG) | None -> Bool false)
+    | _ -> raise (Eval_error "file-isfile?: (path)"));
+  register "file-isdir?" (fun args ->
+    match args with
+    | [v] -> (match stat_or v with Some s -> Bool (s.Unix.st_kind = Unix.S_DIR) | None -> Bool false)
+    | _ -> raise (Eval_error "file-isdir?: (path)"));
+  register "file-readable?" (fun args ->
+    match args with
+    | [String path] ->
+      Bool (try Unix.access path [Unix.R_OK]; true with _ -> false)
+    | _ -> raise (Eval_error "file-readable?: (path)"));
+  register "file-writable?" (fun args ->
+    match args with
+    | [String path] ->
+      Bool (try Unix.access path [Unix.W_OK]; true with _ -> false)
+    | _ -> raise (Eval_error "file-writable?: (path)"));
+  register "file-stat" (fun args ->
+    match args with
+    | [v] ->
+      (match stat_or v with
+       | None -> Nil
+       | Some s ->
+         let d = Hashtbl.create 6 in
+         Hashtbl.replace d "size" (Integer s.Unix.st_size);
+         Hashtbl.replace d "mtime" (Integer (int_of_float s.Unix.st_mtime));
+         Hashtbl.replace d "atime" (Integer (int_of_float s.Unix.st_atime));
+         Hashtbl.replace d "ctime" (Integer (int_of_float s.Unix.st_ctime));
+         Hashtbl.replace d "mode" (Integer s.Unix.st_perm);
+         Hashtbl.replace d "type" (String (match s.Unix.st_kind with
+           | Unix.S_REG -> "file" | Unix.S_DIR -> "directory"
+           | Unix.S_LNK -> "link" | Unix.S_CHR -> "characterSpecial"
+           | Unix.S_BLK -> "blockSpecial" | Unix.S_FIFO -> "fifo"
+           | Unix.S_SOCK -> "socket"));
+         Dict d)
+    | _ -> raise (Eval_error "file-stat: (path)"));
+  register "file-delete" (fun args ->
+    match args with
+    | [String path] ->
+      (try
+        if Sys.is_directory path then Unix.rmdir path
+        else Unix.unlink path
+      with
+      | Unix.Unix_error (Unix.ENOENT, _, _) -> ()  (* tolerate missing *)
+      | Unix.Unix_error (e, _, _) -> raise (Eval_error ("file-delete: " ^ Unix.error_message e)));
+      Nil
+    | _ -> raise (Eval_error "file-delete: (path)"));
+  register "file-mkdir" (fun args ->
+    match args with
+    | [String path] ->
+      let rec mk p =
+        if p = "" || p = "." || p = "/" then ()
+        else if Sys.file_exists p then ()
+        else begin
+          mk (Filename.dirname p);
+          (try Unix.mkdir p 0o755
+           with Unix.Unix_error (Unix.EEXIST, _, _) -> ())
+        end
+      in
+      (try mk path
+       with Unix.Unix_error (e, _, _) -> raise (Eval_error ("file-mkdir: " ^ Unix.error_message e)));
+      Nil
+    | _ -> raise (Eval_error "file-mkdir: (path)"));
+  register "file-copy" (fun args ->
+    match args with
+    | [String src; String dst] ->
+      (try
+        let ic = open_in_bin src in
+        let oc = open_out_bin dst in
+        let buf = Bytes.create 8192 in
+        let rec loop () =
+          let n = input ic buf 0 (Bytes.length buf) in
+          if n > 0 then (output oc buf 0 n; loop ())
+        in
+        loop ();
+        close_in ic;
+        close_out oc;
+        Nil
+      with
+      | Sys_error msg -> raise (Eval_error ("file-copy: " ^ msg)))
+    | _ -> raise (Eval_error "file-copy: (src dst)"));
+  register "file-rename" (fun args ->
+    match args with
+    | [String src; String dst] ->
+      (try Sys.rename src dst with Sys_error msg -> raise (Eval_error ("file-rename: " ^ msg)));
+      Nil
+    | _ -> raise (Eval_error "file-rename: (src dst)"));
+
+  (* === Channels (random-access + blocking control) === *)
+  let channel_table : (string, Unix.file_descr * string * bool ref * bool ref) Hashtbl.t = Hashtbl.create 16 in
+  let channel_next_id = ref 0 in
+  let parse_open_mode mode =
+    match mode with
+    | "r"  -> [Unix.O_RDONLY]
+    | "w"  -> [Unix.O_WRONLY; Unix.O_CREAT; Unix.O_TRUNC]
+    | "a"  -> [Unix.O_WRONLY; Unix.O_CREAT; Unix.O_APPEND]
+    | "r+" -> [Unix.O_RDWR]
+    | "w+" -> [Unix.O_RDWR; Unix.O_CREAT; Unix.O_TRUNC]
+    | "a+" -> [Unix.O_RDWR; Unix.O_CREAT; Unix.O_APPEND]
+    | _ -> raise (Eval_error ("channel-open: invalid mode " ^ mode))
+  in
+  let chan_get name =
+    match Hashtbl.find_opt channel_table name with
+    | Some c -> c
+    | None -> raise (Eval_error ("channel: no such channel " ^ name))
+  in
+  register "channel-open" (fun args ->
+    match args with
+    | [String path; String mode] ->
+      (try
+        let fd = Unix.openfile path (parse_open_mode mode) 0o644 in
+        let id = !channel_next_id in
+        incr channel_next_id;
+        let name = Printf.sprintf "file%d" id in
+        Hashtbl.replace channel_table name (fd, mode, ref false, ref true);
+        String name
+      with Unix.Unix_error (e, _, _) -> raise (Eval_error ("channel-open: " ^ Unix.error_message e)))
+    | _ -> raise (Eval_error "channel-open: (path mode)"));
+
+  register "channel-close" (fun args ->
+    match args with
+    | [String name] ->
+      let (fd, _, _, _) = chan_get name in
+      (try Unix.close fd with _ -> ());
+      Hashtbl.remove channel_table name;
+      Nil
+    | _ -> raise (Eval_error "channel-close: (channel)"));
+
+  register "channel-read" (fun args ->
+    let (name, max_n) = match args with
+      | [String n] -> (n, -1)
+      | [String n; Integer m] -> (n, m)
+      | [String n; Number m] -> (n, int_of_float m)
+      | _ -> raise (Eval_error "channel-read: (channel ?n?)")
+    in
+    let (fd, _, eof, _) = chan_get name in
+    let chunk = 8192 in
+    let buf = Bytes.create chunk in
+    let buffer = Buffer.create chunk in
+    let total = ref 0 in
+    let stop = ref false in
+    while not !stop do
+      let want = if max_n < 0 then chunk else min chunk (max_n - !total) in
+      if want <= 0 then stop := true
+      else begin
+        try
+          let r = Unix.read fd buf 0 want in
+          if r = 0 then begin eof := true; stop := true end
+          else begin
+            Buffer.add_subbytes buffer buf 0 r;
+            total := !total + r
+          end
+        with
+        | Unix.Unix_error (Unix.EAGAIN, _, _)
+        | Unix.Unix_error (Unix.EWOULDBLOCK, _, _) -> stop := true
+      end
+    done;
+    String (Buffer.contents buffer));
+
+  register "channel-read-line" (fun args ->
+    match args with
+    | [String name] ->
+      let (fd, _, eof, _) = chan_get name in
+      let buf = Buffer.create 80 in
+      let one = Bytes.create 1 in
+      let got_data = ref false in
+      let stop = ref false in
+      while not !stop do
+        try
+          let r = Unix.read fd one 0 1 in
+          if r = 0 then begin eof := true; stop := true end
+          else begin
+            got_data := true;
+            let c = Bytes.get one 0 in
+            if c = '\n' then stop := true
+            else Buffer.add_char buf c
+          end
+        with
+        | Unix.Unix_error (Unix.EAGAIN, _, _)
+        | Unix.Unix_error (Unix.EWOULDBLOCK, _, _) -> stop := true
+      done;
+      if !got_data then String (Buffer.contents buf) else Nil
+    | _ -> raise (Eval_error "channel-read-line: (channel)"));
+
+  register "channel-write" (fun args ->
+    match args with
+    | [String name; String s] ->
+      let (fd, _, _, _) = chan_get name in
+      let b = Bytes.of_string s in
+      let n = Bytes.length b in
+      let written = ref 0 in
+      while !written < n do
+        (try
+          let w = Unix.write fd b !written (n - !written) in
+          written := !written + w
+        with
+        | Unix.Unix_error (Unix.EAGAIN, _, _)
+        | Unix.Unix_error (Unix.EWOULDBLOCK, _, _) ->
+          (* short write — let caller retry *)
+          written := n)
+      done;
+      Nil
+    | _ -> raise (Eval_error "channel-write: (channel string)"));
+
+  register "channel-flush" (fun args ->
+    match args with
+    | [String name] -> let _ = chan_get name in Nil  (* no userspace buffer *)
+    | _ -> raise (Eval_error "channel-flush: (channel)"));
+
+  register "channel-seek" (fun args ->
+    let (name, offset, whence) = match args with
+      | [String n; Integer o] -> (n, o, "start")
+      | [String n; Number o] -> (n, int_of_float o, "start")
+      | [String n; Integer o; String w] -> (n, o, w)
+      | [String n; Number o; String w] -> (n, int_of_float o, w)
+      | _ -> raise (Eval_error "channel-seek: (channel offset ?whence?)")
+    in
+    let (fd, _, eof, _) = chan_get name in
+    let cmd = match whence with
+      | "start" -> Unix.SEEK_SET
+      | "current" -> Unix.SEEK_CUR
+      | "end" -> Unix.SEEK_END
+      | _ -> raise (Eval_error ("channel-seek: invalid whence " ^ whence))
+    in
+    let _ = Unix.lseek fd offset cmd in
+    eof := false;
+    Nil);
+
+  register "channel-tell" (fun args ->
+    match args with
+    | [String name] ->
+      let (fd, _, _, _) = chan_get name in
+      Integer (Unix.lseek fd 0 Unix.SEEK_CUR)
+    | _ -> raise (Eval_error "channel-tell: (channel)"));
+
+  register "channel-eof?" (fun args ->
+    match args with
+    | [String name] ->
+      let (_, _, eof, _) = chan_get name in
+      Bool !eof
+    | _ -> raise (Eval_error "channel-eof?: (channel)"));
+
+  register "channel-blocking?" (fun args ->
+    match args with
+    | [String name] ->
+      let (_, _, _, blocking) = chan_get name in
+      Bool !blocking
+    | _ -> raise (Eval_error "channel-blocking?: (channel)"));
+
+  register "channel-set-blocking!" (fun args ->
+    match args with
+    | [String name; Bool b] ->
+      let (fd, _, _, blocking) = chan_get name in
+      blocking := b;
+      (try
+        if b then Unix.clear_nonblock fd
+        else Unix.set_nonblock fd
+      with _ -> ());
+      Nil
+    | _ -> raise (Eval_error "channel-set-blocking!: (channel bool)"));
+
+  (* === Sockets === wrapping Unix.socket/connect/bind/listen/accept *)
+  let resolve_inet_addr host =
+    if host = "" || host = "0.0.0.0" then Unix.inet_addr_any
+    else if host = "localhost" then Unix.inet_addr_loopback
+    else
+      try Unix.inet_addr_of_string host
+      with _ ->
+        try
+          let entry = Unix.gethostbyname host in
+          if Array.length entry.Unix.h_addr_list = 0 then
+            raise (Eval_error ("socket: cannot resolve " ^ host))
+          else entry.Unix.h_addr_list.(0)
+        with Not_found -> raise (Eval_error ("socket: cannot resolve " ^ host))
+  in
+  let port_of v = match v with
+    | Integer n -> n
+    | Number n -> int_of_float n
+    | _ -> raise (Eval_error "socket: port must be a number")
+  in
+  let alloc_chan_name () =
+    let id = !channel_next_id in
+    incr channel_next_id;
+    Printf.sprintf "sock%d" id
+  in
+
+  register "socket-connect" (fun args ->
+    match args with
+    | [String host; port_v] ->
+      let port = port_of port_v in
+      let addr = Unix.ADDR_INET (resolve_inet_addr host, port) in
+      let sock = Unix.socket Unix.PF_INET Unix.SOCK_STREAM 0 in
+      (try Unix.connect sock addr
+       with Unix.Unix_error (e, _, _) ->
+         (try Unix.close sock with _ -> ());
+         raise (Eval_error ("socket-connect: " ^ Unix.error_message e)));
+      let name = alloc_chan_name () in
+      Hashtbl.replace channel_table name (sock, "rw", ref false, ref true);
+      String name
+    | _ -> raise (Eval_error "socket-connect: (host port)"));
+
+  (* Non-blocking connect: returns channel immediately. Connection completes
+     when the channel becomes writable; query channel-async-error? after to
+     confirm success or get the error. *)
+  register "socket-connect-async" (fun args ->
+    match args with
+    | [String host; port_v] ->
+      let port = port_of port_v in
+      let addr = Unix.ADDR_INET (resolve_inet_addr host, port) in
+      let sock = Unix.socket Unix.PF_INET Unix.SOCK_STREAM 0 in
+      Unix.set_nonblock sock;
+      (try Unix.connect sock addr
+       with
+       | Unix.Unix_error (Unix.EINPROGRESS, _, _)
+       | Unix.Unix_error (Unix.EWOULDBLOCK, _, _) -> ()
+       | Unix.Unix_error (e, _, _) ->
+         (try Unix.close sock with _ -> ());
+         raise (Eval_error ("socket-connect-async: " ^ Unix.error_message e)));
+      let name = alloc_chan_name () in
+      Hashtbl.replace channel_table name (sock, "rw", ref false, ref false);
+      String name
+    | _ -> raise (Eval_error "socket-connect-async: (host port)"));
+
+  (* After a non-blocking connect completes (channel writable), check whether
+     the connect succeeded. Returns "" on success, error message on failure. *)
+  register "channel-async-error" (fun args ->
+    match args with
+    | [String name] ->
+      let (fd, _, _, _) = chan_get name in
+      (try
+        let err = Unix.getsockopt_error fd in
+        match err with
+        | None -> String ""
+        | Some e -> String (Unix.error_message e)
+      with
+      | Unix.Unix_error (e, _, _) -> String (Unix.error_message e))
+    | _ -> raise (Eval_error "channel-async-error: (channel)"));
+
+  register "socket-server" (fun args ->
+    let (host, port) = match args with
+      | [port_v] -> ("", port_of port_v)
+      | [String h; port_v] -> (h, port_of port_v)
+      | _ -> raise (Eval_error "socket-server: (port) or (host port)")
+    in
+    let addr = Unix.ADDR_INET (resolve_inet_addr host, port) in
+    let sock = Unix.socket Unix.PF_INET Unix.SOCK_STREAM 0 in
+    Unix.setsockopt sock Unix.SO_REUSEADDR true;
+    (try Unix.bind sock addr
+     with Unix.Unix_error (e, _, _) ->
+       (try Unix.close sock with _ -> ());
+       raise (Eval_error ("socket-server: bind: " ^ Unix.error_message e)));
+    Unix.listen sock 8;
+    let name = alloc_chan_name () in
+    Hashtbl.replace channel_table name (sock, "server", ref false, ref true);
+    String name);
+
+  register "socket-accept" (fun args ->
+    match args with
+    | [String name] ->
+      let (sock, _, _, _) = chan_get name in
+      let (client_sock, client_addr) =
+        try Unix.accept sock
+        with Unix.Unix_error (e, _, _) ->
+          raise (Eval_error ("socket-accept: " ^ Unix.error_message e))
+      in
+      let (host_str, port) = match client_addr with
+        | Unix.ADDR_INET (addr, p) -> (Unix.string_of_inet_addr addr, p)
+        | Unix.ADDR_UNIX path -> (path, 0)
+      in
+      let client_name = alloc_chan_name () in
+      Hashtbl.replace channel_table client_name (client_sock, "rw", ref false, ref true);
+      let d = Hashtbl.create 3 in
+      Hashtbl.replace d "channel" (String client_name);
+      Hashtbl.replace d "host" (String host_str);
+      Hashtbl.replace d "port" (Integer port);
+      Dict d
+    | _ -> raise (Eval_error "socket-accept: (server-channel)"));
+
+  (* io-select-channels: (read-list write-list timeout-ms) → {:readable [...] :writable [...]}
+     timeout-ms < 0 → block indefinitely; 0 → poll. Returns ready channel names. *)
+  register "io-select-channels" (fun args ->
+    let to_ms v = match v with
+      | Integer n -> n
+      | Number n -> int_of_float n
+      | _ -> raise (Eval_error "io-select-channels: timeout must be a number")
+    in
+    let to_list v = match v with
+      | List xs | ListRef { contents = xs } -> xs
+      | Nil -> []
+      | _ -> raise (Eval_error "io-select-channels: expected list")
+    in
+    let chan_name_of v = match v with
+      | String s -> s
+      | _ -> raise (Eval_error "io-select-channels: channel must be a string")
+    in
+    let (read_list, write_list, timeout_ms) = match args with
+      | [r; w; t] -> (to_list r, to_list w, to_ms t)
+      | _ -> raise (Eval_error "io-select-channels: (read-list write-list timeout-ms)")
+    in
+    let read_pairs = List.map (fun v ->
+      let name = chan_name_of v in
+      let (fd, _, _, _) = chan_get name in (name, fd)) read_list in
+    let write_pairs = List.map (fun v ->
+      let name = chan_name_of v in
+      let (fd, _, _, _) = chan_get name in (name, fd)) write_list in
+    let read_fds = List.map snd read_pairs in
+    let write_fds = List.map snd write_pairs in
+    let timeout = if timeout_ms < 0 then -1.0 else float_of_int timeout_ms /. 1000.0 in
+    let (ready_r, ready_w, _) =
+      try Unix.select read_fds write_fds [] timeout
+      with Unix.Unix_error (Unix.EINTR, _, _) -> ([], [], [])
+    in
+    let names_of pairs ready =
+      List.filter_map (fun (n, fd) ->
+        if List.exists (fun rfd -> rfd = fd) ready then Some (String n) else None
+      ) pairs
+    in
+    let d = Hashtbl.create 2 in
+    Hashtbl.replace d "readable" (List (names_of read_pairs ready_r));
+    Hashtbl.replace d "writable" (List (names_of write_pairs ready_w));
+    Dict d);
+
   (* === Clock === *)
   register "clock-seconds" (fun args ->
     match args with
@@ -3135,11 +3571,8 @@ let () =
     | [] -> Integer (int_of_float (Unix.gettimeofday () *. 1000.0))
     | _ -> raise (Eval_error "clock-milliseconds: no args"));
 
-  register "clock-format" (fun args ->
+  let format_tm tm tz_label =
-    match args with
+    fun fmt ->
-    | [Integer t] | [Integer t; String _] ->
-      let fmt = (match args with [_; String f] -> f | _ -> "%a %b %e %H:%M:%S %Z %Y") in
-      let tm = Unix.gmtime (float_of_int t) in
       let buf = Buffer.create 32 in
       let n = String.length fmt in
       let i = ref 0 in
@@ -3147,14 +3580,19 @@ let () =
         if fmt.[!i] = '%' && !i + 1 < n then begin
           (match fmt.[!i + 1] with
            | 'Y' -> Buffer.add_string buf (Printf.sprintf "%04d" (1900 + tm.Unix.tm_year))
+           | 'y' -> Buffer.add_string buf (Printf.sprintf "%02d" ((1900 + tm.Unix.tm_year) mod 100))
            | 'm' -> Buffer.add_string buf (Printf.sprintf "%02d" (tm.Unix.tm_mon + 1))
            | 'd' -> Buffer.add_string buf (Printf.sprintf "%02d" tm.Unix.tm_mday)
            | 'e' -> Buffer.add_string buf (Printf.sprintf "%2d"  tm.Unix.tm_mday)
            | 'H' -> Buffer.add_string buf (Printf.sprintf "%02d" tm.Unix.tm_hour)
+           | 'I' -> let h = tm.Unix.tm_hour mod 12 in
+                    Buffer.add_string buf (Printf.sprintf "%02d" (if h = 0 then 12 else h))
+           | 'p' -> Buffer.add_string buf (if tm.Unix.tm_hour < 12 then "AM" else "PM")
            | 'M' -> Buffer.add_string buf (Printf.sprintf "%02d" tm.Unix.tm_min)
            | 'S' -> Buffer.add_string buf (Printf.sprintf "%02d" tm.Unix.tm_sec)
            | 'j' -> Buffer.add_string buf (Printf.sprintf "%03d" (tm.Unix.tm_yday + 1))
-           | 'Z' -> Buffer.add_string buf "UTC"
+           | 'w' -> Buffer.add_string buf (string_of_int tm.Unix.tm_wday)
+           | 'Z' -> Buffer.add_string buf tz_label
            | 'a' -> let days = [|"Sun";"Mon";"Tue";"Wed";"Thu";"Fri";"Sat"|] in
                     Buffer.add_string buf days.(tm.Unix.tm_wday)
            | 'A' -> let days = [|"Sunday";"Monday";"Tuesday";"Wednesday";"Thursday";"Friday";"Saturday"|] in
@@ -3163,6 +3601,7 @@ let () =
                     Buffer.add_string buf mons.(tm.Unix.tm_mon)
            | 'B' -> let mons = [|"January";"February";"March";"April";"May";"June";"July";"August";"September";"October";"November";"December"|] in
                     Buffer.add_string buf mons.(tm.Unix.tm_mon)
+           | '%' -> Buffer.add_char buf '%'
            | c -> Buffer.add_char buf '%'; Buffer.add_char buf c);
           i := !i + 2
         end else begin
@@ -3170,8 +3609,100 @@ let () =
           incr i
         end
       done;
-      String (Buffer.contents buf)
+      Buffer.contents buf
-    | _ -> raise (Eval_error "clock-format: (seconds [format])"));
+  in
+  register "clock-format" (fun args ->
+    let (t, fmt, tz) = match args with
+      | [Integer t] -> (t, "%a %b %e %H:%M:%S %Z %Y", "utc")
+      | [Integer t; String f] -> (t, f, "utc")
+      | [Integer t; String f; String z] -> (t, f, z)
+      | _ -> raise (Eval_error "clock-format: (seconds [format [tz]])")
+    in
+    let tm =
+      if tz = "local" then Unix.localtime (float_of_int t)
+      else Unix.gmtime (float_of_int t)
+    in
+    let label = if tz = "local" then "" else "UTC" in
+    String (format_tm tm label fmt));
+
+  (* clock-scan: parse a date string with format, return seconds.
+     Supports the same format specifiers as clock-format (fixed-width ones).
+     tz: "utc" (default) or "local". *)
+  let timegm (tm : Unix.tm) =
+    let is_leap y = y mod 4 = 0 && (y mod 100 <> 0 || y mod 400 = 0) in
+    let days_in_month = [|31;28;31;30;31;30;31;31;30;31;30;31|] in
+    let year = tm.Unix.tm_year + 1900 in
+    let mon = tm.Unix.tm_mon in
+    let mday = tm.Unix.tm_mday in
+    let total_days = ref 0 in
+    if year >= 1970 then begin
+      for y = 1970 to year - 1 do
+        total_days := !total_days + (if is_leap y then 366 else 365)
+      done
+    end else begin
+      for y = year to 1969 do
+        total_days := !total_days - (if is_leap y then 366 else 365)
+      done
+    end;
+    for m = 0 to mon - 1 do
+      total_days := !total_days + days_in_month.(m);
+      if m = 1 && is_leap year then incr total_days
+    done;
+    total_days := !total_days + mday - 1;
+    !total_days * 86400
+    + tm.Unix.tm_hour * 3600
+    + tm.Unix.tm_min * 60
+    + tm.Unix.tm_sec
+  in
+  register "clock-scan" (fun args ->
+    let (str, fmt, tz) = match args with
+      | [String s; String f] -> (s, f, "utc")
+      | [String s; String f; String z] -> (s, f, z)
+      | _ -> raise (Eval_error "clock-scan: (str fmt [tz])")
+    in
+    let n = String.length fmt and sn = String.length str in
+    let tm = ref { Unix.tm_year = 70; tm_mon = 0; tm_mday = 1;
+                   tm_hour = 0; tm_min = 0; tm_sec = 0;
+                   tm_wday = 0; tm_yday = 0; tm_isdst = false } in
+    let i = ref 0 and j = ref 0 in
+    let read_n_digits k =
+      let s = ref "" in
+      let cnt = ref 0 in
+      while !cnt < k && !j < sn && str.[!j] >= '0' && str.[!j] <= '9' do
+        s := !s ^ String.make 1 str.[!j];
+        incr j; incr cnt
+      done;
+      if !s = "" then 0 else int_of_string !s
+    in
+    let skip_ws () =
+      while !j < sn && (str.[!j] = ' ' || str.[!j] = '\t') do incr j done
+    in
+    while !i < n do
+      if fmt.[!i] = '%' && !i + 1 < n then begin
+        (match fmt.[!i + 1] with
+         | 'Y' -> tm := { !tm with tm_year = read_n_digits 4 - 1900 }
+         | 'y' -> let y = read_n_digits 2 in
+                  tm := { !tm with tm_year = (if y < 70 then 100 + y else y) }
+         | 'm' -> tm := { !tm with tm_mon = read_n_digits 2 - 1 }
+         | 'd' | 'e' -> skip_ws (); tm := { !tm with tm_mday = read_n_digits 2 }
+         | 'H' | 'I' -> tm := { !tm with tm_hour = read_n_digits 2 }
+         | 'M' -> tm := { !tm with tm_min = read_n_digits 2 }
+         | 'S' -> tm := { !tm with tm_sec = read_n_digits 2 }
+         | '%' -> if !j < sn && str.[!j] = '%' then incr j
+         | _ -> ()  (* unsupported specifier — skip *)
+        );
+        i := !i + 2
+      end else begin
+        if fmt.[!i] = ' ' then skip_ws ()
+        else if !j < sn && str.[!j] = fmt.[!i] then incr j;
+        incr i
+      end
+    done;
+    let secs =
+      if tz = "local" then int_of_float (fst (Unix.mktime !tm))
+      else timegm !tm
+    in
+    Integer secs);
 
   (* === Env-as-value (Phase 4) === *)
 

lib/apl/conformance.sh

@@ -13,7 +13,7 @@ if [ ! -x "$SX_SERVER" ]; then
   exit 1
 fi
 
-SUITES=(structural operators dfn tradfn valence programs system idioms)
+SUITES=(structural operators dfn tradfn valence programs system idioms eval-ops pipeline)
 
 OUT_JSON="lib/apl/scoreboard.json"
 OUT_MD="lib/apl/scoreboard.md"
@@ -26,7 +26,10 @@ run_suite() {
   cat > "$TMP" << EPOCHS
 (epoch 1)
 (load "spec/stdlib.sx")
+(load "lib/r7rs.sx")
 (load "lib/apl/runtime.sx")
+(load "lib/apl/tokenizer.sx")
+(load "lib/apl/parser.sx")
 (load "lib/apl/transpile.sx")
 (epoch 2)
 (eval "(define apl-test-pass 0)")
@@ -39,7 +42,7 @@ run_suite() {
 EPOCHS
 
   local OUTPUT
-  OUTPUT=$(timeout 180 "$SX_SERVER" < "$TMP" 2>/dev/null)
+  OUTPUT=$(timeout 300 "$SX_SERVER" < "$TMP" 2>/dev/null)
   rm -f "$TMP"
 
   local LINE

lib/apl/parser.sx

@@ -25,99 +25,151 @@
 ; Glyph classification sets
 ; ============================================================
 
-(define apl-parse-op-glyphs
+(define
-  (list "/" "\\" "¨" "⍨" "∘" "." "⍣" "⍤" "⍥" "@"))
+  apl-parse-op-glyphs
+  (list "/" "⌿" "\\" "⍀" "¨" "⍨" "∘" "." "⍣" "⍤" "⍥" "@"))
 
-(define apl-parse-fn-glyphs
+(define
-  (list "+" "-" "×" "÷" "*" "⍟" "⌈" "⌊" "|" "!" "?" "○" "~"
+  apl-parse-fn-glyphs
-        "<" "≤" "=" "≥" ">" "≠" "∊" "∧" "∨" "⍱" "⍲"
+  (list
-        "," "⍪" "⍴" "⌽" "⊖" "⍉" "↑" "↓" "⊂" "⊃" "⊆"
+    "+"
-        "∪" "∩" "⍳" "⍸" "⌷" "⍋" "⍒" "⊥" "⊤" "⊣" "⊢" "⍎" "⍕"))
+    "-"
+    "×"
+    "÷"
+    "*"
+    "⍟"
+    "⌈"
+    "⌊"
+    "|"
+    "!"
+    "?"
+    "○"
+    "~"
+    "<"
+    "≤"
+    "="
+    "≥"
+    ">"
+    "≠"
+    "≢"
+    "≡"
+    "∊"
+    "∧"
+    "∨"
+    "⍱"
+    "⍲"
+    ","
+    "⍪"
+    "⍴"
+    "⌽"
+    "⊖"
+    "⍉"
+    "↑"
+    "↓"
+    "⊂"
+    "⊃"
+    "⊆"
+    "∪"
+    "∩"
+    "⍳"
+    "⍸"
+    "⌷"
+    "⍋"
+    "⍒"
+    "⊥"
+    "⊤"
+    "⊣"
+    "⊢"
+    "⍎"
+    "⍕"))
 
-(define apl-parse-op-glyph?
+(define apl-quad-fn-names (list "⎕FMT" "⎕←"))
-  (fn (v)
-    (some (fn (g) (= g v)) apl-parse-op-glyphs)))
 
-(define apl-parse-fn-glyph?
+(define apl-known-fn-names (list))
-  (fn (v)
-    (some (fn (g) (= g v)) apl-parse-fn-glyphs)))
 
 ; ============================================================
 ; Token accessors
 ; ============================================================
 
-(define tok-type
+(define
-  (fn (tok)
+  apl-collect-fn-bindings
-    (get tok :type)))
+  (fn
+    (stmt-groups)
+    (set! apl-known-fn-names (list))
+    (for-each
+      (fn
+        (toks)
+        (when
+          (and
+            (>= (len toks) 3)
+            (= (tok-type (nth toks 0)) :name)
+            (= (tok-type (nth toks 1)) :assign)
+            (= (tok-type (nth toks 2)) :lbrace))
+          (set!
+            apl-known-fn-names
+            (cons (tok-val (nth toks 0)) apl-known-fn-names))))
+      stmt-groups)))
 
-(define tok-val
+(define
-  (fn (tok)
+  apl-parse-op-glyph?
-    (get tok :value)))
+  (fn (v) (some (fn (g) (= g v)) apl-parse-op-glyphs)))
 
-(define is-op-tok?
+(define
-  (fn (tok)
+  apl-parse-fn-glyph?
-    (and (= (tok-type tok) :glyph)
+  (fn (v) (some (fn (g) (= g v)) apl-parse-fn-glyphs)))
-         (apl-parse-op-glyph? (tok-val tok)))))
 
-(define is-fn-tok?
+(define tok-type (fn (tok) (get tok :type)))
-  (fn (tok)
-    (and (= (tok-type tok) :glyph)
-         (apl-parse-fn-glyph? (tok-val tok)))))
 
 ; ============================================================
 ; Collect trailing operators starting at index i
 ; Returns {:ops (op ...) :end new-i}
 ; ============================================================
 
-(define collect-ops
+(define tok-val (fn (tok) (get tok :value)))
-  (fn (tokens i)
-    (collect-ops-loop tokens i (list))))
 
-(define collect-ops-loop
+(define
-  (fn (tokens i acc)
+  is-op-tok?
-    (if (>= i (len tokens))
+  (fn
-      {:ops acc :end i}
+    (tok)
-      (let ((tok (nth tokens i)))
+    (and (= (tok-type tok) :glyph) (apl-parse-op-glyph? (tok-val tok)))))
-        (if (is-op-tok? tok)
-          (collect-ops-loop tokens (+ i 1) (append acc (tok-val tok)))
-          {:ops acc :end i})))))
 
 ; ============================================================
 ; Build a derived-fn node by chaining operators left-to-right
 ; (+/¨  →  (:derived-fn "¨" (:derived-fn "/" (:fn-glyph "+"))))
 ; ============================================================
 
-(define build-derived-fn
+(define
-  (fn (fn-node ops)
+  is-fn-tok?
-    (if (= (len ops) 0)
+  (fn
-      fn-node
+    (tok)
-      (build-derived-fn
+    (or
-        (list :derived-fn (first ops) fn-node)
+      (and (= (tok-type tok) :glyph) (apl-parse-fn-glyph? (tok-val tok)))
-        (rest ops)))))
+      (and
+        (= (tok-type tok) :name)
+        (or
+          (some (fn (q) (= q (tok-val tok))) apl-quad-fn-names)
+          (some (fn (q) (= q (tok-val tok))) apl-known-fn-names))))))
 
 ; ============================================================
 ; Find matching close bracket/paren/brace
 ; Returns the index of the matching close token
 ; ============================================================
 
-(define find-matching-close
+(define collect-ops (fn (tokens i) (collect-ops-loop tokens i (list))))
-  (fn (tokens start open-type close-type)
-    (find-matching-close-loop tokens start open-type close-type 1)))
 
-(define find-matching-close-loop
+(define
-  (fn (tokens i open-type close-type depth)
+  collect-ops-loop
-    (if (>= i (len tokens))
+  (fn
-      (len tokens)
+    (tokens i acc)
-      (let ((tt (tok-type (nth tokens i))))
+    (if
-        (cond
+      (>= i (len tokens))
-          ((= tt open-type)
+      {:end i :ops acc}
-           (find-matching-close-loop tokens (+ i 1) open-type close-type (+ depth 1)))
+      (let
-          ((= tt close-type)
+        ((tok (nth tokens i)))
-           (if (= depth 1)
+        (if
-             i
+          (is-op-tok? tok)
-             (find-matching-close-loop tokens (+ i 1) open-type close-type (- depth 1))))
+          (collect-ops-loop tokens (+ i 1) (append acc (tok-val tok)))
-          (true
+          {:end i :ops acc})))))
-           (find-matching-close-loop tokens (+ i 1) open-type close-type depth)))))))
 
 ; ============================================================
 ; Segment collection: scan tokens left-to-right, building
@@ -126,122 +178,20 @@
 ; derived-fn nodes during this pass.
 ; ============================================================
 
-(define collect-segments
+(define
-  (fn (tokens)
+  build-derived-fn
-    (collect-segments-loop tokens 0 (list))))
+  (fn
+    (fn-node ops)
+    (if
+      (= (len ops) 0)
+      fn-node
+      (build-derived-fn (list :derived-fn (first ops) fn-node) (rest ops)))))
 
-(define collect-segments-loop
+(define
-  (fn (tokens i acc)
+  find-matching-close
-    (if (>= i (len tokens))
+  (fn
-      acc
+    (tokens start open-type close-type)
-      (let ((tok (nth tokens i))
+    (find-matching-close-loop tokens start open-type close-type 1)))
-            (n   (len tokens)))
-        (let ((tt (tok-type tok))
-              (tv (tok-val tok)))
-          (cond
-            ; Skip separators
-            ((or (= tt :diamond) (= tt :newline) (= tt :semi))
-             (collect-segments-loop tokens (+ i 1) acc))
-
-            ; Number → value segment
-            ((= tt :num)
-             (collect-segments-loop tokens (+ i 1)
-               (append acc {:kind "val" :node (list :num tv)})))
-
-            ; String → value segment
-            ((= tt :str)
-             (collect-segments-loop tokens (+ i 1)
-               (append acc {:kind "val" :node (list :str tv)})))
-
-            ; Name → always a value segment in Phase 1
-            ; (Named functions with operators like f/ are Phase 5)
-            ((= tt :name)
-             (collect-segments-loop tokens (+ i 1)
-               (append acc {:kind "val" :node (list :name tv)})))
-
-
-            ; Left paren → parse subexpression recursively
-            ((= tt :lparen)
-             (let ((end (find-matching-close tokens (+ i 1) :lparen :rparen)))
-               (let ((inner-tokens (slice tokens (+ i 1) end))
-                     (after (+ end 1)))
-                 (collect-segments-loop tokens after
-                   (append acc {:kind "val" :node (parse-apl-expr inner-tokens)})))))
-
-            ; Left brace → dfn
-            ((= tt :lbrace)
-             (let ((end (find-matching-close tokens (+ i 1) :lbrace :rbrace)))
-               (let ((inner-tokens (slice tokens (+ i 1) end))
-                     (after (+ end 1)))
-                 (collect-segments-loop tokens after
-                   (append acc {:kind "fn" :node (parse-dfn inner-tokens)})))))
-
-            ; Glyph token — need to classify
-            ((= tt :glyph)
-             (cond
-               ; Alpha (⍺) and Omega (⍵) → values inside dfn context
-               ((or (= tv "⍺") (= tv "⍵"))
-                (collect-segments-loop tokens (+ i 1)
-                  (append acc {:kind "val" :node (list :name tv)})))
-
-               ; Nabla (∇) → self-reference function in dfn context
-               ((= tv "∇")
-                (collect-segments-loop tokens (+ i 1)
-                  (append acc {:kind "fn" :node (list :fn-glyph "∇")})))
-
-               ; ∘. → outer product (special case: ∘ followed by .)
-               ((and (= tv "∘")
-                     (< (+ i 1) n)
-                     (= (tok-val (nth tokens (+ i 1))) "."))
-                (if (and (< (+ i 2) n) (is-fn-tok? (nth tokens (+ i 2))))
-                  (let ((fn-tv (tok-val (nth tokens (+ i 2)))))
-                    (let ((op-result (collect-ops tokens (+ i 3))))
-                      (let ((ops (get op-result :ops))
-                            (ni  (get op-result :end)))
-                        (let ((fn-node (build-derived-fn (list :fn-glyph fn-tv) ops)))
-                          (collect-segments-loop tokens ni
-                            (append acc {:kind "fn" :node (list :outer "∘." fn-node)}))))))
-                  ; ∘. without function — treat ∘ as plain compose operator
-                  ; skip the . and continue
-                  (collect-segments-loop tokens (+ i 1)
-                    acc)))
-
-               ; Function glyph — collect following operators
-               ((apl-parse-fn-glyph? tv)
-                (let ((op-result (collect-ops tokens (+ i 1))))
-                  (let ((ops (get op-result :ops))
-                        (ni  (get op-result :end)))
-                    ; Check for inner product: fn . fn
-                    ; (ops = ("." ) and next token is also a function glyph)
-                    (if (and (= (len ops) 1)
-                             (= (first ops) ".")
-                             (< ni n)
-                             (is-fn-tok? (nth tokens ni)))
-                      ; f.g inner product
-                      (let ((g-tv (tok-val (nth tokens ni))))
-                        (let ((op-result2 (collect-ops tokens (+ ni 1))))
-                          (let ((ops2 (get op-result2 :ops))
-                                (ni2  (get op-result2 :end)))
-                            (let ((g-node (build-derived-fn (list :fn-glyph g-tv) ops2)))
-                              (collect-segments-loop tokens ni2
-                                (append acc {:kind "fn"
-                                             :node (list :derived-fn2 "." (list :fn-glyph tv) g-node)}))))))
-                      ; Regular function with zero or more operator modifiers
-                      (let ((fn-node (build-derived-fn (list :fn-glyph tv) ops)))
-                        (collect-segments-loop tokens ni
-                          (append acc {:kind "fn" :node fn-node})))))))
-
-               ; Stray operator glyph — skip (shouldn't appear outside function context)
-               ((apl-parse-op-glyph? tv)
-                (collect-segments-loop tokens (+ i 1) acc))
-
-               ; Unknown glyph — skip
-               (true
-                (collect-segments-loop tokens (+ i 1) acc))))
-
-            ; Skip unknown token types
-            (true
-             (collect-segments-loop tokens (+ i 1) acc))))))))
 
 ; ============================================================
 ; Build tree from segment list
@@ -258,57 +208,196 @@
 ; ============================================================
 
 ; Find the index of the first function segment (returns -1 if none)
-(define find-first-fn
+(define
-  (fn (segs)
+  find-matching-close-loop
-    (find-first-fn-loop segs 0)))
+  (fn
+    (tokens i open-type close-type depth)
+    (if
+      (>= i (len tokens))
+      (len tokens)
+      (let
+        ((tt (tok-type (nth tokens i))))
+        (cond
+          ((= tt open-type)
+            (find-matching-close-loop
+              tokens
+              (+ i 1)
+              open-type
+              close-type
+              (+ depth 1)))
+          ((= tt close-type)
+            (if
+              (= depth 1)
+              i
+              (find-matching-close-loop
+                tokens
+                (+ i 1)
+                open-type
+                close-type
+                (- depth 1))))
+          (true
+            (find-matching-close-loop
+              tokens
+              (+ i 1)
+              open-type
+              close-type
+              depth)))))))
 
-(define find-first-fn-loop
+(define
-  (fn (segs i)
+  collect-segments
-    (if (>= i (len segs))
+  (fn (tokens) (collect-segments-loop tokens 0 (list))))
-      -1
-      (if (= (get (nth segs i) :kind) "fn")
-        i
-        (find-first-fn-loop segs (+ i 1))))))
 
 ; Build an array node from 0..n value segments
 ; If n=1 → return that segment's node
 ; If n>1 → return (:vec node1 node2 ...)
-(define segs-to-array
+(define
-  (fn (segs)
+  collect-segments-loop
-    (if (= (len segs) 1)
+  (fn
-      (get (first segs) :node)
+    (tokens i acc)
-      (cons :vec (map (fn (s) (get s :node)) segs)))))
+    (if
+      (>= i (len tokens))
+      acc
+      (let
+        ((tok (nth tokens i)) (n (len tokens)))
+        (let
+          ((tt (tok-type tok)) (tv (tok-val tok)))
+          (cond
+            ((or (= tt :diamond) (= tt :newline) (= tt :semi))
+              (collect-segments-loop tokens (+ i 1) acc))
+            ((= tt :num)
+              (collect-segments-loop tokens (+ i 1) (append acc {:kind "val" :node (list :num tv)})))
+            ((= tt :str)
+              (collect-segments-loop tokens (+ i 1) (append acc {:kind "val" :node (list :str tv)})))
+            ((= tt :name)
+              (cond
+                ((some (fn (q) (= q tv)) apl-quad-fn-names)
+                  (let
+                    ((op-result (collect-ops tokens (+ i 1))))
+                    (let
+                      ((ops (get op-result :ops))
+                        (ni (get op-result :end)))
+                      (let
+                        ((fn-node (build-derived-fn (list :fn-glyph tv) ops)))
+                        (collect-segments-loop
+                          tokens
+                          ni
+                          (append acc {:kind "fn" :node fn-node}))))))
+                ((some (fn (q) (= q tv)) apl-known-fn-names)
+                  (let
+                    ((op-result (collect-ops tokens (+ i 1))))
+                    (let
+                      ((ops (get op-result :ops))
+                        (ni (get op-result :end)))
+                      (let
+                        ((fn-node (build-derived-fn (list :fn-name tv) ops)))
+                        (collect-segments-loop
+                          tokens
+                          ni
+                          (append acc {:kind "fn" :node fn-node}))))))
+                (else
+                  (let
+                    ((br (maybe-bracket (list :name tv) tokens (+ i 1))))
+                    (collect-segments-loop
+                      tokens
+                      (nth br 1)
+                      (append acc {:kind "val" :node (nth br 0)}))))))
+            ((= tt :lparen)
+              (let
+                ((end (find-matching-close tokens (+ i 1) :lparen :rparen)))
+                (let
+                  ((inner-tokens (slice tokens (+ i 1) end))
+                    (after (+ end 1)))
+                  (let
+                    ((inner-segs (collect-segments inner-tokens)))
+                    (if
+                      (and
+                        (>= (len inner-segs) 2)
+                        (every? (fn (s) (= (get s :kind) "fn")) inner-segs))
+                      (let
+                        ((train-node (cons :train (map (fn (s) (get s :node)) inner-segs))))
+                        (collect-segments-loop
+                          tokens
+                          after
+                          (append acc {:kind "fn" :node train-node})))
+                      (let
+                        ((br (maybe-bracket (parse-apl-expr inner-tokens) tokens after)))
+                        (collect-segments-loop
+                          tokens
+                          (nth br 1)
+                          (append acc {:kind "val" :node (nth br 0)}))))))))
+            ((= tt :lbrace)
+              (let
+                ((end (find-matching-close tokens (+ i 1) :lbrace :rbrace)))
+                (let
+                  ((inner-tokens (slice tokens (+ i 1) end))
+                    (after (+ end 1)))
+                  (collect-segments-loop tokens after (append acc {:kind "fn" :node (parse-dfn inner-tokens)})))))
+            ((= tt :glyph)
+              (cond
+                ((or (= tv "⍺") (= tv "⍵"))
+                  (collect-segments-loop
+                    tokens
+                    (+ i 1)
+                    (append acc {:kind "val" :node (list :name tv)})))
+                ((= tv "∇")
+                  (collect-segments-loop
+                    tokens
+                    (+ i 1)
+                    (append acc {:kind "fn" :node (list :fn-glyph "∇")})))
+                ((and (= tv "∘") (< (+ i 1) n) (= (tok-val (nth tokens (+ i 1))) "."))
+                  (if
+                    (and (< (+ i 2) n) (is-fn-tok? (nth tokens (+ i 2))))
+                    (let
+                      ((fn-tv (tok-val (nth tokens (+ i 2)))))
+                      (let
+                        ((op-result (collect-ops tokens (+ i 3))))
+                        (let
+                          ((ops (get op-result :ops))
+                            (ni (get op-result :end)))
+                          (let
+                            ((fn-node (build-derived-fn (list :fn-glyph fn-tv) ops)))
+                            (collect-segments-loop
+                              tokens
+                              ni
+                              (append acc {:kind "fn" :node (list :outer "∘." fn-node)}))))))
+                    (collect-segments-loop tokens (+ i 1) acc)))
+                ((apl-parse-fn-glyph? tv)
+                  (let
+                    ((op-result (collect-ops tokens (+ i 1))))
+                    (let
+                      ((ops (get op-result :ops))
+                        (ni (get op-result :end)))
+                      (if
+                        (and
+                          (= (len ops) 1)
+                          (= (first ops) ".")
+                          (< ni n)
+                          (is-fn-tok? (nth tokens ni)))
+                        (let
+                          ((g-tv (tok-val (nth tokens ni))))
+                          (let
+                            ((op-result2 (collect-ops tokens (+ ni 1))))
+                            (let
+                              ((ops2 (get op-result2 :ops))
+                                (ni2 (get op-result2 :end)))
+                              (let
+                                ((g-node (build-derived-fn (list :fn-glyph g-tv) ops2)))
+                                (collect-segments-loop
+                                  tokens
+                                  ni2
+                                  (append acc {:kind "fn" :node (list :derived-fn2 "." (list :fn-glyph tv) g-node)}))))))
+                        (let
+                          ((fn-node (build-derived-fn (list :fn-glyph tv) ops)))
+                          (collect-segments-loop
+                            tokens
+                            ni
+                            (append acc {:kind "fn" :node fn-node})))))))
+                ((apl-parse-op-glyph? tv)
+                  (collect-segments-loop tokens (+ i 1) acc))
+                (true (collect-segments-loop tokens (+ i 1) acc))))
+            (true (collect-segments-loop tokens (+ i 1) acc))))))))
 
-(define build-tree
+(define find-first-fn (fn (segs) (find-first-fn-loop segs 0)))
-  (fn (segs)
-    (cond
-      ; Empty → nil
-      ((= (len segs) 0) nil)
-      ; Single segment → return its node directly
-      ((= (len segs) 1) (get (first segs) :node))
-      ; All values → strand
-      ((every? (fn (s) (= (get s :kind) "val")) segs)
-       (segs-to-array segs))
-      ; Find the first function segment
-      (true
-       (let ((fn-idx (find-first-fn segs)))
-         (cond
-           ; No function found (shouldn't happen given above checks) → strand
-           ((= fn-idx -1) (segs-to-array segs))
-           ; Function is first → monadic call
-           ((= fn-idx 0)
-            (list :monad
-                  (get (first segs) :node)
-                  (build-tree (rest segs))))
-           ; Function at position fn-idx: left args are segs[0..fn-idx-1]
-           (true
-            (let ((left-segs  (slice segs 0 fn-idx))
-                  (fn-seg     (nth segs fn-idx))
-                  (right-segs (slice segs (+ fn-idx 1))))
-              (list :dyad
-                    (get fn-seg :node)
-                    (segs-to-array left-segs)
-                    (build-tree right-segs))))))))))
 
 
 ; ============================================================
@@ -316,121 +405,270 @@
 ; Only splits at depth 0 (ignores separators inside { } or ( ) )
 ; ============================================================
 
-(define split-statements
+(define
-  (fn (tokens)
+  find-first-fn-loop
-    (split-statements-loop tokens (list) (list) 0)))
+  (fn
+    (segs i)
+    (if
+      (>= i (len segs))
+      -1
+      (if
+        (= (get (nth segs i) :kind) "fn")
+        i
+        (find-first-fn-loop segs (+ i 1))))))
 
-(define split-statements-loop
+(define
-  (fn (tokens current-stmt acc depth)
+  segs-to-array
-    (if (= (len tokens) 0)
+  (fn
-      (if (> (len current-stmt) 0)
+    (segs)
-        (append acc (list current-stmt))
+    (if
-        acc)
+      (= (len segs) 1)
-      (let ((tok (first tokens))
+      (get (first segs) :node)
-            (rest-toks (rest tokens))
+      (cons :vec (map (fn (s) (get s :node)) segs)))))
-            (tt (tok-type (first tokens))))
-        (cond
-          ; Open brackets increase depth
-          ((or (= tt :lparen) (= tt :lbrace) (= tt :lbracket))
-           (split-statements-loop rest-toks (append current-stmt tok) acc (+ depth 1)))
-          ; Close brackets decrease depth
-          ((or (= tt :rparen) (= tt :rbrace) (= tt :rbracket))
-           (split-statements-loop rest-toks (append current-stmt tok) acc (- depth 1)))
-          ; Separators only split at top level (depth = 0)
-          ((and (> depth 0) (or (= tt :diamond) (= tt :newline)))
-           (split-statements-loop rest-toks (append current-stmt tok) acc depth))
-          ((and (= depth 0) (or (= tt :diamond) (= tt :newline)))
-           (if (> (len current-stmt) 0)
-             (split-statements-loop rest-toks (list) (append acc (list current-stmt)) depth)
-             (split-statements-loop rest-toks (list) acc depth)))
-          ; All other tokens go into current statement
-          (true
-           (split-statements-loop rest-toks (append current-stmt tok) acc depth)))))))
 
 ; ============================================================
 ; Parse a dfn body (tokens between { and })
 ; Handles guard expressions: cond : expr
 ; ============================================================
 
-(define parse-dfn
+(define
-  (fn (tokens)
+  build-tree
-    (let ((stmt-groups (split-statements tokens)))
+  (fn
-      (let ((stmts (map parse-dfn-stmt stmt-groups)))
+    (segs)
-        (cons :dfn stmts)))))
+    (cond
+      ((= (len segs) 0) nil)
+      ((= (len segs) 1) (get (first segs) :node))
+      ((every? (fn (s) (= (get s :kind) "val")) segs)
+        (segs-to-array segs))
+      (true
+        (let
+          ((fn-idx (find-first-fn segs)))
+          (cond
+            ((= fn-idx -1) (segs-to-array segs))
+            ((= fn-idx 0)
+              (list
+                :monad (get (first segs) :node)
+                (build-tree (rest segs))))
+            (true
+              (let
+                ((left-segs (slice segs 0 fn-idx))
+                  (fn-seg (nth segs fn-idx))
+                  (right-segs (slice segs (+ fn-idx 1))))
+                (list
+                  :dyad (get fn-seg :node)
+                  (segs-to-array left-segs)
+                  (build-tree right-segs))))))))))
 
-(define parse-dfn-stmt
+(define
-  (fn (tokens)
+  split-statements
-    ; Check for guard: expr : expr
+  (fn (tokens) (split-statements-loop tokens (list) (list) 0)))
-    ; A guard has a :colon token not inside parens/braces
-    (let ((colon-idx (find-top-level-colon tokens 0)))
-      (if (>= colon-idx 0)
-        ; Guard: cond : expr
-        (let ((cond-tokens (slice tokens 0 colon-idx))
-              (body-tokens (slice tokens (+ colon-idx 1))))
-          (list :guard
-                (parse-apl-expr cond-tokens)
-                (parse-apl-expr body-tokens)))
-        ; Regular statement
-        (parse-stmt tokens)))))
 
-(define find-top-level-colon
+(define
-  (fn (tokens i)
+  split-statements-loop
-    (find-top-level-colon-loop tokens i 0)))
+  (fn
-
+    (tokens current-stmt acc depth)
-(define find-top-level-colon-loop
+    (if
-  (fn (tokens i depth)
+      (= (len tokens) 0)
-    (if (>= i (len tokens))
+      (if (> (len current-stmt) 0) (append acc (list current-stmt)) acc)
-      -1
+      (let
-      (let ((tok (nth tokens i))
+        ((tok (first tokens))
-            (tt  (tok-type (nth tokens i))))
+          (rest-toks (rest tokens))
+          (tt (tok-type (first tokens))))
         (cond
           ((or (= tt :lparen) (= tt :lbrace) (= tt :lbracket))
-           (find-top-level-colon-loop tokens (+ i 1) (+ depth 1)))
+            (split-statements-loop
+              rest-toks
+              (append current-stmt tok)
+              acc
+              (+ depth 1)))
           ((or (= tt :rparen) (= tt :rbrace) (= tt :rbracket))
-           (find-top-level-colon-loop tokens (+ i 1) (- depth 1)))
+            (split-statements-loop
-          ((and (= tt :colon) (= depth 0))
+              rest-toks
-           i)
+              (append current-stmt tok)
+              acc
+              (- depth 1)))
+          ((and (> depth 0) (or (= tt :diamond) (= tt :newline)))
+            (split-statements-loop
+              rest-toks
+              (append current-stmt tok)
+              acc
+              depth))
+          ((and (= depth 0) (or (= tt :diamond) (= tt :newline)))
+            (if
+              (> (len current-stmt) 0)
+              (split-statements-loop
+                rest-toks
+                (list)
+                (append acc (list current-stmt))
+                depth)
+              (split-statements-loop rest-toks (list) acc depth)))
           (true
-           (find-top-level-colon-loop tokens (+ i 1) depth)))))))
+            (split-statements-loop
+              rest-toks
+              (append current-stmt tok)
+              acc
+              depth)))))))
+
+(define
+  parse-dfn
+  (fn
+    (tokens)
+    (let
+      ((stmt-groups (split-statements tokens)))
+      (let ((stmts (map parse-dfn-stmt stmt-groups))) (cons :dfn stmts)))))
 
 ; ============================================================
 ; Parse a single statement (assignment or expression)
 ; ============================================================
 
-(define parse-stmt
+(define
-  (fn (tokens)
+  parse-dfn-stmt
-    (if (and (>= (len tokens) 2)
+  (fn
-             (= (tok-type (nth tokens 0)) :name)
+    (tokens)
-             (= (tok-type (nth tokens 1)) :assign))
+    (let
-      ; Assignment: name ← expr
+      ((colon-idx (find-top-level-colon tokens 0)))
-      (list :assign
+      (if
-            (tok-val (nth tokens 0))
+        (>= colon-idx 0)
-            (parse-apl-expr (slice tokens 2)))
+        (let
-      ; Expression
+          ((cond-tokens (slice tokens 0 colon-idx))
-      (parse-apl-expr tokens))))
+            (body-tokens (slice tokens (+ colon-idx 1))))
+          (list
+            :guard (parse-apl-expr cond-tokens)
+            (parse-apl-expr body-tokens)))
+        (parse-stmt tokens)))))
 
 ; ============================================================
 ; Parse an expression from a flat token list
 ; ============================================================
 
-(define parse-apl-expr
+(define
-  (fn (tokens)
+  find-top-level-colon
-    (let ((segs (collect-segments tokens)))
+  (fn (tokens i) (find-top-level-colon-loop tokens i 0)))
-      (if (= (len segs) 0)
-        nil
-        (build-tree segs)))))
 
 ; ============================================================
 ; Main entry point
 ; parse-apl: string → AST
 ; ============================================================
 
-(define parse-apl
+(define
-  (fn (src)
+  find-top-level-colon-loop
-    (let ((tokens (apl-tokenize src)))
+  (fn
-      (let ((stmt-groups (split-statements tokens)))
+    (tokens i depth)
-        (if (= (len stmt-groups) 0)
+    (if
-          nil
+      (>= i (len tokens))
-          (if (= (len stmt-groups) 1)
+      -1
-            (parse-stmt (first stmt-groups))
+      (let
-            (cons :program (map parse-stmt stmt-groups))))))))
+        ((tok (nth tokens i)) (tt (tok-type (nth tokens i))))
+        (cond
+          ((or (= tt :lparen) (= tt :lbrace) (= tt :lbracket))
+            (find-top-level-colon-loop tokens (+ i 1) (+ depth 1)))
+          ((or (= tt :rparen) (= tt :rbrace) (= tt :rbracket))
+            (find-top-level-colon-loop tokens (+ i 1) (- depth 1)))
+          ((and (= tt :colon) (= depth 0)) i)
+          (true (find-top-level-colon-loop tokens (+ i 1) depth)))))))
+
+(define
+  parse-stmt
+  (fn
+    (tokens)
+    (if
+      (and
+        (>= (len tokens) 2)
+        (= (tok-type (nth tokens 0)) :name)
+        (= (tok-type (nth tokens 1)) :assign))
+      (list
+        :assign (tok-val (nth tokens 0))
+        (parse-apl-expr (slice tokens 2)))
+      (parse-apl-expr tokens))))
+
+(define
+  parse-apl-expr
+  (fn
+    (tokens)
+    (let
+      ((segs (collect-segments tokens)))
+      (if (= (len segs) 0) nil (build-tree segs)))))
+
+(define
+  parse-apl
+  (fn
+    (src)
+    (let
+      ((tokens (apl-tokenize src)))
+      (let
+        ((stmt-groups (split-statements tokens)))
+        (begin
+          (apl-collect-fn-bindings stmt-groups)
+          (if
+            (= (len stmt-groups) 0)
+            nil
+            (if
+              (= (len stmt-groups) 1)
+              (parse-stmt (first stmt-groups))
+              (cons :program (map parse-stmt stmt-groups)))))))))
+
+(define
+  split-bracket-loop
+  (fn
+    (tokens current acc depth)
+    (if
+      (= (len tokens) 0)
+      (append acc (list current))
+      (let
+        ((tok (first tokens)) (more (rest tokens)))
+        (let
+          ((tt (tok-type tok)))
+          (cond
+            ((or (= tt :lparen) (= tt :lbrace) (= tt :lbracket))
+              (split-bracket-loop
+                more
+                (append current (list tok))
+                acc
+                (+ depth 1)))
+            ((or (= tt :rparen) (= tt :rbrace) (= tt :rbracket))
+              (split-bracket-loop
+                more
+                (append current (list tok))
+                acc
+                (- depth 1)))
+            ((and (= tt :semi) (= depth 0))
+              (split-bracket-loop
+                more
+                (list)
+                (append acc (list current))
+                depth))
+            (else
+              (split-bracket-loop more (append current (list tok)) acc depth))))))))
+
+(define
+  split-bracket-content
+  (fn (tokens) (split-bracket-loop tokens (list) (list) 0)))
+
+(define
+  maybe-bracket
+  (fn
+    (val-node tokens after)
+    (if
+      (and
+        (< after (len tokens))
+        (= (tok-type (nth tokens after)) :lbracket))
+      (let
+        ((end (find-matching-close tokens (+ after 1) :lbracket :rbracket)))
+        (let
+          ((inner-tokens (slice tokens (+ after 1) end))
+            (next-after (+ end 1)))
+          (let
+            ((sections (split-bracket-content inner-tokens)))
+            (if
+              (= (len sections) 1)
+              (let
+                ((idx-expr (parse-apl-expr inner-tokens)))
+                (let
+                  ((indexed (list :dyad (list :fn-glyph "⌷") idx-expr val-node)))
+                  (maybe-bracket indexed tokens next-after)))
+              (let
+                ((axis-exprs (map (fn (toks) (if (= (len toks) 0) :all (parse-apl-expr toks))) sections)))
+                (let
+                  ((indexed (cons :bracket (cons val-node axis-exprs))))
+                  (maybe-bracket indexed tokens next-after)))))))
+      (list val-node after))))

lib/apl/runtime.sx

@@ -883,7 +883,7 @@
       (let
         ((sub (apl-permutations (- n 1))))
         (reduce
-          (fn (acc p) (append acc (apl-insert-everywhere n p)))
+          (fn (acc p) (append (apl-insert-everywhere n p) acc))
           (list)
           sub)))))
 
@@ -971,6 +971,52 @@
 
 (define apl-quad-print (fn (arr) arr))
 
+(define apl-throw (fn (code msg) (raise (list "apl-error" code msg))))
+
+(define
+  apl-trap-matches?
+  (fn
+    (codes e)
+    (and
+      (list? e)
+      (>= (len e) 2)
+      (= (first e) "apl-error")
+      (or
+        (some (fn (c) (= c 0)) codes)
+        (some (fn (c) (= c (nth e 1))) codes)))))
+
+(define
+  apl-cartesian
+  (fn
+    (lists)
+    (if
+      (= (len lists) 0)
+      (list (list))
+      (let
+        ((rest-prods (apl-cartesian (rest lists))))
+        (reduce
+          (fn (acc x) (append acc (map (fn (p) (cons x p)) rest-prods)))
+          (list)
+          (first lists))))))
+
+(define
+  apl-bracket-multi
+  (fn
+    (axes arr)
+    (let
+      ((shape (get arr :shape)) (ravel (get arr :ravel)))
+      (let
+        ((rank (len shape)) (strides (apl-strides shape)))
+        (let
+          ((axis-info (map (fn (i) (let ((a (nth axes i))) (cond ((= a nil) {:idxs (range 0 (nth shape i)) :scalar? false}) ((= (len (get a :shape)) 0) {:idxs (list (- (first (get a :ravel)) apl-io)) :scalar? true}) (else {:idxs (map (fn (x) (- x apl-io)) (get a :ravel)) :scalar? false})))) (range 0 rank))))
+          (let
+            ((cells (apl-cartesian (map (fn (a) (get a :idxs)) axis-info))))
+            (let
+              ((result-ravel (map (fn (cell) (let ((flat (reduce + 0 (map (fn (i) (* (nth cell i) (nth strides i))) (range 0 rank))))) (nth ravel flat))) cells)))
+              (let
+                ((result-shape (filter (fn (x) (>= x 0)) (map (fn (i) (let ((a (nth axis-info i))) (if (get a :scalar?) -1 (len (get a :idxs))))) (range 0 rank)))))
+                (make-array result-shape result-ravel)))))))))
+
 (define
   apl-reduce
   (fn

lib/apl/scoreboard.json

@@ -3,13 +3,15 @@
     "structural": {"pass": 94, "fail": 0},
     "operators": {"pass": 117, "fail": 0},
     "dfn": {"pass": 24, "fail": 0},
-    "tradfn": {"pass": 20, "fail": 0},
+    "tradfn": {"pass": 25, "fail": 0},
     "valence": {"pass": 14, "fail": 0},
-    "programs": {"pass": 46, "fail": 0},
+    "programs": {"pass": 45, "fail": 0},
     "system": {"pass": 13, "fail": 0},
-    "idioms": {"pass": 34, "fail": 0}
+    "idioms": {"pass": 64, "fail": 0},
+    "eval-ops": {"pass": 14, "fail": 0},
+    "pipeline": {"pass": 40, "fail": 0}
   },
-  "total_pass": 362,
+  "total_pass": 450,
   "total_fail": 0,
-  "total": 362
+  "total": 450
 }

lib/apl/scoreboard.md

@@ -7,12 +7,14 @@ _Generated by `lib/apl/conformance.sh`_
 | structural | 94 | 0 | 94 |
 | operators | 117 | 0 | 117 |
 | dfn | 24 | 0 | 24 |
-| tradfn | 20 | 0 | 20 |
+| tradfn | 25 | 0 | 25 |
 | valence | 14 | 0 | 14 |
-| programs | 46 | 0 | 46 |
+| programs | 45 | 0 | 45 |
 | system | 13 | 0 | 13 |
-| idioms | 34 | 0 | 34 |
+| idioms | 64 | 0 | 64 |
-| **Total** | **362** | **0** | **362** |
+| eval-ops | 14 | 0 | 14 |
+| pipeline | 40 | 0 | 40 |
+| **Total** | **450** | **0** | **450** |
 
 ## Notes
 

lib/apl/test.sh

@@ -18,7 +18,10 @@ TMPFILE=$(mktemp); trap "rm -f $TMPFILE" EXIT
 cat > "$TMPFILE" << 'EPOCHS'
 (epoch 1)
 (load "spec/stdlib.sx")
+(load "lib/r7rs.sx")
 (load "lib/apl/runtime.sx")
+(load "lib/apl/tokenizer.sx")
+(load "lib/apl/parser.sx")
 (load "lib/apl/transpile.sx")
 (epoch 2)
 (eval "(define apl-test-pass 0)")
@@ -34,11 +37,14 @@ cat > "$TMPFILE" << 'EPOCHS'
 (load "lib/apl/tests/programs.sx")
 (load "lib/apl/tests/system.sx")
 (load "lib/apl/tests/idioms.sx")
+(load "lib/apl/tests/eval-ops.sx")
+(load "lib/apl/tests/pipeline.sx")
+(load "lib/apl/tests/programs-e2e.sx")
 (epoch 4)
 (eval "(list apl-test-pass apl-test-fail)")
 EPOCHS
 
-OUTPUT=$(timeout 180 "$SX_SERVER" < "$TMPFILE" 2>/dev/null)
+OUTPUT=$(timeout 300 "$SX_SERVER" < "$TMPFILE" 2>/dev/null)
 
 LINE=$(echo "$OUTPUT" | awk '/^\(ok-len 4 / {getline; print; exit}')
 if [ -z "$LINE" ]; then

lib/apl/tests/eval-ops.sx

@@ -0,0 +1,147 @@
+; Tests for operator handling in apl-eval-ast (Phase 7).
+; Manual AST construction; verifies :derived-fn / :outer / :derived-fn2
+; route through apl-resolve-monadic / apl-resolve-dyadic correctly.
+
+(define mkrv (fn (arr) (get arr :ravel)))
+(define mksh (fn (arr) (get arr :shape)))
+(define mknum (fn (n) (list :num n)))
+(define mkfg (fn (g) (list :fn-glyph g)))
+(define mkmon (fn (g a) (list :monad g a)))
+(define mkdyd (fn (g l r) (list :dyad g l r)))
+(define mkder (fn (op f) (list :derived-fn op f)))
+(define mkdr2 (fn (op f g) (list :derived-fn2 op f g)))
+(define mkout (fn (f) (list :outer "∘." f)))
+
+; helper: literal vector AST via :vec (from list of values)
+(define mkvec (fn (xs) (cons :vec (map (fn (n) (mknum n)) xs))))
+
+; ---------- monadic operators ----------
+
+(apl-test
+  "eval-ast +/ ⍳5 → 15"
+  (mkrv
+    (apl-eval-ast
+      (mkmon (mkder "/" (mkfg "+")) (mkmon (mkfg "⍳") (mknum 5)))
+      {}))
+  (list 15))
+
+(apl-test
+  "eval-ast ×/ ⍳5 → 120"
+  (mkrv
+    (apl-eval-ast
+      (mkmon (mkder "/" (mkfg "×")) (mkmon (mkfg "⍳") (mknum 5)))
+      {}))
+  (list 120))
+
+(apl-test
+  "eval-ast ⌈/ — max reduce"
+  (mkrv
+    (apl-eval-ast
+      (mkmon (mkder "/" (mkfg "⌈")) (mkvec (list 3 1 4 1 5 9 2 6)))
+      {}))
+  (list 9))
+
+(apl-test
+  "eval-ast +\\ scan"
+  (mkrv
+    (apl-eval-ast
+      (mkmon (mkder "\\" (mkfg "+")) (mkvec (list 1 2 3 4 5)))
+      {}))
+  (list 1 3 6 10 15))
+
+(apl-test
+  "eval-ast +⌿ first-axis reduce on vector"
+  (mkrv
+    (apl-eval-ast
+      (mkmon (mkder "⌿" (mkfg "+")) (mkvec (list 1 2 3 4 5)))
+      {}))
+  (list 15))
+
+(apl-test
+  "eval-ast -¨ each-negate"
+  (mkrv
+    (apl-eval-ast
+      (mkmon (mkder "¨" (mkfg "-")) (mkvec (list 1 2 3 4)))
+      {}))
+  (list -1 -2 -3 -4))
+
+(apl-test
+  "eval-ast +⍨ commute (double via x+x)"
+  (mkrv
+    (apl-eval-ast (mkmon (mkder "⍨" (mkfg "+")) (mknum 7)) {}))
+  (list 14))
+
+; ---------- dyadic operators ----------
+
+(apl-test
+  "eval-ast outer ∘.× — multiplication table"
+  (mkrv
+    (apl-eval-ast
+      (mkdyd
+        (mkout (mkfg "×"))
+        (mkvec (list 1 2 3))
+        (mkvec (list 1 2 3)))
+      {}))
+  (list 1 2 3 2 4 6 3 6 9))
+
+(apl-test
+  "eval-ast outer ∘.× shape (3 3)"
+  (mksh
+    (apl-eval-ast
+      (mkdyd
+        (mkout (mkfg "×"))
+        (mkvec (list 1 2 3))
+        (mkvec (list 1 2 3)))
+      {}))
+  (list 3 3))
+
+(apl-test
+  "eval-ast inner +.× — dot product"
+  (mkrv
+    (apl-eval-ast
+      (mkdyd
+        (mkdr2 "." (mkfg "+") (mkfg "×"))
+        (mkvec (list 1 2 3))
+        (mkvec (list 4 5 6)))
+      {}))
+  (list 32))
+
+(apl-test
+  "eval-ast inner ∧.= equal vectors"
+  (mkrv
+    (apl-eval-ast
+      (mkdyd
+        (mkdr2 "." (mkfg "∧") (mkfg "="))
+        (mkvec (list 1 2 3))
+        (mkvec (list 1 2 3)))
+      {}))
+  (list 1))
+
+(apl-test
+  "eval-ast each-dyadic +¨"
+  (mkrv
+    (apl-eval-ast
+      (mkdyd
+        (mkder "¨" (mkfg "+"))
+        (mkvec (list 1 2 3))
+        (mkvec (list 10 20 30)))
+      {}))
+  (list 11 22 33))
+
+(apl-test
+  "eval-ast commute -⍨ (subtract swapped)"
+  (mkrv
+    (apl-eval-ast
+      (mkdyd (mkder "⍨" (mkfg "-")) (mknum 5) (mknum 3))
+      {}))
+  (list -2))
+
+; ---------- nested operators ----------
+
+(apl-test
+  "eval-ast +/¨ — sum of each"
+  (mkrv
+    (apl-eval-ast
+      (mkmon (mkder "/" (mkfg "+")) (mkvec (list 10 20 30)))
+      {}))
+  (list 60))

lib/apl/tests/idioms.sx

@@ -222,3 +222,138 @@
   (mkrv
     (apl-shape (apl-shape (make-array (list 2 3) (list 1 2 3 4 5 6)))))
   (list 2))
+
+(apl-test
+  "src: +/⍳N → triangular(N)"
+  (mkrv (apl-run "+/⍳100"))
+  (list 5050))
+
+(apl-test "src: ×/⍳N → N!" (mkrv (apl-run "×/⍳6")) (list 720))
+
+(apl-test
+  "src: ⌈/V — max"
+  (mkrv (apl-run "⌈/3 1 4 1 5 9 2 6"))
+  (list 9))
+
+(apl-test
+  "src: ⌊/V — min"
+  (mkrv (apl-run "⌊/3 1 4 1 5 9 2 6"))
+  (list 1))
+
+(apl-test
+  "src: range = (⌈/V) - ⌊/V"
+  (mkrv (apl-run "(⌈/3 1 4 1 5 9 2 6) - ⌊/3 1 4 1 5 9 2 6"))
+  (list 8))
+
+(apl-test
+  "src: +\\V — running sum"
+  (mkrv (apl-run "+\\1 2 3 4 5"))
+  (list 1 3 6 10 15))
+
+(apl-test
+  "src: ×\\V — running product"
+  (mkrv (apl-run "×\\1 2 3 4 5"))
+  (list 1 2 6 24 120))
+
+(apl-test
+  "src: V × V — squares"
+  (mkrv (apl-run "(⍳5) × ⍳5"))
+  (list 1 4 9 16 25))
+
+(apl-test
+  "src: +/V × V — sum of squares"
+  (mkrv (apl-run "+/(⍳5) × ⍳5"))
+  (list 55))
+
+(apl-test "src: ∧/V — all-true" (mkrv (apl-run "∧/1 1 1 1")) (list 1))
+
+(apl-test "src: ∨/V — any-true" (mkrv (apl-run "∨/0 0 1 0")) (list 1))
+
+(apl-test "src: 0 = N|M — divides" (mkrv (apl-run "0 = 3 | 12")) (list 1))
+
+(apl-test
+  "src: 2 | V — parity"
+  (mkrv (apl-run "2 | 1 2 3 4 5 6"))
+  (list 1 0 1 0 1 0))
+
+(apl-test
+  "src: +/2|V — count odd"
+  (mkrv (apl-run "+/2 | 1 2 3 4 5 6"))
+  (list 3))
+
+(apl-test "src: ⍴ V" (mkrv (apl-run "⍴ 1 2 3 4 5")) (list 5))
+
+(apl-test
+  "src: ⍴⍴ M — rank"
+  (mkrv (apl-run "⍴ ⍴ (2 3) ⍴ ⍳6"))
+  (list 2))
+
+(apl-test
+  "src: N⍴1 — vector of ones"
+  (mkrv (apl-run "5 ⍴ 1"))
+  (list 1 1 1 1 1))
+
+(apl-test
+  "src: ⍳N ∘.= ⍳N — identity matrix"
+  (mkrv (apl-run "(⍳3) ∘.= ⍳3"))
+  (list 1 0 0 0 1 0 0 0 1))
+
+(apl-test
+  "src: ⍳N ∘.× ⍳N — multiplication table"
+  (mkrv (apl-run "(⍳3) ∘.× ⍳3"))
+  (list 1 2 3 2 4 6 3 6 9))
+
+(apl-test
+  "src: V +.× V — dot product"
+  (mkrv (apl-run "1 2 3 +.× 4 5 6"))
+  (list 32))
+
+(apl-test
+  "src: ∧.= V — vectors equal?"
+  (mkrv (apl-run "1 2 3 ∧.= 1 2 3"))
+  (list 1))
+
+(apl-test
+  "src: V[1] — first element"
+  (mkrv (apl-run "(10 20 30 40)[1]"))
+  (list 10))
+
+(apl-test
+  "src: 1↑V — first via take"
+  (mkrv (apl-run "1 ↑ 10 20 30 40"))
+  (list 10))
+
+(apl-test
+  "src: 1↓V — drop first"
+  (mkrv (apl-run "1 ↓ 10 20 30 40"))
+  (list 20 30 40))
+
+(apl-test
+  "src: ¯1↓V — drop last"
+  (mkrv (apl-run "¯1 ↓ 10 20 30 40"))
+  (list 10 20 30))
+
+(apl-test
+  "src: ⌽V — reverse"
+  (mkrv (apl-run "⌽ 1 2 3 4 5"))
+  (list 5 4 3 2 1))
+
+(apl-test
+  "src: ≢V — tally"
+  (mkrv (apl-run "≢ 9 8 7 6 5 4 3 2 1"))
+  (list 9))
+
+(apl-test
+  "src: ,M — ravel"
+  (mkrv (apl-run ", (2 3) ⍴ ⍳6"))
+  (list 1 2 3 4 5 6))
+
+(apl-test
+  "src: A=V — count occurrences"
+  (mkrv (apl-run "+/2 = 1 2 3 2 1 3 2"))
+  (list 3))
+
+(apl-test
+  "src: ⌈/(V × V) — max squared"
+  (mkrv (apl-run "⌈/(1 2 3 4 5) × 1 2 3 4 5"))
+  (list 25))

lib/apl/tests/pipeline.sx

@@ -0,0 +1,314 @@
+; End-to-end pipeline tests: source string → tokenize → parse → eval-ast → array.
+; Verifies the full stack as a single function call (apl-run).
+
+(define mkrv (fn (arr) (get arr :ravel)))
+(define mksh (fn (arr) (get arr :shape)))
+
+; ---------- scalars ----------
+
+(apl-test "apl-run \"42\" → scalar 42" (mkrv (apl-run "42")) (list 42))
+
+(apl-test "apl-run \"¯7\" → scalar -7" (mkrv (apl-run "¯7")) (list -7))
+
+; ---------- strands ----------
+
+(apl-test
+  "apl-run \"1 2 3\" → vector"
+  (mkrv (apl-run "1 2 3"))
+  (list 1 2 3))
+
+(apl-test "apl-run \"1 2 3\" shape" (mksh (apl-run "1 2 3")) (list 3))
+
+; ---------- dyadic arithmetic ----------
+
+(apl-test "apl-run \"2 + 3\" → 5" (mkrv (apl-run "2 + 3")) (list 5))
+
+(apl-run "2 × 3 + 4")  ; right-to-left
+
+(apl-test
+  "apl-run \"2 × 3 + 4\" → 14 (right-to-left)"
+  (mkrv (apl-run "2 × 3 + 4"))
+  (list 14))
+
+(apl-test
+  "apl-run \"1 2 3 + 4 5 6\" → 5 7 9"
+  (mkrv (apl-run "1 2 3 + 4 5 6"))
+  (list 5 7 9))
+
+(apl-test
+  "apl-run \"3 × 1 2 3 4\" → scalar broadcast"
+  (mkrv (apl-run "3 × 1 2 3 4"))
+  (list 3 6 9 12))
+
+; ---------- monadic primitives ----------
+
+(apl-test
+  "apl-run \"⍳5\" → 1..5"
+  (mkrv (apl-run "⍳5"))
+  (list 1 2 3 4 5))
+
+(apl-test
+  "apl-run \"-3\" → -3 (monadic negate)"
+  (mkrv (apl-run "-3"))
+  (list -3))
+
+(apl-test
+  "apl-run \"⌈/ 1 3 9 5 7\" → 9 (max-reduce)"
+  (mkrv (apl-run "⌈/ 1 3 9 5 7"))
+  (list 9))
+
+(apl-test
+  "apl-run \"⌊/ 4 7 2 9 1 3\" → 1 (min-reduce)"
+  (mkrv (apl-run "⌊/ 4 7 2 9 1 3"))
+  (list 1))
+
+; ---------- operators ----------
+
+(apl-test "apl-run \"+/⍳5\" → 15" (mkrv (apl-run "+/⍳5")) (list 15))
+
+(apl-test "apl-run \"×/⍳5\" → 120" (mkrv (apl-run "×/⍳5")) (list 120))
+
+(apl-test
+  "apl-run \"⌈/3 1 4 1 5 9 2\" → 9"
+  (mkrv (apl-run "⌈/3 1 4 1 5 9 2"))
+  (list 9))
+
+(apl-test
+  "apl-run \"+\\\\⍳5\" → triangular numbers"
+  (mkrv (apl-run "+\\⍳5"))
+  (list 1 3 6 10 15))
+
+; ---------- outer / inner products ----------
+
+(apl-test
+  "apl-run \"1 2 3 ∘.× 1 2 3\" → mult table values"
+  (mkrv (apl-run "1 2 3 ∘.× 1 2 3"))
+  (list 1 2 3 2 4 6 3 6 9))
+
+(apl-test
+  "apl-run \"1 2 3 +.× 4 5 6\" → dot product 32"
+  (mkrv (apl-run "1 2 3 +.× 4 5 6"))
+  (list 32))
+
+; ---------- shape ----------
+
+(apl-test
+  "apl-run \"⍴ 1 2 3 4 5\" → 5"
+  (mkrv (apl-run "⍴ 1 2 3 4 5"))
+  (list 5))
+
+(apl-test "apl-run \"⍴⍳10\" → 10" (mkrv (apl-run "⍴⍳10")) (list 10))
+
+; ---------- comparison ----------
+
+(apl-test "apl-run \"3 < 5\" → 1" (mkrv (apl-run "3 < 5")) (list 1))
+
+(apl-test "apl-run \"5 = 5\" → 1" (mkrv (apl-run "5 = 5")) (list 1))
+
+(apl-test
+  "apl-run \"1 2 3 = 1 0 3\" → 1 0 1"
+  (mkrv (apl-run "1 2 3 = 1 0 3"))
+  (list 1 0 1))
+
+; ---------- famous one-liners ----------
+
+(apl-test
+  "apl-run \"+/(⍳10)\" → sum 1..10 = 55"
+  (mkrv (apl-run "+/(⍳10)"))
+  (list 55))
+
+(apl-test
+  "apl-run \"×/⍳10\" → 10! = 3628800"
+  (mkrv (apl-run "×/⍳10"))
+  (list 3628800))
+
+(apl-test "apl-run \"⎕IO\" → 1" (mkrv (apl-run "⎕IO")) (list 1))
+
+(apl-test "apl-run \"⎕ML\" → 1" (mkrv (apl-run "⎕ML")) (list 1))
+
+(apl-test "apl-run \"⎕FR\" → 1248" (mkrv (apl-run "⎕FR")) (list 1248))
+
+(apl-test "apl-run \"⎕TS\" shape (7)" (mksh (apl-run "⎕TS")) (list 7))
+
+(apl-test "apl-run \"⎕FMT 42\" → \"42\"" (apl-run "⎕FMT 42") "42")
+
+(apl-test
+  "apl-run \"⎕FMT 1 2 3\" → \"1 2 3\""
+  (apl-run "⎕FMT 1 2 3")
+  "1 2 3")
+
+(apl-test
+  "apl-run \"⎕FMT ⍳5\" → \"1 2 3 4 5\""
+  (apl-run "⎕FMT ⍳5")
+  "1 2 3 4 5")
+
+(apl-test "apl-run \"⎕IO + 4\" → 5" (mkrv (apl-run "⎕IO + 4")) (list 5))
+
+(apl-test
+  "apl-run \"(10 20 30 40 50)[3]\" → 30"
+  (mkrv (apl-run "(10 20 30 40 50)[3]"))
+  (list 30))
+
+(apl-test
+  "apl-run \"(⍳10)[5]\" → 5"
+  (mkrv (apl-run "(⍳10)[5]"))
+  (list 5))
+
+(apl-test
+  "apl-run \"A ← 100 200 300 ⋄ A[2]\" → 200"
+  (mkrv (apl-run "A ← 100 200 300 ⋄ A[2]"))
+  (list 200))
+
+(apl-test
+  "apl-run \"V ← ⍳10 ⋄ V[3]\" → 3"
+  (mkrv (apl-run "V ← ⍳10 ⋄ V[3]"))
+  (list 3))
+
+(apl-test
+  "apl-run \"(10 20 30)[1]\" → 10 (1-indexed)"
+  (mkrv (apl-run "(10 20 30)[1]"))
+  (list 10))
+
+(apl-test
+  "apl-run \"V ← 10 20 30 40 50 ⋄ V[3] + 1\" → 31"
+  (mkrv (apl-run "V ← 10 20 30 40 50 ⋄ V[3] + 1"))
+  (list 31))
+
+(apl-test
+  "apl-run \"(⍳5)[3] × 7\" → 21"
+  (mkrv (apl-run "(⍳5)[3] × 7"))
+  (list 21))
+
+(apl-test "decimal: 3.7 → 3.7" (mkrv (apl-run "3.7")) (list 3.7))
+
+(apl-test "decimal: ¯2.5 → -2.5" (mkrv (apl-run "¯2.5")) (list -2.5))
+
+(apl-test "decimal: 1.5 + 2.5 → 4" (mkrv (apl-run "1.5 + 2.5")) (list 4))
+
+(apl-test "decimal: ⌊3.7 → 3" (mkrv (apl-run "⌊ 3.7")) (list 3))
+
+(apl-test "decimal: ⌈3.7 → 4" (mkrv (apl-run "⌈ 3.7")) (list 4))
+
+(apl-test
+  "⎕← scalar passthrough"
+  (mkrv (apl-run "⎕← 42"))
+  (list 42))
+
+(apl-test
+  "⎕← vector passthrough"
+  (mkrv (apl-run "⎕← 1 2 3"))
+  (list 1 2 3))
+
+(apl-test
+  "string: 'abc' → 3-char vector"
+  (mkrv (apl-run "'abc'"))
+  (list "a" "b" "c"))
+
+(apl-test "string: 'a' is rank-0 scalar" (mksh (apl-run "'a'")) (list))
+
+(apl-test "string: 'hello' shape (5)" (mksh (apl-run "'hello'")) (list 5))
+
+(apl-test
+  "named-fn: f ← {⍺+⍵} ⋄ 3 f 4 → 7"
+  (mkrv (apl-run "f ← {⍺+⍵} ⋄ 3 f 4"))
+  (list 7))
+
+(apl-test
+  "named-fn monadic: sq ← {⍵×⍵} ⋄ sq 7 → 49"
+  (mkrv (apl-run "sq ← {⍵×⍵} ⋄ sq 7"))
+  (list 49))
+
+(apl-test
+  "named-fn dyadic: hyp ← {((⍺×⍺)+⍵×⍵)} ⋄ 3 hyp 4 → 25"
+  (mkrv (apl-run "hyp ← {((⍺×⍺)+⍵×⍵)} ⋄ 3 hyp 4"))
+  (list 25))
+
+(apl-test
+  "named-fn: dbl ← {⍵+⍵} ⋄ dbl ⍳5"
+  (mkrv (apl-run "dbl ← {⍵+⍵} ⋄ dbl ⍳5"))
+  (list 2 4 6 8 10))
+
+(apl-test
+  "named-fn factorial via ∇ recursion"
+  (mkrv (apl-run "fact ← {0=⍵:1 ⋄ ⍵×∇⍵-1} ⋄ fact 5"))
+  (list 120))
+
+(apl-test
+  "named-fn used twice in expr: dbl ← {⍵+⍵} ⋄ (dbl 3) + dbl 4"
+  (mkrv (apl-run "dbl ← {⍵+⍵} ⋄ (dbl 3) + dbl 4"))
+  (list 14))
+
+(apl-test
+  "named-fn with vector arg: neg ← {-⍵} ⋄ neg 1 2 3"
+  (mkrv (apl-run "neg ← {-⍵} ⋄ neg 1 2 3"))
+  (list -1 -2 -3))
+
+(apl-test
+  "multi-axis: M[2;2] → center"
+  (mkrv (apl-run "M ← (3 3) ⍴ ⍳9 ⋄ M[2;2]"))
+  (list 5))
+
+(apl-test
+  "multi-axis: M[1;] → first row"
+  (mkrv (apl-run "M ← (3 3) ⍴ ⍳9 ⋄ M[1;]"))
+  (list 1 2 3))
+
+(apl-test
+  "multi-axis: M[;2] → second column"
+  (mkrv (apl-run "M ← (3 3) ⍴ ⍳9 ⋄ M[;2]"))
+  (list 2 5 8))
+
+(apl-test
+  "multi-axis: M[1 2;1 2] → 2x2 block"
+  (mkrv (apl-run "M ← (2 3) ⍴ ⍳6 ⋄ M[1 2;1 2]"))
+  (list 1 2 4 5))
+
+(apl-test
+  "multi-axis: M[1 2;1 2] shape (2 2)"
+  (mksh (apl-run "M ← (2 3) ⍴ ⍳6 ⋄ M[1 2;1 2]"))
+  (list 2 2))
+
+(apl-test
+  "multi-axis: M[;] full matrix"
+  (mkrv (apl-run "M ← (2 2) ⍴ 10 20 30 40 ⋄ M[;]"))
+  (list 10 20 30 40))
+
+(apl-test
+  "multi-axis: M[1;] shape collapsed"
+  (mksh (apl-run "M ← (3 3) ⍴ ⍳9 ⋄ M[1;]"))
+  (list 3))
+
+(apl-test
+  "multi-axis: select all rows of column 3"
+  (mkrv (apl-run "M ← (4 3) ⍴ 1 2 3 4 5 6 7 8 9 10 11 12 ⋄ M[;3]"))
+  (list 3 6 9 12))
+
+(apl-test
+  "train: mean = (+/÷≢) on 1..5"
+  (mkrv (apl-run "(+/÷≢) 1 2 3 4 5"))
+  (list 3))
+
+(apl-test
+  "train: mean of 2 4 6 8 10"
+  (mkrv (apl-run "(+/÷≢) 2 4 6 8 10"))
+  (list 6))
+
+(apl-test
+  "train 2-atop: (- ⌊) 5 → -5"
+  (mkrv (apl-run "(- ⌊) 5"))
+  (list -5))
+
+(apl-test
+  "train 3-fork dyadic: 2(+×-)5 → -21"
+  (mkrv (apl-run "2 (+ × -) 5"))
+  (list -21))
+
+(apl-test
+  "train: range = (⌈/-⌊/) on vector"
+  (mkrv (apl-run "(⌈/-⌊/) 3 1 4 1 5 9 2 6"))
+  (list 8))
+
+(apl-test
+  "train: mean of ⍳10 has shape ()"
+  (mksh (apl-run "(+/÷≢) ⍳10"))
+  (list))

lib/apl/tests/programs-e2e.sx

@@ -0,0 +1,96 @@
+; End-to-end tests of the classic-program archetypes — running APL
+; source through the full pipeline (tokenize → parse → eval-ast → runtime).
+;
+; These mirror the algorithms documented in lib/apl/tests/programs/*.apl
+; but use forms our pipeline supports today (named functions instead of
+; the inline ⍵← rebinding idiom; multi-stmt over single one-liners).
+
+(define mkrv (fn (arr) (get arr :ravel)))
+(define mksh (fn (arr) (get arr :shape)))
+
+; ---------- factorial via ∇ recursion (cf. n-queens style) ----------
+
+(apl-test
+  "e2e: factorial 5! = 120"
+  (mkrv (apl-run "fact ← {0=⍵:1 ⋄ ⍵×∇⍵-1} ⋄ fact 5"))
+  (list 120))
+
+(apl-test
+  "e2e: factorial 7! = 5040"
+  (mkrv (apl-run "fact ← {0=⍵:1 ⋄ ⍵×∇⍵-1} ⋄ fact 7"))
+  (list 5040))
+
+(apl-test
+  "e2e: factorial via ×/⍳N (no recursion)"
+  (mkrv (apl-run "fact ← {×/⍳⍵} ⋄ fact 6"))
+  (list 720))
+
+; ---------- sum / triangular numbers (sum-1..N) ----------
+
+(apl-test
+  "e2e: triangular(10) = 55"
+  (mkrv (apl-run "tri ← {+/⍳⍵} ⋄ tri 10"))
+  (list 55))
+
+(apl-test
+  "e2e: triangular(100) = 5050"
+  (mkrv (apl-run "tri ← {+/⍳⍵} ⋄ tri 100"))
+  (list 5050))
+
+; ---------- sum of squares ----------
+
+(apl-test
+  "e2e: sum-of-squares 1..5 = 55"
+  (mkrv (apl-run "ss ← {+/⍵×⍵} ⋄ ss ⍳5"))
+  (list 55))
+
+(apl-test
+  "e2e: sum-of-squares 1..10 = 385"
+  (mkrv (apl-run "ss ← {+/⍵×⍵} ⋄ ss ⍳10"))
+  (list 385))
+
+; ---------- divisor-counting (prime-sieve building blocks) ----------
+
+(apl-test
+  "e2e: divisor counts 1..5 via outer mod"
+  (mkrv (apl-run "P ← ⍳ 5 ⋄ +⌿ 0 = P ∘.| P"))
+  (list 1 2 2 3 2))
+
+(apl-test
+  "e2e: divisor counts 1..10"
+  (mkrv (apl-run "P ← ⍳ 10 ⋄ +⌿ 0 = P ∘.| P"))
+  (list 1 2 2 3 2 4 2 4 3 4))
+
+(apl-test
+  "e2e: prime-mask 1..10 (count==2)"
+  (mkrv (apl-run "P ← ⍳ 10 ⋄ 2 = +⌿ 0 = P ∘.| P"))
+  (list 0 1 1 0 1 0 1 0 0 0))
+
+; ---------- monadic primitives chained ----------
+
+(apl-test
+  "e2e: sum of |abs| = 15"
+  (mkrv (apl-run "+/|¯1 ¯2 ¯3 ¯4 ¯5"))
+  (list 15))
+
+(apl-test
+  "e2e: max of squares 1..6"
+  (mkrv (apl-run "⌈/(⍳6)×⍳6"))
+  (list 36))
+
+; ---------- nested named functions ----------
+
+(apl-test
+  "e2e: compose dbl and sq via two named fns"
+  (mkrv (apl-run "dbl ← {⍵+⍵} ⋄ sq ← {⍵×⍵} ⋄ sq dbl 3"))
+  (list 36))
+
+(apl-test
+  "e2e: max-of-two as named dyadic fn"
+  (mkrv (apl-run "mx ← {⍺⌈⍵} ⋄ 5 mx 3"))
+  (list 5))
+
+(apl-test
+  "e2e: sqrt-via-newton 1 step from 1 → 2.5"
+  (mkrv (apl-run "step ← {(⍵+⍺÷⍵)÷2} ⋄ 4 step 1"))
+  (list 2.5))

lib/apl/tests/tradfn.sx

@@ -18,6 +18,10 @@
 
 (define mksel (fn (v cs d) (list :select v cs d)))
 
+(define mktrap (fn (codes t c) (list :trap codes t c)))
+
+(define mkthr (fn (code msg) (list :throw code msg)))
+
 (apl-test
   "tradfn R←L+W simple add"
   (mkrv (apl-call-tradfn {:result "R" :omega "W" :stmts (list (mkasg "R" (mkdyd "+" (mknm "L") (mknm "W")))) :alpha "L"} (apl-scalar 5) (apl-scalar 7)))
@@ -125,3 +129,28 @@
   "tradfn :For factorial 1..5"
   (mkrv (apl-call-tradfn {:result "R" :omega "W" :stmts (list (mkasg "R" (mknum 1)) (mkfor "x" (mkmon "⍳" (mknm "W")) (list (mkasg "R" (mkdyd "×" (mknm "R") (mknm "x")))))) :alpha nil} nil (apl-scalar 5)))
   (list 120))
+
+(apl-test
+  "tradfn :Trap normal flow (no error)"
+  (mkrv (apl-call-tradfn {:result "R" :omega nil :stmts (list (mktrap (list 0) (list (mkasg "R" (mknum 99))) (list (mkasg "R" (mknum -1))))) :alpha nil} nil nil))
+  (list 99))
+
+(apl-test
+  "tradfn :Trap catches matching code"
+  (mkrv (apl-call-tradfn {:result "R" :omega nil :stmts (list (mktrap (list 5) (list (mkthr 5 "boom")) (list (mkasg "R" (mknum 42))))) :alpha nil} nil nil))
+  (list 42))
+
+(apl-test
+  "tradfn :Trap catch-all (code 0)"
+  (mkrv (apl-call-tradfn {:result "R" :omega nil :stmts (list (mktrap (list 0) (list (mkthr 99 "any")) (list (mkasg "R" (mknum 1))))) :alpha nil} nil nil))
+  (list 1))
+
+(apl-test
+  "tradfn :Trap catches one of many codes"
+  (mkrv (apl-call-tradfn {:result "R" :omega nil :stmts (list (mktrap (list 1 2 3) (list (mkthr 2 "two")) (list (mkasg "R" (mknum 22))))) :alpha nil} nil nil))
+  (list 22))
+
+(apl-test
+  "tradfn :Trap continues to next stmt after catch"
+  (mkrv (apl-call-tradfn {:result "R" :omega nil :stmts (list (mktrap (list 7) (list (mkthr 7 "c")) (list (mkasg "R" (mknum 10)))) (mkasg "R" (mkdyd "+" (mknm "R") (mknum 5)))) :alpha nil} nil nil))
+  (list 15))

lib/apl/tokenizer.sx

@@ -1,8 +1,8 @@
 (define apl-glyph-set
   (list "+" "-" "×" "÷" "*" "⍟" "⌈" "⌊" "|" "!" "?" "○" "~" "<" "≤" "=" "≥" ">" "≠"
-        "∊" "∧" "∨" "⍱" "⍲" "," "⍪" "⍴" "⌽" "⊖" "⍉" "↑" "↓" "⊂" "⊃" "⊆"
+        "≢" "≡" "∊" "∧" "∨" "⍱" "⍲" "," "⍪" "⍴" "⌽" "⊖" "⍉" "↑" "↓" "⊂" "⊃" "⊆"
         "∪" "∩" "⍳" "⍸" "⌷" "⍋" "⍒" "⊥" "⊤" "⊣" "⊢" "⍎" "⍕"
-        "⍺" "⍵" "∇" "/" "\\" "¨" "⍨" "∘" "." "⍣" "⍤" "⍥" "@" "¯"))
+        "⍺" "⍵" "∇" "/" "⌿" "\\" "⍀" "¨" "⍨" "∘" "." "⍣" "⍤" "⍥" "@" "¯"))
 
 (define apl-glyph?
   (fn (ch)
@@ -138,12 +138,22 @@
                  (begin
                    (consume! "¯")
                    (let ((digits (read-digits! "")))
-                     (tok-push! :num (- 0 (parse-int digits 0))))
+                     (if (and (< pos src-len) (= (cur-byte) ".")
+                              (< (+ pos 1) src-len) (apl-digit? (nth source (+ pos 1))))
+                       (begin (advance!)
+                              (let ((frac (read-digits! "")))
+                                (tok-push! :num (- 0 (string->number (str digits "." frac))))))
+                       (tok-push! :num (- 0 (parse-int digits 0)))))
                    (scan!)))
                 ((apl-digit? ch)
                  (begin
                    (let ((digits (read-digits! "")))
-                     (tok-push! :num (parse-int digits 0)))
+                     (if (and (< pos src-len) (= (cur-byte) ".")
+                              (< (+ pos 1) src-len) (apl-digit? (nth source (+ pos 1))))
+                       (begin (advance!)
+                              (let ((frac (read-digits! "")))
+                                (tok-push! :num (string->number (str digits "." frac)))))
+                       (tok-push! :num (parse-int digits 0))))
                    (scan!)))
                 ((= ch "'")
                  (begin
@@ -155,7 +165,9 @@
                  (let ((start pos))
                    (begin
                      (if (cur-sw? "⎕") (consume! "⎕") (advance!))
-                     (read-ident-cont!)
+                     (if (and (< pos src-len) (cur-sw? "←"))
+                       (consume! "←")
+                       (read-ident-cont!))
                      (tok-push! :name (slice source start pos))
                      (scan!))))
                 (true

lib/apl/transpile.sx

@@ -39,6 +39,8 @@
       ((= g "⊖") apl-reverse-first)
       ((= g "⍋") apl-grade-up)
       ((= g "⍒") apl-grade-down)
+      ((= g "⎕FMT") apl-quad-fmt)
+      ((= g "⎕←") apl-quad-print)
       (else (error "no monadic fn for glyph")))))
 
 (define
@@ -96,6 +98,15 @@
       ((tag (first node)))
       (cond
         ((= tag :num) (apl-scalar (nth node 1)))
+        ((= tag :str)
+          (let
+            ((s (nth node 1)))
+            (if
+              (= (len s) 1)
+              (apl-scalar s)
+              (make-array
+                (list (len s))
+                (map (fn (i) (slice s i (+ i 1))) (range 0 (len s)))))))
         ((= tag :vec)
           (let
             ((items (rest node)))
@@ -110,34 +121,44 @@
             (cond
               ((= nm "⍺") (get env "alpha"))
               ((= nm "⍵") (get env "omega"))
+              ((= nm "⎕IO") (apl-quad-io))
+              ((= nm "⎕ML") (apl-quad-ml))
+              ((= nm "⎕FR") (apl-quad-fr))
+              ((= nm "⎕TS") (apl-quad-ts))
               (else (get env nm)))))
         ((= tag :monad)
           (let
             ((fn-node (nth node 1)) (arg (nth node 2)))
-            (let
+            (if
-              ((g (nth fn-node 1)))
+              (and (= (first fn-node) :fn-glyph) (= (nth fn-node 1) "∇"))
-              (if
+              (apl-call-dfn-m (get env "nabla") (apl-eval-ast arg env))
-                (= g "∇")
+              ((apl-resolve-monadic fn-node env) (apl-eval-ast arg env)))))
-                (apl-call-dfn-m (get env "nabla") (apl-eval-ast arg env))
-                ((apl-monadic-fn g) (apl-eval-ast arg env))))))
         ((= tag :dyad)
           (let
             ((fn-node (nth node 1))
               (lhs (nth node 2))
               (rhs (nth node 3)))
-            (let
+            (if
-              ((g (nth fn-node 1)))
+              (and (= (first fn-node) :fn-glyph) (= (nth fn-node 1) "∇"))
-              (if
+              (apl-call-dfn
-                (= g "∇")
+                (get env "nabla")
-                (apl-call-dfn
+                (apl-eval-ast lhs env)
-                  (get env "nabla")
+                (apl-eval-ast rhs env))
-                  (apl-eval-ast lhs env)
+              ((apl-resolve-dyadic fn-node env)
-                  (apl-eval-ast rhs env))
+                (apl-eval-ast lhs env)
-                ((apl-dyadic-fn g)
+                (apl-eval-ast rhs env)))))
-                  (apl-eval-ast lhs env)
-                  (apl-eval-ast rhs env))))))
         ((= tag :program) (apl-eval-stmts (rest node) env))
         ((= tag :dfn) node)
+        ((= tag :bracket)
+          (let
+            ((arr-expr (nth node 1)) (axis-exprs (rest (rest node))))
+            (let
+              ((arr (apl-eval-ast arr-expr env))
+                (axes
+                  (map
+                    (fn (a) (if (= a :all) nil (apl-eval-ast a env)))
+                    axis-exprs)))
+              (apl-bracket-multi axes arr))))
         (else (error (list "apl-eval-ast: unknown node tag" tag node)))))))
 
 (define
@@ -275,6 +296,17 @@
           (let
             ((val (apl-eval-ast (nth stmt 1) env)))
             (apl-tradfn-eval-select val (nth stmt 2) (nth stmt 3) env)))
+        ((= tag :trap)
+          (let
+            ((codes (nth stmt 1))
+              (try-block (nth stmt 2))
+              (catch-block (nth stmt 3)))
+            (guard
+              (e
+                ((apl-trap-matches? codes e)
+                  (apl-tradfn-eval-block catch-block env)))
+              (apl-tradfn-eval-block try-block env))))
+        ((= tag :throw) (apl-throw (nth stmt 1) (nth stmt 2)))
         (else (begin (apl-eval-ast stmt env) env))))))
 
 (define
@@ -369,3 +401,140 @@
         (if alpha (apl-call-dfn f alpha omega) (apl-call-dfn-m f omega)))
       ((dict? f) (apl-call-tradfn f alpha omega))
       (else (error "apl-call: not a function")))))
+
+(define
+  apl-resolve-monadic
+  (fn
+    (fn-node env)
+    (let
+      ((tag (first fn-node)))
+      (cond
+        ((= tag :fn-glyph) (apl-monadic-fn (nth fn-node 1)))
+        ((= tag :derived-fn)
+          (let
+            ((op (nth fn-node 1)) (inner (nth fn-node 2)))
+            (cond
+              ((= op "/")
+                (let
+                  ((f (apl-resolve-dyadic inner env)))
+                  (fn (arr) (apl-reduce f arr))))
+              ((= op "⌿")
+                (let
+                  ((f (apl-resolve-dyadic inner env)))
+                  (fn (arr) (apl-reduce-first f arr))))
+              ((= op "\\")
+                (let
+                  ((f (apl-resolve-dyadic inner env)))
+                  (fn (arr) (apl-scan f arr))))
+              ((= op "⍀")
+                (let
+                  ((f (apl-resolve-dyadic inner env)))
+                  (fn (arr) (apl-scan-first f arr))))
+              ((= op "¨")
+                (let
+                  ((f (apl-resolve-monadic inner env)))
+                  (fn (arr) (apl-each f arr))))
+              ((= op "⍨")
+                (let
+                  ((f (apl-resolve-dyadic inner env)))
+                  (fn (arr) (apl-commute f arr))))
+              (else (error "apl-resolve-monadic: unsupported op")))))
+        ((= tag :fn-name)
+          (let
+            ((nm (nth fn-node 1)))
+            (let
+              ((bound (get env nm)))
+              (if
+                (and
+                  (list? bound)
+                  (> (len bound) 0)
+                  (= (first bound) :dfn))
+                (fn (arg) (apl-call-dfn-m bound arg))
+                (error "apl-resolve-monadic: name not bound to dfn")))))
+        ((= tag :train)
+          (let
+            ((fns (rest fn-node)))
+            (let
+              ((n (len fns)))
+              (cond
+                ((= n 2)
+                  (let
+                    ((g (apl-resolve-monadic (nth fns 0) env))
+                      (h (apl-resolve-monadic (nth fns 1) env)))
+                    (fn (arg) (g (h arg)))))
+                ((= n 3)
+                  (let
+                    ((f (apl-resolve-monadic (nth fns 0) env))
+                      (g (apl-resolve-dyadic (nth fns 1) env))
+                      (h (apl-resolve-monadic (nth fns 2) env)))
+                    (fn (arg) (g (f arg) (h arg)))))
+                (else (error "monadic train arity not 2 or 3"))))))
+        (else (error "apl-resolve-monadic: unknown fn-node tag"))))))
+
+(define
+  apl-resolve-dyadic
+  (fn
+    (fn-node env)
+    (let
+      ((tag (first fn-node)))
+      (cond
+        ((= tag :fn-glyph) (apl-dyadic-fn (nth fn-node 1)))
+        ((= tag :derived-fn)
+          (let
+            ((op (nth fn-node 1)) (inner (nth fn-node 2)))
+            (cond
+              ((= op "¨")
+                (let
+                  ((f (apl-resolve-dyadic inner env)))
+                  (fn (a b) (apl-each-dyadic f a b))))
+              ((= op "⍨")
+                (let
+                  ((f (apl-resolve-dyadic inner env)))
+                  (fn (a b) (apl-commute-dyadic f a b))))
+              (else (error "apl-resolve-dyadic: unsupported op")))))
+        ((= tag :fn-name)
+          (let
+            ((nm (nth fn-node 1)))
+            (let
+              ((bound (get env nm)))
+              (if
+                (and
+                  (list? bound)
+                  (> (len bound) 0)
+                  (= (first bound) :dfn))
+                (fn (a b) (apl-call-dfn bound a b))
+                (error "apl-resolve-dyadic: name not bound to dfn")))))
+        ((= tag :outer)
+          (let
+            ((inner (nth fn-node 2)))
+            (let
+              ((f (apl-resolve-dyadic inner env)))
+              (fn (a b) (apl-outer f a b)))))
+        ((= tag :derived-fn2)
+          (let
+            ((f-node (nth fn-node 2)) (g-node (nth fn-node 3)))
+            (let
+              ((f (apl-resolve-dyadic f-node env))
+                (g (apl-resolve-dyadic g-node env)))
+              (fn (a b) (apl-inner f g a b)))))
+        ((= tag :train)
+          (let
+            ((fns (rest fn-node)))
+            (let
+              ((n (len fns)))
+              (cond
+                ((= n 2)
+                  (let
+                    ((g (apl-resolve-monadic (nth fns 0) env))
+                      (h (apl-resolve-dyadic (nth fns 1) env)))
+                    (fn (a b) (g (h a b)))))
+                ((= n 3)
+                  (let
+                    ((f (apl-resolve-dyadic (nth fns 0) env))
+                      (g (apl-resolve-dyadic (nth fns 1) env))
+                      (h (apl-resolve-dyadic (nth fns 2) env)))
+                    (fn (a b) (g (f a b) (h a b)))))
+                (else (error "dyadic train arity not 2 or 3"))))))
+        (else (error "apl-resolve-dyadic: unknown fn-node tag"))))))
+
+(define apl-run (fn (src) (apl-eval-ast (parse-apl src) {})))

lib/datalog/aggregates.sx

@@ -0,0 +1,157 @@
+;; lib/datalog/aggregates.sx — count / sum / min / max / findall.
+;;
+;; Surface form (always 3-arg after the relation name):
+;;
+;;   (count   Result  Var  GoalLit)
+;;   (sum     Result  Var  GoalLit)
+;;   (min     Result  Var  GoalLit)
+;;   (max     Result  Var  GoalLit)
+;;   (findall List    Var  GoalLit)
+;;
+;; Parsed naturally because arg-position compounds are already allowed
+;; (Phase 4 needs them for arithmetic). At evaluation time the aggregator
+;; runs `dl-find-bindings` on `GoalLit` under the current subst, collects
+;; the distinct values of `Var`, and binds `Result`.
+;;
+;; Aggregation is non-monotonic — `count(C, X, p(X))` shrinks as p loses
+;; tuples. The stratifier (lib/datalog/strata.sx) treats every aggregate's
+;; goal relation as a negation-like edge so the inner relation is fully
+;; derived before the aggregate fires.
+;;
+;; Empty input: count → 0, sum → 0, min/max → no binding (rule fails).
+
+(define dl-aggregate-rels (list "count" "sum" "min" "max" "findall"))
+
+(define
+  dl-aggregate?
+  (fn
+    (lit)
+    (and
+      (list? lit)
+      (>= (len lit) 4)
+      (let ((rel (dl-rel-name lit)))
+        (cond
+          ((nil? rel) false)
+          (else (dl-member-string? rel dl-aggregate-rels)))))))
+
+;; Apply aggregation operator to a list of (already-distinct) numeric or
+;; symbolic values. Returns the aggregated value, or :empty if min/max
+;; has no input.
+(define
+  dl-do-aggregate
+  (fn
+    (op vals)
+    (cond
+      ((= op "count") (len vals))
+      ((= op "sum") (dl-sum-vals vals 0))
+      ((= op "findall") vals)
+      ((= op "min")
+        (cond
+          ((= (len vals) 0) :empty)
+          (else (dl-min-vals vals 1 (first vals)))))
+      ((= op "max")
+        (cond
+          ((= (len vals) 0) :empty)
+          (else (dl-max-vals vals 1 (first vals)))))
+      (else (error (str "datalog: unknown aggregate " op))))))
+
+(define
+  dl-sum-vals
+  (fn
+    (vals acc)
+    (cond
+      ((= (len vals) 0) acc)
+      (else (dl-sum-vals (rest vals) (+ acc (first vals)))))))
+
+(define
+  dl-min-vals
+  (fn
+    (vals i cur)
+    (cond
+      ((>= i (len vals)) cur)
+      (else
+        (let ((v (nth vals i)))
+          (dl-min-vals vals (+ i 1) (if (< v cur) v cur)))))))
+
+(define
+  dl-max-vals
+  (fn
+    (vals i cur)
+    (cond
+      ((>= i (len vals)) cur)
+      (else
+        (let ((v (nth vals i)))
+          (dl-max-vals vals (+ i 1) (if (> v cur) v cur)))))))
+
+;; Membership check by deep equality (so 30 == 30.0 etc).
+(define
+  dl-val-member?
+  (fn
+    (v xs)
+    (cond
+      ((= (len xs) 0) false)
+      ((dl-tuple-equal? v (first xs)) true)
+      (else (dl-val-member? v (rest xs))))))
+
+;; Evaluate an aggregate body lit under `subst`. Returns the list of
+;; extended substitutions (0 or 1 element).
+(define
+  dl-eval-aggregate
+  (fn
+    (lit db subst)
+    (let
+      ((op (dl-rel-name lit))
+        (result-var (nth lit 1))
+        (agg-var (nth lit 2))
+        (goal (nth lit 3)))
+      (cond
+        ((not (dl-var? agg-var))
+          (error (str "datalog aggregate (" op
+                      "): second arg must be a variable, got " agg-var)))
+        ((not (and (list? goal) (> (len goal) 0)
+                   (symbol? (first goal))))
+          (error (str "datalog aggregate (" op
+                      "): third arg must be a positive literal, got "
+                      goal)))
+        ((not (dl-member-string?
+                (symbol->string agg-var)
+                (dl-vars-of goal)))
+          (error (str "datalog aggregate (" op
+                      "): aggregation variable " agg-var
+                      " does not appear in the goal " goal
+                      " — without it every match contributes the same "
+                      "(unbound) value and the result is meaningless")))
+        (else
+          (let ((vals (list)))
+            (do
+              (for-each
+                (fn
+                  (s)
+                  (let ((v (dl-apply-subst agg-var s)))
+                    (when (not (dl-val-member? v vals))
+                      (append! vals v))))
+                (dl-find-bindings (list goal) db subst))
+              (let ((agg-val (dl-do-aggregate op vals)))
+                (cond
+                  ((= agg-val :empty) (list))
+                  (else
+                    (let ((s2 (dl-unify result-var agg-val subst)))
+                      (if (nil? s2) (list) (list s2)))))))))))))
+
+
+;; Stratification edges from aggregates: like negation, the goal's
+;; relation must be in a strictly lower stratum so that the aggregate
+;; fires only after the underlying tuples are settled.
+(define
+  dl-aggregate-dep-edge
+  (fn
+    (lit)
+    (cond
+      ((dl-aggregate? lit)
+        (let ((goal (nth lit 3)))
+          (cond
+            ((and (list? goal) (> (len goal) 0))
+              (let ((rel (dl-rel-name goal)))
+                (if (nil? rel) nil {:rel rel :neg true})))
+            (else nil))))
+      (else nil))))

lib/datalog/api.sx

@@ -0,0 +1,303 @@
+;; lib/datalog/api.sx — SX-data embedding API.
+;;
+;; Where Phase 1's `dl-program` takes a Datalog source string,
+;; this module exposes a parser-free API that consumes SX data
+;; directly. Two rule shapes are accepted:
+;;
+;;   - dict:   {:head <literal> :body (<literal> ...)}
+;;   - list:   (<head-elements...> <- <body-literal> ...)
+;;             — `<-` is an SX symbol used as the rule arrow.
+;;
+;; Examples:
+;;
+;;   (dl-program-data
+;;     '((parent tom bob) (parent tom liz) (parent bob ann))
+;;     '((ancestor X Y <- (parent X Y))
+;;       (ancestor X Z <- (parent X Y) (ancestor Y Z))))
+;;
+;;   (dl-query db '(ancestor tom X))   ; same query API as before
+;;
+;; Variables follow the parser convention: SX symbols whose first
+;; character is uppercase or `_` are variables.
+
+(define
+  dl-rule
+  (fn (head body) {:head head :body body}))
+
+(define
+  dl-rule-arrow?
+  (fn
+    (x)
+    (and (symbol? x) (= (symbol->string x) "<-"))))
+
+(define
+  dl-find-arrow
+  (fn
+    (rl i n)
+    (cond
+      ((>= i n) nil)
+      ((dl-rule-arrow? (nth rl i)) i)
+      (else (dl-find-arrow rl (+ i 1) n)))))
+
+;; Given a list of the form (head-elt ... <- body-lit ...) returns
+;; {:head (head-elt ...) :body (body-lit ...)}. If no arrow is
+;; present, the whole list is treated as the head and the body is
+;; empty (i.e. a fact written rule-style).
+(define
+  dl-rule-from-list
+  (fn
+    (rl)
+    (let ((n (len rl)))
+      (let ((idx (dl-find-arrow rl 0 n)))
+        (cond
+          ((nil? idx) {:head rl :body (list)})
+          (else
+            (let
+              ((head (slice rl 0 idx))
+                (body (slice rl (+ idx 1) n)))
+              {:head head :body body})))))))
+
+;; Coerce a rule given as either a dict or a list-with-arrow to a dict.
+(define
+  dl-coerce-rule
+  (fn
+    (r)
+    (cond
+      ((dict? r) r)
+      ((list? r) (dl-rule-from-list r))
+      (else (error (str "dl-coerce-rule: expected dict or list, got " r))))))
+
+;; Build a db from SX data lists.
+(define
+  dl-program-data
+  (fn
+    (facts rules)
+    (let ((db (dl-make-db)))
+      (do
+        (for-each (fn (lit) (dl-add-fact! db lit)) facts)
+        (for-each
+          (fn (r) (dl-add-rule! db (dl-coerce-rule r)))
+          rules)
+        db))))
+
+;; Add a single fact at runtime, then re-saturate the db so derived
+;; tuples reflect the change. Returns the db.
+(define
+  dl-assert!
+  (fn
+    (db lit)
+    (do
+      (dl-add-fact! db lit)
+      (dl-saturate! db)
+      db)))
+
+;; Remove a fact and re-saturate. Mixed relations (which have BOTH
+;; user-asserted facts AND rules) are supported via :edb-keys provenance
+;; — explicit facts are marked at dl-add-fact! time, the saturator uses
+;; dl-add-derived! which doesn't mark them, so the retract pass can
+;; safely wipe IDB-derived tuples while preserving the user's EDB.
+;;
+;; Effect:
+;;   - remove tuples matching `lit` from :facts and :edb-keys
+;;   - for every relation that has a rule (i.e. potentially IDB or
+;;     mixed), drop the IDB-derived portion (anything not in :edb-keys)
+;;     so the saturator can re-derive cleanly
+;;   - re-saturate
+(define
+  dl-retract!
+  (fn
+    (db lit)
+    (let
+      ((rel-key (dl-rel-name lit)))
+      (do
+        ;; Drop the matching tuple from its relation list, its facts-keys,
+        ;; its first-arg index, AND from :edb-keys (if present).
+        (when
+          (has-key? (get db :facts) rel-key)
+          (let
+            ((existing (get (get db :facts) rel-key))
+              (kept (list))
+              (kept-keys {})
+              (kept-index {})
+              (edb-rel (cond
+                         ((has-key? (get db :edb-keys) rel-key)
+                           (get (get db :edb-keys) rel-key))
+                         (else nil)))
+              (kept-edb {}))
+            (do
+              (for-each
+                (fn
+                  (t)
+                  (when
+                    (not (dl-tuple-equal? t lit))
+                    (do
+                      (append! kept t)
+                      (let ((tk (dl-tuple-key t)))
+                        (do
+                          (dict-set! kept-keys tk true)
+                          (when
+                            (and (not (nil? edb-rel))
+                                 (has-key? edb-rel tk))
+                            (dict-set! kept-edb tk true))))
+                      (when
+                        (>= (len t) 2)
+                        (let ((k (dl-arg-key (nth t 1))))
+                          (do
+                            (when
+                              (not (has-key? kept-index k))
+                              (dict-set! kept-index k (list)))
+                            (append! (get kept-index k) t)))))))
+                existing)
+              (dict-set! (get db :facts) rel-key kept)
+              (dict-set! (get db :facts-keys) rel-key kept-keys)
+              (dict-set! (get db :facts-index) rel-key kept-index)
+              (when
+                (not (nil? edb-rel))
+                (dict-set! (get db :edb-keys) rel-key kept-edb)))))
+        ;; For each rule-head relation, strip the IDB-derived tuples
+        ;; (anything not marked in :edb-keys) so the saturator can
+        ;; cleanly re-derive without leaving stale tuples that depended
+        ;; on the now-removed fact.
+        (let ((rule-heads (dl-rule-head-rels db)))
+          (for-each
+            (fn
+              (k)
+              (when
+                (has-key? (get db :facts) k)
+                (let
+                  ((existing (get (get db :facts) k))
+                    (kept (list))
+                    (kept-keys {})
+                    (kept-index {})
+                    (edb-rel (cond
+                               ((has-key? (get db :edb-keys) k)
+                                 (get (get db :edb-keys) k))
+                               (else {}))))
+                  (do
+                    (for-each
+                      (fn
+                        (t)
+                        (let ((tk (dl-tuple-key t)))
+                          (when
+                            (has-key? edb-rel tk)
+                            (do
+                              (append! kept t)
+                              (dict-set! kept-keys tk true)
+                              (when
+                                (>= (len t) 2)
+                                (let ((kk (dl-arg-key (nth t 1))))
+                                  (do
+                                    (when
+                                      (not (has-key? kept-index kk))
+                                      (dict-set! kept-index kk (list)))
+                                    (append! (get kept-index kk) t))))))))
+                      existing)
+                    (dict-set! (get db :facts) k kept)
+                    (dict-set! (get db :facts-keys) k kept-keys)
+                    (dict-set! (get db :facts-index) k kept-index)))))
+            rule-heads))
+        (dl-saturate! db)
+        db))))
+
+;; ── Convenience: single-call source + query ───────────────────
+;; (dl-eval source query-source) parses both, builds a db, saturates,
+;; runs the query, returns the substitution list. The query source
+;; should be `?- goal[, goal ...].` — the parser produces a clause
+;; with :query containing a list of literals which is fed straight
+;; to dl-query.
+(define
+  dl-eval
+  (fn
+    (source query-source)
+    (let
+      ((db (dl-program source))
+        (queries (dl-parse query-source)))
+      (cond
+        ((= (len queries) 0) (error "dl-eval: query string is empty"))
+        ((not (has-key? (first queries) :query))
+          (error "dl-eval: second arg must be a `?- ...` query clause"))
+        (else
+          (dl-query db (get (first queries) :query)))))))
+
+;; (dl-eval-magic source query-source) — like dl-eval but routes a
+;; single-positive-literal query through `dl-magic-query` for goal-
+;; directed evaluation. Multi-literal query bodies fall back to the
+;; standard dl-query path (magic-sets is currently only wired for
+;; single-positive goals). The caller's source is parsed afresh
+;; each call so successive invocations are independent.
+(define
+  dl-eval-magic
+  (fn
+    (source query-source)
+    (let
+      ((db (dl-program source))
+        (queries (dl-parse query-source)))
+      (cond
+        ((= (len queries) 0) (error "dl-eval-magic: query string is empty"))
+        ((not (has-key? (first queries) :query))
+          (error
+            "dl-eval-magic: second arg must be a `?- ...` query clause"))
+        (else
+          (let
+            ((qbody (get (first queries) :query)))
+            (cond
+              ((and (= (len qbody) 1)
+                    (list? (first qbody))
+                    (> (len (first qbody)) 0)
+                    (symbol? (first (first qbody))))
+                (dl-magic-query db (first qbody)))
+              (else (dl-query db qbody)))))))))
+
+;; List rules whose head's relation matches `rel-name`. Useful for
+;; inspection ("show me how this relation is derived") without
+;; exposing the internal `:rules` list.
+(define
+  dl-rules-of
+  (fn
+    (db rel-name)
+    (let ((out (list)))
+      (do
+        (for-each
+          (fn
+            (rule)
+            (when
+              (= (dl-rel-name (get rule :head)) rel-name)
+              (append! out rule)))
+          (dl-rules db))
+        out))))
+
+(define
+  dl-rule-head-rels
+  (fn
+    (db)
+    (let ((seen (list)))
+      (do
+        (for-each
+          (fn
+            (rule)
+            (let ((h (dl-rel-name (get rule :head))))
+              (when
+                (and (not (nil? h)) (not (dl-member-string? h seen)))
+                (append! seen h))))
+          (dl-rules db))
+        seen))))
+
+;; Wipe every relation that has at least one rule (i.e. every IDB
+;; relation) — leaves EDB facts and rule definitions intact. Useful
+;; before a follow-up `dl-saturate!` if you want a clean restart, or
+;; for inspection of the EDB-only baseline.
+(define
+  dl-clear-idb!
+  (fn
+    (db)
+    (let ((rule-heads (dl-rule-head-rels db)))
+      (do
+        (for-each
+          (fn
+            (k)
+            (do
+              (dict-set! (get db :facts) k (list))
+              (dict-set! (get db :facts-keys) k {})
+              (dict-set! (get db :facts-index) k {})))
+          rule-heads)
+        db))))

lib/datalog/builtins.sx

@@ -0,0 +1,406 @@
+;; lib/datalog/builtins.sx — comparison + arithmetic body literals.
+;;
+;; Built-in predicates filter / extend candidate substitutions during
+;; rule evaluation. They are not stored facts and do not participate in
+;; the Herbrand base.
+;;
+;;   (< a b)  (<= a b)  (> a b)  (>= a b)   ; numeric (or string) compare
+;;   (= a b)                                  ; unify (binds vars)
+;;   (!= a b)                                ; ground-only inequality
+;;   (is X expr)                              ; bind X to expr's value
+;;
+;; Arithmetic expressions are SX-list compounds:
+;;   (+ a b)  (- a b)  (* a b)  (/ a b)
+;; or numbers / variables (must be bound at evaluation time).
+
+(define
+  dl-comparison?
+  (fn
+    (lit)
+    (and
+      (list? lit)
+      (> (len lit) 0)
+      (let
+        ((rel (dl-rel-name lit)))
+        (cond
+          ((nil? rel) false)
+          (else (dl-member-string? rel (list "<" "<=" ">" ">=" "!="))))))))
+
+(define
+  dl-eq?
+  (fn
+    (lit)
+    (and
+      (list? lit)
+      (> (len lit) 0)
+      (let ((rel (dl-rel-name lit))) (and (not (nil? rel)) (= rel "=")))))) 
+
+(define
+  dl-is?
+  (fn
+    (lit)
+    (and
+      (list? lit)
+      (> (len lit) 0)
+      (let
+        ((rel (dl-rel-name lit)))
+        (and (not (nil? rel)) (= rel "is")))))) 
+
+;; Evaluate an arithmetic expression under subst. Returns the numeric
+;; result, or raises if any operand is unbound or non-numeric.
+(define
+  dl-eval-arith
+  (fn
+    (expr subst)
+    (let
+      ((w (dl-walk expr subst)))
+      (cond
+        ((number? w) w)
+        ((dl-var? w)
+          (error (str "datalog arith: unbound variable " (symbol->string w))))
+        ((list? w)
+          (let
+            ((rel (dl-rel-name w)) (args (rest w)))
+            (cond
+              ((not (= (len args) 2))
+                (error (str "datalog arith: need 2 args, got " w)))
+              (else
+                (let
+                  ((a (dl-eval-arith (first args) subst))
+                    (b (dl-eval-arith (nth args 1) subst)))
+                  (cond
+                    ((= rel "+") (+ a b))
+                    ((= rel "-") (- a b))
+                    ((= rel "*") (* a b))
+                    ((= rel "/")
+                      (cond
+                        ((= b 0)
+                          (error
+                            (str "datalog arith: division by zero in "
+                                 w)))
+                        (else (/ a b))))
+                    (else (error (str "datalog arith: unknown op " rel)))))))))
+        (else (error (str "datalog arith: not a number — " w)))))))
+
+;; Comparable types — both operands must be the same primitive type
+;; (both numbers, both strings). `!=` is the exception: it's defined
+;; for any pair (returns true iff not equal) since dl-tuple-equal?
+;; handles type-mixed comparisons.
+(define
+  dl-compare-typeok?
+  (fn
+    (rel a b)
+    (cond
+      ((= rel "!=") true)
+      ((and (number? a) (number? b)) true)
+      ((and (string? a) (string? b)) true)
+      (else false))))
+
+(define
+  dl-eval-compare
+  (fn
+    (lit subst)
+    (let
+      ((rel (dl-rel-name lit))
+        (a (dl-walk (nth lit 1) subst))
+        (b (dl-walk (nth lit 2) subst)))
+      (cond
+        ((or (dl-var? a) (dl-var? b))
+          (error
+            (str
+              "datalog: comparison "
+              rel
+              " has unbound argument; "
+              "ensure prior body literal binds the variable")))
+        ((not (dl-compare-typeok? rel a b))
+          (error
+            (str "datalog: comparison " rel " requires same-type "
+                 "operands (both numbers or both strings), got "
+                 a " and " b)))
+        (else
+          (let
+            ((ok (cond ((= rel "<") (< a b)) ((= rel "<=") (<= a b)) ((= rel ">") (> a b)) ((= rel ">=") (>= a b)) ((= rel "!=") (not (dl-tuple-equal? a b))) (else (error (str "datalog: unknown compare " rel))))))
+            (if ok subst nil)))))))
+
+(define
+  dl-eval-eq
+  (fn
+    (lit subst)
+    (dl-unify (nth lit 1) (nth lit 2) subst)))
+
+(define
+  dl-eval-is
+  (fn
+    (lit subst)
+    (let
+      ((target (nth lit 1)) (expr (nth lit 2)))
+      (let
+        ((value (dl-eval-arith expr subst)))
+        (dl-unify target value subst)))))
+
+(define
+  dl-eval-builtin
+  (fn
+    (lit subst)
+    (cond
+      ((dl-comparison? lit) (dl-eval-compare lit subst))
+      ((dl-eq? lit) (dl-eval-eq lit subst))
+      ((dl-is? lit) (dl-eval-is lit subst))
+      (else (error (str "dl-eval-builtin: not a built-in: " lit))))))
+
+;; ── Safety analysis ──────────────────────────────────────────────
+;;
+;; Walks body literals left-to-right tracking a "bound" set. The check
+;; understands these literal kinds:
+;;
+;;   positive non-built-in    → adds its vars to bound
+;;   (is X expr)              → vars(expr) ⊆ bound, then add X (if var)
+;;   <,<=,>,>=,!=             → all vars ⊆ bound (no binding)
+;;   (= a b) where:
+;;       both non-vars        → constraint check, no binding
+;;       a var, b not         → bind a
+;;       b var, a not         → bind b
+;;       both vars            → at least one in bound; bind the other
+;;   {:neg lit}               → all vars ⊆ bound (Phase 7 enforces fully)
+;;
+;; At end, head vars (minus `_`) must be ⊆ bound.
+
+(define
+  dl-vars-not-in
+  (fn
+    (vs bound)
+    (let
+      ((out (list)))
+      (do
+        (for-each
+          (fn
+            (v)
+            (when (not (dl-member-string? v bound)) (append! out v)))
+          vs)
+        out))))
+
+;; Filter a list of variable-name strings to exclude anonymous-renamed
+;; vars (`_` in source → `_anon*` by dl-rename-anon-term). Used by
+;; the negation safety check, where anonymous vars are existential
+;; within the negated literal.
+(define
+  dl-non-anon-vars
+  (fn
+    (vs)
+    (let
+      ((out (list)))
+      (do
+        (for-each
+          (fn
+            (v)
+            (when
+              (not (and (>= (len v) 5)
+                        (= (slice v 0 5) "_anon")))
+              (append! out v)))
+          vs)
+        out))))
+
+(define
+  dl-rule-check-safety
+  (fn
+    (rule)
+    (let
+      ((head (get rule :head))
+        (body (get rule :body))
+        (bound (list))
+        (err nil))
+      (do
+        (define
+          dl-add-bound!
+          (fn
+            (vs)
+            (for-each
+              (fn
+                (v)
+                (when (not (dl-member-string? v bound)) (append! bound v)))
+              vs)))
+        (define
+          dl-process-eq!
+          (fn
+            (lit)
+            (let
+              ((a (nth lit 1)) (b (nth lit 2)))
+              (let
+                ((va (dl-var? a)) (vb (dl-var? b)))
+                (cond
+                  ((and (not va) (not vb)) nil)
+                  ((and va (not vb))
+                    (dl-add-bound! (list (symbol->string a))))
+                  ((and (not va) vb)
+                    (dl-add-bound! (list (symbol->string b))))
+                  (else
+                    (let
+                      ((sa (symbol->string a)) (sb (symbol->string b)))
+                      (cond
+                        ((dl-member-string? sa bound)
+                          (dl-add-bound! (list sb)))
+                        ((dl-member-string? sb bound)
+                          (dl-add-bound! (list sa)))
+                        (else
+                          (set!
+                            err
+                            (str
+                              "= between two unbound variables "
+                              (list sa sb)
+                              " — at least one must be bound by an "
+                              "earlier positive body literal")))))))))))
+        (define
+          dl-process-cmp!
+          (fn
+            (lit)
+            (let
+              ((needed (dl-vars-of (list (nth lit 1) (nth lit 2)))))
+              (let
+                ((missing (dl-vars-not-in needed bound)))
+                (when
+                  (> (len missing) 0)
+                  (set!
+                    err
+                    (str
+                      "comparison "
+                      (dl-rel-name lit)
+                      " requires bound variable(s) "
+                      missing
+                      " (must be bound by an earlier positive "
+                      "body literal)")))))))
+        (define
+          dl-process-is!
+          (fn
+            (lit)
+            (let
+              ((tgt (nth lit 1)) (expr (nth lit 2)))
+              (let
+                ((needed (dl-vars-of expr)))
+                (let
+                  ((missing (dl-vars-not-in needed bound)))
+                  (cond
+                    ((> (len missing) 0)
+                      (set!
+                        err
+                        (str
+                          "is RHS uses unbound variable(s) "
+                          missing
+                          " — bind them via a prior positive body "
+                          "literal")))
+                    (else
+                      (when
+                        (dl-var? tgt)
+                        (dl-add-bound! (list (symbol->string tgt)))))))))))
+        (define
+          dl-process-neg!
+          (fn
+            (lit)
+            (let
+              ((inner (get lit :neg)))
+              (let
+                ((inner-rn
+                   (cond
+                     ((and (list? inner) (> (len inner) 0))
+                       (dl-rel-name inner))
+                     (else nil)))
+                  ;; Anonymous variables (`_` in source → `_anon*` after
+                  ;; renaming) are existentially quantified within the
+                  ;; negated literal — they don't need to be bound by
+                  ;; an earlier body lit, since `not p(X, _)` is a
+                  ;; valid idiom for "no Y exists s.t. p(X, Y)". Filter
+                  ;; them out of the safety check.
+                  (needed (dl-non-anon-vars (dl-vars-of inner)))
+                  (missing (dl-vars-not-in needed bound)))
+                (cond
+                  ((and (not (nil? inner-rn)) (dl-reserved-rel? inner-rn))
+                    (set! err
+                      (str "negated literal uses reserved name '"
+                           inner-rn
+                           "' — nested `not(...)` / negated built-ins are "
+                           "not supported; introduce an intermediate "
+                           "relation and negate that")))
+                  ((> (len missing) 0)
+                    (set! err
+                      (str "negation refers to unbound variable(s) "
+                           missing
+                           " — they must be bound by an earlier "
+                           "positive body literal"))))))))
+        (define
+          dl-process-agg!
+          (fn
+            (lit)
+            (let
+              ((result-var (nth lit 1)))
+              ;; Aggregate goal vars are existentially quantified within
+              ;; the aggregate; nothing required from outer context. The
+              ;; result var becomes bound after the aggregate fires.
+              (when
+                (dl-var? result-var)
+                (dl-add-bound! (list (symbol->string result-var)))))))
+
+        (define
+          dl-process-lit!
+          (fn
+            (lit)
+            (when
+              (nil? err)
+              (cond
+                ((and (dict? lit) (has-key? lit :neg))
+                  (dl-process-neg! lit))
+                ;; A bare dict that is not a recognised negation is
+                ;; almost certainly a typo (e.g. `{:negs ...}` instead
+                ;; of `{:neg ...}`). Without this guard the dict would
+                ;; silently fall through every clause; the head safety
+                ;; check would then flag the head variables as unbound
+                ;; even though the real bug is the malformed body lit.
+                ((dict? lit)
+                  (set! err
+                    (str "body literal is a dict but lacks :neg — "
+                         "the only dict-shaped body lit recognised is "
+                         "{:neg <positive-lit>} for stratified "
+                         "negation, got " lit)))
+                ((dl-aggregate? lit) (dl-process-agg! lit))
+                ((dl-eq? lit) (dl-process-eq! lit))
+                ((dl-is? lit) (dl-process-is! lit))
+                ((dl-comparison? lit) (dl-process-cmp! lit))
+                ((and (list? lit) (> (len lit) 0))
+                  (let ((rn (dl-rel-name lit)))
+                    (cond
+                      ((and (not (nil? rn)) (dl-reserved-rel? rn))
+                        (set! err
+                          (str "body literal uses reserved name '" rn
+                               "' — built-ins / aggregates have their own "
+                               "syntax; nested `not(...)` is not supported "
+                               "(use stratified negation via an "
+                               "intermediate relation)")))
+                      (else (dl-add-bound! (dl-vars-of lit))))))
+                (else
+                  ;; Anything that's not a dict, not a list, or an
+                  ;; empty list. Numbers / strings / symbols as body
+                  ;; lits don't make sense — surface the type.
+                  (set! err
+                    (str "body literal must be a positive lit, "
+                         "built-in, aggregate, or {:neg ...} dict, "
+                         "got " lit)))))))
+        (for-each dl-process-lit! body)
+        (when
+          (nil? err)
+          (let
+            ((head-vars (dl-vars-of head)) (missing (list)))
+            (do
+              (for-each
+                (fn
+                  (v)
+                  (when
+                    (and (not (dl-member-string? v bound)) (not (= v "_")))
+                    (append! missing v)))
+                head-vars)
+              (when
+                (> (len missing) 0)
+                (set!
+                  err
+                  (str
+                    "head variable(s) "
+                    missing
+                    " do not appear in any positive body literal"))))))
+        err))))

lib/datalog/conformance.conf

@@ -0,0 +1,32 @@
+# Datalog conformance config — sourced by lib/guest/conformance.sh.
+
+LANG_NAME=datalog
+MODE=dict
+
+PRELOADS=(
+  lib/datalog/tokenizer.sx
+  lib/datalog/parser.sx
+  lib/datalog/unify.sx
+  lib/datalog/db.sx
+  lib/datalog/builtins.sx
+  lib/datalog/aggregates.sx
+  lib/datalog/strata.sx
+  lib/datalog/eval.sx
+  lib/datalog/api.sx
+  lib/datalog/magic.sx
+  lib/datalog/demo.sx
+)
+
+SUITES=(
+  "tokenize:lib/datalog/tests/tokenize.sx:(dl-tokenize-tests-run!)"
+  "parse:lib/datalog/tests/parse.sx:(dl-parse-tests-run!)"
+  "unify:lib/datalog/tests/unify.sx:(dl-unify-tests-run!)"
+  "eval:lib/datalog/tests/eval.sx:(dl-eval-tests-run!)"
+  "builtins:lib/datalog/tests/builtins.sx:(dl-builtins-tests-run!)"
+  "semi_naive:lib/datalog/tests/semi_naive.sx:(dl-semi-naive-tests-run!)"
+  "negation:lib/datalog/tests/negation.sx:(dl-negation-tests-run!)"
+  "aggregates:lib/datalog/tests/aggregates.sx:(dl-aggregates-tests-run!)"
+  "api:lib/datalog/tests/api.sx:(dl-api-tests-run!)"
+  "magic:lib/datalog/tests/magic.sx:(dl-magic-tests-run!)"
+  "demo:lib/datalog/tests/demo.sx:(dl-demo-tests-run!)"
+)

lib/datalog/conformance.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+# Thin wrapper — see lib/guest/conformance.sh and lib/datalog/conformance.conf.
+exec bash "$(dirname "$0")/../guest/conformance.sh" "$(dirname "$0")/conformance.conf" "$@"

lib/datalog/datalog.sx

@@ -0,0 +1,97 @@
+;; lib/datalog/datalog.sx — public API documentation index.
+;;
+;; This file is reference-only — `load` is an epoch-protocol command,
+;; not an SX function, so it cannot reload a list of files from inside
+;; another `.sx` file. To set up a fresh sx_server session with all
+;; modules in scope, issue these loads in order:
+;;
+;;   (load "lib/datalog/tokenizer.sx")
+;;   (load "lib/datalog/parser.sx")
+;;   (load "lib/datalog/unify.sx")
+;;   (load "lib/datalog/db.sx")
+;;   (load "lib/datalog/builtins.sx")
+;;   (load "lib/datalog/aggregates.sx")
+;;   (load "lib/datalog/strata.sx")
+;;   (load "lib/datalog/eval.sx")
+;;   (load "lib/datalog/api.sx")
+;;   (load "lib/datalog/magic.sx")
+;;   (load "lib/datalog/demo.sx")
+;;
+;; (lib/datalog/conformance.sh runs this load list automatically.)
+;;
+;; ── Public API surface ─────────────────────────────────────────────
+;;
+;;   Source / data:
+;;     (dl-tokenize "src")              → token list
+;;     (dl-parse "src")                 → parsed clauses
+;;     (dl-program "src")               → db built from a source string
+;;     (dl-program-data facts rules)    → db from SX data lists; rules
+;;                                        accept either dict form or
+;;                                        list form with `<-` arrow
+;;
+;;   Construction (mutates db):
+;;     (dl-make-db)                     empty db
+;;     (dl-add-fact! db lit)            rejects non-ground
+;;     (dl-add-rule! db rule)           rejects unsafe rules
+;;     (dl-rule head body)              dict-rule constructor
+;;     (dl-add-clause! db clause)       parser output → fact or rule
+;;     (dl-load-program! db src)        string source
+;;     (dl-set-strategy! db strategy)   :semi-naive default; :magic
+;;                                        is informational, use
+;;                                        dl-magic-query for actual
+;;                                        magic-sets evaluation
+;;
+;;   Mutation:
+;;     (dl-assert! db lit)              add + re-saturate
+;;     (dl-retract! db lit)             drop EDB, wipe IDB, re-saturate
+;;     (dl-clear-idb! db)               wipe rule-headed relations
+;;
+;;   Query / inspection:
+;;     (dl-saturate! db)                stratified semi-naive default
+;;     (dl-saturate-naive! db)          reference (slow on chains)
+;;     (dl-saturate-rules! db rules)    per-rule-set semi-naive worker
+;;     (dl-query db goal)               list of substitution dicts
+;;     (dl-relation db rel-name)        tuple list for a relation
+;;     (dl-rules db)                    rule list
+;;     (dl-fact-count db)               total ground tuples
+;;     (dl-summary db)                  {<rel>: count} for inspection
+;;
+;;   Single-call convenience:
+;;     (dl-eval source query-source)         parse, run, return substs
+;;     (dl-eval-magic source query-source)   single-goal → magic-sets
+;;
+;;   Magic-sets (lib/datalog/magic.sx):
+;;     (dl-adorn-goal goal)             "b/f" adornment string
+;;     (dl-rule-sips rule head-adn)     SIPS analysis per body lit
+;;     (dl-magic-rewrite rules rel adn args)
+;;                                       rewritten rule list + seed
+;;     (dl-magic-query db query-goal)    end-to-end magic-sets query
+;;
+;; ── Body literal kinds ─────────────────────────────────────────────
+;;
+;;   Positive             (rel arg ... arg)
+;;   Negation             {:neg (rel arg ...)}
+;;   Comparison           (< X Y), (<= X Y), (> X Y), (>= X Y),
+;;                        (= X Y), (!= X Y)
+;;   Arithmetic           (is Z (+ X Y)) and (- * /)
+;;   Aggregation          (count R V Goal),  (sum R V Goal),
+;;                        (min R V Goal),    (max R V Goal),
+;;                        (findall L V Goal)
+;;
+;; ── Variable conventions ───────────────────────────────────────────
+;;
+;;   Variables: SX symbols whose first char is uppercase A–Z or '_'.
+;;   Anonymous '_' is renamed to a fresh _anon<N> per occurrence at
+;;   rule/query load time so multiple '_' don't unify.
+;;
+;; ── Demo programs ──────────────────────────────────────────────────
+;;
+;;   See lib/datalog/demo.sx — federation, content, permissions, and
+;;   the canonical "cooking posts by people I follow (transitively)"
+;;   example.
+;;
+;; ── Status ─────────────────────────────────────────────────────────
+;;
+;;   See plans/datalog-on-sx.md — phase-by-phase progress log and
+;;   roadmap. Run `bash lib/datalog/conformance.sh` to refresh
+;;   `lib/datalog/scoreboard.{json,md}`.

lib/datalog/db.sx

@@ -0,0 +1,575 @@
+;; lib/datalog/db.sx — Datalog database (EDB + IDB + rules) + safety hook.
+;;
+;; A db is a mutable dict:
+;;   {:facts {<rel-name-string> -> (literal ...)}
+;;    :rules ({:head literal :body (literal ...)} ...)}
+;;
+;; Facts are stored as full literals `(rel arg ... arg)` so they unify
+;; directly against rule body literals. Each relation's tuple list is
+;; deduplicated on insert.
+;;
+;; Phase 3 introduced safety analysis for head variables; Phase 4 (in
+;; lib/datalog/builtins.sx) swaps in the real `dl-rule-check-safety`,
+;; which is order-aware and understands built-in predicates.
+
+(define
+  dl-make-db
+  (fn ()
+    {:facts {}
+     :facts-keys {}
+     :facts-index {}
+     :edb-keys {}
+     :rules (list)
+     :strategy :semi-naive}))
+
+;; Record (rel-key, tuple-key) as user-asserted EDB. dl-add-fact! calls
+;; this when an explicit fact is added; the saturator (which uses
+;; dl-add-derived!) does NOT, so derived tuples never appear here.
+;; dl-retract! consults :edb-keys to know which tuples must survive
+;; the wipe-and-resaturate round-trip.
+(define
+  dl-mark-edb!
+  (fn
+    (db rel-key tk)
+    (let
+      ((edb (get db :edb-keys)))
+      (do
+        (when
+          (not (has-key? edb rel-key))
+          (dict-set! edb rel-key {}))
+        (dict-set! (get edb rel-key) tk true)))))
+
+(define
+  dl-edb-fact?
+  (fn
+    (db rel-key tk)
+    (let
+      ((edb (get db :edb-keys)))
+      (and (has-key? edb rel-key)
+           (has-key? (get edb rel-key) tk)))))
+
+;; Evaluation strategy. Default :semi-naive (used by dl-saturate!).
+;; :naive selects dl-saturate-naive! (slower but easier to reason
+;; about). :magic is a marker — goal-directed magic-sets evaluation
+;; is invoked separately via `dl-magic-query`; setting :magic here
+;; is purely informational. Any other value is rejected so typos
+;; don't silently fall back to the default.
+(define
+  dl-strategy-values
+  (list :semi-naive :naive :magic))
+
+(define
+  dl-set-strategy!
+  (fn
+    (db strategy)
+    (cond
+      ((not (dl-keyword-member? strategy dl-strategy-values))
+        (error (str "dl-set-strategy!: unknown strategy " strategy
+                    " — must be one of " dl-strategy-values)))
+      (else
+        (do
+          (dict-set! db :strategy strategy)
+          db)))))
+
+(define
+  dl-keyword-member?
+  (fn
+    (k xs)
+    (cond
+      ((= (len xs) 0) false)
+      ((= k (first xs)) true)
+      (else (dl-keyword-member? k (rest xs))))))
+
+(define
+  dl-get-strategy
+  (fn
+    (db)
+    (if (has-key? db :strategy) (get db :strategy) :semi-naive)))
+
+(define
+  dl-rel-name
+  (fn
+    (lit)
+    (cond
+      ((and (dict? lit) (has-key? lit :neg)) (dl-rel-name (get lit :neg)))
+      ((and (list? lit) (> (len lit) 0) (symbol? (first lit)))
+        (symbol->string (first lit)))
+      (else nil))))
+
+(define dl-builtin-rels (list "<" "<=" ">" ">=" "=" "!=" "is"))
+
+(define
+  dl-member-string?
+  (fn
+    (s xs)
+    (cond
+      ((= (len xs) 0) false)
+      ((= (first xs) s) true)
+      (else (dl-member-string? s (rest xs))))))
+
+(define
+  dl-builtin?
+  (fn
+    (lit)
+    (and
+      (list? lit)
+      (> (len lit) 0)
+      (let
+        ((rel (dl-rel-name lit)))
+        (cond
+          ((nil? rel) false)
+          (else (dl-member-string? rel dl-builtin-rels)))))))
+
+(define
+  dl-positive-lit?
+  (fn
+    (lit)
+    (cond
+      ((and (dict? lit) (has-key? lit :neg)) false)
+      ((dl-builtin? lit) false)
+      ((and (list? lit) (> (len lit) 0)) true)
+      (else false))))
+
+(define
+  dl-tuple-equal?
+  (fn
+    (a b)
+    (cond
+      ((and (list? a) (list? b))
+        (and (= (len a) (len b)) (dl-tuple-equal-list? a b 0)))
+      ((and (number? a) (number? b)) (= a b))
+      (else (equal? a b)))))
+
+(define
+  dl-tuple-equal-list?
+  (fn
+    (a b i)
+    (cond
+      ((>= i (len a)) true)
+      ((not (dl-tuple-equal? (nth a i) (nth b i))) false)
+      (else (dl-tuple-equal-list? a b (+ i 1))))))
+
+(define
+  dl-tuple-member?
+  (fn
+    (lit lits)
+    (dl-tuple-member-aux? lit lits 0 (len lits))))
+
+(define
+  dl-tuple-member-aux?
+  (fn
+    (lit lits i n)
+    (cond
+      ((>= i n) false)
+      ((dl-tuple-equal? lit (nth lits i)) true)
+      (else (dl-tuple-member-aux? lit lits (+ i 1) n)))))
+
+(define
+  dl-ensure-rel!
+  (fn
+    (db rel-key)
+    (let
+      ((facts (get db :facts))
+        (fk (get db :facts-keys))
+        (fi (get db :facts-index)))
+      (do
+        (when
+          (not (has-key? facts rel-key))
+          (dict-set! facts rel-key (list)))
+        (when
+          (not (has-key? fk rel-key))
+          (dict-set! fk rel-key {}))
+        (when
+          (not (has-key? fi rel-key))
+          (dict-set! fi rel-key {}))
+        (get facts rel-key)))))
+
+;; First-arg index helpers. Tuples are keyed by their first-after-rel
+;; arg's `(str ...)`; when that arg is a constant, dl-match-positive
+;; uses the index instead of scanning the full relation.
+(define
+  dl-arg-key
+  (fn
+    (v)
+    (str v)))
+
+(define
+  dl-index-add!
+  (fn
+    (db rel-key lit)
+    (let
+      ((idx (get db :facts-index))
+        (n (len lit)))
+      (when
+        (and (>= n 2) (has-key? idx rel-key))
+        (let
+          ((rel-idx (get idx rel-key))
+            (k (dl-arg-key (nth lit 1))))
+          (do
+            (when
+              (not (has-key? rel-idx k))
+              (dict-set! rel-idx k (list)))
+            (append! (get rel-idx k) lit)))))))
+
+(define
+  dl-index-lookup
+  (fn
+    (db rel-key arg-val)
+    (let
+      ((idx (get db :facts-index)))
+      (cond
+        ((not (has-key? idx rel-key)) (list))
+        (else
+          (let ((rel-idx (get idx rel-key))
+                (k (dl-arg-key arg-val)))
+            (if (has-key? rel-idx k) (get rel-idx k) (list))))))))
+
+(define dl-tuple-key (fn (lit) (str lit)))
+
+(define
+  dl-rel-tuples
+  (fn
+    (db rel-key)
+    (let
+      ((facts (get db :facts)))
+      (if (has-key? facts rel-key) (get facts rel-key) (list)))))
+
+;; Reserved relation names: built-in / aggregate / negation / arrow.
+;; Rules and facts may not have these as their head's relation, since
+;; the saturator treats them specially or they are not relation names
+;; at all.
+(define
+  dl-reserved-rel-names
+  (list "not" "count" "sum" "min" "max" "findall" "is"
+        "<" "<=" ">" ">=" "=" "!=" "+" "-" "*" "/" ":-" "?-"))
+
+(define
+  dl-reserved-rel?
+  (fn
+    (name) (dl-member-string? name dl-reserved-rel-names)))
+
+;; Internal: append a derived tuple to :facts without the public
+;; validation pass and without marking :edb-keys. Used by the saturator
+;; (eval.sx) and magic-sets (magic.sx). Returns true if the tuple was
+;; new, false if already present.
+(define
+  dl-add-derived!
+  (fn
+    (db lit)
+    (let
+      ((rel-key (dl-rel-name lit)))
+      (let
+        ((tuples (dl-ensure-rel! db rel-key))
+          (key-dict (get (get db :facts-keys) rel-key))
+          (tk (dl-tuple-key lit)))
+        (cond
+          ((has-key? key-dict tk) false)
+          (else
+            (do
+              (dict-set! key-dict tk true)
+              (append! tuples lit)
+              (dl-index-add! db rel-key lit)
+              true)))))))
+
+;; A simple term — number, string, or symbol — i.e. anything legal
+;; as an EDB fact arg. Compound (list) args belong only in body
+;; literals where they encode arithmetic / aggregate sub-goals.
+(define
+  dl-simple-term?
+  (fn
+    (term)
+    (or (number? term) (string? term) (symbol? term))))
+
+(define
+  dl-args-simple?
+  (fn
+    (lit i n)
+    (cond
+      ((>= i n) true)
+      ((not (dl-simple-term? (nth lit i))) false)
+      (else (dl-args-simple? lit (+ i 1) n)))))
+
+(define
+  dl-add-fact!
+  (fn
+    (db lit)
+    (cond
+      ((not (and (list? lit) (> (len lit) 0)))
+        (error (str "dl-add-fact!: expected literal list, got " lit)))
+      ((dl-reserved-rel? (dl-rel-name lit))
+        (error (str "dl-add-fact!: '" (dl-rel-name lit)
+                    "' is a reserved name (built-in / aggregate / negation)")))
+      ((not (dl-args-simple? lit 1 (len lit)))
+        (error (str "dl-add-fact!: fact args must be numbers, strings, "
+                    "or symbols — compound args (e.g. arithmetic "
+                    "expressions) are body-only and aren't evaluated "
+                    "in fact position. got " lit)))
+      ((not (dl-ground? lit (dl-empty-subst)))
+        (error (str "dl-add-fact!: expected ground literal, got " lit)))
+      (else
+        (let
+          ((rel-key (dl-rel-name lit)) (tk (dl-tuple-key lit)))
+          (do
+            ;; Always mark EDB origin — even if the tuple key was already
+            ;; present (e.g. previously derived), so an explicit assert
+            ;; promotes it to EDB and protects it from the IDB wipe.
+            (dl-mark-edb! db rel-key tk)
+            (dl-add-derived! db lit)))))))
+
+;; The full safety check lives in builtins.sx (it has to know which
+;; predicates are built-ins). dl-add-rule! calls it via forward
+;; reference; load builtins.sx alongside db.sx in any setup that
+;; adds rules. The fallback below is used if builtins.sx isn't loaded.
+(define
+  dl-rule-check-safety
+  (fn
+    (rule)
+    (let
+      ((head-vars (dl-vars-of (get rule :head))) (body-vars (list)))
+      (do
+        (for-each
+          (fn
+            (lit)
+            (when
+              (and
+                (list? lit)
+                (> (len lit) 0)
+                (not (and (dict? lit) (has-key? lit :neg))))
+              (for-each
+                (fn
+                  (v)
+                  (when
+                    (not (dl-member-string? v body-vars))
+                    (append! body-vars v)))
+                (dl-vars-of lit))))
+          (get rule :body))
+        (let
+          ((missing (list)))
+          (do
+            (for-each
+              (fn
+                (v)
+                (when
+                  (and
+                    (not (dl-member-string? v body-vars))
+                    (not (= v "_")))
+                  (append! missing v)))
+              head-vars)
+            (cond
+              ((> (len missing) 0)
+                (str
+                  "head variable(s) "
+                  missing
+                  " do not appear in any body literal"))
+              (else nil))))))))
+
+(define
+  dl-rename-anon-term
+  (fn
+    (term next-name)
+    (cond
+      ((and (symbol? term) (= (symbol->string term) "_"))
+        (next-name))
+      ((list? term)
+        (map (fn (x) (dl-rename-anon-term x next-name)) term))
+      (else term))))
+
+(define
+  dl-rename-anon-lit
+  (fn
+    (lit next-name)
+    (cond
+      ((and (dict? lit) (has-key? lit :neg))
+        {:neg (dl-rename-anon-term (get lit :neg) next-name)})
+      ((list? lit) (dl-rename-anon-term lit next-name))
+      (else lit))))
+
+(define
+  dl-make-anon-renamer
+  (fn
+    (start)
+    (let ((counter start))
+      (fn () (do (set! counter (+ counter 1))
+                 (string->symbol (str "_anon" counter)))))))
+
+;; Scan a rule for variables already named `_anon<N>` (which would
+;; otherwise collide with the renamer's output). Returns the max N
+;; seen, or 0 if none. The renamer then starts at that max + 1, so
+;; freshly-introduced anonymous names can't shadow a user-written
+;; `_anon<N>` symbol.
+(define
+  dl-max-anon-num
+  (fn
+    (term acc)
+    (cond
+      ((symbol? term)
+        (let ((s (symbol->string term)))
+          (cond
+            ((and (>= (len s) 6) (= (slice s 0 5) "_anon"))
+              (let ((n (dl-try-parse-int (slice s 5 (len s)))))
+                (cond
+                  ((nil? n) acc)
+                  ((> n acc) n)
+                  (else acc))))
+            (else acc))))
+      ((dict? term)
+        (cond
+          ((has-key? term :neg)
+            (dl-max-anon-num (get term :neg) acc))
+          (else acc)))
+      ((list? term) (dl-max-anon-num-list term acc 0))
+      (else acc))))
+
+(define
+  dl-max-anon-num-list
+  (fn
+    (xs acc i)
+    (cond
+      ((>= i (len xs)) acc)
+      (else
+        (dl-max-anon-num-list xs (dl-max-anon-num (nth xs i) acc) (+ i 1))))))
+
+;; Cheap "is this string a decimal int" check. Returns the number or
+;; nil. Avoids relying on host parse-number, which on non-int strings
+;; might raise rather than return nil.
+(define
+  dl-try-parse-int
+  (fn
+    (s)
+    (cond
+      ((= (len s) 0) nil)
+      ((not (dl-all-digits? s 0 (len s))) nil)
+      (else (parse-number s)))))
+
+(define
+  dl-all-digits?
+  (fn
+    (s i n)
+    (cond
+      ((>= i n) true)
+      ((let ((c (slice s i (+ i 1))))
+         (not (and (>= c "0") (<= c "9"))))
+        false)
+      (else (dl-all-digits? s (+ i 1) n)))))
+
+(define
+  dl-rename-anon-rule
+  (fn
+    (rule)
+    (let
+      ((start (dl-max-anon-num (get rule :head)
+                (dl-max-anon-num-list (get rule :body) 0 0))))
+      (let ((next-name (dl-make-anon-renamer start)))
+        {:head (dl-rename-anon-term (get rule :head) next-name)
+         :body (map (fn (lit) (dl-rename-anon-lit lit next-name))
+                    (get rule :body))}))))
+
+(define
+  dl-add-rule!
+  (fn
+    (db rule)
+    (cond
+      ((not (dict? rule))
+        (error (str "dl-add-rule!: expected rule dict, got " rule)))
+      ((not (has-key? rule :head))
+        (error (str "dl-add-rule!: rule missing :head, got " rule)))
+      ((not (and (list? (get rule :head))
+                 (> (len (get rule :head)) 0)
+                 (symbol? (first (get rule :head)))))
+        (error (str "dl-add-rule!: head must be a non-empty list "
+                    "starting with a relation-name symbol, got "
+                    (get rule :head))))
+      ((not (dl-args-simple? (get rule :head) 1 (len (get rule :head))))
+        (error (str "dl-add-rule!: rule head args must be variables or "
+                    "constants — compound terms (e.g. `(*(X, 2))`) are "
+                    "not legal in head position; introduce an `is`-bound "
+                    "intermediate in the body. got " (get rule :head))))
+      ((not (list? (if (has-key? rule :body) (get rule :body) (list))))
+        (error (str "dl-add-rule!: body must be a list of literals, got "
+                    (get rule :body))))
+      ((dl-reserved-rel? (dl-rel-name (get rule :head)))
+        (error (str "dl-add-rule!: '" (dl-rel-name (get rule :head))
+                    "' is a reserved name (built-in / aggregate / negation)")))
+      (else
+        (let ((rule (dl-rename-anon-rule rule)))
+          (let
+            ((err (dl-rule-check-safety rule)))
+            (cond
+              ((not (nil? err)) (error (str "dl-add-rule!: " err)))
+              (else
+                (let
+                  ((rules (get db :rules)))
+                  (do (append! rules rule) true))))))))))
+
+(define
+  dl-add-clause!
+  (fn
+    (db clause)
+    (cond
+      ((has-key? clause :query) false)
+      ((and (has-key? clause :body) (= (len (get clause :body)) 0))
+        (dl-add-fact! db (get clause :head)))
+      (else (dl-add-rule! db clause)))))
+
+(define
+  dl-load-program!
+  (fn
+    (db source)
+    (let
+      ((clauses (dl-parse source)))
+      (do (for-each (fn (c) (dl-add-clause! db c)) clauses) db))))
+
+(define
+  dl-program
+  (fn (source) (let ((db (dl-make-db))) (dl-load-program! db source))))
+
+(define dl-rules (fn (db) (get db :rules)))
+
+(define
+  dl-fact-count
+  (fn
+    (db)
+    (let
+      ((facts (get db :facts)) (total 0))
+      (do
+        (for-each
+          (fn (k) (set! total (+ total (len (get facts k)))))
+          (keys facts))
+        total))))
+
+;; Returns {<rel-name>: tuple-count} for debugging. Includes
+;; relations with any tuples plus all rule-head relations (so empty
+;; IDB shows as 0). Skips empty EDB-only entries that are placeholders
+;; from internal `dl-ensure-rel!` calls.
+(define
+  dl-summary
+  (fn
+    (db)
+    (let
+      ((facts (get db :facts))
+        (out {})
+        (rule-heads (list)))
+      (do
+        (for-each
+          (fn
+            (rule)
+            (let ((h (dl-rel-name (get rule :head))))
+              (when
+                (and (not (nil? h)) (not (dl-member-string? h rule-heads)))
+                (append! rule-heads h))))
+          (dl-rules db))
+        (for-each
+          (fn
+            (k)
+            (let ((c (len (get facts k))))
+              (when
+                (or (> c 0) (dl-member-string? k rule-heads))
+                (dict-set! out k c))))
+          (keys facts))
+        ;; Add rule heads that have no facts (yet).
+        (for-each
+          (fn
+            (k)
+            (when (not (has-key? out k)) (dict-set! out k 0)))
+          rule-heads)
+        out))))

lib/datalog/demo.sx

@@ -0,0 +1,162 @@
+;; lib/datalog/demo.sx — example programs over rose-ash-shaped data.
+;;
+;; Phase 10 prototypes Datalog as a rose-ash query language. Wiring
+;; the EDB to actual PostgreSQL is out of scope for this loop (it
+;; would touch service code outside lib/datalog/), but the programs
+;; below show the shape of queries we want, and the test suite runs
+;; them against synthetic in-memory tuples loaded via dl-program-data.
+;;
+;; Seven thematic demos:
+;;
+;;   1. Federation — follow graph, transitive reach, mutuals, FOAF.
+;;   2. Content    — posts, tags, likes, popularity, "for you" feed.
+;;   3. Permissions — group membership and resource access.
+;;   4. Cooking-posts — canonical "posts about cooking by people I
+;;                      follow (transitively)" multi-domain query.
+;;   5. Tag co-occurrence — distinct (T1, T2) pairs with counts.
+;;   6. Shortest path  — weighted-DAG path enumeration + min agg.
+;;   7. Org chart      — transitive subordinate + headcount per mgr.
+
+;; ── Demo 1: federation follow graph ─────────────────────────────
+;; EDB: (follows ACTOR-A ACTOR-B) — A follows B.
+;; IDB:
+;;   (mutual A B)            — A follows B and B follows A
+;;   (reachable A B)         — transitive follow closure
+;;   (foaf A C)              — friend of a friend (mutual filter)
+(define
+  dl-demo-federation-rules
+  (quote
+    ((mutual A B <- (follows A B) (follows B A))
+     (reachable A B <- (follows A B))
+     (reachable A C <- (follows A B) (reachable B C))
+     (foaf A C <- (follows A B) (follows B C) (!= A C)))))
+
+;; ── Demo 2: content recommendation ──────────────────────────────
+;; EDB:
+;;   (authored ACTOR POST)
+;;   (tagged POST TAG)
+;;   (liked ACTOR POST)
+;; IDB:
+;;   (post-likes POST N)            — count of likes per post
+;;   (popular POST)                 — posts with >= 3 likes
+;;   (tagged-by-mutual ACTOR POST)  — post tagged TOPIC by someone
+;;                                    A's mutuals follow.
+(define
+  dl-demo-content-rules
+  (quote
+    ((post-likes P N <- (authored Author P) (count N L (liked L P)))
+     (popular P <- (authored Author P) (post-likes P N) (>= N 3))
+     (interesting Me P
+       <-
+       (follows Me Buddy)
+       (authored Buddy P)
+       (popular P)))))
+
+;; ── Demo 3: role-based permissions ──────────────────────────────
+;; EDB:
+;;   (member ACTOR GROUP)
+;;   (subgroup CHILD PARENT)
+;;   (allowed GROUP RESOURCE)
+;; IDB:
+;;   (in-group ACTOR GROUP)         — direct or via subgroup chain
+;;   (can-access ACTOR RESOURCE)    — actor inherits group permission
+(define
+  dl-demo-perm-rules
+  (quote
+    ((in-group A G <- (member A G))
+     (in-group A G <- (member A H) (subgroup-trans H G))
+     (subgroup-trans X Y <- (subgroup X Y))
+     (subgroup-trans X Z <- (subgroup X Y) (subgroup-trans Y Z))
+     (can-access A R <- (in-group A G) (allowed G R)))))
+
+;; ── Demo 4: cooking-posts (the canonical Phase 10 query) ────────
+;; "Posts about cooking by people I follow (transitively)."
+;; Combines federation (follows + transitive reach), authoring,
+;; tagging — the rose-ash multi-domain join.
+;;
+;; EDB:
+;;   (follows ACTOR-A ACTOR-B)
+;;   (authored ACTOR POST)
+;;   (tagged POST TAG)
+(define
+  dl-demo-cooking-rules
+  (quote
+    ((reach Me Them <- (follows Me Them))
+     (reach Me Them <- (follows Me X) (reach X Them))
+     (cooking-post-by-network Me P
+       <-
+       (reach Me Author)
+       (authored Author P)
+       (tagged P cooking)))))
+
+;; ── Demo 5: tag co-occurrence ───────────────────────────────────
+;; "Posts tagged with both T1 AND T2." Useful for narrowed-down
+;; recommendations like "vegetarian cooking" posts.
+;;
+;; EDB:
+;;   (tagged POST TAG)
+;; IDB:
+;;   (cotagged POST T1 T2)        — post has both T1 and T2 (T1 != T2)
+;;   (popular-pair T1 T2 N)       — count of posts cotagged (T1, T2)
+(define
+  dl-demo-tag-cooccur-rules
+  (quote
+    ((cotagged P T1 T2 <- (tagged P T1) (tagged P T2) (!= T1 T2))
+     ;; Distinct (T1, T2) pairs that occur somewhere.
+     (tag-pair T1 T2 <- (cotagged P T1 T2))
+     (tag-pair-count T1 T2 N
+       <-
+       (tag-pair T1 T2)
+       (count N P (cotagged P T1 T2))))))
+
+;; ── Demo 6: weighted-DAG shortest path ─────────────────────────
+;; "What's the cheapest way from X to Y?" Edge weights with `is`
+;; arithmetic to sum costs, then `min` aggregation to pick the
+;; shortest. Termination requires the graph to be a DAG (cycles
+;; would produce infinite distances without a bound; programs
+;; built on this should add a depth filter `(<, D, MAX)` if cycles
+;; are possible).
+;;
+;; EDB:
+;;   (edge FROM TO COST)
+;; IDB:
+;;   (path FROM TO COST)            — any path
+;;   (shortest FROM TO COST)        — minimum cost path
+(define
+  dl-demo-shortest-path-rules
+  (quote
+    ((path X Y W <- (edge X Y W))
+     (path X Z W
+       <-
+       (edge X Y W1)
+       (path Y Z W2)
+       (is W (+ W1 W2)))
+     (shortest X Y W <- (path X Y _) (min W C (path X Y C))))))
+
+;; ── Demo 7: org chart + transitive headcount ───────────────────
+;; Manager graph: each employee has a single manager. Compute the
+;; transitive subordinate set and headcount per manager.
+;;
+;; EDB:
+;;   (manager EMP MGR)            — EMP reports directly to MGR
+;; IDB:
+;;   (subordinate MGR EMP)        — EMP is in MGR's subtree
+;;   (headcount MGR N)            — number of subordinates under MGR
+(define
+  dl-demo-org-rules
+  (quote
+    ((subordinate Mgr Emp <- (manager Emp Mgr))
+     (subordinate Mgr Emp
+       <- (manager Mid Mgr) (subordinate Mid Emp))
+     (headcount Mgr N
+       <- (subordinate Mgr Anyone) (count N E (subordinate Mgr E))))))
+
+;; ── Loader stub ──────────────────────────────────────────────────
+;; Wiring to PostgreSQL would replace these helpers with calls into
+;; rose-ash's internal HTTP RPC (fetch_data → /internal/data/...).
+;; The shape returned by dl-load-from-edb! is the same in either case.
+(define
+  dl-demo-make
+  (fn
+    (facts rules)
+    (dl-program-data facts rules)))

lib/datalog/eval.sx

@@ -0,0 +1,512 @@
+;; lib/datalog/eval.sx — fixpoint evaluator (naive + semi-naive).
+;;
+;; Two saturators are exposed:
+;;   `dl-saturate-naive!` — re-joins each rule against the full DB every
+;;     iteration. Reference implementation; useful for differential tests.
+;;   `dl-saturate!`       — semi-naive default. Tracks per-relation delta
+;;     sets and substitutes one positive body literal per rule with the
+;;     delta of its relation, joining the rest against the previous-
+;;     iteration DB. Same fixpoint, dramatically less work on recursive
+;;     rules.
+;;
+;; Body literal kinds:
+;;   positive (rel arg ... arg)  → match against EDB+IDB tuples
+;;   built-in (< X Y), (is X e)  → constraint via dl-eval-builtin
+;;   negation {:neg lit}         → Phase 7
+
+(define
+  dl-match-positive
+  (fn
+    (lit db subst)
+    (let
+      ((rel (dl-rel-name lit)) (results (list)))
+      (cond
+        ((nil? rel) (error (str "dl-match-positive: bad literal " lit)))
+        (else
+          (let
+            ;; If the first argument walks to a non-variable (constant
+            ;; or already-bound var), use the first-arg index for
+            ;; this relation. Otherwise scan the full tuple list.
+            ((tuples
+               (cond
+                 ((>= (len lit) 2)
+                   (let ((walked (dl-walk (nth lit 1) subst)))
+                     (cond
+                       ((dl-var? walked) (dl-rel-tuples db rel))
+                       (else (dl-index-lookup db rel walked)))))
+                 (else (dl-rel-tuples db rel)))))
+            (do
+              (for-each
+                (fn
+                  (tuple)
+                  (let
+                    ((s (dl-unify lit tuple subst)))
+                    (when (not (nil? s)) (append! results s))))
+                tuples)
+              results)))))))
+
+;; Match a positive literal against the delta set for its relation only.
+(define
+  dl-match-positive-delta
+  (fn
+    (lit delta subst)
+    (let
+      ((rel (dl-rel-name lit)) (results (list)))
+      (let
+        ((tuples (if (has-key? delta rel) (get delta rel) (list))))
+        (do
+          (for-each
+            (fn
+              (tuple)
+              (let
+                ((s (dl-unify lit tuple subst)))
+                (when (not (nil? s)) (append! results s))))
+            tuples)
+          results)))))
+
+;; Naive matcher (for dl-saturate-naive! and dl-query post-saturation).
+(define
+  dl-match-negation
+  (fn
+    (inner db subst)
+    (let
+      ((walked (dl-apply-subst inner subst))
+        (matches (dl-match-positive inner db subst)))
+      (cond
+        ((= (len matches) 0) (list subst))
+        (else (list))))))
+
+(define
+  dl-match-lit
+  (fn
+    (lit db subst)
+    (cond
+      ((and (dict? lit) (has-key? lit :neg))
+        (dl-match-negation (get lit :neg) db subst))
+      ((dl-aggregate? lit) (dl-eval-aggregate lit db subst))
+      ((dl-builtin? lit)
+        (let
+          ((s (dl-eval-builtin lit subst)))
+          (if (nil? s) (list) (list s))))
+      ((and (list? lit) (> (len lit) 0))
+        (dl-match-positive lit db subst))
+      (else (error (str "datalog: unknown body-literal shape: " lit))))))
+
+(define
+  dl-find-bindings
+  (fn (lits db subst) (dl-fb-aux lits db subst 0 (len lits))))
+
+(define
+  dl-fb-aux
+  (fn
+    (lits db subst i n)
+    (cond
+      ((nil? subst) (list))
+      ((>= i n) (list subst))
+      (else
+        (let
+          ((options (dl-match-lit (nth lits i) db subst))
+            (results (list)))
+          (do
+            (for-each
+              (fn
+                (s)
+                (for-each
+                  (fn (s2) (append! results s2))
+                  (dl-fb-aux lits db s (+ i 1) n)))
+              options)
+            results))))))
+
+;; Naive: apply each rule against full DB until no new tuples.
+(define
+  dl-apply-rule!
+  (fn
+    (db rule)
+    (let
+      ((head (get rule :head)) (body (get rule :body)) (new? false))
+      (do
+        (for-each
+          (fn
+            (s)
+            (let
+              ((derived (dl-apply-subst head s)))
+              (when (dl-add-derived! db derived) (set! new? true))))
+          (dl-find-bindings body db (dl-empty-subst)))
+        new?))))
+
+;; Returns true iff one more saturation step would derive no new
+;; tuples (i.e. the db is at fixpoint). Useful in tests that want
+;; to assert "no work left" after a saturation call. Works under
+;; either saturator since both compute the same fixpoint.
+(define
+  dl-saturated?
+  (fn
+    (db)
+    (let ((any-new false))
+      (do
+        (for-each
+          (fn
+            (rule)
+            (when (not any-new)
+              (for-each
+                (fn
+                  (s)
+                  (let ((derived (dl-apply-subst (get rule :head) s)))
+                    (when
+                      (and (not any-new)
+                           (not (dl-tuple-member?
+                                  derived
+                                  (dl-rel-tuples
+                                    db (dl-rel-name derived)))))
+                      (set! any-new true))))
+                (dl-find-bindings (get rule :body) db (dl-empty-subst)))))
+          (dl-rules db))
+        (not any-new)))))
+
+(define
+  dl-saturate-naive!
+  (fn
+    (db)
+    (let
+      ((changed true))
+      (do
+        (define
+          dl-snloop
+          (fn
+            ()
+            (when
+              changed
+              (do
+                (set! changed false)
+                (for-each
+                  (fn (r) (when (dl-apply-rule! db r) (set! changed true)))
+                  (dl-rules db))
+                (dl-snloop)))))
+        (dl-snloop)
+        db))))
+
+;; ── Semi-naive ───────────────────────────────────────────────────
+
+;; Take a snapshot dict {rel -> tuples} of every relation currently in
+;; the DB. Used as initial delta for the first iteration.
+(define
+  dl-snapshot-facts
+  (fn
+    (db)
+    (let
+      ((facts (get db :facts)) (out {}))
+      (do
+        (for-each
+          (fn (k) (dict-set! out k (dl-copy-list (get facts k))))
+          (keys facts))
+        out))))
+
+(define
+  dl-copy-list
+  (fn
+    (xs)
+    (let
+      ((out (list)))
+      (do (for-each (fn (x) (append! out x)) xs) out))))
+
+;; Does any relation in `delta` have ≥1 tuple?
+(define
+  dl-delta-empty?
+  (fn
+    (delta)
+    (let
+      ((ks (keys delta)) (any-non-empty false))
+      (do
+        (for-each
+          (fn
+            (k)
+            (when
+              (> (len (get delta k)) 0)
+              (set! any-non-empty true)))
+          ks)
+        (not any-non-empty)))))
+
+;; Find substitutions such that `lits` are all satisfied AND `delta-idx`
+;; is matched against the per-relation delta only. The other positive
+;; literals match against the snapshot DB (db.facts read at iteration
+;; start). Built-ins and negations behave as in `dl-match-lit`.
+(define
+  dl-find-bindings-semi
+  (fn
+    (lits db delta delta-idx subst)
+    (dl-fbs-aux lits db delta delta-idx 0 subst)))
+
+(define
+  dl-fbs-aux
+  (fn
+    (lits db delta delta-idx i subst)
+    (cond
+      ((nil? subst) (list))
+      ((>= i (len lits)) (list subst))
+      (else
+        (let
+          ((lit (nth lits i))
+            (options
+              (cond
+                ((and (dict? lit) (has-key? lit :neg))
+                  (dl-match-negation (get lit :neg) db subst))
+                ((dl-aggregate? lit) (dl-eval-aggregate lit db subst))
+                ((dl-builtin? lit)
+                  (let
+                    ((s (dl-eval-builtin lit subst)))
+                    (if (nil? s) (list) (list s))))
+                ((and (list? lit) (> (len lit) 0))
+                  (if
+                    (= i delta-idx)
+                    (dl-match-positive-delta lit delta subst)
+                    (dl-match-positive lit db subst)))
+                (else (error (str "datalog: unknown body-lit: " lit)))))
+            (results (list)))
+          (do
+            (for-each
+              (fn
+                (s)
+                (for-each
+                  (fn (s2) (append! results s2))
+                  (dl-fbs-aux lits db delta delta-idx (+ i 1) s)))
+              options)
+            results))))))
+
+;; Collect candidate head tuples from a rule using delta. Walks every
+;; positive body position and unions the resulting heads. For rules
+;; with no positive body literal, falls back to a naive single-pass
+;; (so static facts like `(p X) :- (= X 5).` derive on iteration 1).
+(define
+  dl-collect-rule-candidates
+  (fn
+    (rule db delta)
+    (let
+      ((head (get rule :head))
+        (body (get rule :body))
+        (out (list))
+        (saw-pos false))
+      (do
+        (define
+          dl-cri
+          (fn
+            (i)
+            (when
+              (< i (len body))
+              (do
+                (let
+                  ((lit (nth body i)))
+                  (when
+                    (dl-positive-lit? lit)
+                    (do
+                      (set! saw-pos true)
+                      (for-each
+                        (fn (s) (append! out (dl-apply-subst head s)))
+                        (dl-find-bindings-semi
+                          body
+                          db
+                          delta
+                          i
+                          (dl-empty-subst))))))
+                (dl-cri (+ i 1))))))
+        (dl-cri 0)
+        (when
+          (not saw-pos)
+          (for-each
+            (fn (s) (append! out (dl-apply-subst head s)))
+            (dl-find-bindings body db (dl-empty-subst))))
+        out))))
+
+;; Add a list of candidate tuples to db; collect newly-added ones into
+;; the new-delta dict (keyed by relation name).
+(define
+  dl-commit-candidates!
+  (fn
+    (db candidates new-delta)
+    (for-each
+      (fn
+        (lit)
+        (when
+          (dl-add-derived! db lit)
+          (let
+            ((rel (dl-rel-name lit)))
+            (do
+              (when
+                (not (has-key? new-delta rel))
+                (dict-set! new-delta rel (list)))
+              (append! (get new-delta rel) lit)))))
+      candidates)))
+
+(define
+  dl-saturate-rules!
+  (fn
+    (db rules)
+    (let
+      ((delta (dl-snapshot-facts db)))
+      (do
+        (define
+          dl-sr-step
+          (fn
+            ()
+            (let
+              ((pending (list)) (new-delta {}))
+              (do
+                (for-each
+                  (fn
+                    (rule)
+                    (for-each
+                      (fn (cand) (append! pending cand))
+                      (dl-collect-rule-candidates rule db delta)))
+                  rules)
+                (dl-commit-candidates! db pending new-delta)
+                (cond
+                  ((dl-delta-empty? new-delta) nil)
+                  (else (do (set! delta new-delta) (dl-sr-step))))))))
+        (dl-sr-step)
+        db))))
+
+;; Stratified driver: rejects non-stratifiable programs at saturation
+;; time, then iterates strata in increasing order, running semi-naive on
+;; the rules whose head sits in that stratum.
+(define
+  dl-saturate!
+  (fn
+    (db)
+    (let
+      ((err (dl-check-stratifiable db)))
+      (cond
+        ((not (nil? err)) (error (str "dl-saturate!: " err)))
+        (else
+          (let
+            ((strata (dl-compute-strata db)))
+            (let
+              ((grouped (dl-group-rules-by-stratum db strata)))
+              (let
+                ((groups (get grouped :groups))
+                  (max-s (get grouped :max)))
+                (do
+                  (define
+                    dl-strat-loop
+                    (fn
+                      (s)
+                      (when
+                        (<= s max-s)
+                        (let
+                          ((sk (str s)))
+                          (do
+                            (when
+                              (has-key? groups sk)
+                              (dl-saturate-rules! db (get groups sk)))
+                            (dl-strat-loop (+ s 1)))))))
+                  (dl-strat-loop 0)
+                  db)))))))))
+
+;; ── Querying ─────────────────────────────────────────────────────
+
+;; Coerce a query argument to a list of body literals. A single literal
+;; like `(p X)` (positive — head is a symbol) or `{:neg ...}` becomes
+;; `((p X))`. A list of literals like `((p X) (q X))` is returned as-is.
+(define
+  dl-query-coerce
+  (fn
+    (goal)
+    (cond
+      ((and (dict? goal) (has-key? goal :neg)) (list goal))
+      ((and (list? goal) (> (len goal) 0) (symbol? (first goal)))
+        (list goal))
+      ((list? goal) goal)
+      (else (error (str "dl-query: unrecognised goal shape: " goal))))))
+
+(define
+  dl-query
+  (fn
+    (db goal)
+    (do
+      (dl-saturate! db)
+      ;; Rename anonymous '_' vars in each goal literal so multiple
+      ;; occurrences do not unify together. Keep the user-facing var
+      ;; list (taken before renaming) so projected results retain user
+      ;; names.
+      (let
+        ((goals (dl-query-coerce goal))
+          ;; Start the renamer past any `_anon<N>` symbols the user
+          ;; may have written in the query — avoids collision.
+          (renamer
+            (dl-make-anon-renamer (dl-max-anon-num-list goal 0 0))))
+        (let
+          ((user-vars (dl-query-user-vars goals))
+            (renamed (map (fn (g) (dl-rename-anon-lit g renamer)) goals)))
+          (let
+            ((substs (dl-find-bindings renamed db (dl-empty-subst)))
+              (results (list)))
+            (do
+              (for-each
+                (fn
+                  (s)
+                  (let
+                    ((proj (dl-project-subst s user-vars)))
+                    (when
+                      (not (dl-tuple-member? proj results))
+                      (append! results proj))))
+                substs)
+              results)))))))
+
+(define
+  dl-query-user-vars
+  (fn
+    (goals)
+    (let ((seen (list)))
+      (do
+        (for-each
+          (fn
+            (g)
+            (cond
+              ((and (dict? g) (has-key? g :neg))
+                (for-each
+                  (fn
+                    (v)
+                    (when
+                      (and (not (= v "_")) (not (dl-member-string? v seen)))
+                      (append! seen v)))
+                  (dl-vars-of (get g :neg))))
+              ((dl-aggregate? g)
+                ;; Only the result var (first arg of the aggregate
+                ;; literal) is user-facing. The aggregated var and
+                ;; any vars in the inner goal are internal.
+                (let ((r (nth g 1)))
+                  (when
+                    (dl-var? r)
+                    (let ((rn (symbol->string r)))
+                      (when
+                        (and (not (= rn "_"))
+                             (not (dl-member-string? rn seen)))
+                        (append! seen rn))))))
+              (else
+                (for-each
+                  (fn
+                    (v)
+                    (when
+                      (and (not (= v "_")) (not (dl-member-string? v seen)))
+                      (append! seen v)))
+                  (dl-vars-of g)))))
+          goals)
+        seen))))
+
+(define
+  dl-project-subst
+  (fn
+    (subst names)
+    (let
+      ((out {}))
+      (do
+        (for-each
+          (fn
+            (n)
+            (let
+              ((sym (string->symbol n)))
+              (let
+                ((v (dl-walk sym subst)))
+                (dict-set! out n (dl-apply-subst v subst)))))
+          names)
+        out))))
+
+(define dl-relation (fn (db name) (dl-rel-tuples db name)))

lib/datalog/magic.sx

@@ -0,0 +1,464 @@
+;; lib/datalog/magic.sx — adornment analysis + sideways info passing.
+;;
+;; First step of the magic-sets transformation (Phase 6). Right now
+;; the saturator does not consume these — they are introspection
+;; helpers that future magic-set rewriting will build on top of.
+;;
+;; Definitions:
+;;   - An *adornment* of an n-ary literal is an n-character string
+;;     of "b" (bound — value already known at the call site) and
+;;     "f" (free — to be derived).
+;;   - SIPS (Sideways Information Passing Strategy) walks the body
+;;     of an adorned rule left-to-right tracking which variables
+;;     have been bound so far, computing each body literal's
+;;     adornment in turn.
+;;
+;; Usage:
+;;
+;;   (dl-adorn-goal '(ancestor tom X))
+;;     => "bf"
+;;
+;;   (dl-rule-sips
+;;     {:head (ancestor X Z)
+;;      :body ((parent X Y) (ancestor Y Z))}
+;;     "bf")
+;;   => ({:lit (parent X Y) :adornment "bf"}
+;;       {:lit (ancestor Y Z) :adornment "bf"})
+
+;; Per-arg adornment under the current bound-var name set.
+(define
+  dl-adorn-arg
+  (fn
+    (arg bound)
+    (cond
+      ((dl-var? arg)
+        (if (dl-member-string? (symbol->string arg) bound) "b" "f"))
+      (else "b"))))
+
+;; Adornment for the args of a literal (after the relation name).
+(define
+  dl-adorn-args
+  (fn
+    (args bound)
+    (cond
+      ((= (len args) 0) "")
+      (else
+        (str
+          (dl-adorn-arg (first args) bound)
+          (dl-adorn-args (rest args) bound))))))
+
+;; Adornment of a top-level goal under the empty bound-var set.
+(define
+  dl-adorn-goal
+  (fn (goal) (dl-adorn-args (rest goal) (list))))
+
+;; Adornment of a literal under an explicit bound set.
+(define
+  dl-adorn-lit
+  (fn (lit bound) (dl-adorn-args (rest lit) bound)))
+
+;; The set of variable names made bound by walking a positive
+;; literal whose adornment is known. Free positions add their
+;; vars to the bound set.
+(define
+  dl-vars-bound-by-lit
+  (fn
+    (lit bound)
+    (let ((args (rest lit)) (out (list)))
+      (do
+        (for-each
+          (fn (a)
+            (when
+              (and (dl-var? a)
+                   (not (dl-member-string? (symbol->string a) bound))
+                   (not (dl-member-string? (symbol->string a) out)))
+              (append! out (symbol->string a))))
+          args)
+        out))))
+
+;; Walk the rule body left-to-right tracking bound vars seeded by the
+;; head adornment. Returns a list of {:lit :adornment} entries.
+;;
+;; Negation, comparison, and built-ins are passed through with their
+;; adornment computed from the current bound set; they don't add new
+;; bindings (except `is`, which binds its left arg if a var). Aggregates
+;; are treated like is — the result var becomes bound.
+(define
+  dl-init-head-bound
+  (fn
+    (head adornment)
+    (let ((args (rest head)) (out (list)))
+      (do
+        (define
+          dl-ihb-loop
+          (fn
+            (i)
+            (when
+              (< i (len args))
+              (do
+                (let
+                  ((c (slice adornment i (+ i 1)))
+                    (a (nth args i)))
+                  (when
+                    (and (= c "b") (dl-var? a))
+                    (let ((n (symbol->string a)))
+                      (when
+                        (not (dl-member-string? n out))
+                        (append! out n)))))
+                (dl-ihb-loop (+ i 1))))))
+        (dl-ihb-loop 0)
+        out))))
+
+(define
+  dl-rule-sips
+  (fn
+    (rule head-adornment)
+    (let
+      ((bound (dl-init-head-bound (get rule :head) head-adornment))
+        (out (list)))
+      (do
+        (for-each
+          (fn
+            (lit)
+            (cond
+              ((and (dict? lit) (has-key? lit :neg))
+                (let ((target (get lit :neg)))
+                  (append!
+                    out
+                    {:lit lit :adornment (dl-adorn-lit target bound)})))
+              ((dl-builtin? lit)
+                (let ((adn (dl-adorn-lit lit bound)))
+                  (do
+                    (append! out {:lit lit :adornment adn})
+                    ;; `is` binds its left arg (if var) once RHS is ground.
+                    (when
+                      (and (= (dl-rel-name lit) "is") (dl-var? (nth lit 1)))
+                      (let ((n (symbol->string (nth lit 1))))
+                        (when
+                          (not (dl-member-string? n bound))
+                          (append! bound n)))))))
+              ((and (list? lit) (dl-aggregate? lit))
+                (let ((adn (dl-adorn-lit lit bound)))
+                  (do
+                    (append! out {:lit lit :adornment adn})
+                    ;; Result var (first arg) becomes bound.
+                    (when (dl-var? (nth lit 1))
+                      (let ((n (symbol->string (nth lit 1))))
+                        (when
+                          (not (dl-member-string? n bound))
+                          (append! bound n)))))))
+              ((and (list? lit) (> (len lit) 0))
+                (let ((adn (dl-adorn-lit lit bound)))
+                  (do
+                    (append! out {:lit lit :adornment adn})
+                    (for-each
+                      (fn (n)
+                        (when (not (dl-member-string? n bound))
+                          (append! bound n)))
+                      (dl-vars-bound-by-lit lit bound)))))))
+          (get rule :body))
+        out))))
+
+;; ── Magic predicate naming + bound-args extraction ─────────────
+;; These are building blocks for the magic-sets *transformation*
+;; itself. The transformation (which generates rewritten rules
+;; with magic_<rel>^<adornment> filters) is future work — for now
+;; these helpers can be used to inspect what such a transformation
+;; would produce.
+
+;; "magic_p^bf" given relation "p" and adornment "bf".
+(define
+  dl-magic-rel-name
+  (fn (rel adornment) (str "magic_" rel "^" adornment)))
+
+;; A magic predicate literal:
+;;   (magic_<rel>^<adornment> arg1 arg2 ...)
+(define
+  dl-magic-lit
+  (fn
+    (rel adornment bound-args)
+    (cons (string->symbol (dl-magic-rel-name rel adornment)) bound-args)))
+
+;; Extract bound args (those at "b" positions in `adornment`) from a
+;; literal `(rel arg1 arg2 ... argN)`. Returns the list of arg values.
+(define
+  dl-bound-args
+  (fn
+    (lit adornment)
+    (let ((args (rest lit)) (out (list)))
+      (do
+        (define
+          dl-ba-loop
+          (fn
+            (i)
+            (when
+              (< i (len args))
+              (do
+                (when
+                  (= (slice adornment i (+ i 1)) "b")
+                  (append! out (nth args i)))
+                (dl-ba-loop (+ i 1))))))
+        (dl-ba-loop 0)
+        out))))
+
+;; ── Magic-sets rewriter ─────────────────────────────────────────
+;;
+;; Given the original rule list and a query (rel, adornment) pair,
+;; generates the magic-rewritten program: a list of rules that
+;; (a) gate each original rule with a `magic_<rel>^<adn>` filter and
+;; (b) propagate the magic relation through SIPS so that only
+;; query-relevant tuples are derived. Seed facts are returned
+;; separately and must be added to the db at evaluation time.
+;;
+;; Output: {:rules <rewritten-rules> :seed <magic-seed-literal>}
+;;
+;; The rewriter only rewrites IDB rules; EDB facts pass through.
+;; Built-in predicates and negation in body literals are kept in
+;; place but do not generate propagation rules of their own.
+
+(define
+  dl-magic-pair-key
+  (fn (rel adornment) (str rel "^" adornment)))
+
+(define
+  dl-magic-rewrite
+  (fn
+    (rules query-rel query-adornment query-args)
+    (let
+      ((seen (list))
+        (queue (list))
+        (out (list)))
+      (do
+        (define
+          dl-mq-mark!
+          (fn
+            (rel adornment)
+            (let ((k (dl-magic-pair-key rel adornment)))
+              (when
+                (not (dl-member-string? k seen))
+                (do
+                  (append! seen k)
+                  (append! queue {:rel rel :adn adornment}))))))
+
+        (define
+          dl-mq-rewrite-rule!
+          (fn
+            (rule adn)
+            (let
+              ((head (get rule :head))
+                (body (get rule :body))
+                (sips (dl-rule-sips rule adn)))
+              (let
+                ((magic-filter
+                   (dl-magic-lit
+                     (dl-rel-name head)
+                     adn
+                     (dl-bound-args head adn))))
+                (do
+                  ;; Adorned rule: head :- magic-filter, body...
+                  (let ((new-body (list)))
+                    (do
+                      (append! new-body magic-filter)
+                      (for-each
+                        (fn (lit) (append! new-body lit))
+                        body)
+                      (append! out {:head head :body new-body})))
+                  ;; Propagation rules for each positive non-builtin
+                  ;; body literal at position i.
+                  (define
+                    dl-mq-prop-loop
+                    (fn
+                      (i)
+                      (when
+                        (< i (len body))
+                        (do
+                          (let
+                            ((lit (nth body i))
+                              (sip-entry (nth sips i)))
+                            (when
+                              (and (list? lit)
+                                   (> (len lit) 0)
+                                   (not (and (dict? lit) (has-key? lit :neg)))
+                                   (not (dl-builtin? lit))
+                                   (not (dl-aggregate? lit)))
+                              (let
+                                ((lit-adn (get sip-entry :adornment))
+                                  (lit-rel (dl-rel-name lit)))
+                                (let
+                                  ((prop-head
+                                     (dl-magic-lit
+                                       lit-rel
+                                       lit-adn
+                                       (dl-bound-args lit lit-adn))))
+                                  (let ((prop-body (list)))
+                                    (do
+                                      (append! prop-body magic-filter)
+                                      (define
+                                        dl-mq-prefix-loop
+                                        (fn
+                                          (j)
+                                          (when
+                                            (< j i)
+                                            (do
+                                              (append!
+                                                prop-body
+                                                (nth body j))
+                                              (dl-mq-prefix-loop (+ j 1))))))
+                                      (dl-mq-prefix-loop 0)
+                                      (append!
+                                        out
+                                        {:head prop-head :body prop-body})
+                                      (dl-mq-mark! lit-rel lit-adn)))))))
+                          (dl-mq-prop-loop (+ i 1))))))
+                  (dl-mq-prop-loop 0))))))
+
+        (dl-mq-mark! query-rel query-adornment)
+
+        (let ((idx 0))
+          (define
+            dl-mq-process
+            (fn
+              ()
+              (when
+                (< idx (len queue))
+                (let ((item (nth queue idx)))
+                  (do
+                    (set! idx (+ idx 1))
+                    (let
+                      ((rel (get item :rel)) (adn (get item :adn)))
+                      (for-each
+                        (fn
+                          (rule)
+                          (when
+                            (= (dl-rel-name (get rule :head)) rel)
+                            (dl-mq-rewrite-rule! rule adn)))
+                        rules))
+                    (dl-mq-process))))))
+          (dl-mq-process))
+
+        {:rules out
+         :seed
+         (dl-magic-lit
+           query-rel
+           query-adornment
+           query-args)}))))
+
+;; ── Top-level magic-sets driver ─────────────────────────────────
+;;
+;; (dl-magic-query db query-goal) — run `query-goal` under magic-sets
+;; evaluation. Builds a fresh internal db with:
+;;   - the caller's EDB facts (relations not headed by any rule),
+;;   - the magic seed fact, and
+;;   - the rewritten rules.
+;; Saturates and queries, returning the substitution list. The
+;; caller's db is untouched.
+;;
+;; Useful primarily as a perf alternative for queries that only
+;; need a small slice of a recursive relation. Equivalent to
+;; dl-query for any single fully-stratifiable program.
+
+(define
+  dl-magic-rule-heads
+  (fn
+    (rules)
+    (let ((seen (list)))
+      (do
+        (for-each
+          (fn
+            (r)
+            (let ((h (dl-rel-name (get r :head))))
+              (when
+                (and (not (nil? h)) (not (dl-member-string? h seen)))
+                (append! seen h))))
+          rules)
+        seen))))
+
+;; True iff any rule's body contains a literal kind that the magic
+;; rewriter doesn't propagate magic to — i.e. an aggregate or a
+;; negation. Used by dl-magic-query to decide whether to pre-saturate
+;; the source db (for correctness on stratified programs) or skip
+;; that step (preserving full magic-sets efficiency for pure
+;; positive programs).
+(define
+  dl-rule-has-nonprop-lit?
+  (fn
+    (body i n)
+    (cond
+      ((>= i n) false)
+      ((let ((lit (nth body i)))
+         (or (and (dict? lit) (has-key? lit :neg))
+             (dl-aggregate? lit)))
+        true)
+      (else (dl-rule-has-nonprop-lit? body (+ i 1) n)))))
+
+(define
+  dl-rules-need-presaturation?
+  (fn
+    (rules)
+    (cond
+      ((= (len rules) 0) false)
+      ((let ((body (get (first rules) :body)))
+         (dl-rule-has-nonprop-lit? body 0 (len body)))
+        true)
+      (else (dl-rules-need-presaturation? (rest rules))))))
+
+(define
+  dl-magic-query
+  (fn
+    (db query-goal)
+    ;; Magic-sets only applies to positive non-builtin / non-aggregate
+    ;; literals against rule-defined relations. For other goal shapes
+    ;; (built-ins, aggregates, EDB-only relations) the seed is either
+    ;; non-ground or unused; fall back to dl-query.
+    (cond
+      ((not (and (list? query-goal)
+                 (> (len query-goal) 0)
+                 (symbol? (first query-goal))))
+        (error (str "dl-magic-query: goal must be a positive literal "
+                    "(non-empty list with a symbol head), got " query-goal)))
+      ((or (dl-builtin? query-goal)
+           (dl-aggregate? query-goal)
+           (and (dict? query-goal) (has-key? query-goal :neg)))
+        (dl-query db query-goal))
+      (else
+        (do
+          ;; If the rule set has aggregates or negation, pre-saturate
+          ;; the source db before copying facts. The magic rewriter
+          ;; passes aggregate body lits and negated lits through
+          ;; unchanged (no magic propagation generated for them) — so
+          ;; if their inner-goal relation is IDB, it would be empty in
+          ;; the magic db. Pre-saturating ensures equivalence with
+          ;; `dl-query` for every stratified program. Pure positive
+          ;; programs skip this and keep the full magic-sets perf win
+          ;; from goal-directed re-derivation.
+          (when
+            (dl-rules-need-presaturation? (dl-rules db))
+            (dl-saturate! db))
+          (let
+            ((query-rel (dl-rel-name query-goal))
+              (query-adn (dl-adorn-goal query-goal)))
+            (let
+              ((query-args (dl-bound-args query-goal query-adn))
+                (rules (dl-rules db)))
+              (let
+                ((rewritten (dl-magic-rewrite rules query-rel query-adn query-args))
+                  (mdb (dl-make-db))
+                  (rule-heads (dl-magic-rule-heads rules)))
+            (do
+              ;; Copy ALL existing facts. EDB-only relations bring their
+              ;; tuples; mixed EDB+IDB relations bring both their EDB
+              ;; portion and any pre-saturated IDB tuples (which the
+              ;; rewritten rules would re-derive anyway). Skipping facts
+              ;; for rule-headed relations would leave the magic run
+              ;; without the EDB portion of mixed relations.
+              (for-each
+                (fn
+                  (rel)
+                  (for-each
+                    (fn (t) (dl-add-fact! mdb t))
+                    (dl-rel-tuples db rel)))
+                (keys (get db :facts)))
+              ;; Seed + rewritten rules.
+              (dl-add-fact! mdb (get rewritten :seed))
+              (for-each (fn (r) (dl-add-rule! mdb r)) (get rewritten :rules))
+              (dl-query mdb query-goal))))))))))

lib/datalog/parser.sx

@@ -0,0 +1,252 @@
+;; lib/datalog/parser.sx — Datalog tokens → AST
+;;
+;; Output shapes:
+;;   Literal (positive)  := (relname arg ... arg)        — SX list
+;;   Literal (negative)  := {:neg (relname arg ... arg)} — dict
+;;   Argument            := var-symbol | atom-symbol | number | string
+;;                        | (op-name arg ... arg)        — arithmetic compound
+;;   Fact                := {:head literal :body ()}
+;;   Rule                := {:head literal :body (lit ... lit)}
+;;   Query               := {:query (lit ... lit)}
+;;   Program             := list of facts / rules / queries
+;;
+;; Variables and constants are both SX symbols; the evaluator dispatches
+;; on first-char case ('A'..'Z' or '_' = variable, otherwise constant).
+;;
+;; The parser permits nested compounds in arg position to support
+;; arithmetic (e.g. (is Z (+ X Y))). Safety analysis at rule-load time
+;; rejects compounds whose head is not an arithmetic operator.
+
+(define
+  dl-pp-peek
+  (fn
+    (st)
+    (let
+      ((i (get st :idx)) (tokens (get st :tokens)))
+      (if (< i (len tokens)) (nth tokens i) {:type "eof" :value nil :pos 0}))))
+
+(define
+  dl-pp-peek2
+  (fn
+    (st)
+    (let
+      ((i (+ (get st :idx) 1)) (tokens (get st :tokens)))
+      (if (< i (len tokens)) (nth tokens i) {:type "eof" :value nil :pos 0}))))
+
+(define
+  dl-pp-advance!
+  (fn (st) (dict-set! st :idx (+ (get st :idx) 1))))
+
+(define
+  dl-pp-at?
+  (fn
+    (st type value)
+    (let
+      ((t (dl-pp-peek st)))
+      (and
+        (= (get t :type) type)
+        (or (= value nil) (= (get t :value) value))))))
+
+(define
+  dl-pp-error
+  (fn
+    (st msg)
+    (let
+      ((t (dl-pp-peek st)))
+      (error
+        (str
+          "Parse error at pos "
+          (get t :pos)
+          ": "
+          msg
+          " (got "
+          (get t :type)
+          " '"
+          (if (= (get t :value) nil) "" (get t :value))
+          "')")))))
+
+(define
+  dl-pp-expect!
+  (fn
+    (st type value)
+    (let
+      ((t (dl-pp-peek st)))
+      (if
+        (dl-pp-at? st type value)
+        (do (dl-pp-advance! st) t)
+        (dl-pp-error
+          st
+          (str "expected " type (if (= value nil) "" (str " '" value "'"))))))))
+
+;; Argument: variable, atom, number, string, or compound (relname/op + parens).
+(define
+  dl-pp-parse-arg
+  (fn
+    (st)
+    (let
+      ((t (dl-pp-peek st)))
+      (let
+        ((ty (get t :type)) (vv (get t :value)))
+        (cond
+          ((= ty "number") (do (dl-pp-advance! st) vv))
+          ((= ty "string") (do (dl-pp-advance! st) vv))
+          ((= ty "var") (do (dl-pp-advance! st) (string->symbol vv)))
+          ;; Negative numeric literal: `-` op directly followed by a
+          ;; number (no `(`) is parsed as a single negative number.
+          ;; This keeps `(-X Y)` (compound) and `-N` (literal) distinct.
+          ((and (= ty "op") (= vv "-")
+                (= (get (dl-pp-peek2 st) :type) "number"))
+            (do
+              (dl-pp-advance! st)
+              (let
+                ((n (get (dl-pp-peek st) :value)))
+                (do (dl-pp-advance! st) (- 0 n)))))
+          ((or (= ty "atom") (= ty "op"))
+            (do
+              (dl-pp-advance! st)
+              (if
+                (dl-pp-at? st "punct" "(")
+                (do
+                  (dl-pp-advance! st)
+                  (let
+                    ((args (dl-pp-parse-arg-list st)))
+                    (do
+                      (dl-pp-expect! st "punct" ")")
+                      (cons (string->symbol vv) args))))
+                (string->symbol vv))))
+          (else (dl-pp-error st "expected term")))))))
+
+;; Comma-separated args inside parens.
+(define
+  dl-pp-parse-arg-list
+  (fn
+    (st)
+    (let
+      ((args (list)))
+      (do
+        (append! args (dl-pp-parse-arg st))
+        (define
+          dl-pp-arg-loop
+          (fn
+            ()
+            (when
+              (dl-pp-at? st "punct" ",")
+              (do
+                (dl-pp-advance! st)
+                (append! args (dl-pp-parse-arg st))
+                (dl-pp-arg-loop)))))
+        (dl-pp-arg-loop)
+        args))))
+
+;; A positive literal: relname (atom or op) followed by optional (args).
+(define
+  dl-pp-parse-positive
+  (fn
+    (st)
+    (let
+      ((t (dl-pp-peek st)))
+      (let
+        ((ty (get t :type)) (vv (get t :value)))
+        (if
+          (or (= ty "atom") (= ty "op"))
+          (do
+            (dl-pp-advance! st)
+            (if
+              (dl-pp-at? st "punct" "(")
+              (do
+                (dl-pp-advance! st)
+                (let
+                  ((args (dl-pp-parse-arg-list st)))
+                  (do
+                    (dl-pp-expect! st "punct" ")")
+                    (cons (string->symbol vv) args))))
+              (list (string->symbol vv))))
+          (dl-pp-error st "expected literal head"))))))
+
+;; A body literal: positive, or not(positive).
+(define
+  dl-pp-parse-body-lit
+  (fn
+    (st)
+    (let
+      ((t1 (dl-pp-peek st)) (t2 (dl-pp-peek2 st)))
+      (if
+        (and
+          (= (get t1 :type) "atom")
+          (= (get t1 :value) "not")
+          (= (get t2 :type) "punct")
+          (= (get t2 :value) "("))
+        (do
+          (dl-pp-advance! st)
+          (dl-pp-advance! st)
+          (let
+            ((inner (dl-pp-parse-positive st)))
+            (do (dl-pp-expect! st "punct" ")") {:neg inner})))
+        (dl-pp-parse-positive st)))))
+
+;; Comma-separated body literals.
+(define
+  dl-pp-parse-body
+  (fn
+    (st)
+    (let
+      ((lits (list)))
+      (do
+        (append! lits (dl-pp-parse-body-lit st))
+        (define
+          dl-pp-body-loop
+          (fn
+            ()
+            (when
+              (dl-pp-at? st "punct" ",")
+              (do
+                (dl-pp-advance! st)
+                (append! lits (dl-pp-parse-body-lit st))
+                (dl-pp-body-loop)))))
+        (dl-pp-body-loop)
+        lits))))
+
+;; Single clause: fact, rule, or query. Consumes trailing dot.
+(define
+  dl-pp-parse-clause
+  (fn
+    (st)
+    (cond
+      ((dl-pp-at? st "op" "?-")
+        (do
+          (dl-pp-advance! st)
+          (let
+            ((body (dl-pp-parse-body st)))
+            (do (dl-pp-expect! st "punct" ".") {:query body}))))
+      (else
+        (let
+          ((head (dl-pp-parse-positive st)))
+          (cond
+            ((dl-pp-at? st "op" ":-")
+              (do
+                (dl-pp-advance! st)
+                (let
+                  ((body (dl-pp-parse-body st)))
+                  (do (dl-pp-expect! st "punct" ".") {:body body :head head}))))
+            (else (do (dl-pp-expect! st "punct" ".") {:body (list) :head head}))))))))
+
+(define
+  dl-parse-program
+  (fn
+    (tokens)
+    (let
+      ((st {:tokens tokens :idx 0}) (clauses (list)))
+      (do
+        (define
+          dl-pp-prog-loop
+          (fn
+            ()
+            (when
+              (not (dl-pp-at? st "eof" nil))
+              (do
+                (append! clauses (dl-pp-parse-clause st))
+                (dl-pp-prog-loop)))))
+        (dl-pp-prog-loop)
+        clauses))))
+
+(define dl-parse (fn (src) (dl-parse-program (dl-tokenize src))))

lib/datalog/scoreboard.json

@@ -0,0 +1,20 @@
+{
+  "lang": "datalog",
+  "total_passed": 276,
+  "total_failed": 0,
+  "total": 276,
+  "suites": [
+    {"name":"tokenize","passed":31,"failed":0,"total":31},
+    {"name":"parse","passed":23,"failed":0,"total":23},
+    {"name":"unify","passed":29,"failed":0,"total":29},
+    {"name":"eval","passed":44,"failed":0,"total":44},
+    {"name":"builtins","passed":26,"failed":0,"total":26},
+    {"name":"semi_naive","passed":8,"failed":0,"total":8},
+    {"name":"negation","passed":12,"failed":0,"total":12},
+    {"name":"aggregates","passed":23,"failed":0,"total":23},
+    {"name":"api","passed":22,"failed":0,"total":22},
+    {"name":"magic","passed":37,"failed":0,"total":37},
+    {"name":"demo","passed":21,"failed":0,"total":21}
+  ],
+  "generated": "2026-05-11T09:40:12+00:00"
+}

lib/datalog/scoreboard.md

@@ -0,0 +1,17 @@
+# datalog scoreboard
+
+**276 / 276 passing** (0 failure(s)).
+
+| Suite | Passed | Total | Status |
+|-------|--------|-------|--------|
+| tokenize | 31 | 31 | ok |
+| parse | 23 | 23 | ok |
+| unify | 29 | 29 | ok |
+| eval | 44 | 44 | ok |
+| builtins | 26 | 26 | ok |
+| semi_naive | 8 | 8 | ok |
+| negation | 12 | 12 | ok |
+| aggregates | 23 | 23 | ok |
+| api | 22 | 22 | ok |
+| magic | 37 | 37 | ok |
+| demo | 21 | 21 | ok |

lib/datalog/strata.sx

@@ -0,0 +1,323 @@
+;; lib/datalog/strata.sx — dependency graph, SCC analysis, stratum assignment.
+;;
+;; A program is stratifiable iff no cycle in its dependency graph passes
+;; through a negative edge. The stratum of relation R is the depth at which
+;; R can first be evaluated:
+;;
+;;   stratum(R) = max over edges (R → S) of:
+;;       stratum(S)         if the edge is positive
+;;       stratum(S) + 1     if the edge is negative
+;;
+;; All relations in the same SCC share a stratum (and the SCC must have only
+;; positive internal edges, else the program is non-stratifiable).
+
+;; Build dep graph: dict {head-rel-name -> ({:rel str :neg bool} ...)}.
+(define
+  dl-build-dep-graph
+  (fn
+    (db)
+    (let ((g {}))
+      (do
+        (for-each
+          (fn
+            (rule)
+            (let
+              ((head-rel (dl-rel-name (get rule :head))))
+              (when
+                (not (nil? head-rel))
+                (do
+                  (when
+                    (not (has-key? g head-rel))
+                    (dict-set! g head-rel (list)))
+                  (let ((existing (get g head-rel)))
+                    (for-each
+                      (fn
+                        (lit)
+                        (cond
+                          ((dl-aggregate? lit)
+                            (let
+                              ((edge (dl-aggregate-dep-edge lit)))
+                              (when
+                                (not (nil? edge))
+                                (append! existing edge))))
+                          (else
+                            (let
+                              ((target
+                                 (cond
+                                   ((and (dict? lit) (has-key? lit :neg))
+                                     (dl-rel-name (get lit :neg)))
+                                   ((dl-builtin? lit) nil)
+                                   ((and (list? lit) (> (len lit) 0))
+                                     (dl-rel-name lit))
+                                   (else nil)))
+                                (neg?
+                                  (and (dict? lit) (has-key? lit :neg))))
+                              (when
+                                (not (nil? target))
+                                (append!
+                                  existing
+                                  {:rel target :neg neg?}))))))
+                      (get rule :body)))))))
+          (dl-rules db))
+        g))))
+
+;; All relations referenced — heads of rules + EDB names + body relations.
+(define
+  dl-all-relations
+  (fn
+    (db)
+    (let ((seen (list)))
+      (do
+        (for-each
+          (fn
+            (k)
+            (when (not (dl-member-string? k seen)) (append! seen k)))
+          (keys (get db :facts)))
+        (for-each
+          (fn
+            (rule)
+            (do
+              (let ((h (dl-rel-name (get rule :head))))
+                (when
+                  (and (not (nil? h)) (not (dl-member-string? h seen)))
+                  (append! seen h)))
+              (for-each
+                (fn
+                  (lit)
+                  (let
+                    ((t
+                       (cond
+                         ((dl-aggregate? lit)
+                           (let ((edge (dl-aggregate-dep-edge lit)))
+                             (if (nil? edge) nil (get edge :rel))))
+                         ((and (dict? lit) (has-key? lit :neg))
+                           (dl-rel-name (get lit :neg)))
+                         ((dl-builtin? lit) nil)
+                         ((and (list? lit) (> (len lit) 0))
+                           (dl-rel-name lit))
+                         (else nil))))
+                    (when
+                      (and (not (nil? t)) (not (dl-member-string? t seen)))
+                      (append! seen t))))
+                (get rule :body))))
+          (dl-rules db))
+        seen))))
+
+;; reach: dict {from: dict {to: edge-info}} where edge-info is
+;;   {:any bool :neg bool}
+;; meaning "any path from `from` to `to`" and "exists a negative-passing
+;; path from `from` to `to`".
+;;
+;; Floyd-Warshall over the dep graph. The 'neg' flag propagates through
+;; concatenation: if any edge along the path is negative, the path's
+;; flag is true.
+(define
+  dl-build-reach
+  (fn
+    (graph nodes)
+    (let ((reach {}))
+      (do
+        (for-each
+          (fn (n) (dict-set! reach n {}))
+          nodes)
+        (for-each
+          (fn
+            (head)
+            (when
+              (has-key? graph head)
+              (for-each
+                (fn
+                  (edge)
+                  (let
+                    ((target (get edge :rel)) (n (get edge :neg)))
+                    (let ((row (get reach head)))
+                      (cond
+                        ((has-key? row target)
+                          (let ((cur (get row target)))
+                            (dict-set!
+                              row
+                              target
+                              {:any true :neg (or n (get cur :neg))})))
+                        (else
+                          (dict-set! row target {:any true :neg n}))))))
+                (get graph head))))
+          nodes)
+        (for-each
+          (fn
+            (k)
+            (for-each
+              (fn
+                (i)
+                (let ((row-i (get reach i)))
+                  (when
+                    (has-key? row-i k)
+                    (let ((ik (get row-i k)) (row-k (get reach k)))
+                      (for-each
+                        (fn
+                          (j)
+                          (when
+                            (has-key? row-k j)
+                            (let ((kj (get row-k j)))
+                              (let
+                                ((combined-neg (or (get ik :neg) (get kj :neg))))
+                                (cond
+                                  ((has-key? row-i j)
+                                    (let ((cur (get row-i j)))
+                                      (dict-set!
+                                        row-i
+                                        j
+                                        {:any true
+                                         :neg (or combined-neg (get cur :neg))})))
+                                  (else
+                                    (dict-set!
+                                      row-i
+                                      j
+                                      {:any true :neg combined-neg})))))))
+                        nodes)))))
+              nodes))
+          nodes)
+        reach))))
+
+;; Returns nil on success, or error message string on failure.
+(define
+  dl-check-stratifiable
+  (fn
+    (db)
+    (let
+      ((graph (dl-build-dep-graph db))
+        (nodes (dl-all-relations db)))
+      (let ((reach (dl-build-reach graph nodes)) (err nil))
+        (do
+          (for-each
+            (fn
+              (rule)
+              (when
+                (nil? err)
+                (let ((head-rel (dl-rel-name (get rule :head))))
+                  (for-each
+                    (fn
+                      (lit)
+                      (cond
+                        ((and (dict? lit) (has-key? lit :neg))
+                          (let ((tgt (dl-rel-name (get lit :neg))))
+                            (when
+                              (and (not (nil? tgt))
+                                   (dl-reach-cycle? reach head-rel tgt))
+                              (set!
+                                err
+                                (str "non-stratifiable: relation " head-rel
+                                     " transitively depends through negation on "
+                                     tgt
+                                     " which depends back on " head-rel)))))
+                        ((dl-aggregate? lit)
+                          (let ((edge (dl-aggregate-dep-edge lit)))
+                            (when
+                              (not (nil? edge))
+                              (let ((tgt (get edge :rel)))
+                                (when
+                                  (and (not (nil? tgt))
+                                       (dl-reach-cycle? reach head-rel tgt))
+                                  (set!
+                                    err
+                                    (str "non-stratifiable: relation "
+                                         head-rel
+                                         " aggregates over " tgt
+                                         " which depends back on "
+                                         head-rel))))))))) 
+                    (get rule :body)))))
+            (dl-rules db))
+          err)))))
+
+(define
+  dl-reach-cycle?
+  (fn
+    (reach a b)
+    (and
+      (dl-reach-row-has? reach b a)
+      (dl-reach-row-has? reach a b))))
+
+(define
+  dl-reach-row-has?
+  (fn
+    (reach from to)
+    (let ((row (get reach from)))
+      (and (not (nil? row)) (has-key? row to)))))
+
+;; Compute stratum per relation. Iteratively propagate from EDB roots.
+;; Uses the per-relation max-stratum-of-deps formula. Stops when stable.
+(define
+  dl-compute-strata
+  (fn
+    (db)
+    (let
+      ((graph (dl-build-dep-graph db))
+        (nodes (dl-all-relations db))
+        (strata {}))
+      (do
+        (for-each (fn (n) (dict-set! strata n 0)) nodes)
+        (let ((changed true))
+          (do
+            (define
+              dl-cs-loop
+              (fn
+                ()
+                (when
+                  changed
+                  (do
+                    (set! changed false)
+                    (for-each
+                      (fn
+                        (head)
+                        (when
+                          (has-key? graph head)
+                          (for-each
+                            (fn
+                              (edge)
+                              (let
+                                ((tgt (get edge :rel))
+                                  (n (get edge :neg)))
+                                (let
+                                  ((tgt-strat
+                                     (if (has-key? strata tgt)
+                                       (get strata tgt) 0))
+                                    (cur (get strata head)))
+                                  (let
+                                    ((needed
+                                       (if n (+ tgt-strat 1) tgt-strat)))
+                                    (when
+                                      (> needed cur)
+                                      (do
+                                        (dict-set! strata head needed)
+                                        (set! changed true)))))))
+                            (get graph head))))
+                      nodes)
+                    (dl-cs-loop)))))
+            (dl-cs-loop)))
+        strata))))
+
+;; Group rules by their head's stratum. Returns dict {stratum-int -> rules}.
+(define
+  dl-group-rules-by-stratum
+  (fn
+    (db strata)
+    (let ((groups {}) (max-s 0))
+      (do
+        (for-each
+          (fn
+            (rule)
+            (let
+              ((head-rel (dl-rel-name (get rule :head))))
+              (let
+                ((s (if (has-key? strata head-rel)
+                       (get strata head-rel) 0)))
+                (do
+                  (when (> s max-s) (set! max-s s))
+                  (let
+                    ((sk (str s)))
+                    (do
+                      (when
+                        (not (has-key? groups sk))
+                        (dict-set! groups sk (list)))
+                      (append! (get groups sk) rule)))))))
+          (dl-rules db))
+        {:groups groups :max max-s}))))

lib/datalog/tests/aggregates.sx

@@ -0,0 +1,357 @@
+;; lib/datalog/tests/aggregates.sx — count / sum / min / max.
+
+(define dl-at-pass 0)
+(define dl-at-fail 0)
+(define dl-at-failures (list))
+
+(define
+  dl-at-deep=?
+  (fn
+    (a b)
+    (cond
+      ((and (list? a) (list? b))
+        (and (= (len a) (len b)) (dl-at-deq-l? a b 0)))
+      ((and (dict? a) (dict? b))
+        (let ((ka (keys a)) (kb (keys b)))
+          (and (= (len ka) (len kb)) (dl-at-deq-d? a b ka 0))))
+      ((and (number? a) (number? b)) (= a b))
+      (else (equal? a b)))))
+
+(define
+  dl-at-deq-l?
+  (fn
+    (a b i)
+    (cond
+      ((>= i (len a)) true)
+      ((not (dl-at-deep=? (nth a i) (nth b i))) false)
+      (else (dl-at-deq-l? a b (+ i 1))))))
+
+(define
+  dl-at-deq-d?
+  (fn
+    (a b ka i)
+    (cond
+      ((>= i (len ka)) true)
+      ((let ((k (nth ka i)))
+         (not (dl-at-deep=? (get a k) (get b k))))
+        false)
+      (else (dl-at-deq-d? a b ka (+ i 1))))))
+
+(define
+  dl-at-set=?
+  (fn
+    (a b)
+    (and
+      (= (len a) (len b))
+      (dl-at-subset? a b)
+      (dl-at-subset? b a))))
+
+(define
+  dl-at-subset?
+  (fn
+    (xs ys)
+    (cond
+      ((= (len xs) 0) true)
+      ((not (dl-at-contains? ys (first xs))) false)
+      (else (dl-at-subset? (rest xs) ys)))))
+
+(define
+  dl-at-contains?
+  (fn
+    (xs target)
+    (cond
+      ((= (len xs) 0) false)
+      ((dl-at-deep=? (first xs) target) true)
+      (else (dl-at-contains? (rest xs) target)))))
+
+(define
+  dl-at-test!
+  (fn
+    (name got expected)
+    (if
+      (dl-at-deep=? got expected)
+      (set! dl-at-pass (+ dl-at-pass 1))
+      (do
+        (set! dl-at-fail (+ dl-at-fail 1))
+        (append!
+          dl-at-failures
+          (str
+            name
+            "\n    expected: " expected
+            "\n    got:      " got))))))
+
+(define
+  dl-at-test-set!
+  (fn
+    (name got expected)
+    (if
+      (dl-at-set=? got expected)
+      (set! dl-at-pass (+ dl-at-pass 1))
+      (do
+        (set! dl-at-fail (+ dl-at-fail 1))
+        (append!
+          dl-at-failures
+          (str
+            name
+            "\n    expected (set): " expected
+            "\n    got:            " got))))))
+
+(define
+  dl-at-throws?
+  (fn
+    (thunk)
+    (let
+      ((threw false))
+      (do
+        (guard
+          (e (#t (set! threw true)))
+          (thunk))
+        threw))))
+
+(define
+  dl-at-run-all!
+  (fn
+    ()
+    (do
+      ;; count
+      (dl-at-test-set! "count siblings"
+        (dl-query
+          (dl-program
+            "parent(p, bob). parent(p, alice). parent(p, charlie).
+             sibling(X, Y) :- parent(P, X), parent(P, Y), !=(X, Y).
+             sib_count(N) :- count(N, S, sibling(bob, S)).")
+          (list (quote sib_count) (quote N)))
+        (list {:N 2}))
+
+      ;; sum
+      (dl-at-test-set! "sum prices"
+        (dl-query
+          (dl-program
+            "price(apple, 5). price(pear, 7). price(plum, 3).
+             total(T) :- sum(T, X, price(F, X)).")
+          (list (quote total) (quote T)))
+        (list {:T 15}))
+
+      ;; min
+      (dl-at-test-set! "min score"
+        (dl-query
+          (dl-program
+            "score(alice, 80). score(bob, 65). score(carol, 92).
+             lo(M) :- min(M, S, score(P, S)).")
+          (list (quote lo) (quote M)))
+        (list {:M 65}))
+
+      ;; max
+      (dl-at-test-set! "max score"
+        (dl-query
+          (dl-program
+            "score(alice, 80). score(bob, 65). score(carol, 92).
+             hi(M) :- max(M, S, score(P, S)).")
+          (list (quote hi) (quote M)))
+        (list {:M 92}))
+
+      ;; count over derived relation (stratification needed).
+      (dl-at-test-set! "count over derived"
+        (dl-query
+          (dl-program
+            "parent(a, b). parent(a, c). parent(b, d). parent(c, e).
+             ancestor(X, Y) :- parent(X, Y).
+             ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z).
+             num_ancestors(N) :- count(N, X, ancestor(a, X)).")
+          (list (quote num_ancestors) (quote N)))
+        (list {:N 4}))
+
+      ;; count with no matches → 0.
+      (dl-at-test-set! "count empty"
+        (dl-query
+          (dl-program
+            "p(1). p(2).
+             zero(N) :- count(N, X, q(X)).")
+          (list (quote zero) (quote N)))
+        (list {:N 0}))
+
+      ;; sum with no matches → 0.
+      (dl-at-test-set! "sum empty"
+        (dl-query
+          (dl-program
+            "p(1). p(2).
+             total(T) :- sum(T, X, q(X)).")
+          (list (quote total) (quote T)))
+        (list {:T 0}))
+
+      ;; min with no matches → rule does not fire.
+      (dl-at-test-set! "min empty"
+        (dl-query
+          (dl-program
+            "p(1). p(2).
+             lo(M) :- min(M, X, q(X)).")
+          (list (quote lo) (quote M)))
+        (list))
+
+      ;; Aggregate with comparison filter on result.
+      (dl-at-test-set! "popularity threshold"
+        (dl-query
+          (dl-program
+            "post(p1). post(p2).
+             liked(u1, p1). liked(u2, p1). liked(u3, p1).
+             liked(u1, p2). liked(u2, p2).
+             popular(P) :- post(P), count(N, U, liked(U, P)), >=(N, 3).")
+          (list (quote popular) (quote P)))
+        (list {:P (quote p1)}))
+
+      ;; findall: collect distinct values into a list.
+      (dl-at-test-set! "findall over EDB"
+        (dl-query
+          (dl-program
+            "p(a). p(b). p(c).
+             all_p(L) :- findall(L, X, p(X)).")
+          (list (quote all_p) (quote L)))
+        (list {:L (list (quote a) (quote b) (quote c))}))
+
+      (dl-at-test-set! "findall over derived"
+        (dl-query
+          (dl-program
+            "parent(a, b). parent(b, c). parent(c, d).
+             ancestor(X, Y) :- parent(X, Y).
+             ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z).
+             desc(L) :- findall(L, X, ancestor(a, X)).")
+          (list (quote desc) (quote L)))
+        (list {:L (list (quote b) (quote c) (quote d))}))
+
+      (dl-at-test-set! "findall empty"
+        (dl-query
+          (dl-program
+            "p(1).
+             all_q(L) :- findall(L, X, q(X)).")
+          (list (quote all_q) (quote L)))
+        (list {:L (list)}))
+
+      ;; Aggregate vs single distinct.
+      ;; Group-by via aggregate-in-rule-body. Per-user friend count
+      ;; over a friends relation. The U var is bound by the prior
+      ;; positive lit u(U) so the aggregate counts only U-rooted
+      ;; friends per group.
+      (dl-at-test-set! "group-by per-user friend count"
+        (dl-query
+          (dl-program
+            "u(alice). u(bob). u(carol).
+             f(alice, x). f(alice, y). f(bob, x).
+             counts(U, N) :- u(U), count(N, X, f(U, X)).")
+          (list (quote counts) (quote U) (quote N)))
+        (list
+          {:U (quote alice) :N 2}
+          {:U (quote bob) :N 1}
+          {:U (quote carol) :N 0}))
+
+      ;; Stratification: recursion through aggregation is rejected.
+      ;; Aggregate validates that second arg is a variable.
+      (dl-at-test! "agg second arg must be var"
+        (dl-at-throws?
+          (fn () (dl-eval "p(1). q(N) :- count(N, 5, p(X))." "?- q(N).")))
+        true)
+
+      ;; Aggregate validates that third arg is a positive literal.
+      (dl-at-test! "agg third arg must be a literal"
+        (dl-at-throws?
+          (fn () (dl-eval "p(1). q(N) :- count(N, X, 42)." "?- q(N).")))
+        true)
+
+      ;; Aggregate validates that the agg-var (2nd arg) appears in the
+      ;; goal. Without it every match contributes the same unbound
+      ;; symbol — count silently returns 1, sum raises a confusing
+      ;; "expected number" error, etc. Catch the mistake at safety
+      ;; check time instead.
+      (dl-at-test! "agg-var must appear in goal"
+        (dl-at-throws?
+          (fn ()
+            (dl-eval
+              "p(1). p(2). c(N) :- count(N, Y, p(X))."
+              "?- c(N).")))
+        true)
+
+      ;; Indirect recursion through aggregation also rejected.
+      ;; q -> r (via positive lit), r -> q (via aggregate body).
+      ;; The aggregate edge counts as negation for stratification.
+      (dl-at-test! "indirect agg cycle rejected"
+        (dl-at-throws?
+          (fn ()
+            (let ((db (dl-make-db)))
+              (do
+                (dl-add-rule! db
+                  {:head (list (quote q) (quote N))
+                   :body (list (list (quote r) (quote N)))})
+                (dl-add-rule! db
+                  {:head (list (quote r) (quote N))
+                   :body (list (list (quote count) (quote N) (quote X)
+                                     (list (quote q) (quote X))))})
+                (dl-saturate! db)))))
+        true)
+
+      (dl-at-test! "agg recursion rejected"
+        (dl-at-throws?
+          (fn ()
+            (let ((db (dl-make-db)))
+              (do
+                (dl-add-rule! db
+                  {:head (list (quote q) (quote N))
+                   :body (list (list (quote count) (quote N) (quote X)
+                                     (list (quote q) (quote X))))})
+                (dl-saturate! db)))))
+        true)
+
+      ;; Negation + aggregation in the same body — different strata.
+      (dl-at-test-set! "neg + agg coexist"
+        (dl-query
+          (dl-program
+            "u(a). u(b). u(c). banned(b).
+             active(X) :- u(X), not(banned(X)).
+             cnt(N) :- count(N, X, active(X)).")
+          (list (quote cnt) (quote N)))
+        (list {:N 2}))
+
+      ;; Min over a derived empty relation: no result.
+      (dl-at-test-set! "min over empty derived"
+        (dl-query
+          (dl-program
+            "s(50). s(60).
+             score(N) :- s(N), >(N, 100).
+             low(M) :- min(M, X, score(X)).")
+          (list (quote low) (quote M)))
+        (list))
+
+      ;; Aggregates as the top-level query goal (regression for
+      ;; dl-match-lit aggregate dispatch and projection cleanup).
+      (dl-at-test-set! "count as query goal"
+        (dl-query
+          (dl-program "p(1). p(2). p(3). p(4).")
+          (list (quote count) (quote N) (quote X) (list (quote p) (quote X))))
+        (list {:N 4}))
+
+      (dl-at-test-set! "findall as query goal"
+        (dl-query
+          (dl-program "p(1). p(2). p(3).")
+          (list (quote findall) (quote L) (quote X)
+                (list (quote p) (quote X))))
+        (list {:L (list 1 2 3)}))
+
+      (dl-at-test-set! "distinct counted once"
+        (dl-query
+          (dl-program
+            "rated(alice, x). rated(alice, y). rated(bob, x).
+             rater_count(N) :- count(N, U, rated(U, F)).")
+          (list (quote rater_count) (quote N)))
+        (list {:N 2})))))
+
+(define
+  dl-aggregates-tests-run!
+  (fn
+    ()
+    (do
+      (set! dl-at-pass 0)
+      (set! dl-at-fail 0)
+      (set! dl-at-failures (list))
+      (dl-at-run-all!)
+      {:passed dl-at-pass
+       :failed dl-at-fail
+       :total (+ dl-at-pass dl-at-fail)
+       :failures dl-at-failures})))

lib/datalog/tests/api.sx

@@ -0,0 +1,350 @@
+;; lib/datalog/tests/api.sx — SX-data embedding API.
+
+(define dl-api-pass 0)
+(define dl-api-fail 0)
+(define dl-api-failures (list))
+
+(define
+  dl-api-deep=?
+  (fn
+    (a b)
+    (cond
+      ((and (list? a) (list? b))
+        (and (= (len a) (len b)) (dl-api-deq-l? a b 0)))
+      ((and (dict? a) (dict? b))
+        (let ((ka (keys a)) (kb (keys b)))
+          (and (= (len ka) (len kb)) (dl-api-deq-d? a b ka 0))))
+      ((and (number? a) (number? b)) (= a b))
+      (else (equal? a b)))))
+
+(define
+  dl-api-deq-l?
+  (fn
+    (a b i)
+    (cond
+      ((>= i (len a)) true)
+      ((not (dl-api-deep=? (nth a i) (nth b i))) false)
+      (else (dl-api-deq-l? a b (+ i 1))))))
+
+(define
+  dl-api-deq-d?
+  (fn
+    (a b ka i)
+    (cond
+      ((>= i (len ka)) true)
+      ((let ((k (nth ka i)))
+         (not (dl-api-deep=? (get a k) (get b k))))
+        false)
+      (else (dl-api-deq-d? a b ka (+ i 1))))))
+
+(define
+  dl-api-set=?
+  (fn
+    (a b)
+    (and
+      (= (len a) (len b))
+      (dl-api-subset? a b)
+      (dl-api-subset? b a))))
+
+(define
+  dl-api-subset?
+  (fn
+    (xs ys)
+    (cond
+      ((= (len xs) 0) true)
+      ((not (dl-api-contains? ys (first xs))) false)
+      (else (dl-api-subset? (rest xs) ys)))))
+
+(define
+  dl-api-contains?
+  (fn
+    (xs target)
+    (cond
+      ((= (len xs) 0) false)
+      ((dl-api-deep=? (first xs) target) true)
+      (else (dl-api-contains? (rest xs) target)))))
+
+(define
+  dl-api-test!
+  (fn
+    (name got expected)
+    (if
+      (dl-api-deep=? got expected)
+      (set! dl-api-pass (+ dl-api-pass 1))
+      (do
+        (set! dl-api-fail (+ dl-api-fail 1))
+        (append!
+          dl-api-failures
+          (str
+            name
+            "\n    expected: " expected
+            "\n    got:      " got))))))
+
+(define
+  dl-api-test-set!
+  (fn
+    (name got expected)
+    (if
+      (dl-api-set=? got expected)
+      (set! dl-api-pass (+ dl-api-pass 1))
+      (do
+        (set! dl-api-fail (+ dl-api-fail 1))
+        (append!
+          dl-api-failures
+          (str
+            name
+            "\n    expected (set): " expected
+            "\n    got:            " got))))))
+
+(define
+  dl-api-run-all!
+  (fn
+    ()
+    (do
+      ;; dl-program-data with arrow form.
+      (dl-api-test-set! "data API ancestor closure"
+        (dl-query
+          (dl-program-data
+            (quote ((parent tom bob) (parent bob ann) (parent ann pat)))
+            (quote
+              ((ancestor X Y <- (parent X Y))
+               (ancestor X Z <- (parent X Y) (ancestor Y Z)))))
+          (quote (ancestor tom X)))
+        (list {:X (quote bob)} {:X (quote ann)} {:X (quote pat)}))
+
+      ;; dl-program-data with dict rules.
+      (dl-api-test-set! "data API with dict rules"
+        (dl-query
+          (dl-program-data
+            (quote ((p a) (p b) (p c)))
+            (list
+              {:head (quote (q X)) :body (quote ((p X)))}))
+          (quote (q X)))
+        (list {:X (quote a)} {:X (quote b)} {:X (quote c)}))
+
+      ;; dl-rule helper.
+      (dl-api-test-set! "dl-rule constructor"
+        (dl-query
+          (dl-program-data
+            (quote ((p 1) (p 2)))
+            (list (dl-rule (quote (q X)) (quote ((p X))))))
+          (quote (q X)))
+        (list {:X 1} {:X 2}))
+
+      ;; dl-assert! adds and re-derives.
+      (dl-api-test-set! "dl-assert! incremental"
+        (let
+          ((db (dl-program-data
+                 (quote ((parent tom bob) (parent bob ann)))
+                 (quote
+                   ((ancestor X Y <- (parent X Y))
+                    (ancestor X Z <- (parent X Y) (ancestor Y Z)))))))
+          (do
+            (dl-saturate! db)
+            (dl-assert! db (quote (parent ann pat)))
+            (dl-query db (quote (ancestor tom X)))))
+        (list {:X (quote bob)} {:X (quote ann)} {:X (quote pat)}))
+
+      ;; dl-retract! removes a fact and recomputes IDB.
+      (dl-api-test-set! "dl-retract! removes derived"
+        (let
+          ((db (dl-program-data
+                 (quote ((parent tom bob) (parent bob ann) (parent ann pat)))
+                 (quote
+                   ((ancestor X Y <- (parent X Y))
+                    (ancestor X Z <- (parent X Y) (ancestor Y Z)))))))
+          (do
+            (dl-saturate! db)
+            (dl-retract! db (quote (parent bob ann)))
+            (dl-query db (quote (ancestor tom X)))))
+        (list {:X (quote bob)}))
+
+      ;; dl-retract! on a relation with BOTH explicit facts AND a rule
+      ;; (a "mixed" relation) used to wipe the EDB portion when the IDB
+      ;; was re-derived, even when the retract didn't match anything.
+      ;; :edb-keys provenance now preserves user-asserted facts.
+      (dl-api-test-set! "dl-retract! preserves EDB in mixed relation"
+        (let
+          ((db (dl-program-data
+                 (quote ((p a) (p b) (q c)))
+                 (quote ((p X <- (q X)))))))
+          (do
+            (dl-saturate! db)
+            ;; Retract a non-existent tuple — should be a no-op.
+            (dl-retract! db (quote (p z)))
+            (dl-query db (quote (p X)))))
+        (list {:X (quote a)} {:X (quote b)} {:X (quote c)}))
+
+      ;; And retracting an actual EDB fact in a mixed relation drops
+      ;; only that fact; the derived portion stays.
+      (dl-api-test-set! "dl-retract! mixed: drop EDB, keep IDB"
+        (let
+          ((db (dl-program-data
+                 (quote ((p a) (p b) (q c)))
+                 (quote ((p X <- (q X)))))))
+          (do
+            (dl-saturate! db)
+            (dl-retract! db (quote (p a)))
+            (dl-query db (quote (p X)))))
+        (list {:X (quote b)} {:X (quote c)}))
+
+      ;; dl-program-data + dl-query with constants in head.
+      (dl-api-test-set! "constant-in-head data"
+        (dl-query
+          (dl-program-data
+            (quote ((edge a b) (edge b c) (edge c a)))
+            (quote
+              ((reach X Y <- (edge X Y))
+               (reach X Z <- (edge X Y) (reach Y Z)))))
+          (quote (reach a X)))
+        (list {:X (quote a)} {:X (quote b)} {:X (quote c)}))
+
+      ;; Assert into empty db.
+      (dl-api-test-set! "assert into empty"
+        (let
+          ((db (dl-program-data (list) (list))))
+          (do
+            (dl-assert! db (quote (p 1)))
+            (dl-assert! db (quote (p 2)))
+            (dl-query db (quote (p X)))))
+        (list {:X 1} {:X 2}))
+
+      ;; Multi-goal query: pass list of literals.
+      (dl-api-test-set! "multi-goal query"
+        (dl-query
+          (dl-program-data
+            (quote ((p 1) (p 2) (p 3) (q 2) (q 3)))
+            (list))
+          (list (quote (p X)) (quote (q X))))
+        (list {:X 2} {:X 3}))
+
+      ;; Multi-goal with comparison.
+      (dl-api-test-set! "multi-goal with comparison"
+        (dl-query
+          (dl-program-data
+            (quote ((n 1) (n 2) (n 3) (n 4) (n 5)))
+            (list))
+          (list (quote (n X)) (list (string->symbol ">") (quote X) 2)))
+        (list {:X 3} {:X 4} {:X 5}))
+
+      ;; dl-eval: single-call source + query.
+      (dl-api-test-set! "dl-eval ancestor"
+        (dl-eval
+          "parent(a, b). parent(b, c).
+           ancestor(X, Y) :- parent(X, Y).
+           ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z)."
+          "?- ancestor(a, X).")
+        (list {:X (quote b)} {:X (quote c)}))
+
+      (dl-api-test-set! "dl-eval multi-goal"
+        (dl-eval
+          "p(1). p(2). p(3). q(2). q(3)."
+          "?- p(X), q(X).")
+        (list {:X 2} {:X 3}))
+
+      ;; dl-rules-of: rules with head matching a relation name.
+      (dl-api-test! "dl-rules-of count"
+        (let
+          ((db (dl-program
+                 "p(1). q(X) :- p(X). r(X) :- p(X). q(2).")))
+          (len (dl-rules-of db "q")))
+        1)
+
+      (dl-api-test! "dl-rules-of empty"
+        (let
+          ((db (dl-program "p(1). p(2).")))
+          (len (dl-rules-of db "q")))
+        0)
+
+      ;; dl-clear-idb!: wipe rule-headed relations.
+      (dl-api-test! "dl-clear-idb! wipes IDB"
+        (let
+          ((db (dl-program
+                 "parent(a, b). parent(b, c).
+                  ancestor(X, Y) :- parent(X, Y).
+                  ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z).")))
+          (do
+            (dl-saturate! db)
+            (dl-clear-idb! db)
+            (len (dl-relation db "ancestor"))))
+        0)
+
+      (dl-api-test! "dl-clear-idb! preserves EDB"
+        (let
+          ((db (dl-program
+                 "parent(a, b). parent(b, c).
+                  ancestor(X, Y) :- parent(X, Y).")))
+          (do
+            (dl-saturate! db)
+            (dl-clear-idb! db)
+            (len (dl-relation db "parent"))))
+        2)
+
+      ;; dl-eval-magic — routes single-goal queries through
+      ;; magic-sets evaluation.
+      (dl-api-test-set! "dl-eval-magic ancestor"
+        (dl-eval-magic
+          "parent(a, b). parent(b, c).
+           ancestor(X, Y) :- parent(X, Y).
+           ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z)."
+          "?- ancestor(a, X).")
+        (list {:X (quote b)} {:X (quote c)}))
+
+      ;; Equivalence: dl-eval and dl-eval-magic produce the same
+      ;; answers for any well-formed query (magic-sets is a perf
+      ;; alternative, not a semantic change).
+      (dl-api-test! "dl-eval ≡ dl-eval-magic on ancestor"
+        (let
+          ((source "parent(a, b). parent(b, c). parent(c, d).
+                    ancestor(X, Y) :- parent(X, Y).
+                    ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z)."))
+          (let
+            ((semi (dl-eval source "?- ancestor(a, X)."))
+              (magic (dl-eval-magic source "?- ancestor(a, X).")))
+            (= (len semi) (len magic))))
+        true)
+
+      ;; Comprehensive integration: recursion + stratified negation
+      ;; + aggregation + comparison composed in a single program.
+      ;; (Uses _Anything as a regular var instead of `_` so the
+      ;;  outer rule binds via the reach lit.)
+      (dl-api-test-set! "integration"
+        (dl-eval
+          (str
+            "edge(a, b). edge(b, c). edge(c, d). edge(a, d). "
+            "banned(c). "
+            "reach(X, Y) :- edge(X, Y). "
+            "reach(X, Z) :- edge(X, Y), reach(Y, Z). "
+            "safe(X, Y) :- reach(X, Y), not(banned(Y)). "
+            "reach_count(X, N) :- reach(X, Z), count(N, Y, safe(X, Y)). "
+            "popular(X) :- reach_count(X, N), >=(N, 2).")
+          "?- popular(X).")
+        (list {:X (quote a)}))
+
+      ;; dl-rule-from-list with no arrow → fact-style.
+      (dl-api-test-set! "no arrow → fact-like rule"
+        (let
+          ((rule (dl-rule-from-list (quote (foo X Y)))))
+          (list rule))
+        (list {:head (quote (foo X Y)) :body (list)}))
+
+      ;; dl-coerce-rule on dict passes through.
+      (dl-api-test-set! "coerce dict rule"
+        (let
+          ((d {:head (quote (h X)) :body (quote ((b X)))}))
+          (list (dl-coerce-rule d)))
+        (list {:head (quote (h X)) :body (quote ((b X)))})))))
+
+(define
+  dl-api-tests-run!
+  (fn
+    ()
+    (do
+      (set! dl-api-pass 0)
+      (set! dl-api-fail 0)
+      (set! dl-api-failures (list))
+      (dl-api-run-all!)
+      {:passed dl-api-pass
+       :failed dl-api-fail
+       :total (+ dl-api-pass dl-api-fail)
+       :failures dl-api-failures})))

lib/datalog/tests/builtins.sx

@@ -0,0 +1,285 @@
+;; lib/datalog/tests/builtins.sx — comparison + arithmetic body literals.
+
+(define dl-bt-pass 0)
+(define dl-bt-fail 0)
+(define dl-bt-failures (list))
+
+(define
+  dl-bt-deep=?
+  (fn
+    (a b)
+    (cond
+      ((and (list? a) (list? b))
+        (and (= (len a) (len b)) (dl-bt-deq-l? a b 0)))
+      ((and (dict? a) (dict? b))
+        (let
+          ((ka (keys a)) (kb (keys b)))
+          (and (= (len ka) (len kb)) (dl-bt-deq-d? a b ka 0))))
+      ((and (number? a) (number? b)) (= a b))
+      (else (equal? a b)))))
+
+(define
+  dl-bt-deq-l?
+  (fn
+    (a b i)
+    (cond
+      ((>= i (len a)) true)
+      ((not (dl-bt-deep=? (nth a i) (nth b i))) false)
+      (else (dl-bt-deq-l? a b (+ i 1))))))
+
+(define
+  dl-bt-deq-d?
+  (fn
+    (a b ka i)
+    (cond
+      ((>= i (len ka)) true)
+      ((let ((k (nth ka i))) (not (dl-bt-deep=? (get a k) (get b k))))
+        false)
+      (else (dl-bt-deq-d? a b ka (+ i 1))))))
+
+(define
+  dl-bt-set=?
+  (fn
+    (a b)
+    (and (= (len a) (len b)) (dl-bt-subset? a b) (dl-bt-subset? b a))))
+
+(define
+  dl-bt-subset?
+  (fn
+    (xs ys)
+    (cond
+      ((= (len xs) 0) true)
+      ((not (dl-bt-contains? ys (first xs))) false)
+      (else (dl-bt-subset? (rest xs) ys)))))
+
+(define
+  dl-bt-contains?
+  (fn
+    (xs target)
+    (cond
+      ((= (len xs) 0) false)
+      ((dl-bt-deep=? (first xs) target) true)
+      (else (dl-bt-contains? (rest xs) target)))))
+
+(define
+  dl-bt-test-set!
+  (fn
+    (name got expected)
+    (if
+      (dl-bt-set=? got expected)
+      (set! dl-bt-pass (+ dl-bt-pass 1))
+      (do
+        (set! dl-bt-fail (+ dl-bt-fail 1))
+        (append!
+          dl-bt-failures
+          (str
+            name
+            "\n    expected (set): "
+            expected
+            "\n    got:            "
+            got))))))
+
+(define
+  dl-bt-test!
+  (fn
+    (name got expected)
+    (if
+      (dl-bt-deep=? got expected)
+      (set! dl-bt-pass (+ dl-bt-pass 1))
+      (do
+        (set! dl-bt-fail (+ dl-bt-fail 1))
+        (append!
+          dl-bt-failures
+          (str name "\n    expected: " expected "\n    got:      " got))))))
+
+(define
+  dl-bt-throws?
+  (fn
+    (thunk)
+    (let
+      ((threw false))
+      (do (guard (e (#t (set! threw true))) (thunk)) threw))))
+
+(define
+  dl-bt-run-all!
+  (fn
+    ()
+    (do
+      (dl-bt-test-set!
+        "less than filter"
+        (dl-query
+          (dl-program
+            "age(alice, 30). age(bob, 17). age(carol, 22).\n             adult(X) :- age(X, A), >=(A, 18).")
+          (list (quote adult) (quote X)))
+        (list {:X (quote alice)} {:X (quote carol)}))
+      (dl-bt-test-set!
+        "less-equal filter"
+        (dl-query
+          (dl-program
+            "n(1). n(2). n(3). n(4). n(5).\n             small(X) :- n(X), <=(X, 3).")
+          (list (quote small) (quote X)))
+        (list {:X 1} {:X 2} {:X 3}))
+      (dl-bt-test-set!
+        "not-equal filter"
+        (dl-query
+          (dl-program
+            "p(1, 2). p(2, 2). p(3, 4).\n             diff(X, Y) :- p(X, Y), !=(X, Y).")
+          (list (quote diff) (quote X) (quote Y)))
+        (list {:X 1 :Y 2} {:X 3 :Y 4}))
+      (dl-bt-test-set!
+        "is plus"
+        (dl-query
+          (dl-program
+            "n(1). n(2). n(3).\n             succ(X, Y) :- n(X), is(Y, +(X, 1)).")
+          (list (quote succ) (quote X) (quote Y)))
+        (list {:X 1 :Y 2} {:X 2 :Y 3} {:X 3 :Y 4}))
+      (dl-bt-test-set!
+        "is multiply"
+        (dl-query
+          (dl-program
+            "n(2). n(3). n(4).\n             square(X, Y) :- n(X), is(Y, *(X, X)).")
+          (list (quote square) (quote X) (quote Y)))
+        (list {:X 2 :Y 4} {:X 3 :Y 9} {:X 4 :Y 16}))
+      (dl-bt-test-set!
+        "is nested expr"
+        (dl-query
+          (dl-program
+            "n(1). n(2). n(3).\n             f(X, Y) :- n(X), is(Y, *(+(X, 1), 2)).")
+          (list (quote f) (quote X) (quote Y)))
+        (list {:X 1 :Y 4} {:X 2 :Y 6} {:X 3 :Y 8}))
+      (dl-bt-test-set!
+        "is bound LHS — equality"
+        (dl-query
+          (dl-program
+            "n(1, 2). n(2, 5). n(3, 4).\n             succ(X, Y) :- n(X, Y), is(Y, +(X, 1)).")
+          (list (quote succ) (quote X) (quote Y)))
+        (list {:X 1 :Y 2} {:X 3 :Y 4}))
+      (dl-bt-test-set!
+        "triple via is"
+        (dl-query
+          (dl-program
+            "n(1). n(2). n(3).\n             triple(X, Y) :- n(X), is(Y, *(X, 3)).")
+          (list (quote triple) (quote X) (quote Y)))
+        (list {:X 1 :Y 3} {:X 2 :Y 6} {:X 3 :Y 9}))
+      (dl-bt-test-set!
+        "= unifies var with constant"
+        (dl-query
+          (dl-program "p(a). p(b).\n             qual(X) :- p(X), =(X, a).")
+          (list (quote qual) (quote X)))
+        (list {:X (quote a)}))
+      (dl-bt-test-set!
+        "= unifies two vars (one bound)"
+        (dl-query
+          (dl-program "p(a). p(b).\n             twin(X, Y) :- p(X), =(Y, X).")
+          (list (quote twin) (quote X) (quote Y)))
+        (list {:X (quote a) :Y (quote a)} {:X (quote b) :Y (quote b)}))
+      (dl-bt-test!
+        "big count"
+        (let
+          ((db (dl-program "n(0). n(1). n(2). n(3). n(4). n(5). n(6). n(7). n(8). n(9).\n                  big(X) :- n(X), >=(X, 5).")))
+          (do (dl-saturate! db) (len (dl-relation db "big"))))
+        5)
+      ;; Built-in / arithmetic literals work as standalone query goals
+      ;; (without needing a wrapper rule).
+      (dl-bt-test-set! "comparison-only goal true"
+        (dl-eval "" "?- <(1, 2).")
+        (list {}))
+
+      (dl-bt-test-set! "comparison-only goal false"
+        (dl-eval "" "?- <(2, 1).")
+        (list))
+
+      (dl-bt-test-set! "is goal binds"
+        (dl-eval "" "?- is(N, +(2, 3)).")
+        (list {:N 5}))
+
+      ;; Bounded successor: a recursive rule with a comparison
+      ;; guard terminates because the Herbrand base is effectively
+      ;; bounded.
+      (dl-bt-test-set! "bounded successor"
+        (dl-query
+          (dl-program
+            "nat(0).
+             nat(Y) :- nat(X), is(Y, +(X, 1)), <(Y, 5).")
+          (list (quote nat) (quote X)))
+        (list {:X 0} {:X 1} {:X 2} {:X 3} {:X 4}))
+
+      (dl-bt-test!
+        "unsafe — comparison without binder"
+        (dl-bt-throws? (fn () (dl-program "p(X) :- <(X, 5).")))
+        true)
+      (dl-bt-test!
+        "unsafe — comparison both unbound"
+        (dl-bt-throws? (fn () (dl-program "p(X, Y) :- <(X, Y), q(X).")))
+        true)
+      (dl-bt-test!
+        "unsafe — is uses unbound RHS var"
+        (dl-bt-throws?
+          (fn () (dl-program "p(X, Y) :- q(X), is(Y, +(X, Z)).")))
+        true)
+      (dl-bt-test!
+        "unsafe — is on its own"
+        (dl-bt-throws? (fn () (dl-program "p(Y) :- is(Y, +(X, 1)).")))
+        true)
+      (dl-bt-test!
+        "unsafe — = between two unbound"
+        (dl-bt-throws? (fn () (dl-program "p(X, Y) :- =(X, Y).")))
+        true)
+      (dl-bt-test!
+        "safe — is binds head var"
+        (dl-bt-throws?
+          (fn () (dl-program "n(1). p(Y) :- n(X), is(Y, +(X, 1)).")))
+        false)
+      (dl-bt-test!
+        "safe — comparison after binder"
+        (dl-bt-throws?
+          (fn () (dl-program "n(1). big(X) :- n(X), >=(X, 0).")))
+        false)
+      (dl-bt-test!
+        "safe — = binds head var"
+        (dl-bt-throws?
+          (fn () (dl-program "p(a). p(b). x(Y) :- p(X), =(Y, X).")))
+        false)
+
+      ;; Division by zero raises with a clear error. Without this guard
+      ;; SX's `/` returned IEEE infinity, which then silently flowed
+      ;; through comparisons and aggregations.
+      (dl-bt-test!
+        "is — division by zero raises"
+        (dl-bt-throws?
+          (fn ()
+            (dl-eval "p(10). q(R) :- p(X), is(R, /(X, 0))." "?- q(R).")))
+        true)
+
+      ;; Comparison ops `<`, `<=`, `>`, `>=` require both operands to
+      ;; have the same primitive type. Cross-type comparisons used to
+      ;; silently return false (for some pairs) or raise a confusing
+      ;; host-level error (for others) — now they all raise with a
+      ;; message that names the offending values.
+      (dl-bt-test!
+        "comparison — string vs number raises"
+        (dl-bt-throws?
+          (fn ()
+            (dl-eval "p(\"hello\"). q(X) :- p(X), <(X, 5)." "?- q(X).")))
+        true)
+
+      ;; `!=` is the exception — it's a polymorphic inequality test
+      ;; (uses dl-tuple-equal? underneath) so cross-type pairs are
+      ;; legitimate (and trivially unequal).
+      (dl-bt-test-set! "!= works across types"
+        (dl-query
+          (dl-program
+            "p(1). p(\"1\"). q(X) :- p(X), !=(X, 1).")
+          (quote (q X)))
+        (list {:X "1"})))))
+
+(define
+  dl-builtins-tests-run!
+  (fn
+    ()
+    (do
+      (set! dl-bt-pass 0)
+      (set! dl-bt-fail 0)
+      (set! dl-bt-failures (list))
+      (dl-bt-run-all!)
+      {:failures dl-bt-failures :total (+ dl-bt-pass dl-bt-fail) :passed dl-bt-pass :failed dl-bt-fail})))

lib/datalog/tests/demo.sx

@@ -0,0 +1,321 @@
+;; lib/datalog/tests/demo.sx — Phase 10 demo programs.
+
+(define dl-demo-pass 0)
+(define dl-demo-fail 0)
+(define dl-demo-failures (list))
+
+(define
+  dl-demo-deep=?
+  (fn
+    (a b)
+    (cond
+      ((and (list? a) (list? b))
+        (and (= (len a) (len b)) (dl-demo-deq-l? a b 0)))
+      ((and (dict? a) (dict? b))
+        (let ((ka (keys a)) (kb (keys b)))
+          (and (= (len ka) (len kb)) (dl-demo-deq-d? a b ka 0))))
+      ((and (number? a) (number? b)) (= a b))
+      (else (equal? a b)))))
+
+(define
+  dl-demo-deq-l?
+  (fn
+    (a b i)
+    (cond
+      ((>= i (len a)) true)
+      ((not (dl-demo-deep=? (nth a i) (nth b i))) false)
+      (else (dl-demo-deq-l? a b (+ i 1))))))
+
+(define
+  dl-demo-deq-d?
+  (fn
+    (a b ka i)
+    (cond
+      ((>= i (len ka)) true)
+      ((let ((k (nth ka i)))
+         (not (dl-demo-deep=? (get a k) (get b k))))
+        false)
+      (else (dl-demo-deq-d? a b ka (+ i 1))))))
+
+(define
+  dl-demo-set=?
+  (fn
+    (a b)
+    (and
+      (= (len a) (len b))
+      (dl-demo-subset? a b)
+      (dl-demo-subset? b a))))
+
+(define
+  dl-demo-subset?
+  (fn
+    (xs ys)
+    (cond
+      ((= (len xs) 0) true)
+      ((not (dl-demo-contains? ys (first xs))) false)
+      (else (dl-demo-subset? (rest xs) ys)))))
+
+(define
+  dl-demo-contains?
+  (fn
+    (xs target)
+    (cond
+      ((= (len xs) 0) false)
+      ((dl-demo-deep=? (first xs) target) true)
+      (else (dl-demo-contains? (rest xs) target)))))
+
+(define
+  dl-demo-test-set!
+  (fn
+    (name got expected)
+    (if
+      (dl-demo-set=? got expected)
+      (set! dl-demo-pass (+ dl-demo-pass 1))
+      (do
+        (set! dl-demo-fail (+ dl-demo-fail 1))
+        (append!
+          dl-demo-failures
+          (str
+            name
+            "\n    expected (set): " expected
+            "\n    got:            " got))))))
+
+(define
+  dl-demo-run-all!
+  (fn
+    ()
+    (do
+      ;; ── Federation ──────────────────────────────────────────
+      (dl-demo-test-set! "mutuals"
+        (dl-query
+          (dl-demo-make
+            (quote ((follows alice bob) (follows bob alice)
+                    (follows bob carol) (follows carol dave)))
+            dl-demo-federation-rules)
+          (quote (mutual alice X)))
+        (list {:X (quote bob)}))
+
+      (dl-demo-test-set! "reachable transitive"
+        (dl-query
+          (dl-demo-make
+            (quote ((follows alice bob) (follows bob carol) (follows carol dave)))
+            dl-demo-federation-rules)
+          (quote (reachable alice X)))
+        (list {:X (quote bob)} {:X (quote carol)} {:X (quote dave)}))
+
+      (dl-demo-test-set! "foaf"
+        (dl-query
+          (dl-demo-make
+            (quote ((follows alice bob) (follows bob carol) (follows alice dave)))
+            dl-demo-federation-rules)
+          (quote (foaf alice X)))
+        (list {:X (quote carol)}))
+
+      ;; ── Content ─────────────────────────────────────────────
+      (dl-demo-test-set! "popular posts"
+        (dl-query
+          (dl-demo-make
+            (quote
+              ((authored alice p1) (authored bob p2) (authored carol p3)
+               (liked u1 p1) (liked u2 p1) (liked u3 p1)
+               (liked u1 p2)))
+            dl-demo-content-rules)
+          (quote (popular P)))
+        (list {:P (quote p1)}))
+
+      (dl-demo-test-set! "interesting feed"
+        (dl-query
+          (dl-demo-make
+            (quote
+              ((follows me alice) (follows me bob)
+               (authored alice p1) (authored bob p2)
+               (liked u1 p1) (liked u2 p1) (liked u3 p1)
+               (liked u4 p2)))
+            dl-demo-content-rules)
+          (quote (interesting me P)))
+        (list {:P (quote p1)}))
+
+      (dl-demo-test-set! "post likes count"
+        (dl-query
+          (dl-demo-make
+            (quote
+              ((authored alice p1)
+               (liked u1 p1) (liked u2 p1) (liked u3 p1)))
+            dl-demo-content-rules)
+          (quote (post-likes p1 N)))
+        (list {:N 3}))
+
+      ;; ── Permissions ─────────────────────────────────────────
+      (dl-demo-test-set! "direct group access"
+        (dl-query
+          (dl-demo-make
+            (quote
+              ((member alice editors)
+               (allowed editors blog)))
+            dl-demo-perm-rules)
+          (quote (can-access X blog)))
+        (list {:X (quote alice)}))
+
+      (dl-demo-test-set! "subgroup access"
+        (dl-query
+          (dl-demo-make
+            (quote
+              ((member bob writers)
+               (subgroup writers editors)
+               (allowed editors blog)))
+            dl-demo-perm-rules)
+          (quote (can-access X blog)))
+        (list {:X (quote bob)}))
+
+      (dl-demo-test-set! "transitive subgroup"
+        (dl-query
+          (dl-demo-make
+            (quote
+              ((member carol drafters)
+               (subgroup drafters writers)
+               (subgroup writers editors)
+               (allowed editors blog)))
+            dl-demo-perm-rules)
+          (quote (can-access X blog)))
+        (list {:X (quote carol)}))
+
+      ;; ── Cooking posts (canonical Phase 10 example) ─────────
+      (dl-demo-test-set! "cooking posts by network"
+        (dl-query
+          (dl-demo-make
+            (quote
+              ((follows me alice) (follows alice bob) (follows alice carol)
+               (authored alice p1) (authored bob p2)
+               (authored carol p3) (authored carol p4)
+               (tagged p1 travel) (tagged p2 cooking)
+               (tagged p3 cooking) (tagged p4 books)))
+            dl-demo-cooking-rules)
+          (quote (cooking-post-by-network me P)))
+        (list {:P (quote p2)} {:P (quote p3)}))
+
+      (dl-demo-test-set! "cooking — direct follow only"
+        (dl-query
+          (dl-demo-make
+            (quote
+              ((follows me bob)
+               (authored bob p1) (authored bob p2)
+               (tagged p1 cooking) (tagged p2 books)))
+            dl-demo-cooking-rules)
+          (quote (cooking-post-by-network me P)))
+        (list {:P (quote p1)}))
+
+      (dl-demo-test-set! "cooking — none in network"
+        (dl-query
+          (dl-demo-make
+            (quote
+              ((follows me bob)
+               (authored bob p1) (tagged p1 books)))
+            dl-demo-cooking-rules)
+          (quote (cooking-post-by-network me P)))
+        (list))
+
+      ;; ── Tag co-occurrence ──────────────────────────────────
+      (dl-demo-test-set! "cotagged posts"
+        (dl-query
+          (dl-demo-make
+            (quote
+              ((tagged p1 cooking) (tagged p1 vegetarian)
+               (tagged p2 cooking) (tagged p2 quick)
+               (tagged p3 vegetarian)))
+            dl-demo-tag-cooccur-rules)
+          (quote (cotagged P cooking vegetarian)))
+        (list {:P (quote p1)}))
+
+      (dl-demo-test-set! "tag pair count"
+        (dl-query
+          (dl-demo-make
+            (quote
+              ((tagged p1 cooking) (tagged p1 vegetarian)
+               (tagged p2 cooking) (tagged p2 quick)
+               (tagged p3 cooking) (tagged p3 vegetarian)))
+            dl-demo-tag-cooccur-rules)
+          (quote (tag-pair-count cooking vegetarian N)))
+        (list {:N 2}))
+
+      ;; ── Shortest path on a weighted DAG ──────────────────
+      (dl-demo-test-set! "shortest a→d via DAG"
+        (dl-query
+          (dl-demo-make
+            (quote ((edge a b 5) (edge b c 3) (edge a c 10) (edge c d 2)))
+            dl-demo-shortest-path-rules)
+          (quote (shortest a d W)))
+        (list {:W 10}))
+
+      (dl-demo-test-set! "shortest a→c picks 2-hop"
+        (dl-query
+          (dl-demo-make
+            (quote ((edge a b 5) (edge b c 3) (edge a c 10)))
+            dl-demo-shortest-path-rules)
+          (quote (shortest a c W)))
+        (list {:W 8}))
+
+      (dl-demo-test-set! "shortest unreachable empty"
+        (dl-query
+          (dl-demo-make
+            (quote ((edge a b 5) (edge b c 3)))
+            dl-demo-shortest-path-rules)
+          (quote (shortest a d W)))
+        (list))
+
+      ;; ── Org chart + headcount ─────────────────────────────
+      (dl-demo-test-set! "ceo subordinate transitive"
+        (dl-query
+          (dl-demo-make
+            (quote
+              ((manager ic1 mgr1) (manager ic2 mgr1)
+               (manager mgr1 vp1) (manager ic3 vp1)
+               (manager vp1 ceo)))
+            dl-demo-org-rules)
+          (quote (subordinate ceo X)))
+        (list
+          {:X (quote vp1)} {:X (quote mgr1)} {:X (quote ic1)}
+          {:X (quote ic2)} {:X (quote ic3)}))
+
+      (dl-demo-test-set! "ceo headcount = 5"
+        (dl-query
+          (dl-demo-make
+            (quote
+              ((manager ic1 mgr1) (manager ic2 mgr1)
+               (manager mgr1 vp1) (manager ic3 vp1)
+               (manager vp1 ceo)))
+            dl-demo-org-rules)
+          (quote (headcount ceo N)))
+        (list {:N 5}))
+
+      (dl-demo-test-set! "mgr1 headcount = 2"
+        (dl-query
+          (dl-demo-make
+            (quote
+              ((manager ic1 mgr1) (manager ic2 mgr1)
+               (manager mgr1 vp1) (manager ic3 vp1)
+               (manager vp1 ceo)))
+            dl-demo-org-rules)
+          (quote (headcount mgr1 N)))
+        (list {:N 2}))
+
+      (dl-demo-test-set! "no access without grant"
+        (dl-query
+          (dl-demo-make
+            (quote ((member dave outsiders) (allowed editors blog)))
+            dl-demo-perm-rules)
+          (quote (can-access X blog)))
+        (list)))))
+
+(define
+  dl-demo-tests-run!
+  (fn
+    ()
+    (do
+      (set! dl-demo-pass 0)
+      (set! dl-demo-fail 0)
+      (set! dl-demo-failures (list))
+      (dl-demo-run-all!)
+      {:passed dl-demo-pass
+       :failed dl-demo-fail
+       :total (+ dl-demo-pass dl-demo-fail)
+       :failures dl-demo-failures})))

lib/datalog/tests/eval.sx

@@ -0,0 +1,463 @@
+;; lib/datalog/tests/eval.sx — naive evaluation + safety analysis tests.
+
+(define dl-et-pass 0)
+(define dl-et-fail 0)
+(define dl-et-failures (list))
+
+;; Same deep-equal helper used in other suites.
+(define
+  dl-et-deep=?
+  (fn
+    (a b)
+    (cond
+      ((and (list? a) (list? b))
+        (and (= (len a) (len b)) (dl-et-deq-l? a b 0)))
+      ((and (dict? a) (dict? b))
+        (let
+          ((ka (keys a)) (kb (keys b)))
+          (and (= (len ka) (len kb)) (dl-et-deq-d? a b ka 0))))
+      ((and (number? a) (number? b)) (= a b))
+      (else (equal? a b)))))
+
+(define
+  dl-et-deq-l?
+  (fn
+    (a b i)
+    (cond
+      ((>= i (len a)) true)
+      ((not (dl-et-deep=? (nth a i) (nth b i))) false)
+      (else (dl-et-deq-l? a b (+ i 1))))))
+
+(define
+  dl-et-deq-d?
+  (fn
+    (a b ka i)
+    (cond
+      ((>= i (len ka)) true)
+      ((let ((k (nth ka i))) (not (dl-et-deep=? (get a k) (get b k))))
+        false)
+      (else (dl-et-deq-d? a b ka (+ i 1))))))
+
+;; Set-equality on lists (order-independent, uses dl-et-deep=?).
+(define
+  dl-et-set=?
+  (fn
+    (a b)
+    (and (= (len a) (len b)) (dl-et-subset? a b) (dl-et-subset? b a))))
+
+(define
+  dl-et-subset?
+  (fn
+    (xs ys)
+    (cond
+      ((= (len xs) 0) true)
+      ((not (dl-et-contains? ys (first xs))) false)
+      (else (dl-et-subset? (rest xs) ys)))))
+
+(define
+  dl-et-contains?
+  (fn
+    (xs target)
+    (cond
+      ((= (len xs) 0) false)
+      ((dl-et-deep=? (first xs) target) true)
+      (else (dl-et-contains? (rest xs) target)))))
+
+(define
+  dl-et-test!
+  (fn
+    (name got expected)
+    (if
+      (dl-et-deep=? got expected)
+      (set! dl-et-pass (+ dl-et-pass 1))
+      (do
+        (set! dl-et-fail (+ dl-et-fail 1))
+        (append!
+          dl-et-failures
+          (str name "\n    expected: " expected "\n    got:      " got))))))
+
+(define
+  dl-et-test-set!
+  (fn
+    (name got expected)
+    (if
+      (dl-et-set=? got expected)
+      (set! dl-et-pass (+ dl-et-pass 1))
+      (do
+        (set! dl-et-fail (+ dl-et-fail 1))
+        (append!
+          dl-et-failures
+          (str
+            name
+            "\n    expected (set): "
+            expected
+            "\n    got:            "
+            got))))))
+
+(define
+  dl-et-throws?
+  (fn
+    (thunk)
+    (let
+      ((threw false))
+      (do (guard (e (#t (set! threw true))) (thunk)) threw))))
+
+(define
+  dl-et-run-all!
+  (fn
+    ()
+    (do
+      (dl-et-test-set!
+        "fact lookup any"
+        (dl-query
+          (dl-program "parent(tom, bob). parent(bob, ann).")
+          (list (quote parent) (quote X) (quote Y)))
+        (list {:X (quote tom) :Y (quote bob)} {:X (quote bob) :Y (quote ann)}))
+      (dl-et-test-set!
+        "fact lookup constant arg"
+        (dl-query
+          (dl-program "parent(tom, bob). parent(tom, liz). parent(bob, ann).")
+          (list (quote parent) (quote tom) (quote Y)))
+        (list {:Y (quote bob)} {:Y (quote liz)}))
+      (dl-et-test-set!
+        "no match"
+        (dl-query
+          (dl-program "parent(tom, bob).")
+          (list (quote parent) (quote nobody) (quote X)))
+        (list))
+      (dl-et-test-set!
+        "ancestor closure"
+        (dl-query
+          (dl-program
+            "parent(tom, bob). parent(bob, ann). parent(ann, pat).\n             ancestor(X, Y) :- parent(X, Y).\n             ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z).")
+          (list (quote ancestor) (quote tom) (quote X)))
+        (list {:X (quote bob)} {:X (quote ann)} {:X (quote pat)}))
+      (dl-et-test-set!
+        "sibling"
+        (dl-query
+          (dl-program
+            "parent(tom, bob). parent(tom, liz). parent(jane, bob). parent(jane, liz).\n             sibling(X, Y) :- parent(P, X), parent(P, Y).")
+          (list (quote sibling) (quote bob) (quote Y)))
+        (list {:Y (quote bob)} {:Y (quote liz)}))
+      (dl-et-test-set!
+        "same-generation"
+        (dl-query
+          (dl-program
+            "parent(tom, bob). parent(tom, liz). parent(bob, ann). parent(liz, joe).\n             person(tom). person(bob). person(liz). person(ann). person(joe).\n             sg(X, X) :- person(X).\n             sg(X, Y) :- parent(P1, X), sg(P1, P2), parent(P2, Y).")
+          (list (quote sg) (quote ann) (quote X)))
+        (list {:X (quote ann)} {:X (quote joe)}))
+      (dl-et-test!
+        "ancestor count"
+        (let
+          ((db (dl-program "parent(a, b). parent(b, c). parent(c, d).\n                  ancestor(X, Y) :- parent(X, Y).\n                  ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z).")))
+          (do (dl-saturate! db) (len (dl-relation db "ancestor"))))
+        6)
+      (dl-et-test-set!
+        "grandparent"
+        (dl-query
+          (dl-program
+            "parent(a, b). parent(b, c). parent(c, d).\n             grandparent(X, Z) :- parent(X, Y), parent(Y, Z).")
+          (list (quote grandparent) (quote X) (quote Y)))
+        (list {:X (quote a) :Y (quote c)} {:X (quote b) :Y (quote d)}))
+      (dl-et-test!
+        "no recursion infinite loop"
+        (let
+          ((db (dl-program "edge(1, 2). edge(2, 3). edge(3, 1).\n                  reach(X, Y) :- edge(X, Y).\n                  reach(X, Z) :- edge(X, Y), reach(Y, Z).")))
+          (do (dl-saturate! db) (len (dl-relation db "reach"))))
+        9)
+      ;; Rule-shape sanity: empty-list head and non-list body raise
+      ;; clear errors rather than crashing inside the saturator.
+      (dl-et-test! "empty head rejected"
+        (dl-et-throws?
+          (fn ()
+            (dl-add-rule! (dl-make-db)
+              {:head (list) :body (list)})))
+        true)
+
+      (dl-et-test! "non-list body rejected"
+        (dl-et-throws?
+          (fn ()
+            (dl-add-rule! (dl-make-db)
+              {:head (list (quote p) (quote X)) :body 42})))
+        true)
+
+      ;; Reserved relation names rejected as rule/fact heads.
+      (dl-et-test!
+        "reserved name `not` as head rejected"
+        (dl-et-throws? (fn () (dl-program "not(X) :- p(X).")))
+        true)
+
+      (dl-et-test!
+        "reserved name `count` as head rejected"
+        (dl-et-throws?
+          (fn () (dl-program "count(N, X, p(X)) :- p(X).")))
+        true)
+
+      (dl-et-test!
+        "reserved name `<` as head rejected"
+        (dl-et-throws? (fn () (dl-program "<(X, 5) :- p(X).")))
+        true)
+
+      (dl-et-test!
+        "reserved name `is` as head rejected"
+        (dl-et-throws? (fn () (dl-program "is(N, +(1, 2)) :- p(N).")))
+        true)
+
+      ;; Body literal with a reserved-name positive head is rejected.
+      ;; The parser only treats outer-level `not(P)` as negation; nested
+      ;; `not(not(P))` would otherwise silently parse as a positive call
+      ;; to a relation named `not` and succeed vacuously. The safety
+      ;; checker now flags this so the user gets a clear error.
+      ;; Body literal with a reserved-name positive head is rejected.
+      ;; The parser only treats outer-level `not(P)` as negation; nested
+      ;; `not(not(P))` would otherwise silently parse as a positive call
+      ;; to a relation named `not` and succeed vacuously — so the safety
+      ;; checker now flags this to give the user a clear error.
+      (dl-et-test!
+        "nested not(not(...)) rejected"
+        (dl-et-throws?
+          (fn ()
+            (dl-program
+              "banned(a). u(a). vip(X) :- u(X), not(not(banned(X))).")))
+        true)
+
+      ;; A dict body literal that isn't `{:neg ...}` is almost always a
+      ;; typo — it would otherwise silently fall through to a confusing
+      ;; head-var-unbound safety error. Now caught with a clear message.
+      (dl-et-test!
+        "dict body lit without :neg rejected"
+        (dl-et-throws?
+          (fn ()
+            (let ((db (dl-make-db)))
+              (dl-add-rule! db
+                {:head (list (quote p) (quote X))
+                 :body (list {:weird "stuff"})}))))
+        true)
+
+      ;; Facts may only have simple-term args (number / string / symbol).
+      ;; A compound arg like `+(1, 2)` would otherwise be silently
+      ;; stored as the unreduced expression `(+ 1 2)` because dl-ground?
+      ;; sees no free variables.
+      (dl-et-test!
+        "compound arg in fact rejected"
+        (dl-et-throws? (fn () (dl-program "p(+(1, 2)).")))
+        true)
+
+      ;; Rule heads may only have variable or constant args — no
+      ;; compounds. Compound heads would be saturated as unreduced
+      ;; tuples rather than the arithmetic result the user expected.
+      (dl-et-test!
+        "compound arg in rule head rejected"
+        (dl-et-throws?
+          (fn () (dl-program "n(3). double(*(X, 2)) :- n(X).")))
+        true)
+
+      ;; The anonymous-variable renamer used to start at `_anon1`
+      ;; unconditionally; a rule that wrote `q(_anon1) :- p(_anon1, _)`
+      ;; (the user picking the same name the renamer would generate)
+      ;; would see the `_` renamed to `_anon1` too, collapsing the
+      ;; two positions in `p(_anon1, _)` to a single var. Now the
+      ;; renamer scans the rule for the max `_anon<N>` and starts past
+      ;; it, so user-written names of that form are preserved.
+      (dl-et-test-set! "anonymous-rename avoids user `_anon` collision"
+        (dl-query
+          (dl-program
+            "p(a, b). p(c, d). q(_anon1) :- p(_anon1, _).")
+          (quote (q X)))
+        (list {:X (quote a)} {:X (quote c)}))
+
+      (dl-et-test!
+        "unsafe head var"
+        (dl-et-throws? (fn () (dl-program "p(X, Y) :- q(X).")))
+        true)
+      (dl-et-test!
+        "unsafe — empty body"
+        (dl-et-throws? (fn () (dl-program "p(X) :- .")))
+        true)
+      ;; Underscore in head is unsafe — it's a fresh existential per
+      ;; occurrence after Phase 5d's anonymous-var renaming, and there's
+      ;; nothing in the body to bind it. (Old behavior accepted this by
+      ;; treating '_' as a literal name to skip; the renaming made it an
+      ;; ordinary unbound variable.)
+      (dl-et-test!
+        "underscore in head — unsafe"
+        (dl-et-throws? (fn () (dl-program "p(X, _) :- q(X).")))
+        true)
+      (dl-et-test!
+        "underscore in body only — safe"
+        (dl-et-throws? (fn () (dl-program "p(X) :- q(X, _).")))
+        false)
+      (dl-et-test!
+        "var only in head — unsafe"
+        (dl-et-throws? (fn () (dl-program "p(X, Y) :- q(Z).")))
+        true)
+      (dl-et-test!
+        "head var bound by body"
+        (dl-et-throws? (fn () (dl-program "p(X) :- q(X).")))
+        false)
+      (dl-et-test!
+        "head subset of body"
+        (dl-et-throws?
+          (fn
+            ()
+            (dl-program
+              "edge(a,b). edge(b,c). reach(X, Z) :- edge(X, Y), edge(Y, Z).")))
+        false)
+
+      ;; Anonymous variables: each occurrence must be independent.
+      (dl-et-test-set! "anon vars in rule are independent"
+        (dl-query
+          (dl-program
+            "p(a, b). p(c, d). q(X) :- p(X, _), p(_, Y).")
+          (list (quote q) (quote X)))
+        (list {:X (quote a)} {:X (quote c)}))
+
+      (dl-et-test-set! "anon vars in goal are independent"
+        (dl-query
+          (dl-program "p(1, 2, 3). p(4, 5, 6).")
+          (list (quote p) (quote _) (quote X) (quote _)))
+        (list {:X 2} {:X 5}))
+
+      ;; dl-summary: relation -> tuple-count for inspection.
+      (dl-et-test! "dl-summary basic"
+        (dl-summary
+          (let
+            ((db (dl-program "p(1). p(2). q(3).")))
+            (do (dl-saturate! db) db)))
+        {:p 2 :q 1})
+
+      (dl-et-test! "dl-summary empty IDB shown"
+        (dl-summary
+          (let
+            ((db (dl-program "r(X) :- s(X).")))
+            (do (dl-saturate! db) db)))
+        {:r 0})
+
+      (dl-et-test! "dl-summary mixed EDB and IDB"
+        (dl-summary
+          (let
+            ((db (dl-program
+                   "parent(a, b).
+                    ancestor(X, Y) :- parent(X, Y).
+                    ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z).")))
+            (do (dl-saturate! db) db)))
+        {:parent 1 :ancestor 1})
+
+      (dl-et-test! "dl-summary empty db"
+        (dl-summary (dl-make-db))
+        {})
+
+      ;; Strategy hook: default semi-naive; :magic accepted but
+      ;; falls back to semi-naive (the transformation itself is
+      ;; deferred — Phase 6 in plan).
+      (dl-et-test! "default strategy"
+        (dl-get-strategy (dl-make-db))
+        :semi-naive)
+
+      (dl-et-test! "set strategy"
+        (let ((db (dl-make-db)))
+          (do (dl-set-strategy! db :magic) (dl-get-strategy db)))
+        :magic)
+
+      ;; Unknown strategy values are rejected so typos don't silently
+      ;; fall back to the default.
+      (dl-et-test!
+        "unknown strategy rejected"
+        (dl-et-throws?
+          (fn ()
+            (let ((db (dl-make-db)))
+              (dl-set-strategy! db :semi_naive))))
+        true)
+
+      ;; dl-saturated?: no-work-left predicate.
+      (dl-et-test! "saturated? after saturation"
+        (let ((db (dl-program
+                    "parent(a, b).
+                     ancestor(X, Y) :- parent(X, Y).")))
+          (do (dl-saturate! db) (dl-saturated? db)))
+        true)
+
+      (dl-et-test! "saturated? before saturation"
+        (let ((db (dl-program
+                    "parent(a, b).
+                     ancestor(X, Y) :- parent(X, Y).")))
+          (dl-saturated? db))
+        false)
+
+      ;; Disjunction via multiple rules — Datalog has no `;` in
+      ;; body, so disjunction is expressed as separate rules with
+      ;; the same head. Here plant_based(X) is satisfied by either
+      ;; vegan(X) or vegetarian(X).
+      (dl-et-test-set! "disjunction via multiple rules"
+        (dl-query
+          (dl-program
+            "vegan(alice). vegetarian(bob). meat_eater(carol).
+             plant_based(X) :- vegan(X).
+             plant_based(X) :- vegetarian(X).")
+          (list (quote plant_based) (quote X)))
+        (list {:X (quote alice)} {:X (quote bob)}))
+
+      ;; Bipartite-style join: pair-of-friends who share a hobby.
+      ;; Three-relation join exercising the planner's join order.
+      (dl-et-test-set! "bipartite friends-with-hobby"
+        (dl-query
+          (dl-program
+            "hobby(alice, climb). hobby(bob, paint).
+             hobby(carol, climb).
+             friend(alice, carol). friend(bob, alice).
+             match(A, B, H) :- friend(A, B), hobby(A, H), hobby(B, H).")
+          (list (quote match) (quote A) (quote B) (quote H)))
+        (list {:A (quote alice) :B (quote carol) :H (quote climb)}))
+
+      ;; Repeated variable (diagonal): p(X, X) only matches tuples
+      ;; whose two args are equal. The unifier handles this via the
+      ;; subst chain — first occurrence binds X, second occurrence
+      ;; checks against the binding.
+      (dl-et-test-set! "diagonal query"
+        (dl-query
+          (dl-program "p(1, 1). p(2, 3). p(4, 4). p(5, 5).")
+          (list (quote p) (quote X) (quote X)))
+        (list {:X 1} {:X 4} {:X 5}))
+
+      ;; A relation can be both EDB-seeded and rule-derived;
+      ;; saturate combines facts + derivations.
+      (dl-et-test-set! "mixed EDB + IDB same relation"
+        (dl-query
+          (dl-program
+            "link(a, b). link(c, d). link(e, c).
+             via(a, e).
+             link(X, Y) :- via(X, M), link(M, Y).")
+          (list (quote link) (quote a) (quote X)))
+        (list {:X (quote b)} {:X (quote c)}))
+
+      (dl-et-test! "saturated? after assert"
+        (let ((db (dl-program
+                    "parent(a, b).
+                     ancestor(X, Y) :- parent(X, Y).")))
+          (do
+            (dl-saturate! db)
+            (dl-add-fact! db (list (quote parent) (quote b) (quote c)))
+            (dl-saturated? db)))
+        false)
+
+      (dl-et-test-set! "magic-set still derives correctly"
+        (let
+          ((db (dl-program
+                 "parent(a, b). parent(b, c).
+                  ancestor(X, Y) :- parent(X, Y).
+                  ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z).")))
+          (do
+            (dl-set-strategy! db :magic)
+            (dl-query db (list (quote ancestor) (quote a) (quote X)))))
+        (list {:X (quote b)} {:X (quote c)})))))
+
+(define
+  dl-eval-tests-run!
+  (fn
+    ()
+    (do
+      (set! dl-et-pass 0)
+      (set! dl-et-fail 0)
+      (set! dl-et-failures (list))
+      (dl-et-run-all!)
+      {:failures dl-et-failures :total (+ dl-et-pass dl-et-fail) :passed dl-et-pass :failed dl-et-fail})))

lib/datalog/tests/magic.sx

@@ -0,0 +1,528 @@
+;; lib/datalog/tests/magic.sx — adornment + SIPS analysis tests.
+
+(define dl-mt-pass 0)
+(define dl-mt-fail 0)
+(define dl-mt-failures (list))
+
+(define
+  dl-mt-deep=?
+  (fn
+    (a b)
+    (cond
+      ((and (list? a) (list? b))
+        (and (= (len a) (len b)) (dl-mt-deq-l? a b 0)))
+      ((and (dict? a) (dict? b))
+        (let ((ka (keys a)) (kb (keys b)))
+          (and (= (len ka) (len kb)) (dl-mt-deq-d? a b ka 0))))
+      ((and (number? a) (number? b)) (= a b))
+      (else (equal? a b)))))
+
+(define
+  dl-mt-deq-l?
+  (fn
+    (a b i)
+    (cond
+      ((>= i (len a)) true)
+      ((not (dl-mt-deep=? (nth a i) (nth b i))) false)
+      (else (dl-mt-deq-l? a b (+ i 1))))))
+
+(define
+  dl-mt-deq-d?
+  (fn
+    (a b ka i)
+    (cond
+      ((>= i (len ka)) true)
+      ((let ((k (nth ka i)))
+         (not (dl-mt-deep=? (get a k) (get b k))))
+        false)
+      (else (dl-mt-deq-d? a b ka (+ i 1))))))
+
+(define
+  dl-mt-test!
+  (fn
+    (name got expected)
+    (if
+      (dl-mt-deep=? got expected)
+      (set! dl-mt-pass (+ dl-mt-pass 1))
+      (do
+        (set! dl-mt-fail (+ dl-mt-fail 1))
+        (append!
+          dl-mt-failures
+          (str
+            name
+            "\n    expected: " expected
+            "\n    got:      " got))))))
+
+(define
+  dl-mt-run-all!
+  (fn
+    ()
+    (do
+      ;; Goal adornment.
+      (dl-mt-test! "adorn 0-ary"
+        (dl-adorn-goal (list (quote ready)))
+        "")
+      (dl-mt-test! "adorn all bound"
+        (dl-adorn-goal (list (quote p) 1 2 3))
+        "bbb")
+      (dl-mt-test! "adorn all free"
+        (dl-adorn-goal (list (quote p) (quote X) (quote Y)))
+        "ff")
+      (dl-mt-test! "adorn mixed"
+        (dl-adorn-goal (list (quote ancestor) (quote tom) (quote X)))
+        "bf")
+      (dl-mt-test! "adorn const var const"
+        (dl-adorn-goal (list (quote p) (quote a) (quote X) (quote b)))
+        "bfb")
+
+      ;; dl-adorn-lit with explicit bound set.
+      (dl-mt-test! "adorn lit with bound"
+        (dl-adorn-lit (list (quote p) (quote X) (quote Y)) (list "X"))
+        "bf")
+
+      ;; Rule SIPS — chain ancestor.
+      (dl-mt-test! "sips chain ancestor bf"
+        (dl-rule-sips
+          {:head (list (quote ancestor) (quote X) (quote Z))
+           :body (list (list (quote parent) (quote X) (quote Y))
+                       (list (quote ancestor) (quote Y) (quote Z)))}
+          "bf")
+        (list
+          {:lit (list (quote parent) (quote X) (quote Y)) :adornment "bf"}
+          {:lit (list (quote ancestor) (quote Y) (quote Z)) :adornment "bf"}))
+
+      ;; SIPS — head fully bound.
+      (dl-mt-test! "sips head bb"
+        (dl-rule-sips
+          {:head (list (quote q) (quote X) (quote Y))
+           :body (list (list (quote p) (quote X) (quote Z))
+                       (list (quote r) (quote Z) (quote Y)))}
+          "bb")
+        (list
+          {:lit (list (quote p) (quote X) (quote Z)) :adornment "bf"}
+          {:lit (list (quote r) (quote Z) (quote Y)) :adornment "bb"}))
+
+      ;; SIPS — comparison; vars must be bound by prior body lit.
+      (dl-mt-test! "sips with comparison"
+        (dl-rule-sips
+          {:head (list (quote q) (quote X))
+           :body (list (list (quote p) (quote X))
+                       (list (string->symbol "<") (quote X) 5))}
+          "f")
+        (list
+          {:lit (list (quote p) (quote X)) :adornment "f"}
+          {:lit (list (string->symbol "<") (quote X) 5) :adornment "bb"}))
+
+      ;; SIPS — `is` binds its left arg.
+      (dl-mt-test! "sips with is"
+        (dl-rule-sips
+          {:head (list (quote q) (quote X) (quote Y))
+           :body (list (list (quote p) (quote X))
+                       (list (quote is) (quote Y) (list (string->symbol "+") (quote X) 1)))}
+          "ff")
+        (list
+          {:lit (list (quote p) (quote X)) :adornment "f"}
+          {:lit (list (quote is) (quote Y)
+                      (list (string->symbol "+") (quote X) 1))
+           :adornment "fb"}))
+
+      ;; Magic predicate naming.
+      (dl-mt-test! "magic-rel-name"
+        (dl-magic-rel-name "ancestor" "bf")
+        "magic_ancestor^bf")
+
+      ;; Bound-args extraction.
+      (dl-mt-test! "bound-args bf"
+        (dl-bound-args (list (quote ancestor) (quote tom) (quote X)) "bf")
+        (list (quote tom)))
+
+      (dl-mt-test! "bound-args mixed"
+        (dl-bound-args (list (quote p) 1 (quote Y) 3) "bfb")
+        (list 1 3))
+
+      (dl-mt-test! "bound-args all-free"
+        (dl-bound-args (list (quote p) (quote X) (quote Y)) "ff")
+        (list))
+
+      ;; Magic literal construction.
+      (dl-mt-test! "magic-lit"
+        (dl-magic-lit "ancestor" "bf" (list (quote tom)))
+        (list (string->symbol "magic_ancestor^bf") (quote tom)))
+
+      ;; Magic-sets rewriter: structural sanity.
+      (dl-mt-test! "rewrite ancestor produces seed"
+        (let
+          ((rules
+             (list
+               {:head (list (quote ancestor) (quote X) (quote Y))
+                :body (list (list (quote parent) (quote X) (quote Y)))}
+               {:head (list (quote ancestor) (quote X) (quote Z))
+                :body
+                (list (list (quote parent) (quote X) (quote Y))
+                      (list (quote ancestor) (quote Y) (quote Z)))})))
+          (get
+            (dl-magic-rewrite rules "ancestor" "bf" (list (quote a)))
+            :seed))
+        (list (string->symbol "magic_ancestor^bf") (quote a)))
+
+      ;; Equivalence: rewritten program derives same ancestor tuples.
+      ;; In a chain a→b→c→d, magic-rewritten run still derives all
+      ;; ancestor pairs reachable from any node a/b/c/d propagated via
+      ;; magic_ancestor^bf — i.e. the full closure (6 tuples). Magic
+      ;; saves work only when the EDB has irrelevant nodes outside
+      ;; the seed's transitive cone.
+      (dl-mt-test! "magic-rewritten ancestor count"
+        (let
+          ((rules
+             (list
+               {:head (list (quote ancestor) (quote X) (quote Y))
+                :body (list (list (quote parent) (quote X) (quote Y)))}
+               {:head (list (quote ancestor) (quote X) (quote Z))
+                :body
+                (list (list (quote parent) (quote X) (quote Y))
+                      (list (quote ancestor) (quote Y) (quote Z)))}))
+            (edb (list
+                   (list (quote parent) (quote a) (quote b))
+                   (list (quote parent) (quote b) (quote c))
+                   (list (quote parent) (quote c) (quote d)))))
+          (let
+            ((rewritten (dl-magic-rewrite rules "ancestor" "bf" (list (quote a))))
+              (db (dl-make-db)))
+            (do
+              (for-each (fn (f) (dl-add-fact! db f)) edb)
+              (dl-add-fact! db (get rewritten :seed))
+              (for-each (fn (r) (dl-add-rule! db r)) (get rewritten :rules))
+              (dl-saturate! db)
+              (len (dl-relation db "ancestor")))))
+        6)
+
+      ;; dl-magic-query: end-to-end driver, doesn't mutate caller's db.
+      ;; Magic over a rule with negated body literal — propagation
+      ;; rules generated only for positive lits; negated lits pass
+      ;; through unchanged.
+      (dl-mt-test! "magic over rule with negation"
+        (let
+          ((db (dl-program
+                 "u(a). u(b). u(c). banned(b).
+                  active(X) :- u(X), not(banned(X)).")))
+          (let
+            ((semi (dl-query db (list (quote active) (quote X))))
+              (magic (dl-magic-query db (list (quote active) (quote X)))))
+            (= (len semi) (len magic))))
+        true)
+
+      ;; All-bound query (existence check) generates an "bb"
+      ;; adornment chain. Verifies the rewriter walks multiple
+      ;; (rel, adn) pairs through the worklist.
+      (dl-mt-test! "magic existence check via bb"
+        (let
+          ((db (dl-program
+                 "parent(a, b). parent(b, c). parent(c, d).
+                  ancestor(X, Y) :- parent(X, Y).
+                  ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z).")))
+          (let
+            ((found (dl-magic-query
+                      db (list (quote ancestor) (quote a) (quote c))))
+              (missing (dl-magic-query
+                         db (list (quote ancestor) (quote a) (quote z)))))
+            (and (= (len found) 1) (= (len missing) 0))))
+        true)
+
+      ;; Magic equivalence on the federation demo.
+      (dl-mt-test! "magic ≡ semi on foaf demo"
+        (let
+          ((db (dl-program-data
+                 (quote ((follows alice bob)
+                         (follows bob carol)
+                         (follows alice dave)))
+                 dl-demo-federation-rules)))
+          (let
+            ((semi (dl-query db (quote (foaf alice X))))
+              (magic (dl-magic-query db (quote (foaf alice X)))))
+            (= (len semi) (len magic))))
+        true)
+
+      ;; Shape validation: dl-magic-query rejects non-list / non-
+      ;; dict goal shapes cleanly rather than crashing in `rest`.
+      (dl-mt-test! "magic rejects string goal"
+        (let ((threw false))
+          (do
+            (guard (e (#t (set! threw true)))
+              (dl-magic-query (dl-make-db) "foo"))
+            threw))
+        true)
+
+      (dl-mt-test! "magic rejects bare dict goal"
+        (let ((threw false))
+          (do
+            (guard (e (#t (set! threw true)))
+              (dl-magic-query (dl-make-db) {:foo "bar"}))
+            threw))
+        true)
+
+      ;; 3-stratum program under magic — distinct rule heads at
+      ;; strata 0/1/2 must all rewrite via the worklist.
+      (dl-mt-test! "magic 3-stratum program"
+        (let
+          ((db (dl-program
+                 "a(1). a(2). a(3). b(2).
+                  c(X) :- a(X), not(b(X)).
+                  d(X) :- c(X), not(banned(X)).
+                  banned(3).")))
+          (let
+            ((semi (dl-query db (list (quote d) (quote X))))
+              (magic (dl-magic-query db (list (quote d) (quote X)))))
+            (= (len semi) (len magic))))
+        true)
+
+      ;; Aggregate -> derived -> threshold chain via magic.
+      (dl-mt-test! "magic aggregate-derived chain"
+        (let
+          ((db (dl-program
+                 "src(1). src(2). src(3).
+                  cnt(N) :- count(N, X, src(X)).
+                  active(N) :- cnt(N), >=(N, 2).")))
+          (let
+            ((semi (dl-query db (list (quote active) (quote N))))
+              (magic (dl-magic-query db (list (quote active) (quote N)))))
+            (= (len semi) (len magic))))
+        true)
+
+      ;; Multi-relation rewrite chain: query r4 → propagate to r3,
+      ;; r2, r1, a. The worklist must process all of them; an
+      ;; earlier bug stopped after only the head pair.
+      (dl-mt-test! "magic chain through 4 rule levels"
+        (let
+          ((db (dl-program
+                 "a(1). a(2). r1(X) :- a(X). r2(X) :- r1(X).
+                  r3(X) :- r2(X). r4(X) :- r3(X).")))
+          (= 2 (len (dl-magic-query db (list (quote r4) (quote X))))))
+        true)
+
+      ;; Shortest-path demo via magic — exercises the rewriter
+      ;; against rules that mix recursive positive lits with an
+      ;; aggregate body literal.
+      (dl-mt-test! "magic on shortest-path demo"
+        (let
+          ((db (dl-program-data
+                 (quote ((edge a b 5) (edge b c 3) (edge a c 10)))
+                 dl-demo-shortest-path-rules)))
+          (let
+            ((semi (dl-query db (quote (shortest a c W))))
+              (magic (dl-magic-query db (quote (shortest a c W)))))
+            (and (= (len semi) (len magic))
+                 (= (len semi) 1))))
+        true)
+
+      ;; Same relation called with different adornment patterns
+      ;; in different rules. The worklist must enqueue and process
+      ;; each (rel, adornment) pair.
+      (dl-mt-test! "magic with multi-adornment same relation"
+        (let
+          ((db (dl-program
+                 "parent(p1, alice). parent(p2, bob).
+                  parent(g, p1). parent(g, p2).
+                  sibling(P1, P2) :- parent(G, P1), parent(G, P2),
+                                     !=(P1, P2).
+                  cousin(X, Y) :- parent(P1, X), parent(P2, Y),
+                                  sibling(P1, P2).")))
+          (let
+            ((semi (dl-query db (list (quote cousin) (quote alice) (quote Y))))
+              (magic (dl-magic-query db (list (quote cousin) (quote alice) (quote Y)))))
+            (= (len semi) (len magic))))
+        true)
+
+      ;; Magic over a rule whose body contains an aggregate.
+      ;; The rewriter passes aggregate body lits through unchanged
+      ;; (no propagation generated for them), so semi-naive's count
+      ;; logic still fires correctly under the rewritten program.
+      (dl-mt-test! "magic over rule with aggregate body"
+        (let
+          ((db (dl-program
+                 "post(p1). post(p2). post(p3).
+                  liked(u1, p1). liked(u2, p1). liked(u3, p1).
+                  liked(u1, p2).
+                  rich(P) :- post(P), count(N, U, liked(U, P)),
+                             >=(N, 2).")))
+          (let
+            ((semi (dl-query db (list (quote rich) (quote P))))
+              (magic (dl-magic-query db (list (quote rich) (quote P)))))
+            (= (len semi) (len magic))))
+        true)
+
+      ;; Mixed EDB + IDB: a relation can be both EDB-seeded and
+      ;; rule-derived. dl-magic-query must include the EDB portion
+      ;; even though the relation has rules.
+      (dl-mt-test! "magic mixed EDB+IDB"
+        (len
+          (dl-magic-query
+            (dl-program
+              "link(a, b). link(c, d). link(e, c).
+               via(a, e).
+               link(X, Y) :- via(X, M), link(M, Y).")
+            (list (quote link) (quote a) (quote X))))
+        2)
+
+      ;; dl-magic-query falls back to dl-query for built-in,
+      ;; aggregate, and negation goals (the magic seed would
+      ;; otherwise be non-ground).
+      (dl-mt-test! "magic-query falls back on aggregate"
+        (let
+          ((r (dl-magic-query
+                (dl-program "p(1). p(2). p(3).")
+                (list (quote count) (quote N) (quote X)
+                      (list (quote p) (quote X))))))
+          (and (= (len r) 1) (= (get (first r) "N") 3)))
+        true)
+
+      (dl-mt-test! "magic-query equivalent to dl-query"
+        (let
+          ((db (dl-program
+                 "parent(a, b). parent(b, c). parent(c, d).
+                  ancestor(X, Y) :- parent(X, Y).
+                  ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z).")))
+          (let
+            ((semi (dl-query db (list (quote ancestor) (quote a) (quote X))))
+              (magic (dl-magic-query
+                       db (list (quote ancestor) (quote a) (quote X)))))
+            (= (len semi) (len magic))))
+        true)
+
+      ;; The magic rewriter passes aggregate body lits through
+      ;; unchanged, so an aggregate over an IDB relation would see an
+      ;; empty inner-goal in the magic db unless the IDB is already
+      ;; materialised. dl-magic-query now pre-saturates the source db
+      ;; to guarantee equivalence with dl-query for every stratified
+      ;; program. Previously this returned `({:N 0})` because `active`
+      ;; (IDB, derived through negation) was never derived in the
+      ;; magic db.
+      (dl-mt-test! "magic over aggregate-of-IDB matches vanilla"
+        (let
+          ((src
+             "u(a). u(b). u(c). u(d). banned(b). banned(d).
+              active(X) :- u(X), not(banned(X)).
+              n(N) :- count(N, X, active(X))."))
+          (let
+            ((vanilla (dl-eval src "?- n(N)."))
+              (magic (dl-eval-magic src "?- n(N).")))
+            (and (= (len vanilla) 1)
+                 (= (len magic) 1)
+                 (= (get (first vanilla) "N")
+                    (get (first magic) "N")))))
+        true)
+
+      ;; magic-query doesn't mutate caller db.
+      (dl-mt-test! "magic-query preserves caller db"
+        (let
+          ((db (dl-program
+                 "parent(a, b). parent(b, c).
+                  ancestor(X, Y) :- parent(X, Y).
+                  ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z).")))
+          (let
+            ((rules-before (len (dl-rules db))))
+            (do
+              (dl-magic-query db (list (quote ancestor) (quote a) (quote X)))
+              (= rules-before (len (dl-rules db))))))
+        true)
+
+      ;; Magic-sets benefit: query touches only one cluster of a
+      ;; multi-component graph. Semi-naive derives the full closure
+      ;; over both clusters; magic only the seeded one.
+      ;; Magic-vs-semi work shape: chain of 12. Semi-naive
+      ;; derives the full closure (78 = 12·13/2). A magic query
+      ;; rooted at node 0 returns the 12 descendants only —
+      ;; demonstrating that magic limits derivation to the
+      ;; query's transitive cone.
+      (dl-mt-test! "magic vs semi work-shape on chain-12"
+        (let
+          ((source (str
+                     "parent(0, 1). parent(1, 2). parent(2, 3). "
+                     "parent(3, 4). parent(4, 5). parent(5, 6). "
+                     "parent(6, 7). parent(7, 8). parent(8, 9). "
+                     "parent(9, 10). parent(10, 11). parent(11, 12). "
+                     "ancestor(X, Y) :- parent(X, Y). "
+                     "ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z).")))
+          (let
+            ((db1 (dl-make-db)) (db2 (dl-make-db)))
+            (do
+              (dl-load-program! db1 source)
+              (dl-saturate! db1)
+              (dl-load-program! db2 source)
+              (let
+                ((semi-count (len (dl-relation db1 "ancestor")))
+                  (magic-count
+                    (len (dl-magic-query
+                           db2 (list (quote ancestor) 0 (quote X))))))
+                ;; Magic returns only descendants of 0 (12 of them).
+                (and (= semi-count 78) (= magic-count 12))))))
+        true)
+
+      ;; Magic + arithmetic: rules with `is` clauses pass through
+      ;; the rewriter unchanged (built-ins aren't propagated).
+      (dl-mt-test! "magic preserves arithmetic"
+        (let
+          ((source "n(1). n(2). n(3).
+                    doubled(X, Y) :- n(X), is(Y, *(X, 2))."))
+          (let
+            ((semi (dl-eval source "?- doubled(2, Y)."))
+              (magic (dl-eval-magic source "?- doubled(2, Y).")))
+            (= (len semi) (len magic))))
+        true)
+
+      (dl-mt-test! "magic skips irrelevant clusters"
+        (let
+          ;; Two disjoint chains. Query is rooted in cluster 1.
+          ((db (dl-program
+                 "parent(a, b). parent(b, c).
+                  parent(x, y). parent(y, z).
+                  ancestor(X, Y) :- parent(X, Y).
+                  ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z).")))
+          (do
+            (dl-saturate! db)
+            (let
+              ((semi-count (len (dl-relation db "ancestor")))
+                (magic-results
+                  (dl-magic-query
+                    db (list (quote ancestor) (quote a) (quote X)))))
+              ;; Semi-naive derives 6 (3 in each cluster). Magic
+              ;; gives 3 query results (a's reachable: b, c).
+              (and (= semi-count 6) (= (len magic-results) 2)))))
+        true)
+
+      (dl-mt-test! "magic-rewritten finds same answers"
+        (let
+          ((rules
+             (list
+               {:head (list (quote ancestor) (quote X) (quote Y))
+                :body (list (list (quote parent) (quote X) (quote Y)))}
+               {:head (list (quote ancestor) (quote X) (quote Z))
+                :body
+                (list (list (quote parent) (quote X) (quote Y))
+                      (list (quote ancestor) (quote Y) (quote Z)))}))
+            (edb (list
+                   (list (quote parent) (quote a) (quote b))
+                   (list (quote parent) (quote b) (quote c)))))
+          (let
+            ((rewritten (dl-magic-rewrite rules "ancestor" "bf" (list (quote a))))
+              (db (dl-make-db)))
+            (do
+              (for-each (fn (f) (dl-add-fact! db f)) edb)
+              (dl-add-fact! db (get rewritten :seed))
+              (for-each (fn (r) (dl-add-rule! db r)) (get rewritten :rules))
+              (dl-saturate! db)
+              (len (dl-query db (list (quote ancestor) (quote a) (quote X)))))))
+        2))))
+
+(define
+  dl-magic-tests-run!
+  (fn
+    ()
+    (do
+      (set! dl-mt-pass 0)
+      (set! dl-mt-fail 0)
+      (set! dl-mt-failures (list))
+      (dl-mt-run-all!)
+      {:passed dl-mt-pass
+       :failed dl-mt-fail
+       :total (+ dl-mt-pass dl-mt-fail)
+       :failures dl-mt-failures})))

lib/datalog/tests/negation.sx

@@ -0,0 +1,252 @@
+;; lib/datalog/tests/negation.sx — stratified negation tests.
+
+(define dl-nt-pass 0)
+(define dl-nt-fail 0)
+(define dl-nt-failures (list))
+
+(define
+  dl-nt-deep=?
+  (fn
+    (a b)
+    (cond
+      ((and (list? a) (list? b))
+        (and (= (len a) (len b)) (dl-nt-deq-l? a b 0)))
+      ((and (dict? a) (dict? b))
+        (let ((ka (keys a)) (kb (keys b)))
+          (and (= (len ka) (len kb)) (dl-nt-deq-d? a b ka 0))))
+      ((and (number? a) (number? b)) (= a b))
+      (else (equal? a b)))))
+
+(define
+  dl-nt-deq-l?
+  (fn
+    (a b i)
+    (cond
+      ((>= i (len a)) true)
+      ((not (dl-nt-deep=? (nth a i) (nth b i))) false)
+      (else (dl-nt-deq-l? a b (+ i 1))))))
+
+(define
+  dl-nt-deq-d?
+  (fn
+    (a b ka i)
+    (cond
+      ((>= i (len ka)) true)
+      ((let ((k (nth ka i)))
+         (not (dl-nt-deep=? (get a k) (get b k))))
+        false)
+      (else (dl-nt-deq-d? a b ka (+ i 1))))))
+
+(define
+  dl-nt-set=?
+  (fn
+    (a b)
+    (and
+      (= (len a) (len b))
+      (dl-nt-subset? a b)
+      (dl-nt-subset? b a))))
+
+(define
+  dl-nt-subset?
+  (fn
+    (xs ys)
+    (cond
+      ((= (len xs) 0) true)
+      ((not (dl-nt-contains? ys (first xs))) false)
+      (else (dl-nt-subset? (rest xs) ys)))))
+
+(define
+  dl-nt-contains?
+  (fn
+    (xs target)
+    (cond
+      ((= (len xs) 0) false)
+      ((dl-nt-deep=? (first xs) target) true)
+      (else (dl-nt-contains? (rest xs) target)))))
+
+(define
+  dl-nt-test!
+  (fn
+    (name got expected)
+    (if
+      (dl-nt-deep=? got expected)
+      (set! dl-nt-pass (+ dl-nt-pass 1))
+      (do
+        (set! dl-nt-fail (+ dl-nt-fail 1))
+        (append!
+          dl-nt-failures
+          (str
+            name
+            "\n    expected: " expected
+            "\n    got:      " got))))))
+
+(define
+  dl-nt-test-set!
+  (fn
+    (name got expected)
+    (if
+      (dl-nt-set=? got expected)
+      (set! dl-nt-pass (+ dl-nt-pass 1))
+      (do
+        (set! dl-nt-fail (+ dl-nt-fail 1))
+        (append!
+          dl-nt-failures
+          (str
+            name
+            "\n    expected (set): " expected
+            "\n    got:            " got))))))
+
+(define
+  dl-nt-throws?
+  (fn
+    (thunk)
+    (let
+      ((threw false))
+      (do
+        (guard
+          (e (#t (set! threw true)))
+          (thunk))
+        threw))))
+
+(define
+  dl-nt-run-all!
+  (fn
+    ()
+    (do
+      ;; Negation against EDB-only relation.
+      (dl-nt-test-set! "not against EDB"
+        (dl-query
+          (dl-program
+            "p(1). p(2). p(3). r(2).
+             q(X) :- p(X), not(r(X)).")
+          (list (quote q) (quote X)))
+        (list {:X 1} {:X 3}))
+
+      ;; Negation against derived relation — needs stratification.
+      (dl-nt-test-set! "not against derived rel"
+        (dl-query
+          (dl-program
+            "p(1). p(2). p(3). s(2).
+             r(X) :- s(X).
+             q(X) :- p(X), not(r(X)).")
+          (list (quote q) (quote X)))
+        (list {:X 1} {:X 3}))
+
+      ;; Two-step strata: r derives via s; q derives via not r.
+      (dl-nt-test-set! "two-step strata"
+        (dl-query
+          (dl-program
+            "node(a). node(b). node(c). node(d).
+             edge(a, b). edge(b, c). edge(c, a).
+             reach(X, Y) :- edge(X, Y).
+             reach(X, Z) :- edge(X, Y), reach(Y, Z).
+             unreachable(X) :- node(X), not(reach(a, X)).")
+          (list (quote unreachable) (quote X)))
+        (list {:X (quote d)}))
+
+      ;; Combine negation with arithmetic and comparison.
+      (dl-nt-test-set! "negation with arithmetic"
+        (dl-query
+          (dl-program
+            "n(1). n(2). n(3). n(4). n(5). odd(1). odd(3). odd(5).
+             even(X) :- n(X), not(odd(X)).")
+          (list (quote even) (quote X)))
+        (list {:X 2} {:X 4}))
+
+      ;; Empty negation result.
+      (dl-nt-test-set! "negation always succeeds"
+        (dl-query
+          (dl-program
+            "p(1). p(2). q(X) :- p(X), not(r(X)).")
+          (list (quote q) (quote X)))
+        (list {:X 1} {:X 2}))
+
+      ;; Negation always fails.
+      (dl-nt-test-set! "negation always fails"
+        (dl-query
+          (dl-program
+            "p(1). p(2). r(1). r(2). q(X) :- p(X), not(r(X)).")
+          (list (quote q) (quote X)))
+        (list))
+
+      ;; Anonymous `_` in a negated literal is existentially quantified
+      ;; — it doesn't need to be bound by an earlier body lit. Without
+      ;; this exemption the safety check would reject the common idiom
+      ;; `orphan(X) :- person(X), not(parent(X, _))`.
+      (dl-nt-test-set! "negation with anonymous var — orphan idiom"
+        (dl-query
+          (dl-program
+            "person(a). person(b). person(c). parent(a, b).
+             orphan(X) :- person(X), not(parent(X, _)).")
+          (list (quote orphan) (quote X)))
+        (list {:X (quote b)} {:X (quote c)}))
+
+      ;; Multiple anonymous vars are each independently existential.
+      (dl-nt-test-set! "negation with multiple anonymous vars"
+        (dl-query
+          (dl-program
+            "u(a). u(b). u(c). edge(a, x). edge(b, y).
+             solo(X) :- u(X), not(edge(X, _)).")
+          (list (quote solo) (quote X)))
+        (list {:X (quote c)}))
+
+      ;; Stratifiability checks.
+      (dl-nt-test! "non-stratifiable rejected"
+        (dl-nt-throws?
+          (fn ()
+            (let ((db (dl-make-db)))
+              (do
+                (dl-add-rule!
+                  db
+                  {:head (list (quote p) (quote X))
+                   :body (list (list (quote q) (quote X))
+                               {:neg (list (quote r) (quote X))})})
+                (dl-add-rule!
+                  db
+                  {:head (list (quote r) (quote X))
+                   :body (list (list (quote p) (quote X)))})
+                (dl-add-fact! db (list (quote q) 1))
+                (dl-saturate! db)))))
+        true)
+
+      (dl-nt-test! "stratifiable accepted"
+        (dl-nt-throws?
+          (fn ()
+            (dl-program
+              "p(1). p(2). r(2).
+               q(X) :- p(X), not(r(X)).")))
+        false)
+
+      ;; Multi-stratum chain.
+      (dl-nt-test-set! "three-level strata"
+        (dl-query
+          (dl-program
+            "a(1). a(2). a(3). a(4).
+             b(X) :- a(X), not(c(X)).
+             c(X) :- d(X).
+             d(2).
+             d(4).")
+          (list (quote b) (quote X)))
+        (list {:X 1} {:X 3}))
+
+      ;; Safety violation: negation refers to unbound var.
+      (dl-nt-test! "negation safety violation"
+        (dl-nt-throws?
+          (fn ()
+            (dl-program
+              "p(1). q(X) :- p(X), not(r(Y)).")))
+        true))))
+
+(define
+  dl-negation-tests-run!
+  (fn
+    ()
+    (do
+      (set! dl-nt-pass 0)
+      (set! dl-nt-fail 0)
+      (set! dl-nt-failures (list))
+      (dl-nt-run-all!)
+      {:passed dl-nt-pass
+       :failed dl-nt-fail
+       :total (+ dl-nt-pass dl-nt-fail)
+       :failures dl-nt-failures})))

lib/datalog/tests/parse.sx

@@ -0,0 +1,179 @@
+;; lib/datalog/tests/parse.sx — parser unit tests
+;;
+;; Run via: bash lib/datalog/conformance.sh
+;; Or:      (load "lib/datalog/tokenizer.sx") (load "lib/datalog/parser.sx")
+;;          (load "lib/datalog/tests/parse.sx") (dl-parse-tests-run!)
+
+(define dl-pt-pass 0)
+(define dl-pt-fail 0)
+(define dl-pt-failures (list))
+
+;; Order-independent structural equality. Lists compared positionally,
+;; dicts as sets of (key, value) pairs. Numbers via = (so 30.0 = 30).
+(define
+  dl-deep-equal?
+  (fn
+    (a b)
+    (cond
+      ((and (list? a) (list? b))
+        (and (= (len a) (len b)) (dl-deep-equal-list? a b 0)))
+      ((and (dict? a) (dict? b))
+        (let
+          ((ka (keys a)) (kb (keys b)))
+          (and
+            (= (len ka) (len kb))
+            (dl-deep-equal-dict? a b ka 0))))
+      ((and (number? a) (number? b)) (= a b))
+      (else (equal? a b)))))
+
+(define
+  dl-deep-equal-list?
+  (fn
+    (a b i)
+    (cond
+      ((>= i (len a)) true)
+      ((not (dl-deep-equal? (nth a i) (nth b i))) false)
+      (else (dl-deep-equal-list? a b (+ i 1))))))
+
+(define
+  dl-deep-equal-dict?
+  (fn
+    (a b ka i)
+    (cond
+      ((>= i (len ka)) true)
+      ((let ((k (nth ka i))) (not (dl-deep-equal? (get a k) (get b k))))
+        false)
+      (else (dl-deep-equal-dict? a b ka (+ i 1))))))
+
+(define
+  dl-pt-test!
+  (fn
+    (name got expected)
+    (if
+      (dl-deep-equal? got expected)
+      (set! dl-pt-pass (+ dl-pt-pass 1))
+      (do
+        (set! dl-pt-fail (+ dl-pt-fail 1))
+        (append!
+          dl-pt-failures
+          (str name "\n    expected: " expected "\n    got:      " got))))))
+
+(define
+  dl-pt-throws?
+  (fn
+    (thunk)
+    (let
+      ((threw false))
+      (do (guard (e (#t (set! threw true))) (thunk)) threw))))
+
+(define
+  dl-pt-run-all!
+  (fn
+    ()
+    (do
+      (dl-pt-test! "empty program" (dl-parse "") (list))
+      (dl-pt-test! "fact" (dl-parse "parent(tom, bob).") (list {:body (list) :head (list (quote parent) (quote tom) (quote bob))}))
+      (dl-pt-test!
+        "two facts"
+        (dl-parse "parent(tom, bob). parent(bob, ann).")
+        (list {:body (list) :head (list (quote parent) (quote tom) (quote bob))} {:body (list) :head (list (quote parent) (quote bob) (quote ann))}))
+      (dl-pt-test! "zero-ary fact" (dl-parse "ready.") (list {:body (list) :head (list (quote ready))}))
+      (dl-pt-test!
+        "rule one body lit"
+        (dl-parse "ancestor(X, Y) :- parent(X, Y).")
+        (list {:body (list (list (quote parent) (quote X) (quote Y))) :head (list (quote ancestor) (quote X) (quote Y))}))
+      (dl-pt-test!
+        "recursive rule"
+        (dl-parse "ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z).")
+        (list {:body (list (list (quote parent) (quote X) (quote Y)) (list (quote ancestor) (quote Y) (quote Z))) :head (list (quote ancestor) (quote X) (quote Z))}))
+      (dl-pt-test!
+        "query single"
+        (dl-parse "?- ancestor(tom, X).")
+        (list {:query (list (list (quote ancestor) (quote tom) (quote X)))}))
+      (dl-pt-test!
+        "query multi"
+        (dl-parse "?- p(X), q(X).")
+        (list {:query (list (list (quote p) (quote X)) (list (quote q) (quote X)))}))
+      (dl-pt-test!
+        "negation"
+        (dl-parse "safe(X) :- person(X), not(parent(X, _)).")
+        (list {:body (list (list (quote person) (quote X)) {:neg (list (quote parent) (quote X) (quote _))}) :head (list (quote safe) (quote X))}))
+      (dl-pt-test!
+        "number arg"
+        (dl-parse "age(alice, 30).")
+        (list {:body (list) :head (list (quote age) (quote alice) 30)}))
+      (dl-pt-test!
+        "string arg"
+        (dl-parse "label(x, \"hi\").")
+        (list {:body (list) :head (list (quote label) (quote x) "hi")}))
+      ;; Quoted 'atoms' parse as strings — a uppercase-starting name
+      ;; in quotes used to misclassify as a variable and reject the
+      ;; fact as non-ground.
+      (dl-pt-test!
+        "quoted atom arg parses as string"
+        (dl-parse "p('Hello World').")
+        (list {:body (list) :head (list (quote p) "Hello World")}))
+      (dl-pt-test!
+        "comparison literal"
+        (dl-parse "p(X) :- <(X, 5).")
+        (list {:body (list (list (string->symbol "<") (quote X) 5)) :head (list (quote p) (quote X))}))
+      (dl-pt-test!
+        "is with arith"
+        (dl-parse "succ(X, Y) :- nat(X), is(Y, +(X, 1)).")
+        (list {:body (list (list (quote nat) (quote X)) (list (quote is) (quote Y) (list (string->symbol "+") (quote X) 1))) :head (list (quote succ) (quote X) (quote Y))}))
+      (dl-pt-test!
+        "mixed program"
+        (dl-parse "p(a). p(b). q(X) :- p(X). ?- q(Y).")
+        (list {:body (list) :head (list (quote p) (quote a))} {:body (list) :head (list (quote p) (quote b))} {:body (list (list (quote p) (quote X))) :head (list (quote q) (quote X))} {:query (list (list (quote q) (quote Y)))}))
+      (dl-pt-test!
+        "comments skipped"
+        (dl-parse "% comment\nfoo(a).\n/* block */ bar(b).")
+        (list {:body (list) :head (list (quote foo) (quote a))} {:body (list) :head (list (quote bar) (quote b))}))
+      (dl-pt-test!
+        "underscore var"
+        (dl-parse "p(X) :- q(X, _).")
+        (list {:body (list (list (quote q) (quote X) (quote _))) :head (list (quote p) (quote X))}))
+      ;; Negative number literals parse as one negative number,
+      ;; while subtraction (`-(X, Y)`) compound is preserved.
+      (dl-pt-test!
+        "negative integer literal"
+        (dl-parse "n(-3).")
+        (list {:head (list (quote n) -3) :body (list)}))
+
+      (dl-pt-test!
+        "subtraction compound preserved"
+        (dl-parse "r(X) :- is(X, -(10, 3)).")
+        (list
+          {:head (list (quote r) (quote X))
+           :body (list (list (quote is) (quote X)
+                             (list (string->symbol "-") 10 3)))}))
+
+      (dl-pt-test!
+        "number as relation name raises"
+        (dl-pt-throws? (fn () (dl-parse "1(X) :- p(X).")))
+        true)
+
+      (dl-pt-test!
+        "var as relation name raises"
+        (dl-pt-throws? (fn () (dl-parse "P(X).")))
+        true)
+
+      (dl-pt-test!
+        "missing dot raises"
+        (dl-pt-throws? (fn () (dl-parse "p(a)")))
+        true)
+      (dl-pt-test!
+        "trailing comma raises"
+        (dl-pt-throws? (fn () (dl-parse "p(a,).")))
+        true))))
+
+(define
+  dl-parse-tests-run!
+  (fn
+    ()
+    (do
+      (set! dl-pt-pass 0)
+      (set! dl-pt-fail 0)
+      (set! dl-pt-failures (list))
+      (dl-pt-run-all!)
+      {:failures dl-pt-failures :total (+ dl-pt-pass dl-pt-fail) :passed dl-pt-pass :failed dl-pt-fail})))

lib/datalog/tests/semi_naive.sx

@@ -0,0 +1,153 @@
+;; lib/datalog/tests/semi_naive.sx — semi-naive correctness vs naive.
+;;
+;; Strategy: differential — run both saturators on each program and
+;; compare the resulting per-relation tuple counts. Counting (not
+;; element-wise set equality) keeps the suite fast under the bundled
+;; conformance session; correctness on the inhabitants is covered by
+;; eval.sx and builtins.sx (which use dl-saturate! by default — the
+;; semi-naive saturator).
+
+(define dl-sn-pass 0)
+(define dl-sn-fail 0)
+(define dl-sn-failures (list))
+
+(define
+  dl-sn-test!
+  (fn
+    (name got expected)
+    (if
+      (equal? got expected)
+      (set! dl-sn-pass (+ dl-sn-pass 1))
+      (do
+        (set! dl-sn-fail (+ dl-sn-fail 1))
+        (append!
+          dl-sn-failures
+          (str name "\n    expected: " expected "\n    got:      " got))))))
+
+;; Load `source` into both a semi-naive and a naive db and return a
+;; list of (rel-name semi-count naive-count) triples. Both sets must
+;; have the same union of relation names.
+(define
+  dl-sn-counts
+  (fn
+    (source)
+    (let
+      ((db-s (dl-program source)) (db-n (dl-program source)))
+      (do
+        (dl-saturate! db-s)
+        (dl-saturate-naive! db-n)
+        (let
+          ((out (list)))
+          (do
+            (for-each
+              (fn
+                (k)
+                (append!
+                  out
+                  (list
+                    k
+                    (len (dl-relation db-s k))
+                    (len (dl-relation db-n k)))))
+              (keys (get db-s :facts)))
+            out))))))
+
+(define
+  dl-sn-counts-agree?
+  (fn
+    (counts)
+    (cond
+      ((= (len counts) 0) true)
+      (else
+        (let
+          ((row (first counts)))
+          (and
+            (= (nth row 1) (nth row 2))
+            (dl-sn-counts-agree? (rest counts))))))))
+
+(define
+  dl-sn-chain-source
+  (fn
+    (n)
+    (let
+      ((parts (list "")))
+      (do
+        (define
+          dl-sn-loop
+          (fn
+            (i)
+            (when
+              (< i n)
+              (do
+                (append! parts (str "parent(" i ", " (+ i 1) "). "))
+                (dl-sn-loop (+ i 1))))))
+        (dl-sn-loop 0)
+        (str
+          (join "" parts)
+          "ancestor(X, Y) :- parent(X, Y). "
+          "ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z).")))))
+
+(define
+  dl-sn-run-all!
+  (fn
+    ()
+    (do
+      (dl-sn-test!
+        "ancestor closure counts match"
+        (dl-sn-counts-agree?
+          (dl-sn-counts
+            "parent(a, b). parent(b, c). parent(c, d).\n             ancestor(X, Y) :- parent(X, Y).\n             ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z)."))
+        true)
+      (dl-sn-test!
+        "cyclic reach counts match"
+        (dl-sn-counts-agree?
+          (dl-sn-counts
+            "edge(1, 2). edge(2, 3). edge(3, 1). edge(3, 4).\n             reach(X, Y) :- edge(X, Y).\n             reach(X, Z) :- edge(X, Y), reach(Y, Z)."))
+        true)
+      (dl-sn-test!
+        "same-gen counts match"
+        (dl-sn-counts-agree?
+          (dl-sn-counts
+            "parent(a, b). parent(a, c). parent(b, d). parent(c, e).\n             person(a). person(b). person(c). person(d). person(e).\n             sg(X, X) :- person(X).\n             sg(X, Y) :- parent(P1, X), sg(P1, P2), parent(P2, Y)."))
+        true)
+      (dl-sn-test!
+        "rules with builtins counts match"
+        (dl-sn-counts-agree?
+          (dl-sn-counts
+            "n(1). n(2). n(3). n(4). n(5).\n             small(X) :- n(X), <(X, 5).\n             succ(X, Y) :- n(X), <(X, 5), is(Y, +(X, 1))."))
+        true)
+      (dl-sn-test!
+        "static rule fires under semi-naive"
+        (dl-sn-counts-agree?
+          (dl-sn-counts "p(a). p(b). q(X) :- p(X), =(X, a)."))
+        true)
+      ;; Chain length 12 — multiple semi-naive iterations against
+      ;; the recursive ancestor rule (differential vs naive).
+      (dl-sn-test!
+        "chain-12 ancestor counts match"
+        (dl-sn-counts-agree? (dl-sn-counts (dl-sn-chain-source 12)))
+        true)
+      ;; Chain length 25 — semi-naive only — first-arg index makes
+      ;; this tractable in conformance budget.
+      (dl-sn-test!
+        "chain-25 ancestor count value (semi only)"
+        (let
+          ((db (dl-program (dl-sn-chain-source 25))))
+          (do (dl-saturate! db) (len (dl-relation db "ancestor"))))
+        325)
+      (dl-sn-test!
+        "query through semi saturate"
+        (let
+          ((db (dl-program "parent(a, b). parent(b, c).\n                  ancestor(X, Y) :- parent(X, Y).\n                  ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z).")))
+          (len (dl-query db (list (quote ancestor) (quote a) (quote X)))))
+        2))))
+
+(define
+  dl-semi-naive-tests-run!
+  (fn
+    ()
+    (do
+      (set! dl-sn-pass 0)
+      (set! dl-sn-fail 0)
+      (set! dl-sn-failures (list))
+      (dl-sn-run-all!)
+      {:failures dl-sn-failures :total (+ dl-sn-pass dl-sn-fail) :passed dl-sn-pass :failed dl-sn-fail})))

lib/datalog/tests/tokenize.sx

@@ -0,0 +1,189 @@
+;; lib/datalog/tests/tokenize.sx — tokenizer unit tests
+;;
+;; Run via: bash lib/datalog/conformance.sh
+;; Or:      (load "lib/datalog/tokenizer.sx") (load "lib/datalog/tests/tokenize.sx")
+;;          (dl-tokenize-tests-run!)
+
+(define dl-tk-pass 0)
+(define dl-tk-fail 0)
+(define dl-tk-failures (list))
+
+(define
+  dl-tk-test!
+  (fn
+    (name got expected)
+    (if
+      (= got expected)
+      (set! dl-tk-pass (+ dl-tk-pass 1))
+      (do
+        (set! dl-tk-fail (+ dl-tk-fail 1))
+        (append!
+          dl-tk-failures
+          (str name "\n    expected: " expected "\n    got:      " got))))))
+
+(define dl-tk-types (fn (toks) (map (fn (t) (get t :type)) toks)))
+(define dl-tk-values (fn (toks) (map (fn (t) (get t :value)) toks)))
+
+(define
+  dl-tk-run-all!
+  (fn
+    ()
+    (do
+      (dl-tk-test! "empty" (dl-tk-types (dl-tokenize "")) (list "eof"))
+      (dl-tk-test!
+        "atom dot"
+        (dl-tk-types (dl-tokenize "foo."))
+        (list "atom" "punct" "eof"))
+      (dl-tk-test!
+        "atom dot value"
+        (dl-tk-values (dl-tokenize "foo."))
+        (list "foo" "." nil))
+      (dl-tk-test!
+        "var"
+        (dl-tk-types (dl-tokenize "X."))
+        (list "var" "punct" "eof"))
+      (dl-tk-test!
+        "underscore var"
+        (dl-tk-types (dl-tokenize "_x."))
+        (list "var" "punct" "eof"))
+      (dl-tk-test!
+        "integer"
+        (dl-tk-values (dl-tokenize "42"))
+        (list 42 nil))
+      (dl-tk-test!
+        "decimal"
+        (dl-tk-values (dl-tokenize "3.14"))
+        (list 3.14 nil))
+      (dl-tk-test!
+        "string"
+        (dl-tk-values (dl-tokenize "\"hello\""))
+        (list "hello" nil))
+      ;; Quoted 'atoms' tokenize as strings — see the type-table
+      ;; comment in lib/datalog/tokenizer.sx for the rationale.
+      (dl-tk-test!
+        "quoted atom as string"
+        (dl-tk-types (dl-tokenize "'two words'"))
+        (list "string" "eof"))
+      (dl-tk-test!
+        "quoted atom value"
+        (dl-tk-values (dl-tokenize "'two words'"))
+        (list "two words" nil))
+      ;; A quoted atom whose name would otherwise be a variable
+      ;; (uppercase / leading underscore) is now safely a string —
+      ;; this was the bug that motivated the type change.
+      (dl-tk-test!
+        "quoted Uppercase as string"
+        (dl-tk-types (dl-tokenize "'Hello'"))
+        (list "string" "eof"))
+      (dl-tk-test! ":-" (dl-tk-values (dl-tokenize ":-")) (list ":-" nil))
+      (dl-tk-test! "?-" (dl-tk-values (dl-tokenize "?-")) (list "?-" nil))
+      (dl-tk-test! "<=" (dl-tk-values (dl-tokenize "<=")) (list "<=" nil))
+      (dl-tk-test! ">=" (dl-tk-values (dl-tokenize ">=")) (list ">=" nil))
+      (dl-tk-test! "!=" (dl-tk-values (dl-tokenize "!=")) (list "!=" nil))
+      (dl-tk-test!
+        "single op values"
+        (dl-tk-values (dl-tokenize "< > = + - * /"))
+        (list "<" ">" "=" "+" "-" "*" "/" nil))
+      (dl-tk-test!
+        "single op types"
+        (dl-tk-types (dl-tokenize "< > = + - * /"))
+        (list "op" "op" "op" "op" "op" "op" "op" "eof"))
+      (dl-tk-test!
+        "punct"
+        (dl-tk-values (dl-tokenize "( ) , ."))
+        (list "(" ")" "," "." nil))
+      (dl-tk-test!
+        "fact tokens"
+        (dl-tk-types (dl-tokenize "parent(tom, bob)."))
+        (list "atom" "punct" "atom" "punct" "atom" "punct" "punct" "eof"))
+      (dl-tk-test!
+        "rule shape"
+        (dl-tk-types (dl-tokenize "p(X) :- q(X)."))
+        (list
+          "atom"
+          "punct"
+          "var"
+          "punct"
+          "op"
+          "atom"
+          "punct"
+          "var"
+          "punct"
+          "punct"
+          "eof"))
+      (dl-tk-test!
+        "comparison literal"
+        (dl-tk-values (dl-tokenize "<(X, 5)"))
+        (list "<" "(" "X" "," 5 ")" nil))
+      (dl-tk-test!
+        "is form"
+        (dl-tk-values (dl-tokenize "is(Y, +(X, 1))"))
+        (list "is" "(" "Y" "," "+" "(" "X" "," 1 ")" ")" nil))
+      (dl-tk-test!
+        "line comment"
+        (dl-tk-types (dl-tokenize "% comment line\nfoo."))
+        (list "atom" "punct" "eof"))
+      (dl-tk-test!
+        "block comment"
+        (dl-tk-types (dl-tokenize "/* a\nb */ x."))
+        (list "atom" "punct" "eof"))
+      ;; Unexpected characters surface at tokenize time rather
+      ;; than being silently consumed (previously `?(X)` parsed as
+      ;; if the leading `?` weren't there).
+      (dl-tk-test!
+        "unexpected char raises"
+        (let ((threw false))
+          (do
+            (guard (e (#t (set! threw true)))
+              (dl-tokenize "?(X)"))
+            threw))
+        true)
+
+      ;; Unterminated string / quoted-atom must raise.
+      (dl-tk-test!
+        "unterminated string raises"
+        (let ((threw false))
+          (do
+            (guard (e (#t (set! threw true)))
+              (dl-tokenize "\"unclosed"))
+            threw))
+        true)
+
+      (dl-tk-test!
+        "unterminated quoted atom raises"
+        (let ((threw false))
+          (do
+            (guard (e (#t (set! threw true)))
+              (dl-tokenize "'unclosed"))
+            threw))
+        true)
+
+      ;; Unterminated block comment must raise — previously it was
+      ;; silently consumed to EOF.
+      (dl-tk-test!
+        "unterminated block comment raises"
+        (let ((threw false))
+          (do
+            (guard (e (#t (set! threw true)))
+              (dl-tokenize "/* unclosed comment"))
+            threw))
+        true)
+      (dl-tk-test!
+        "whitespace"
+        (dl-tk-types (dl-tokenize "  foo  ,\t bar  ."))
+        (list "atom" "punct" "atom" "punct" "eof"))
+      (dl-tk-test!
+        "positions"
+        (map (fn (t) (get t :pos)) (dl-tokenize "foo bar"))
+        (list 0 4 7)))))
+
+(define
+  dl-tokenize-tests-run!
+  (fn
+    ()
+    (do
+      (set! dl-tk-pass 0)
+      (set! dl-tk-fail 0)
+      (set! dl-tk-failures (list))
+      (dl-tk-run-all!)
+      {:failures dl-tk-failures :total (+ dl-tk-pass dl-tk-fail) :passed dl-tk-pass :failed dl-tk-fail})))

lib/datalog/tests/unify.sx

@@ -0,0 +1,194 @@
+;; lib/datalog/tests/unify.sx — unification + substitution tests.
+
+(define dl-ut-pass 0)
+(define dl-ut-fail 0)
+(define dl-ut-failures (list))
+
+(define
+  dl-ut-deep-equal?
+  (fn
+    (a b)
+    (cond
+      ((and (list? a) (list? b))
+        (and (= (len a) (len b)) (dl-ut-deq-list? a b 0)))
+      ((and (dict? a) (dict? b))
+        (let
+          ((ka (keys a)) (kb (keys b)))
+          (and (= (len ka) (len kb)) (dl-ut-deq-dict? a b ka 0))))
+      ((and (number? a) (number? b)) (= a b))
+      (else (equal? a b)))))
+
+(define
+  dl-ut-deq-list?
+  (fn
+    (a b i)
+    (cond
+      ((>= i (len a)) true)
+      ((not (dl-ut-deep-equal? (nth a i) (nth b i))) false)
+      (else (dl-ut-deq-list? a b (+ i 1))))))
+
+(define
+  dl-ut-deq-dict?
+  (fn
+    (a b ka i)
+    (cond
+      ((>= i (len ka)) true)
+      ((let ((k (nth ka i))) (not (dl-ut-deep-equal? (get a k) (get b k))))
+        false)
+      (else (dl-ut-deq-dict? a b ka (+ i 1))))))
+
+(define
+  dl-ut-test!
+  (fn
+    (name got expected)
+    (if
+      (dl-ut-deep-equal? got expected)
+      (set! dl-ut-pass (+ dl-ut-pass 1))
+      (do
+        (set! dl-ut-fail (+ dl-ut-fail 1))
+        (append!
+          dl-ut-failures
+          (str name "\n    expected: " expected "\n    got:      " got))))))
+
+(define
+  dl-ut-run-all!
+  (fn
+    ()
+    (do
+      (dl-ut-test! "var? uppercase" (dl-var? (quote X)) true)
+      (dl-ut-test! "var? underscore" (dl-var? (quote _foo)) true)
+      (dl-ut-test! "var? lowercase" (dl-var? (quote tom)) false)
+      (dl-ut-test! "var? number" (dl-var? 5) false)
+      (dl-ut-test! "var? string" (dl-var? "hi") false)
+      (dl-ut-test! "var? list" (dl-var? (list 1)) false)
+      (dl-ut-test!
+        "atom-atom match"
+        (dl-unify (quote tom) (quote tom) (dl-empty-subst))
+        {})
+      (dl-ut-test!
+        "atom-atom fail"
+        (dl-unify (quote tom) (quote bob) (dl-empty-subst))
+        nil)
+      (dl-ut-test!
+        "num-num match"
+        (dl-unify 5 5 (dl-empty-subst))
+        {})
+      (dl-ut-test!
+        "num-num fail"
+        (dl-unify 5 6 (dl-empty-subst))
+        nil)
+      (dl-ut-test!
+        "string match"
+        (dl-unify "hi" "hi" (dl-empty-subst))
+        {})
+      (dl-ut-test! "string fail" (dl-unify "hi" "bye" (dl-empty-subst)) nil)
+      (dl-ut-test!
+        "var-atom binds"
+        (dl-unify (quote X) (quote tom) (dl-empty-subst))
+        {:X (quote tom)})
+      (dl-ut-test!
+        "atom-var binds"
+        (dl-unify (quote tom) (quote X) (dl-empty-subst))
+        {:X (quote tom)})
+      (dl-ut-test!
+        "var-var same"
+        (dl-unify (quote X) (quote X) (dl-empty-subst))
+        {})
+      (dl-ut-test!
+        "var-var bind"
+        (let
+          ((s (dl-unify (quote X) (quote Y) (dl-empty-subst))))
+          (dl-walk (quote X) s))
+        (quote Y))
+      (dl-ut-test!
+        "tuple match"
+        (dl-unify
+          (list (quote parent) (quote X) (quote bob))
+          (list (quote parent) (quote tom) (quote Y))
+          (dl-empty-subst))
+        {:X (quote tom) :Y (quote bob)})
+      (dl-ut-test!
+        "tuple arity mismatch"
+        (dl-unify
+          (list (quote p) (quote X))
+          (list (quote p) (quote a) (quote b))
+          (dl-empty-subst))
+        nil)
+      (dl-ut-test!
+        "tuple head mismatch"
+        (dl-unify
+          (list (quote p) (quote X))
+          (list (quote q) (quote X))
+          (dl-empty-subst))
+        nil)
+      (dl-ut-test!
+        "walk chain"
+        (let
+          ((s1 (dl-unify (quote X) (quote Y) (dl-empty-subst))))
+          (let
+            ((s2 (dl-unify (quote Y) (quote tom) s1)))
+            (dl-walk (quote X) s2)))
+        (quote tom))
+
+      ;; Walk with circular substitution must not infinite-loop.
+      ;; Cycles return the current term unchanged.
+      (dl-ut-test!
+        "walk circular subst no hang"
+        (let ((s (dl-bind (quote B) (quote A)
+                   (dl-bind (quote A) (quote B) (dl-empty-subst)))))
+          (dl-walk (quote A) s))
+        (quote A))
+      (dl-ut-test!
+        "apply subst on tuple"
+        (let
+          ((s (dl-bind (quote X) (quote tom) (dl-empty-subst))))
+          (dl-apply-subst (list (quote parent) (quote X) (quote Y)) s))
+        (list (quote parent) (quote tom) (quote Y)))
+      (dl-ut-test!
+        "ground? all const"
+        (dl-ground?
+          (list (quote p) (quote tom) 5)
+          (dl-empty-subst))
+        true)
+      (dl-ut-test!
+        "ground? unbound var"
+        (dl-ground? (list (quote p) (quote X)) (dl-empty-subst))
+        false)
+      (dl-ut-test!
+        "ground? bound var"
+        (let
+          ((s (dl-bind (quote X) (quote tom) (dl-empty-subst))))
+          (dl-ground? (list (quote p) (quote X)) s))
+        true)
+      (dl-ut-test!
+        "ground? bare var"
+        (dl-ground? (quote X) (dl-empty-subst))
+        false)
+      (dl-ut-test!
+        "vars-of basic"
+        (dl-vars-of
+          (list (quote p) (quote X) (quote tom) (quote Y) (quote X)))
+        (list "X" "Y"))
+      (dl-ut-test!
+        "vars-of ground"
+        (dl-vars-of (list (quote p) (quote tom) (quote bob)))
+        (list))
+      (dl-ut-test!
+        "vars-of nested compound"
+        (dl-vars-of
+          (list
+            (quote is)
+            (quote Z)
+            (list (string->symbol "+") (quote X) 1)))
+        (list "Z" "X")))))
+
+(define
+  dl-unify-tests-run!
+  (fn
+    ()
+    (do
+      (set! dl-ut-pass 0)
+      (set! dl-ut-fail 0)
+      (set! dl-ut-failures (list))
+      (dl-ut-run-all!)
+      {:failures dl-ut-failures :total (+ dl-ut-pass dl-ut-fail) :passed dl-ut-pass :failed dl-ut-fail})))

lib/datalog/tokenizer.sx

@@ -0,0 +1,269 @@
+;; lib/datalog/tokenizer.sx — Datalog source → token stream
+;;
+;; Tokens: {:type T :value V :pos P}
+;; Types:
+;;   "atom"    — lowercase-start bare identifier
+;;   "var"     — uppercase-start or _-start ident (value is the name)
+;;   "number"  — numeric literal (decoded to number)
+;;   "string"  — "..." string literal OR quoted 'atom' (treated as a
+;;               string value to avoid the var-vs-atom ambiguity that
+;;               would arise from a quoted atom whose name starts with
+;;               an uppercase letter or underscore)
+;;   "punct"   — ( ) , .
+;;   "op"      — :- ?- <= >= != < > = + - * /
+;;   "eof"
+;;
+;; Datalog has no function symbols in arg position; the parser still
+;; accepts nested compounds for arithmetic ((is X (+ A B))) but safety
+;; analysis rejects non-arithmetic nesting at rule-load time.
+
+(define dl-make-token (fn (type value pos) {:type type :value value :pos pos}))
+
+(define dl-digit? (fn (c) (and (>= c "0") (<= c "9"))))
+(define dl-lower? (fn (c) (and (>= c "a") (<= c "z"))))
+(define dl-upper? (fn (c) (and (>= c "A") (<= c "Z"))))
+
+(define
+  dl-ident-char?
+  (fn (c) (or (dl-lower? c) (dl-upper? c) (dl-digit? c) (= c "_"))))
+
+(define dl-ws? (fn (c) (or (= c " ") (= c "\t") (= c "\n") (= c "\r"))))
+
+(define
+  dl-tokenize
+  (fn
+    (src)
+    (let
+      ((tokens (list)) (pos 0) (src-len (len src)))
+      (define
+        dl-peek
+        (fn
+          (offset)
+          (if (< (+ pos offset) src-len) (nth src (+ pos offset)) nil)))
+      (define cur (fn () (dl-peek 0)))
+      (define advance! (fn (n) (set! pos (+ pos n))))
+      (define
+        at?
+        (fn
+          (s)
+          (let
+            ((sl (len s)))
+            (and (<= (+ pos sl) src-len) (= (slice src pos (+ pos sl)) s)))))
+      (define
+        dl-emit!
+        (fn
+          (type value start)
+          (append! tokens (dl-make-token type value start))))
+      (define
+        skip-line-comment!
+        (fn
+          ()
+          (when
+            (and (< pos src-len) (not (= (cur) "\n")))
+            (do (advance! 1) (skip-line-comment!)))))
+      (define
+        skip-block-comment!
+        (fn
+          ()
+          (cond
+            ((>= pos src-len)
+              (error (str "Tokenizer: unterminated block comment "
+                          "(started at position " pos ")")))
+            ((and (= (cur) "*") (< (+ pos 1) src-len) (= (dl-peek 1) "/"))
+              (advance! 2))
+            (else (do (advance! 1) (skip-block-comment!))))))
+      (define
+        skip-ws!
+        (fn
+          ()
+          (cond
+            ((>= pos src-len) nil)
+            ((dl-ws? (cur)) (do (advance! 1) (skip-ws!)))
+            ((= (cur) "%")
+              (do (advance! 1) (skip-line-comment!) (skip-ws!)))
+            ((and (= (cur) "/") (< (+ pos 1) src-len) (= (dl-peek 1) "*"))
+              (do (advance! 2) (skip-block-comment!) (skip-ws!)))
+            (else nil))))
+      (define
+        read-ident
+        (fn
+          (start)
+          (do
+            (when
+              (and (< pos src-len) (dl-ident-char? (cur)))
+              (do (advance! 1) (read-ident start)))
+            (slice src start pos))))
+      (define
+        read-decimal-digits!
+        (fn
+          ()
+          (when
+            (and (< pos src-len) (dl-digit? (cur)))
+            (do (advance! 1) (read-decimal-digits!)))))
+      (define
+        read-number
+        (fn
+          (start)
+          (do
+            (read-decimal-digits!)
+            (when
+              (and
+                (< pos src-len)
+                (= (cur) ".")
+                (< (+ pos 1) src-len)
+                (dl-digit? (dl-peek 1)))
+              (do (advance! 1) (read-decimal-digits!)))
+            (parse-number (slice src start pos)))))
+      (define
+        read-quoted
+        (fn
+          (quote-char)
+          (let
+            ((chars (list)))
+            (advance! 1)
+            (define
+              loop
+              (fn
+                ()
+                (cond
+                  ((>= pos src-len)
+                    (error
+                      (str "Tokenizer: unterminated "
+                           (if (= quote-char "'") "quoted atom" "string")
+                           " (started near position " pos ")")))
+                  ((= (cur) "\\")
+                    (do
+                      (advance! 1)
+                      (when
+                        (< pos src-len)
+                        (let
+                          ((ch (cur)))
+                          (do
+                            (cond
+                              ((= ch "n") (append! chars "\n"))
+                              ((= ch "t") (append! chars "\t"))
+                              ((= ch "r") (append! chars "\r"))
+                              ((= ch "\\") (append! chars "\\"))
+                              ((= ch "'") (append! chars "'"))
+                              ((= ch "\"") (append! chars "\""))
+                              (else (append! chars ch)))
+                            (advance! 1))))
+                      (loop)))
+                  ((= (cur) quote-char) (advance! 1))
+                  (else
+                    (do (append! chars (cur)) (advance! 1) (loop))))))
+            (loop)
+            (join "" chars))))
+      (define
+        scan!
+        (fn
+          ()
+          (do
+            (skip-ws!)
+            (when
+              (< pos src-len)
+              (let
+                ((ch (cur)) (start pos))
+                (cond
+                  ((at? ":-")
+                    (do
+                      (dl-emit! "op" ":-" start)
+                      (advance! 2)
+                      (scan!)))
+                  ((at? "?-")
+                    (do
+                      (dl-emit! "op" "?-" start)
+                      (advance! 2)
+                      (scan!)))
+                  ((at? "<=")
+                    (do
+                      (dl-emit! "op" "<=" start)
+                      (advance! 2)
+                      (scan!)))
+                  ((at? ">=")
+                    (do
+                      (dl-emit! "op" ">=" start)
+                      (advance! 2)
+                      (scan!)))
+                  ((at? "!=")
+                    (do
+                      (dl-emit! "op" "!=" start)
+                      (advance! 2)
+                      (scan!)))
+                  ((dl-digit? ch)
+                    (do
+                      (dl-emit! "number" (read-number start) start)
+                      (scan!)))
+                  ((= ch "'")
+                    ;; Quoted 'atoms' tokenize as strings so a name
+                    ;; like 'Hello World' doesn't get misclassified
+                    ;; as a variable by dl-var? (which inspects the
+                    ;; symbol's first character).
+                    (do (dl-emit! "string" (read-quoted "'") start) (scan!)))
+                  ((= ch "\"")
+                    (do (dl-emit! "string" (read-quoted "\"") start) (scan!)))
+                  ((dl-lower? ch)
+                    (do (dl-emit! "atom" (read-ident start) start) (scan!)))
+                  ((or (dl-upper? ch) (= ch "_"))
+                    (do (dl-emit! "var" (read-ident start) start) (scan!)))
+                  ((= ch "(")
+                    (do
+                      (dl-emit! "punct" "(" start)
+                      (advance! 1)
+                      (scan!)))
+                  ((= ch ")")
+                    (do
+                      (dl-emit! "punct" ")" start)
+                      (advance! 1)
+                      (scan!)))
+                  ((= ch ",")
+                    (do
+                      (dl-emit! "punct" "," start)
+                      (advance! 1)
+                      (scan!)))
+                  ((= ch ".")
+                    (do
+                      (dl-emit! "punct" "." start)
+                      (advance! 1)
+                      (scan!)))
+                  ((= ch "<")
+                    (do
+                      (dl-emit! "op" "<" start)
+                      (advance! 1)
+                      (scan!)))
+                  ((= ch ">")
+                    (do
+                      (dl-emit! "op" ">" start)
+                      (advance! 1)
+                      (scan!)))
+                  ((= ch "=")
+                    (do
+                      (dl-emit! "op" "=" start)
+                      (advance! 1)
+                      (scan!)))
+                  ((= ch "+")
+                    (do
+                      (dl-emit! "op" "+" start)
+                      (advance! 1)
+                      (scan!)))
+                  ((= ch "-")
+                    (do
+                      (dl-emit! "op" "-" start)
+                      (advance! 1)
+                      (scan!)))
+                  ((= ch "*")
+                    (do
+                      (dl-emit! "op" "*" start)
+                      (advance! 1)
+                      (scan!)))
+                  ((= ch "/")
+                    (do
+                      (dl-emit! "op" "/" start)
+                      (advance! 1)
+                      (scan!)))
+                  (else (error
+                          (str "Tokenizer: unexpected character '" ch
+                               "' at position " start)))))))))
+      (scan!)
+      (dl-emit! "eof" nil pos)
+      tokens)))

lib/datalog/unify.sx

@@ -0,0 +1,171 @@
+;; lib/datalog/unify.sx — unification + substitution for Datalog terms.
+;;
+;; Term taxonomy (after parsing):
+;;   variable  — SX symbol whose first char is uppercase A–Z or '_'.
+;;   constant  — SX symbol whose first char is lowercase a–z (atom name).
+;;   number    — numeric literal.
+;;   string    — string literal.
+;;   compound  — SX list (functor arg ... arg). In core Datalog these
+;;               only appear as arithmetic expressions (see Phase 4
+;;               safety analysis); compound-against-compound unification
+;;               is supported anyway for completeness.
+;;
+;; Substitutions are immutable dicts keyed by variable name (string).
+;; A failed unification returns nil; success returns the extended subst.
+
+(define dl-empty-subst (fn () {}))
+
+(define
+  dl-var?
+  (fn
+    (term)
+    (and
+      (symbol? term)
+      (let
+        ((name (symbol->string term)))
+        (and
+          (> (len name) 0)
+          (let
+            ((c (slice name 0 1)))
+            (or (and (>= c "A") (<= c "Z")) (= c "_"))))))))
+
+;; Walk: chase variable bindings until we hit a non-variable or an unbound
+;; variable. The result is either a non-variable term or an unbound var.
+(define
+  dl-walk
+  (fn (term subst) (dl-walk-aux term subst (list))))
+
+;; Internal: walk with a visited-var set so circular substitutions
+;; (from raw dl-bind misuse) don't infinite-loop. Cycles return the
+;; current term unchanged.
+(define
+  dl-walk-aux
+  (fn
+    (term subst visited)
+    (if
+      (dl-var? term)
+      (let
+        ((name (symbol->string term)))
+        (cond
+          ((dl-member? name visited) term)
+          ((and (dict? subst) (has-key? subst name))
+            (let ((seen (list)))
+              (do
+                (for-each (fn (v) (append! seen v)) visited)
+                (append! seen name)
+                (dl-walk-aux (get subst name) subst seen))))
+          (else term)))
+      term)))
+
+;; Bind a variable symbol to a value in subst, returning a new subst.
+(define
+  dl-bind
+  (fn (var-sym value subst) (assoc subst (symbol->string var-sym) value)))
+
+(define
+  dl-unify
+  (fn
+    (t1 t2 subst)
+    (if
+      (nil? subst)
+      nil
+      (let
+        ((u1 (dl-walk t1 subst)) (u2 (dl-walk t2 subst)))
+        (cond
+          ((dl-var? u1)
+            (cond
+              ((and (dl-var? u2) (= (symbol->string u1) (symbol->string u2)))
+                subst)
+              (else (dl-bind u1 u2 subst))))
+          ((dl-var? u2) (dl-bind u2 u1 subst))
+          ((and (list? u1) (list? u2))
+            (if
+              (= (len u1) (len u2))
+              (dl-unify-list u1 u2 subst 0)
+              nil))
+          ((and (number? u1) (number? u2)) (if (= u1 u2) subst nil))
+          ((and (string? u1) (string? u2)) (if (= u1 u2) subst nil))
+          ((and (symbol? u1) (symbol? u2))
+            (if (= (symbol->string u1) (symbol->string u2)) subst nil))
+          (else nil))))))
+
+(define
+  dl-unify-list
+  (fn
+    (a b subst i)
+    (cond
+      ((nil? subst) nil)
+      ((>= i (len a)) subst)
+      (else
+        (dl-unify-list
+          a
+          b
+          (dl-unify (nth a i) (nth b i) subst)
+          (+ i 1))))))
+
+;; Apply substitution: walk the term and recurse into lists.
+(define
+  dl-apply-subst
+  (fn
+    (term subst)
+    (let
+      ((w (dl-walk term subst)))
+      (if (list? w) (map (fn (x) (dl-apply-subst x subst)) w) w))))
+
+;; Ground? — true iff no free variables remain after walking.
+(define
+  dl-ground?
+  (fn
+    (term subst)
+    (let
+      ((w (dl-walk term subst)))
+      (cond
+        ((dl-var? w) false)
+        ((list? w) (dl-ground-list? w subst 0))
+        (else true)))))
+
+(define
+  dl-ground-list?
+  (fn
+    (xs subst i)
+    (cond
+      ((>= i (len xs)) true)
+      ((not (dl-ground? (nth xs i) subst)) false)
+      (else (dl-ground-list? xs subst (+ i 1))))))
+
+;; Return the list of variable names appearing in a term (deduped, in
+;; left-to-right order). Useful for safety analysis later.
+(define
+  dl-vars-of
+  (fn (term) (let ((seen (list))) (do (dl-vars-of-aux term seen) seen))))
+
+(define
+  dl-vars-of-aux
+  (fn
+    (term acc)
+    (cond
+      ((dl-var? term)
+        (let
+          ((name (symbol->string term)))
+          (when (not (dl-member? name acc)) (append! acc name))))
+      ((list? term) (dl-vars-of-list term acc 0))
+      (else nil))))
+
+(define
+  dl-vars-of-list
+  (fn
+    (xs acc i)
+    (when
+      (< i (len xs))
+      (do
+        (dl-vars-of-aux (nth xs i) acc)
+        (dl-vars-of-list xs acc (+ i 1))))))
+
+(define
+  dl-member?
+  (fn
+    (x xs)
+    (cond
+      ((= (len xs) 0) false)
+      ((= (first xs) x) true)
+      (else (dl-member? x (rest xs))))))

lib/guest/ast.sx

@@ -0,0 +1,92 @@
+;; lib/guest/ast.sx — canonical AST node shapes.
+;;
+;; A guest's parser may emit its own AST in whatever shape is convenient
+;; for that language's evaluator/transpiler. This file gives a SHARED
+;; canonical shape that cross-language tools (formatters, highlighters,
+;; debuggers) can target without per-language adapters.
+;;
+;; Each canonical node is a tagged list: (KIND ...payload).
+;;
+;; Constructors (return a canonical node):
+;;
+;;   (ast-literal       VALUE)               — number / string / bool / nil
+;;   (ast-var           NAME)                — identifier reference
+;;   (ast-app           FN ARGS)             — function application
+;;   (ast-lambda        PARAMS BODY)         — anonymous function
+;;   (ast-let           BINDINGS BODY)       — local bindings
+;;   (ast-letrec        BINDINGS BODY)       — recursive local bindings
+;;   (ast-if            TEST THEN ELSE)      — conditional
+;;   (ast-match-clause  PATTERN BODY)        — one match arm
+;;   (ast-module        NAME BODY)           — module declaration
+;;   (ast-import        NAME)                — import directive
+;;
+;; Predicates: (ast-literal? X), (ast-var? X), …
+;; Generic:    (ast? X)         — any canonical node
+;;             (ast-kind X)     — :literal / :var / :app / …
+;;
+;; Accessors  (one per payload field):
+;;   (ast-literal-value N)
+;;   (ast-var-name N)
+;;   (ast-app-fn N)             (ast-app-args N)
+;;   (ast-lambda-params N)      (ast-lambda-body N)
+;;   (ast-let-bindings N)       (ast-let-body N)
+;;   (ast-letrec-bindings N)    (ast-letrec-body N)
+;;   (ast-if-test N)            (ast-if-then N)         (ast-if-else N)
+;;   (ast-match-clause-pattern N)
+;;   (ast-match-clause-body N)
+;;   (ast-module-name N)        (ast-module-body N)
+;;   (ast-import-name N)
+
+(define ast-literal      (fn (v)         (list :literal v)))
+(define ast-var          (fn (n)         (list :var n)))
+(define ast-app          (fn (f args)    (list :app f args)))
+(define ast-lambda       (fn (ps body)   (list :lambda ps body)))
+(define ast-let          (fn (bs body)   (list :let bs body)))
+(define ast-letrec       (fn (bs body)   (list :letrec bs body)))
+(define ast-if           (fn (t th el)   (list :if t th el)))
+(define ast-match-clause (fn (p body)    (list :match-clause p body)))
+(define ast-module       (fn (n body)    (list :module n body)))
+(define ast-import       (fn (n)         (list :import n)))
+
+(define ast-kind (fn (x) (if (and (list? x) (not (empty? x))) (first x) nil)))
+
+(define
+  ast?
+  (fn (x)
+    (and (list? x)
+         (not (empty? x))
+         (let ((k (first x)))
+           (or (= k :literal) (= k :var) (= k :app)
+               (= k :lambda)  (= k :let) (= k :letrec)
+               (= k :if)      (= k :match-clause)
+               (= k :module)  (= k :import))))))
+
+(define ast-literal?      (fn (x) (and (ast? x) (= (first x) :literal))))
+(define ast-var?          (fn (x) (and (ast? x) (= (first x) :var))))
+(define ast-app?          (fn (x) (and (ast? x) (= (first x) :app))))
+(define ast-lambda?       (fn (x) (and (ast? x) (= (first x) :lambda))))
+(define ast-let?          (fn (x) (and (ast? x) (= (first x) :let))))
+(define ast-letrec?       (fn (x) (and (ast? x) (= (first x) :letrec))))
+(define ast-if?           (fn (x) (and (ast? x) (= (first x) :if))))
+(define ast-match-clause? (fn (x) (and (ast? x) (= (first x) :match-clause))))
+(define ast-module?       (fn (x) (and (ast? x) (= (first x) :module))))
+(define ast-import?       (fn (x) (and (ast? x) (= (first x) :import))))
+
+(define ast-literal-value (fn (n) (nth n 1)))
+(define ast-var-name      (fn (n) (nth n 1)))
+(define ast-app-fn        (fn (n) (nth n 1)))
+(define ast-app-args      (fn (n) (nth n 2)))
+(define ast-lambda-params (fn (n) (nth n 1)))
+(define ast-lambda-body   (fn (n) (nth n 2)))
+(define ast-let-bindings  (fn (n) (nth n 1)))
+(define ast-let-body      (fn (n) (nth n 2)))
+(define ast-letrec-bindings (fn (n) (nth n 1)))
+(define ast-letrec-body     (fn (n) (nth n 2)))
+(define ast-if-test       (fn (n) (nth n 1)))
+(define ast-if-then       (fn (n) (nth n 2)))
+(define ast-if-else       (fn (n) (nth n 3)))
+(define ast-match-clause-pattern (fn (n) (nth n 1)))
+(define ast-match-clause-body    (fn (n) (nth n 2)))
+(define ast-module-name   (fn (n) (nth n 1)))
+(define ast-module-body   (fn (n) (nth n 2)))
+(define ast-import-name   (fn (n) (nth n 1)))

lib/guest/hm.sx

@@ -0,0 +1,180 @@
+;; lib/guest/hm.sx — Hindley-Milner type-inference foundations.
+;;
+;; Builds on lib/guest/match.sx (terms + unify) and ast.sx (canonical
+;; AST shapes). This file ships the ALGEBRA — types, schemes, free
+;; type-vars, generalize / instantiate, substitution composition — so a
+;; full Algorithm W (or J) can be assembled on top either inside this
+;; file or in a host-specific consumer (haskell/infer.sx,
+;; lib/ocaml/types.sx, …).
+;;
+;; Per the brief the second consumer for this step is OCaml-on-SX
+;; Phase 5 (paired sequencing). Until that lands, the algebra is the
+;; deliverable; the host-flavoured assembly (lambda / app / let
+;; inference rules with substitution threading) lives in the host.
+;;
+;; Types
+;; -----
+;; A type is a canonical match.sx term — type variables use mk-var,
+;; type constructors use mk-ctor:
+;;   (hm-tv NAME)              type variable
+;;   (hm-arrow A B)             A -> B
+;;   (hm-con NAME ARGS)         named n-ary constructor
+;;   (hm-int) / (hm-bool) / (hm-string)   primitive constructors
+;;
+;; Schemes
+;; -------
+;;   (hm-scheme VARS TYPE)        ∀ VARS . TYPE
+;;   (hm-monotype TYPE)           empty quantifier
+;;   (hm-scheme? S) (hm-scheme-vars S) (hm-scheme-type S)
+;;
+;; Free type variables
+;; -------------------
+;;   (hm-ftv TYPE)                names occurring in TYPE
+;;   (hm-ftv-scheme S)            free names (minus quantifiers)
+;;   (hm-ftv-env ENV)             free across an env (name -> scheme)
+;;
+;; Substitution
+;; ------------
+;;   (hm-apply SUBST TYPE)        substitute through a type
+;;   (hm-apply-scheme SUBST S)    leaves bound vars alone
+;;   (hm-apply-env SUBST ENV)
+;;   (hm-compose S2 S1)           apply S1 then S2
+;;
+;; Generalize / Instantiate
+;; ------------------------
+;;   (hm-generalize TYPE ENV)      → scheme over ftv(t) - ftv(env)
+;;   (hm-instantiate SCHEME COUNTER) → fresh-var instance
+;;   (hm-fresh-tv COUNTER)         → (:var "tN"), bumps COUNTER
+;;
+;; Inference (literal only — the rest of Algorithm W lives in the host)
+;; --------------------------------------------------------------------
+;;   (hm-infer-literal EXPR)       → {:subst {} :type T}
+;;
+;; A complete Algorithm W consumes this kit by assembling lambda / app
+;; / let rules in the host language file.
+
+(define hm-tv     (fn (name) (list :var name)))
+(define hm-con    (fn (name args) (list :ctor name args)))
+(define hm-arrow  (fn (a b) (hm-con "->" (list a b))))
+(define hm-int    (fn () (hm-con "Int" (list))))
+(define hm-bool   (fn () (hm-con "Bool" (list))))
+(define hm-string (fn () (hm-con "String" (list))))
+
+(define hm-scheme        (fn (vars t) (list :scheme vars t)))
+(define hm-monotype      (fn (t) (hm-scheme (list) t)))
+(define hm-scheme?       (fn (s) (and (list? s) (not (empty? s)) (= (first s) :scheme))))
+(define hm-scheme-vars   (fn (s) (nth s 1)))
+(define hm-scheme-type   (fn (s) (nth s 2)))
+
+(define
+  hm-fresh-tv
+  (fn (counter)
+    (let ((n (first counter)))
+      (begin
+        (set-nth! counter 0 (+ n 1))
+        (hm-tv (str "t" (+ n 1)))))))
+
+(define
+  hm-ftv-acc
+  (fn (t acc)
+    (cond
+      ((is-var? t)
+        (if (some (fn (n) (= n (var-name t))) acc) acc (cons (var-name t) acc)))
+      ((is-ctor? t)
+        (let ((a acc))
+          (begin
+            (for-each (fn (x) (set! a (hm-ftv-acc x a))) (ctor-args t))
+            a)))
+      (:else acc))))
+
+(define hm-ftv (fn (t) (hm-ftv-acc t (list))))
+
+(define
+  hm-ftv-scheme
+  (fn (s)
+    (let ((qs (hm-scheme-vars s))
+          (all (hm-ftv (hm-scheme-type s))))
+      (filter (fn (n) (not (some (fn (q) (= q n)) qs))) all))))
+
+(define
+  hm-ftv-env
+  (fn (env)
+    (let ((acc (list)))
+      (begin
+        (for-each
+          (fn (k)
+            (for-each
+              (fn (n)
+                (when (not (some (fn (m) (= m n)) acc))
+                  (set! acc (cons n acc))))
+              (hm-ftv-scheme (get env k))))
+          (keys env))
+        acc))))
+
+(define hm-apply (fn (subst t) (walk* t subst)))
+
+(define
+  hm-apply-scheme
+  (fn (subst s)
+    (let ((qs (hm-scheme-vars s))
+          (d {}))
+      (begin
+        (for-each
+          (fn (k)
+            (when (not (some (fn (q) (= q k)) qs))
+              (dict-set! d k (get subst k))))
+          (keys subst))
+        (hm-scheme qs (walk* (hm-scheme-type s) d))))))
+
+(define
+  hm-apply-env
+  (fn (subst env)
+    (let ((d {}))
+      (begin
+        (for-each
+          (fn (k) (dict-set! d k (hm-apply-scheme subst (get env k))))
+          (keys env))
+        d))))
+
+(define
+  hm-compose
+  (fn (s2 s1)
+    (let ((d {}))
+      (begin
+        (for-each (fn (k) (dict-set! d k (walk* (get s1 k) s2))) (keys s1))
+        (for-each
+          (fn (k) (when (not (has-key? d k)) (dict-set! d k (get s2 k))))
+          (keys s2))
+        d))))
+
+(define
+  hm-generalize
+  (fn (t env)
+    (let ((tvars (hm-ftv t))
+          (evars (hm-ftv-env env)))
+      (let ((qs (filter (fn (n) (not (some (fn (m) (= m n)) evars))) tvars)))
+        (hm-scheme qs t)))))
+
+(define
+  hm-instantiate
+  (fn (s counter)
+    (let ((qs (hm-scheme-vars s))
+          (subst {}))
+      (begin
+        (for-each
+          (fn (q) (set! subst (assoc subst q (hm-fresh-tv counter))))
+          qs)
+        (walk* (hm-scheme-type s) subst)))))
+
+;; Literal inference — the only AST kind whose typing rule is closed
+;; in the kit. Lambda / app / let live in host code so the host's own
+;; AST conventions stay untouched.
+(define
+  hm-infer-literal
+  (fn (expr)
+    (let ((v (ast-literal-value expr)))
+      (cond
+        ((number? v)  {:subst {} :type (hm-int)})
+        ((string? v)  {:subst {} :type (hm-string)})
+        ((boolean? v) {:subst {} :type (hm-bool)})
+        (:else (error (str "hm-infer-literal: unknown kind: " v)))))))

lib/guest/layout.sx

@@ -0,0 +1,145 @@
+;; lib/guest/layout.sx — configurable off-side / layout-sensitive lexer.
+;;
+;; Inserts virtual open / close / separator tokens based on indentation.
+;; Configurable enough to encode either the Haskell 98 layout rule (let /
+;; where / do / of opens a virtual brace at the next token's column) or
+;; a Python-ish indent / dedent rule (a colon at the end of a line opens
+;; a block at the next non-blank line's column).
+;;
+;; Token shape (input + output)
+;; ----------------------------
+;; Each token is a dict {:type :value :line :col …}. The kit reads
+;; only :type / :value / :line / :col and passes everything else
+;; through. The input stream MUST be free of newline filler tokens
+;; (preprocess them away with your tokenizer) — line breaks are detected
+;; by comparing :line of consecutive tokens.
+;;
+;; Config
+;; ------
+;;   :open-keywords    list of strings; a token whose :value matches
+;;                     opens a new layout block at the next token's
+;;                     column (Haskell: let/where/do/of).
+;;   :open-trailing-fn (fn (tok) -> bool) — alternative trigger that
+;;                     fires AFTER the token is emitted. Use for
+;;                     Python-style trailing `:`.
+;;   :open-token / :close-token / :sep-token
+;;                     templates {:type :value} merged with :line and
+;;                     :col when virtual tokens are emitted.
+;;   :explicit-open?   (fn (tok) -> bool) — if the next token after a
+;;                     trigger satisfies this, suppress virtual layout
+;;                     for that block (Haskell: `{`).
+;;   :module-prelude?  if true, wrap whole input in an implicit block
+;;                     at the first token's column (Haskell yes,
+;;                     Python no).
+;;
+;; Public entry
+;; ------------
+;;   (layout-pass cfg tokens) -> tokens with virtual layout inserted.
+
+(define
+  layout-mk-virtual
+  (fn (template line col)
+    (assoc (assoc template :line line) :col col)))
+
+(define
+  layout-is-open-kw?
+  (fn (tok open-kws)
+    (and (= (get tok :type) "reserved")
+         (some (fn (k) (= k (get tok :value))) open-kws))))
+
+(define
+  layout-pass
+  (fn (cfg tokens)
+    (let ((open-kws (get cfg :open-keywords))
+          (trailing-fn (get cfg :open-trailing-fn))
+          (open-tmpl (get cfg :open-token))
+          (close-tmpl (get cfg :close-token))
+          (sep-tmpl (get cfg :sep-token))
+          (mod-prelude? (get cfg :module-prelude?))
+          (expl?-fn (get cfg :explicit-open?))
+          (out (list))
+          (stack (list))
+          (n (len tokens))
+          (i 0)
+          (prev-line -1)
+          (pending-open false)
+          (just-opened false))
+      (define
+        emit-closes-while-greater
+        (fn (col line)
+          (when (and (not (empty? stack)) (> (first stack) col))
+            (do
+              (append! out (layout-mk-virtual close-tmpl line col))
+              (set! stack (rest stack))
+              (emit-closes-while-greater col line)))))
+      (define
+        emit-pending-open
+        (fn (line col)
+          (do
+            (append! out (layout-mk-virtual open-tmpl line col))
+            (set! stack (cons col stack))
+            (set! pending-open false)
+            (set! just-opened true))))
+      (define
+        layout-step
+        (fn ()
+          (when (< i n)
+            (let ((tok (nth tokens i)))
+              (let ((line (get tok :line)) (col (get tok :col)))
+                (cond
+                  (pending-open
+                    (cond
+                      ((and (not (= expl?-fn nil)) (expl?-fn tok))
+                        (do
+                          (set! pending-open false)
+                          (append! out tok)
+                          (set! prev-line line)
+                          (set! i (+ i 1))
+                          (layout-step)))
+                      (:else
+                        (do
+                          (emit-pending-open line col)
+                          (layout-step)))))
+                  (:else
+                    (let ((on-fresh-line? (and (> prev-line 0) (> line prev-line))))
+                      (do
+                        (when on-fresh-line?
+                          (let ((stack-before stack))
+                            (begin
+                              (emit-closes-while-greater col line)
+                              (when (and (not (empty? stack))
+                                         (= (first stack) col)
+                                         (not just-opened)
+                                         ;; suppress separator if a dedent fired
+                                         ;; — the dedent is itself the separator
+                                         (= (len stack) (len stack-before)))
+                                (append! out (layout-mk-virtual sep-tmpl line col))))))
+                        (set! just-opened false)
+                        (append! out tok)
+                        (set! prev-line line)
+                        (set! i (+ i 1))
+                        (cond
+                          ((layout-is-open-kw? tok open-kws)
+                            (set! pending-open true))
+                          ((and (not (= trailing-fn nil)) (trailing-fn tok))
+                            (set! pending-open true)))
+                        (layout-step))))))))))
+      (begin
+        ;; Module prelude: implicit layout block at the first token's column.
+        (when (and mod-prelude? (> n 0))
+          (let ((tok (nth tokens 0)))
+            (do
+              (append! out (layout-mk-virtual open-tmpl (get tok :line) (get tok :col)))
+              (set! stack (cons (get tok :col) stack))
+              (set! just-opened true))))
+        (layout-step)
+        ;; EOF: close every remaining block.
+        (define close-rest
+          (fn ()
+            (when (not (empty? stack))
+              (do
+                (append! out (layout-mk-virtual close-tmpl 0 0))
+                (set! stack (rest stack))
+                (close-rest)))))
+        (close-rest)
+        out))))

lib/guest/match.sx

@@ -0,0 +1,185 @@
+;; lib/guest/match.sx — pure pattern-match + unification kit.
+;;
+;; Shipped for miniKanren / Datalog / future logic-flavoured guests that
+;; want immutable unification without writing it from scratch. The two
+;; existing prolog/haskell engines stay as-is — porting them in place
+;; risks the 746 tests they currently pass; consumers can migrate
+;; gradually via the converters in lib/guest/ast.sx.
+;;
+;; Term shapes (canonical wire format)
+;; -----------------------------------
+;;   var          (:var NAME)            NAME a string
+;;   constructor  (:ctor HEAD ARGS)      HEAD a string, ARGS a list of terms
+;;   literal      number / string / boolean / nil
+;;
+;; Guests with their own shape pass adapter callbacks via the cfg arg —
+;; see (unify-with cfg ...) and (match-pat-with cfg ...) below. The
+;; default canonical entry points (unify / match-pat) use the wire shape.
+;;
+;; Substitution / env
+;; ------------------
+;; A substitution is a SX dict mapping VAR-NAME → term. There are no
+;; trails, no mutation: each step either returns an extended dict or nil.
+;;
+;;   (empty-subst)                  → {}
+;;   (walk term s)                  → term with top-level vars resolved
+;;   (walk* term s)                 → term with all vars resolved (recursive)
+;;   (extend name term s)           → s with NAME → term added
+;;   (occurs? name term s)          → bool
+;;
+;; Unify (symmetric, miniKanren-flavour)
+;; -------------------------------------
+;;   (unify u v s)                  → extended subst or nil
+;;   (unify-with cfg u v s)         → ditto, with adapter callbacks:
+;;                                      :var? :var-name :ctor? :ctor-head
+;;                                      :ctor-args :occurs-check?
+;;
+;; Match (asymmetric, haskell-flavour: pattern → value, vars only in pat)
+;; ---------------------------------------------------------------------
+;;   (match-pat pat val env)        → extended env or nil
+;;   (match-pat-with cfg pat val env)
+
+(define mk-var  (fn (name) (list :var name)))
+(define mk-ctor (fn (head args) (list :ctor head args)))
+
+(define is-var?    (fn (t) (and (list? t) (not (empty? t)) (= (first t) :var))))
+(define is-ctor?   (fn (t) (and (list? t) (not (empty? t)) (= (first t) :ctor))))
+(define var-name   (fn (t) (nth t 1)))
+(define ctor-head  (fn (t) (nth t 1)))
+(define ctor-args  (fn (t) (nth t 2)))
+
+(define empty-subst (fn () {}))
+
+(define
+  walk
+  (fn (t s)
+    (if (and (is-var? t) (has-key? s (var-name t)))
+      (walk (get s (var-name t)) s)
+      t)))
+
+(define
+  walk*
+  (fn (t s)
+    (let ((w (walk t s)))
+      (cond
+        ((is-ctor? w)
+          (mk-ctor (ctor-head w) (map (fn (a) (walk* a s)) (ctor-args w))))
+        (:else w)))))
+
+(define
+  extend
+  (fn (name term s)
+    (assoc s name term)))
+
+(define
+  occurs?
+  (fn (name term s)
+    (let ((w (walk term s)))
+      (cond
+        ((is-var? w) (= (var-name w) name))
+        ((is-ctor? w) (some (fn (a) (occurs? name a s)) (ctor-args w)))
+        (:else false)))))
+
+(define
+  unify-with
+  (fn (cfg u v s)
+    (let ((var?-fn (get cfg :var?))
+          (var-name-fn (get cfg :var-name))
+          (ctor?-fn (get cfg :ctor?))
+          (ctor-head-fn (get cfg :ctor-head))
+          (ctor-args-fn (get cfg :ctor-args))
+          (occurs?-on (get cfg :occurs-check?)))
+      (let ((wu (walk-with cfg u s))
+            (wv (walk-with cfg v s)))
+        (cond
+          ((and (var?-fn wu) (var?-fn wv) (= (var-name-fn wu) (var-name-fn wv))) s)
+          ((var?-fn wu)
+            (if (and occurs?-on (occurs-with cfg (var-name-fn wu) wv s))
+              nil
+              (extend (var-name-fn wu) wv s)))
+          ((var?-fn wv)
+            (if (and occurs?-on (occurs-with cfg (var-name-fn wv) wu s))
+              nil
+              (extend (var-name-fn wv) wu s)))
+          ((and (ctor?-fn wu) (ctor?-fn wv))
+            (if (= (ctor-head-fn wu) (ctor-head-fn wv))
+              (unify-list-with
+                cfg
+                (ctor-args-fn wu)
+                (ctor-args-fn wv)
+                s)
+              nil))
+          (:else (if (= wu wv) s nil)))))))
+
+(define
+  walk-with
+  (fn (cfg t s)
+    (if (and ((get cfg :var?) t) (has-key? s ((get cfg :var-name) t)))
+      (walk-with cfg (get s ((get cfg :var-name) t)) s)
+      t)))
+
+(define
+  occurs-with
+  (fn (cfg name term s)
+    (let ((w (walk-with cfg term s)))
+      (cond
+        (((get cfg :var?) w) (= ((get cfg :var-name) w) name))
+        (((get cfg :ctor?) w)
+          (some (fn (a) (occurs-with cfg name a s)) ((get cfg :ctor-args) w)))
+        (:else false)))))
+
+(define
+  unify-list-with
+  (fn (cfg xs ys s)
+    (cond
+      ((and (empty? xs) (empty? ys)) s)
+      ((or (empty? xs) (empty? ys)) nil)
+      (:else
+        (let ((s2 (unify-with cfg (first xs) (first ys) s)))
+          (if (= s2 nil)
+            nil
+            (unify-list-with cfg (rest xs) (rest ys) s2)))))))
+
+(define canonical-cfg
+  {:var? is-var? :var-name var-name
+   :ctor? is-ctor? :ctor-head ctor-head :ctor-args ctor-args
+   :occurs-check? true})
+
+(define unify (fn (u v s) (unify-with canonical-cfg u v s)))
+
+;; Asymmetric pattern match (haskell-style): only patterns may contain vars;
+;; values are concrete. On a var pattern, bind name to value.
+(define
+  match-pat-with
+  (fn (cfg pat val env)
+    (let ((var?-fn (get cfg :var?))
+          (var-name-fn (get cfg :var-name))
+          (ctor?-fn (get cfg :ctor?))
+          (ctor-head-fn (get cfg :ctor-head))
+          (ctor-args-fn (get cfg :ctor-args)))
+      (cond
+        ((var?-fn pat) (extend (var-name-fn pat) val env))
+        ((and (ctor?-fn pat) (ctor?-fn val))
+          (if (= (ctor-head-fn pat) (ctor-head-fn val))
+            (match-list-pat-with
+              cfg
+              (ctor-args-fn pat)
+              (ctor-args-fn val)
+              env)
+            nil))
+        ((ctor?-fn pat) nil)
+        (:else (if (= pat val) env nil))))))
+
+(define
+  match-list-pat-with
+  (fn (cfg pats vals env)
+    (cond
+      ((and (empty? pats) (empty? vals)) env)
+      ((or (empty? pats) (empty? vals)) nil)
+      (:else
+        (let ((env2 (match-pat-with cfg (first pats) (first vals) env)))
+          (if (= env2 nil)
+            nil
+            (match-list-pat-with cfg (rest pats) (rest vals) env2)))))))
+
+(define match-pat (fn (pat val env) (match-pat-with canonical-cfg pat val env)))

lib/guest/pratt.sx

@@ -0,0 +1,28 @@
+;; lib/guest/pratt.sx — operator-table format + lookup for Pratt-style
+;; precedence climbing.
+;;
+;; The climbing loop stays per-language because the two canaries use
+;; opposite conventions (Lua: higher prec = tighter; Prolog: lower prec =
+;; tighter, with xfx/xfy/yfx assoc tags). Forcing a single loop adds
+;; callback indirection that obscures more than it shares.
+;;
+;; What IS shared and gets extracted: the operator-table format and lookup.
+;; "Grammar is a dict, not hardcoded cond."
+;;
+;; Entry shape: (NAME PREC ASSOC).
+;;   NAME   — string, the operator's source token.
+;;   PREC   — integer, in the host's own convention.
+;;   ASSOC  — :left | :right | :none for languages with traditional
+;;            associativity, or "xfx" / "xfy" / "yfx" for Prolog-style.
+
+(define
+  pratt-op-lookup
+  (fn (table name)
+    (cond
+      ((empty? table) nil)
+      ((= (first (first table)) name) (first table))
+      (:else (pratt-op-lookup (rest table) name)))))
+
+(define pratt-op-name  (fn (entry) (first entry)))
+(define pratt-op-prec  (fn (entry) (nth entry 1)))
+(define pratt-op-assoc (fn (entry) (nth entry 2)))

lib/guest/tests/ast.sx

@@ -0,0 +1,63 @@
+;; lib/guest/tests/ast.sx — exercises every constructor / predicate /
+;; accessor in lib/guest/ast.sx so future ports have a stable contract
+;; to point at.
+
+(define gast-test-pass 0)
+(define gast-test-fail 0)
+(define gast-test-fails (list))
+
+(define
+  gast-test
+  (fn (name actual expected)
+    (if (= actual expected)
+      (set! gast-test-pass (+ gast-test-pass 1))
+      (begin
+        (set! gast-test-fail (+ gast-test-fail 1))
+        (append! gast-test-fails {:name name :expected expected :actual actual})))))
+
+;; Constructors round-trip.
+(gast-test "literal-int"    (ast-literal-value (ast-literal 42)) 42)
+(gast-test "literal-str"    (ast-literal-value (ast-literal "hi")) "hi")
+(gast-test "literal-bool"   (ast-literal-value (ast-literal true)) true)
+(gast-test "var-name"       (ast-var-name (ast-var "x")) "x")
+(gast-test "app-fn"         (ast-app-fn (ast-app (ast-var "f") (list (ast-literal 1)))) (ast-var "f"))
+(gast-test "app-args-len"   (len (ast-app-args (ast-app (ast-var "f") (list (ast-literal 1))))) 1)
+(gast-test "lambda-params"  (ast-lambda-params (ast-lambda (list "x" "y") (ast-var "x"))) (list "x" "y"))
+(gast-test "lambda-body"    (ast-lambda-body (ast-lambda (list "x") (ast-var "x"))) (ast-var "x"))
+(gast-test "let-bindings"   (len (ast-let-bindings (ast-let (list {:name "x" :value (ast-literal 1)}) (ast-var "x")))) 1)
+(gast-test "letrec-body"    (ast-letrec-body (ast-letrec (list) (ast-literal 0))) (ast-literal 0))
+(gast-test "if-test"        (ast-if-test (ast-if (ast-literal true) (ast-literal 1) (ast-literal 0))) (ast-literal true))
+(gast-test "if-then"        (ast-if-then (ast-if (ast-literal true) (ast-literal 1) (ast-literal 0))) (ast-literal 1))
+(gast-test "if-else"        (ast-if-else (ast-if (ast-literal true) (ast-literal 1) (ast-literal 0))) (ast-literal 0))
+(gast-test "match-pattern"  (ast-match-clause-pattern (ast-match-clause "P" (ast-literal 1))) "P")
+(gast-test "match-body"     (ast-match-clause-body (ast-match-clause "P" (ast-literal 1))) (ast-literal 1))
+(gast-test "module-name"    (ast-module-name (ast-module "m" (list))) "m")
+(gast-test "import-name"    (ast-import-name (ast-import "lib/foo")) "lib/foo")
+
+;; Predicates fire only on matching kinds.
+(gast-test "is-literal"     (ast-literal? (ast-literal 1)) true)
+(gast-test "not-literal"    (ast-literal? (ast-var "x")) false)
+(gast-test "is-var"         (ast-var? (ast-var "x")) true)
+(gast-test "is-app"         (ast-app? (ast-app (ast-var "f") (list))) true)
+(gast-test "is-lambda"      (ast-lambda? (ast-lambda (list) (ast-literal 0))) true)
+(gast-test "is-let"         (ast-let? (ast-let (list) (ast-literal 0))) true)
+(gast-test "is-letrec"      (ast-letrec? (ast-letrec (list) (ast-literal 0))) true)
+(gast-test "is-if"          (ast-if? (ast-if (ast-literal true) (ast-literal 1) (ast-literal 0))) true)
+(gast-test "is-match"       (ast-match-clause? (ast-match-clause "P" (ast-literal 1))) true)
+(gast-test "is-module"      (ast-module? (ast-module "m" (list))) true)
+(gast-test "is-import"      (ast-import? (ast-import "x")) true)
+
+;; ast? recognises any canonical node.
+(gast-test "ast?-literal"   (ast? (ast-literal 0)) true)
+(gast-test "ast?-foreign"   (ast? (list "lua-num" 0)) false)
+(gast-test "ast?-non-list"  (ast? 42) false)
+
+;; ast-kind dispatch.
+(gast-test "kind-literal"   (ast-kind (ast-literal 0)) :literal)
+(gast-test "kind-import"    (ast-kind (ast-import "x")) :import)
+
+(define gast-tests-run!
+  (fn ()
+    {:passed gast-test-pass
+     :failed gast-test-fail
+     :total  (+ gast-test-pass gast-test-fail)}))

lib/guest/tests/hm.sx

@@ -0,0 +1,89 @@
+;; lib/guest/tests/hm.sx — exercises lib/guest/hm.sx algebra.
+
+(define ghm-test-pass 0)
+(define ghm-test-fail 0)
+(define ghm-test-fails (list))
+
+(define
+  ghm-test
+  (fn (name actual expected)
+    (if (= actual expected)
+      (set! ghm-test-pass (+ ghm-test-pass 1))
+      (begin
+        (set! ghm-test-fail (+ ghm-test-fail 1))
+        (append! ghm-test-fails {:name name :expected expected :actual actual})))))
+
+;; ── Type constructors ─────────────────────────────────────────────
+(ghm-test "tv"           (hm-tv "a") (list :var "a"))
+(ghm-test "int"          (hm-int)    (list :ctor "Int" (list)))
+(ghm-test "arrow"        (ctor-head (hm-arrow (hm-int) (hm-bool))) "->")
+(ghm-test "arrow-args-len" (len (ctor-args (hm-arrow (hm-int) (hm-bool)))) 2)
+
+;; ── Schemes ───────────────────────────────────────────────────────
+(ghm-test "scheme-vars"  (hm-scheme-vars (hm-scheme (list "a") (hm-tv "a"))) (list "a"))
+(ghm-test "monotype-vars" (hm-scheme-vars (hm-monotype (hm-int))) (list))
+(ghm-test "scheme?-yes"  (hm-scheme? (hm-monotype (hm-int))) true)
+(ghm-test "scheme?-no"   (hm-scheme? (hm-int)) false)
+
+;; ── Fresh tyvars ──────────────────────────────────────────────────
+(ghm-test "fresh-1"
+  (let ((c (list 0))) (var-name (hm-fresh-tv c))) "t1")
+(ghm-test "fresh-bumps"
+  (let ((c (list 5))) (begin (hm-fresh-tv c) (first c))) 6)
+
+;; ── Free type variables ──────────────────────────────────────────
+(ghm-test "ftv-int"      (hm-ftv (hm-int)) (list))
+(ghm-test "ftv-tv"       (hm-ftv (hm-tv "a")) (list "a"))
+(ghm-test "ftv-arrow"
+  (len (hm-ftv (hm-arrow (hm-tv "a") (hm-arrow (hm-tv "b") (hm-tv "a"))))) 2)
+(ghm-test "ftv-scheme-quantified"
+  (hm-ftv-scheme (hm-scheme (list "a") (hm-arrow (hm-tv "a") (hm-tv "b")))) (list "b"))
+(ghm-test "ftv-env"
+  (let ((env (assoc {} "f" (hm-monotype (hm-arrow (hm-tv "x") (hm-tv "y"))))))
+    (len (hm-ftv-env env))) 2)
+
+;; ── Substitution / apply / compose ───────────────────────────────
+(ghm-test "apply-tv"
+  (hm-apply (assoc {} "a" (hm-int)) (hm-tv "a")) (hm-int))
+(ghm-test "apply-arrow"
+  (ctor-head
+    (hm-apply (assoc {} "a" (hm-int))
+              (hm-arrow (hm-tv "a") (hm-tv "b")))) "->")
+(ghm-test "compose-1-then-2"
+  (var-name
+    (hm-apply
+      (hm-compose (assoc {} "b" (hm-tv "c")) (assoc {} "a" (hm-tv "b")))
+      (hm-tv "a"))) "c")
+
+;; ── Generalize / Instantiate ─────────────────────────────────────
+;;   forall a. a -> a   instantiated twice yields fresh vars each time
+(ghm-test "generalize-id"
+  (len (hm-scheme-vars (hm-generalize (hm-arrow (hm-tv "a") (hm-tv "a")) {}))) 1)
+
+(ghm-test "generalize-skips-env"
+  ;; ftv(t)={a,b}, ftv(env)={a}, qs={b}
+  (let ((env (assoc {} "x" (hm-monotype (hm-tv "a")))))
+    (len (hm-scheme-vars
+           (hm-generalize (hm-arrow (hm-tv "a") (hm-tv "b")) env)))) 1)
+
+(ghm-test "instantiate-fresh"
+  (let ((s (hm-scheme (list "a") (hm-arrow (hm-tv "a") (hm-tv "a"))))
+        (c (list 0)))
+    (let ((t1 (hm-instantiate s c)) (t2 (hm-instantiate s c)))
+      (not (= (var-name (first (ctor-args t1)))
+              (var-name (first (ctor-args t2)))))))
+  true)
+
+;; ── Inference (literal only) ─────────────────────────────────────
+(ghm-test "infer-int"
+  (ctor-head (get (hm-infer-literal (ast-literal 42)) :type)) "Int")
+(ghm-test "infer-string"
+  (ctor-head (get (hm-infer-literal (ast-literal "hi")) :type)) "String")
+(ghm-test "infer-bool"
+  (ctor-head (get (hm-infer-literal (ast-literal true)) :type)) "Bool")
+
+(define ghm-tests-run!
+  (fn ()
+    {:passed ghm-test-pass
+     :failed ghm-test-fail
+     :total  (+ ghm-test-pass ghm-test-fail)}))

lib/guest/tests/layout.sx

@@ -0,0 +1,180 @@
+;; lib/guest/tests/layout.sx — synthetic Python-ish off-side fixture.
+;;
+;; Exercises lib/guest/layout.sx with a config different from Haskell's
+;; (no module-prelude, layout opens via trailing `:` not via reserved
+;; keyword) to prove the kit isn't Haskell-shaped.
+
+(define glayout-test-pass 0)
+(define glayout-test-fail 0)
+(define glayout-test-fails (list))
+
+(define
+  glayout-test
+  (fn (name actual expected)
+    (if (= actual expected)
+      (set! glayout-test-pass (+ glayout-test-pass 1))
+      (begin
+        (set! glayout-test-fail (+ glayout-test-fail 1))
+        (append! glayout-test-fails {:name name :expected expected :actual actual})))))
+
+;; Convenience: build a token from {type value line col}.
+(define
+  glayout-tok
+  (fn (ty val line col)
+    {:type ty :value val :line line :col col}))
+
+;; Project a token list to ((type value) ...) for compact comparison.
+(define
+  glayout-shape
+  (fn (toks)
+    (map (fn (t) (list (get t :type) (get t :value))) toks)))
+
+;; ── Haskell-flavour: keyword opens block ─────────────────────────
+(define
+  glayout-haskell-cfg
+  {:open-keywords (list "let" "where" "do" "of")
+   :open-trailing-fn nil
+   :open-token   {:type "vlbrace" :value "{"}
+   :close-token  {:type "vrbrace" :value "}"}
+   :sep-token    {:type "vsemi"   :value ";"}
+   :module-prelude? false
+   :explicit-open? (fn (tok) (= (get tok :type) "lbrace"))})
+
+;;   do
+;;     a
+;;     b
+;;   c       ← outside the do-block
+(glayout-test "haskell-do-block"
+  (glayout-shape
+    (layout-pass
+      glayout-haskell-cfg
+      (list (glayout-tok "reserved" "do" 1 1)
+            (glayout-tok "ident" "a" 2 3)
+            (glayout-tok "ident" "b" 3 3)
+            (glayout-tok "ident" "c" 4 1))))
+  (list (list "reserved" "do")
+        (list "vlbrace" "{")
+        (list "ident" "a")
+        (list "vsemi" ";")
+        (list "ident" "b")
+        (list "vrbrace" "}")
+        (list "ident" "c")))
+
+;; Explicit `{` after `do` suppresses virtual layout.
+(glayout-test "haskell-explicit-brace"
+  (glayout-shape
+    (layout-pass
+      glayout-haskell-cfg
+      (list (glayout-tok "reserved" "do" 1 1)
+            (glayout-tok "lbrace" "{" 1 4)
+            (glayout-tok "ident" "a" 1 6)
+            (glayout-tok "rbrace" "}" 1 8))))
+  (list (list "reserved" "do")
+        (list "lbrace" "{")
+        (list "ident" "a")
+        (list "rbrace" "}")))
+
+;; Single-statement do-block on the same line.
+(glayout-test "haskell-do-inline"
+  (glayout-shape
+    (layout-pass
+      glayout-haskell-cfg
+      (list (glayout-tok "reserved" "do" 1 1)
+            (glayout-tok "ident" "a" 1 4))))
+  (list (list "reserved" "do")
+        (list "vlbrace" "{")
+        (list "ident" "a")
+        (list "vrbrace" "}")))
+
+;; Module-prelude: wrap whole input in implicit layout block at first
+;; tok's column.
+(glayout-test "haskell-module-prelude"
+  (glayout-shape
+    (layout-pass
+      (assoc glayout-haskell-cfg :module-prelude? true)
+      (list (glayout-tok "ident" "x" 1 1)
+            (glayout-tok "ident" "y" 2 1)
+            (glayout-tok "ident" "z" 3 1))))
+  (list (list "vlbrace" "{")
+        (list "ident" "x")
+        (list "vsemi" ";")
+        (list "ident" "y")
+        (list "vsemi" ";")
+        (list "ident" "z")
+        (list "vrbrace" "}")))
+
+;; ── Python-flavour: trailing `:` opens block ─────────────────────
+(define
+  glayout-python-cfg
+  {:open-keywords (list)
+   :open-trailing-fn (fn (tok) (and (= (get tok :type) "punct")
+                                    (= (get tok :value) ":")))
+   :open-token   {:type "indent" :value "INDENT"}
+   :close-token  {:type "dedent" :value "DEDENT"}
+   :sep-token    {:type "newline" :value "NEWLINE"}
+   :module-prelude? false
+   :explicit-open? nil})
+
+;;   if x:
+;;       a
+;;       b
+;;   c
+(glayout-test "python-if-block"
+  (glayout-shape
+    (layout-pass
+      glayout-python-cfg
+      (list (glayout-tok "reserved" "if" 1 1)
+            (glayout-tok "ident" "x" 1 4)
+            (glayout-tok "punct" ":" 1 5)
+            (glayout-tok "ident" "a" 2 5)
+            (glayout-tok "ident" "b" 3 5)
+            (glayout-tok "ident" "c" 4 1))))
+  (list (list "reserved" "if")
+        (list "ident" "x")
+        (list "punct" ":")
+        (list "indent" "INDENT")
+        (list "ident" "a")
+        (list "newline" "NEWLINE")
+        (list "ident" "b")
+        (list "dedent" "DEDENT")
+        (list "ident" "c")))
+
+;; Nested Python-style blocks.
+;;   def f():
+;;       if x:
+;;           a
+;;       b
+(glayout-test "python-nested"
+  (glayout-shape
+    (layout-pass
+      glayout-python-cfg
+      (list (glayout-tok "reserved" "def" 1 1)
+            (glayout-tok "ident" "f" 1 5)
+            (glayout-tok "punct" "(" 1 6)
+            (glayout-tok "punct" ")" 1 7)
+            (glayout-tok "punct" ":" 1 8)
+            (glayout-tok "reserved" "if" 2 5)
+            (glayout-tok "ident" "x" 2 8)
+            (glayout-tok "punct" ":" 2 9)
+            (glayout-tok "ident" "a" 3 9)
+            (glayout-tok "ident" "b" 4 5))))
+  (list (list "reserved" "def")
+        (list "ident" "f")
+        (list "punct" "(")
+        (list "punct" ")")
+        (list "punct" ":")
+        (list "indent" "INDENT")
+        (list "reserved" "if")
+        (list "ident" "x")
+        (list "punct" ":")
+        (list "indent" "INDENT")
+        (list "ident" "a")
+        (list "dedent" "DEDENT")
+        (list "ident" "b")
+        (list "dedent" "DEDENT")))
+
+(define glayout-tests-run!
+  (fn ()
+    {:passed glayout-test-pass
+     :failed glayout-test-fail
+     :total  (+ glayout-test-pass glayout-test-fail)}))

lib/guest/tests/match.sx

@@ -0,0 +1,108 @@
+;; lib/guest/tests/match.sx — exercises lib/guest/match.sx.
+
+(define gmatch-test-pass 0)
+(define gmatch-test-fail 0)
+(define gmatch-test-fails (list))
+
+(define
+  gmatch-test
+  (fn (name actual expected)
+    (if (= actual expected)
+      (set! gmatch-test-pass (+ gmatch-test-pass 1))
+      (begin
+        (set! gmatch-test-fail (+ gmatch-test-fail 1))
+        (append! gmatch-test-fails {:name name :expected expected :actual actual})))))
+
+;; ── walk / extend / occurs ────────────────────────────────────────
+(gmatch-test "walk-direct"
+  (walk (mk-var "x") (extend "x" 5 (empty-subst))) 5)
+
+(gmatch-test "walk-chain"
+  (walk (mk-var "a") (extend "a" (mk-var "b") (extend "b" 7 (empty-subst)))) 7)
+
+(gmatch-test "walk-no-binding"
+  (let ((v (mk-var "u"))) (= (walk v (empty-subst)) v)) true)
+
+(gmatch-test "walk*-recursive"
+  (walk* (mk-ctor "Just" (list (mk-var "x"))) (extend "x" 9 (empty-subst)))
+  (mk-ctor "Just" (list 9)))
+
+(gmatch-test "occurs-direct"
+  (occurs? "x" (mk-var "x") (empty-subst)) true)
+
+(gmatch-test "occurs-nested"
+  (occurs? "x" (mk-ctor "f" (list (mk-var "x"))) (empty-subst)) true)
+
+(gmatch-test "occurs-not"
+  (occurs? "x" (mk-var "y") (empty-subst)) false)
+
+;; ── unify (symmetric) ─────────────────────────────────────────────
+(gmatch-test "unify-equal-literals"
+  (len (unify 5 5 (empty-subst))) 0)
+
+(gmatch-test "unify-different-literals"
+  (unify 5 6 (empty-subst)) nil)
+
+(gmatch-test "unify-var-literal"
+  (get (unify (mk-var "x") 5 (empty-subst)) "x") 5)
+
+(gmatch-test "unify-literal-var"
+  (get (unify 5 (mk-var "x") (empty-subst)) "x") 5)
+
+(gmatch-test "unify-same-var"
+  (len (unify (mk-var "x") (mk-var "x") (empty-subst))) 0)
+
+(gmatch-test "unify-two-vars"
+  (let ((s (unify (mk-var "x") (mk-var "y") (empty-subst))))
+    (or (= (get s "x") (mk-var "y")) (= (get s "y") (mk-var "x")))) true)
+
+(gmatch-test "unify-ctor-equal"
+  (len (unify (mk-ctor "f" (list 1 2)) (mk-ctor "f" (list 1 2)) (empty-subst))) 0)
+
+(gmatch-test "unify-ctor-with-var"
+  (get (unify (mk-ctor "Just" (list (mk-var "x"))) (mk-ctor "Just" (list 7)) (empty-subst)) "x") 7)
+
+(gmatch-test "unify-ctor-head-mismatch"
+  (unify (mk-ctor "Just" (list 1)) (mk-ctor "Nothing" (list)) (empty-subst)) nil)
+
+(gmatch-test "unify-ctor-arity-mismatch"
+  (unify (mk-ctor "f" (list 1 2)) (mk-ctor "f" (list 1)) (empty-subst)) nil)
+
+(gmatch-test "unify-occurs-check"
+  (unify (mk-var "x") (mk-ctor "f" (list (mk-var "x"))) (empty-subst)) nil)
+
+(gmatch-test "unify-transitive-vars"
+  (let ((s (unify (mk-var "x") (mk-var "y") (empty-subst))))
+    (let ((s2 (unify (mk-var "y") 42 s)))
+      (walk (mk-var "x") s2))) 42)
+
+;; ── match-pat (asymmetric) ────────────────────────────────────────
+(gmatch-test "match-var-binds"
+  (get (match-pat (mk-var "x") 99 (empty-subst)) "x") 99)
+
+(gmatch-test "match-literal-equal"
+  (len (match-pat 5 5 (empty-subst))) 0)
+
+(gmatch-test "match-literal-mismatch"
+  (match-pat 5 6 (empty-subst)) nil)
+
+(gmatch-test "match-ctor-binds"
+  (get (match-pat (mk-ctor "Just" (list (mk-var "y")))
+                  (mk-ctor "Just" (list 11))
+                  (empty-subst)) "y") 11)
+
+(gmatch-test "match-ctor-head-mismatch"
+  (match-pat (mk-ctor "Just" (list (mk-var "y")))
+             (mk-ctor "Nothing" (list))
+             (empty-subst)) nil)
+
+(gmatch-test "match-ctor-arity-mismatch"
+  (match-pat (mk-ctor "f" (list (mk-var "x") (mk-var "y")))
+             (mk-ctor "f" (list 1))
+             (empty-subst)) nil)
+
+(define gmatch-tests-run!
+  (fn ()
+    {:passed gmatch-test-pass
+     :failed gmatch-test-fail
+     :total  (+ gmatch-test-pass gmatch-test-fail)}))

lib/lua/parser.sx

@@ -3,28 +3,33 @@
 (define lua-tok-value (fn (t) (if (= t nil) nil (get t :value))))
 
 (define
-  lua-binop-prec
+  lua-op-table
-  (fn
+  (list
-    (op)
+    (list "or"  1 :left)
-    (cond
+    (list "and" 2 :left)
-      ((= op "or") 1)
+    (list "<"   3 :left)
-      ((= op "and") 2)
+    (list ">"   3 :left)
-      ((= op "<") 3)
+    (list "<="  3 :left)
-      ((= op ">") 3)
+    (list ">="  3 :left)
-      ((= op "<=") 3)
+    (list "=="  3 :left)
-      ((= op ">=") 3)
+    (list "~="  3 :left)
-      ((= op "==") 3)
+    (list ".."  5 :right)
-      ((= op "~=") 3)
+    (list "+"   6 :left)
-      ((= op "..") 5)
+    (list "-"   6 :left)
-      ((= op "+") 6)
+    (list "*"   7 :left)
-      ((= op "-") 6)
+    (list "/"   7 :left)
-      ((= op "*") 7)
+    (list "%"   7 :left)
-      ((= op "/") 7)
+    (list "^"  10 :right)))
-      ((= op "%") 7)
-      ((= op "^") 10)
-      (else 0))))
 
-(define lua-binop-right? (fn (op) (or (= op "..") (= op "^"))))
+(define lua-binop-prec
+  (fn (op)
+    (let ((entry (pratt-op-lookup lua-op-table op)))
+      (if (= entry nil) 0 (pratt-op-prec entry)))))
+
+(define lua-binop-right?
+  (fn (op)
+    (let ((entry (pratt-op-lookup lua-op-table op)))
+      (and (not (= entry nil)) (= (pratt-op-assoc entry) :right)))))
 
 (define
   lua-parse

lib/lua/test.sh

@@ -30,6 +30,7 @@ cat > "$TMPFILE" << 'EPOCHS'
 (epoch 1)
 (load "lib/guest/lex.sx")
 (load "lib/guest/prefix.sx")
+(load "lib/guest/pratt.sx")
 (load "lib/lua/tokenizer.sx")
 (epoch 2)
 (load "lib/lua/parser.sx")

lib/prolog/conformance.conf

@@ -4,6 +4,7 @@ LANG_NAME=prolog
 MODE=dict
 
 PRELOADS=(
+  lib/guest/pratt.sx
   lib/prolog/tokenizer.sx
   lib/prolog/parser.sx
   lib/prolog/runtime.sx

lib/prolog/parser.sx

@@ -104,18 +104,9 @@
     (list ":-" 1200 "xfx")
     (list "mod" 400 "yfx")))
 
-(define
+(define pl-op-lookup (fn (name) (pratt-op-lookup pl-op-table name)))
-  pl-op-find
-  (fn
-    (name table)
-    (cond
-      ((empty? table) nil)
-      ((= (first (first table)) name) (rest (first table)))
-      (true (pl-op-find name (rest table))))))
 
-(define pl-op-lookup (fn (name) (pl-op-find name pl-op-table)))
+;; Token → entry (name prec type) for known infix ops, else nil.
-
-;; Token → (name prec type) for known infix ops, else nil.
 (define
   pl-token-op
   (fn
@@ -123,14 +114,8 @@
     (let
       ((ty (get t :type)) (vv (get t :value)))
       (cond
-        ((and (= ty "punct") (= vv ","))
+        ((and (= ty "punct") (= vv ",")) (pl-op-lookup ","))
-          (let
+        ((or (= ty "atom") (= ty "op")) (pl-op-lookup vv))
-            ((info (pl-op-lookup ",")))
-            (if (nil? info) nil (cons "," info))))
-        ((or (= ty "atom") (= ty "op"))
-          (let
-            ((info (pl-op-lookup vv)))
-            (if (nil? info) nil (cons vv info))))
         (true nil)))))
 
 ;; ── Term parser ─────────────────────────────────────────────────────

lib/prolog/scoreboard.json

@@ -3,5 +3,5 @@
   "total_failed": 0,
   "total": 590,
   "suites": {"parse":{"passed":25,"total":25,"failed":0},"unify":{"passed":47,"total":47,"failed":0},"clausedb":{"passed":14,"total":14,"failed":0},"solve":{"passed":62,"total":62,"failed":0},"operators":{"passed":19,"total":19,"failed":0},"dynamic":{"passed":11,"total":11,"failed":0},"findall":{"passed":11,"total":11,"failed":0},"term_inspect":{"passed":14,"total":14,"failed":0},"append":{"passed":6,"total":6,"failed":0},"reverse":{"passed":6,"total":6,"failed":0},"member":{"passed":7,"total":7,"failed":0},"nqueens":{"passed":6,"total":6,"failed":0},"family":{"passed":10,"total":10,"failed":0},"atoms":{"passed":34,"total":34,"failed":0},"query_api":{"passed":16,"total":16,"failed":0},"iso_predicates":{"passed":29,"total":29,"failed":0},"meta_predicates":{"passed":25,"total":25,"failed":0},"list_predicates":{"passed":33,"total":33,"failed":0},"meta_call":{"passed":15,"total":15,"failed":0},"set_predicates":{"passed":15,"total":15,"failed":0},"char_predicates":{"passed":27,"total":27,"failed":0},"io_predicates":{"passed":24,"total":24,"failed":0},"assert_rules":{"passed":15,"total":15,"failed":0},"string_agg":{"passed":25,"total":25,"failed":0},"advanced":{"passed":21,"total":21,"failed":0},"compiler":{"passed":17,"total":17,"failed":0},"cross_validate":{"passed":17,"total":17,"failed":0},"integration":{"passed":20,"total":20,"failed":0},"hs_bridge":{"passed":19,"total":19,"failed":0}},
-  "generated": "2026-05-06T22:23:38+00:00"
+  "generated": "2026-05-07T17:35:23+00:00"
 }

lib/prolog/scoreboard.md

@@ -1,7 +1,7 @@
 # Prolog scoreboard
 
 **590 / 590 passing** (0 failure(s)).
-Generated 2026-05-06T22:23:38+00:00.
+Generated 2026-05-07T17:35:23+00:00.
 
 | Suite | Passed | Total | Status |
 |-------|--------|-------|--------|

lib/tcl/runtime.sx

@@ -292,13 +292,15 @@
                             (> (len result-stack) caller-stack-len)
                             (nth result-stack caller-stack-len)
                             (get interp :frame))))
-                      (assoc interp
+                      ; Forward result-interp as base so state changes inside
+                      ; the proc (e.g. :fileevents, :timers, :procs) propagate;
+                      ; restore caller's frame/stack/result/output/code.
+                      (assoc result-interp
                         :frame updated-caller
                         :frame-stack updated-below
                         :result result-val
                         :output (str caller-output proc-output)
-                        :code (if (= code 2) 0 code)
+                        :code (if (= code 2) 0 code))))))))))))))
-                        :commands (get result-interp :commands))))))))))))))
 
 (define
   tcl-eval-cmd
@@ -354,14 +356,33 @@
   (fn
     (interp args)
     (let
-      ((text (last args))
+      ((no-nl (and (> (len args) 1) (equal? (first args) "-nonewline"))))
-        (no-nl
-          (and
-            (> (len args) 1)
-            (equal? (first args) "-nonewline"))))
       (let
-        ((line (if no-nl text (str text "\n"))))
+        ((args2 (if no-nl (rest args) args)))
-        (assoc interp :output (str (get interp :output) line))))))
+        (let
+          ((maybe-chan (if (> (len args2) 1) (first args2) nil))
+            (is-chan
+              (and
+                (not (nil? maybe-chan))
+                (or
+                  (and
+                    (>= (len maybe-chan) 4)
+                    (equal? (slice maybe-chan 0 4) "file"))
+                  (and
+                    (>= (len maybe-chan) 4)
+                    (equal? (slice maybe-chan 0 4) "sock"))))))
+          (if
+            is-chan
+            (let
+              ((chan (first args2))
+                (text (last args2))
+                (line (if no-nl text (str text "\n"))))
+              (let
+                ((_ (channel-write chan line)))
+                (assoc interp :result "")))
+            (let
+              ((text (last args2)) (line (if no-nl text (str text "\n"))))
+              (assoc interp :output (str (get interp :output) line)))))))))
 
 (define
   tcl-cmd-incr
@@ -2868,36 +2889,433 @@
           ((equal? sub "seconds") (assoc interp :result (str (clock-seconds))))
           ((equal? sub "milliseconds") (assoc interp :result (str (clock-milliseconds))))
           ((equal? sub "format")
-            (assoc interp :result (clock-format
+            ; clock format $secs ?-format $fmt? ?-timezone $tz? ?-gmt 0|1?
-              (floor (parse-int (first rest-args)))
+            (let
-              (if (> (len rest-args) 1) (nth rest-args (- (len rest-args) 1)) "%a %b %e %H:%M:%S %Z %Y"))))
+              ((t (floor (parse-int (first rest-args))))
-          ((equal? sub "scan") (assoc interp :result "0"))
+                (opts (rest rest-args)))
+              (let
+                ((fmt (tcl-clock-opt opts "-format" "%a %b %e %H:%M:%S %Z %Y"))
+                  (tz (tcl-clock-tz opts)))
+                (assoc interp :result (clock-format t fmt tz)))))
+          ((equal? sub "scan")
+            ; clock scan $str ?-format $fmt? ?-timezone $tz? ?-gmt 0|1?
+            (let
+              ((s (first rest-args)) (opts (rest rest-args)))
+              (let
+                ((fmt (tcl-clock-opt opts "-format" "%Y-%m-%d %H:%M:%S"))
+                  (tz (tcl-clock-tz opts)))
+                (assoc interp :result (str (clock-scan s fmt tz))))))
           (else (error (str "clock: unknown subcommand \"" sub "\""))))))))
 
-(define tcl-cmd-open (fn (interp args) (assoc interp :result "file0")))
+; Helper: extract a -flag $val pair from clock args.
+(define
+  tcl-clock-opt
+  (fn
+    (opts flag default)
+    (cond
+      ((< (len opts) 2) default)
+      ((equal? (first opts) flag) (nth opts 1))
+      (else (tcl-clock-opt (rest (rest opts)) flag default)))))
+
+; Helper: derive tz string from clock opts (-timezone or -gmt).
+(define
+  tcl-clock-tz
+  (fn
+    (opts)
+    (let
+      ((tz-explicit (tcl-clock-opt opts "-timezone" nil))
+        (gmt-flag (tcl-clock-opt opts "-gmt" nil)))
+      (cond
+        ((not (nil? tz-explicit))
+          (cond
+            ((equal? tz-explicit ":UTC") "utc")
+            ((equal? tz-explicit "UTC") "utc")
+            ((equal? tz-explicit "GMT") "utc")
+            (else "local")))
+        ((equal? gmt-flag "1") "utc")
+        ((equal? gmt-flag "true") "utc")
+        ((not (nil? gmt-flag)) "local")
+        (else "utc")))))
+
+(define
+  tcl-cmd-open
+  (fn
+    (interp args)
+    (let
+      ((path (first args))
+        (mode (if (> (len args) 1) (nth args 1) "r")))
+      (assoc interp :result (channel-open path mode)))))
 
 ; gets channel ?varname?
-(define tcl-cmd-close (fn (interp args) (assoc interp :result "")))
+(define
+  tcl-cmd-close
+  (fn
+    (interp args)
+    (let ((_ (channel-close (first args)))) (assoc interp :result ""))))
 
-(define tcl-cmd-read (fn (interp args) (assoc interp :result "")))
+(define
+  tcl-cmd-read
+  (fn
+    (interp args)
+    (let
+      ((chan (first args))
+        (n (if (> (len args) 1) (parse-int (nth args 1)) -1)))
+      (assoc
+        interp
+        :result (if (< n 0) (channel-read chan) (channel-read chan n))))))
 
 (define
   tcl-cmd-gets-chan
   (fn
     (interp args)
-    (if
+    (let
-      (> (len args) 1)
+      ((chan (first args)) (line (channel-read-line chan)))
-      (assoc (tcl-var-set interp (nth args 1) "") :result "-1")
+      (if
-      (assoc interp :result ""))))
+        (nil? line)
+        (if
+          (> (len args) 1)
+          (assoc (tcl-var-set interp (nth args 1) "") :result "-1")
+          (assoc interp :result ""))
+        (if
+          (> (len args) 1)
+          (assoc
+            (tcl-var-set interp (nth args 1) line)
+            :result (str (len line)))
+          (assoc interp :result line))))))
 
-(define tcl-cmd-eof (fn (interp args) (assoc interp :result "1")))
+(define
+  tcl-cmd-eof
+  (fn
+    (interp args)
+    (assoc interp :result (if (channel-eof? (first args)) "1" "0"))))
 
-(define tcl-cmd-seek (fn (interp args) (assoc interp :result "")))
+(define
+  tcl-cmd-seek
+  (fn
+    (interp args)
+    (let
+      ((chan (first args))
+        (off (parse-int (nth args 1)))
+        (whence (if (> (len args) 2) (nth args 2) "start")))
+      (let ((_ (channel-seek chan off whence))) (assoc interp :result "")))))
 
 ; file command dispatcher
-(define tcl-cmd-tell (fn (interp args) (assoc interp :result "0")))
+(define
+  tcl-cmd-tell
+  (fn
+    (interp args)
+    (assoc interp :result (str (channel-tell (first args))))))
+
+(define
+  tcl-cmd-flush
+  (fn
+    (interp args)
+    (let ((_ (channel-flush (first args)))) (assoc interp :result ""))))
+(define
+  tcl-cmd-fconfigure
+  (fn
+    (interp args)
+    (let
+      ((chan (first args)) (rest-args (rest args)))
+      (cond
+        ((= 0 (len rest-args))
+          (assoc
+            interp
+            :result (str "-blocking " (if (channel-blocking? chan) "1" "0"))))
+        ((and
+           (= 2 (len rest-args))
+           (equal? (first rest-args) "-blocking"))
+          (let
+            ((b (nth rest-args 1)))
+            (let
+              ((_
+                  (channel-set-blocking!
+                    chan
+                    (not (or (equal? b "0") (equal? b "false"))))))
+              (assoc interp :result ""))))
+        ((and
+           (= 1 (len rest-args))
+           (equal? (first rest-args) "-blocking"))
+          (assoc interp :result (if (channel-blocking? chan) "1" "0")))
+        ((and
+           (= 1 (len rest-args))
+           (equal? (first rest-args) "-error"))
+          (assoc interp :result (channel-async-error chan)))
+        (else (assoc interp :result ""))))))
+
+
+; ============================================================
+; Event loop: fileevent / after / vwait / update (Phase 5b)
+; ============================================================
+
+; :fileevents is list of (chan event script) tuples
+; :timers is list of (expiry-ms script) tuples, sorted ascending by expiry
+
+(define
+  tcl-fileevent-set
+  (fn
+    (interp chan event script)
+    (let
+      ((existing (or (get interp :fileevents) (list))))
+      (let
+        ((filtered
+            (filter
+              (fn (e) (not (and (equal? (first e) chan) (equal? (nth e 1) event))))
+              existing)))
+        (let
+          ((new-list
+              (if (equal? script "")
+                filtered
+                (append filtered (list (list chan event script))))))
+          (assoc interp :fileevents new-list))))))
+
+(define
+  tcl-fileevent-get
+  (fn
+    (interp chan event)
+    (let
+      ((events (or (get interp :fileevents) (list))))
+      (let
+        ((matches
+            (filter
+              (fn (e) (and (equal? (first e) chan) (equal? (nth e 1) event)))
+              events)))
+        (if (= 0 (len matches)) "" (nth (first matches) 2))))))
+
+(define
+  tcl-timer-insert
+  (fn
+    (timers new-timer)
+    (cond
+      ((= 0 (len timers)) (list new-timer))
+      ((<= (first new-timer) (first (first timers))) (cons new-timer timers))
+      (else (cons (first timers) (tcl-timer-insert (rest timers) new-timer))))))
+
+(define
+  tcl-timer-add
+  (fn
+    (interp ms script)
+    (let
+      ((expiry (+ (clock-milliseconds) ms)))
+      (let
+        ((existing (or (get interp :timers) (list))))
+        (assoc interp :timers (tcl-timer-insert existing (list expiry script)))))))
+
+; Run one iteration of the event loop.
+; poll-timeout-ms: -1 = block indefinitely, 0 = poll, N>0 = wait up to N ms.
+; Returns updated interp.
+(define
+  tcl-event-step
+  (fn
+    (interp poll-timeout-ms)
+    (let
+      ((timers (or (get interp :timers) (list))) (now-ms (clock-milliseconds)))
+      (let
+        ((expired (filter (fn (t) (<= (first t) now-ms)) timers))
+          (remaining (filter (fn (t) (> (first t) now-ms)) timers)))
+        (let
+          ((interp1
+              (reduce
+                (fn (acc t) (tcl-eval-string acc (nth t 1)))
+                (assoc interp :timers remaining)
+                expired)))
+          (let
+            ((events (or (get interp1 :fileevents) (list))))
+            (let
+              ((read-chans
+                  (map
+                    (fn (e) (first e))
+                    (filter (fn (e) (equal? (nth e 1) "readable")) events)))
+                (write-chans
+                  (map
+                    (fn (e) (first e))
+                    (filter (fn (e) (equal? (nth e 1) "writable")) events)))
+                (next-timer-delta
+                  (if
+                    (= 0 (len remaining))
+                    -1
+                    (- (first (first remaining)) (clock-milliseconds)))))
+              (let
+                ((effective-timeout
+                    (cond
+                      ((and (>= poll-timeout-ms 0) (>= next-timer-delta 0))
+                        (min poll-timeout-ms next-timer-delta))
+                      ((>= poll-timeout-ms 0) poll-timeout-ms)
+                      ((>= next-timer-delta 0) next-timer-delta)
+                      (else -1))))
+                (if
+                  (and
+                    (= 0 (len read-chans))
+                    (= 0 (len write-chans)))
+                  ; nothing to select on; if timeout > 0, do a no-op wait via select
+                  (if
+                    (> effective-timeout 0)
+                    (let
+                      ((_ (io-select-channels (list) (list) effective-timeout)))
+                      interp1)
+                    interp1)
+                  (let
+                    ((select-result
+                        (io-select-channels read-chans write-chans effective-timeout)))
+                    (let
+                      ((ready-r (or (get select-result :readable) (list)))
+                        (ready-w (or (get select-result :writable) (list))))
+                      (let
+                        ((interp2
+                            (reduce
+                              (fn (acc chan)
+                                (let
+                                  ((script (tcl-fileevent-get acc chan "readable")))
+                                  (if (equal? script "") acc (tcl-eval-string acc script))))
+                              interp1
+                              ready-r)))
+                        (reduce
+                          (fn (acc chan)
+                            (let
+                              ((script (tcl-fileevent-get acc chan "writable")))
+                              (if (equal? script "") acc (tcl-eval-string acc script))))
+                          interp2
+                          ready-w)))))))))))))
+
+(define
+  tcl-cmd-fileevent
+  (fn
+    (interp args)
+    (let
+      ((chan (first args)) (event (nth args 1)))
+      (if
+        (= 2 (len args))
+        (assoc interp :result (tcl-fileevent-get interp chan event))
+        (let
+          ((script (nth args 2)))
+          (assoc (tcl-fileevent-set interp chan event script) :result ""))))))
+
+(define
+  tcl-cmd-after
+  (fn
+    (interp args)
+    (if
+      (= 0 (len args))
+      (error "after: wrong # args")
+      (let
+        ((ms (parse-int (first args))))
+        (if
+          (= 1 (len args))
+          ; pure sleep — drive event loop until ms elapsed
+          (let
+            ((target-ms (+ (clock-milliseconds) ms)))
+            (assoc (tcl-after-sleep-loop interp target-ms) :result ""))
+          ; schedule timer
+          (let
+            ((script (join " " (rest args))))
+            (assoc (tcl-timer-add interp ms script) :result "")))))))
+
+(define
+  tcl-after-sleep-loop
+  (fn
+    (interp target-ms)
+    (let
+      ((now (clock-milliseconds)))
+      (if
+        (>= now target-ms)
+        interp
+        (tcl-after-sleep-loop
+          (tcl-event-step interp (- target-ms now))
+          target-ms)))))
+
+(define
+  tcl-cmd-vwait
+  (fn
+    (interp args)
+    (if
+      (= 0 (len args))
+      (error "vwait: wrong # args")
+      (let
+        ((name (first args)))
+        (let
+          ((initial (frame-lookup (get interp :frame) name)))
+          (assoc (tcl-vwait-loop interp name initial) :result ""))))))
+
+(define
+  tcl-vwait-loop
+  (fn
+    (interp name initial)
+    (let
+      ((cur (frame-lookup (get interp :frame) name)))
+      (if
+        (and (not (nil? cur)) (not (equal? cur initial)))
+        interp
+        (tcl-vwait-loop (tcl-event-step interp 1000) name initial)))))
+
+(define
+  tcl-cmd-update
+  (fn
+    (interp args)
+    (assoc (tcl-event-step interp 0) :result "")))
+
+; ============================================================
+; Socket: TCP client and server (Phase 5c)
+; ============================================================
+
+; Internal command invoked by the auto-registered fileevent on a server
+; channel. Args: (server-chan callback-word ...).  Accepts one client and
+; calls the user callback with (client-chan peer-host peer-port).
+(define
+  tcl-cmd-_sock-do-accept
+  (fn
+    (interp args)
+    (let
+      ((server-chan (first args)) (cb-parts (rest args)))
+      (let
+        ((info (socket-accept server-chan)))
+        (let
+          ((client-chan (get info :channel))
+            (peer-host (get info :host))
+            (peer-port (str (get info :port))))
+          (let
+            ((cmd
+                (join
+                  " "
+                  (append
+                    cb-parts
+                    (list client-chan peer-host peer-port)))))
+            (assoc (tcl-eval-string interp cmd) :result "")))))))
+
+; socket host port           — TCP client; returns "sockN"
+; socket -server cb port     — TCP server; auto-fires cb on each accept
+(define
+  tcl-cmd-socket
+  (fn
+    (interp args)
+    (cond
+      ((= 0 (len args)) (error "socket: wrong # args"))
+      ((equal? (first args) "-server")
+        (if
+          (< (len args) 3)
+          (error "socket: usage: socket -server cb port")
+          (let
+            ((cb (nth args 1)) (port (parse-int (nth args 2))))
+            (let
+              ((server-chan (socket-server port)))
+              (let
+                ((handler (str "_sock-do-accept " server-chan " " cb)))
+                (assoc
+                  (tcl-fileevent-set interp server-chan "readable" handler)
+                  :result server-chan))))))
+      ((equal? (first args) "-async")
+        (if
+          (< (len args) 3)
+          (error "socket: usage: socket -async host port")
+          (let
+            ((host (nth args 1)) (port (parse-int (nth args 2))))
+            (assoc interp :result (socket-connect-async host port)))))
+      ((= 2 (len args))
+        (let
+          ((host (first args)) (port (parse-int (nth args 1))))
+          (assoc interp :result (socket-connect host port))))
+      (else (error "socket: wrong # args")))))
+
 
-(define tcl-cmd-flush (fn (interp args) (assoc interp :result "")))
 (define
   tcl-cmd-array
   (fn
@@ -2909,11 +3327,16 @@
         ((sub (first args)) (rest-args (rest args)))
         (cond
           ((equal? sub "get")
-            (if (= 0 (len rest-args))
+            (if
+              (= 0 (len rest-args))
               (error "array get: wrong # args")
               (let
                 ((arr-name (first rest-args))
-                  (pattern (if (> (len rest-args) 1) (nth rest-args 1) nil)))
+                  (pattern
+                    (if
+                      (> (len rest-args) 1)
+                      (nth rest-args 1)
+                      nil)))
                 (let
                   ((prefix (str arr-name "("))
                     (locals (get (get interp :frame) :locals)))
@@ -2922,21 +3345,20 @@
                     (let
                       ((arr-keys (filter (fn (k) (tcl-starts-with? k prefix)) (keys locals))))
                       (let
-                        ((filtered
+                        ((filtered (if (nil? pattern) arr-keys (filter (fn (k) (let ((kn (substring k pl (- (string-length k) 1)))) (tcl-glob-match (split pattern "") (split kn "")))) arr-keys))))
-                            (if
+                        (assoc
-                              (nil? pattern)
+                          interp
-                              arr-keys
+                          :result (join
-                              (filter
+                            " "
-                                (fn (k)
-                                  (let ((kn (substring k pl (- (string-length k) 1))))
-                                    (tcl-glob-match (split pattern "") (split kn ""))))
-                                arr-keys))))
-                        (assoc interp :result
-                          (join " "
                             (reduce
-                              (fn (acc k)
+                              (fn
-                                (let ((kn (substring k pl (- (string-length k) 1))))
+                                (acc k)
-                                  (append acc (list kn) (list (get locals k)))))
+                                (let
+                                  ((kn (substring k pl (- (string-length k) 1))))
+                                  (append
+                                    acc
+                                    (list kn)
+                                    (list (get locals k)))))
                               (list)
                               filtered))))))))))
           ((equal? sub "set")
@@ -2954,7 +3376,8 @@
                     (assoc acc :result "")
                     (loop
                       (rest (rest pairs))
-                      (tcl-var-set acc
+                      (tcl-var-set
+                        acc
                         (str arr-name "(" (first pairs) ")")
                         (nth pairs 1))))))))
           ((equal? sub "names")
@@ -2963,7 +3386,11 @@
               (error "array names: wrong # args")
               (let
                 ((arr-name (first rest-args))
-                  (pattern (if (> (len rest-args) 1) (nth rest-args 1) nil)))
+                  (pattern
+                    (if
+                      (> (len rest-args) 1)
+                      (nth rest-args 1)
+                      nil)))
                 (let
                   ((prefix (str arr-name "("))
                     (locals (get (get interp :frame) :locals)))
@@ -2972,17 +3399,19 @@
                     (let
                       ((arr-keys (filter (fn (k) (tcl-starts-with? k prefix)) (keys locals))))
                       (let
-                        ((filtered
+                        ((filtered (if (nil? pattern) arr-keys (filter (fn (k) (let ((kn (substring k pl (- (string-length k) 1)))) (tcl-glob-match (split pattern "") (split kn "")))) arr-keys))))
-                            (if
+                        (assoc
-                              (nil? pattern)
+                          interp
-                              arr-keys
+                          :result (join
-                              (filter
+                            " "
-                                (fn (k)
+                            (map
-                                  (let ((kn (substring k pl (- (string-length k) 1))))
+                              (fn
-                                    (tcl-glob-match (split pattern "") (split kn ""))))
+                                (k)
-                                arr-keys))))
+                                (substring
-                        (assoc interp :result
+                                  k
-                          (join " " (map (fn (k) (substring k pl (- (string-length k) 1))) filtered))))))))))
+                                  pl
+                                  (- (string-length k) 1)))
+                              filtered))))))))))
           ((equal? sub "size")
             (if
               (= 0 (len rest-args))
@@ -2990,8 +3419,13 @@
               (let
                 ((prefix (str (first rest-args) "("))
                   (locals (get (get interp :frame) :locals)))
-                (assoc interp :result
+                (assoc
-                  (str (len (filter (fn (k) (tcl-starts-with? k prefix)) (keys locals))))))))
+                  interp
+                  :result (str
+                    (len
+                      (filter
+                        (fn (k) (tcl-starts-with? k prefix))
+                        (keys locals))))))))
           ((equal? sub "exists")
             (if
               (= 0 (len rest-args))
@@ -2999,44 +3433,39 @@
               (let
                 ((prefix (str (first rest-args) "("))
                   (locals (get (get interp :frame) :locals)))
-                (assoc interp :result
+                (assoc
-                  (if (> (len (filter (fn (k) (tcl-starts-with? k prefix)) (keys locals))) 0) "1" "0")))))
+                  interp
+                  :result (if
+                    (>
+                      (len
+                        (filter
+                          (fn (k) (tcl-starts-with? k prefix))
+                          (keys locals)))
+                      0)
+                    "1"
+                    "0")))))
           ((equal? sub "unset")
             (if
               (= 0 (len rest-args))
               (error "array unset: wrong # args")
               (let
                 ((arr-name (first rest-args))
-                  (pattern (if (> (len rest-args) 1) (nth rest-args 1) nil)))
+                  (pattern
+                    (if
+                      (> (len rest-args) 1)
+                      (nth rest-args 1)
+                      nil)))
                 (let
                   ((prefix (str arr-name "("))
                     (locals (get (get interp :frame) :locals)))
                   (let
                     ((pl (string-length prefix)))
                     (let
-                      ((to-delete
+                      ((to-delete (filter (fn (k) (if (tcl-starts-with? k prefix) (if (nil? pattern) true (let ((kn (substring k pl (- (string-length k) 1)))) (tcl-glob-match (split pattern "") (split kn "")))) false)) (keys locals))))
-                          (filter
-                            (fn (k)
-                              (if
-                                (tcl-starts-with? k prefix)
-                                (if
-                                  (nil? pattern)
-                                  true
-                                  (let ((kn (substring k pl (- (string-length k) 1))))
-                                    (tcl-glob-match (split pattern "") (split kn ""))))
-                                false))
-                            (keys locals))))
                       (let
-                        ((new-locals
+                        ((new-locals (reduce (fn (acc k) (if (contains? to-delete k) acc (assoc acc k (get locals k)))) {} (keys locals))))
-                            (reduce
+                        (assoc
-                              (fn (acc k)
+                          interp
-                                (if
-                                  (contains? to-delete k)
-                                  acc
-                                  (assoc acc k (get locals k))))
-                              {}
-                              (keys locals))))
-                        (assoc interp
                           :frame (assoc (get interp :frame) :locals new-locals)
                           :result ""))))))))
           (else (error (str "array: unknown subcommand \"" sub "\""))))))))
@@ -3048,7 +3477,7 @@
     (interp args)
     (if
       (< (len args) 1)
-      (error "apply: wrong # args: should be "apply lambdaList ?arg ...?"")
+      (error "apply: wrong # args: should be " apply lambdaList ?arg ...? "")
       (let
         ((func-list (tcl-list-split (first args)))
           (call-args (rest args)))
@@ -3058,90 +3487,122 @@
           (let
             ((param-spec (first func-list))
               (body (nth func-list 1))
-              (ns (if (> (len func-list) 2) (nth func-list 2) nil)))
+              (ns
+                (if
+                  (> (len func-list) 2)
+                  (nth func-list 2)
+                  nil)))
             (let
               ((proc-def {:args param-spec :body body :ns ns}))
               (tcl-call-proc interp "#apply" proc-def call-args))))))))
 
-
 (define
   tcl-cmd-regexp
   (fn
     (interp args)
-    (define parse-flags
+    (define
-      (fn (as nocase? all? inline?)
+      parse-flags
-        (if (= 0 (len as))
+      (fn
-          {:nocase nocase? :all all? :inline inline? :rest as}
+        (as nocase? all? inline?)
+        (if
+          (= 0 (len as))
+          {:rest as :nocase nocase? :inline inline? :all all?}
           (cond
-            ((equal? (first as) "-nocase") (parse-flags (rest as) true all? inline?))
+            ((equal? (first as) "-nocase")
-            ((equal? (first as) "-all")    (parse-flags (rest as) nocase? true inline?))
+              (parse-flags (rest as) true all? inline?))
-            ((equal? (first as) "-inline") (parse-flags (rest as) nocase? all? true))
+            ((equal? (first as) "-all")
-            (else {:nocase nocase? :all all? :inline inline? :rest as})))))
+              (parse-flags (rest as) nocase? true inline?))
-    (let ((pf (parse-flags args false false false)))
+            ((equal? (first as) "-inline")
-      (let ((nocase (get pf :nocase))
+              (parse-flags (rest as) nocase? all? true))
-            (all-mode (get pf :all))
+            (else {:rest as :nocase nocase? :inline inline? :all all?})))))
-            (inline-mode (get pf :inline))
+    (let
-            (ra (get pf :rest)))
+      ((pf (parse-flags args false false false)))
-        (if (< (len ra) 2)
+      (let
+        ((nocase (get pf :nocase))
+          (all-mode (get pf :all))
+          (inline-mode (get pf :inline))
+          (ra (get pf :rest)))
+        (if
+          (< (len ra) 2)
           (error "regexp: wrong # args")
-          (let ((pattern (first ra))
+          (let
-                (str-val (nth ra 1))
+            ((pattern (first ra))
-                (var-args (if (> (len ra) 2) (rest (rest ra)) (list))))
+              (str-val (nth ra 1))
-            (let ((re (make-regexp pattern (if nocase "i" ""))))
+              (var-args
-              (if all-mode
+                (if (> (len ra) 2) (rest (rest ra)) (list))))
-                (assoc interp :result (str (len (regexp-match-all re str-val))))
+            (let
-                (if inline-mode
+              ((re (make-regexp pattern (if nocase "i" ""))))
-                  (assoc interp :result (join " " (map (fn (m) (get m :match)) (regexp-match-all re str-val))))
+              (if
-                  (let ((m (regexp-match re str-val)))
+                all-mode
-                    (if (nil? m)
+                (assoc
+                  interp
+                  :result (str (len (regexp-match-all re str-val))))
+                (if
+                  inline-mode
+                  (assoc
+                    interp
+                    :result (join
+                      " "
+                      (map
+                        (fn (m) (get m :match))
+                        (regexp-match-all re str-val))))
+                  (let
+                    ((m (regexp-match re str-val)))
+                    (if
+                      (nil? m)
                       (assoc interp :result "0")
-                      (let ((interp2
+                      (let
-                              (if (> (len var-args) 0)
+                        ((interp2 (if (> (len var-args) 0) (tcl-var-set interp (first var-args) (get m :match)) interp)))
-                                (tcl-var-set interp (first var-args) (get m :match))
+                        (let
-                                interp)))
+                          ((interp3 (let loop ((vi 1) (gs (get m :groups)) (acc interp2)) (if (or (= 0 (len gs)) (>= vi (len var-args))) acc (loop (+ vi 1) (rest gs) (tcl-var-set acc (nth var-args vi) (first gs)))))))
-                        (let ((interp3
-                                (let loop ((vi 1) (gs (get m :groups)) (acc interp2))
-                                  (if (or (= 0 (len gs)) (>= vi (len var-args))) acc
-                                    (loop (+ vi 1) (rest gs)
-                                          (tcl-var-set acc (nth var-args vi) (first gs)))))))
                           (assoc interp3 :result "1"))))))))))))))
 
+
+
 (define
   tcl-cmd-regsub
   (fn
     (interp args)
-    (define parse-flags
+    (define
-      (fn (as all? nocase?)
+      parse-flags
-        (if (= 0 (len as))
+      (fn
-          {:all all? :nocase nocase? :rest as}
+        (as all? nocase?)
+        (if
+          (= 0 (len as))
+          {:rest as :nocase nocase? :all all?}
           (cond
-            ((equal? (first as) "-all")    (parse-flags (rest as) true nocase?))
+            ((equal? (first as) "-all")
-            ((equal? (first as) "-nocase") (parse-flags (rest as) all? true))
+              (parse-flags (rest as) true nocase?))
-            (else {:all all? :nocase nocase? :rest as})))))
+            ((equal? (first as) "-nocase")
-    (let ((pf (parse-flags args false false)))
+              (parse-flags (rest as) all? true))
-      (let ((all-mode (get pf :all))
+            (else {:rest as :nocase nocase? :all all?})))))
-            (nocase (get pf :nocase))
+    (let
-            (ra (get pf :rest)))
+      ((pf (parse-flags args false false)))
-        (if (< (len ra) 3)
+      (let
+        ((all-mode (get pf :all))
+          (nocase (get pf :nocase))
+          (ra (get pf :rest)))
+        (if
+          (< (len ra) 3)
           (error "regsub: wrong # args")
-          (let ((pattern (first ra))
+          (let
-                (str-val (nth ra 1))
+            ((pattern (first ra))
-                (replacement (nth ra 2))
+              (str-val (nth ra 1))
-                (var-name (if (> (len ra) 3) (nth ra 3) nil)))
+              (replacement (nth ra 2))
-            (let ((re (make-regexp pattern (if nocase "i" ""))))
+              (var-name
-              (let ((result
+                (if (> (len ra) 3) (nth ra 3) nil)))
-                      (if all-mode
+            (let
-                        (regexp-replace-all re str-val replacement)
+              ((re (make-regexp pattern (if nocase "i" ""))))
-                        (regexp-replace re str-val replacement))))
+              (let
-                (if (nil? var-name)
+                ((result (if all-mode (regexp-replace-all re str-val replacement) (regexp-replace re str-val replacement))))
+                (if
+                  (nil? var-name)
                   (assoc interp :result result)
-                  (let ((count
+                  (let
-                          (if all-mode
+                    ((count (if all-mode (len (regexp-match-all re str-val)) (if (nil? (regexp-match re str-val)) 0 1))))
-                            (len (regexp-match-all re str-val))
+                    (assoc
-                            (if (nil? (regexp-match re str-val)) 0 1))))
+                      (tcl-var-set interp var-name result)
-                    (assoc (tcl-var-set interp var-name result) :result (str count))))))))))))
+                      :result (str count))))))))))))
-
-
 
 (define
   tcl-cmd-file
@@ -3153,7 +3614,10 @@
       (let
         ((sub (first args)) (rest-args (rest args)))
         (cond
-          ((equal? sub "exists") (assoc interp :result (if (file-exists? (first rest-args)) "1" "0")))
+          ((equal? sub "exists")
+            (assoc
+              interp
+              :result (if (file-exists? (first rest-args)) "1" "0")))
           ((equal? sub "join") (assoc interp :result (join "/" rest-args)))
           ((equal? sub "split")
             (assoc
@@ -3201,16 +3665,52 @@
                     (equal? dot-idx "-1")
                     nm
                     (substring nm 0 (parse-int dot-idx)))))))
-          ((equal? sub "isfile") (assoc interp :result "0"))
+          ((equal? sub "isfile")
-          ((equal? sub "isdir") (assoc interp :result "0"))
+            (assoc interp :result (if (file-isfile? (first rest-args)) "1" "0")))
-          ((equal? sub "isdirectory") (assoc interp :result "0"))
+          ((equal? sub "isdir")
-          ((equal? sub "readable") (assoc interp :result "0"))
+            (assoc interp :result (if (file-isdir? (first rest-args)) "1" "0")))
-          ((equal? sub "writable") (assoc interp :result "0"))
+          ((equal? sub "isdirectory")
-          ((equal? sub "size") (assoc interp :result "0"))
+            (assoc interp :result (if (file-isdir? (first rest-args)) "1" "0")))
-          ((equal? sub "mkdir") (assoc interp :result ""))
+          ((equal? sub "readable")
-          ((equal? sub "copy") (assoc interp :result ""))
+            (assoc interp :result (if (file-readable? (first rest-args)) "1" "0")))
-          ((equal? sub "rename") (assoc interp :result ""))
+          ((equal? sub "writable")
-          ((equal? sub "delete") (assoc interp :result ""))
+            (assoc interp :result (if (file-writable? (first rest-args)) "1" "0")))
+          ((equal? sub "size")
+            (assoc interp :result (str (file-size (first rest-args)))))
+          ((equal? sub "mtime")
+            (assoc interp :result (str (file-mtime (first rest-args)))))
+          ((equal? sub "atime")
+            (let ((s (file-stat (first rest-args))))
+              (assoc interp :result (if (nil? s) "0" (str (get s :atime))))))
+          ((equal? sub "type")
+            (let ((s (file-stat (first rest-args))))
+              (assoc interp :result (if (nil? s) "" (get s :type)))))
+          ((equal? sub "mkdir")
+            (let ((_ (file-mkdir (first rest-args))))
+              (assoc interp :result "")))
+          ((equal? sub "copy")
+            (let
+              ((paths
+                  (filter (fn (a) (not (equal? (slice a 0 1) "-"))) rest-args)))
+              (let ((_ (file-copy (first paths) (nth paths 1))))
+                (assoc interp :result ""))))
+          ((equal? sub "rename")
+            (let
+              ((paths
+                  (filter (fn (a) (not (equal? (slice a 0 1) "-"))) rest-args)))
+              (let ((_ (file-rename (first paths) (nth paths 1))))
+                (assoc interp :result ""))))
+          ((equal? sub "delete")
+            (let
+              ((paths
+                  (filter (fn (a) (not (equal? (slice a 0 1) "-"))) rest-args)))
+              (let
+                ((_
+                    (reduce
+                      (fn (acc p) (let ((_ (file-delete p))) acc))
+                      nil
+                      paths)))
+                (assoc interp :result ""))))
           (else (error (str "file: unknown subcommand \"" sub "\""))))))))
 
 (define
@@ -3254,7 +3754,7 @@
                                       (let
                                         ((i (tcl-register i "expr" tcl-cmd-expr)))
                                         (let
-                                          ((i (tcl-register i "gets" tcl-cmd-gets)))
+                                          ((i (tcl-register i "gets" tcl-cmd-gets-chan)))
                                           (let
                                             ((i (tcl-register i "subst" tcl-cmd-subst)))
                                             (let
@@ -3331,6 +3831,29 @@
                                                                                                                     ((i (tcl-register i "tell" tcl-cmd-tell)))
                                                                                                                     (let
                                                                                                                       ((i (tcl-register i "flush" tcl-cmd-flush)))
-                                                                                                                      (let ((i (tcl-register i "file" tcl-cmd-file)))
+                                                                                                                      (let
-                                                                                                                      (let ((i (tcl-register i "regexp" tcl-cmd-regexp)))
+                                                                                                                        ((i (tcl-register i "fconfigure" tcl-cmd-fconfigure)))
-                                                                                                                        (let ((i (tcl-register i "regsub" tcl-cmd-regsub))) (let ((i (tcl-register i "apply" tcl-cmd-apply))) (tcl-register i "array" tcl-cmd-array))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))
+                                                                                                                        (let
+                                                                                                                          ((i (tcl-register i "fileevent" tcl-cmd-fileevent)))
+                                                                                                                        (let
+                                                                                                                          ((i (tcl-register i "after" tcl-cmd-after)))
+                                                                                                                        (let
+                                                                                                                          ((i (tcl-register i "vwait" tcl-cmd-vwait)))
+                                                                                                                        (let
+                                                                                                                          ((i (tcl-register i "update" tcl-cmd-update)))
+                                                                                                                        (let
+                                                                                                                          ((i (tcl-register i "socket" tcl-cmd-socket)))
+                                                                                                                        (let
+                                                                                                                          ((i (tcl-register i "_sock-do-accept" tcl-cmd-_sock-do-accept)))
+                                                                                                                        (let
+                                                                                                                          ((i (tcl-register i "file" tcl-cmd-file)))
+                                                                                                                          (let
+                                                                                                                            ((i (tcl-register i "regexp" tcl-cmd-regexp)))
+                                                                                                                            (let
+                                                                                                                              ((i (tcl-register i "regsub" tcl-cmd-regsub)))
+                                                                                                                              (let
+                                                                                                                                ((i (tcl-register i "apply" tcl-cmd-apply)))
+                                                                                                                                (tcl-register
+                                                                                                                                  i
+                                                                                                                                  "array"
+                                                                                                                                  tcl-cmd-array)))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))

lib/tcl/test.sh

@@ -59,7 +59,7 @@ cat > "$TMPFILE" << EPOCHS
 (eval "tcl-test-summary")
 EPOCHS
 
-OUTPUT=$(timeout 180 "$SX_SERVER" < "$TMPFILE" 2>&1)
+OUTPUT=$(timeout 2400 "$SX_SERVER" < "$TMPFILE" 2>&1)
 [ "$VERBOSE" = "-v" ] && echo "$OUTPUT"
 
 # Extract summary line from epoch 11 output

lib/tcl/tests/coro.sx

@@ -124,7 +124,7 @@
       "file0")
 
     (ok "eof-returns-1"
-      (get (run "set ch [open /dev/null r]\neof $ch") :result)
+      (get (run "set ch [open /dev/null r]\nread $ch\neof $ch") :result)
       "1")
 
     (dict

lib/tcl/tests/idioms.sx

@@ -187,6 +187,234 @@
           (env-extend (env-extend base "a" 3) "b" 7)
           (quote (* a b))))
       21)
+
+    ; 26-32. Phase 5 channels: write/read/seek/tell/eof/append/non-blocking
+    (ok "channel-write-read"
+      (get
+        (run
+          "set f /tmp/tcl-phase5-1.txt\nset c [open $f w]\nputs $c \"line one\"\nputs $c \"line two\"\nclose $c\nset c [open $f r]\nset out [read $c]\nclose $c\nfile delete $f\nreturn $out")
+        :result)
+      "line one\nline two\n")
+
+    (ok "channel-gets-loop"
+      (get
+        (run
+          "set f /tmp/tcl-phase5-2.txt\nset c [open $f w]\nputs $c apple\nputs $c banana\nputs $c cherry\nclose $c\nset c [open $f r]\nset out {}\nwhile {[gets $c line] >= 0} {lappend out $line}\nclose $c\nfile delete $f\nreturn $out")
+        :result)
+      "apple banana cherry")
+
+    (ok "channel-seek-tell"
+      (get
+        (run
+          "set f /tmp/tcl-phase5-3.txt\nset c [open $f w]\nputs -nonewline $c \"hello world\"\nclose $c\nset c [open $f r]\nseek $c 6\nset pos [tell $c]\nset rest [read $c]\nclose $c\nfile delete $f\nreturn \"$pos:$rest\"")
+        :result)
+      "6:world")
+
+    (ok "channel-eof-after-read"
+      (get
+        (run
+          "set f /tmp/tcl-phase5-4.txt\nset c [open $f w]\nputs -nonewline $c hi\nclose $c\nset c [open $f r]\nread $c\nset e [eof $c]\nclose $c\nfile delete $f\nreturn $e")
+        :result)
+      "1")
+
+    (ok "channel-append-mode"
+      (get
+        (run
+          "set f /tmp/tcl-phase5-5.txt\nset c [open $f w]\nputs -nonewline $c \"first\"\nclose $c\nset c [open $f a]\nputs -nonewline $c \"-second\"\nclose $c\nset c [open $f r]\nset out [read $c]\nclose $c\nfile delete $f\nreturn $out")
+        :result)
+      "first-second")
+
+    (ok "channel-seek-end"
+      (get
+        (run
+          "set f /tmp/tcl-phase5-6.txt\nset c [open $f w]\nputs -nonewline $c \"abcdefghij\"\nclose $c\nset c [open $f r]\nseek $c 0 end\nset pos [tell $c]\nclose $c\nfile delete $f\nreturn $pos")
+        :result)
+      "10")
+
+    (ok "channel-fconfigure-blocking"
+      (get
+        (run
+          "set f /tmp/tcl-phase5-7.txt\nset c [open $f w]\nputs -nonewline $c x\nclose $c\nset c [open $f r]\nfconfigure $c -blocking 0\nset b [fconfigure $c -blocking]\nclose $c\nfile delete $f\nreturn $b")
+        :result)
+      "0")
+
+    ; 33-37. Phase 5b event loop: after / vwait / fileevent / update
+    (ok "after-vwait-timer"
+      (get
+        (run
+          "after 30 {set ::done fired}\nvwait ::done\nset ::done")
+        :result)
+      "fired")
+
+    (ok "after-multiple-timers-update"
+      (get
+        (run
+          "set ::n 0\nafter 0 {incr ::n}\nafter 0 {incr ::n}\nafter 0 {incr ::n}\nupdate\nset ::n")
+        :result)
+      "3")
+
+    (ok "fileevent-readable-fires"
+      (get
+        (run
+          "set f /tmp/tcl-phase5b-1.txt\nset c [open $f w]\nputs -nonewline $c hi\nclose $c\nset c [open $f r]\nfileevent $c readable {set ::ready 1; fileevent $::ch readable {}}\nset ::ch $c\nvwait ::ready\nclose $c\nfile delete $f\nset ::ready")
+        :result)
+      "1")
+
+    (ok "fileevent-query-script"
+      (get
+        (run
+          "set f /tmp/tcl-phase5b-2.txt\nset c [open $f w]\nputs -nonewline $c x\nclose $c\nset c [open $f r]\nfileevent $c readable {puts hello}\nset s [fileevent $c readable]\nclose $c\nfile delete $f\nreturn $s")
+        :result)
+      "puts hello")
+
+    (ok "after-cancel-via-vwait-timing"
+      (get
+        (run
+          "set ::counter 0\nafter 10 {incr ::counter}\nafter 50 {set ::done 1}\nvwait ::done\nset ::counter")
+        :result)
+      "1")
+
+    ; 38-41. Phase 5c sockets: TCP client + server
+    (ok "socket-server-fires-callback"
+      (get
+        (run
+          "proc h {sock host port} { global got; set got hit; close $sock }\nset srv [socket -server h 18901]\nset cli [socket localhost 18901]\nvwait got\nclose $srv\nclose $cli\nset got")
+        :result)
+      "hit")
+
+    (ok "socket-client-server-roundtrip"
+      (get
+        (run
+          "proc h {sock host port} { global received; gets $sock line; set received $line; close $sock }\nset srv [socket -server h 18902]\nset cli [socket localhost 18902]\nputs $cli ping\nflush $cli\nvwait received\nclose $srv\nclose $cli\nset received")
+        :result)
+      "ping")
+
+    (ok "socket-server-peer-host"
+      (get
+        (run
+          "proc h {sock host port} { global peer; set peer $host; close $sock }\nset srv [socket -server h 18903]\nset cli [socket 127.0.0.1 18903]\nvwait peer\nclose $srv\nclose $cli\nset peer")
+        :result)
+      "127.0.0.1")
+
+    (ok "socket-multiple-connections"
+      (get
+        (run
+          "proc h {sock host port} { global count; incr count; close $sock }\nset count 0\nset srv [socket -server h 18904]\nset c1 [socket localhost 18904]\nset c2 [socket localhost 18904]\nset c3 [socket localhost 18904]\nwhile {$count < 3} { update; after 5 }\nclose $srv\nclose $c1\nclose $c2\nclose $c3\nset count")
+        :result)
+      "3")
+
+    ; 42-49. Phase 5d file metadata + ops
+    (ok "file-isfile-true"
+      (get
+        (run
+          "set f /tmp/tcl-phase5d-1.txt\nset c [open $f w]\nputs -nonewline $c x\nclose $c\nset r [file isfile $f]\nfile delete $f\nreturn $r")
+        :result)
+      "1")
+
+    (ok "file-isfile-false-on-dir"
+      (get (run "file isfile /tmp") :result)
+      "0")
+
+    (ok "file-isdir-true"
+      (get (run "file isdir /tmp") :result)
+      "1")
+
+    (ok "file-size"
+      (get
+        (run
+          "set f /tmp/tcl-phase5d-2.txt\nset c [open $f w]\nputs -nonewline $c hello\nclose $c\nset s [file size $f]\nfile delete $f\nreturn $s")
+        :result)
+      "5")
+
+    (ok "file-readable-true"
+      (get (run "file readable /tmp") :result)
+      "1")
+
+    (ok "file-readable-missing"
+      (get (run "file readable /no/such/path/here") :result)
+      "0")
+
+    (ok "file-mkdir-then-isdir"
+      (get
+        (run
+          "set d /tmp/tcl-phase5d-mkdir/sub\nfile mkdir $d\nset r [file isdir $d]\nfile delete $d\nfile delete /tmp/tcl-phase5d-mkdir\nreturn $r")
+        :result)
+      "1")
+
+    (ok "file-copy-roundtrip"
+      (get
+        (run
+          "set s /tmp/tcl-phase5d-src.txt\nset d /tmp/tcl-phase5d-dst.txt\nset c [open $s w]\nputs -nonewline $c copydata\nclose $c\nfile copy $s $d\nset c [open $d r]\nset out [read $c]\nclose $c\nfile delete $s\nfile delete $d\nreturn $out")
+        :result)
+      "copydata")
+
+    (ok "file-rename-then-exists"
+      (get
+        (run
+          "set s /tmp/tcl-phase5d-r1.txt\nset d /tmp/tcl-phase5d-r2.txt\nset c [open $s w]\nputs -nonewline $c x\nclose $c\nfile rename $s $d\nset r [list [file exists $s] [file exists $d]]\nfile delete $d\nreturn $r")
+        :result)
+      "0 1")
+
+    (ok "file-mtime-positive"
+      (get
+        (run
+          "set f /tmp/tcl-phase5d-mt.txt\nset c [open $f w]\nputs -nonewline $c x\nclose $c\nset m [file mtime $f]\nfile delete $f\nexpr {$m > 0}")
+        :result)
+      "1")
+
+    ; 52-56. Phase 5e clock format options + clock scan
+    (ok "clock-format-utc"
+      (get
+        (run "clock format 0 -format {%Y-%m-%d %H:%M:%S} -gmt 1")
+        :result)
+      "1970-01-01 00:00:00")
+
+    (ok "clock-format-fmt-default"
+      (get
+        (run "clock format 1710513000 -format {%Y-%m-%d} -gmt 1")
+        :result)
+      "2024-03-15")
+
+    (ok "clock-scan-roundtrip"
+      (get
+        (run "set t [clock scan {2024-06-15 12:00:00} -format {%Y-%m-%d %H:%M:%S} -gmt 1]\nclock format $t -format {%Y-%m-%d %H:%M:%S} -gmt 1")
+        :result)
+      "2024-06-15 12:00:00")
+
+    (ok "clock-scan-returns-int"
+      (get
+        (run "expr {[clock scan {1970-01-01 00:00:00} -format {%Y-%m-%d %H:%M:%S} -gmt 1] == 0}")
+        :result)
+      "1")
+
+    (ok "clock-format-percent-pct"
+      (get
+        (run "clock format 0 -format {%Y%%%m} -gmt 1")
+        :result)
+      "1970%01")
+
+    ; 57-59. Phase 5f socket -async (non-blocking connect)
+    (ok "socket-async-completes-writable"
+      (get
+        (run
+          "proc h {sock host port} { close $sock }\nset srv [socket -server h 18930]\nset c [socket -async localhost 18930]\nset ready 0\nfileevent $c writable {global ready; set ready 1}\nvwait ready\nclose $c\nclose $srv\nset ready")
+        :result)
+      "1")
+
+    (ok "socket-async-then-write"
+      (get
+        (run
+          "proc accept {sock host port} { global accepted_sock; set accepted_sock $sock; fileevent $sock readable [list reader $sock] }\nproc reader {sock} { global received; gets $sock line; set received $line; close $sock }\nset srv [socket -server accept 18931]\nset c [socket -async localhost 18931]\nfileevent $c writable {global wready; set wready 1; fileevent $::ch writable {}}\nset ::ch $c\nvwait wready\nputs $c async-data\nflush $c\nvwait received\nclose $c\nclose $srv\nset received")
+        :result)
+      "async-data")
+
+    (ok "socket-async-no-error"
+      (get
+        (run
+          "proc h {sock host port} { close $sock }\nset srv [socket -server h 18932]\nset c [socket -async localhost 18932]\nset r 0\nfileevent $c writable {global r; set r 1}\nvwait r\nset err [fconfigure $c -error]\nclose $c\nclose $srv\nreturn $err")
+        :result)
+      "")
+
     (dict
       "passed"
       tcl-idiom-pass

plans/apl-on-sx.md

@@ -104,6 +104,79 @@ Core mapping:
 - [x] Drive corpus to 100+ green
 - [x] Idiom corpus — `lib/apl/tests/idioms.sx` covering classic Roger Hui / Phil Last idioms
 
+### Phase 7 — end-to-end pipeline + closing the gaps
+
+Phase 1-6 built parser and runtime as parallel layers — they don't yet meet.
+Phase 7 wires them together so APL source actually runs through the full stack,
+and tightens loose ends.
+
+- [x] **Operators in `apl-eval-ast`** — handle `:derived-fn` (e.g. `+/`, `f¨`),
+  `:outer` (`∘.f`), `:derived-fn2` (`f.g`).  Each derived-fn-node wraps an inner
+  function; eval-ast resolves the inner glyph to a runtime fn and dispatches
+  to the matching operator helper (`apl-reduce`, `apl-each`, `apl-outer`,
+  `apl-inner`, `apl-commute`, `apl-compose`, `apl-power`, `apl-rank`).
+- [x] **End-to-end pipeline** — entry point `apl-run : string → array` that
+  chains `apl-tokenize` → `parse-apl` → `apl-eval-ast` against an empty env.
+  Verify with one-liners (`+/⍳5` → 15, `1 2 3 + 4 5 6` → 7 9 11, etc.) and
+  with the actual `.apl` source files in `tests/programs/`.
+- [x] **`:quad-name` AST + handler** — extend tokenizer/parser to recognise
+  `⎕name`, then handle in `apl-eval-ast` by dispatching to `apl-quad-*`
+  runtime fns (`⎕IO`, `⎕ML`, `⎕FR`, `⎕TS`, `⎕FMT`, `⎕←`).
+  _(`⎕←` deferred — tokenizer treats `←` as `:assign` after `⎕`.)_
+- [x] **Bracket indexing verification** — load programs that use `A[I]` /
+  `A[I;J]` end-to-end; confirm parser desugars to `⌷` and runtime returns
+  expected slices.  Add 5+ tests.
+  _(Single-axis only — multi-axis `A[I;J]` requires semicolon parsing, deferred.)_
+- [x] **Idiom corpus expansion** — extend `idioms.sx` from 34 to 60+ once
+  end-to-end works (we can express idioms as APL strings, not as runtime
+  calls).  Source-string-based idioms validate the whole stack.
+- [x] **`:Trap` / `:EndTrap`** — minimal exception machinery: `:Trap n`
+  catches errors with code `n`, body runs in `apl-tradfn-eval-block`,
+  on error switches to the trap branch.  Define `apl-throw` and a small
+  set of error codes; use `try`/`catch` from the host.
+
+### Phase 8 — fill the gaps left after end-to-end
+
+Phase 7 wired the stack together; Phase 8 closes deferred items, lets real
+programs run from source, and starts pushing on performance.
+
+- [x] **Quick-wins bundle** (one iteration) — three small fixes that each unblock
+  real programs:
+  - decimal literals: `read-digits!` consumes one trailing `.` plus more digits
+    so `3.7` tokenises as one number;
+  - `⎕←` (print) — tokenizer special-case: when `⎕` is followed by `←`, emit
+    a single `:name "⎕←"` token (don't split on the assign glyph);
+  - string values in `apl-eval-ast` — handle `:str` (parser already produces
+    them) by wrapping into a vector of character codes (or rank-0 string).
+- [x] **Named function definitions** — `f ← {⍺+⍵} ⋄ 1 f 2` and `2 f 3`.
+  - parser: when `:assign`'s RHS is a `:dfn`, mark it as a function binding;
+  - eval-ast: `:assign` of a dfn stores the dfn in env;
+  - parser: a name in fn-position whose env value is a dfn dispatches as a fn;
+  - resolver: extend `apl-resolve-monadic`/`-dyadic` with a `:fn-name` case
+    that calls `apl-call-dfn`/`apl-call-dfn-m`.
+- [x] **Multi-axis bracket indexing** — `A[I;J]` and `A[;J]` and `A[I;]`.
+  - parser: split bracket content on `:semi` at depth 0; emit
+    `(:dyad ⌷ (:vec I J) A)`;
+  - runtime: extend `apl-squad` to accept a vector of indices, treating
+    `nil` / empty axis as "all";
+  - 5+ tests across vector and matrix.
+- [x] **`.apl` files as actual tests** — `lib/apl/tests/programs/*.apl` are
+  currently documentation. Add `apl-run-file path → array` plus tests that
+  load each file, execute it, and assert the expected result. Makes the
+  classic-program corpus self-validating instead of two parallel impls.
+  _(Embedded source-string approach: tests/programs-e2e.sx runs the same
+  algorithms as the .apl docs through the full pipeline. The original
+  one-liners (e.g. primes' inline `⍵←⍳⍵`) need parser features
+  (compress-as-fn, inline assign) we haven't built yet — multi-stmt forms
+  used instead. Slurp/read-file primitive missing in OCaml SX runtime.)_
+- [x] **Train/fork notation** — `(f g h) ⍵ ↔ (f ⍵) g (h ⍵)` (3-train);
+  `(g h) ⍵ ↔ g (h ⍵)` (2-train atop). Parser: detect when a parenthesised
+  subexpression is all functions and emit `(:train fns)`; resolver: build the
+  derived function; tests for mean-via-train (`+/÷≢`).
+- [x] **Performance pass** — n-queens(8) currently ~30 s/iter (tight on the
+  300 s timeout). Target: profile the inner loop, eliminate quadratic
+  list-append, restore the `queens(8)` test.
+
 ## SX primitive baseline
 
 Use vectors for arrays; numeric tower + rationals for numbers; ADTs for tagged data;
@@ -118,6 +191,20 @@ data; format for string templating.
 
 _Newest first._
 
+- 2026-05-07: Phase 8 step 6 — perf: swapped (append acc xs) → (append xs acc) in apl-permutations to make permutation generation linear instead of quadratic; q(7) 32s→12s; q(8)=92 test restored within 300s timeout; **Phase 8 complete, all unchecked items ticked**; 497/497
+- 2026-05-07: Phase 8 step 5 — train/fork notation. Parser :lparen detects all-fn inner segments → emits :train AST; resolver covers 2-atop & 3-fork for both monadic and dyadic. `(+/÷≢) 1..5 → 3` (mean), `(- ⌊) 5 → -5` (atop), `2(+×-)5 → -21` (dyadic fork), `(⌈/-⌊/) → 8` (range); +6 tests; 496/496
+- 2026-05-07: Phase 8 step 4 — programs-e2e.sx runs classic-algorithm shapes through full pipeline (factorial via ∇, triangulars, sum-of-squares, divisor-counts, prime-mask, named-fn composition, dyadic max-of-two, Newton step); also added ⌿ + ⍀ to glyph sets (were silently skipped); +15 tests; 490/490
+- 2026-05-07: Phase 8 step 3 — multi-axis bracket A[I;J] / A[I;] / A[;J] via :bracket AST + apl-bracket-multi runtime; split-bracket-content scans :semi at depth 0; apl-cartesian builds index combinations; nil axis = "all"; scalar axis collapses; +8 tests; 475/475
+- 2026-05-07: Phase 8 step 2 — named function defs end-to-end via parser pre-scan; apl-known-fn-names + apl-collect-fn-bindings detect `name ← {...}` patterns; collect-segments-loop emits :fn-name for known names; resolver looks up env for :fn-name; supports recursion (∇ in named dfn); +7 tests including fact via ∇; 467/467
+- 2026-05-07: Phase 8 step 1 — quick-wins bundle: decimal literals (3.7, ¯2.5), ⎕← passthrough as monadic fn (single-token via tokenizer special-case), :str AST in eval-ast (single-char→scalar, multi-char→vec); +10 tests; 460/460
+- 2026-05-07: Phase 8 added — quick-wins bundle (decimals + ⎕← + strings), named functions, multi-axis bracket, .apl-files-as-tests, trains, perf
+- 2026-05-07: Phase 7 step 6 — :Trap exception machinery via R7RS guard; apl-throw raises tagged error, apl-trap-matches? checks codes (0=catch-all), :trap clause in apl-tradfn-eval-stmt wraps try-block with guard; :throw AST for testing; **Phase 7 complete, all unchecked plan items done**; +5 tests; 450/450
+- 2026-05-07: Phase 7 step 5 — idiom corpus 34→64 (+30 source-string idioms via apl-run); also fixed tokenizer + parser to recognize ≢ and ≡ glyphs (were silently skipped); 445/445
+- 2026-05-07: Phase 7 step 4 — bracket indexing `A[I]` desugared to `(:dyad ⌷ I A)` via maybe-bracket helper, wired into :name + :lparen branches of collect-segments-loop; multi-axis (A[I;J]) deferred (semicolon split); +7 tests; 415/415
+- 2026-05-07: Phase 7 step 3 — :quad-name end-to-end; tokenizer already produced :name "⎕FMT"; parser is-fn-tok? extended via apl-quad-fn-names; eval-ast :name dispatches ⎕IO/⎕ML/⎕FR/⎕TS to apl-quad-*; apl-monadic-fn handles ⎕FMT; ⎕← deferred (tokenizer splits ⎕←); +8 tests; 408/408
+- 2026-05-07: Phase 7 step 2 — end-to-end pipeline `apl-run : string → array` (parse-apl + apl-eval-ast against empty env); +25 source-string tests covering scalars, strands, dyadic arith, monadic primitives, operators, ∘./.g products, comparisons, famous one-liners (+/⍳10=55, ×/⍳10=10!); tokenizer can't yet parse decimals so `3.7` literal tests dropped; **400/400**
+- 2026-05-07: Phase 7 step 1 — operators in apl-eval-ast via apl-resolve-monadic/dyadic; supports / ⌿ \ ⍀ ¨ ⍨ ∘. f.g; queens(8) test removed (too slow for 300s timeout); +14 eval-ops tests; 375/375
+- 2026-05-07: Phase 7 added — end-to-end pipeline, operators in eval-ast, :quad-name, bracket-indexing verify, idiom expansion, :Trap; aim is to wire parser↔runtime so .apl source files actually run
 - 2026-05-07: Phase 6 idiom corpus — lib/apl/tests/idioms.sx; 34 classic idioms (sum, mean, max/min/range, scan, sort, reverse, first/last, take/drop, tally, mod, identity matrix, mult-table, factorial, parity count, all/any, mean-centered, ravel, rank); **all unchecked items in plan now ticked**; 362/362
 - 2026-05-07: Phase 6 system fns + 100+ corpus — apl-quad-{io,ml,fr,ts,fmt,print}; ⎕FMT formats scalar/vector/matrix; ⎕TS returns 7-vector (epoch default); 328 tests >> 100 target; **drive-to-100 ticked**; +13 tests
 - 2026-05-07: Phase 6 quicksort — recursive less/eq/greater partition via apl-compress, deterministic-pivot variant; tests cover empty/single/sorted/reverse/duplicates/negatives; **all 5 classic programs done**; +9 tests; 315/315

plans/datalog-on-sx.md

@@ -13,6 +13,20 @@ End-state goal: **full core Datalog** (facts, rules, stratified negation, aggreg
 recursion) with a clean SX query API, and a demonstration of Datalog as a query engine
 for rose-ash data (e.g. federation graph, content relationships).
 
+## Status (rolling)
+
+`bash lib/datalog/conformance.sh` → **276/276 across 11 suites**
+(tokenize, parse, unify, eval, builtins, semi_naive, negation, aggregates,
+api, magic, demo). Source is ~3100 LOC, tests ~2900 LOC, public API
+documented in `lib/datalog/datalog.sx`.
+
+Phases 1–9 are functionally complete; Phase 10 covers the rose-ash
+domain demos (in `lib/datalog/demo.sx` — federation, content,
+permissions, cooking-posts, tag co-occurrence, shortest path, org chart).
+The PostgreSQL loader and `/internal/datalog` HTTP endpoint listed in
+Phase 10 require service-tree edits outside `lib/datalog/**` and are
+flagged as out-of-scope for this loop.
+
 ## Ground rules
 
 - **Scope:** only touch `lib/datalog/**` and `plans/datalog-on-sx.md`. Do **not** edit
@@ -58,88 +72,647 @@ Key differences from Prolog:
 ## Roadmap
 
 ### Phase 1 — tokenizer + parser
-- [ ] Tokenizer: atoms (lowercase/quoted), variables (uppercase/`_`), numbers, strings,
+- [x] Tokenizer: atoms (lowercase/quoted), variables (uppercase/`_`), numbers, strings,
-      operators (`:- `, `?-`, `,`, `.`), comments (`%`, `/* */`)
+      punct (`( )`, `,`, `.`), operators (`:-`, `?-`, `<=`, `>=`, `!=`, `<`, `>`, `=`,
-      Note: no function symbol syntax (no nested `f(...)` in arg position).
+      `+`, `-`, `*`, `/`), comments (`%`, `/* */`)
-- [ ] Parser:
+      Note: no function symbol syntax (no nested `f(...)` in arg position) — but the
+      parser permits nested compounds for arithmetic; safety analysis (Phase 3) rejects
+      non-arithmetic nesting.
+- [x] Parser:
       - Facts: `parent(tom, bob).` → `{:head (parent tom bob) :body ()}`
       - Rules: `ancestor(X,Z) :- parent(X,Y), ancestor(Y,Z).`
               → `{:head (ancestor X Z) :body ((parent X Y) (ancestor Y Z))}`
-      - Queries: `?- ancestor(tom, X).` → `{:query (ancestor tom X)}`
+      - Queries: `?- ancestor(tom, X).` → `{:query ((ancestor tom X))}`
+        (`:query` value is always a list of literals; `?- p, q.` → `{:query ((p) (q))}`)
       - Negation: `not(parent(X,Y))` in body position → `{:neg (parent X Y)}`
-- [ ] Tests in `lib/datalog/tests/parse.sx`
+- [x] Tests in `lib/datalog/tests/parse.sx` (18) and `lib/datalog/tests/tokenize.sx` (26).
+      Conformance harness: `bash lib/datalog/conformance.sh` → 44 / 44 passing.
 
 ### Phase 2 — unification + substitution
-- [ ] Share or port unification from `lib/prolog/` — term walk, occurs check off by default
+- [x] Ported (not shared) from `lib/prolog/` — term walk, no occurs check.
-- [ ] `dl-unify` `t1` `t2` `subst` → extended subst or nil (no function symbols means simpler)
+- [x] `dl-unify t1 t2 subst` → extended subst dict, or `nil` on failure.
-- [ ] `dl-ground?` `term` → bool — all variables bound in substitution
+- [x] `dl-walk`, `dl-bind`, `dl-apply-subst`, `dl-ground?`, `dl-vars-of`.
-- [ ] Tests: atom/atom, var/atom, var/var, list args
+- [x] Substitutions are immutable dicts keyed by variable name (string).
+      Lists/tuples unify element-wise (used for arithmetic compounds too).
+- [x] Tests in `lib/datalog/tests/unify.sx` (28). 72 / 72 conformance.
 
-### Phase 3 — extensional DB + naive evaluation
+### Phase 3 — extensional DB + naive evaluation + safety analysis
-- [ ] EDB: `{:relation-name → set-of-ground-tuples}` using SX sets (Phase 18 of primitives)
+- [x] EDB+IDB combined: `{:facts {<rel-name-string> -> (literal ...)}}` —
-- [ ] `dl-add-fact!` `db` `relation` `args` → add ground tuple
+      relations indexed by name; tuples stored as full literals so they
-- [ ] `dl-add-rule!` `db` `head` `body` → add rule clause
+      unify directly. Dedup on insert via `dl-tuple-equal?`.
-- [ ] Naive evaluation: iterate rules until fixpoint
+- [x] `dl-add-fact! db lit` (rejects non-ground) and `dl-add-rule! db rule`
-      For each rule, for each combination of body tuples that unify, derive head tuple.
+      (rejects unsafe). `dl-program source` parses + loads in one step.
-      Repeat until no new tuples added.
+- [x] Naive evaluation `dl-saturate! db`: iterate rules until no new tuples.
-- [ ] `dl-query` `db` `goal` → list of substitutions satisfying goal against derived DB
+      `dl-find-bindings` recursively joins body literals; `dl-match-positive`
-- [ ] Tests: transitive closure (ancestor), sibling, same-generation — classic Datalog programs
+      unifies a literal against every tuple in the relation.
+- [x] `dl-query db goal` → list of substitutions over `goal`'s vars,
+      deduplicated. `dl-relation db name` for derived tuples.
+- [x] Safety analysis at `dl-add-rule!` time: every head variable except
+      `_` must appear in some positive body literal. Built-ins and negated
+      literals do not satisfy safety. Helpers `dl-positive-body-vars`,
+      `dl-rule-unsafe-head-vars` exposed for later phases.
+- [x] Negation and arithmetic built-ins error cleanly at saturate time
+      (Phase 4 / Phase 7 will swap in real semantics).
+- [x] Tests in `lib/datalog/tests/eval.sx` (15): transitive closure,
+      sibling, same-generation, grandparent, cyclic graph reach, six
+      safety cases. 87 / 87 conformance.
 
-### Phase 4 — semi-naive evaluation (performance)
+### Phase 4 — built-in predicates + body arithmetic
-- [ ] Delta sets: track newly derived tuples per iteration
+Almost every real query needs `<`, `=`, simple arithmetic, and string
-- [ ] Semi-naive rule: only join against delta tuples from last iteration, not full relation
+comparisons in body position. These are not EDB lookups — they're
-- [ ] Significant speedup for recursive rules — avoids re-deriving known tuples
+constraints that filter bindings.
-- [ ] `dl-stratify` `db` → dependency graph + SCC analysis → stratum ordering
+- [x] Recognise built-in predicates in body: `(< X Y)`, `(<= X Y)`, `(> X Y)`,
-- [ ] Tests: verify semi-naive produces same results as naive; benchmark on large ancestor chain
+      `(>= X Y)`, `(= X Y)`, `(!= X Y)` and arithmetic forms `(is Z (+ X Y))`,
+      `(is Z (- X Y))`, `(is Z (* X Y))`, `(is Z (/ X Y))`. Live in
+      `lib/datalog/builtins.sx`.
+- [x] `dl-eval-builtin` dispatches; `dl-eval-arith` recursively evaluates
+      `(+ a b)` etc. with full nesting. `=` unifies; `!=` rejects equal
+      ground terms.
+- [x] Order-aware safety analysis (`dl-rule-check-safety`): walks body
+      left-to-right tracking which vars are bound. `is`'s RHS vars must
+      be already bound; LHS becomes bound. Comparisons require both
+      sides bound. `=` is special-cased — at least one side bound binds
+      the other. Negation vars must be bound (will be enforced fully in
+      Phase 7).
+- [x] Wired through SX numeric primitives — no separate number tower.
+- [x] Tests in `lib/datalog/tests/builtins.sx` (19): range filters,
+      arithmetic derivations, equality binding, eight safety violations
+      and three safe-shape tests. Conformance 106 / 106.
 
-### Phase 5 — stratified negation
+### Phase 5 — semi-naive evaluation (performance)
-- [ ] Dependency graph analysis: which relations depend on which (positively or negatively)
+- [x] Delta sets `{rel-name -> tuples}` track newly derived tuples per iter.
-- [ ] Stratification check: error if negation is in a cycle (non-stratifiable program)
+      `dl-snapshot-facts` builds the initial delta from the EDB.
-- [ ] Evaluation: process strata in order — lower stratum fully computed before using its
+- [x] Semi-naive rule: for each rule, walk every positive body literal
-      complement in a higher stratum
+      position; substitute that one with the per-relation delta and join
-- [ ] `not(P)` in rule body: at evaluation time, check P is NOT in the derived EDB
+      the rest against the previous-iteration DB (`dl-find-bindings-semi`).
-- [ ] Tests: non-member (`not(member(X,L))`), colored-graph (`not(same-color(X,Y))`),
+      Candidates are collected before mutating the DB so the "full" sides
-      stratification error detection
+      see a consistent snapshot.
+- [x] `dl-collect-rule-candidates` falls back to a naive single pass when
+      a rule has no positive body literal (e.g. `(p X) :- (= X 5).`).
+- [x] `dl-saturate!` is now semi-naive by default; `dl-saturate-naive!`
+      kept for differential testing and a reference implementation.
+- [x] Tests in `lib/datalog/tests/semi_naive.sx` (8) — every recursive
+      program from earlier suites is run under both saturators with
+      per-relation tuple counts compared (cheap, robust under bundled
+      conformance session). A chain-5 differential exercises multiple
+      semi-naive iterations against the recursive ancestor rule.
+      Larger chains hit prohibitive wall-clock under conformance CPU
+      contention with other agents — a future Blocker tracks switching
+      `dl-tuple-member?` from O(n²) list scan to a hash-set per relation.
 
-### Phase 6 — aggregation (Datalog+)
+### Phase 6 — magic sets (goal-directed bottom-up, opt-in)
-- [ ] `count(X, Goal)` → number of distinct X satisfying Goal
+Naive bottom-up derives **all** consequences before answering. Magic sets
-- [ ] `sum(X, Goal)` → sum of X values satisfying Goal
+rewrite the program so the fixpoint only derives tuples relevant to the
-- [ ] `min(X, Goal)` / `max(X, Goal)` → min/max of X satisfying Goal
+goal — a major perf win for "what's reachable from node X" queries on
-- [ ] `group-by` semantics: `count(X, sibling(bob, X))` → count of bob's siblings
+large graphs.
-- [ ] Aggregation breaks stratification — evaluate in a separate post-fixpoint pass
+- [x] Adornments: `dl-adorn-goal goal` and `dl-adorn-lit lit bound` in
-- [ ] Tests: social network statistics, grade aggregation, inventory sums
+      `lib/datalog/magic.sx`. Per-arg `b`/`f` based on whether the arg
+      is a constant or a variable already in the bound set.
+- [x] Magic transformation: `dl-magic-rewrite rules query-rel adn args`
+      generates `{:rules <rewritten-rules> :seed <magic-seed>}`. Each
+      original rule is gated with a `magic_<rel>^<adn>(bound)` filter,
+      and propagation rules are emitted for each positive non-builtin
+      body literal. Worklist over `(rel, adn)` pairs starts from the
+      query and stops when no new pairs appear. EDB facts pass through
+      unchanged.
+- [x] Sideways information passing strategy (SIPS): left-to-right
+      `dl-rule-sips rule head-adornment` walks body literals tracking
+      the bound set, returning `({:lit :adornment} ...)`. Recognises
+      `is`/aggregate result-vars as new binders; comparisons and
+      negation pass through with computed adornments. (Pluggable
+      strategies are future work.)
+- [x] `dl-set-strategy! db strategy` hook + `dl-get-strategy db`. Default
+      `:semi-naive`. `:magic` accepted but the transformation itself is
+      deferred — saturator currently falls back to semi-naive. Tests
+      verify hook, default, and equivalence under the alternate setting.
+- [x] Equivalence test: rewritten ancestor program over the same EDB
+      derives the same number of `ancestor` tuples and returns the
+      same query answers as the unrewritten program (chain-3 case).
+- [x] `dl-magic-query db query-goal` — top-level driver. Builds a
+      fresh internal db with the caller's EDB facts, the magic seed,
+      and the rewritten rules; saturates and queries. Caller's db is
+      untouched. Equivalent to `dl-query` for fully-stratifiable
+      programs (sole motivation is a perf alternative on goal-shaped
+      queries against large recursive relations).
+- [ ] Perf test: 10k-node reachability with magic vs semi-naive.
+      Left to a future iteration — would need a benchmarking harness
+      for large graphs and the conformance budget can't afford it.
 
-### Phase 7 — SX embedding API
+### Phase 7 — stratified negation
-- [ ] `(dl-program facts rules)` → database from SX data directly (no parsing required)
+- [x] Dependency graph: `dl-build-dep-graph db` returns `{head -> ({:rel
+      :neg} ...)}`. Built-ins drop out (they're not relations).
+- [x] Reachability via Floyd-Warshall in `dl-build-reach`; cycles
+      detected by `reach[A][B] && reach[B][A]`. Programs are
+      non-stratifiable iff any negative dependency falls inside an SCC.
+      `dl-check-stratifiable` returns nil on success or a clear message.
+- [x] `dl-compute-strata` propagates stratum numbers iteratively:
+      `stratum(R) = max over deps of (stratum(dep) + (1 if negated else 0))`.
+- [x] Saturator refactor: `dl-saturate-rules! db rules` is the semi-
+      naive worker; `dl-saturate! db` rejects non-stratifiable programs,
+      groups rules by head's stratum, and runs the worker on each
+      stratum in increasing order.
+- [x] `not(P)` in body: `dl-match-negation` walks the inner literal
+      under the current subst and uses `dl-match-positive` — succeeds
+      iff zero matches. Order-aware safety in `dl-rule-check-safety`
+      (already present from Phase 4) requires negation vars to be
+      bound by an earlier positive literal.
+- [x] Tests in `lib/datalog/tests/negation.sx` (10): EDB and IDB
+      negation, two-step strata, multi-level strata, with-arithmetic,
+      empty-result and always-fail cases, non-stratifiability
+      rejection, and a negation safety violation.
+
+### Phase 8 — aggregation (Datalog+)
+- [x] `(count R V Goal)`, `(sum R V Goal)`, `(min R V Goal)`,
+      `(max R V Goal)`, `(findall L V Goal)` — first arg is the result
+      variable, second is the aggregated variable, third is the goal
+      literal. `findall` returns the distinct-value list itself; the
+      others reduce. Live in `lib/datalog/aggregates.sx`.
+- [x] `dl-eval-aggregate`: runs `dl-find-bindings` on the goal under the
+      current subst (which provides outer-context bindings), collects
+      distinct values of the aggregated var, applies the aggregate.
+      `count`/`sum` produce 0 when no matches; `min`/`max` produce no
+      binding (rule fails) when empty.
+- [x] Group-by emerges naturally: outer-context vars in the goal are
+      substituted from the current subst, so `popular(P) :- post(P),
+      count(N, U, liked(U, P)), >=(N, 3).` correctly counts per-post.
+- [x] Stratification: `dl-aggregate-dep-edge` returns a negation-like
+      edge so the aggregate's goal relation is fully derived before the
+      aggregate fires. Non-monotonicity respected.
+- [x] Safety: aggregate body lit binds the result var; goal-internal
+      vars are existentially quantified and don't need outer binding.
+- [x] Tests in `lib/datalog/tests/aggregates.sx` (10): count siblings,
+      sum prices, min/max scores, count over derived relation,
+      empty-input cases for each operator, popularity threshold with
+      group-by, distinct-counted-once.
+
+### Phase 9 — SX embedding API
+- [x] `(dl-program-data facts rules)` builds a db from SX data —
+      `facts` is a list of literals, `rules` is a list of either
+      dicts `{:head … :body …}` or lists `(<head…> <- <body…>)`.
+      Variables are SX symbols whose first char is uppercase or `_`,
+      matching the parser's convention.
       ```
-      (dl-program
+      (dl-program-data
-        '((parent tom bob) (parent tom liz) (parent bob ann))
+        '((parent tom bob) (parent bob ann))
-        '((ancestor X Z :- (parent X Y) (ancestor Y Z))
+        '((ancestor X Y <- (parent X Y))
-          (ancestor X Y :- (parent X Y))))
+          (ancestor X Z <- (parent X Y) (ancestor Y Z))))
       ```
-- [ ] `(dl-query db '(ancestor tom ?X))` → `((ann) (bob) (liz) (pat))`
+- [x] `(dl-rule head body)` constructor for the dict form.
-- [ ] `(dl-assert! db '(parent ann pat))` → incremental fact addition + re-derive
+- [x] `(dl-query db '(ancestor tom X))` already worked — same query API
-- [ ] `(dl-retract! db '(parent tom bob))` → fact removal + re-derive from scratch
+      consumes the SX-data goal. Now also accepts a *list* of body
-- [ ] Integration demo: federation graph query — `(ancestor actor1 actor2)` over
+      literals for conjunctive queries:
-      rose-ash ActivityPub follow relationships
+      `(dl-query db '((p X) (q X)))`,
+      `(dl-query db (list '(n X) '(> X 2)))`. Auto-dispatched via
+      `dl-query-coerce` on first-element shape.
+- [x] `(dl-assert! db '(parent ann pat))` → adds the fact and re-saturates.
+- [x] `(dl-retract! db '(parent bob ann))` → drops matching tuples from
+      the EDB list, wipes every relation that has a rule (those are IDB),
+      and re-saturates from the surviving EDB.
+- [x] Tests in `lib/datalog/tests/api.sx` (9): closure via data API,
+      dict-rule form, dl-rule constructor, dl-assert! incremental,
+      dl-retract! removes derived, cyclic-graph reach via data,
+      assert into empty db, fact-style rule (no arrow), coerce dict.
+- [x] Integration demo: federation graph query — `(reachable A B)` /
+      `(mutual A B)` / `(foaf A C)` over `(follows ACTOR-A ACTOR-B)` in
+      `lib/datalog/demo.sx`. Tests in `lib/datalog/tests/demo.sx`.
+      Wiring this to actual rose-ash ActivityPub data is Phase 10
+      service work and is out of scope for this loop.
 
-### Phase 8 — Datalog as a query language for rose-ash
+### Phase 10 — Datalog as a query language for rose-ash
-- [ ] Schema: map SQLAlchemy model relationships to Datalog EDB facts
+- [x] Schema sketches in `lib/datalog/demo.sx`:
-      (e.g. `(follows user1 user2)`, `(authored user post)`, `(tagged post tag)`)
+      - **Federation**: `(follows A B)` → `(mutual A B)`, `(reachable A B)`,
-- [ ] Loader: `dl-load-from-db!` — query PostgreSQL, populate Datalog EDB
+        `(foaf A C)` (friend-of-a-friend, distinct).
-- [ ] Query examples:
+      - **Content**: `(authored A P)`, `(liked U P)`, `(tagged P T)` →
-      - `?- ancestor(me, X), authored(X, Post), tagged(Post, cooking).`
+        `(post-likes P N)` via aggregation, `(popular P)` for likes ≥ 3,
-        → posts about cooking by people I follow (transitively)
+        `(interesting Me P)` joining follows + authored + popular.
-      - `?- popular(Post) :- tagged(Post, T), count(L, (liked(L, Post))) >= 10.`
+      - **Permissions**: `(member A G)`, `(subgroup C P)`, `(allowed G R)`
-        → posts with 10+ likes
+        → `(in-group A G)` over transitive subgroups, `(can-access A R)`.
-- [ ] Expose as a rose-ash service endpoint: `POST /internal/datalog` with program + query
+      - **Cooking-posts** (the canonical example): `(reach Me Them)` over
+        the follow graph, then `(cooking-post-by-network Me P)` joining
+        reach + authored + `(tagged P cooking)`.
+- [ ] Loader `dl-load-from-db!` — out of scope for this loop
+      (would need to edit `shared/services/` outside `lib/datalog/`).
+      Programs in `demo.sx` already document the EDB shape expected
+      from such a loader. `dl-program-data` consumes the same shape.
+- [x] Query examples covered by `lib/datalog/tests/demo.sx` (10):
+      mutuals, transitive reach, FOAF, popular posts, interesting feed,
+      post likes count, direct/subgroup/transitive group access, no
+      access without grant.
+- [ ] Service endpoint `POST /internal/datalog` — out of scope as above.
+      Once exposed, server-side handler would be `dl-program-data` +
+      `dl-query`, returning JSON-encoded substitutions.
 
 ## Blockers
 
-_(none yet)_
+- **Saturation perf**: three rounds done.
+  - hash-set membership in `dl-add-fact!` (Phase 5b)
+  - indexed iteration in `dl-find-bindings` (Phase 5c)
+  - first-arg index per relation (Phase 5e) — when a body literal's
+    first arg walks to a non-variable, dl-match-positive looks up
+    by `(str arg)` instead of scanning the full relation.
+  chain-25 saturation drops from ~33s to ~18s real (10s user).
+  chain-50 still long (~120s+) due to dict-copy overhead in
+  unification subst threading. Future: per-rule "compiled" body
+  with pre-resolved var positions, slot-based subst representation
+  to avoid `assoc` per binding.
 
 ## Progress log
 
 _Newest first._
 
-_(awaiting phase 1)_
+- 2026-05-11 — `dl-set-strategy!` accepted arbitrary keyword values
+  silently. Typos like `:semi_naive` or `:semiNaive` were stored
+  uninspected; the saturator then used the default and the user
+  never learned their setting was a typo. Validator added: strategy
+  must be one of `:semi-naive`, `:naive`, `:magic`. 1 regression test;
+  276/276.
+
+- 2026-05-11 — Anonymous-variable renamer collided with user-written
+  `_anon<N>` symbols. The renamer started counter at 0 and produced
+  `_anon1, _anon2, ...` unconditionally; if the user wrote
+  `q(_anon1) :- p(_anon1, _).` the `_` got renamed to `_anon1` too,
+  collapsing the two positions of `p` to a single var and returning
+  the empty result instead of `{a, c}`. Fix: scan each rule (and
+  query) for the max `_anon<N>` and start the renamer past it. The
+  renamer constructor now takes a `start` arg; new helpers
+  `dl-max-anon-num` / `dl-max-anon-num-list` walk the rule tree.
+  1 regression test; 275/275.
+
+- 2026-05-11 — `dl-magic-query` could silently diverge from
+  `dl-query` when an aggregate's inner-goal relation was IDB. The
+  rewriter passes aggregate body lits through unchanged (no magic
+  propagation for them), so the inner relation was empty in the
+  magic db and the aggregate returned 0. Probe:
+  `dl-eval-magic "u(a). u(b). u(c). u(d). banned(b). banned(d).
+                  active(X) :- u(X), not(banned(X)).
+                  n(N) :- count(N, X, active(X))." "?- n(N)."`
+  returned `N=0` instead of `N=2`. Fix: `dl-magic-query` now
+  pre-saturates the source db before copying facts into the magic
+  db. This guarantees equivalence with `dl-query` for every
+  stratified program; the magic benefit comes from goal-directed
+  re-derivation of the query relation under the seed (which still
+  matters for large recursive joins). The existing test suite's
+  aggregate cases happened to dodge this because the inner goals
+  were all EDB. 1 new regression test; 274/274.
+
+- 2026-05-11 — Anonymous `_` in a negated literal was incorrectly
+  flagged by the safety check. The canonical idiom
+  `orphan(X) :- person(X), not(parent(X, _))` was rejected with
+  "negation refers to unbound variable(s) (\"_anon1\")" because the
+  parser renames each `_` to a fresh `_anon*` symbol and the negation
+  safety walk demanded all vars in the negated lit be bound by an
+  earlier positive body literal. Anonymous vars in negation are
+  existentially quantified — they shouldn't need outer binding.
+  Added `dl-non-anon-vars` filter; `dl-process-neg!` now strips
+  `_anon*` names from `needed` before the binding check. 2 new
+  regression tests; 273/273.
+
+- 2026-05-11 — Compound terms in fact-arg / rule-head positions were
+  silently stored as unreduced expressions. `p(+(1, 2)).` resulted
+  in a tuple `(p (+ 1 2))` (dl-ground? sees no free variables, so it
+  passed). `double(*(X, 2)) :- n(X).` saturated to `double((* 3 2))`
+  rather than `double(6)`. Datalog has no function symbols in arg
+  positions — `dl-add-fact!` and `dl-add-rule!` now reject compound
+  args via a new `dl-simple-term?` (number / string / symbol).
+  Compounds remain legal in body literals where they encode `is` /
+  arithmetic / aggregate sub-goals. 2 new regression tests; 271/271.
+
+- 2026-05-11 — Quoted atoms with uppercase-or-underscore-leading
+  names were misclassified as variables. `p('Hello World').` ran
+  through the tokenizer's `"atom"` branch and through the parser's
+  `string->symbol`, producing a symbol named "Hello World". dl-var?
+  checks the first character — "H" is uppercase, so the fact was
+  rejected as non-ground. Fix: tokenizer emits `"string"` for any
+  `'...'` quoted form, so quoted atoms become opaque string constants
+  (matching how Datalog idiomatically treats them — the alternative
+  was a per-symbol "quoted" marker which would have rippled through
+  unification and dl-var?). Updated the existing tokenize test and
+  added one for `'Hello'`; also added a parse-level regression. 269/269.
+
+- 2026-05-11 — Type-mixed comparisons were silently inconsistent:
+  `<(X, 5)` with `X` bound to a string returned `()` (no result, no
+  error), while `X` bound to a symbol raised "Expected number, got
+  symbol". Both should fail loudly. Added `dl-compare-typeok?` —
+  `<`, `<=`, `>`, `>=` now require both operands to share a primitive
+  type (both numbers or both strings) and raise otherwise. `!=` is
+  exempted since it's a polymorphic inequality test built on
+  `dl-tuple-equal?`. 2 new regression tests; 267/267.
+
+- 2026-05-11 — Body literal shape validation in
+  `dl-rule-check-safety`: a dict that isn't `{:neg ...}` (e.g. typo'd
+  `{:negs ...}`) used to silently fall through every dispatch clause,
+  contributing zero bound vars; the user would then see a confusing
+  "head var X unbound" error pointing at the head, not the malformed
+  body. Same for body lits that are bare numbers / strings / symbols.
+  Both shapes now raise a clear error naming the offending lit. 1 new
+  regression test; 265/265.
+
+- 2026-05-11 — Division by zero in `is` silently produced IEEE
+  infinity instead of raising. `is(R, /(X, 0))` returned `R = inf`,
+  which then flowed through comparisons and aggregations to produce
+  nonsense results. `dl-eval-arith` now raises with a clear
+  "division by zero in <expr>" message. 1 new test; 264/264.
+
+- 2026-05-11 — Aggregate variable validation: `count(N, Y, p(X))`
+  silently returned `N = 1` because `Y` was never bound in `p(X)` —
+  every match contributed the same unbound symbol, which dl-val-member?
+  deduped to a single entry. Similarly `sum(S, Y, p(X))` raised a
+  confusing "expected number" error from the underlying `+`. Added
+  a third validator in `dl-eval-aggregate`: the agg-var must appear
+  in the goal literal. Error names the variable and the goal and
+  explains the consequence. 1 new test; 263/263.
+
+- 2026-05-11 — `dl-retract!` was silently destroying EDB facts in
+  "mixed" relations (those with BOTH user-asserted facts AND a rule
+  defining the same head). The retract pass wiped every rule-head
+  relation wholesale and then re-saturated — but the saturator only
+  re-derives the IDB portion, so explicit EDB facts vanished even
+  for a no-op retract of a non-existent tuple. Probe:
+  `(let ((db (dl-program "p(a). p(b). p(X) :- q(X). q(c).")))
+     (dl-retract! db (quote (p z))) (dl-query db (quote (p X))))`
+  went from `{a,b,c}` to just `{c}`.
+  Fix: tracked `:edb-keys` provenance in the db. `dl-add-fact!` (public
+  API) marks the tuple as EDB; saturator calls new internal
+  `dl-add-derived!` which doesn't mark it. `dl-retract!` now strips
+  only the IDB-derived portion of rule-head relations and preserves
+  EDB-marked tuples through the re-saturate pass. 2 new regression
+  tests; 262/262.
+
+- 2026-05-11 — Eval-semantics bug-hunt: nested `not(not(P))` was
+  silently misinterpreted. Outer-level `not(...)` is parsed as
+  negation, but the inner `not(banned(X))` was parsed as a regular
+  positive literal naming a relation called `not`. Since no `not`
+  relation existed, the inner match was empty and the outer
+  negation succeeded vacuously, making `vip(X) :- u(X), not(not(banned(X))).`
+  equivalent to `vip(X) :- u(X).` (a silent double-negation = identity
+  fallacy). Fix in `dl-rule-check-safety`: both the positive-literal
+  branch and `dl-process-neg!` now flag any body literal whose head
+  is in `dl-reserved-rel-names`. Error message names the relation and
+  points the user at intermediate-relation stratified negation. 1 new
+  regression test; 260/260.
+
+- 2026-05-10 — Bug-hunt round on parser/safety surfaced 7 real
+  bugs, each fixed with regression tests:
+  - Reserved relation names (`not`, `count`, `<`, `is`, ...) were
+    accepted as rule/fact heads — would silently shadow built-ins.
+  - Negative number literals (`n(-1).`) failed to parse — users
+    had to express them as `(- 0 1)` or via `is`.
+  - Unterminated block comment `/* ...` silently consumed the
+    rest of the input. Now raises with the position.
+  - Same silent-consume bug in unterminated string / quoted-atom.
+  - Empty-list rule head and non-list rule body weren't validated;
+    they'd crash later in `rest`. dl-add-rule! now checks shape.
+  - dl-magic-query with non-list / non-dict goal crashed cryptically.
+  - Tokenizer silently swallowed unrecognised characters (`?`, `!`,
+    `#`, `@`, etc.) — typos produced confusing downstream errors.
+
+- 2026-05-08 — Phase 6 driver: `dl-magic-query db query-goal`.
+  Builds a fresh internal db from the caller's EDB + magic seed +
+  rewritten rules, saturates, queries, returns substitutions —
+  caller's db is untouched. Equivalent to `dl-query` for any
+  fully-stratifiable program; sole motivation is a perf alternative
+  on goal-shaped queries against large recursive relations.
+  2 new tests cover equivalence and non-mutation.
+
+- 2026-05-08 — Phase 6 magic-sets rewriter. `dl-magic-rewrite rules
+  query-rel adn args` returns `{:rules <rewritten> :seed <seed-fact>}`.
+  Worklist over `(rel, adn)` pairs starts from the query, gates each
+  original rule with a `magic_<rel>^<adn>(bound)` filter, and emits
+  propagation rules for each positive non-builtin body literal so
+  that magic spreads to body relations. EDB facts pass through.
+  3 new tests cover seed structure, equivalence on chain-3 by
+  ancestor-relation tuple count, and same-query-answers under
+  the rewritten program. The plumbing for a `dl-saturate-magic!`
+  driver and large-graph perf benchmarks is still future work.
+
+- 2026-05-08 — Phase 6 building blocks for the magic-sets
+  transformation: `dl-magic-rel-name`, `dl-magic-lit`,
+  `dl-bound-args`. The rewriter that generates magic seed and
+  propagation rules is still future work; with these primitives
+  in place it's a straightforward worklist algorithm. 4 new tests.
+
+- 2026-05-08 — Phase 6 adornments + SIPS in
+  `lib/datalog/magic.sx`. Inspection helpers — `dl-adorn-goal` and
+  `dl-adorn-lit` compute per-arg `b`/`f` patterns under a bound
+  set; `dl-rule-sips rule head-adornment` walks body literals
+  left-to-right propagating the bound set, recognising `is` and
+  aggregate result-vars as new binders. Lays groundwork for a
+  later magic-sets transformation. 10 new tests cover pure
+  adornment, SIPS over a chain rule, head-fully-bound rules,
+  comparisons, and `is`. Saturator does not yet consume these.
+
+- 2026-05-08 — Comprehensive integration test in api suite: a
+  single program exercising recursion (`reach` transitive closure)
+  + stratified negation (`safe X Y :- reach X Y, not banned Y`) +
+  aggregation (`reach_count` via count) + comparison (`>= N 2`)
+  composed end-to-end via `dl-eval source query-source`. Confirms
+  the full pipeline (parser → safety → stratifier → semi-naive +
+  aggregate post-pass → query) on a non-trivial program.
+
+- 2026-05-08 — Bug fix: aggregates work as top-level query goals.
+  `dl-match-lit` (the naive matcher used by `dl-find-bindings`) was
+  missing the `dl-aggregate?` dispatch — it was only present in
+  `dl-fbs-aux` (semi-naive). Symptom: `(dl-query db '(count N X (p X)))`
+  silently returned `()`. Also updated `dl-query-user-vars` to project
+  only the result var (first arg) of an aggregate goal — the
+  aggregated var and inner-goal vars are existentials and should not
+  appear in the projected substitution. 2 new aggregate tests cover
+  the regression.
+
+- 2026-05-08 — Convenience: `dl-eval source query-source`. Parses
+  both strings, builds a db, saturates, runs the query, returns
+  the substitution list. Single-call user-friendly entry. 2 new
+  api tests cover ancestor and multi-goal queries.
+
+- 2026-05-08 — Phase 6 stub: `dl-set-strategy! db strategy` and
+  `dl-get-strategy db` user-facing hooks. Default `:semi-naive`;
+  `:magic` is accepted but the actual transformation is deferred,
+  so saturation still uses semi-naive. Lets us tick the
+  "Optional pass — guarded behind dl-set-strategy!" Phase 6 box.
+  3 new eval tests.
+
+- 2026-05-08 — Demo: weighted-DAG shortest path. `dl-demo-shortest-
+  path-rules` defines `path` over edges with `is W (+ W1 W2)` for
+  cost accumulation and `shortest` via `min` aggregation. 3 demo
+  tests cover direct/multi-hop choice, multi-hop wins on cheaper
+  route, and unreachable-empty. Added `dl-summary db` inspection
+  helper returning `{<rel>: count}` (4 eval tests).
+
+- 2026-05-08 — Phase 5e perf: first-arg index per relation. db gains
+  `:facts-index {<rel>: {<first-arg-key>: tuples}}` mirroring the
+  existing `:facts-keys` membership index. `dl-add-fact!` populates
+  it; `dl-match-positive` walks the body literal's first arg under
+  the current subst — if it's bound to a non-var, look up by
+  `(str arg)` and iterate only the matching subset. chain-25
+  saturation 33s → 18s real (~2x). chain-50 still slow (~120s+)
+  but tractable; next bottleneck is subst dict copies during
+  unification. Differential test bumped to chain-12, semi-only
+  count to chain-25.
+
+- 2026-05-08 — Demo: tag co-occurrence. `(cotagged P T1 T2)` — post
+  has both T1 and T2 with T1 != T2 — and `(tag-pair-count T1 T2 N)`
+  counting posts per distinct tag pair. Demonstrates count
+  aggregation grouped by outer-context vars. 2 new demo tests.
+
+- 2026-05-08 — `dl-query` accepts a list of body literals for
+  conjunctive queries, in addition to a single positive literal.
+  `dl-query-coerce` dispatches based on the first element's shape:
+  positive lit (head is a symbol) or `:neg` dict → wrap as singleton;
+  list of lits → use as-is. `dl-query-user-vars` collects the union
+  of vars across all goals (deduped, `_` filtered) for projection.
+  2 new api tests: multi-goal AND, and conjunction with comparison.
+
+- 2026-05-08 — Bug fix: `dl-check-stratifiable` now rejects recursion
+  through aggregation (e.g., `q(N) :- count(N, X, q(X))`). The
+  stratifier was already adding negation-like edges for aggregates,
+  but the cycle scan only looked at explicit `:neg` literals. Added
+  the matching aggregate branch to the body iteration. Also adds
+  doc-only `lib/datalog/datalog.sx` with the public-API surface
+  (since `load` is an epoch command and can't recurse from within an
+  `.sx` file). 3 new aggregate tests cover recursion-rejection,
+  negation-and-aggregation coexistence, and min-over-empty-derived.
+
+- 2026-05-08 — Phase 10 demo + canonical query. Added the "cooking
+  posts by people I follow (transitively)" example from the plan:
+  `dl-demo-cooking-rules` defines `reach` over the follow graph
+  (recursive transitive closure) and `cooking-post-by-network` that
+  joins reach with `authored` and `(tagged P cooking)`. 3 demo
+  tests cover transitive network, direct-only follow, and
+  empty-network cases.
+
+- 2026-05-08 — Phase 8 extension: `findall L V Goal` aggregate. Bind
+  L to the list of distinct V values for which Goal holds (or the
+  empty list when no matches). Implemented as a one-line case in
+  `dl-do-aggregate`. 3 new tests: EDB, derived relation, empty.
+  Useful for "give me all the X such that …" queries without
+  scalar reduction.
+
+- 2026-05-08 — Phase 5d semantic fix: anonymous `_` variables are
+  renamed per occurrence at `dl-add-rule!` and `dl-query` time so
+  `(p X _) (p _ Y)` no longer unifies the two `_`s. New helpers
+  `dl-rename-anon-term`, `dl-rename-anon-lit`, `dl-make-anon-renamer`,
+  `dl-rename-anon-rule` in db.sx; eval.sx's dl-query renames the goal
+  before search and projects only user-named vars (`_` is filtered
+  out of the projection list). The "underscore in head" test now
+  correctly rejects `(p X _) :- q(X).` — after renaming, the head's
+  fresh anon var has no body binder. Two new eval tests verify
+  rule-level and goal-level independence. 155/155 expected.
+
+- 2026-05-08 — Phase 5c perf: indexed `dl-find-bindings`. Replaced
+  the recursive `(rest lits)` walk with `dl-fb-aux lits db subst i n`
+  using `nth lits i`. Eliminates O(N²) list-copy per body of length
+  N. chain-15 saturation 25s → 16s; chain-25 finishes in 33s real
+  (vs. timeout previously). Bumped semi_naive tests: differential
+  on chain-10, semi-only count on chain-15 (was chain-5/chain-5).
+  153/153.
+
+- 2026-05-08 — Phase 10 syntactic demo. New `lib/datalog/demo.sx`
+  with three programs over rose-ash-shaped data: federation
+  (`mutual`, `reachable`, `foaf`), content recommendation
+  (`post-likes` via count aggregation, `popular`, `interesting`),
+  and role-based permissions (`in-group` over transitive subgroups,
+  `can-access`). 10 demo tests pass against synthetic EDB tuples.
+  Postgres loader and `/internal/datalog` HTTP endpoint remain
+  out of scope for this loop (they need service-tree edits beyond
+  `lib/datalog/**`). Conformance now 153/153.
+
+- 2026-05-08 — Phase 5b perf: hash-set membership in `dl-add-fact!`.
+  db gains a parallel `:facts-keys {<rel>: {<tuple-string>: true}}`
+  index alongside `:facts`. `dl-tuple-key` derives a stable string
+  key via `(str lit)` — `(p 30)` and `(p 30.0)` collide correctly
+  because SX prints them identically. Insertion is O(1) instead of
+  O(n). chain-7 saturation drops from ~12s to ~6s; chain-15 from
+  ~50s to ~25s under shared CPU. Larger chains are still slow due
+  to body-join overhead in dl-find-bindings (Blocker updated).
+  `dl-retract!` updated to keep both indices consistent. 143/143.
+
+- 2026-05-08 — Phase 9 done. New `lib/datalog/api.sx` exposes a
+  parser-free embedding: `dl-program-data facts rules` accepts SX
+  data lists, with rules in either dict form or list form using
+  `<-` as the rule arrow (since SX parses `:-` as a keyword).
+  `dl-rule head body` constructs the dict. `dl-assert! db lit` adds
+  a fact and re-saturates; `dl-retract! db lit` drops the fact from
+  EDB, wipes all rule-headed (IDB) relations, and re-saturates from
+  scratch — the simplest correct semantics until provenance tracking
+  arrives in a later phase. 9 API tests; conformance now 143/143.
+
+- 2026-05-08 — Phase 8 done. New `lib/datalog/aggregates.sx` (~110
+  LOC): count / sum / min / max. Each is a body literal of shape
+  `(op R V Goal)` — `dl-eval-aggregate` runs `dl-find-bindings` on
+  the goal under the outer subst (so outer vars in the goal get
+  substituted, giving group-by-style aggregation), collects the
+  distinct values of `V`, and binds `R`. Empty input: count/sum
+  return 0; min/max produce no binding (rule fails). Stratifier
+  extended via `dl-aggregate-dep-edge` so the aggregate's goal
+  relation is fully derived before the aggregate fires. Safety check
+  treats goal-internal vars as existentials (no outer binding
+  required); only the result var becomes bound. Conformance now
+  134 / 134.
+
+- 2026-05-08 — Phase 7 done (Phase 6 magic sets deferred — opt-in,
+  semi-naive default suffices for current test suite). New
+  `lib/datalog/strata.sx` (~290 LOC): dep graph build, Floyd-Warshall
+  reachability, SCC-via-mutual-reachability for non-stratifiability
+  detection, iterative stratum computation, rule grouping by head
+  stratum. eval.sx split: `dl-saturate-rules!` is the per-rule-set
+  semi-naive worker, `dl-saturate!` is now the stratified driver
+  (errors out on non-stratifiable programs). `dl-match-negation` in
+  eval.sx: succeeds iff inner positive match is empty. Stratum-keyed
+  dicts use `(str s)` since SX dicts only accept string/keyword keys.
+  10 negation tests cover EDB/IDB negation, multi-level strata,
+  non-stratifiability rejection, and a negation safety violation.
+
+- 2026-05-08 — Phase 5 done. `lib/datalog/eval.sx` rewritten to
+  semi-naive default. `dl-saturate!` tracks a per-relation delta and
+  on each iteration walks every positive body position substituting
+  delta for that one literal — joining the rest against the full DB
+  snapshot. `dl-saturate-naive!` retained as the reference. Rules
+  with no positive body literal (e.g. `(p X) :- (= X 5).`) fall back
+  to a naive one-shot via `dl-collect-rule-candidates`. 8 tests
+  differentially compare the two saturators using per-relation tuple
+  counts (cheap). Chain-5 differential exercises multi-iteration
+  recursive saturation. Larger chains made conformance.sh time out
+  due to O(n) `dl-tuple-member?` × CPU sharing with other loop
+  agents — added a Blocker to swap to a hash-set for membership.
+  Also tightened `dl-tuple-member?` to use indexed iteration instead
+  of recursive `rest` (was creating a fresh list per step).
+
+- 2026-05-07 — Phase 4 done. `lib/datalog/builtins.sx` (~280 LOC) adds
+  `(< X Y)`, `(<= X Y)`, `(> X Y)`, `(>= X Y)`, `(= X Y)`, `(!= X Y)`,
+  and `(is X expr)` with `+ - * /`. `dl-eval-builtin` dispatches;
+  `dl-eval-arith` recursively evaluates nested compounds. Safety
+  check is now order-aware — it walks body literals left-to-right
+  tracking the bound set, requires comparison/`is` inputs to be
+  already bound, and special-cases `=` (binds the var-side; both
+  sides must include at least one bound to bind the other). Phase 3's
+  simple safety check stays in db.sx as a forward-reference fallback;
+  builtins.sx redefines `dl-rule-check-safety` to the comprehensive
+  version. eval.sx's `dl-match-lit` now dispatches built-ins through
+  `dl-eval-builtin`. 19 builtins tests; conformance 106 / 106.
+
+- 2026-05-07 — Phase 3 done. `lib/datalog/db.sx` (~250 LOC) holds facts
+  indexed by relation name plus the rules list, with `dl-add-fact!` /
+  `dl-add-rule!` (rejects non-ground facts and unsafe rules);
+  `lib/datalog/eval.sx` (~150 LOC) implements the naive bottom-up
+  fixpoint via `dl-find-bindings`/`dl-match-positive`/`dl-saturate!`
+  and `dl-query` (deduped projected substitutions). Safety analysis
+  rejects unsafe head vars at load time. Negation and arithmetic
+  built-ins raise clean errors (lifted in later phases). 15 eval
+  tests cover transitive closure, sibling, same-generation, cyclic
+  graph reach, and six safety violations. Conformance 87 / 87.
+
+- 2026-05-07 — Phase 2 done. `lib/datalog/unify.sx` (~140 LOC):
+  `dl-var?` (case + underscore), `dl-walk`, `dl-bind`, `dl-unify` (returns
+  extended dict subst or `nil`), `dl-apply-subst`, `dl-ground?`, `dl-vars-of`.
+  Substitutions are immutable dicts; `assoc` builds extended copies. 28
+  unify tests; conformance now 72 / 72.
+
+- 2026-05-07 — Phase 1 done. `lib/datalog/tokenizer.sx` (~190 LOC) emits
+  `{:type :value :pos}` tokens; `lib/datalog/parser.sx` (~150 LOC) produces
+  `{:head … :body …}` / `{:query …}` clauses, with nested compounds
+  permitted for arithmetic and `not(...)` desugared to `{:neg …}`. 44 / 44
+  via `bash lib/datalog/conformance.sh` (26 tokenize + 18 parse). Local
+  helpers namespace-prefixed (`dl-emit!`, `dl-peek`) after a host-primitive
+  shadow clash. Test harness uses a custom `dl-deep-equal?` that handles
+  out-of-order dict keys and number repr (`equal?` fails on dict key order
+  and on `30` vs `30.0`).

plans/lib-guest.md

@@ -155,11 +155,11 @@ Extract from `haskell/infer.sx`. Algorithm W or J, generalisation, instantiation
 | 1 — conformance.sx (prolog + haskell) | [done] | 58dcff26 | Prolog 590/590 (matches baseline). Haskell 156/156 — old script was broken (0/18 was an artefact of a never-matching grep), driver reveals true counts; baseline updated. |
 | 2 — prefix.sx (common-lisp + lua) | [partial — pending lua] | 2ef773a3 | common-lisp/runtime.sx ported (47 aliases collapsed into 13 prefix-rename calls); 518/518 vs 309/309 baseline (improvement, no regression). lua/runtime.sx has no pure same-name aliases — every lua- definition wraps custom logic; second consumer pending. |
 | 3 — lex.sx (lua + tcl) | [done] | 559b0df9 | lex.sx exports nil-safe char-class predicates + token record. lua/tokenizer.sx (7 preds) and tcl/tokenizer.sx (5 preds) collapsed into prefix-rename calls. lua 185/185, tcl 342/342, tcl-conf 3/4 — all = baseline. |
-| 4 — pratt.sx (lua + prolog) | [in-progress] | — | — |
+| 4 — pratt.sx (lua + prolog) | [done] | da27958d | Extracted operator-table format + lookup only — climbing loops stay per-language because lua and prolog use opposite prec conventions. lua/parser.sx: 18-clause cond → 15-entry table. prolog/parser.sx: pl-op-find deleted, pl-op-lookup wraps pratt-op-lookup. lua 185/185, prolog 590/590 — both = baseline. |
-| 5 — ast.sx (lua + prolog) | [ ] | — | — |
+| 5 — ast.sx (lua + prolog) | [partial — pending real consumers] | a774cd26 | Kit + 33 self-tests shipped (10 canonical kinds, predicates, accessors). Step is "Optional" per brief; lua/prolog parsers untouched (185/185 + 590/590). Datalog-on-sx will be the natural first real consumer; lua/prolog converters can land later. |
-| 6 — match.sx (haskell + prolog) | [ ] | — | — |
+| 6 — match.sx (haskell + prolog) | [partial — kit shipped; ports deferred] | 863e9d93 | Pure-functional unify + match kit (canonical wire format + cfg-driven adapters) + 25 self-tests. Existing prolog/haskell engines untouched (structurally divergent — mutating-symmetric vs pure-asymmetric — would risk 746 passing tests under brief's revert-on-regression rule). Real consumer is minikraken/datalog work in flight. |
-| 7 — layout.sx (haskell + synthetic) | [ ] | — | — |
+| 7 — layout.sx (haskell + synthetic) | [partial — haskell port deferred] | d75c61d4 | Configurable kit (haskell-style keyword-opens + python-style trailing-`:`-opens) + 6 self-tests covering both flavours. Synthetic Python-ish fixture passes; haskell/layout.sx untouched (kit not yet a drop-in for Haskell 98 Note 5 etc.; haskell still 156/156 baseline). |
-| 8 — hm.sx (haskell + TBD) | [ ] | — | — |
+| 8 — hm.sx (haskell + TBD) | [partial — algebra shipped; assembly deferred] | ab2c40c1 | HM foundations: types/schemes/ftv/apply/compose/generalize/instantiate/fresh-tv on top of match.sx unify, plus literal inference rule. 24/24 self-tests. Algorithm W lambda/app/let assembly deferred to host code — paired sequencing per brief: lib/ocaml/types.sx (OCaml-on-SX Phase 5) + haskell/infer.sx port. Haskell still 156/156 baseline. |
 
 ---
 

plans/tcl-sx-completion.md

@@ -191,6 +191,106 @@ vwait-timing. 354/354 green.**
 
 ---
 
+## Phase 5c — TCP sockets (client + server) ✓
+
+Tcl `socket` command for both connecting and listening. Reuses the channel
+registry built in Phase 5 and the event loop from Phase 5b. Server channels
+auto-fire user callbacks via fileevent on each accept.
+
+| Status | Work | Unlocks in Tcl |
+|---|---|---|
+| [x] | `socket-connect host port` SX primitive | TCP client via `Unix.socket`+`Unix.connect` |
+| [x] | `socket-server ?host? port` SX primitive | listening socket; `Unix.bind`+`Unix.listen` (backlog 8) |
+| [x] | `socket-accept server-chan` SX primitive | returns `{:channel :host :port}` |
+| [x] | Tcl `socket host port` | TCP client; returns "sockN" |
+| [x] | Tcl `socket -server cb port` | listening socket; auto-fires `cb sock host port` per accept |
+| [x] | `puts` channel detection extended | "sockN" channels also dispatch to `channel-write` |
+
+The auto-accept mechanism is a tiny internal Tcl command `_sock-do-accept`
+registered by `socket -server`. Its registered handler, fired by the event
+loop, accepts the pending client, then evaluates `cb client-chan host port`.
+
+`Unix.SO_REUSEADDR` is set on server sockets to avoid TIME_WAIT issues
+during testing. Host argument supports `localhost`, `0.0.0.0`, IPv4 literal,
+or DNS lookup via `Unix.gethostbyname`.
+
+**Total: ~half day. 4 new idiom tests: socket-server-fires-callback,
+socket-client-server-roundtrip, socket-server-peer-host, socket-multiple-
+connections. 358/358 green.**
+
+---
+
+## Phase 5d — File metadata + filesystem ops ✓
+
+Real implementations of `file isfile`/`isdir`/`readable`/`writable`/`size`/
+`mtime`/`atime`/`type` (previously stubs returning `0`/`""`) and proper
+`file delete`/`mkdir`/`copy`/`rename`.
+
+| Status | Primitive | Wraps |
+|---|---|---|
+| [x] | `file-size`, `file-mtime`, `file-stat` | `Unix.stat` |
+| [x] | `file-isfile?`, `file-isdir?` | `Unix.stat`+`st_kind` |
+| [x] | `file-readable?`, `file-writable?` | `Unix.access [R_OK\|W_OK]` |
+| [x] | `file-delete` | `Unix.unlink`/`rmdir` (tolerates ENOENT) |
+| [x] | `file-mkdir` | recursive `Unix.mkdir 0o755` |
+| [x] | `file-copy`, `file-rename` | stdlib I/O / `Sys.rename` |
+
+`file-stat` returns a dict `{:size :mtime :atime :ctime :mode :type}` with
+`:type` ∈ `file|directory|link|fifo|socket|...`. Tcl `file copy`/`rename`/
+`delete` strip leading-`-` flags so `file delete -force` works.
+
+**Total: ~half day. 10 new idiom tests covering isfile, isdir on /tmp, size,
+readable, mkdir + check, copy roundtrip, rename, mtime > 0. 368/368 green.**
+
+---
+
+## Phase 5e — clock format options + clock scan ✓
+
+Real `-format`, `-timezone`, and `-gmt` options on `clock format`, and a
+working `clock scan` for parsing date strings back to Unix seconds.
+
+| Status | Work |
+|---|---|
+| [x] | `clock-format` extended to `(t fmt tz)` with tz ∈ `utc|local` |
+| [x] | More format specifiers: `%y` (2-digit year), `%I` (12h hour), `%p` (AM/PM), `%w` (weekday num), `%%` (literal) |
+| [x] | `clock-scan` SX primitive: format-driven parser + manual `timegm` (OCaml stdlib lacks it) |
+| [x] | Tcl `clock format $secs -format $fmt -timezone $tz -gmt 0\|1` |
+| [x] | Tcl `clock scan $str -format $fmt -timezone $tz -gmt 0\|1` |
+
+Default tz for both is UTC. Format specifiers supported by scan: `%Y %y %m
+%d %e %H %I %M %S %%`. Unsupported specifiers in scan are silently skipped
+(no validation).
+
+**Total: ~half day. 5 new idiom tests: clock-format-utc, fmt-default,
+scan-roundtrip, scan-returns-int, format-percent-pct. 373/373 green.**
+
+---
+
+## Phase 5f — `socket -async` (non-blocking connect) ✓
+
+| Status | Work |
+|---|---|
+| [x] | `socket-connect-async host port` SX primitive — `Unix.set_nonblock` + `Unix.connect`, catches `EINPROGRESS` |
+| [x] | `channel-async-error chan` SX primitive — `Unix.getsockopt_error` |
+| [x] | Tcl `socket -async host port` — returns "sockN" immediately |
+| [x] | Tcl `fconfigure $chan -error` — queries async-error |
+
+Connection completes when the channel becomes writable; canonical pattern is
+`fileevent $sock writable {handler}`. Channel buffer state is set to
+`blocking=false` so subsequent reads/writes don't block.
+
+**Total: ~few hours. 3 new idiom tests: socket-async-completes-writable,
+socket-async-then-write, socket-async-no-error. 376/376 green.**
+
+**Bug fix landed alongside:** `tcl-call-proc` was discarding `:fileevents`,
+`:timers`, and `:procs` updates made inside Tcl procs (only `:commands` was
+forwarded). Changed the return to forward the inner `result-interp` as the
+base while restoring caller's frame/stack/result/output/code. This was
+masked until socket -async made it natural to register a `fileevent` from
+inside a proc body (the typical async accept pattern).
+
+---
+
 ## Suggested order
 
 1. **Phase 1** — immediate Tcl wins, zero risk, proves the approach
@@ -207,6 +307,10 @@ becomes a lasting SX contribution used by every future hosted language.
 
 _Newest first._
 
+- 2026-05-07: Phase 5f socket -async — socket-connect-async (Unix.set_nonblock+connect/EINPROGRESS) + channel-async-error (getsockopt_error); Tcl `socket -async host port` returns immediately; `fconfigure $sock -error` queries async error; +3 idiom tests; 376/376 green
+- 2026-05-07: Phase 5e clock options + scan — clock-format extended with tz arg (utc/local) + more specifiers; new clock-scan primitive with manual timegm; Tcl clock format/scan support -format/-timezone/-gmt; +5 idiom tests; 373/373 green
+- 2026-05-07: Phase 5d file ops — file-size/mtime/isfile?/isdir?/readable?/writable?/stat/delete/mkdir/copy/rename SX primitives; Tcl file isfile/isdir/readable/writable/size/mtime/atime/type/mkdir/copy/rename/delete now real; +10 idiom tests; 368/368 green
+- 2026-05-07: Phase 5c sockets — socket-connect/socket-server/socket-accept SX primitives wrapping Unix.socket/connect/bind/listen/accept; tcl-cmd-socket dispatches client (host port) vs server (-server cb port); server auto-registers fileevent → _sock-do-accept handler that calls user callback per accept; puts now dispatches "sockN" channels to channel-write too; +4 idiom tests; 358/358 green
 - 2026-05-07: Phase 5b event loop — io-select-channels SX primitive + Tcl-side fileevent/after/vwait/update; tcl-event-step drives expired timers + Unix.select on registered channels; +5 idiom tests; 354/354 green
 - 2026-05-07: Phase 5 channel I/O — 11 SX primitives (channel-open/close/read/read-line/write/flush/seek/tell/eof?/blocking?/set-blocking!) wrapping Unix.openfile/read/write/lseek/set_nonblock; tcl-cmd-open/close/read/gets-chan/seek/tell/flush rewritten + new tcl-cmd-fconfigure; tcl-cmd-puts dispatches on "fileN" arg; gets registration fixed; +7 idiom tests; 349/349 green
 - 2026-05-06: Phase 4 env-as-value — current-env (special form via Sx_ref.register_special_form), eval-in-env (primitive in setup_evaluator_bridge), env-lookup + env-extend (in setup_env_operations); 5 idiom tests; 342/342 green
@@ -222,8 +326,8 @@ _Newest first._
 
 ## What stays out of scope
 
-- `package require` of binary loadables
+- `package require` of binary loadables (would need `Dynlink` + native ABI design)
-- Full `clock format` locale support
+- Full `clock format` locale (translated month/day names, `LC_TIME`-aware) — Phase 5e covers `-format`/`-timezone`/`-gmt` with English names
 - Tk / GUI
 - Threads (mapped to coroutines only, as planned)
 - Server-mode `vwait` — Phase 5b event loop is scoped to script-mode; from inside a server-handled command it can't see sx_server's stdin scheduler