lua: re-apply arch's GUEST-lex prefix-rename refactor on top of merged loops/lua

Lua now joins tcl/ocaml/kernel/common-lisp in consuming lib/guest/lex.sx via
prefix-rename. Removes 28 lines of duplicated character-class helpers
(lua-make-token, lua-digit?, lua-hex-digit?, lua-letter?, lua-ident-start?,
lua-ident-char?, lua-ws?) and replaces with the 8-line prefix-rename block.

The byte-table additions from loops/lua (__ascii-tok, __lua-127-255-tok,
lua-byte-to-char) are preserved at the top of tokenizer.sx — those provide
Lua's 8-bit-clean string semantics on top of the shared lex layer.

test.sh updated to preload lib/guest/lex.sx + lib/guest/prefix.sx before
lua sources, matching the load order arch's pre-merge test.sh used.

393/395 maintained. The 2 pre-existing failures are unrelated:
- math.random(n) primitive arity issue
- os.clock returns rational instead of number (SX division semantics)

Skipped from the planned follow-up: delay/force port. Arch's lua-force was
defined but never referenced anywhere — dead code, not worth porting.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

.claude/scheduled_tasks.lock

@@ -0,0 +1 @@
+{"sessionId":"31c80255-eb92-43e4-8997-84ad84e27326","pid":90960,"procStart":"564684","acquiredAt":1777049890282}

hosts/ocaml/bin/run_tests.ml

@@ -1279,7 +1279,7 @@ let run_foundation_tests () =
   assert_true "sx_truthy \"\"" (Bool (sx_truthy (String "")));
   assert_eq "not truthy nil" (Bool false) (Bool (sx_truthy Nil));
   assert_eq "not truthy false" (Bool false) (Bool (sx_truthy (Bool false)));
-  let l = { l_params = ["x"]; l_body = Symbol "x"; l_closure = Sx_types.make_env (); l_name = None; l_compiled = None } in
+  let l = { l_params = ["x"]; l_body = Symbol "x"; l_closure = Sx_types.make_env (); l_name = None; l_compiled = None; l_call_count = 0; l_uid = Sx_types.next_lambda_uid () } in
   assert_true "is_lambda" (Bool (Sx_types.is_lambda (Lambda l)));
   ignore (Sx_types.set_lambda_name (Lambda l) "my-fn");
   assert_eq "lambda name mutated" (String "my-fn") (lambda_name (Lambda l))

hosts/ocaml/browser/sx_browser.ml

@@ -676,7 +676,11 @@ let () =
   let rec deep_equal a b =
     match a, b with
     | Nil, Nil -> true | Bool a, Bool b -> a = b
-    | Number a, Number b -> a = b | String a, String b -> a = b
+    | Integer a, Integer b -> a = b
+    | Number a, Number b -> a = b
+    | Integer a, Number b -> float_of_int a = b
+    | Number a, Integer b -> a = float_of_int b
+    | String a, String b -> a = b
     | Symbol a, Symbol b -> a = b | Keyword a, Keyword b -> a = b
     | (List a | ListRef { contents = a }), (List b | ListRef { contents = b }) ->
       List.length a = List.length b && List.for_all2 deep_equal a b

hosts/ocaml/lib/sx_primitives.ml

@@ -528,6 +528,183 @@ let () =
     | [Rational (_, d)] -> Integer d
     | [Integer _]       -> Integer 1
     | _ -> raise (Eval_error "denominator: expected rational or integer"));
+  (* printf-spec: apply one Tcl/printf format spec to one arg.
+     spec is like "%5.2f", "%-10s", "%x", "%c", "%d". Always starts with %
+     and ends with the conversion char.  Supports d i u x X o c s f e g.
+     Coerces arg to the right type per conversion. *)
+  register "printf-spec" (fun args ->
+    let spec_str, arg = match args with
+      | [String s; v] -> (s, v)
+      | _ -> raise (Eval_error "printf-spec: (spec arg)")
+    in
+    let n = String.length spec_str in
+    if n < 2 || spec_str.[0] <> '%' then
+      raise (Eval_error ("printf-spec: invalid spec " ^ spec_str));
+    let type_char = spec_str.[n - 1] in
+    let to_int v = match v with
+      | Integer i -> i
+      | Number f -> int_of_float f
+      | String s ->
+        let s = String.trim s in
+        (try int_of_string s
+         with _ ->
+           try int_of_float (float_of_string s)
+           with _ -> 0)
+      | Bool true -> 1 | Bool false -> 0
+      | _ -> 0
+    in
+    let to_float v = match v with
+      | Number f -> f
+      | Integer i -> float_of_int i
+      | String s ->
+        let s = String.trim s in
+        (try float_of_string s with _ -> 0.0)
+      | _ -> 0.0
+    in
+    let to_string v = match v with
+      | String s -> s
+      | Integer i -> string_of_int i
+      | Number f -> Sx_types.format_number f
+      | Bool true -> "1" | Bool false -> "0"
+      | Nil -> ""
+      | _ -> Sx_types.inspect v
+    in
+    try
+      match type_char with
+      | 'd' | 'i' ->
+        let fmt = Scanf.format_from_string spec_str "%d" in
+        String (Printf.sprintf fmt (to_int arg))
+      | 'u' ->
+        let fmt = Scanf.format_from_string spec_str "%u" in
+        String (Printf.sprintf fmt (to_int arg))
+      | 'x' ->
+        let fmt = Scanf.format_from_string spec_str "%x" in
+        String (Printf.sprintf fmt (to_int arg))
+      | 'X' ->
+        let fmt = Scanf.format_from_string spec_str "%X" in
+        String (Printf.sprintf fmt (to_int arg))
+      | 'o' ->
+        let fmt = Scanf.format_from_string spec_str "%o" in
+        String (Printf.sprintf fmt (to_int arg))
+      | 'c' ->
+        let n_val = to_int arg in
+        let body = String.sub spec_str 0 (n - 1) in
+        let fmt = Scanf.format_from_string (body ^ "s") "%s" in
+        String (Printf.sprintf fmt (String.make 1 (Char.chr (n_val land 0xff))))
+      | 's' ->
+        let fmt = Scanf.format_from_string spec_str "%s" in
+        String (Printf.sprintf fmt (to_string arg))
+      | 'f' ->
+        let fmt = Scanf.format_from_string spec_str "%f" in
+        String (Printf.sprintf fmt (to_float arg))
+      | 'e' ->
+        let fmt = Scanf.format_from_string spec_str "%e" in
+        String (Printf.sprintf fmt (to_float arg))
+      | 'E' ->
+        let fmt = Scanf.format_from_string spec_str "%E" in
+        String (Printf.sprintf fmt (to_float arg))
+      | 'g' ->
+        let fmt = Scanf.format_from_string spec_str "%g" in
+        String (Printf.sprintf fmt (to_float arg))
+      | 'G' ->
+        let fmt = Scanf.format_from_string spec_str "%G" in
+        String (Printf.sprintf fmt (to_float arg))
+      | _ -> raise (Eval_error ("printf-spec: unsupported conversion " ^ String.make 1 type_char))
+    with
+    | Eval_error _ as e -> raise e
+    | _ -> raise (Eval_error ("printf-spec: invalid format " ^ spec_str)));
+
+  (* scan-spec: apply one Tcl/scanf format spec to a string.
+     Returns (consumed-count . parsed-value), or nil on failure. *)
+  register "scan-spec" (fun args ->
+    let spec_str, str = match args with
+      | [String s; String input] -> (s, input)
+      | _ -> raise (Eval_error "scan-spec: (spec input)")
+    in
+    let n = String.length spec_str in
+    if n < 2 || spec_str.[0] <> '%' then
+      raise (Eval_error ("scan-spec: invalid spec " ^ spec_str));
+    let type_char = spec_str.[n - 1] in
+    let len = String.length str in
+    (* skip leading whitespace for non-%c/%s conversions *)
+    let i = ref 0 in
+    if type_char <> 'c' then
+      while !i < len && (str.[!i] = ' ' || str.[!i] = '\t' || str.[!i] = '\n') do incr i done;
+    let start = !i in
+    try
+      match type_char with
+      | 'd' | 'i' ->
+        let j = ref !i in
+        if !j < len && (str.[!j] = '-' || str.[!j] = '+') then incr j;
+        while !j < len && str.[!j] >= '0' && str.[!j] <= '9' do incr j done;
+        if !j > start && (str.[start] >= '0' && str.[start] <= '9'
+                         || (!j > start + 1 && (str.[start] = '-' || str.[start] = '+'))) then
+          let n_val = int_of_string (String.sub str start (!j - start)) in
+          let d = Hashtbl.create 2 in
+          Hashtbl.replace d "value" (Integer n_val);
+          Hashtbl.replace d "consumed" (Integer !j);
+          Dict d
+        else Nil
+      | 'x' | 'X' ->
+        let j = ref !i in
+        while !j < len &&
+              ((str.[!j] >= '0' && str.[!j] <= '9') ||
+               (str.[!j] >= 'a' && str.[!j] <= 'f') ||
+               (str.[!j] >= 'A' && str.[!j] <= 'F')) do incr j done;
+        if !j > start then
+          let n_val = int_of_string ("0x" ^ String.sub str start (!j - start)) in
+          let d = Hashtbl.create 2 in
+          Hashtbl.replace d "value" (Integer n_val);
+          Hashtbl.replace d "consumed" (Integer !j);
+          Dict d
+        else Nil
+      | 'o' ->
+        let j = ref !i in
+        while !j < len && str.[!j] >= '0' && str.[!j] <= '7' do incr j done;
+        if !j > start then
+          let n_val = int_of_string ("0o" ^ String.sub str start (!j - start)) in
+          let d = Hashtbl.create 2 in
+          Hashtbl.replace d "value" (Integer n_val);
+          Hashtbl.replace d "consumed" (Integer !j);
+          Dict d
+        else Nil
+      | 'f' | 'e' | 'g' ->
+        let j = ref !i in
+        if !j < len && (str.[!j] = '-' || str.[!j] = '+') then incr j;
+        while !j < len && ((str.[!j] >= '0' && str.[!j] <= '9') || str.[!j] = '.') do incr j done;
+        if !j < len && (str.[!j] = 'e' || str.[!j] = 'E') then begin
+          incr j;
+          if !j < len && (str.[!j] = '-' || str.[!j] = '+') then incr j;
+          while !j < len && str.[!j] >= '0' && str.[!j] <= '9' do incr j done
+        end;
+        if !j > start then
+          let f_val = float_of_string (String.sub str start (!j - start)) in
+          let d = Hashtbl.create 2 in
+          Hashtbl.replace d "value" (Number f_val);
+          Hashtbl.replace d "consumed" (Integer !j);
+          Dict d
+        else Nil
+      | 's' ->
+        let j = ref !i in
+        while !j < len && str.[!j] <> ' ' && str.[!j] <> '\t' && str.[!j] <> '\n' do incr j done;
+        if !j > start then
+          let d = Hashtbl.create 2 in
+          Hashtbl.replace d "value" (String (String.sub str start (!j - start)));
+          Hashtbl.replace d "consumed" (Integer !j);
+          Dict d
+        else Nil
+      | 'c' ->
+        if !i < len then
+          let d = Hashtbl.create 2 in
+          Hashtbl.replace d "value" (Integer (Char.code str.[!i]));
+          Hashtbl.replace d "consumed" (Integer (!i + 1));
+          Dict d
+        else Nil
+      | _ -> raise (Eval_error ("scan-spec: unsupported conversion " ^ String.make 1 type_char))
+    with
+    | Eval_error _ as e -> raise e
+    | _ -> Nil);
+
   register "parse-int" (fun args ->
     let parse_leading_int s =
       let len = String.length s in
@@ -582,11 +759,22 @@ let () =
       (List lb | ListRef { contents = lb }) ->
       List.length la = List.length lb &&
       List.for_all2 safe_eq la lb
-    (* Dict: check __host_handle for DOM node identity *)
+    (* Dict: __host_handle identity for DOM-wrapped dicts; otherwise
+       structural equality over keys + values. *)
     | Dict a, Dict b ->
       (match Hashtbl.find_opt a "__host_handle", Hashtbl.find_opt b "__host_handle" with
        | Some (Number ha), Some (Number hb) -> ha = hb
-       | _ -> false)
+       | Some _, _ | _, Some _ -> false
+       | None, None ->
+         Hashtbl.length a = Hashtbl.length b &&
+         (let eq = ref true in
+          Hashtbl.iter (fun k v ->
+            if !eq then
+              match Hashtbl.find_opt b k with
+              | Some v' -> if not (safe_eq v v') then eq := false
+              | None -> eq := false
+          ) a;
+          !eq))
     (* Records: same type + structurally equal fields *)
     | Record a, Record b ->
       a.r_type.rt_uid = b.r_type.rt_uid &&
@@ -3399,6 +3587,204 @@ let () =
       Nil
     | _ -> raise (Eval_error "channel-set-blocking!: (channel bool)"));
 
+  (* === Exec === run an external process; capture stdout *)
+  register "exec-process" (fun args ->
+    let items = match args with
+      | [List xs] | [ListRef { contents = xs }] -> xs
+      | _ -> raise (Eval_error "exec-process: (cmd-list)")
+    in
+    let argv = Array.of_list (List.map (function
+      | String s -> s
+      | v -> Sx_types.inspect v
+    ) items) in
+    if Array.length argv = 0 then raise (Eval_error "exec: empty command");
+    let (out_r, out_w) = Unix.pipe () in
+    let (err_r, err_w) = Unix.pipe () in
+    let pid =
+      try Unix.create_process argv.(0) argv Unix.stdin out_w err_w
+      with Unix.Unix_error (e, _, _) ->
+        Unix.close out_r; Unix.close out_w;
+        Unix.close err_r; Unix.close err_w;
+        raise (Eval_error ("exec: " ^ Unix.error_message e))
+    in
+    Unix.close out_w;
+    Unix.close err_w;
+    let buf = Buffer.create 256 in
+    let errbuf = Buffer.create 64 in
+    let chunk = Bytes.create 4096 in
+    let read_all fd target =
+      try
+        let stop = ref false in
+        while not !stop do
+          let n = Unix.read fd chunk 0 (Bytes.length chunk) in
+          if n = 0 then stop := true
+          else Buffer.add_subbytes target chunk 0 n
+        done
+      with _ -> ()
+    in
+    read_all out_r buf;
+    read_all err_r errbuf;
+    Unix.close out_r;
+    Unix.close err_r;
+    let (_, status) = Unix.waitpid [] pid in
+    let exit_code = match status with
+      | Unix.WEXITED n -> n
+      | Unix.WSIGNALED _ | Unix.WSTOPPED _ -> 1
+    in
+    let s = Buffer.contents buf in
+    let trimmed =
+      if String.length s > 0 && s.[String.length s - 1] = '\n'
+      then String.sub s 0 (String.length s - 1) else s
+    in
+    if exit_code <> 0 then
+      raise (Eval_error ("exec: child exited " ^ string_of_int exit_code
+                         ^ (if Buffer.length errbuf > 0
+                            then ": " ^ Buffer.contents errbuf
+                            else "")))
+    else String trimmed);
+
+  (* exec-pipeline: takes a list of words like Tcl `exec` would receive.
+     Recognizes `|` as a stage separator and `> file`, `>> file`, `< file`,
+     `2>@1` (stderr→stdout), `2> file`. Returns trimmed stdout of the last
+     stage; raises Eval_error if the last stage exits non-zero. *)
+  register "exec-pipeline" (fun args ->
+    let items = match args with
+      | [List xs] | [ListRef { contents = xs }] -> xs
+      | _ -> raise (Eval_error "exec-pipeline: (word-list)")
+    in
+    let words = List.map (function
+      | String s -> s
+      | v -> Sx_types.inspect v
+    ) items in
+    if words = [] then raise (Eval_error "exec: empty command");
+    let split_stages ws =
+      let rec loop acc cur = function
+        | [] -> List.rev (List.rev cur :: acc)
+        | "|" :: rest -> loop (List.rev cur :: acc) [] rest
+        | w :: rest -> loop acc (w :: cur) rest
+      in
+      loop [] [] ws
+    in
+    let extract_redirs ws =
+      let in_path = ref None in
+      let out_path = ref None in
+      let out_append = ref false in
+      let err_path = ref None in
+      let merge_err = ref false in
+      let cleaned = ref [] in
+      let rec loop = function
+        | [] -> ()
+        | "<" :: p :: rest -> in_path := Some p; loop rest
+        | ">" :: p :: rest -> out_path := Some p; out_append := false; loop rest
+        | ">>" :: p :: rest -> out_path := Some p; out_append := true; loop rest
+        | "2>@1" :: rest -> merge_err := true; loop rest
+        | "2>" :: p :: rest -> err_path := Some p; loop rest
+        | w :: rest -> cleaned := w :: !cleaned; loop rest
+      in
+      loop ws;
+      (List.rev !cleaned, !in_path, !out_path, !out_append, !err_path, !merge_err)
+    in
+    let stages = List.map extract_redirs (split_stages words) in
+    if stages = [] then raise (Eval_error "exec: no stages");
+    let n = List.length stages in
+    let pipes = Array.init (max 0 (n - 1)) (fun _ -> Unix.pipe ()) in
+    let (final_r, final_w) = Unix.pipe () in
+    let (errstash_r, errstash_w) = Unix.pipe () in
+    let pids = ref [] in
+    let close_safe fd = try Unix.close fd with _ -> () in
+    let open_in_redir = function
+      | None -> Unix.stdin
+      | Some path ->
+        (try Unix.openfile path [Unix.O_RDONLY] 0o644
+         with Unix.Unix_error (e, _, _) ->
+           raise (Eval_error ("exec: open <" ^ path ^ ": " ^ Unix.error_message e)))
+    in
+    let open_out_redir path append =
+      let flags = Unix.O_WRONLY :: Unix.O_CREAT :: (if append then [Unix.O_APPEND] else [Unix.O_TRUNC]) in
+      try Unix.openfile path flags 0o644
+      with Unix.Unix_error (e, _, _) ->
+        raise (Eval_error ("exec: open >" ^ path ^ ": " ^ Unix.error_message e))
+    in
+    let stages_arr = Array.of_list stages in
+    (try
+      Array.iteri (fun i (cleaned, ip, op, app, ep, merge) ->
+        if cleaned = [] then raise (Eval_error "exec: empty stage in pipeline");
+        let argv = Array.of_list cleaned in
+        let stdin_fd =
+          if i = 0 then open_in_redir ip
+          else fst pipes.(i - 1)
+        in
+        let stdout_fd =
+          if i = n - 1 then
+            (match op with
+             | None -> final_w
+             | Some path -> open_out_redir path app)
+          else snd pipes.(i)
+        in
+        let stderr_fd =
+          if merge then stdout_fd
+          else (match ep with
+                | None -> if i = n - 1 then errstash_w else Unix.stderr
+                | Some path -> open_out_redir path false)
+        in
+        let pid =
+          try Unix.create_process argv.(0) argv stdin_fd stdout_fd stderr_fd
+          with Unix.Unix_error (e, _, _) ->
+            raise (Eval_error ("exec: " ^ argv.(0) ^ ": " ^ Unix.error_message e))
+        in
+        pids := pid :: !pids;
+        if i > 0 then close_safe (fst pipes.(i - 1));
+        if i < n - 1 then close_safe (snd pipes.(i));
+        if i = 0 && ip <> None then close_safe stdin_fd;
+        if i = n - 1 && op <> None then close_safe stdout_fd;
+        if not merge && ep <> None then close_safe stderr_fd
+      ) stages_arr
+    with e ->
+      close_safe final_r; close_safe final_w;
+      close_safe errstash_r; close_safe errstash_w;
+      Array.iter (fun (a,b) -> close_safe a; close_safe b) pipes;
+      raise e);
+    close_safe final_w;
+    close_safe errstash_w;
+    let buf = Buffer.create 256 in
+    let errbuf = Buffer.create 64 in
+    let chunk = Bytes.create 4096 in
+    let read_all fd target =
+      try
+        let stop = ref false in
+        while not !stop do
+          let r = Unix.read fd chunk 0 (Bytes.length chunk) in
+          if r = 0 then stop := true
+          else Buffer.add_subbytes target chunk 0 r
+        done
+      with _ -> ()
+    in
+    read_all final_r buf;
+    read_all errstash_r errbuf;
+    close_safe final_r;
+    close_safe errstash_r;
+    let exit_codes = List.rev_map (fun pid ->
+      let (_, st) = Unix.waitpid [] pid in
+      match st with
+      | Unix.WEXITED c -> c
+      | _ -> 1
+    ) !pids in
+    let final_code = match List.rev exit_codes with
+      | [] -> 0
+      | last :: _ -> last
+    in
+    let s = Buffer.contents buf in
+    let trimmed =
+      if String.length s > 0 && s.[String.length s - 1] = '\n'
+      then String.sub s 0 (String.length s - 1) else s
+    in
+    if final_code <> 0 then
+      raise (Eval_error ("exec: pipeline last stage exited " ^ string_of_int final_code
+                         ^ (if Buffer.length errbuf > 0
+                            then ": " ^ Buffer.contents errbuf
+                            else "")))
+    else String trimmed);
+
   (* === Sockets === wrapping Unix.socket/connect/bind/listen/accept *)
   let resolve_inet_addr host =
     if host = "" || host = "0.0.0.0" then Unix.inet_addr_any
@@ -3734,4 +4120,42 @@ let () =
         | k :: v :: rest -> ignore (env_bind child (value_to_string k) v); add_bindings rest
         | [_] -> raise (Eval_error "env-extend: odd number of key-val pairs") in
       add_bindings pairs;
-      Env child)
+      Env child);
+
+  (* JIT cache control & observability — backed by refs in sx_types.ml to
+     avoid creating a sx_primitives → sx_vm dependency cycle. sx_vm reads
+     these refs to decide when to JIT. *)
+  register "jit-stats" (fun _args ->
+    let d = Hashtbl.create 8 in
+    Hashtbl.replace d "threshold"        (Number (float_of_int !Sx_types.jit_threshold));
+    Hashtbl.replace d "budget"           (Number (float_of_int !Sx_types.jit_budget));
+    Hashtbl.replace d "cache-size"       (Number (float_of_int (Sx_types.jit_cache_size ())));
+    Hashtbl.replace d "compiled"         (Number (float_of_int !Sx_types.jit_compiled_count));
+    Hashtbl.replace d "compile-failed"   (Number (float_of_int !Sx_types.jit_skipped_count));
+    Hashtbl.replace d "below-threshold"  (Number (float_of_int !Sx_types.jit_threshold_skipped_count));
+    Hashtbl.replace d "evicted"          (Number (float_of_int !Sx_types.jit_evicted_count));
+    Dict d);
+  register "jit-set-threshold!" (fun args ->
+    match args with
+    | [Number n] -> Sx_types.jit_threshold := int_of_float n; Nil
+    | [Integer n] -> Sx_types.jit_threshold := n; Nil
+    | _ -> raise (Eval_error "jit-set-threshold!: (n) where n is integer"));
+  register "jit-set-budget!" (fun args ->
+    match args with
+    | [Number n] -> Sx_types.jit_budget := int_of_float n; Nil
+    | [Integer n] -> Sx_types.jit_budget := n; Nil
+    | _ -> raise (Eval_error "jit-set-budget!: (n) where n is integer"));
+  register "jit-reset-cache!" (fun _args ->
+    (* Phase 3 manual cache reset — clear all compiled VmClosures.
+       Hot paths will re-JIT on next call (after re-hitting threshold). *)
+    Queue.iter (fun (_, v) ->
+      match v with Lambda l -> l.l_compiled <- None | _ -> ()
+    ) Sx_types.jit_cache_queue;
+    Queue.clear Sx_types.jit_cache_queue;
+    Nil);
+  register "jit-reset-counters!" (fun _args ->
+    Sx_types.jit_compiled_count := 0;
+    Sx_types.jit_skipped_count := 0;
+    Sx_types.jit_threshold_skipped_count := 0;
+    Sx_types.jit_evicted_count := 0;
+    Nil)

hosts/ocaml/lib/sx_types.ml

@@ -138,6 +138,8 @@ and lambda = {
   l_closure : env;
   mutable l_name : string option;
   mutable l_compiled : vm_closure option;  (** Lazy JIT cache *)
+  mutable l_call_count : int;              (** Tiered-compilation counter — JIT after threshold calls *)
+  l_uid : int;                             (** Unique identity for LRU cache tracking *)
 }
 
 and component = {
@@ -444,12 +446,60 @@ let unwrap_env_val = function
   | Env e -> e
   | _ -> raise (Eval_error "make_lambda: expected env for closure")
 
+(* Lambda UID — minted on construction, used as LRU cache key (Phase 2). *)
+let lambda_uid_counter = ref 0
+let next_lambda_uid () = incr lambda_uid_counter; !lambda_uid_counter
+
 let make_lambda params body closure =
   let ps = match params with
     | List items -> List.map value_to_string items
     | _ -> value_to_string_list params
   in
-  Lambda { l_params = ps; l_body = body; l_closure = unwrap_env_val closure; l_name = None; l_compiled = None }
+  Lambda { l_params = ps; l_body = body; l_closure = unwrap_env_val closure; l_name = None; l_compiled = None; l_call_count = 0; l_uid = next_lambda_uid () }
+
+(** {1 JIT cache control}
+
+    Tiered compilation: only JIT a lambda after it's been called [jit_threshold]
+    times. This filters out one-shot lambdas (test harness, dynamic eval, REPLs)
+    so they never enter the JIT cache. Counters are exposed to SX as [(jit-stats)].
+
+    These live here (in sx_types) rather than sx_vm so [sx_primitives] can read
+    them without creating a sx_primitives → sx_vm dependency cycle. *)
+let jit_threshold = ref 4
+let jit_compiled_count = ref 0
+let jit_skipped_count = ref 0
+let jit_threshold_skipped_count = ref 0
+
+(** {2 JIT cache LRU eviction — Phase 2}
+
+    Once a lambda crosses the threshold, its [l_compiled] slot is filled.
+    To bound memory under unbounded compilation pressure, track all live
+    compiled lambdas in FIFO order, and evict from the head when the count
+    exceeds [jit_budget].
+
+    [lambda_uid_counter] mints unique identities on lambda creation; the
+    LRU queue holds these IDs paired with a back-reference to the lambda
+    so we can clear its [l_compiled] slot on eviction.
+
+    Budget of 0 = no cache (disable JIT entirely).
+    Budget of [max_int] = unbounded (legacy behaviour). Default 5000 is
+    a generous ceiling for any realistic page; the test harness compiles
+    ~3000 distinct one-shot lambdas in a full run but tiered compilation
+    (Phase 1) means most never enter the cache, so steady-state count
+    stays small.
+
+    [lambda_uid_counter] and [next_lambda_uid] are defined above
+    [make_lambda] (which uses them on construction). *)
+let jit_budget = ref 5000
+let jit_evicted_count = ref 0
+
+(** Live compiled lambdas in FIFO order — front is oldest, back is newest.
+    Each entry is (uid, lambda); on eviction we clear lambda.l_compiled and
+    drop from the queue. Using a mutable Queue rather than a hand-rolled
+    linked list because eviction is amortised O(1) at the head and inserts
+    are O(1) at the tail. *)
+let jit_cache_queue : (int * value) Queue.t = Queue.create ()
+let jit_cache_size () = Queue.length jit_cache_queue
 
 let make_component name params has_children body closure affinity =
   let n = value_to_string name in

hosts/ocaml/lib/sx_vm.ml

@@ -57,6 +57,9 @@ let () = Sx_types._convert_vm_suspension := (fun exn ->
 let jit_compile_ref : (lambda -> (string, value) Hashtbl.t -> vm_closure option) ref =
   ref (fun _ _ -> None)
 
+(* JIT threshold and counters live in Sx_types so primitives can read them
+   without creating a sx_primitives → sx_vm dependency cycle. *)
+
 (** Sentinel closure indicating JIT compilation was attempted and failed.
     Prevents retrying compilation on every call. *)
 let jit_failed_sentinel = {
@@ -364,13 +367,29 @@ and vm_call vm f args =
      | None ->
        if l.l_name <> None
        then begin
-         l.l_compiled <- Some jit_failed_sentinel;
-         match !jit_compile_ref l vm.globals with
-         | Some cl ->
-           l.l_compiled <- Some cl;
-           push_closure_frame vm cl args
-         | None ->
+         l.l_call_count <- l.l_call_count + 1;
+         if l.l_call_count >= !Sx_types.jit_threshold && !Sx_types.jit_budget > 0 then begin
+           l.l_compiled <- Some jit_failed_sentinel;
+           match !jit_compile_ref l vm.globals with
+           | Some cl ->
+             incr Sx_types.jit_compiled_count;
+             l.l_compiled <- Some cl;
+             (* Phase 2 LRU: track this compiled lambda; if cache exceeds budget,
+                evict the oldest by clearing its l_compiled slot. *)
+             Queue.add (l.l_uid, Lambda l) Sx_types.jit_cache_queue;
+             while Queue.length Sx_types.jit_cache_queue > !Sx_types.jit_budget do
+               (match Queue.pop Sx_types.jit_cache_queue with
+                | (_, Lambda ev_l) -> ev_l.l_compiled <- None; incr Sx_types.jit_evicted_count
+                | _ -> ())
+             done;
+             push_closure_frame vm cl args
+           | None ->
+             incr Sx_types.jit_skipped_count;
+             push vm (cek_call_or_suspend vm f (List args))
+         end else begin
+           incr Sx_types.jit_threshold_skipped_count;
            push vm (cek_call_or_suspend vm f (List args))
+         end
        end
        else
          push vm (cek_call_or_suspend vm f (List args)))

lib/apl/parser.sx

@@ -270,6 +270,15 @@
               (collect-segments-loop tokens (+ i 1) (append acc {:kind "val" :node (list :str tv)})))
             ((= tt :name)
               (cond
+                ((and (< (+ i 1) (len tokens)) (= (tok-type (nth tokens (+ i 1))) :assign))
+                  (let
+                    ((rhs-tokens (slice tokens (+ i 2) (len tokens))))
+                    (let
+                      ((rhs-expr (parse-apl-expr rhs-tokens)))
+                      (collect-segments-loop
+                        tokens
+                        (len tokens)
+                        (append acc {:kind "val" :node (list :assign-expr tv rhs-expr)})))))
                 ((some (fn (q) (= q tv)) apl-quad-fn-names)
                   (let
                     ((op-result (collect-ops tokens (+ i 1))))
@@ -335,10 +344,22 @@
             ((= tt :glyph)
               (cond
                 ((or (= tv "⍺") (= tv "⍵"))
-                  (collect-segments-loop
-                    tokens
-                    (+ i 1)
-                    (append acc {:kind "val" :node (list :name tv)})))
+                  (if
+                    (and
+                      (< (+ i 1) (len tokens))
+                      (= (tok-type (nth tokens (+ i 1))) :assign))
+                    (let
+                      ((rhs-tokens (slice tokens (+ i 2) (len tokens))))
+                      (let
+                        ((rhs-expr (parse-apl-expr rhs-tokens)))
+                        (collect-segments-loop
+                          tokens
+                          (len tokens)
+                          (append acc {:kind "val" :node (list :assign-expr tv rhs-expr)}))))
+                    (collect-segments-loop
+                      tokens
+                      (+ i 1)
+                      (append acc {:kind "val" :node (list :name tv)}))))
                 ((= tv "∇")
                   (collect-segments-loop
                     tokens
@@ -393,7 +414,23 @@
                             ni
                             (append acc {:kind "fn" :node fn-node})))))))
                 ((apl-parse-op-glyph? tv)
-                  (collect-segments-loop tokens (+ i 1) acc))
+                  (if
+                    (or (= tv "/") (= tv "⌿") (= tv "\\") (= tv "⍀"))
+                    (let
+                      ((next-i (+ i 1)))
+                      (let
+                        ((next-tok (if (< next-i n) (nth tokens next-i) nil)))
+                        (let
+                          ((mod (if (and next-tok (= (tok-type next-tok) :glyph) (or (= (get next-tok :value) "⍨") (= (get next-tok :value) "¨"))) (get next-tok :value) nil))
+                            (base-fn-node (list :fn-glyph tv)))
+                          (let
+                            ((node (if mod (list :derived-fn mod base-fn-node) base-fn-node))
+                              (advance (if mod 2 1)))
+                            (collect-segments-loop
+                              tokens
+                              (+ i advance)
+                              (append acc {:kind "fn" :node node}))))))
+                    (collect-segments-loop tokens (+ i 1) acc)))
                 (true (collect-segments-loop tokens (+ i 1) acc))))
             (true (collect-segments-loop tokens (+ i 1) acc))))))))
 

lib/apl/runtime.sx

@@ -65,10 +65,30 @@
             (get a :shape)
             (map (fn (x) (f x sv)) (get a :ravel)))))
       (else
-        (if
-          (equal? (get a :shape) (get b :shape))
-          (make-array (get a :shape) (map f (get a :ravel) (get b :ravel)))
-          (error "length error: shape mismatch"))))))
+        (let
+          ((a-shape (get a :shape)) (b-shape (get b :shape)))
+          (cond
+            ((equal? a-shape b-shape)
+              (make-array a-shape (map f (get a :ravel) (get b :ravel))))
+            ((and (= (len a-shape) 1) (> (len b-shape) 1))
+              (make-array
+                (append a-shape b-shape)
+                (flatten
+                  (map
+                    (fn
+                      (x)
+                      (get (broadcast-dyadic f (apl-scalar x) b) :ravel))
+                    (get a :ravel)))))
+            ((and (= (len b-shape) 1) (> (len a-shape) 1))
+              (make-array
+                (append a-shape b-shape)
+                (flatten
+                  (map
+                    (fn
+                      (acell)
+                      (get (broadcast-dyadic f (apl-scalar acell) b) :ravel))
+                    (get a :ravel)))))
+            (else (error "length error: shape mismatch"))))))))
 
 ; ============================================================
 ; Arithmetic primitives
@@ -808,6 +828,125 @@
           ((picked (map (fn (i) (nth arr-ravel i)) kept)))
           (make-array (list (len picked)) picked))))))
 
+(define
+  apl-compress-first
+  (fn
+    (mask arr)
+    (let
+      ((mask-ravel (get mask :ravel))
+        (shape (get arr :shape))
+        (ravel (get arr :ravel)))
+      (if
+        (< (len shape) 2)
+        (apl-compress mask arr)
+        (let
+          ((rows (first shape)) (cols (last shape)))
+          (let
+            ((kept-rows (filter (fn (i) (not (= 0 (nth mask-ravel i)))) (range 0 rows))))
+            (let
+              ((new-ravel (reduce (fn (acc r) (append acc (map (fn (j) (nth ravel (+ (* r cols) j))) (range 0 cols)))) (list) kept-rows)))
+              (make-array (cons (len kept-rows) (rest shape)) new-ravel))))))))
+
+(define
+  apl-where
+  (fn
+    (arr)
+    (let
+      ((ravel (get arr :ravel)) (io (disclose (apl-quad-io))))
+      (let
+        ((indices (filter (fn (i) (not (= (nth ravel i) 0))) (range 0 (len ravel)))))
+        (apl-vector (map (fn (i) (+ i io)) indices))))))
+
+(define
+  apl-interval-index
+  (fn
+    (breaks vals)
+    (let
+      ((b-ravel (get breaks :ravel))
+        (v-ravel
+          (if (scalar? vals) (list (disclose vals)) (get vals :ravel))))
+      (let
+        ((result (map (fn (y) (len (filter (fn (b) (<= b y)) b-ravel))) v-ravel)))
+        (if
+          (scalar? vals)
+          (apl-scalar (first result))
+          (make-array (get vals :shape) result))))))
+
+(define
+  apl-unique
+  (fn
+    (arr)
+    (let
+      ((ravel (if (scalar? arr) (list (disclose arr)) (get arr :ravel))))
+      (let
+        ((dedup (reduce (fn (acc x) (if (index-of acc x) acc (append acc (list x)))) (list) ravel)))
+        (apl-vector dedup)))))
+
+(define
+  apl-union
+  (fn
+    (a b)
+    (let
+      ((a-ravel (if (scalar? a) (list (disclose a)) (get a :ravel)))
+        (b-ravel (if (scalar? b) (list (disclose b)) (get b :ravel))))
+      (let
+        ((a-dedup (reduce (fn (acc x) (if (index-of acc x) acc (append acc (list x)))) (list) a-ravel)))
+        (let
+          ((b-extra (filter (fn (x) (not (index-of a-dedup x))) b-ravel)))
+          (let
+            ((b-extra-dedup (reduce (fn (acc x) (if (index-of acc x) acc (append acc (list x)))) (list) b-extra)))
+            (apl-vector (append a-dedup b-extra-dedup))))))))
+
+(define
+  apl-intersect
+  (fn
+    (a b)
+    (let
+      ((a-ravel (if (scalar? a) (list (disclose a)) (get a :ravel)))
+        (b-ravel (if (scalar? b) (list (disclose b)) (get b :ravel))))
+      (apl-vector (filter (fn (x) (index-of b-ravel x)) a-ravel)))))
+
+(define
+  apl-decode
+  (fn
+    (base digits)
+    (let
+      ((d-ravel (if (scalar? digits) (list (disclose digits)) (get digits :ravel))))
+      (let
+        ((d-len (len d-ravel)))
+        (let
+          ((b-ravel (if (scalar? base) (let ((b (disclose base))) (map (fn (i) b) (range 0 d-len))) (get base :ravel))))
+          (let
+            ((result (reduce (fn (acc i) (if (= i 0) (nth d-ravel 0) (+ (* acc (nth b-ravel i)) (nth d-ravel i)))) 0 (range 0 d-len))))
+            (apl-scalar result)))))))
+
+(define
+  apl-encode
+  (fn
+    (base val)
+    (let
+      ((b-ravel (if (scalar? base) (list (disclose base)) (get base :ravel)))
+        (n (if (scalar? val) (disclose val) (first (get val :ravel)))))
+      (let
+        ((b-len (len b-ravel)))
+        (let
+          ((result (reduce (fn (acc-and-n i) (let ((acc (first acc-and-n)) (rem (nth acc-and-n 1))) (let ((b (nth b-ravel (- (- b-len 1) i)))) (if (= b 0) (list (cons rem acc) 0) (list (cons (modulo rem b) acc) (floor (/ rem b))))))) (list (list) n) (range 0 b-len))))
+          (apl-vector (first result)))))))
+
+(define
+  apl-partition
+  (fn
+    (mask val)
+    (let
+      ((m-ravel (if (scalar? mask) (list (disclose mask)) (get mask :ravel)))
+        (v-ravel
+          (if (scalar? val) (list (disclose val)) (get val :ravel))))
+      (let
+        ((n (len m-ravel)))
+        (let
+          ((built (reduce (fn (acc-and-prev i) (let ((acc (first acc-and-prev)) (prev (nth acc-and-prev 1))) (let ((mi (nth m-ravel i)) (vi (nth v-ravel i))) (cond ((= mi 0) (list acc 0)) ((> mi prev) (list (append acc (list (list vi))) mi)) (else (let ((idx (- (len acc) 1))) (list (append (slice acc 0 idx) (list (append (nth acc idx) (list vi)))) mi))))))) (list (list) 0) (range 0 n))))
+          (apl-vector (map (fn (part) (apl-vector part)) (first built))))))))
+
 (define
   apl-primes
   (fn
@@ -985,6 +1124,28 @@
         (some (fn (c) (= c 0)) codes)
         (some (fn (c) (= c (nth e 1))) codes)))))
 
+(define apl-rng-state 12345)
+
+(define apl-rng-seed! (fn (s) (set! apl-rng-state s)))
+
+(define
+  apl-rng-next!
+  (fn
+    ()
+    (begin
+      (set!
+        apl-rng-state
+        (mod (+ (* apl-rng-state 1103515245) 12345) 2147483648))
+      apl-rng-state)))
+
+(define
+  apl-roll
+  (fn
+    (arr)
+    (let
+      ((n (if (scalar? arr) (first (get arr :ravel)) (first (get arr :ravel)))))
+      (apl-scalar (+ apl-io (mod (apl-rng-next!) n))))))
+
 (define
   apl-cartesian
   (fn
@@ -1033,11 +1194,9 @@
             (if
               (= n 0)
               (apl-scalar 0)
-              (apl-scalar
-                (reduce
-                  (fn (a b) (disclose (f (apl-scalar a) (apl-scalar b))))
-                  (first ravel)
-                  (rest ravel)))))
+              (let
+                ((rr (reduce (fn (a b) (let ((wa (if (= (type-of a) "dict") a (apl-scalar a))) (wb (if (= (type-of b) "dict") b (apl-scalar b)))) (let ((r (f wa wb))) (if (scalar? r) (disclose r) r)))) (first ravel) (rest ravel))))
+                (if (= (type-of rr) "dict") rr (apl-scalar rr)))))
           (let
             ((last-dim (last shape))
               (pre-shape (take shape (- (len shape) 1)))
@@ -1059,7 +1218,13 @@
                       (reduce
                         (fn
                           (a b)
-                          (disclose (f (apl-scalar a) (apl-scalar b))))
+                          (let
+                            ((wa (if (= (type-of a) "dict") a (apl-scalar a)))
+                              (wb
+                                (if (= (type-of b) "dict") b (apl-scalar b))))
+                            (let
+                              ((r (f wa wb)))
+                              (if (scalar? r) (disclose r) r))))
                         (first elems)
                         (rest elems)))))
                 (range 0 pre-size)))))))))
@@ -1200,13 +1365,29 @@
     (cond
       ((and (scalar? a) (scalar? b)) (apl-scalar (disclose (f a b))))
       ((scalar? a)
-        (make-array
-          (get b :shape)
-          (map (fn (x) (disclose (f a (apl-scalar x)))) (get b :ravel))))
+        (let
+          ((a-eff (let ((d (disclose a))) (if (= (type-of d) "dict") d a))))
+          (make-array
+            (get b :shape)
+            (map
+              (fn
+                (x)
+                (let
+                  ((r (f a-eff (apl-scalar x))))
+                  (if (scalar? r) (disclose r) r)))
+              (get b :ravel)))))
       ((scalar? b)
-        (make-array
-          (get a :shape)
-          (map (fn (x) (disclose (f (apl-scalar x) b))) (get a :ravel))))
+        (let
+          ((b-eff (let ((d (disclose b))) (if (= (type-of d) "dict") d b))))
+          (make-array
+            (get a :shape)
+            (map
+              (fn
+                (x)
+                (let
+                  ((r (f (apl-scalar x) b-eff)))
+                  (if (scalar? r) (disclose r) r)))
+              (get a :ravel)))))
       (else
         (if
           (equal? (get a :shape) (get b :shape))
@@ -1227,16 +1408,22 @@
         (b-shape (get b :shape))
         (a-ravel (get a :ravel))
         (b-ravel (get b :ravel)))
-      (make-array
-        (append a-shape b-shape)
-        (flatten
-          (map
-            (fn
-              (x)
-              (map
-                (fn (y) (disclose (f (apl-scalar x) (apl-scalar y))))
-                b-ravel))
-            a-ravel))))))
+      (let
+        ((wrap (fn (x) (if (= (type-of x) "dict") x (apl-scalar x)))))
+        (make-array
+          (append a-shape b-shape)
+          (flatten
+            (map
+              (fn
+                (x)
+                (map
+                  (fn
+                    (y)
+                    (let
+                      ((r (f (wrap x) (wrap y))))
+                      (if (scalar? r) (disclose r) r)))
+                  b-ravel))
+              a-ravel)))))))
 
 (define
   apl-inner
@@ -1260,25 +1447,12 @@
               ((a-pre-size (reduce * 1 a-pre))
                 (b-post-size (reduce * 1 b-post))
                 (new-shape (append a-pre b-post)))
-              (make-array
-                new-shape
-                (flatten
-                  (map
-                    (fn
-                      (i)
-                      (map
-                        (fn
-                          (j)
-                          (let
-                            ((pairs (map (fn (k) (disclose (g (apl-scalar (nth a-ravel (+ (* i inner-dim) k))) (apl-scalar (nth b-ravel (+ (* k b-post-size) j)))))) (range 0 inner-dim))))
-                            (reduce
-                              (fn
-                                (x y)
-                                (disclose (f (apl-scalar x) (apl-scalar y))))
-                              (first pairs)
-                              (rest pairs))))
-                        (range 0 b-post-size)))
-                    (range 0 a-pre-size)))))))))))
+              (let
+                ((result (make-array new-shape (flatten (map (fn (i) (map (fn (j) (let ((pairs (map (fn (k) (let ((a-elem (nth a-ravel (+ (* i inner-dim) k))) (b-elem (nth b-ravel (+ (* k b-post-size) j)))) (let ((a-cell (if (= (type-of a-elem) "dict") (nth (get a-elem :ravel) j) a-elem)) (b-cell (if (= (type-of b-elem) "dict") (nth (get b-elem :ravel) 0) b-elem))) (disclose (g (apl-scalar a-cell) (apl-scalar b-cell)))))) (range 0 inner-dim)))) (reduce (fn (x y) (let ((wx (if (= (type-of x) "dict") x (apl-scalar x))) (wy (if (= (type-of y) "dict") y (apl-scalar y)))) (let ((r (f wx wy))) (if (scalar? r) (disclose r) r)))) (first pairs) (rest pairs)))) (range 0 b-post-size))) (range 0 a-pre-size))))))
+                (if
+                  (some (fn (x) (= (type-of x) "dict")) a-ravel)
+                  (enclose result)
+                  result)))))))))
 
 (define apl-commute (fn (f x) (f x x)))
 

lib/apl/tests/pipeline.sx

@@ -312,3 +312,376 @@
   "train: mean of ⍳10 has shape ()"
   (mksh (apl-run "(+/÷≢) ⍳10"))
   (list))
+
+(apl-test
+  "compress: 1 0 1 0 1 / 10 20 30 40 50"
+  (mkrv (apl-run "1 0 1 0 1 / 10 20 30 40 50"))
+  (list 10 30 50))
+
+(apl-test
+  "compress: empty mask → empty"
+  (mkrv (apl-run "0 0 0 / 1 2 3"))
+  (list))
+
+(apl-test
+  "primes via classic idiom (multi-stmt)"
+  (mkrv (apl-run "P ← ⍳ 30 ⋄ (2 = +⌿ 0 = P ∘.| P) / P"))
+  (list 2 3 5 7 11 13 17 19 23 29))
+
+(apl-test
+  "primes via classic idiom (n=20)"
+  (mkrv (apl-run "P ← ⍳ 20 ⋄ (2 = +⌿ 0 = P ∘.| P) / P"))
+  (list 2 3 5 7 11 13 17 19))
+
+(apl-test
+  "compress: filter even values"
+  (mkrv (apl-run "(0 = 2 | 1 2 3 4 5 6) / 1 2 3 4 5 6"))
+  (list 2 4 6))
+
+(apl-test "inline-assign: x ← 5" (mkrv (apl-run "x ← 5")) (list 5))
+
+(apl-test
+  "inline-assign: (2×x) + x←10 → 30"
+  (mkrv (apl-run "(2 × x) + x ← 10"))
+  (list 30))
+
+(apl-test
+  "inline-assign primes one-liner: (2=+⌿0=a∘.|a)/a←⍳30"
+  (mkrv (apl-run "(2 = +⌿ 0 = a ∘.| a) / a ← ⍳ 30"))
+  (list 2 3 5 7 11 13 17 19 23 29))
+
+(apl-test
+  "inline-assign: x is reusable — x + x ← 7 → 14"
+  (mkrv (apl-run "x + x ← 7"))
+  (list 14))
+
+(apl-test
+  "inline-assign in dfn: f ← {x + x ← ⍵} ⋄ f 8 → 16"
+  (mkrv (apl-run "f ← {x + x ← ⍵} ⋄ f 8"))
+  (list 16))
+
+(begin (apl-rng-seed! 42) nil)
+
+(apl-test
+  "?10 with seed 42 → 8 (deterministic)"
+  (mkrv (apl-run "?10"))
+  (list 8))
+
+(apl-test "?10 next call → 5" (mkrv (apl-run "?10")) (list 5))
+
+(apl-test
+  "?100 stays in range"
+  (let ((v (first (mkrv (apl-run "?100"))))) (and (>= v 1) (<= v 100)))
+  true)
+
+(begin (apl-rng-seed! 42) nil)
+
+(apl-test
+  "?10 with re-seed 42 → 8 (reproducible)"
+  (mkrv (apl-run "?10"))
+  (list 8))
+
+(apl-test
+  "apl-run-file: load primes.apl returns dfn AST"
+  (first (apl-run-file "lib/apl/tests/programs/primes.apl"))
+  :dfn)
+
+(apl-test
+  "apl-run-file: life.apl parses without error"
+  (first (apl-run-file "lib/apl/tests/programs/life.apl"))
+  :dfn)
+
+(apl-test
+  "apl-run-file: quicksort.apl parses without error"
+  (first (apl-run-file "lib/apl/tests/programs/quicksort.apl"))
+  :dfn)
+
+(apl-test
+  "apl-run-file: source-then-call returns primes count"
+  (mksh
+    (apl-run
+      (str (file-read "lib/apl/tests/programs/primes.apl") " ⋄ primes 30")))
+  (list 10))
+
+(apl-test
+  "primes one-liner with ⍵-rebind: primes 30"
+  (mkrv
+    (apl-run "primes ← {(2=+⌿0=⍵∘.|⍵)/⍵←⍳⍵} ⋄ primes 30"))
+  (list 2 3 5 7 11 13 17 19 23 29))
+
+(apl-test
+  "primes one-liner: primes 50"
+  (mkrv
+    (apl-run "primes ← {(2=+⌿0=⍵∘.|⍵)/⍵←⍳⍵} ⋄ primes 50"))
+  (list 2 3 5 7 11 13 17 19 23 29 31 37 41 43 47))
+
+(apl-test
+  "primes.apl loaded + called via apl-run-file"
+  (mkrv
+    (apl-run
+      (str (file-read "lib/apl/tests/programs/primes.apl") " ⋄ primes 20")))
+  (list 2 3 5 7 11 13 17 19))
+
+(apl-test
+  "primes.apl loaded — count of primes ≤ 100"
+  (first
+    (mksh
+      (apl-run
+        (str
+          (file-read "lib/apl/tests/programs/primes.apl")
+          " ⋄ primes 100"))))
+  25)
+
+(apl-test
+  "⍉ monadic transpose 2x3 → 3x2"
+  (mkrv (apl-run "⍉ (2 3) ⍴ ⍳6"))
+  (list 1 4 2 5 3 6))
+
+(apl-test
+  "⍉ transpose shape (3 2)"
+  (mksh (apl-run "⍉ (2 3) ⍴ ⍳6"))
+  (list 3 2))
+
+(apl-test "⊢ monadic identity" (mkrv (apl-run "⊢ 1 2 3")) (list 1 2 3))
+
+(apl-test
+  "5 ⊣ 1 2 3 → 5 (left)"
+  (mkrv (apl-run "5 ⊣ 1 2 3"))
+  (list 5))
+
+(apl-test
+  "5 ⊢ 1 2 3 → 1 2 3 (right)"
+  (mkrv (apl-run "5 ⊢ 1 2 3"))
+  (list 1 2 3))
+
+(apl-test "⍕ 42 → \"42\" (alias for ⎕FMT)" (apl-run "⍕ 42") "42")
+
+(begin
+  (apl-test
+    "⍸ where: indices of truthy cells"
+    (mkrv (apl-run "⍸ 0 1 0 1 1"))
+    (list 2 4 5))
+  (apl-test
+    "⍸ where: leading truthy"
+    (mkrv (apl-run "⍸ 1 0 0 1 1"))
+    (list 1 4 5))
+  (apl-test
+    "⍸ where: all-zero → empty"
+    (mkrv (apl-run "⍸ 0 0 0"))
+    (list))
+  (apl-test
+    "⍸ where: all-truthy"
+    (mkrv (apl-run "⍸ 1 1 1"))
+    (list 1 2 3))
+  (apl-test
+    "⍸ where: ⎕IO=1 (1-based)"
+    (mkrv (apl-run "⍸ (⍳5)=3"))
+    (list 3))
+  (apl-test
+    "⍸ interval-index: 2 4 6 ⍸ 5 → 2"
+    (mkrv (apl-run "2 4 6 ⍸ 5"))
+    (list 2))
+  (apl-test
+    "⍸ interval-index: 2 4 6 ⍸ 1 3 5 6 7 → 0 1 2 3 3"
+    (mkrv (apl-run "2 4 6 ⍸ 1 3 5 6 7"))
+    (list 0 1 2 3 3))
+  (apl-test
+    "⍸ interval-index: ⍳5 ⍸ 3 → 3"
+    (mkrv (apl-run "(⍳5) ⍸ 3"))
+    (list 3))
+  (apl-test
+    "⍸ interval-index: y below all → 0"
+    (mkrv (apl-run "10 20 30 ⍸ 5"))
+    (list 0))
+  (apl-test
+    "⍸ interval-index: y above all → len breaks"
+    (mkrv (apl-run "10 20 30 ⍸ 100"))
+    (list 3)))
+
+(begin
+  (apl-test
+    "∪ unique: dedup keeps first-occurrence order"
+    (mkrv (apl-run "∪ 1 2 1 3 2 1 4"))
+    (list 1 2 3 4))
+  (apl-test
+    "∪ unique: already-unique unchanged"
+    (mkrv (apl-run "∪ 5 4 3 2 1"))
+    (list 5 4 3 2 1))
+  (apl-test "∪ unique: scalar" (mkrv (apl-run "∪ 7")) (list 7))
+  (apl-test
+    "∪ unique: string mississippi → misp"
+    (mkrv (apl-run "∪ 'mississippi'"))
+    (list "m" "i" "s" "p"))
+  (apl-test
+    "∪ union: 1 2 3 ∪ 3 4 5 → 1 2 3 4 5"
+    (mkrv (apl-run "1 2 3 ∪ 3 4 5"))
+    (list 1 2 3 4 5))
+  (apl-test
+    "∪ union: dedups left side too"
+    (mkrv (apl-run "1 2 1 ∪ 1 3 2"))
+    (list 1 2 3))
+  (apl-test
+    "∪ union: disjoint → catenated"
+    (mkrv (apl-run "1 2 ∪ 3 4"))
+    (list 1 2 3 4))
+  (apl-test
+    "∩ intersection: 1 2 3 4 ∩ 2 4 6 → 2 4"
+    (mkrv (apl-run "1 2 3 4 ∩ 2 4 6"))
+    (list 2 4))
+  (apl-test
+    "∩ intersection: disjoint → empty"
+    (mkrv (apl-run "1 2 3 ∩ 4 5 6"))
+    (list))
+  (apl-test
+    "∩ intersection: preserves left order"
+    (mkrv (apl-run "(⍳5) ∩ 5 3 1"))
+    (list 1 3 5))
+  (apl-test
+    "∩ intersection: identical"
+    (mkrv (apl-run "1 2 3 ∩ 1 2 3"))
+    (list 1 2 3))
+  (apl-test
+    "∪/∩ identity: A ∪ A = ∪A"
+    (mkrv (apl-run "1 2 1 ∪ 1 2 1"))
+    (list 1 2)))
+
+(begin
+  (apl-test
+    "⊥ decode: 2 2 2 ⊥ 1 0 1 → 5"
+    (mkrv (apl-run "2 2 2 ⊥ 1 0 1"))
+    (list 5))
+  (apl-test
+    "⊥ decode: 10 10 10 ⊥ 1 2 3 → 123"
+    (mkrv (apl-run "10 10 10 ⊥ 1 2 3"))
+    (list 123))
+  (apl-test
+    "⊥ decode: 24 60 60 ⊥ 2 3 4 → 7384 (mixed-radix HMS)"
+    (mkrv (apl-run "24 60 60 ⊥ 2 3 4"))
+    (list 7384))
+  (apl-test
+    "⊥ decode: scalar base 2 ⊥ 1 0 1 0 → 10"
+    (mkrv (apl-run "2 ⊥ 1 0 1 0"))
+    (list 10))
+  (apl-test
+    "⊥ decode: 16 16 ⊥ 15 15 → 255"
+    (mkrv (apl-run "16 16 ⊥ 15 15"))
+    (list 255))
+  (apl-test
+    "⊤ encode: 2 2 2 ⊤ 5 → 1 0 1"
+    (mkrv (apl-run "2 2 2 ⊤ 5"))
+    (list 1 0 1))
+  (apl-test
+    "⊤ encode: 24 60 60 ⊤ 7384 → 2 3 4 (HMS)"
+    (mkrv (apl-run "24 60 60 ⊤ 7384"))
+    (list 2 3 4))
+  (apl-test
+    "⊤ encode: 2 2 2 2 ⊤ 13 → 1 1 0 1"
+    (mkrv (apl-run "2 2 2 2 ⊤ 13"))
+    (list 1 1 0 1))
+  (apl-test
+    "⊤ encode: 10 10 ⊤ 42 → 4 2"
+    (mkrv (apl-run "10 10 ⊤ 42"))
+    (list 4 2))
+  (apl-test
+    "⊤ encode: round-trip B⊥(B⊤N) = N"
+    (mkrv (apl-run "24 60 60 ⊥ 24 60 60 ⊤ 7384"))
+    (list 7384))
+  (apl-test
+    "⊥ decode: round-trip B⊤(B⊥V) = V"
+    (mkrv (apl-run "2 2 2 ⊤ 2 2 2 ⊥ 1 0 1"))
+    (list 1 0 1)))
+
+(begin
+  (define
+    mk-parts
+    (fn (s) (map (fn (p) (get p :ravel)) (get (apl-run s) :ravel))))
+  (apl-test
+    "⊆ partition: 1 1 0 1 1 ⊆ 'abcde' → ('ab' 'de')"
+    (mk-parts "1 1 0 1 1 ⊆ 'abcde'")
+    (list (list "a" "b") (list "d" "e")))
+  (apl-test
+    "⊆ partition: 1 0 0 1 1 ⊆ ⍳5 → ((1) (4 5))"
+    (mk-parts "1 0 0 1 1 ⊆ ⍳5")
+    (list (list 1) (list 4 5)))
+  (apl-test
+    "⊆ partition: all-zero mask → empty"
+    (len (get (apl-run "0 0 0 ⊆ 1 2 3") :ravel))
+    0)
+  (apl-test
+    "⊆ partition: all-one mask → single partition"
+    (mk-parts "1 1 1 ⊆ 7 8 9")
+    (list (list 7 8 9)))
+  (apl-test
+    "⊆ partition: strict increase 1 2 starts new"
+    (mk-parts "1 2 ⊆ 10 20")
+    (list (list 10) (list 20)))
+  (apl-test
+    "⊆ partition: same level continues 2 2 → one partition"
+    (mk-parts "2 2 ⊆ 10 20")
+    (list (list 10 20)))
+  (apl-test
+    "⊆ partition: 0 separates"
+    (mk-parts "1 1 0 0 1 ⊆ 1 2 3 4 5")
+    (list (list 1 2) (list 5)))
+  (apl-test
+    "⊆ partition: outer length matches partition count"
+    (len (get (apl-run "1 0 1 0 1 ⊆ ⍳5") :ravel))
+    3))
+
+(begin
+  (apl-test
+    "⍎ execute: ⍎ '1 + 2' → 3"
+    (mkrv (apl-run "⍎ '1 + 2'"))
+    (list 3))
+  (apl-test
+    "⍎ execute: ⍎ '+/⍳10' → 55"
+    (mkrv (apl-run "⍎ '+/⍳10'"))
+    (list 55))
+  (apl-test
+    "⍎ execute: ⍎ '⌈/ 1 3 9 5 7' → 9"
+    (mkrv (apl-run "⍎ '⌈/ 1 3 9 5 7'"))
+    (list 9))
+  (apl-test
+    "⍎ execute: ⍎ '⍳5' → 1..5"
+    (mkrv (apl-run "⍎ '⍳5'"))
+    (list 1 2 3 4 5))
+  (apl-test
+    "⍎ execute: ⍎ '×/⍳5' → 120"
+    (mkrv (apl-run "⍎ '×/⍳5'"))
+    (list 120))
+  (apl-test
+    "⍎ execute: round-trip ⍎ ⎕FMT 42 → 42"
+    (mkrv (apl-run "⍎ ⎕FMT 42"))
+    (list 42))
+  (apl-test
+    "⍎ execute: nested ⍎ ⍎"
+    (mkrv (apl-run "⍎ '⍎ ''2 × 3'''"))
+    (list 6))
+  (apl-test
+    "⍎ execute: with assignment side-effect"
+    (mkrv (apl-run "⍎ 'q ← 99 ⋄ q + 1'"))
+    (list 100)))
+
+(begin
+  (apl-test
+    "het-inner: 1 ⍵ ∨.∧ X — result is enclosed (5 5)"
+    (let
+      ((r (apl-run "B ← 5 5 ⍴ 0 0 0 0 0 0 0 0 0 0 0 1 1 1 0 0 0 0 0 0 0 0 0 0 0 ⋄ X ← 3 4 = +/ +/ ¯1 0 1 ∘.⊖ ¯1 0 1 ⌽¨ ⊂B ⋄ 1 B ∨.∧ X")))
+      (list
+        (len (get r :shape))
+        (= (type-of (first (get r :ravel))) "dict")))
+    (list 0 true))
+  (apl-test
+    "het-inner: ⊃ unwraps to (5 5) board"
+    (mksh
+      (apl-run
+        "B ← 5 5 ⍴ 0 0 0 0 0 0 0 0 0 0 0 1 1 1 0 0 0 0 0 0 0 0 0 0 0 ⋄ X ← 3 4 = +/ +/ ¯1 0 1 ∘.⊖ ¯1 0 1 ⌽¨ ⊂B ⋄ ⊃ 1 B ∨.∧ X"))
+    (list 5 5))
+  (apl-test
+    "het-inner: homogeneous inner product unaffected"
+    (mkrv (apl-run "1 2 3 +.× 4 5 6"))
+    (list 32))
+  (apl-test
+    "het-inner: matrix inner product unaffected"
+    (mkrv (apl-run "(2 2 ⍴ 1 2 3 4) +.× 2 2 ⍴ 5 6 7 8"))
+    (list 19 22 43 50)))

lib/apl/tests/programs-e2e.sx

@@ -94,3 +94,96 @@
   "e2e: sqrt-via-newton 1 step from 1 → 2.5"
   (mkrv (apl-run "step ← {(⍵+⍺÷⍵)÷2} ⋄ 4 step 1"))
   (list 2.5))
+
+(begin
+  (apl-test
+    "life.apl: blinker 5×5 → vertical blinker"
+    (mkrv
+      (apl-run
+        "life ← {⊃1 ⍵ ∨.∧ 3 4 = +/ +/ ¯1 0 1 ∘.⊖ ¯1 0 1 ⌽¨ ⊂⍵} ⋄ life 5 5 ⍴ 0 0 0 0 0 0 0 0 0 0 0 1 1 1 0 0 0 0 0 0 0 0 0 0 0"))
+    (list 0 0 0 0 0 0 0 1 0 0 0 0 1 0 0 0 0 1 0 0 0 0 0 0 0))
+  (apl-test
+    "life.apl: blinker oscillates (period 2)"
+    (mkrv
+      (apl-run
+        "life ← {⊃1 ⍵ ∨.∧ 3 4 = +/ +/ ¯1 0 1 ∘.⊖ ¯1 0 1 ⌽¨ ⊂⍵} ⋄ life life 5 5 ⍴ 0 0 0 0 0 0 0 0 0 0 0 1 1 1 0 0 0 0 0 0 0 0 0 0 0"))
+    (list 0 0 0 0 0 0 0 0 0 0 0 1 1 1 0 0 0 0 0 0 0 0 0 0 0))
+  (apl-test
+    "life.apl: 2×2 block stable"
+    (mkrv
+      (apl-run
+        "life ← {⊃1 ⍵ ∨.∧ 3 4 = +/ +/ ¯1 0 1 ∘.⊖ ¯1 0 1 ⌽¨ ⊂⍵} ⋄ life 4 4 ⍴ 0 0 0 0 0 1 1 0 0 1 1 0 0 0 0 0"))
+    (list 0 0 0 0 0 1 1 0 0 1 1 0 0 0 0 0))
+  (apl-test
+    "life.apl: empty grid stays empty"
+    (mkrv
+      (apl-run
+        "life ← {⊃1 ⍵ ∨.∧ 3 4 = +/ +/ ¯1 0 1 ∘.⊖ ¯1 0 1 ⌽¨ ⊂⍵} ⋄ life 5 5 ⍴ 0"))
+    (list 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0))
+  (apl-test
+    "life.apl: source-file as-written runs"
+    (let
+      ((dfn (apl-run-file "lib/apl/tests/programs/life.apl"))
+        (board
+          (apl-run "5 5 ⍴ 0 0 0 0 0 0 0 0 0 0 0 1 1 1 0 0 0 0 0 0 0 0 0 0 0")))
+      (get (apl-call-dfn-m dfn board) :ravel))
+    (list 0 0 0 0 0 0 0 1 0 0 0 0 1 0 0 0 0 1 0 0 0 0 0 0 0)))
+
+(begin
+  (apl-test
+    "quicksort.apl: 11-element with duplicates"
+    (begin
+      (apl-rng-seed! 42)
+      (mkrv
+        (apl-run
+          "quicksort ← {1≥≢⍵:⍵ ⋄ p←⍵⌷⍨?≢⍵ ⋄ (∇⍵⌿⍨⍵<p),(p=⍵)/⍵,∇⍵⌿⍨⍵>p} ⋄ quicksort 3 1 4 1 5 9 2 6 5 3 5")))
+    (list 1 1 2 3 3 4 5 5 5 6 9))
+  (apl-test
+    "quicksort.apl: already sorted"
+    (begin
+      (apl-rng-seed! 42)
+      (mkrv
+        (apl-run
+          "quicksort ← {1≥≢⍵:⍵ ⋄ p←⍵⌷⍨?≢⍵ ⋄ (∇⍵⌿⍨⍵<p),(p=⍵)/⍵,∇⍵⌿⍨⍵>p} ⋄ quicksort 1 2 3 4 5")))
+    (list 1 2 3 4 5))
+  (apl-test
+    "quicksort.apl: reverse sorted"
+    (begin
+      (apl-rng-seed! 42)
+      (mkrv
+        (apl-run
+          "quicksort ← {1≥≢⍵:⍵ ⋄ p←⍵⌷⍨?≢⍵ ⋄ (∇⍵⌿⍨⍵<p),(p=⍵)/⍵,∇⍵⌿⍨⍵>p} ⋄ quicksort 5 4 3 2 1")))
+    (list 1 2 3 4 5))
+  (apl-test
+    "quicksort.apl: all equal"
+    (begin
+      (apl-rng-seed! 42)
+      (mkrv
+        (apl-run
+          "quicksort ← {1≥≢⍵:⍵ ⋄ p←⍵⌷⍨?≢⍵ ⋄ (∇⍵⌿⍨⍵<p),(p=⍵)/⍵,∇⍵⌿⍨⍵>p} ⋄ quicksort 7 7 7 7")))
+    (list 7 7 7 7))
+  (apl-test
+    "quicksort.apl: single element"
+    (begin
+      (apl-rng-seed! 42)
+      (mkrv
+        (apl-run
+          "quicksort ← {1≥≢⍵:⍵ ⋄ p←⍵⌷⍨?≢⍵ ⋄ (∇⍵⌿⍨⍵<p),(p=⍵)/⍵,∇⍵⌿⍨⍵>p} ⋄ quicksort ,42")))
+    (list 42))
+  (apl-test
+    "quicksort.apl: matches grade-up"
+    (begin
+      (apl-rng-seed! 42)
+      (mkrv
+        (apl-run
+          "V ← 8 3 1 9 2 7 5 6 4 ⋄ quicksort ← {1≥≢⍵:⍵ ⋄ p←⍵⌷⍨?≢⍵ ⋄ (∇⍵⌿⍨⍵<p),(p=⍵)/⍵,∇⍵⌿⍨⍵>p} ⋄ quicksort V")))
+    (list 1 2 3 4 5 6 7 8 9))
+  (apl-test
+    "quicksort.apl: source-file as-written runs"
+    (begin
+      (apl-rng-seed! 42)
+      (let
+        ((dfn (apl-run-file "lib/apl/tests/programs/quicksort.apl"))
+          (vec (apl-run "5 2 8 1 9 3 7 4 6")))
+        (get (apl-call-dfn-m dfn vec) :ravel)))
+    (list 1 2 3 4 5 6 7 8 9)))

lib/apl/tests/programs.sx

@@ -252,8 +252,6 @@
 
 (apl-test "queens 7 → 40 solutions" (mkrv (apl-queens 7)) (list 40))
 
-(apl-test "queens 8 → 92 solutions" (mkrv (apl-queens 8)) (list 92))
-
 (apl-test "permutations of 3 has 6" (len (apl-permutations 3)) 6)
 
 (apl-test "permutations of 4 has 24" (len (apl-permutations 4)) 24)

lib/apl/tests/programs/life.apl

@@ -8,9 +8,9 @@
 ⍝   ¯1 0 1 ⌽¨ ⊂⍵      : produce 3 horizontally-shifted copies
 ⍝   ¯1 0 1 ∘.⊖ …      : outer-product with vertical shifts → 3×3 = 9 shifts
 ⍝   +/ +/ …           : sum the 9 boards element-wise → neighbor-count + self
-⍝   3 4 = …           : boolean — count is exactly 3 or exactly 4
+⍝   3 4 = …           : leading-axis-extended boolean — count is 3 (born) or 4 (survive)
 ⍝   1 ⍵ ∨.∧ …         : "alive next" iff (count=3) or (alive AND count=4)
-⍝   ⊃ …               : disclose back to a 2D board
+⍝   ⊃ …               : disclose the enclosed result back to a 2D board
 ⍝
 ⍝ Rules in plain language:
 ⍝   - dead cell + 3 live neighbors → born

lib/apl/tokenizer.sx

@@ -19,162 +19,180 @@
              (and (>= ch "A") (<= ch "Z"))
              (= ch "_")))))
 
-(define apl-tokenize
-  (fn (source)
-    (let ((pos 0)
-          (src-len (len source))
-          (tokens (list)))
-
-      (define tok-push!
-        (fn (type value)
-          (append! tokens {:type type :value value})))
-
-      (define cur-sw?
-        (fn (ch)
+(define
+  apl-tokenize
+  (fn
+    (source)
+    (let
+      ((pos 0) (src-len (len source)) (tokens (list)))
+      (define tok-push! (fn (type value) (append! tokens {:value value :type type})))
+      (define
+        cur-sw?
+        (fn
+          (ch)
           (and (< pos src-len) (starts-with? (slice source pos) ch))))
-
-      (define cur-byte
-        (fn ()
-          (if (< pos src-len) (nth source pos) nil)))
-
-      (define advance!
-        (fn ()
-          (set! pos (+ pos 1))))
-
-      (define consume!
-        (fn (ch)
-          (set! pos (+ pos (len ch)))))
-
-      (define find-glyph
-        (fn ()
-          (let ((rem (slice source pos)))
-            (let ((matches (filter (fn (g) (starts-with? rem g)) apl-glyph-set)))
+      (define cur-byte (fn () (if (< pos src-len) (nth source pos) nil)))
+      (define advance! (fn () (set! pos (+ pos 1))))
+      (define consume! (fn (ch) (set! pos (+ pos (len ch)))))
+      (define
+        find-glyph
+        (fn
+          ()
+          (let
+            ((rem (slice source pos)))
+            (let
+              ((matches (filter (fn (g) (starts-with? rem g)) apl-glyph-set)))
               (if (> (len matches) 0) (first matches) nil)))))
-
-      (define read-digits!
-        (fn (acc)
-          (if (and (< pos src-len) (apl-digit? (cur-byte)))
-            (let ((ch (cur-byte)))
-              (begin
-                (advance!)
-                (read-digits! (str acc ch))))
+      (define
+        read-digits!
+        (fn
+          (acc)
+          (if
+            (and (< pos src-len) (apl-digit? (cur-byte)))
+            (let
+              ((ch (cur-byte)))
+              (begin (advance!) (read-digits! (str acc ch))))
             acc)))
-
-      (define read-ident-cont!
-        (fn ()
-          (when (and (< pos src-len)
-                     (let ((ch (cur-byte)))
-                       (or (apl-alpha? ch) (apl-digit? ch))))
-            (begin
-              (advance!)
-              (read-ident-cont!)))))
-
-      (define read-string!
-        (fn (acc)
+      (define
+        read-ident-cont!
+        (fn
+          ()
+          (when
+            (and
+              (< pos src-len)
+              (let
+                ((ch (cur-byte)))
+                (or (apl-alpha? ch) (apl-digit? ch))))
+            (begin (advance!) (read-ident-cont!)))))
+      (define
+        read-string!
+        (fn
+          (acc)
           (cond
             ((>= pos src-len) acc)
             ((cur-sw? "'")
-             (if (and (< (+ pos 1) src-len) (cur-sw? "'"))
-               (begin
-                 (advance!)
-                 (advance!)
-                 (read-string! (str acc "'")))
-               (begin (advance!) acc)))
+              (if
+                (and (< (+ pos 1) src-len) (cur-sw? "'"))
+                (begin (advance!) (advance!) (read-string! (str acc "'")))
+                (begin (advance!) acc)))
             (true
-             (let ((ch (cur-byte)))
-               (begin
-                 (advance!)
-                 (read-string! (str acc ch))))))))
-
-      (define skip-line!
-        (fn ()
-          (when (and (< pos src-len) (not (cur-sw? "\n")))
-            (begin
-              (advance!)
-              (skip-line!)))))
-
-      (define scan!
-        (fn ()
-          (when (< pos src-len)
-            (let ((ch (cur-byte)))
+              (let
+                ((ch (cur-byte)))
+                (begin (advance!) (read-string! (str acc ch))))))))
+      (define
+        skip-line!
+        (fn
+          ()
+          (when
+            (and (< pos src-len) (not (cur-sw? "\n")))
+            (begin (advance!) (skip-line!)))))
+      (define
+        scan!
+        (fn
+          ()
+          (when
+            (< pos src-len)
+            (let
+              ((ch (cur-byte)))
               (cond
                 ((or (= ch " ") (= ch "\t") (= ch "\r"))
-                 (begin (advance!) (scan!)))
+                  (begin (advance!) (scan!)))
                 ((= ch "\n")
-                 (begin (advance!) (tok-push! :newline nil) (scan!)))
-                ((cur-sw? "⍝")
-                 (begin (skip-line!) (scan!)))
+                  (begin (advance!) (tok-push! :newline nil) (scan!)))
+                ((cur-sw? "⍝") (begin (skip-line!) (scan!)))
                 ((cur-sw? "⋄")
-                 (begin (consume! "⋄") (tok-push! :diamond nil) (scan!)))
+                  (begin (consume! "⋄") (tok-push! :diamond nil) (scan!)))
                 ((= ch "(")
-                 (begin (advance!) (tok-push! :lparen nil) (scan!)))
+                  (begin (advance!) (tok-push! :lparen nil) (scan!)))
                 ((= ch ")")
-                 (begin (advance!) (tok-push! :rparen nil) (scan!)))
+                  (begin (advance!) (tok-push! :rparen nil) (scan!)))
                 ((= ch "[")
-                 (begin (advance!) (tok-push! :lbracket nil) (scan!)))
+                  (begin (advance!) (tok-push! :lbracket nil) (scan!)))
                 ((= ch "]")
-                 (begin (advance!) (tok-push! :rbracket nil) (scan!)))
+                  (begin (advance!) (tok-push! :rbracket nil) (scan!)))
                 ((= ch "{")
-                 (begin (advance!) (tok-push! :lbrace nil) (scan!)))
+                  (begin (advance!) (tok-push! :lbrace nil) (scan!)))
                 ((= ch "}")
-                 (begin (advance!) (tok-push! :rbrace nil) (scan!)))
+                  (begin (advance!) (tok-push! :rbrace nil) (scan!)))
                 ((= ch ";")
-                 (begin (advance!) (tok-push! :semi nil) (scan!)))
+                  (begin (advance!) (tok-push! :semi nil) (scan!)))
                 ((cur-sw? "←")
-                 (begin (consume! "←") (tok-push! :assign nil) (scan!)))
+                  (begin (consume! "←") (tok-push! :assign nil) (scan!)))
                 ((= ch ":")
-                 (let ((start pos))
-                   (begin
-                     (advance!)
-                     (if (and (< pos src-len) (apl-alpha? (cur-byte)))
-                       (begin
-                         (read-ident-cont!)
-                         (tok-push! :keyword (slice source start pos)))
-                       (tok-push! :colon nil))
-                     (scan!))))
-                ((and (cur-sw? "¯")
-                      (< (+ pos (len "¯")) src-len)
-                      (apl-digit? (nth source (+ pos (len "¯")))))
-                 (begin
-                   (consume! "¯")
-                   (let ((digits (read-digits! "")))
-                     (if (and (< pos src-len) (= (cur-byte) ".")
-                              (< (+ pos 1) src-len) (apl-digit? (nth source (+ pos 1))))
-                       (begin (advance!)
-                              (let ((frac (read-digits! "")))
-                                (tok-push! :num (- 0 (string->number (str digits "." frac))))))
-                       (tok-push! :num (- 0 (parse-int digits 0)))))
-                   (scan!)))
+                  (let
+                    ((start pos))
+                    (begin
+                      (advance!)
+                      (if
+                        (and (< pos src-len) (apl-alpha? (cur-byte)))
+                        (begin
+                          (read-ident-cont!)
+                          (tok-push! :keyword (slice source start pos)))
+                        (tok-push! :colon nil))
+                      (scan!))))
+                ((and (cur-sw? "¯") (< (+ pos (len "¯")) src-len) (apl-digit? (nth source (+ pos (len "¯")))))
+                  (begin
+                    (consume! "¯")
+                    (let
+                      ((digits (read-digits! "")))
+                      (if
+                        (and
+                          (< pos src-len)
+                          (= (cur-byte) ".")
+                          (< (+ pos 1) src-len)
+                          (apl-digit? (nth source (+ pos 1))))
+                        (begin
+                          (advance!)
+                          (let
+                            ((frac (read-digits! "")))
+                            (tok-push!
+                              :num (- 0 (string->number (str digits "." frac))))))
+                        (tok-push! :num (- 0 (parse-int digits 0)))))
+                    (scan!)))
                 ((apl-digit? ch)
-                 (begin
-                   (let ((digits (read-digits! "")))
-                     (if (and (< pos src-len) (= (cur-byte) ".")
-                              (< (+ pos 1) src-len) (apl-digit? (nth source (+ pos 1))))
-                       (begin (advance!)
-                              (let ((frac (read-digits! "")))
-                                (tok-push! :num (string->number (str digits "." frac)))))
-                       (tok-push! :num (parse-int digits 0))))
-                   (scan!)))
+                  (begin
+                    (let
+                      ((digits (read-digits! "")))
+                      (if
+                        (and
+                          (< pos src-len)
+                          (= (cur-byte) ".")
+                          (< (+ pos 1) src-len)
+                          (apl-digit? (nth source (+ pos 1))))
+                        (begin
+                          (advance!)
+                          (let
+                            ((frac (read-digits! "")))
+                            (tok-push!
+                              :num (string->number (str digits "." frac)))))
+                        (tok-push! :num (parse-int digits 0))))
+                    (scan!)))
                 ((= ch "'")
-                 (begin
-                   (advance!)
-                   (let ((s (read-string! "")))
-                     (tok-push! :str s))
-                   (scan!)))
+                  (begin
+                    (advance!)
+                    (let ((s (read-string! ""))) (tok-push! :str s))
+                    (scan!)))
                 ((or (apl-alpha? ch) (cur-sw? "⎕"))
-                 (let ((start pos))
-                   (begin
-                     (if (cur-sw? "⎕") (consume! "⎕") (advance!))
-                     (if (and (< pos src-len) (cur-sw? "←"))
-                       (consume! "←")
-                       (read-ident-cont!))
-                     (tok-push! :name (slice source start pos))
-                     (scan!))))
+                  (let
+                    ((start pos))
+                    (begin
+                      (if
+                        (cur-sw? "⎕")
+                        (begin
+                          (consume! "⎕")
+                          (if
+                            (and (< pos src-len) (cur-sw? "←"))
+                            (consume! "←")
+                            (read-ident-cont!)))
+                        (begin (advance!) (read-ident-cont!)))
+                      (tok-push! :name (slice source start pos))
+                      (scan!))))
                 (true
-                 (let ((g (find-glyph)))
-                   (if g
-                     (begin (consume! g) (tok-push! :glyph g) (scan!))
-                     (begin (advance!) (scan!))))))))))
-
+                  (let
+                    ((g (find-glyph)))
+                    (if
+                      g
+                      (begin (consume! g) (tok-push! :glyph g) (scan!))
+                      (begin (advance!) (scan!))))))))))
       (scan!)
       tokens)))

lib/apl/transpile.sx

@@ -39,8 +39,16 @@
       ((= g "⊖") apl-reverse-first)
       ((= g "⍋") apl-grade-up)
       ((= g "⍒") apl-grade-down)
+      ((= g "?") apl-roll)
+      ((= g "⍉") apl-transpose)
+      ((= g "⊢") (fn (a) a))
+      ((= g "⊣") (fn (a) a))
+      ((= g "⍕") apl-quad-fmt)
       ((= g "⎕FMT") apl-quad-fmt)
       ((= g "⎕←") apl-quad-print)
+      ((= g "⍸") apl-where)
+      ((= g "∪") apl-unique)
+      ((= g "⍎") apl-execute)
       (else (error "no monadic fn for glyph")))))
 
 (define
@@ -80,6 +88,17 @@
       ((= g "∊") apl-member)
       ((= g "⍳") apl-index-of)
       ((= g "~") apl-without)
+      ((= g "/") apl-compress)
+      ((= g "⌿") apl-compress-first)
+      ((= g "⍉") apl-transpose-dyadic)
+      ((= g "⊢") (fn (a b) b))
+      ((= g "⊣") (fn (a b) a))
+      ((= g "⍸") apl-interval-index)
+      ((= g "∪") apl-union)
+      ((= g "∩") apl-intersect)
+      ((= g "⊥") apl-decode)
+      ((= g "⊤") apl-encode)
+      ((= g "⊆") apl-partition)
       (else (error "no dyadic fn for glyph")))))
 
 (define
@@ -114,13 +133,26 @@
               ((vals (map (fn (n) (apl-eval-ast n env)) items)))
               (make-array
                 (list (len vals))
-                (map (fn (v) (first (get v :ravel))) vals)))))
+                (map
+                  (fn
+                    (v)
+                    (if
+                      (= (len (get v :shape)) 0)
+                      (first (get v :ravel))
+                      v))
+                  vals)))))
         ((= tag :name)
           (let
             ((nm (nth node 1)))
             (cond
-              ((= nm "⍺") (get env "alpha"))
-              ((= nm "⍵") (get env "omega"))
+              ((= nm "⍺")
+                (let
+                  ((v (get env "⍺")))
+                  (if (= v nil) (get env "alpha") v)))
+              ((= nm "⍵")
+                (let
+                  ((v (get env "⍵")))
+                  (if (= v nil) (get env "omega") v)))
               ((= nm "⎕IO") (apl-quad-io))
               ((= nm "⎕ML") (apl-quad-ml))
               ((= nm "⎕FR") (apl-quad-fr))
@@ -132,7 +164,11 @@
             (if
               (and (= (first fn-node) :fn-glyph) (= (nth fn-node 1) "∇"))
               (apl-call-dfn-m (get env "nabla") (apl-eval-ast arg env))
-              ((apl-resolve-monadic fn-node env) (apl-eval-ast arg env)))))
+              (let
+                ((arg-val (apl-eval-ast arg env)))
+                (let
+                  ((new-env (if (and (list? arg) (> (len arg) 0) (= (first arg) :assign-expr)) (assoc env (nth arg 1) arg-val) env)))
+                  ((apl-resolve-monadic fn-node new-env) arg-val))))))
         ((= tag :dyad)
           (let
             ((fn-node (nth node 1))
@@ -144,9 +180,13 @@
                 (get env "nabla")
                 (apl-eval-ast lhs env)
                 (apl-eval-ast rhs env))
-              ((apl-resolve-dyadic fn-node env)
-                (apl-eval-ast lhs env)
-                (apl-eval-ast rhs env)))))
+              (let
+                ((rhs-val (apl-eval-ast rhs env)))
+                (let
+                  ((new-env (if (and (list? rhs) (> (len rhs) 0) (= (first rhs) :assign-expr)) (assoc env (nth rhs 1) rhs-val) env)))
+                  ((apl-resolve-dyadic fn-node new-env)
+                    (apl-eval-ast lhs new-env)
+                    rhs-val))))))
         ((= tag :program) (apl-eval-stmts (rest node) env))
         ((= tag :dfn) node)
         ((= tag :bracket)
@@ -159,6 +199,8 @@
                     (fn (a) (if (= a :all) nil (apl-eval-ast a env)))
                     axis-exprs)))
               (apl-bracket-multi axes arr))))
+        ((= tag :assign-expr) (apl-eval-ast (nth node 2) env))
+        ((= tag :assign) (apl-eval-ast (nth node 2) env))
         (else (error (list "apl-eval-ast: unknown node tag" tag node)))))))
 
 (define
@@ -538,3 +580,13 @@
         (else (error "apl-resolve-dyadic: unknown fn-node tag"))))))
 
 (define apl-run (fn (src) (apl-eval-ast (parse-apl src) {})))
+
+(define apl-run-file (fn (path) (apl-run (file-read path))))
+
+(define
+  apl-execute
+  (fn
+    (arr)
+    (let
+      ((src (cond ((string? arr) arr) ((scalar? arr) (disclose arr)) (else (reduce str "" (get arr :ravel))))))
+      (apl-run src))))

lib/common-lisp/clos.sx

@@ -330,37 +330,22 @@
                     false))))))
         (check-all 0)))))
 
-;; Precedence distance: how far class-name is from spec-name up the hierarchy.
-(define
-  clos-specificity
-  (let
-    ((registry clos-class-registry))
-    (fn
-      (class-name spec-name)
-      (define
-        walk
-        (fn
-          (cn depth)
-          (if
-            (= cn spec-name)
-            depth
-            (let
-              ((rec (get registry cn)))
-              (if
-                (nil? rec)
-                nil
-                (let
-                  ((results (map (fn (p) (walk p (+ depth 1))) (get rec "parents"))))
-                  (let
-                    ((non-nil (filter (fn (x) (not (nil? x))) results)))
-                    (if
-                      (empty? non-nil)
-                      nil
-                      (reduce
-                        (fn (a b) (if (< a b) a b))
-                        (first non-nil)
-                        (rest non-nil))))))))))
-      (walk class-name 0))))
+;; CLOS-side adapter for lib/guest/reflective/class-chain.sx. Classes
+;; live in clos-class-registry; :parents is a list of parent class
+;; names (CLOS supports multiple inheritance).
+(define clos-class-cfg
+  {:parents-of (fn (cn)
+                 (let ((rec (clos-find-class cn)))
+                   (cond ((nil? rec) (list))
+                         (:else (or (get rec "parents") (list))))))
+   :class?     (fn (n) (not (nil? (clos-find-class n))))})
+
+;; Precedence distance: how far class-name is from spec-name up the
+;; hierarchy. Delegates to refl-class-chain-depth-with which handles
+;; the multi-parent DFS with min-depth selection.
+(define clos-specificity
+  (fn (class-name spec-name)
+    (refl-class-chain-depth-with clos-class-cfg class-name spec-name)))
 
 (define
   clos-method-more-specific?

lib/common-lisp/test.sh

@@ -368,7 +368,7 @@ run_program_suite \
 
 # ── Phase 4: CLOS unit tests ─────────────────────────────────────────────────
 CLOS_FILE=$(mktemp); trap "rm -f $CLOS_FILE" EXIT
-printf '(epoch 1)\n(load "spec/stdlib.sx")\n(epoch 2)\n(load "lib/common-lisp/runtime.sx")\n(epoch 3)\n(load "lib/common-lisp/clos.sx")\n(epoch 4)\n(load "lib/common-lisp/tests/clos.sx")\n(epoch 5)\n(eval "passed")\n(epoch 6)\n(eval "failed")\n(epoch 7)\n(eval "failures")\n' > "$CLOS_FILE"
+printf '(epoch 1)\n(load "spec/stdlib.sx")\n(epoch 2)\n(load "lib/common-lisp/runtime.sx")\n(epoch 3)\n(load "lib/guest/reflective/class-chain.sx")\n(load "lib/common-lisp/clos.sx")\n(epoch 4)\n(load "lib/common-lisp/tests/clos.sx")\n(epoch 5)\n(eval "passed")\n(epoch 6)\n(eval "failed")\n(epoch 7)\n(eval "failures")\n' > "$CLOS_FILE"
 CLOS_OUT=$(timeout 30 "$SX_SERVER" < "$CLOS_FILE" 2>/dev/null)
 rm -f "$CLOS_FILE"
 CLOS_PASSED=$(echo "$CLOS_OUT" | grep -A1 "^(ok-len 5 " | tail -1 || true)
@@ -389,7 +389,7 @@ fi
 run_clos_suite() {
   local prog="$1" pass_var="$2" fail_var="$3" failures_var="$4"
   local PROG_FILE=$(mktemp)
-  printf '(epoch 1)\n(load "spec/stdlib.sx")\n(epoch 2)\n(load "lib/common-lisp/runtime.sx")\n(epoch 3)\n(load "lib/common-lisp/clos.sx")\n(epoch 4)\n(load "%s")\n(epoch 5)\n(eval "%s")\n(epoch 6)\n(eval "%s")\n(epoch 7)\n(eval "%s")\n' \
+  printf '(epoch 1)\n(load "spec/stdlib.sx")\n(epoch 2)\n(load "lib/common-lisp/runtime.sx")\n(epoch 3)\n(load "lib/guest/reflective/class-chain.sx")\n(load "lib/common-lisp/clos.sx")\n(epoch 4)\n(load "%s")\n(epoch 5)\n(eval "%s")\n(epoch 6)\n(eval "%s")\n(epoch 7)\n(eval "%s")\n' \
     "$prog" "$pass_var" "$fail_var" "$failures_var" > "$PROG_FILE"
   local OUT; OUT=$(timeout 20 "$SX_SERVER" < "$PROG_FILE" 2>/dev/null)
   rm -f "$PROG_FILE"

lib/datalog/aggregates.sx

@@ -0,0 +1,157 @@
+;; lib/datalog/aggregates.sx — count / sum / min / max / findall.
+;;
+;; Surface form (always 3-arg after the relation name):
+;;
+;;   (count   Result  Var  GoalLit)
+;;   (sum     Result  Var  GoalLit)
+;;   (min     Result  Var  GoalLit)
+;;   (max     Result  Var  GoalLit)
+;;   (findall List    Var  GoalLit)
+;;
+;; Parsed naturally because arg-position compounds are already allowed
+;; (Phase 4 needs them for arithmetic). At evaluation time the aggregator
+;; runs `dl-find-bindings` on `GoalLit` under the current subst, collects
+;; the distinct values of `Var`, and binds `Result`.
+;;
+;; Aggregation is non-monotonic — `count(C, X, p(X))` shrinks as p loses
+;; tuples. The stratifier (lib/datalog/strata.sx) treats every aggregate's
+;; goal relation as a negation-like edge so the inner relation is fully
+;; derived before the aggregate fires.
+;;
+;; Empty input: count → 0, sum → 0, min/max → no binding (rule fails).
+
+(define dl-aggregate-rels (list "count" "sum" "min" "max" "findall"))
+
+(define
+  dl-aggregate?
+  (fn
+    (lit)
+    (and
+      (list? lit)
+      (>= (len lit) 4)
+      (let ((rel (dl-rel-name lit)))
+        (cond
+          ((nil? rel) false)
+          (else (dl-member-string? rel dl-aggregate-rels)))))))
+
+;; Apply aggregation operator to a list of (already-distinct) numeric or
+;; symbolic values. Returns the aggregated value, or :empty if min/max
+;; has no input.
+(define
+  dl-do-aggregate
+  (fn
+    (op vals)
+    (cond
+      ((= op "count") (len vals))
+      ((= op "sum") (dl-sum-vals vals 0))
+      ((= op "findall") vals)
+      ((= op "min")
+        (cond
+          ((= (len vals) 0) :empty)
+          (else (dl-min-vals vals 1 (first vals)))))
+      ((= op "max")
+        (cond
+          ((= (len vals) 0) :empty)
+          (else (dl-max-vals vals 1 (first vals)))))
+      (else (error (str "datalog: unknown aggregate " op))))))
+
+(define
+  dl-sum-vals
+  (fn
+    (vals acc)
+    (cond
+      ((= (len vals) 0) acc)
+      (else (dl-sum-vals (rest vals) (+ acc (first vals)))))))
+
+(define
+  dl-min-vals
+  (fn
+    (vals i cur)
+    (cond
+      ((>= i (len vals)) cur)
+      (else
+        (let ((v (nth vals i)))
+          (dl-min-vals vals (+ i 1) (if (< v cur) v cur)))))))
+
+(define
+  dl-max-vals
+  (fn
+    (vals i cur)
+    (cond
+      ((>= i (len vals)) cur)
+      (else
+        (let ((v (nth vals i)))
+          (dl-max-vals vals (+ i 1) (if (> v cur) v cur)))))))
+
+;; Membership check by deep equality (so 30 == 30.0 etc).
+(define
+  dl-val-member?
+  (fn
+    (v xs)
+    (cond
+      ((= (len xs) 0) false)
+      ((dl-tuple-equal? v (first xs)) true)
+      (else (dl-val-member? v (rest xs))))))
+
+;; Evaluate an aggregate body lit under `subst`. Returns the list of
+;; extended substitutions (0 or 1 element).
+(define
+  dl-eval-aggregate
+  (fn
+    (lit db subst)
+    (let
+      ((op (dl-rel-name lit))
+        (result-var (nth lit 1))
+        (agg-var (nth lit 2))
+        (goal (nth lit 3)))
+      (cond
+        ((not (dl-var? agg-var))
+          (error (str "datalog aggregate (" op
+                      "): second arg must be a variable, got " agg-var)))
+        ((not (and (list? goal) (> (len goal) 0)
+                   (symbol? (first goal))))
+          (error (str "datalog aggregate (" op
+                      "): third arg must be a positive literal, got "
+                      goal)))
+        ((not (dl-member-string?
+                (symbol->string agg-var)
+                (dl-vars-of goal)))
+          (error (str "datalog aggregate (" op
+                      "): aggregation variable " agg-var
+                      " does not appear in the goal " goal
+                      " — without it every match contributes the same "
+                      "(unbound) value and the result is meaningless")))
+        (else
+          (let ((vals (list)))
+            (do
+              (for-each
+                (fn
+                  (s)
+                  (let ((v (dl-apply-subst agg-var s)))
+                    (when (not (dl-val-member? v vals))
+                      (append! vals v))))
+                (dl-find-bindings (list goal) db subst))
+              (let ((agg-val (dl-do-aggregate op vals)))
+                (cond
+                  ((= agg-val :empty) (list))
+                  (else
+                    (let ((s2 (dl-unify result-var agg-val subst)))
+                      (if (nil? s2) (list) (list s2)))))))))))))
+
+
+;; Stratification edges from aggregates: like negation, the goal's
+;; relation must be in a strictly lower stratum so that the aggregate
+;; fires only after the underlying tuples are settled.
+(define
+  dl-aggregate-dep-edge
+  (fn
+    (lit)
+    (cond
+      ((dl-aggregate? lit)
+        (let ((goal (nth lit 3)))
+          (cond
+            ((and (list? goal) (> (len goal) 0))
+              (let ((rel (dl-rel-name goal)))
+                (if (nil? rel) nil {:rel rel :neg true})))
+            (else nil))))
+      (else nil))))

lib/datalog/api.sx

@@ -0,0 +1,303 @@
+;; lib/datalog/api.sx — SX-data embedding API.
+;;
+;; Where Phase 1's `dl-program` takes a Datalog source string,
+;; this module exposes a parser-free API that consumes SX data
+;; directly. Two rule shapes are accepted:
+;;
+;;   - dict:   {:head <literal> :body (<literal> ...)}
+;;   - list:   (<head-elements...> <- <body-literal> ...)
+;;             — `<-` is an SX symbol used as the rule arrow.
+;;
+;; Examples:
+;;
+;;   (dl-program-data
+;;     '((parent tom bob) (parent tom liz) (parent bob ann))
+;;     '((ancestor X Y <- (parent X Y))
+;;       (ancestor X Z <- (parent X Y) (ancestor Y Z))))
+;;
+;;   (dl-query db '(ancestor tom X))   ; same query API as before
+;;
+;; Variables follow the parser convention: SX symbols whose first
+;; character is uppercase or `_` are variables.
+
+(define
+  dl-rule
+  (fn (head body) {:head head :body body}))
+
+(define
+  dl-rule-arrow?
+  (fn
+    (x)
+    (and (symbol? x) (= (symbol->string x) "<-"))))
+
+(define
+  dl-find-arrow
+  (fn
+    (rl i n)
+    (cond
+      ((>= i n) nil)
+      ((dl-rule-arrow? (nth rl i)) i)
+      (else (dl-find-arrow rl (+ i 1) n)))))
+
+;; Given a list of the form (head-elt ... <- body-lit ...) returns
+;; {:head (head-elt ...) :body (body-lit ...)}. If no arrow is
+;; present, the whole list is treated as the head and the body is
+;; empty (i.e. a fact written rule-style).
+(define
+  dl-rule-from-list
+  (fn
+    (rl)
+    (let ((n (len rl)))
+      (let ((idx (dl-find-arrow rl 0 n)))
+        (cond
+          ((nil? idx) {:head rl :body (list)})
+          (else
+            (let
+              ((head (slice rl 0 idx))
+                (body (slice rl (+ idx 1) n)))
+              {:head head :body body})))))))
+
+;; Coerce a rule given as either a dict or a list-with-arrow to a dict.
+(define
+  dl-coerce-rule
+  (fn
+    (r)
+    (cond
+      ((dict? r) r)
+      ((list? r) (dl-rule-from-list r))
+      (else (error (str "dl-coerce-rule: expected dict or list, got " r))))))
+
+;; Build a db from SX data lists.
+(define
+  dl-program-data
+  (fn
+    (facts rules)
+    (let ((db (dl-make-db)))
+      (do
+        (for-each (fn (lit) (dl-add-fact! db lit)) facts)
+        (for-each
+          (fn (r) (dl-add-rule! db (dl-coerce-rule r)))
+          rules)
+        db))))
+
+;; Add a single fact at runtime, then re-saturate the db so derived
+;; tuples reflect the change. Returns the db.
+(define
+  dl-assert!
+  (fn
+    (db lit)
+    (do
+      (dl-add-fact! db lit)
+      (dl-saturate! db)
+      db)))
+
+;; Remove a fact and re-saturate. Mixed relations (which have BOTH
+;; user-asserted facts AND rules) are supported via :edb-keys provenance
+;; — explicit facts are marked at dl-add-fact! time, the saturator uses
+;; dl-add-derived! which doesn't mark them, so the retract pass can
+;; safely wipe IDB-derived tuples while preserving the user's EDB.
+;;
+;; Effect:
+;;   - remove tuples matching `lit` from :facts and :edb-keys
+;;   - for every relation that has a rule (i.e. potentially IDB or
+;;     mixed), drop the IDB-derived portion (anything not in :edb-keys)
+;;     so the saturator can re-derive cleanly
+;;   - re-saturate
+(define
+  dl-retract!
+  (fn
+    (db lit)
+    (let
+      ((rel-key (dl-rel-name lit)))
+      (do
+        ;; Drop the matching tuple from its relation list, its facts-keys,
+        ;; its first-arg index, AND from :edb-keys (if present).
+        (when
+          (has-key? (get db :facts) rel-key)
+          (let
+            ((existing (get (get db :facts) rel-key))
+              (kept (list))
+              (kept-keys {})
+              (kept-index {})
+              (edb-rel (cond
+                         ((has-key? (get db :edb-keys) rel-key)
+                           (get (get db :edb-keys) rel-key))
+                         (else nil)))
+              (kept-edb {}))
+            (do
+              (for-each
+                (fn
+                  (t)
+                  (when
+                    (not (dl-tuple-equal? t lit))
+                    (do
+                      (append! kept t)
+                      (let ((tk (dl-tuple-key t)))
+                        (do
+                          (dict-set! kept-keys tk true)
+                          (when
+                            (and (not (nil? edb-rel))
+                                 (has-key? edb-rel tk))
+                            (dict-set! kept-edb tk true))))
+                      (when
+                        (>= (len t) 2)
+                        (let ((k (dl-arg-key (nth t 1))))
+                          (do
+                            (when
+                              (not (has-key? kept-index k))
+                              (dict-set! kept-index k (list)))
+                            (append! (get kept-index k) t)))))))
+                existing)
+              (dict-set! (get db :facts) rel-key kept)
+              (dict-set! (get db :facts-keys) rel-key kept-keys)
+              (dict-set! (get db :facts-index) rel-key kept-index)
+              (when
+                (not (nil? edb-rel))
+                (dict-set! (get db :edb-keys) rel-key kept-edb)))))
+        ;; For each rule-head relation, strip the IDB-derived tuples
+        ;; (anything not marked in :edb-keys) so the saturator can
+        ;; cleanly re-derive without leaving stale tuples that depended
+        ;; on the now-removed fact.
+        (let ((rule-heads (dl-rule-head-rels db)))
+          (for-each
+            (fn
+              (k)
+              (when
+                (has-key? (get db :facts) k)
+                (let
+                  ((existing (get (get db :facts) k))
+                    (kept (list))
+                    (kept-keys {})
+                    (kept-index {})
+                    (edb-rel (cond
+                               ((has-key? (get db :edb-keys) k)
+                                 (get (get db :edb-keys) k))
+                               (else {}))))
+                  (do
+                    (for-each
+                      (fn
+                        (t)
+                        (let ((tk (dl-tuple-key t)))
+                          (when
+                            (has-key? edb-rel tk)
+                            (do
+                              (append! kept t)
+                              (dict-set! kept-keys tk true)
+                              (when
+                                (>= (len t) 2)
+                                (let ((kk (dl-arg-key (nth t 1))))
+                                  (do
+                                    (when
+                                      (not (has-key? kept-index kk))
+                                      (dict-set! kept-index kk (list)))
+                                    (append! (get kept-index kk) t))))))))
+                      existing)
+                    (dict-set! (get db :facts) k kept)
+                    (dict-set! (get db :facts-keys) k kept-keys)
+                    (dict-set! (get db :facts-index) k kept-index)))))
+            rule-heads))
+        (dl-saturate! db)
+        db))))
+
+;; ── Convenience: single-call source + query ───────────────────
+;; (dl-eval source query-source) parses both, builds a db, saturates,
+;; runs the query, returns the substitution list. The query source
+;; should be `?- goal[, goal ...].` — the parser produces a clause
+;; with :query containing a list of literals which is fed straight
+;; to dl-query.
+(define
+  dl-eval
+  (fn
+    (source query-source)
+    (let
+      ((db (dl-program source))
+        (queries (dl-parse query-source)))
+      (cond
+        ((= (len queries) 0) (error "dl-eval: query string is empty"))
+        ((not (has-key? (first queries) :query))
+          (error "dl-eval: second arg must be a `?- ...` query clause"))
+        (else
+          (dl-query db (get (first queries) :query)))))))
+
+;; (dl-eval-magic source query-source) — like dl-eval but routes a
+;; single-positive-literal query through `dl-magic-query` for goal-
+;; directed evaluation. Multi-literal query bodies fall back to the
+;; standard dl-query path (magic-sets is currently only wired for
+;; single-positive goals). The caller's source is parsed afresh
+;; each call so successive invocations are independent.
+(define
+  dl-eval-magic
+  (fn
+    (source query-source)
+    (let
+      ((db (dl-program source))
+        (queries (dl-parse query-source)))
+      (cond
+        ((= (len queries) 0) (error "dl-eval-magic: query string is empty"))
+        ((not (has-key? (first queries) :query))
+          (error
+            "dl-eval-magic: second arg must be a `?- ...` query clause"))
+        (else
+          (let
+            ((qbody (get (first queries) :query)))
+            (cond
+              ((and (= (len qbody) 1)
+                    (list? (first qbody))
+                    (> (len (first qbody)) 0)
+                    (symbol? (first (first qbody))))
+                (dl-magic-query db (first qbody)))
+              (else (dl-query db qbody)))))))))
+
+;; List rules whose head's relation matches `rel-name`. Useful for
+;; inspection ("show me how this relation is derived") without
+;; exposing the internal `:rules` list.
+(define
+  dl-rules-of
+  (fn
+    (db rel-name)
+    (let ((out (list)))
+      (do
+        (for-each
+          (fn
+            (rule)
+            (when
+              (= (dl-rel-name (get rule :head)) rel-name)
+              (append! out rule)))
+          (dl-rules db))
+        out))))
+
+(define
+  dl-rule-head-rels
+  (fn
+    (db)
+    (let ((seen (list)))
+      (do
+        (for-each
+          (fn
+            (rule)
+            (let ((h (dl-rel-name (get rule :head))))
+              (when
+                (and (not (nil? h)) (not (dl-member-string? h seen)))
+                (append! seen h))))
+          (dl-rules db))
+        seen))))
+
+;; Wipe every relation that has at least one rule (i.e. every IDB
+;; relation) — leaves EDB facts and rule definitions intact. Useful
+;; before a follow-up `dl-saturate!` if you want a clean restart, or
+;; for inspection of the EDB-only baseline.
+(define
+  dl-clear-idb!
+  (fn
+    (db)
+    (let ((rule-heads (dl-rule-head-rels db)))
+      (do
+        (for-each
+          (fn
+            (k)
+            (do
+              (dict-set! (get db :facts) k (list))
+              (dict-set! (get db :facts-keys) k {})
+              (dict-set! (get db :facts-index) k {})))
+          rule-heads)
+        db))))

lib/datalog/builtins.sx

@@ -0,0 +1,406 @@
+;; lib/datalog/builtins.sx — comparison + arithmetic body literals.
+;;
+;; Built-in predicates filter / extend candidate substitutions during
+;; rule evaluation. They are not stored facts and do not participate in
+;; the Herbrand base.
+;;
+;;   (< a b)  (<= a b)  (> a b)  (>= a b)   ; numeric (or string) compare
+;;   (= a b)                                  ; unify (binds vars)
+;;   (!= a b)                                ; ground-only inequality
+;;   (is X expr)                              ; bind X to expr's value
+;;
+;; Arithmetic expressions are SX-list compounds:
+;;   (+ a b)  (- a b)  (* a b)  (/ a b)
+;; or numbers / variables (must be bound at evaluation time).
+
+(define
+  dl-comparison?
+  (fn
+    (lit)
+    (and
+      (list? lit)
+      (> (len lit) 0)
+      (let
+        ((rel (dl-rel-name lit)))
+        (cond
+          ((nil? rel) false)
+          (else (dl-member-string? rel (list "<" "<=" ">" ">=" "!="))))))))
+
+(define
+  dl-eq?
+  (fn
+    (lit)
+    (and
+      (list? lit)
+      (> (len lit) 0)
+      (let ((rel (dl-rel-name lit))) (and (not (nil? rel)) (= rel "=")))))) 
+
+(define
+  dl-is?
+  (fn
+    (lit)
+    (and
+      (list? lit)
+      (> (len lit) 0)
+      (let
+        ((rel (dl-rel-name lit)))
+        (and (not (nil? rel)) (= rel "is")))))) 
+
+;; Evaluate an arithmetic expression under subst. Returns the numeric
+;; result, or raises if any operand is unbound or non-numeric.
+(define
+  dl-eval-arith
+  (fn
+    (expr subst)
+    (let
+      ((w (dl-walk expr subst)))
+      (cond
+        ((number? w) w)
+        ((dl-var? w)
+          (error (str "datalog arith: unbound variable " (symbol->string w))))
+        ((list? w)
+          (let
+            ((rel (dl-rel-name w)) (args (rest w)))
+            (cond
+              ((not (= (len args) 2))
+                (error (str "datalog arith: need 2 args, got " w)))
+              (else
+                (let
+                  ((a (dl-eval-arith (first args) subst))
+                    (b (dl-eval-arith (nth args 1) subst)))
+                  (cond
+                    ((= rel "+") (+ a b))
+                    ((= rel "-") (- a b))
+                    ((= rel "*") (* a b))
+                    ((= rel "/")
+                      (cond
+                        ((= b 0)
+                          (error
+                            (str "datalog arith: division by zero in "
+                                 w)))
+                        (else (/ a b))))
+                    (else (error (str "datalog arith: unknown op " rel)))))))))
+        (else (error (str "datalog arith: not a number — " w)))))))
+
+;; Comparable types — both operands must be the same primitive type
+;; (both numbers, both strings). `!=` is the exception: it's defined
+;; for any pair (returns true iff not equal) since dl-tuple-equal?
+;; handles type-mixed comparisons.
+(define
+  dl-compare-typeok?
+  (fn
+    (rel a b)
+    (cond
+      ((= rel "!=") true)
+      ((and (number? a) (number? b)) true)
+      ((and (string? a) (string? b)) true)
+      (else false))))
+
+(define
+  dl-eval-compare
+  (fn
+    (lit subst)
+    (let
+      ((rel (dl-rel-name lit))
+        (a (dl-walk (nth lit 1) subst))
+        (b (dl-walk (nth lit 2) subst)))
+      (cond
+        ((or (dl-var? a) (dl-var? b))
+          (error
+            (str
+              "datalog: comparison "
+              rel
+              " has unbound argument; "
+              "ensure prior body literal binds the variable")))
+        ((not (dl-compare-typeok? rel a b))
+          (error
+            (str "datalog: comparison " rel " requires same-type "
+                 "operands (both numbers or both strings), got "
+                 a " and " b)))
+        (else
+          (let
+            ((ok (cond ((= rel "<") (< a b)) ((= rel "<=") (<= a b)) ((= rel ">") (> a b)) ((= rel ">=") (>= a b)) ((= rel "!=") (not (dl-tuple-equal? a b))) (else (error (str "datalog: unknown compare " rel))))))
+            (if ok subst nil)))))))
+
+(define
+  dl-eval-eq
+  (fn
+    (lit subst)
+    (dl-unify (nth lit 1) (nth lit 2) subst)))
+
+(define
+  dl-eval-is
+  (fn
+    (lit subst)
+    (let
+      ((target (nth lit 1)) (expr (nth lit 2)))
+      (let
+        ((value (dl-eval-arith expr subst)))
+        (dl-unify target value subst)))))
+
+(define
+  dl-eval-builtin
+  (fn
+    (lit subst)
+    (cond
+      ((dl-comparison? lit) (dl-eval-compare lit subst))
+      ((dl-eq? lit) (dl-eval-eq lit subst))
+      ((dl-is? lit) (dl-eval-is lit subst))
+      (else (error (str "dl-eval-builtin: not a built-in: " lit))))))
+
+;; ── Safety analysis ──────────────────────────────────────────────
+;;
+;; Walks body literals left-to-right tracking a "bound" set. The check
+;; understands these literal kinds:
+;;
+;;   positive non-built-in    → adds its vars to bound
+;;   (is X expr)              → vars(expr) ⊆ bound, then add X (if var)
+;;   <,<=,>,>=,!=             → all vars ⊆ bound (no binding)
+;;   (= a b) where:
+;;       both non-vars        → constraint check, no binding
+;;       a var, b not         → bind a
+;;       b var, a not         → bind b
+;;       both vars            → at least one in bound; bind the other
+;;   {:neg lit}               → all vars ⊆ bound (Phase 7 enforces fully)
+;;
+;; At end, head vars (minus `_`) must be ⊆ bound.
+
+(define
+  dl-vars-not-in
+  (fn
+    (vs bound)
+    (let
+      ((out (list)))
+      (do
+        (for-each
+          (fn
+            (v)
+            (when (not (dl-member-string? v bound)) (append! out v)))
+          vs)
+        out))))
+
+;; Filter a list of variable-name strings to exclude anonymous-renamed
+;; vars (`_` in source → `_anon*` by dl-rename-anon-term). Used by
+;; the negation safety check, where anonymous vars are existential
+;; within the negated literal.
+(define
+  dl-non-anon-vars
+  (fn
+    (vs)
+    (let
+      ((out (list)))
+      (do
+        (for-each
+          (fn
+            (v)
+            (when
+              (not (and (>= (len v) 5)
+                        (= (slice v 0 5) "_anon")))
+              (append! out v)))
+          vs)
+        out))))
+
+(define
+  dl-rule-check-safety
+  (fn
+    (rule)
+    (let
+      ((head (get rule :head))
+        (body (get rule :body))
+        (bound (list))
+        (err nil))
+      (do
+        (define
+          dl-add-bound!
+          (fn
+            (vs)
+            (for-each
+              (fn
+                (v)
+                (when (not (dl-member-string? v bound)) (append! bound v)))
+              vs)))
+        (define
+          dl-process-eq!
+          (fn
+            (lit)
+            (let
+              ((a (nth lit 1)) (b (nth lit 2)))
+              (let
+                ((va (dl-var? a)) (vb (dl-var? b)))
+                (cond
+                  ((and (not va) (not vb)) nil)
+                  ((and va (not vb))
+                    (dl-add-bound! (list (symbol->string a))))
+                  ((and (not va) vb)
+                    (dl-add-bound! (list (symbol->string b))))
+                  (else
+                    (let
+                      ((sa (symbol->string a)) (sb (symbol->string b)))
+                      (cond
+                        ((dl-member-string? sa bound)
+                          (dl-add-bound! (list sb)))
+                        ((dl-member-string? sb bound)
+                          (dl-add-bound! (list sa)))
+                        (else
+                          (set!
+                            err
+                            (str
+                              "= between two unbound variables "
+                              (list sa sb)
+                              " — at least one must be bound by an "
+                              "earlier positive body literal")))))))))))
+        (define
+          dl-process-cmp!
+          (fn
+            (lit)
+            (let
+              ((needed (dl-vars-of (list (nth lit 1) (nth lit 2)))))
+              (let
+                ((missing (dl-vars-not-in needed bound)))
+                (when
+                  (> (len missing) 0)
+                  (set!
+                    err
+                    (str
+                      "comparison "
+                      (dl-rel-name lit)
+                      " requires bound variable(s) "
+                      missing
+                      " (must be bound by an earlier positive "
+                      "body literal)")))))))
+        (define
+          dl-process-is!
+          (fn
+            (lit)
+            (let
+              ((tgt (nth lit 1)) (expr (nth lit 2)))
+              (let
+                ((needed (dl-vars-of expr)))
+                (let
+                  ((missing (dl-vars-not-in needed bound)))
+                  (cond
+                    ((> (len missing) 0)
+                      (set!
+                        err
+                        (str
+                          "is RHS uses unbound variable(s) "
+                          missing
+                          " — bind them via a prior positive body "
+                          "literal")))
+                    (else
+                      (when
+                        (dl-var? tgt)
+                        (dl-add-bound! (list (symbol->string tgt)))))))))))
+        (define
+          dl-process-neg!
+          (fn
+            (lit)
+            (let
+              ((inner (get lit :neg)))
+              (let
+                ((inner-rn
+                   (cond
+                     ((and (list? inner) (> (len inner) 0))
+                       (dl-rel-name inner))
+                     (else nil)))
+                  ;; Anonymous variables (`_` in source → `_anon*` after
+                  ;; renaming) are existentially quantified within the
+                  ;; negated literal — they don't need to be bound by
+                  ;; an earlier body lit, since `not p(X, _)` is a
+                  ;; valid idiom for "no Y exists s.t. p(X, Y)". Filter
+                  ;; them out of the safety check.
+                  (needed (dl-non-anon-vars (dl-vars-of inner)))
+                  (missing (dl-vars-not-in needed bound)))
+                (cond
+                  ((and (not (nil? inner-rn)) (dl-reserved-rel? inner-rn))
+                    (set! err
+                      (str "negated literal uses reserved name '"
+                           inner-rn
+                           "' — nested `not(...)` / negated built-ins are "
+                           "not supported; introduce an intermediate "
+                           "relation and negate that")))
+                  ((> (len missing) 0)
+                    (set! err
+                      (str "negation refers to unbound variable(s) "
+                           missing
+                           " — they must be bound by an earlier "
+                           "positive body literal"))))))))
+        (define
+          dl-process-agg!
+          (fn
+            (lit)
+            (let
+              ((result-var (nth lit 1)))
+              ;; Aggregate goal vars are existentially quantified within
+              ;; the aggregate; nothing required from outer context. The
+              ;; result var becomes bound after the aggregate fires.
+              (when
+                (dl-var? result-var)
+                (dl-add-bound! (list (symbol->string result-var)))))))
+
+        (define
+          dl-process-lit!
+          (fn
+            (lit)
+            (when
+              (nil? err)
+              (cond
+                ((and (dict? lit) (has-key? lit :neg))
+                  (dl-process-neg! lit))
+                ;; A bare dict that is not a recognised negation is
+                ;; almost certainly a typo (e.g. `{:negs ...}` instead
+                ;; of `{:neg ...}`). Without this guard the dict would
+                ;; silently fall through every clause; the head safety
+                ;; check would then flag the head variables as unbound
+                ;; even though the real bug is the malformed body lit.
+                ((dict? lit)
+                  (set! err
+                    (str "body literal is a dict but lacks :neg — "
+                         "the only dict-shaped body lit recognised is "
+                         "{:neg <positive-lit>} for stratified "
+                         "negation, got " lit)))
+                ((dl-aggregate? lit) (dl-process-agg! lit))
+                ((dl-eq? lit) (dl-process-eq! lit))
+                ((dl-is? lit) (dl-process-is! lit))
+                ((dl-comparison? lit) (dl-process-cmp! lit))
+                ((and (list? lit) (> (len lit) 0))
+                  (let ((rn (dl-rel-name lit)))
+                    (cond
+                      ((and (not (nil? rn)) (dl-reserved-rel? rn))
+                        (set! err
+                          (str "body literal uses reserved name '" rn
+                               "' — built-ins / aggregates have their own "
+                               "syntax; nested `not(...)` is not supported "
+                               "(use stratified negation via an "
+                               "intermediate relation)")))
+                      (else (dl-add-bound! (dl-vars-of lit))))))
+                (else
+                  ;; Anything that's not a dict, not a list, or an
+                  ;; empty list. Numbers / strings / symbols as body
+                  ;; lits don't make sense — surface the type.
+                  (set! err
+                    (str "body literal must be a positive lit, "
+                         "built-in, aggregate, or {:neg ...} dict, "
+                         "got " lit)))))))
+        (for-each dl-process-lit! body)
+        (when
+          (nil? err)
+          (let
+            ((head-vars (dl-vars-of head)) (missing (list)))
+            (do
+              (for-each
+                (fn
+                  (v)
+                  (when
+                    (and (not (dl-member-string? v bound)) (not (= v "_")))
+                    (append! missing v)))
+                head-vars)
+              (when
+                (> (len missing) 0)
+                (set!
+                  err
+                  (str
+                    "head variable(s) "
+                    missing
+                    " do not appear in any positive body literal"))))))
+        err))))

lib/datalog/conformance.conf

@@ -0,0 +1,32 @@
+# Datalog conformance config — sourced by lib/guest/conformance.sh.
+
+LANG_NAME=datalog
+MODE=dict
+
+PRELOADS=(
+  lib/datalog/tokenizer.sx
+  lib/datalog/parser.sx
+  lib/datalog/unify.sx
+  lib/datalog/db.sx
+  lib/datalog/builtins.sx
+  lib/datalog/aggregates.sx
+  lib/datalog/strata.sx
+  lib/datalog/eval.sx
+  lib/datalog/api.sx
+  lib/datalog/magic.sx
+  lib/datalog/demo.sx
+)
+
+SUITES=(
+  "tokenize:lib/datalog/tests/tokenize.sx:(dl-tokenize-tests-run!)"
+  "parse:lib/datalog/tests/parse.sx:(dl-parse-tests-run!)"
+  "unify:lib/datalog/tests/unify.sx:(dl-unify-tests-run!)"
+  "eval:lib/datalog/tests/eval.sx:(dl-eval-tests-run!)"
+  "builtins:lib/datalog/tests/builtins.sx:(dl-builtins-tests-run!)"
+  "semi_naive:lib/datalog/tests/semi_naive.sx:(dl-semi-naive-tests-run!)"
+  "negation:lib/datalog/tests/negation.sx:(dl-negation-tests-run!)"
+  "aggregates:lib/datalog/tests/aggregates.sx:(dl-aggregates-tests-run!)"
+  "api:lib/datalog/tests/api.sx:(dl-api-tests-run!)"
+  "magic:lib/datalog/tests/magic.sx:(dl-magic-tests-run!)"
+  "demo:lib/datalog/tests/demo.sx:(dl-demo-tests-run!)"
+)

lib/datalog/conformance.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+# Thin wrapper — see lib/guest/conformance.sh and lib/datalog/conformance.conf.
+exec bash "$(dirname "$0")/../guest/conformance.sh" "$(dirname "$0")/conformance.conf" "$@"

lib/datalog/datalog.sx

@@ -0,0 +1,97 @@
+;; lib/datalog/datalog.sx — public API documentation index.
+;;
+;; This file is reference-only — `load` is an epoch-protocol command,
+;; not an SX function, so it cannot reload a list of files from inside
+;; another `.sx` file. To set up a fresh sx_server session with all
+;; modules in scope, issue these loads in order:
+;;
+;;   (load "lib/datalog/tokenizer.sx")
+;;   (load "lib/datalog/parser.sx")
+;;   (load "lib/datalog/unify.sx")
+;;   (load "lib/datalog/db.sx")
+;;   (load "lib/datalog/builtins.sx")
+;;   (load "lib/datalog/aggregates.sx")
+;;   (load "lib/datalog/strata.sx")
+;;   (load "lib/datalog/eval.sx")
+;;   (load "lib/datalog/api.sx")
+;;   (load "lib/datalog/magic.sx")
+;;   (load "lib/datalog/demo.sx")
+;;
+;; (lib/datalog/conformance.sh runs this load list automatically.)
+;;
+;; ── Public API surface ─────────────────────────────────────────────
+;;
+;;   Source / data:
+;;     (dl-tokenize "src")              → token list
+;;     (dl-parse "src")                 → parsed clauses
+;;     (dl-program "src")               → db built from a source string
+;;     (dl-program-data facts rules)    → db from SX data lists; rules
+;;                                        accept either dict form or
+;;                                        list form with `<-` arrow
+;;
+;;   Construction (mutates db):
+;;     (dl-make-db)                     empty db
+;;     (dl-add-fact! db lit)            rejects non-ground
+;;     (dl-add-rule! db rule)           rejects unsafe rules
+;;     (dl-rule head body)              dict-rule constructor
+;;     (dl-add-clause! db clause)       parser output → fact or rule
+;;     (dl-load-program! db src)        string source
+;;     (dl-set-strategy! db strategy)   :semi-naive default; :magic
+;;                                        is informational, use
+;;                                        dl-magic-query for actual
+;;                                        magic-sets evaluation
+;;
+;;   Mutation:
+;;     (dl-assert! db lit)              add + re-saturate
+;;     (dl-retract! db lit)             drop EDB, wipe IDB, re-saturate
+;;     (dl-clear-idb! db)               wipe rule-headed relations
+;;
+;;   Query / inspection:
+;;     (dl-saturate! db)                stratified semi-naive default
+;;     (dl-saturate-naive! db)          reference (slow on chains)
+;;     (dl-saturate-rules! db rules)    per-rule-set semi-naive worker
+;;     (dl-query db goal)               list of substitution dicts
+;;     (dl-relation db rel-name)        tuple list for a relation
+;;     (dl-rules db)                    rule list
+;;     (dl-fact-count db)               total ground tuples
+;;     (dl-summary db)                  {<rel>: count} for inspection
+;;
+;;   Single-call convenience:
+;;     (dl-eval source query-source)         parse, run, return substs
+;;     (dl-eval-magic source query-source)   single-goal → magic-sets
+;;
+;;   Magic-sets (lib/datalog/magic.sx):
+;;     (dl-adorn-goal goal)             "b/f" adornment string
+;;     (dl-rule-sips rule head-adn)     SIPS analysis per body lit
+;;     (dl-magic-rewrite rules rel adn args)
+;;                                       rewritten rule list + seed
+;;     (dl-magic-query db query-goal)    end-to-end magic-sets query
+;;
+;; ── Body literal kinds ─────────────────────────────────────────────
+;;
+;;   Positive             (rel arg ... arg)
+;;   Negation             {:neg (rel arg ...)}
+;;   Comparison           (< X Y), (<= X Y), (> X Y), (>= X Y),
+;;                        (= X Y), (!= X Y)
+;;   Arithmetic           (is Z (+ X Y)) and (- * /)
+;;   Aggregation          (count R V Goal),  (sum R V Goal),
+;;                        (min R V Goal),    (max R V Goal),
+;;                        (findall L V Goal)
+;;
+;; ── Variable conventions ───────────────────────────────────────────
+;;
+;;   Variables: SX symbols whose first char is uppercase A–Z or '_'.
+;;   Anonymous '_' is renamed to a fresh _anon<N> per occurrence at
+;;   rule/query load time so multiple '_' don't unify.
+;;
+;; ── Demo programs ──────────────────────────────────────────────────
+;;
+;;   See lib/datalog/demo.sx — federation, content, permissions, and
+;;   the canonical "cooking posts by people I follow (transitively)"
+;;   example.
+;;
+;; ── Status ─────────────────────────────────────────────────────────
+;;
+;;   See plans/datalog-on-sx.md — phase-by-phase progress log and
+;;   roadmap. Run `bash lib/datalog/conformance.sh` to refresh
+;;   `lib/datalog/scoreboard.{json,md}`.

lib/datalog/db.sx

@@ -0,0 +1,575 @@
+;; lib/datalog/db.sx — Datalog database (EDB + IDB + rules) + safety hook.
+;;
+;; A db is a mutable dict:
+;;   {:facts {<rel-name-string> -> (literal ...)}
+;;    :rules ({:head literal :body (literal ...)} ...)}
+;;
+;; Facts are stored as full literals `(rel arg ... arg)` so they unify
+;; directly against rule body literals. Each relation's tuple list is
+;; deduplicated on insert.
+;;
+;; Phase 3 introduced safety analysis for head variables; Phase 4 (in
+;; lib/datalog/builtins.sx) swaps in the real `dl-rule-check-safety`,
+;; which is order-aware and understands built-in predicates.
+
+(define
+  dl-make-db
+  (fn ()
+    {:facts {}
+     :facts-keys {}
+     :facts-index {}
+     :edb-keys {}
+     :rules (list)
+     :strategy :semi-naive}))
+
+;; Record (rel-key, tuple-key) as user-asserted EDB. dl-add-fact! calls
+;; this when an explicit fact is added; the saturator (which uses
+;; dl-add-derived!) does NOT, so derived tuples never appear here.
+;; dl-retract! consults :edb-keys to know which tuples must survive
+;; the wipe-and-resaturate round-trip.
+(define
+  dl-mark-edb!
+  (fn
+    (db rel-key tk)
+    (let
+      ((edb (get db :edb-keys)))
+      (do
+        (when
+          (not (has-key? edb rel-key))
+          (dict-set! edb rel-key {}))
+        (dict-set! (get edb rel-key) tk true)))))
+
+(define
+  dl-edb-fact?
+  (fn
+    (db rel-key tk)
+    (let
+      ((edb (get db :edb-keys)))
+      (and (has-key? edb rel-key)
+           (has-key? (get edb rel-key) tk)))))
+
+;; Evaluation strategy. Default :semi-naive (used by dl-saturate!).
+;; :naive selects dl-saturate-naive! (slower but easier to reason
+;; about). :magic is a marker — goal-directed magic-sets evaluation
+;; is invoked separately via `dl-magic-query`; setting :magic here
+;; is purely informational. Any other value is rejected so typos
+;; don't silently fall back to the default.
+(define
+  dl-strategy-values
+  (list :semi-naive :naive :magic))
+
+(define
+  dl-set-strategy!
+  (fn
+    (db strategy)
+    (cond
+      ((not (dl-keyword-member? strategy dl-strategy-values))
+        (error (str "dl-set-strategy!: unknown strategy " strategy
+                    " — must be one of " dl-strategy-values)))
+      (else
+        (do
+          (dict-set! db :strategy strategy)
+          db)))))
+
+(define
+  dl-keyword-member?
+  (fn
+    (k xs)
+    (cond
+      ((= (len xs) 0) false)
+      ((= k (first xs)) true)
+      (else (dl-keyword-member? k (rest xs))))))
+
+(define
+  dl-get-strategy
+  (fn
+    (db)
+    (if (has-key? db :strategy) (get db :strategy) :semi-naive)))
+
+(define
+  dl-rel-name
+  (fn
+    (lit)
+    (cond
+      ((and (dict? lit) (has-key? lit :neg)) (dl-rel-name (get lit :neg)))
+      ((and (list? lit) (> (len lit) 0) (symbol? (first lit)))
+        (symbol->string (first lit)))
+      (else nil))))
+
+(define dl-builtin-rels (list "<" "<=" ">" ">=" "=" "!=" "is"))
+
+(define
+  dl-member-string?
+  (fn
+    (s xs)
+    (cond
+      ((= (len xs) 0) false)
+      ((= (first xs) s) true)
+      (else (dl-member-string? s (rest xs))))))
+
+(define
+  dl-builtin?
+  (fn
+    (lit)
+    (and
+      (list? lit)
+      (> (len lit) 0)
+      (let
+        ((rel (dl-rel-name lit)))
+        (cond
+          ((nil? rel) false)
+          (else (dl-member-string? rel dl-builtin-rels)))))))
+
+(define
+  dl-positive-lit?
+  (fn
+    (lit)
+    (cond
+      ((and (dict? lit) (has-key? lit :neg)) false)
+      ((dl-builtin? lit) false)
+      ((and (list? lit) (> (len lit) 0)) true)
+      (else false))))
+
+(define
+  dl-tuple-equal?
+  (fn
+    (a b)
+    (cond
+      ((and (list? a) (list? b))
+        (and (= (len a) (len b)) (dl-tuple-equal-list? a b 0)))
+      ((and (number? a) (number? b)) (= a b))
+      (else (equal? a b)))))
+
+(define
+  dl-tuple-equal-list?
+  (fn
+    (a b i)
+    (cond
+      ((>= i (len a)) true)
+      ((not (dl-tuple-equal? (nth a i) (nth b i))) false)
+      (else (dl-tuple-equal-list? a b (+ i 1))))))
+
+(define
+  dl-tuple-member?
+  (fn
+    (lit lits)
+    (dl-tuple-member-aux? lit lits 0 (len lits))))
+
+(define
+  dl-tuple-member-aux?
+  (fn
+    (lit lits i n)
+    (cond
+      ((>= i n) false)
+      ((dl-tuple-equal? lit (nth lits i)) true)
+      (else (dl-tuple-member-aux? lit lits (+ i 1) n)))))
+
+(define
+  dl-ensure-rel!
+  (fn
+    (db rel-key)
+    (let
+      ((facts (get db :facts))
+        (fk (get db :facts-keys))
+        (fi (get db :facts-index)))
+      (do
+        (when
+          (not (has-key? facts rel-key))
+          (dict-set! facts rel-key (list)))
+        (when
+          (not (has-key? fk rel-key))
+          (dict-set! fk rel-key {}))
+        (when
+          (not (has-key? fi rel-key))
+          (dict-set! fi rel-key {}))
+        (get facts rel-key)))))
+
+;; First-arg index helpers. Tuples are keyed by their first-after-rel
+;; arg's `(str ...)`; when that arg is a constant, dl-match-positive
+;; uses the index instead of scanning the full relation.
+(define
+  dl-arg-key
+  (fn
+    (v)
+    (str v)))
+
+(define
+  dl-index-add!
+  (fn
+    (db rel-key lit)
+    (let
+      ((idx (get db :facts-index))
+        (n (len lit)))
+      (when
+        (and (>= n 2) (has-key? idx rel-key))
+        (let
+          ((rel-idx (get idx rel-key))
+            (k (dl-arg-key (nth lit 1))))
+          (do
+            (when
+              (not (has-key? rel-idx k))
+              (dict-set! rel-idx k (list)))
+            (append! (get rel-idx k) lit)))))))
+
+(define
+  dl-index-lookup
+  (fn
+    (db rel-key arg-val)
+    (let
+      ((idx (get db :facts-index)))
+      (cond
+        ((not (has-key? idx rel-key)) (list))
+        (else
+          (let ((rel-idx (get idx rel-key))
+                (k (dl-arg-key arg-val)))
+            (if (has-key? rel-idx k) (get rel-idx k) (list))))))))
+
+(define dl-tuple-key (fn (lit) (str lit)))
+
+(define
+  dl-rel-tuples
+  (fn
+    (db rel-key)
+    (let
+      ((facts (get db :facts)))
+      (if (has-key? facts rel-key) (get facts rel-key) (list)))))
+
+;; Reserved relation names: built-in / aggregate / negation / arrow.
+;; Rules and facts may not have these as their head's relation, since
+;; the saturator treats them specially or they are not relation names
+;; at all.
+(define
+  dl-reserved-rel-names
+  (list "not" "count" "sum" "min" "max" "findall" "is"
+        "<" "<=" ">" ">=" "=" "!=" "+" "-" "*" "/" ":-" "?-"))
+
+(define
+  dl-reserved-rel?
+  (fn
+    (name) (dl-member-string? name dl-reserved-rel-names)))
+
+;; Internal: append a derived tuple to :facts without the public
+;; validation pass and without marking :edb-keys. Used by the saturator
+;; (eval.sx) and magic-sets (magic.sx). Returns true if the tuple was
+;; new, false if already present.
+(define
+  dl-add-derived!
+  (fn
+    (db lit)
+    (let
+      ((rel-key (dl-rel-name lit)))
+      (let
+        ((tuples (dl-ensure-rel! db rel-key))
+          (key-dict (get (get db :facts-keys) rel-key))
+          (tk (dl-tuple-key lit)))
+        (cond
+          ((has-key? key-dict tk) false)
+          (else
+            (do
+              (dict-set! key-dict tk true)
+              (append! tuples lit)
+              (dl-index-add! db rel-key lit)
+              true)))))))
+
+;; A simple term — number, string, or symbol — i.e. anything legal
+;; as an EDB fact arg. Compound (list) args belong only in body
+;; literals where they encode arithmetic / aggregate sub-goals.
+(define
+  dl-simple-term?
+  (fn
+    (term)
+    (or (number? term) (string? term) (symbol? term))))
+
+(define
+  dl-args-simple?
+  (fn
+    (lit i n)
+    (cond
+      ((>= i n) true)
+      ((not (dl-simple-term? (nth lit i))) false)
+      (else (dl-args-simple? lit (+ i 1) n)))))
+
+(define
+  dl-add-fact!
+  (fn
+    (db lit)
+    (cond
+      ((not (and (list? lit) (> (len lit) 0)))
+        (error (str "dl-add-fact!: expected literal list, got " lit)))
+      ((dl-reserved-rel? (dl-rel-name lit))
+        (error (str "dl-add-fact!: '" (dl-rel-name lit)
+                    "' is a reserved name (built-in / aggregate / negation)")))
+      ((not (dl-args-simple? lit 1 (len lit)))
+        (error (str "dl-add-fact!: fact args must be numbers, strings, "
+                    "or symbols — compound args (e.g. arithmetic "
+                    "expressions) are body-only and aren't evaluated "
+                    "in fact position. got " lit)))
+      ((not (dl-ground? lit (dl-empty-subst)))
+        (error (str "dl-add-fact!: expected ground literal, got " lit)))
+      (else
+        (let
+          ((rel-key (dl-rel-name lit)) (tk (dl-tuple-key lit)))
+          (do
+            ;; Always mark EDB origin — even if the tuple key was already
+            ;; present (e.g. previously derived), so an explicit assert
+            ;; promotes it to EDB and protects it from the IDB wipe.
+            (dl-mark-edb! db rel-key tk)
+            (dl-add-derived! db lit)))))))
+
+;; The full safety check lives in builtins.sx (it has to know which
+;; predicates are built-ins). dl-add-rule! calls it via forward
+;; reference; load builtins.sx alongside db.sx in any setup that
+;; adds rules. The fallback below is used if builtins.sx isn't loaded.
+(define
+  dl-rule-check-safety
+  (fn
+    (rule)
+    (let
+      ((head-vars (dl-vars-of (get rule :head))) (body-vars (list)))
+      (do
+        (for-each
+          (fn
+            (lit)
+            (when
+              (and
+                (list? lit)
+                (> (len lit) 0)
+                (not (and (dict? lit) (has-key? lit :neg))))
+              (for-each
+                (fn
+                  (v)
+                  (when
+                    (not (dl-member-string? v body-vars))
+                    (append! body-vars v)))
+                (dl-vars-of lit))))
+          (get rule :body))
+        (let
+          ((missing (list)))
+          (do
+            (for-each
+              (fn
+                (v)
+                (when
+                  (and
+                    (not (dl-member-string? v body-vars))
+                    (not (= v "_")))
+                  (append! missing v)))
+              head-vars)
+            (cond
+              ((> (len missing) 0)
+                (str
+                  "head variable(s) "
+                  missing
+                  " do not appear in any body literal"))
+              (else nil))))))))
+
+(define
+  dl-rename-anon-term
+  (fn
+    (term next-name)
+    (cond
+      ((and (symbol? term) (= (symbol->string term) "_"))
+        (next-name))
+      ((list? term)
+        (map (fn (x) (dl-rename-anon-term x next-name)) term))
+      (else term))))
+
+(define
+  dl-rename-anon-lit
+  (fn
+    (lit next-name)
+    (cond
+      ((and (dict? lit) (has-key? lit :neg))
+        {:neg (dl-rename-anon-term (get lit :neg) next-name)})
+      ((list? lit) (dl-rename-anon-term lit next-name))
+      (else lit))))
+
+(define
+  dl-make-anon-renamer
+  (fn
+    (start)
+    (let ((counter start))
+      (fn () (do (set! counter (+ counter 1))
+                 (string->symbol (str "_anon" counter)))))))
+
+;; Scan a rule for variables already named `_anon<N>` (which would
+;; otherwise collide with the renamer's output). Returns the max N
+;; seen, or 0 if none. The renamer then starts at that max + 1, so
+;; freshly-introduced anonymous names can't shadow a user-written
+;; `_anon<N>` symbol.
+(define
+  dl-max-anon-num
+  (fn
+    (term acc)
+    (cond
+      ((symbol? term)
+        (let ((s (symbol->string term)))
+          (cond
+            ((and (>= (len s) 6) (= (slice s 0 5) "_anon"))
+              (let ((n (dl-try-parse-int (slice s 5 (len s)))))
+                (cond
+                  ((nil? n) acc)
+                  ((> n acc) n)
+                  (else acc))))
+            (else acc))))
+      ((dict? term)
+        (cond
+          ((has-key? term :neg)
+            (dl-max-anon-num (get term :neg) acc))
+          (else acc)))
+      ((list? term) (dl-max-anon-num-list term acc 0))
+      (else acc))))
+
+(define
+  dl-max-anon-num-list
+  (fn
+    (xs acc i)
+    (cond
+      ((>= i (len xs)) acc)
+      (else
+        (dl-max-anon-num-list xs (dl-max-anon-num (nth xs i) acc) (+ i 1))))))
+
+;; Cheap "is this string a decimal int" check. Returns the number or
+;; nil. Avoids relying on host parse-number, which on non-int strings
+;; might raise rather than return nil.
+(define
+  dl-try-parse-int
+  (fn
+    (s)
+    (cond
+      ((= (len s) 0) nil)
+      ((not (dl-all-digits? s 0 (len s))) nil)
+      (else (parse-number s)))))
+
+(define
+  dl-all-digits?
+  (fn
+    (s i n)
+    (cond
+      ((>= i n) true)
+      ((let ((c (slice s i (+ i 1))))
+         (not (and (>= c "0") (<= c "9"))))
+        false)
+      (else (dl-all-digits? s (+ i 1) n)))))
+
+(define
+  dl-rename-anon-rule
+  (fn
+    (rule)
+    (let
+      ((start (dl-max-anon-num (get rule :head)
+                (dl-max-anon-num-list (get rule :body) 0 0))))
+      (let ((next-name (dl-make-anon-renamer start)))
+        {:head (dl-rename-anon-term (get rule :head) next-name)
+         :body (map (fn (lit) (dl-rename-anon-lit lit next-name))
+                    (get rule :body))}))))
+
+(define
+  dl-add-rule!
+  (fn
+    (db rule)
+    (cond
+      ((not (dict? rule))
+        (error (str "dl-add-rule!: expected rule dict, got " rule)))
+      ((not (has-key? rule :head))
+        (error (str "dl-add-rule!: rule missing :head, got " rule)))
+      ((not (and (list? (get rule :head))
+                 (> (len (get rule :head)) 0)
+                 (symbol? (first (get rule :head)))))
+        (error (str "dl-add-rule!: head must be a non-empty list "
+                    "starting with a relation-name symbol, got "
+                    (get rule :head))))
+      ((not (dl-args-simple? (get rule :head) 1 (len (get rule :head))))
+        (error (str "dl-add-rule!: rule head args must be variables or "
+                    "constants — compound terms (e.g. `(*(X, 2))`) are "
+                    "not legal in head position; introduce an `is`-bound "
+                    "intermediate in the body. got " (get rule :head))))
+      ((not (list? (if (has-key? rule :body) (get rule :body) (list))))
+        (error (str "dl-add-rule!: body must be a list of literals, got "
+                    (get rule :body))))
+      ((dl-reserved-rel? (dl-rel-name (get rule :head)))
+        (error (str "dl-add-rule!: '" (dl-rel-name (get rule :head))
+                    "' is a reserved name (built-in / aggregate / negation)")))
+      (else
+        (let ((rule (dl-rename-anon-rule rule)))
+          (let
+            ((err (dl-rule-check-safety rule)))
+            (cond
+              ((not (nil? err)) (error (str "dl-add-rule!: " err)))
+              (else
+                (let
+                  ((rules (get db :rules)))
+                  (do (append! rules rule) true))))))))))
+
+(define
+  dl-add-clause!
+  (fn
+    (db clause)
+    (cond
+      ((has-key? clause :query) false)
+      ((and (has-key? clause :body) (= (len (get clause :body)) 0))
+        (dl-add-fact! db (get clause :head)))
+      (else (dl-add-rule! db clause)))))
+
+(define
+  dl-load-program!
+  (fn
+    (db source)
+    (let
+      ((clauses (dl-parse source)))
+      (do (for-each (fn (c) (dl-add-clause! db c)) clauses) db))))
+
+(define
+  dl-program
+  (fn (source) (let ((db (dl-make-db))) (dl-load-program! db source))))
+
+(define dl-rules (fn (db) (get db :rules)))
+
+(define
+  dl-fact-count
+  (fn
+    (db)
+    (let
+      ((facts (get db :facts)) (total 0))
+      (do
+        (for-each
+          (fn (k) (set! total (+ total (len (get facts k)))))
+          (keys facts))
+        total))))
+
+;; Returns {<rel-name>: tuple-count} for debugging. Includes
+;; relations with any tuples plus all rule-head relations (so empty
+;; IDB shows as 0). Skips empty EDB-only entries that are placeholders
+;; from internal `dl-ensure-rel!` calls.
+(define
+  dl-summary
+  (fn
+    (db)
+    (let
+      ((facts (get db :facts))
+        (out {})
+        (rule-heads (list)))
+      (do
+        (for-each
+          (fn
+            (rule)
+            (let ((h (dl-rel-name (get rule :head))))
+              (when
+                (and (not (nil? h)) (not (dl-member-string? h rule-heads)))
+                (append! rule-heads h))))
+          (dl-rules db))
+        (for-each
+          (fn
+            (k)
+            (let ((c (len (get facts k))))
+              (when
+                (or (> c 0) (dl-member-string? k rule-heads))
+                (dict-set! out k c))))
+          (keys facts))
+        ;; Add rule heads that have no facts (yet).
+        (for-each
+          (fn
+            (k)
+            (when (not (has-key? out k)) (dict-set! out k 0)))
+          rule-heads)
+        out))))

lib/datalog/demo.sx

@@ -0,0 +1,162 @@
+;; lib/datalog/demo.sx — example programs over rose-ash-shaped data.
+;;
+;; Phase 10 prototypes Datalog as a rose-ash query language. Wiring
+;; the EDB to actual PostgreSQL is out of scope for this loop (it
+;; would touch service code outside lib/datalog/), but the programs
+;; below show the shape of queries we want, and the test suite runs
+;; them against synthetic in-memory tuples loaded via dl-program-data.
+;;
+;; Seven thematic demos:
+;;
+;;   1. Federation — follow graph, transitive reach, mutuals, FOAF.
+;;   2. Content    — posts, tags, likes, popularity, "for you" feed.
+;;   3. Permissions — group membership and resource access.
+;;   4. Cooking-posts — canonical "posts about cooking by people I
+;;                      follow (transitively)" multi-domain query.
+;;   5. Tag co-occurrence — distinct (T1, T2) pairs with counts.
+;;   6. Shortest path  — weighted-DAG path enumeration + min agg.
+;;   7. Org chart      — transitive subordinate + headcount per mgr.
+
+;; ── Demo 1: federation follow graph ─────────────────────────────
+;; EDB: (follows ACTOR-A ACTOR-B) — A follows B.
+;; IDB:
+;;   (mutual A B)            — A follows B and B follows A
+;;   (reachable A B)         — transitive follow closure
+;;   (foaf A C)              — friend of a friend (mutual filter)
+(define
+  dl-demo-federation-rules
+  (quote
+    ((mutual A B <- (follows A B) (follows B A))
+     (reachable A B <- (follows A B))
+     (reachable A C <- (follows A B) (reachable B C))
+     (foaf A C <- (follows A B) (follows B C) (!= A C)))))
+
+;; ── Demo 2: content recommendation ──────────────────────────────
+;; EDB:
+;;   (authored ACTOR POST)
+;;   (tagged POST TAG)
+;;   (liked ACTOR POST)
+;; IDB:
+;;   (post-likes POST N)            — count of likes per post
+;;   (popular POST)                 — posts with >= 3 likes
+;;   (tagged-by-mutual ACTOR POST)  — post tagged TOPIC by someone
+;;                                    A's mutuals follow.
+(define
+  dl-demo-content-rules
+  (quote
+    ((post-likes P N <- (authored Author P) (count N L (liked L P)))
+     (popular P <- (authored Author P) (post-likes P N) (>= N 3))
+     (interesting Me P
+       <-
+       (follows Me Buddy)
+       (authored Buddy P)
+       (popular P)))))
+
+;; ── Demo 3: role-based permissions ──────────────────────────────
+;; EDB:
+;;   (member ACTOR GROUP)
+;;   (subgroup CHILD PARENT)
+;;   (allowed GROUP RESOURCE)
+;; IDB:
+;;   (in-group ACTOR GROUP)         — direct or via subgroup chain
+;;   (can-access ACTOR RESOURCE)    — actor inherits group permission
+(define
+  dl-demo-perm-rules
+  (quote
+    ((in-group A G <- (member A G))
+     (in-group A G <- (member A H) (subgroup-trans H G))
+     (subgroup-trans X Y <- (subgroup X Y))
+     (subgroup-trans X Z <- (subgroup X Y) (subgroup-trans Y Z))
+     (can-access A R <- (in-group A G) (allowed G R)))))
+
+;; ── Demo 4: cooking-posts (the canonical Phase 10 query) ────────
+;; "Posts about cooking by people I follow (transitively)."
+;; Combines federation (follows + transitive reach), authoring,
+;; tagging — the rose-ash multi-domain join.
+;;
+;; EDB:
+;;   (follows ACTOR-A ACTOR-B)
+;;   (authored ACTOR POST)
+;;   (tagged POST TAG)
+(define
+  dl-demo-cooking-rules
+  (quote
+    ((reach Me Them <- (follows Me Them))
+     (reach Me Them <- (follows Me X) (reach X Them))
+     (cooking-post-by-network Me P
+       <-
+       (reach Me Author)
+       (authored Author P)
+       (tagged P cooking)))))
+
+;; ── Demo 5: tag co-occurrence ───────────────────────────────────
+;; "Posts tagged with both T1 AND T2." Useful for narrowed-down
+;; recommendations like "vegetarian cooking" posts.
+;;
+;; EDB:
+;;   (tagged POST TAG)
+;; IDB:
+;;   (cotagged POST T1 T2)        — post has both T1 and T2 (T1 != T2)
+;;   (popular-pair T1 T2 N)       — count of posts cotagged (T1, T2)
+(define
+  dl-demo-tag-cooccur-rules
+  (quote
+    ((cotagged P T1 T2 <- (tagged P T1) (tagged P T2) (!= T1 T2))
+     ;; Distinct (T1, T2) pairs that occur somewhere.
+     (tag-pair T1 T2 <- (cotagged P T1 T2))
+     (tag-pair-count T1 T2 N
+       <-
+       (tag-pair T1 T2)
+       (count N P (cotagged P T1 T2))))))
+
+;; ── Demo 6: weighted-DAG shortest path ─────────────────────────
+;; "What's the cheapest way from X to Y?" Edge weights with `is`
+;; arithmetic to sum costs, then `min` aggregation to pick the
+;; shortest. Termination requires the graph to be a DAG (cycles
+;; would produce infinite distances without a bound; programs
+;; built on this should add a depth filter `(<, D, MAX)` if cycles
+;; are possible).
+;;
+;; EDB:
+;;   (edge FROM TO COST)
+;; IDB:
+;;   (path FROM TO COST)            — any path
+;;   (shortest FROM TO COST)        — minimum cost path
+(define
+  dl-demo-shortest-path-rules
+  (quote
+    ((path X Y W <- (edge X Y W))
+     (path X Z W
+       <-
+       (edge X Y W1)
+       (path Y Z W2)
+       (is W (+ W1 W2)))
+     (shortest X Y W <- (path X Y _) (min W C (path X Y C))))))
+
+;; ── Demo 7: org chart + transitive headcount ───────────────────
+;; Manager graph: each employee has a single manager. Compute the
+;; transitive subordinate set and headcount per manager.
+;;
+;; EDB:
+;;   (manager EMP MGR)            — EMP reports directly to MGR
+;; IDB:
+;;   (subordinate MGR EMP)        — EMP is in MGR's subtree
+;;   (headcount MGR N)            — number of subordinates under MGR
+(define
+  dl-demo-org-rules
+  (quote
+    ((subordinate Mgr Emp <- (manager Emp Mgr))
+     (subordinate Mgr Emp
+       <- (manager Mid Mgr) (subordinate Mid Emp))
+     (headcount Mgr N
+       <- (subordinate Mgr Anyone) (count N E (subordinate Mgr E))))))
+
+;; ── Loader stub ──────────────────────────────────────────────────
+;; Wiring to PostgreSQL would replace these helpers with calls into
+;; rose-ash's internal HTTP RPC (fetch_data → /internal/data/...).
+;; The shape returned by dl-load-from-edb! is the same in either case.
+(define
+  dl-demo-make
+  (fn
+    (facts rules)
+    (dl-program-data facts rules)))

lib/datalog/eval.sx

@@ -0,0 +1,512 @@
+;; lib/datalog/eval.sx — fixpoint evaluator (naive + semi-naive).
+;;
+;; Two saturators are exposed:
+;;   `dl-saturate-naive!` — re-joins each rule against the full DB every
+;;     iteration. Reference implementation; useful for differential tests.
+;;   `dl-saturate!`       — semi-naive default. Tracks per-relation delta
+;;     sets and substitutes one positive body literal per rule with the
+;;     delta of its relation, joining the rest against the previous-
+;;     iteration DB. Same fixpoint, dramatically less work on recursive
+;;     rules.
+;;
+;; Body literal kinds:
+;;   positive (rel arg ... arg)  → match against EDB+IDB tuples
+;;   built-in (< X Y), (is X e)  → constraint via dl-eval-builtin
+;;   negation {:neg lit}         → Phase 7
+
+(define
+  dl-match-positive
+  (fn
+    (lit db subst)
+    (let
+      ((rel (dl-rel-name lit)) (results (list)))
+      (cond
+        ((nil? rel) (error (str "dl-match-positive: bad literal " lit)))
+        (else
+          (let
+            ;; If the first argument walks to a non-variable (constant
+            ;; or already-bound var), use the first-arg index for
+            ;; this relation. Otherwise scan the full tuple list.
+            ((tuples
+               (cond
+                 ((>= (len lit) 2)
+                   (let ((walked (dl-walk (nth lit 1) subst)))
+                     (cond
+                       ((dl-var? walked) (dl-rel-tuples db rel))
+                       (else (dl-index-lookup db rel walked)))))
+                 (else (dl-rel-tuples db rel)))))
+            (do
+              (for-each
+                (fn
+                  (tuple)
+                  (let
+                    ((s (dl-unify lit tuple subst)))
+                    (when (not (nil? s)) (append! results s))))
+                tuples)
+              results)))))))
+
+;; Match a positive literal against the delta set for its relation only.
+(define
+  dl-match-positive-delta
+  (fn
+    (lit delta subst)
+    (let
+      ((rel (dl-rel-name lit)) (results (list)))
+      (let
+        ((tuples (if (has-key? delta rel) (get delta rel) (list))))
+        (do
+          (for-each
+            (fn
+              (tuple)
+              (let
+                ((s (dl-unify lit tuple subst)))
+                (when (not (nil? s)) (append! results s))))
+            tuples)
+          results)))))
+
+;; Naive matcher (for dl-saturate-naive! and dl-query post-saturation).
+(define
+  dl-match-negation
+  (fn
+    (inner db subst)
+    (let
+      ((walked (dl-apply-subst inner subst))
+        (matches (dl-match-positive inner db subst)))
+      (cond
+        ((= (len matches) 0) (list subst))
+        (else (list))))))
+
+(define
+  dl-match-lit
+  (fn
+    (lit db subst)
+    (cond
+      ((and (dict? lit) (has-key? lit :neg))
+        (dl-match-negation (get lit :neg) db subst))
+      ((dl-aggregate? lit) (dl-eval-aggregate lit db subst))
+      ((dl-builtin? lit)
+        (let
+          ((s (dl-eval-builtin lit subst)))
+          (if (nil? s) (list) (list s))))
+      ((and (list? lit) (> (len lit) 0))
+        (dl-match-positive lit db subst))
+      (else (error (str "datalog: unknown body-literal shape: " lit))))))
+
+(define
+  dl-find-bindings
+  (fn (lits db subst) (dl-fb-aux lits db subst 0 (len lits))))
+
+(define
+  dl-fb-aux
+  (fn
+    (lits db subst i n)
+    (cond
+      ((nil? subst) (list))
+      ((>= i n) (list subst))
+      (else
+        (let
+          ((options (dl-match-lit (nth lits i) db subst))
+            (results (list)))
+          (do
+            (for-each
+              (fn
+                (s)
+                (for-each
+                  (fn (s2) (append! results s2))
+                  (dl-fb-aux lits db s (+ i 1) n)))
+              options)
+            results))))))
+
+;; Naive: apply each rule against full DB until no new tuples.
+(define
+  dl-apply-rule!
+  (fn
+    (db rule)
+    (let
+      ((head (get rule :head)) (body (get rule :body)) (new? false))
+      (do
+        (for-each
+          (fn
+            (s)
+            (let
+              ((derived (dl-apply-subst head s)))
+              (when (dl-add-derived! db derived) (set! new? true))))
+          (dl-find-bindings body db (dl-empty-subst)))
+        new?))))
+
+;; Returns true iff one more saturation step would derive no new
+;; tuples (i.e. the db is at fixpoint). Useful in tests that want
+;; to assert "no work left" after a saturation call. Works under
+;; either saturator since both compute the same fixpoint.
+(define
+  dl-saturated?
+  (fn
+    (db)
+    (let ((any-new false))
+      (do
+        (for-each
+          (fn
+            (rule)
+            (when (not any-new)
+              (for-each
+                (fn
+                  (s)
+                  (let ((derived (dl-apply-subst (get rule :head) s)))
+                    (when
+                      (and (not any-new)
+                           (not (dl-tuple-member?
+                                  derived
+                                  (dl-rel-tuples
+                                    db (dl-rel-name derived)))))
+                      (set! any-new true))))
+                (dl-find-bindings (get rule :body) db (dl-empty-subst)))))
+          (dl-rules db))
+        (not any-new)))))
+
+(define
+  dl-saturate-naive!
+  (fn
+    (db)
+    (let
+      ((changed true))
+      (do
+        (define
+          dl-snloop
+          (fn
+            ()
+            (when
+              changed
+              (do
+                (set! changed false)
+                (for-each
+                  (fn (r) (when (dl-apply-rule! db r) (set! changed true)))
+                  (dl-rules db))
+                (dl-snloop)))))
+        (dl-snloop)
+        db))))
+
+;; ── Semi-naive ───────────────────────────────────────────────────
+
+;; Take a snapshot dict {rel -> tuples} of every relation currently in
+;; the DB. Used as initial delta for the first iteration.
+(define
+  dl-snapshot-facts
+  (fn
+    (db)
+    (let
+      ((facts (get db :facts)) (out {}))
+      (do
+        (for-each
+          (fn (k) (dict-set! out k (dl-copy-list (get facts k))))
+          (keys facts))
+        out))))
+
+(define
+  dl-copy-list
+  (fn
+    (xs)
+    (let
+      ((out (list)))
+      (do (for-each (fn (x) (append! out x)) xs) out))))
+
+;; Does any relation in `delta` have ≥1 tuple?
+(define
+  dl-delta-empty?
+  (fn
+    (delta)
+    (let
+      ((ks (keys delta)) (any-non-empty false))
+      (do
+        (for-each
+          (fn
+            (k)
+            (when
+              (> (len (get delta k)) 0)
+              (set! any-non-empty true)))
+          ks)
+        (not any-non-empty)))))
+
+;; Find substitutions such that `lits` are all satisfied AND `delta-idx`
+;; is matched against the per-relation delta only. The other positive
+;; literals match against the snapshot DB (db.facts read at iteration
+;; start). Built-ins and negations behave as in `dl-match-lit`.
+(define
+  dl-find-bindings-semi
+  (fn
+    (lits db delta delta-idx subst)
+    (dl-fbs-aux lits db delta delta-idx 0 subst)))
+
+(define
+  dl-fbs-aux
+  (fn
+    (lits db delta delta-idx i subst)
+    (cond
+      ((nil? subst) (list))
+      ((>= i (len lits)) (list subst))
+      (else
+        (let
+          ((lit (nth lits i))
+            (options
+              (cond
+                ((and (dict? lit) (has-key? lit :neg))
+                  (dl-match-negation (get lit :neg) db subst))
+                ((dl-aggregate? lit) (dl-eval-aggregate lit db subst))
+                ((dl-builtin? lit)
+                  (let
+                    ((s (dl-eval-builtin lit subst)))
+                    (if (nil? s) (list) (list s))))
+                ((and (list? lit) (> (len lit) 0))
+                  (if
+                    (= i delta-idx)
+                    (dl-match-positive-delta lit delta subst)
+                    (dl-match-positive lit db subst)))
+                (else (error (str "datalog: unknown body-lit: " lit)))))
+            (results (list)))
+          (do
+            (for-each
+              (fn
+                (s)
+                (for-each
+                  (fn (s2) (append! results s2))
+                  (dl-fbs-aux lits db delta delta-idx (+ i 1) s)))
+              options)
+            results))))))
+
+;; Collect candidate head tuples from a rule using delta. Walks every
+;; positive body position and unions the resulting heads. For rules
+;; with no positive body literal, falls back to a naive single-pass
+;; (so static facts like `(p X) :- (= X 5).` derive on iteration 1).
+(define
+  dl-collect-rule-candidates
+  (fn
+    (rule db delta)
+    (let
+      ((head (get rule :head))
+        (body (get rule :body))
+        (out (list))
+        (saw-pos false))
+      (do
+        (define
+          dl-cri
+          (fn
+            (i)
+            (when
+              (< i (len body))
+              (do
+                (let
+                  ((lit (nth body i)))
+                  (when
+                    (dl-positive-lit? lit)
+                    (do
+                      (set! saw-pos true)
+                      (for-each
+                        (fn (s) (append! out (dl-apply-subst head s)))
+                        (dl-find-bindings-semi
+                          body
+                          db
+                          delta
+                          i
+                          (dl-empty-subst))))))
+                (dl-cri (+ i 1))))))
+        (dl-cri 0)
+        (when
+          (not saw-pos)
+          (for-each
+            (fn (s) (append! out (dl-apply-subst head s)))
+            (dl-find-bindings body db (dl-empty-subst))))
+        out))))
+
+;; Add a list of candidate tuples to db; collect newly-added ones into
+;; the new-delta dict (keyed by relation name).
+(define
+  dl-commit-candidates!
+  (fn
+    (db candidates new-delta)
+    (for-each
+      (fn
+        (lit)
+        (when
+          (dl-add-derived! db lit)
+          (let
+            ((rel (dl-rel-name lit)))
+            (do
+              (when
+                (not (has-key? new-delta rel))
+                (dict-set! new-delta rel (list)))
+              (append! (get new-delta rel) lit)))))
+      candidates)))
+
+(define
+  dl-saturate-rules!
+  (fn
+    (db rules)
+    (let
+      ((delta (dl-snapshot-facts db)))
+      (do
+        (define
+          dl-sr-step
+          (fn
+            ()
+            (let
+              ((pending (list)) (new-delta {}))
+              (do
+                (for-each
+                  (fn
+                    (rule)
+                    (for-each
+                      (fn (cand) (append! pending cand))
+                      (dl-collect-rule-candidates rule db delta)))
+                  rules)
+                (dl-commit-candidates! db pending new-delta)
+                (cond
+                  ((dl-delta-empty? new-delta) nil)
+                  (else (do (set! delta new-delta) (dl-sr-step))))))))
+        (dl-sr-step)
+        db))))
+
+;; Stratified driver: rejects non-stratifiable programs at saturation
+;; time, then iterates strata in increasing order, running semi-naive on
+;; the rules whose head sits in that stratum.
+(define
+  dl-saturate!
+  (fn
+    (db)
+    (let
+      ((err (dl-check-stratifiable db)))
+      (cond
+        ((not (nil? err)) (error (str "dl-saturate!: " err)))
+        (else
+          (let
+            ((strata (dl-compute-strata db)))
+            (let
+              ((grouped (dl-group-rules-by-stratum db strata)))
+              (let
+                ((groups (get grouped :groups))
+                  (max-s (get grouped :max)))
+                (do
+                  (define
+                    dl-strat-loop
+                    (fn
+                      (s)
+                      (when
+                        (<= s max-s)
+                        (let
+                          ((sk (str s)))
+                          (do
+                            (when
+                              (has-key? groups sk)
+                              (dl-saturate-rules! db (get groups sk)))
+                            (dl-strat-loop (+ s 1)))))))
+                  (dl-strat-loop 0)
+                  db)))))))))
+
+;; ── Querying ─────────────────────────────────────────────────────
+
+;; Coerce a query argument to a list of body literals. A single literal
+;; like `(p X)` (positive — head is a symbol) or `{:neg ...}` becomes
+;; `((p X))`. A list of literals like `((p X) (q X))` is returned as-is.
+(define
+  dl-query-coerce
+  (fn
+    (goal)
+    (cond
+      ((and (dict? goal) (has-key? goal :neg)) (list goal))
+      ((and (list? goal) (> (len goal) 0) (symbol? (first goal)))
+        (list goal))
+      ((list? goal) goal)
+      (else (error (str "dl-query: unrecognised goal shape: " goal))))))
+
+(define
+  dl-query
+  (fn
+    (db goal)
+    (do
+      (dl-saturate! db)
+      ;; Rename anonymous '_' vars in each goal literal so multiple
+      ;; occurrences do not unify together. Keep the user-facing var
+      ;; list (taken before renaming) so projected results retain user
+      ;; names.
+      (let
+        ((goals (dl-query-coerce goal))
+          ;; Start the renamer past any `_anon<N>` symbols the user
+          ;; may have written in the query — avoids collision.
+          (renamer
+            (dl-make-anon-renamer (dl-max-anon-num-list goal 0 0))))
+        (let
+          ((user-vars (dl-query-user-vars goals))
+            (renamed (map (fn (g) (dl-rename-anon-lit g renamer)) goals)))
+          (let
+            ((substs (dl-find-bindings renamed db (dl-empty-subst)))
+              (results (list)))
+            (do
+              (for-each
+                (fn
+                  (s)
+                  (let
+                    ((proj (dl-project-subst s user-vars)))
+                    (when
+                      (not (dl-tuple-member? proj results))
+                      (append! results proj))))
+                substs)
+              results)))))))
+
+(define
+  dl-query-user-vars
+  (fn
+    (goals)
+    (let ((seen (list)))
+      (do
+        (for-each
+          (fn
+            (g)
+            (cond
+              ((and (dict? g) (has-key? g :neg))
+                (for-each
+                  (fn
+                    (v)
+                    (when
+                      (and (not (= v "_")) (not (dl-member-string? v seen)))
+                      (append! seen v)))
+                  (dl-vars-of (get g :neg))))
+              ((dl-aggregate? g)
+                ;; Only the result var (first arg of the aggregate
+                ;; literal) is user-facing. The aggregated var and
+                ;; any vars in the inner goal are internal.
+                (let ((r (nth g 1)))
+                  (when
+                    (dl-var? r)
+                    (let ((rn (symbol->string r)))
+                      (when
+                        (and (not (= rn "_"))
+                             (not (dl-member-string? rn seen)))
+                        (append! seen rn))))))
+              (else
+                (for-each
+                  (fn
+                    (v)
+                    (when
+                      (and (not (= v "_")) (not (dl-member-string? v seen)))
+                      (append! seen v)))
+                  (dl-vars-of g)))))
+          goals)
+        seen))))
+
+(define
+  dl-project-subst
+  (fn
+    (subst names)
+    (let
+      ((out {}))
+      (do
+        (for-each
+          (fn
+            (n)
+            (let
+              ((sym (string->symbol n)))
+              (let
+                ((v (dl-walk sym subst)))
+                (dict-set! out n (dl-apply-subst v subst)))))
+          names)
+        out))))
+
+(define dl-relation (fn (db name) (dl-rel-tuples db name)))

lib/datalog/magic.sx

@@ -0,0 +1,464 @@
+;; lib/datalog/magic.sx — adornment analysis + sideways info passing.
+;;
+;; First step of the magic-sets transformation (Phase 6). Right now
+;; the saturator does not consume these — they are introspection
+;; helpers that future magic-set rewriting will build on top of.
+;;
+;; Definitions:
+;;   - An *adornment* of an n-ary literal is an n-character string
+;;     of "b" (bound — value already known at the call site) and
+;;     "f" (free — to be derived).
+;;   - SIPS (Sideways Information Passing Strategy) walks the body
+;;     of an adorned rule left-to-right tracking which variables
+;;     have been bound so far, computing each body literal's
+;;     adornment in turn.
+;;
+;; Usage:
+;;
+;;   (dl-adorn-goal '(ancestor tom X))
+;;     => "bf"
+;;
+;;   (dl-rule-sips
+;;     {:head (ancestor X Z)
+;;      :body ((parent X Y) (ancestor Y Z))}
+;;     "bf")
+;;   => ({:lit (parent X Y) :adornment "bf"}
+;;       {:lit (ancestor Y Z) :adornment "bf"})
+
+;; Per-arg adornment under the current bound-var name set.
+(define
+  dl-adorn-arg
+  (fn
+    (arg bound)
+    (cond
+      ((dl-var? arg)
+        (if (dl-member-string? (symbol->string arg) bound) "b" "f"))
+      (else "b"))))
+
+;; Adornment for the args of a literal (after the relation name).
+(define
+  dl-adorn-args
+  (fn
+    (args bound)
+    (cond
+      ((= (len args) 0) "")
+      (else
+        (str
+          (dl-adorn-arg (first args) bound)
+          (dl-adorn-args (rest args) bound))))))
+
+;; Adornment of a top-level goal under the empty bound-var set.
+(define
+  dl-adorn-goal
+  (fn (goal) (dl-adorn-args (rest goal) (list))))
+
+;; Adornment of a literal under an explicit bound set.
+(define
+  dl-adorn-lit
+  (fn (lit bound) (dl-adorn-args (rest lit) bound)))
+
+;; The set of variable names made bound by walking a positive
+;; literal whose adornment is known. Free positions add their
+;; vars to the bound set.
+(define
+  dl-vars-bound-by-lit
+  (fn
+    (lit bound)
+    (let ((args (rest lit)) (out (list)))
+      (do
+        (for-each
+          (fn (a)
+            (when
+              (and (dl-var? a)
+                   (not (dl-member-string? (symbol->string a) bound))
+                   (not (dl-member-string? (symbol->string a) out)))
+              (append! out (symbol->string a))))
+          args)
+        out))))
+
+;; Walk the rule body left-to-right tracking bound vars seeded by the
+;; head adornment. Returns a list of {:lit :adornment} entries.
+;;
+;; Negation, comparison, and built-ins are passed through with their
+;; adornment computed from the current bound set; they don't add new
+;; bindings (except `is`, which binds its left arg if a var). Aggregates
+;; are treated like is — the result var becomes bound.
+(define
+  dl-init-head-bound
+  (fn
+    (head adornment)
+    (let ((args (rest head)) (out (list)))
+      (do
+        (define
+          dl-ihb-loop
+          (fn
+            (i)
+            (when
+              (< i (len args))
+              (do
+                (let
+                  ((c (slice adornment i (+ i 1)))
+                    (a (nth args i)))
+                  (when
+                    (and (= c "b") (dl-var? a))
+                    (let ((n (symbol->string a)))
+                      (when
+                        (not (dl-member-string? n out))
+                        (append! out n)))))
+                (dl-ihb-loop (+ i 1))))))
+        (dl-ihb-loop 0)
+        out))))
+
+(define
+  dl-rule-sips
+  (fn
+    (rule head-adornment)
+    (let
+      ((bound (dl-init-head-bound (get rule :head) head-adornment))
+        (out (list)))
+      (do
+        (for-each
+          (fn
+            (lit)
+            (cond
+              ((and (dict? lit) (has-key? lit :neg))
+                (let ((target (get lit :neg)))
+                  (append!
+                    out
+                    {:lit lit :adornment (dl-adorn-lit target bound)})))
+              ((dl-builtin? lit)
+                (let ((adn (dl-adorn-lit lit bound)))
+                  (do
+                    (append! out {:lit lit :adornment adn})
+                    ;; `is` binds its left arg (if var) once RHS is ground.
+                    (when
+                      (and (= (dl-rel-name lit) "is") (dl-var? (nth lit 1)))
+                      (let ((n (symbol->string (nth lit 1))))
+                        (when
+                          (not (dl-member-string? n bound))
+                          (append! bound n)))))))
+              ((and (list? lit) (dl-aggregate? lit))
+                (let ((adn (dl-adorn-lit lit bound)))
+                  (do
+                    (append! out {:lit lit :adornment adn})
+                    ;; Result var (first arg) becomes bound.
+                    (when (dl-var? (nth lit 1))
+                      (let ((n (symbol->string (nth lit 1))))
+                        (when
+                          (not (dl-member-string? n bound))
+                          (append! bound n)))))))
+              ((and (list? lit) (> (len lit) 0))
+                (let ((adn (dl-adorn-lit lit bound)))
+                  (do
+                    (append! out {:lit lit :adornment adn})
+                    (for-each
+                      (fn (n)
+                        (when (not (dl-member-string? n bound))
+                          (append! bound n)))
+                      (dl-vars-bound-by-lit lit bound)))))))
+          (get rule :body))
+        out))))
+
+;; ── Magic predicate naming + bound-args extraction ─────────────
+;; These are building blocks for the magic-sets *transformation*
+;; itself. The transformation (which generates rewritten rules
+;; with magic_<rel>^<adornment> filters) is future work — for now
+;; these helpers can be used to inspect what such a transformation
+;; would produce.
+
+;; "magic_p^bf" given relation "p" and adornment "bf".
+(define
+  dl-magic-rel-name
+  (fn (rel adornment) (str "magic_" rel "^" adornment)))
+
+;; A magic predicate literal:
+;;   (magic_<rel>^<adornment> arg1 arg2 ...)
+(define
+  dl-magic-lit
+  (fn
+    (rel adornment bound-args)
+    (cons (string->symbol (dl-magic-rel-name rel adornment)) bound-args)))
+
+;; Extract bound args (those at "b" positions in `adornment`) from a
+;; literal `(rel arg1 arg2 ... argN)`. Returns the list of arg values.
+(define
+  dl-bound-args
+  (fn
+    (lit adornment)
+    (let ((args (rest lit)) (out (list)))
+      (do
+        (define
+          dl-ba-loop
+          (fn
+            (i)
+            (when
+              (< i (len args))
+              (do
+                (when
+                  (= (slice adornment i (+ i 1)) "b")
+                  (append! out (nth args i)))
+                (dl-ba-loop (+ i 1))))))
+        (dl-ba-loop 0)
+        out))))
+
+;; ── Magic-sets rewriter ─────────────────────────────────────────
+;;
+;; Given the original rule list and a query (rel, adornment) pair,
+;; generates the magic-rewritten program: a list of rules that
+;; (a) gate each original rule with a `magic_<rel>^<adn>` filter and
+;; (b) propagate the magic relation through SIPS so that only
+;; query-relevant tuples are derived. Seed facts are returned
+;; separately and must be added to the db at evaluation time.
+;;
+;; Output: {:rules <rewritten-rules> :seed <magic-seed-literal>}
+;;
+;; The rewriter only rewrites IDB rules; EDB facts pass through.
+;; Built-in predicates and negation in body literals are kept in
+;; place but do not generate propagation rules of their own.
+
+(define
+  dl-magic-pair-key
+  (fn (rel adornment) (str rel "^" adornment)))
+
+(define
+  dl-magic-rewrite
+  (fn
+    (rules query-rel query-adornment query-args)
+    (let
+      ((seen (list))
+        (queue (list))
+        (out (list)))
+      (do
+        (define
+          dl-mq-mark!
+          (fn
+            (rel adornment)
+            (let ((k (dl-magic-pair-key rel adornment)))
+              (when
+                (not (dl-member-string? k seen))
+                (do
+                  (append! seen k)
+                  (append! queue {:rel rel :adn adornment}))))))
+
+        (define
+          dl-mq-rewrite-rule!
+          (fn
+            (rule adn)
+            (let
+              ((head (get rule :head))
+                (body (get rule :body))
+                (sips (dl-rule-sips rule adn)))
+              (let
+                ((magic-filter
+                   (dl-magic-lit
+                     (dl-rel-name head)
+                     adn
+                     (dl-bound-args head adn))))
+                (do
+                  ;; Adorned rule: head :- magic-filter, body...
+                  (let ((new-body (list)))
+                    (do
+                      (append! new-body magic-filter)
+                      (for-each
+                        (fn (lit) (append! new-body lit))
+                        body)
+                      (append! out {:head head :body new-body})))
+                  ;; Propagation rules for each positive non-builtin
+                  ;; body literal at position i.
+                  (define
+                    dl-mq-prop-loop
+                    (fn
+                      (i)
+                      (when
+                        (< i (len body))
+                        (do
+                          (let
+                            ((lit (nth body i))
+                              (sip-entry (nth sips i)))
+                            (when
+                              (and (list? lit)
+                                   (> (len lit) 0)
+                                   (not (and (dict? lit) (has-key? lit :neg)))
+                                   (not (dl-builtin? lit))
+                                   (not (dl-aggregate? lit)))
+                              (let
+                                ((lit-adn (get sip-entry :adornment))
+                                  (lit-rel (dl-rel-name lit)))
+                                (let
+                                  ((prop-head
+                                     (dl-magic-lit
+                                       lit-rel
+                                       lit-adn
+                                       (dl-bound-args lit lit-adn))))
+                                  (let ((prop-body (list)))
+                                    (do
+                                      (append! prop-body magic-filter)
+                                      (define
+                                        dl-mq-prefix-loop
+                                        (fn
+                                          (j)
+                                          (when
+                                            (< j i)
+                                            (do
+                                              (append!
+                                                prop-body
+                                                (nth body j))
+                                              (dl-mq-prefix-loop (+ j 1))))))
+                                      (dl-mq-prefix-loop 0)
+                                      (append!
+                                        out
+                                        {:head prop-head :body prop-body})
+                                      (dl-mq-mark! lit-rel lit-adn)))))))
+                          (dl-mq-prop-loop (+ i 1))))))
+                  (dl-mq-prop-loop 0))))))
+
+        (dl-mq-mark! query-rel query-adornment)
+
+        (let ((idx 0))
+          (define
+            dl-mq-process
+            (fn
+              ()
+              (when
+                (< idx (len queue))
+                (let ((item (nth queue idx)))
+                  (do
+                    (set! idx (+ idx 1))
+                    (let
+                      ((rel (get item :rel)) (adn (get item :adn)))
+                      (for-each
+                        (fn
+                          (rule)
+                          (when
+                            (= (dl-rel-name (get rule :head)) rel)
+                            (dl-mq-rewrite-rule! rule adn)))
+                        rules))
+                    (dl-mq-process))))))
+          (dl-mq-process))
+
+        {:rules out
+         :seed
+         (dl-magic-lit
+           query-rel
+           query-adornment
+           query-args)}))))
+
+;; ── Top-level magic-sets driver ─────────────────────────────────
+;;
+;; (dl-magic-query db query-goal) — run `query-goal` under magic-sets
+;; evaluation. Builds a fresh internal db with:
+;;   - the caller's EDB facts (relations not headed by any rule),
+;;   - the magic seed fact, and
+;;   - the rewritten rules.
+;; Saturates and queries, returning the substitution list. The
+;; caller's db is untouched.
+;;
+;; Useful primarily as a perf alternative for queries that only
+;; need a small slice of a recursive relation. Equivalent to
+;; dl-query for any single fully-stratifiable program.
+
+(define
+  dl-magic-rule-heads
+  (fn
+    (rules)
+    (let ((seen (list)))
+      (do
+        (for-each
+          (fn
+            (r)
+            (let ((h (dl-rel-name (get r :head))))
+              (when
+                (and (not (nil? h)) (not (dl-member-string? h seen)))
+                (append! seen h))))
+          rules)
+        seen))))
+
+;; True iff any rule's body contains a literal kind that the magic
+;; rewriter doesn't propagate magic to — i.e. an aggregate or a
+;; negation. Used by dl-magic-query to decide whether to pre-saturate
+;; the source db (for correctness on stratified programs) or skip
+;; that step (preserving full magic-sets efficiency for pure
+;; positive programs).
+(define
+  dl-rule-has-nonprop-lit?
+  (fn
+    (body i n)
+    (cond
+      ((>= i n) false)
+      ((let ((lit (nth body i)))
+         (or (and (dict? lit) (has-key? lit :neg))
+             (dl-aggregate? lit)))
+        true)
+      (else (dl-rule-has-nonprop-lit? body (+ i 1) n)))))
+
+(define
+  dl-rules-need-presaturation?
+  (fn
+    (rules)
+    (cond
+      ((= (len rules) 0) false)
+      ((let ((body (get (first rules) :body)))
+         (dl-rule-has-nonprop-lit? body 0 (len body)))
+        true)
+      (else (dl-rules-need-presaturation? (rest rules))))))
+
+(define
+  dl-magic-query
+  (fn
+    (db query-goal)
+    ;; Magic-sets only applies to positive non-builtin / non-aggregate
+    ;; literals against rule-defined relations. For other goal shapes
+    ;; (built-ins, aggregates, EDB-only relations) the seed is either
+    ;; non-ground or unused; fall back to dl-query.
+    (cond
+      ((not (and (list? query-goal)
+                 (> (len query-goal) 0)
+                 (symbol? (first query-goal))))
+        (error (str "dl-magic-query: goal must be a positive literal "
+                    "(non-empty list with a symbol head), got " query-goal)))
+      ((or (dl-builtin? query-goal)
+           (dl-aggregate? query-goal)
+           (and (dict? query-goal) (has-key? query-goal :neg)))
+        (dl-query db query-goal))
+      (else
+        (do
+          ;; If the rule set has aggregates or negation, pre-saturate
+          ;; the source db before copying facts. The magic rewriter
+          ;; passes aggregate body lits and negated lits through
+          ;; unchanged (no magic propagation generated for them) — so
+          ;; if their inner-goal relation is IDB, it would be empty in
+          ;; the magic db. Pre-saturating ensures equivalence with
+          ;; `dl-query` for every stratified program. Pure positive
+          ;; programs skip this and keep the full magic-sets perf win
+          ;; from goal-directed re-derivation.
+          (when
+            (dl-rules-need-presaturation? (dl-rules db))
+            (dl-saturate! db))
+          (let
+            ((query-rel (dl-rel-name query-goal))
+              (query-adn (dl-adorn-goal query-goal)))
+            (let
+              ((query-args (dl-bound-args query-goal query-adn))
+                (rules (dl-rules db)))
+              (let
+                ((rewritten (dl-magic-rewrite rules query-rel query-adn query-args))
+                  (mdb (dl-make-db))
+                  (rule-heads (dl-magic-rule-heads rules)))
+            (do
+              ;; Copy ALL existing facts. EDB-only relations bring their
+              ;; tuples; mixed EDB+IDB relations bring both their EDB
+              ;; portion and any pre-saturated IDB tuples (which the
+              ;; rewritten rules would re-derive anyway). Skipping facts
+              ;; for rule-headed relations would leave the magic run
+              ;; without the EDB portion of mixed relations.
+              (for-each
+                (fn
+                  (rel)
+                  (for-each
+                    (fn (t) (dl-add-fact! mdb t))
+                    (dl-rel-tuples db rel)))
+                (keys (get db :facts)))
+              ;; Seed + rewritten rules.
+              (dl-add-fact! mdb (get rewritten :seed))
+              (for-each (fn (r) (dl-add-rule! mdb r)) (get rewritten :rules))
+              (dl-query mdb query-goal))))))))))

lib/datalog/parser.sx

@@ -0,0 +1,252 @@
+;; lib/datalog/parser.sx — Datalog tokens → AST
+;;
+;; Output shapes:
+;;   Literal (positive)  := (relname arg ... arg)        — SX list
+;;   Literal (negative)  := {:neg (relname arg ... arg)} — dict
+;;   Argument            := var-symbol | atom-symbol | number | string
+;;                        | (op-name arg ... arg)        — arithmetic compound
+;;   Fact                := {:head literal :body ()}
+;;   Rule                := {:head literal :body (lit ... lit)}
+;;   Query               := {:query (lit ... lit)}
+;;   Program             := list of facts / rules / queries
+;;
+;; Variables and constants are both SX symbols; the evaluator dispatches
+;; on first-char case ('A'..'Z' or '_' = variable, otherwise constant).
+;;
+;; The parser permits nested compounds in arg position to support
+;; arithmetic (e.g. (is Z (+ X Y))). Safety analysis at rule-load time
+;; rejects compounds whose head is not an arithmetic operator.
+
+(define
+  dl-pp-peek
+  (fn
+    (st)
+    (let
+      ((i (get st :idx)) (tokens (get st :tokens)))
+      (if (< i (len tokens)) (nth tokens i) {:type "eof" :value nil :pos 0}))))
+
+(define
+  dl-pp-peek2
+  (fn
+    (st)
+    (let
+      ((i (+ (get st :idx) 1)) (tokens (get st :tokens)))
+      (if (< i (len tokens)) (nth tokens i) {:type "eof" :value nil :pos 0}))))
+
+(define
+  dl-pp-advance!
+  (fn (st) (dict-set! st :idx (+ (get st :idx) 1))))
+
+(define
+  dl-pp-at?
+  (fn
+    (st type value)
+    (let
+      ((t (dl-pp-peek st)))
+      (and
+        (= (get t :type) type)
+        (or (= value nil) (= (get t :value) value))))))
+
+(define
+  dl-pp-error
+  (fn
+    (st msg)
+    (let
+      ((t (dl-pp-peek st)))
+      (error
+        (str
+          "Parse error at pos "
+          (get t :pos)
+          ": "
+          msg
+          " (got "
+          (get t :type)
+          " '"
+          (if (= (get t :value) nil) "" (get t :value))
+          "')")))))
+
+(define
+  dl-pp-expect!
+  (fn
+    (st type value)
+    (let
+      ((t (dl-pp-peek st)))
+      (if
+        (dl-pp-at? st type value)
+        (do (dl-pp-advance! st) t)
+        (dl-pp-error
+          st
+          (str "expected " type (if (= value nil) "" (str " '" value "'"))))))))
+
+;; Argument: variable, atom, number, string, or compound (relname/op + parens).
+(define
+  dl-pp-parse-arg
+  (fn
+    (st)
+    (let
+      ((t (dl-pp-peek st)))
+      (let
+        ((ty (get t :type)) (vv (get t :value)))
+        (cond
+          ((= ty "number") (do (dl-pp-advance! st) vv))
+          ((= ty "string") (do (dl-pp-advance! st) vv))
+          ((= ty "var") (do (dl-pp-advance! st) (string->symbol vv)))
+          ;; Negative numeric literal: `-` op directly followed by a
+          ;; number (no `(`) is parsed as a single negative number.
+          ;; This keeps `(-X Y)` (compound) and `-N` (literal) distinct.
+          ((and (= ty "op") (= vv "-")
+                (= (get (dl-pp-peek2 st) :type) "number"))
+            (do
+              (dl-pp-advance! st)
+              (let
+                ((n (get (dl-pp-peek st) :value)))
+                (do (dl-pp-advance! st) (- 0 n)))))
+          ((or (= ty "atom") (= ty "op"))
+            (do
+              (dl-pp-advance! st)
+              (if
+                (dl-pp-at? st "punct" "(")
+                (do
+                  (dl-pp-advance! st)
+                  (let
+                    ((args (dl-pp-parse-arg-list st)))
+                    (do
+                      (dl-pp-expect! st "punct" ")")
+                      (cons (string->symbol vv) args))))
+                (string->symbol vv))))
+          (else (dl-pp-error st "expected term")))))))
+
+;; Comma-separated args inside parens.
+(define
+  dl-pp-parse-arg-list
+  (fn
+    (st)
+    (let
+      ((args (list)))
+      (do
+        (append! args (dl-pp-parse-arg st))
+        (define
+          dl-pp-arg-loop
+          (fn
+            ()
+            (when
+              (dl-pp-at? st "punct" ",")
+              (do
+                (dl-pp-advance! st)
+                (append! args (dl-pp-parse-arg st))
+                (dl-pp-arg-loop)))))
+        (dl-pp-arg-loop)
+        args))))
+
+;; A positive literal: relname (atom or op) followed by optional (args).
+(define
+  dl-pp-parse-positive
+  (fn
+    (st)
+    (let
+      ((t (dl-pp-peek st)))
+      (let
+        ((ty (get t :type)) (vv (get t :value)))
+        (if
+          (or (= ty "atom") (= ty "op"))
+          (do
+            (dl-pp-advance! st)
+            (if
+              (dl-pp-at? st "punct" "(")
+              (do
+                (dl-pp-advance! st)
+                (let
+                  ((args (dl-pp-parse-arg-list st)))
+                  (do
+                    (dl-pp-expect! st "punct" ")")
+                    (cons (string->symbol vv) args))))
+              (list (string->symbol vv))))
+          (dl-pp-error st "expected literal head"))))))
+
+;; A body literal: positive, or not(positive).
+(define
+  dl-pp-parse-body-lit
+  (fn
+    (st)
+    (let
+      ((t1 (dl-pp-peek st)) (t2 (dl-pp-peek2 st)))
+      (if
+        (and
+          (= (get t1 :type) "atom")
+          (= (get t1 :value) "not")
+          (= (get t2 :type) "punct")
+          (= (get t2 :value) "("))
+        (do
+          (dl-pp-advance! st)
+          (dl-pp-advance! st)
+          (let
+            ((inner (dl-pp-parse-positive st)))
+            (do (dl-pp-expect! st "punct" ")") {:neg inner})))
+        (dl-pp-parse-positive st)))))
+
+;; Comma-separated body literals.
+(define
+  dl-pp-parse-body
+  (fn
+    (st)
+    (let
+      ((lits (list)))
+      (do
+        (append! lits (dl-pp-parse-body-lit st))
+        (define
+          dl-pp-body-loop
+          (fn
+            ()
+            (when
+              (dl-pp-at? st "punct" ",")
+              (do
+                (dl-pp-advance! st)
+                (append! lits (dl-pp-parse-body-lit st))
+                (dl-pp-body-loop)))))
+        (dl-pp-body-loop)
+        lits))))
+
+;; Single clause: fact, rule, or query. Consumes trailing dot.
+(define
+  dl-pp-parse-clause
+  (fn
+    (st)
+    (cond
+      ((dl-pp-at? st "op" "?-")
+        (do
+          (dl-pp-advance! st)
+          (let
+            ((body (dl-pp-parse-body st)))
+            (do (dl-pp-expect! st "punct" ".") {:query body}))))
+      (else
+        (let
+          ((head (dl-pp-parse-positive st)))
+          (cond
+            ((dl-pp-at? st "op" ":-")
+              (do
+                (dl-pp-advance! st)
+                (let
+                  ((body (dl-pp-parse-body st)))
+                  (do (dl-pp-expect! st "punct" ".") {:body body :head head}))))
+            (else (do (dl-pp-expect! st "punct" ".") {:body (list) :head head}))))))))
+
+(define
+  dl-parse-program
+  (fn
+    (tokens)
+    (let
+      ((st {:tokens tokens :idx 0}) (clauses (list)))
+      (do
+        (define
+          dl-pp-prog-loop
+          (fn
+            ()
+            (when
+              (not (dl-pp-at? st "eof" nil))
+              (do
+                (append! clauses (dl-pp-parse-clause st))
+                (dl-pp-prog-loop)))))
+        (dl-pp-prog-loop)
+        clauses))))
+
+(define dl-parse (fn (src) (dl-parse-program (dl-tokenize src))))

lib/datalog/scoreboard.json

@@ -0,0 +1,20 @@
+{
+  "lang": "datalog",
+  "total_passed": 276,
+  "total_failed": 0,
+  "total": 276,
+  "suites": [
+    {"name":"tokenize","passed":31,"failed":0,"total":31},
+    {"name":"parse","passed":23,"failed":0,"total":23},
+    {"name":"unify","passed":29,"failed":0,"total":29},
+    {"name":"eval","passed":44,"failed":0,"total":44},
+    {"name":"builtins","passed":26,"failed":0,"total":26},
+    {"name":"semi_naive","passed":8,"failed":0,"total":8},
+    {"name":"negation","passed":12,"failed":0,"total":12},
+    {"name":"aggregates","passed":23,"failed":0,"total":23},
+    {"name":"api","passed":22,"failed":0,"total":22},
+    {"name":"magic","passed":37,"failed":0,"total":37},
+    {"name":"demo","passed":21,"failed":0,"total":21}
+  ],
+  "generated": "2026-05-11T09:40:12+00:00"
+}

lib/datalog/scoreboard.md

@@ -0,0 +1,17 @@
+# datalog scoreboard
+
+**276 / 276 passing** (0 failure(s)).
+
+| Suite | Passed | Total | Status |
+|-------|--------|-------|--------|
+| tokenize | 31 | 31 | ok |
+| parse | 23 | 23 | ok |
+| unify | 29 | 29 | ok |
+| eval | 44 | 44 | ok |
+| builtins | 26 | 26 | ok |
+| semi_naive | 8 | 8 | ok |
+| negation | 12 | 12 | ok |
+| aggregates | 23 | 23 | ok |
+| api | 22 | 22 | ok |
+| magic | 37 | 37 | ok |
+| demo | 21 | 21 | ok |

lib/datalog/strata.sx

@@ -0,0 +1,323 @@
+;; lib/datalog/strata.sx — dependency graph, SCC analysis, stratum assignment.
+;;
+;; A program is stratifiable iff no cycle in its dependency graph passes
+;; through a negative edge. The stratum of relation R is the depth at which
+;; R can first be evaluated:
+;;
+;;   stratum(R) = max over edges (R → S) of:
+;;       stratum(S)         if the edge is positive
+;;       stratum(S) + 1     if the edge is negative
+;;
+;; All relations in the same SCC share a stratum (and the SCC must have only
+;; positive internal edges, else the program is non-stratifiable).
+
+;; Build dep graph: dict {head-rel-name -> ({:rel str :neg bool} ...)}.
+(define
+  dl-build-dep-graph
+  (fn
+    (db)
+    (let ((g {}))
+      (do
+        (for-each
+          (fn
+            (rule)
+            (let
+              ((head-rel (dl-rel-name (get rule :head))))
+              (when
+                (not (nil? head-rel))
+                (do
+                  (when
+                    (not (has-key? g head-rel))
+                    (dict-set! g head-rel (list)))
+                  (let ((existing (get g head-rel)))
+                    (for-each
+                      (fn
+                        (lit)
+                        (cond
+                          ((dl-aggregate? lit)
+                            (let
+                              ((edge (dl-aggregate-dep-edge lit)))
+                              (when
+                                (not (nil? edge))
+                                (append! existing edge))))
+                          (else
+                            (let
+                              ((target
+                                 (cond
+                                   ((and (dict? lit) (has-key? lit :neg))
+                                     (dl-rel-name (get lit :neg)))
+                                   ((dl-builtin? lit) nil)
+                                   ((and (list? lit) (> (len lit) 0))
+                                     (dl-rel-name lit))
+                                   (else nil)))
+                                (neg?
+                                  (and (dict? lit) (has-key? lit :neg))))
+                              (when
+                                (not (nil? target))
+                                (append!
+                                  existing
+                                  {:rel target :neg neg?}))))))
+                      (get rule :body)))))))
+          (dl-rules db))
+        g))))
+
+;; All relations referenced — heads of rules + EDB names + body relations.
+(define
+  dl-all-relations
+  (fn
+    (db)
+    (let ((seen (list)))
+      (do
+        (for-each
+          (fn
+            (k)
+            (when (not (dl-member-string? k seen)) (append! seen k)))
+          (keys (get db :facts)))
+        (for-each
+          (fn
+            (rule)
+            (do
+              (let ((h (dl-rel-name (get rule :head))))
+                (when
+                  (and (not (nil? h)) (not (dl-member-string? h seen)))
+                  (append! seen h)))
+              (for-each
+                (fn
+                  (lit)
+                  (let
+                    ((t
+                       (cond
+                         ((dl-aggregate? lit)
+                           (let ((edge (dl-aggregate-dep-edge lit)))
+                             (if (nil? edge) nil (get edge :rel))))
+                         ((and (dict? lit) (has-key? lit :neg))
+                           (dl-rel-name (get lit :neg)))
+                         ((dl-builtin? lit) nil)
+                         ((and (list? lit) (> (len lit) 0))
+                           (dl-rel-name lit))
+                         (else nil))))
+                    (when
+                      (and (not (nil? t)) (not (dl-member-string? t seen)))
+                      (append! seen t))))
+                (get rule :body))))
+          (dl-rules db))
+        seen))))
+
+;; reach: dict {from: dict {to: edge-info}} where edge-info is
+;;   {:any bool :neg bool}
+;; meaning "any path from `from` to `to`" and "exists a negative-passing
+;; path from `from` to `to`".
+;;
+;; Floyd-Warshall over the dep graph. The 'neg' flag propagates through
+;; concatenation: if any edge along the path is negative, the path's
+;; flag is true.
+(define
+  dl-build-reach
+  (fn
+    (graph nodes)
+    (let ((reach {}))
+      (do
+        (for-each
+          (fn (n) (dict-set! reach n {}))
+          nodes)
+        (for-each
+          (fn
+            (head)
+            (when
+              (has-key? graph head)
+              (for-each
+                (fn
+                  (edge)
+                  (let
+                    ((target (get edge :rel)) (n (get edge :neg)))
+                    (let ((row (get reach head)))
+                      (cond
+                        ((has-key? row target)
+                          (let ((cur (get row target)))
+                            (dict-set!
+                              row
+                              target
+                              {:any true :neg (or n (get cur :neg))})))
+                        (else
+                          (dict-set! row target {:any true :neg n}))))))
+                (get graph head))))
+          nodes)
+        (for-each
+          (fn
+            (k)
+            (for-each
+              (fn
+                (i)
+                (let ((row-i (get reach i)))
+                  (when
+                    (has-key? row-i k)
+                    (let ((ik (get row-i k)) (row-k (get reach k)))
+                      (for-each
+                        (fn
+                          (j)
+                          (when
+                            (has-key? row-k j)
+                            (let ((kj (get row-k j)))
+                              (let
+                                ((combined-neg (or (get ik :neg) (get kj :neg))))
+                                (cond
+                                  ((has-key? row-i j)
+                                    (let ((cur (get row-i j)))
+                                      (dict-set!
+                                        row-i
+                                        j
+                                        {:any true
+                                         :neg (or combined-neg (get cur :neg))})))
+                                  (else
+                                    (dict-set!
+                                      row-i
+                                      j
+                                      {:any true :neg combined-neg})))))))
+                        nodes)))))
+              nodes))
+          nodes)
+        reach))))
+
+;; Returns nil on success, or error message string on failure.
+(define
+  dl-check-stratifiable
+  (fn
+    (db)
+    (let
+      ((graph (dl-build-dep-graph db))
+        (nodes (dl-all-relations db)))
+      (let ((reach (dl-build-reach graph nodes)) (err nil))
+        (do
+          (for-each
+            (fn
+              (rule)
+              (when
+                (nil? err)
+                (let ((head-rel (dl-rel-name (get rule :head))))
+                  (for-each
+                    (fn
+                      (lit)
+                      (cond
+                        ((and (dict? lit) (has-key? lit :neg))
+                          (let ((tgt (dl-rel-name (get lit :neg))))
+                            (when
+                              (and (not (nil? tgt))
+                                   (dl-reach-cycle? reach head-rel tgt))
+                              (set!
+                                err
+                                (str "non-stratifiable: relation " head-rel
+                                     " transitively depends through negation on "
+                                     tgt
+                                     " which depends back on " head-rel)))))
+                        ((dl-aggregate? lit)
+                          (let ((edge (dl-aggregate-dep-edge lit)))
+                            (when
+                              (not (nil? edge))
+                              (let ((tgt (get edge :rel)))
+                                (when
+                                  (and (not (nil? tgt))
+                                       (dl-reach-cycle? reach head-rel tgt))
+                                  (set!
+                                    err
+                                    (str "non-stratifiable: relation "
+                                         head-rel
+                                         " aggregates over " tgt
+                                         " which depends back on "
+                                         head-rel))))))))) 
+                    (get rule :body)))))
+            (dl-rules db))
+          err)))))
+
+(define
+  dl-reach-cycle?
+  (fn
+    (reach a b)
+    (and
+      (dl-reach-row-has? reach b a)
+      (dl-reach-row-has? reach a b))))
+
+(define
+  dl-reach-row-has?
+  (fn
+    (reach from to)
+    (let ((row (get reach from)))
+      (and (not (nil? row)) (has-key? row to)))))
+
+;; Compute stratum per relation. Iteratively propagate from EDB roots.
+;; Uses the per-relation max-stratum-of-deps formula. Stops when stable.
+(define
+  dl-compute-strata
+  (fn
+    (db)
+    (let
+      ((graph (dl-build-dep-graph db))
+        (nodes (dl-all-relations db))
+        (strata {}))
+      (do
+        (for-each (fn (n) (dict-set! strata n 0)) nodes)
+        (let ((changed true))
+          (do
+            (define
+              dl-cs-loop
+              (fn
+                ()
+                (when
+                  changed
+                  (do
+                    (set! changed false)
+                    (for-each
+                      (fn
+                        (head)
+                        (when
+                          (has-key? graph head)
+                          (for-each
+                            (fn
+                              (edge)
+                              (let
+                                ((tgt (get edge :rel))
+                                  (n (get edge :neg)))
+                                (let
+                                  ((tgt-strat
+                                     (if (has-key? strata tgt)
+                                       (get strata tgt) 0))
+                                    (cur (get strata head)))
+                                  (let
+                                    ((needed
+                                       (if n (+ tgt-strat 1) tgt-strat)))
+                                    (when
+                                      (> needed cur)
+                                      (do
+                                        (dict-set! strata head needed)
+                                        (set! changed true)))))))
+                            (get graph head))))
+                      nodes)
+                    (dl-cs-loop)))))
+            (dl-cs-loop)))
+        strata))))
+
+;; Group rules by their head's stratum. Returns dict {stratum-int -> rules}.
+(define
+  dl-group-rules-by-stratum
+  (fn
+    (db strata)
+    (let ((groups {}) (max-s 0))
+      (do
+        (for-each
+          (fn
+            (rule)
+            (let
+              ((head-rel (dl-rel-name (get rule :head))))
+              (let
+                ((s (if (has-key? strata head-rel)
+                       (get strata head-rel) 0)))
+                (do
+                  (when (> s max-s) (set! max-s s))
+                  (let
+                    ((sk (str s)))
+                    (do
+                      (when
+                        (not (has-key? groups sk))
+                        (dict-set! groups sk (list)))
+                      (append! (get groups sk) rule)))))))
+          (dl-rules db))
+        {:groups groups :max max-s}))))

lib/datalog/tests/aggregates.sx

@@ -0,0 +1,357 @@
+;; lib/datalog/tests/aggregates.sx — count / sum / min / max.
+
+(define dl-at-pass 0)
+(define dl-at-fail 0)
+(define dl-at-failures (list))
+
+(define
+  dl-at-deep=?
+  (fn
+    (a b)
+    (cond
+      ((and (list? a) (list? b))
+        (and (= (len a) (len b)) (dl-at-deq-l? a b 0)))
+      ((and (dict? a) (dict? b))
+        (let ((ka (keys a)) (kb (keys b)))
+          (and (= (len ka) (len kb)) (dl-at-deq-d? a b ka 0))))
+      ((and (number? a) (number? b)) (= a b))
+      (else (equal? a b)))))
+
+(define
+  dl-at-deq-l?
+  (fn
+    (a b i)
+    (cond
+      ((>= i (len a)) true)
+      ((not (dl-at-deep=? (nth a i) (nth b i))) false)
+      (else (dl-at-deq-l? a b (+ i 1))))))
+
+(define
+  dl-at-deq-d?
+  (fn
+    (a b ka i)
+    (cond
+      ((>= i (len ka)) true)
+      ((let ((k (nth ka i)))
+         (not (dl-at-deep=? (get a k) (get b k))))
+        false)
+      (else (dl-at-deq-d? a b ka (+ i 1))))))
+
+(define
+  dl-at-set=?
+  (fn
+    (a b)
+    (and
+      (= (len a) (len b))
+      (dl-at-subset? a b)
+      (dl-at-subset? b a))))
+
+(define
+  dl-at-subset?
+  (fn
+    (xs ys)
+    (cond
+      ((= (len xs) 0) true)
+      ((not (dl-at-contains? ys (first xs))) false)
+      (else (dl-at-subset? (rest xs) ys)))))
+
+(define
+  dl-at-contains?
+  (fn
+    (xs target)
+    (cond
+      ((= (len xs) 0) false)
+      ((dl-at-deep=? (first xs) target) true)
+      (else (dl-at-contains? (rest xs) target)))))
+
+(define
+  dl-at-test!
+  (fn
+    (name got expected)
+    (if
+      (dl-at-deep=? got expected)
+      (set! dl-at-pass (+ dl-at-pass 1))
+      (do
+        (set! dl-at-fail (+ dl-at-fail 1))
+        (append!
+          dl-at-failures
+          (str
+            name
+            "\n    expected: " expected
+            "\n    got:      " got))))))
+
+(define
+  dl-at-test-set!
+  (fn
+    (name got expected)
+    (if
+      (dl-at-set=? got expected)
+      (set! dl-at-pass (+ dl-at-pass 1))
+      (do
+        (set! dl-at-fail (+ dl-at-fail 1))
+        (append!
+          dl-at-failures
+          (str
+            name
+            "\n    expected (set): " expected
+            "\n    got:            " got))))))
+
+(define
+  dl-at-throws?
+  (fn
+    (thunk)
+    (let
+      ((threw false))
+      (do
+        (guard
+          (e (#t (set! threw true)))
+          (thunk))
+        threw))))
+
+(define
+  dl-at-run-all!
+  (fn
+    ()
+    (do
+      ;; count
+      (dl-at-test-set! "count siblings"
+        (dl-query
+          (dl-program
+            "parent(p, bob). parent(p, alice). parent(p, charlie).
+             sibling(X, Y) :- parent(P, X), parent(P, Y), !=(X, Y).
+             sib_count(N) :- count(N, S, sibling(bob, S)).")
+          (list (quote sib_count) (quote N)))
+        (list {:N 2}))
+
+      ;; sum
+      (dl-at-test-set! "sum prices"
+        (dl-query
+          (dl-program
+            "price(apple, 5). price(pear, 7). price(plum, 3).
+             total(T) :- sum(T, X, price(F, X)).")
+          (list (quote total) (quote T)))
+        (list {:T 15}))
+
+      ;; min
+      (dl-at-test-set! "min score"
+        (dl-query
+          (dl-program
+            "score(alice, 80). score(bob, 65). score(carol, 92).
+             lo(M) :- min(M, S, score(P, S)).")
+          (list (quote lo) (quote M)))
+        (list {:M 65}))
+
+      ;; max
+      (dl-at-test-set! "max score"
+        (dl-query
+          (dl-program
+            "score(alice, 80). score(bob, 65). score(carol, 92).
+             hi(M) :- max(M, S, score(P, S)).")
+          (list (quote hi) (quote M)))
+        (list {:M 92}))
+
+      ;; count over derived relation (stratification needed).
+      (dl-at-test-set! "count over derived"
+        (dl-query
+          (dl-program
+            "parent(a, b). parent(a, c). parent(b, d). parent(c, e).
+             ancestor(X, Y) :- parent(X, Y).
+             ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z).
+             num_ancestors(N) :- count(N, X, ancestor(a, X)).")
+          (list (quote num_ancestors) (quote N)))
+        (list {:N 4}))
+
+      ;; count with no matches → 0.
+      (dl-at-test-set! "count empty"
+        (dl-query
+          (dl-program
+            "p(1). p(2).
+             zero(N) :- count(N, X, q(X)).")
+          (list (quote zero) (quote N)))
+        (list {:N 0}))
+
+      ;; sum with no matches → 0.
+      (dl-at-test-set! "sum empty"
+        (dl-query
+          (dl-program
+            "p(1). p(2).
+             total(T) :- sum(T, X, q(X)).")
+          (list (quote total) (quote T)))
+        (list {:T 0}))
+
+      ;; min with no matches → rule does not fire.
+      (dl-at-test-set! "min empty"
+        (dl-query
+          (dl-program
+            "p(1). p(2).
+             lo(M) :- min(M, X, q(X)).")
+          (list (quote lo) (quote M)))
+        (list))
+
+      ;; Aggregate with comparison filter on result.
+      (dl-at-test-set! "popularity threshold"
+        (dl-query
+          (dl-program
+            "post(p1). post(p2).
+             liked(u1, p1). liked(u2, p1). liked(u3, p1).
+             liked(u1, p2). liked(u2, p2).
+             popular(P) :- post(P), count(N, U, liked(U, P)), >=(N, 3).")
+          (list (quote popular) (quote P)))
+        (list {:P (quote p1)}))
+
+      ;; findall: collect distinct values into a list.
+      (dl-at-test-set! "findall over EDB"
+        (dl-query
+          (dl-program
+            "p(a). p(b). p(c).
+             all_p(L) :- findall(L, X, p(X)).")
+          (list (quote all_p) (quote L)))
+        (list {:L (list (quote a) (quote b) (quote c))}))
+
+      (dl-at-test-set! "findall over derived"
+        (dl-query
+          (dl-program
+            "parent(a, b). parent(b, c). parent(c, d).
+             ancestor(X, Y) :- parent(X, Y).
+             ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z).
+             desc(L) :- findall(L, X, ancestor(a, X)).")
+          (list (quote desc) (quote L)))
+        (list {:L (list (quote b) (quote c) (quote d))}))
+
+      (dl-at-test-set! "findall empty"
+        (dl-query
+          (dl-program
+            "p(1).
+             all_q(L) :- findall(L, X, q(X)).")
+          (list (quote all_q) (quote L)))
+        (list {:L (list)}))
+
+      ;; Aggregate vs single distinct.
+      ;; Group-by via aggregate-in-rule-body. Per-user friend count
+      ;; over a friends relation. The U var is bound by the prior
+      ;; positive lit u(U) so the aggregate counts only U-rooted
+      ;; friends per group.
+      (dl-at-test-set! "group-by per-user friend count"
+        (dl-query
+          (dl-program
+            "u(alice). u(bob). u(carol).
+             f(alice, x). f(alice, y). f(bob, x).
+             counts(U, N) :- u(U), count(N, X, f(U, X)).")
+          (list (quote counts) (quote U) (quote N)))
+        (list
+          {:U (quote alice) :N 2}
+          {:U (quote bob) :N 1}
+          {:U (quote carol) :N 0}))
+
+      ;; Stratification: recursion through aggregation is rejected.
+      ;; Aggregate validates that second arg is a variable.
+      (dl-at-test! "agg second arg must be var"
+        (dl-at-throws?
+          (fn () (dl-eval "p(1). q(N) :- count(N, 5, p(X))." "?- q(N).")))
+        true)
+
+      ;; Aggregate validates that third arg is a positive literal.
+      (dl-at-test! "agg third arg must be a literal"
+        (dl-at-throws?
+          (fn () (dl-eval "p(1). q(N) :- count(N, X, 42)." "?- q(N).")))
+        true)
+
+      ;; Aggregate validates that the agg-var (2nd arg) appears in the
+      ;; goal. Without it every match contributes the same unbound
+      ;; symbol — count silently returns 1, sum raises a confusing
+      ;; "expected number" error, etc. Catch the mistake at safety
+      ;; check time instead.
+      (dl-at-test! "agg-var must appear in goal"
+        (dl-at-throws?
+          (fn ()
+            (dl-eval
+              "p(1). p(2). c(N) :- count(N, Y, p(X))."
+              "?- c(N).")))
+        true)
+
+      ;; Indirect recursion through aggregation also rejected.
+      ;; q -> r (via positive lit), r -> q (via aggregate body).
+      ;; The aggregate edge counts as negation for stratification.
+      (dl-at-test! "indirect agg cycle rejected"
+        (dl-at-throws?
+          (fn ()
+            (let ((db (dl-make-db)))
+              (do
+                (dl-add-rule! db
+                  {:head (list (quote q) (quote N))
+                   :body (list (list (quote r) (quote N)))})
+                (dl-add-rule! db
+                  {:head (list (quote r) (quote N))
+                   :body (list (list (quote count) (quote N) (quote X)
+                                     (list (quote q) (quote X))))})
+                (dl-saturate! db)))))
+        true)
+
+      (dl-at-test! "agg recursion rejected"
+        (dl-at-throws?
+          (fn ()
+            (let ((db (dl-make-db)))
+              (do
+                (dl-add-rule! db
+                  {:head (list (quote q) (quote N))
+                   :body (list (list (quote count) (quote N) (quote X)
+                                     (list (quote q) (quote X))))})
+                (dl-saturate! db)))))
+        true)
+
+      ;; Negation + aggregation in the same body — different strata.
+      (dl-at-test-set! "neg + agg coexist"
+        (dl-query
+          (dl-program
+            "u(a). u(b). u(c). banned(b).
+             active(X) :- u(X), not(banned(X)).
+             cnt(N) :- count(N, X, active(X)).")
+          (list (quote cnt) (quote N)))
+        (list {:N 2}))
+
+      ;; Min over a derived empty relation: no result.
+      (dl-at-test-set! "min over empty derived"
+        (dl-query
+          (dl-program
+            "s(50). s(60).
+             score(N) :- s(N), >(N, 100).
+             low(M) :- min(M, X, score(X)).")
+          (list (quote low) (quote M)))
+        (list))
+
+      ;; Aggregates as the top-level query goal (regression for
+      ;; dl-match-lit aggregate dispatch and projection cleanup).
+      (dl-at-test-set! "count as query goal"
+        (dl-query
+          (dl-program "p(1). p(2). p(3). p(4).")
+          (list (quote count) (quote N) (quote X) (list (quote p) (quote X))))
+        (list {:N 4}))
+
+      (dl-at-test-set! "findall as query goal"
+        (dl-query
+          (dl-program "p(1). p(2). p(3).")
+          (list (quote findall) (quote L) (quote X)
+                (list (quote p) (quote X))))
+        (list {:L (list 1 2 3)}))
+
+      (dl-at-test-set! "distinct counted once"
+        (dl-query
+          (dl-program
+            "rated(alice, x). rated(alice, y). rated(bob, x).
+             rater_count(N) :- count(N, U, rated(U, F)).")
+          (list (quote rater_count) (quote N)))
+        (list {:N 2})))))
+
+(define
+  dl-aggregates-tests-run!
+  (fn
+    ()
+    (do
+      (set! dl-at-pass 0)
+      (set! dl-at-fail 0)
+      (set! dl-at-failures (list))
+      (dl-at-run-all!)
+      {:passed dl-at-pass
+       :failed dl-at-fail
+       :total (+ dl-at-pass dl-at-fail)
+       :failures dl-at-failures})))

lib/datalog/tests/api.sx

@@ -0,0 +1,350 @@
+;; lib/datalog/tests/api.sx — SX-data embedding API.
+
+(define dl-api-pass 0)
+(define dl-api-fail 0)
+(define dl-api-failures (list))
+
+(define
+  dl-api-deep=?
+  (fn
+    (a b)
+    (cond
+      ((and (list? a) (list? b))
+        (and (= (len a) (len b)) (dl-api-deq-l? a b 0)))
+      ((and (dict? a) (dict? b))
+        (let ((ka (keys a)) (kb (keys b)))
+          (and (= (len ka) (len kb)) (dl-api-deq-d? a b ka 0))))
+      ((and (number? a) (number? b)) (= a b))
+      (else (equal? a b)))))
+
+(define
+  dl-api-deq-l?
+  (fn
+    (a b i)
+    (cond
+      ((>= i (len a)) true)
+      ((not (dl-api-deep=? (nth a i) (nth b i))) false)
+      (else (dl-api-deq-l? a b (+ i 1))))))
+
+(define
+  dl-api-deq-d?
+  (fn
+    (a b ka i)
+    (cond
+      ((>= i (len ka)) true)
+      ((let ((k (nth ka i)))
+         (not (dl-api-deep=? (get a k) (get b k))))
+        false)
+      (else (dl-api-deq-d? a b ka (+ i 1))))))
+
+(define
+  dl-api-set=?
+  (fn
+    (a b)
+    (and
+      (= (len a) (len b))
+      (dl-api-subset? a b)
+      (dl-api-subset? b a))))
+
+(define
+  dl-api-subset?
+  (fn
+    (xs ys)
+    (cond
+      ((= (len xs) 0) true)
+      ((not (dl-api-contains? ys (first xs))) false)
+      (else (dl-api-subset? (rest xs) ys)))))
+
+(define
+  dl-api-contains?
+  (fn
+    (xs target)
+    (cond
+      ((= (len xs) 0) false)
+      ((dl-api-deep=? (first xs) target) true)
+      (else (dl-api-contains? (rest xs) target)))))
+
+(define
+  dl-api-test!
+  (fn
+    (name got expected)
+    (if
+      (dl-api-deep=? got expected)
+      (set! dl-api-pass (+ dl-api-pass 1))
+      (do
+        (set! dl-api-fail (+ dl-api-fail 1))
+        (append!
+          dl-api-failures
+          (str
+            name
+            "\n    expected: " expected
+            "\n    got:      " got))))))
+
+(define
+  dl-api-test-set!
+  (fn
+    (name got expected)
+    (if
+      (dl-api-set=? got expected)
+      (set! dl-api-pass (+ dl-api-pass 1))
+      (do
+        (set! dl-api-fail (+ dl-api-fail 1))
+        (append!
+          dl-api-failures
+          (str
+            name
+            "\n    expected (set): " expected
+            "\n    got:            " got))))))
+
+(define
+  dl-api-run-all!
+  (fn
+    ()
+    (do
+      ;; dl-program-data with arrow form.
+      (dl-api-test-set! "data API ancestor closure"
+        (dl-query
+          (dl-program-data
+            (quote ((parent tom bob) (parent bob ann) (parent ann pat)))
+            (quote
+              ((ancestor X Y <- (parent X Y))
+               (ancestor X Z <- (parent X Y) (ancestor Y Z)))))
+          (quote (ancestor tom X)))
+        (list {:X (quote bob)} {:X (quote ann)} {:X (quote pat)}))
+
+      ;; dl-program-data with dict rules.
+      (dl-api-test-set! "data API with dict rules"
+        (dl-query
+          (dl-program-data
+            (quote ((p a) (p b) (p c)))
+            (list
+              {:head (quote (q X)) :body (quote ((p X)))}))
+          (quote (q X)))
+        (list {:X (quote a)} {:X (quote b)} {:X (quote c)}))
+
+      ;; dl-rule helper.
+      (dl-api-test-set! "dl-rule constructor"
+        (dl-query
+          (dl-program-data
+            (quote ((p 1) (p 2)))
+            (list (dl-rule (quote (q X)) (quote ((p X))))))
+          (quote (q X)))
+        (list {:X 1} {:X 2}))
+
+      ;; dl-assert! adds and re-derives.
+      (dl-api-test-set! "dl-assert! incremental"
+        (let
+          ((db (dl-program-data
+                 (quote ((parent tom bob) (parent bob ann)))
+                 (quote
+                   ((ancestor X Y <- (parent X Y))
+                    (ancestor X Z <- (parent X Y) (ancestor Y Z)))))))
+          (do
+            (dl-saturate! db)
+            (dl-assert! db (quote (parent ann pat)))
+            (dl-query db (quote (ancestor tom X)))))
+        (list {:X (quote bob)} {:X (quote ann)} {:X (quote pat)}))
+
+      ;; dl-retract! removes a fact and recomputes IDB.
+      (dl-api-test-set! "dl-retract! removes derived"
+        (let
+          ((db (dl-program-data
+                 (quote ((parent tom bob) (parent bob ann) (parent ann pat)))
+                 (quote
+                   ((ancestor X Y <- (parent X Y))
+                    (ancestor X Z <- (parent X Y) (ancestor Y Z)))))))
+          (do
+            (dl-saturate! db)
+            (dl-retract! db (quote (parent bob ann)))
+            (dl-query db (quote (ancestor tom X)))))
+        (list {:X (quote bob)}))
+
+      ;; dl-retract! on a relation with BOTH explicit facts AND a rule
+      ;; (a "mixed" relation) used to wipe the EDB portion when the IDB
+      ;; was re-derived, even when the retract didn't match anything.
+      ;; :edb-keys provenance now preserves user-asserted facts.
+      (dl-api-test-set! "dl-retract! preserves EDB in mixed relation"
+        (let
+          ((db (dl-program-data
+                 (quote ((p a) (p b) (q c)))
+                 (quote ((p X <- (q X)))))))
+          (do
+            (dl-saturate! db)
+            ;; Retract a non-existent tuple — should be a no-op.
+            (dl-retract! db (quote (p z)))
+            (dl-query db (quote (p X)))))
+        (list {:X (quote a)} {:X (quote b)} {:X (quote c)}))
+
+      ;; And retracting an actual EDB fact in a mixed relation drops
+      ;; only that fact; the derived portion stays.
+      (dl-api-test-set! "dl-retract! mixed: drop EDB, keep IDB"
+        (let
+          ((db (dl-program-data
+                 (quote ((p a) (p b) (q c)))
+                 (quote ((p X <- (q X)))))))
+          (do
+            (dl-saturate! db)
+            (dl-retract! db (quote (p a)))
+            (dl-query db (quote (p X)))))
+        (list {:X (quote b)} {:X (quote c)}))
+
+      ;; dl-program-data + dl-query with constants in head.
+      (dl-api-test-set! "constant-in-head data"
+        (dl-query
+          (dl-program-data
+            (quote ((edge a b) (edge b c) (edge c a)))
+            (quote
+              ((reach X Y <- (edge X Y))
+               (reach X Z <- (edge X Y) (reach Y Z)))))
+          (quote (reach a X)))
+        (list {:X (quote a)} {:X (quote b)} {:X (quote c)}))
+
+      ;; Assert into empty db.
+      (dl-api-test-set! "assert into empty"
+        (let
+          ((db (dl-program-data (list) (list))))
+          (do
+            (dl-assert! db (quote (p 1)))
+            (dl-assert! db (quote (p 2)))
+            (dl-query db (quote (p X)))))
+        (list {:X 1} {:X 2}))
+
+      ;; Multi-goal query: pass list of literals.
+      (dl-api-test-set! "multi-goal query"
+        (dl-query
+          (dl-program-data
+            (quote ((p 1) (p 2) (p 3) (q 2) (q 3)))
+            (list))
+          (list (quote (p X)) (quote (q X))))
+        (list {:X 2} {:X 3}))
+
+      ;; Multi-goal with comparison.
+      (dl-api-test-set! "multi-goal with comparison"
+        (dl-query
+          (dl-program-data
+            (quote ((n 1) (n 2) (n 3) (n 4) (n 5)))
+            (list))
+          (list (quote (n X)) (list (string->symbol ">") (quote X) 2)))
+        (list {:X 3} {:X 4} {:X 5}))
+
+      ;; dl-eval: single-call source + query.
+      (dl-api-test-set! "dl-eval ancestor"
+        (dl-eval
+          "parent(a, b). parent(b, c).
+           ancestor(X, Y) :- parent(X, Y).
+           ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z)."
+          "?- ancestor(a, X).")
+        (list {:X (quote b)} {:X (quote c)}))
+
+      (dl-api-test-set! "dl-eval multi-goal"
+        (dl-eval
+          "p(1). p(2). p(3). q(2). q(3)."
+          "?- p(X), q(X).")
+        (list {:X 2} {:X 3}))
+
+      ;; dl-rules-of: rules with head matching a relation name.
+      (dl-api-test! "dl-rules-of count"
+        (let
+          ((db (dl-program
+                 "p(1). q(X) :- p(X). r(X) :- p(X). q(2).")))
+          (len (dl-rules-of db "q")))
+        1)
+
+      (dl-api-test! "dl-rules-of empty"
+        (let
+          ((db (dl-program "p(1). p(2).")))
+          (len (dl-rules-of db "q")))
+        0)
+
+      ;; dl-clear-idb!: wipe rule-headed relations.
+      (dl-api-test! "dl-clear-idb! wipes IDB"
+        (let
+          ((db (dl-program
+                 "parent(a, b). parent(b, c).
+                  ancestor(X, Y) :- parent(X, Y).
+                  ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z).")))
+          (do
+            (dl-saturate! db)
+            (dl-clear-idb! db)
+            (len (dl-relation db "ancestor"))))
+        0)
+
+      (dl-api-test! "dl-clear-idb! preserves EDB"
+        (let
+          ((db (dl-program
+                 "parent(a, b). parent(b, c).
+                  ancestor(X, Y) :- parent(X, Y).")))
+          (do
+            (dl-saturate! db)
+            (dl-clear-idb! db)
+            (len (dl-relation db "parent"))))
+        2)
+
+      ;; dl-eval-magic — routes single-goal queries through
+      ;; magic-sets evaluation.
+      (dl-api-test-set! "dl-eval-magic ancestor"
+        (dl-eval-magic
+          "parent(a, b). parent(b, c).
+           ancestor(X, Y) :- parent(X, Y).
+           ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z)."
+          "?- ancestor(a, X).")
+        (list {:X (quote b)} {:X (quote c)}))
+
+      ;; Equivalence: dl-eval and dl-eval-magic produce the same
+      ;; answers for any well-formed query (magic-sets is a perf
+      ;; alternative, not a semantic change).
+      (dl-api-test! "dl-eval ≡ dl-eval-magic on ancestor"
+        (let
+          ((source "parent(a, b). parent(b, c). parent(c, d).
+                    ancestor(X, Y) :- parent(X, Y).
+                    ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z)."))
+          (let
+            ((semi (dl-eval source "?- ancestor(a, X)."))
+              (magic (dl-eval-magic source "?- ancestor(a, X).")))
+            (= (len semi) (len magic))))
+        true)
+
+      ;; Comprehensive integration: recursion + stratified negation
+      ;; + aggregation + comparison composed in a single program.
+      ;; (Uses _Anything as a regular var instead of `_` so the
+      ;;  outer rule binds via the reach lit.)
+      (dl-api-test-set! "integration"
+        (dl-eval
+          (str
+            "edge(a, b). edge(b, c). edge(c, d). edge(a, d). "
+            "banned(c). "
+            "reach(X, Y) :- edge(X, Y). "
+            "reach(X, Z) :- edge(X, Y), reach(Y, Z). "
+            "safe(X, Y) :- reach(X, Y), not(banned(Y)). "
+            "reach_count(X, N) :- reach(X, Z), count(N, Y, safe(X, Y)). "
+            "popular(X) :- reach_count(X, N), >=(N, 2).")
+          "?- popular(X).")
+        (list {:X (quote a)}))
+
+      ;; dl-rule-from-list with no arrow → fact-style.
+      (dl-api-test-set! "no arrow → fact-like rule"
+        (let
+          ((rule (dl-rule-from-list (quote (foo X Y)))))
+          (list rule))
+        (list {:head (quote (foo X Y)) :body (list)}))
+
+      ;; dl-coerce-rule on dict passes through.
+      (dl-api-test-set! "coerce dict rule"
+        (let
+          ((d {:head (quote (h X)) :body (quote ((b X)))}))
+          (list (dl-coerce-rule d)))
+        (list {:head (quote (h X)) :body (quote ((b X)))})))))
+
+(define
+  dl-api-tests-run!
+  (fn
+    ()
+    (do
+      (set! dl-api-pass 0)
+      (set! dl-api-fail 0)
+      (set! dl-api-failures (list))
+      (dl-api-run-all!)
+      {:passed dl-api-pass
+       :failed dl-api-fail
+       :total (+ dl-api-pass dl-api-fail)
+       :failures dl-api-failures})))

lib/datalog/tests/builtins.sx

@@ -0,0 +1,285 @@
+;; lib/datalog/tests/builtins.sx — comparison + arithmetic body literals.
+
+(define dl-bt-pass 0)
+(define dl-bt-fail 0)
+(define dl-bt-failures (list))
+
+(define
+  dl-bt-deep=?
+  (fn
+    (a b)
+    (cond
+      ((and (list? a) (list? b))
+        (and (= (len a) (len b)) (dl-bt-deq-l? a b 0)))
+      ((and (dict? a) (dict? b))
+        (let
+          ((ka (keys a)) (kb (keys b)))
+          (and (= (len ka) (len kb)) (dl-bt-deq-d? a b ka 0))))
+      ((and (number? a) (number? b)) (= a b))
+      (else (equal? a b)))))
+
+(define
+  dl-bt-deq-l?
+  (fn
+    (a b i)
+    (cond
+      ((>= i (len a)) true)
+      ((not (dl-bt-deep=? (nth a i) (nth b i))) false)
+      (else (dl-bt-deq-l? a b (+ i 1))))))
+
+(define
+  dl-bt-deq-d?
+  (fn
+    (a b ka i)
+    (cond
+      ((>= i (len ka)) true)
+      ((let ((k (nth ka i))) (not (dl-bt-deep=? (get a k) (get b k))))
+        false)
+      (else (dl-bt-deq-d? a b ka (+ i 1))))))
+
+(define
+  dl-bt-set=?
+  (fn
+    (a b)
+    (and (= (len a) (len b)) (dl-bt-subset? a b) (dl-bt-subset? b a))))
+
+(define
+  dl-bt-subset?
+  (fn
+    (xs ys)
+    (cond
+      ((= (len xs) 0) true)
+      ((not (dl-bt-contains? ys (first xs))) false)
+      (else (dl-bt-subset? (rest xs) ys)))))
+
+(define
+  dl-bt-contains?
+  (fn
+    (xs target)
+    (cond
+      ((= (len xs) 0) false)
+      ((dl-bt-deep=? (first xs) target) true)
+      (else (dl-bt-contains? (rest xs) target)))))
+
+(define
+  dl-bt-test-set!
+  (fn
+    (name got expected)
+    (if
+      (dl-bt-set=? got expected)
+      (set! dl-bt-pass (+ dl-bt-pass 1))
+      (do
+        (set! dl-bt-fail (+ dl-bt-fail 1))
+        (append!
+          dl-bt-failures
+          (str
+            name
+            "\n    expected (set): "
+            expected
+            "\n    got:            "
+            got))))))
+
+(define
+  dl-bt-test!
+  (fn
+    (name got expected)
+    (if
+      (dl-bt-deep=? got expected)
+      (set! dl-bt-pass (+ dl-bt-pass 1))
+      (do
+        (set! dl-bt-fail (+ dl-bt-fail 1))
+        (append!
+          dl-bt-failures
+          (str name "\n    expected: " expected "\n    got:      " got))))))
+
+(define
+  dl-bt-throws?
+  (fn
+    (thunk)
+    (let
+      ((threw false))
+      (do (guard (e (#t (set! threw true))) (thunk)) threw))))
+
+(define
+  dl-bt-run-all!
+  (fn
+    ()
+    (do
+      (dl-bt-test-set!
+        "less than filter"
+        (dl-query
+          (dl-program
+            "age(alice, 30). age(bob, 17). age(carol, 22).\n             adult(X) :- age(X, A), >=(A, 18).")
+          (list (quote adult) (quote X)))
+        (list {:X (quote alice)} {:X (quote carol)}))
+      (dl-bt-test-set!
+        "less-equal filter"
+        (dl-query
+          (dl-program
+            "n(1). n(2). n(3). n(4). n(5).\n             small(X) :- n(X), <=(X, 3).")
+          (list (quote small) (quote X)))
+        (list {:X 1} {:X 2} {:X 3}))
+      (dl-bt-test-set!
+        "not-equal filter"
+        (dl-query
+          (dl-program
+            "p(1, 2). p(2, 2). p(3, 4).\n             diff(X, Y) :- p(X, Y), !=(X, Y).")
+          (list (quote diff) (quote X) (quote Y)))
+        (list {:X 1 :Y 2} {:X 3 :Y 4}))
+      (dl-bt-test-set!
+        "is plus"
+        (dl-query
+          (dl-program
+            "n(1). n(2). n(3).\n             succ(X, Y) :- n(X), is(Y, +(X, 1)).")
+          (list (quote succ) (quote X) (quote Y)))
+        (list {:X 1 :Y 2} {:X 2 :Y 3} {:X 3 :Y 4}))
+      (dl-bt-test-set!
+        "is multiply"
+        (dl-query
+          (dl-program
+            "n(2). n(3). n(4).\n             square(X, Y) :- n(X), is(Y, *(X, X)).")
+          (list (quote square) (quote X) (quote Y)))
+        (list {:X 2 :Y 4} {:X 3 :Y 9} {:X 4 :Y 16}))
+      (dl-bt-test-set!
+        "is nested expr"
+        (dl-query
+          (dl-program
+            "n(1). n(2). n(3).\n             f(X, Y) :- n(X), is(Y, *(+(X, 1), 2)).")
+          (list (quote f) (quote X) (quote Y)))
+        (list {:X 1 :Y 4} {:X 2 :Y 6} {:X 3 :Y 8}))
+      (dl-bt-test-set!
+        "is bound LHS — equality"
+        (dl-query
+          (dl-program
+            "n(1, 2). n(2, 5). n(3, 4).\n             succ(X, Y) :- n(X, Y), is(Y, +(X, 1)).")
+          (list (quote succ) (quote X) (quote Y)))
+        (list {:X 1 :Y 2} {:X 3 :Y 4}))
+      (dl-bt-test-set!
+        "triple via is"
+        (dl-query
+          (dl-program
+            "n(1). n(2). n(3).\n             triple(X, Y) :- n(X), is(Y, *(X, 3)).")
+          (list (quote triple) (quote X) (quote Y)))
+        (list {:X 1 :Y 3} {:X 2 :Y 6} {:X 3 :Y 9}))
+      (dl-bt-test-set!
+        "= unifies var with constant"
+        (dl-query
+          (dl-program "p(a). p(b).\n             qual(X) :- p(X), =(X, a).")
+          (list (quote qual) (quote X)))
+        (list {:X (quote a)}))
+      (dl-bt-test-set!
+        "= unifies two vars (one bound)"
+        (dl-query
+          (dl-program "p(a). p(b).\n             twin(X, Y) :- p(X), =(Y, X).")
+          (list (quote twin) (quote X) (quote Y)))
+        (list {:X (quote a) :Y (quote a)} {:X (quote b) :Y (quote b)}))
+      (dl-bt-test!
+        "big count"
+        (let
+          ((db (dl-program "n(0). n(1). n(2). n(3). n(4). n(5). n(6). n(7). n(8). n(9).\n                  big(X) :- n(X), >=(X, 5).")))
+          (do (dl-saturate! db) (len (dl-relation db "big"))))
+        5)
+      ;; Built-in / arithmetic literals work as standalone query goals
+      ;; (without needing a wrapper rule).
+      (dl-bt-test-set! "comparison-only goal true"
+        (dl-eval "" "?- <(1, 2).")
+        (list {}))
+
+      (dl-bt-test-set! "comparison-only goal false"
+        (dl-eval "" "?- <(2, 1).")
+        (list))
+
+      (dl-bt-test-set! "is goal binds"
+        (dl-eval "" "?- is(N, +(2, 3)).")
+        (list {:N 5}))
+
+      ;; Bounded successor: a recursive rule with a comparison
+      ;; guard terminates because the Herbrand base is effectively
+      ;; bounded.
+      (dl-bt-test-set! "bounded successor"
+        (dl-query
+          (dl-program
+            "nat(0).
+             nat(Y) :- nat(X), is(Y, +(X, 1)), <(Y, 5).")
+          (list (quote nat) (quote X)))
+        (list {:X 0} {:X 1} {:X 2} {:X 3} {:X 4}))
+
+      (dl-bt-test!
+        "unsafe — comparison without binder"
+        (dl-bt-throws? (fn () (dl-program "p(X) :- <(X, 5).")))
+        true)
+      (dl-bt-test!
+        "unsafe — comparison both unbound"
+        (dl-bt-throws? (fn () (dl-program "p(X, Y) :- <(X, Y), q(X).")))
+        true)
+      (dl-bt-test!
+        "unsafe — is uses unbound RHS var"
+        (dl-bt-throws?
+          (fn () (dl-program "p(X, Y) :- q(X), is(Y, +(X, Z)).")))
+        true)
+      (dl-bt-test!
+        "unsafe — is on its own"
+        (dl-bt-throws? (fn () (dl-program "p(Y) :- is(Y, +(X, 1)).")))
+        true)
+      (dl-bt-test!
+        "unsafe — = between two unbound"
+        (dl-bt-throws? (fn () (dl-program "p(X, Y) :- =(X, Y).")))
+        true)
+      (dl-bt-test!
+        "safe — is binds head var"
+        (dl-bt-throws?
+          (fn () (dl-program "n(1). p(Y) :- n(X), is(Y, +(X, 1)).")))
+        false)
+      (dl-bt-test!
+        "safe — comparison after binder"
+        (dl-bt-throws?
+          (fn () (dl-program "n(1). big(X) :- n(X), >=(X, 0).")))
+        false)
+      (dl-bt-test!
+        "safe — = binds head var"
+        (dl-bt-throws?
+          (fn () (dl-program "p(a). p(b). x(Y) :- p(X), =(Y, X).")))
+        false)
+
+      ;; Division by zero raises with a clear error. Without this guard
+      ;; SX's `/` returned IEEE infinity, which then silently flowed
+      ;; through comparisons and aggregations.
+      (dl-bt-test!
+        "is — division by zero raises"
+        (dl-bt-throws?
+          (fn ()
+            (dl-eval "p(10). q(R) :- p(X), is(R, /(X, 0))." "?- q(R).")))
+        true)
+
+      ;; Comparison ops `<`, `<=`, `>`, `>=` require both operands to
+      ;; have the same primitive type. Cross-type comparisons used to
+      ;; silently return false (for some pairs) or raise a confusing
+      ;; host-level error (for others) — now they all raise with a
+      ;; message that names the offending values.
+      (dl-bt-test!
+        "comparison — string vs number raises"
+        (dl-bt-throws?
+          (fn ()
+            (dl-eval "p(\"hello\"). q(X) :- p(X), <(X, 5)." "?- q(X).")))
+        true)
+
+      ;; `!=` is the exception — it's a polymorphic inequality test
+      ;; (uses dl-tuple-equal? underneath) so cross-type pairs are
+      ;; legitimate (and trivially unequal).
+      (dl-bt-test-set! "!= works across types"
+        (dl-query
+          (dl-program
+            "p(1). p(\"1\"). q(X) :- p(X), !=(X, 1).")
+          (quote (q X)))
+        (list {:X "1"})))))
+
+(define
+  dl-builtins-tests-run!
+  (fn
+    ()
+    (do
+      (set! dl-bt-pass 0)
+      (set! dl-bt-fail 0)
+      (set! dl-bt-failures (list))
+      (dl-bt-run-all!)
+      {:failures dl-bt-failures :total (+ dl-bt-pass dl-bt-fail) :passed dl-bt-pass :failed dl-bt-fail})))

lib/datalog/tests/demo.sx

@@ -0,0 +1,321 @@
+;; lib/datalog/tests/demo.sx — Phase 10 demo programs.
+
+(define dl-demo-pass 0)
+(define dl-demo-fail 0)
+(define dl-demo-failures (list))
+
+(define
+  dl-demo-deep=?
+  (fn
+    (a b)
+    (cond
+      ((and (list? a) (list? b))
+        (and (= (len a) (len b)) (dl-demo-deq-l? a b 0)))
+      ((and (dict? a) (dict? b))
+        (let ((ka (keys a)) (kb (keys b)))
+          (and (= (len ka) (len kb)) (dl-demo-deq-d? a b ka 0))))
+      ((and (number? a) (number? b)) (= a b))
+      (else (equal? a b)))))
+
+(define
+  dl-demo-deq-l?
+  (fn
+    (a b i)
+    (cond
+      ((>= i (len a)) true)
+      ((not (dl-demo-deep=? (nth a i) (nth b i))) false)
+      (else (dl-demo-deq-l? a b (+ i 1))))))
+
+(define
+  dl-demo-deq-d?
+  (fn
+    (a b ka i)
+    (cond
+      ((>= i (len ka)) true)
+      ((let ((k (nth ka i)))
+         (not (dl-demo-deep=? (get a k) (get b k))))
+        false)
+      (else (dl-demo-deq-d? a b ka (+ i 1))))))
+
+(define
+  dl-demo-set=?
+  (fn
+    (a b)
+    (and
+      (= (len a) (len b))
+      (dl-demo-subset? a b)
+      (dl-demo-subset? b a))))
+
+(define
+  dl-demo-subset?
+  (fn
+    (xs ys)
+    (cond
+      ((= (len xs) 0) true)
+      ((not (dl-demo-contains? ys (first xs))) false)
+      (else (dl-demo-subset? (rest xs) ys)))))
+
+(define
+  dl-demo-contains?
+  (fn
+    (xs target)
+    (cond
+      ((= (len xs) 0) false)
+      ((dl-demo-deep=? (first xs) target) true)
+      (else (dl-demo-contains? (rest xs) target)))))
+
+(define
+  dl-demo-test-set!
+  (fn
+    (name got expected)
+    (if
+      (dl-demo-set=? got expected)
+      (set! dl-demo-pass (+ dl-demo-pass 1))
+      (do
+        (set! dl-demo-fail (+ dl-demo-fail 1))
+        (append!
+          dl-demo-failures
+          (str
+            name
+            "\n    expected (set): " expected
+            "\n    got:            " got))))))
+
+(define
+  dl-demo-run-all!
+  (fn
+    ()
+    (do
+      ;; ── Federation ──────────────────────────────────────────
+      (dl-demo-test-set! "mutuals"
+        (dl-query
+          (dl-demo-make
+            (quote ((follows alice bob) (follows bob alice)
+                    (follows bob carol) (follows carol dave)))
+            dl-demo-federation-rules)
+          (quote (mutual alice X)))
+        (list {:X (quote bob)}))
+
+      (dl-demo-test-set! "reachable transitive"
+        (dl-query
+          (dl-demo-make
+            (quote ((follows alice bob) (follows bob carol) (follows carol dave)))
+            dl-demo-federation-rules)
+          (quote (reachable alice X)))
+        (list {:X (quote bob)} {:X (quote carol)} {:X (quote dave)}))
+
+      (dl-demo-test-set! "foaf"
+        (dl-query
+          (dl-demo-make
+            (quote ((follows alice bob) (follows bob carol) (follows alice dave)))
+            dl-demo-federation-rules)
+          (quote (foaf alice X)))
+        (list {:X (quote carol)}))
+
+      ;; ── Content ─────────────────────────────────────────────
+      (dl-demo-test-set! "popular posts"
+        (dl-query
+          (dl-demo-make
+            (quote
+              ((authored alice p1) (authored bob p2) (authored carol p3)
+               (liked u1 p1) (liked u2 p1) (liked u3 p1)
+               (liked u1 p2)))
+            dl-demo-content-rules)
+          (quote (popular P)))
+        (list {:P (quote p1)}))
+
+      (dl-demo-test-set! "interesting feed"
+        (dl-query
+          (dl-demo-make
+            (quote
+              ((follows me alice) (follows me bob)
+               (authored alice p1) (authored bob p2)
+               (liked u1 p1) (liked u2 p1) (liked u3 p1)
+               (liked u4 p2)))
+            dl-demo-content-rules)
+          (quote (interesting me P)))
+        (list {:P (quote p1)}))
+
+      (dl-demo-test-set! "post likes count"
+        (dl-query
+          (dl-demo-make
+            (quote
+              ((authored alice p1)
+               (liked u1 p1) (liked u2 p1) (liked u3 p1)))
+            dl-demo-content-rules)
+          (quote (post-likes p1 N)))
+        (list {:N 3}))
+
+      ;; ── Permissions ─────────────────────────────────────────
+      (dl-demo-test-set! "direct group access"
+        (dl-query
+          (dl-demo-make
+            (quote
+              ((member alice editors)
+               (allowed editors blog)))
+            dl-demo-perm-rules)
+          (quote (can-access X blog)))
+        (list {:X (quote alice)}))
+
+      (dl-demo-test-set! "subgroup access"
+        (dl-query
+          (dl-demo-make
+            (quote
+              ((member bob writers)
+               (subgroup writers editors)
+               (allowed editors blog)))
+            dl-demo-perm-rules)
+          (quote (can-access X blog)))
+        (list {:X (quote bob)}))
+
+      (dl-demo-test-set! "transitive subgroup"
+        (dl-query
+          (dl-demo-make
+            (quote
+              ((member carol drafters)
+               (subgroup drafters writers)
+               (subgroup writers editors)
+               (allowed editors blog)))
+            dl-demo-perm-rules)
+          (quote (can-access X blog)))
+        (list {:X (quote carol)}))
+
+      ;; ── Cooking posts (canonical Phase 10 example) ─────────
+      (dl-demo-test-set! "cooking posts by network"
+        (dl-query
+          (dl-demo-make
+            (quote
+              ((follows me alice) (follows alice bob) (follows alice carol)
+               (authored alice p1) (authored bob p2)
+               (authored carol p3) (authored carol p4)
+               (tagged p1 travel) (tagged p2 cooking)
+               (tagged p3 cooking) (tagged p4 books)))
+            dl-demo-cooking-rules)
+          (quote (cooking-post-by-network me P)))
+        (list {:P (quote p2)} {:P (quote p3)}))
+
+      (dl-demo-test-set! "cooking — direct follow only"
+        (dl-query
+          (dl-demo-make
+            (quote
+              ((follows me bob)
+               (authored bob p1) (authored bob p2)
+               (tagged p1 cooking) (tagged p2 books)))
+            dl-demo-cooking-rules)
+          (quote (cooking-post-by-network me P)))
+        (list {:P (quote p1)}))
+
+      (dl-demo-test-set! "cooking — none in network"
+        (dl-query
+          (dl-demo-make
+            (quote
+              ((follows me bob)
+               (authored bob p1) (tagged p1 books)))
+            dl-demo-cooking-rules)
+          (quote (cooking-post-by-network me P)))
+        (list))
+
+      ;; ── Tag co-occurrence ──────────────────────────────────
+      (dl-demo-test-set! "cotagged posts"
+        (dl-query
+          (dl-demo-make
+            (quote
+              ((tagged p1 cooking) (tagged p1 vegetarian)
+               (tagged p2 cooking) (tagged p2 quick)
+               (tagged p3 vegetarian)))
+            dl-demo-tag-cooccur-rules)
+          (quote (cotagged P cooking vegetarian)))
+        (list {:P (quote p1)}))
+
+      (dl-demo-test-set! "tag pair count"
+        (dl-query
+          (dl-demo-make
+            (quote
+              ((tagged p1 cooking) (tagged p1 vegetarian)
+               (tagged p2 cooking) (tagged p2 quick)
+               (tagged p3 cooking) (tagged p3 vegetarian)))
+            dl-demo-tag-cooccur-rules)
+          (quote (tag-pair-count cooking vegetarian N)))
+        (list {:N 2}))
+
+      ;; ── Shortest path on a weighted DAG ──────────────────
+      (dl-demo-test-set! "shortest a→d via DAG"
+        (dl-query
+          (dl-demo-make
+            (quote ((edge a b 5) (edge b c 3) (edge a c 10) (edge c d 2)))
+            dl-demo-shortest-path-rules)
+          (quote (shortest a d W)))
+        (list {:W 10}))
+
+      (dl-demo-test-set! "shortest a→c picks 2-hop"
+        (dl-query
+          (dl-demo-make
+            (quote ((edge a b 5) (edge b c 3) (edge a c 10)))
+            dl-demo-shortest-path-rules)
+          (quote (shortest a c W)))
+        (list {:W 8}))
+
+      (dl-demo-test-set! "shortest unreachable empty"
+        (dl-query
+          (dl-demo-make
+            (quote ((edge a b 5) (edge b c 3)))
+            dl-demo-shortest-path-rules)
+          (quote (shortest a d W)))
+        (list))
+
+      ;; ── Org chart + headcount ─────────────────────────────
+      (dl-demo-test-set! "ceo subordinate transitive"
+        (dl-query
+          (dl-demo-make
+            (quote
+              ((manager ic1 mgr1) (manager ic2 mgr1)
+               (manager mgr1 vp1) (manager ic3 vp1)
+               (manager vp1 ceo)))
+            dl-demo-org-rules)
+          (quote (subordinate ceo X)))
+        (list
+          {:X (quote vp1)} {:X (quote mgr1)} {:X (quote ic1)}
+          {:X (quote ic2)} {:X (quote ic3)}))
+
+      (dl-demo-test-set! "ceo headcount = 5"
+        (dl-query
+          (dl-demo-make
+            (quote
+              ((manager ic1 mgr1) (manager ic2 mgr1)
+               (manager mgr1 vp1) (manager ic3 vp1)
+               (manager vp1 ceo)))
+            dl-demo-org-rules)
+          (quote (headcount ceo N)))
+        (list {:N 5}))
+
+      (dl-demo-test-set! "mgr1 headcount = 2"
+        (dl-query
+          (dl-demo-make
+            (quote
+              ((manager ic1 mgr1) (manager ic2 mgr1)
+               (manager mgr1 vp1) (manager ic3 vp1)
+               (manager vp1 ceo)))
+            dl-demo-org-rules)
+          (quote (headcount mgr1 N)))
+        (list {:N 2}))
+
+      (dl-demo-test-set! "no access without grant"
+        (dl-query
+          (dl-demo-make
+            (quote ((member dave outsiders) (allowed editors blog)))
+            dl-demo-perm-rules)
+          (quote (can-access X blog)))
+        (list)))))
+
+(define
+  dl-demo-tests-run!
+  (fn
+    ()
+    (do
+      (set! dl-demo-pass 0)
+      (set! dl-demo-fail 0)
+      (set! dl-demo-failures (list))
+      (dl-demo-run-all!)
+      {:passed dl-demo-pass
+       :failed dl-demo-fail
+       :total (+ dl-demo-pass dl-demo-fail)
+       :failures dl-demo-failures})))

lib/datalog/tests/eval.sx

@@ -0,0 +1,463 @@
+;; lib/datalog/tests/eval.sx — naive evaluation + safety analysis tests.
+
+(define dl-et-pass 0)
+(define dl-et-fail 0)
+(define dl-et-failures (list))
+
+;; Same deep-equal helper used in other suites.
+(define
+  dl-et-deep=?
+  (fn
+    (a b)
+    (cond
+      ((and (list? a) (list? b))
+        (and (= (len a) (len b)) (dl-et-deq-l? a b 0)))
+      ((and (dict? a) (dict? b))
+        (let
+          ((ka (keys a)) (kb (keys b)))
+          (and (= (len ka) (len kb)) (dl-et-deq-d? a b ka 0))))
+      ((and (number? a) (number? b)) (= a b))
+      (else (equal? a b)))))
+
+(define
+  dl-et-deq-l?
+  (fn
+    (a b i)
+    (cond
+      ((>= i (len a)) true)
+      ((not (dl-et-deep=? (nth a i) (nth b i))) false)
+      (else (dl-et-deq-l? a b (+ i 1))))))
+
+(define
+  dl-et-deq-d?
+  (fn
+    (a b ka i)
+    (cond
+      ((>= i (len ka)) true)
+      ((let ((k (nth ka i))) (not (dl-et-deep=? (get a k) (get b k))))
+        false)
+      (else (dl-et-deq-d? a b ka (+ i 1))))))
+
+;; Set-equality on lists (order-independent, uses dl-et-deep=?).
+(define
+  dl-et-set=?
+  (fn
+    (a b)
+    (and (= (len a) (len b)) (dl-et-subset? a b) (dl-et-subset? b a))))
+
+(define
+  dl-et-subset?
+  (fn
+    (xs ys)
+    (cond
+      ((= (len xs) 0) true)
+      ((not (dl-et-contains? ys (first xs))) false)
+      (else (dl-et-subset? (rest xs) ys)))))
+
+(define
+  dl-et-contains?
+  (fn
+    (xs target)
+    (cond
+      ((= (len xs) 0) false)
+      ((dl-et-deep=? (first xs) target) true)
+      (else (dl-et-contains? (rest xs) target)))))
+
+(define
+  dl-et-test!
+  (fn
+    (name got expected)
+    (if
+      (dl-et-deep=? got expected)
+      (set! dl-et-pass (+ dl-et-pass 1))
+      (do
+        (set! dl-et-fail (+ dl-et-fail 1))
+        (append!
+          dl-et-failures
+          (str name "\n    expected: " expected "\n    got:      " got))))))
+
+(define
+  dl-et-test-set!
+  (fn
+    (name got expected)
+    (if
+      (dl-et-set=? got expected)
+      (set! dl-et-pass (+ dl-et-pass 1))
+      (do
+        (set! dl-et-fail (+ dl-et-fail 1))
+        (append!
+          dl-et-failures
+          (str
+            name
+            "\n    expected (set): "
+            expected
+            "\n    got:            "
+            got))))))
+
+(define
+  dl-et-throws?
+  (fn
+    (thunk)
+    (let
+      ((threw false))
+      (do (guard (e (#t (set! threw true))) (thunk)) threw))))
+
+(define
+  dl-et-run-all!
+  (fn
+    ()
+    (do
+      (dl-et-test-set!
+        "fact lookup any"
+        (dl-query
+          (dl-program "parent(tom, bob). parent(bob, ann).")
+          (list (quote parent) (quote X) (quote Y)))
+        (list {:X (quote tom) :Y (quote bob)} {:X (quote bob) :Y (quote ann)}))
+      (dl-et-test-set!
+        "fact lookup constant arg"
+        (dl-query
+          (dl-program "parent(tom, bob). parent(tom, liz). parent(bob, ann).")
+          (list (quote parent) (quote tom) (quote Y)))
+        (list {:Y (quote bob)} {:Y (quote liz)}))
+      (dl-et-test-set!
+        "no match"
+        (dl-query
+          (dl-program "parent(tom, bob).")
+          (list (quote parent) (quote nobody) (quote X)))
+        (list))
+      (dl-et-test-set!
+        "ancestor closure"
+        (dl-query
+          (dl-program
+            "parent(tom, bob). parent(bob, ann). parent(ann, pat).\n             ancestor(X, Y) :- parent(X, Y).\n             ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z).")
+          (list (quote ancestor) (quote tom) (quote X)))
+        (list {:X (quote bob)} {:X (quote ann)} {:X (quote pat)}))
+      (dl-et-test-set!
+        "sibling"
+        (dl-query
+          (dl-program
+            "parent(tom, bob). parent(tom, liz). parent(jane, bob). parent(jane, liz).\n             sibling(X, Y) :- parent(P, X), parent(P, Y).")
+          (list (quote sibling) (quote bob) (quote Y)))
+        (list {:Y (quote bob)} {:Y (quote liz)}))
+      (dl-et-test-set!
+        "same-generation"
+        (dl-query
+          (dl-program
+            "parent(tom, bob). parent(tom, liz). parent(bob, ann). parent(liz, joe).\n             person(tom). person(bob). person(liz). person(ann). person(joe).\n             sg(X, X) :- person(X).\n             sg(X, Y) :- parent(P1, X), sg(P1, P2), parent(P2, Y).")
+          (list (quote sg) (quote ann) (quote X)))
+        (list {:X (quote ann)} {:X (quote joe)}))
+      (dl-et-test!
+        "ancestor count"
+        (let
+          ((db (dl-program "parent(a, b). parent(b, c). parent(c, d).\n                  ancestor(X, Y) :- parent(X, Y).\n                  ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z).")))
+          (do (dl-saturate! db) (len (dl-relation db "ancestor"))))
+        6)
+      (dl-et-test-set!
+        "grandparent"
+        (dl-query
+          (dl-program
+            "parent(a, b). parent(b, c). parent(c, d).\n             grandparent(X, Z) :- parent(X, Y), parent(Y, Z).")
+          (list (quote grandparent) (quote X) (quote Y)))
+        (list {:X (quote a) :Y (quote c)} {:X (quote b) :Y (quote d)}))
+      (dl-et-test!
+        "no recursion infinite loop"
+        (let
+          ((db (dl-program "edge(1, 2). edge(2, 3). edge(3, 1).\n                  reach(X, Y) :- edge(X, Y).\n                  reach(X, Z) :- edge(X, Y), reach(Y, Z).")))
+          (do (dl-saturate! db) (len (dl-relation db "reach"))))
+        9)
+      ;; Rule-shape sanity: empty-list head and non-list body raise
+      ;; clear errors rather than crashing inside the saturator.
+      (dl-et-test! "empty head rejected"
+        (dl-et-throws?
+          (fn ()
+            (dl-add-rule! (dl-make-db)
+              {:head (list) :body (list)})))
+        true)
+
+      (dl-et-test! "non-list body rejected"
+        (dl-et-throws?
+          (fn ()
+            (dl-add-rule! (dl-make-db)
+              {:head (list (quote p) (quote X)) :body 42})))
+        true)
+
+      ;; Reserved relation names rejected as rule/fact heads.
+      (dl-et-test!
+        "reserved name `not` as head rejected"
+        (dl-et-throws? (fn () (dl-program "not(X) :- p(X).")))
+        true)
+
+      (dl-et-test!
+        "reserved name `count` as head rejected"
+        (dl-et-throws?
+          (fn () (dl-program "count(N, X, p(X)) :- p(X).")))
+        true)
+
+      (dl-et-test!
+        "reserved name `<` as head rejected"
+        (dl-et-throws? (fn () (dl-program "<(X, 5) :- p(X).")))
+        true)
+
+      (dl-et-test!
+        "reserved name `is` as head rejected"
+        (dl-et-throws? (fn () (dl-program "is(N, +(1, 2)) :- p(N).")))
+        true)
+
+      ;; Body literal with a reserved-name positive head is rejected.
+      ;; The parser only treats outer-level `not(P)` as negation; nested
+      ;; `not(not(P))` would otherwise silently parse as a positive call
+      ;; to a relation named `not` and succeed vacuously. The safety
+      ;; checker now flags this so the user gets a clear error.
+      ;; Body literal with a reserved-name positive head is rejected.
+      ;; The parser only treats outer-level `not(P)` as negation; nested
+      ;; `not(not(P))` would otherwise silently parse as a positive call
+      ;; to a relation named `not` and succeed vacuously — so the safety
+      ;; checker now flags this to give the user a clear error.
+      (dl-et-test!
+        "nested not(not(...)) rejected"
+        (dl-et-throws?
+          (fn ()
+            (dl-program
+              "banned(a). u(a). vip(X) :- u(X), not(not(banned(X))).")))
+        true)
+
+      ;; A dict body literal that isn't `{:neg ...}` is almost always a
+      ;; typo — it would otherwise silently fall through to a confusing
+      ;; head-var-unbound safety error. Now caught with a clear message.
+      (dl-et-test!
+        "dict body lit without :neg rejected"
+        (dl-et-throws?
+          (fn ()
+            (let ((db (dl-make-db)))
+              (dl-add-rule! db
+                {:head (list (quote p) (quote X))
+                 :body (list {:weird "stuff"})}))))
+        true)
+
+      ;; Facts may only have simple-term args (number / string / symbol).
+      ;; A compound arg like `+(1, 2)` would otherwise be silently
+      ;; stored as the unreduced expression `(+ 1 2)` because dl-ground?
+      ;; sees no free variables.
+      (dl-et-test!
+        "compound arg in fact rejected"
+        (dl-et-throws? (fn () (dl-program "p(+(1, 2)).")))
+        true)
+
+      ;; Rule heads may only have variable or constant args — no
+      ;; compounds. Compound heads would be saturated as unreduced
+      ;; tuples rather than the arithmetic result the user expected.
+      (dl-et-test!
+        "compound arg in rule head rejected"
+        (dl-et-throws?
+          (fn () (dl-program "n(3). double(*(X, 2)) :- n(X).")))
+        true)
+
+      ;; The anonymous-variable renamer used to start at `_anon1`
+      ;; unconditionally; a rule that wrote `q(_anon1) :- p(_anon1, _)`
+      ;; (the user picking the same name the renamer would generate)
+      ;; would see the `_` renamed to `_anon1` too, collapsing the
+      ;; two positions in `p(_anon1, _)` to a single var. Now the
+      ;; renamer scans the rule for the max `_anon<N>` and starts past
+      ;; it, so user-written names of that form are preserved.
+      (dl-et-test-set! "anonymous-rename avoids user `_anon` collision"
+        (dl-query
+          (dl-program
+            "p(a, b). p(c, d). q(_anon1) :- p(_anon1, _).")
+          (quote (q X)))
+        (list {:X (quote a)} {:X (quote c)}))
+
+      (dl-et-test!
+        "unsafe head var"
+        (dl-et-throws? (fn () (dl-program "p(X, Y) :- q(X).")))
+        true)
+      (dl-et-test!
+        "unsafe — empty body"
+        (dl-et-throws? (fn () (dl-program "p(X) :- .")))
+        true)
+      ;; Underscore in head is unsafe — it's a fresh existential per
+      ;; occurrence after Phase 5d's anonymous-var renaming, and there's
+      ;; nothing in the body to bind it. (Old behavior accepted this by
+      ;; treating '_' as a literal name to skip; the renaming made it an
+      ;; ordinary unbound variable.)
+      (dl-et-test!
+        "underscore in head — unsafe"
+        (dl-et-throws? (fn () (dl-program "p(X, _) :- q(X).")))
+        true)
+      (dl-et-test!
+        "underscore in body only — safe"
+        (dl-et-throws? (fn () (dl-program "p(X) :- q(X, _).")))
+        false)
+      (dl-et-test!
+        "var only in head — unsafe"
+        (dl-et-throws? (fn () (dl-program "p(X, Y) :- q(Z).")))
+        true)
+      (dl-et-test!
+        "head var bound by body"
+        (dl-et-throws? (fn () (dl-program "p(X) :- q(X).")))
+        false)
+      (dl-et-test!
+        "head subset of body"
+        (dl-et-throws?
+          (fn
+            ()
+            (dl-program
+              "edge(a,b). edge(b,c). reach(X, Z) :- edge(X, Y), edge(Y, Z).")))
+        false)
+
+      ;; Anonymous variables: each occurrence must be independent.
+      (dl-et-test-set! "anon vars in rule are independent"
+        (dl-query
+          (dl-program
+            "p(a, b). p(c, d). q(X) :- p(X, _), p(_, Y).")
+          (list (quote q) (quote X)))
+        (list {:X (quote a)} {:X (quote c)}))
+
+      (dl-et-test-set! "anon vars in goal are independent"
+        (dl-query
+          (dl-program "p(1, 2, 3). p(4, 5, 6).")
+          (list (quote p) (quote _) (quote X) (quote _)))
+        (list {:X 2} {:X 5}))
+
+      ;; dl-summary: relation -> tuple-count for inspection.
+      (dl-et-test! "dl-summary basic"
+        (dl-summary
+          (let
+            ((db (dl-program "p(1). p(2). q(3).")))
+            (do (dl-saturate! db) db)))
+        {:p 2 :q 1})
+
+      (dl-et-test! "dl-summary empty IDB shown"
+        (dl-summary
+          (let
+            ((db (dl-program "r(X) :- s(X).")))
+            (do (dl-saturate! db) db)))
+        {:r 0})
+
+      (dl-et-test! "dl-summary mixed EDB and IDB"
+        (dl-summary
+          (let
+            ((db (dl-program
+                   "parent(a, b).
+                    ancestor(X, Y) :- parent(X, Y).
+                    ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z).")))
+            (do (dl-saturate! db) db)))
+        {:parent 1 :ancestor 1})
+
+      (dl-et-test! "dl-summary empty db"
+        (dl-summary (dl-make-db))
+        {})
+
+      ;; Strategy hook: default semi-naive; :magic accepted but
+      ;; falls back to semi-naive (the transformation itself is
+      ;; deferred — Phase 6 in plan).
+      (dl-et-test! "default strategy"
+        (dl-get-strategy (dl-make-db))
+        :semi-naive)
+
+      (dl-et-test! "set strategy"
+        (let ((db (dl-make-db)))
+          (do (dl-set-strategy! db :magic) (dl-get-strategy db)))
+        :magic)
+
+      ;; Unknown strategy values are rejected so typos don't silently
+      ;; fall back to the default.
+      (dl-et-test!
+        "unknown strategy rejected"
+        (dl-et-throws?
+          (fn ()
+            (let ((db (dl-make-db)))
+              (dl-set-strategy! db :semi_naive))))
+        true)
+
+      ;; dl-saturated?: no-work-left predicate.
+      (dl-et-test! "saturated? after saturation"
+        (let ((db (dl-program
+                    "parent(a, b).
+                     ancestor(X, Y) :- parent(X, Y).")))
+          (do (dl-saturate! db) (dl-saturated? db)))
+        true)
+
+      (dl-et-test! "saturated? before saturation"
+        (let ((db (dl-program
+                    "parent(a, b).
+                     ancestor(X, Y) :- parent(X, Y).")))
+          (dl-saturated? db))
+        false)
+
+      ;; Disjunction via multiple rules — Datalog has no `;` in
+      ;; body, so disjunction is expressed as separate rules with
+      ;; the same head. Here plant_based(X) is satisfied by either
+      ;; vegan(X) or vegetarian(X).
+      (dl-et-test-set! "disjunction via multiple rules"
+        (dl-query
+          (dl-program
+            "vegan(alice). vegetarian(bob). meat_eater(carol).
+             plant_based(X) :- vegan(X).
+             plant_based(X) :- vegetarian(X).")
+          (list (quote plant_based) (quote X)))
+        (list {:X (quote alice)} {:X (quote bob)}))
+
+      ;; Bipartite-style join: pair-of-friends who share a hobby.
+      ;; Three-relation join exercising the planner's join order.
+      (dl-et-test-set! "bipartite friends-with-hobby"
+        (dl-query
+          (dl-program
+            "hobby(alice, climb). hobby(bob, paint).
+             hobby(carol, climb).
+             friend(alice, carol). friend(bob, alice).
+             match(A, B, H) :- friend(A, B), hobby(A, H), hobby(B, H).")
+          (list (quote match) (quote A) (quote B) (quote H)))
+        (list {:A (quote alice) :B (quote carol) :H (quote climb)}))
+
+      ;; Repeated variable (diagonal): p(X, X) only matches tuples
+      ;; whose two args are equal. The unifier handles this via the
+      ;; subst chain — first occurrence binds X, second occurrence
+      ;; checks against the binding.
+      (dl-et-test-set! "diagonal query"
+        (dl-query
+          (dl-program "p(1, 1). p(2, 3). p(4, 4). p(5, 5).")
+          (list (quote p) (quote X) (quote X)))
+        (list {:X 1} {:X 4} {:X 5}))
+
+      ;; A relation can be both EDB-seeded and rule-derived;
+      ;; saturate combines facts + derivations.
+      (dl-et-test-set! "mixed EDB + IDB same relation"
+        (dl-query
+          (dl-program
+            "link(a, b). link(c, d). link(e, c).
+             via(a, e).
+             link(X, Y) :- via(X, M), link(M, Y).")
+          (list (quote link) (quote a) (quote X)))
+        (list {:X (quote b)} {:X (quote c)}))
+
+      (dl-et-test! "saturated? after assert"
+        (let ((db (dl-program
+                    "parent(a, b).
+                     ancestor(X, Y) :- parent(X, Y).")))
+          (do
+            (dl-saturate! db)
+            (dl-add-fact! db (list (quote parent) (quote b) (quote c)))
+            (dl-saturated? db)))
+        false)
+
+      (dl-et-test-set! "magic-set still derives correctly"
+        (let
+          ((db (dl-program
+                 "parent(a, b). parent(b, c).
+                  ancestor(X, Y) :- parent(X, Y).
+                  ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z).")))
+          (do
+            (dl-set-strategy! db :magic)
+            (dl-query db (list (quote ancestor) (quote a) (quote X)))))
+        (list {:X (quote b)} {:X (quote c)})))))
+
+(define
+  dl-eval-tests-run!
+  (fn
+    ()
+    (do
+      (set! dl-et-pass 0)
+      (set! dl-et-fail 0)
+      (set! dl-et-failures (list))
+      (dl-et-run-all!)
+      {:failures dl-et-failures :total (+ dl-et-pass dl-et-fail) :passed dl-et-pass :failed dl-et-fail})))

lib/datalog/tests/magic.sx

@@ -0,0 +1,528 @@
+;; lib/datalog/tests/magic.sx — adornment + SIPS analysis tests.
+
+(define dl-mt-pass 0)
+(define dl-mt-fail 0)
+(define dl-mt-failures (list))
+
+(define
+  dl-mt-deep=?
+  (fn
+    (a b)
+    (cond
+      ((and (list? a) (list? b))
+        (and (= (len a) (len b)) (dl-mt-deq-l? a b 0)))
+      ((and (dict? a) (dict? b))
+        (let ((ka (keys a)) (kb (keys b)))
+          (and (= (len ka) (len kb)) (dl-mt-deq-d? a b ka 0))))
+      ((and (number? a) (number? b)) (= a b))
+      (else (equal? a b)))))
+
+(define
+  dl-mt-deq-l?
+  (fn
+    (a b i)
+    (cond
+      ((>= i (len a)) true)
+      ((not (dl-mt-deep=? (nth a i) (nth b i))) false)
+      (else (dl-mt-deq-l? a b (+ i 1))))))
+
+(define
+  dl-mt-deq-d?
+  (fn
+    (a b ka i)
+    (cond
+      ((>= i (len ka)) true)
+      ((let ((k (nth ka i)))
+         (not (dl-mt-deep=? (get a k) (get b k))))
+        false)
+      (else (dl-mt-deq-d? a b ka (+ i 1))))))
+
+(define
+  dl-mt-test!
+  (fn
+    (name got expected)
+    (if
+      (dl-mt-deep=? got expected)
+      (set! dl-mt-pass (+ dl-mt-pass 1))
+      (do
+        (set! dl-mt-fail (+ dl-mt-fail 1))
+        (append!
+          dl-mt-failures
+          (str
+            name
+            "\n    expected: " expected
+            "\n    got:      " got))))))
+
+(define
+  dl-mt-run-all!
+  (fn
+    ()
+    (do
+      ;; Goal adornment.
+      (dl-mt-test! "adorn 0-ary"
+        (dl-adorn-goal (list (quote ready)))
+        "")
+      (dl-mt-test! "adorn all bound"
+        (dl-adorn-goal (list (quote p) 1 2 3))
+        "bbb")
+      (dl-mt-test! "adorn all free"
+        (dl-adorn-goal (list (quote p) (quote X) (quote Y)))
+        "ff")
+      (dl-mt-test! "adorn mixed"
+        (dl-adorn-goal (list (quote ancestor) (quote tom) (quote X)))
+        "bf")
+      (dl-mt-test! "adorn const var const"
+        (dl-adorn-goal (list (quote p) (quote a) (quote X) (quote b)))
+        "bfb")
+
+      ;; dl-adorn-lit with explicit bound set.
+      (dl-mt-test! "adorn lit with bound"
+        (dl-adorn-lit (list (quote p) (quote X) (quote Y)) (list "X"))
+        "bf")
+
+      ;; Rule SIPS — chain ancestor.
+      (dl-mt-test! "sips chain ancestor bf"
+        (dl-rule-sips
+          {:head (list (quote ancestor) (quote X) (quote Z))
+           :body (list (list (quote parent) (quote X) (quote Y))
+                       (list (quote ancestor) (quote Y) (quote Z)))}
+          "bf")
+        (list
+          {:lit (list (quote parent) (quote X) (quote Y)) :adornment "bf"}
+          {:lit (list (quote ancestor) (quote Y) (quote Z)) :adornment "bf"}))
+
+      ;; SIPS — head fully bound.
+      (dl-mt-test! "sips head bb"
+        (dl-rule-sips
+          {:head (list (quote q) (quote X) (quote Y))
+           :body (list (list (quote p) (quote X) (quote Z))
+                       (list (quote r) (quote Z) (quote Y)))}
+          "bb")
+        (list
+          {:lit (list (quote p) (quote X) (quote Z)) :adornment "bf"}
+          {:lit (list (quote r) (quote Z) (quote Y)) :adornment "bb"}))
+
+      ;; SIPS — comparison; vars must be bound by prior body lit.
+      (dl-mt-test! "sips with comparison"
+        (dl-rule-sips
+          {:head (list (quote q) (quote X))
+           :body (list (list (quote p) (quote X))
+                       (list (string->symbol "<") (quote X) 5))}
+          "f")
+        (list
+          {:lit (list (quote p) (quote X)) :adornment "f"}
+          {:lit (list (string->symbol "<") (quote X) 5) :adornment "bb"}))
+
+      ;; SIPS — `is` binds its left arg.
+      (dl-mt-test! "sips with is"
+        (dl-rule-sips
+          {:head (list (quote q) (quote X) (quote Y))
+           :body (list (list (quote p) (quote X))
+                       (list (quote is) (quote Y) (list (string->symbol "+") (quote X) 1)))}
+          "ff")
+        (list
+          {:lit (list (quote p) (quote X)) :adornment "f"}
+          {:lit (list (quote is) (quote Y)
+                      (list (string->symbol "+") (quote X) 1))
+           :adornment "fb"}))
+
+      ;; Magic predicate naming.
+      (dl-mt-test! "magic-rel-name"
+        (dl-magic-rel-name "ancestor" "bf")
+        "magic_ancestor^bf")
+
+      ;; Bound-args extraction.
+      (dl-mt-test! "bound-args bf"
+        (dl-bound-args (list (quote ancestor) (quote tom) (quote X)) "bf")
+        (list (quote tom)))
+
+      (dl-mt-test! "bound-args mixed"
+        (dl-bound-args (list (quote p) 1 (quote Y) 3) "bfb")
+        (list 1 3))
+
+      (dl-mt-test! "bound-args all-free"
+        (dl-bound-args (list (quote p) (quote X) (quote Y)) "ff")
+        (list))
+
+      ;; Magic literal construction.
+      (dl-mt-test! "magic-lit"
+        (dl-magic-lit "ancestor" "bf" (list (quote tom)))
+        (list (string->symbol "magic_ancestor^bf") (quote tom)))
+
+      ;; Magic-sets rewriter: structural sanity.
+      (dl-mt-test! "rewrite ancestor produces seed"
+        (let
+          ((rules
+             (list
+               {:head (list (quote ancestor) (quote X) (quote Y))
+                :body (list (list (quote parent) (quote X) (quote Y)))}
+               {:head (list (quote ancestor) (quote X) (quote Z))
+                :body
+                (list (list (quote parent) (quote X) (quote Y))
+                      (list (quote ancestor) (quote Y) (quote Z)))})))
+          (get
+            (dl-magic-rewrite rules "ancestor" "bf" (list (quote a)))
+            :seed))
+        (list (string->symbol "magic_ancestor^bf") (quote a)))
+
+      ;; Equivalence: rewritten program derives same ancestor tuples.
+      ;; In a chain a→b→c→d, magic-rewritten run still derives all
+      ;; ancestor pairs reachable from any node a/b/c/d propagated via
+      ;; magic_ancestor^bf — i.e. the full closure (6 tuples). Magic
+      ;; saves work only when the EDB has irrelevant nodes outside
+      ;; the seed's transitive cone.
+      (dl-mt-test! "magic-rewritten ancestor count"
+        (let
+          ((rules
+             (list
+               {:head (list (quote ancestor) (quote X) (quote Y))
+                :body (list (list (quote parent) (quote X) (quote Y)))}
+               {:head (list (quote ancestor) (quote X) (quote Z))
+                :body
+                (list (list (quote parent) (quote X) (quote Y))
+                      (list (quote ancestor) (quote Y) (quote Z)))}))
+            (edb (list
+                   (list (quote parent) (quote a) (quote b))
+                   (list (quote parent) (quote b) (quote c))
+                   (list (quote parent) (quote c) (quote d)))))
+          (let
+            ((rewritten (dl-magic-rewrite rules "ancestor" "bf" (list (quote a))))
+              (db (dl-make-db)))
+            (do
+              (for-each (fn (f) (dl-add-fact! db f)) edb)
+              (dl-add-fact! db (get rewritten :seed))
+              (for-each (fn (r) (dl-add-rule! db r)) (get rewritten :rules))
+              (dl-saturate! db)
+              (len (dl-relation db "ancestor")))))
+        6)
+
+      ;; dl-magic-query: end-to-end driver, doesn't mutate caller's db.
+      ;; Magic over a rule with negated body literal — propagation
+      ;; rules generated only for positive lits; negated lits pass
+      ;; through unchanged.
+      (dl-mt-test! "magic over rule with negation"
+        (let
+          ((db (dl-program
+                 "u(a). u(b). u(c). banned(b).
+                  active(X) :- u(X), not(banned(X)).")))
+          (let
+            ((semi (dl-query db (list (quote active) (quote X))))
+              (magic (dl-magic-query db (list (quote active) (quote X)))))
+            (= (len semi) (len magic))))
+        true)
+
+      ;; All-bound query (existence check) generates an "bb"
+      ;; adornment chain. Verifies the rewriter walks multiple
+      ;; (rel, adn) pairs through the worklist.
+      (dl-mt-test! "magic existence check via bb"
+        (let
+          ((db (dl-program
+                 "parent(a, b). parent(b, c). parent(c, d).
+                  ancestor(X, Y) :- parent(X, Y).
+                  ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z).")))
+          (let
+            ((found (dl-magic-query
+                      db (list (quote ancestor) (quote a) (quote c))))
+              (missing (dl-magic-query
+                         db (list (quote ancestor) (quote a) (quote z)))))
+            (and (= (len found) 1) (= (len missing) 0))))
+        true)
+
+      ;; Magic equivalence on the federation demo.
+      (dl-mt-test! "magic ≡ semi on foaf demo"
+        (let
+          ((db (dl-program-data
+                 (quote ((follows alice bob)
+                         (follows bob carol)
+                         (follows alice dave)))
+                 dl-demo-federation-rules)))
+          (let
+            ((semi (dl-query db (quote (foaf alice X))))
+              (magic (dl-magic-query db (quote (foaf alice X)))))
+            (= (len semi) (len magic))))
+        true)
+
+      ;; Shape validation: dl-magic-query rejects non-list / non-
+      ;; dict goal shapes cleanly rather than crashing in `rest`.
+      (dl-mt-test! "magic rejects string goal"
+        (let ((threw false))
+          (do
+            (guard (e (#t (set! threw true)))
+              (dl-magic-query (dl-make-db) "foo"))
+            threw))
+        true)
+
+      (dl-mt-test! "magic rejects bare dict goal"
+        (let ((threw false))
+          (do
+            (guard (e (#t (set! threw true)))
+              (dl-magic-query (dl-make-db) {:foo "bar"}))
+            threw))
+        true)
+
+      ;; 3-stratum program under magic — distinct rule heads at
+      ;; strata 0/1/2 must all rewrite via the worklist.
+      (dl-mt-test! "magic 3-stratum program"
+        (let
+          ((db (dl-program
+                 "a(1). a(2). a(3). b(2).
+                  c(X) :- a(X), not(b(X)).
+                  d(X) :- c(X), not(banned(X)).
+                  banned(3).")))
+          (let
+            ((semi (dl-query db (list (quote d) (quote X))))
+              (magic (dl-magic-query db (list (quote d) (quote X)))))
+            (= (len semi) (len magic))))
+        true)
+
+      ;; Aggregate -> derived -> threshold chain via magic.
+      (dl-mt-test! "magic aggregate-derived chain"
+        (let
+          ((db (dl-program
+                 "src(1). src(2). src(3).
+                  cnt(N) :- count(N, X, src(X)).
+                  active(N) :- cnt(N), >=(N, 2).")))
+          (let
+            ((semi (dl-query db (list (quote active) (quote N))))
+              (magic (dl-magic-query db (list (quote active) (quote N)))))
+            (= (len semi) (len magic))))
+        true)
+
+      ;; Multi-relation rewrite chain: query r4 → propagate to r3,
+      ;; r2, r1, a. The worklist must process all of them; an
+      ;; earlier bug stopped after only the head pair.
+      (dl-mt-test! "magic chain through 4 rule levels"
+        (let
+          ((db (dl-program
+                 "a(1). a(2). r1(X) :- a(X). r2(X) :- r1(X).
+                  r3(X) :- r2(X). r4(X) :- r3(X).")))
+          (= 2 (len (dl-magic-query db (list (quote r4) (quote X))))))
+        true)
+
+      ;; Shortest-path demo via magic — exercises the rewriter
+      ;; against rules that mix recursive positive lits with an
+      ;; aggregate body literal.
+      (dl-mt-test! "magic on shortest-path demo"
+        (let
+          ((db (dl-program-data
+                 (quote ((edge a b 5) (edge b c 3) (edge a c 10)))
+                 dl-demo-shortest-path-rules)))
+          (let
+            ((semi (dl-query db (quote (shortest a c W))))
+              (magic (dl-magic-query db (quote (shortest a c W)))))
+            (and (= (len semi) (len magic))
+                 (= (len semi) 1))))
+        true)
+
+      ;; Same relation called with different adornment patterns
+      ;; in different rules. The worklist must enqueue and process
+      ;; each (rel, adornment) pair.
+      (dl-mt-test! "magic with multi-adornment same relation"
+        (let
+          ((db (dl-program
+                 "parent(p1, alice). parent(p2, bob).
+                  parent(g, p1). parent(g, p2).
+                  sibling(P1, P2) :- parent(G, P1), parent(G, P2),
+                                     !=(P1, P2).
+                  cousin(X, Y) :- parent(P1, X), parent(P2, Y),
+                                  sibling(P1, P2).")))
+          (let
+            ((semi (dl-query db (list (quote cousin) (quote alice) (quote Y))))
+              (magic (dl-magic-query db (list (quote cousin) (quote alice) (quote Y)))))
+            (= (len semi) (len magic))))
+        true)
+
+      ;; Magic over a rule whose body contains an aggregate.
+      ;; The rewriter passes aggregate body lits through unchanged
+      ;; (no propagation generated for them), so semi-naive's count
+      ;; logic still fires correctly under the rewritten program.
+      (dl-mt-test! "magic over rule with aggregate body"
+        (let
+          ((db (dl-program
+                 "post(p1). post(p2). post(p3).
+                  liked(u1, p1). liked(u2, p1). liked(u3, p1).
+                  liked(u1, p2).
+                  rich(P) :- post(P), count(N, U, liked(U, P)),
+                             >=(N, 2).")))
+          (let
+            ((semi (dl-query db (list (quote rich) (quote P))))
+              (magic (dl-magic-query db (list (quote rich) (quote P)))))
+            (= (len semi) (len magic))))
+        true)
+
+      ;; Mixed EDB + IDB: a relation can be both EDB-seeded and
+      ;; rule-derived. dl-magic-query must include the EDB portion
+      ;; even though the relation has rules.
+      (dl-mt-test! "magic mixed EDB+IDB"
+        (len
+          (dl-magic-query
+            (dl-program
+              "link(a, b). link(c, d). link(e, c).
+               via(a, e).
+               link(X, Y) :- via(X, M), link(M, Y).")
+            (list (quote link) (quote a) (quote X))))
+        2)
+
+      ;; dl-magic-query falls back to dl-query for built-in,
+      ;; aggregate, and negation goals (the magic seed would
+      ;; otherwise be non-ground).
+      (dl-mt-test! "magic-query falls back on aggregate"
+        (let
+          ((r (dl-magic-query
+                (dl-program "p(1). p(2). p(3).")
+                (list (quote count) (quote N) (quote X)
+                      (list (quote p) (quote X))))))
+          (and (= (len r) 1) (= (get (first r) "N") 3)))
+        true)
+
+      (dl-mt-test! "magic-query equivalent to dl-query"
+        (let
+          ((db (dl-program
+                 "parent(a, b). parent(b, c). parent(c, d).
+                  ancestor(X, Y) :- parent(X, Y).
+                  ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z).")))
+          (let
+            ((semi (dl-query db (list (quote ancestor) (quote a) (quote X))))
+              (magic (dl-magic-query
+                       db (list (quote ancestor) (quote a) (quote X)))))
+            (= (len semi) (len magic))))
+        true)
+
+      ;; The magic rewriter passes aggregate body lits through
+      ;; unchanged, so an aggregate over an IDB relation would see an
+      ;; empty inner-goal in the magic db unless the IDB is already
+      ;; materialised. dl-magic-query now pre-saturates the source db
+      ;; to guarantee equivalence with dl-query for every stratified
+      ;; program. Previously this returned `({:N 0})` because `active`
+      ;; (IDB, derived through negation) was never derived in the
+      ;; magic db.
+      (dl-mt-test! "magic over aggregate-of-IDB matches vanilla"
+        (let
+          ((src
+             "u(a). u(b). u(c). u(d). banned(b). banned(d).
+              active(X) :- u(X), not(banned(X)).
+              n(N) :- count(N, X, active(X))."))
+          (let
+            ((vanilla (dl-eval src "?- n(N)."))
+              (magic (dl-eval-magic src "?- n(N).")))
+            (and (= (len vanilla) 1)
+                 (= (len magic) 1)
+                 (= (get (first vanilla) "N")
+                    (get (first magic) "N")))))
+        true)
+
+      ;; magic-query doesn't mutate caller db.
+      (dl-mt-test! "magic-query preserves caller db"
+        (let
+          ((db (dl-program
+                 "parent(a, b). parent(b, c).
+                  ancestor(X, Y) :- parent(X, Y).
+                  ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z).")))
+          (let
+            ((rules-before (len (dl-rules db))))
+            (do
+              (dl-magic-query db (list (quote ancestor) (quote a) (quote X)))
+              (= rules-before (len (dl-rules db))))))
+        true)
+
+      ;; Magic-sets benefit: query touches only one cluster of a
+      ;; multi-component graph. Semi-naive derives the full closure
+      ;; over both clusters; magic only the seeded one.
+      ;; Magic-vs-semi work shape: chain of 12. Semi-naive
+      ;; derives the full closure (78 = 12·13/2). A magic query
+      ;; rooted at node 0 returns the 12 descendants only —
+      ;; demonstrating that magic limits derivation to the
+      ;; query's transitive cone.
+      (dl-mt-test! "magic vs semi work-shape on chain-12"
+        (let
+          ((source (str
+                     "parent(0, 1). parent(1, 2). parent(2, 3). "
+                     "parent(3, 4). parent(4, 5). parent(5, 6). "
+                     "parent(6, 7). parent(7, 8). parent(8, 9). "
+                     "parent(9, 10). parent(10, 11). parent(11, 12). "
+                     "ancestor(X, Y) :- parent(X, Y). "
+                     "ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z).")))
+          (let
+            ((db1 (dl-make-db)) (db2 (dl-make-db)))
+            (do
+              (dl-load-program! db1 source)
+              (dl-saturate! db1)
+              (dl-load-program! db2 source)
+              (let
+                ((semi-count (len (dl-relation db1 "ancestor")))
+                  (magic-count
+                    (len (dl-magic-query
+                           db2 (list (quote ancestor) 0 (quote X))))))
+                ;; Magic returns only descendants of 0 (12 of them).
+                (and (= semi-count 78) (= magic-count 12))))))
+        true)
+
+      ;; Magic + arithmetic: rules with `is` clauses pass through
+      ;; the rewriter unchanged (built-ins aren't propagated).
+      (dl-mt-test! "magic preserves arithmetic"
+        (let
+          ((source "n(1). n(2). n(3).
+                    doubled(X, Y) :- n(X), is(Y, *(X, 2))."))
+          (let
+            ((semi (dl-eval source "?- doubled(2, Y)."))
+              (magic (dl-eval-magic source "?- doubled(2, Y).")))
+            (= (len semi) (len magic))))
+        true)
+
+      (dl-mt-test! "magic skips irrelevant clusters"
+        (let
+          ;; Two disjoint chains. Query is rooted in cluster 1.
+          ((db (dl-program
+                 "parent(a, b). parent(b, c).
+                  parent(x, y). parent(y, z).
+                  ancestor(X, Y) :- parent(X, Y).
+                  ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z).")))
+          (do
+            (dl-saturate! db)
+            (let
+              ((semi-count (len (dl-relation db "ancestor")))
+                (magic-results
+                  (dl-magic-query
+                    db (list (quote ancestor) (quote a) (quote X)))))
+              ;; Semi-naive derives 6 (3 in each cluster). Magic
+              ;; gives 3 query results (a's reachable: b, c).
+              (and (= semi-count 6) (= (len magic-results) 2)))))
+        true)
+
+      (dl-mt-test! "magic-rewritten finds same answers"
+        (let
+          ((rules
+             (list
+               {:head (list (quote ancestor) (quote X) (quote Y))
+                :body (list (list (quote parent) (quote X) (quote Y)))}
+               {:head (list (quote ancestor) (quote X) (quote Z))
+                :body
+                (list (list (quote parent) (quote X) (quote Y))
+                      (list (quote ancestor) (quote Y) (quote Z)))}))
+            (edb (list
+                   (list (quote parent) (quote a) (quote b))
+                   (list (quote parent) (quote b) (quote c)))))
+          (let
+            ((rewritten (dl-magic-rewrite rules "ancestor" "bf" (list (quote a))))
+              (db (dl-make-db)))
+            (do
+              (for-each (fn (f) (dl-add-fact! db f)) edb)
+              (dl-add-fact! db (get rewritten :seed))
+              (for-each (fn (r) (dl-add-rule! db r)) (get rewritten :rules))
+              (dl-saturate! db)
+              (len (dl-query db (list (quote ancestor) (quote a) (quote X)))))))
+        2))))
+
+(define
+  dl-magic-tests-run!
+  (fn
+    ()
+    (do
+      (set! dl-mt-pass 0)
+      (set! dl-mt-fail 0)
+      (set! dl-mt-failures (list))
+      (dl-mt-run-all!)
+      {:passed dl-mt-pass
+       :failed dl-mt-fail
+       :total (+ dl-mt-pass dl-mt-fail)
+       :failures dl-mt-failures})))

lib/datalog/tests/negation.sx

@@ -0,0 +1,252 @@
+;; lib/datalog/tests/negation.sx — stratified negation tests.
+
+(define dl-nt-pass 0)
+(define dl-nt-fail 0)
+(define dl-nt-failures (list))
+
+(define
+  dl-nt-deep=?
+  (fn
+    (a b)
+    (cond
+      ((and (list? a) (list? b))
+        (and (= (len a) (len b)) (dl-nt-deq-l? a b 0)))
+      ((and (dict? a) (dict? b))
+        (let ((ka (keys a)) (kb (keys b)))
+          (and (= (len ka) (len kb)) (dl-nt-deq-d? a b ka 0))))
+      ((and (number? a) (number? b)) (= a b))
+      (else (equal? a b)))))
+
+(define
+  dl-nt-deq-l?
+  (fn
+    (a b i)
+    (cond
+      ((>= i (len a)) true)
+      ((not (dl-nt-deep=? (nth a i) (nth b i))) false)
+      (else (dl-nt-deq-l? a b (+ i 1))))))
+
+(define
+  dl-nt-deq-d?
+  (fn
+    (a b ka i)
+    (cond
+      ((>= i (len ka)) true)
+      ((let ((k (nth ka i)))
+         (not (dl-nt-deep=? (get a k) (get b k))))
+        false)
+      (else (dl-nt-deq-d? a b ka (+ i 1))))))
+
+(define
+  dl-nt-set=?
+  (fn
+    (a b)
+    (and
+      (= (len a) (len b))
+      (dl-nt-subset? a b)
+      (dl-nt-subset? b a))))
+
+(define
+  dl-nt-subset?
+  (fn
+    (xs ys)
+    (cond
+      ((= (len xs) 0) true)
+      ((not (dl-nt-contains? ys (first xs))) false)
+      (else (dl-nt-subset? (rest xs) ys)))))
+
+(define
+  dl-nt-contains?
+  (fn
+    (xs target)
+    (cond
+      ((= (len xs) 0) false)
+      ((dl-nt-deep=? (first xs) target) true)
+      (else (dl-nt-contains? (rest xs) target)))))
+
+(define
+  dl-nt-test!
+  (fn
+    (name got expected)
+    (if
+      (dl-nt-deep=? got expected)
+      (set! dl-nt-pass (+ dl-nt-pass 1))
+      (do
+        (set! dl-nt-fail (+ dl-nt-fail 1))
+        (append!
+          dl-nt-failures
+          (str
+            name
+            "\n    expected: " expected
+            "\n    got:      " got))))))
+
+(define
+  dl-nt-test-set!
+  (fn
+    (name got expected)
+    (if
+      (dl-nt-set=? got expected)
+      (set! dl-nt-pass (+ dl-nt-pass 1))
+      (do
+        (set! dl-nt-fail (+ dl-nt-fail 1))
+        (append!
+          dl-nt-failures
+          (str
+            name
+            "\n    expected (set): " expected
+            "\n    got:            " got))))))
+
+(define
+  dl-nt-throws?
+  (fn
+    (thunk)
+    (let
+      ((threw false))
+      (do
+        (guard
+          (e (#t (set! threw true)))
+          (thunk))
+        threw))))
+
+(define
+  dl-nt-run-all!
+  (fn
+    ()
+    (do
+      ;; Negation against EDB-only relation.
+      (dl-nt-test-set! "not against EDB"
+        (dl-query
+          (dl-program
+            "p(1). p(2). p(3). r(2).
+             q(X) :- p(X), not(r(X)).")
+          (list (quote q) (quote X)))
+        (list {:X 1} {:X 3}))
+
+      ;; Negation against derived relation — needs stratification.
+      (dl-nt-test-set! "not against derived rel"
+        (dl-query
+          (dl-program
+            "p(1). p(2). p(3). s(2).
+             r(X) :- s(X).
+             q(X) :- p(X), not(r(X)).")
+          (list (quote q) (quote X)))
+        (list {:X 1} {:X 3}))
+
+      ;; Two-step strata: r derives via s; q derives via not r.
+      (dl-nt-test-set! "two-step strata"
+        (dl-query
+          (dl-program
+            "node(a). node(b). node(c). node(d).
+             edge(a, b). edge(b, c). edge(c, a).
+             reach(X, Y) :- edge(X, Y).
+             reach(X, Z) :- edge(X, Y), reach(Y, Z).
+             unreachable(X) :- node(X), not(reach(a, X)).")
+          (list (quote unreachable) (quote X)))
+        (list {:X (quote d)}))
+
+      ;; Combine negation with arithmetic and comparison.
+      (dl-nt-test-set! "negation with arithmetic"
+        (dl-query
+          (dl-program
+            "n(1). n(2). n(3). n(4). n(5). odd(1). odd(3). odd(5).
+             even(X) :- n(X), not(odd(X)).")
+          (list (quote even) (quote X)))
+        (list {:X 2} {:X 4}))
+
+      ;; Empty negation result.
+      (dl-nt-test-set! "negation always succeeds"
+        (dl-query
+          (dl-program
+            "p(1). p(2). q(X) :- p(X), not(r(X)).")
+          (list (quote q) (quote X)))
+        (list {:X 1} {:X 2}))
+
+      ;; Negation always fails.
+      (dl-nt-test-set! "negation always fails"
+        (dl-query
+          (dl-program
+            "p(1). p(2). r(1). r(2). q(X) :- p(X), not(r(X)).")
+          (list (quote q) (quote X)))
+        (list))
+
+      ;; Anonymous `_` in a negated literal is existentially quantified
+      ;; — it doesn't need to be bound by an earlier body lit. Without
+      ;; this exemption the safety check would reject the common idiom
+      ;; `orphan(X) :- person(X), not(parent(X, _))`.
+      (dl-nt-test-set! "negation with anonymous var — orphan idiom"
+        (dl-query
+          (dl-program
+            "person(a). person(b). person(c). parent(a, b).
+             orphan(X) :- person(X), not(parent(X, _)).")
+          (list (quote orphan) (quote X)))
+        (list {:X (quote b)} {:X (quote c)}))
+
+      ;; Multiple anonymous vars are each independently existential.
+      (dl-nt-test-set! "negation with multiple anonymous vars"
+        (dl-query
+          (dl-program
+            "u(a). u(b). u(c). edge(a, x). edge(b, y).
+             solo(X) :- u(X), not(edge(X, _)).")
+          (list (quote solo) (quote X)))
+        (list {:X (quote c)}))
+
+      ;; Stratifiability checks.
+      (dl-nt-test! "non-stratifiable rejected"
+        (dl-nt-throws?
+          (fn ()
+            (let ((db (dl-make-db)))
+              (do
+                (dl-add-rule!
+                  db
+                  {:head (list (quote p) (quote X))
+                   :body (list (list (quote q) (quote X))
+                               {:neg (list (quote r) (quote X))})})
+                (dl-add-rule!
+                  db
+                  {:head (list (quote r) (quote X))
+                   :body (list (list (quote p) (quote X)))})
+                (dl-add-fact! db (list (quote q) 1))
+                (dl-saturate! db)))))
+        true)
+
+      (dl-nt-test! "stratifiable accepted"
+        (dl-nt-throws?
+          (fn ()
+            (dl-program
+              "p(1). p(2). r(2).
+               q(X) :- p(X), not(r(X)).")))
+        false)
+
+      ;; Multi-stratum chain.
+      (dl-nt-test-set! "three-level strata"
+        (dl-query
+          (dl-program
+            "a(1). a(2). a(3). a(4).
+             b(X) :- a(X), not(c(X)).
+             c(X) :- d(X).
+             d(2).
+             d(4).")
+          (list (quote b) (quote X)))
+        (list {:X 1} {:X 3}))
+
+      ;; Safety violation: negation refers to unbound var.
+      (dl-nt-test! "negation safety violation"
+        (dl-nt-throws?
+          (fn ()
+            (dl-program
+              "p(1). q(X) :- p(X), not(r(Y)).")))
+        true))))
+
+(define
+  dl-negation-tests-run!
+  (fn
+    ()
+    (do
+      (set! dl-nt-pass 0)
+      (set! dl-nt-fail 0)
+      (set! dl-nt-failures (list))
+      (dl-nt-run-all!)
+      {:passed dl-nt-pass
+       :failed dl-nt-fail
+       :total (+ dl-nt-pass dl-nt-fail)
+       :failures dl-nt-failures})))

lib/datalog/tests/parse.sx

@@ -0,0 +1,179 @@
+;; lib/datalog/tests/parse.sx — parser unit tests
+;;
+;; Run via: bash lib/datalog/conformance.sh
+;; Or:      (load "lib/datalog/tokenizer.sx") (load "lib/datalog/parser.sx")
+;;          (load "lib/datalog/tests/parse.sx") (dl-parse-tests-run!)
+
+(define dl-pt-pass 0)
+(define dl-pt-fail 0)
+(define dl-pt-failures (list))
+
+;; Order-independent structural equality. Lists compared positionally,
+;; dicts as sets of (key, value) pairs. Numbers via = (so 30.0 = 30).
+(define
+  dl-deep-equal?
+  (fn
+    (a b)
+    (cond
+      ((and (list? a) (list? b))
+        (and (= (len a) (len b)) (dl-deep-equal-list? a b 0)))
+      ((and (dict? a) (dict? b))
+        (let
+          ((ka (keys a)) (kb (keys b)))
+          (and
+            (= (len ka) (len kb))
+            (dl-deep-equal-dict? a b ka 0))))
+      ((and (number? a) (number? b)) (= a b))
+      (else (equal? a b)))))
+
+(define
+  dl-deep-equal-list?
+  (fn
+    (a b i)
+    (cond
+      ((>= i (len a)) true)
+      ((not (dl-deep-equal? (nth a i) (nth b i))) false)
+      (else (dl-deep-equal-list? a b (+ i 1))))))
+
+(define
+  dl-deep-equal-dict?
+  (fn
+    (a b ka i)
+    (cond
+      ((>= i (len ka)) true)
+      ((let ((k (nth ka i))) (not (dl-deep-equal? (get a k) (get b k))))
+        false)
+      (else (dl-deep-equal-dict? a b ka (+ i 1))))))
+
+(define
+  dl-pt-test!
+  (fn
+    (name got expected)
+    (if
+      (dl-deep-equal? got expected)
+      (set! dl-pt-pass (+ dl-pt-pass 1))
+      (do
+        (set! dl-pt-fail (+ dl-pt-fail 1))
+        (append!
+          dl-pt-failures
+          (str name "\n    expected: " expected "\n    got:      " got))))))
+
+(define
+  dl-pt-throws?
+  (fn
+    (thunk)
+    (let
+      ((threw false))
+      (do (guard (e (#t (set! threw true))) (thunk)) threw))))
+
+(define
+  dl-pt-run-all!
+  (fn
+    ()
+    (do
+      (dl-pt-test! "empty program" (dl-parse "") (list))
+      (dl-pt-test! "fact" (dl-parse "parent(tom, bob).") (list {:body (list) :head (list (quote parent) (quote tom) (quote bob))}))
+      (dl-pt-test!
+        "two facts"
+        (dl-parse "parent(tom, bob). parent(bob, ann).")
+        (list {:body (list) :head (list (quote parent) (quote tom) (quote bob))} {:body (list) :head (list (quote parent) (quote bob) (quote ann))}))
+      (dl-pt-test! "zero-ary fact" (dl-parse "ready.") (list {:body (list) :head (list (quote ready))}))
+      (dl-pt-test!
+        "rule one body lit"
+        (dl-parse "ancestor(X, Y) :- parent(X, Y).")
+        (list {:body (list (list (quote parent) (quote X) (quote Y))) :head (list (quote ancestor) (quote X) (quote Y))}))
+      (dl-pt-test!
+        "recursive rule"
+        (dl-parse "ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z).")
+        (list {:body (list (list (quote parent) (quote X) (quote Y)) (list (quote ancestor) (quote Y) (quote Z))) :head (list (quote ancestor) (quote X) (quote Z))}))
+      (dl-pt-test!
+        "query single"
+        (dl-parse "?- ancestor(tom, X).")
+        (list {:query (list (list (quote ancestor) (quote tom) (quote X)))}))
+      (dl-pt-test!
+        "query multi"
+        (dl-parse "?- p(X), q(X).")
+        (list {:query (list (list (quote p) (quote X)) (list (quote q) (quote X)))}))
+      (dl-pt-test!
+        "negation"
+        (dl-parse "safe(X) :- person(X), not(parent(X, _)).")
+        (list {:body (list (list (quote person) (quote X)) {:neg (list (quote parent) (quote X) (quote _))}) :head (list (quote safe) (quote X))}))
+      (dl-pt-test!
+        "number arg"
+        (dl-parse "age(alice, 30).")
+        (list {:body (list) :head (list (quote age) (quote alice) 30)}))
+      (dl-pt-test!
+        "string arg"
+        (dl-parse "label(x, \"hi\").")
+        (list {:body (list) :head (list (quote label) (quote x) "hi")}))
+      ;; Quoted 'atoms' parse as strings — a uppercase-starting name
+      ;; in quotes used to misclassify as a variable and reject the
+      ;; fact as non-ground.
+      (dl-pt-test!
+        "quoted atom arg parses as string"
+        (dl-parse "p('Hello World').")
+        (list {:body (list) :head (list (quote p) "Hello World")}))
+      (dl-pt-test!
+        "comparison literal"
+        (dl-parse "p(X) :- <(X, 5).")
+        (list {:body (list (list (string->symbol "<") (quote X) 5)) :head (list (quote p) (quote X))}))
+      (dl-pt-test!
+        "is with arith"
+        (dl-parse "succ(X, Y) :- nat(X), is(Y, +(X, 1)).")
+        (list {:body (list (list (quote nat) (quote X)) (list (quote is) (quote Y) (list (string->symbol "+") (quote X) 1))) :head (list (quote succ) (quote X) (quote Y))}))
+      (dl-pt-test!
+        "mixed program"
+        (dl-parse "p(a). p(b). q(X) :- p(X). ?- q(Y).")
+        (list {:body (list) :head (list (quote p) (quote a))} {:body (list) :head (list (quote p) (quote b))} {:body (list (list (quote p) (quote X))) :head (list (quote q) (quote X))} {:query (list (list (quote q) (quote Y)))}))
+      (dl-pt-test!
+        "comments skipped"
+        (dl-parse "% comment\nfoo(a).\n/* block */ bar(b).")
+        (list {:body (list) :head (list (quote foo) (quote a))} {:body (list) :head (list (quote bar) (quote b))}))
+      (dl-pt-test!
+        "underscore var"
+        (dl-parse "p(X) :- q(X, _).")
+        (list {:body (list (list (quote q) (quote X) (quote _))) :head (list (quote p) (quote X))}))
+      ;; Negative number literals parse as one negative number,
+      ;; while subtraction (`-(X, Y)`) compound is preserved.
+      (dl-pt-test!
+        "negative integer literal"
+        (dl-parse "n(-3).")
+        (list {:head (list (quote n) -3) :body (list)}))
+
+      (dl-pt-test!
+        "subtraction compound preserved"
+        (dl-parse "r(X) :- is(X, -(10, 3)).")
+        (list
+          {:head (list (quote r) (quote X))
+           :body (list (list (quote is) (quote X)
+                             (list (string->symbol "-") 10 3)))}))
+
+      (dl-pt-test!
+        "number as relation name raises"
+        (dl-pt-throws? (fn () (dl-parse "1(X) :- p(X).")))
+        true)
+
+      (dl-pt-test!
+        "var as relation name raises"
+        (dl-pt-throws? (fn () (dl-parse "P(X).")))
+        true)
+
+      (dl-pt-test!
+        "missing dot raises"
+        (dl-pt-throws? (fn () (dl-parse "p(a)")))
+        true)
+      (dl-pt-test!
+        "trailing comma raises"
+        (dl-pt-throws? (fn () (dl-parse "p(a,).")))
+        true))))
+
+(define
+  dl-parse-tests-run!
+  (fn
+    ()
+    (do
+      (set! dl-pt-pass 0)
+      (set! dl-pt-fail 0)
+      (set! dl-pt-failures (list))
+      (dl-pt-run-all!)
+      {:failures dl-pt-failures :total (+ dl-pt-pass dl-pt-fail) :passed dl-pt-pass :failed dl-pt-fail})))

lib/datalog/tests/semi_naive.sx

@@ -0,0 +1,153 @@
+;; lib/datalog/tests/semi_naive.sx — semi-naive correctness vs naive.
+;;
+;; Strategy: differential — run both saturators on each program and
+;; compare the resulting per-relation tuple counts. Counting (not
+;; element-wise set equality) keeps the suite fast under the bundled
+;; conformance session; correctness on the inhabitants is covered by
+;; eval.sx and builtins.sx (which use dl-saturate! by default — the
+;; semi-naive saturator).
+
+(define dl-sn-pass 0)
+(define dl-sn-fail 0)
+(define dl-sn-failures (list))
+
+(define
+  dl-sn-test!
+  (fn
+    (name got expected)
+    (if
+      (equal? got expected)
+      (set! dl-sn-pass (+ dl-sn-pass 1))
+      (do
+        (set! dl-sn-fail (+ dl-sn-fail 1))
+        (append!
+          dl-sn-failures
+          (str name "\n    expected: " expected "\n    got:      " got))))))
+
+;; Load `source` into both a semi-naive and a naive db and return a
+;; list of (rel-name semi-count naive-count) triples. Both sets must
+;; have the same union of relation names.
+(define
+  dl-sn-counts
+  (fn
+    (source)
+    (let
+      ((db-s (dl-program source)) (db-n (dl-program source)))
+      (do
+        (dl-saturate! db-s)
+        (dl-saturate-naive! db-n)
+        (let
+          ((out (list)))
+          (do
+            (for-each
+              (fn
+                (k)
+                (append!
+                  out
+                  (list
+                    k
+                    (len (dl-relation db-s k))
+                    (len (dl-relation db-n k)))))
+              (keys (get db-s :facts)))
+            out))))))
+
+(define
+  dl-sn-counts-agree?
+  (fn
+    (counts)
+    (cond
+      ((= (len counts) 0) true)
+      (else
+        (let
+          ((row (first counts)))
+          (and
+            (= (nth row 1) (nth row 2))
+            (dl-sn-counts-agree? (rest counts))))))))
+
+(define
+  dl-sn-chain-source
+  (fn
+    (n)
+    (let
+      ((parts (list "")))
+      (do
+        (define
+          dl-sn-loop
+          (fn
+            (i)
+            (when
+              (< i n)
+              (do
+                (append! parts (str "parent(" i ", " (+ i 1) "). "))
+                (dl-sn-loop (+ i 1))))))
+        (dl-sn-loop 0)
+        (str
+          (join "" parts)
+          "ancestor(X, Y) :- parent(X, Y). "
+          "ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z).")))))
+
+(define
+  dl-sn-run-all!
+  (fn
+    ()
+    (do
+      (dl-sn-test!
+        "ancestor closure counts match"
+        (dl-sn-counts-agree?
+          (dl-sn-counts
+            "parent(a, b). parent(b, c). parent(c, d).\n             ancestor(X, Y) :- parent(X, Y).\n             ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z)."))
+        true)
+      (dl-sn-test!
+        "cyclic reach counts match"
+        (dl-sn-counts-agree?
+          (dl-sn-counts
+            "edge(1, 2). edge(2, 3). edge(3, 1). edge(3, 4).\n             reach(X, Y) :- edge(X, Y).\n             reach(X, Z) :- edge(X, Y), reach(Y, Z)."))
+        true)
+      (dl-sn-test!
+        "same-gen counts match"
+        (dl-sn-counts-agree?
+          (dl-sn-counts
+            "parent(a, b). parent(a, c). parent(b, d). parent(c, e).\n             person(a). person(b). person(c). person(d). person(e).\n             sg(X, X) :- person(X).\n             sg(X, Y) :- parent(P1, X), sg(P1, P2), parent(P2, Y)."))
+        true)
+      (dl-sn-test!
+        "rules with builtins counts match"
+        (dl-sn-counts-agree?
+          (dl-sn-counts
+            "n(1). n(2). n(3). n(4). n(5).\n             small(X) :- n(X), <(X, 5).\n             succ(X, Y) :- n(X), <(X, 5), is(Y, +(X, 1))."))
+        true)
+      (dl-sn-test!
+        "static rule fires under semi-naive"
+        (dl-sn-counts-agree?
+          (dl-sn-counts "p(a). p(b). q(X) :- p(X), =(X, a)."))
+        true)
+      ;; Chain length 12 — multiple semi-naive iterations against
+      ;; the recursive ancestor rule (differential vs naive).
+      (dl-sn-test!
+        "chain-12 ancestor counts match"
+        (dl-sn-counts-agree? (dl-sn-counts (dl-sn-chain-source 12)))
+        true)
+      ;; Chain length 25 — semi-naive only — first-arg index makes
+      ;; this tractable in conformance budget.
+      (dl-sn-test!
+        "chain-25 ancestor count value (semi only)"
+        (let
+          ((db (dl-program (dl-sn-chain-source 25))))
+          (do (dl-saturate! db) (len (dl-relation db "ancestor"))))
+        325)
+      (dl-sn-test!
+        "query through semi saturate"
+        (let
+          ((db (dl-program "parent(a, b). parent(b, c).\n                  ancestor(X, Y) :- parent(X, Y).\n                  ancestor(X, Z) :- parent(X, Y), ancestor(Y, Z).")))
+          (len (dl-query db (list (quote ancestor) (quote a) (quote X)))))
+        2))))
+
+(define
+  dl-semi-naive-tests-run!
+  (fn
+    ()
+    (do
+      (set! dl-sn-pass 0)
+      (set! dl-sn-fail 0)
+      (set! dl-sn-failures (list))
+      (dl-sn-run-all!)
+      {:failures dl-sn-failures :total (+ dl-sn-pass dl-sn-fail) :passed dl-sn-pass :failed dl-sn-fail})))

lib/datalog/tests/tokenize.sx

@@ -0,0 +1,189 @@
+;; lib/datalog/tests/tokenize.sx — tokenizer unit tests
+;;
+;; Run via: bash lib/datalog/conformance.sh
+;; Or:      (load "lib/datalog/tokenizer.sx") (load "lib/datalog/tests/tokenize.sx")
+;;          (dl-tokenize-tests-run!)
+
+(define dl-tk-pass 0)
+(define dl-tk-fail 0)
+(define dl-tk-failures (list))
+
+(define
+  dl-tk-test!
+  (fn
+    (name got expected)
+    (if
+      (= got expected)
+      (set! dl-tk-pass (+ dl-tk-pass 1))
+      (do
+        (set! dl-tk-fail (+ dl-tk-fail 1))
+        (append!
+          dl-tk-failures
+          (str name "\n    expected: " expected "\n    got:      " got))))))
+
+(define dl-tk-types (fn (toks) (map (fn (t) (get t :type)) toks)))
+(define dl-tk-values (fn (toks) (map (fn (t) (get t :value)) toks)))
+
+(define
+  dl-tk-run-all!
+  (fn
+    ()
+    (do
+      (dl-tk-test! "empty" (dl-tk-types (dl-tokenize "")) (list "eof"))
+      (dl-tk-test!
+        "atom dot"
+        (dl-tk-types (dl-tokenize "foo."))
+        (list "atom" "punct" "eof"))
+      (dl-tk-test!
+        "atom dot value"
+        (dl-tk-values (dl-tokenize "foo."))
+        (list "foo" "." nil))
+      (dl-tk-test!
+        "var"
+        (dl-tk-types (dl-tokenize "X."))
+        (list "var" "punct" "eof"))
+      (dl-tk-test!
+        "underscore var"
+        (dl-tk-types (dl-tokenize "_x."))
+        (list "var" "punct" "eof"))
+      (dl-tk-test!
+        "integer"
+        (dl-tk-values (dl-tokenize "42"))
+        (list 42 nil))
+      (dl-tk-test!
+        "decimal"
+        (dl-tk-values (dl-tokenize "3.14"))
+        (list 3.14 nil))
+      (dl-tk-test!
+        "string"
+        (dl-tk-values (dl-tokenize "\"hello\""))
+        (list "hello" nil))
+      ;; Quoted 'atoms' tokenize as strings — see the type-table
+      ;; comment in lib/datalog/tokenizer.sx for the rationale.
+      (dl-tk-test!
+        "quoted atom as string"
+        (dl-tk-types (dl-tokenize "'two words'"))
+        (list "string" "eof"))
+      (dl-tk-test!
+        "quoted atom value"
+        (dl-tk-values (dl-tokenize "'two words'"))
+        (list "two words" nil))
+      ;; A quoted atom whose name would otherwise be a variable
+      ;; (uppercase / leading underscore) is now safely a string —
+      ;; this was the bug that motivated the type change.
+      (dl-tk-test!
+        "quoted Uppercase as string"
+        (dl-tk-types (dl-tokenize "'Hello'"))
+        (list "string" "eof"))
+      (dl-tk-test! ":-" (dl-tk-values (dl-tokenize ":-")) (list ":-" nil))
+      (dl-tk-test! "?-" (dl-tk-values (dl-tokenize "?-")) (list "?-" nil))
+      (dl-tk-test! "<=" (dl-tk-values (dl-tokenize "<=")) (list "<=" nil))
+      (dl-tk-test! ">=" (dl-tk-values (dl-tokenize ">=")) (list ">=" nil))
+      (dl-tk-test! "!=" (dl-tk-values (dl-tokenize "!=")) (list "!=" nil))
+      (dl-tk-test!
+        "single op values"
+        (dl-tk-values (dl-tokenize "< > = + - * /"))
+        (list "<" ">" "=" "+" "-" "*" "/" nil))
+      (dl-tk-test!
+        "single op types"
+        (dl-tk-types (dl-tokenize "< > = + - * /"))
+        (list "op" "op" "op" "op" "op" "op" "op" "eof"))
+      (dl-tk-test!
+        "punct"
+        (dl-tk-values (dl-tokenize "( ) , ."))
+        (list "(" ")" "," "." nil))
+      (dl-tk-test!
+        "fact tokens"
+        (dl-tk-types (dl-tokenize "parent(tom, bob)."))
+        (list "atom" "punct" "atom" "punct" "atom" "punct" "punct" "eof"))
+      (dl-tk-test!
+        "rule shape"
+        (dl-tk-types (dl-tokenize "p(X) :- q(X)."))
+        (list
+          "atom"
+          "punct"
+          "var"
+          "punct"
+          "op"
+          "atom"
+          "punct"
+          "var"
+          "punct"
+          "punct"
+          "eof"))
+      (dl-tk-test!
+        "comparison literal"
+        (dl-tk-values (dl-tokenize "<(X, 5)"))
+        (list "<" "(" "X" "," 5 ")" nil))
+      (dl-tk-test!
+        "is form"
+        (dl-tk-values (dl-tokenize "is(Y, +(X, 1))"))
+        (list "is" "(" "Y" "," "+" "(" "X" "," 1 ")" ")" nil))
+      (dl-tk-test!
+        "line comment"
+        (dl-tk-types (dl-tokenize "% comment line\nfoo."))
+        (list "atom" "punct" "eof"))
+      (dl-tk-test!
+        "block comment"
+        (dl-tk-types (dl-tokenize "/* a\nb */ x."))
+        (list "atom" "punct" "eof"))
+      ;; Unexpected characters surface at tokenize time rather
+      ;; than being silently consumed (previously `?(X)` parsed as
+      ;; if the leading `?` weren't there).
+      (dl-tk-test!
+        "unexpected char raises"
+        (let ((threw false))
+          (do
+            (guard (e (#t (set! threw true)))
+              (dl-tokenize "?(X)"))
+            threw))
+        true)
+
+      ;; Unterminated string / quoted-atom must raise.
+      (dl-tk-test!
+        "unterminated string raises"
+        (let ((threw false))
+          (do
+            (guard (e (#t (set! threw true)))
+              (dl-tokenize "\"unclosed"))
+            threw))
+        true)
+
+      (dl-tk-test!
+        "unterminated quoted atom raises"
+        (let ((threw false))
+          (do
+            (guard (e (#t (set! threw true)))
+              (dl-tokenize "'unclosed"))
+            threw))
+        true)
+
+      ;; Unterminated block comment must raise — previously it was
+      ;; silently consumed to EOF.
+      (dl-tk-test!
+        "unterminated block comment raises"
+        (let ((threw false))
+          (do
+            (guard (e (#t (set! threw true)))
+              (dl-tokenize "/* unclosed comment"))
+            threw))
+        true)
+      (dl-tk-test!
+        "whitespace"
+        (dl-tk-types (dl-tokenize "  foo  ,\t bar  ."))
+        (list "atom" "punct" "atom" "punct" "eof"))
+      (dl-tk-test!
+        "positions"
+        (map (fn (t) (get t :pos)) (dl-tokenize "foo bar"))
+        (list 0 4 7)))))
+
+(define
+  dl-tokenize-tests-run!
+  (fn
+    ()
+    (do
+      (set! dl-tk-pass 0)
+      (set! dl-tk-fail 0)
+      (set! dl-tk-failures (list))
+      (dl-tk-run-all!)
+      {:failures dl-tk-failures :total (+ dl-tk-pass dl-tk-fail) :passed dl-tk-pass :failed dl-tk-fail})))

lib/datalog/tests/unify.sx

@@ -0,0 +1,194 @@
+;; lib/datalog/tests/unify.sx — unification + substitution tests.
+
+(define dl-ut-pass 0)
+(define dl-ut-fail 0)
+(define dl-ut-failures (list))
+
+(define
+  dl-ut-deep-equal?
+  (fn
+    (a b)
+    (cond
+      ((and (list? a) (list? b))
+        (and (= (len a) (len b)) (dl-ut-deq-list? a b 0)))
+      ((and (dict? a) (dict? b))
+        (let
+          ((ka (keys a)) (kb (keys b)))
+          (and (= (len ka) (len kb)) (dl-ut-deq-dict? a b ka 0))))
+      ((and (number? a) (number? b)) (= a b))
+      (else (equal? a b)))))
+
+(define
+  dl-ut-deq-list?
+  (fn
+    (a b i)
+    (cond
+      ((>= i (len a)) true)
+      ((not (dl-ut-deep-equal? (nth a i) (nth b i))) false)
+      (else (dl-ut-deq-list? a b (+ i 1))))))
+
+(define
+  dl-ut-deq-dict?
+  (fn
+    (a b ka i)
+    (cond
+      ((>= i (len ka)) true)
+      ((let ((k (nth ka i))) (not (dl-ut-deep-equal? (get a k) (get b k))))
+        false)
+      (else (dl-ut-deq-dict? a b ka (+ i 1))))))
+
+(define
+  dl-ut-test!
+  (fn
+    (name got expected)
+    (if
+      (dl-ut-deep-equal? got expected)
+      (set! dl-ut-pass (+ dl-ut-pass 1))
+      (do
+        (set! dl-ut-fail (+ dl-ut-fail 1))
+        (append!
+          dl-ut-failures
+          (str name "\n    expected: " expected "\n    got:      " got))))))
+
+(define
+  dl-ut-run-all!
+  (fn
+    ()
+    (do
+      (dl-ut-test! "var? uppercase" (dl-var? (quote X)) true)
+      (dl-ut-test! "var? underscore" (dl-var? (quote _foo)) true)
+      (dl-ut-test! "var? lowercase" (dl-var? (quote tom)) false)
+      (dl-ut-test! "var? number" (dl-var? 5) false)
+      (dl-ut-test! "var? string" (dl-var? "hi") false)
+      (dl-ut-test! "var? list" (dl-var? (list 1)) false)
+      (dl-ut-test!
+        "atom-atom match"
+        (dl-unify (quote tom) (quote tom) (dl-empty-subst))
+        {})
+      (dl-ut-test!
+        "atom-atom fail"
+        (dl-unify (quote tom) (quote bob) (dl-empty-subst))
+        nil)
+      (dl-ut-test!
+        "num-num match"
+        (dl-unify 5 5 (dl-empty-subst))
+        {})
+      (dl-ut-test!
+        "num-num fail"
+        (dl-unify 5 6 (dl-empty-subst))
+        nil)
+      (dl-ut-test!
+        "string match"
+        (dl-unify "hi" "hi" (dl-empty-subst))
+        {})
+      (dl-ut-test! "string fail" (dl-unify "hi" "bye" (dl-empty-subst)) nil)
+      (dl-ut-test!
+        "var-atom binds"
+        (dl-unify (quote X) (quote tom) (dl-empty-subst))
+        {:X (quote tom)})
+      (dl-ut-test!
+        "atom-var binds"
+        (dl-unify (quote tom) (quote X) (dl-empty-subst))
+        {:X (quote tom)})
+      (dl-ut-test!
+        "var-var same"
+        (dl-unify (quote X) (quote X) (dl-empty-subst))
+        {})
+      (dl-ut-test!
+        "var-var bind"
+        (let
+          ((s (dl-unify (quote X) (quote Y) (dl-empty-subst))))
+          (dl-walk (quote X) s))
+        (quote Y))
+      (dl-ut-test!
+        "tuple match"
+        (dl-unify
+          (list (quote parent) (quote X) (quote bob))
+          (list (quote parent) (quote tom) (quote Y))
+          (dl-empty-subst))
+        {:X (quote tom) :Y (quote bob)})
+      (dl-ut-test!
+        "tuple arity mismatch"
+        (dl-unify
+          (list (quote p) (quote X))
+          (list (quote p) (quote a) (quote b))
+          (dl-empty-subst))
+        nil)
+      (dl-ut-test!
+        "tuple head mismatch"
+        (dl-unify
+          (list (quote p) (quote X))
+          (list (quote q) (quote X))
+          (dl-empty-subst))
+        nil)
+      (dl-ut-test!
+        "walk chain"
+        (let
+          ((s1 (dl-unify (quote X) (quote Y) (dl-empty-subst))))
+          (let
+            ((s2 (dl-unify (quote Y) (quote tom) s1)))
+            (dl-walk (quote X) s2)))
+        (quote tom))
+
+      ;; Walk with circular substitution must not infinite-loop.
+      ;; Cycles return the current term unchanged.
+      (dl-ut-test!
+        "walk circular subst no hang"
+        (let ((s (dl-bind (quote B) (quote A)
+                   (dl-bind (quote A) (quote B) (dl-empty-subst)))))
+          (dl-walk (quote A) s))
+        (quote A))
+      (dl-ut-test!
+        "apply subst on tuple"
+        (let
+          ((s (dl-bind (quote X) (quote tom) (dl-empty-subst))))
+          (dl-apply-subst (list (quote parent) (quote X) (quote Y)) s))
+        (list (quote parent) (quote tom) (quote Y)))
+      (dl-ut-test!
+        "ground? all const"
+        (dl-ground?
+          (list (quote p) (quote tom) 5)
+          (dl-empty-subst))
+        true)
+      (dl-ut-test!
+        "ground? unbound var"
+        (dl-ground? (list (quote p) (quote X)) (dl-empty-subst))
+        false)
+      (dl-ut-test!
+        "ground? bound var"
+        (let
+          ((s (dl-bind (quote X) (quote tom) (dl-empty-subst))))
+          (dl-ground? (list (quote p) (quote X)) s))
+        true)
+      (dl-ut-test!
+        "ground? bare var"
+        (dl-ground? (quote X) (dl-empty-subst))
+        false)
+      (dl-ut-test!
+        "vars-of basic"
+        (dl-vars-of
+          (list (quote p) (quote X) (quote tom) (quote Y) (quote X)))
+        (list "X" "Y"))
+      (dl-ut-test!
+        "vars-of ground"
+        (dl-vars-of (list (quote p) (quote tom) (quote bob)))
+        (list))
+      (dl-ut-test!
+        "vars-of nested compound"
+        (dl-vars-of
+          (list
+            (quote is)
+            (quote Z)
+            (list (string->symbol "+") (quote X) 1)))
+        (list "Z" "X")))))
+
+(define
+  dl-unify-tests-run!
+  (fn
+    ()
+    (do
+      (set! dl-ut-pass 0)
+      (set! dl-ut-fail 0)
+      (set! dl-ut-failures (list))
+      (dl-ut-run-all!)
+      {:failures dl-ut-failures :total (+ dl-ut-pass dl-ut-fail) :passed dl-ut-pass :failed dl-ut-fail})))

lib/datalog/tokenizer.sx

@@ -0,0 +1,269 @@
+;; lib/datalog/tokenizer.sx — Datalog source → token stream
+;;
+;; Tokens: {:type T :value V :pos P}
+;; Types:
+;;   "atom"    — lowercase-start bare identifier
+;;   "var"     — uppercase-start or _-start ident (value is the name)
+;;   "number"  — numeric literal (decoded to number)
+;;   "string"  — "..." string literal OR quoted 'atom' (treated as a
+;;               string value to avoid the var-vs-atom ambiguity that
+;;               would arise from a quoted atom whose name starts with
+;;               an uppercase letter or underscore)
+;;   "punct"   — ( ) , .
+;;   "op"      — :- ?- <= >= != < > = + - * /
+;;   "eof"
+;;
+;; Datalog has no function symbols in arg position; the parser still
+;; accepts nested compounds for arithmetic ((is X (+ A B))) but safety
+;; analysis rejects non-arithmetic nesting at rule-load time.
+
+(define dl-make-token (fn (type value pos) {:type type :value value :pos pos}))
+
+(define dl-digit? (fn (c) (and (>= c "0") (<= c "9"))))
+(define dl-lower? (fn (c) (and (>= c "a") (<= c "z"))))
+(define dl-upper? (fn (c) (and (>= c "A") (<= c "Z"))))
+
+(define
+  dl-ident-char?
+  (fn (c) (or (dl-lower? c) (dl-upper? c) (dl-digit? c) (= c "_"))))
+
+(define dl-ws? (fn (c) (or (= c " ") (= c "\t") (= c "\n") (= c "\r"))))
+
+(define
+  dl-tokenize
+  (fn
+    (src)
+    (let
+      ((tokens (list)) (pos 0) (src-len (len src)))
+      (define
+        dl-peek
+        (fn
+          (offset)
+          (if (< (+ pos offset) src-len) (nth src (+ pos offset)) nil)))
+      (define cur (fn () (dl-peek 0)))
+      (define advance! (fn (n) (set! pos (+ pos n))))
+      (define
+        at?
+        (fn
+          (s)
+          (let
+            ((sl (len s)))
+            (and (<= (+ pos sl) src-len) (= (slice src pos (+ pos sl)) s)))))
+      (define
+        dl-emit!
+        (fn
+          (type value start)
+          (append! tokens (dl-make-token type value start))))
+      (define
+        skip-line-comment!
+        (fn
+          ()
+          (when
+            (and (< pos src-len) (not (= (cur) "\n")))
+            (do (advance! 1) (skip-line-comment!)))))
+      (define
+        skip-block-comment!
+        (fn
+          ()
+          (cond
+            ((>= pos src-len)
+              (error (str "Tokenizer: unterminated block comment "
+                          "(started at position " pos ")")))
+            ((and (= (cur) "*") (< (+ pos 1) src-len) (= (dl-peek 1) "/"))
+              (advance! 2))
+            (else (do (advance! 1) (skip-block-comment!))))))
+      (define
+        skip-ws!
+        (fn
+          ()
+          (cond
+            ((>= pos src-len) nil)
+            ((dl-ws? (cur)) (do (advance! 1) (skip-ws!)))
+            ((= (cur) "%")
+              (do (advance! 1) (skip-line-comment!) (skip-ws!)))
+            ((and (= (cur) "/") (< (+ pos 1) src-len) (= (dl-peek 1) "*"))
+              (do (advance! 2) (skip-block-comment!) (skip-ws!)))
+            (else nil))))
+      (define
+        read-ident
+        (fn
+          (start)
+          (do
+            (when
+              (and (< pos src-len) (dl-ident-char? (cur)))
+              (do (advance! 1) (read-ident start)))
+            (slice src start pos))))
+      (define
+        read-decimal-digits!
+        (fn
+          ()
+          (when
+            (and (< pos src-len) (dl-digit? (cur)))
+            (do (advance! 1) (read-decimal-digits!)))))
+      (define
+        read-number
+        (fn
+          (start)
+          (do
+            (read-decimal-digits!)
+            (when
+              (and
+                (< pos src-len)
+                (= (cur) ".")
+                (< (+ pos 1) src-len)
+                (dl-digit? (dl-peek 1)))
+              (do (advance! 1) (read-decimal-digits!)))
+            (parse-number (slice src start pos)))))
+      (define
+        read-quoted
+        (fn
+          (quote-char)
+          (let
+            ((chars (list)))
+            (advance! 1)
+            (define
+              loop
+              (fn
+                ()
+                (cond
+                  ((>= pos src-len)
+                    (error
+                      (str "Tokenizer: unterminated "
+                           (if (= quote-char "'") "quoted atom" "string")
+                           " (started near position " pos ")")))
+                  ((= (cur) "\\")
+                    (do
+                      (advance! 1)
+                      (when
+                        (< pos src-len)
+                        (let
+                          ((ch (cur)))
+                          (do
+                            (cond
+                              ((= ch "n") (append! chars "\n"))
+                              ((= ch "t") (append! chars "\t"))
+                              ((= ch "r") (append! chars "\r"))
+                              ((= ch "\\") (append! chars "\\"))
+                              ((= ch "'") (append! chars "'"))
+                              ((= ch "\"") (append! chars "\""))
+                              (else (append! chars ch)))
+                            (advance! 1))))
+                      (loop)))
+                  ((= (cur) quote-char) (advance! 1))
+                  (else
+                    (do (append! chars (cur)) (advance! 1) (loop))))))
+            (loop)
+            (join "" chars))))
+      (define
+        scan!
+        (fn
+          ()
+          (do
+            (skip-ws!)
+            (when
+              (< pos src-len)
+              (let
+                ((ch (cur)) (start pos))
+                (cond
+                  ((at? ":-")
+                    (do
+                      (dl-emit! "op" ":-" start)
+                      (advance! 2)
+                      (scan!)))
+                  ((at? "?-")
+                    (do
+                      (dl-emit! "op" "?-" start)
+                      (advance! 2)
+                      (scan!)))
+                  ((at? "<=")
+                    (do
+                      (dl-emit! "op" "<=" start)
+                      (advance! 2)
+                      (scan!)))
+                  ((at? ">=")
+                    (do
+                      (dl-emit! "op" ">=" start)
+                      (advance! 2)
+                      (scan!)))
+                  ((at? "!=")
+                    (do
+                      (dl-emit! "op" "!=" start)
+                      (advance! 2)
+                      (scan!)))
+                  ((dl-digit? ch)
+                    (do
+                      (dl-emit! "number" (read-number start) start)
+                      (scan!)))
+                  ((= ch "'")
+                    ;; Quoted 'atoms' tokenize as strings so a name
+                    ;; like 'Hello World' doesn't get misclassified
+                    ;; as a variable by dl-var? (which inspects the
+                    ;; symbol's first character).
+                    (do (dl-emit! "string" (read-quoted "'") start) (scan!)))
+                  ((= ch "\"")
+                    (do (dl-emit! "string" (read-quoted "\"") start) (scan!)))
+                  ((dl-lower? ch)
+                    (do (dl-emit! "atom" (read-ident start) start) (scan!)))
+                  ((or (dl-upper? ch) (= ch "_"))
+                    (do (dl-emit! "var" (read-ident start) start) (scan!)))
+                  ((= ch "(")
+                    (do
+                      (dl-emit! "punct" "(" start)
+                      (advance! 1)
+                      (scan!)))
+                  ((= ch ")")
+                    (do
+                      (dl-emit! "punct" ")" start)
+                      (advance! 1)
+                      (scan!)))
+                  ((= ch ",")
+                    (do
+                      (dl-emit! "punct" "," start)
+                      (advance! 1)
+                      (scan!)))
+                  ((= ch ".")
+                    (do
+                      (dl-emit! "punct" "." start)
+                      (advance! 1)
+                      (scan!)))
+                  ((= ch "<")
+                    (do
+                      (dl-emit! "op" "<" start)
+                      (advance! 1)
+                      (scan!)))
+                  ((= ch ">")
+                    (do
+                      (dl-emit! "op" ">" start)
+                      (advance! 1)
+                      (scan!)))
+                  ((= ch "=")
+                    (do
+                      (dl-emit! "op" "=" start)
+                      (advance! 1)
+                      (scan!)))
+                  ((= ch "+")
+                    (do
+                      (dl-emit! "op" "+" start)
+                      (advance! 1)
+                      (scan!)))
+                  ((= ch "-")
+                    (do
+                      (dl-emit! "op" "-" start)
+                      (advance! 1)
+                      (scan!)))
+                  ((= ch "*")
+                    (do
+                      (dl-emit! "op" "*" start)
+                      (advance! 1)
+                      (scan!)))
+                  ((= ch "/")
+                    (do
+                      (dl-emit! "op" "/" start)
+                      (advance! 1)
+                      (scan!)))
+                  (else (error
+                          (str "Tokenizer: unexpected character '" ch
+                               "' at position " start)))))))))
+      (scan!)
+      (dl-emit! "eof" nil pos)
+      tokens)))

lib/datalog/unify.sx

@@ -0,0 +1,171 @@
+;; lib/datalog/unify.sx — unification + substitution for Datalog terms.
+;;
+;; Term taxonomy (after parsing):
+;;   variable  — SX symbol whose first char is uppercase A–Z or '_'.
+;;   constant  — SX symbol whose first char is lowercase a–z (atom name).
+;;   number    — numeric literal.
+;;   string    — string literal.
+;;   compound  — SX list (functor arg ... arg). In core Datalog these
+;;               only appear as arithmetic expressions (see Phase 4
+;;               safety analysis); compound-against-compound unification
+;;               is supported anyway for completeness.
+;;
+;; Substitutions are immutable dicts keyed by variable name (string).
+;; A failed unification returns nil; success returns the extended subst.
+
+(define dl-empty-subst (fn () {}))
+
+(define
+  dl-var?
+  (fn
+    (term)
+    (and
+      (symbol? term)
+      (let
+        ((name (symbol->string term)))
+        (and
+          (> (len name) 0)
+          (let
+            ((c (slice name 0 1)))
+            (or (and (>= c "A") (<= c "Z")) (= c "_"))))))))
+
+;; Walk: chase variable bindings until we hit a non-variable or an unbound
+;; variable. The result is either a non-variable term or an unbound var.
+(define
+  dl-walk
+  (fn (term subst) (dl-walk-aux term subst (list))))
+
+;; Internal: walk with a visited-var set so circular substitutions
+;; (from raw dl-bind misuse) don't infinite-loop. Cycles return the
+;; current term unchanged.
+(define
+  dl-walk-aux
+  (fn
+    (term subst visited)
+    (if
+      (dl-var? term)
+      (let
+        ((name (symbol->string term)))
+        (cond
+          ((dl-member? name visited) term)
+          ((and (dict? subst) (has-key? subst name))
+            (let ((seen (list)))
+              (do
+                (for-each (fn (v) (append! seen v)) visited)
+                (append! seen name)
+                (dl-walk-aux (get subst name) subst seen))))
+          (else term)))
+      term)))
+
+;; Bind a variable symbol to a value in subst, returning a new subst.
+(define
+  dl-bind
+  (fn (var-sym value subst) (assoc subst (symbol->string var-sym) value)))
+
+(define
+  dl-unify
+  (fn
+    (t1 t2 subst)
+    (if
+      (nil? subst)
+      nil
+      (let
+        ((u1 (dl-walk t1 subst)) (u2 (dl-walk t2 subst)))
+        (cond
+          ((dl-var? u1)
+            (cond
+              ((and (dl-var? u2) (= (symbol->string u1) (symbol->string u2)))
+                subst)
+              (else (dl-bind u1 u2 subst))))
+          ((dl-var? u2) (dl-bind u2 u1 subst))
+          ((and (list? u1) (list? u2))
+            (if
+              (= (len u1) (len u2))
+              (dl-unify-list u1 u2 subst 0)
+              nil))
+          ((and (number? u1) (number? u2)) (if (= u1 u2) subst nil))
+          ((and (string? u1) (string? u2)) (if (= u1 u2) subst nil))
+          ((and (symbol? u1) (symbol? u2))
+            (if (= (symbol->string u1) (symbol->string u2)) subst nil))
+          (else nil))))))
+
+(define
+  dl-unify-list
+  (fn
+    (a b subst i)
+    (cond
+      ((nil? subst) nil)
+      ((>= i (len a)) subst)
+      (else
+        (dl-unify-list
+          a
+          b
+          (dl-unify (nth a i) (nth b i) subst)
+          (+ i 1))))))
+
+;; Apply substitution: walk the term and recurse into lists.
+(define
+  dl-apply-subst
+  (fn
+    (term subst)
+    (let
+      ((w (dl-walk term subst)))
+      (if (list? w) (map (fn (x) (dl-apply-subst x subst)) w) w))))
+
+;; Ground? — true iff no free variables remain after walking.
+(define
+  dl-ground?
+  (fn
+    (term subst)
+    (let
+      ((w (dl-walk term subst)))
+      (cond
+        ((dl-var? w) false)
+        ((list? w) (dl-ground-list? w subst 0))
+        (else true)))))
+
+(define
+  dl-ground-list?
+  (fn
+    (xs subst i)
+    (cond
+      ((>= i (len xs)) true)
+      ((not (dl-ground? (nth xs i) subst)) false)
+      (else (dl-ground-list? xs subst (+ i 1))))))
+
+;; Return the list of variable names appearing in a term (deduped, in
+;; left-to-right order). Useful for safety analysis later.
+(define
+  dl-vars-of
+  (fn (term) (let ((seen (list))) (do (dl-vars-of-aux term seen) seen))))
+
+(define
+  dl-vars-of-aux
+  (fn
+    (term acc)
+    (cond
+      ((dl-var? term)
+        (let
+          ((name (symbol->string term)))
+          (when (not (dl-member? name acc)) (append! acc name))))
+      ((list? term) (dl-vars-of-list term acc 0))
+      (else nil))))
+
+(define
+  dl-vars-of-list
+  (fn
+    (xs acc i)
+    (when
+      (< i (len xs))
+      (do
+        (dl-vars-of-aux (nth xs i) acc)
+        (dl-vars-of-list xs acc (+ i 1))))))
+
+(define
+  dl-member?
+  (fn
+    (x xs)
+    (cond
+      ((= (len xs) 0) false)
+      ((= (first xs) x) true)
+      (else (dl-member? x (rest xs))))))

lib/erlang/conformance.sh

@@ -76,7 +76,7 @@ cat > "$TMPFILE" << 'EPOCHS'
 (eval "(list er-fib-test-pass er-fib-test-count)")
 EPOCHS
 
-timeout 120 "$SX_SERVER" < "$TMPFILE" > "$OUTFILE" 2>&1
+timeout 600 "$SX_SERVER" < "$TMPFILE" > "$OUTFILE" 2>&1
 
 # Parse "(N M)" from the line after each "(ok-len <epoch> ...)" marker.
 parse_pair() {

lib/erlang/scoreboard.json

@@ -1,16 +1,16 @@
 {
   "language": "erlang",
-  "total_pass": 0,
-  "total": 0,
+  "total_pass": 530,
+  "total": 530,
   "suites": [
-    {"name":"tokenize","pass":0,"total":0,"status":"ok"},
-    {"name":"parse","pass":0,"total":0,"status":"ok"},
-    {"name":"eval","pass":0,"total":0,"status":"ok"},
-    {"name":"runtime","pass":0,"total":0,"status":"ok"},
-    {"name":"ring","pass":0,"total":0,"status":"ok"},
-    {"name":"ping-pong","pass":0,"total":0,"status":"ok"},
-    {"name":"bank","pass":0,"total":0,"status":"ok"},
-    {"name":"echo","pass":0,"total":0,"status":"ok"},
-    {"name":"fib","pass":0,"total":0,"status":"ok"}
+    {"name":"tokenize","pass":62,"total":62,"status":"ok"},
+    {"name":"parse","pass":52,"total":52,"status":"ok"},
+    {"name":"eval","pass":346,"total":346,"status":"ok"},
+    {"name":"runtime","pass":39,"total":39,"status":"ok"},
+    {"name":"ring","pass":4,"total":4,"status":"ok"},
+    {"name":"ping-pong","pass":4,"total":4,"status":"ok"},
+    {"name":"bank","pass":8,"total":8,"status":"ok"},
+    {"name":"echo","pass":7,"total":7,"status":"ok"},
+    {"name":"fib","pass":8,"total":8,"status":"ok"}
   ]
 }

lib/erlang/scoreboard.md

@@ -1,18 +1,18 @@
 # Erlang-on-SX Scoreboard
 
-**Total: 0 / 0 tests passing**
+**Total: 530 / 530 tests passing**
 
 |  | Suite | Pass | Total |
 |---|---|---|---|
-| ✅ | tokenize | 0 | 0 |
-| ✅ | parse | 0 | 0 |
-| ✅ | eval | 0 | 0 |
-| ✅ | runtime | 0 | 0 |
-| ✅ | ring | 0 | 0 |
-| ✅ | ping-pong | 0 | 0 |
-| ✅ | bank | 0 | 0 |
-| ✅ | echo | 0 | 0 |
-| ✅ | fib | 0 | 0 |
+| ✅ | tokenize | 62 | 62 |
+| ✅ | parse | 52 | 52 |
+| ✅ | eval | 346 | 346 |
+| ✅ | runtime | 39 | 39 |
+| ✅ | ring | 4 | 4 |
+| ✅ | ping-pong | 4 | 4 |
+| ✅ | bank | 8 | 8 |
+| ✅ | echo | 7 | 7 |
+| ✅ | fib | 8 | 8 |
 
 
 Generated by `lib/erlang/conformance.sh`.

lib/forth/ans-tests/README.md

@@ -0,0 +1,14 @@
+ANS Forth conformance tests — vendored from
+https://github.com/gerryjackson/forth2012-test-suite (master, commit-locked
+on first fetch: 2026-04-24).
+
+Files in this directory are pristine copies of upstream — do not edit them.
+They are consumed by the conformance runner in `lib/forth/conformance.sh`.
+
+- `tester.fr` — John Hayes' test harness (`T{ ... -> ... }T`). (C) 1995
+  Johns Hopkins APL, distributable under its notice.
+- `core.fr` — Core word set tests (Hayes, ~1000 lines).
+- `coreexttest.fth` — Core Extension tests (Gerry Jackson).
+
+Only `core.fr` is expected to run green end-to-end for Phase 3; the others
+stay parked until later phases.

lib/forth/ans-tests/coreexttest.fth

@@ -0,0 +1,775 @@
+\ To test the ANS Forth Core Extension word set
+
+\ This program was written by Gerry Jackson in 2006, with contributions from
+\ others where indicated, and is in the public domain - it can be distributed
+\ and/or modified in any way but please retain this notice.
+
+\ This program is distributed in the hope that it will be useful,
+\ but WITHOUT ANY WARRANTY; without even the implied warranty of
+\ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+
+\ The tests are not claimed to be comprehensive or correct 
+
+\ ------------------------------------------------------------------------------
+\ Version 0.15 1 August 2025 Added two tests to VALUE
+\         0.14 21 July 2022 Updated first line of BUFFER: test as recommended
+\              in issue 32
+\         0.13 28 October 2015
+\              Replace <FALSE> and <TRUE> with FALSE and TRUE to avoid
+\              dependence on Core tests
+\              Moved SAVE-INPUT and RESTORE-INPUT tests in a file to filetest.fth
+\              Use of 2VARIABLE (from optional wordset) replaced with CREATE.
+\              Minor lower to upper case conversions.
+\              Calls to COMPARE replaced by S= (in utilities.fth) to avoid use
+\              of a word from an optional word set.
+\              UNUSED tests revised as UNUSED UNUSED = may return FALSE when an
+\              implementation has the data stack sharing unused dataspace.
+\              Double number input dependency removed from the HOLDS tests.
+\              Minor case sensitivities removed in definition names.
+\         0.11 25 April 2015
+\              Added tests for PARSE-NAME HOLDS BUFFER:
+\              S\" tests added
+\              DEFER IS ACTION-OF DEFER! DEFER@ tests added
+\              Empty CASE statement test added
+\              [COMPILE] tests removed because it is obsolescent in Forth 2012
+\         0.10 1 August 2014
+\             Added tests contributed by James Bowman for:
+\                <> U> 0<> 0> NIP TUCK ROLL PICK 2>R 2R@ 2R>
+\                HEX WITHIN UNUSED AGAIN MARKER
+\             Added tests for:
+\                .R U.R ERASE PAD REFILL SOURCE-ID 
+\             Removed ABORT from NeverExecuted to enable Win32
+\             to continue after failure of RESTORE-INPUT.
+\             Removed max-intx which is no longer used.
+\         0.7 6 June 2012 Extra CASE test added
+\         0.6 1 April 2012 Tests placed in the public domain.
+\             SAVE-INPUT & RESTORE-INPUT tests, position
+\             of T{ moved so that tests work with ttester.fs
+\             CONVERT test deleted - obsolete word removed from Forth 200X
+\             IMMEDIATE VALUEs tested
+\             RECURSE with :NONAME tested
+\             PARSE and .( tested
+\             Parsing behaviour of C" added
+\         0.5 14 September 2011 Removed the double [ELSE] from the
+\             initial SAVE-INPUT & RESTORE-INPUT test
+\         0.4 30 November 2009  max-int replaced with max-intx to
+\             avoid redefinition warnings.
+\         0.3  6 March 2009 { and } replaced with T{ and }T
+\                           CONVERT test now independent of cell size
+\         0.2  20 April 2007 ANS Forth words changed to upper case
+\                            Tests qd3 to qd6 by Reinhold Straub
+\         0.1  Oct 2006 First version released
+\ -----------------------------------------------------------------------------
+\ The tests are based on John Hayes test program for the core word set
+
+\ Words tested in this file are:
+\     .( .R 0<> 0> 2>R 2R> 2R@ :NONAME <> ?DO AGAIN C" CASE COMPILE, ENDCASE
+\     ENDOF ERASE FALSE HEX MARKER NIP OF PAD PARSE PICK REFILL
+\     RESTORE-INPUT ROLL SAVE-INPUT SOURCE-ID TO TRUE TUCK U.R U> UNUSED
+\     VALUE WITHIN [COMPILE]
+
+\ Words not tested or partially tested:
+\     \ because it has been extensively used already and is, hence, unnecessary
+\     REFILL and SOURCE-ID from the user input device which are not possible
+\     when testing from a file such as this one
+\     UNUSED (partially tested) as the value returned is system dependent
+\     Obsolescent words #TIB CONVERT EXPECT QUERY SPAN TIB as they have been
+\     removed from the Forth 2012 standard
+
+\ Results from words that output to the user output device have to visually
+\ checked for correctness. These are .R U.R .(
+
+\ -----------------------------------------------------------------------------
+\ Assumptions & dependencies:
+\     - tester.fr (or ttester.fs), errorreport.fth and utilities.fth have been
+\       included prior to this file
+\     - the Core word set available
+\ -----------------------------------------------------------------------------
+TESTING Core Extension words
+
+DECIMAL
+
+TESTING TRUE FALSE
+
+T{ TRUE  -> 0 INVERT }T
+T{ FALSE -> 0 }T
+
+\ -----------------------------------------------------------------------------
+TESTING <> U>   (contributed by James Bowman)
+
+T{ 0 0 <> -> FALSE }T
+T{ 1 1 <> -> FALSE }T
+T{ -1 -1 <> -> FALSE }T
+T{ 1 0 <> -> TRUE }T
+T{ -1 0 <> -> TRUE }T
+T{ 0 1 <> -> TRUE }T
+T{ 0 -1 <> -> TRUE }T
+
+T{ 0 1 U> -> FALSE }T
+T{ 1 2 U> -> FALSE }T
+T{ 0 MID-UINT U> -> FALSE }T
+T{ 0 MAX-UINT U> -> FALSE }T
+T{ MID-UINT MAX-UINT U> -> FALSE }T
+T{ 0 0 U> -> FALSE }T
+T{ 1 1 U> -> FALSE }T
+T{ 1 0 U> -> TRUE }T
+T{ 2 1 U> -> TRUE }T
+T{ MID-UINT 0 U> -> TRUE }T
+T{ MAX-UINT 0 U> -> TRUE }T
+T{ MAX-UINT MID-UINT U> -> TRUE }T
+
+\ -----------------------------------------------------------------------------
+TESTING 0<> 0>   (contributed by James Bowman)
+
+T{ 0 0<> -> FALSE }T
+T{ 1 0<> -> TRUE }T
+T{ 2 0<> -> TRUE }T
+T{ -1 0<> -> TRUE }T
+T{ MAX-UINT 0<> -> TRUE }T
+T{ MIN-INT 0<> -> TRUE }T
+T{ MAX-INT 0<> -> TRUE }T
+
+T{ 0 0> -> FALSE }T
+T{ -1 0> -> FALSE }T
+T{ MIN-INT 0> -> FALSE }T
+T{ 1 0> -> TRUE }T
+T{ MAX-INT 0> -> TRUE }T
+
+\ -----------------------------------------------------------------------------
+TESTING NIP TUCK ROLL PICK   (contributed by James Bowman)
+
+T{ 1 2 NIP -> 2 }T
+T{ 1 2 3 NIP -> 1 3 }T
+
+T{ 1 2 TUCK -> 2 1 2 }T
+T{ 1 2 3 TUCK -> 1 3 2 3 }T
+
+T{ : RO5 100 200 300 400 500 ; -> }T
+T{ RO5 3 ROLL -> 100 300 400 500 200 }T
+T{ RO5 2 ROLL -> RO5 ROT }T
+T{ RO5 1 ROLL -> RO5 SWAP }T
+T{ RO5 0 ROLL -> RO5 }T
+
+T{ RO5 2 PICK -> 100 200 300 400 500 300 }T
+T{ RO5 1 PICK -> RO5 OVER }T
+T{ RO5 0 PICK -> RO5 DUP }T
+
+\ -----------------------------------------------------------------------------
+TESTING 2>R 2R@ 2R>   (contributed by James Bowman)
+
+T{ : RR0 2>R 100 R> R> ; -> }T
+T{ 300 400 RR0 -> 100 400 300 }T
+T{ 200 300 400 RR0 -> 200 100 400 300 }T
+
+T{ : RR1 2>R 100 2R@ R> R> ; -> }T
+T{ 300 400 RR1 -> 100 300 400 400 300 }T
+T{ 200 300 400 RR1 -> 200 100 300 400 400 300 }T
+
+T{ : RR2 2>R 100 2R> ; -> }T
+T{ 300 400 RR2 -> 100 300 400 }T
+T{ 200 300 400 RR2 -> 200 100 300 400 }T
+
+\ -----------------------------------------------------------------------------
+TESTING HEX   (contributed by James Bowman)
+
+T{ BASE @ HEX BASE @ DECIMAL BASE @ - SWAP BASE ! -> 6 }T
+
+\ -----------------------------------------------------------------------------
+TESTING WITHIN   (contributed by James Bowman)
+
+T{ 0 0 0 WITHIN -> FALSE }T
+T{ 0 0 MID-UINT WITHIN -> TRUE }T
+T{ 0 0 MID-UINT+1 WITHIN -> TRUE }T
+T{ 0 0 MAX-UINT WITHIN -> TRUE }T
+T{ 0 MID-UINT 0 WITHIN -> FALSE }T
+T{ 0 MID-UINT MID-UINT WITHIN -> FALSE }T
+T{ 0 MID-UINT MID-UINT+1 WITHIN -> FALSE }T
+T{ 0 MID-UINT MAX-UINT WITHIN -> FALSE }T
+T{ 0 MID-UINT+1 0 WITHIN -> FALSE }T
+T{ 0 MID-UINT+1 MID-UINT WITHIN -> TRUE }T
+T{ 0 MID-UINT+1 MID-UINT+1 WITHIN -> FALSE }T
+T{ 0 MID-UINT+1 MAX-UINT WITHIN -> FALSE }T
+T{ 0 MAX-UINT 0 WITHIN -> FALSE }T
+T{ 0 MAX-UINT MID-UINT WITHIN -> TRUE }T
+T{ 0 MAX-UINT MID-UINT+1 WITHIN -> TRUE }T
+T{ 0 MAX-UINT MAX-UINT WITHIN -> FALSE }T
+T{ MID-UINT 0 0 WITHIN -> FALSE }T
+T{ MID-UINT 0 MID-UINT WITHIN -> FALSE }T
+T{ MID-UINT 0 MID-UINT+1 WITHIN -> TRUE }T
+T{ MID-UINT 0 MAX-UINT WITHIN -> TRUE }T
+T{ MID-UINT MID-UINT 0 WITHIN -> TRUE }T
+T{ MID-UINT MID-UINT MID-UINT WITHIN -> FALSE }T
+T{ MID-UINT MID-UINT MID-UINT+1 WITHIN -> TRUE }T
+T{ MID-UINT MID-UINT MAX-UINT WITHIN -> TRUE }T
+T{ MID-UINT MID-UINT+1 0 WITHIN -> FALSE }T
+T{ MID-UINT MID-UINT+1 MID-UINT WITHIN -> FALSE }T
+T{ MID-UINT MID-UINT+1 MID-UINT+1 WITHIN -> FALSE }T
+T{ MID-UINT MID-UINT+1 MAX-UINT WITHIN -> FALSE }T
+T{ MID-UINT MAX-UINT 0 WITHIN -> FALSE }T
+T{ MID-UINT MAX-UINT MID-UINT WITHIN -> FALSE }T
+T{ MID-UINT MAX-UINT MID-UINT+1 WITHIN -> TRUE }T
+T{ MID-UINT MAX-UINT MAX-UINT WITHIN -> FALSE }T
+T{ MID-UINT+1 0 0 WITHIN -> FALSE }T
+T{ MID-UINT+1 0 MID-UINT WITHIN -> FALSE }T
+T{ MID-UINT+1 0 MID-UINT+1 WITHIN -> FALSE }T
+T{ MID-UINT+1 0 MAX-UINT WITHIN -> TRUE }T
+T{ MID-UINT+1 MID-UINT 0 WITHIN -> TRUE }T
+T{ MID-UINT+1 MID-UINT MID-UINT WITHIN -> FALSE }T
+T{ MID-UINT+1 MID-UINT MID-UINT+1 WITHIN -> FALSE }T
+T{ MID-UINT+1 MID-UINT MAX-UINT WITHIN -> TRUE }T
+T{ MID-UINT+1 MID-UINT+1 0 WITHIN -> TRUE }T
+T{ MID-UINT+1 MID-UINT+1 MID-UINT WITHIN -> TRUE }T
+T{ MID-UINT+1 MID-UINT+1 MID-UINT+1 WITHIN -> FALSE }T
+T{ MID-UINT+1 MID-UINT+1 MAX-UINT WITHIN -> TRUE }T
+T{ MID-UINT+1 MAX-UINT 0 WITHIN -> FALSE }T
+T{ MID-UINT+1 MAX-UINT MID-UINT WITHIN -> FALSE }T
+T{ MID-UINT+1 MAX-UINT MID-UINT+1 WITHIN -> FALSE }T
+T{ MID-UINT+1 MAX-UINT MAX-UINT WITHIN -> FALSE }T
+T{ MAX-UINT 0 0 WITHIN -> FALSE }T
+T{ MAX-UINT 0 MID-UINT WITHIN -> FALSE }T
+T{ MAX-UINT 0 MID-UINT+1 WITHIN -> FALSE }T
+T{ MAX-UINT 0 MAX-UINT WITHIN -> FALSE }T
+T{ MAX-UINT MID-UINT 0 WITHIN -> TRUE }T
+T{ MAX-UINT MID-UINT MID-UINT WITHIN -> FALSE }T
+T{ MAX-UINT MID-UINT MID-UINT+1 WITHIN -> FALSE }T
+T{ MAX-UINT MID-UINT MAX-UINT WITHIN -> FALSE }T
+T{ MAX-UINT MID-UINT+1 0 WITHIN -> TRUE }T
+T{ MAX-UINT MID-UINT+1 MID-UINT WITHIN -> TRUE }T
+T{ MAX-UINT MID-UINT+1 MID-UINT+1 WITHIN -> FALSE }T
+T{ MAX-UINT MID-UINT+1 MAX-UINT WITHIN -> FALSE }T
+T{ MAX-UINT MAX-UINT 0 WITHIN -> TRUE }T
+T{ MAX-UINT MAX-UINT MID-UINT WITHIN -> TRUE }T
+T{ MAX-UINT MAX-UINT MID-UINT+1 WITHIN -> TRUE }T
+T{ MAX-UINT MAX-UINT MAX-UINT WITHIN -> FALSE }T
+
+T{ MIN-INT MIN-INT MIN-INT WITHIN -> FALSE }T
+T{ MIN-INT MIN-INT 0 WITHIN -> TRUE }T
+T{ MIN-INT MIN-INT 1 WITHIN -> TRUE }T
+T{ MIN-INT MIN-INT MAX-INT WITHIN -> TRUE }T
+T{ MIN-INT 0 MIN-INT WITHIN -> FALSE }T
+T{ MIN-INT 0 0 WITHIN -> FALSE }T
+T{ MIN-INT 0 1 WITHIN -> FALSE }T
+T{ MIN-INT 0 MAX-INT WITHIN -> FALSE }T
+T{ MIN-INT 1 MIN-INT WITHIN -> FALSE }T
+T{ MIN-INT 1 0 WITHIN -> TRUE }T
+T{ MIN-INT 1 1 WITHIN -> FALSE }T
+T{ MIN-INT 1 MAX-INT WITHIN -> FALSE }T
+T{ MIN-INT MAX-INT MIN-INT WITHIN -> FALSE }T
+T{ MIN-INT MAX-INT 0 WITHIN -> TRUE }T
+T{ MIN-INT MAX-INT 1 WITHIN -> TRUE }T
+T{ MIN-INT MAX-INT MAX-INT WITHIN -> FALSE }T
+T{ 0 MIN-INT MIN-INT WITHIN -> FALSE }T
+T{ 0 MIN-INT 0 WITHIN -> FALSE }T
+T{ 0 MIN-INT 1 WITHIN -> TRUE }T
+T{ 0 MIN-INT MAX-INT WITHIN -> TRUE }T
+T{ 0 0 MIN-INT WITHIN -> TRUE }T
+T{ 0 0 0 WITHIN -> FALSE }T
+T{ 0 0 1 WITHIN -> TRUE }T
+T{ 0 0 MAX-INT WITHIN -> TRUE }T
+T{ 0 1 MIN-INT WITHIN -> FALSE }T
+T{ 0 1 0 WITHIN -> FALSE }T
+T{ 0 1 1 WITHIN -> FALSE }T
+T{ 0 1 MAX-INT WITHIN -> FALSE }T
+T{ 0 MAX-INT MIN-INT WITHIN -> FALSE }T
+T{ 0 MAX-INT 0 WITHIN -> FALSE }T
+T{ 0 MAX-INT 1 WITHIN -> TRUE }T
+T{ 0 MAX-INT MAX-INT WITHIN -> FALSE }T
+T{ 1 MIN-INT MIN-INT WITHIN -> FALSE }T
+T{ 1 MIN-INT 0 WITHIN -> FALSE }T
+T{ 1 MIN-INT 1 WITHIN -> FALSE }T
+T{ 1 MIN-INT MAX-INT WITHIN -> TRUE }T
+T{ 1 0 MIN-INT WITHIN -> TRUE }T
+T{ 1 0 0 WITHIN -> FALSE }T
+T{ 1 0 1 WITHIN -> FALSE }T
+T{ 1 0 MAX-INT WITHIN -> TRUE }T
+T{ 1 1 MIN-INT WITHIN -> TRUE }T
+T{ 1 1 0 WITHIN -> TRUE }T
+T{ 1 1 1 WITHIN -> FALSE }T
+T{ 1 1 MAX-INT WITHIN -> TRUE }T
+T{ 1 MAX-INT MIN-INT WITHIN -> FALSE }T
+T{ 1 MAX-INT 0 WITHIN -> FALSE }T
+T{ 1 MAX-INT 1 WITHIN -> FALSE }T
+T{ 1 MAX-INT MAX-INT WITHIN -> FALSE }T
+T{ MAX-INT MIN-INT MIN-INT WITHIN -> FALSE }T
+T{ MAX-INT MIN-INT 0 WITHIN -> FALSE }T
+T{ MAX-INT MIN-INT 1 WITHIN -> FALSE }T
+T{ MAX-INT MIN-INT MAX-INT WITHIN -> FALSE }T
+T{ MAX-INT 0 MIN-INT WITHIN -> TRUE }T
+T{ MAX-INT 0 0 WITHIN -> FALSE }T
+T{ MAX-INT 0 1 WITHIN -> FALSE }T
+T{ MAX-INT 0 MAX-INT WITHIN -> FALSE }T
+T{ MAX-INT 1 MIN-INT WITHIN -> TRUE }T
+T{ MAX-INT 1 0 WITHIN -> TRUE }T
+T{ MAX-INT 1 1 WITHIN -> FALSE }T
+T{ MAX-INT 1 MAX-INT WITHIN -> FALSE }T
+T{ MAX-INT MAX-INT MIN-INT WITHIN -> TRUE }T
+T{ MAX-INT MAX-INT 0 WITHIN -> TRUE }T
+T{ MAX-INT MAX-INT 1 WITHIN -> TRUE }T
+T{ MAX-INT MAX-INT MAX-INT WITHIN -> FALSE }T
+
+\ -----------------------------------------------------------------------------
+TESTING UNUSED  (contributed by James Bowman & Peter Knaggs)
+
+VARIABLE UNUSED0
+T{ UNUSED DROP -> }T                  
+T{ ALIGN UNUSED UNUSED0 ! 0 , UNUSED CELL+ UNUSED0 @ = -> TRUE }T
+T{ UNUSED UNUSED0 ! 0 C, UNUSED CHAR+ UNUSED0 @ =
+         -> TRUE }T  \ aligned -> unaligned
+T{ UNUSED UNUSED0 ! 0 C, UNUSED CHAR+ UNUSED0 @ = -> TRUE }T  \ unaligned -> ?
+
+\ -----------------------------------------------------------------------------
+TESTING AGAIN   (contributed by James Bowman)
+
+T{ : AG0 701 BEGIN DUP 7 MOD 0= IF EXIT THEN 1+ AGAIN ; -> }T
+T{ AG0 -> 707 }T
+
+\ -----------------------------------------------------------------------------
+TESTING MARKER   (contributed by James Bowman)
+
+T{ : MA? BL WORD FIND NIP 0<> ; -> }T
+T{ MARKER MA0 -> }T
+T{ : MA1 111 ; -> }T
+T{ MARKER MA2 -> }T
+T{ : MA1 222 ; -> }T
+T{ MA? MA0 MA? MA1 MA? MA2 -> TRUE TRUE TRUE }T
+T{ MA1 MA2 MA1 -> 222 111 }T
+T{ MA? MA0 MA? MA1 MA? MA2 -> TRUE TRUE FALSE }T
+T{ MA0 -> }T
+T{ MA? MA0 MA? MA1 MA? MA2 -> FALSE FALSE FALSE }T
+
+\ -----------------------------------------------------------------------------
+TESTING ?DO
+
+: QD ?DO I LOOP ;
+T{ 789 789 QD -> }T
+T{ -9876 -9876 QD -> }T
+T{ 5 0 QD -> 0 1 2 3 4 }T
+
+: QD1 ?DO I 10 +LOOP ;
+T{ 50 1 QD1 -> 1 11 21 31 41 }T
+T{ 50 0 QD1 -> 0 10 20 30 40 }T
+
+: QD2 ?DO I 3 > IF LEAVE ELSE I THEN LOOP ;
+T{ 5 -1 QD2 -> -1 0 1 2 3 }T
+
+: QD3 ?DO I 1 +LOOP ;
+T{ 4  4 QD3 -> }T
+T{ 4  1 QD3 -> 1 2 3 }T
+T{ 2 -1 QD3 -> -1 0 1 }T
+
+: QD4 ?DO I -1 +LOOP ;
+T{  4 4 QD4 -> }T
+T{  1 4 QD4 -> 4 3 2 1 }T
+T{ -1 2 QD4 -> 2 1 0 -1 }T
+
+: QD5 ?DO I -10 +LOOP ;
+T{   1 50 QD5 -> 50 40 30 20 10 }T
+T{   0 50 QD5 -> 50 40 30 20 10 0 }T
+T{ -25 10 QD5 -> 10 0 -10 -20 }T
+
+VARIABLE ITERS
+VARIABLE INCRMNT
+
+: QD6 ( limit start increment -- )
+   INCRMNT !
+   0 ITERS !
+   ?DO
+      1 ITERS +!
+      I
+      ITERS @  6 = IF LEAVE THEN
+      INCRMNT @
+   +LOOP ITERS @
+;
+
+T{  4  4 -1 QD6 -> 0 }T
+T{  1  4 -1 QD6 -> 4 3 2 1 4 }T
+T{  4  1 -1 QD6 -> 1 0 -1 -2 -3 -4 6 }T
+T{  4  1  0 QD6 -> 1 1 1 1 1 1 6 }T
+T{  0  0  0 QD6 -> 0 }T
+T{  1  4  0 QD6 -> 4 4 4 4 4 4 6 }T
+T{  1  4  1 QD6 -> 4 5 6 7 8 9 6 }T
+T{  4  1  1 QD6 -> 1 2 3 3 }T
+T{  4  4  1 QD6 -> 0 }T
+T{  2 -1 -1 QD6 -> -1 -2 -3 -4 -5 -6 6 }T
+T{ -1  2 -1 QD6 -> 2 1 0 -1 4 }T
+T{  2 -1  0 QD6 -> -1 -1 -1 -1 -1 -1 6 }T
+T{ -1  2  0 QD6 -> 2 2 2 2 2 2 6 }T
+T{ -1  2  1 QD6 -> 2 3 4 5 6 7 6 }T
+T{  2 -1  1 QD6 -> -1 0 1 3 }T
+
+\ -----------------------------------------------------------------------------
+TESTING BUFFER:
+
+T{ 2 CELLS BUFFER: BUF:TEST -> }T
+T{ BUF:TEST DUP ALIGNED = -> TRUE }T
+T{ 111 BUF:TEST ! 222 BUF:TEST CELL+ ! -> }T
+T{ BUF:TEST @ BUF:TEST CELL+ @ -> 111 222 }T
+
+\ -----------------------------------------------------------------------------
+TESTING VALUE TO
+
+T{ 111 VALUE VAL1 -999 VALUE VAL2 -> }T
+T{ VAL1 -> 111 }T
+T{ VAL2 -> -999 }T
+T{ 222 TO VAL1 -> }T
+T{ VAL1 -> 222 }T
+T{ : VD1 VAL1 ; -> }T
+T{ VD1 -> 222 }T
+T{ : VD2 TO VAL2 ; -> }T
+T{ VAL2 -> -999 }T
+T{ -333 VD2 -> }T
+T{ VAL2 -> -333 }T
+T{ VAL1 -> 222 }T
+T{ 444 TO VAL1 -> }T
+T{ VD1 -> 444 }T
+T{ 123 VALUE VAL3 IMMEDIATE VAL3 -> 123 }T
+T{ : VD3 VAL3 LITERAL ; VD3 -> 123 }T
+
+\ -----------------------------------------------------------------------------
+TESTING CASE OF ENDOF ENDCASE
+
+: CS1 CASE 1 OF 111 ENDOF
+           2 OF 222 ENDOF
+           3 OF 333 ENDOF
+           >R 999 R>
+      ENDCASE
+;
+
+T{ 1 CS1 -> 111 }T
+T{ 2 CS1 -> 222 }T
+T{ 3 CS1 -> 333 }T
+T{ 4 CS1 -> 999 }T
+
+\ Nested CASE's
+
+: CS2 >R CASE -1 OF CASE R@ 1 OF 100 ENDOF
+                            2 OF 200 ENDOF
+                           >R -300 R>
+                    ENDCASE
+                 ENDOF
+              -2 OF CASE R@ 1 OF -99  ENDOF
+                            >R -199 R>
+                    ENDCASE
+                 ENDOF
+                 >R 299 R>
+         ENDCASE R> DROP
+;
+
+T{ -1 1 CS2 ->  100 }T
+T{ -1 2 CS2 ->  200 }T
+T{ -1 3 CS2 -> -300 }T
+T{ -2 1 CS2 -> -99  }T
+T{ -2 2 CS2 -> -199 }T
+T{  0 2 CS2 ->  299 }T
+
+\ Boolean short circuiting using CASE
+
+: CS3  ( N1 -- N2 )
+   CASE 1- FALSE OF 11 ENDOF
+        1- FALSE OF 22 ENDOF
+        1- FALSE OF 33 ENDOF
+        44 SWAP
+   ENDCASE
+;
+
+T{ 1 CS3 -> 11 }T
+T{ 2 CS3 -> 22 }T
+T{ 3 CS3 -> 33 }T
+T{ 9 CS3 -> 44 }T
+
+\ Empty CASE statements with/without default
+
+T{ : CS4 CASE ENDCASE ; 1 CS4 -> }T
+T{ : CS5 CASE 2 SWAP ENDCASE ; 1 CS5 -> 2 }T
+T{ : CS6 CASE 1 OF ENDOF 2 ENDCASE ; 1 CS6 -> }T
+T{ : CS7 CASE 3 OF ENDOF 2 ENDCASE ; 1 CS7 -> 1 }T
+
+\ -----------------------------------------------------------------------------
+TESTING :NONAME RECURSE
+
+VARIABLE NN1
+VARIABLE NN2
+:NONAME 1234 ; NN1 !
+:NONAME 9876 ; NN2 !
+T{ NN1 @ EXECUTE -> 1234 }T
+T{ NN2 @ EXECUTE -> 9876 }T
+
+T{ :NONAME ( n -- 0,1,..n ) DUP IF DUP >R 1- RECURSE R> THEN ;
+   CONSTANT RN1 -> }T
+T{ 0 RN1 EXECUTE -> 0 }T
+T{ 4 RN1 EXECUTE -> 0 1 2 3 4 }T
+
+:NONAME  ( n -- n1 )    \ Multiple RECURSEs in one definition
+   1- DUP
+   CASE 0 OF EXIT ENDOF
+        1 OF 11 SWAP RECURSE ENDOF
+        2 OF 22 SWAP RECURSE ENDOF
+        3 OF 33 SWAP RECURSE ENDOF
+        DROP ABS RECURSE EXIT
+   ENDCASE
+; CONSTANT RN2
+
+T{  1 RN2 EXECUTE -> 0 }T
+T{  2 RN2 EXECUTE -> 11 0 }T
+T{  4 RN2 EXECUTE -> 33 22 11 0 }T
+T{ 25 RN2 EXECUTE -> 33 22 11 0 }T
+
+\ -----------------------------------------------------------------------------
+TESTING C"
+
+T{ : CQ1 C" 123" ; -> }T
+T{ CQ1 COUNT EVALUATE -> 123 }T
+T{ : CQ2 C" " ; -> }T
+T{ CQ2 COUNT EVALUATE -> }T
+T{ : CQ3 C" 2345"COUNT EVALUATE ; CQ3 -> 2345 }T
+
+\ -----------------------------------------------------------------------------
+TESTING COMPILE,
+
+:NONAME DUP + ; CONSTANT DUP+
+T{ : Q DUP+ COMPILE, ; -> }T
+T{ : AS1 [ Q ] ; -> }T
+T{ 123 AS1 -> 246 }T
+
+\ -----------------------------------------------------------------------------
+\ Cannot automatically test SAVE-INPUT and RESTORE-INPUT from a console source
+
+TESTING SAVE-INPUT and RESTORE-INPUT with a string source
+
+VARIABLE SI_INC 0 SI_INC !
+
+: SI1
+   SI_INC @ >IN +!
+   15 SI_INC !
+;
+
+: S$ S" SAVE-INPUT SI1 RESTORE-INPUT 12345" ;
+
+T{ S$ EVALUATE SI_INC @ -> 0 2345 15 }T
+
+\ -----------------------------------------------------------------------------
+TESTING .(
+
+CR CR .( Output from .() 
+T{ CR .( You should see -9876: ) -9876 . -> }T
+T{ CR .( and again: ).( -9876)CR -> }T
+
+CR CR .( On the next 2 lines you should see First then Second messages:)
+T{ : DOTP  CR ." Second message via ." [CHAR] " EMIT    \ Check .( is immediate
+     [ CR ] .( First message via .( ) ; DOTP -> }T
+CR CR
+T{ : IMM? BL WORD FIND NIP ; IMM? .( -> 1 }T
+
+\ -----------------------------------------------------------------------------
+TESTING .R and U.R - has to handle different cell sizes
+
+\ Create some large integers just below/above MAX and Min INTs
+MAX-INT 73 79 */ CONSTANT LI1
+MIN-INT 71 73 */ CONSTANT LI2
+
+LI1 0 <# #S #> NIP CONSTANT LENLI1
+
+: (.R&U.R)  ( u1 u2 -- )  \ u1 <= string length, u2 is required indentation
+   TUCK + >R
+   LI1 OVER SPACES  . CR R@    LI1 SWAP  .R CR
+   LI2 OVER SPACES  . CR R@ 1+ LI2 SWAP  .R CR
+   LI1 OVER SPACES U. CR R@    LI1 SWAP U.R CR
+   LI2 SWAP SPACES U. CR R>    LI2 SWAP U.R CR
+;
+
+: .R&U.R  ( -- )
+   CR ." You should see lines duplicated:" CR
+   ." indented by 0 spaces" CR 0      0 (.R&U.R) CR
+   ." indented by 0 spaces" CR LENLI1 0 (.R&U.R) CR \ Just fits required width
+   ." indented by 5 spaces" CR LENLI1 5 (.R&U.R) CR
+;
+
+CR CR .( Output from .R and U.R)
+T{ .R&U.R -> }T
+
+\ -----------------------------------------------------------------------------
+TESTING PAD ERASE
+\ Must handle different size characters i.e. 1 CHARS >= 1 
+
+84 CONSTANT CHARS/PAD      \ Minimum size of PAD in chars
+CHARS/PAD CHARS CONSTANT AUS/PAD
+: CHECKPAD  ( caddr u ch -- f )  \ f = TRUE if u chars = ch
+   SWAP 0
+   ?DO
+      OVER I CHARS + C@ OVER <>
+      IF 2DROP UNLOOP FALSE EXIT THEN
+   LOOP  
+   2DROP TRUE
+;
+
+T{ PAD DROP -> }T
+T{ 0 INVERT PAD C! -> }T
+T{ PAD C@ CONSTANT MAXCHAR -> }T
+T{ PAD CHARS/PAD 2DUP MAXCHAR FILL MAXCHAR CHECKPAD -> TRUE }T
+T{ PAD CHARS/PAD 2DUP CHARS ERASE 0 CHECKPAD -> TRUE }T
+T{ PAD CHARS/PAD 2DUP MAXCHAR FILL PAD 0 ERASE MAXCHAR CHECKPAD -> TRUE }T
+T{ PAD 43 CHARS + 9 CHARS ERASE -> }T
+T{ PAD 43 MAXCHAR CHECKPAD -> TRUE }T
+T{ PAD 43 CHARS + 9 0 CHECKPAD -> TRUE }T
+T{ PAD 52 CHARS + CHARS/PAD 52 - MAXCHAR CHECKPAD -> TRUE }T
+
+\ Check that use of WORD and pictured numeric output do not corrupt PAD
+\ Minimum size of buffers for these are 33 chars and (2*n)+2 chars respectively
+\ where n is number of bits per cell
+
+PAD CHARS/PAD ERASE
+2 BASE !
+MAX-UINT MAX-UINT <# #S CHAR 1 DUP HOLD HOLD #> 2DROP
+DECIMAL
+BL WORD 12345678123456781234567812345678 DROP
+T{ PAD CHARS/PAD 0 CHECKPAD -> TRUE }T
+
+\ -----------------------------------------------------------------------------
+TESTING PARSE
+
+T{ CHAR | PARSE 1234| DUP ROT ROT EVALUATE -> 4 1234 }T
+T{ CHAR ^ PARSE  23 45 ^ DUP ROT ROT EVALUATE -> 7 23 45 }T
+: PA1 [CHAR] $ PARSE DUP >R PAD SWAP CHARS MOVE PAD R> ;
+T{ PA1 3456
+   DUP ROT ROT EVALUATE -> 4 3456 }T
+T{ CHAR A PARSE A SWAP DROP -> 0 }T
+T{ CHAR Z PARSE
+   SWAP DROP -> 0 }T
+T{ CHAR " PARSE 4567 "DUP ROT ROT EVALUATE -> 5 4567 }T
+ 
+\ -----------------------------------------------------------------------------
+TESTING PARSE-NAME  (Forth 2012)
+\ Adapted from the PARSE-NAME RfD tests
+
+T{ PARSE-NAME abcd  STR1  S= -> TRUE }T        \ No leading spaces
+T{ PARSE-NAME      abcde STR2 S= -> TRUE }T    \ Leading spaces
+
+\ Test empty parse area, new lines are necessary
+T{ PARSE-NAME
+  NIP -> 0 }T
+\ Empty parse area with spaces after PARSE-NAME
+T{ PARSE-NAME         
+  NIP -> 0 }T
+
+T{ : PARSE-NAME-TEST ( "name1" "name2" -- n )
+    PARSE-NAME PARSE-NAME S= ; -> }T
+T{ PARSE-NAME-TEST abcd abcd  -> TRUE }T
+T{ PARSE-NAME-TEST abcd   abcd  -> TRUE }T  \ Leading spaces
+T{ PARSE-NAME-TEST abcde abcdf -> FALSE }T
+T{ PARSE-NAME-TEST abcdf abcde -> FALSE }T
+T{ PARSE-NAME-TEST abcde abcde
+   -> TRUE }T         \ Parse to end of line
+T{ PARSE-NAME-TEST abcde           abcde         
+   -> TRUE }T         \ Leading and trailing spaces
+
+\ -----------------------------------------------------------------------------
+TESTING DEFER DEFER@ DEFER! IS ACTION-OF (Forth 2012)
+\ Adapted from the Forth 200X RfD tests
+
+T{ DEFER DEFER1 -> }T
+T{ : MY-DEFER DEFER ; -> }T
+T{ : IS-DEFER1 IS DEFER1 ; -> }T
+T{ : ACTION-DEFER1 ACTION-OF DEFER1 ; -> }T
+T{ : DEF! DEFER! ; -> }T
+T{ : DEF@ DEFER@ ; -> }T
+
+T{ ' * ' DEFER1 DEFER! -> }T
+T{ 2 3 DEFER1 -> 6 }T
+T{ ' DEFER1 DEFER@ -> ' * }T
+T{ ' DEFER1 DEF@ -> ' * }T
+T{ ACTION-OF DEFER1 -> ' * }T
+T{ ACTION-DEFER1 -> ' * }T
+T{ ' + IS DEFER1 -> }T
+T{ 1 2 DEFER1 -> 3 }T
+T{ ' DEFER1 DEFER@ -> ' + }T
+T{ ' DEFER1 DEF@ -> ' + }T
+T{ ACTION-OF DEFER1 -> ' + }T
+T{ ACTION-DEFER1 -> ' + }T
+T{ ' - IS-DEFER1 -> }T
+T{ 1 2 DEFER1 -> -1 }T
+T{ ' DEFER1 DEFER@ -> ' - }T
+T{ ' DEFER1 DEF@ -> ' - }T
+T{ ACTION-OF DEFER1 -> ' - }T
+T{ ACTION-DEFER1 -> ' - }T
+
+T{ MY-DEFER DEFER2 -> }T
+T{ ' DUP IS DEFER2 -> }T
+T{ 1 DEFER2 -> 1 1 }T
+
+\ -----------------------------------------------------------------------------
+TESTING HOLDS  (Forth 2012)
+
+: HTEST S" Testing HOLDS" ;
+: HTEST2 S" works" ;
+: HTEST3 S" Testing HOLDS works 123" ;
+T{ 0 0 <#  HTEST HOLDS #> HTEST S= -> TRUE }T
+T{ 123 0 <# #S BL HOLD HTEST2 HOLDS BL HOLD HTEST HOLDS #>
+   HTEST3 S= -> TRUE }T
+T{ : HLD HOLDS ; -> }T
+T{ 0 0 <#  HTEST HLD #> HTEST S= -> TRUE }T
+
+\ -----------------------------------------------------------------------------
+TESTING REFILL SOURCE-ID
+\ REFILL and SOURCE-ID from the user input device can't be tested from a file,
+\ can only be tested from a string via EVALUATE
+
+T{ : RF1  S" REFILL" EVALUATE ; RF1 -> FALSE }T
+T{ : SID1  S" SOURCE-ID" EVALUATE ; SID1 -> -1 }T
+
+\ ------------------------------------------------------------------------------
+TESTING S\"  (Forth 2012 compilation mode)
+\ Extended the Forth 200X RfD tests
+\ Note this tests the Core Ext definition of S\" which has unedfined
+\ interpretation semantics. S\" in interpretation mode is tested in the tests on
+\ the File-Access word set
+
+T{ : SSQ1 S\" abc" S" abc" S= ; -> }T  \ No escapes
+T{ SSQ1 -> TRUE }T
+T{ : SSQ2 S\" " ; SSQ2 SWAP DROP -> 0 }T    \ Empty string
+
+T{ : SSQ3 S\" \a\b\e\f\l\m\q\r\t\v\x0F0\x1Fa\xaBx\z\"\\" ; -> }T
+T{ SSQ3 SWAP DROP          ->  20 }T    \ String length
+T{ SSQ3 DROP            C@ ->   7 }T    \ \a   BEL  Bell
+T{ SSQ3 DROP  1 CHARS + C@ ->   8 }T    \ \b   BS   Backspace
+T{ SSQ3 DROP  2 CHARS + C@ ->  27 }T    \ \e   ESC  Escape
+T{ SSQ3 DROP  3 CHARS + C@ ->  12 }T    \ \f   FF   Form feed
+T{ SSQ3 DROP  4 CHARS + C@ ->  10 }T    \ \l   LF   Line feed
+T{ SSQ3 DROP  5 CHARS + C@ ->  13 }T    \ \m        CR of CR/LF pair
+T{ SSQ3 DROP  6 CHARS + C@ ->  10 }T    \           LF of CR/LF pair
+T{ SSQ3 DROP  7 CHARS + C@ ->  34 }T    \ \q   "    Double Quote
+T{ SSQ3 DROP  8 CHARS + C@ ->  13 }T    \ \r   CR   Carriage Return
+T{ SSQ3 DROP  9 CHARS + C@ ->   9 }T    \ \t   TAB  Horizontal Tab
+T{ SSQ3 DROP 10 CHARS + C@ ->  11 }T    \ \v   VT   Vertical Tab
+T{ SSQ3 DROP 11 CHARS + C@ ->  15 }T    \ \x0F      Given Char
+T{ SSQ3 DROP 12 CHARS + C@ ->  48 }T    \ 0    0    Digit follow on
+T{ SSQ3 DROP 13 CHARS + C@ ->  31 }T    \ \x1F      Given Char
+T{ SSQ3 DROP 14 CHARS + C@ ->  97 }T    \ a    a    Hex follow on
+T{ SSQ3 DROP 15 CHARS + C@ -> 171 }T    \ \xaB      Insensitive Given Char
+T{ SSQ3 DROP 16 CHARS + C@ -> 120 }T    \ x    x    Non hex follow on
+T{ SSQ3 DROP 17 CHARS + C@ ->   0 }T    \ \z   NUL  No Character
+T{ SSQ3 DROP 18 CHARS + C@ ->  34 }T    \ \"   "    Double Quote
+T{ SSQ3 DROP 19 CHARS + C@ ->  92 }T    \ \\   \    Back Slash
+
+\ The above does not test \n as this is a system dependent value.
+\ Check it displays a new line
+CR .( The next test should display:)
+CR .( One line...)
+CR .( another line)
+T{ : SSQ4 S\" \nOne line...\nanotherLine\n" TYPE ; SSQ4 -> }T
+
+\ Test bare escapable characters appear as themselves
+T{ : SSQ5 S\" abeflmnqrtvxz" S" abeflmnqrtvxz" S= ; SSQ5 -> TRUE }T
+
+T{ : SSQ6 S\" a\""2DROP 1111 ; SSQ6 -> 1111 }T \ Parsing behaviour
+
+T{ : SSQ7  S\" 111 : SSQ8 S\\\" 222\" EVALUATE ; SSQ8 333" EVALUATE ; -> }T
+T{ SSQ7 -> 111 222 333 }T
+T{ : SSQ9  S\" 11 : SSQ10 S\\\" \\x32\\x32\" EVALUATE ; SSQ10 33" EVALUATE ; -> }T
+T{ SSQ9 -> 11 22 33 }T
+
+\ -----------------------------------------------------------------------------
+CORE-EXT-ERRORS SET-ERROR-COUNT
+
+CR .( End of Core Extension word tests) CR
+
+

lib/forth/ans-tests/tester.fr

@@ -0,0 +1,66 @@
+\ From: John Hayes S1I
+\ Subject: tester.fr
+\ Date: Mon, 27 Nov 95 13:10:09 PST  
+
+\ (C) 1995 JOHNS HOPKINS UNIVERSITY / APPLIED PHYSICS LABORATORY
+\ MAY BE DISTRIBUTED FREELY AS LONG AS THIS COPYRIGHT NOTICE REMAINS.
+\ VERSION 1.2
+
+\ 24/11/2015 Replaced Core Ext word <> with = 0=
+\ 31/3/2015 Variable #ERRORS added and incremented for each error reported.
+\ 22/1/09 The words { and } have been changed to T{ and }T respectively to
+\ agree with the Forth 200X file ttester.fs. This avoids clashes with
+\ locals using { ... } and the FSL use of } 
+
+HEX
+
+\ SET THE FOLLOWING FLAG TO TRUE FOR MORE VERBOSE OUTPUT; THIS MAY
+\ ALLOW YOU TO TELL WHICH TEST CAUSED YOUR SYSTEM TO HANG.
+VARIABLE VERBOSE
+   FALSE VERBOSE !
+\   TRUE VERBOSE !
+
+: EMPTY-STACK   \ ( ... -- ) EMPTY STACK: HANDLES UNDERFLOWED STACK TOO.
+   DEPTH ?DUP IF DUP 0< IF NEGATE 0 DO 0 LOOP ELSE 0 DO DROP LOOP THEN THEN ;
+
+VARIABLE #ERRORS 0 #ERRORS !
+
+: ERROR      \ ( C-ADDR U -- ) DISPLAY AN ERROR MESSAGE FOLLOWED BY
+      \ THE LINE THAT HAD THE ERROR.
+   CR TYPE SOURCE TYPE       \ DISPLAY LINE CORRESPONDING TO ERROR
+   EMPTY-STACK               \ THROW AWAY EVERY THING ELSE
+   #ERRORS @ 1 + #ERRORS !
+\   QUIT  \ *** Uncomment this line to QUIT on an error
+;
+
+VARIABLE ACTUAL-DEPTH         \ STACK RECORD
+CREATE ACTUAL-RESULTS 20 CELLS ALLOT
+
+: T{      \ ( -- ) SYNTACTIC SUGAR.
+   ;
+
+: ->      \ ( ... -- ) RECORD DEPTH AND CONTENT OF STACK.
+   DEPTH DUP ACTUAL-DEPTH !      \ RECORD DEPTH
+   ?DUP IF            \ IF THERE IS SOMETHING ON STACK
+      0 DO ACTUAL-RESULTS I CELLS + ! LOOP \ SAVE THEM
+   THEN ;
+
+: }T      \ ( ... -- ) COMPARE STACK (EXPECTED) CONTENTS WITH SAVED
+      \ (ACTUAL) CONTENTS.
+   DEPTH ACTUAL-DEPTH @ = IF      \ IF DEPTHS MATCH
+      DEPTH ?DUP IF         \ IF THERE IS SOMETHING ON THE STACK
+         0  DO            \ FOR EACH STACK ITEM
+           ACTUAL-RESULTS I CELLS + @   \ COMPARE ACTUAL WITH EXPECTED
+           = 0= IF S" INCORRECT RESULT: " ERROR LEAVE THEN
+         LOOP
+      THEN
+   ELSE               \ DEPTH MISMATCH
+      S" WRONG NUMBER OF RESULTS: " ERROR
+   THEN ;
+
+: TESTING   \ ( -- ) TALKING COMMENT.
+  SOURCE VERBOSE @
+   IF DUP >R TYPE CR R> >IN !
+   ELSE >IN ! DROP [CHAR] * EMIT
+   THEN ;
+

lib/forth/conformance.sh

@@ -0,0 +1,170 @@
+#!/usr/bin/env bash
+# Run the Hayes/Gerry-Jackson Core conformance suite against our Forth
+# interpreter and emit scoreboard.json + scoreboard.md.
+#
+# Method:
+#   1. Preprocess lib/forth/ans-tests/core.fr — strip \ comments, ( ... )
+#      comments, and TESTING … metadata lines.
+#   2. Split into chunks ending at each `}T` so an error in one test
+#      chunk doesn't abort the run.
+#   3. Emit an SX file that exposes those chunks as a list.
+#   4. Run our Forth + hayes-runner under sx_server; record pass/fail/error.
+
+set -e
+FORTH_DIR="$(cd "$(dirname "$0")" && pwd)"
+ROOT="$(cd "$FORTH_DIR/../.." && pwd)"
+SX_SERVER="${SX_SERVER:-/root/rose-ash/hosts/ocaml/_build/default/bin/sx_server.exe}"
+SOURCE="$FORTH_DIR/ans-tests/core.fr"
+OUT_JSON="$FORTH_DIR/scoreboard.json"
+OUT_MD="$FORTH_DIR/scoreboard.md"
+TMP="$(mktemp -d)"
+PREPROC="$TMP/preproc.forth"
+CHUNKS_SX="$TMP/chunks.sx"
+
+cd "$ROOT"
+
+# 1. preprocess
+awk '
+{
+  line = $0
+  # protect POSTPONE \ so the comment-strip below leaves the literal \ alone
+  gsub(/POSTPONE[ \t]+\\/, "POSTPONE @@BS@@", line)
+  # strip leading/embedded \ line comments (must be \ followed by space or EOL)
+  gsub(/(^|[ \t])\\([ \t].*|$)/, " ", line)
+  # strip ( ... ) block comments that sit on one line
+  gsub(/\([^)]*\)/, " ", line)
+  # strip TESTING … metadata lines (rest of line, incl. bare TESTING)
+  sub(/TESTING([ \t].*)?$/, " ", line)
+  # restore the protected backslash
+  gsub(/@@BS@@/, "\\", line)
+  print line
+}' "$SOURCE" > "$PREPROC"
+
+# 2 + 3: split into chunks at each `}T` and emit as a SX file
+#
+# Cap chunks via MAX_CHUNKS env (default 638 = full Hayes Core). Lower
+# it temporarily if later tests regress into an infinite loop while you
+# are iterating on primitives.
+MAX_CHUNKS="${MAX_CHUNKS:-638}"
+
+MAX_CHUNKS="$MAX_CHUNKS" python3 - "$PREPROC" "$CHUNKS_SX" <<'PY'
+import os, re, sys
+preproc_path, out_path = sys.argv[1], sys.argv[2]
+max_chunks = int(os.environ.get("MAX_CHUNKS", "590"))
+text = open(preproc_path).read()
+# keep the `}T` attached to the preceding chunk
+parts = re.split(r'(\}T)', text)
+chunks = []
+buf = ""
+for p in parts:
+    buf += p
+    if p == "}T":
+        s = buf.strip()
+        if s:
+            chunks.append(s)
+        buf = ""
+if buf.strip():
+    chunks.append(buf.strip())
+chunks = chunks[:max_chunks]
+
+def esc(s):
+    s = s.replace('\\', '\\\\').replace('"', '\\"')
+    s = s.replace('\r', ' ').replace('\n', ' ')
+    s = re.sub(r'\s+', ' ', s).strip()
+    return s
+
+with open(out_path, "w") as f:
+    f.write("(define hayes-chunks (list\n")
+    for c in chunks:
+        f.write('  "' + esc(c) + '"\n')
+    f.write("))\n\n")
+    f.write("(define\n")
+    f.write("  hayes-run-all\n")
+    f.write("  (fn\n")
+    f.write("    ()\n")
+    f.write("    (hayes-reset!)\n")
+    f.write("    (let ((s (hayes-boot)))\n")
+    f.write("      (for-each (fn (c) (hayes-run-chunk s c)) hayes-chunks))\n")
+    f.write("    (hayes-summary)))\n")
+PY
+
+# 4. run it
+OUT=$(printf '(epoch 1)\n(load "lib/forth/runtime.sx")\n(epoch 2)\n(load "lib/forth/reader.sx")\n(epoch 3)\n(load "lib/forth/interpreter.sx")\n(epoch 4)\n(load "lib/forth/compiler.sx")\n(epoch 5)\n(load "lib/forth/hayes-runner.sx")\n(epoch 6)\n(load "%s")\n(epoch 7)\n(eval "(hayes-run-all)")\n' "$CHUNKS_SX" \
+  | timeout 180 "$SX_SERVER" 2>&1)
+STATUS=$?
+
+SUMMARY=$(printf '%s\n' "$OUT" | awk '/^\{:pass / {print; exit}')
+PASS=$(printf '%s' "$SUMMARY" | sed -n 's/.*:pass \([0-9-]*\).*/\1/p')
+FAIL=$(printf '%s' "$SUMMARY" | sed -n 's/.*:fail \([0-9-]*\).*/\1/p')
+ERR=$(printf '%s' "$SUMMARY" | sed -n 's/.*:error \([0-9-]*\).*/\1/p')
+TOTAL=$(printf '%s' "$SUMMARY" | sed -n 's/.*:total \([0-9-]*\).*/\1/p')
+CHUNK_COUNT=$(grep -c '^  "' "$CHUNKS_SX" || echo 0)
+TOTAL_AVAILABLE=$(grep -c '}T' "$PREPROC" || echo 0)
+
+NOW="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+
+if [ -z "$PASS" ]; then
+  PASS=0; FAIL=0; ERR=0; TOTAL=0
+  NOTE="runner halted before completing (timeout or SX error)"
+else
+  NOTE="completed"
+fi
+
+PCT=0
+if [ "$TOTAL" -gt 0 ]; then
+  PCT=$((PASS * 100 / TOTAL))
+fi
+
+cat > "$OUT_JSON" <<JSON
+{
+  "source": "gerryjackson/forth2012-test-suite src/core.fr",
+  "generated_at": "$NOW",
+  "chunks_available": $TOTAL_AVAILABLE,
+  "chunks_fed": $CHUNK_COUNT,
+  "total": $TOTAL,
+  "pass": $PASS,
+  "fail": $FAIL,
+  "error": $ERR,
+  "percent": $PCT,
+  "note": "$NOTE"
+}
+JSON
+
+cat > "$OUT_MD" <<MD
+# Forth Hayes Core scoreboard
+
+| metric            | value |
+| ----------------- | ----: |
+| chunks available  | $TOTAL_AVAILABLE |
+| chunks fed        | $CHUNK_COUNT |
+| total             | $TOTAL |
+| pass              | $PASS |
+| fail              | $FAIL |
+| error             | $ERR |
+| percent           | ${PCT}% |
+
+- **Source**: \`gerryjackson/forth2012-test-suite\` \`src/core.fr\`
+- **Generated**: $NOW
+- **Note**: $NOTE
+
+A "chunk" is any preprocessed segment ending at a \`}T\` (every Hayes test
+is one chunk, plus the small declaration blocks between tests).
+The runner catches raised errors at chunk boundaries so one bad chunk
+does not abort the rest. \`error\` covers chunks that raised; \`fail\`
+covers tests whose \`->\` / \`}T\` comparison mismatched.
+
+### Chunk cap
+
+\`conformance.sh\` processes the first \`\$MAX_CHUNKS\` chunks (default
+**638**, i.e. the whole Hayes Core file). Lower the cap temporarily
+while iterating on primitives if a regression re-opens an infinite
+loop in later tests.
+MD
+
+echo "$SUMMARY"
+echo "Scoreboard: $OUT_JSON"
+echo "           $OUT_MD"
+
+if [ "$STATUS" -ne 0 ] && [ "$TOTAL" -eq 0 ]; then
+  exit 1
+fi

lib/forth/hayes-runner.sx

@@ -0,0 +1,158 @@
+;; Hayes conformance test runner.
+;; Installs T{ -> }T as Forth primitives that snapshot and compare dstack,
+;; plus stub TESTING / HEX / DECIMAL so the Hayes Core file can stream
+;; through the interpreter without halting on unsupported metadata words.
+
+(define hayes-pass 0)
+(define hayes-fail 0)
+(define hayes-error 0)
+(define hayes-start-depth 0)
+(define hayes-actual (list))
+(define hayes-actual-set false)
+(define hayes-failures (list))
+(define hayes-first-error "")
+(define hayes-error-hist (dict))
+
+(define
+  hayes-reset!
+  (fn
+    ()
+    (set! hayes-pass 0)
+    (set! hayes-fail 0)
+    (set! hayes-error 0)
+    (set! hayes-start-depth 0)
+    (set! hayes-actual (list))
+    (set! hayes-actual-set false)
+    (set! hayes-failures (list))
+    (set! hayes-first-error "")
+    (set! hayes-error-hist (dict))))
+
+(define
+  hayes-slice
+  (fn
+    (state base)
+    (let
+      ((n (- (forth-depth state) base)))
+      (if (<= n 0) (list) (take (get state "dstack") n)))))
+
+(define
+  hayes-truncate!
+  (fn
+    (state base)
+    (let
+      ((n (- (forth-depth state) base)))
+      (when (> n 0) (dict-set! state "dstack" (drop (get state "dstack") n))))))
+
+(define
+  hayes-install!
+  (fn
+    (state)
+    (forth-def-prim!
+      state
+      "T{"
+      (fn
+        (s)
+        (set! hayes-start-depth (forth-depth s))
+        (set! hayes-actual-set false)
+        (set! hayes-actual (list))))
+    (forth-def-prim!
+      state
+      "->"
+      (fn
+        (s)
+        (set! hayes-actual (hayes-slice s hayes-start-depth))
+        (set! hayes-actual-set true)
+        (hayes-truncate! s hayes-start-depth)))
+    (forth-def-prim!
+      state
+      "}T"
+      (fn
+        (s)
+        (let
+          ((expected (hayes-slice s hayes-start-depth)))
+          (hayes-truncate! s hayes-start-depth)
+          (if
+            (and hayes-actual-set (= expected hayes-actual))
+            (set! hayes-pass (+ hayes-pass 1))
+            (begin
+              (set! hayes-fail (+ hayes-fail 1))
+              (set!
+                hayes-failures
+                (concat
+                  hayes-failures
+                  (list
+                    (dict
+                      "kind"
+                      "fail"
+                      "expected"
+                      (str expected)
+                      "actual"
+                      (str hayes-actual))))))))))
+    (forth-def-prim! state "TESTING" (fn (s) nil))
+    ;; HEX/DECIMAL are real primitives now (runtime.sx) — no stub needed.
+    state))
+
+(define
+  hayes-boot
+  (fn () (let ((s (forth-boot))) (hayes-install! s) (hayes-reset!) s)))
+
+;; Run a single preprocessed chunk (string of Forth source) on the shared
+;; state. Catch any raised error and move on — the chunk boundary is a
+;; safe resume point.
+(define
+  hayes-bump-error-key!
+  (fn
+    (err)
+    (let
+      ((msg (str err)))
+      (let
+        ((space-idx (index-of msg " ")))
+        (let
+          ((key
+            (if
+              (> space-idx 0)
+              (substr msg 0 space-idx)
+              msg)))
+          (dict-set!
+            hayes-error-hist
+            key
+            (+ 1 (or (get hayes-error-hist key) 0))))))))
+
+(define
+  hayes-run-chunk
+  (fn
+    (state src)
+    (guard
+      (err
+        ((= 1 1)
+          (begin
+            (set! hayes-error (+ hayes-error 1))
+            (when
+              (= (len hayes-first-error) 0)
+              (set! hayes-first-error (str err)))
+            (hayes-bump-error-key! err)
+            (dict-set! state "dstack" (list))
+            (dict-set! state "rstack" (list))
+            (dict-set! state "compiling" false)
+            (dict-set! state "current-def" nil)
+            (dict-set! state "cstack" (list))
+            (dict-set! state "input" (list)))))
+      (forth-interpret state src))))
+
+(define
+  hayes-summary
+  (fn
+    ()
+    (dict
+      "pass"
+      hayes-pass
+      "fail"
+      hayes-fail
+      "error"
+      hayes-error
+      "total"
+      (+ (+ hayes-pass hayes-fail) hayes-error)
+      "first-error"
+      hayes-first-error
+      "error-hist"
+      hayes-error-hist)))

lib/forth/interpreter.sx

@@ -5,7 +5,39 @@
 
 (define
   forth-execute-word
-  (fn (state word) (let ((body (get word "body"))) (body state))))
+  (fn
+    (state word)
+    (dict-set! word "call-count" (+ 1 (or (get word "call-count") 0)))
+    (let ((body (get word "body"))) (body state))))
+
+(define
+  forth-hot-words
+  (fn
+    (state threshold)
+    (forth-hot-walk
+      (keys (get state "dict"))
+      (get state "dict")
+      threshold
+      (list))))
+
+(define
+  forth-hot-walk
+  (fn
+    (names dict threshold acc)
+    (if
+      (= (len names) 0)
+      acc
+      (let
+        ((n (first names)))
+        (let
+          ((w (get dict n)))
+          (let
+            ((c (or (get w "call-count") 0)))
+            (forth-hot-walk
+              (rest names)
+              dict
+              threshold
+              (if (>= c threshold) (cons (list n c) acc) acc))))))))
 
 (define
   forth-interpret-token
@@ -17,7 +49,7 @@
         (not (nil? w))
         (forth-execute-word state w)
         (let
-          ((n (forth-parse-number tok (get state "base"))))
+          ((n (forth-parse-number tok (get (get state "vars") "base"))))
           (if
             (not (nil? n))
             (forth-push state n)

lib/forth/scoreboard.json

@@ -0,0 +1,12 @@
+{
+  "source": "gerryjackson/forth2012-test-suite src/core.fr",
+  "generated_at": "2026-05-05T21:30:21Z",
+  "chunks_available": 638,
+  "chunks_fed": 638,
+  "total": 638,
+  "pass": 632,
+  "fail": 6,
+  "error": 0,
+  "percent": 99,
+  "note": "completed"
+}

lib/forth/scoreboard.md

@@ -0,0 +1,28 @@
+# Forth Hayes Core scoreboard
+
+| metric            | value |
+| ----------------- | ----: |
+| chunks available  | 638 |
+| chunks fed        | 638 |
+| total             | 638 |
+| pass              | 632 |
+| fail              | 6 |
+| error             | 0 |
+| percent           | 99% |
+
+- **Source**: `gerryjackson/forth2012-test-suite` `src/core.fr`
+- **Generated**: 2026-05-05T21:30:21Z
+- **Note**: completed
+
+A "chunk" is any preprocessed segment ending at a `}T` (every Hayes test
+is one chunk, plus the small declaration blocks between tests).
+The runner catches raised errors at chunk boundaries so one bad chunk
+does not abort the rest. `error` covers chunks that raised; `fail`
+covers tests whose `->` / `}T` comparison mismatched.
+
+### Chunk cap
+
+`conformance.sh` processes the first `$MAX_CHUNKS` chunks (default
+**638**, i.e. the whole Hayes Core file). Lower the cap temporarily
+while iterating on primitives if a regression re-opens an infinite
+loop in later tests.

lib/forth/tests/test-phase3.sx

@@ -0,0 +1,239 @@
+;; Phase 3 — control flow (IF/ELSE/THEN, BEGIN/UNTIL/WHILE/REPEAT/AGAIN,
+;; DO/LOOP, return stack). Grows as each control construct lands.
+
+(define forth-p3-passed 0)
+(define forth-p3-failed 0)
+(define forth-p3-failures (list))
+
+(define
+  forth-p3-assert
+  (fn
+    (label expected actual)
+    (if
+      (= expected actual)
+      (set! forth-p3-passed (+ forth-p3-passed 1))
+      (begin
+        (set! forth-p3-failed (+ forth-p3-failed 1))
+        (set!
+          forth-p3-failures
+          (concat
+            forth-p3-failures
+            (list
+              (str label ": expected " (str expected) " got " (str actual)))))))))
+
+(define
+  forth-p3-check-stack
+  (fn
+    (label src expected)
+    (let ((r (forth-run src))) (forth-p3-assert label expected (nth r 2)))))
+
+(define
+  forth-p3-if-tests
+  (fn
+    ()
+    (forth-p3-check-stack
+      "IF taken (-1)"
+      ": Q -1 IF 10 THEN ; Q"
+      (list 10))
+    (forth-p3-check-stack
+      "IF not taken (0)"
+      ": Q 0 IF 10 THEN ; Q"
+      (list))
+    (forth-p3-check-stack
+      "IF with non-zero truthy"
+      ": Q 42 IF 10 THEN ; Q"
+      (list 10))
+    (forth-p3-check-stack
+      "IF ELSE — true branch"
+      ": Q -1 IF 10 ELSE 20 THEN ; Q"
+      (list 10))
+    (forth-p3-check-stack
+      "IF ELSE — false branch"
+      ": Q 0 IF 10 ELSE 20 THEN ; Q"
+      (list 20))
+    (forth-p3-check-stack
+      "IF consumes flag"
+      ": Q IF 1 ELSE 2 THEN ; 0 Q"
+      (list 2))
+    (forth-p3-check-stack
+      "absolute value via IF"
+      ": ABS2 DUP 0 < IF NEGATE THEN ; -7 ABS2"
+      (list 7))
+    (forth-p3-check-stack
+      "abs leaves positive alone"
+      ": ABS2 DUP 0 < IF NEGATE THEN ; 7 ABS2"
+      (list 7))
+    (forth-p3-check-stack
+      "sign: negative"
+      ": SIGN DUP 0 < IF DROP -1 ELSE DROP 1 THEN ; -3 SIGN"
+      (list -1))
+    (forth-p3-check-stack
+      "sign: positive"
+      ": SIGN DUP 0 < IF DROP -1 ELSE DROP 1 THEN ; 3 SIGN"
+      (list 1))
+    (forth-p3-check-stack
+      "nested IF (both true)"
+      ": Q 1 IF 1 IF 10 ELSE 20 THEN ELSE 30 THEN ; Q"
+      (list 10))
+    (forth-p3-check-stack
+      "nested IF (inner false)"
+      ": Q 1 IF 0 IF 10 ELSE 20 THEN ELSE 30 THEN ; Q"
+      (list 20))
+    (forth-p3-check-stack
+      "nested IF (outer false)"
+      ": Q 0 IF 0 IF 10 ELSE 20 THEN ELSE 30 THEN ; Q"
+      (list 30))
+    (forth-p3-check-stack
+      "IF before other ops"
+      ": Q 1 IF 5 ELSE 6 THEN 2 * ; Q"
+      (list 10))
+    (forth-p3-check-stack
+      "IF in chained def"
+      ": POS? 0 > ;
+       : DOUBLE-IF-POS DUP POS? IF 2 * THEN ;
+       3 DOUBLE-IF-POS"
+      (list 6))
+    (forth-p3-check-stack
+      "empty then branch"
+      ": Q 1 IF THEN 99 ; Q"
+      (list 99))
+    (forth-p3-check-stack
+      "empty else branch"
+      ": Q 0 IF 99 ELSE THEN ; Q"
+      (list))
+    (forth-p3-check-stack
+      "sequential IF blocks"
+      ": Q -1 IF 1 THEN -1 IF 2 THEN ; Q"
+      (list 1 2))))
+
+(define
+  forth-p3-loop-tests
+  (fn
+    ()
+    (forth-p3-check-stack
+      "BEGIN UNTIL (countdown to zero)"
+      ": CD BEGIN 1- DUP 0 = UNTIL ; 3 CD"
+      (list 0))
+    (forth-p3-check-stack
+      "BEGIN UNTIL — single pass (UNTIL true immediately)"
+      ": Q BEGIN -1 UNTIL 42 ; Q"
+      (list 42))
+    (forth-p3-check-stack
+      "BEGIN UNTIL — accumulate sum 1+2+3"
+      ": SUM3 0 3 BEGIN TUCK + SWAP 1- DUP 0 = UNTIL DROP ; SUM3"
+      (list 6))
+    (forth-p3-check-stack
+      "BEGIN WHILE REPEAT — triangular sum 5"
+      ": TRI 0 5 BEGIN DUP 0 > WHILE TUCK + SWAP 1- REPEAT DROP ; TRI"
+      (list 15))
+    (forth-p3-check-stack
+      "BEGIN WHILE REPEAT — zero iterations"
+      ": TRI 0 0 BEGIN DUP 0 > WHILE TUCK + SWAP 1- REPEAT DROP ; TRI"
+      (list 0))
+    (forth-p3-check-stack
+      "BEGIN WHILE REPEAT — one iteration"
+      ": TRI 0 1 BEGIN DUP 0 > WHILE TUCK + SWAP 1- REPEAT DROP ; TRI"
+      (list 1))
+    (forth-p3-check-stack
+      "nested BEGIN UNTIL"
+      ": INNER BEGIN 1- DUP 0 = UNTIL DROP ;
+       : OUTER BEGIN 3 INNER 1- DUP 0 = UNTIL ;
+       2 OUTER"
+      (list 0))
+    (forth-p3-check-stack
+      "BEGIN UNTIL after colon prefix"
+      ": TEN 10 ;
+       : CD TEN BEGIN 1- DUP 0 = UNTIL ;
+       CD"
+      (list 0))
+    (forth-p3-check-stack
+      "WHILE inside IF branch"
+      ": Q 1 IF 0 3 BEGIN DUP 0 > WHILE TUCK + SWAP 1- REPEAT DROP ELSE 99 THEN ; Q"
+      (list 6))))
+
+(define
+  forth-p3-do-tests
+  (fn
+    ()
+    (forth-p3-check-stack
+      "DO LOOP — simple sum 0..4"
+      ": SUM 0 5 0 DO I + LOOP ; SUM"
+      (list 10))
+    (forth-p3-check-stack
+      "DO LOOP — 10..14 sum using I"
+      ": SUM 0 15 10 DO I + LOOP ; SUM"
+      (list 60))
+    (forth-p3-check-stack
+      "DO LOOP — limit = start runs one pass"
+      ": SUM 0 5 5 DO I + LOOP ; SUM"
+      (list 5))
+    (forth-p3-check-stack
+      "DO LOOP — count iterations"
+      ": COUNT 0 4 0 DO 1+ LOOP ; COUNT"
+      (list 4))
+    (forth-p3-check-stack
+      "DO LOOP — nested, I inner / J outer"
+      ": MATRIX 0 3 0 DO 3 0 DO I J + + LOOP LOOP ; MATRIX"
+      (list 18))
+    (forth-p3-check-stack
+      "DO LOOP — I used in arithmetic"
+      ": DBL 0 5 1 DO I 2 * + LOOP ; DBL"
+      (list 20))
+    (forth-p3-check-stack
+      "+LOOP — count by 2"
+      ": Q 0 10 0 DO I + 2 +LOOP ; Q"
+      (list 20))
+    (forth-p3-check-stack
+      "+LOOP — count by 3"
+      ": Q 0 10 0 DO I + 3 +LOOP ; Q"
+      (list 18))
+    (forth-p3-check-stack
+      "+LOOP — negative step"
+      ": Q 0 0 10 DO I + -1 +LOOP ; Q"
+      (list 55))
+    (forth-p3-check-stack
+      "LEAVE — early exit at I=3"
+      ": Q 0 10 0 DO I 3 = IF LEAVE THEN I + LOOP ; Q"
+      (list 3))
+    (forth-p3-check-stack
+      "LEAVE — in nested loop exits only inner"
+      ": Q 0 3 0 DO 5 0 DO I 2 = IF LEAVE THEN I + LOOP LOOP ; Q"
+      (list 3))
+    (forth-p3-check-stack
+      "DO LOOP preserves outer stack"
+      ": Q 99 5 0 DO I + LOOP ; Q"
+      (list 109))
+    (forth-p3-check-stack
+      ">R R>"
+      ": Q 7 >R 11 R> ; Q"
+      (list 11 7))
+    (forth-p3-check-stack
+      ">R R@ R>"
+      ": Q 7 >R R@ R> ; Q"
+      (list 7 7))
+    (forth-p3-check-stack
+      "2>R 2R>"
+      ": Q 1 2 2>R 99 2R> ; Q"
+      (list 99 1 2))
+    (forth-p3-check-stack
+      "2>R 2R@ 2R>"
+      ": Q 3 4 2>R 2R@ 2R> ; Q"
+      (list 3 4 3 4))))
+
+(define
+  forth-p3-run-all
+  (fn
+    ()
+    (set! forth-p3-passed 0)
+    (set! forth-p3-failed 0)
+    (set! forth-p3-failures (list))
+    (forth-p3-if-tests)
+    (forth-p3-loop-tests)
+    (forth-p3-do-tests)
+    (dict
+      "passed"
+      forth-p3-passed
+      "failed"
+      forth-p3-failed
+      "failures"
+      forth-p3-failures)))

lib/forth/tests/test-phase4.sx

@@ -0,0 +1,268 @@
+;; Phase 4 — strings + more Core.
+;; Uses the byte-memory model on state ("mem" dict + "here" cursor).
+
+(define forth-p4-passed 0)
+(define forth-p4-failed 0)
+(define forth-p4-failures (list))
+
+(define
+  forth-p4-assert
+  (fn
+    (label expected actual)
+    (if
+      (= expected actual)
+      (set! forth-p4-passed (+ forth-p4-passed 1))
+      (begin
+        (set! forth-p4-failed (+ forth-p4-failed 1))
+        (set!
+          forth-p4-failures
+          (concat
+            forth-p4-failures
+            (list
+              (str label ": expected " (str expected) " got " (str actual)))))))))
+
+(define
+  forth-p4-check-output
+  (fn
+    (label src expected)
+    (let ((r (forth-run src))) (forth-p4-assert label expected (nth r 1)))))
+
+(define
+  forth-p4-check-stack-size
+  (fn
+    (label src expected-n)
+    (let
+      ((r (forth-run src)))
+      (forth-p4-assert label expected-n (len (nth r 2))))))
+
+(define
+  forth-p4-check-top
+  (fn
+    (label src expected)
+    (let
+      ((r (forth-run src)))
+      (let
+        ((stk (nth r 2)))
+        (forth-p4-assert label expected (nth stk (- (len stk) 1)))))))
+
+(define
+  forth-p4-check-typed
+  (fn
+    (label src expected)
+    (forth-p4-check-output label (str src " TYPE") expected)))
+
+(define
+  forth-p4-string-tests
+  (fn
+    ()
+    (forth-p4-check-typed
+      "S\" + TYPE — hello"
+      "S\" HELLO\""
+      "HELLO")
+    (forth-p4-check-typed
+      "S\" + TYPE — two words"
+      "S\" HELLO WORLD\""
+      "HELLO WORLD")
+    (forth-p4-check-typed
+      "S\" + TYPE — empty"
+      "S\" \""
+      "")
+    (forth-p4-check-typed
+      "S\" + TYPE — single char"
+      "S\" X\""
+      "X")
+    (forth-p4-check-stack-size
+      "S\" pushes (addr len)"
+      "S\" HI\""
+      2)
+    (forth-p4-check-top
+      "S\" length is correct"
+      "S\" HELLO\""
+      5)
+    (forth-p4-check-output
+      ".\" prints at interpret time"
+      ".\" HELLO\""
+      "HELLO")
+    (forth-p4-check-output
+      ".\" in colon def"
+      ": GREET .\" HI \" ; GREET GREET"
+      "HI HI ")))
+
+(define
+  forth-p4-count-tests
+  (fn
+    ()
+    (forth-p4-check-typed
+      "C\" + COUNT + TYPE"
+      "C\" ABC\" COUNT"
+      "ABC")
+    (forth-p4-check-typed
+      "C\" then COUNT leaves right len"
+      "C\" HI THERE\" COUNT"
+      "HI THERE")))
+
+(define
+  forth-p4-fill-tests
+  (fn
+    ()
+    (forth-p4-check-typed
+      "FILL overwrites prefix bytes"
+      "S\" ABCDE\" 2DUP DROP 3 65 FILL"
+      "AAADE")
+    (forth-p4-check-typed
+      "BLANK sets spaces"
+      "S\" XYZAB\" 2DUP DROP 3 BLANK"
+      "   AB")))
+
+(define
+  forth-p4-cmove-tests
+  (fn
+    ()
+    (forth-p4-check-output
+      "CMOVE copies HELLO forward"
+      ": MKH 72 0 C! 69 1 C! 76 2 C! 76 3 C! 79 4 C! ;
+       : T MKH 0 10 5 CMOVE 10 5 TYPE ; T"
+      "HELLO")
+    (forth-p4-check-output
+      "CMOVE> copies overlapping backward"
+      ": MKA 65 0 C! 66 1 C! 67 2 C! ;
+       : T MKA 0 1 2 CMOVE> 0 3 TYPE ; T"
+      "AAB")
+    (forth-p4-check-output
+      "MOVE picks direction for overlap"
+      ": MKA 65 0 C! 66 1 C! 67 2 C! ;
+       : T MKA 0 1 2 MOVE 0 3 TYPE ; T"
+      "AAB")))
+
+(define
+  forth-p4-charplus-tests
+  (fn
+    ()
+    (forth-p4-check-top
+      "CHAR+ increments"
+      "5 CHAR+"
+      6)))
+
+(define
+  forth-p4-char-tests
+  (fn
+    ()
+    (forth-p4-check-top "CHAR A -> 65" "CHAR A" 65)
+    (forth-p4-check-top "CHAR x -> 120" "CHAR x" 120)
+    (forth-p4-check-top "CHAR takes only first char" "CHAR HELLO" 72)
+    (forth-p4-check-top
+      "[CHAR] compiles literal"
+      ": AA [CHAR] A ; AA"
+      65)
+    (forth-p4-check-top
+      "[CHAR] reads past IMMEDIATE"
+      ": ZZ [CHAR] Z ; ZZ"
+      90)
+    (forth-p4-check-stack-size
+      "[CHAR] doesn't leak at compile time"
+      ": FOO [CHAR] A ; "
+      0)))
+
+(define
+  forth-p4-key-accept-tests
+  (fn
+    ()
+    (let
+      ((r (forth-run "1000 2 ACCEPT")))
+      (let ((stk (nth r 2))) (forth-p4-assert "ACCEPT empty buf -> 0" (list 0) stk)))))
+
+(define
+  forth-p4-shift-tests
+  (fn
+    ()
+    (forth-p4-check-top "1 0 LSHIFT" "1 0 LSHIFT" 1)
+    (forth-p4-check-top "1 1 LSHIFT" "1 1 LSHIFT" 2)
+    (forth-p4-check-top "1 2 LSHIFT" "1 2 LSHIFT" 4)
+    (forth-p4-check-top "1 15 LSHIFT" "1 15 LSHIFT" 32768)
+    (forth-p4-check-top "1 31 LSHIFT" "1 31 LSHIFT" -2147483648)
+    (forth-p4-check-top "1 0 RSHIFT" "1 0 RSHIFT" 1)
+    (forth-p4-check-top "1 1 RSHIFT" "1 1 RSHIFT" 0)
+    (forth-p4-check-top "2 1 RSHIFT" "2 1 RSHIFT" 1)
+    (forth-p4-check-top "4 2 RSHIFT" "4 2 RSHIFT" 1)
+    (forth-p4-check-top "-1 1 RSHIFT (logical, not arithmetic)" "-1 1 RSHIFT" 2147483647)
+    (forth-p4-check-top "MSB via 1S 1 RSHIFT INVERT" "0 INVERT 1 RSHIFT INVERT" -2147483648)))
+
+(define
+  forth-p4-sp-tests
+  (fn
+    ()
+    (forth-p4-check-top "SP@ returns depth (0)" "SP@" 0)
+    (forth-p4-check-top
+      "SP@ after pushes"
+      "1 2 3 SP@ SWAP DROP SWAP DROP SWAP DROP"
+      3)
+    (forth-p4-check-stack-size
+      "SP! truncates"
+      "1 2 3 4 5 2 SP!"
+      2)
+    (forth-p4-check-top
+      "SP! leaves base items intact"
+      "1 2 3 4 5 2 SP!"
+      2)))
+
+(define
+  forth-p4-base-tests
+  (fn
+    ()
+    (forth-p4-check-top
+      "BASE default is 10"
+      "BASE @"
+      10)
+    (forth-p4-check-top
+      "HEX switches base to 16"
+      "HEX BASE @"
+      16)
+    (forth-p4-check-top
+      "DECIMAL resets to 10"
+      "HEX DECIMAL BASE @"
+      10)
+    (forth-p4-check-top
+      "HEX parses 10 as 16"
+      "HEX 10"
+      16)
+    (forth-p4-check-top
+      "HEX parses FF as 255"
+      "HEX FF"
+      255)
+    (forth-p4-check-top
+      "DECIMAL parses 10 as 10"
+      "HEX DECIMAL 10"
+      10)
+    (forth-p4-check-top
+      "OCTAL parses 17 as 15"
+      "OCTAL 17"
+      15)
+    (forth-p4-check-top
+      "BASE @ ; 16 BASE ! ; BASE @"
+      "BASE @ 16 BASE ! BASE @ SWAP DROP"
+      16)))
+
+(define
+  forth-p4-run-all
+  (fn
+    ()
+    (set! forth-p4-passed 0)
+    (set! forth-p4-failed 0)
+    (set! forth-p4-failures (list))
+    (forth-p4-string-tests)
+    (forth-p4-count-tests)
+    (forth-p4-fill-tests)
+    (forth-p4-cmove-tests)
+    (forth-p4-charplus-tests)
+    (forth-p4-char-tests)
+    (forth-p4-key-accept-tests)
+    (forth-p4-base-tests)
+    (forth-p4-shift-tests)
+    (forth-p4-sp-tests)
+    (dict
+      "passed"
+      forth-p4-passed
+      "failed"
+      forth-p4-failed
+      "failures"
+      forth-p4-failures)))

lib/forth/tests/test-phase5.sx

@@ -0,0 +1,333 @@
+;; Phase 5 — Core Extension + memory primitives.
+
+(define forth-p5-passed 0)
+(define forth-p5-failed 0)
+(define forth-p5-failures (list))
+
+(define
+  forth-p5-assert
+  (fn
+    (label expected actual)
+    (if
+      (= expected actual)
+      (set! forth-p5-passed (+ forth-p5-passed 1))
+      (begin
+        (set! forth-p5-failed (+ forth-p5-failed 1))
+        (set!
+          forth-p5-failures
+          (concat
+            forth-p5-failures
+            (list
+              (str label ": expected " (str expected) " got " (str actual)))))))))
+
+(define
+  forth-p5-check-stack
+  (fn
+    (label src expected)
+    (let ((r (forth-run src))) (forth-p5-assert label expected (nth r 2)))))
+
+(define
+  forth-p5-check-top
+  (fn
+    (label src expected)
+    (let
+      ((r (forth-run src)))
+      (let
+        ((stk (nth r 2)))
+        (forth-p5-assert label expected (nth stk (- (len stk) 1)))))))
+
+(define
+  forth-p5-create-tests
+  (fn
+    ()
+    (forth-p5-check-top
+      "CREATE pushes HERE-at-creation"
+      "HERE CREATE FOO FOO ="
+      -1)
+    (forth-p5-check-top
+      "CREATE + ALLOT advances HERE"
+      "HERE 5 ALLOT HERE SWAP -"
+      5)
+    (forth-p5-check-top
+      "CREATE + , stores cell"
+      "CREATE FOO 42 , FOO @"
+      42)
+    (forth-p5-check-stack
+      "CREATE multiple ,"
+      "CREATE TBL 1 , 2 , 3 , TBL @ TBL CELL+ @ TBL CELL+ CELL+ @"
+      (list 1 2 3))
+    (forth-p5-check-top
+      "C, stores byte"
+      "CREATE B 65 C, 66 C, B C@"
+      65)))
+
+(define
+  forth-p5-unsigned-tests
+  (fn
+    ()
+    (forth-p5-check-top "1 2 U<" "1 2 U<" -1)
+    (forth-p5-check-top "2 1 U<" "2 1 U<" 0)
+    (forth-p5-check-top "0 1 U<" "0 1 U<" -1)
+    (forth-p5-check-top "-1 1 U< (since -1 unsigned is huge)" "-1 1 U<" 0)
+    (forth-p5-check-top "1 -1 U<" "1 -1 U<" -1)
+    (forth-p5-check-top "1 2 U>" "1 2 U>" 0)
+    (forth-p5-check-top "-1 1 U>" "-1 1 U>" -1)))
+
+(define
+  forth-p5-2bang-tests
+  (fn
+    ()
+    (forth-p5-check-stack
+      "2! / 2@"
+      "CREATE X 0 , 0 , 11 22 X 2! X 2@"
+      (list 11 22))))
+
+(define
+  forth-p5-mixed-tests
+  (fn
+    ()
+    (forth-p5-check-stack "S>D positive" "5 S>D" (list 5 0))
+    (forth-p5-check-stack "S>D negative" "-5 S>D" (list -5 -1))
+    (forth-p5-check-stack "S>D zero" "0 S>D" (list 0 0))
+    (forth-p5-check-top "D>S keeps low" "5 0 D>S" 5)
+    (forth-p5-check-stack "M* small positive" "3 4 M*" (list 12 0))
+    (forth-p5-check-stack "M* negative" "-3 4 M*" (list -12 -1))
+    (forth-p5-check-stack
+      "M* negative * negative"
+      "-3 -4 M*"
+      (list 12 0))
+    (forth-p5-check-stack "UM* small" "3 4 UM*" (list 12 0))
+    (forth-p5-check-stack
+      "UM/MOD: 100 0 / 5"
+      "100 0 5 UM/MOD"
+      (list 0 20))
+    (forth-p5-check-stack
+      "FM/MOD: -7 / 2 floored"
+      "-7 -1 2 FM/MOD"
+      (list 1 -4))
+    (forth-p5-check-stack
+      "SM/REM: -7 / 2 truncated"
+      "-7 -1 2 SM/REM"
+      (list -1 -3))
+    (forth-p5-check-top "*/ truncated" "7 11 13 */" 5)
+    (forth-p5-check-stack "*/MOD" "7 11 13 */MOD" (list 12 5))))
+
+(define
+  forth-p5-double-tests
+  (fn
+    ()
+    (forth-p5-check-stack "D+ small" "5 0 7 0 D+" (list 12 0))
+    (forth-p5-check-stack "D+ negative" "-5 -1 -3 -1 D+" (list -8 -1))
+    (forth-p5-check-stack "D- small" "10 0 3 0 D-" (list 7 0))
+    (forth-p5-check-stack "DNEGATE positive" "5 0 DNEGATE" (list -5 -1))
+    (forth-p5-check-stack "DNEGATE negative" "-5 -1 DNEGATE" (list 5 0))
+    (forth-p5-check-stack "DABS negative" "-7 -1 DABS" (list 7 0))
+    (forth-p5-check-stack "DABS positive" "7 0 DABS" (list 7 0))
+    (forth-p5-check-top "D= equal" "5 0 5 0 D=" -1)
+    (forth-p5-check-top "D= unequal lo" "5 0 7 0 D=" 0)
+    (forth-p5-check-top "D= unequal hi" "5 0 5 1 D=" 0)
+    (forth-p5-check-top "D< lt" "5 0 7 0 D<" -1)
+    (forth-p5-check-top "D< gt" "7 0 5 0 D<" 0)
+    (forth-p5-check-top "D0= zero" "0 0 D0=" -1)
+    (forth-p5-check-top "D0= nonzero" "5 0 D0=" 0)
+    (forth-p5-check-top "D0< neg" "-5 -1 D0<" -1)
+    (forth-p5-check-top "D0< pos" "5 0 D0<" 0)
+    (forth-p5-check-stack "DMAX" "5 0 7 0 DMAX" (list 7 0))
+    (forth-p5-check-stack "DMIN" "5 0 7 0 DMIN" (list 5 0))))
+
+(define
+  forth-p5-format-tests
+  (fn
+    ()
+    (forth-p4-check-output-passthrough
+      "U. prints with trailing space"
+      "123 U."
+      "123 ")
+    (forth-p4-check-output-passthrough
+      "<# #S #> TYPE — decimal"
+      "123 0 <# #S #> TYPE"
+      "123")
+    (forth-p4-check-output-passthrough
+      "<# #S #> TYPE — hex"
+      "255 HEX 0 <# #S #> TYPE"
+      "FF")
+    (forth-p4-check-output-passthrough
+      "<# # # #> partial"
+      "1234 0 <# # # #> TYPE"
+      "34")
+    (forth-p4-check-output-passthrough
+      "SIGN holds minus"
+      "<# -1 SIGN -1 SIGN 0 0 #> TYPE"
+      "--")
+    (forth-p4-check-output-passthrough
+      ".R right-justifies"
+      "42 5 .R"
+      "   42")
+    (forth-p4-check-output-passthrough
+      ".R negative"
+      "-42 5 .R"
+      "  -42")
+    (forth-p4-check-output-passthrough
+      "U.R"
+      "42 5 U.R"
+      "   42")
+    (forth-p4-check-output-passthrough
+      "HOLD char"
+      "<# 0 0 65 HOLD #> TYPE"
+      "A")))
+
+(define
+  forth-p5-dict-tests
+  (fn
+    ()
+    (forth-p5-check-top
+      "EXECUTE via tick"
+      ": INC 1+ ; 9 ' INC EXECUTE"
+      10)
+    (forth-p5-check-top
+      "['] inside def"
+      ": DUB 2* ; : APPLY ['] DUB EXECUTE ; 5 APPLY"
+      10)
+    (forth-p5-check-top
+      ">BODY of CREATE word"
+      "CREATE C 99 , ' C >BODY @"
+      99)
+    (forth-p5-check-stack
+      "WORD parses next token to counted-string"
+      ": A 5 ; BL WORD A COUNT TYPE"
+      (list))
+    (forth-p5-check-top
+      "FIND on known word -> non-zero"
+      ": A 5 ; BL WORD A FIND SWAP DROP"
+      -1)))
+
+(define
+  forth-p5-state-tests
+  (fn
+    ()
+    (forth-p5-check-top
+      "STATE @ in interpret mode"
+      "STATE @"
+      0)
+    (forth-p5-check-top
+      "STATE @ via IMMEDIATE inside compile"
+      ": GT8 STATE @ ; IMMEDIATE : T GT8 LITERAL ; T"
+      -1)
+    (forth-p5-check-top
+      "[ ] LITERAL captures"
+      ": SEVEN [ 7 ] LITERAL ; SEVEN"
+      7)
+    (forth-p5-check-top
+      "EVALUATE in interpret mode"
+      "S\" 5 7 +\" EVALUATE"
+      12)
+    (forth-p5-check-top
+      "EVALUATE inside def"
+      ": A 100 ; : B S\" A\" EVALUATE ; B"
+      100)))
+
+(define
+  forth-p5-misc-tests
+  (fn
+    ()
+    (forth-p5-check-top "WITHIN inclusive lower" "3 2 10 WITHIN" -1)
+    (forth-p5-check-top "WITHIN exclusive upper" "10 2 10 WITHIN" 0)
+    (forth-p5-check-top "WITHIN below range" "1 2 10 WITHIN" 0)
+    (forth-p5-check-top "WITHIN at lower" "2 2 10 WITHIN" -1)
+    (forth-p5-check-top
+      "EXIT leaves colon-def early"
+      ": F 5 EXIT 99 ; F"
+      5)
+    (forth-p5-check-stack
+      "EXIT in IF branch"
+      ": F 5 0 IF DROP 99 EXIT THEN ; F"
+      (list 5))
+    (forth-p5-check-top
+      "UNLOOP + EXIT in DO"
+      ": SUM 0 10 0 DO I 5 = IF I UNLOOP EXIT THEN LOOP ; SUM"
+      5)))
+
+(define
+  forth-p5-fa-tests
+  (fn
+    ()
+    (forth-p5-check-top
+      "R/O R/W W/O constants"
+      "R/O R/W W/O + +"
+      3)
+    (forth-p5-check-top
+      "CREATE-FILE returns ior=0"
+      "CREATE PAD 50 ALLOT PAD S\" /tmp/test.fxf\" ROT SWAP CMOVE S\" /tmp/test.fxf\" R/W CREATE-FILE SWAP DROP"
+      0)
+    (forth-p5-check-top
+      "WRITE-FILE then CLOSE"
+      "S\" /tmp/t2.fxf\" R/W CREATE-FILE DROP >R S\" HI\" R@ WRITE-FILE R> CLOSE-FILE +"
+      0)
+    (forth-p5-check-top
+      "OPEN-FILE on unknown path returns ior!=0"
+      "S\" /tmp/nope.fxf\" R/O OPEN-FILE SWAP DROP 0 ="
+      0)))
+
+(define
+  forth-p5-string-tests
+  (fn
+    ()
+    (forth-p5-check-top "COMPARE equal" "S\" ABC\" S\" ABC\" COMPARE" 0)
+    (forth-p5-check-top "COMPARE less" "S\" ABC\" S\" ABD\" COMPARE" -1)
+    (forth-p5-check-top "COMPARE greater" "S\" ABD\" S\" ABC\" COMPARE" 1)
+    (forth-p5-check-top
+      "COMPARE prefix less"
+      "S\" AB\" S\" ABC\" COMPARE"
+      -1)
+    (forth-p5-check-top
+      "COMPARE prefix greater"
+      "S\" ABC\" S\" AB\" COMPARE"
+      1)
+    (forth-p5-check-top
+      "SEARCH found flag"
+      "S\" HELLO WORLD\" S\" WORLD\" SEARCH"
+      -1)
+    (forth-p5-check-top
+      "SEARCH not found flag"
+      "S\" HELLO\" S\" XYZ\" SEARCH"
+      0)
+    (forth-p5-check-top
+      "SEARCH empty needle flag"
+      "S\" HELLO\" S\" \" SEARCH"
+      -1)
+    (forth-p5-check-top
+      "SLITERAL via [ S\" ... \" ]"
+      ": A [ S\" HI\" ] SLITERAL ; A SWAP DROP"
+      2)))
+
+(define
+  forth-p4-check-output-passthrough
+  (fn
+    (label src expected)
+    (let ((r (forth-run src))) (forth-p5-assert label expected (nth r 1)))))
+
+(define
+  forth-p5-run-all
+  (fn
+    ()
+    (set! forth-p5-passed 0)
+    (set! forth-p5-failed 0)
+    (set! forth-p5-failures (list))
+    (forth-p5-create-tests)
+    (forth-p5-unsigned-tests)
+    (forth-p5-2bang-tests)
+    (forth-p5-mixed-tests)
+    (forth-p5-double-tests)
+    (forth-p5-format-tests)
+    (forth-p5-dict-tests)
+    (forth-p5-state-tests)
+    (forth-p5-misc-tests)
+    (forth-p5-fa-tests)
+    (forth-p5-string-tests)
+    (dict
+      "passed"
+      forth-p5-passed
+      "failed"
+      forth-p5-failed
+      "failures"
+      forth-p5-failures)))

lib/guest/reflective/class-chain.sx

@@ -0,0 +1,129 @@
+;; lib/guest/reflective/class-chain.sx — class inheritance walker.
+;;
+;; Extracted from Smalltalk's `st-method-lookup-walk` (single-parent
+;; class chain for message-send dispatch) and CLOS's `clos-specificity`
+;; (multi-parent class graph for method-precedence distance). Both walk
+;; a class-name → parent-name(s) graph applying a probe at each node;
+;; the cfg adapter normalises single-parent and multi-parent classes
+;; into a uniform `:parents-of` callback that returns a (possibly
+;; empty) list of parent class names.
+;;
+;; Adapter cfg
+;; -----------
+;;   :parents-of  — fn (class-name) → list of parent class names.
+;;                  Empty list = no parents (root). Single-parent guests
+;;                  return a 1-element list; multi-parent guests (CLOS)
+;;                  may return any number.
+;;   :class?      — fn (name) → bool. False short-circuits the walk —
+;;                  used to skip non-existent class names mid-chain.
+;;
+;; Public API
+;;   (refl-class-chain-find-with CFG CLASS-NAME PROBE)
+;;     Depth-first walk from CLASS-NAME up its parent chain. At each
+;;     class, calls `(probe class-name)`. Returns the first non-nil
+;;     probe result, or nil if no class produces one. Probes evaluate
+;;     left-to-right across siblings in multi-parent guests.
+;;
+;;   (refl-class-chain-depth-with CFG CLASS-NAME ANCESTOR-NAME)
+;;     Minimum hop count from CLASS-NAME to ANCESTOR-NAME along any
+;;     parent path. CLASS-NAME itself counts as depth 0. Returns nil
+;;     if ANCESTOR-NAME is unreachable.
+;;
+;;   (refl-class-chain-ancestors-with CFG CLASS-NAME)
+;;     Flat list of all reachable ancestor names in DFS order (no
+;;     dedup; multi-parent guests may want to dedup themselves).
+;;
+;; Consumer migrations
+;; -------------------
+;; - Smalltalk: see `lib/smalltalk/runtime.sx` — `st-method-lookup-walk`
+;;   becomes a one-line probe through `refl-class-chain-find-with`.
+;; - CLOS: see `lib/common-lisp/clos.sx` — `clos-specificity` becomes a
+;;   thin wrapper around `refl-class-chain-depth-with`.
+
+(define
+  refl-find-in-parents-with
+  (fn
+    (cfg parents probe)
+    (cond
+      ((or (nil? parents) (= (length parents) 0)) nil)
+      (:else
+        (let
+          ((hit (refl-class-chain-find-with cfg (first parents) probe)))
+          (cond
+            ((not (nil? hit)) hit)
+            (:else (refl-find-in-parents-with cfg (rest parents) probe))))))))
+
+(define
+  refl-class-chain-find-with
+  (fn
+    (cfg class-name probe)
+    (cond
+      ((nil? class-name) nil)
+      ((not ((get cfg :class?) class-name)) nil)
+      (:else
+        (let
+          ((hit (probe class-name)))
+          (cond
+            ((not (nil? hit)) hit)
+            (:else
+              (refl-find-in-parents-with
+                cfg
+                ((get cfg :parents-of) class-name)
+                probe))))))))
+
+(define
+  refl-class-chain-depth-walk
+  (fn
+    (cfg cur target depth)
+    (cond
+      ((= cur target) depth)
+      ((nil? cur) nil)
+      ((not ((get cfg :class?) cur)) nil)
+      (:else
+        (let
+          ((parents ((get cfg :parents-of) cur)))
+          (let
+            ((results (map (fn (p) (refl-class-chain-depth-walk cfg p target (+ depth 1))) parents)))
+            (let
+              ((non-nil (filter (fn (x) (not (nil? x))) results)))
+              (cond
+                ((or (nil? non-nil) (= (length non-nil) 0)) nil)
+                (:else
+                  (reduce
+                    (fn (a b) (if (< a b) a b))
+                    (first non-nil)
+                    (rest non-nil)))))))))))
+
+(define
+  refl-class-chain-depth-with
+  (fn
+    (cfg class-name ancestor-name)
+    (refl-class-chain-depth-walk cfg class-name ancestor-name 0)))
+
+(define
+  refl-class-chain-ancestors-with
+  (fn (cfg class-name) (refl-ancestors-walk cfg class-name (list))))
+
+(define
+  refl-ancestors-walk
+  (fn
+    (cfg cn acc)
+    (cond
+      ((nil? cn) acc)
+      ((not ((get cfg :class?) cn)) acc)
+      (:else
+        (let
+          ((parents ((get cfg :parents-of) cn)))
+          (refl-ancestors-walk-list cfg parents (append acc (list cn))))))))
+
+(define
+  refl-ancestors-walk-list
+  (fn
+    (cfg parents acc)
+    (cond
+      ((or (nil? parents) (= (length parents) 0)) acc)
+      (:else
+        (refl-ancestors-walk-list
+          cfg
+          (rest parents)
+          (refl-ancestors-walk cfg (first parents) acc))))))

lib/guest/reflective/env.sx

@@ -0,0 +1,159 @@
+;; lib/guest/reflective/env.sx — first-class environment kit.
+;;
+;; Extracted from Kernel-on-SX (lib/kernel/eval.sx) when Tcl's
+;; uplevel/upvar machinery (lib/tcl/runtime.sx) materialised as a
+;; second consumer needing the same scope-chain semantics.
+;;
+;; Canonical wire shape
+;; --------------------
+;;   {:refl-tag :env :bindings DICT :parent ENV-OR-NIL}
+;;
+;;   - :bindings is a mutable SX dict keyed by symbol name.
+;;   - :parent is either another env or nil (root).
+;;   - Lookup walks the parent chain until a hit or nil.
+;;   - Default cfg uses dict-set! to mutate bindings in place.
+;;
+;; Consumers with their own shape (e.g., Tcl's {:level :locals :parent})
+;; pass an adapter cfg dict — same trick as lib/guest/match.sx's cfg
+;; for unification over guest-specific term shapes.
+;;
+;; Adapter cfg keys
+;; ----------------
+;;   :bindings-of  — fn (scope) → DICT
+;;   :parent-of    — fn (scope) → SCOPE-OR-NIL
+;;   :extend       — fn (scope) → SCOPE         (push a fresh child)
+;;   :bind!        — fn (scope name val) → scope (functional or mutable)
+;;   :env?         — fn (v) → bool              (predicate; cheap shape check)
+;;
+;; Public API — canonical shape, mutable, raises on miss
+;;
+;;   (refl-make-env)
+;;   (refl-extend-env PARENT)
+;;   (refl-env? V)
+;;   (refl-env-bind! ENV NAME VAL)
+;;   (refl-env-has? ENV NAME)
+;;   (refl-env-lookup ENV NAME)
+;;   (refl-env-lookup-or-nil ENV NAME)
+;;
+;; Public API — adapter-cfg, any shape
+;;
+;;   (refl-env-extend-with CFG SCOPE)
+;;   (refl-env-bind!-with  CFG SCOPE NAME VAL)
+;;   (refl-env-has?-with   CFG SCOPE NAME)
+;;   (refl-env-lookup-with CFG SCOPE NAME)
+;;   (refl-env-lookup-or-nil-with CFG SCOPE NAME)
+;;   (refl-env-find-frame-with CFG SCOPE NAME)
+;;     — returns the scope in the chain that contains NAME (or nil).
+;;       Consumers needing source-frame mutation use this.
+;;
+;;   (refl-canonical-cfg) — the default cfg, exposed so consumers
+;;                          can compare or extend it.
+
+;; ── Canonical-shape predicates and constructors ─────────────────
+
+(define refl-env? (fn (v) (and (dict? v) (= (get v :refl-tag) :env))))
+
+(define refl-make-env (fn () {:parent nil :refl-tag :env :bindings {}}))
+
+(define refl-extend-env (fn (parent) {:parent parent :refl-tag :env :bindings {}}))
+
+(define
+  refl-env-bind!
+  (fn (env name val) (dict-set! (get env :bindings) name val) env))
+
+(define
+  refl-env-has?
+  (fn
+    (env name)
+    (cond
+      ((nil? env) false)
+      ((not (refl-env? env)) false)
+      ((dict-has? (get env :bindings) name) true)
+      (:else (refl-env-has? (get env :parent) name)))))
+
+(define
+  refl-env-lookup
+  (fn
+    (env name)
+    (cond
+      ((nil? env) (error (str "refl-env-lookup: unbound symbol: " name)))
+      ((not (refl-env? env))
+        (error (str "refl-env-lookup: corrupt env: " env)))
+      ((dict-has? (get env :bindings) name) (get (get env :bindings) name))
+      (:else (refl-env-lookup (get env :parent) name)))))
+
+(define
+  refl-env-lookup-or-nil
+  (fn
+    (env name)
+    (cond
+      ((nil? env) nil)
+      ((not (refl-env? env)) nil)
+      ((dict-has? (get env :bindings) name) (get (get env :bindings) name))
+      (:else (refl-env-lookup-or-nil (get env :parent) name)))))
+
+;; ── Adapter-cfg variants — any wire shape ───────────────────────
+
+(define refl-env-extend-with (fn (cfg scope) ((get cfg :extend) scope)))
+
+(define
+  refl-env-bind!-with
+  (fn (cfg scope name val) ((get cfg :bind!) scope name val)))
+
+(define
+  refl-env-has?-with
+  (fn
+    (cfg scope name)
+    (cond
+      ((nil? scope) false)
+      ((not ((get cfg :env?) scope)) false)
+      ((dict-has? ((get cfg :bindings-of) scope) name) true)
+      (:else (refl-env-has?-with cfg ((get cfg :parent-of) scope) name)))))
+
+(define
+  refl-env-lookup-with
+  (fn
+    (cfg scope name)
+    (cond
+      ((nil? scope) (error (str "refl-env-lookup: unbound symbol: " name)))
+      ((not ((get cfg :env?) scope))
+        (error (str "refl-env-lookup: corrupt scope: " scope)))
+      ((dict-has? ((get cfg :bindings-of) scope) name)
+        (get ((get cfg :bindings-of) scope) name))
+      (:else (refl-env-lookup-with cfg ((get cfg :parent-of) scope) name)))))
+
+(define
+  refl-env-lookup-or-nil-with
+  (fn
+    (cfg scope name)
+    (cond
+      ((nil? scope) nil)
+      ((not ((get cfg :env?) scope)) nil)
+      ((dict-has? ((get cfg :bindings-of) scope) name)
+        (get ((get cfg :bindings-of) scope) name))
+      (:else
+        (refl-env-lookup-or-nil-with cfg ((get cfg :parent-of) scope) name)))))
+
+;; Returns the SCOPE in the chain that contains NAME, or nil if no
+;; scope binds it. Consumers (e.g. Smalltalk) use this to mutate the
+;; binding at its source frame rather than introducing a new shadow
+;; binding at the current frame. Pairs with `refl-env-lookup-with`
+;; for callers that need both the value and the defining scope.
+
+(define refl-env-find-frame-with
+  (fn (cfg scope name)
+    (cond
+      ((nil? scope) nil)
+      ((not ((get cfg :env?) scope)) nil)
+      ((dict-has? ((get cfg :bindings-of) scope) name) scope)
+      (:else
+        (refl-env-find-frame-with cfg ((get cfg :parent-of) scope) name)))))
+
+(define refl-env-find-frame
+  (fn (env name) (refl-env-find-frame-with refl-canonical-cfg env name)))
+
+;; ── Default canonical cfg ───────────────────────────────────────
+;; Exposed so consumers can use it explicitly, compose with it, or
+;; check adapter-correctness against the canonical implementation.
+
+(define refl-canonical-cfg {:bind! (fn (e n v) (refl-env-bind! e n v)) :parent-of (fn (e) (get e :parent)) :env? (fn (v) (refl-env? v)) :bindings-of (fn (e) (get e :bindings)) :extend (fn (e) (refl-extend-env e))})

lib/guest/reflective/quoting.sx

@@ -0,0 +1,77 @@
+;; lib/guest/reflective/quoting.sx — quasiquote walker.
+;;
+;; Extracted from Kernel's `knl-quasi-walk` and Scheme's `scm-quasi-walk`,
+;; which differ only in:
+;;   - the unquote keyword name (Kernel: "$unquote" / "$unquote-splicing";
+;;     Scheme: "unquote" / "unquote-splicing")
+;;   - the host evaluator function (`kernel-eval` vs `scheme-eval`)
+;;
+;; Algorithm is identical. Adapter cfg parameterises the two
+;; language-specific knobs.
+;;
+;; Adapter cfg keys
+;; ----------------
+;;   :unquote-name           — string, name of the unquote keyword
+;;   :unquote-splicing-name  — string, name of the splice keyword
+;;   :eval                   — fn (form env) → value
+;;
+;; Public API
+;;   (refl-quasi-walk-with CFG FORM ENV)
+;;     Top-level walker. Returns FORM with unquotes evaluated in ENV.
+;;
+;;   (refl-quasi-walk-list-with CFG FORMS ENV)
+;;     Walks a list of forms, splicing unquote-splicing results inline.
+;;
+;;   (refl-quasi-list-concat XS YS)
+;;     Pure-SX list append (no host append/append! needed).
+
+(define
+  refl-quasi-list-concat
+  (fn
+    (xs ys)
+    (cond
+      ((or (nil? xs) (= (length xs) 0)) ys)
+      (:else (cons (first xs) (refl-quasi-list-concat (rest xs) ys))))))
+
+(define
+  refl-quasi-walk-with
+  (fn
+    (cfg form env)
+    (cond
+      ((not (list? form)) form)
+      ((= (length form) 0) form)
+      ((and (string? (first form)) (= (first form) (get cfg :unquote-name)))
+        (cond
+          ((not (= (length form) 2))
+            (error
+              (str (get cfg :unquote-name) ": expects exactly 1 argument")))
+          (:else ((get cfg :eval) (nth form 1) env))))
+      (:else (refl-quasi-walk-list-with cfg form env)))))
+
+(define
+  refl-quasi-walk-list-with
+  (fn
+    (cfg forms env)
+    (cond
+      ((or (nil? forms) (= (length forms) 0)) (list))
+      (:else
+        (let
+          ((head (first forms)))
+          (cond
+            ((and (list? head) (= (length head) 2) (string? (first head)) (= (first head) (get cfg :unquote-splicing-name)))
+              (let
+                ((spliced ((get cfg :eval) (nth head 1) env)))
+                (cond
+                  ((not (list? spliced))
+                    (error
+                      (str
+                        (get cfg :unquote-splicing-name)
+                        ": value must be a list")))
+                  (:else
+                    (refl-quasi-list-concat
+                      spliced
+                      (refl-quasi-walk-list-with cfg (rest forms) env))))))
+            (:else
+              (cons
+                (refl-quasi-walk-with cfg head env)
+                (refl-quasi-walk-list-with cfg (rest forms) env)))))))))

lib/guest/test-runner.sx

@@ -0,0 +1,50 @@
+;; lib/guest/test-runner.sx — per-suite test harness for guest test files.
+;;
+;; Across the codebase 142+ test files implement the identical four-form
+;; boilerplate: `<X>-test-pass`, `<X>-test-fail`, `<X>-test-fails`, and
+;; an `<X>-test` recording function. Only the prefix differs. This kit
+;; collapses the boilerplate to a per-suite mutable dict + a recording
+;; helper, so each test file goes from ~12 lines of harness to ~3:
+;;
+;;   (define ke-suite (refl-make-test-suite))
+;;   (define ke-test  (fn (n a e) (refl-test ke-suite n a e)))
+;;   (define ke-tests-run! (fn () (refl-test-report ke-suite)))
+;;
+;; The suite is a mutable dict `{:pass N :fail N :fails LIST}`. Each
+;; failed assertion appends `{:name NAME :expected EXPECTED :actual ACT}`
+;; to :fails — same shape every existing harness already produces.
+;;
+;; The `:fails` list is mutated in place via `append!`, so callers who
+;; have a reference to it see the same updates. (Same semantic the
+;; existing per-suite globals had — just held in the suite dict now.)
+;;
+;; Public API
+;;   (refl-make-test-suite)              — fresh suite
+;;   (refl-test SUITE NAME ACT EXP)      — record one assertion
+;;   (refl-test-report SUITE)            — return {:total :passed :failed :fails}
+;;   (refl-test-pass? SUITE)             — convenience: all green?
+;;   (refl-test-suite? V)                — predicate
+
+(define refl-make-test-suite (fn () {:fail 0 :pass 0 :fails (list)}))
+
+(define
+  refl-test-suite?
+  (fn
+    (v)
+    (and (dict? v) (number? (get v :pass)) (number? (get v :fail)))))
+
+(define
+  refl-test
+  (fn
+    (suite name actual expected)
+    (cond
+      ((= actual expected)
+        (dict-set! suite :pass (+ (get suite :pass) 1)))
+      (:else
+        (begin
+          (dict-set! suite :fail (+ (get suite :fail) 1))
+          (append! (get suite :fails) {:name name :actual actual :expected expected}))))))
+
+(define refl-test-report (fn (suite) {:total (+ (get suite :pass) (get suite :fail)) :passed (get suite :pass) :failed (get suite :fail) :fails (get suite :fails)}))
+
+(define refl-test-pass? (fn (suite) (= (get suite :fail) 0)))

lib/hyperscript/compiler.sx

@@ -226,6 +226,28 @@
                             value)
                           (list (quote set!) (hs-to-sx target) value)))))))
               (true (list (quote set!) (hs-to-sx target) value)))))))
+    ;; Throttle/debounce extraction state — module-level so they don't get
+    ;; redefined on every emit-on call (which was causing JIT churn). Set
+    ;; via _strip-throttle-debounce at the start of each emit-on, used in
+    ;; the handler-build step inside scan-on.
+    (define _throttle-ms nil)
+    (define _debounce-ms nil)
+    (define
+      _strip-throttle-debounce
+      (fn
+        (lst)
+        (cond
+          ((<= (len lst) 1) lst)
+          ((= (first lst) :throttle)
+            (do
+              (set! _throttle-ms (nth lst 1))
+              (_strip-throttle-debounce (rest (rest lst)))))
+          ((= (first lst) :debounce)
+            (do
+              (set! _debounce-ms (nth lst 1))
+              (_strip-throttle-debounce (rest (rest lst)))))
+          (true
+            (cons (first lst) (_strip-throttle-debounce (rest lst)))))))
     (define
       emit-on
       (fn
@@ -234,6 +256,8 @@
           ((parts (rest ast)))
           (let
             ((event-name (first parts)))
+            (set! _throttle-ms nil)
+            (set! _debounce-ms nil)
             (define
               scan-on
               (fn
@@ -266,6 +290,13 @@
                                   ((wrapped-body (if catch-info (let ((var (make-symbol (nth catch-info 0))) (catch-body (hs-to-sx (nth catch-info 1)))) (if finally-info (list (quote let) (list (list (quote __hs-exc) nil) (list (quote __hs-reraise) false)) (list (quote do) (list (quote guard) (list var (list true (list (quote let) (list (list var (list (quote host-hs-normalize-exc) var))) (list (quote guard) (list (quote __inner-exc) (list true (list (quote do) (list (quote set!) (quote __hs-exc) (quote __inner-exc)) (list (quote set!) (quote __hs-reraise) true)))) catch-body)))) compiled-body) (hs-to-sx finally-info) (list (quote when) (quote __hs-reraise) (list (quote raise) (quote __hs-exc))))) (list (quote let) (list (list (quote __hs-exc) nil) (list (quote __hs-reraise) false)) (list (quote do) (list (quote guard) (list var (list true (list (quote let) (list (list var (list (quote host-hs-normalize-exc) var))) (list (quote guard) (list (quote __inner-exc) (list true (list (quote do) (list (quote set!) (quote __hs-exc) (quote __inner-exc)) (list (quote set!) (quote __hs-reraise) true)))) catch-body)))) compiled-body) (list (quote when) (quote __hs-reraise) (list (quote raise) (quote __hs-exc))))))) (if finally-info (list (quote do) compiled-body (hs-to-sx finally-info)) compiled-body))))
                                   (let
                                     ((handler (let ((uses-the-result? (fn (expr) (cond ((= expr (quote the-result)) true) ((list? expr) (some (fn (x) (uses-the-result? x)) expr)) (true false))))) (let ((base-handler (list (quote fn) (list (quote event)) (if (uses-the-result? wrapped-body) (list (quote let) (list (list (quote the-result) nil)) wrapped-body) wrapped-body)))) (if count-filter-info (let ((mn (get count-filter-info "min")) (mx (get count-filter-info "max"))) (list (quote let) (list (list (quote __hs-count) 0)) (list (quote fn) (list (quote event)) (list (quote begin) (list (quote set!) (quote __hs-count) (list (quote +) (quote __hs-count) 1)) (list (quote when) (if (= mx -1) (list (quote >=) (quote __hs-count) mn) (list (quote and) (list (quote >=) (quote __hs-count) mn) (list (quote <=) (quote __hs-count) mx))) (nth base-handler 2)))))) base-handler)))))
+                                    (let
+                                      ((handler (cond
+                                                  (_throttle-ms
+                                                    (list (quote hs-throttle!) handler (hs-to-sx _throttle-ms)))
+                                                  (_debounce-ms
+                                                    (list (quote hs-debounce!) handler (hs-to-sx _debounce-ms)))
+                                                  (true handler))))
                                     (let
                                       ((on-call (if every? (list (quote hs-on-every) target event-name handler) (list (quote hs-on) target event-name handler))))
                                       (cond
@@ -325,7 +356,7 @@
                                                       (first pair)
                                                       handler))
                                                   or-sources)))
-                                            on-call)))))))))))))
+                                            on-call))))))))))))))
                   ((= (first items) :from)
                     (scan-on
                       (rest (rest items))
@@ -469,7 +500,7 @@
                       count-filter-info
                       elsewhere?
                       or-sources)))))
-            (scan-on (rest parts) nil nil false nil nil nil nil nil false nil)))))
+            (scan-on (_strip-throttle-debounce (rest parts)) nil nil false nil nil nil nil nil false nil)))))
     (define
       emit-send
       (fn
@@ -2490,6 +2521,15 @@
                             (quote fn)
                             (list (quote it))
                             (hs-to-sx body))))
+                      ((and (list? expr) (= (first expr) (quote attr)))
+                        (list
+                          (quote hs-attr-watch!)
+                          (hs-to-sx (nth expr 2))
+                          (nth expr 1)
+                          (list
+                            (quote fn)
+                            (list (quote it))
+                            (hs-to-sx body))))
                       (true nil))))
                 ((= head (quote init))
                   (list

lib/hyperscript/parser.sx

@@ -1358,7 +1358,17 @@
                           cls
                           (first extra-classes)
                           tgt))
-                      ((match-kw "for")
+                      ((and
+                          (= (tp-type) "keyword") (= (tp-val) "for")
+                          ;; Only consume 'for' as a duration clause if the next
+                          ;; token is NOT '<ident> in ...' — that pattern is a
+                          ;; for-in loop, not a toggle duration.
+                          (not
+                            (and
+                              (> (len tokens) (+ p 2))
+                              (= (get (nth tokens (+ p 1)) "type") "ident")
+                              (= (get (nth tokens (+ p 2)) "value") "in")))
+                          (do (adv!) true))
                         (let
                           ((dur (parse-expr)))
                           (list (quote toggle-class-for) cls tgt dur)))
@@ -3090,7 +3100,17 @@
                               (= (tp-val) "queue"))
                             (do (adv!) (adv!)))
                           (let
-                            ((every? (match-kw "every")))
+                            ((every? (match-kw "every"))
+                             (throttle-ms nil)
+                             (debounce-ms nil))
+                            ;; 'throttled at <duration>' / 'debounced at <duration>'
+                            ;; — parsed as handler modifiers, captured as :throttle / :debounce parts.
+                            (when (and (= (tp-type) "ident") (= (tp-val) "throttled"))
+                              (adv!)
+                              (when (match-kw "at") (set! throttle-ms (parse-expr))))
+                            (when (and (= (tp-type) "ident") (= (tp-val) "debounced"))
+                              (adv!)
+                              (when (match-kw "at") (set! debounce-ms (parse-expr))))
                             (let
                               ((having (if (or h-margin h-threshold) (dict "margin" h-margin "threshold" h-threshold) nil)))
                               (let
@@ -3105,6 +3125,10 @@
                                   (match-kw "end")
                                   (let
                                     ((parts (list (quote on) event-name)))
+                                    (let
+                                      ((parts (if throttle-ms (append parts (list :throttle throttle-ms)) parts)))
+                                    (let
+                                      ((parts (if debounce-ms (append parts (list :debounce debounce-ms)) parts)))
                                     (let
                                       ((parts (if every? (append parts (list :every true)) parts)))
                                       (let
@@ -3127,7 +3151,7 @@
                                                         ((parts (if finally-clause (append parts (list :finally finally-clause)) parts)))
                                                         (let
                                                           ((parts (append parts (list (if (> (len event-vars) 0) (cons (quote do) (append (map (fn (nm) (list (quote ref) nm)) event-vars) (if (and (list? body) (= (first body) (quote do))) (rest body) (list body)))) body)))))
-                                                          parts))))))))))))))))))))))))))
+                                                          parts))))))))))))))))))))))))))))
       (define
         parse-init-feat
         (fn
@@ -3177,6 +3201,7 @@
             (or
               (= (tp-type) "hat")
               (= (tp-type) "local")
+              (= (tp-type) "attr")
               (and (= (tp-type) "keyword") (= (tp-val) "dom")))
             (let
               ((expr (parse-expr)))

lib/hyperscript/runtime.sx

@@ -12,6 +12,29 @@
 
 ;; Register an event listener. Returns unlisten function.
 ;; (hs-on target event-name handler) → unlisten-fn
+(begin
+  (define _hs-config-log-all false)
+  (define _hs-log-captured (list))
+  (define
+    hs-set-log-all!
+    (fn (flag) (set! _hs-config-log-all (if flag true false))))
+  (define hs-get-log-captured (fn () _hs-log-captured))
+  (define
+    hs-clear-log-captured!
+    (fn () (begin (set! _hs-log-captured (list)) nil)))
+  (define
+    hs-log-event!
+    (fn
+      (msg)
+      (when
+        _hs-config-log-all
+        (begin
+          (set! _hs-log-captured (append _hs-log-captured (list msg)))
+          (host-call (host-global "console") "log" msg)
+          nil)))))
+
+;; Run an initializer function immediately.
+;; (hs-init thunk) — called at element boot time
 (define
   hs-each
   (fn
@@ -22,17 +45,52 @@
 ;; (hs-init thunk) — called at element boot time
 (define meta (host-new "Object"))
 
-;; Run an initializer function immediately.
-;; (hs-init thunk) — called at element boot time
-(define
-  hs-on-every
-  (fn (target event-name handler) (dom-listen target event-name handler)))
-
 ;; ── Async / timing ──────────────────────────────────────────────
 
 ;; Wait for a duration in milliseconds.
 ;; In hyperscript, wait is async-transparent — execution pauses.
 ;; Here we use perform/IO suspension for true pause semantics.
+(define
+  hs-on-every
+  (fn (target event-name handler) (dom-listen target event-name handler)))
+
+;; Throttle: drops events that arrive within the window. First event fires
+;; immediately; subsequent events within `ms` of the previous fire are dropped.
+;; Returns a wrapped handler suitable for hs-on / hs-on-every.
+(define
+  hs-throttle!
+  (fn
+    (handler ms)
+    (let
+      ((__hs-last-fire 0))
+      (fn
+        (event)
+        (let
+          ((__hs-now (host-call (host-global "Date") "now")))
+          (when
+            (>= (- __hs-now __hs-last-fire) ms)
+            (set! __hs-last-fire __hs-now)
+            (handler event)))))))
+
+;; Debounce: waits until `ms` has elapsed since the last event before firing.
+;; In our synchronous test mock no time passes, so the timer fires immediately
+;; via setTimeout(_, 0); the wrapped handler still gets called once per burst.
+(define
+  hs-debounce!
+  (fn
+    (handler ms)
+    (let
+      ((__hs-timer nil))
+      (fn
+        (event)
+        (when __hs-timer (host-call (host-global "window") "clearTimeout" __hs-timer))
+        (set! __hs-timer
+          (host-call (host-global "window") "setTimeout"
+            (host-new-function (list "ev") "return arguments[0](arguments[1]);")
+            ms handler event))))))
+
+;; Wait for a DOM event on a target.
+;; (hs-wait-for target event-name) — suspends until event fires
 (define
   _hs-on-caller
   (let
@@ -45,8 +103,7 @@
       (host-set! _ctx "meta" _m)
       _ctx)))
 
-;; Wait for a DOM event on a target.
-;; (hs-wait-for target event-name) — suspends until event fires
+;; Wait for CSS transitions/animations to settle on an element.
 (define
   hs-on
   (fn
@@ -66,14 +123,14 @@
               (append prev (list unlisten)))
             unlisten))))))
 
-;; Wait for CSS transitions/animations to settle on an element.
+;; ── Class manipulation ──────────────────────────────────────────
+
+;; Toggle a single class on an element.
 (define
   hs-on-every
   (fn (target event-name handler) (dom-listen target event-name handler)))
 
-;; ── Class manipulation ──────────────────────────────────────────
-
-;; Toggle a single class on an element.
+;; Toggle between two classes — exactly one is active at a time.
 (define
   hs-on-intersection-attach!
   (fn
@@ -89,7 +146,8 @@
           (host-call observer "observe" target)
           observer)))))
 
-;; Toggle between two classes — exactly one is active at a time.
+;; Take a class from siblings — add to target, remove from others.
+;; (hs-take! target cls) — like radio button class behavior
 (define
   hs-on-mutation-attach!
   (fn
@@ -110,19 +168,18 @@
             (host-call observer "observe" target opts)
             observer))))))
 
-;; Take a class from siblings — add to target, remove from others.
-;; (hs-take! target cls) — like radio button class behavior
-(define hs-init (fn (thunk) (thunk)))
-
 ;; ── DOM insertion ───────────────────────────────────────────────
 
 ;; Put content at a position relative to a target.
 ;; pos: "into" | "before" | "after"
-(define hs-wait (fn (ms) (perform (list (quote io-sleep) ms))))
+(define hs-init (fn (thunk) (thunk)))
 
 ;; ── Navigation / traversal ──────────────────────────────────────
 
 ;; Navigate to a URL.
+(define hs-wait (fn (ms) (perform (list (quote io-sleep) ms))))
+
+;; Find next sibling matching a selector (or any sibling).
 (begin
   (define
     hs-wait-for
@@ -135,7 +192,7 @@
       (target event-name timeout-ms)
       (perform (list (quote io-wait-event) target event-name timeout-ms)))))
 
-;; Find next sibling matching a selector (or any sibling).
+;; Find previous sibling matching a selector.
 (define
   hs-settle
   (fn
@@ -143,7 +200,7 @@
     (hs-null-raise! target)
     (when (not (nil? target)) (perform (list (quote io-settle) target)))))
 
-;; Find previous sibling matching a selector.
+;; First element matching selector within a scope.
 (define
   hs-toggle-class!
   (fn
@@ -153,7 +210,7 @@
       (not (nil? target))
       (host-call (host-get target "classList") "toggle" cls))))
 
-;; First element matching selector within a scope.
+;; Last element matching selector.
 (define
   hs-toggle-var-cycle!
   (fn
@@ -175,7 +232,7 @@
           var-name
           (if (= idx -1) (first values) (nth values (mod (+ idx 1) n))))))))
 
-;; Last element matching selector.
+;; First/last within a specific scope.
 (define
   hs-toggle-between!
   (fn
@@ -188,7 +245,6 @@
         (do (dom-remove-class target cls1) (dom-add-class target cls2))
         (do (dom-remove-class target cls2) (dom-add-class target cls1))))))
 
-;; First/last within a specific scope.
 (define
   hs-toggle-style!
   (fn
@@ -212,6 +268,9 @@
             (dom-set-style target prop "hidden")
             (dom-set-style target prop "")))))))
 
+;; ── Iteration ───────────────────────────────────────────────────
+
+;; Repeat a thunk N times.
 (define
   hs-toggle-style-between!
   (fn
@@ -223,9 +282,7 @@
         (dom-set-style target prop val2)
         (dom-set-style target prop val1)))))
 
-;; ── Iteration ───────────────────────────────────────────────────
-
-;; Repeat a thunk N times.
+;; Repeat forever (until break — relies on exception/continuation).
 (define
   hs-toggle-style-cycle!
   (fn
@@ -246,7 +303,10 @@
             (true (find-next (rest remaining))))))
       (dom-set-style target prop (find-next vals)))))
 
-;; Repeat forever (until break — relies on exception/continuation).
+;; ── Fetch ───────────────────────────────────────────────────────
+
+;; Fetch a URL, parse response according to format.
+;; (hs-fetch url format) — format is "json" | "text" | "html"
 (define
   hs-take!
   (fn
@@ -269,8 +329,7 @@
             (when with-cls (dom-remove-class target with-cls))))
         (let
           ((attr-val (if (> (len extra) 0) (first extra) nil))
-            (with-val
-              (if (> (len extra) 1) (nth extra 1) nil)))
+            (with-val (if (> (len extra) 1) (nth extra 1) nil)))
           (do
             (for-each
               (fn
@@ -287,10 +346,10 @@
               (dom-set-attr target name attr-val)
               (dom-set-attr target name ""))))))))
 
-;; ── Fetch ───────────────────────────────────────────────────────
+;; ── Type coercion ───────────────────────────────────────────────
 
-;; Fetch a URL, parse response according to format.
-;; (hs-fetch url format) — format is "json" | "text" | "html"
+;; Coerce a value to a type by name.
+;; (hs-coerce value type-name) — type-name is "Int", "Float", "String", etc.
 (begin
   (define
     hs-element?
@@ -447,10 +506,10 @@
                     (dom-insert-adjacent-html target "beforeend" value)
                     (hs-boot-subtree! target)))))))))))
 
-;; ── Type coercion ───────────────────────────────────────────────
+;; ── Object creation ─────────────────────────────────────────────
 
-;; Coerce a value to a type by name.
-;; (hs-coerce value type-name) — type-name is "Int", "Float", "String", etc.
+;; Make a new object of a given type.
+;; (hs-make type-name) — creates empty object/collection
 (define
   hs-add-to!
   (fn
@@ -464,10 +523,11 @@
       ((hs-is-set? target) (do (host-call target "add" value) target))
       (true (do (host-call target "push" value) target)))))
 
-;; ── Object creation ─────────────────────────────────────────────
+;; ── Behavior installation ───────────────────────────────────────
 
-;; Make a new object of a given type.
-;; (hs-make type-name) — creates empty object/collection
+;; Install a behavior on an element.
+;; A behavior is a function that takes (me ...params) and sets up features.
+;; (hs-install behavior-fn me ...args)
 (define
   hs-remove-from!
   (fn
@@ -477,11 +537,10 @@
       ((hs-is-set? target) (do (host-call target "delete" value) target))
       (true (host-call target "splice" (host-call target "indexOf" value) 1)))))
 
-;; ── Behavior installation ───────────────────────────────────────
+;; ── Measurement ─────────────────────────────────────────────────
 
-;; Install a behavior on an element.
-;; A behavior is a function that takes (me ...params) and sets up features.
-;; (hs-install behavior-fn me ...args)
+;; Measure an element's bounding rect, store as local variables.
+;; Returns a dict with x, y, width, height, top, left, right, bottom.
 (define
   hs-splice-at!
   (fn
@@ -494,10 +553,7 @@
           ((i (if (< idx 0) (+ n idx) idx)))
           (cond
             ((or (< i 0) (>= i n)) target)
-            (true
-              (concat
-                (slice target 0 i)
-                (slice target (+ i 1) n))))))
+            (true (concat (slice target 0 i) (slice target (+ i 1) n))))))
       (do
         (when
           target
@@ -508,10 +564,10 @@
               (host-call target "splice" i 1))))
         target))))
 
-;; ── Measurement ─────────────────────────────────────────────────
-
-;; Measure an element's bounding rect, store as local variables.
-;; Returns a dict with x, y, width, height, top, left, right, bottom.
+;; Return the current text selection as a string.  In the browser this is
+;; `window.getSelection().toString()`.  In the mock test runner, a test
+;; setup stashes the desired selection text at `window.__test_selection`
+;; and the fallback path returns that so tests can assert on the result.
 (define
   hs-index
   (fn
@@ -523,10 +579,11 @@
       ((string? obj) (nth obj key))
       (true (host-get obj key)))))
 
-;; Return the current text selection as a string.  In the browser this is
-;; `window.getSelection().toString()`.  In the mock test runner, a test
-;; setup stashes the desired selection text at `window.__test_selection`
-;; and the fallback path returns that so tests can assert on the result.
+
+;; ── Transition ──────────────────────────────────────────────────
+
+;; Transition a CSS property to a value, optionally with duration.
+;; (hs-transition target prop value duration)
 (define
   hs-put-at!
   (fn
@@ -548,11 +605,6 @@
                 ((= pos "start") (host-call target "unshift" value)))
               target)))))))
 
-
-;; ── Transition ──────────────────────────────────────────────────
-
-;; Transition a CSS property to a value, optionally with duration.
-;; (hs-transition target prop value duration)
 (define
   hs-dict-without
   (fn
@@ -589,6 +641,11 @@
       ((w (host-global "window")))
       (if w (host-call w "prompt" msg) nil))))
 
+
+;; ── Transition ──────────────────────────────────────────────────
+
+;; Transition a CSS property to a value, optionally with duration.
+;; (hs-transition target prop value duration)
 (define
   hs-answer
   (fn
@@ -597,11 +654,6 @@
       ((w (host-global "window")))
       (if w (if (host-call w "confirm" msg) yes-val no-val) no-val))))
 
-
-;; ── Transition ──────────────────────────────────────────────────
-
-;; Transition a CSS property to a value, optionally with duration.
-;; (hs-transition target prop value duration)
 (define
   hs-answer-alert
   (fn
@@ -662,6 +714,10 @@
             (if (nil? sel) "" (host-call sel "toString" (list))))
           stash)))))
 
+
+
+
+
 (define
   hs-reset!
   (fn
@@ -708,10 +764,6 @@
                   (when default-val (dom-set-prop target "value" default-val)))))
             (true nil)))))))
 
-
-
-
-
 (define
   hs-next
   (fn
@@ -730,7 +782,8 @@
               ((dom-matches? el sel) el)
               (true (find-next (dom-next-sibling el))))))
         (find-next sibling)))))
-
+;; ── Sandbox/test runtime additions ──────────────────────────────
+;; Property access — dot notation and .length
 (define
   hs-previous
   (fn
@@ -749,10 +802,9 @@
               ((dom-matches? el sel) el)
               (true (find-prev (dom-get-prop el "previousElementSibling"))))))
         (find-prev sibling)))))
-;; ── Sandbox/test runtime additions ──────────────────────────────
-;; Property access — dot notation and .length
-(define _hs-last-query-sel nil)
 ;; DOM query stub — sandbox returns empty list
+(define _hs-last-query-sel nil)
+;; Method dispatch — obj.method(args)
 (define
   hs-null-raise!
   (fn
@@ -763,7 +815,9 @@
         ((msg (str "'" (or (host-get (host-global "window") "_hs_last_query_sel") "target") "' is null")))
         (host-set! (host-global "window") "_hs_null_error" msg)
         (guard (_null-e (true nil)) (raise msg))))))
-;; Method dispatch — obj.method(args)
+
+;; ── 0.9.90 features ─────────────────────────────────────────────
+;; beep! — debug logging, returns value unchanged
 (define
   hs-empty-raise!
   (fn
@@ -777,9 +831,7 @@
         ((msg (str "'" (or (host-get (host-global "window") "_hs_last_query_sel") "target") "' is null")))
         (host-set! (host-global "window") "_hs_null_error" msg)
         (guard (_null-e (true nil)) (raise msg))))))
-
-;; ── 0.9.90 features ─────────────────────────────────────────────
-;; beep! — debug logging, returns value unchanged
+;; Property-based is — check obj.key truthiness
 (define
   hs-query-all-checked
   (fn
@@ -787,14 +839,14 @@
     (let
       ((result (hs-query-all sel)))
       (do (hs-empty-raise! result) result))))
-;; Property-based is — check obj.key truthiness
+;; Array slicing (inclusive both ends)
 (define
   hs-dispatch!
   (fn
     (target event detail)
     (hs-null-raise! target)
     (when (not (nil? target)) (dom-dispatch target event detail))))
-;; Array slicing (inclusive both ends)
+;; Collection: sorted by
 (define
   hs-query-all
   (fn
@@ -802,7 +854,7 @@
     (do
       (host-set! (host-global "window") "_hs_last_query_sel" sel)
       (dom-query-all (dom-document) sel))))
-;; Collection: sorted by
+;; Collection: sorted by descending
 (define
   hs-query-all-in
   (fn
@@ -811,17 +863,17 @@
       (nil? target)
       (hs-query-all sel)
       (host-call target "querySelectorAll" sel))))
-;; Collection: sorted by descending
+;; Collection: split by
 (define
   hs-list-set
   (fn
     (lst idx val)
     (append (take lst idx) (cons val (drop lst (+ idx 1))))))
-;; Collection: split by
+;; Collection: joined by
 (define
   hs-to-number
   (fn (v) (if (number? v) v (or (parse-number (str v)) 0))))
-;; Collection: joined by
+
 (define
   hs-query-first
   (fn
@@ -951,7 +1003,7 @@
                   ((= (str ex) "hs-continue") (do-loop (rest remaining)))
                   (true (raise ex))))))))
       (do-loop items))))
-
+;; Collection: joined by
 (begin
   (define
     hs-append
@@ -992,7 +1044,7 @@
               (host-get value "outerHTML")
               (str value))))
         (true nil)))))
-;; Collection: joined by
+
 (define
   hs-sender
   (fn
@@ -1084,6 +1136,7 @@
               (hs-host-to-sx (perform (list "io-parse-json" raw))))
             ((= fmt "number")
               (hs-to-number (perform (list "io-parse-text" raw))))
+            ((= fmt "html") (perform (list "io-parse-html" raw)))
             (true (perform (list "io-parse-text" raw)))))))))
 
 (define hs-fetch (fn (url format) (hs-fetch-impl url format false)))
@@ -1623,14 +1676,10 @@
                 ((ch (substring sel i (+ i 1))))
                 (cond
                   ((= ch ".")
-                    (do
-                      (flush!)
-                      (set! mode "class")
-                      (walk (+ i 1))))
+                    (do (flush!) (set! mode "class") (walk (+ i 1))))
                   ((= ch "#")
                     (do (flush!) (set! mode "id") (walk (+ i 1))))
-                  (true
-                    (do (set! cur (str cur ch)) (walk (+ i 1)))))))))
+                  (true (do (set! cur (str cur ch)) (walk (+ i 1)))))))))
         (walk 0)
         (flush!)
         {:tag tag :classes classes :id id}))))
@@ -1724,11 +1773,11 @@
     (value type-name)
     (if (nil? value) false (hs-type-check value type-name))))
 
+
 (define
   hs-strict-eq
   (fn (a b) (and (= (type-of a) (type-of b)) (= a b))))
 
-
 (define
   hs-id=
   (fn
@@ -1760,6 +1809,20 @@
       ((nil? suffix) false)
       (true (ends-with? (str s) (str suffix))))))
 
+(define
+  hs-attr-watch!
+  (fn
+    (target attr-name handler)
+    (let
+      ((mo-class (host-get (host-global "window") "MutationObserver")))
+      (when
+        mo-class
+        (let
+          ((cb (fn (records observer) (for-each (fn (rec) (when (= (host-get rec "attributeName") attr-name) (handler (host-call target "getAttribute" attr-name)))) records))))
+          (let
+            ((mo (host-new "MutationObserver" cb)))
+            (host-call mo "observe" target {:attributeFilter (list attr-name) :attributes true})))))))
+
 (define
   hs-scoped-set!
   (fn
@@ -1805,10 +1868,7 @@
       ((and (dict? a) (dict? b))
         (let
           ((pos (host-call a "compareDocumentPosition" b)))
-          (if
-            (number? pos)
-            (not (= 0 (mod (/ pos 4) 2)))
-            false)))
+          (if (number? pos) (not (= 0 (mod (/ pos 4) 2))) false)))
       (true (< (str a) (str b))))))
 
 (define
@@ -1929,10 +1989,7 @@
       ((and (dict? a) (dict? b))
         (let
           ((pos (host-call a "compareDocumentPosition" b)))
-          (if
-            (number? pos)
-            (not (= 0 (mod (/ pos 4) 2)))
-            false)))
+          (if (number? pos) (not (= 0 (mod (/ pos 4) 2))) false)))
       (true (< (str a) (str b))))))
 
 (define
@@ -1985,9 +2042,7 @@
 
 (define
   hs-morph-char
-  (fn
-    (s p)
-    (if (or (< p 0) (>= p (string-length s))) nil (nth s p))))
+  (fn (s p) (if (or (< p 0) (>= p (string-length s))) nil (nth s p))))
 
 (define
   hs-morph-index-from
@@ -2015,10 +2070,7 @@
         (q)
         (let
           ((c (hs-morph-char s q)))
-          (if
-            (and c (< (index-of stop c) 0))
-            (loop (+ q 1))
-            q))))
+          (if (and c (< (index-of stop c) 0)) (loop (+ q 1)) q))))
     (let ((e (loop p))) (list (substring s p e) e))))
 
 (define
@@ -2060,9 +2112,7 @@
                                 (append
                                   acc
                                   (list
-                                    (list
-                                      name
-                                      (substring s (+ p4 1) close)))))))
+                                    (list name (substring s (+ p4 1) close)))))))
                           ((= c2 "'")
                             (let
                               ((close (hs-morph-index-from s "'" (+ p4 1))))
@@ -2072,9 +2122,7 @@
                                 (append
                                   acc
                                   (list
-                                    (list
-                                      name
-                                      (substring s (+ p4 1) close)))))))
+                                    (list name (substring s (+ p4 1) close)))))))
                           (true
                             (let
                               ((r2 (hs-morph-read-until s p4 " \t\n/>")))
@@ -2158,9 +2206,7 @@
               (for-each
                 (fn
                   (c)
-                  (when
-                    (> (string-length c) 0)
-                    (dom-add-class el c)))
+                  (when (> (string-length c) 0) (dom-add-class el c)))
                 (split v " ")))
             ((and keep-id (= n "id")) nil)
             (true (dom-set-attr el n v)))))
@@ -2261,8 +2307,7 @@
           ((parts (split resolved ":")))
           (let
             ((prop (first parts))
-              (val
-                (if (> (len parts) 1) (nth parts 1) nil)))
+              (val (if (> (len parts) 1) (nth parts 1) nil)))
             (cond
               ((and (not (= prop "display")) (not (= prop "opacity")) (not (= prop "visibility")) (not (= prop "hidden")) (not (= prop "class-hidden")) (not (= prop "class-invisible")) (not (= prop "class-opacity")) (not (= prop "details")) (not (= prop "dialog")) (dict-has? _hs-hide-strategies prop))
                 (let
@@ -2302,8 +2347,7 @@
           ((parts (split resolved ":")))
           (let
             ((prop (first parts))
-              (val
-                (if (> (len parts) 1) (nth parts 1) nil)))
+              (val (if (> (len parts) 1) (nth parts 1) nil)))
             (cond
               ((and (not (= prop "display")) (not (= prop "opacity")) (not (= prop "visibility")) (not (= prop "hidden")) (not (= prop "class-hidden")) (not (= prop "class-invisible")) (not (= prop "class-opacity")) (not (= prop "details")) (not (= prop "dialog")) (dict-has? _hs-hide-strategies prop))
                 (let
@@ -2408,14 +2452,10 @@
                             (if
                               (= depth 1)
                               j
-                              (find-close
-                                (+ j 1)
-                                (- depth 1)))
+                              (find-close (+ j 1) (- depth 1)))
                             (if
                               (= (nth raw j) "{")
-                              (find-close
-                                (+ j 1)
-                                (+ depth 1))
+                              (find-close (+ j 1) (+ depth 1))
                               (find-close (+ j 1) depth))))))
                     (let
                       ((close (find-close start 1)))
@@ -2526,10 +2566,7 @@
               (if
                 (= (len lst) 0)
                 -1
-                (if
-                  (= (first lst) item)
-                  i
-                  (idx-loop (rest lst) (+ i 1))))))
+                (if (= (first lst) item) i (idx-loop (rest lst) (+ i 1))))))
           (idx-loop obj 0)))
       (true
         (let
@@ -2621,8 +2658,7 @@
                 (cond
                   ((= end "hs-pick-end") n)
                   ((= end "hs-pick-start") 0)
-                  ((and (number? end) (< end 0))
-                    (max 0 (+ n end)))
+                  ((and (number? end) (< end 0)) (max 0 (+ n end)))
                   (true end))))
             (cond
               ((string? col) (slice col s e))
@@ -2802,6 +2838,8 @@
   hs-sorted-by-desc
   (fn (col key-fn) (reverse (hs-sorted-by col key-fn))))
 
+;; ── SourceInfo API ────────────────────────────────────────────────
+
 (define
   hs-dom-has-var?
   (fn
@@ -2821,8 +2859,6 @@
       ((store (host-get el "__hs_vars")))
       (if (nil? store) nil (host-get store name)))))
 
-;; ── SourceInfo API ────────────────────────────────────────────────
-
 (define
   hs-dom-set-var-raw!
   (fn
@@ -2913,7 +2949,12 @@
 
 (define
   hs-null-error!
-  (fn (selector) (raise (str "'" selector "' is null"))))
+  (fn
+    (selector)
+    (let
+      ((msg (str "'" selector "' is null")))
+      (host-set! (host-global "window") "_hs_null_error" msg)
+      (guard (_null-e (true nil)) (raise msg)))))
 
 (define
   hs-named-target
@@ -2933,9 +2974,7 @@
       ((results (hs-query-all selector)))
       (if
         (and
-          (or
-            (nil? results)
-            (and (list? results) (= (len results) 0)))
+          (or (nil? results) (and (list? results) (= (len results) 0)))
           (string? selector)
           (> (len selector) 0)
           (= (substring selector 0 1) "#"))
@@ -3203,97 +3242,112 @@
 
 (define hs-token-op? (fn (tok) (dict-get tok :op)))
 
+;; ── WebSocket / socket feature ───────────────────────────────────
+
 (define
   hs-try-json-parse
-  (fn (data) (if (string? data) (guard (_e nil) (json-parse data)) nil)))
-
-(define
-  hs-socket-normalise-url
-  (fn
-    (url)
-    (if
-      (or (starts-with? url "ws://") (starts-with? url "wss://"))
-      url
-      (let
-        ((proto (host-get (host-global "location") "protocol"))
-          (host-str (host-get (host-global "location") "host")))
-        (let
-          ((scheme (if (= proto "https:") "wss://" "ws://")))
-          (str scheme host-str url))))))
-
-(define
-  hs-socket-bind-name!
-  (fn
-    (name-path wrapper)
-    (let
-      ((win (host-global "window")))
-      (if
-        (= (len name-path) 1)
-        (host-set! win (first name-path) wrapper)
-        (do
-          (when
-            (nil? (host-get win (first name-path)))
-            (host-set! win (first name-path) (host-new "Object")))
-          (host-set!
-            (host-get win (first name-path))
-            (nth name-path 1)
-            wrapper))))))
+  (fn (s) (host-call (host-global "JSON") "parse" s)))
 
 (define
   hs-socket-resolve-rpc!
   (fn
-    (wrapper data)
+    (wrapper msg)
     (let
-      ((iid (host-get data "iid")))
-      (when
-        (not (nil? iid))
-        (let
-          ((pending (host-get wrapper "_pending")))
-          (when
-            (not (nil? pending))
-            (let
-              ((entry (host-get pending iid)))
-              (when
-                (not (nil? entry))
-                (host-set! pending iid nil)
-                (if
-                  (not (nil? (host-get data "throw")))
-                  (host-call-fn
-                    (host-get entry "reject")
-                    (list (host-get data "throw")))
-                  (host-call-fn
-                    (host-get entry "resolve")
-                    (list (host-get data "return"))))))))))))
+      ((pending (host-get wrapper "pending")) (iid (host-get msg "iid")))
+      (let
+        ((resolver (host-get pending iid)))
+        (when
+          (not (nil? resolver))
+          (if
+            (not (nil? (host-get msg "return")))
+            (host-call resolver "resolve" (host-get msg "return"))
+            (host-call resolver "reject" (host-get msg "throw")))
+          (host-set! pending iid nil))))))
 
 (define
   hs-socket-register!
   (fn
-    (name-path url timeout on-message-handler json?)
+    (name-path url timeout-ms handler json?)
     (let
-      ((norm-url (hs-socket-normalise-url url)))
+      ((ws-url (cond ((or (starts-with? url "ws://") (starts-with? url "wss://")) url) (true (let ((proto (host-get (host-global "location") "protocol")) (h (host-get (host-global "location") "host"))) (str (if (= proto "https:") "wss:" "ws:") "//" h url))))))
       (let
-        ((wrapper (host-new "Object")))
-        (do
-          (host-set! wrapper "_url" norm-url)
-          (host-set! wrapper "_timeout" (if (nil? timeout) 0 timeout))
-          (host-set! wrapper "_pending" (host-new "Object"))
-          (host-set! wrapper "_closed" false)
+        ((ws (host-new "WebSocket" ws-url)))
+        (let
+          ((wrapper (host-new "Object")))
+          (host-set! wrapper "raw" ws)
+          (host-set! wrapper "url" ws-url)
+          (host-set! wrapper "timeout" timeout-ms)
+          (host-set! wrapper "pending" (host-new "Object"))
+          (host-set! wrapper "handler" handler)
+          (host-set! wrapper "json?" json?)
+          (host-set! wrapper "closed?" false)
+          (host-set! wrapper "closedFlag" nil)
           (let
-            ((ws (host-new "WebSocket" norm-url)))
-            (do
-              (host-set! wrapper "_ws" ws)
-              (let
-                ((msg-handler (host-callback (fn (evt) (do (let ((parsed (hs-try-json-parse (host-get evt "data")))) (when (and (not (nil? parsed)) (not (nil? (host-get parsed "iid")))) (hs-socket-resolve-rpc! wrapper parsed))) (when (not (nil? on-message-handler)) (if json? (let ((data (hs-try-json-parse (host-get evt "data")))) (when (not (nil? data)) (on-message-handler data))) (on-message-handler evt))))))))
-                (do
-                  (host-set! ws "onmessage" msg-handler)
-                  (host-set! wrapper "_onmessage_handler" msg-handler)
-                  (host-set!
-                    ws
-                    "onclose"
-                    (host-callback
-                      (fn (e) (host-set! wrapper "_closed" true))))
-                  (host-call-fn
-                    (host-global "_hsSetupSocket")
-                    (list wrapper))
-                  (hs-socket-bind-name! name-path wrapper)
-                  wrapper)))))))))
+            ((proxy-factory (host-global "_hs_make_rpc_proxy")))
+            (when
+              proxy-factory
+              (host-set!
+                wrapper
+                "rpc"
+                (host-call proxy-factory "call" nil wrapper))))
+          (host-set!
+            ws
+            "onmessage"
+            (host-callback
+              (fn
+                (event)
+                (let
+                  ((data (host-get event "data")))
+                  (let
+                    ((parsed (hs-try-json-parse data)))
+                    (cond
+                      ((and (not (nil? parsed)) (not (nil? (host-get parsed "iid"))))
+                        (hs-socket-resolve-rpc! wrapper parsed))
+                      ((not (nil? handler))
+                        (if
+                          json?
+                          (if
+                            (not (nil? parsed))
+                            (handler parsed)
+                            (error "Received non-JSON message"))
+                          (handler event)))))))))
+          (host-call
+            ws
+            "addEventListener"
+            "close"
+            (host-callback
+              (fn
+                (evt)
+                (host-set! wrapper "closedFlag" "1"))))
+          (host-set!
+            wrapper
+            "dispatchEvent"
+            (host-callback
+              (fn
+                (evt)
+                (let
+                  ((payload (host-new "Object")))
+                  (host-set! payload "type" (host-get evt "type"))
+                  (host-call
+                    (host-get wrapper "raw")
+                    "send"
+                    (host-call
+                      (host-global "JSON")
+                      "stringify"
+                      payload))))))
+          (define
+            bind-path!
+            (fn
+              (obj path)
+              (if
+                (= (len path) 1)
+                (host-set! obj (first path) wrapper)
+                (let
+                  ((key (first path)) (rest-path (rest path)))
+                  (let
+                    ((next (or (host-get obj key) (host-new "Object"))))
+                    (host-set! obj key next)
+                    (bind-path! next rest-path))))))
+          (bind-path! (host-global "window") name-path)
+          wrapper)))))
+

lib/hyperscript/tokenizer.sx

@@ -855,4 +855,230 @@
                 :else (do (t-advance! 1) (scan-template!)))))))
       (scan-template!)
       (t-emit! "eof" nil)
-      tokens)))
+      tokens)))
+
+;; ── Stream wrapper for upstream-style stateful tokenizer API ───────────────
+;;
+;; Upstream _hyperscript exposes a Tokens object with cursor + follow-set
+;; semantics on _hyperscript.internals.tokenizer. Our hs-tokenize returns a
+;; flat list; the stream wrapper adds the stateful operations.
+;;
+;; Type names map ours → upstream's (e.g. "ident" → "IDENTIFIER").
+
+(define
+  hs-stream-type-map
+  (fn
+    (t)
+    (cond
+      ((= t "ident") "IDENTIFIER")
+      ((= t "number") "NUMBER")
+      ((= t "string") "STRING")
+      ((= t "class") "CLASS_REF")
+      ((= t "id") "ID_REF")
+      ((= t "attr") "ATTRIBUTE_REF")
+      ((= t "style") "STYLE_REF")
+      ((= t "whitespace") "WHITESPACE")
+      ((= t "op") "OPERATOR")
+      ((= t "eof") "EOF")
+      (true (upcase t)))))
+
+;; Create a stream from a source string.
+;; Returns a dict — mutable via dict-set!.
+(define
+  hs-stream
+  (fn
+    (src)
+    {:tokens (hs-tokenize src) :pos 0 :follows (list) :last-match nil :last-ws nil}))
+
+;; Skip whitespace tokens, advancing pos to the next non-WS token.
+;; Captures the last skipped whitespace value into :last-ws.
+(define
+  hs-stream-skip-ws!
+  (fn
+    (s)
+    (let
+      ((tokens (get s :tokens)))
+      (define
+        loop
+        (fn
+          ()
+          (let
+            ((p (get s :pos)))
+            (when
+              (and (< p (len tokens))
+                   (= (get (nth tokens p) :type) "whitespace"))
+              (do
+                (dict-set! s :last-ws (get (nth tokens p) :value))
+                (dict-set! s :pos (+ p 1))
+                (loop))))))
+      (loop))))
+
+;; Current token (after skipping whitespace).
+(define
+  hs-stream-current
+  (fn
+    (s)
+    (do
+      (hs-stream-skip-ws! s)
+      (let
+        ((tokens (get s :tokens)) (p (get s :pos)))
+        (if (< p (len tokens)) (nth tokens p) nil)))))
+
+;; Returns the current token if its value matches; advances and updates
+;; :last-match. Returns nil otherwise (no advance).
+;; Honors the follow set: tokens whose value is in :follows do NOT match.
+(define
+  hs-stream-match
+  (fn
+    (s value)
+    (let
+      ((cur (hs-stream-current s)))
+      (cond
+        ((nil? cur) nil)
+        ((some (fn (f) (= f value)) (get s :follows)) nil)
+        ((= (get cur :value) value)
+          (do
+            (dict-set! s :pos (+ (get s :pos) 1))
+            (dict-set! s :last-match cur)
+            cur))
+        (true nil)))))
+
+;; Match by upstream-style type name. Accepts any number of allowed types.
+(define
+  hs-stream-match-type
+  (fn
+    (s &rest types)
+    (let
+      ((cur (hs-stream-current s)))
+      (cond
+        ((nil? cur) nil)
+        ((some (fn (t) (= (hs-stream-type-map (get cur :type)) t)) types)
+          (do
+            (dict-set! s :pos (+ (get s :pos) 1))
+            (dict-set! s :last-match cur)
+            cur))
+        (true nil)))))
+
+;; Match if value is one of the given names.
+(define
+  hs-stream-match-any
+  (fn
+    (s &rest names)
+    (let
+      ((cur (hs-stream-current s)))
+      (cond
+        ((nil? cur) nil)
+        ((some (fn (n) (= (get cur :value) n)) names)
+          (do
+            (dict-set! s :pos (+ (get s :pos) 1))
+            (dict-set! s :last-match cur)
+            cur))
+        (true nil)))))
+
+;; Match an op token whose value is in the list.
+(define
+  hs-stream-match-any-op
+  (fn
+    (s &rest ops)
+    (let
+      ((cur (hs-stream-current s)))
+      (cond
+        ((nil? cur) nil)
+        ((and (= (get cur :type) "op")
+              (some (fn (o) (= (get cur :value) o)) ops))
+          (do
+            (dict-set! s :pos (+ (get s :pos) 1))
+            (dict-set! s :last-match cur)
+            cur))
+        (true nil)))))
+
+;; Peek N non-WS tokens ahead. Returns the token if its value matches; nil otherwise.
+(define
+  hs-stream-peek
+  (fn
+    (s value offset)
+    (let
+      ((tokens (get s :tokens)))
+      (define
+        skip-n-non-ws
+        (fn
+          (p remaining)
+          (cond
+            ((>= p (len tokens)) -1)
+            ((= (get (nth tokens p) :type) "whitespace")
+              (skip-n-non-ws (+ p 1) remaining))
+            ((= remaining 0) p)
+            (true (skip-n-non-ws (+ p 1) (- remaining 1))))))
+      (let
+        ((p (skip-n-non-ws (get s :pos) offset)))
+        (if (and (>= p 0) (< p (len tokens))
+                 (= (get (nth tokens p) :value) value))
+          (nth tokens p)
+          nil)))))
+
+;; Consume tokens until one whose value matches the marker. Returns
+;; the consumed list (excluding the marker). Marker becomes current.
+(define
+  hs-stream-consume-until
+  (fn
+    (s marker)
+    (let
+      ((tokens (get s :tokens)) (out (list)))
+      (define
+        loop
+        (fn
+          (acc)
+          (let
+            ((p (get s :pos)))
+            (cond
+              ((>= p (len tokens)) acc)
+              ((= (get (nth tokens p) :value) marker) acc)
+              (true
+                (do
+                  (dict-set! s :pos (+ p 1))
+                  (loop (append acc (list (nth tokens p))))))))))
+      (loop out))))
+
+;; Consume until the next whitespace token; returns the consumed list.
+(define
+  hs-stream-consume-until-ws
+  (fn
+    (s)
+    (let
+      ((tokens (get s :tokens)))
+      (define
+        loop
+        (fn
+          (acc)
+          (let
+            ((p (get s :pos)))
+            (cond
+              ((>= p (len tokens)) acc)
+              ((= (get (nth tokens p) :type) "whitespace") acc)
+              (true
+                (do
+                  (dict-set! s :pos (+ p 1))
+                  (loop (append acc (list (nth tokens p))))))))))
+      (loop (list)))))
+
+;; Follow-set management.
+(define hs-stream-push-follow! (fn (s v) (dict-set! s :follows (cons v (get s :follows)))))
+(define
+  hs-stream-pop-follow!
+  (fn (s) (let ((f (get s :follows))) (when (> (len f) 0) (dict-set! s :follows (rest f))))))
+(define
+  hs-stream-push-follows!
+  (fn (s vs) (for-each (fn (v) (hs-stream-push-follow! s v)) vs)))
+(define
+  hs-stream-pop-follows!
+  (fn (s n) (when (> n 0) (do (hs-stream-pop-follow! s) (hs-stream-pop-follows! s (- n 1))))))
+(define
+  hs-stream-clear-follows!
+  (fn (s) (let ((saved (get s :follows))) (do (dict-set! s :follows (list)) saved))))
+(define
+  hs-stream-restore-follows!
+  (fn (s saved) (dict-set! s :follows saved)))
+
+;; Last-consumed token / whitespace.
+(define hs-stream-last-match (fn (s) (get s :last-match)))
+(define hs-stream-last-ws (fn (s) (get s :last-ws)))

lib/jit.sx

@@ -0,0 +1,89 @@
+;; lib/jit.sx — SX-level convenience wrappers over the JIT cache control
+;; primitives (jit-stats, jit-set-threshold!, jit-set-budget!, jit-reset-cache!,
+;; jit-reset-counters!). Host-specific implementations live in
+;; hosts/<host>/lib/sx_*.ml; the API surface is portable across hosts.
+
+;; with-jit-threshold — temporarily set the JIT call-count threshold for
+;; the duration of body, restoring the previous value on exit. Useful for
+;; sections that want eager compilation (threshold=1) or want to skip JIT
+;; entirely (threshold=999999) for diagnostic comparison.
+(defmacro
+  with-jit-threshold
+  (n &rest body)
+  `(let
+     ((__old (get (jit-stats) "threshold")))
+     (jit-set-threshold! ,n)
+     (let
+       ((__r (do ,@body)))
+       (jit-set-threshold! __old)
+       __r)))
+
+;; with-jit-budget — temporarily set the LRU cache budget. Setting to 0
+;; disables JIT entirely (everything falls through to the interpreter);
+;; large values are effectively unbounded.
+(defmacro
+  with-jit-budget
+  (n &rest body)
+  `(let
+     ((__old (get (jit-stats) "budget")))
+     (jit-set-budget! ,n)
+     (let
+       ((__r (do ,@body)))
+       (jit-set-budget! __old)
+       __r)))
+
+;; with-fresh-jit — clear the cache before body, run body, clear again
+;; after. Use between sessions / request batches / test suites where you
+;; want deterministic timing free of carryover.
+(defmacro
+  with-fresh-jit
+  (&rest body)
+  `(let
+     ((__r (do (jit-reset-cache!) ,@body)))
+     (jit-reset-cache!)
+     __r))
+
+;; jit-report — human-readable summary of current JIT state. Returns a
+;; string suitable for logging.
+(define
+  jit-report
+  (fn
+    ()
+    (let
+      ((s (jit-stats)))
+      (let
+        ((compiled (get s "compiled"))
+          (skipped (get s "below-threshold"))
+          (failed (get s "compile-failed"))
+          (evicted (get s "evicted"))
+          (cache-size (get s "cache-size"))
+          (budget (get s "budget"))
+          (threshold (get s "threshold")))
+        (let
+          ((total (+ compiled skipped failed)))
+          (str
+            "jit: " cache-size "/" budget " cached "
+            "(thr=" threshold ") · "
+            compiled " compiled, "
+            skipped " below-thr, "
+            failed " failed, "
+            evicted " evicted "
+            "(" (if (> total 0) (* 100 (/ compiled total)) 0) "% compile rate)"))))))
+
+;; jit-disable! / jit-enable! — convenience helpers. Disabling sets budget
+;; to 0 which causes the VM to skip JIT entirely on the next call. Enable
+;; restores the budget to its previous value (or 5000 if no previous).
+(define _jit-saved-budget (list 5000))
+
+(define
+  jit-disable!
+  (fn
+    ()
+    (set! _jit-saved-budget (list (get (jit-stats) "budget")))
+    (jit-set-budget! 0)))
+
+(define
+  jit-enable!
+  (fn
+    ()
+    (jit-set-budget! (first _jit-saved-budget))))

lib/js/lexer.sx

@@ -29,6 +29,16 @@
       (and (>= c "a") (<= c "f"))
       (and (>= c "A") (<= c "F")))))
 
+(define
+  js-hex-value
+  (fn
+    (c)
+    (cond
+      ((and (>= c "0") (<= c "9")) (- (char-code c) 48))
+      ((and (>= c "a") (<= c "f")) (- (char-code c) 87))
+      ((and (>= c "A") (<= c "F")) (- (char-code c) 55))
+      (else 0))))
+
 (define
   js-letter?
   (fn (c) (or (and (>= c "a") (<= c "z")) (and (>= c "A") (<= c "Z")))))
@@ -37,9 +47,9 @@
 
 (define js-ident-char? (fn (c) (or (js-ident-start? c) (js-digit? c))))
 
+;; ── Reserved words ────────────────────────────────────────────────
 (define js-ws? (fn (c) (or (= c " ") (= c "\t") (= c "\n") (= c "\r"))))
 
-;; ── Reserved words ────────────────────────────────────────────────
 (define
   js-keywords
   (list
@@ -86,15 +96,18 @@
     "await"
     "of"))
 
+;; ── Main tokenizer ────────────────────────────────────────────────
 (define js-keyword? (fn (word) (contains? js-keywords word)))
 
-;; ── Main tokenizer ────────────────────────────────────────────────
 (define
   js-tokenize
   (fn
     (src)
     (let
-      ((tokens (list)) (pos 0) (src-len (len src)))
+      ((tokens (list))
+        (pos 0)
+        (src-len (len src))
+        (nl-before false))
       (define
         js-peek
         (fn
@@ -109,11 +122,7 @@
           (let
             ((sl (len s)))
             (and (<= (+ pos sl) src-len) (= (slice src pos (+ pos sl)) s)))))
-      (define
-        js-emit!
-        (fn
-          (type value start)
-          (append! tokens (js-make-token type value start))))
+      (define js-emit! (fn (type value start) (append! tokens {:nl nl-before :type type :value value :pos start})))
       (define
         skip-line-comment!
         (fn
@@ -136,7 +145,13 @@
           ()
           (cond
             ((>= pos src-len) nil)
-            ((js-ws? (cur)) (do (advance! 1) (skip-ws!)))
+            ((js-ws? (cur))
+              (do
+                (when
+                  (or (= (cur) "\n") (= (cur) "\r"))
+                  (set! nl-before true))
+                (advance! 1)
+                (skip-ws!)))
             ((and (= (cur) "/") (< (+ pos 1) src-len) (= (js-peek 1) "/"))
               (do (advance! 2) (skip-line-comment!) (skip-ws!)))
             ((and (= (cur) "/") (< (+ pos 1) src-len) (= (js-peek 1) "*"))
@@ -254,11 +269,55 @@
                               ((= ch "b") (append! chars "\\b"))
                               ((= ch "f") (append! chars "\\f"))
                               ((= ch "v") (append! chars "\\v"))
+                              ((= ch "u")
+                                (if
+                                  (and
+                                    (< (+ pos 4) src-len)
+                                    (js-hex-digit? (js-peek 1))
+                                    (js-hex-digit? (js-peek 2))
+                                    (js-hex-digit? (js-peek 3))
+                                    (js-hex-digit? (js-peek 4)))
+                                  (do
+                                    (append!
+                                      chars
+                                      (char-from-code
+                                        (+
+                                          (*
+                                            4096
+                                            (js-hex-value
+                                              (js-peek 1)))
+                                          (*
+                                            256
+                                            (js-hex-value
+                                              (js-peek 2)))
+                                          (*
+                                            16
+                                            (js-hex-value
+                                              (js-peek 3)))
+                                          (js-hex-value (js-peek 4)))))
+                                    (advance! 4))
+                                  (append! chars ch)))
+                              ((= ch "x")
+                                (if
+                                  (and
+                                    (< (+ pos 2) src-len)
+                                    (js-hex-digit? (js-peek 1))
+                                    (js-hex-digit? (js-peek 2)))
+                                  (do
+                                    (append!
+                                      chars
+                                      (char-from-code
+                                        (+
+                                          (* 16 (js-hex-value (js-peek 1)))
+                                          (js-hex-value (js-peek 2)))))
+                                    (advance! 2))
+                                  (append! chars ch)))
                               (else (append! chars ch)))
                             (advance! 1))))
                       (loop)))
                   ((= (cur) quote-char) (advance! 1))
-                  (else (do (append! chars (cur)) (advance! 1) (loop))))))
+                  (else
+                    (do (append! chars (cur)) (advance! 1) (loop))))))
             (loop)
             (join "" chars))))
       (define
@@ -289,7 +348,8 @@
                       ()
                       (cond
                         ((>= pos src-len) nil)
-                        ((and (= (cur) "}") (= depth 1)) (advance! 1))
+                        ((and (= (cur) "}") (= depth 1))
+                          (advance! 1))
                         ((= (cur) "}")
                           (do
                             (append! buf (cur))
@@ -325,7 +385,9 @@
                                             (advance! 1)))
                                         (sloop)))
                                     ((= (cur) q)
-                                      (do (append! buf (cur)) (advance! 1)))
+                                      (do
+                                        (append! buf (cur))
+                                        (advance! 1)))
                                     (else
                                       (do
                                         (append! buf (cur))
@@ -334,7 +396,10 @@
                               (sloop)
                               (expr-loop))))
                         (else
-                          (do (append! buf (cur)) (advance! 1) (expr-loop))))))
+                          (do
+                            (append! buf (cur))
+                            (advance! 1)
+                            (expr-loop))))))
                   (expr-loop)
                   (join "" buf))))
             (define
@@ -376,14 +441,17 @@
                               (else (append! chars ch)))
                             (advance! 1))))
                       (loop)))
-                  (else (do (append! chars (cur)) (advance! 1) (loop))))))
+                  (else
+                    (do (append! chars (cur)) (advance! 1) (loop))))))
             (loop)
             (flush-chars!)
             (if
               (= (len parts) 0)
               ""
               (if
-                (and (= (len parts) 1) (= (nth (nth parts 0) 0) "str"))
+                (and
+                  (= (len parts) 1)
+                  (= (nth (nth parts 0) 0) "str"))
                 (nth (nth parts 0) 1)
                 parts)))))
       (define
@@ -399,7 +467,7 @@
                 ((ty (dict-get tk "type")) (vv (dict-get tk "value")))
                 (cond
                   ((= ty "punct")
-                    (and (not (= vv ")")) (not (= vv "]"))))
+                    (and (not (= vv ")")) (not (= vv "]")) (not (= vv "}"))))
                   ((= ty "op") true)
                   ((= ty "keyword")
                     (contains?
@@ -453,9 +521,13 @@
                       (append! buf (cur))
                       (advance! 1)
                       (body-loop)))
-                  ((and (= (cur) "/") (not in-class)) (advance! 1))
+                  ((and (= (cur) "/") (not in-class))
+                    (advance! 1))
                   (else
-                    (begin (append! buf (cur)) (advance! 1) (body-loop))))))
+                    (begin
+                      (append! buf (cur))
+                      (advance! 1)
+                      (body-loop))))))
             (body-loop)
             (let
               ((flags-buf (list)))
@@ -470,7 +542,7 @@
                       (advance! 1)
                       (flags-loop)))))
               (flags-loop)
-              {:pattern (join "" buf) :flags (join "" flags-buf)}))))
+              {:flags (join "" flags-buf) :pattern (join "" buf)}))))
       (define
         try-op-4!
         (fn
@@ -510,64 +582,113 @@
         (fn
           (start)
           (cond
-            ((at? "==") (do (js-emit! "op" "==" start) (advance! 2) true))
-            ((at? "!=") (do (js-emit! "op" "!=" start) (advance! 2) true))
-            ((at? "<=") (do (js-emit! "op" "<=" start) (advance! 2) true))
-            ((at? ">=") (do (js-emit! "op" ">=" start) (advance! 2) true))
-            ((at? "&&") (do (js-emit! "op" "&&" start) (advance! 2) true))
-            ((at? "||") (do (js-emit! "op" "||" start) (advance! 2) true))
-            ((at? "??") (do (js-emit! "op" "??" start) (advance! 2) true))
-            ((at? "=>") (do (js-emit! "op" "=>" start) (advance! 2) true))
-            ((at? "**") (do (js-emit! "op" "**" start) (advance! 2) true))
-            ((at? "<<") (do (js-emit! "op" "<<" start) (advance! 2) true))
-            ((at? ">>") (do (js-emit! "op" ">>" start) (advance! 2) true))
-            ((at? "++") (do (js-emit! "op" "++" start) (advance! 2) true))
-            ((at? "--") (do (js-emit! "op" "--" start) (advance! 2) true))
-            ((at? "+=") (do (js-emit! "op" "+=" start) (advance! 2) true))
-            ((at? "-=") (do (js-emit! "op" "-=" start) (advance! 2) true))
-            ((at? "*=") (do (js-emit! "op" "*=" start) (advance! 2) true))
-            ((at? "/=") (do (js-emit! "op" "/=" start) (advance! 2) true))
-            ((at? "%=") (do (js-emit! "op" "%=" start) (advance! 2) true))
-            ((at? "&=") (do (js-emit! "op" "&=" start) (advance! 2) true))
-            ((at? "|=") (do (js-emit! "op" "|=" start) (advance! 2) true))
-            ((at? "^=") (do (js-emit! "op" "^=" start) (advance! 2) true))
-            ((at? "?.") (do (js-emit! "op" "?." start) (advance! 2) true))
+            ((at? "==")
+              (do (js-emit! "op" "==" start) (advance! 2) true))
+            ((at? "!=")
+              (do (js-emit! "op" "!=" start) (advance! 2) true))
+            ((at? "<=")
+              (do (js-emit! "op" "<=" start) (advance! 2) true))
+            ((at? ">=")
+              (do (js-emit! "op" ">=" start) (advance! 2) true))
+            ((at? "&&")
+              (do (js-emit! "op" "&&" start) (advance! 2) true))
+            ((at? "||")
+              (do (js-emit! "op" "||" start) (advance! 2) true))
+            ((at? "??")
+              (do (js-emit! "op" "??" start) (advance! 2) true))
+            ((at? "=>")
+              (do (js-emit! "op" "=>" start) (advance! 2) true))
+            ((at? "**")
+              (do (js-emit! "op" "**" start) (advance! 2) true))
+            ((at? "<<")
+              (do (js-emit! "op" "<<" start) (advance! 2) true))
+            ((at? ">>")
+              (do (js-emit! "op" ">>" start) (advance! 2) true))
+            ((at? "++")
+              (do (js-emit! "op" "++" start) (advance! 2) true))
+            ((at? "--")
+              (do (js-emit! "op" "--" start) (advance! 2) true))
+            ((at? "+=")
+              (do (js-emit! "op" "+=" start) (advance! 2) true))
+            ((at? "-=")
+              (do (js-emit! "op" "-=" start) (advance! 2) true))
+            ((at? "*=")
+              (do (js-emit! "op" "*=" start) (advance! 2) true))
+            ((at? "/=")
+              (do (js-emit! "op" "/=" start) (advance! 2) true))
+            ((at? "%=")
+              (do (js-emit! "op" "%=" start) (advance! 2) true))
+            ((at? "&=")
+              (do (js-emit! "op" "&=" start) (advance! 2) true))
+            ((at? "|=")
+              (do (js-emit! "op" "|=" start) (advance! 2) true))
+            ((at? "^=")
+              (do (js-emit! "op" "^=" start) (advance! 2) true))
+            ((at? "?.")
+              (do (js-emit! "op" "?." start) (advance! 2) true))
             (else false))))
       (define
         emit-one-op!
         (fn
           (ch start)
           (cond
-            ((= ch "(") (do (js-emit! "punct" "(" start) (advance! 1)))
-            ((= ch ")") (do (js-emit! "punct" ")" start) (advance! 1)))
-            ((= ch "[") (do (js-emit! "punct" "[" start) (advance! 1)))
-            ((= ch "]") (do (js-emit! "punct" "]" start) (advance! 1)))
-            ((= ch "{") (do (js-emit! "punct" "{" start) (advance! 1)))
-            ((= ch "}") (do (js-emit! "punct" "}" start) (advance! 1)))
-            ((= ch ",") (do (js-emit! "punct" "," start) (advance! 1)))
-            ((= ch ";") (do (js-emit! "punct" ";" start) (advance! 1)))
-            ((= ch ":") (do (js-emit! "punct" ":" start) (advance! 1)))
-            ((= ch ".") (do (js-emit! "punct" "." start) (advance! 1)))
-            ((= ch "?") (do (js-emit! "op" "?" start) (advance! 1)))
-            ((= ch "+") (do (js-emit! "op" "+" start) (advance! 1)))
-            ((= ch "-") (do (js-emit! "op" "-" start) (advance! 1)))
-            ((= ch "*") (do (js-emit! "op" "*" start) (advance! 1)))
-            ((= ch "/") (do (js-emit! "op" "/" start) (advance! 1)))
-            ((= ch "%") (do (js-emit! "op" "%" start) (advance! 1)))
-            ((= ch "=") (do (js-emit! "op" "=" start) (advance! 1)))
-            ((= ch "<") (do (js-emit! "op" "<" start) (advance! 1)))
-            ((= ch ">") (do (js-emit! "op" ">" start) (advance! 1)))
-            ((= ch "!") (do (js-emit! "op" "!" start) (advance! 1)))
-            ((= ch "&") (do (js-emit! "op" "&" start) (advance! 1)))
-            ((= ch "|") (do (js-emit! "op" "|" start) (advance! 1)))
-            ((= ch "^") (do (js-emit! "op" "^" start) (advance! 1)))
-            ((= ch "~") (do (js-emit! "op" "~" start) (advance! 1)))
+            ((= ch "(")
+              (do (js-emit! "punct" "(" start) (advance! 1)))
+            ((= ch ")")
+              (do (js-emit! "punct" ")" start) (advance! 1)))
+            ((= ch "[")
+              (do (js-emit! "punct" "[" start) (advance! 1)))
+            ((= ch "]")
+              (do (js-emit! "punct" "]" start) (advance! 1)))
+            ((= ch "{")
+              (do (js-emit! "punct" "{" start) (advance! 1)))
+            ((= ch "}")
+              (do (js-emit! "punct" "}" start) (advance! 1)))
+            ((= ch ",")
+              (do (js-emit! "punct" "," start) (advance! 1)))
+            ((= ch ";")
+              (do (js-emit! "punct" ";" start) (advance! 1)))
+            ((= ch ":")
+              (do (js-emit! "punct" ":" start) (advance! 1)))
+            ((= ch ".")
+              (do (js-emit! "punct" "." start) (advance! 1)))
+            ((= ch "?")
+              (do (js-emit! "op" "?" start) (advance! 1)))
+            ((= ch "+")
+              (do (js-emit! "op" "+" start) (advance! 1)))
+            ((= ch "-")
+              (do (js-emit! "op" "-" start) (advance! 1)))
+            ((= ch "*")
+              (do (js-emit! "op" "*" start) (advance! 1)))
+            ((= ch "/")
+              (do (js-emit! "op" "/" start) (advance! 1)))
+            ((= ch "%")
+              (do (js-emit! "op" "%" start) (advance! 1)))
+            ((= ch "=")
+              (do (js-emit! "op" "=" start) (advance! 1)))
+            ((= ch "<")
+              (do (js-emit! "op" "<" start) (advance! 1)))
+            ((= ch ">")
+              (do (js-emit! "op" ">" start) (advance! 1)))
+            ((= ch "!")
+              (do (js-emit! "op" "!" start) (advance! 1)))
+            ((= ch "&")
+              (do (js-emit! "op" "&" start) (advance! 1)))
+            ((= ch "|")
+              (do (js-emit! "op" "|" start) (advance! 1)))
+            ((= ch "^")
+              (do (js-emit! "op" "^" start) (advance! 1)))
+            ((= ch "~")
+              (do (js-emit! "op" "~" start) (advance! 1)))
+            ((= ch "\\")
+              (error "Unexpected char '\\' in source"))
             (else (advance! 1)))))
       (define
         scan!
         (fn
           ()
           (do
+            (set! nl-before false)
             (skip-ws!)
             (when
               (< pos src-len)

lib/js/parser.sx

@@ -153,6 +153,32 @@
           (do (jp-advance! st) (list (quote js-ident) "this")))
         ((and (= (get t :type) "keyword") (= (get t :value) "new"))
           (do (jp-advance! st) (jp-parse-new-expr st)))
+        ((and (= (get t :type) "keyword") (= (get t :value) "function"))
+          (do
+            (jp-advance! st)
+            (let
+              ((nm
+                 (if
+                   (= (get (jp-peek st) :type) "ident")
+                   (let ((n (get (jp-peek st) :value))) (do (jp-advance! st) n))
+                   nil)))
+              (let
+                ((params (jp-parse-param-list st)))
+                (let
+                  ((body (jp-parse-fn-body st)))
+                  (list (quote js-funcexpr) nm params body))))))
+        ((and (= (get t :type) "keyword") (= (get t :value) "true"))
+          (do (jp-advance! st) (list (quote js-bool) true)))
+        ((and (= (get t :type) "keyword") (= (get t :value) "false"))
+          (do (jp-advance! st) (list (quote js-bool) false)))
+        ((and (= (get t :type) "keyword") (= (get t :value) "null"))
+          (do (jp-advance! st) (list (quote js-null))))
+        ((and (= (get t :type) "keyword") (= (get t :value) "undefined"))
+          (do (jp-advance! st) (list (quote js-undef))))
+        ((= (get t :type) "number")
+          (do (jp-advance! st) (list (quote js-num) (get t :value))))
+        ((= (get t :type) "string")
+          (do (jp-advance! st) (list (quote js-str) (get t :value))))
         ((and (= (get t :type) "punct") (= (get t :value) "("))
           (jp-parse-paren-or-arrow st))
         (else
@@ -211,7 +237,7 @@
               (let
                 ((params (jp-parse-param-list st)))
                 (let
-                  ((body (jp-parse-block st)))
+                  ((body (jp-parse-fn-body st)))
                   (list (quote js-funcexpr-async) nm params body))))))
         ((= (get t :type) "ident")
           (do
@@ -363,7 +389,7 @@
               (let
                 ((params (jp-parse-param-list st)))
                 (let
-                  ((body (jp-parse-block st)))
+                  ((body (jp-parse-fn-body st)))
                   (list (quote js-funcexpr) nm params body))))))
         ((= (get t :type) "ident")
           (do
@@ -418,16 +444,51 @@
               (dict-set! st :idx saved)
               (jp-advance! st)
               (let
-                ((e (jp-parse-assignment st)))
+                ((e (jp-parse-comma-seq st)))
                 (jp-expect! st "punct" ")")
-                e)))
+                (jp-paren-wrap e))))
           (do
             (dict-set! st :idx saved)
             (jp-advance! st)
             (let
-              ((e (jp-parse-assignment st)))
+              ((e (jp-parse-comma-seq st)))
               (jp-expect! st "punct" ")")
-              e)))))))
+              (jp-paren-wrap e))))))))
+
+(define
+  jp-paren-wrap
+  (fn
+    (e)
+    (cond
+      ((and (list? e) (= (first e) (quote js-unop)))
+        (list (quote js-paren) e))
+      (else e))))
+
+(define
+  jp-parse-comma-seq
+  (fn
+    (st)
+    (let
+      ((first-expr (jp-parse-assignment st)))
+      (if
+        (jp-at? st "punct" ",")
+        (jp-parse-comma-seq-rest st (list first-expr))
+        first-expr))))
+
+(define
+  jp-parse-comma-seq-rest
+  (fn
+    (st acc)
+    (do
+      (jp-advance! st)
+      (let
+        ((next-expr (jp-parse-assignment st)))
+        (let
+          ((acc2 (append acc (list next-expr))))
+          (if
+            (jp-at? st "punct" ",")
+            (jp-parse-comma-seq-rest st acc2)
+            (cons (quote js-comma) (list acc2))))))))
 
 (define
   jp-collect-params
@@ -485,6 +546,11 @@
     (st elems)
     (cond
       ((jp-at? st "punct" "]") nil)
+      ((jp-at? st "punct" ",")
+        (begin
+          (append! elems (list (quote js-undef)))
+          (jp-advance! st)
+          (jp-array-loop st elems)))
       (else
         (begin
           (cond
@@ -558,6 +624,20 @@
             (jp-advance! st)
             (jp-expect! st "punct" ":")
             (append! kvs {:value (jp-parse-assignment st) :key (get t :value)})))
+        ((and (= (get t :type) "punct") (= (get t :value) "["))
+          (do
+            (jp-advance! st)
+            (let
+              ((key-expr (jp-parse-assignment st)))
+              (jp-expect! st "punct" "]")
+              (jp-expect! st "punct" ":")
+              (append!
+                kvs
+                {:value (jp-parse-assignment st) :computed-key key-expr :key ""}))))
+        ((and (= (get t :type) "punct") (= (get t :value) "..."))
+          (do
+            (jp-advance! st)
+            (append! kvs {:spread (jp-parse-assignment st)})))
         (else (error (str "Unexpected in object: " (get t :type))))))))
 
 (define
@@ -629,7 +709,7 @@
                       st
                       (list (quote js-optchain-member) left (get t :value))))
                   (error "expected ident, [ or ( after ?.")))))))
-      ((or (jp-at? st "op" "++") (jp-at? st "op" "--"))
+      ((and (or (jp-at? st "op" "++") (jp-at? st "op" "--")) (not (jp-token-nl? st)))
         (let
           ((op (get (jp-peek st) :value)))
           (jp-advance! st)
@@ -682,6 +762,12 @@
             (cond
               ((< prec 0) left)
               ((< prec min-prec) left)
+              ((and (= op "**") (list? left) (= (first left) (quote js-unop)))
+                (error
+                  (str
+                    "SyntaxError: Unary operator '"
+                    (nth left 1)
+                    "' used immediately before exponentiation expression")))
               (else
                 (do
                   (jp-advance! st)
@@ -835,6 +921,12 @@
   jp-eat-semi
   (fn (st) (if (jp-at? st "punct" ";") (do (jp-advance! st) nil) nil)))
 
+(define
+  jp-token-nl?
+  (fn
+    (st)
+    (let ((tok (jp-peek st))) (if tok (= (get tok :nl) true) false))))
+
 (define
   jp-parse-vardecl
   (fn
@@ -1052,15 +1144,63 @@
         ((c (jp-parse-assignment st)))
         (do
           (jp-expect! st "punct" ")")
+          (jp-disallow-decl-stmt! st "if")
           (let
             ((t (jp-parse-stmt st)))
             (if
               (jp-at? st "keyword" "else")
               (do
                 (jp-advance! st)
+                (jp-disallow-decl-stmt! st "else")
                 (list (quote js-if) c t (jp-parse-stmt st)))
               (list (quote js-if) c t nil))))))))
 
+(define
+  jp-disallow-decl-stmt!
+  (fn
+    (st context)
+    (let
+      ((t (jp-peek st)))
+      (cond
+        ((and (= (get t :type) "keyword")
+              (or (= (get t :value) "let")
+                  (= (get t :value) "const")
+                  (= (get t :value) "function")
+                  (= (get t :value) "class")))
+          (cond
+            ((and (= (get t :value) "let")
+                  (or (= (get (jp-peek-at st 1) :type) "ident")
+                      (and (= (get (jp-peek-at st 1) :type) "punct")
+                           (or (= (get (jp-peek-at st 1) :value) "[")
+                               (= (get (jp-peek-at st 1) :value) "{")))))
+              (error
+                (str
+                  "SyntaxError: Lexical declaration cannot appear in single-statement context: "
+                  context)))
+            ((or (= (get t :value) "const")
+                 (= (get t :value) "function")
+                 (= (get t :value) "class"))
+              (error
+                (str
+                  "SyntaxError: "
+                  (get t :value)
+                  " declaration cannot appear in single-statement context: "
+                  context)))
+            (else nil)))
+        (else nil)))))
+
+(define
+  jp-bump!
+  (fn
+    (st key)
+    (dict-set! st key (+ (get st key) 1))))
+
+(define
+  jp-decr!
+  (fn
+    (st key)
+    (dict-set! st key (- (get st key) 1))))
+
 (define
   jp-parse-while-stmt
   (fn
@@ -1072,7 +1212,11 @@
         ((c (jp-parse-assignment st)))
         (do
           (jp-expect! st "punct" ")")
-          (let ((body (jp-parse-stmt st))) (list (quote js-while) c body)))))))
+          (jp-disallow-decl-stmt! st "while")
+          (jp-bump! st :loop-depth)
+          (let ((body (jp-parse-stmt st)))
+            (jp-decr! st :loop-depth)
+            (list (quote js-while) c body)))))))
 
 (define
   jp-parse-do-while-stmt
@@ -1080,8 +1224,11 @@
     (st)
     (do
       (jp-advance! st)
+      (jp-disallow-decl-stmt! st "do")
+      (jp-bump! st :loop-depth)
       (let
         ((body (jp-parse-stmt st)))
+        (jp-decr! st :loop-depth)
         (do
           (if
             (jp-at? st "keyword" "while")
@@ -1126,8 +1273,11 @@
                   (let
                     ((iter (jp-parse-assignment st)))
                     (jp-expect! st "punct" ")")
+                    (jp-disallow-decl-stmt! st "for-of/in")
+                    (jp-bump! st :loop-depth)
                     (let
                       ((body (jp-parse-stmt st)))
+                      (jp-decr! st :loop-depth)
                       (list (quote js-for-of-in) iter-kind ident iter body)))))))
           (else
             (let
@@ -1138,8 +1288,11 @@
                 (let
                   ((step (if (jp-at? st "punct" ")") nil (jp-parse-assignment st))))
                   (jp-expect! st "punct" ")")
+                  (jp-disallow-decl-stmt! st "for")
+                  (jp-bump! st :loop-depth)
                   (let
                     ((body (jp-parse-stmt st)))
+                    (jp-decr! st :loop-depth)
                     (list (quote js-for) init cond-ast step body)))))))))))
 
 (define
@@ -1162,10 +1315,14 @@
     (st)
     (do
       (jp-advance! st)
+      (when
+        (= (get st :fn-depth) 0)
+        (error "SyntaxError: Illegal return statement"))
       (if
         (or
           (jp-at? st "punct" ";")
           (jp-at? st "punct" "}")
+          (jp-token-nl? st)
           (jp-at? st "eof" nil))
         (do (jp-eat-semi st) (list (quote js-return) nil))
         (let
@@ -1188,7 +1345,7 @@
           (let
             ((params (jp-parse-param-list st)))
             (let
-              ((body (jp-parse-block st)))
+              ((body (jp-parse-fn-body st)))
               (list (quote js-funcdecl) nm params body))))))))
 
 (define
@@ -1207,7 +1364,7 @@
           (let
             ((params (jp-parse-param-list st)))
             (let
-              ((body (jp-parse-block st)))
+              ((body (jp-parse-fn-body st)))
               (list (quote js-funcdecl-async) nm params body))))))))
 
 (define
@@ -1256,7 +1413,7 @@
           (let
             ((params (jp-parse-param-list st)))
             (let
-              ((body (jp-parse-block st)))
+              ((body (jp-parse-fn-body st)))
               (list
                 (quote js-method)
                 (if static? "static" "instance")
@@ -1284,9 +1441,11 @@
       ((disc (jp-parse-assignment st)))
       (jp-expect! st "punct" ")")
       (jp-expect! st "punct" "{")
+      (jp-bump! st :switch-depth)
       (let
         ((cases (list)))
         (jp-parse-switch-cases st cases)
+        (jp-decr! st :switch-depth)
         (jp-expect! st "punct" "}")
         (list (quote js-switch) disc cases)))))
 
@@ -1362,9 +1521,40 @@
       ((jp-at? st "keyword" "for") (jp-parse-for-stmt st))
       ((jp-at? st "keyword" "return") (jp-parse-return-stmt st))
       ((jp-at? st "keyword" "break")
-        (do (jp-advance! st) (jp-eat-semi st) (list (quote js-break))))
+        (do
+          (jp-advance! st)
+          (cond
+            ((= (get (jp-peek st) :type) "ident")
+              (do (jp-advance! st) (jp-eat-semi st) (list (quote js-break))))
+            (else
+              (do
+                (when
+                  (and (= (get st :loop-depth) 0) (= (get st :switch-depth) 0))
+                  (error "SyntaxError: Illegal break statement"))
+                (jp-eat-semi st)
+                (list (quote js-break)))))))
       ((jp-at? st "keyword" "continue")
-        (do (jp-advance! st) (jp-eat-semi st) (list (quote js-continue))))
+        (do
+          (jp-advance! st)
+          (cond
+            ((= (get (jp-peek st) :type) "ident")
+              (do (jp-advance! st) (jp-eat-semi st) (list (quote js-continue))))
+            (else
+              (do
+                (when
+                  (= (get st :loop-depth) 0)
+                  (error "SyntaxError: Illegal continue statement"))
+                (jp-eat-semi st)
+                (list (quote js-continue)))))))
+      ((and
+         (= (get (jp-peek st) :type) "ident")
+         (= (get (jp-peek-at st 1) :type) "punct")
+         (= (get (jp-peek-at st 1) :value) ":"))
+        (do
+          (jp-advance! st)
+          (jp-advance! st)
+          (jp-disallow-decl-stmt! st "label")
+          (jp-parse-stmt st)))
       ((jp-at? st "keyword" "class") (jp-parse-class-decl st))
       ((jp-at? st "keyword" "throw") (jp-parse-throw-stmt st))
       ((jp-at? st "keyword" "try") (jp-parse-try-stmt st))
@@ -1374,7 +1564,7 @@
       ((jp-at? st "keyword" "switch") (jp-parse-switch-stmt st))
       (else
         (let
-          ((e (jp-parse-assignment st)))
+          ((e (jp-parse-comma-seq st)))
           (do (jp-eat-semi st) (list (quote js-exprstmt) e)))))))
 
 (define
@@ -1400,10 +1590,33 @@
   jp-parse-arrow-body
   (fn
     (st)
-    (if
-      (jp-at? st "punct" "{")
-      (jp-parse-block st)
-      (jp-parse-assignment st))))
+    (jp-bump! st :fn-depth)
+    (let
+      ((saved-loop (get st :loop-depth)) (saved-switch (get st :switch-depth)))
+      (dict-set! st :loop-depth 0)
+      (dict-set! st :switch-depth 0)
+      (let
+        ((body (if (jp-at? st "punct" "{") (jp-parse-block st) (jp-parse-assignment st))))
+        (jp-decr! st :fn-depth)
+        (dict-set! st :loop-depth saved-loop)
+        (dict-set! st :switch-depth saved-switch)
+        body))))
+
+(define
+  jp-parse-fn-body
+  (fn
+    (st)
+    (jp-bump! st :fn-depth)
+    (let
+      ((saved-loop (get st :loop-depth)) (saved-switch (get st :switch-depth)))
+      (dict-set! st :loop-depth 0)
+      (dict-set! st :switch-depth 0)
+      (let
+        ((body (jp-parse-block st)))
+        (jp-decr! st :fn-depth)
+        (dict-set! st :loop-depth saved-loop)
+        (dict-set! st :switch-depth saved-switch)
+        body))))
 
 (define
   js-parse
@@ -1414,7 +1627,7 @@
         (= (len tokens) 0)
         (and (= (len tokens) 1) (= (get (nth tokens 0) :type) "eof")))
       (list (quote js-program) (list))
-      (let ((st {:idx 0 :tokens tokens :arrow-candidate true})) (jp-parse-program st)))))
+      (let ((st {:idx 0 :tokens tokens :arrow-candidate true :loop-depth 0 :switch-depth 0 :fn-depth 0})) (jp-parse-program st)))))
 
 (define
   js-parse-expr
@@ -1427,4 +1640,4 @@
           (= (len tokens) 0)
           (and (= (len tokens) 1) (= (get (nth tokens 0) :type) "eof")))
         (list)
-        (let ((st {:idx 0 :tokens tokens :arrow-candidate true})) (jp-parse-assignment st))))))
+        (let ((st {:idx 0 :tokens tokens :arrow-candidate true :loop-depth 0 :switch-depth 0 :fn-depth 0})) (jp-parse-assignment st))))))

lib/js/test.sh

@@ -1486,6 +1486,24 @@ cat > "$TMPFILE" << 'EPOCHS'
 (eval "(get (RegExp \"hello\" \"gi\") \"global\")")
 (epoch 6032)
 (eval "(get (RegExp \"foo\" \"i\") \"ignoreCase\")")
+;; ── Phase 1.ASI: automatic semicolon insertion ─────────────────
+(epoch 4200)
+(eval "(js-eval \"function f() { return\n42\n} f()\")")
+(epoch 4201)
+(eval "(js-eval \"function g() { return 42 } g()\")")
+(epoch 4202)
+(eval "(let ((toks (js-tokenize \"a\nb\"))) (get (nth toks 1) :nl))")
+(epoch 4203)
+(eval "(let ((toks (js-tokenize \"a b\"))) (get (nth toks 1) :nl))")
+
+(epoch 4300)
+(eval "(js-eval \"var x = 5; x\")")
+(epoch 4301)
+(eval "(js-eval \"function f() { return x; var x = 42; } f()\")")
+(epoch 4302)
+(eval "(js-eval \"function f() { var y = 7; return y; } f()\")")
+(epoch 4303)
+(eval "(js-eval \"function f() { var z; z = 3; return z; } f()\")")
 
 EPOCHS
 
@@ -2280,6 +2298,16 @@ check 6025 "set delete→size 0"     '0'
 check 6030 "RegExp? result"        'true'
 check 6031 "RegExp global flag"    'true'
 check 6032 "RegExp ignoreCase"     'true'
+# ── Phase 1.ASI: automatic semicolon insertion ────────────────────
+check 4200 "return+newline → undefined"    '"js-undefined"'
+check 4201 "return+space+val → val"        '42'
+check 4202 "nl-before flag set after newline" 'true'
+check 4203 "nl-before flag false on same line" 'false'
+
+check 4300 "var decl program-level"         '5'
+check 4301 "var hoisted before use → undef" '"js-undefined"'
+check 4302 "var in function body"           '7'
+check 4303 "var then set in function"       '3'
 
 TOTAL=$((PASS + FAIL))
 if [ $FAIL -eq 0 ]; then

lib/js/test262-runner.py

@@ -52,7 +52,7 @@ UPSTREAM = REPO / "lib" / "js" / "test262-upstream"
 TEST_ROOT = UPSTREAM / "test"
 HARNESS_DIR = UPSTREAM / "harness"
 
-DEFAULT_PER_TEST_TIMEOUT_S = 5.0
+DEFAULT_PER_TEST_TIMEOUT_S = 15.0
 DEFAULT_BATCH_TIMEOUT_S = 120
 
 # Cache dir for precomputed SX source of harness JS (one file per Python run).
@@ -134,6 +134,9 @@ var verifyProperty = function (obj, name, desc, opts) {
   }
 };
 var verifyPrimordialProperty = verifyProperty;
+var verifyEqualTo = function (obj, name, value) {
+  assert.sameValue(obj[name], value, name + " equals");
+};
 var verifyNotEnumerable = function (o, n, v, w, x) { };
 var verifyNotWritable = function (o, n, v, w, x) { };
 var verifyNotConfigurable = function (o, n, v, w, x) { };
@@ -146,6 +149,50 @@ var isConstructor = function (f) {
   // Best-effort: built-in functions and arrows aren't; declared `function` decls are.
   return false;
 };
+// $DONE / asyncTest — async-flag tests call $DONE(err) to signal completion.
+// Since we drain microtasks synchronously, $DONE is just a final-assertion sink.
+var $DONE = function (err) {
+  if (err) { throw new Test262Error((err && err.message) || err); }
+};
+var asyncTest = function (testFunc) {
+  Promise.resolve(testFunc()).then(function () { $DONE(); }, function (e) { $DONE(e); });
+};
+// promiseHelper.js include — used by Promise.all/race tests for ordering checks.
+var checkSequence = function (arr, message) {
+  for (var i = 0; i < arr.length; i = i + 1) {
+    if (arr[i] !== (i + 1)) {
+      throw new Test262Error((message || "Sequence") + " expected " + (i+1) + " at index " + i + " but got " + arr[i]);
+    }
+  }
+  return true;
+};
+var checkSettledPromises = function (settleds, expected, message) {
+  var msg = message ? message + " " : "";
+  if (settleds.length !== expected.length) {
+    throw new Test262Error(msg + "lengths differ: " + settleds.length + " vs " + expected.length);
+  }
+  for (var i = 0; i < settleds.length; i = i + 1) {
+    if (settleds[i].status !== expected[i].status) {
+      throw new Test262Error(msg + "status[" + i + "]: " + settleds[i].status + " vs " + expected[i].status);
+    }
+    if (expected[i].status === "fulfilled" && settleds[i].value !== expected[i].value) {
+      throw new Test262Error(msg + "value[" + i + "]: " + settleds[i].value + " vs " + expected[i].value);
+    }
+    if (expected[i].status === "rejected" && settleds[i].reason !== expected[i].reason) {
+      throw new Test262Error(msg + "reason[" + i + "]: " + settleds[i].reason + " vs " + expected[i].reason);
+    }
+  }
+};
+// decimalToHexString.js include — used by URI/escape tests.
+var decimalToHexString = function (n) {
+  var hex = "0123456789ABCDEF";
+  if (n < 0) { n = n + 65536; }
+  return hex[(n >> 12) & 15] + hex[(n >> 8) & 15] + hex[(n >> 4) & 15] + hex[n & 15];
+};
+var decimalToPercentHexString = function (n) {
+  var hex = "0123456789ABCDEF";
+  return "%" + hex[(n >> 4) & 15] + hex[n & 15];
+};
 // Trivial helper for tests that use Array.isArray-like functionality
 // (many tests reach for it via compareArray)
 """
@@ -358,6 +405,8 @@ def classify_negative_result(fm: Frontmatter, kind: str, payload: str):
             or ("expected" in low and "got" in low)
             or "js-transpile-unop" in low
             or "js-transpile-binop" in low
+            or "js-transpile-assign" in low
+            or "js-transpile" in low
             or "js-compound-update" in low
             or "parse" in low
         ):
@@ -1012,11 +1061,45 @@ def _worker_run(args):
 # ---------------------------------------------------------------------------
 
 
+_HARNESS_INCLUDE_CACHE: dict = {}
+
+# Only inline these small harness files per-test. Large ones like propertyHelper.js
+# multiply js-eval/JIT cost by ~5-10x and push tests over the per-test timeout.
+_INLINE_INCLUDES = {"nans.js", "sta.js", "byteConversionValues.js", "compareArray.js"}
+
+
+def _load_harness_include(name: str) -> str:
+    """Read an upstream harness include file (e.g. nans.js).
+    Returns empty string if the file isn't present.
+    """
+    if name in _HARNESS_INCLUDE_CACHE:
+        return _HARNESS_INCLUDE_CACHE[name]
+    path = HARNESS_DIR / name
+    try:
+        src = path.read_text()
+    except OSError:
+        src = ""
+    _HARNESS_INCLUDE_CACHE[name] = src
+    return src
+
+
 def assemble_source(t):
     """Return JS source to feed to js-eval. Harness is preloaded, so we only
-    append the test source (plus negative-test prep if needed).
+    append the test source (plus a small allowlist of per-test includes).
     """
-    return t.src
+    if not getattr(t.fm, "includes", None):
+        return t.src
+    parts = []
+    for inc in t.fm.includes:
+        if inc not in _INLINE_INCLUDES:
+            continue
+        chunk = _load_harness_include(inc)
+        if chunk:
+            parts.append(chunk)
+    if not parts:
+        return t.src
+    parts.append(t.src)
+    return "\n".join(parts)
 
 
 def aggregate(results):
@@ -1194,7 +1277,7 @@ def main(argv):
     shards = [[] for _ in range(n_workers)]
     for i, t in enumerate(tests):
         shards[i % n_workers].append(
-            (t.rel, t.category, t.src, t.fm.negative_phase, t.fm.negative_type)
+            (t.rel, t.category, assemble_source(t), t.fm.negative_phase, t.fm.negative_type)
         )
 
     t_run_start = time.monotonic()

lib/js/test262-scoreboard.json

@@ -1,137 +1,53 @@
 {
   "totals": {
-    "pass": 162,
-    "fail": 128,
-    "skip": 1597,
-    "timeout": 10,
-    "total": 1897,
-    "runnable": 300,
-    "pass_rate": 54.0
+    "pass": 4,
+    "fail": 10,
+    "skip": 16,
+    "timeout": 0,
+    "total": 30,
+    "runnable": 14,
+    "pass_rate": 28.6
   },
   "categories": [
     {
-      "category": "built-ins/Math",
-      "total": 327,
-      "pass": 43,
-      "fail": 56,
-      "skip": 227,
-      "timeout": 1,
-      "pass_rate": 43.0,
+      "category": "built-ins/Function",
+      "total": 30,
+      "pass": 4,
+      "fail": 10,
+      "skip": 16,
+      "timeout": 0,
+      "pass_rate": 28.6,
       "top_failures": [
         [
-          "TypeError: not a function",
-          36
-        ],
-        [
-          "Test262Error (assertion failed)",
-          20
-        ],
-        [
-          "Timeout",
-          1
-        ]
-      ]
-    },
-    {
-      "category": "built-ins/Number",
-      "total": 340,
-      "pass": 77,
-      "fail": 19,
-      "skip": 240,
-      "timeout": 4,
-      "pass_rate": 77.0,
-      "top_failures": [
-        [
-          "Test262Error (assertion failed)",
-          19
-        ],
-        [
-          "Timeout",
+          "SyntaxError (parse/unsupported syntax)",
           4
-        ]
-      ]
-    },
-    {
-      "category": "built-ins/String",
-      "total": 1223,
-      "pass": 42,
-      "fail": 53,
-      "skip": 1123,
-      "timeout": 5,
-      "pass_rate": 42.0,
-      "top_failures": [
-        [
-          "Test262Error (assertion failed)",
-          44
-        ],
-        [
-          "Timeout",
-          5
         ],
         [
           "ReferenceError (undefined symbol)",
-          2
+          3
         ],
         [
-          "Unhandled: Not callable: {:__proto__ {:toLowerCase <lambda(&rest, args)",
-          2
-        ],
-        [
-          "Unhandled: Not callable: \\\\\\",
-          2
+          "TypeError (other)",
+          3
         ]
       ]
-    },
-    {
-      "category": "built-ins/StringIteratorPrototype",
-      "total": 7,
-      "pass": 0,
-      "fail": 0,
-      "skip": 7,
-      "timeout": 0,
-      "pass_rate": 0.0,
-      "top_failures": []
     }
   ],
   "top_failure_modes": [
     [
-      "Test262Error (assertion failed)",
-      83
-    ],
-    [
-      "TypeError: not a function",
-      36
-    ],
-    [
-      "Timeout",
-      10
+      "SyntaxError (parse/unsupported syntax)",
+      4
     ],
     [
       "ReferenceError (undefined symbol)",
-      2
+      3
     ],
     [
-      "Unhandled: Not callable: {:__proto__ {:toLowerCase <lambda(&rest, args)",
-      2
-    ],
-    [
-      "Unhandled: Not callable: \\\\\\",
-      2
-    ],
-    [
-      "SyntaxError (parse/unsupported syntax)",
-      1
-    ],
-    [
-      "Unhandled: Not callable: {:__proto__ {:valueOf <lambda()> :propertyIsEn",
-      1
-    ],
-    [
-      "Unhandled: js-transpile-binop: unsupported op: >>>\\",
-      1
+      "TypeError (other)",
+      3
     ]
   ],
   "pinned_commit": "d5e73fc8d2c663554fb72e2380a8c2bc1a318a33",
-  "elapsed_seconds": 274.5,
+  "elapsed_seconds": 11.2,
   "workers": 1
 }

lib/js/test262-scoreboard.md

@@ -1,47 +1,26 @@
 # test262 scoreboard
 
 Pinned commit: `d5e73fc8d2c663554fb72e2380a8c2bc1a318a33`
-Wall time: 274.5s
+Wall time: 11.2s
 
-**Total:** 162/300 runnable passed (54.0%). Raw: pass=162 fail=128 skip=1597 timeout=10 total=1897.
+**Total:** 4/14 runnable passed (28.6%). Raw: pass=4 fail=10 skip=16 timeout=0 total=30.
 
 ## Top failure modes
 
-- **83x** Test262Error (assertion failed)
-- **36x** TypeError: not a function
-- **10x** Timeout
-- **2x** ReferenceError (undefined symbol)
-- **2x** Unhandled: Not callable: {:__proto__ {:toLowerCase <lambda(&rest, args)
-- **2x** Unhandled: Not callable: \\\
-- **1x** SyntaxError (parse/unsupported syntax)
-- **1x** Unhandled: Not callable: {:__proto__ {:valueOf <lambda()> :propertyIsEn
-- **1x** Unhandled: js-transpile-binop: unsupported op: >>>\
+- **4x** SyntaxError (parse/unsupported syntax)
+- **3x** ReferenceError (undefined symbol)
+- **3x** TypeError (other)
 
 ## Categories (worst pass-rate first, min 10 runnable)
 
 | Category | Pass | Fail | Skip | Timeout | Total | Pass % |
 |---|---:|---:|---:|---:|---:|---:|
-| built-ins/String | 42 | 53 | 1123 | 5 | 1223 | 42.0% |
-| built-ins/Math | 43 | 56 | 227 | 1 | 327 | 43.0% |
-| built-ins/Number | 77 | 19 | 240 | 4 | 340 | 77.0% |
+| built-ins/Function | 4 | 10 | 16 | 0 | 30 | 28.6% |
 
 ## Per-category top failures (min 10 runnable, worst first)
 
-### built-ins/String  (42/100 — 42.0%)
+### built-ins/Function  (4/14 — 28.6%)
 
-- **44x** Test262Error (assertion failed)
-- **5x** Timeout
-- **2x** ReferenceError (undefined symbol)
-- **2x** Unhandled: Not callable: {:__proto__ {:toLowerCase <lambda(&rest, args)
-- **2x** Unhandled: Not callable: \\\
-
-### built-ins/Math  (43/100 — 43.0%)
-
-- **36x** TypeError: not a function
-- **20x** Test262Error (assertion failed)
-- **1x** Timeout
-
-### built-ins/Number  (77/100 — 77.0%)
-
-- **19x** Test262Error (assertion failed)
-- **4x** Timeout
+- **4x** SyntaxError (parse/unsupported syntax)
+- **3x** ReferenceError (undefined symbol)
+- **3x** TypeError (other)

lib/js/transpile.sx

@@ -98,6 +98,7 @@
             (list (js-sym "js-regex-new") (nth ast 1) (nth ast 2)))
           ((js-tag? ast "js-null") nil)
           ((js-tag? ast "js-undef") (list (js-sym "quote") :js-undefined))
+          ((js-tag? ast "js-paren") (js-transpile (nth ast 1)))
           ((js-tag? ast "js-ident") (js-transpile-ident (nth ast 1)))
           ((js-tag? ast "js-unop")
             (js-transpile-unop (nth ast 1) (nth ast 2)))
@@ -116,7 +117,8 @@
           ((js-tag? ast "js-arrow")
             (js-transpile-arrow (nth ast 1) (nth ast 2)))
           ((js-tag? ast "js-program") (js-transpile-stmts (nth ast 1)))
-          ((js-tag? ast "js-block") (js-transpile-stmts (nth ast 1)))
+          ((js-tag? ast "js-block")
+            (cons (js-sym "begin") (js-transpile-stmt-list (nth ast 1))))
           ((js-tag? ast "js-exprstmt") (js-transpile (nth ast 1)))
           ((js-tag? ast "js-empty") nil)
           ((js-tag? ast "js-var")
@@ -164,6 +166,8 @@
             (js-transpile-new (nth ast 1) (nth ast 2)))
           ((js-tag? ast "js-class")
             (js-transpile-class (nth ast 1) (nth ast 2) (nth ast 3)))
+          ((js-tag? ast "js-comma")
+            (cons (js-sym "begin") (map js-transpile (nth ast 1))))
           ((js-tag? ast "js-throw") (js-transpile-throw (nth ast 1)))
           ((js-tag? ast "js-try")
             (js-transpile-try (nth ast 1) (nth ast 2) (nth ast 3)))
@@ -221,7 +225,23 @@
               (js-sym "js-delete-prop")
               (js-transpile (nth arg 1))
               (js-transpile (nth arg 2))))
+          ((js-tag? arg "js-ident") false)
+          ((js-tag? arg "js-paren") (js-transpile-unop op (nth arg 1)))
           (else true)))
+      ((and (= op "typeof") (js-tag? arg "js-ident"))
+        (let
+          ((name (nth arg 1)))
+          (list
+            (js-sym "if")
+            (list
+              (js-sym "or")
+              (list
+                (js-sym "env-has?")
+                (list (js-sym "current-env"))
+                name)
+              (list (js-sym "dict-has?") (js-sym "js-global") name))
+            (list (js-sym "js-typeof") (js-transpile arg))
+            "undefined")))
       (else
         (let
           ((a (js-transpile arg)))
@@ -231,7 +251,8 @@
             ((= op "!") (list (js-sym "js-not") a))
             ((= op "~") (list (js-sym "js-bitnot") a))
             ((= op "typeof") (list (js-sym "js-typeof") a))
-            ((= op "void") (list (js-sym "quote") :js-undefined))
+            ((= op "void")
+              (list (js-sym "begin") a (list (js-sym "quote") :js-undefined)))
             (else (error (str "js-transpile-unop: unsupported op: " op)))))))))
 
 ;; ── Array literal ─────────────────────────────────────────────────
@@ -295,6 +316,21 @@
               (list (js-sym "js-undefined?") (js-sym "_a")))
             (js-transpile r)
             (js-sym "_a"))))
+      ((= op ">>>")
+        (list
+          (js-sym "js-unsigned-rshift")
+          (js-transpile l)
+          (js-transpile r)))
+      ((= op "<<")
+        (list (js-sym "js-shl") (js-transpile l) (js-transpile r)))
+      ((= op ">>")
+        (list (js-sym "js-shr") (js-transpile l) (js-transpile r)))
+      ((= op "&")
+        (list (js-sym "js-bitand") (js-transpile l) (js-transpile r)))
+      ((= op "|")
+        (list (js-sym "js-bitor") (js-transpile l) (js-transpile r)))
+      ((= op "^")
+        (list (js-sym "js-bitxor") (js-transpile l) (js-transpile r)))
       (else (error (str "js-transpile-binop: unsupported op: " op))))))
 
 ;; ── Object literal ────────────────────────────────────────────────
@@ -373,7 +409,19 @@
     (list
       (js-sym "js-new-call")
       (js-transpile callee)
-      (cons (js-sym "list") (map js-transpile args)))))
+      (cond
+        ((js-has-spread? args)
+          (cons
+            (js-sym "js-array-spread-build")
+            (map
+              (fn
+                (e)
+                (if
+                  (js-tag? e "js-spread")
+                  (list (js-sym "list") "js-spread" (js-transpile (nth e 1)))
+                  (list (js-sym "list") "js-value" (js-transpile e))))
+              args)))
+        (else (cons (js-sym "js-args") (map js-transpile args)))))))
 
 (define
   js-transpile-array
@@ -391,7 +439,7 @@
               (list (js-sym "list") "js-spread" (js-transpile (nth e 1)))
               (list (js-sym "list") "js-value" (js-transpile e))))
           elts))
-      (cons (js-sym "list") (map js-transpile elts)))))
+      (cons (js-sym "js-make-list") (map js-transpile elts)))))
 
 (define
   js-has-spread?
@@ -421,7 +469,7 @@
               (list (js-sym "list") "js-spread" (js-transpile (nth e 1)))
               (list (js-sym "list") "js-value" (js-transpile e))))
           args))
-      (cons (js-sym "list") (map js-transpile args)))))
+      (cons (js-sym "js-args") (map js-transpile args)))))
 
 ;; Transpile a JS expression string to SX source text (for inspection
 ;; in tests). Useful for asserting the exact emitted tree.
@@ -431,18 +479,28 @@
     (entries)
     (list
       (js-sym "let")
-      (list (list (js-sym "_obj") (list (js-sym "dict"))))
+      (list (list (js-sym "_obj") (list (js-sym "js-make-obj"))))
       (cons
         (js-sym "begin")
         (append
           (map
             (fn
               (entry)
-              (list
-                (js-sym "dict-set!")
-                (js-sym "_obj")
-                (get entry :key)
-                (js-transpile (get entry :value))))
+              (cond
+                ((contains? (keys entry) :spread)
+                  (list
+                    (js-sym "js-obj-spread!")
+                    (js-sym "_obj")
+                    (js-transpile (get entry :spread))))
+                (else
+                  (list
+                    (js-sym "js-obj-set!")
+                    (js-sym "_obj")
+                    (if
+                      (contains? (keys entry) :computed-key)
+                      (list (js-sym "js-to-string") (js-transpile (get entry :computed-key)))
+                      (get entry :key))
+                    (js-transpile (get entry :value))))))
             entries)
           (list (js-sym "_obj")))))))
 
@@ -486,6 +544,95 @@
                 (append inits (list (js-transpile body))))))))
       (list (js-sym "fn") param-syms body-tr))))
 
+(define
+  js-collect-var-decl-names
+  (fn
+    (decls)
+    (cond
+      ((empty? decls) (list))
+      ((js-tag? (first decls) "js-vardecl")
+        (cons
+          (nth (first decls) 1)
+          (js-collect-var-decl-names (rest decls))))
+      (else (js-collect-var-decl-names (rest decls))))))
+
+(define
+  js-collect-var-names
+  (fn
+    (stmts)
+    (cond
+      ((empty? stmts) (list))
+      (else
+        (append
+          (js-collect-var-names-stmt (first stmts))
+          (js-collect-var-names (rest stmts)))))))
+
+(define
+  js-collect-var-names-stmt
+  (fn
+    (stmt)
+    (cond
+      ((not (list? stmt)) (list))
+      ((and (js-tag? stmt "js-var") (= (nth stmt 1) "var"))
+        (js-collect-var-decl-names (nth stmt 2)))
+      ((js-tag? stmt "js-block") (js-collect-var-names (nth stmt 1)))
+      ((js-tag? stmt "js-for")
+        (append
+          (js-collect-var-names-stmt (nth stmt 1))
+          (js-collect-var-names-stmt (nth stmt 4))))
+      ((js-tag? stmt "js-for-of-in")
+        (js-collect-var-names-stmt (nth stmt 4)))
+      ((js-tag? stmt "js-while")
+        (js-collect-var-names-stmt (nth stmt 2)))
+      ((js-tag? stmt "js-do-while")
+        (js-collect-var-names-stmt (nth stmt 1)))
+      ((js-tag? stmt "js-if")
+        (append
+          (js-collect-var-names-stmt (nth stmt 2))
+          (if (>= (len stmt) 4) (js-collect-var-names-stmt (nth stmt 3)) (list))))
+      ((js-tag? stmt "js-try")
+        (append
+          (js-collect-var-names-stmt (nth stmt 1))
+          (if (and (>= (len stmt) 3) (list? (nth stmt 2)))
+              (js-collect-var-names-stmt (nth (nth stmt 2) 2))
+              (list))
+          (if (>= (len stmt) 4) (js-collect-var-names-stmt (nth stmt 3)) (list))))
+      ((js-tag? stmt "js-switch")
+        (js-collect-var-names-cases (nth stmt 2)))
+      (else (list)))))
+
+(define
+  js-collect-var-names-cases
+  (fn
+    (cases)
+    (cond
+      ((empty? cases) (list))
+      (else
+        (append
+          (js-collect-var-names (nth (first cases) 2))
+          (js-collect-var-names-cases (rest cases)))))))
+
+(define
+  js-dedup-names
+  (fn
+    (names seen)
+    (cond
+      ((empty? names) (list))
+      ((some (fn (s) (= s (first names))) seen)
+        (js-dedup-names (rest names) seen))
+      (else
+        (cons
+          (first names)
+          (js-dedup-names (rest names) (cons (first names) seen)))))))
+
+(define
+  js-var-hoist-forms
+  (fn
+    (names)
+    (map
+      (fn (name) (list (js-sym "define") (js-sym name) :js-undefined))
+      names)))
+
 (define
   js-transpile-tpl
   (fn
@@ -577,6 +724,12 @@
             (list (js-sym "js-undefined?") lhs-expr))
           rhs-expr
           lhs-expr))
+      ((= op "<<=") (list (js-sym "js-shl") lhs-expr rhs-expr))
+      ((= op ">>=") (list (js-sym "js-shr") lhs-expr rhs-expr))
+      ((= op ">>>=") (list (js-sym "js-unsigned-rshift") lhs-expr rhs-expr))
+      ((= op "&=") (list (js-sym "js-bitand") lhs-expr rhs-expr))
+      ((= op "|=") (list (js-sym "js-bitor") lhs-expr rhs-expr))
+      ((= op "^=") (list (js-sym "js-bitxor") lhs-expr rhs-expr))
       (else (error (str "js-compound-update: unsupported op: " op))))))
 
 (define
@@ -806,7 +959,7 @@
                 (if
                   (= iter-kind "of")
                   (list (js-sym "js-iterable-to-list") iter-sx)
-                  (list (js-sym "js-object-keys") iter-sx))))
+                  (list (js-sym "js-for-in-keys") iter-sx))))
             (list
               (js-sym "for-each")
               (list
@@ -835,7 +988,7 @@
   (fn
     (params)
     (cond
-      ((empty? params) (list))
+      ((empty? params) (list (js-sym "&rest") (js-sym "__extra_args__")))
       ((and (list? (first params)) (js-tag? (first params) "js-rest"))
         (list (js-sym "&rest") (js-sym (nth (first params) 1))))
       (else
@@ -843,6 +996,27 @@
           (js-param-sym (first params))
           (js-build-param-list (rest params)))))))
 
+(define
+  js-arguments-build-form
+  (fn
+    (params)
+    (list (js-sym "js-list-copy") (js-arguments-build-form-raw params))))
+
+(define
+  js-arguments-build-form-raw
+  (fn
+    (params)
+    (cond
+      ((empty? params)
+        (js-sym "__extra_args__"))
+      ((and (list? (first params)) (js-tag? (first params) "js-rest"))
+        (js-sym (nth (first params) 1)))
+      (else
+        (list
+          (js-sym "cons")
+          (js-param-sym (first params))
+          (js-arguments-build-form-raw (rest params)))))))
+
 (define
   js-param-init-forms
   (fn
@@ -876,7 +1050,7 @@
   (fn
     (stmts)
     (let
-      ((hoisted (js-collect-funcdecls stmts)))
+      ((hoisted (append (js-var-hoist-forms (js-dedup-names (js-collect-var-names stmts) (list))) (js-collect-funcdecls stmts))))
       (let
         ((rest-stmts (js-transpile-stmt-list stmts)))
         (cons (js-sym "begin") (append hoisted rest-stmts))))))
@@ -935,12 +1109,12 @@
 
 (define
   js-transpile-var
-  (fn (kind decls) (cons (js-sym "begin") (js-vardecl-forms kind decls))))
+  (fn (kind decls) (cons (js-sym "begin") (js-vardecl-forms decls (= kind "var")))))
 
 (define
   js-vardecl-forms
   (fn
-    (kind decls)
+    (decls is-var)
     (cond
       ((empty? decls) (list))
       (else
@@ -950,10 +1124,10 @@
             ((js-tag? d "js-vardecl")
               (cons
                 (list
-                  (js-sym "define")
+                  (js-sym (if is-var "set!" "define"))
                   (js-sym (nth d 1))
                   (js-transpile (nth d 2)))
-                (js-vardecl-forms kind (rest decls))))
+                (js-vardecl-forms (rest decls) is-var)))
             ((js-tag? d "js-vardecl-obj")
               (let
                 ((names (nth d 1))
@@ -964,7 +1138,7 @@
                   (js-vardecl-obj-forms
                     names
                     tmp-sym
-                    (js-vardecl-forms kind (rest decls))))))
+                    (js-vardecl-forms (rest decls) is-var)))))
             ((js-tag? d "js-vardecl-arr")
               (let
                 ((names (nth d 1))
@@ -976,7 +1150,7 @@
                     names
                     tmp-sym
                     0
-                    (js-vardecl-forms kind (rest decls))))))
+                    (js-vardecl-forms (rest decls) is-var)))))
             (else (error "js-vardecl-forms: unexpected decl"))))))))
 
 (define
@@ -1276,7 +1450,28 @@
     (let
       ((body-tr (js-transpile body)))
       (let
-        ((with-catch (cond ((= catch-part nil) body-tr) (else (let ((pname (nth catch-part 0)) (cbody (nth catch-part 1))) (list (js-sym "guard") (list (if (= pname nil) (js-sym "__exc__") (js-sym pname)) (list (js-sym "else") (js-transpile cbody))) body-tr))))))
+        ((with-catch
+           (cond
+             ((= catch-part nil) body-tr)
+             (else
+               (let
+                 ((pname (nth catch-part 0))
+                   (cbody (nth catch-part 1))
+                   (raw-sym (js-sym "__raw_exc__")))
+                 (list
+                   (js-sym "guard")
+                   (list
+                     raw-sym
+                     (list
+                       (js-sym "else")
+                       (cond
+                         ((= pname nil) (js-transpile cbody))
+                         (else
+                           (list
+                             (js-sym "let")
+                             (list (list (js-sym pname) (list (js-sym "js-wrap-exn") raw-sym)))
+                             (js-transpile cbody))))))
+                   body-tr))))))
         (cond
           ((= finally-part nil) with-catch)
           (else
@@ -1297,7 +1492,7 @@
           (if
             (and (list? body) (js-tag? body "js-block"))
             (let
-              ((hoisted (js-collect-funcdecls (nth body 1))))
+              ((hoisted (append (js-var-hoist-forms (js-dedup-names (js-collect-var-names (nth body 1)) (list))) (js-collect-funcdecls (nth body 1)))))
               (append hoisted (js-transpile-stmt-list (nth body 1))))
             (list (js-transpile body)))))
       (list
@@ -1305,7 +1500,9 @@
         param-syms
         (list
           (js-sym "let")
-          (list (list (js-sym "this") (list (js-sym "js-this"))))
+          (list
+            (list (js-sym "this") (list (js-sym "js-this")))
+            (list (js-sym "arguments") (js-arguments-build-form params)))
           (list
             (js-sym "let")
             (list
@@ -1316,7 +1513,7 @@
                   (list
                     (js-sym "fn")
                     (list (js-sym "__return__"))
-                    (cons (js-sym "begin") (append inits body-forms))))))
+                    (cons (js-sym "begin") (append (append inits body-forms) (list nil)))))))
             (list
               (js-sym "if")
               (list (js-sym "=") (js-sym "__r__") nil)
@@ -1333,7 +1530,7 @@
           (if
             (and (list? body) (js-tag? body "js-block"))
             (let
-              ((hoisted (js-collect-funcdecls (nth body 1))))
+              ((hoisted (append (js-var-hoist-forms (js-dedup-names (js-collect-var-names (nth body 1)) (list))) (js-collect-funcdecls (nth body 1)))))
               (append hoisted (js-transpile-stmt-list (nth body 1))))
             (list (js-transpile body)))))
       (list
@@ -1396,13 +1593,43 @@
         param-syms
         (list (js-sym "js-async-wrap") (list (js-sym "fn") (list) body-tr))))))
 
+(define
+  js-internal-key?
+  (fn
+    (k)
+    (and
+      (string? k)
+      (>= (len k) 4)
+      (= (substring k 0 2) "__")
+      (= (substring k (- (len k) 2) (len k)) "__"))))
+
+(define
+  js-display
+  (fn
+    (val)
+    (cond
+      ((dict? val)
+        (let
+          ((cleaned (dict)))
+          (begin
+            (for-each
+              (fn
+                (k)
+                (when
+                  (not (js-internal-key? k))
+                  (dict-set! cleaned k (js-display (get val k)))))
+              (keys val))
+            cleaned)))
+      ((list? val) (map js-display val))
+      (else val))))
+
 (define
   js-eval
   (fn
     (src)
     (let
-      ((result (eval-expr (js-transpile (js-parse (js-tokenize src))))))
+      ((result (eval-expr (list (quote let) (list (list (js-sym "this") (list (js-sym "js-this")))) (js-transpile (js-parse (js-tokenize src)))))))
       (js-drain-microtasks!)
-      result)))
+      (js-display result))))
 
 (define js-compile-expr (fn (src) (js-transpile (js-parse-expr src))))

lib/kernel/eval.sx

@@ -0,0 +1,214 @@
+;; lib/kernel/eval.sx — Kernel evaluator.
+;;
+;; The evaluator is `lookup-and-combine`: there are no hardcoded special
+;; forms. Even $if / $define! / $lambda are ordinary operatives bound in
+;; the standard environment (Phase 4). This file builds the dispatch
+;; machinery and the operative/applicative tagged-value protocol.
+;;
+;; Tagged values
+;; -------------
+;;   {:refl-tag :env :bindings DICT :parent PARENT-OR-NIL}
+;;     A first-class Kernel environment. Bindings is a mutable SX dict
+;;     keyed by symbol name; parent walks up the lookup chain. Shape
+;;     and operations are inherited from lib/guest/reflective/env.sx
+;;     (canonical wire shape) — Kernel-side names are thin wrappers.
+;;
+;;   {:knl-tag :operative :impl FN}
+;;     Primitive operative. FN receives (args dyn-env) — args are the
+;;     UN-evaluated argument expressions, dyn-env is the calling env.
+;;
+;;   {:knl-tag :operative :params P :env-param EP :body B :static-env SE}
+;;     User-defined operative (built by $vau). Same tag; dispatch in
+;;     kernel-call-operative forks on which keys are present.
+;;
+;;   {:knl-tag :applicative :underlying OP}
+;;     An applicative wraps an operative. Calls evaluate args first,
+;;     then forward to the underlying operative.
+;;
+;; The env-param of a user operative may be the sentinel :knl-ignore,
+;; in which case the dynamic env is not bound.
+;;
+;; Public API
+;;   (kernel-eval EXPR ENV)                     — primary entry
+;;   (kernel-combine COMBINER ARGS DYN-ENV)
+;;   (kernel-call-operative OP ARGS DYN-ENV)
+;;   (kernel-bind-params! ENV PARAMS ARGS)
+;;   (kernel-make-env) / (kernel-extend-env P)
+;;   (kernel-env-bind! E N V) / (kernel-env-lookup E N)
+;;   (kernel-env-has? E N) / (kernel-env? V)
+;;   (kernel-make-primitive-operative IMPL)
+;;   (kernel-make-primitive-applicative IMPL)
+;;   (kernel-make-user-operative PARAMS EPARAM BODY STATIC-ENV)
+;;   (kernel-wrap OP) / (kernel-unwrap APP)
+;;   (kernel-operative? V) / (kernel-applicative? V) / (kernel-combiner? V)
+;;
+;; Consumes: lib/kernel/parser.sx (kernel-string?, kernel-string-value)
+
+;; ── Environments — delegated to lib/guest/reflective/env.sx ──────
+;; The env values themselves now carry `:refl-tag :env` (shared with the
+;; reflective kit). Kernel's API names stay; bodies are thin wrappers.
+
+(define kernel-env?        refl-env?)
+(define kernel-make-env    refl-make-env)
+(define kernel-extend-env  refl-extend-env)
+(define kernel-env-bind!   refl-env-bind!)
+(define kernel-env-has?    refl-env-has?)
+(define kernel-env-lookup  refl-env-lookup)
+
+;; ── Tagged-value constructors and predicates ─────────────────────
+
+(define kernel-make-primitive-operative (fn (impl) {:impl impl :knl-tag :operative}))
+
+(define
+  kernel-make-user-operative
+  (fn (params eparam body static-env) {:knl-tag :operative :static-env static-env :params params :body body :env-param eparam}))
+
+(define
+  kernel-operative?
+  (fn (v) (and (dict? v) (= (get v :knl-tag) :operative))))
+
+(define
+  kernel-applicative?
+  (fn (v) (and (dict? v) (= (get v :knl-tag) :applicative))))
+
+(define
+  kernel-combiner?
+  (fn (v) (or (kernel-operative? v) (kernel-applicative? v))))
+
+(define
+  kernel-wrap
+  (fn
+    (op)
+    (cond
+      ((kernel-operative? op) {:knl-tag :applicative :underlying op})
+      (:else (error "kernel-wrap: argument must be an operative")))))
+
+(define
+  kernel-unwrap
+  (fn
+    (app)
+    (cond
+      ((kernel-applicative? app) (get app :underlying))
+      (:else (error "kernel-unwrap: argument must be an applicative")))))
+
+(define
+  kernel-make-primitive-applicative
+  (fn
+    (impl)
+    (kernel-wrap
+      (kernel-make-primitive-operative (fn (args dyn-env) (impl args))))))
+
+;; As above, but IMPL receives (args dyn-env). Used by combinators that
+;; re-enter the evaluator (map, filter, reduce, apply, eval, ...).
+(define kernel-make-primitive-applicative-with-env
+  (fn (impl)
+    (kernel-wrap
+      (kernel-make-primitive-operative
+        (fn (args dyn-env) (impl args dyn-env))))))
+
+;; ── The evaluator ────────────────────────────────────────────────
+
+(define
+  kernel-eval
+  (fn
+    (expr env)
+    (cond
+      ((number? expr) expr)
+      ((boolean? expr) expr)
+      ((nil? expr) expr)
+      ((kernel-string? expr) (kernel-string-value expr))
+      ((string? expr) (kernel-env-lookup env expr))
+      ((list? expr)
+        (cond
+          ((= (length expr) 0) expr)
+          (:else
+            (let
+              ((combiner (kernel-eval (first expr) env))
+                (args (rest expr)))
+              (kernel-combine combiner args env)))))
+      (:else (error (str "kernel-eval: unknown form: " expr))))))
+
+(define
+  kernel-combine
+  (fn
+    (combiner args dyn-env)
+    (cond
+      ((kernel-operative? combiner)
+        (kernel-call-operative combiner args dyn-env))
+      ((kernel-applicative? combiner)
+        (kernel-combine
+          (get combiner :underlying)
+          (kernel-eval-args args dyn-env)
+          dyn-env))
+      (:else (error (str "kernel-eval: not a combiner: " combiner))))))
+
+;; Operatives may be primitive (:impl is a host fn) or user-defined
+;; (carry :params / :env-param / :body / :static-env). The dispatch
+;; fork is here so kernel-combine stays small.
+(define
+  kernel-call-operative
+  (fn
+    (op args dyn-env)
+    (cond
+      ((dict-has? op :impl) ((get op :impl) args dyn-env))
+      ((dict-has? op :body)
+        (let
+          ((local (kernel-extend-env (get op :static-env))))
+          (kernel-bind-params! local (get op :params) args)
+          (let
+            ((eparam (get op :env-param)))
+            (when
+              (not (= eparam :knl-ignore))
+              (kernel-env-bind! local eparam dyn-env)))
+          ;; :body is a list of forms — evaluate in sequence, return last.
+          (knl-eval-body (get op :body) local)))
+      (:else (error "kernel-call-operative: malformed operative")))))
+
+(define knl-eval-body
+  (fn (forms env)
+    (cond
+      ((= (length forms) 1) (kernel-eval (first forms) env))
+      (:else
+        (begin
+          (kernel-eval (first forms) env)
+          (knl-eval-body (rest forms) env))))))
+
+;; Phase 3 supports a flat parameter list only — destructuring later.
+(define
+  kernel-bind-params!
+  (fn
+    (env params args)
+    (cond
+      ((or (nil? params) (= (length params) 0))
+        (cond
+          ((or (nil? args) (= (length args) 0)) nil)
+          (:else (error "kernel-call: too many arguments"))))
+      ((or (nil? args) (= (length args) 0))
+        (error "kernel-call: too few arguments"))
+      (:else
+        (begin
+          (kernel-env-bind! env (first params) (first args))
+          (kernel-bind-params! env (rest params) (rest args)))))))
+
+(define
+  kernel-eval-args
+  (fn
+    (args env)
+    (cond
+      ((or (nil? args) (= (length args) 0)) (list))
+      (:else
+        (cons
+          (kernel-eval (first args) env)
+          (kernel-eval-args (rest args) env))))))
+
+(define
+  kernel-eval-program
+  (fn
+    (forms env)
+    (cond
+      ((or (nil? forms) (= (length forms) 0)) nil)
+      ((= (length forms) 1) (kernel-eval (first forms) env))
+      (:else
+        (begin
+          (kernel-eval (first forms) env)
+          (kernel-eval-program (rest forms) env))))))

lib/kernel/parser.sx

@@ -0,0 +1,253 @@
+;; lib/kernel/parser.sx — Kernel s-expression reader.
+;;
+;; Reads R-1RK lexical syntax: numbers, strings, symbols, booleans (#t/#f),
+;; the empty list (), nested lists, and ; line comments. Reader macros
+;; (' ` , ,@) deferred to Phase 6 per the plan.
+;;
+;; Public AST shape:
+;;   number      → SX number
+;;   #t / #f     → SX true / false
+;;   ()          → SX empty list (Kernel's nil — the empty list)
+;;   "..."       → {:knl-string "..."}    wrapped to distinguish from symbols
+;;   foo         → "foo"                  bare SX string is a Kernel symbol
+;;   (a b c)     → SX list of forms
+;;
+;; Public API:
+;;   (kernel-parse SRC)      — first form; errors on extra trailing input
+;;   (kernel-parse-all SRC)  — all top-level forms, as SX list
+;;   (kernel-string? V)      — recognise wrapped string literal
+;;   (kernel-string-value V) — extract the underlying string
+;;
+;; Consumes: lib/guest/lex.sx (lex-digit?, lex-whitespace?)
+
+(define kernel-string-make (fn (s) {:knl-string s}))
+(define
+  kernel-string?
+  (fn (v) (and (dict? v) (string? (get v :knl-string)))))
+(define kernel-string-value (fn (v) (get v :knl-string)))
+
+;; Atom delimiters: characters that end a symbol or numeric token.
+(define
+  knl-delim?
+  (fn
+    (c)
+    (or
+      (nil? c)
+      (lex-whitespace? c)
+      (= c "(")
+      (= c ")")
+      (= c "\"")
+      (= c ";")
+      (= c "'")
+      (= c "`")
+      (= c ","))))
+
+;; Numeric grammar:  [+-]? (digit+ ('.' digit+)? | '.' digit+) ([eE][+-]?digit+)?
+(define
+  knl-numeric?
+  (fn
+    (s)
+    (let
+      ((n (string-length s)))
+      (cond
+        ((= n 0) false)
+        (:else
+          (let
+            ((c0 (substring s 0 1)))
+            (let
+              ((start (if (or (= c0 "+") (= c0 "-")) 1 0)))
+              (knl-num-body? s start n))))))))
+
+(define
+  knl-num-body?
+  (fn
+    (s start n)
+    (cond
+      ((>= start n) false)
+      ((= (substring s start (+ start 1)) ".")
+        (knl-num-need-digits? s (+ start 1) n false))
+      ((lex-digit? (substring s start (+ start 1)))
+        (knl-num-int-tail? s (+ start 1) n))
+      (:else false))))
+
+(define
+  knl-num-int-tail?
+  (fn
+    (s i n)
+    (cond
+      ((>= i n) true)
+      ((lex-digit? (substring s i (+ i 1)))
+        (knl-num-int-tail? s (+ i 1) n))
+      ((= (substring s i (+ i 1)) ".")
+        (knl-num-need-digits? s (+ i 1) n true))
+      ((or (= (substring s i (+ i 1)) "e") (= (substring s i (+ i 1)) "E"))
+        (knl-num-exp-sign? s (+ i 1) n))
+      (:else false))))
+
+(define
+  knl-num-need-digits?
+  (fn
+    (s i n had-int)
+    (cond
+      ((>= i n) had-int)
+      ((lex-digit? (substring s i (+ i 1)))
+        (knl-num-frac-tail? s (+ i 1) n))
+      (:else false))))
+
+(define
+  knl-num-frac-tail?
+  (fn
+    (s i n)
+    (cond
+      ((>= i n) true)
+      ((lex-digit? (substring s i (+ i 1)))
+        (knl-num-frac-tail? s (+ i 1) n))
+      ((or (= (substring s i (+ i 1)) "e") (= (substring s i (+ i 1)) "E"))
+        (knl-num-exp-sign? s (+ i 1) n))
+      (:else false))))
+
+(define
+  knl-num-exp-sign?
+  (fn
+    (s i n)
+    (cond
+      ((>= i n) false)
+      ((or (= (substring s i (+ i 1)) "+") (= (substring s i (+ i 1)) "-"))
+        (knl-num-exp-digits? s (+ i 1) n false))
+      (:else (knl-num-exp-digits? s i n false)))))
+
+(define
+  knl-num-exp-digits?
+  (fn
+    (s i n had)
+    (cond
+      ((>= i n) had)
+      ((lex-digit? (substring s i (+ i 1)))
+        (knl-num-exp-digits? s (+ i 1) n true))
+      (:else false))))
+
+;; Reader: a closure over (src, pos). Exposes :read-form and :read-all.
+(define
+  knl-make-reader
+  (fn
+    (src)
+    (let
+      ((pos 0) (n (string-length src)))
+      (define
+        at
+        (fn () (if (< pos n) (substring src pos (+ pos 1)) nil)))
+      (define adv (fn () (set! pos (+ pos 1))))
+      (define
+        skip-line
+        (fn () (when (and (at) (not (= (at) "\n"))) (adv) (skip-line))))
+      (define
+        skip-ws
+        (fn
+          ()
+          (cond
+            ((nil? (at)) nil)
+            ((lex-whitespace? (at)) (do (adv) (skip-ws)))
+            ((= (at) ";") (do (adv) (skip-line) (skip-ws)))
+            (:else nil))))
+      (define
+        read-string-body
+        (fn
+          (acc)
+          (cond
+            ((nil? (at)) (error "kernel-parse: unterminated string"))
+            ((= (at) "\"") (do (adv) acc))
+            ((= (at) "\\")
+              (do
+                (adv)
+                (let
+                  ((c (at)))
+                  (when (nil? c) (error "kernel-parse: trailing backslash"))
+                  (adv)
+                  (read-string-body
+                    (str
+                      acc
+                      (cond
+                        ((= c "n") "\n")
+                        ((= c "t") "\t")
+                        ((= c "r") "\r")
+                        ((= c "\"") "\"")
+                        ((= c "\\") "\\")
+                        (:else c)))))))
+            (:else
+              (let ((c (at))) (adv) (read-string-body (str acc c)))))))
+      (define
+        read-atom-body
+        (fn
+          (acc)
+          (cond
+            ((knl-delim? (at)) acc)
+            (:else (let ((c (at))) (adv) (read-atom-body (str acc c)))))))
+      (define
+        classify-atom
+        (fn
+          (s)
+          (cond
+            ((= s "#t") true)
+            ((= s "#f") false)
+            ((knl-numeric? s) (string->number s))
+            (:else s))))
+      (define
+        read-form
+        (fn
+          ()
+          (skip-ws)
+          (cond
+            ((nil? (at)) :knl-eof)
+            ((= (at) ")") (error "kernel-parse: unexpected ')'"))
+            ((= (at) "(") (do (adv) (read-list (list))))
+            ((= (at) "\"")
+              (do (adv) (kernel-string-make (read-string-body ""))))
+            ((= (at) "'")
+              (do (adv) (list "$quote" (read-form))))
+            ((= (at) "`")
+              (do (adv) (list "$quasiquote" (read-form))))
+            ((= (at) ",")
+              (do (adv)
+                  (cond
+                    ((= (at) "@")
+                      (do (adv) (list "$unquote-splicing" (read-form))))
+                    (:else (list "$unquote" (read-form))))))
+            (:else (classify-atom (read-atom-body ""))))))
+      (define
+        read-list
+        (fn
+          (acc)
+          (skip-ws)
+          (cond
+            ((nil? (at)) (error "kernel-parse: unterminated list"))
+            ((= (at) ")") (do (adv) acc))
+            (:else (read-list (append acc (list (read-form))))))))
+      (define
+        read-all
+        (fn
+          (acc)
+          (skip-ws)
+          (if (nil? (at)) acc (read-all (append acc (list (read-form)))))))
+      {:read-form read-form :read-all read-all})))
+
+(define
+  kernel-parse-all
+  (fn (src) ((get (knl-make-reader src) :read-all) (list))))
+
+(define
+  kernel-parse
+  (fn
+    (src)
+    (let
+      ((r (knl-make-reader src)))
+      (let
+        ((form ((get r :read-form))))
+        (cond
+          ((= form :knl-eof) (error "kernel-parse: empty input"))
+          (:else
+            (let
+              ((next ((get r :read-form))))
+              (if
+                (= next :knl-eof)
+                form
+                (error "kernel-parse: trailing input after first form")))))))))

lib/kernel/runtime.sx

@@ -0,0 +1,881 @@
+;; lib/kernel/runtime.sx — the operative–applicative substrate and the
+;; standard Kernel environment.
+;;
+;; Phase 3 supplied four user-visible combiners ($vau, $lambda, wrap,
+;; unwrap). Phase 4 fills out the rest of the R-1RK core: $if, $define!,
+;; $sequence, eval, make-environment, get-current-environment, plus
+;; arithmetic, equality, list/pair, and boolean primitives — enough to
+;; write factorial.
+;;
+;; The standard env is built by EXTENDING the base env, not replacing
+;; it. So `kernel-standard-env` includes everything from `kernel-base-env`.
+;;
+;; Public API
+;;   (kernel-base-env)       — Phase 3 combiners
+;;   (kernel-standard-env)   — Phase 4 standard environment
+
+(define
+  knl-eparam-sentinel
+  (fn
+    (sym)
+    (cond
+      ((= sym "_") :knl-ignore)
+      ((= sym "#ignore") :knl-ignore)
+      (:else sym))))
+
+(define
+  knl-formals-ok?
+  (fn
+    (formals)
+    (cond
+      ((not (list? formals)) false)
+      ((= (length formals) 0) true)
+      ((string? (first formals)) (knl-formals-ok? (rest formals)))
+      (:else false))))
+
+;; ── $vau ─────────────────────────────────────────────────────────
+
+(define
+  kernel-vau-impl
+  (fn
+    (args dyn-env)
+    (cond
+      ((< (length args) 3)
+        (error "$vau: expects (formals env-param body...)"))
+      (:else
+        (let
+          ((formals (first args))
+            (eparam-raw (nth args 1))
+            (body-forms (rest (rest args))))
+          (cond
+            ((not (knl-formals-ok? formals))
+              (error "$vau: formals must be a list of symbols"))
+            ((not (string? eparam-raw))
+              (error "$vau: env-param must be a symbol"))
+            (:else
+              (kernel-make-user-operative
+                formals
+                (knl-eparam-sentinel eparam-raw)
+                body-forms
+                dyn-env))))))))
+
+(define
+  kernel-vau-operative
+  (kernel-make-primitive-operative kernel-vau-impl))
+
+;; ── $lambda ──────────────────────────────────────────────────────
+
+(define
+  kernel-lambda-impl
+  (fn
+    (args dyn-env)
+    (cond
+      ((< (length args) 2)
+        (error "$lambda: expects (formals body...)"))
+      (:else
+        (let
+          ((formals (first args)) (body-forms (rest args)))
+          (cond
+            ((not (knl-formals-ok? formals))
+              (error "$lambda: formals must be a list of symbols"))
+            (:else
+              (kernel-wrap
+                (kernel-make-user-operative
+                  formals
+                  :knl-ignore
+                  body-forms
+                  dyn-env)))))))))
+
+(define
+  kernel-lambda-operative
+  (kernel-make-primitive-operative kernel-lambda-impl))
+
+;; ── wrap / unwrap / predicates ───────────────────────────────────
+
+(define
+  kernel-wrap-applicative
+  (kernel-make-primitive-applicative
+    (fn
+      (args)
+      (cond
+        ((not (= (length args) 1))
+          (error "wrap: expects exactly 1 argument"))
+        (:else (kernel-wrap (first args)))))))
+
+(define
+  kernel-unwrap-applicative
+  (kernel-make-primitive-applicative
+    (fn
+      (args)
+      (cond
+        ((not (= (length args) 1))
+          (error "unwrap: expects exactly 1 argument"))
+        (:else (kernel-unwrap (first args)))))))
+
+(define
+  kernel-operative?-applicative
+  (kernel-make-primitive-applicative
+    (fn (args) (kernel-operative? (first args)))))
+
+(define
+  kernel-applicative?-applicative
+  (kernel-make-primitive-applicative
+    (fn (args) (kernel-applicative? (first args)))))
+
+(define
+  kernel-base-env
+  (fn
+    ()
+    (let
+      ((env (kernel-make-env)))
+      (kernel-env-bind! env "$vau" kernel-vau-operative)
+      (kernel-env-bind! env "$lambda" kernel-lambda-operative)
+      (kernel-env-bind! env "wrap" kernel-wrap-applicative)
+      (kernel-env-bind! env "unwrap" kernel-unwrap-applicative)
+      (kernel-env-bind! env "operative?" kernel-operative?-applicative)
+      (kernel-env-bind! env "applicative?" kernel-applicative?-applicative)
+      env)))
+
+;; ── $if / $define! / $sequence ───────────────────────────────────
+
+(define
+  kernel-if-operative
+  (kernel-make-primitive-operative
+    (fn
+      (args dyn-env)
+      (cond
+        ((not (= (length args) 3))
+          (error "$if: expects (condition then-expr else-expr)"))
+        (:else
+          (let
+            ((c (kernel-eval (first args) dyn-env)))
+            (if
+              c
+              (kernel-eval (nth args 1) dyn-env)
+              (kernel-eval (nth args 2) dyn-env))))))))
+
+(define
+  kernel-define!-operative
+  (kernel-make-primitive-operative
+    (fn
+      (args dyn-env)
+      (cond
+        ((not (= (length args) 2))
+          (error "$define!: expects (name expr)"))
+        ((not (string? (first args)))
+          (error "$define!: name must be a symbol"))
+        (:else
+          (let
+            ((v (kernel-eval (nth args 1) dyn-env)))
+            (kernel-env-bind! dyn-env (first args) v)
+            v))))))
+
+(define
+  kernel-sequence-operative
+  (kernel-make-primitive-operative
+    (fn
+      (args dyn-env)
+      (cond
+        ((or (nil? args) (= (length args) 0)) nil)
+        ((= (length args) 1) (kernel-eval (first args) dyn-env))
+        (:else
+          (begin
+            (kernel-eval (first args) dyn-env)
+            ((get kernel-sequence-operative :impl) (rest args) dyn-env)))))))
+
+;; ── eval / make-environment / get-current-environment ───────────
+
+(define
+  kernel-quote-operative
+  (kernel-make-primitive-operative
+    (fn
+      (args dyn-env)
+      (cond
+        ((not (= (length args) 1)) (error "$quote: expects 1 argument"))
+        (:else (first args))))))
+
+;; Kernel-side adapter for lib/guest/reflective/quoting.sx.
+;; Kernel uses $unquote / $unquote-splicing (dollar-prefixed) and the
+;; host-level kernel-eval as the evaluator. The walker algorithm
+;; itself is shared with Scheme via the kit.
+(define knl-quasi-cfg
+  {:unquote-name "$unquote"
+   :unquote-splicing-name "$unquote-splicing"
+   :eval (fn (form env) (kernel-eval form env))})
+
+(define knl-quasi-walk
+  (fn (form dyn-env)
+    (refl-quasi-walk-with knl-quasi-cfg form dyn-env)))
+
+;; $cond — multi-clause branch.
+;;   ($cond (TEST1 EXPR1 ...) (TEST2 EXPR2 ...) ...)
+;; Evaluates each TEST in order; first truthy one runs its EXPRs (in
+;; sequence) and returns the last; if no TEST is truthy, returns nil.
+;; A clause with TEST = `else` always matches (sugar for $if's default).
+(define knl-cond-impl
+  (fn (clauses dyn-env)
+    (cond
+      ((or (nil? clauses) (= (length clauses) 0)) nil)
+      (:else
+        (let ((clause (first clauses)))
+          (cond
+            ((not (list? clause))
+              (error "$cond: each clause must be a list"))
+            ((= (length clause) 0)
+              (error "$cond: empty clause"))
+            ((and (string? (first clause)) (= (first clause) "else"))
+              (knl-cond-eval-body (rest clause) dyn-env))
+            (:else
+              (let ((test-val (kernel-eval (first clause) dyn-env)))
+                (cond
+                  (test-val (knl-cond-eval-body (rest clause) dyn-env))
+                  (:else (knl-cond-impl (rest clauses) dyn-env)))))))))))
+
+(define knl-cond-eval-body
+  (fn (body dyn-env)
+    (cond
+      ((or (nil? body) (= (length body) 0)) nil)
+      ((= (length body) 1) (kernel-eval (first body) dyn-env))
+      (:else
+        (begin
+          (kernel-eval (first body) dyn-env)
+          (knl-cond-eval-body (rest body) dyn-env))))))
+
+(define kernel-cond-operative
+  (kernel-make-primitive-operative
+    (fn (args dyn-env) (knl-cond-impl args dyn-env))))
+
+;; $when COND BODY... — evaluate body iff COND is truthy; else nil.
+(define kernel-when-operative
+  (kernel-make-primitive-operative
+    (fn (args dyn-env)
+      (cond
+        ((< (length args) 1)
+          (error "$when: expects (cond body...)"))
+        (:else
+          (let ((c (kernel-eval (first args) dyn-env)))
+            (cond
+              (c (knl-cond-eval-body (rest args) dyn-env))
+              (:else nil))))))))
+
+;; $and? — short-circuit AND. Operative (not applicative) so untaken
+;; clauses are NOT evaluated. Empty $and? returns true (the identity).
+(define knl-and?-impl
+  (fn (args dyn-env)
+    (cond
+      ((or (nil? args) (= (length args) 0)) true)
+      ((= (length args) 1) (kernel-eval (first args) dyn-env))
+      (:else
+        (let ((v (kernel-eval (first args) dyn-env)))
+          (cond
+            (v (knl-and?-impl (rest args) dyn-env))
+            (:else v)))))))
+
+(define kernel-and?-operative
+  (kernel-make-primitive-operative knl-and?-impl))
+
+;; $or? — short-circuit OR. Operative; untaken clauses NOT evaluated.
+;; Empty $or? returns false (the identity).
+(define knl-or?-impl
+  (fn (args dyn-env)
+    (cond
+      ((or (nil? args) (= (length args) 0)) false)
+      ((= (length args) 1) (kernel-eval (first args) dyn-env))
+      (:else
+        (let ((v (kernel-eval (first args) dyn-env)))
+          (cond
+            (v v)
+            (:else (knl-or?-impl (rest args) dyn-env))))))))
+
+(define kernel-or?-operative
+  (kernel-make-primitive-operative knl-or?-impl))
+
+;; $unless COND BODY... — evaluate body iff COND is falsy; else nil.
+(define kernel-unless-operative
+  (kernel-make-primitive-operative
+    (fn (args dyn-env)
+      (cond
+        ((< (length args) 1)
+          (error "$unless: expects (cond body...)"))
+        (:else
+          (let ((c (kernel-eval (first args) dyn-env)))
+            (cond
+              (c nil)
+              (:else (knl-cond-eval-body (rest args) dyn-env)))))))))
+
+(define kernel-quasiquote-operative
+  (kernel-make-primitive-operative
+    (fn (args dyn-env)
+      (cond
+        ((not (= (length args) 1))
+          (error "$quasiquote: expects exactly 1 argument"))
+        (:else (knl-quasi-walk (first args) dyn-env))))))
+
+(define
+  kernel-eval-applicative
+  (kernel-make-primitive-applicative
+    (fn
+      (args)
+      (cond
+        ((not (= (length args) 2))
+          (error "eval: expects (expr env)"))
+        ((not (kernel-env? (nth args 1)))
+          (error "eval: second arg must be a kernel env"))
+        (:else (kernel-eval (first args) (nth args 1)))))))
+
+(define
+  kernel-make-environment-applicative
+  (kernel-make-primitive-applicative
+    (fn
+      (args)
+      (cond
+        ((= (length args) 0) (kernel-make-env))
+        ((= (length args) 1)
+          (cond
+            ((not (kernel-env? (first args)))
+              (error "make-environment: parent must be a kernel env"))
+            (:else (kernel-extend-env (first args)))))
+        (:else (error "make-environment: 0 or 1 argument"))))))
+
+;; ── arithmetic and comparison (binary; trivial to extend later) ─
+
+(define
+  kernel-get-current-env-operative
+  (kernel-make-primitive-operative
+    (fn
+      (args dyn-env)
+      (cond
+        ((not (= (length args) 0))
+          (error "get-current-environment: expects 0 arguments"))
+        (:else dyn-env)))))
+
+(define
+  knl-bin-app
+  (fn
+    (name f)
+    (kernel-make-primitive-applicative
+      (fn
+        (args)
+        (cond
+          ((not (= (length args) 2))
+            (error (str name ": expects 2 arguments")))
+          (:else (f (first args) (nth args 1))))))))
+
+;; Variadic left-fold helper. ZERO-RES is the identity (`(+)` → 0);
+;; ONE-FN handles single-arg case (`(- x)` negates; `(+ x)` returns x).
+(define knl-fold-step
+  (fn (f acc rest-args)
+    (cond
+      ((or (nil? rest-args) (= (length rest-args) 0)) acc)
+      (:else
+        (knl-fold-step f (f acc (first rest-args)) (rest rest-args))))))
+
+(define knl-fold-app
+  (fn (name f zero-res one-fn)
+    (kernel-make-primitive-applicative
+      (fn (args)
+        (cond
+          ((= (length args) 0) zero-res)
+          ((= (length args) 1) (one-fn (first args)))
+          (:else (knl-fold-step f (first args) (rest args))))))))
+
+;; Variadic n-ary chained comparison: `(< 1 2 3)` ≡ `(< 1 2)` AND `(< 2 3)`.
+(define knl-chain-step
+  (fn (cmp prev rest-args)
+    (cond
+      ((or (nil? rest-args) (= (length rest-args) 0)) true)
+      (:else
+        (let ((next (first rest-args)))
+          (cond
+            ((cmp prev next)
+              (knl-chain-step cmp next (rest rest-args)))
+            (:else false)))))))
+
+(define knl-chain-cmp
+  (fn (name cmp)
+    (kernel-make-primitive-applicative
+      (fn (args)
+        (cond
+          ((< (length args) 2)
+            (error (str name ": expects at least 2 arguments")))
+          (:else (knl-chain-step cmp (first args) (rest args))))))))
+
+;; ── list / pair primitives ──────────────────────────────────────
+
+(define
+  knl-unary-app
+  (fn
+    (name f)
+    (kernel-make-primitive-applicative
+      (fn
+        (args)
+        (cond
+          ((not (= (length args) 1))
+            (error (str name ": expects 1 argument")))
+          (:else (f (first args))))))))
+
+(define kernel-cons-applicative (knl-bin-app "cons" (fn (a b) (cons a b))))
+
+(define
+  kernel-car-applicative
+  (knl-unary-app
+    "car"
+    (fn
+      (xs)
+      (cond
+        ((or (nil? xs) (and (list? xs) (= (length xs) 0)))
+          (error "car: empty list"))
+        (:else (first xs))))))
+
+(define
+  kernel-cdr-applicative
+  (knl-unary-app
+    "cdr"
+    (fn
+      (xs)
+      (cond
+        ((or (nil? xs) (and (list? xs) (= (length xs) 0)))
+          (error "cdr: empty list"))
+        (:else (rest xs))))))
+
+(define
+  kernel-list-applicative
+  (kernel-make-primitive-applicative (fn (args) args)))
+
+(define
+  kernel-length-applicative
+  (knl-unary-app "length" (fn (xs) (length xs))))
+
+(define
+  kernel-null?-applicative
+  (knl-unary-app
+    "null?"
+    (fn (v) (or (nil? v) (and (list? v) (= (length v) 0))))))
+
+;; ── boolean / equality ──────────────────────────────────────────
+
+(define
+  kernel-pair?-applicative
+  (knl-unary-app
+    "pair?"
+    (fn (v) (and (list? v) (> (length v) 0)))))
+
+(define knl-append-step
+  (fn (xs ys)
+    (cond
+      ((or (nil? xs) (= (length xs) 0)) ys)
+      (:else (cons (first xs) (knl-append-step (rest xs) ys))))))
+
+(define knl-all-lists?
+  (fn (xs)
+    (cond
+      ((or (nil? xs) (= (length xs) 0)) true)
+      ((list? (first xs)) (knl-all-lists? (rest xs)))
+      (:else false))))
+
+(define knl-append-all
+  (fn (lists)
+    (cond
+      ((or (nil? lists) (= (length lists) 0)) (list))
+      ((= (length lists) 1) (first lists))
+      (:else
+        (knl-append-step (first lists)
+                         (knl-append-all (rest lists)))))))
+
+(define kernel-append-applicative
+  (kernel-make-primitive-applicative
+    (fn (args)
+      (cond
+        ((knl-all-lists? args) (knl-append-all args))
+        (:else (error "append: all arguments must be lists"))))))
+
+(define knl-reverse-step
+  (fn (xs acc)
+    (cond
+      ((or (nil? xs) (= (length xs) 0)) acc)
+      (:else (knl-reverse-step (rest xs) (cons (first xs) acc))))))
+
+(define kernel-reverse-applicative
+  (knl-unary-app "reverse"
+    (fn (xs)
+      (cond
+        ((not (list? xs)) (error "reverse: argument must be a list"))
+        (:else (knl-reverse-step xs (list)))))))
+
+(define kernel-not-applicative (knl-unary-app "not" (fn (v) (not v))))
+
+;; Type predicates (Kernel-visible). Note `string?` covers BOTH symbols
+;; and string-literals in our representation (symbols are bare SX
+;; strings); a `kernel-string?` applicative distinguishes the two if
+;; needed.
+(define kernel-number?-applicative
+  (knl-unary-app "number?" (fn (v) (number? v))))
+(define kernel-string?-applicative
+  (knl-unary-app "string?" (fn (v) (string? v))))
+(define kernel-list?-applicative
+  (knl-unary-app "list?" (fn (v) (list? v))))
+(define kernel-boolean?-applicative
+  (knl-unary-app "boolean?" (fn (v) (boolean? v))))
+(define kernel-symbol?-applicative
+  (knl-unary-app "symbol?" (fn (v) (string? v))))
+
+(define kernel-eq?-applicative (knl-bin-app "eq?" (fn (a b) (= a b))))
+
+;; ── the standard environment ────────────────────────────────────
+
+(define
+  kernel-equal?-applicative
+  (knl-bin-app "equal?" (fn (a b) (= a b))))
+
+;; ── List combinators: map / filter / reduce ─────────────────────
+;; These re-enter the evaluator on each element, so they use the
+;; with-env applicative constructor.
+
+;; When the combiner is an applicative, we MUST unwrap before calling
+;; — otherwise kernel-combine will re-evaluate the already-evaluated
+;; element values (and crash if an element is itself a list).
+(define knl-apply-op
+  (fn (combiner)
+    (cond
+      ((kernel-applicative? combiner) (kernel-unwrap combiner))
+      (:else combiner))))
+
+(define knl-map-step
+  (fn (fn-val xs dyn-env)
+    (let ((op (knl-apply-op fn-val)))
+      (knl-map-walk op xs dyn-env))))
+
+(define knl-map-walk
+  (fn (op xs dyn-env)
+    (cond
+      ((or (nil? xs) (= (length xs) 0)) (list))
+      (:else
+        (cons (kernel-combine op (list (first xs)) dyn-env)
+              (knl-map-walk op (rest xs) dyn-env))))))
+
+(define kernel-map-applicative
+  (kernel-make-primitive-applicative-with-env
+    (fn (args dyn-env)
+      (cond
+        ((not (= (length args) 2))
+          (error "map: expects (fn list)"))
+        ((not (kernel-combiner? (first args)))
+          (error "map: first arg must be a combiner"))
+        ((not (list? (nth args 1)))
+          (error "map: second arg must be a list"))
+        (:else (knl-map-step (first args) (nth args 1) dyn-env))))))
+
+(define knl-filter-step
+  (fn (pred xs dyn-env)
+    (knl-filter-walk (knl-apply-op pred) xs dyn-env)))
+
+(define knl-filter-walk
+  (fn (op xs dyn-env)
+    (cond
+      ((or (nil? xs) (= (length xs) 0)) (list))
+      (:else
+        (let ((keep? (kernel-combine op (list (first xs)) dyn-env)))
+          (cond
+            (keep?
+              (cons (first xs) (knl-filter-walk op (rest xs) dyn-env)))
+            (:else (knl-filter-walk op (rest xs) dyn-env))))))))
+
+(define kernel-filter-applicative
+  (kernel-make-primitive-applicative-with-env
+    (fn (args dyn-env)
+      (cond
+        ((not (= (length args) 2))
+          (error "filter: expects (pred list)"))
+        ((not (kernel-combiner? (first args)))
+          (error "filter: first arg must be a combiner"))
+        ((not (list? (nth args 1)))
+          (error "filter: second arg must be a list"))
+        (:else (knl-filter-step (first args) (nth args 1) dyn-env))))))
+
+(define knl-reduce-step
+  (fn (fn-val xs acc dyn-env)
+    (knl-reduce-walk (knl-apply-op fn-val) xs acc dyn-env)))
+
+(define knl-reduce-walk
+  (fn (op xs acc dyn-env)
+    (cond
+      ((or (nil? xs) (= (length xs) 0)) acc)
+      (:else
+        (knl-reduce-walk
+          op
+          (rest xs)
+          (kernel-combine op (list acc (first xs)) dyn-env)
+          dyn-env)))))
+
+;; (apply COMBINER ARGS-LIST) — call COMBINER with the elements of
+;; ARGS-LIST as arguments. The Kernel canonical use: turn a constructed
+;; list of values into a function call. We skip the applicative's
+;; auto-eval step (via unwrap) because ARGS-LIST is already values, not
+;; expressions; for a bare operative, we pass through directly.
+(define kernel-apply-applicative
+  (kernel-make-primitive-applicative-with-env
+    (fn (args dyn-env)
+      (cond
+        ((not (= (length args) 2))
+          (error "apply: expects (combiner args-list)"))
+        ((not (kernel-combiner? (first args)))
+          (error "apply: first arg must be a combiner"))
+        ((not (list? (nth args 1)))
+          (error "apply: second arg must be a list"))
+        (:else
+          (let ((op (cond
+                      ((kernel-applicative? (first args))
+                        (kernel-unwrap (first args)))
+                      (:else (first args)))))
+            (kernel-combine op (nth args 1) dyn-env)))))))
+
+(define kernel-reduce-applicative
+  (kernel-make-primitive-applicative-with-env
+    (fn (args dyn-env)
+      (cond
+        ((not (= (length args) 3))
+          (error "reduce: expects (fn init list)"))
+        ((not (kernel-combiner? (first args)))
+          (error "reduce: first arg must be a combiner"))
+        ((not (list? (nth args 2)))
+          (error "reduce: third arg must be a list"))
+        (:else
+          (knl-reduce-step (first args) (nth args 2)
+                           (nth args 1) dyn-env))))))
+
+;; ── Encapsulations: Kernel's opaque-type idiom ──────────────────
+;;
+;; (make-encapsulation-type) → (encapsulator predicate decapsulator)
+;;
+;; Each call returns three applicatives over a fresh family identity.
+;; - (encapsulator V) → an opaque wrapper around V.
+;; - (predicate V)    → true iff V was wrapped by THIS family.
+;; - (decapsulator W) → the inner value; errors on wrong family.
+;;
+;; Family identity is a fresh empty dict; SX compares dicts by reference,
+;; so two `(make-encapsulation-type)` calls return distinct families.
+;;
+;; Pattern usage (Phase 5 lacks destructuring, so accessors are explicit):
+;;   ($define! triple (make-encapsulation-type))
+;;   ($define! wrap-promise   (car  triple))
+;;   ($define! promise?       (car  (cdr triple)))
+;;   ($define! unwrap-promise (car  (cdr (cdr triple))))
+
+(define kernel-make-encap-type-impl
+  (fn (args)
+    (cond
+      ((not (= (length args) 0))
+        (error "make-encapsulation-type: expects 0 arguments"))
+      (:else
+        (let ((family {}))
+          (let ((encap
+                  (kernel-make-primitive-applicative
+                    (fn (vargs)
+                      (cond
+                        ((not (= (length vargs) 1))
+                          (error "encapsulator: expects 1 argument"))
+                        (:else
+                          {:knl-tag :encap
+                           :family family
+                           :value (first vargs)})))))
+                (pred
+                  (kernel-make-primitive-applicative
+                    (fn (vargs)
+                      (cond
+                        ((not (= (length vargs) 1))
+                          (error "predicate: expects 1 argument"))
+                        (:else
+                          (let ((v (first vargs)))
+                            (and (dict? v)
+                                 (= (get v :knl-tag) :encap)
+                                 (= (get v :family) family))))))))
+                (decap
+                  (kernel-make-primitive-applicative
+                    (fn (vargs)
+                      (cond
+                        ((not (= (length vargs) 1))
+                          (error "decapsulator: expects 1 argument"))
+                        (:else
+                          (let ((v (first vargs)))
+                            (cond
+                              ((not (and (dict? v)
+                                         (= (get v :knl-tag) :encap)))
+                                (error "decapsulator: not an encapsulation"))
+                              ((not (= (get v :family) family))
+                                (error "decapsulator: wrong family"))
+                              (:else (get v :value))))))))))
+            (list encap pred decap)))))))
+
+(define kernel-make-encap-type-applicative
+  (kernel-make-primitive-applicative kernel-make-encap-type-impl))
+
+;; ── Hygiene: $let, $define-in!, make-environment ────────────────
+;;
+;; Kernel-on-SX is hygienic *by default* because user-defined operatives
+;; (Phase 3) bind their formals + any $define! in a CHILD env extending
+;; the operative's static-env, never the dyn-env. The caller's env is
+;; only mutated when code explicitly says so (e.g. `(eval expr env-arg)`).
+;;
+;; Phase 6 adds two helpers that make the property easy to lean on:
+;;
+;;   ($let ((NAME EXPR) ...) BODY)
+;;     Evaluates each EXPR in the calling env, binds NAME in a fresh
+;;     child env, evaluates BODY in that child env. NAMES don't leak.
+;;
+;;   ($define-in! ENV NAME EXPR)
+;;     Binds NAME=value-of-EXPR in the *specified* env, not the dyn-env.
+;;     Useful for operatives that need to mutate a sandbox env without
+;;     touching their caller's env.
+;;
+;; Shutt's full scope-set / frame-stamp hygiene (lifted symbols carrying
+;; provenance markers so introduced bindings can shadow without
+;; capturing) is research-grade and not implemented here. Notes for
+;; `lib/guest/reflective/hygiene.sx` candidate API below the std env.
+
+(define knl-bind-let-vals!
+  (fn (local bindings dyn-env)
+    (cond
+      ((or (nil? bindings) (= (length bindings) 0)) nil)
+      (:else
+        (let ((b (first bindings)))
+          (cond
+            ((not (and (list? b) (= (length b) 2)))
+              (error "$let: each binding must be (name expr)"))
+            ((not (string? (first b)))
+              (error "$let: binding name must be a symbol"))
+            (:else
+              (begin
+                (kernel-env-bind! local
+                                  (first b)
+                                  (kernel-eval (nth b 1) dyn-env))
+                (knl-bind-let-vals! local (rest bindings) dyn-env)))))))))
+
+(define kernel-let-operative
+  (kernel-make-primitive-operative
+    (fn (args dyn-env)
+      (cond
+        ((< (length args) 2)
+          (error "$let: expects (bindings body...)"))
+        ((not (list? (first args)))
+          (error "$let: bindings must be a list"))
+        (:else
+          (let ((local (kernel-extend-env dyn-env)))
+            (knl-bind-let-vals! local (first args) dyn-env)
+            (knl-eval-body (rest args) local)))))))
+
+;; $let* — sequential let. Each binding sees prior names in scope.
+;; Implemented by nesting envs one per binding; the body runs in the
+;; innermost env, so later bindings shadow earlier ones if names repeat.
+(define knl-let*-step
+  (fn (bindings env body-forms)
+    (cond
+      ((or (nil? bindings) (= (length bindings) 0))
+        (knl-eval-body body-forms env))
+      (:else
+        (let ((b (first bindings)))
+          (cond
+            ((not (and (list? b) (= (length b) 2)))
+              (error "$let*: each binding must be (name expr)"))
+            ((not (string? (first b)))
+              (error "$let*: binding name must be a symbol"))
+            (:else
+              (let ((child (kernel-extend-env env)))
+                (kernel-env-bind! child
+                                  (first b)
+                                  (kernel-eval (nth b 1) env))
+                (knl-let*-step (rest bindings) child body-forms)))))))))
+
+(define kernel-let*-operative
+  (kernel-make-primitive-operative
+    (fn (args dyn-env)
+      (cond
+        ((< (length args) 2)
+          (error "$let*: expects (bindings body...)"))
+        ((not (list? (first args)))
+          (error "$let*: bindings must be a list"))
+        (:else
+          (knl-let*-step (first args) dyn-env (rest args)))))))
+
+(define kernel-define-in!-operative
+  (kernel-make-primitive-operative
+    (fn (args dyn-env)
+      (cond
+        ((not (= (length args) 3))
+          (error "$define-in!: expects (env-expr name expr)"))
+        ((not (string? (nth args 1)))
+          (error "$define-in!: name must be a symbol"))
+        (:else
+          (let ((target (kernel-eval (first args) dyn-env)))
+            (cond
+              ((not (kernel-env? target))
+                (error "$define-in!: first arg must evaluate to an env"))
+              (:else
+                (let ((v (kernel-eval (nth args 2) dyn-env)))
+                  (kernel-env-bind! target (nth args 1) v)
+                  v)))))))))
+
+(define
+  kernel-standard-env
+  (fn
+    ()
+    (let
+      ((env (kernel-base-env)))
+      (kernel-env-bind! env "$if" kernel-if-operative)
+      (kernel-env-bind! env "$define!" kernel-define!-operative)
+      (kernel-env-bind! env "$sequence" kernel-sequence-operative)
+      (kernel-env-bind! env "$quote"    kernel-quote-operative)
+      (kernel-env-bind! env "$quasiquote" kernel-quasiquote-operative)
+      (kernel-env-bind! env "$cond"   kernel-cond-operative)
+      (kernel-env-bind! env "$when"   kernel-when-operative)
+      (kernel-env-bind! env "$unless" kernel-unless-operative)
+      (kernel-env-bind! env "$and?"   kernel-and?-operative)
+      (kernel-env-bind! env "$or?"    kernel-or?-operative)
+      (kernel-env-bind! env "eval" kernel-eval-applicative)
+      (kernel-env-bind!
+        env
+        "make-environment"
+        kernel-make-environment-applicative)
+      (kernel-env-bind!
+        env
+        "get-current-environment"
+        kernel-get-current-env-operative)
+      (kernel-env-bind! env "+"
+        (knl-fold-app "+" (fn (a b) (+ a b)) 0 (fn (x) x)))
+      (kernel-env-bind! env "-"
+        (knl-fold-app "-" (fn (a b) (- a b)) 0 (fn (x) (- 0 x))))
+      (kernel-env-bind! env "*"
+        (knl-fold-app "*" (fn (a b) (* a b)) 1 (fn (x) x)))
+      (kernel-env-bind! env "/"
+        (knl-fold-app "/" (fn (a b) (/ a b)) 1 (fn (x) (/ 1 x))))
+      (kernel-env-bind! env "<"   (knl-chain-cmp "<"   (fn (a b) (< a b))))
+      (kernel-env-bind! env ">"   (knl-chain-cmp ">"   (fn (a b) (> a b))))
+      (kernel-env-bind! env "<=?" (knl-chain-cmp "<=?" (fn (a b) (<= a b))))
+      (kernel-env-bind! env ">=?" (knl-chain-cmp ">=?" (fn (a b) (>= a b))))
+      (kernel-env-bind! env "=?" kernel-eq?-applicative)
+      (kernel-env-bind! env "equal?" kernel-equal?-applicative)
+      (kernel-env-bind! env "eq?" kernel-eq?-applicative)
+      (kernel-env-bind! env "cons" kernel-cons-applicative)
+      (kernel-env-bind! env "car" kernel-car-applicative)
+      (kernel-env-bind! env "cdr" kernel-cdr-applicative)
+      (kernel-env-bind! env "list" kernel-list-applicative)
+      (kernel-env-bind! env "length" kernel-length-applicative)
+      (kernel-env-bind! env "null?" kernel-null?-applicative)
+      (kernel-env-bind! env "pair?" kernel-pair?-applicative)
+      (kernel-env-bind! env "map"    kernel-map-applicative)
+      (kernel-env-bind! env "filter" kernel-filter-applicative)
+      (kernel-env-bind! env "reduce" kernel-reduce-applicative)
+      (kernel-env-bind! env "apply"   kernel-apply-applicative)
+      (kernel-env-bind! env "append"  kernel-append-applicative)
+      (kernel-env-bind! env "reverse" kernel-reverse-applicative)
+      (kernel-env-bind! env "number?"  kernel-number?-applicative)
+      (kernel-env-bind! env "string?"  kernel-string?-applicative)
+      (kernel-env-bind! env "list?"    kernel-list?-applicative)
+      (kernel-env-bind! env "boolean?" kernel-boolean?-applicative)
+      (kernel-env-bind! env "symbol?"  kernel-symbol?-applicative)
+      (kernel-env-bind! env "not" kernel-not-applicative)
+      (kernel-env-bind! env "make-encapsulation-type"
+                            kernel-make-encap-type-applicative)
+      (kernel-env-bind! env "$let"         kernel-let-operative)
+      (kernel-env-bind! env "$let*"        kernel-let*-operative)
+      (kernel-env-bind! env "$define-in!"  kernel-define-in!-operative)
+      env)))

lib/kernel/tests/encap.sx

@@ -0,0 +1,171 @@
+;; lib/kernel/tests/encap.sx — exercises make-encapsulation-type.
+;;
+;; The Phase 5 Kernel idiom: build opaque types whose constructor,
+;; predicate, and accessor are all standard Kernel applicatives. The
+;; identity is per-call, so two `(make-encapsulation-type)` calls
+;; produce non-interchangeable families.
+
+(define ken-suite (refl-make-test-suite))
+(define ken-test  (fn (n a e) (refl-test ken-suite n a e)))
+
+(define ken-eval-in (fn (src env) (kernel-eval (kernel-parse src) env)))
+
+;; A helper that builds a standard env with `encap`/`pred?`/`decap`
+;; bound from a single call to make-encapsulation-type.
+(define
+  ken-make-encap-env
+  (fn
+    ()
+    (let
+      ((env (kernel-standard-env)))
+      (ken-eval-in "($define! triple (make-encapsulation-type))" env)
+      (ken-eval-in "($define! encap (car triple))" env)
+      (ken-eval-in "($define! pred? (car (cdr triple)))" env)
+      (ken-eval-in "($define! decap (car (cdr (cdr triple))))" env)
+      env)))
+
+;; ── construction ────────────────────────────────────────────────
+(ken-test
+  "make: returns 3-element list"
+  (ken-eval-in "(length (make-encapsulation-type))" (kernel-standard-env))
+  3)
+
+(ken-test
+  "make: first is applicative"
+  (kernel-applicative?
+    (ken-eval-in "(car (make-encapsulation-type))" (kernel-standard-env)))
+  true)
+
+(ken-test
+  "make: second is applicative"
+  (kernel-applicative?
+    (ken-eval-in
+      "(car (cdr (make-encapsulation-type)))"
+      (kernel-standard-env)))
+  true)
+
+(ken-test
+  "make: third is applicative"
+  (kernel-applicative?
+    (ken-eval-in
+      "(car (cdr (cdr (make-encapsulation-type))))"
+      (kernel-standard-env)))
+  true)
+
+;; ── round-trip ──────────────────────────────────────────────────
+(ken-test
+  "round-trip: number"
+  (ken-eval-in "(decap (encap 42))" (ken-make-encap-env))
+  42)
+
+(ken-test
+  "round-trip: string"
+  (ken-eval-in "(decap (encap ($quote hello)))" (ken-make-encap-env))
+  "hello")
+
+(ken-test
+  "round-trip: list"
+  (ken-eval-in "(decap (encap (list 1 2 3)))" (ken-make-encap-env))
+  (list 1 2 3))
+
+;; ── predicate ───────────────────────────────────────────────────
+(ken-test
+  "pred?: wrapped value"
+  (ken-eval-in "(pred? (encap 1))" (ken-make-encap-env))
+  true)
+
+(ken-test
+  "pred?: raw value"
+  (ken-eval-in "(pred? 1)" (ken-make-encap-env))
+  false)
+
+(ken-test
+  "pred?: raw string"
+  (ken-eval-in "(pred? ($quote foo))" (ken-make-encap-env))
+  false)
+
+(ken-test
+  "pred?: raw list"
+  (ken-eval-in "(pred? (list))" (ken-make-encap-env))
+  false)
+
+;; ── opacity: different families are not interchangeable ─────────
+(ken-test
+  "opacity: foreign value rejected by predicate"
+  (let
+    ((env (kernel-standard-env)))
+    (ken-eval-in "($define! tA (make-encapsulation-type))" env)
+    (ken-eval-in "($define! tB (make-encapsulation-type))" env)
+    (ken-eval-in "($define! encA (car tA))" env)
+    (ken-eval-in "($define! predB (car (cdr tB)))" env)
+    (ken-eval-in "(predB (encA 42))" env))
+  false)
+
+(ken-test
+  "opacity: decap rejects foreign value"
+  (let
+    ((env (kernel-standard-env)))
+    (ken-eval-in "($define! tA (make-encapsulation-type))" env)
+    (ken-eval-in "($define! tB (make-encapsulation-type))" env)
+    (ken-eval-in "($define! encA (car tA))" env)
+    (ken-eval-in "($define! decapB (car (cdr (cdr tB))))" env)
+    (guard (e (true :raised)) (ken-eval-in "(decapB (encA 42))" env)))
+  :raised)
+
+(ken-test
+  "opacity: decap rejects raw value"
+  (guard
+    (e (true :raised))
+    (ken-eval-in "(decap 42)" (ken-make-encap-env)))
+  :raised)
+
+;; ── promise: classic Kernel encapsulation use case ──────────────
+;; A "promise" wraps a thunk to compute on demand and memoises the
+;; first result. Built entirely with the standard encap idiom.
+(ken-test
+  "promise: force returns thunk result"
+  (let
+    ((env (kernel-standard-env)))
+    (ken-eval-in
+      "($sequence\n         ($define! ptriple (make-encapsulation-type))\n         ($define! make-promise (car ptriple))\n         ($define! promise? (car (cdr ptriple)))\n         ($define! decode-promise (car (cdr (cdr ptriple))))\n         ($define! force ($lambda (p) ((decode-promise p))))\n         ($define! delay ($lambda (thunk) (make-promise thunk)))\n         (force (delay ($lambda () (+ 19 23)))))"
+      env))
+  42)
+
+(ken-test
+  "promise: promise? recognises its own type"
+  (let
+    ((env (kernel-standard-env)))
+    (ken-eval-in
+      "($sequence\n         ($define! ptriple (make-encapsulation-type))\n         ($define! make-promise (car ptriple))\n         ($define! promise? (car (cdr ptriple)))\n         (promise? (make-promise ($lambda () 42))))"
+      env))
+  true)
+
+(ken-test
+  "promise: promise? false on plain value"
+  (let
+    ((env (kernel-standard-env)))
+    (ken-eval-in
+      "($sequence\n         ($define! ptriple (make-encapsulation-type))\n         ($define! promise? (car (cdr ptriple)))\n         (promise? 99))"
+      env))
+  false)
+
+;; ── independent families don't leak ─────────────────────────────
+(ken-test
+  "two families: distinct identity"
+  (let
+    ((env (kernel-standard-env)))
+    (ken-eval-in
+      "($sequence\n         ($define! t1 (make-encapsulation-type))\n         ($define! t2 (make-encapsulation-type))\n         ($define! enc1 (car t1))\n         ($define! pred2 (car (cdr t2)))\n         (pred2 (enc1 ($quote stuff))))"
+      env))
+  false)
+
+(ken-test
+  "same family: re-bound shares identity"
+  (let
+    ((env (kernel-standard-env)))
+    (ken-eval-in
+      "($sequence\n         ($define! t (make-encapsulation-type))\n         ($define! e (car t))\n         ($define! p (car (cdr t)))\n         ($define! d (car (cdr (cdr t))))\n         (list (p (e 7)) (d (e 7))))"
+      env))
+  (list true 7))
+
+(define ken-tests-run! (fn () (refl-test-report ken-suite)))

lib/kernel/tests/eval.sx

@@ -0,0 +1,258 @@
+;; lib/kernel/tests/eval.sx — exercises lib/kernel/eval.sx.
+;;
+;; Phase 2 covers literal evaluation, symbol lookup, and combiner
+;; dispatch (operative vs applicative). Standard-environment operatives
+;; ($if, $define!, $lambda, …) arrive in Phase 4, so tests build a
+;; minimal env on the fly and verify the dispatch contract directly.
+
+(define ke-suite (refl-make-test-suite))
+(define ke-test  (fn (n a e) (refl-test ke-suite n a e)))
+
+;; ── helpers ──────────────────────────────────────────────────────
+
+(define ke-eval-src (fn (src env) (kernel-eval (kernel-parse src) env)))
+
+(define
+  ke-make-test-env
+  (fn
+    ()
+    (let
+      ((env (kernel-make-env)))
+      (kernel-env-bind!
+        env
+        "+"
+        (kernel-make-primitive-applicative
+          (fn (args) (+ (first args) (nth args 1)))))
+      (kernel-env-bind!
+        env
+        "list"
+        (kernel-make-primitive-applicative (fn (args) args)))
+      (kernel-env-bind!
+        env
+        "$quote"
+        (kernel-make-primitive-operative (fn (args dyn-env) (first args))))
+      (kernel-env-bind!
+        env
+        "$if"
+        (kernel-make-primitive-operative
+          (fn
+            (args dyn-env)
+            (if
+              (kernel-eval (first args) dyn-env)
+              (kernel-eval (nth args 1) dyn-env)
+              (kernel-eval (nth args 2) dyn-env)))))
+      env)))
+
+;; ── literal evaluation ───────────────────────────────────────────
+(ke-test "lit: number" (ke-eval-src "42" (kernel-make-env)) 42)
+(ke-test "lit: zero" (ke-eval-src "0" (kernel-make-env)) 0)
+(ke-test "lit: float" (ke-eval-src "3.14" (kernel-make-env)) 3.14)
+(ke-test "lit: true" (ke-eval-src "#t" (kernel-make-env)) true)
+(ke-test "lit: false" (ke-eval-src "#f" (kernel-make-env)) false)
+(ke-test "lit: string" (ke-eval-src "\"hello\"" (kernel-make-env)) "hello")
+(ke-test "lit: empty list" (ke-eval-src "()" (kernel-make-env)) (list))
+
+;; ── symbol lookup ────────────────────────────────────────────────
+(ke-test
+  "sym: bound to number"
+  (let
+    ((env (kernel-make-env)))
+    (kernel-env-bind! env "x" 100)
+    (ke-eval-src "x" env))
+  100)
+
+(ke-test
+  "sym: bound to string"
+  (let
+    ((env (kernel-make-env)))
+    (kernel-env-bind! env "name" "kernel")
+    (ke-eval-src "name" env))
+  "kernel")
+
+(ke-test
+  "sym: parent-chain lookup"
+  (let
+    ((p (kernel-make-env)))
+    (kernel-env-bind! p "outer" 1)
+    (let
+      ((c (kernel-extend-env p)))
+      (kernel-env-bind! c "inner" 2)
+      (+ (ke-eval-src "outer" c) (ke-eval-src "inner" c))))
+  3)
+
+(ke-test
+  "sym: child shadows parent"
+  (let
+    ((p (kernel-make-env)))
+    (kernel-env-bind! p "x" 1)
+    (let
+      ((c (kernel-extend-env p)))
+      (kernel-env-bind! c "x" 2)
+      (ke-eval-src "x" c)))
+  2)
+
+(ke-test
+  "env-has?: present"
+  (let
+    ((env (kernel-make-env)))
+    (kernel-env-bind! env "x" 1)
+    (kernel-env-has? env "x"))
+  true)
+
+(ke-test
+  "env-has?: missing"
+  (kernel-env-has? (kernel-make-env) "nope")
+  false)
+
+;; ── tagged-value predicates ─────────────────────────────────────
+(ke-test
+  "tag: operative?"
+  (kernel-operative? (kernel-make-primitive-operative (fn (a e) nil)))
+  true)
+
+(ke-test
+  "tag: applicative?"
+  (kernel-applicative? (kernel-make-primitive-applicative (fn (a) nil)))
+  true)
+
+(ke-test
+  "tag: combiner? operative"
+  (kernel-combiner? (kernel-make-primitive-operative (fn (a e) nil)))
+  true)
+
+(ke-test
+  "tag: combiner? applicative"
+  (kernel-combiner? (kernel-make-primitive-applicative (fn (a) nil)))
+  true)
+
+(ke-test "tag: combiner? number" (kernel-combiner? 42) false)
+
+(ke-test "tag: number is not operative" (kernel-operative? 42) false)
+
+;; ── wrap / unwrap ────────────────────────────────────────────────
+(ke-test
+  "wrap+unwrap roundtrip"
+  (let
+    ((op (kernel-make-primitive-operative (fn (a e) :sentinel))))
+    (= (kernel-unwrap (kernel-wrap op)) op))
+  true)
+
+(ke-test
+  "wrap produces applicative"
+  (kernel-applicative?
+    (kernel-wrap (kernel-make-primitive-operative (fn (a e) nil))))
+  true)
+
+(ke-test
+  "unwrap of primitive-applicative is operative"
+  (kernel-operative?
+    (kernel-unwrap (kernel-make-primitive-applicative (fn (a) nil))))
+  true)
+
+;; ── combiner dispatch — applicatives evaluate their args ─────────
+(ke-test
+  "applicative: simple call"
+  (ke-eval-src "(+ 2 3)" (ke-make-test-env))
+  5)
+
+(ke-test
+  "applicative: nested"
+  (ke-eval-src "(+ (+ 1 2) (+ 3 4))" (ke-make-test-env))
+  10)
+
+(ke-test
+  "applicative: receives evaluated args"
+  (let
+    ((env (ke-make-test-env)))
+    (kernel-env-bind! env "x" 10)
+    (kernel-env-bind! env "y" 20)
+    (ke-eval-src "(+ x y)" env))
+  30)
+
+(ke-test
+  "applicative: list builds an SX list of values"
+  (let
+    ((env (ke-make-test-env)))
+    (kernel-env-bind! env "a" 1)
+    (kernel-env-bind! env "b" 2)
+    (ke-eval-src "(list a b 99)" env))
+  (list 1 2 99))
+
+;; ── combiner dispatch — operatives DO NOT evaluate their args ───
+(ke-test
+  "operative: $quote returns symbol unevaluated"
+  (ke-eval-src "($quote foo)" (ke-make-test-env))
+  "foo")
+
+(ke-test
+  "operative: $quote returns list unevaluated"
+  (ke-eval-src "($quote (+ 1 2))" (ke-make-test-env))
+  (list "+" 1 2))
+
+(ke-test
+  "operative: $if true branch"
+  (ke-eval-src "($if #t 1 2)" (ke-make-test-env))
+  1)
+
+(ke-test
+  "operative: $if false branch"
+  (ke-eval-src "($if #f 1 2)" (ke-make-test-env))
+  2)
+
+(ke-test
+  "operative: $if doesn't eval untaken branch"
+  (ke-eval-src "($if #t 99 unbound)" (ke-make-test-env))
+  99)
+
+(ke-test
+  "operative: $if takes dynamic env for branches"
+  (let
+    ((env (ke-make-test-env)))
+    (kernel-env-bind! env "x" 7)
+    (ke-eval-src "($if #t x 0)" env))
+  7)
+
+;; ── operative built ON-THE-FLY can inspect raw expressions ──────
+(ke-test
+  "operative: sees raw symbol head"
+  (let
+    ((env (kernel-make-env)))
+    (kernel-env-bind!
+      env
+      "head"
+      (kernel-make-primitive-operative (fn (args dyn-env) (first args))))
+    (ke-eval-src "(head (+ 1 2))" env))
+  (list "+" 1 2))
+
+(ke-test
+  "operative: sees dynamic env"
+  (let
+    ((env (kernel-make-env)))
+    (kernel-env-bind! env "x" 999)
+    (kernel-env-bind!
+      env
+      "$probe"
+      (kernel-make-primitive-operative
+        (fn (args dyn-env) (kernel-env-lookup dyn-env "x"))))
+    (ke-eval-src "($probe ignored)" env))
+  999)
+
+;; ── error cases ──────────────────────────────────────────────────
+(ke-test
+  "error: unbound symbol"
+  (guard
+    (e (true :raised))
+    (kernel-eval (kernel-parse "nope") (kernel-make-env)))
+  :raised)
+
+(ke-test
+  "error: combine non-combiner"
+  (guard
+    (e (true :raised))
+    (let
+      ((env (kernel-make-env)))
+      (kernel-env-bind! env "x" 42)
+      (kernel-eval (kernel-parse "(x 1)") env)))
+  :raised)
+
+(define ke-tests-run! (fn () (refl-test-report ke-suite)))

lib/kernel/tests/hygiene.sx

@@ -0,0 +1,208 @@
+;; lib/kernel/tests/hygiene.sx — exercises Phase 6 hygiene helpers.
+;;
+;; Kernel-on-SX is hygienic by default: $vau/$lambda close over their
+;; static env, and bind their formals (plus any $define!s in the body)
+;; in a CHILD env. The caller's env is only mutated when user code
+;; explicitly threads the env-param through `eval` or `$define-in!`.
+;;
+;; These tests verify the property, plus the Phase 6 helpers ($let and
+;; $define-in!). Shutt's full scope-set hygiene (lifted symbols with
+;; provenance markers) is research-grade and is NOT implemented — see
+;; the plan's reflective-API notes for the proposed approach.
+
+(define kh-suite (refl-make-test-suite))
+(define kh-test  (fn (n a e) (refl-test kh-suite n a e)))
+
+(define kh-eval-in (fn (src env) (kernel-eval (kernel-parse src) env)))
+
+;; ── Default hygiene: $define! inside operative body stays local ─
+
+(kh-test
+  "hygiene: vau body $define! doesn't escape"
+  (let
+    ((env (kernel-standard-env)))
+    (kh-eval-in "($define! x 1)" env)
+    (kh-eval-in
+      "($define! my-op ($vau () _ ($sequence ($define! x 999) x)))"
+      env)
+    (kh-eval-in "(my-op)" env)
+    (kh-eval-in "x" env))
+  1)
+
+(kh-test
+  "hygiene: vau body $define! visible inside body"
+  (let
+    ((env (kernel-standard-env)))
+    (kh-eval-in "($define! x 1)" env)
+    (kh-eval-in
+      "($define! my-op ($vau () _ ($sequence ($define! x 999) x)))"
+      env)
+    (kh-eval-in "(my-op)" env))
+  999)
+
+(kh-test
+  "hygiene: lambda body $define! doesn't escape"
+  (let
+    ((env (kernel-standard-env)))
+    (kh-eval-in "($define! y 50)" env)
+    (kh-eval-in "($define! f ($lambda () ($sequence ($define! y 7) y)))" env)
+    (kh-eval-in "(f)" env)
+    (kh-eval-in "y" env))
+  50)
+
+(kh-test
+  "hygiene: caller's binding visible inside operative"
+  (let
+    ((env (kernel-standard-env)))
+    (kh-eval-in "($define! caller-x 88)" env)
+    (kh-eval-in "($define! my-op ($vau () _ caller-x))" env)
+    (kh-eval-in "(my-op)" env))
+  88)
+
+;; ── $let — proper hygienic scoping ──────────────────────────────
+
+(kh-test
+  "let: returns body value"
+  (kh-eval-in "($let ((x 5)) (+ x 1))" (kernel-standard-env))
+  6)
+
+(kh-test
+  "let: multiple bindings"
+  (kh-eval-in "($let ((x 3) (y 4)) (+ x y))" (kernel-standard-env))
+  7)
+
+(kh-test
+  "let: bindings shadow outer"
+  (let
+    ((env (kernel-standard-env)))
+    (kh-eval-in "($define! x 1)" env)
+    (kh-eval-in "($let ((x 99)) x)" env))
+  99)
+
+(kh-test
+  "let: bindings don't leak after"
+  (let
+    ((env (kernel-standard-env)))
+    (kh-eval-in "($define! x 1)" env)
+    (kh-eval-in "($let ((x 99)) x)" env)
+    (kh-eval-in "x" env))
+  1)
+
+(kh-test
+  "let: parallel — RHS sees outer, not inner"
+  (let
+    ((env (kernel-standard-env)))
+    (kh-eval-in "($define! x 1)" env)
+    (kh-eval-in "($let ((x 10) (y x)) y)" env))
+  1)
+
+(kh-test
+  "let: nested"
+  (kh-eval-in "($let ((x 1)) ($let ((y 2)) (+ x y)))" (kernel-standard-env))
+  3)
+
+(kh-test
+  "let: error on malformed binding"
+  (guard
+    (e (true :raised))
+    (kh-eval-in "($let ((x)) x)" (kernel-standard-env)))
+  :raised)
+
+(kh-test
+  "let: error on non-symbol name"
+  (guard
+    (e (true :raised))
+    (kh-eval-in "($let ((1 2)) 1)" (kernel-standard-env)))
+  :raised)
+
+;; ── $define-in! — explicit env targeting ────────────────────────
+
+(kh-test
+  "define-in!: binds in chosen env, not dyn-env"
+  (let
+    ((env (kernel-standard-env)))
+    (kh-eval-in "($define! sandbox (make-environment))" env)
+    (kh-eval-in "($define-in! sandbox z 77)" env)
+    (kernel-env-has? (kh-eval-in "sandbox" env) "z"))
+  true)
+
+(kh-test
+  "define-in!: doesn't pollute caller"
+  (let
+    ((env (kernel-standard-env)))
+    (kh-eval-in "($define! sandbox (make-environment))" env)
+    (kh-eval-in "($define-in! sandbox z 77)" env)
+    (kernel-env-has? env "z"))
+  false)
+
+(kh-test
+  "define-in!: error on non-env target"
+  (guard
+    (e (true :raised))
+    (let
+      ((env (kernel-standard-env)))
+      (kh-eval-in "($define-in! 42 x 1)" env)))
+  :raised)
+
+;; ── Closure does NOT see post-definition caller binds ───────────
+;; The classic "lexical scope wins over dynamic" test.
+
+(kh-test
+  "lexical: closure sees its own static env"
+  (let
+    ((env (kernel-standard-env)))
+    (kh-eval-in "($define! x 1)" env)
+    (kh-eval-in "($define! get-x ($lambda () x))" env)
+    (kh-eval-in "($define! x 999)" env)
+    (kh-eval-in "(get-x)" env))
+  999)
+
+(kh-test
+  "lexical: $let-bound name invisible outside"
+  (guard
+    (e (true :raised))
+    (let
+      ((env (kernel-standard-env)))
+      (kh-eval-in "($let ((private 42)) private)" env)
+      (kh-eval-in "private" env)))
+  :raised)
+
+;; ── Operative + $let: hygiene compose ───────────────────────────
+
+(kh-test
+  "let-inside-vau: temp doesn't escape body"
+  (let
+    ((env (kernel-standard-env)))
+    (kh-eval-in "($define! x 1)" env)
+    (kh-eval-in "($define! op ($vau () _ ($let ((x 5)) x)))" env)
+    (kh-eval-in "(op)" env)
+    (kh-eval-in "x" env))
+  1)
+
+;; ── $let* — sequential let ──────────────────────────────────────
+(kh-test "let*: empty bindings"
+  (kh-eval-in "($let* () 42)" (kernel-standard-env)) 42)
+(kh-test "let*: single binding"
+  (kh-eval-in "($let* ((x 5)) (+ x 1))" (kernel-standard-env)) 6)
+(kh-test "let*: later sees earlier"
+  (kh-eval-in "($let* ((x 1) (y (+ x 1)) (z (+ y 1))) z)"
+              (kernel-standard-env)) 3)
+(kh-test "let*: bindings don't leak after"
+  (let ((env (kernel-standard-env)))
+    (kh-eval-in "($define! x 1)" env)
+    (kh-eval-in "($let* ((x 99) (y (+ x 1))) y)" env)
+    (kh-eval-in "x" env)) 1)
+(kh-test "let*: same-name later binding shadows earlier"
+  (kh-eval-in "($let* ((x 1) (x 2)) x)" (kernel-standard-env)) 2)
+(kh-test "let*: multi-expression body"
+  (kh-eval-in "($let* ((x 5)) ($define! double (+ x x)) double)"
+              (kernel-standard-env)) 10)
+(kh-test "let*: error on malformed binding"
+  (guard (e (true :raised))
+    (kh-eval-in "($let* ((x)) x)" (kernel-standard-env)))
+  :raised)
+(kh-test "let: multi-body"
+  (kh-eval-in "($let ((x 5)) ($define! tmp (+ x 1)) tmp)"
+              (kernel-standard-env)) 6)
+
+(define kh-tests-run! (fn () (refl-test-report kh-suite)))

lib/kernel/tests/metacircular.sx

@@ -0,0 +1,150 @@
+;; lib/kernel/tests/metacircular.sx — Kernel-in-Kernel demo.
+;;
+;; Demonstrates reflective completeness: a Kernel program implements
+;; a recognisable subset of Kernel's own evaluation rules and produces
+;; matching values for a battery of test programs.
+;;
+;; This is a SHALLOW metacircular: it dispatches on expression shape
+;; itself (numbers, booleans, lists, symbols), recursively meta-evals
+;; each argument of an applicative call, and delegates only to the
+;; host evaluator for the leaf cases (operatives, symbol lookup). The
+;; point is to show that env-as-value, first-class operatives, and
+;; first-class evaluators all line up — enough so a Kernel program
+;; can itself reason about Kernel programs.
+
+(define kmc-suite (refl-make-test-suite))
+(define kmc-test  (fn (n a e) (refl-test kmc-suite n a e)))
+
+;; Build a Kernel env with m-eval and m-apply defined. The two refer
+;; to each other and to standard primitives, so we use the standard
+;; env as the static-env for both.
+(define
+  kmc-make-env
+  (fn
+    ()
+    (let
+      ((env (kernel-standard-env)))
+      (kernel-eval
+        (kernel-parse
+          "($define! m-eval\n           ($lambda (expr env)\n             ($cond\n               ((number? expr) expr)\n               ((boolean? expr) expr)\n               ((null? expr) expr)\n               ((symbol? expr) (eval expr env))\n               ((list? expr)\n                 ($let ((head-val (m-eval (car expr) env)))\n                   ($cond\n                     ((applicative? head-val)\n                       (apply head-val\n                         (map ($lambda (a) (m-eval a env)) (cdr expr))))\n                     (else (eval expr env)))))\n               (else expr))))")
+        env)
+      env)))
+
+(define
+  kmc-eval
+  (fn
+    (src)
+    (let
+      ((env (kmc-make-env)))
+      (kernel-eval
+        (kernel-parse
+          (str "(m-eval (quote " src ") (get-current-environment))"))
+        env))))
+
+;; ── literals self-evaluate via m-eval ──────────────────────────
+(kmc-test
+  "m-eval: integer literal"
+  (kernel-eval
+    (kernel-parse "(m-eval 42 (get-current-environment))")
+    (kmc-make-env))
+  42)
+
+(kmc-test
+  "m-eval: boolean true"
+  (kernel-eval
+    (kernel-parse "(m-eval #t (get-current-environment))")
+    (kmc-make-env))
+  true)
+
+(kmc-test
+  "m-eval: boolean false"
+  (kernel-eval
+    (kernel-parse "(m-eval #f (get-current-environment))")
+    (kmc-make-env))
+  false)
+
+(kmc-test
+  "m-eval: empty list"
+  (kernel-eval
+    (kernel-parse "(m-eval () (get-current-environment))")
+    (kmc-make-env))
+  (list))
+
+;; ── symbol lookup goes through env ─────────────────────────────
+(kmc-test
+  "m-eval: symbol lookup"
+  (let
+    ((env (kmc-make-env)))
+    (kernel-eval (kernel-parse "($define! shared-x 99)") env)
+    (kernel-eval
+      (kernel-parse "(m-eval ($quote shared-x) (get-current-environment))")
+      env))
+  99)
+
+;; ── applicative calls are dispatched by m-eval recursively ─────
+(kmc-test
+  "m-eval: addition"
+  (kernel-eval
+    (kernel-parse "(m-eval ($quote (+ 1 2)) (get-current-environment))")
+    (kmc-make-env))
+  3)
+
+(kmc-test
+  "m-eval: nested arithmetic"
+  (kernel-eval
+    (kernel-parse
+      "(m-eval ($quote (+ (* 2 3) (- 10 4))) (get-current-environment))")
+    (kmc-make-env))
+  12)
+
+(kmc-test
+  "m-eval: variadic +"
+  (kernel-eval
+    (kernel-parse "(m-eval ($quote (+ 1 2 3 4 5)) (get-current-environment))")
+    (kmc-make-env))
+  15)
+
+(kmc-test
+  "m-eval: list construction"
+  (kernel-eval
+    (kernel-parse "(m-eval ($quote (list 1 2 3)) (get-current-environment))")
+    (kmc-make-env))
+  (list 1 2 3))
+
+(kmc-test "m-eval: cons reverse-style"
+  (kernel-eval
+    (kernel-parse "(m-eval ($quote (cons 0 (list 1 2))) (get-current-environment))")
+    (kmc-make-env)) (list 0 1 2))
+
+(kmc-test "m-eval: nested apply"
+  (kernel-eval
+    (kernel-parse "(m-eval ($quote (apply + (list 10 20 30))) (get-current-environment))")
+    (kmc-make-env)) 60)
+
+;; ── operatives delegate to host eval (transparently for the caller) ─
+(kmc-test
+  "m-eval: $if true branch (via delegation)"
+  (kernel-eval
+    (kernel-parse "(m-eval ($quote ($if #t 1 2)) (get-current-environment))")
+    (kmc-make-env))
+  1)
+
+(kmc-test
+  "m-eval: $if false branch"
+  (kernel-eval
+    (kernel-parse "(m-eval ($quote ($if #f 1 2)) (get-current-environment))")
+    (kmc-make-env))
+  2)
+
+;; ── m-eval can call a user-defined lambda ──────────────────────
+(kmc-test
+  "m-eval: user lambda call"
+  (let
+    ((env (kmc-make-env)))
+    (kernel-eval (kernel-parse "($define! sq ($lambda (x) (* x x)))") env)
+    (kernel-eval
+      (kernel-parse "(m-eval ($quote (sq 7)) (get-current-environment))")
+      env))
+  49)
+
+(define kmc-tests-run! (fn () (refl-test-report kmc-suite)))

lib/kernel/tests/parse.sx

@@ -0,0 +1,146 @@
+;; lib/kernel/tests/parse.sx — exercises lib/kernel/parser.sx.
+
+(define knl-suite (refl-make-test-suite))
+(define knl-test  (fn (n a e) (refl-test knl-suite n a e)))
+
+;; ── atoms: numbers ────────────────────────────────────────────────
+(knl-test "num: integer" (kernel-parse "42") 42)
+(knl-test "num: zero" (kernel-parse "0") 0)
+(knl-test "num: negative integer" (kernel-parse "-7") -7)
+(knl-test "num: positive sign" (kernel-parse "+5") 5)
+(knl-test "num: float" (kernel-parse "3.14") 3.14)
+(knl-test "num: negative float" (kernel-parse "-2.5") -2.5)
+(knl-test "num: leading dot" (kernel-parse ".5") 0.5)
+(knl-test "num: exponent" (kernel-parse "1e3") 1000)
+(knl-test "num: exponent with sign" (kernel-parse "2.5e-1") 0.25)
+(knl-test "num: capital E exponent" (kernel-parse "1E2") 100)
+
+;; ── atoms: booleans ───────────────────────────────────────────────
+(knl-test "bool: true" (kernel-parse "#t") true)
+(knl-test "bool: false" (kernel-parse "#f") false)
+
+;; ── atoms: empty list (Kernel nil) ────────────────────────────────
+(knl-test "nil: ()" (kernel-parse "()") (list))
+(knl-test "nil: (= () (list))" (= (kernel-parse "()") (list)) true)
+
+;; ── atoms: symbols ────────────────────────────────────────────────
+(knl-test "sym: word" (kernel-parse "foo") "foo")
+(knl-test "sym: hyphenated" (kernel-parse "foo-bar") "foo-bar")
+(knl-test "sym: dollar-bang" (kernel-parse "$define!") "$define!")
+(knl-test "sym: question" (kernel-parse "null?") "null?")
+(knl-test "sym: lt-eq" (kernel-parse "<=") "<=")
+(knl-test "sym: bare plus" (kernel-parse "+") "+")
+(knl-test "sym: bare minus" (kernel-parse "-") "-")
+(knl-test "sym: plus-letter" (kernel-parse "+a") "+a")
+(knl-test "sym: arrow" (kernel-parse "->") "->")
+(knl-test "sym: dot-prefixed" (kernel-parse ".foo") ".foo")
+
+;; ── atoms: strings ────────────────────────────────────────────────
+(knl-test "str: empty" (kernel-string-value (kernel-parse "\"\"")) "")
+(knl-test
+  "str: hello"
+  (kernel-string-value (kernel-parse "\"hello\""))
+  "hello")
+(knl-test "str: predicate" (kernel-string? (kernel-parse "\"x\"")) true)
+(knl-test "str: not symbol" (kernel-string? (kernel-parse "x")) false)
+(knl-test
+  "str: escape newline"
+  (kernel-string-value (kernel-parse "\"a\\nb\""))
+  "a\nb")
+(knl-test
+  "str: escape tab"
+  (kernel-string-value (kernel-parse "\"a\\tb\""))
+  "a\tb")
+(knl-test
+  "str: escape quote"
+  (kernel-string-value (kernel-parse "\"a\\\"b\""))
+  "a\"b")
+(knl-test
+  "str: escape backslash"
+  (kernel-string-value (kernel-parse "\"a\\\\b\""))
+  "a\\b")
+
+;; ── lists ─────────────────────────────────────────────────────────
+(knl-test "list: flat" (kernel-parse "(a b c)") (list "a" "b" "c"))
+(knl-test
+  "list: nested"
+  (kernel-parse "(a (b c) d)")
+  (list "a" (list "b" "c") "d"))
+(knl-test
+  "list: deeply nested"
+  (kernel-parse "(((x)))")
+  (list (list (list "x"))))
+(knl-test
+  "list: mixed atoms"
+  (kernel-parse "(1 #t foo)")
+  (list 1 true "foo"))
+(knl-test
+  "list: empty inside"
+  (kernel-parse "(a () b)")
+  (list "a" (list) "b"))
+
+;; ── whitespace + comments ─────────────────────────────────────────
+(knl-test "ws: leading" (kernel-parse "   42") 42)
+(knl-test "ws: trailing" (kernel-parse "42   ") 42)
+(knl-test "ws: tabs/newlines" (kernel-parse "\n\t  42 \n") 42)
+(knl-test "comment: line" (kernel-parse "; nope\n42") 42)
+(knl-test "comment: trailing" (kernel-parse "42 ; tail") 42)
+(knl-test
+  "comment: inside list"
+  (kernel-parse "(a ; mid\n  b)")
+  (list "a" "b"))
+
+;; ── parse-all ─────────────────────────────────────────────────────
+(knl-test "all: empty input" (kernel-parse-all "") (list))
+(knl-test "all: only whitespace" (kernel-parse-all "   ") (list))
+(knl-test "all: only comment" (kernel-parse-all "; nope") (list))
+(knl-test
+  "all: three forms"
+  (kernel-parse-all "1 2 3")
+  (list 1 2 3))
+(knl-test
+  "all: mixed"
+  (kernel-parse-all "($if #t 1 2) foo")
+  (list (list "$if" true 1 2) "foo"))
+
+;; ── classic Kernel programs (smoke) ───────────────────────────────
+(knl-test
+  "klisp: vau form"
+  (kernel-parse "($vau (x e) e (eval x e))")
+  (list "$vau" (list "x" "e") "e" (list "eval" "x" "e")))
+(knl-test
+  "klisp: define lambda"
+  (kernel-parse "($define! sq ($lambda (x) (* x x)))")
+  (list "$define!" "sq" (list "$lambda" (list "x") (list "*" "x" "x"))))
+
+;; ── round-trip identity for primitive symbols ─────────────────────
+(knl-test "identity: $vau" (kernel-parse "$vau") "$vau")
+(knl-test "identity: $lambda" (kernel-parse "$lambda") "$lambda")
+(knl-test "identity: wrap" (kernel-parse "wrap") "wrap")
+(knl-test "identity: unwrap" (kernel-parse "unwrap") "unwrap")
+
+;; ── reader macros ─────────────────────────────────────────────────
+(knl-test "reader: 'foo  → ($quote foo)"
+  (kernel-parse "'foo") (list "$quote" "foo"))
+(knl-test "reader: '(a b c)"
+  (kernel-parse "'(a b c)") (list "$quote" (list "a" "b" "c")))
+(knl-test "reader: nested quotes"
+  (kernel-parse "''x")
+  (list "$quote" (list "$quote" "x")))
+(knl-test "reader: ` quasiquote"
+  (kernel-parse "`x") (list "$quasiquote" "x"))
+(knl-test "reader: , unquote"
+  (kernel-parse ",x") (list "$unquote" "x"))
+(knl-test "reader: ,@ unquote-splicing"
+  (kernel-parse ",@x") (list "$unquote-splicing" "x"))
+(knl-test "reader: quasi-mix"
+  (kernel-parse "`(a ,b ,@c)")
+  (list "$quasiquote"
+        (list "a"
+              (list "$unquote" "b")
+              (list "$unquote-splicing" "c"))))
+(knl-test "reader: quote separates from neighbouring atom"
+  (kernel-parse "(a 'b c)")
+  (list "a" (list "$quote" "b") "c"))
+
+(define knl-tests-run! (fn () (refl-test-report knl-suite)))

lib/kernel/tests/standard.sx

@@ -0,0 +1,433 @@
+;; lib/kernel/tests/standard.sx — exercises the Kernel standard env.
+;;
+;; Phase 4 tests verify that the standard env is rich enough to run
+;; classic Kernel programs: factorial via recursion, list operations,
+;; first-class environment manipulation. Each test starts from a fresh
+;; standard env via `(kernel-standard-env)`.
+
+(define ks-suite (refl-make-test-suite))
+(define ks-test  (fn (n a e) (refl-test ks-suite n a e)))
+
+(define
+  ks-eval
+  (fn (src) (kernel-eval (kernel-parse src) (kernel-standard-env))))
+
+(define ks-eval-in (fn (src env) (kernel-eval (kernel-parse src) env)))
+
+(define
+  ks-eval-all
+  (fn (src env) (kernel-eval-program (kernel-parse-all src) env)))
+
+;; ── $if ──────────────────────────────────────────────────────────
+(ks-test "if: true branch" (ks-eval "($if #t 1 2)") 1)
+(ks-test "if: false branch" (ks-eval "($if #f 1 2)") 2)
+(ks-test "if: predicate"
+  (ks-eval "($if (<=? 1 2) ($quote yes) ($quote no))") "yes")
+(ks-test
+  "if: untaken branch not evaluated"
+  (ks-eval "($if #t 42 nope)")
+  42)
+
+;; ── $define! + arithmetic ───────────────────────────────────────
+(ks-test
+  "define!: returns value"
+  (let ((env (kernel-standard-env))) (ks-eval-in "($define! x 5)" env))
+  5)
+
+(ks-test
+  "define!: bound in env"
+  (let
+    ((env (kernel-standard-env)))
+    (ks-eval-in "($define! x 5)" env)
+    (ks-eval-in "x" env))
+  5)
+
+(ks-test "arith: +" (ks-eval "(+ 2 3)") 5)
+(ks-test "arith: -" (ks-eval "(- 10 4)") 6)
+(ks-test "arith: *" (ks-eval "(* 6 7)") 42)
+(ks-test "arith: /" (ks-eval "(/ 20 5)") 4)
+(ks-test "cmp: <  true" (ks-eval "(< 1 2)") true)
+(ks-test "cmp: <  false" (ks-eval "(< 2 1)") false)
+(ks-test "cmp: >=" (ks-eval "(>=? 2 2)") true)
+(ks-test "cmp: <=" (ks-eval "(<=? 2 3)") true)
+(ks-test "cmp: =" (ks-eval "(=? 7 7)") true)
+
+;; ── $sequence ────────────────────────────────────────────────────
+(ks-test "sequence: empty" (ks-eval "($sequence)") nil)
+(ks-test "sequence: single" (ks-eval "($sequence 99)") 99)
+(ks-test
+  "sequence: multi-effect"
+  (let
+    ((env (kernel-standard-env)))
+    (ks-eval-in "($sequence ($define! a 1) ($define! b 2) (+ a b))" env))
+  3)
+
+;; ── list primitives ──────────────────────────────────────────────
+(ks-test
+  "list: builds"
+  (ks-eval "(list 1 2 3)")
+  (list 1 2 3))
+(ks-test "list: empty" (ks-eval "(list)") (list))
+(ks-test
+  "cons: prepend"
+  (ks-eval "(cons 0 (list 1 2 3))")
+  (list 0 1 2 3))
+(ks-test "car: head" (ks-eval "(car (list 10 20 30))") 10)
+(ks-test
+  "cdr: tail"
+  (ks-eval "(cdr (list 10 20 30))")
+  (list 20 30))
+(ks-test "length: 3" (ks-eval "(length (list 1 2 3))") 3)
+(ks-test "length: 0" (ks-eval "(length (list))") 0)
+(ks-test "null?: empty" (ks-eval "(null? (list))") true)
+(ks-test "null?: nonempty" (ks-eval "(null? (list 1))") false)
+(ks-test "pair?: empty" (ks-eval "(pair? (list))") false)
+(ks-test "pair?: nonempty" (ks-eval "(pair? (list 1))") true)
+
+;; ── $quote ───────────────────────────────────────────────────────
+(ks-test "quote: symbol" (ks-eval "($quote foo)") "foo")
+(ks-test
+  "quote: list"
+  (ks-eval "($quote (+ 1 2))")
+  (list "+" 1 2))
+
+;; ── boolean / not ────────────────────────────────────────────────
+(ks-test "not: true" (ks-eval "(not #t)") false)
+(ks-test "not: false" (ks-eval "(not #f)") true)
+
+;; ── factorial ────────────────────────────────────────────────────
+(ks-test
+  "factorial: 5!"
+  (let
+    ((env (kernel-standard-env)))
+    (ks-eval-in
+      "($define! factorial ($lambda (n) ($if (<=? n 1) 1 (* n (factorial (- n 1))))))"
+      env)
+    (ks-eval-in "(factorial 5)" env))
+  120)
+
+(ks-test
+  "factorial: 0! = 1"
+  (let
+    ((env (kernel-standard-env)))
+    (ks-eval-in
+      "($define! factorial ($lambda (n) ($if (<=? n 1) 1 (* n (factorial (- n 1))))))"
+      env)
+    (ks-eval-in "(factorial 0)" env))
+  1)
+
+(ks-test
+  "factorial: 10!"
+  (let
+    ((env (kernel-standard-env)))
+    (ks-eval-in
+      "($define! factorial ($lambda (n) ($if (<=? n 1) 1 (* n (factorial (- n 1))))))"
+      env)
+    (ks-eval-in "(factorial 10)" env))
+  3628800)
+
+;; ── recursive list operations ────────────────────────────────────
+(ks-test
+  "sum: recursive over list"
+  (let
+    ((env (kernel-standard-env)))
+    (ks-eval-in
+      "($define! sum ($lambda (xs) ($if (null? xs) 0 (+ (car xs) (sum (cdr xs))))))"
+      env)
+    (ks-eval-in "(sum (list 1 2 3 4 5))" env))
+  15)
+
+(ks-test
+  "len: recursive count"
+  (let
+    ((env (kernel-standard-env)))
+    (ks-eval-in
+      "($define! mylen ($lambda (xs) ($if (null? xs) 0 (+ 1 (mylen (cdr xs))))))"
+      env)
+    (ks-eval-in "(mylen (list 1 2 3 4))" env))
+  4)
+
+(ks-test
+  "map-add1: build new list"
+  (let
+    ((env (kernel-standard-env)))
+    (ks-eval-in
+      "($define! add1-all ($lambda (xs) ($if (null? xs) (list) (cons (+ 1 (car xs)) (add1-all (cdr xs))))))"
+      env)
+    (ks-eval-in "(add1-all (list 10 20 30))" env))
+  (list 11 21 31))
+
+;; ── eval as a first-class applicative ────────────────────────────
+(ks-test
+  "eval: applies to constructed form"
+  (ks-eval "(eval (list ($quote +) 2 3) (get-current-environment))")
+  5)
+
+(ks-test
+  "eval: with a fresh make-environment"
+  (guard
+    (e (true :raised))
+    (ks-eval "(eval ($quote (+ 1 2)) (make-environment))"))
+  :raised)
+
+(ks-test
+  "eval: in extended env sees parent's bindings"
+  (let
+    ((env (kernel-standard-env)))
+    (ks-eval-in "($define! shared 7)" env)
+    (ks-eval-in
+      "(eval ($quote shared) (make-environment (get-current-environment)))"
+      env))
+  7)
+
+;; ── get-current-environment ──────────────────────────────────────
+(ks-test
+  "get-current-environment: returns env"
+  (kernel-env? (ks-eval "(get-current-environment)"))
+  true)
+
+(ks-test
+  "get-current-environment: contains $if"
+  (let
+    ((env (ks-eval "(get-current-environment)")))
+    (kernel-env-has? env "$if"))
+  true)
+
+(ks-test
+  "make-environment: empty"
+  (let ((env (ks-eval "(make-environment)"))) (kernel-env-has? env "$if"))
+  false)
+
+(ks-test
+  "make-environment: child sees parent"
+  (let
+    ((env (kernel-standard-env)))
+    (ks-eval-in "($define! marker 123)" env)
+    (let
+      ((child (ks-eval-in "(make-environment (get-current-environment))" env)))
+      (kernel-env-has? child "marker")))
+  true)
+
+;; ── closures and lexical scope ───────────────────────────────────
+(ks-test
+  "closure: captures binding"
+  (let
+    ((env (kernel-standard-env)))
+    (ks-eval-in
+      "($define! make-adder ($lambda (n) ($lambda (x) (+ x n))))"
+      env)
+    (ks-eval-in "($define! add5 (make-adder 5))" env)
+    (ks-eval-in "(add5 10)" env))
+  15)
+
+(ks-test
+  "closure: nested lookups"
+  (let
+    ((env (kernel-standard-env)))
+    (ks-eval-in
+      "($define! curry-add ($lambda (a) ($lambda (b) ($lambda (c) (+ a (+ b c))))))"
+      env)
+    (ks-eval-in "(((curry-add 1) 2) 3)" env))
+  6)
+
+;; ── operative defined in standard env can reach $define! ─────────
+(ks-test
+  "custom: define-via-vau"
+  (let
+    ((env (kernel-standard-env)))
+    (ks-eval-in
+      "($define! $let-it ($vau (name expr) e ($sequence ($define! tmp (eval expr e)) (eval (list ($quote $define!) name (list ($quote $quote) tmp)) e) tmp)))"
+      env)
+    (ks-eval-in "($let-it z 77)" env)
+    (ks-eval-in "z" env))
+  77)
+
+;; ── quasiquote ──────────────────────────────────────────────────
+(ks-test "qq: plain atom"      (ks-eval "`hello") "hello")
+(ks-test "qq: plain list"      (ks-eval "`(a b c)") (list "a" "b" "c"))
+(ks-test "qq: unquote splices value"
+  (let ((env (kernel-standard-env)))
+    (ks-eval-in "($define! x 42)" env)
+    (ks-eval-in "`(a ,x b)" env)) (list "a" 42 "b"))
+(ks-test "qq: unquote-splicing splices list"
+  (let ((env (kernel-standard-env)))
+    (ks-eval-in "($define! xs (list 1 2 3))" env)
+    (ks-eval-in "`(a ,@xs b)" env)) (list "a" 1 2 3 "b"))
+(ks-test "qq: unquote-splicing at end"
+  (let ((env (kernel-standard-env)))
+    (ks-eval-in "($define! xs (list 9 8))" env)
+    (ks-eval-in "`(a b ,@xs)" env)) (list "a" "b" 9 8))
+(ks-test "qq: unquote-splicing at start"
+  (let ((env (kernel-standard-env)))
+    (ks-eval-in "($define! xs (list 1 2))" env)
+    (ks-eval-in "`(,@xs c)" env)) (list 1 2 "c"))
+(ks-test "qq: nested list with unquote inside"
+  (let ((env (kernel-standard-env)))
+    (ks-eval-in "($define! x 5)" env)
+    (ks-eval-in "`(a (b ,x) c)" env))
+  (list "a" (list "b" 5) "c"))
+(ks-test "qq: error on bare unquote-splicing into non-list"
+  (let ((env (kernel-standard-env)))
+    (ks-eval-in "($define! x 42)" env)
+    (guard (e (true :raised))
+      (ks-eval-in "`(a ,@x b)" env)))
+  :raised)
+
+;; ── $cond / $when / $unless ─────────────────────────────────────
+(ks-test "cond: first match"
+  (ks-eval "($cond (#f 1) (#t 2) (#t 3))") 2)
+(ks-test "cond: else fallback"
+  (ks-eval "($cond (#f 1) (else 99))") 99)
+(ks-test "cond: no match returns nil"
+  (ks-eval "($cond (#f 1) (#f 2))") nil)
+(ks-test "cond: empty clauses returns nil"
+  (ks-eval "($cond)") nil)
+(ks-test "cond: multi-expr body"
+  (ks-eval "($cond (#t 1 2 3))") 3)
+(ks-test "cond: doesn't evaluate untaken clauses"
+  ;; If the second clause's test were evaluated, the unbound `nope` would error.
+  (ks-eval "($cond (#t 7) (nope ignored))") 7)
+(ks-test "cond: predicate evaluation"
+  (let ((env (kernel-standard-env)))
+    (ks-eval-in "($define! n 5)" env)
+    (ks-eval-in "($cond ((< n 0) ($quote negative)) ((= n 0) ($quote zero)) (else ($quote positive)))" env))
+  "positive")
+
+(ks-test "when: true runs body"
+  (ks-eval "($when #t 1 2 3)") 3)
+(ks-test "when: false returns nil"
+  (ks-eval "($when #f 1 2 3)") nil)
+(ks-test "when: skips body when false"
+  (ks-eval "($when #f nope)") nil)
+
+(ks-test "unless: false runs body"
+  (ks-eval "($unless #f 99)") 99)
+(ks-test "unless: true returns nil"
+  (ks-eval "($unless #t 99)") nil)
+(ks-test "unless: skips body when true"
+  (ks-eval "($unless #t nope)") nil)
+
+;; ── $and? / $or? short-circuit ──────────────────────────────────
+(ks-test "and: empty returns true"   (ks-eval "($and?)") true)
+(ks-test "and: single returns value" (ks-eval "($and? 42)") 42)
+(ks-test "and: all true returns last"
+  (ks-eval "($and? 1 2 3)") 3)
+(ks-test "and: first false short-circuits"
+  (ks-eval "($and? #f nope)") false)
+(ks-test "and: false in middle short-circuits"
+  (ks-eval "($and? 1 #f nope)") false)
+(ks-test "or: empty returns false"  (ks-eval "($or?)") false)
+(ks-test "or: single returns value" (ks-eval "($or? 42)") 42)
+(ks-test "or: first truthy short-circuits"
+  (ks-eval "($or? 99 nope)") 99)
+(ks-test "or: all false returns last"
+  (ks-eval "($or? #f #f #f)") false)
+(ks-test "or: middle truthy"
+  (ks-eval "($or? #f 42 nope)") 42)
+
+;; ── variadic arithmetic ─────────────────────────────────────────
+(ks-test "+: zero args = 0"    (ks-eval "(+)") 0)
+(ks-test "+: one arg = arg"    (ks-eval "(+ 7)") 7)
+(ks-test "+: two args"         (ks-eval "(+ 3 4)") 7)
+(ks-test "+: five args"        (ks-eval "(+ 1 2 3 4 5)") 15)
+
+(ks-test "*: zero args = 1"    (ks-eval "(*)") 1)
+(ks-test "*: one arg"          (ks-eval "(* 7)") 7)
+(ks-test "*: four args"        (ks-eval "(* 1 2 3 4)") 24)
+
+(ks-test "-: one arg negates"  (ks-eval "(- 10)") -10)
+(ks-test "-: two args"         (ks-eval "(- 10 3)") 7)
+(ks-test "-: four args fold"   (ks-eval "(- 100 1 2 3)") 94)
+
+(ks-test "/: two args"         (ks-eval "(/ 20 5)") 4)
+(ks-test "/: three args fold"  (ks-eval "(/ 100 2 5)") 10)
+
+;; ── variadic chained comparison ─────────────────────────────────
+(ks-test "<: chained ascending"  (ks-eval "(< 1 2 3 4 5)") true)
+(ks-test "<: not strict"          (ks-eval "(< 1 2 2 3)") false)
+(ks-test "<: anti-monotonic"      (ks-eval "(< 5 3)") false)
+(ks-test ">: chained descending" (ks-eval "(> 5 4 3 2 1)") true)
+(ks-test "<=? ascending equals"   (ks-eval "(<=? 1 1 2 3 3)") true)
+(ks-test "<=? violation"          (ks-eval "(<=? 1 2 1)") false)
+(ks-test ">=? descending equals" (ks-eval "(>=? 3 3 2 1)") true)
+
+;; ── list combinators ────────────────────────────────────────────
+(ks-test "map: square"
+  (ks-eval "(map ($lambda (x) (* x x)) (list 1 2 3 4))")
+  (list 1 4 9 16))
+(ks-test "map: empty list"
+  (ks-eval "(map ($lambda (x) x) (list))") (list))
+(ks-test "map: identity preserves"
+  (ks-eval "(map ($lambda (x) x) (list 1 2 3))") (list 1 2 3))
+(ks-test "map: with closure over outer"
+  (let ((env (kernel-standard-env)))
+    (ks-eval-in "($define! k 10)" env)
+    (ks-eval-in "(map ($lambda (x) (+ x k)) (list 1 2 3))" env))
+  (list 11 12 13))
+
+(ks-test "filter: positives"
+  (ks-eval "(filter ($lambda (x) (< 0 x)) (list -2 -1 0 1 2))")
+  (list 1 2))
+(ks-test "filter: empty result"
+  (ks-eval "(filter ($lambda (x) #f) (list 1 2 3))") (list))
+(ks-test "filter: all match"
+  (ks-eval "(filter ($lambda (x) #t) (list 1 2 3))") (list 1 2 3))
+
+(ks-test "reduce: sum"
+  (ks-eval "(reduce ($lambda (a b) (+ a b)) 0 (list 1 2 3 4 5))") 15)
+(ks-test "reduce: product"
+  (ks-eval "(reduce ($lambda (a b) (* a b)) 1 (list 1 2 3 4))") 24)
+(ks-test "reduce: empty returns init"
+  (ks-eval "(reduce ($lambda (a b) (+ a b)) 42 (list))") 42)
+(ks-test "reduce: build list"
+  (ks-eval "(reduce ($lambda (acc x) (cons x acc)) () (list 1 2 3))")
+  (list 3 2 1))
+
+;; ── apply ────────────────────────────────────────────────────────
+(ks-test "apply: + over list"
+  (ks-eval "(apply + (list 1 2 3 4 5))") 15)
+(ks-test "apply: lambda"
+  (ks-eval "(apply ($lambda (a b c) (* a (+ b c))) (list 2 3 4))") 14)
+(ks-test "apply: list identity"
+  (ks-eval "(apply list (list 1 2 3))") (list 1 2 3))
+(ks-test "apply: empty args list"
+  (ks-eval "(apply + (list))") 0)
+(ks-test "apply: single arg list"
+  (ks-eval "(apply ($lambda (x) (* x 10)) (list 7))") 70)
+(ks-test "apply: built via map+apply"
+  ;; (apply + (map ($lambda (x) (* x x)) (list 1 2 3))) → 1+4+9 = 14
+  (ks-eval
+    "(apply + (map ($lambda (x) (* x x)) (list 1 2 3)))") 14)
+(ks-test "apply: error on non-list args"
+  (guard (e (true :raised))
+    (ks-eval "(apply + 5)"))
+  :raised)
+
+;; ── append / reverse ────────────────────────────────────────────
+(ks-test "append: two lists"
+  (ks-eval "(append (list 1 2) (list 3 4))") (list 1 2 3 4))
+(ks-test "append: three lists"
+  (ks-eval "(append (list 1) (list 2) (list 3))") (list 1 2 3))
+(ks-test "append: empty list"
+  (ks-eval "(append)") (list))
+(ks-test "append: one list"
+  (ks-eval "(append (list 1 2 3))") (list 1 2 3))
+(ks-test "append: empty + nonempty"
+  (ks-eval "(append (list) (list 1 2))") (list 1 2))
+(ks-test "append: nonempty + empty"
+  (ks-eval "(append (list 1 2) (list))") (list 1 2))
+(ks-test "append: error on non-list"
+  (guard (e (true :raised))
+    (ks-eval "(append (list 1) 5)"))
+  :raised)
+
+(ks-test "reverse: four elements"
+  (ks-eval "(reverse (list 1 2 3 4))") (list 4 3 2 1))
+(ks-test "reverse: empty"
+  (ks-eval "(reverse (list))") (list))
+(ks-test "reverse: single"
+  (ks-eval "(reverse (list 99))") (list 99))
+(ks-test "reverse: double reverse is identity"
+  (ks-eval "(reverse (reverse (list 1 2 3)))") (list 1 2 3))
+
+(define ks-tests-run! (fn () (refl-test-report ks-suite)))

lib/kernel/tests/vau.sx

@@ -0,0 +1,297 @@
+;; lib/kernel/tests/vau.sx — exercises lib/kernel/runtime.sx.
+;;
+;; Verifies the Phase 3 promise: user-defined operatives and applicatives
+;; constructible from inside the language. Tests build a Kernel
+;; base-env, bind a few helper applicatives (+, *, list, =, $if), and
+;; run programs that construct and use custom combiners.
+
+(define kv-suite (refl-make-test-suite))
+(define kv-test  (fn (n a e) (refl-test kv-suite n a e)))
+
+(define kv-eval-src (fn (src env) (kernel-eval (kernel-parse src) env)))
+
+(define
+  kv-make-env
+  (fn
+    ()
+    (let
+      ((env (kernel-base-env)))
+      (kernel-env-bind!
+        env
+        "+"
+        (kernel-make-primitive-applicative
+          (fn (args) (+ (first args) (nth args 1)))))
+      (kernel-env-bind!
+        env
+        "*"
+        (kernel-make-primitive-applicative
+          (fn (args) (* (first args) (nth args 1)))))
+      (kernel-env-bind!
+        env
+        "-"
+        (kernel-make-primitive-applicative
+          (fn (args) (- (first args) (nth args 1)))))
+      (kernel-env-bind!
+        env
+        "="
+        (kernel-make-primitive-applicative
+          (fn (args) (= (first args) (nth args 1)))))
+      (kernel-env-bind!
+        env
+        "list"
+        (kernel-make-primitive-applicative (fn (args) args)))
+      (kernel-env-bind!
+        env
+        "cons"
+        (kernel-make-primitive-applicative
+          (fn (args) (cons (first args) (nth args 1)))))
+      (kernel-env-bind!
+        env
+        "$quote"
+        (kernel-make-primitive-operative (fn (args dyn-env) (first args))))
+      (kernel-env-bind!
+        env
+        "$if"
+        (kernel-make-primitive-operative
+          (fn
+            (args dyn-env)
+            (if
+              (kernel-eval (first args) dyn-env)
+              (kernel-eval (nth args 1) dyn-env)
+              (kernel-eval (nth args 2) dyn-env)))))
+      env)))
+
+;; ── $vau: builds an operative ───────────────────────────────────
+(kv-test
+  "vau: identity returns first arg unevaluated"
+  (kv-eval-src "(($vau (a) _ a) hello)" (kv-make-env))
+  "hello")
+
+(kv-test
+  "vau: returns args as raw expressions"
+  (kv-eval-src "(($vau (a b) _ (list a b)) (+ 1 2) (+ 3 4))" (kv-make-env))
+  (list (list "+" 1 2) (list "+" 3 4)))
+
+(kv-test
+  "vau: env-param is a kernel env"
+  (kernel-env? (kv-eval-src "(($vau () e e))" (kv-make-env)))
+  true)
+
+(kv-test
+  "vau: returns operative"
+  (kernel-operative? (kv-eval-src "($vau (x) _ x)" (kv-make-env)))
+  true)
+
+(kv-test
+  "vau: returns operative not applicative"
+  (kernel-applicative? (kv-eval-src "($vau (x) _ x)" (kv-make-env)))
+  false)
+
+(kv-test
+  "vau: zero-arg body"
+  (kv-eval-src "(($vau () _ 42))" (kv-make-env))
+  42)
+
+(kv-test
+  "vau: static-env closure captured"
+  (let
+    ((outer (kv-make-env)))
+    (kernel-env-bind! outer "captured" 17)
+    (let
+      ((op (kv-eval-src "($vau () _ captured)" outer))
+        (caller (kv-make-env)))
+      (kernel-env-bind! caller "captured" 99)
+      (kernel-combine op (list) caller)))
+  17)
+
+(kv-test
+  "vau: env-param exposes caller's dynamic env"
+  (let
+    ((outer (kv-make-env)))
+    (kernel-env-bind! outer "x" 1)
+    (let
+      ((op (kv-eval-src "($vau () e e)" outer)) (caller (kv-make-env)))
+      (kernel-env-bind! caller "x" 2)
+      (let
+        ((e-val (kernel-combine op (list) caller)))
+        (kernel-env-lookup e-val "x"))))
+  2)
+
+;; ── $lambda: applicatives evaluate their args ───────────────────
+(kv-test
+  "lambda: identity"
+  (kv-eval-src "(($lambda (x) x) 42)" (kv-make-env))
+  42)
+
+(kv-test
+  "lambda: addition"
+  (kv-eval-src "(($lambda (x y) (+ x y)) 3 4)" (kv-make-env))
+  7)
+
+(kv-test
+  "lambda: args are evaluated before bind"
+  (kv-eval-src "(($lambda (x) x) (+ 2 3))" (kv-make-env))
+  5)
+
+(kv-test
+  "lambda: zero args"
+  (kv-eval-src "(($lambda () 99))" (kv-make-env))
+  99)
+
+(kv-test
+  "lambda: returns applicative"
+  (kernel-applicative? (kv-eval-src "($lambda (x) x)" (kv-make-env)))
+  true)
+
+(kv-test
+  "lambda: returns applicative not operative"
+  (kernel-operative? (kv-eval-src "($lambda (x) x)" (kv-make-env)))
+  false)
+
+(kv-test
+  "lambda: higher-order"
+  (kv-eval-src "(($lambda (f) (f 10)) ($lambda (x) (+ x 1)))" (kv-make-env))
+  11)
+
+;; ── wrap / unwrap as user-callable applicatives ─────────────────
+
+(kv-test
+  "wrap: makes applicative from operative"
+  (kernel-applicative? (kv-eval-src "(wrap ($vau (x) _ x))" (kv-make-env)))
+  true)
+
+(kv-test
+  "wrap: result evaluates its arg"
+  (kv-eval-src "((wrap ($vau (x) _ x)) (+ 1 2))" (kv-make-env))
+  3)
+
+(kv-test
+  "unwrap: extracts operative from applicative"
+  (kernel-operative? (kv-eval-src "(unwrap ($lambda (x) x))" (kv-make-env)))
+  true)
+
+(kv-test
+  "wrap/unwrap roundtrip preserves identity"
+  (kv-eval-src
+    "(($lambda (op) (= op (unwrap (wrap op)))) ($vau (x) _ x))"
+    (kv-make-env))
+  true)
+
+;; ── operative? / applicative? as user-visible predicates ────────
+
+(kv-test
+  "operative? on vau result"
+  (kv-eval-src "(operative? ($vau (x) _ x))" (kv-make-env))
+  true)
+
+(kv-test
+  "operative? on lambda result"
+  (kv-eval-src "(operative? ($lambda (x) x))" (kv-make-env))
+  false)
+
+(kv-test
+  "applicative? on lambda result"
+  (kv-eval-src "(applicative? ($lambda (x) x))" (kv-make-env))
+  true)
+
+(kv-test
+  "applicative? on vau result"
+  (kv-eval-src "(applicative? ($vau (x) _ x))" (kv-make-env))
+  false)
+
+(kv-test
+  "operative? on number"
+  (kv-eval-src "(operative? 42)" (kv-make-env))
+  false)
+
+;; ── Build BOTH layers from user code ────────────────────────────
+;; The headline Phase 3 test: defining an operative on top of an
+;; applicative defined on top of a vau.
+
+(kv-test
+  "custom: applicative + operative compose"
+  (let
+    ((env (kv-make-env)))
+    (kernel-env-bind! env "square" (kv-eval-src "($lambda (x) (* x x))" env))
+    (kv-eval-src "(square 4)" env))
+  16)
+
+(kv-test "custom: operative captures argument syntax"
+  ;; ($capture x) returns the raw expression `x`, regardless of value.
+  (let ((env (kv-make-env)))
+    (kernel-env-bind! env "$capture"
+      (kv-eval-src "($vau (form) _ form)" env))
+    (kv-eval-src "($capture (+ 1 2))" env))
+  (list "+" 1 2))
+
+(kv-test "custom: applicative re-wraps an operative"
+  ;; Build a captured operative, then wrap it into an applicative that
+  ;; evaluates args before re-entry. This exercises wrap+$vau composed.
+  (let ((env (kv-make-env)))
+    (kernel-env-bind! env "id-app"
+      (kv-eval-src "(wrap ($vau (x) _ x))" env))
+    (kv-eval-src "(id-app (+ 10 20))" env))
+  30)
+
+;; ── Error cases ──────────────────────────────────────────────────
+
+(kv-test
+  "vau: rejects non-list formals"
+  (guard (e (true :raised)) (kv-eval-src "($vau x _ x)" (kv-make-env)))
+  :raised)
+
+(kv-test
+  "vau: rejects non-symbol formal"
+  (guard (e (true :raised)) (kv-eval-src "($vau (1) _ x)" (kv-make-env)))
+  :raised)
+
+(kv-test
+  "vau: rejects non-symbol env-param"
+  (guard (e (true :raised)) (kv-eval-src "($vau (x) 7 x)" (kv-make-env)))
+  :raised)
+
+(kv-test
+  "vau: too few args at call site"
+  (guard
+    (e (true :raised))
+    (kv-eval-src "(($vau (x y) _ x) 1)" (kv-make-env)))
+  :raised)
+
+(kv-test
+  "vau: too many args at call site"
+  (guard
+    (e (true :raised))
+    (kv-eval-src "(($vau (x) _ x) 1 2)" (kv-make-env)))
+  :raised)
+
+(kv-test
+  "wrap: rejects non-operative"
+  (guard (e (true :raised)) (kv-eval-src "(wrap 42)" (kv-make-env)))
+  :raised)
+
+(kv-test
+  "unwrap: rejects non-applicative"
+  (guard (e (true :raised)) (kv-eval-src "(unwrap 42)" (kv-make-env)))
+  :raised)
+
+;; ── Multi-expression body (implicit $sequence) ──────────────────
+
+(kv-test "lambda: two body forms — value of last"
+  (kv-eval-src "(($lambda (n) (+ n 1) (+ n 10)) 5)" (kv-make-env)) 15)
+
+(kv-test "lambda: three body forms"
+  (kv-eval-src "(($lambda (n) n (+ n 1) (+ n 2)) 10)" (kv-make-env)) 12)
+
+(kv-test "vau: two body forms"
+  (kv-eval-src "(($vau (a b) _ a (list a b)) 7 8)" (kv-make-env))
+  (list 7 8))
+
+(kv-test "lambda: $define! in early body visible in later body"
+  (kv-eval-src
+    "(($lambda (n) ($define! double (+ n n)) double) 6)"
+    (kv-make-env)) 12)
+
+(kv-test "lambda: zero-arg multi-body"
+  (kv-eval-src "(($lambda () 1 2 3))" (kv-make-env)) 3)
+
+(define kv-tests-run! (fn () (refl-test-report kv-suite)))

lib/lua/conformance.py

@@ -0,0 +1,348 @@
+#!/usr/bin/env python3
+"""lua-conformance — run the PUC-Rio Lua 5.1 test suite against Lua-on-SX.
+
+Walks lib/lua/lua-tests/*.lua, evaluates each via `lua-eval-ast` on a
+long-lived sx_server.exe subprocess, classifies pass/fail/timeout per file,
+and writes lib/lua/scoreboard.{json,md}.
+
+Modelled on lib/js/test262-runner.py but much simpler: each Lua test file is
+its own unit (they're self-contained assertion scripts; they pass if they
+complete without raising). No harness stub, no frontmatter, no worker pool.
+
+Usage:
+    python3 lib/lua/conformance.py
+    python3 lib/lua/conformance.py --filter locals
+    python3 lib/lua/conformance.py --per-test-timeout 3 -v
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import select
+import subprocess
+import sys
+import time
+from collections import Counter
+from pathlib import Path
+
+REPO = Path(__file__).resolve().parents[2]
+SX_SERVER_PRIMARY = REPO / "hosts" / "ocaml" / "_build" / "default" / "bin" / "sx_server.exe"
+SX_SERVER_FALLBACK = Path("/root/rose-ash/hosts/ocaml/_build/default/bin/sx_server.exe")
+TESTS_DIR = REPO / "lib" / "lua" / "lua-tests"
+
+DEFAULT_TIMEOUT = 8.0
+
+# Files that require facilities we don't (and won't soon) support.
+# Still classified as skip rather than fail so the scoreboard stays honest.
+HARDCODED_SKIP = {
+    "all.lua":        "driver uses dofile to chain other tests",
+    "api.lua":        "requires testC (C debug library)",
+    "checktable.lua": "internal debug helpers",
+    "code.lua":       "bytecode inspection via debug library",
+    "db.lua":         "debug library",
+    "files.lua":      "io library",
+    "gc.lua":         "collectgarbage / finalisers",
+    "main.lua":       "standalone interpreter driver",
+}
+
+RX_OK_INLINE = re.compile(r"^\(ok (\d+) (.*)\)\s*$")
+RX_OK_LEN = re.compile(r"^\(ok-len (\d+) \d+\)\s*$")
+RX_ERR = re.compile(r"^\(error (\d+) (.*)\)\s*$")
+
+
+def pick_sx_server() -> Path:
+    if SX_SERVER_PRIMARY.exists():
+        return SX_SERVER_PRIMARY
+    return SX_SERVER_FALLBACK
+
+
+def sx_escape_nested(s: str) -> str:
+    """Two-level escape: (eval "(lua-eval-ast \"<src>\")").
+
+    Outer literal is consumed by `eval` then the inner literal by `lua-eval-ast`.
+    """
+    inner = (
+        s.replace("\\", "\\\\")
+        .replace('"', '\\"')
+        .replace("\n", "\\n")
+        .replace("\r", "\\r")
+        .replace("\t", "\\t")
+    )
+    return inner.replace("\\", "\\\\").replace('"', '\\"')
+
+
+def classify_error(msg: str) -> str:
+    m = msg.lower()
+    sym = re.search(r"undefined symbol:\s*\\?\"?([^\"\s)]+)", msg, re.I)
+    if sym:
+        return f"undefined symbol: {sym.group(1).strip(chr(34))}"
+    if "undefined symbol" in m:
+        return "undefined symbol"
+    if "lua: arith" in m:
+        return "arith type error"
+    if "lua-transpile" in m:
+        return "transpile: unsupported node"
+    if "lua-parse" in m:
+        return "parse error"
+    if "lua-tokenize" in m:
+        return "tokenize error"
+    if "unknown node" in m:
+        return "unknown AST node"
+    if "not yet supported" in m:
+        return "not yet supported"
+    if "nth: index out" in m or "nth:" in m:
+        return "nth index error"
+    if "timeout" in m:
+        return "timeout"
+    # Strip SX-side wrapping and trim
+    trimmed = msg.strip('"').strip()
+    return f"other: {trimmed[:80]}"
+
+
+class Session:
+    def __init__(self, sx_server: Path, timeout: float):
+        self.sx_server = sx_server
+        self.timeout = timeout
+        self.proc: subprocess.Popen | None = None
+        self._buf = b""
+        self._fd = -1
+
+    def start(self) -> None:
+        self.proc = subprocess.Popen(
+            [str(self.sx_server)],
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.DEVNULL,
+            cwd=str(REPO),
+            bufsize=0,
+        )
+        self._fd = self.proc.stdout.fileno()
+        self._buf = b""
+        os.set_blocking(self._fd, False)
+        self._wait_for("(ready)", timeout=15.0)
+        self._run(1, '(load "lib/lua/tokenizer.sx")', 60)
+        self._run(2, '(load "lib/lua/parser.sx")', 60)
+        self._run(3, '(load "lib/lua/runtime.sx")', 60)
+        self._run(4, '(load "lib/lua/transpile.sx")', 60)
+
+    def stop(self) -> None:
+        if self.proc is None:
+            return
+        try:
+            self.proc.stdin.close()
+        except Exception:
+            pass
+        try:
+            self.proc.terminate()
+            self.proc.wait(timeout=3)
+        except Exception:
+            try:
+                self.proc.kill()
+            except Exception:
+                pass
+        self.proc = None
+
+    def _readline(self, timeout: float) -> str | None:
+        deadline = time.monotonic() + timeout
+        while True:
+            nl = self._buf.find(b"\n")
+            if nl >= 0:
+                line = self._buf[: nl + 1]
+                self._buf = self._buf[nl + 1 :]
+                return line.decode("utf-8", errors="replace")
+            remaining = deadline - time.monotonic()
+            if remaining <= 0:
+                raise TimeoutError("readline timeout")
+            try:
+                rlist, _, _ = select.select([self._fd], [], [], remaining)
+            except (OSError, ValueError):
+                return None
+            if not rlist:
+                raise TimeoutError("readline timeout")
+            try:
+                chunk = os.read(self._fd, 65536)
+            except (BlockingIOError, InterruptedError):
+                continue
+            except OSError:
+                return None
+            if not chunk:
+                if self._buf:
+                    rv = self._buf.decode("utf-8", errors="replace")
+                    self._buf = b""
+                    return rv
+                return None
+            self._buf += chunk
+
+    def _wait_for(self, token: str, timeout: float) -> None:
+        start = time.monotonic()
+        while time.monotonic() - start < timeout:
+            line = self._readline(timeout - (time.monotonic() - start))
+            if line is None:
+                raise RuntimeError("sx_server closed stdout before ready")
+            if token in line:
+                return
+        raise TimeoutError(f"timeout waiting for {token}")
+
+    def _run(self, epoch: int, cmd: str, timeout: float):
+        payload = f"(epoch {epoch})\n{cmd}\n".encode("utf-8")
+        try:
+            self.proc.stdin.write(payload)
+            self.proc.stdin.flush()
+        except (BrokenPipeError, OSError):
+            raise RuntimeError("sx_server stdin closed")
+        deadline = time.monotonic() + timeout
+        while time.monotonic() < deadline:
+            remaining = deadline - time.monotonic()
+            if remaining <= 0:
+                raise TimeoutError(f"epoch {epoch} timeout")
+            line = self._readline(remaining)
+            if line is None:
+                raise RuntimeError("sx_server closed stdout mid-epoch")
+            m = RX_OK_INLINE.match(line)
+            if m and int(m.group(1)) == epoch:
+                return "ok", m.group(2)
+            m = RX_OK_LEN.match(line)
+            if m and int(m.group(1)) == epoch:
+                val = self._readline(deadline - time.monotonic()) or ""
+                return "ok", val.rstrip("\n")
+            m = RX_ERR.match(line)
+            if m and int(m.group(1)) == epoch:
+                return "error", m.group(2)
+        raise TimeoutError(f"epoch {epoch} timeout")
+
+    def run_lua(self, epoch: int, src: str):
+        escaped = sx_escape_nested(src)
+        cmd = f'(eval "(lua-eval-ast \\"{escaped}\\")")'
+        return self._run(epoch, cmd, self.timeout)
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--per-test-timeout", type=float, default=DEFAULT_TIMEOUT)
+    ap.add_argument("--filter", type=str, default=None,
+                    help="only run tests whose filename contains this substring")
+    ap.add_argument("-v", "--verbose", action="store_true")
+    ap.add_argument("--no-scoreboard", action="store_true",
+                    help="do not write scoreboard.{json,md}")
+    args = ap.parse_args()
+
+    sx_server = pick_sx_server()
+    if not sx_server.exists():
+        print(f"ERROR: sx_server not found at {sx_server}", file=sys.stderr)
+        return 1
+    if not TESTS_DIR.exists():
+        print(f"ERROR: no tests dir at {TESTS_DIR}", file=sys.stderr)
+        return 1
+
+    tests = sorted(TESTS_DIR.glob("*.lua"))
+    if args.filter:
+        tests = [p for p in tests if args.filter in p.name]
+    if not tests:
+        print("No tests matched.", file=sys.stderr)
+        return 1
+
+    print(f"Running {len(tests)} Lua test file(s)…", file=sys.stderr)
+    session = Session(sx_server, args.per_test_timeout)
+    session.start()
+
+    results = []
+    failure_modes: Counter = Counter()
+
+    try:
+        for i, path in enumerate(tests, start=1):
+            name = path.name
+            skip_reason = HARDCODED_SKIP.get(name)
+            if skip_reason:
+                results.append({"name": name, "status": "skip", "reason": skip_reason, "ms": 0})
+                if args.verbose:
+                    print(f"  - {name}: SKIP ({skip_reason})")
+                continue
+
+            try:
+                src = path.read_text(encoding="utf-8")
+            except UnicodeDecodeError:
+                src = path.read_text(encoding="latin-1")
+            t0 = time.monotonic()
+            try:
+                kind, payload = session.run_lua(100 + i, src)
+                ms = int((time.monotonic() - t0) * 1000)
+                if kind == "ok":
+                    results.append({"name": name, "status": "pass", "reason": "", "ms": ms})
+                    if args.verbose:
+                        print(f"  + {name}: PASS ({ms}ms)")
+                else:
+                    reason = classify_error(payload)
+                    failure_modes[reason] += 1
+                    results.append({"name": name, "status": "fail", "reason": reason, "ms": ms})
+                    if args.verbose:
+                        print(f"  - {name}: FAIL — {reason}")
+            except TimeoutError:
+                ms = int((time.monotonic() - t0) * 1000)
+                failure_modes["timeout"] += 1
+                results.append({"name": name, "status": "timeout", "reason": "per-test timeout",
+                                "ms": ms})
+                if args.verbose:
+                    print(f"  - {name}: TIMEOUT ({ms}ms)")
+                # Restart after a timeout to shed any stuck state.
+                session.stop()
+                session.start()
+    finally:
+        session.stop()
+
+    n_pass = sum(1 for r in results if r["status"] == "pass")
+    n_fail = sum(1 for r in results if r["status"] == "fail")
+    n_timeout = sum(1 for r in results if r["status"] == "timeout")
+    n_skip = sum(1 for r in results if r["status"] == "skip")
+    n_total = len(results)
+    n_runnable = n_total - n_skip
+    pct = (n_pass / n_runnable * 100.0) if n_runnable else 0.0
+
+    print()
+    print(f"Lua-on-SX conformance: {n_pass}/{n_runnable} runnable pass ({pct:.1f}%)  "
+          f"fail={n_fail} timeout={n_timeout} skip={n_skip} total={n_total}")
+    if failure_modes:
+        print("Top failure modes:")
+        for mode, count in failure_modes.most_common(10):
+            print(f"  {count}x  {mode}")
+
+    if not args.no_scoreboard:
+        sb = {
+            "totals": {
+                "pass": n_pass, "fail": n_fail, "timeout": n_timeout,
+                "skip": n_skip, "total": n_total, "runnable": n_runnable,
+                "pass_rate": round(pct, 1),
+            },
+            "top_failure_modes": failure_modes.most_common(20),
+            "results": results,
+        }
+        (REPO / "lib" / "lua" / "scoreboard.json").write_text(
+            json.dumps(sb, indent=2), encoding="utf-8"
+        )
+        md = [
+            "# Lua-on-SX conformance scoreboard",
+            "",
+            f"**Pass rate:** {n_pass}/{n_runnable} runnable ({pct:.1f}%)",
+            f"fail={n_fail} timeout={n_timeout} skip={n_skip} total={n_total}",
+            "",
+            "## Top failure modes",
+            "",
+        ]
+        for mode, count in failure_modes.most_common(10):
+            md.append(f"- **{count}x** {mode}")
+        md.extend(["", "## Per-test results", "",
+                   "| Test | Status | Reason | ms |",
+                   "|---|---|---|---:|"])
+        for r in results:
+            reason = r["reason"] or "-"
+            md.append(f"| {r['name']} | {r['status']} | {reason} | {r['ms']} |")
+        (REPO / "lib" / "lua" / "scoreboard.md").write_text(
+            "\n".join(md) + "\n", encoding="utf-8"
+        )
+
+    return 0 if (n_fail == 0 and n_timeout == 0) else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

lib/lua/conformance.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+# Lua-on-SX conformance runner — walks lib/lua/lua-tests/*.lua, runs each via
+# `lua-eval-ast` on a long-lived sx_server.exe subprocess, classifies
+# pass/fail/timeout, and writes lib/lua/scoreboard.{json,md}.
+#
+# Usage:
+#   bash lib/lua/conformance.sh               # full suite
+#   bash lib/lua/conformance.sh --filter sort # filter by filename substring
+#   bash lib/lua/conformance.sh -v            # per-file verbose
+
+set -uo pipefail
+cd "$(git rev-parse --show-toplevel)"
+exec python3 lib/lua/conformance.py "$@"

lib/lua/lua-tests/README

@@ -0,0 +1,41 @@
+This tarball contains the official test scripts for Lua 5.1.
+Unlike Lua itself, these tests do not aim portability, small footprint,
+or easy of use. (Their main goal is to try to crash Lua.) They are not
+intended for general use. You are wellcome to use them, but expect to
+have to "dirt your hands".
+
+The tarball should expand in the following contents:
+  - several .lua scripts with the tests
+  - a main "all.lua" Lua script that invokes all the other scripts
+  - a subdirectory "libs" with an empty subdirectory "libs/P1",
+    to be used by the scripts
+  - a subdirectory "etc" with some extra files
+
+To run the tests, do as follows:
+
+- go to the test directory
+
+- set LUA_PATH to "?;./?.lua" (or, better yet, set LUA_PATH to "./?.lua;;"
+  and LUA_INIT to "package.path = '?;'..package.path")
+
+- run "lua all.lua"
+
+
+--------------------------------------------
+Internal tests
+--------------------------------------------
+
+Some tests need a special library, "testC", that gives access to
+several internal structures in Lua.
+This library is only available when Lua is compiled in debug mode.
+The scripts automatically detect its absence and skip those tests.
+
+If you want to run these tests, move etc/ltests.c and etc/ltests.h to
+the directory with the source Lua files, and recompile Lua with
+the option -DLUA_USER_H='"ltests.h"' (or its equivalent to define
+LUA_USER_H as the string "ltests.h", including the quotes). This
+option not only adds the testC library, but it adds several other
+internal tests as well. After the recompilation, run the tests
+as before.
+
+

lib/lua/lua-tests/all.lua

@@ -0,0 +1,137 @@
+#!../lua
+
+math.randomseed(0)
+
+collectgarbage("setstepmul", 180)
+collectgarbage("setpause", 190)
+
+
+--[=[
+  example of a long [comment],
+  [[spanning several [lines]]]
+
+]=]
+
+print("current path:\n  " .. string.gsub(package.path, ";", "\n  "))
+
+
+local msgs = {}
+function Message (m)
+  print(m)
+  msgs[#msgs+1] = string.sub(m, 3, -3)
+end
+
+
+local c = os.clock()
+
+assert(os.setlocale"C")
+
+local T,print,gcinfo,format,write,assert,type =
+      T,print,gcinfo,string.format,io.write,assert,type
+
+local function formatmem (m)
+  if m < 1024 then return m
+  else
+    m = m/1024 - m/1024%1
+    if m < 1024 then return m.."K"
+    else
+      m = m/1024 - m/1024%1
+      return m.."M"
+    end
+  end
+end
+
+local showmem = function ()
+  if not T then
+    print(format("    ---- total memory: %s ----\n", formatmem(gcinfo())))
+  else
+    T.checkmemory()
+    local a,b,c = T.totalmem()
+    local d,e = gcinfo()
+    print(format(
+  "\n    ---- total memory: %s (%dK), max use: %s,  blocks: %d\n",
+                        formatmem(a),  d,      formatmem(c),           b))
+  end
+end
+
+
+--
+-- redefine dofile to run files through dump/undump
+--
+dofile = function (n)
+  showmem()
+  local f = assert(loadfile(n))
+  local b = string.dump(f)
+  f = assert(loadstring(b))
+  return f()
+end
+
+dofile('main.lua')
+
+do
+  local u = newproxy(true)
+  local newproxy, stderr = newproxy, io.stderr
+  getmetatable(u).__gc = function (o)
+    stderr:write'.'
+    newproxy(o)
+  end
+end
+
+local f = assert(loadfile('gc.lua'))
+f()
+dofile('db.lua')
+assert(dofile('calls.lua') == deep and deep)
+dofile('strings.lua')
+dofile('literals.lua')
+assert(dofile('attrib.lua') == 27)
+assert(dofile('locals.lua') == 5)
+dofile('constructs.lua')
+dofile('code.lua')
+do
+  local f = coroutine.wrap(assert(loadfile('big.lua')))
+  assert(f() == 'b')
+  assert(f() == 'a')
+end
+dofile('nextvar.lua')
+dofile('pm.lua')
+dofile('api.lua')
+assert(dofile('events.lua') == 12)
+dofile('vararg.lua')
+dofile('closure.lua')
+dofile('errors.lua')
+dofile('math.lua')
+dofile('sort.lua')
+assert(dofile('verybig.lua') == 10); collectgarbage()
+dofile('files.lua')
+
+if #msgs > 0 then
+  print("\ntests not performed:")
+  for i=1,#msgs do
+    print(msgs[i])
+  end
+  print()
+end
+
+print("final OK !!!")
+print('cleaning all!!!!')
+
+debug.sethook(function (a) assert(type(a) == 'string') end, "cr")
+
+local _G, collectgarbage, showmem, print, format, clock =
+      _G, collectgarbage, showmem, print, format, os.clock
+
+local a={}
+for n in pairs(_G) do a[n] = 1 end
+a.tostring = nil
+a.___Glob = nil
+for n in pairs(a) do _G[n] = nil end
+
+a = nil
+collectgarbage()
+collectgarbage()
+collectgarbage()
+collectgarbage()
+collectgarbage()
+collectgarbage();showmem()
+
+print(format("\n\ntotal time: %.2f\n", clock()-c))

lib/lua/lua-tests/api.lua

@@ -0,0 +1,711 @@
+
+if T==nil then
+  (Message or print)('\a\n >>> testC not active: skipping API tests <<<\n\a')
+  return
+end
+
+
+
+function tcheck (t1, t2)
+  table.remove(t1, 1)  -- remove code
+  assert(table.getn(t1) == table.getn(t2))
+  for i=1,table.getn(t1) do assert(t1[i] == t2[i]) end
+end
+
+function pack(...) return arg end
+
+
+print('testing C API')
+
+-- testing allignment
+a = T.d2s(12458954321123)
+assert(string.len(a) == 8)   -- sizeof(double)
+assert(T.s2d(a) == 12458954321123)
+
+a,b,c = T.testC("pushnum 1; pushnum 2; pushnum 3; return 2")
+assert(a == 2 and b == 3 and not c)
+
+-- test that all trues are equal
+a,b,c = T.testC("pushbool 1; pushbool 2; pushbool 0; return 3")
+assert(a == b and a == true and c == false)
+a,b,c = T.testC"pushbool 0; pushbool 10; pushnil;\
+                      tobool -3; tobool -3; tobool -3; return 3"
+assert(a==0 and b==1 and c==0)
+
+
+a,b,c = T.testC("gettop; return 2", 10, 20, 30, 40)
+assert(a == 40 and b == 5 and not c)
+
+t = pack(T.testC("settop 5; gettop; return .", 2, 3))
+tcheck(t, {n=4,2,3})
+
+t = pack(T.testC("settop 0; settop 15; return 10", 3, 1, 23))
+assert(t.n == 10 and t[1] == nil and t[10] == nil)
+
+t = pack(T.testC("remove -2; gettop; return .", 2, 3, 4))
+tcheck(t, {n=2,2,4})
+
+t = pack(T.testC("insert -1; gettop; return .", 2, 3))
+tcheck(t, {n=2,2,3})
+
+t = pack(T.testC("insert 3; gettop; return .", 2, 3, 4, 5))
+tcheck(t, {n=4,2,5,3,4})
+
+t = pack(T.testC("replace 2; gettop; return .", 2, 3, 4, 5))
+tcheck(t, {n=3,5,3,4})
+
+t = pack(T.testC("replace -2; gettop; return .", 2, 3, 4, 5))
+tcheck(t, {n=3,2,3,5})
+
+t = pack(T.testC("remove 3; gettop; return .", 2, 3, 4, 5))
+tcheck(t, {n=3,2,4,5})
+
+t = pack(T.testC("insert 3; pushvalue 3; remove 3; pushvalue 2; remove 2; \
+                  insert 2; pushvalue 1; remove 1; insert 1; \
+      insert -2; pushvalue -2; remove -3; gettop; return .",
+      2, 3, 4, 5, 10, 40, 90))
+tcheck(t, {n=7,2,3,4,5,10,40,90})
+
+t = pack(T.testC("concat 5; gettop; return .", "alo", 2, 3, "joao", 12))
+tcheck(t, {n=1,"alo23joao12"})
+
+-- testing MULTRET
+t = pack(T.testC("rawcall 2,-1; gettop; return .",
+     function (a,b) return 1,2,3,4,a,b end, "alo", "joao"))
+tcheck(t, {n=6,1,2,3,4,"alo", "joao"})
+
+do  -- test returning more results than fit in the caller stack
+  local a = {}
+  for i=1,1000 do a[i] = true end; a[999] = 10
+  local b = T.testC([[call 1 -1; pop 1; tostring -1; return 1]], unpack, a)
+  assert(b == "10")
+end
+
+
+-- testing lessthan
+assert(T.testC("lessthan 2 5, return 1", 3, 2, 2, 4, 2, 2))
+assert(T.testC("lessthan 5 2, return 1", 4, 2, 2, 3, 2, 2))
+assert(not T.testC("lessthan 2 -3, return 1", "4", "2", "2", "3", "2", "2"))
+assert(not T.testC("lessthan -3 2, return 1", "3", "2", "2", "4", "2", "2"))
+
+local b = {__lt = function (a,b) return a[1] < b[1] end}
+local a1,a3,a4 = setmetatable({1}, b),
+                 setmetatable({3}, b),
+                 setmetatable({4}, b)
+assert(T.testC("lessthan 2 5, return 1", a3, 2, 2, a4, 2, 2))
+assert(T.testC("lessthan 5 -6, return 1", a4, 2, 2, a3, 2, 2))
+a,b = T.testC("lessthan 5 -6, return 2", a1, 2, 2, a3, 2, 20)
+assert(a == 20 and b == false)
+
+
+-- testing lua_is
+
+function count (x, n)
+  n = n or 2
+  local prog = [[
+    isnumber %d;
+    isstring %d;
+    isfunction %d;
+    iscfunction %d;
+    istable %d;
+    isuserdata %d;
+    isnil %d;
+    isnull %d;
+    return 8
+  ]]
+  prog = string.format(prog, n, n, n, n, n, n, n, n)
+  local a,b,c,d,e,f,g,h = T.testC(prog, x)
+  return a+b+c+d+e+f+g+(100*h)
+end
+
+assert(count(3) == 2)
+assert(count('alo') == 1)
+assert(count('32') == 2)
+assert(count({}) == 1)
+assert(count(print) == 2)
+assert(count(function () end) == 1)
+assert(count(nil) == 1)
+assert(count(io.stdin) == 1)
+assert(count(nil, 15) == 100)
+
+-- testing lua_to...
+
+function to (s, x, n)
+  n = n or 2
+  return T.testC(string.format("%s %d; return 1", s, n), x)
+end
+
+assert(to("tostring", {}) == nil)
+assert(to("tostring", "alo") == "alo")
+assert(to("tostring", 12) == "12")
+assert(to("tostring", 12, 3) == nil)
+assert(to("objsize", {}) == 0)
+assert(to("objsize", "alo\0\0a") == 6)
+assert(to("objsize", T.newuserdata(0)) == 0)
+assert(to("objsize", T.newuserdata(101)) == 101)
+assert(to("objsize", 12) == 2)
+assert(to("objsize", 12, 3) == 0)
+assert(to("tonumber", {}) == 0)
+assert(to("tonumber", "12") == 12)
+assert(to("tonumber", "s2") == 0)
+assert(to("tonumber", 1, 20) == 0)
+a = to("tocfunction", math.deg)
+assert(a(3) == math.deg(3) and a ~= math.deg)
+
+
+-- testing errors
+
+a = T.testC([[
+  loadstring 2; call 0,1;
+  pushvalue 3; insert -2; call 1, 1;
+  call 0, 0;
+  return 1
+]], "x=150", function (a) assert(a==nil); return 3 end)
+
+assert(type(a) == 'string' and x == 150)
+
+function check3(p, ...)
+  assert(arg.n == 3)
+  assert(string.find(arg[3], p))
+end
+check3(":1:", T.testC("loadstring 2; gettop; return .", "x="))
+check3("cannot read", T.testC("loadfile 2; gettop; return .", "."))
+check3("cannot open xxxx", T.testC("loadfile 2; gettop; return .", "xxxx"))
+
+-- testing table access
+
+a = {x=0, y=12}
+x, y = T.testC("gettable 2; pushvalue 4; gettable 2; return 2",
+                a, 3, "y", 4, "x")
+assert(x == 0 and y == 12)
+T.testC("settable -5", a, 3, 4, "x", 15)
+assert(a.x == 15)
+a[a] = print
+x = T.testC("gettable 2; return 1", a)  -- table and key are the same object!
+assert(x == print)
+T.testC("settable 2", a, "x")    -- table and key are the same object!
+assert(a[a] == "x")
+
+b = setmetatable({p = a}, {})
+getmetatable(b).__index = function (t, i) return t.p[i] end
+k, x = T.testC("gettable 3, return 2", 4, b, 20, 35, "x")
+assert(x == 15 and k == 35)
+getmetatable(b).__index = function (t, i) return a[i] end
+getmetatable(b).__newindex = function (t, i,v ) a[i] = v end
+y = T.testC("insert 2; gettable -5; return 1", 2, 3, 4, "y", b)
+assert(y == 12)
+k = T.testC("settable -5, return 1", b, 3, 4, "x", 16)
+assert(a.x == 16 and k == 4)
+a[b] = 'xuxu'
+y = T.testC("gettable 2, return 1", b)
+assert(y == 'xuxu')
+T.testC("settable 2", b, 19)
+assert(a[b] == 19)
+
+-- testing next
+a = {}
+t = pack(T.testC("next; gettop; return .", a, nil))
+tcheck(t, {n=1,a})
+a = {a=3}
+t = pack(T.testC("next; gettop; return .", a, nil))
+tcheck(t, {n=3,a,'a',3})
+t = pack(T.testC("next; pop 1; next; gettop; return .", a, nil))
+tcheck(t, {n=1,a})
+
+
+
+-- testing upvalues
+
+do
+  local A = T.testC[[ pushnum 10; pushnum 20; pushcclosure 2; return 1]]
+  t, b, c = A([[pushvalue U0; pushvalue U1; pushvalue U2; return 3]])
+  assert(b == 10 and c == 20 and type(t) == 'table')
+  a, b = A([[tostring U3; tonumber U4; return 2]])
+  assert(a == nil and b == 0)
+  A([[pushnum 100; pushnum 200; replace U2; replace U1]])
+  b, c = A([[pushvalue U1; pushvalue U2; return 2]])
+  assert(b == 100 and c == 200)
+  A([[replace U2; replace U1]], {x=1}, {x=2})
+  b, c = A([[pushvalue U1; pushvalue U2; return 2]])
+  assert(b.x == 1 and c.x == 2)
+  T.checkmemory()
+end
+
+local f = T.testC[[ pushnum 10; pushnum 20; pushcclosure 2; return 1]]
+assert(T.upvalue(f, 1) == 10 and
+       T.upvalue(f, 2) == 20 and
+       T.upvalue(f, 3) == nil)
+T.upvalue(f, 2, "xuxu")
+assert(T.upvalue(f, 2) == "xuxu")
+
+
+-- testing environments
+
+assert(T.testC"pushvalue G; return 1" == _G)
+assert(T.testC"pushvalue E; return 1" == _G)
+local a = {}
+T.testC("replace E; return 1", a)
+assert(T.testC"pushvalue G; return 1" == _G)
+assert(T.testC"pushvalue E; return 1" == a)
+assert(debug.getfenv(T.testC) == a)
+assert(debug.getfenv(T.upvalue) == _G)
+-- userdata inherit environment
+local u = T.testC"newuserdata 0; return 1"
+assert(debug.getfenv(u) == a)
+-- functions inherit environment
+u = T.testC"pushcclosure 0; return 1"
+assert(debug.getfenv(u) == a)
+debug.setfenv(T.testC, _G)
+assert(T.testC"pushvalue E; return 1" == _G)
+
+local b = newproxy()
+assert(debug.getfenv(b) == _G)
+assert(debug.setfenv(b, a))
+assert(debug.getfenv(b) == a)
+
+
+
+-- testing locks (refs)
+
+-- reuse of references
+local i = T.ref{}
+T.unref(i)
+assert(T.ref{} == i)
+
+Arr = {}
+Lim = 100
+for i=1,Lim do   -- lock many objects
+  Arr[i] = T.ref({})
+end
+
+assert(T.ref(nil) == -1 and T.getref(-1) == nil)
+T.unref(-1); T.unref(-1)
+
+for i=1,Lim do   -- unlock all them
+  T.unref(Arr[i])
+end
+
+function printlocks ()
+  local n = T.testC("gettable R; return 1", "n")
+  print("n", n)
+  for i=0,n do
+    print(i, T.testC("gettable R; return 1", i))
+  end
+end
+
+
+for i=1,Lim do   -- lock many objects
+  Arr[i] = T.ref({})
+end
+
+for i=1,Lim,2 do   -- unlock half of them
+  T.unref(Arr[i])
+end
+
+assert(type(T.getref(Arr[2])) == 'table')
+
+
+assert(T.getref(-1) == nil)
+
+
+a = T.ref({})
+
+collectgarbage()
+
+assert(type(T.getref(a)) == 'table')
+
+
+-- colect in cl the `val' of all collected userdata
+tt = {}
+cl = {n=0}
+A = nil; B = nil
+local F
+F = function (x)
+  local udval = T.udataval(x)
+  table.insert(cl, udval)
+  local d = T.newuserdata(100)   -- cria lixo
+  d = nil
+  assert(debug.getmetatable(x).__gc == F)
+  loadstring("table.insert({}, {})")()   -- cria mais lixo
+  collectgarbage()   -- forca coleta de lixo durante coleta!
+  assert(debug.getmetatable(x).__gc == F)   -- coleta anterior nao melou isso?
+  local dummy = {}    -- cria lixo durante coleta
+  if A ~= nil then
+    assert(type(A) == "userdata")
+    assert(T.udataval(A) == B)
+    debug.getmetatable(A)    -- just acess it
+  end
+  A = x   -- ressucita userdata
+  B = udval
+  return 1,2,3
+end
+tt.__gc = F
+
+-- test whether udate collection frees memory in the right time
+do
+  collectgarbage();
+  collectgarbage();
+  local x = collectgarbage("count");
+  local a = T.newuserdata(5001)
+  assert(T.testC("objsize 2; return 1", a) == 5001)
+  assert(collectgarbage("count") >= x+4) 
+  a = nil
+  collectgarbage();
+  assert(collectgarbage("count") <= x+1)
+  -- udata without finalizer
+  x = collectgarbage("count")
+  collectgarbage("stop")
+  for i=1,1000 do newproxy(false) end
+  assert(collectgarbage("count") > x+10)
+  collectgarbage()
+  assert(collectgarbage("count") <= x+1)
+  -- udata with finalizer
+  x = collectgarbage("count")
+  collectgarbage()
+  collectgarbage("stop")
+  a = newproxy(true)
+  getmetatable(a).__gc = function () end
+  for i=1,1000 do newproxy(a) end
+  assert(collectgarbage("count") >= x+10)
+  collectgarbage()  -- this collection only calls TM, without freeing memory
+  assert(collectgarbage("count") >= x+10)
+  collectgarbage()  -- now frees memory
+  assert(collectgarbage("count") <= x+1)
+end
+
+
+collectgarbage("stop")
+
+-- create 3 userdatas with tag `tt'
+a = T.newuserdata(0); debug.setmetatable(a, tt); na = T.udataval(a)
+b = T.newuserdata(0); debug.setmetatable(b, tt); nb = T.udataval(b)
+c = T.newuserdata(0); debug.setmetatable(c, tt); nc = T.udataval(c)
+
+-- create userdata without meta table
+x = T.newuserdata(4)
+y = T.newuserdata(0)
+
+assert(debug.getmetatable(x) == nil and debug.getmetatable(y) == nil)
+
+d=T.ref(a);
+e=T.ref(b);
+f=T.ref(c);
+t = {T.getref(d), T.getref(e), T.getref(f)}
+assert(t[1] == a and t[2] == b and t[3] == c)
+
+t=nil; a=nil; c=nil;
+T.unref(e); T.unref(f)
+
+collectgarbage()
+
+-- check that unref objects have been collected
+assert(table.getn(cl) == 1 and cl[1] == nc)
+
+x = T.getref(d)
+assert(type(x) == 'userdata' and debug.getmetatable(x) == tt)
+x =nil
+tt.b = b  -- create cycle
+tt=nil    -- frees tt for GC
+A = nil
+b = nil
+T.unref(d);
+n5 = T.newuserdata(0)
+debug.setmetatable(n5, {__gc=F})
+n5 = T.udataval(n5)
+collectgarbage()
+assert(table.getn(cl) == 4)
+-- check order of collection
+assert(cl[2] == n5 and cl[3] == nb and cl[4] == na)
+
+
+a, na = {}, {}
+for i=30,1,-1 do
+  a[i] = T.newuserdata(0)
+  debug.setmetatable(a[i], {__gc=F})
+  na[i] = T.udataval(a[i])
+end
+cl = {}
+a = nil; collectgarbage()
+assert(table.getn(cl) == 30)
+for i=1,30 do assert(cl[i] == na[i]) end
+na = nil
+
+
+for i=2,Lim,2 do   -- unlock the other half
+  T.unref(Arr[i])
+end
+
+x = T.newuserdata(41); debug.setmetatable(x, {__gc=F})
+assert(T.testC("objsize 2; return 1", x) == 41)
+cl = {}
+a = {[x] = 1}
+x = T.udataval(x)
+collectgarbage()
+-- old `x' cannot be collected (`a' still uses it)
+assert(table.getn(cl) == 0)
+for n in pairs(a) do a[n] = nil end
+collectgarbage()
+assert(table.getn(cl) == 1 and cl[1] == x)   -- old `x' must be collected
+
+-- testing lua_equal
+assert(T.testC("equal 2 4; return 1", print, 1, print, 20))
+assert(T.testC("equal 3 2; return 1", 'alo', "alo"))
+assert(T.testC("equal 2 3; return 1", nil, nil))
+assert(not T.testC("equal 2 3; return 1", {}, {}))
+assert(not T.testC("equal 2 3; return 1"))
+assert(not T.testC("equal 2 3; return 1", 3))
+
+-- testing lua_equal with fallbacks
+do
+  local map = {}
+  local t = {__eq = function (a,b) return map[a] == map[b] end}
+  local function f(x)
+    local u = T.newuserdata(0)
+    debug.setmetatable(u, t)
+    map[u] = x
+    return u
+  end
+  assert(f(10) == f(10))
+  assert(f(10) ~= f(11))
+  assert(T.testC("equal 2 3; return 1", f(10), f(10)))
+  assert(not T.testC("equal 2 3; return 1", f(10), f(20)))
+  t.__eq = nil
+  assert(f(10) ~= f(10))
+end
+
+print'+'
+
+
+
+-------------------------------------------------------------------------
+do   -- testing errors during GC
+  local a = {}
+  for i=1,20 do
+    a[i] = T.newuserdata(i)   -- creates several udata
+  end
+  for i=1,20,2 do   -- mark half of them to raise error during GC
+    debug.setmetatable(a[i], {__gc = function (x) error("error inside gc") end})
+  end
+  for i=2,20,2 do   -- mark the other half to count and to create more garbage
+    debug.setmetatable(a[i], {__gc = function (x) loadstring("A=A+1")() end})
+  end
+  _G.A = 0
+  a = 0
+  while 1 do
+  if xpcall(collectgarbage, function (s) a=a+1 end) then
+    break   -- stop if no more errors
+  end
+  end
+  assert(a == 10)  -- number of errors
+  assert(A == 10)  -- number of normal collections
+end
+-------------------------------------------------------------------------
+-- test for userdata vals
+do
+  local a = {}; local lim = 30
+  for i=0,lim do a[i] = T.pushuserdata(i) end
+  for i=0,lim do assert(T.udataval(a[i]) == i) end
+  for i=0,lim do assert(T.pushuserdata(i) == a[i]) end
+  for i=0,lim do a[a[i]] = i end
+  for i=0,lim do a[T.pushuserdata(i)] = i end
+  assert(type(tostring(a[1])) == "string")
+end
+
+
+-------------------------------------------------------------------------
+-- testing multiple states
+T.closestate(T.newstate());
+L1 = T.newstate()
+assert(L1)
+assert(pack(T.doremote(L1, "function f () return 'alo', 3 end; f()")).n == 0)
+
+a, b = T.doremote(L1, "return f()")
+assert(a == 'alo' and b == '3')
+
+T.doremote(L1, "_ERRORMESSAGE = nil")
+-- error: `sin' is not defined
+a, b = T.doremote(L1, "return sin(1)")
+assert(a == nil and b == 2)   -- 2 == run-time error
+
+-- error: syntax error
+a, b, c = T.doremote(L1, "return a+")
+assert(a == nil and b == 3 and type(c) == "string")   -- 3 == syntax error
+
+T.loadlib(L1)
+a, b = T.doremote(L1, [[
+  a = strlibopen()
+  a = packageopen()
+  a = baselibopen(); assert(a == _G and require("_G") == a)
+  a = iolibopen(); assert(type(a.read) == "function")
+  assert(require("io") == a)
+  a = tablibopen(); assert(type(a.insert) == "function")
+  a = dblibopen(); assert(type(a.getlocal) == "function")
+  a = mathlibopen(); assert(type(a.sin) == "function")
+  return string.sub('okinama', 1, 2)
+]])
+assert(a == "ok")
+
+T.closestate(L1);
+
+L1 = T.newstate()
+T.loadlib(L1)
+T.doremote(L1, "a = {}")
+T.testC(L1, [[pushstring a; gettable G; pushstring x; pushnum 1;
+             settable -3]])
+assert(T.doremote(L1, "return a.x") == "1")
+
+T.closestate(L1)
+
+L1 = nil
+
+print('+')
+
+-------------------------------------------------------------------------
+-- testing memory limits
+-------------------------------------------------------------------------
+collectgarbage()
+T.totalmem(T.totalmem()+5000)   -- set low memory limit (+5k)
+assert(not pcall(loadstring"local a={}; for i=1,100000 do a[i]=i end"))
+T.totalmem(1000000000)          -- restore high limit
+
+
+local function stack(x) if x>0 then stack(x-1) end end
+
+-- test memory errors; increase memory limit in small steps, so that
+-- we get memory errors in different parts of a given task, up to there
+-- is enough memory to complete the task without errors
+function testamem (s, f)
+  collectgarbage()
+  stack(10)    -- ensure minimum stack size
+  local M = T.totalmem()
+  local oldM = M
+  local a,b = nil
+  while 1 do
+    M = M+3   -- increase memory limit in small steps
+    T.totalmem(M)
+    a, b = pcall(f)
+    if a and b then break end       -- stop when no more errors
+    collectgarbage()
+    if not a and not string.find(b, "memory") then   -- `real' error?
+      T.totalmem(1000000000)  -- restore high limit
+      error(b, 0)
+    end
+  end
+  T.totalmem(1000000000)  -- restore high limit
+  print("\nlimit for " .. s .. ": " .. M-oldM)
+  return b
+end
+
+
+-- testing memory errors when creating a new state
+
+b = testamem("state creation", T.newstate)
+T.closestate(b);  -- close new state
+
+
+-- testing threads
+
+function expand (n,s)
+  if n==0 then return "" end
+  local e = string.rep("=", n)
+  return string.format("T.doonnewstack([%s[ %s;\n collectgarbage(); %s]%s])\n",
+                              e, s, expand(n-1,s), e)
+end
+
+G=0; collectgarbage(); a =collectgarbage("count")
+loadstring(expand(20,"G=G+1"))()
+assert(G==20); collectgarbage();  -- assert(gcinfo() <= a+1)
+
+testamem("thread creation", function ()
+  return T.doonnewstack("x=1") == 0  -- try to create thread
+end)
+
+
+-- testing memory x compiler
+
+testamem("loadstring", function ()
+  return loadstring("x=1")  -- try to do a loadstring
+end)
+
+
+local testprog = [[
+local function foo () return end
+local t = {"x"}
+a = "aaa"
+for _, v in ipairs(t) do a=a..v end
+return true
+]]
+
+-- testing memory x dofile
+_G.a = nil
+local t =os.tmpname()
+local f = assert(io.open(t, "w"))
+f:write(testprog)
+f:close()
+testamem("dofile", function ()
+  local a = loadfile(t)
+  return a and a()
+end)
+assert(os.remove(t))
+assert(_G.a == "aaax")
+
+
+-- other generic tests
+
+testamem("string creation", function ()
+  local a, b = string.gsub("alo alo", "(a)", function (x) return x..'b' end)
+  return (a == 'ablo ablo')
+end)
+
+testamem("dump/undump", function ()
+  local a = loadstring(testprog)
+  local b = a and string.dump(a)
+  a = b and loadstring(b)
+  return a and a()
+end)
+
+local t = os.tmpname()
+testamem("file creation", function ()
+  local f = assert(io.open(t, 'w'))
+  assert (not io.open"nomenaoexistente")
+  io.close(f);
+  return not loadfile'nomenaoexistente'
+end)
+assert(os.remove(t))
+
+testamem("table creation", function ()
+  local a, lim = {}, 10
+  for i=1,lim do a[i] = i; a[i..'a'] = {} end
+  return (type(a[lim..'a']) == 'table' and a[lim] == lim)
+end)
+
+local a = 1
+close = nil
+testamem("closure creation", function ()
+  function close (b,c)
+   return function (x) return a+b+c+x end
+  end
+  return (close(2,3)(4) == 10)
+end)
+
+testamem("coroutines", function ()
+  local a = coroutine.wrap(function ()
+              coroutine.yield(string.rep("a", 10))
+              return {}
+            end)
+  assert(string.len(a()) == 10)
+  return a()
+end)
+
+print'+'
+
+-- testing some auxlib functions
+assert(T.gsub("alo.alo.uhuh.", ".", "//") == "alo//alo//uhuh//")
+assert(T.gsub("alo.alo.uhuh.", "alo", "//") == "//.//.uhuh.")
+assert(T.gsub("", "alo", "//") == "")
+assert(T.gsub("...", ".", "/.") == "/././.")
+assert(T.gsub("...", "...", "") == "")
+
+
+print'OK'
+

lib/lua/lua-tests/attrib.lua

@@ -0,0 +1,339 @@
+do --[
+
+print "testing require"
+
+assert(require"string" == string)
+assert(require"math" == math)
+assert(require"table" == table)
+assert(require"io" == io)
+assert(require"os" == os)
+assert(require"debug" == debug)
+assert(require"coroutine" == coroutine)
+
+assert(type(package.path) == "string")
+assert(type(package.cpath) == "string")
+assert(type(package.loaded) == "table")
+assert(type(package.preload) == "table")
+
+
+local DIR = "libs/"
+
+local function createfiles (files, preextras, posextras)
+  for n,c in pairs(files) do
+    io.output(DIR..n)
+    io.write(string.format(preextras, n))
+    io.write(c)
+    io.write(string.format(posextras, n))
+    io.close(io.output())
+  end
+end
+
+function removefiles (files)
+  for n in pairs(files) do
+    os.remove(DIR..n)
+  end
+end
+
+local files = {
+  ["A.lua"] = "",
+  ["B.lua"] = "assert(...=='B');require 'A'",
+  ["A.lc"] = "",
+  ["A"] = "",
+  ["L"] = "",
+  ["XXxX"] = "",
+  ["C.lua"] = "package.loaded[...] = 25; require'C'"
+}
+
+AA = nil
+local extras = [[
+NAME = '%s'
+REQUIRED = ...
+return AA]]
+
+createfiles(files, "", extras)
+
+
+local oldpath = package.path
+
+package.path = string.gsub("D/?.lua;D/?.lc;D/?;D/??x?;D/L", "D/", DIR)
+
+local try = function (p, n, r)
+  NAME = nil
+  local rr = require(p)
+  assert(NAME == n)
+  assert(REQUIRED == p)
+  assert(rr == r)
+end
+
+assert(require"C" == 25)
+assert(require"C" == 25)
+AA = nil
+try('B', 'B.lua', true)
+assert(package.loaded.B)
+assert(require"B" == true)
+assert(package.loaded.A)
+package.loaded.A = nil
+try('B', nil, true)   -- should not reload package
+try('A', 'A.lua', true)
+package.loaded.A = nil
+os.remove(DIR..'A.lua')
+AA = {}
+try('A', 'A.lc', AA)  -- now must find second option
+assert(require("A") == AA)
+AA = false
+try('K', 'L', false)     -- default option
+try('K', 'L', false)     -- default option (should reload it)
+assert(rawget(_G, "_REQUIREDNAME") == nil)
+
+AA = "x"
+try("X", "XXxX", AA)
+
+
+removefiles(files)
+
+
+-- testing require of sub-packages
+
+package.path = string.gsub("D/?.lua;D/?/init.lua", "D/", DIR)
+
+files = {
+  ["P1/init.lua"] = "AA = 10",
+  ["P1/xuxu.lua"] = "AA = 20",
+}
+
+createfiles(files, "module(..., package.seeall)\n", "")
+AA = 0
+
+local m = assert(require"P1")
+assert(m == P1 and m._NAME == "P1" and AA == 0 and m.AA == 10)
+assert(require"P1" == P1 and P1 == m)
+assert(require"P1" == P1)
+assert(P1._PACKAGE == "")
+
+local m = assert(require"P1.xuxu")
+assert(m == P1.xuxu and m._NAME == "P1.xuxu" and AA == 0 and m.AA == 20)
+assert(require"P1.xuxu" == P1.xuxu and P1.xuxu == m)
+assert(require"P1.xuxu" == P1.xuxu)
+assert(require"P1" == P1)
+assert(P1.xuxu._PACKAGE == "P1.")
+assert(P1.AA == 10 and P1._PACKAGE == "")
+assert(P1._G == _G and P1.xuxu._G == _G)
+
+
+
+removefiles(files)
+
+
+package.path = ""
+assert(not pcall(require, "file_does_not_exist"))
+package.path = "??\0?"
+assert(not pcall(require, "file_does_not_exist1"))
+
+package.path = oldpath
+
+-- check 'require' error message
+local fname = "file_does_not_exist2"
+local m, err = pcall(require, fname)
+for t in string.gmatch(package.path..";"..package.cpath, "[^;]+") do
+  t = string.gsub(t, "?", fname)
+  assert(string.find(err, t, 1, true))
+end
+
+
+local function import(...)
+  local f = {...}
+  return function (m)
+    for i=1, #f do m[f[i]] = _G[f[i]] end
+  end
+end
+
+local assert, module, package = assert, module, package
+X = nil; x = 0; assert(_G.x == 0)   -- `x' must be a global variable
+module"X"; x = 1; assert(_M.x == 1)
+module"X.a.b.c"; x = 2; assert(_M.x == 2)
+module("X.a.b", package.seeall); x = 3
+assert(X._NAME == "X" and X.a.b.c._NAME == "X.a.b.c" and X.a.b._NAME == "X.a.b")
+assert(X._M == X and X.a.b.c._M == X.a.b.c and X.a.b._M == X.a.b)
+assert(X.x == 1 and X.a.b.c.x == 2 and X.a.b.x == 3)
+assert(X._PACKAGE == "" and X.a.b.c._PACKAGE == "X.a.b." and
+       X.a.b._PACKAGE == "X.a.")
+assert(_PACKAGE.."c" == "X.a.c")
+assert(X.a._NAME == nil and X.a._M == nil)
+module("X.a", import("X")) ; x = 4
+assert(X.a._NAME == "X.a" and X.a.x == 4 and X.a._M == X.a)
+module("X.a.b", package.seeall); assert(x == 3); x = 5
+assert(_NAME == "X.a.b" and X.a.b.x == 5)
+
+assert(X._G == nil and X.a._G == nil and X.a.b._G == _G and X.a.b.c._G == nil)
+
+setfenv(1, _G)
+assert(x == 0)
+
+assert(not pcall(module, "x"))
+assert(not pcall(module, "math.sin"))
+
+
+-- testing C libraries
+
+
+local p = ""   -- On Mac OS X, redefine this to "_"
+
+-- assert(loadlib == package.loadlib)   -- only for compatibility
+local f, err, when = package.loadlib("libs/lib1.so", p.."luaopen_lib1")
+if not f then
+  (Message or print)('\a\n >>> cannot load dynamic library <<<\n\a')
+  print(err, when)
+else
+  f()   -- open library
+  assert(require("lib1") == lib1)
+  collectgarbage()
+  assert(lib1.id("x") == "x")
+  f = assert(package.loadlib("libs/lib1.so", p.."anotherfunc"))
+  assert(f(10, 20) == "1020\n")
+  f, err, when = package.loadlib("libs/lib1.so", p.."xuxu")
+  assert(not f and type(err) == "string" and when == "init")
+  package.cpath = "libs/?.so"
+  require"lib2"
+  assert(lib2.id("x") == "x")
+  local fs = require"lib1.sub"
+  assert(fs == lib1.sub and next(lib1.sub) == nil)
+  module("lib2", package.seeall)
+  f = require"-lib2"
+  assert(f.id("x") == "x" and _M == f and _NAME == "lib2")
+  module("lib1.sub", package.seeall)
+  assert(_M == fs)
+  setfenv(1, _G)
+ 
+end
+f, err, when = package.loadlib("donotexist", p.."xuxu")
+assert(not f and type(err) == "string" and (when == "open" or when == "absent"))
+
+
+-- testing preload
+
+do
+  local p = package
+  package = {}
+  p.preload.pl = function (...)
+    module(...)
+    function xuxu (x) return x+20 end
+  end
+
+  require"pl"
+  assert(require"pl" == pl)
+  assert(pl.xuxu(10) == 30)
+
+  package = p
+  assert(type(package.path) == "string")
+end
+
+
+
+end  --]
+
+print('+')
+
+print("testing assignments, logical operators, and constructors")
+
+local res, res2 = 27
+
+a, b = 1, 2+3
+assert(a==1 and b==5)
+a={}
+function f() return 10, 11, 12 end
+a.x, b, a[1] = 1, 2, f()
+assert(a.x==1 and b==2 and a[1]==10)
+a[f()], b, a[f()+3] = f(), a, 'x'
+assert(a[10] == 10 and b == a and a[13] == 'x')
+
+do
+  local f = function (n) local x = {}; for i=1,n do x[i]=i end;
+                         return unpack(x) end;
+  local a,b,c
+  a,b = 0, f(1)
+  assert(a == 0 and b == 1)
+  A,b = 0, f(1)
+  assert(A == 0 and b == 1)
+  a,b,c = 0,5,f(4)
+  assert(a==0 and b==5 and c==1)
+  a,b,c = 0,5,f(0)
+  assert(a==0 and b==5 and c==nil)
+end
+
+
+a, b, c, d = 1 and nil, 1 or nil, (1 and (nil or 1)), 6
+assert(not a and b and c and d==6)
+
+d = 20
+a, b, c, d = f()
+assert(a==10 and b==11 and c==12 and d==nil)
+a,b = f(), 1, 2, 3, f()
+assert(a==10 and b==1)
+
+assert(a<b == false and a>b == true)
+assert((10 and 2) == 2)
+assert((10 or 2) == 10)
+assert((10 or assert(nil)) == 10)
+assert(not (nil and assert(nil)))
+assert((nil or "alo") == "alo")
+assert((nil and 10) == nil)
+assert((false and 10) == false)
+assert((true or 10) == true)
+assert((false or 10) == 10)
+assert(false ~= nil)
+assert(nil ~= false)
+assert(not nil == true)
+assert(not not nil == false)
+assert(not not 1 == true)
+assert(not not a == true)
+assert(not not (6 or nil) == true)
+assert(not not (nil and 56) == false)
+assert(not not (nil and true) == false)
+print('+')
+
+a = {}
+a[true] = 20
+a[false] = 10
+assert(a[1<2] == 20 and a[1>2] == 10)
+
+function f(a) return a end
+
+local a = {}
+for i=3000,-3000,-1 do a[i] = i; end
+a[10e30] = "alo"; a[true] = 10; a[false] = 20
+assert(a[10e30] == 'alo' and a[not 1] == 20 and a[10<20] == 10)
+for i=3000,-3000,-1 do assert(a[i] == i); end
+a[print] = assert
+a[f] = print
+a[a] = a
+assert(a[a][a][a][a][print] == assert)
+a[print](a[a[f]] == a[print])
+a = nil
+
+a = {10,9,8,7,6,5,4,3,2; [-3]='a', [f]=print, a='a', b='ab'}
+a, a.x, a.y = a, a[-3]
+assert(a[1]==10 and a[-3]==a.a and a[f]==print and a.x=='a' and not a.y)
+a[1], f(a)[2], b, c = {['alo']=assert}, 10, a[1], a[f], 6, 10, 23, f(a), 2
+a[1].alo(a[2]==10 and b==10 and c==print)
+
+a[2^31] = 10; a[2^31+1] = 11; a[-2^31] = 12;
+a[2^32] = 13; a[-2^32] = 14; a[2^32+1] = 15; a[10^33] = 16;
+
+assert(a[2^31] == 10 and a[2^31+1] == 11 and a[-2^31] == 12 and
+       a[2^32] == 13 and a[-2^32] == 14 and a[2^32+1] == 15 and
+       a[10^33] == 16)
+
+a = nil
+
+
+do
+  local a,i,j,b
+  a = {'a', 'b'}; i=1; j=2; b=a
+  i, a[i], a, j, a[j], a[i+j] = j, i, i, b, j, i
+  assert(i == 2 and b[1] == 1 and a == 1 and j == b and b[2] == 2 and
+         b[3] == 1)
+end
+
+print('OK')
+
+return res

lib/lua/lua-tests/big.lua

@@ -0,0 +1,381 @@
+print "testing string length overflow"
+
+local longs = string.rep("\0", 2^25)
+local function catter (i)
+  return assert(loadstring(
+    string.format("return function(a) return a%s end",
+                     string.rep("..a", i-1))))()
+end
+rep129 = catter(129)
+local a, b = pcall(rep129, longs)
+assert(not a and string.find(b, "overflow"))
+print('+')
+
+
+require "checktable"
+
+--[[ lots of empty lines (to force SETLINEW)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+--]]
+
+
+a,b = nil,nil
+while not b do
+if a then
+b = {  -- lots of strings (to force JMPW and PUSHCONSTANTW)
+"n1", "n2", "n3", "n4", "n5", "n6", "n7", "n8", "n9", "n10",
+"n11", "n12", "j301", "j302", "j303", "j304", "j305", "j306", "j307", "j308",
+"j309", "a310", "n311", "n312", "n313", "n314", "n315", "n316", "n317", "n318",
+"n319", "n320", "n321", "n322", "n323", "n324", "n325", "n326", "n327", "n328",
+"a329", "n330", "n331", "n332", "n333", "n334", "n335", "n336", "n337", "n338",
+"n339", "n340", "n341", "z342", "n343", "n344", "n345", "n346", "n347", "n348",
+"n349", "n350", "n351", "n352", "r353", "n354", "n355", "n356", "n357", "n358",
+"n359", "n360", "n361", "n362", "n363", "n364", "n365", "n366", "z367", "n368",
+"n369", "n370", "n371", "n372", "n373", "n374", "n375", "a376", "n377", "n378",
+"n379", "n380", "n381", "n382", "n383", "n384", "n385", "n386", "n387", "n388",
+"n389", "n390", "n391", "n392", "n393", "n394", "n395", "n396", "n397", "n398",
+"n399", "n400", "n13", "n14", "n15", "n16", "n17", "n18", "n19", "n20",
+"n21", "n22", "n23", "a24", "n25", "n26", "n27", "n28", "n29", "j30",
+"n31", "n32", "n33", "n34", "n35", "n36", "n37", "n38", "n39", "n40",
+"n41", "n42", "n43", "n44", "n45", "n46", "n47", "n48", "n49", "n50",
+"n51", "n52", "n53", "n54", "n55", "n56", "n57", "n58", "n59", "n60",
+"n61", "n62", "n63", "n64", "n65", "a66", "z67", "n68", "n69", "n70",
+"n71", "n72", "n73", "n74", "n75", "n76", "n77", "n78", "n79", "n80",
+"n81", "n82", "n83", "n84", "n85", "n86", "n87", "n88", "n89", "n90",
+"n91", "n92", "n93", "n94", "n95", "n96", "n97", "n98", "n99", "n100",
+"n201", "n202", "n203", "n204", "n205", "n206", "n207", "n208", "n209", "n210",
+"n211", "n212", "n213", "n214", "n215", "n216", "n217", "n218", "n219", "n220",
+"n221", "n222", "n223", "n224", "n225", "n226", "n227", "n228", "n229", "n230",
+"n231", "n232", "n233", "n234", "n235", "n236", "n237", "n238", "n239", "a240",
+"a241", "a242", "a243", "a244", "a245", "a246", "a247", "a248", "a249", "n250",
+"n251", "n252", "n253", "n254", "n255", "n256", "n257", "n258", "n259", "n260",
+"n261", "n262", "n263", "n264", "n265", "n266", "n267", "n268", "n269", "n270",
+"n271", "n272", "n273", "n274", "n275", "n276", "n277", "n278", "n279", "n280",
+"n281", "n282", "n283", "n284", "n285", "n286", "n287", "n288", "n289", "n290",
+"n291", "n292", "n293", "n294", "n295", "n296", "n297", "n298", "n299"
+; x=23}
+else a = 1 end
+
+
+end
+
+assert(b.x == 23)
+print('+')
+
+stat(b)
+
+repeat
+a = {
+n1 = 1.5, n2 = 2.5, n3 = 3.5, n4 = 4.5, n5 = 5.5, n6 = 6.5, n7 = 7.5,
+n8 = 8.5, n9 = 9.5, n10 = 10.5, n11 = 11.5, n12 = 12.5,
+j301 = 301.5, j302 = 302.5, j303 = 303.5, j304 = 304.5, j305 = 305.5,
+j306 = 306.5, j307 = 307.5, j308 = 308.5, j309 = 309.5, a310 = 310.5,
+n311 = 311.5, n312 = 312.5, n313 = 313.5, n314 = 314.5, n315 = 315.5,
+n316 = 316.5, n317 = 317.5, n318 = 318.5, n319 = 319.5, n320 = 320.5,
+n321 = 321.5, n322 = 322.5, n323 = 323.5, n324 = 324.5, n325 = 325.5,
+n326 = 326.5, n327 = 327.5, n328 = 328.5, a329 = 329.5, n330 = 330.5,
+n331 = 331.5, n332 = 332.5, n333 = 333.5, n334 = 334.5, n335 = 335.5,
+n336 = 336.5, n337 = 337.5, n338 = 338.5, n339 = 339.5, n340 = 340.5,
+n341 = 341.5, z342 = 342.5, n343 = 343.5, n344 = 344.5, n345 = 345.5,
+n346 = 346.5, n347 = 347.5, n348 = 348.5, n349 = 349.5, n350 = 350.5,
+n351 = 351.5, n352 = 352.5, r353 = 353.5, n354 = 354.5, n355 = 355.5,
+n356 = 356.5, n357 = 357.5, n358 = 358.5, n359 = 359.5, n360 = 360.5,
+n361 = 361.5, n362 = 362.5, n363 = 363.5, n364 = 364.5, n365 = 365.5,
+n366 = 366.5, z367 = 367.5, n368 = 368.5, n369 = 369.5, n370 = 370.5,
+n371 = 371.5, n372 = 372.5, n373 = 373.5, n374 = 374.5, n375 = 375.5,
+a376 = 376.5, n377 = 377.5, n378 = 378.5, n379 = 379.5, n380 = 380.5,
+n381 = 381.5, n382 = 382.5, n383 = 383.5, n384 = 384.5, n385 = 385.5,
+n386 = 386.5, n387 = 387.5, n388 = 388.5, n389 = 389.5, n390 = 390.5,
+n391 = 391.5, n392 = 392.5, n393 = 393.5, n394 = 394.5, n395 = 395.5,
+n396 = 396.5, n397 = 397.5, n398 = 398.5, n399 = 399.5, n400 = 400.5,
+n13 = 13.5, n14 = 14.5, n15 = 15.5, n16 = 16.5, n17 = 17.5,
+n18 = 18.5, n19 = 19.5, n20 = 20.5, n21 = 21.5, n22 = 22.5,
+n23 = 23.5, a24 = 24.5, n25 = 25.5, n26 = 26.5, n27 = 27.5,
+n28 = 28.5, n29 = 29.5, j30 = 30.5, n31 = 31.5, n32 = 32.5,
+n33 = 33.5, n34 = 34.5, n35 = 35.5, n36 = 36.5, n37 = 37.5,
+n38 = 38.5, n39 = 39.5, n40 = 40.5, n41 = 41.5, n42 = 42.5,
+n43 = 43.5, n44 = 44.5, n45 = 45.5, n46 = 46.5, n47 = 47.5,
+n48 = 48.5, n49 = 49.5, n50 = 50.5, n51 = 51.5, n52 = 52.5,
+n53 = 53.5, n54 = 54.5, n55 = 55.5, n56 = 56.5, n57 = 57.5,
+n58 = 58.5, n59 = 59.5, n60 = 60.5, n61 = 61.5, n62 = 62.5,
+n63 = 63.5, n64 = 64.5, n65 = 65.5, a66 = 66.5, z67 = 67.5,
+n68 = 68.5, n69 = 69.5, n70 = 70.5, n71 = 71.5, n72 = 72.5,
+n73 = 73.5, n74 = 74.5, n75 = 75.5, n76 = 76.5, n77 = 77.5,
+n78 = 78.5, n79 = 79.5, n80 = 80.5, n81 = 81.5, n82 = 82.5,
+n83 = 83.5, n84 = 84.5, n85 = 85.5, n86 = 86.5, n87 = 87.5,
+n88 = 88.5, n89 = 89.5, n90 = 90.5, n91 = 91.5, n92 = 92.5,
+n93 = 93.5, n94 = 94.5, n95 = 95.5, n96 = 96.5, n97 = 97.5,
+n98 = 98.5, n99 = 99.5, n100 = 100.5, n201 = 201.5, n202 = 202.5,
+n203 = 203.5, n204 = 204.5, n205 = 205.5, n206 = 206.5, n207 = 207.5,
+n208 = 208.5, n209 = 209.5, n210 = 210.5, n211 = 211.5, n212 = 212.5,
+n213 = 213.5, n214 = 214.5, n215 = 215.5, n216 = 216.5, n217 = 217.5,
+n218 = 218.5, n219 = 219.5, n220 = 220.5, n221 = 221.5, n222 = 222.5,
+n223 = 223.5, n224 = 224.5, n225 = 225.5, n226 = 226.5, n227 = 227.5,
+n228 = 228.5, n229 = 229.5, n230 = 230.5, n231 = 231.5, n232 = 232.5,
+n233 = 233.5, n234 = 234.5, n235 = 235.5, n236 = 236.5, n237 = 237.5,
+n238 = 238.5, n239 = 239.5, a240 = 240.5, a241 = 241.5, a242 = 242.5,
+a243 = 243.5, a244 = 244.5, a245 = 245.5, a246 = 246.5, a247 = 247.5,
+a248 = 248.5, a249 = 249.5, n250 = 250.5, n251 = 251.5, n252 = 252.5,
+n253 = 253.5, n254 = 254.5, n255 = 255.5, n256 = 256.5, n257 = 257.5,
+n258 = 258.5, n259 = 259.5, n260 = 260.5, n261 = 261.5, n262 = 262.5,
+n263 = 263.5, n264 = 264.5, n265 = 265.5, n266 = 266.5, n267 = 267.5,
+n268 = 268.5, n269 = 269.5, n270 = 270.5, n271 = 271.5, n272 = 272.5,
+n273 = 273.5, n274 = 274.5, n275 = 275.5, n276 = 276.5, n277 = 277.5,
+n278 = 278.5, n279 = 279.5, n280 = 280.5, n281 = 281.5, n282 = 282.5,
+n283 = 283.5, n284 = 284.5, n285 = 285.5, n286 = 286.5, n287 = 287.5,
+n288 = 288.5, n289 = 289.5, n290 = 290.5, n291 = 291.5, n292 = 292.5,
+n293 = 293.5, n294 = 294.5, n295 = 295.5, n296 = 296.5, n297 = 297.5,
+n298 = 298.5, n299 = 299.5, j300 = 300} or 1
+until 1
+
+assert(a.n299 == 299.5)
+xxx = 1
+assert(xxx == 1)
+
+stat(a)
+
+function a:findfield (f)
+  local i,v = next(self, nil)
+  while i ~= f do
+    if not i then return end
+    i,v = next(self, i)
+  end
+  return v
+end
+
+local ii = 0
+i = 1
+while b[i] do
+  local r = a:findfield(b[i]);
+  assert(a[b[i]] == r)
+  ii = math.max(ii,i)
+  i = i+1
+end
+
+assert(ii == 299)
+
+function xxxx (x) coroutine.yield('b'); return ii+x end
+
+assert(xxxx(10) == 309)
+
+a = nil
+b = nil
+a1 = nil
+
+print("tables with table indices:")
+i = 1; a={}
+while i <= 1023 do a[{}] = i; i=i+1 end
+stat(a)
+a = nil
+
+print("tables with function indices:")
+a={}
+for i=1,511 do local x; a[function () return x end] = i end
+stat(a)
+a = nil
+
+print'OK'
+
+return 'a'

lib/lua/lua-tests/calls.lua

@@ -0,0 +1,294 @@
+print("testing functions and calls")
+
+-- get the opportunity to test 'type' too ;)
+
+assert(type(1<2) == 'boolean')
+assert(type(true) == 'boolean' and type(false) == 'boolean')
+assert(type(nil) == 'nil' and type(-3) == 'number' and type'x' == 'string' and
+       type{} == 'table' and type(type) == 'function')
+
+assert(type(assert) == type(print))
+f = nil
+function f (x) return a:x (x) end
+assert(type(f) == 'function')
+
+
+-- testing local-function recursion
+fact = false
+do
+  local res = 1
+  local function fact (n)
+    if n==0 then return res
+    else return n*fact(n-1)
+    end
+  end
+  assert(fact(5) == 120)
+end
+assert(fact == false)
+
+-- testing declarations
+a = {i = 10}
+self = 20
+function a:x (x) return x+self.i end
+function a.y (x) return x+self end
+
+assert(a:x(1)+10 == a.y(1))
+
+a.t = {i=-100}
+a["t"].x = function (self, a,b) return self.i+a+b end
+
+assert(a.t:x(2,3) == -95)
+
+do
+  local a = {x=0}
+  function a:add (x) self.x, a.y = self.x+x, 20; return self end
+  assert(a:add(10):add(20):add(30).x == 60 and a.y == 20)
+end
+
+local a = {b={c={}}}
+
+function a.b.c.f1 (x) return x+1 end
+function a.b.c:f2 (x,y) self[x] = y end
+assert(a.b.c.f1(4) == 5)
+a.b.c:f2('k', 12); assert(a.b.c.k == 12)
+
+print('+')
+
+t = nil   -- 'declare' t
+function f(a,b,c) local d = 'a'; t={a,b,c,d} end
+
+f(      -- this line change must be valid
+  1,2)
+assert(t[1] == 1 and t[2] == 2 and t[3] == nil and t[4] == 'a')
+f(1,2,   -- this one too
+      3,4)
+assert(t[1] == 1 and t[2] == 2 and t[3] == 3 and t[4] == 'a')
+
+function fat(x)
+  if x <= 1 then return 1
+  else return x*loadstring("return fat(" .. x-1 .. ")")()
+  end
+end
+
+assert(loadstring "loadstring 'assert(fat(6)==720)' () ")()
+a = loadstring('return fat(5), 3')
+a,b = a()
+assert(a == 120 and b == 3)
+print('+')
+
+function err_on_n (n)
+  if n==0 then error(); exit(1);
+  else err_on_n (n-1); exit(1);
+  end
+end
+
+do
+  function dummy (n)
+    if n > 0 then
+      assert(not pcall(err_on_n, n))
+      dummy(n-1)
+    end
+  end
+end
+
+dummy(10)
+
+function deep (n)
+  if n>0 then deep(n-1) end
+end
+deep(10)
+deep(200)
+
+-- testing tail call
+function deep (n) if n>0 then return deep(n-1) else return 101 end end
+assert(deep(30000) == 101)
+a = {}
+function a:deep (n) if n>0 then return self:deep(n-1) else return 101 end end
+assert(a:deep(30000) == 101)
+
+print('+')
+
+
+a = nil
+(function (x) a=x end)(23)
+assert(a == 23 and (function (x) return x*2 end)(20) == 40)
+
+
+local x,y,z,a
+a = {}; lim = 2000
+for i=1, lim do a[i]=i end
+assert(select(lim, unpack(a)) == lim and select('#', unpack(a)) == lim)
+x = unpack(a)
+assert(x == 1)
+x = {unpack(a)}
+assert(table.getn(x) == lim and x[1] == 1 and x[lim] == lim)
+x = {unpack(a, lim-2)}
+assert(table.getn(x) == 3 and x[1] == lim-2 and x[3] == lim)
+x = {unpack(a, 10, 6)}
+assert(next(x) == nil)   -- no elements
+x = {unpack(a, 11, 10)}
+assert(next(x) == nil)   -- no elements
+x,y = unpack(a, 10, 10)
+assert(x == 10 and y == nil)
+x,y,z = unpack(a, 10, 11)
+assert(x == 10 and y == 11 and z == nil)
+a,x = unpack{1}
+assert(a==1 and x==nil)
+a,x = unpack({1,2}, 1, 1)
+assert(a==1 and x==nil)
+
+
+-- testing closures
+
+-- fixed-point operator
+Y = function (le)
+      local function a (f)
+        return le(function (x) return f(f)(x) end)
+      end
+      return a(a)
+    end
+
+
+-- non-recursive factorial
+
+F = function (f)
+      return function (n)
+               if n == 0 then return 1
+               else return n*f(n-1) end
+             end
+    end
+
+fat = Y(F)
+
+assert(fat(0) == 1 and fat(4) == 24 and Y(F)(5)==5*Y(F)(4))
+
+local function g (z)
+  local function f (a,b,c,d)
+    return function (x,y) return a+b+c+d+a+x+y+z end
+  end
+  return f(z,z+1,z+2,z+3)
+end
+
+f = g(10)
+assert(f(9, 16) == 10+11+12+13+10+9+16+10)
+
+Y, F, f = nil
+print('+')
+
+-- testing multiple returns
+
+function unlpack (t, i)
+  i = i or 1
+  if (i <= table.getn(t)) then
+    return t[i], unlpack(t, i+1)
+  end
+end
+
+function equaltab (t1, t2)
+  assert(table.getn(t1) == table.getn(t2))
+  for i,v1 in ipairs(t1) do
+    assert(v1 == t2[i])
+  end
+end
+
+local function pack (...)
+  local x = {...}
+  x.n = select('#', ...)
+  return x
+end
+
+function f() return 1,2,30,4 end
+function ret2 (a,b) return a,b end
+
+local a,b,c,d = unlpack{1,2,3}
+assert(a==1 and b==2 and c==3 and d==nil)
+a = {1,2,3,4,false,10,'alo',false,assert}
+equaltab(pack(unlpack(a)), a)
+equaltab(pack(unlpack(a), -1), {1,-1})
+a,b,c,d = ret2(f()), ret2(f())
+assert(a==1 and b==1 and c==2 and d==nil)
+a,b,c,d = unlpack(pack(ret2(f()), ret2(f())))
+assert(a==1 and b==1 and c==2 and d==nil)
+a,b,c,d = unlpack(pack(ret2(f()), (ret2(f()))))
+assert(a==1 and b==1 and c==nil and d==nil)
+
+a = ret2{ unlpack{1,2,3}, unlpack{3,2,1}, unlpack{"a", "b"}}
+assert(a[1] == 1 and a[2] == 3 and a[3] == "a" and a[4] == "b")
+
+
+-- testing calls with 'incorrect' arguments
+rawget({}, "x", 1)
+rawset({}, "x", 1, 2)
+assert(math.sin(1,2) == math.sin(1))
+table.sort({10,9,8,4,19,23,0,0}, function (a,b) return a<b end, "extra arg")
+
+
+-- test for generic load
+x = "-- a comment\0\0\0\n  x = 10 + \n23; \
+     local a = function () x = 'hi' end; \
+     return '\0'"
+local i = 0
+function read1 (x)
+  return function ()
+    collectgarbage()
+    i=i+1
+    return string.sub(x, i, i)
+  end
+end
+
+a = assert(load(read1(x), "modname"))
+assert(a() == "\0" and _G.x == 33)
+assert(debug.getinfo(a).source == "modname")
+
+x = string.dump(loadstring("x = 1; return x"))
+i = 0
+a = assert(load(read1(x)))
+assert(a() == 1 and _G.x == 1)
+
+i = 0
+local a, b = load(read1("*a = 123"))
+assert(not a and type(b) == "string" and i == 2)
+
+a, b = load(function () error("hhi") end)
+assert(not a and string.find(b, "hhi"))
+
+-- test generic load with nested functions
+x = [[
+  return function (x)
+    return function (y)
+     return function (z)
+       return x+y+z
+     end
+   end
+  end
+]]
+
+a = assert(load(read1(x)))
+assert(a()(2)(3)(10) == 15)
+
+
+-- test for dump/undump with upvalues
+local a, b = 20, 30
+x = loadstring(string.dump(function (x)
+  if x == "set" then a = 10+b; b = b+1 else
+  return a
+  end
+end))
+assert(x() == nil)
+assert(debug.setupvalue(x, 1, "hi") == "a")
+assert(x() == "hi")
+assert(debug.setupvalue(x, 2, 13) == "b")
+assert(not debug.setupvalue(x, 3, 10))   -- only 2 upvalues
+x("set")
+assert(x() == 23)
+x("set")
+assert(x() == 24)
+
+
+-- test for bug in parameter adjustment
+assert((function () return nil end)(4) == nil)
+assert((function () local a; return a end)(4) == nil)
+assert((function (a) return a end)() == nil)
+
+print('OK')
+return deep

lib/lua/lua-tests/checktable.lua

@@ -0,0 +1,77 @@
+
+assert(rawget(_G, "stat") == nil)  -- module not loaded before
+
+if T == nil then
+  stat = function () print"`querytab' nao ativo" end
+  return
+end
+
+
+function checktable (t)
+  local asize, hsize, ff = T.querytab(t)
+  local l = {}
+  for i=0,hsize-1 do
+    local key,val,next = T.querytab(t, i + asize)
+    if key == nil then
+      assert(l[i] == nil and val==nil and next==nil)
+    elseif key == "<undef>" then
+      assert(val==nil)
+    else
+      assert(t[key] == val)
+      local mp = T.hash(key, t)
+      if l[i] then
+        assert(l[i] == mp)
+      elseif mp ~= i then
+        l[i] = mp
+      else  -- list head
+        l[mp] = {mp}   -- first element
+        while next do
+          assert(ff <= next and next < hsize)
+          if l[next] then assert(l[next] == mp) else l[next] = mp end
+          table.insert(l[mp], next)
+          key,val,next = T.querytab(t, next)
+          assert(key)
+        end
+      end
+    end
+  end
+  l.asize = asize; l.hsize = hsize; l.ff = ff
+  return l
+end
+
+function mostra (t)
+  local asize, hsize, ff = T.querytab(t)
+  print(asize, hsize, ff)
+  print'------'
+  for i=0,asize-1 do
+    local _, v = T.querytab(t, i)
+    print(string.format("[%d] -", i), v)
+  end
+  print'------'
+  for i=0,hsize-1 do
+    print(i, T.querytab(t, i+asize))
+  end
+  print'-------------'
+end
+
+function stat (t)
+  t = checktable(t)
+  local nelem, nlist = 0, 0
+  local maxlist = {}
+  for i=0,t.hsize-1 do
+    if type(t[i]) == 'table' then
+      local n = table.getn(t[i])
+      nlist = nlist+1
+      nelem = nelem + n
+      if not maxlist[n] then maxlist[n] = 0 end
+      maxlist[n] = maxlist[n]+1
+    end
+  end
+  print(string.format("hsize=%d  elements=%d  load=%.2f  med.len=%.2f (asize=%d)",
+          t.hsize, nelem, nelem/t.hsize, nelem/nlist, t.asize))
+  for i=1,table.getn(maxlist) do
+    local n = maxlist[i] or 0
+    print(string.format("%5d %10d %.2f%%", i, n, n*100/nlist))
+  end
+end
+

lib/lua/lua-tests/closure.lua

@@ -0,0 +1,422 @@
+print "testing closures and coroutines"
+
+local A,B = 0,{g=10}
+function f(x)
+  local a = {}
+  for i=1,1000 do
+    local y = 0
+    do
+      a[i] = function () B.g = B.g+1; y = y+x; return y+A end
+    end
+  end
+  local dummy = function () return a[A] end
+  collectgarbage()
+  A = 1; assert(dummy() == a[1]); A = 0;
+  assert(a[1]() == x)
+  assert(a[3]() == x)
+  collectgarbage()
+  assert(B.g == 12)
+  return a
+end
+
+a = f(10)
+-- force a GC in this level
+local x = {[1] = {}}   -- to detect a GC
+setmetatable(x, {__mode = 'kv'})
+while x[1] do   -- repeat until GC
+  local a = A..A..A..A  -- create garbage
+  A = A+1
+end
+assert(a[1]() == 20+A)
+assert(a[1]() == 30+A)
+assert(a[2]() == 10+A)
+collectgarbage()
+assert(a[2]() == 20+A)
+assert(a[2]() == 30+A)
+assert(a[3]() == 20+A)
+assert(a[8]() == 10+A)
+assert(getmetatable(x).__mode == 'kv')
+assert(B.g == 19)
+
+-- testing closures with 'for' control variable
+a = {}
+for i=1,10 do
+  a[i] = {set = function(x) i=x end, get = function () return i end}
+  if i == 3 then break end
+end
+assert(a[4] == nil)
+a[1].set(10)
+assert(a[2].get() == 2)
+a[2].set('a')
+assert(a[3].get() == 3)
+assert(a[2].get() == 'a')
+
+a = {}
+for i, k in pairs{'a', 'b'} do
+  a[i] = {set = function(x, y) i=x; k=y end,
+          get = function () return i, k end}
+  if i == 2 then break end
+end
+a[1].set(10, 20)
+local r,s = a[2].get()
+assert(r == 2 and s == 'b')
+r,s = a[1].get()
+assert(r == 10 and s == 20)
+a[2].set('a', 'b')
+r,s = a[2].get()
+assert(r == "a" and s == "b")
+
+
+-- testing closures with 'for' control variable x break
+for i=1,3 do
+  f = function () return i end
+  break
+end
+assert(f() == 1)
+
+for k, v in pairs{"a", "b"} do
+  f = function () return k, v end
+  break
+end
+assert(({f()})[1] == 1)
+assert(({f()})[2] == "a")
+
+
+-- testing closure x break x return x errors
+
+local b
+function f(x)
+  local first = 1
+  while 1 do
+    if x == 3 and not first then return end
+    local a = 'xuxu'
+    b = function (op, y)
+          if op == 'set' then
+            a = x+y
+          else
+            return a
+          end
+        end
+    if x == 1 then do break end
+    elseif x == 2 then return
+    else if x ~= 3 then error() end
+    end
+    first = nil
+  end
+end
+
+for i=1,3 do
+  f(i)
+  assert(b('get') == 'xuxu')
+  b('set', 10); assert(b('get') == 10+i)
+  b = nil
+end
+
+pcall(f, 4);
+assert(b('get') == 'xuxu')
+b('set', 10); assert(b('get') == 14)
+
+
+local w
+-- testing multi-level closure
+function f(x)
+  return function (y)
+    return function (z) return w+x+y+z end
+  end
+end
+
+y = f(10)
+w = 1.345
+assert(y(20)(30) == 60+w)
+
+-- testing closures x repeat-until
+
+local a = {}
+local i = 1
+repeat
+  local x = i
+  a[i] = function () i = x+1; return x end
+until i > 10 or a[i]() ~= x
+assert(i == 11 and a[1]() == 1 and a[3]() == 3 and i == 4)
+
+print'+'
+
+
+-- test for correctly closing upvalues in tail calls of vararg functions
+local function t ()
+  local function c(a,b) assert(a=="test" and b=="OK") end
+  local function v(f, ...) c("test", f() ~= 1 and "FAILED" or "OK") end
+  local x = 1
+  return v(function() return x end)
+end
+t()
+
+
+-- coroutine tests
+
+local f
+
+assert(coroutine.running() == nil)
+
+
+-- tests for global environment
+
+local function foo (a)
+  setfenv(0, a)
+  coroutine.yield(getfenv())
+  assert(getfenv(0) == a)
+  assert(getfenv(1) == _G)
+  assert(getfenv(loadstring"") == a)
+  return getfenv()
+end
+
+f = coroutine.wrap(foo)
+local a = {}
+assert(f(a) == _G)
+local a,b = pcall(f)
+assert(a and b == _G)
+
+
+-- tests for multiple yield/resume arguments
+
+local function eqtab (t1, t2)
+  assert(table.getn(t1) == table.getn(t2))
+  for i,v in ipairs(t1) do
+    assert(t2[i] == v)
+  end
+end
+
+_G.x = nil   -- declare x
+function foo (a, ...)
+  assert(coroutine.running() == f)
+  assert(coroutine.status(f) == "running")
+  local arg = {...}
+  for i=1,table.getn(arg) do
+    _G.x = {coroutine.yield(unpack(arg[i]))}
+  end
+  return unpack(a)
+end
+
+f = coroutine.create(foo)
+assert(type(f) == "thread" and coroutine.status(f) == "suspended")
+assert(string.find(tostring(f), "thread"))
+local s,a,b,c,d
+s,a,b,c,d = coroutine.resume(f, {1,2,3}, {}, {1}, {'a', 'b', 'c'})
+assert(s and a == nil and coroutine.status(f) == "suspended")
+s,a,b,c,d = coroutine.resume(f)
+eqtab(_G.x, {})
+assert(s and a == 1 and b == nil)
+s,a,b,c,d = coroutine.resume(f, 1, 2, 3)
+eqtab(_G.x, {1, 2, 3})
+assert(s and a == 'a' and b == 'b' and c == 'c' and d == nil)
+s,a,b,c,d = coroutine.resume(f, "xuxu")
+eqtab(_G.x, {"xuxu"})
+assert(s and a == 1 and b == 2 and c == 3 and d == nil)
+assert(coroutine.status(f) == "dead")
+s, a = coroutine.resume(f, "xuxu")
+assert(not s and string.find(a, "dead") and coroutine.status(f) == "dead")
+
+
+-- yields in tail calls
+local function foo (i) return coroutine.yield(i) end
+f = coroutine.wrap(function ()
+  for i=1,10 do
+    assert(foo(i) == _G.x)
+  end
+  return 'a'
+end)
+for i=1,10 do _G.x = i; assert(f(i) == i) end
+_G.x = 'xuxu'; assert(f('xuxu') == 'a')
+
+-- recursive
+function pf (n, i)
+  coroutine.yield(n)
+  pf(n*i, i+1)
+end
+
+f = coroutine.wrap(pf)
+local s=1
+for i=1,10 do
+  assert(f(1, 1) == s)
+  s = s*i
+end
+
+-- sieve
+function gen (n)
+  return coroutine.wrap(function ()
+    for i=2,n do coroutine.yield(i) end
+  end)
+end
+
+
+function filter (p, g)
+  return coroutine.wrap(function ()
+    while 1 do
+      local n = g()
+      if n == nil then return end
+      if math.mod(n, p) ~= 0 then coroutine.yield(n) end
+    end
+  end)
+end
+
+local x = gen(100)
+local a = {}
+while 1 do
+  local n = x()
+  if n == nil then break end
+  table.insert(a, n)
+  x = filter(n, x)
+end
+
+assert(table.getn(a) == 25 and a[table.getn(a)] == 97)
+
+
+-- errors in coroutines
+function foo ()
+  assert(debug.getinfo(1).currentline == debug.getinfo(foo).linedefined + 1)
+  assert(debug.getinfo(2).currentline == debug.getinfo(goo).linedefined)
+  coroutine.yield(3)
+  error(foo)
+end
+
+function goo() foo() end
+x = coroutine.wrap(goo)
+assert(x() == 3)
+local a,b = pcall(x)
+assert(not a and b == foo)
+
+x = coroutine.create(goo)
+a,b = coroutine.resume(x)
+assert(a and b == 3)
+a,b = coroutine.resume(x)
+assert(not a and b == foo and coroutine.status(x) == "dead")
+a,b = coroutine.resume(x)
+assert(not a and string.find(b, "dead") and coroutine.status(x) == "dead")
+
+
+-- co-routines x for loop
+function all (a, n, k)
+  if k == 0 then coroutine.yield(a)
+  else
+    for i=1,n do
+      a[k] = i
+      all(a, n, k-1)
+    end
+  end
+end
+
+local a = 0
+for t in coroutine.wrap(function () all({}, 5, 4) end) do
+  a = a+1
+end
+assert(a == 5^4)
+
+
+-- access to locals of collected corroutines
+local C = {}; setmetatable(C, {__mode = "kv"})
+local x = coroutine.wrap (function ()
+            local a = 10
+            local function f () a = a+10; return a end
+            while true do
+              a = a+1
+              coroutine.yield(f)
+            end
+          end)
+
+C[1] = x;
+
+local f = x()
+assert(f() == 21 and x()() == 32 and x() == f)
+x = nil
+collectgarbage()
+assert(C[1] == nil)
+assert(f() == 43 and f() == 53)
+
+
+-- old bug: attempt to resume itself
+
+function co_func (current_co)
+  assert(coroutine.running() == current_co)
+  assert(coroutine.resume(current_co) == false)
+  assert(coroutine.resume(current_co) == false)
+  return 10
+end
+
+local co = coroutine.create(co_func)
+local a,b = coroutine.resume(co, co)
+assert(a == true and b == 10)
+assert(coroutine.resume(co, co) == false)
+assert(coroutine.resume(co, co) == false)
+
+-- access to locals of erroneous coroutines
+local x = coroutine.create (function ()
+            local a = 10
+            _G.f = function () a=a+1; return a end
+            error('x')
+          end)
+
+assert(not coroutine.resume(x))
+-- overwrite previous position of local `a'
+assert(not coroutine.resume(x, 1, 1, 1, 1, 1, 1, 1))
+assert(_G.f() == 11)
+assert(_G.f() == 12)
+
+
+if not T then
+  (Message or print)('\a\n >>> testC not active: skipping yield/hook tests <<<\n\a')
+else
+
+  local turn
+  
+  function fact (t, x)
+    assert(turn == t)
+    if x == 0 then return 1
+    else return x*fact(t, x-1)
+    end
+  end
+
+  local A,B,a,b = 0,0,0,0
+
+  local x = coroutine.create(function ()
+    T.setyhook("", 2)
+    A = fact("A", 10)
+  end)
+
+  local y = coroutine.create(function ()
+    T.setyhook("", 3)
+    B = fact("B", 11)
+  end)
+
+  while A==0 or B==0 do
+    if A==0 then turn = "A"; T.resume(x) end
+    if B==0 then turn = "B"; T.resume(y) end
+  end
+
+  assert(B/A == 11)
+end
+
+
+-- leaving a pending coroutine open
+_X = coroutine.wrap(function ()
+      local a = 10
+      local x = function () a = a+1 end
+      coroutine.yield()
+    end)
+
+_X()
+
+
+-- coroutine environments
+co = coroutine.create(function ()
+       coroutine.yield(getfenv(0))
+       return loadstring("return a")()
+     end)
+
+a = {a = 15}
+debug.setfenv(co, a)
+assert(debug.getfenv(co) == a)
+assert(select(2, coroutine.resume(co)) == a)
+assert(select(2, coroutine.resume(co)) == a.a)
+
+
+print'OK'

lib/lua/lua-tests/code.lua

@@ -0,0 +1,143 @@
+
+if T==nil then
+  (Message or print)('\a\n >>> testC not active: skipping opcode tests <<<\n\a')
+  return
+end
+print "testing code generation and optimizations"
+
+
+-- this code gave an error for the code checker
+do
+  local function f (a)
+  for k,v,w in a do end
+  end
+end
+
+
+function check (f, ...)
+  local c = T.listcode(f)
+  for i=1, arg.n do
+    -- print(arg[i], c[i])
+    assert(string.find(c[i], '- '..arg[i]..' *%d'))
+  end
+  assert(c[arg.n+2] == nil)
+end
+
+
+function checkequal (a, b)
+  a = T.listcode(a)
+  b = T.listcode(b)
+  for i = 1, table.getn(a) do
+    a[i] = string.gsub(a[i], '%b()', '')   -- remove line number
+    b[i] = string.gsub(b[i], '%b()', '')   -- remove line number
+    assert(a[i] == b[i])
+  end
+end
+
+
+-- some basic instructions
+check(function ()
+  (function () end){f()}
+end, 'CLOSURE', 'NEWTABLE', 'GETGLOBAL', 'CALL', 'SETLIST', 'CALL', 'RETURN')
+
+
+-- sequence of LOADNILs
+check(function ()
+  local a,b,c
+  local d; local e;
+  a = nil; d=nil
+end, 'RETURN')
+
+
+-- single return
+check (function (a,b,c) return a end, 'RETURN')
+
+
+-- infinite loops
+check(function () while true do local a = -1 end end,
+'LOADK', 'JMP', 'RETURN')
+
+check(function () while 1 do local a = -1 end end,
+'LOADK', 'JMP', 'RETURN')
+
+check(function () repeat local x = 1 until false end,
+'LOADK', 'JMP', 'RETURN')
+
+check(function () repeat local x until nil end,
+'LOADNIL', 'JMP', 'RETURN')
+
+check(function () repeat local x = 1 until true end,
+'LOADK', 'RETURN')
+
+
+-- concat optimization
+check(function (a,b,c,d) return a..b..c..d end,
+  'MOVE', 'MOVE', 'MOVE', 'MOVE', 'CONCAT', 'RETURN')
+
+-- not
+check(function () return not not nil end, 'LOADBOOL', 'RETURN')
+check(function () return not not false end, 'LOADBOOL', 'RETURN')
+check(function () return not not true end, 'LOADBOOL', 'RETURN')
+check(function () return not not 1 end, 'LOADBOOL', 'RETURN')
+
+-- direct access to locals
+check(function ()
+  local a,b,c,d
+  a = b*2
+  c[4], a[b] = -((a + d/-20.5 - a[b]) ^ a.x), b
+end,
+  'MUL',
+  'DIV', 'ADD', 'GETTABLE', 'SUB', 'GETTABLE', 'POW',
+    'UNM', 'SETTABLE', 'SETTABLE', 'RETURN')
+
+
+-- direct access to constants
+check(function ()
+  local a,b
+  a.x = 0
+  a.x = b
+  a[b] = 'y'
+  a = 1 - a
+  b = 1/a
+  b = 5+4
+  a[true] = false
+end,
+  'SETTABLE', 'SETTABLE', 'SETTABLE', 'SUB', 'DIV', 'LOADK',
+  'SETTABLE', 'RETURN')
+
+local function f () return -((2^8 + -(-1)) % 8)/2 * 4 - 3 end
+
+check(f, 'LOADK', 'RETURN')
+assert(f() == -5)
+
+check(function ()
+  local a,b,c
+  b[c], a = c, b
+  b[a], a = c, b
+  a, b = c, a
+  a = a
+end, 
+  'MOVE', 'MOVE', 'SETTABLE',
+  'MOVE', 'MOVE', 'MOVE', 'SETTABLE',
+  'MOVE', 'MOVE', 'MOVE',
+  -- no code for a = a
+  'RETURN')
+
+
+-- x == nil , x ~= nil
+checkequal(function () if (a==nil) then a=1 end; if a~=nil then a=1 end end,
+           function () if (a==9) then a=1 end; if a~=9 then a=1 end end)
+
+check(function () if a==nil then a=1 end end,
+'GETGLOBAL', 'EQ', 'JMP', 'LOADK', 'SETGLOBAL', 'RETURN')
+
+-- de morgan
+checkequal(function () local a; if not (a or b) then b=a end end,
+           function () local a; if (not a and not b) then b=a end end)
+
+checkequal(function (l) local a; return 0 <= a and a <= l end,
+           function (l) local a; return not (not(a >= 0) or not(a <= l)) end)
+
+
+print 'OK'
+

lib/lua/lua-tests/constructs.lua

@@ -0,0 +1,240 @@
+print "testing syntax"
+
+-- testing priorities
+
+assert(2^3^2 == 2^(3^2));
+assert(2^3*4 == (2^3)*4);
+assert(2^-2 == 1/4 and -2^- -2 == - - -4);
+assert(not nil and 2 and not(2>3 or 3<2));
+assert(-3-1-5 == 0+0-9);
+assert(-2^2 == -4 and (-2)^2 == 4 and 2*2-3-1 == 0);
+assert(2*1+3/3 == 3 and 1+2 .. 3*1 == "33");
+assert(not(2+1 > 3*1) and "a".."b" > "a");
+
+assert(not ((true or false) and nil))
+assert(      true or false  and nil)
+
+local a,b = 1,nil;
+assert(-(1 or 2) == -1 and (1 and 2)+(-1.25 or -4) == 0.75);
+x = ((b or a)+1 == 2 and (10 or a)+1 == 11); assert(x);
+x = (((2<3) or 1) == true and (2<3 and 4) == 4); assert(x);
+
+x,y=1,2;
+assert((x>y) and x or y == 2);
+x,y=2,1;
+assert((x>y) and x or y == 2);
+
+assert(1234567890 == tonumber('1234567890') and 1234567890+1 == 1234567891)
+
+
+-- silly loops
+repeat until 1; repeat until true;
+while false do end; while nil do end;
+
+do  -- test old bug (first name could not be an `upvalue')
+ local a; function f(x) x={a=1}; x={x=1}; x={G=1} end
+end
+
+function f (i)
+  if type(i) ~= 'number' then return i,'jojo'; end;
+  if i > 0 then return i, f(i-1); end;
+end
+
+x = {f(3), f(5), f(10);};
+assert(x[1] == 3 and x[2] == 5 and x[3] == 10 and x[4] == 9 and x[12] == 1);
+assert(x[nil] == nil)
+x = {f'alo', f'xixi', nil};
+assert(x[1] == 'alo' and x[2] == 'xixi' and x[3] == nil);
+x = {f'alo'..'xixi'};
+assert(x[1] == 'aloxixi')
+x = {f{}}
+assert(x[2] == 'jojo' and type(x[1]) == 'table')
+
+
+local f = function (i)
+  if i < 10 then return 'a';
+  elseif i < 20 then return 'b';
+  elseif i < 30 then return 'c';
+  end;
+end
+
+assert(f(3) == 'a' and f(12) == 'b' and f(26) == 'c' and f(100) == nil)
+
+for i=1,1000 do break; end;
+n=100;
+i=3;
+t = {};
+a=nil
+while not a do
+  a=0; for i=1,n do for i=i,1,-1 do a=a+1; t[i]=1; end; end;
+end
+assert(a == n*(n+1)/2 and i==3);
+assert(t[1] and t[n] and not t[0] and not t[n+1])
+
+function f(b)
+  local x = 1;
+  repeat
+    local a;
+    if b==1 then local b=1; x=10; break
+    elseif b==2 then x=20; break;
+    elseif b==3 then x=30;
+    else local a,b,c,d=math.sin(1); x=x+1;
+    end
+  until x>=12;
+  return x;
+end;
+
+assert(f(1) == 10 and f(2) == 20 and f(3) == 30 and f(4)==12)
+
+
+local f = function (i)
+  if i < 10 then return 'a'
+  elseif i < 20 then return 'b'
+  elseif i < 30 then return 'c'
+  else return 8
+  end
+end
+
+assert(f(3) == 'a' and f(12) == 'b' and f(26) == 'c' and f(100) == 8)
+
+local a, b = nil, 23
+x = {f(100)*2+3 or a, a or b+2}
+assert(x[1] == 19 and x[2] == 25)
+x = {f=2+3 or a, a = b+2}
+assert(x.f == 5 and x.a == 25)
+
+a={y=1}
+x = {a.y}
+assert(x[1] == 1)
+
+function f(i)
+  while 1 do
+    if i>0 then i=i-1;
+    else return; end;
+  end;
+end;
+
+function g(i)
+  while 1 do
+    if i>0 then i=i-1
+    else return end
+  end
+end
+
+f(10); g(10);
+
+do
+  function f () return 1,2,3; end
+  local a, b, c = f();
+  assert(a==1 and b==2 and c==3)
+  a, b, c = (f());
+  assert(a==1 and b==nil and c==nil)
+end
+
+local a,b = 3 and f();
+assert(a==1 and b==nil)
+
+function g() f(); return; end;
+assert(g() == nil)
+function g() return nil or f() end
+a,b = g()
+assert(a==1 and b==nil)
+
+print'+';
+
+
+f = [[
+return function ( a , b , c , d , e )
+  local x = a >= b or c or ( d and e ) or nil
+  return x
+end , { a = 1 , b = 2 >= 1 , } or { 1 };
+]]
+f = string.gsub(f, "%s+", "\n");   -- force a SETLINE between opcodes
+f,a = loadstring(f)();
+assert(a.a == 1 and a.b)
+
+function g (a,b,c,d,e)
+  if not (a>=b or c or d and e or nil) then return 0; else return 1; end;
+end
+
+function h (a,b,c,d,e)
+  while (a>=b or c or (d and e) or nil) do return 1; end;
+  return 0;
+end;
+
+assert(f(2,1) == true and g(2,1) == 1 and h(2,1) == 1)
+assert(f(1,2,'a') == 'a' and g(1,2,'a') == 1 and h(1,2,'a') == 1)
+assert(f(1,2,'a')
+~=          -- force SETLINE before nil
+nil, "")
+assert(f(1,2,'a') == 'a' and g(1,2,'a') == 1 and h(1,2,'a') == 1)
+assert(f(1,2,nil,1,'x') == 'x' and g(1,2,nil,1,'x') == 1 and
+                                   h(1,2,nil,1,'x') == 1)
+assert(f(1,2,nil,nil,'x') == nil and g(1,2,nil,nil,'x') == 0 and
+                                     h(1,2,nil,nil,'x') == 0)
+assert(f(1,2,nil,1,nil) == nil and g(1,2,nil,1,nil) == 0 and
+                                   h(1,2,nil,1,nil) == 0)
+
+assert(1 and 2<3 == true and 2<3 and 'a'<'b' == true)
+x = 2<3 and not 3; assert(x==false)
+x = 2<1 or (2>1 and 'a'); assert(x=='a')
+
+
+do
+  local a; if nil then a=1; else a=2; end;    -- this nil comes as PUSHNIL 2
+  assert(a==2)
+end
+
+function F(a)
+  assert(debug.getinfo(1, "n").name == 'F')
+  return a,2,3
+end
+
+a,b = F(1)~=nil; assert(a == true and b == nil);
+a,b = F(nil)==nil; assert(a == true and b == nil)
+
+----------------------------------------------------------------
+-- creates all combinations of 
+-- [not] ([not] arg op [not] (arg op [not] arg ))
+-- and tests each one
+
+function ID(x) return x end
+
+function f(t, i)
+  local b = t.n
+  local res = math.mod(math.floor(i/c), b)+1
+  c = c*b
+  return t[res]
+end
+
+local arg = {" ( 1 < 2 ) ", " ( 1 >= 2 ) ", " F ( ) ", "  nil "; n=4}
+
+local op = {" and ", " or ", " == ", " ~= "; n=4}
+
+local neg = {" ", " not "; n=2}
+
+local i = 0
+repeat
+  c = 1
+  local s = f(neg, i)..'ID('..f(neg, i)..f(arg, i)..f(op, i)..
+            f(neg, i)..'ID('..f(arg, i)..f(op, i)..f(neg, i)..f(arg, i)..'))'
+  local s1 = string.gsub(s, 'ID', '')
+  K,X,NX,WX1,WX2 = nil
+  s = string.format([[
+      local a = %s
+      local b = not %s
+      K = b
+      local xxx; 
+      if %s then X = a  else X = b end
+      if %s then NX = b  else NX = a end
+      while %s do WX1 = a; break end
+      while %s do WX2 = a; break end
+      repeat if (%s) then break end; assert(b)  until not(%s)
+  ]], s1, s, s1, s, s1, s, s1, s, s)
+  assert(loadstring(s))()
+  assert(X and not NX and not WX1 == K and not WX2 == K)
+  if math.mod(i,4000) == 0 then print('+') end
+  i = i+1
+until i==c
+
+print'OK'

lib/lua/lua-tests/db.lua

@@ -0,0 +1,499 @@
+-- testing debug library
+
+local function dostring(s) return assert(loadstring(s))() end
+
+print"testing debug library and debug information"
+
+do
+local a=1
+end
+
+function test (s, l, p)
+  collectgarbage()   -- avoid gc during trace
+  local function f (event, line)
+    assert(event == 'line')
+    local l = table.remove(l, 1)
+    if p then print(l, line) end
+    assert(l == line, "wrong trace!!")
+  end
+  debug.sethook(f,"l"); loadstring(s)(); debug.sethook()
+  assert(table.getn(l) == 0)
+end
+
+
+do
+  local a = debug.getinfo(print)
+  assert(a.what == "C" and a.short_src == "[C]")
+  local b = debug.getinfo(test, "SfL")
+  assert(b.name == nil and b.what == "Lua" and b.linedefined == 11 and
+         b.lastlinedefined == b.linedefined + 10 and
+         b.func == test and not string.find(b.short_src, "%["))
+  assert(b.activelines[b.linedefined + 1] and
+         b.activelines[b.lastlinedefined])
+  assert(not b.activelines[b.linedefined] and
+         not b.activelines[b.lastlinedefined + 1])
+end
+
+
+-- test file and string names truncation
+a = "function f () end"
+local function dostring (s, x) return loadstring(s, x)() end
+dostring(a)
+assert(debug.getinfo(f).short_src == string.format('[string "%s"]', a))
+dostring(a..string.format("; %s\n=1", string.rep('p', 400)))
+assert(string.find(debug.getinfo(f).short_src, '^%[string [^\n]*%.%.%."%]$'))
+dostring("\n"..a)
+assert(debug.getinfo(f).short_src == '[string "..."]')
+dostring(a, "")
+assert(debug.getinfo(f).short_src == '[string ""]')
+dostring(a, "@xuxu")
+assert(debug.getinfo(f).short_src == "xuxu")
+dostring(a, "@"..string.rep('p', 1000)..'t')
+assert(string.find(debug.getinfo(f).short_src, "^%.%.%.p*t$"))
+dostring(a, "=xuxu")
+assert(debug.getinfo(f).short_src == "xuxu")
+dostring(a, string.format("=%s", string.rep('x', 500)))
+assert(string.find(debug.getinfo(f).short_src, "^x*"))
+dostring(a, "=")
+assert(debug.getinfo(f).short_src == "")
+a = nil; f = nil;
+
+
+repeat
+  local g = {x = function ()
+    local a = debug.getinfo(2)
+    assert(a.name == 'f' and a.namewhat == 'local')
+    a = debug.getinfo(1)
+    assert(a.name == 'x' and a.namewhat == 'field')
+    return 'xixi'
+  end}
+  local f = function () return 1+1 and (not 1 or g.x()) end
+  assert(f() == 'xixi')
+  g = debug.getinfo(f)
+  assert(g.what == "Lua" and g.func == f and g.namewhat == "" and not g.name)
+
+  function f (x, name)   -- local!
+    name = name or 'f'
+    local a = debug.getinfo(1)
+    assert(a.name == name and a.namewhat == 'local')
+    return x
+  end
+
+  -- breaks in different conditions
+  if 3>4 then break end; f()
+  if 3<4 then a=1 else break end; f()
+  while 1 do local x=10; break end; f()
+  local b = 1
+  if 3>4 then return math.sin(1) end; f()
+  a = 3<4; f()
+  a = 3<4 or 1; f()
+  repeat local x=20; if 4>3 then f() else break end; f() until 1
+  g = {}
+  f(g).x = f(2) and f(10)+f(9)
+  assert(g.x == f(19))
+  function g(x) if not x then return 3 end return (x('a', 'x')) end
+  assert(g(f) == 'a')
+until 1
+
+test([[if
+math.sin(1)
+then
+  a=1
+else
+  a=2
+end
+]], {2,4,7})
+
+test([[--
+if nil then
+  a=1
+else
+  a=2
+end
+]], {2,5,6})
+
+test([[a=1
+repeat
+  a=a+1
+until a==3
+]], {1,3,4,3,4})
+
+test([[ do
+  return
+end
+]], {2})
+
+test([[local a
+a=1
+while a<=3 do
+  a=a+1
+end
+]], {2,3,4,3,4,3,4,3,5})
+
+test([[while math.sin(1) do
+  if math.sin(1)
+  then
+    break
+  end
+end
+a=1]], {1,2,4,7})
+
+test([[for i=1,3 do
+  a=i
+end
+]], {1,2,1,2,1,2,1,3})
+
+test([[for i,v in pairs{'a','b'} do
+  a=i..v
+end
+]], {1,2,1,2,1,3})
+
+test([[for i=1,4 do a=1 end]], {1,1,1,1,1})
+
+
+
+print'+'
+
+a = {}; L = nil
+local glob = 1
+local oldglob = glob
+debug.sethook(function (e,l)
+  collectgarbage()   -- force GC during a hook
+  local f, m, c = debug.gethook()
+  assert(m == 'crl' and c == 0)
+  if e == "line" then
+    if glob ~= oldglob then
+      L = l-1   -- get the first line where "glob" has changed
+      oldglob = glob
+    end
+  elseif e == "call" then
+      local f = debug.getinfo(2, "f").func
+      a[f] = 1
+  else assert(e == "return")
+  end
+end, "crl")
+
+function f(a,b)
+  collectgarbage()
+  local _, x = debug.getlocal(1, 1)
+  local _, y = debug.getlocal(1, 2)
+  assert(x == a and y == b)
+  assert(debug.setlocal(2, 3, "pera") == "AA".."AA")
+  assert(debug.setlocal(2, 4, "maçã") == "B")
+  x = debug.getinfo(2)
+  assert(x.func == g and x.what == "Lua" and x.name == 'g' and
+         x.nups == 0 and string.find(x.source, "^@.*db%.lua"))
+  glob = glob+1
+  assert(debug.getinfo(1, "l").currentline == L+1)
+  assert(debug.getinfo(1, "l").currentline == L+2)
+end
+
+function foo()
+  glob = glob+1
+  assert(debug.getinfo(1, "l").currentline == L+1)
+end; foo()  -- set L
+-- check line counting inside strings and empty lines
+
+_ = 'alo\
+alo' .. [[
+
+]]
+--[[
+]]
+assert(debug.getinfo(1, "l").currentline == L+11)  -- check count of lines
+
+
+function g(...)
+  do local a,b,c; a=math.sin(40); end
+  local feijao
+  local AAAA,B = "xuxu", "mamão"
+  f(AAAA,B)
+  assert(AAAA == "pera" and B == "maçã")
+  do
+     local B = 13
+     local x,y = debug.getlocal(1,5)
+     assert(x == 'B' and y == 13)
+  end
+end
+
+g()
+
+
+assert(a[f] and a[g] and a[assert] and a[debug.getlocal] and not a[print])
+
+
+-- tests for manipulating non-registered locals (C and Lua temporaries)
+
+local n, v = debug.getlocal(0, 1)
+assert(v == 0 and n == "(*temporary)")
+local n, v = debug.getlocal(0, 2)
+assert(v == 2 and n == "(*temporary)")
+assert(not debug.getlocal(0, 3))
+assert(not debug.getlocal(0, 0))
+
+function f()
+  assert(select(2, debug.getlocal(2,3)) == 1)
+  assert(not debug.getlocal(2,4))
+  debug.setlocal(2, 3, 10)
+  return 20
+end
+
+function g(a,b) return (a+1) + f() end
+
+assert(g(0,0) == 30)
+ 
+
+debug.sethook(nil);
+assert(debug.gethook() == nil)
+
+
+-- testing access to function arguments
+
+X = nil
+a = {}
+function a:f (a, b, ...) local c = 13 end
+debug.sethook(function (e)
+  assert(e == "call")
+  dostring("XX = 12")  -- test dostring inside hooks
+  -- testing errors inside hooks
+  assert(not pcall(loadstring("a='joao'+1")))
+  debug.sethook(function (e, l) 
+    assert(debug.getinfo(2, "l").currentline == l)
+    local f,m,c = debug.gethook()
+    assert(e == "line")
+    assert(m == 'l' and c == 0)
+    debug.sethook(nil)  -- hook is called only once
+    assert(not X)       -- check that
+    X = {}; local i = 1
+    local x,y
+    while 1 do
+      x,y = debug.getlocal(2, i)
+      if x==nil then break end
+      X[x] = y
+      i = i+1
+    end
+  end, "l")
+end, "c")
+
+a:f(1,2,3,4,5)
+assert(X.self == a and X.a == 1   and X.b == 2 and X.arg.n == 3 and X.c == nil)
+assert(XX == 12)
+assert(debug.gethook() == nil)
+
+
+-- testing upvalue access
+local function getupvalues (f)
+  local t = {}
+  local i = 1
+  while true do
+    local name, value = debug.getupvalue(f, i)
+    if not name then break end
+    assert(not t[name])
+    t[name] = value
+    i = i + 1
+  end
+  return t
+end
+
+local a,b,c = 1,2,3
+local function foo1 (a) b = a; return c end
+local function foo2 (x) a = x; return c+b end
+assert(debug.getupvalue(foo1, 3) == nil)
+assert(debug.getupvalue(foo1, 0) == nil)
+assert(debug.setupvalue(foo1, 3, "xuxu") == nil)
+local t = getupvalues(foo1)
+assert(t.a == nil and t.b == 2 and t.c == 3)
+t = getupvalues(foo2)
+assert(t.a == 1 and t.b == 2 and t.c == 3)
+assert(debug.setupvalue(foo1, 1, "xuxu") == "b")
+assert(({debug.getupvalue(foo2, 3)})[2] == "xuxu")
+-- cannot manipulate C upvalues from Lua
+assert(debug.getupvalue(io.read, 1) == nil)  
+assert(debug.setupvalue(io.read, 1, 10) == nil)  
+
+
+-- testing count hooks
+local a=0
+debug.sethook(function (e) a=a+1 end, "", 1)
+a=0; for i=1,1000 do end; assert(1000 < a and a < 1012)
+debug.sethook(function (e) a=a+1 end, "", 4)
+a=0; for i=1,1000 do end; assert(250 < a and a < 255)
+local f,m,c = debug.gethook()
+assert(m == "" and c == 4)
+debug.sethook(function (e) a=a+1 end, "", 4000)
+a=0; for i=1,1000 do end; assert(a == 0)
+debug.sethook(print, "", 2^24 - 1)   -- count upperbound
+local f,m,c = debug.gethook()
+assert(({debug.gethook()})[3] == 2^24 - 1)
+debug.sethook()
+
+
+-- tests for tail calls
+local function f (x)
+  if x then
+    assert(debug.getinfo(1, "S").what == "Lua")
+    local tail = debug.getinfo(2)
+    assert(not pcall(getfenv, 3))
+    assert(tail.what == "tail" and tail.short_src == "(tail call)" and
+           tail.linedefined == -1 and tail.func == nil)
+    assert(debug.getinfo(3, "f").func == g1)
+    assert(getfenv(3))
+    assert(debug.getinfo(4, "S").what == "tail")
+    assert(not pcall(getfenv, 5))
+    assert(debug.getinfo(5, "S").what == "main")
+    assert(getfenv(5))
+    print"+"
+    end
+end
+
+function g(x) return f(x) end
+
+function g1(x) g(x) end
+
+local function h (x) local f=g1; return f(x) end
+
+h(true)
+
+local b = {}
+debug.sethook(function (e) table.insert(b, e) end, "cr")
+h(false)
+debug.sethook()
+local res = {"return",   -- first return (from sethook)
+  "call", "call", "call", "call",
+  "return", "tail return", "return", "tail return",
+  "call",    -- last call (to sethook)
+}
+for _, k in ipairs(res) do assert(k == table.remove(b, 1)) end
+
+
+lim = 30000
+local function foo (x)
+  if x==0 then
+    assert(debug.getinfo(lim+2).what == "main")
+    for i=2,lim do assert(debug.getinfo(i, "S").what == "tail") end
+  else return foo(x-1)
+  end
+end
+
+foo(lim)
+
+
+print"+"
+
+
+-- testing traceback
+
+assert(debug.traceback(print) == print)
+assert(debug.traceback(print, 4) == print)
+assert(string.find(debug.traceback("hi", 4), "^hi\n"))
+assert(string.find(debug.traceback("hi"), "^hi\n"))
+assert(not string.find(debug.traceback("hi"), "'traceback'"))
+assert(string.find(debug.traceback("hi", 0), "'traceback'"))
+assert(string.find(debug.traceback(), "^stack traceback:\n"))
+
+-- testing debugging of coroutines
+
+local function checktraceback (co, p)
+  local tb = debug.traceback(co)
+  local i = 0
+  for l in string.gmatch(tb, "[^\n]+\n?") do
+    assert(i == 0 or string.find(l, p[i]))
+    i = i+1
+  end
+  assert(p[i] == nil)
+end
+
+
+local function f (n)
+  if n > 0 then return f(n-1)
+  else coroutine.yield() end
+end
+
+local co = coroutine.create(f)
+coroutine.resume(co, 3)
+checktraceback(co, {"yield", "db.lua", "tail", "tail", "tail"})
+
+
+co = coroutine.create(function (x)
+       local a = 1
+       coroutine.yield(debug.getinfo(1, "l"))
+       coroutine.yield(debug.getinfo(1, "l").currentline)
+       return a
+     end)
+
+local tr = {}
+local foo = function (e, l) table.insert(tr, l) end
+debug.sethook(co, foo, "l")
+
+local _, l = coroutine.resume(co, 10)
+local x = debug.getinfo(co, 1, "lfLS")
+assert(x.currentline == l.currentline and x.activelines[x.currentline])
+assert(type(x.func) == "function")
+for i=x.linedefined + 1, x.lastlinedefined do
+  assert(x.activelines[i])
+  x.activelines[i] = nil
+end
+assert(next(x.activelines) == nil)   -- no 'extra' elements
+assert(debug.getinfo(co, 2) == nil)
+local a,b = debug.getlocal(co, 1, 1)
+assert(a == "x" and b == 10)
+a,b = debug.getlocal(co, 1, 2)
+assert(a == "a" and b == 1)
+debug.setlocal(co, 1, 2, "hi")
+assert(debug.gethook(co) == foo)
+assert(table.getn(tr) == 2 and
+       tr[1] == l.currentline-1 and tr[2] == l.currentline)
+
+a,b,c = pcall(coroutine.resume, co)
+assert(a and b and c == l.currentline+1)
+checktraceback(co, {"yield", "in function <"})
+
+a,b = coroutine.resume(co)
+assert(a and b == "hi")
+assert(table.getn(tr) == 4 and tr[4] == l.currentline+2)
+assert(debug.gethook(co) == foo)
+assert(debug.gethook() == nil)
+checktraceback(co, {})
+
+
+-- check traceback of suspended (or dead with error) coroutines
+
+function f(i) if i==0 then error(i) else coroutine.yield(); f(i-1) end end
+
+co = coroutine.create(function (x) f(x) end)
+a, b = coroutine.resume(co, 3)
+t = {"'yield'", "'f'", "in function <"}
+while coroutine.status(co) == "suspended" do
+  checktraceback(co, t)
+  a, b = coroutine.resume(co)
+  table.insert(t, 2, "'f'")   -- one more recursive call to 'f'
+end
+t[1] = "'error'"
+checktraceback(co, t)
+
+
+-- test acessing line numbers of a coroutine from a resume inside
+-- a C function (this is a known bug in Lua 5.0)
+
+local function g(x)
+    coroutine.yield(x)
+end
+
+local function f (i)
+  debug.sethook(function () end, "l")
+  for j=1,1000 do
+    g(i+j)
+  end
+end
+
+local co = coroutine.wrap(f)
+co(10)
+pcall(co)
+pcall(co)
+
+
+assert(type(debug.getregistry()) == "table")
+
+
+print"OK"
+