From 17d6f58cc5aa3146e1a5b866e5edda7a7b1ae822 Mon Sep 17 00:00:00 2001
From: giles <giles.bradshaw@rose-ash.com>
Date: Fri, 8 May 2026 14:41:05 +0000
Subject: [PATCH] datalog: dl-magic-rewrite worklist now drains across rule
 chains (239/239)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Real bug: the worklist used (set! queue (rest queue)) to pop the
head, which left queue bound to a fresh empty list as soon as the
last item was popped. Subsequent (append! queue ...) was a no-op
on the empty list — so when the head's rewrite generated new
(rel, adn) pairs to enqueue, they vanished. Multi-relation
programs (e.g. shortest -> path -> edge, or chained derived
relations) only had their head's rules rewritten; downstream
rules silently dropped.

Fix: use an index-based loop (idx 0 → len queue), with append!
adding to the same list. Items added after the current pointer
are picked up in subsequent iterations.

2 new regression tests:
- 4-level chain (a → r1 → r2 → r3 → r4) under magic returns 2
- shortest-path demo via magic equals dl-query (1 result)
---
 lib/datalog/magic.sx        | 41 +++++++++++++++++++------------------
 lib/datalog/scoreboard.json |  8 ++++----
 lib/datalog/scoreboard.md   |  4 ++--
 lib/datalog/tests/magic.sx  | 26 +++++++++++++++++++++++
 4 files changed, 53 insertions(+), 26 deletions(-)

diff --git a/lib/datalog/magic.sx b/lib/datalog/magic.sx
index 97168a4e..ffdf4cf4 100644
--- a/lib/datalog/magic.sx
+++ b/lib/datalog/magic.sx
@@ -314,26 +314,27 @@
 
         (dl-mq-mark! query-rel query-adornment)
 
-        (define
-          dl-mq-process
-          (fn
-            ()
-            (when
-              (> (len queue) 0)
-              (let ((item (first queue)))
-                (do
-                  (set! queue (rest queue))
-                  (let
-                    ((rel (get item :rel)) (adn (get item :adn)))
-                    (for-each
-                      (fn
-                        (rule)
-                        (when
-                          (= (dl-rel-name (get rule :head)) rel)
-                          (dl-mq-rewrite-rule! rule adn)))
-                      rules))
-                  (dl-mq-process))))))
-        (dl-mq-process)
+        (let ((idx 0))
+          (define
+            dl-mq-process
+            (fn
+              ()
+              (when
+                (< idx (len queue))
+                (let ((item (nth queue idx)))
+                  (do
+                    (set! idx (+ idx 1))
+                    (let
+                      ((rel (get item :rel)) (adn (get item :adn)))
+                      (for-each
+                        (fn
+                          (rule)
+                          (when
+                            (= (dl-rel-name (get rule :head)) rel)
+                            (dl-mq-rewrite-rule! rule adn)))
+                        rules))
+                    (dl-mq-process))))))
+          (dl-mq-process))
 
         {:rules out
          :seed
diff --git a/lib/datalog/scoreboard.json b/lib/datalog/scoreboard.json
index 60f7f14d..c5e44ea7 100644
--- a/lib/datalog/scoreboard.json
+++ b/lib/datalog/scoreboard.json
@@ -1,8 +1,8 @@
 {
   "lang": "datalog",
-  "total_passed": 237,
+  "total_passed": 239,
   "total_failed": 0,
-  "total": 237,
+  "total": 239,
   "suites": [
     {"name":"tokenize","passed":26,"failed":0,"total":26},
     {"name":"parse","passed":20,"failed":0,"total":20},
@@ -13,8 +13,8 @@
     {"name":"negation","passed":10,"failed":0,"total":10},
     {"name":"aggregates","passed":20,"failed":0,"total":20},
     {"name":"api","passed":20,"failed":0,"total":20},
-    {"name":"magic","passed":29,"failed":0,"total":29},
+    {"name":"magic","passed":31,"failed":0,"total":31},
     {"name":"demo","passed":21,"failed":0,"total":21}
   ],
-  "generated": "2026-05-08T14:29:08+00:00"
+  "generated": "2026-05-08T14:40:44+00:00"
 }
diff --git a/lib/datalog/scoreboard.md b/lib/datalog/scoreboard.md
index cec0e73e..6e176c8b 100644
--- a/lib/datalog/scoreboard.md
+++ b/lib/datalog/scoreboard.md
@@ -1,6 +1,6 @@
 # datalog scoreboard
 
-**237 / 237 passing** (0 failure(s)).
+**239 / 239 passing** (0 failure(s)).
 
 | Suite | Passed | Total | Status |
 |-------|--------|-------|--------|
@@ -13,5 +13,5 @@
 | negation | 10 | 10 | ok |
 | aggregates | 20 | 20 | ok |
 | api | 20 | 20 | ok |
-| magic | 29 | 29 | ok |
+| magic | 31 | 31 | ok |
 | demo | 21 | 21 | ok |
diff --git a/lib/datalog/tests/magic.sx b/lib/datalog/tests/magic.sx
index 985ba4dc..00edf4da 100644
--- a/lib/datalog/tests/magic.sx
+++ b/lib/datalog/tests/magic.sx
@@ -242,6 +242,32 @@
             (= (len semi) (len magic))))
         true)
 
+      ;; Multi-relation rewrite chain: query r4 → propagate to r3,
+      ;; r2, r1, a. The worklist must process all of them; an
+      ;; earlier bug stopped after only the head pair.
+      (dl-mt-test! "magic chain through 4 rule levels"
+        (let
+          ((db (dl-program
+                 "a(1). a(2). r1(X) :- a(X). r2(X) :- r1(X).
+                  r3(X) :- r2(X). r4(X) :- r3(X).")))
+          (= 2 (len (dl-magic-query db (list (quote r4) (quote X))))))
+        true)
+
+      ;; Shortest-path demo via magic — exercises the rewriter
+      ;; against rules that mix recursive positive lits with an
+      ;; aggregate body literal.
+      (dl-mt-test! "magic on shortest-path demo"
+        (let
+          ((db (dl-program-data
+                 (quote ((edge a b 5) (edge b c 3) (edge a c 10)))
+                 dl-demo-shortest-path-rules)))
+          (let
+            ((semi (dl-query db (quote (shortest a c W))))
+              (magic (dl-magic-query db (quote (shortest a c W)))))
+            (and (= (len semi) (len magic))
+                 (= (len semi) 1))))
+        true)
+
       ;; Magic over a rule whose body contains an aggregate.
       ;; The rewriter passes aggregate body lits through unchanged
       ;; (no propagation generated for them), so semi-naive's count