diff --git a/lib/artdag/conformance.sh b/lib/artdag/conformance.sh index 050b6687..c2d19bba 100755 --- a/lib/artdag/conformance.sh +++ b/lib/artdag/conformance.sh @@ -13,7 +13,7 @@ if [ ! -x "$SX_SERVER" ]; then exit 1 fi -SUITES=(dag analyze plan execute optimize fed cost serialize) +SUITES=(dag analyze plan execute optimize fed cost serialize stats) OUT_JSON="lib/artdag/scoreboard.json" OUT_MD="lib/artdag/scoreboard.md" @@ -49,6 +49,7 @@ run_suite() { (load "lib/artdag/federation.sx") (load "lib/artdag/cost.sx") (load "lib/artdag/serialize.sx") +(load "lib/artdag/stats.sx") (epoch 2) (eval "(define artdag-test-pass 0)") (eval "(define artdag-test-fail 0)") diff --git a/lib/artdag/scoreboard.json b/lib/artdag/scoreboard.json index f46251b7..2b4698d7 100644 --- a/lib/artdag/scoreboard.json +++ b/lib/artdag/scoreboard.json @@ -7,9 +7,10 @@ "optimize": {"pass": 22, "fail": 0}, "fed": {"pass": 15, "fail": 0}, "cost": {"pass": 13, "fail": 0}, - "serialize": {"pass": 13, "fail": 0} + "serialize": {"pass": 13, "fail": 0}, + "stats": {"pass": 12, "fail": 0} }, - "total_pass": 132, + "total_pass": 144, "total_fail": 0, - "total": 132 + "total": 144 } diff --git a/lib/artdag/scoreboard.md b/lib/artdag/scoreboard.md index 5d1a483c..5e61f292 100644 --- a/lib/artdag/scoreboard.md +++ b/lib/artdag/scoreboard.md @@ -12,4 +12,5 @@ _Generated by `lib/artdag/conformance.sh`_ | fed | 15 | 0 | 15 | | cost | 13 | 0 | 13 | | serialize | 13 | 0 | 13 | -| **Total** | **132** | **0** | **132** | +| stats | 12 | 0 | 12 | +| **Total** | **144** | **0** | **144** | diff --git a/lib/artdag/stats.sx b/lib/artdag/stats.sx new file mode 100644 index 00000000..5ddf1e10 --- /dev/null +++ b/lib/artdag/stats.sx @@ -0,0 +1,51 @@ +; lib/artdag/stats.sx — observability over an execution: cache hit ratio and the +; compute work saved by memoization (weighted by the cost model). An exec is the +; {:results :recomputed :hits} record returned by artdag/execute. Depends on +; execute.sx (exec accessors) and cost.sx (artdag/-node-cost). + +(define + artdag/exec-total + (fn (exec) (+ (artdag/recompute-count exec) (artdag/hit-count exec)))) + +; fraction of executed nodes served from cache (0 when nothing ran). +(define + artdag/hit-ratio + (fn + (exec) + (let + ((n (artdag/exec-total exec))) + (if (= n 0) 0 (/ (artdag/hit-count exec) n))))) + +(define + artdag/-sum-cost + (fn + (dag cost-fn ids) + (reduce + (fn (s id) (+ s (artdag/-node-cost dag cost-fn id))) + 0 + ids))) + +; weighted compute work that actually ran this execution. +(define + artdag/work-recomputed + (fn + (exec dag cost-fn) + (artdag/-sum-cost dag cost-fn (get exec :recomputed)))) + +; weighted compute work avoided by cache hits. +(define + artdag/work-saved + (fn (exec dag cost-fn) (artdag/-sum-cost dag cost-fn (get exec :hits)))) + +; fraction of total weighted work that the cache saved (0 when no work at all). +(define + artdag/savings-ratio + (fn + (exec dag cost-fn) + (let + ((saved (artdag/work-saved exec dag cost-fn)) + (ran (artdag/work-recomputed exec dag cost-fn))) + (if (= (+ saved ran) 0) 0 (/ saved (+ saved ran)))))) + +; compact summary dict for logging. +(define artdag/exec-summary (fn (exec dag cost-fn) {:work-saved (artdag/work-saved exec dag cost-fn) :recomputed (artdag/recompute-count exec) :total (artdag/exec-total exec) :work-ran (artdag/work-recomputed exec dag cost-fn) :hits (artdag/hit-count exec)})) diff --git a/lib/artdag/tests/stats.sx b/lib/artdag/tests/stats.sx new file mode 100644 index 00000000..c143bff1 --- /dev/null +++ b/lib/artdag/tests/stats.sx @@ -0,0 +1,150 @@ +; execution stats: hit ratio + memoized work saved (cost-weighted). + +(define st-RT (artdag/op-table-runner {:in (fn (p i) (get p :v)) :add (fn (p i) (+ (nth i 0) (nth i 1))) :inc (fn (p i) (+ 1 (first i)))})) + +(define + st-D + (artdag/build + (list + (list "p" "in" (list) {:v 10}) + (list "q" "in" (list) {:v 20}) + (list "b" "inc" (list "p") {}) + (list "c" "inc" (list "q") {}) + (list "d" "add" (list "b" "c") {} true)))) + +; same shape, leaf q changed -> dirty closure {q,c,d} +(define + st-D2 + (artdag/build + (list + (list "p" "in" (list) {:v 10}) + (list "q" "in" (list) {:v 21}) + (list "b" "inc" (list "p") {}) + (list "c" "inc" (list "q") {}) + (list "d" "add" (list "b" "c") {} true)))) + +(define st-W (artdag/op-cost {:add 5 :inc 2})) + +; ---- cold run ---- + +(artdag-test + "cold run: hit ratio is zero" + (let + ((cache (persist/open))) + (artdag/hit-ratio (artdag/run st-D st-RT cache))) + 0) + +(artdag-test + "cold run: nothing saved" + (let + ((cache (persist/open))) + (artdag/work-saved (artdag/run st-D st-RT cache) st-D artdag/const-cost)) + 0) + +(artdag-test + "cold run: all work runs" + (let + ((cache (persist/open))) + (artdag/work-recomputed + (artdag/run st-D st-RT cache) + st-D + artdag/const-cost)) + 5) + +(artdag-test + "cold run: weighted work ran" + (let + ((cache (persist/open))) + (artdag/work-recomputed (artdag/run st-D st-RT cache) st-D st-W)) + 11) + +; ---- warm rerun ---- + +(artdag-test + "warm rerun: hit ratio is one" + (let + ((cache (persist/open))) + (begin + (artdag/run st-D st-RT cache) + (artdag/hit-ratio (artdag/run st-D st-RT cache)))) + 1) + +(artdag-test + "warm rerun: savings ratio is one" + (let + ((cache (persist/open))) + (begin + (artdag/run st-D st-RT cache) + (artdag/savings-ratio + (artdag/run st-D st-RT cache) + st-D + artdag/const-cost))) + 1) + +(artdag-test + "warm rerun: all weighted work saved" + (let + ((cache (persist/open))) + (begin + (artdag/run st-D st-RT cache) + (artdag/work-saved (artdag/run st-D st-RT cache) st-D st-W))) + 11) + +; ---- partial (incremental) ---- + +(artdag-test + "incremental: total is every node" + (let + ((cache (persist/open))) + (begin + (artdag/run st-D st-RT cache) + (artdag/exec-total (artdag/run st-D2 st-RT cache)))) + 5) + +(artdag-test + "incremental: saved work counts unchanged nodes" + (let + ((cache (persist/open))) + (begin + (artdag/run st-D st-RT cache) + (artdag/work-saved + (artdag/run st-D2 st-RT cache) + st-D2 + artdag/const-cost))) + 2) + +(artdag-test + "incremental: ran work counts dirty closure" + (let + ((cache (persist/open))) + (begin + (artdag/run st-D st-RT cache) + (artdag/work-recomputed + (artdag/run st-D2 st-RT cache) + st-D2 + artdag/const-cost))) + 3) + +(artdag-test + "summary reports recompute count" + (let + ((cache (persist/open))) + (get + (artdag/exec-summary + (artdag/run st-D st-RT cache) + st-D + artdag/const-cost) + :recomputed)) + 5) + +(artdag-test + "summary reports total" + (let + ((cache (persist/open))) + (get + (artdag/exec-summary + (artdag/run st-D st-RT cache) + st-D + artdag/const-cost) + :total)) + 5) diff --git a/plans/artdag-on-sx.md b/plans/artdag-on-sx.md index b8019bee..1a105d98 100644 --- a/plans/artdag-on-sx.md +++ b/plans/artdag-on-sx.md @@ -30,7 +30,7 @@ edges. ## Status (rolling) -`bash lib/artdag/conformance.sh` → **132/132** (8 suites: dag, analyze, plan, execute, optimize, fed, cost, serialize) +`bash lib/artdag/conformance.sh` → **144/144** (9 suites: dag, analyze, plan, execute, optimize, fed, cost, serialize, stats) Base roadmap (Phases 1–6) COMPLETE. Now extending. @@ -138,6 +138,12 @@ lib/artdag/optimize.sx lib/artdag/federation.sx ## Progress log +- **Ext: execution stats / cache analytics** (stats suite 12/12, total 144/144). + `lib/artdag/stats.sx` over an exec record: `hit-ratio`, `work-recomputed`/`work-saved` + (cost-weighted via the cost model), `savings-ratio`, and `exec-summary`. Cold run = + 0 hit ratio / all work ran; warm rerun = ratio 1 / all work saved; incremental = saved + work counts unchanged nodes, ran work counts the dirty closure. + - **Ext: optimize composition pass** (optimize suite 22/22, total 132/132). `artdag/optimize entries outputs fusible?` fuses the entry list then DCEs against the output names (sinks survive fusion since they're never absorbed) — fewer nodes,