diff --git a/lib/artdag/conformance.sh b/lib/artdag/conformance.sh index 0551abb3..ff32eaf2 100755 --- a/lib/artdag/conformance.sh +++ b/lib/artdag/conformance.sh @@ -13,7 +13,7 @@ if [ ! -x "$SX_SERVER" ]; then exit 1 fi -SUITES=(dag analyze plan execute optimize fed) +SUITES=(dag analyze plan execute optimize fed cost) OUT_JSON="lib/artdag/scoreboard.json" OUT_MD="lib/artdag/scoreboard.md" @@ -47,6 +47,7 @@ run_suite() { (load "lib/artdag/execute.sx") (load "lib/artdag/optimize.sx") (load "lib/artdag/federation.sx") +(load "lib/artdag/cost.sx") (epoch 2) (eval "(define artdag-test-pass 0)") (eval "(define artdag-test-fail 0)") diff --git a/lib/artdag/cost.sx b/lib/artdag/cost.sx new file mode 100644 index 00000000..6cbf9547 --- /dev/null +++ b/lib/artdag/cost.sx @@ -0,0 +1,66 @@ +; lib/artdag/cost.sx — cost model for the scheduler: per-node weights, critical +; path (min makespan with unlimited parallelism), plan makespan under batching/cap, +; total serial work, and the resulting speedup. Costs come from an injected +; cost-fn (op params) -> number so media-op costs stay opaque. Depends on dag.sx. + +(define artdag/const-cost (fn (op params) 1)) + +(define + artdag/op-cost + (fn + (table) + (fn (op params) (if (has-key? table op) (get table op) 1)))) + +(define + artdag/-node-cost + (fn + (dag cost-fn id) + (let + ((n (artdag/dag-get dag id))) + (cost-fn (artdag/node-op n) (artdag/node-params n))))) + +(define + artdag/-max + (fn (xs) (reduce (fn (mx x) (if (> x mx) x mx)) 0 xs))) + +; longest weighted path through the dag = makespan with unlimited workers. +(define + artdag/critical-path + (fn + (dag cost-fn) + (let + ((ft (reduce (fn (m id) (let ((maxdep (artdag/-max (map (fn (d) (get m d)) (artdag/node-inputs (artdag/dag-get dag id)))))) (assoc m id (+ (artdag/-node-cost dag cost-fn id) maxdep)))) {} (artdag/dag-order dag)))) + (artdag/-max (map (fn (id) (get ft id)) (keys ft)))))) + +; estimated wall-clock for a plan: each batch runs in parallel (costs its +; slowest node), batches run in sequence. +(define + artdag/makespan + (fn + (dag plan cost-fn) + (reduce + (fn + (total batch) + (+ + total + (artdag/-max + (map (fn (id) (artdag/-node-cost dag cost-fn id)) batch)))) + 0 + plan))) + +; total serial work = sum of all node costs. +(define + artdag/total-work + (fn + (dag cost-fn) + (reduce + (fn (s id) (+ s (artdag/-node-cost dag cost-fn id))) + 0 + (keys (artdag/dag-nodes dag))))) + +; speedup of a plan vs running everything serially. +(define + artdag/speedup + (fn + (dag plan cost-fn) + (/ (artdag/total-work dag cost-fn) (artdag/makespan dag plan cost-fn)))) diff --git a/lib/artdag/scoreboard.json b/lib/artdag/scoreboard.json index 1c4bedd3..e707a6d3 100644 --- a/lib/artdag/scoreboard.json +++ b/lib/artdag/scoreboard.json @@ -5,9 +5,10 @@ "plan": {"pass": 18, "fail": 0}, "execute": {"pass": 15, "fail": 0}, "optimize": {"pass": 18, "fail": 0}, - "fed": {"pass": 15, "fail": 0} + "fed": {"pass": 15, "fail": 0}, + "cost": {"pass": 13, "fail": 0} }, - "total_pass": 102, + "total_pass": 115, "total_fail": 0, - "total": 102 + "total": 115 } diff --git a/lib/artdag/scoreboard.md b/lib/artdag/scoreboard.md index e5d97fb9..6440f9d5 100644 --- a/lib/artdag/scoreboard.md +++ b/lib/artdag/scoreboard.md @@ -10,4 +10,5 @@ _Generated by `lib/artdag/conformance.sh`_ | execute | 15 | 0 | 15 | | optimize | 18 | 0 | 18 | | fed | 15 | 0 | 15 | -| **Total** | **102** | **0** | **102** | +| cost | 13 | 0 | 13 | +| **Total** | **115** | **0** | **115** | diff --git a/lib/artdag/tests/cost.sx b/lib/artdag/tests/cost.sx new file mode 100644 index 00000000..fbc71da5 --- /dev/null +++ b/lib/artdag/tests/cost.sx @@ -0,0 +1,117 @@ +; cost model: critical path, makespan under cap, total work, speedup. + +(define + cost-CHAIN + (artdag/build + (list + (list "a" "in" (list) {}) + (list "b" "f" (list "a") {}) + (list "c" "f" (list "b") {}) + (list "d" "f" (list "c") {})))) + +(define + cost-DIA + (artdag/build + (list + (list "a" "in" (list) {}) + (list "b" "f" (list "a") {}) + (list "c" "g" (list "a") {}) + (list "d" "add" (list "b" "c") {} true)))) + +(define cost-W (artdag/op-cost {:f 2 :add 5})) + +; ---- unit cost ---- + +(artdag-test + "critical path: chain is its length" + (artdag/critical-path cost-CHAIN artdag/const-cost) + 4) + +(artdag-test + "critical path: diamond longest path" + (artdag/critical-path cost-DIA artdag/const-cost) + 3) + +(artdag-test + "total work: unit cost equals node count" + (artdag/total-work cost-DIA artdag/const-cost) + 4) + +(artdag-test + "single node critical path is its cost" + (artdag/critical-path + (artdag/build (list (list "a" "in" (list) {}))) + artdag/const-cost) + 1) + +; ---- makespan vs cap ---- + +(artdag-test + "full plan makespan equals critical path" + (artdag/makespan + cost-DIA + (artdag/plan cost-DIA 0) + artdag/const-cost) + (artdag/critical-path cost-DIA artdag/const-cost)) + +(artdag-test + "serial plan makespan equals total work" + (artdag/makespan + cost-DIA + (artdag/plan cost-DIA 1) + artdag/const-cost) + (artdag/total-work cost-DIA artdag/const-cost)) + +(artdag-test + "capped makespan is never below the critical path" + (>= + (artdag/makespan + cost-DIA + (artdag/plan cost-DIA 1) + artdag/const-cost) + (artdag/critical-path cost-DIA artdag/const-cost)) + true) + +; ---- weighted costs ---- + +(artdag-test + "weighted critical path follows heavy ops" + (artdag/critical-path cost-DIA cost-W) + 8) + +(artdag-test + "weighted total work sums all node costs" + (artdag/total-work cost-DIA cost-W) + 9) + +(artdag-test + "op-cost defaults unknown ops to 1" + (artdag/total-work + (artdag/build (list (list "a" "in" (list) {}))) + cost-W) + 1) + +(artdag-test + "weighted full-plan makespan equals critical path" + (artdag/makespan cost-DIA (artdag/plan cost-DIA 0) cost-W) + (artdag/critical-path cost-DIA cost-W)) + +; ---- speedup ---- + +(artdag-test + "serial plan has no speedup" + (artdag/speedup + cost-DIA + (artdag/plan cost-DIA 1) + artdag/const-cost) + 1) + +(artdag-test + "parallel plan beats serial" + (> + (artdag/speedup + cost-DIA + (artdag/plan cost-DIA 0) + artdag/const-cost) + 1) + true) diff --git a/plans/artdag-on-sx.md b/plans/artdag-on-sx.md index 95e168b0..af9ee3a7 100644 --- a/plans/artdag-on-sx.md +++ b/plans/artdag-on-sx.md @@ -30,7 +30,9 @@ edges. ## Status (rolling) -`bash lib/artdag/conformance.sh` → **102/102** (6 suites: dag, analyze, plan, execute, optimize, fed) +`bash lib/artdag/conformance.sh` → **115/115** (7 suites: dag, analyze, plan, execute, optimize, fed, cost) + +Base roadmap (Phases 1–6) COMPLETE. Now extending. ## Ground rules @@ -136,6 +138,15 @@ lib/artdag/optimize.sx lib/artdag/federation.sx ## Progress log +- **Ext: cost-based scheduling** (cost suite 13/13, total 115/115). + `lib/artdag/cost.sx`: an injected `cost-fn (op params)` keeps media-op costs opaque + (`const-cost`, `op-cost table`). `critical-path` = longest weighted path (finish-time + fold over topo order) = min makespan with unlimited workers. `makespan dag plan + cost-fn` sums each batch's slowest node — full plan (cap 0) makespan == critical + path, serial (cap 1) == `total-work`. `speedup` = total-work / makespan. Verified + weighted paths follow heavy ops and capped makespan never dips below the critical + path. + - **Phase 6 — Federation (shared content-addressed cache)** (fed suite 15/15, total 102/102). `lib/artdag/federation.sx`: an instance = `{:cache :prov {cid->origin-peer}}`. `fed-export` dumps the whole cache as `{:cid :result :peer}`