diff --git a/lib/artdag/conformance.sh b/lib/artdag/conformance.sh index dcccfa4a..0551abb3 100755 --- a/lib/artdag/conformance.sh +++ b/lib/artdag/conformance.sh @@ -13,7 +13,7 @@ if [ ! -x "$SX_SERVER" ]; then exit 1 fi -SUITES=(dag analyze plan execute optimize) +SUITES=(dag analyze plan execute optimize fed) OUT_JSON="lib/artdag/scoreboard.json" OUT_MD="lib/artdag/scoreboard.md" @@ -46,6 +46,7 @@ run_suite() { (load "lib/artdag/plan.sx") (load "lib/artdag/execute.sx") (load "lib/artdag/optimize.sx") +(load "lib/artdag/federation.sx") (epoch 2) (eval "(define artdag-test-pass 0)") (eval "(define artdag-test-fail 0)") diff --git a/lib/artdag/federation.sx b/lib/artdag/federation.sx new file mode 100644 index 00000000..15780b46 --- /dev/null +++ b/lib/artdag/federation.sx @@ -0,0 +1,75 @@ +; lib/artdag/federation.sx — Phase 6: shared content-addressed cache across +; instances (the L2-registry analog). Because content-ids are global, a result +; computed on one instance is reusable on another by id. Imports are trust-gated +; and carry provenance so a peer's results can be invalidated when trust is +; withdrawn. Transport is injected (mock in tests). Depends on dag.sx, execute.sx +; (the cache is a lib/persist/ kv backend) — federation tracks provenance beside it. + +; an instance: a persist kv cache + a provenance map {cid -> origin-peer}. +(define artdag/fed-open (fn () {:cache (persist/open) :prov {}})) +(define artdag/fed-cache (fn (fed) (get fed :cache))) +(define artdag/fed-prov (fn (fed) (get fed :prov))) + +(define + artdag/-dict-remove + (fn + (d key) + (reduce + (fn (acc k) (if (= k key) acc (assoc acc k (get d k)))) + {} + (keys d)))) + +; export every cached result as a bundle of {:cid :result :peer}, tagged with +; the exporting instance's peer id (the result's origin/provenance). +(define + artdag/fed-export + (fn + (fed peer-id) + (map (fn (cid) {:peer peer-id :cid cid :result (persist/kv-get (get fed :cache) cid)}) (persist/kv-keys (get fed :cache))))) + +; import a bundle, accepting only records from trusted peers (trust gating) and +; recording each accepted result's provenance. Returns the updated instance. +(define + artdag/fed-import + (fn + (fed bundle trusted?) + (reduce + (fn + (f rec) + (if + (trusted? (get rec :peer)) + (begin + (persist/kv-put (get f :cache) (get rec :cid) (get rec :result)) + {:cache (get f :cache) :prov (assoc (get f :prov) (get rec :cid) (get rec :peer))}) + f)) + fed + bundle))) + +; pull from a peer through an injected transport (fetch-fn peer-id -> bundle). +(define + artdag/fed-pull + (fn + (fed fetch-fn peer-id trusted?) + (artdag/fed-import fed (fetch-fn peer-id) trusted?))) + +; invalidate: drop every cached result provenanced to a peer (trust withdrawn), +; from both the cache and the provenance map. Locally-computed results (no +; provenance) are untouched. Returns the updated instance. +(define + artdag/fed-invalidate + (fn + (fed peer-id) + (reduce + (fn + (f cid) + (if + (= (get (get f :prov) cid) peer-id) + (begin (persist/kv-delete (get f :cache) cid) {:cache (get f :cache) :prov (artdag/-dict-remove (get f :prov) cid)}) + f)) + fed + (keys (get fed :prov))))) + +; convenience: run a dag against an instance's cache. +(define + artdag/fed-run + (fn (fed dag runner) (artdag/run dag runner (artdag/fed-cache fed)))) diff --git a/lib/artdag/scoreboard.json b/lib/artdag/scoreboard.json index 7be1073f..1c4bedd3 100644 --- a/lib/artdag/scoreboard.json +++ b/lib/artdag/scoreboard.json @@ -4,9 +4,10 @@ "analyze": {"pass": 16, "fail": 0}, "plan": {"pass": 18, "fail": 0}, "execute": {"pass": 15, "fail": 0}, - "optimize": {"pass": 18, "fail": 0} + "optimize": {"pass": 18, "fail": 0}, + "fed": {"pass": 15, "fail": 0} }, - "total_pass": 87, + "total_pass": 102, "total_fail": 0, - "total": 87 + "total": 102 } diff --git a/lib/artdag/scoreboard.md b/lib/artdag/scoreboard.md index 14f8c5c8..e5d97fb9 100644 --- a/lib/artdag/scoreboard.md +++ b/lib/artdag/scoreboard.md @@ -9,4 +9,5 @@ _Generated by `lib/artdag/conformance.sh`_ | plan | 18 | 0 | 18 | | execute | 15 | 0 | 15 | | optimize | 18 | 0 | 18 | -| **Total** | **87** | **0** | **87** | +| fed | 15 | 0 | 15 | +| **Total** | **102** | **0** | **102** | diff --git a/lib/artdag/tests/fed.sx b/lib/artdag/tests/fed.sx new file mode 100644 index 00000000..981bcfcd --- /dev/null +++ b/lib/artdag/tests/fed.sx @@ -0,0 +1,157 @@ +; Phase 6 — federation: shared content-addressed cache, trust gating, invalidation. + +(define fed-BASE (artdag/op-table-runner {:in (fn (params inputs) (get params :v)) :add (fn (params inputs) (+ (nth inputs 0) (nth inputs 1))) :inc (fn (params inputs) (+ 1 (first inputs)))})) + +(define + fed-D + (artdag/build + (list + (list "p" "in" (list) {:v 10}) + (list "q" "in" (list) {:v 20}) + (list "b" "inc" (list "p") {}) + (list "c" "inc" (list "q") {}) + (list "d" "add" (list "b" "c") {} true)))) + +(define fed-trust-A (fn (p) (= p "A"))) +(define fed-trust-none (fn (p) false)) + +; a warmed instance A and its export bundle (origin peer "A"). +(define fed-A (artdag/fed-open)) +(define fed-warm (artdag/fed-run fed-A fed-D fed-BASE)) +(define fed-bundle (artdag/fed-export fed-A "A")) + +; ---- export ---- + +(artdag-test + "export: bundle covers every cached node" + (len fed-bundle) + 5) + +; ---- remote cache hit ---- + +(artdag-test + "trusted import enables remote cache hit (no recompute)" + (artdag/recompute-count + (artdag/fed-run + (artdag/fed-import (artdag/fed-open) fed-bundle fed-trust-A) + fed-D + fed-BASE)) + 0) + +(artdag-test + "trusted import: every node is a hit" + (artdag/hit-count + (artdag/fed-run + (artdag/fed-import (artdag/fed-open) fed-bundle fed-trust-A) + fed-D + fed-BASE)) + 5) + +(artdag-test + "remote hit yields correct result" + (artdag/result-of + (artdag/fed-run + (artdag/fed-import (artdag/fed-open) fed-bundle fed-trust-A) + fed-D + fed-BASE) + (artdag/dag-id fed-D "d")) + 32) + +; ---- trust gating ---- + +(artdag-test + "untrusted peer is rejected (recompute everything)" + (artdag/recompute-count + (artdag/fed-run + (artdag/fed-import (artdag/fed-open) fed-bundle fed-trust-none) + fed-D + fed-BASE)) + 5) + +(artdag-test + "trust gating: untrusted records never enter the cache" + (let + ((B (artdag/fed-import (artdag/fed-open) (cons {:peer "C" :cid "node:foreign" :result 99} fed-bundle) fed-trust-A))) + (persist/kv-has? (artdag/fed-cache B) "node:foreign")) + false) + +(artdag-test + "trust gating: trusted records still admitted alongside rejected" + (let + ((B (artdag/fed-import (artdag/fed-open) (cons {:peer "C" :cid "node:foreign" :result 99} fed-bundle) fed-trust-A))) + (persist/kv-has? (artdag/fed-cache B) (artdag/dag-id fed-D "d"))) + true) + +; ---- provenance ---- + +(artdag-test + "provenance is recorded for imported results" + (get + (artdag/fed-prov + (artdag/fed-import (artdag/fed-open) fed-bundle fed-trust-A)) + (artdag/dag-id fed-D "d")) + "A") + +(artdag-test + "locally computed results carry no provenance" + (len (keys (artdag/fed-prov fed-A))) + 0) + +; ---- injected transport ---- + +(artdag-test + "fed-pull imports via an injected fetch transport" + (artdag/recompute-count + (artdag/fed-run + (artdag/fed-pull + (artdag/fed-open) + (fn (peer) fed-bundle) + "A" + fed-trust-A) + fed-D + fed-BASE)) + 0) + +; ---- invalidation ---- + +(artdag-test + "invalidation drops a peer's results (recompute again)" + (let + ((B (artdag/fed-import (artdag/fed-open) fed-bundle fed-trust-A))) + (artdag/recompute-count + (artdag/fed-run (artdag/fed-invalidate B "A") fed-D fed-BASE))) + 5) + +(artdag-test + "invalidation: recomputed result still correct" + (let + ((B (artdag/fed-import (artdag/fed-open) fed-bundle fed-trust-A))) + (artdag/result-of + (artdag/fed-run (artdag/fed-invalidate B "A") fed-D fed-BASE) + (artdag/dag-id fed-D "d"))) + 32) + +(artdag-test + "invalidation: provenance map is cleared for that peer" + (let + ((B (artdag/fed-import (artdag/fed-open) fed-bundle fed-trust-A))) + (len (keys (artdag/fed-prov (artdag/fed-invalidate B "A"))))) + 0) + +(artdag-test + "invalidation is peer-scoped: other peers' results survive" + (let + ((B (artdag/fed-import (artdag/fed-open) (cons {:peer "C" :cid "node:fromC" :result 7} fed-bundle) (fn (p) true)))) + (persist/kv-has? + (artdag/fed-cache (artdag/fed-invalidate B "A")) + "node:fromC")) + true) + +(artdag-test + "invalidation is peer-scoped: target peer's results removed" + (let + ((B (artdag/fed-import (artdag/fed-open) (cons {:peer "C" :cid "node:fromC" :result 7} fed-bundle) (fn (p) true)))) + (persist/kv-has? + (artdag/fed-cache (artdag/fed-invalidate B "A")) + (artdag/dag-id fed-D "d"))) + false) diff --git a/plans/artdag-on-sx.md b/plans/artdag-on-sx.md index 0ebb38b4..95e168b0 100644 --- a/plans/artdag-on-sx.md +++ b/plans/artdag-on-sx.md @@ -30,7 +30,7 @@ edges. ## Status (rolling) -`bash lib/artdag/conformance.sh` → **87/87** (5 suites: dag, analyze, plan, execute, optimize) +`bash lib/artdag/conformance.sh` → **102/102** (6 suites: dag, analyze, plan, execute, optimize, fed) ## Ground rules @@ -127,15 +127,27 @@ lib/artdag/optimize.sx lib/artdag/federation.sx ## Phase 6 — Federation (shared content-addressed cache) -- [ ] a result computed on one instance is reusable on another by content-id (the +- [x] a result computed on one instance is reusable on another by content-id (the L2-registry analog): export/import `{content-id → result}` with provenance -- [ ] trust gating — accept a remote result only from a trusted peer (mirror the +- [x] trust gating — accept a remote result only from a trusted peer (mirror the fed trust shape; mock the transport in tests) -- [ ] revocation/invalidation — drop a remote result if its provenance is withdrawn -- [ ] `lib/artdag/tests/fed.sx` — remote cache hit, trust gating, invalidation +- [x] revocation/invalidation — drop a remote result if its provenance is withdrawn +- [x] `lib/artdag/tests/fed.sx` — remote cache hit, trust gating, invalidation ## Progress log +- **Phase 6 — Federation (shared content-addressed cache)** (fed suite 15/15, total + 102/102). `lib/artdag/federation.sx`: an instance = `{:cache :prov + {cid->origin-peer}}`. `fed-export` dumps the whole cache as `{:cid :result :peer}` + records tagged with the exporter's id; `fed-import` accepts only records from + trusted peers (trust gating) and records provenance; `fed-pull` imports via an + injected `fetch-fn(peer-id)` transport (mocked in tests). Because content-ids are + global, a trusted import makes the importer's run a pure cache hit (recompute 0) — + the L2-registry analog. `fed-invalidate peer` drops every result provenanced to a + peer from cache + prov (trust withdrawn → recompute), peer-scoped (other peers' + results survive) and leaving locally-computed (un-provenanced) results untouched. + ALL 6 PHASES COMPLETE. + - **Phase 5 — Effect-pipeline optimization** (optimize suite 18/18, total 87/87). `lib/artdag/optimize.sx`: `artdag/dce dag outputs` keeps only the outputs plus their transitive ancestors (via analyze), preserving surviving content-ids.