blogimport: published-posts source contract + blog-side draft (76/76)
Some checks failed
Test, Build, and Deploy / test-build-deploy (push) Failing after 55s
Some checks failed
Test, Build, and Deploy / test-build-deploy (push) Failing after 55s
source.sx refactored to a single published-posts batch query returning full rows (incl. lexical) — the existing post-by-id/slug DTO lacks lexical (sx_content/html only), so the canonical lexical->blocks path needs a dedicated migration provider. backfill-ids! now filters client-side (no extra query). drafts/published-posts.sx + drafts/README.md: paste-ready blog-app change (defquery + SqlBlogService.list_published_posts returning rows incl. raw lexical). README updated. source 21/21; total 76/76. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -17,11 +17,14 @@
|
||||
; here with dream-json-parse into the SX dict shape blogimport/lex-blocks expects.
|
||||
; (If a handler returns :lexical already-structured, it is used as-is.)
|
||||
;
|
||||
; REQUIRED BLOG-SIDE ADDITION (the one gap): blog/queries.sx exposes fetch-by-id/slug
|
||||
; but NO enumeration query. The corpus (Q-D2 = every published post) needs a
|
||||
; `published-posts` query returning the published ids/slugs (Python: list_posts(
|
||||
; status="published"), blog/bp/blog/ghost_db.py:102). Flagged for the blog app; mocked
|
||||
; in tests. Until it exists, callers can pass an explicit id list to backfill-ids!.
|
||||
; REQUIRED BLOG-SIDE ADDITION (the one gap — draft in drafts/published-posts.sx):
|
||||
; the migration needs a `published-posts` query that returns full published-post ROWS
|
||||
; INCLUDING the raw `:lexical` body. The existing post-by-id/slug providers return a
|
||||
; PostDTO that carries sx_content/html but NOT lexical (blog/services/__init__.py
|
||||
; _post_to_dto), so they cannot feed the canonical lexical->blocks converter. One new
|
||||
; provider (Python list_published_posts over list_posts(status="published"),
|
||||
; blog/bp/blog/ghost_db.py:102) covers both enumeration AND bodies in one batch call.
|
||||
; Mocked here against that contract; see drafts/ for the paste-ready blog-side change.
|
||||
|
||||
(define blogimport/dep-json-parse dream-json-parse)
|
||||
|
||||
@@ -47,45 +50,34 @@
|
||||
:authors (or (get row :authors) (list))
|
||||
:lexical (blogimport/parse-lexical (get row :lexical))}))
|
||||
|
||||
; --- fetch one post via an internal-data query ----------------------------------
|
||||
; --- the published-post rows from the live source (one batch query) -------------
|
||||
(define
|
||||
blogimport/fetch-post
|
||||
(fn (fetch-fn query params)
|
||||
(blogimport/parse-row (fetch-fn query params))))
|
||||
|
||||
; --- enumerate published post ids (needs the `published-posts` query) -----------
|
||||
(define
|
||||
blogimport/published-ids
|
||||
blogimport/source-rows
|
||||
(fn (fetch-fn) (fetch-fn "published-posts" {})))
|
||||
|
||||
; --- fetch all published posts as importer `post` dicts -------------------------
|
||||
; --- all published posts as importer `post` dicts -------------------------------
|
||||
(define
|
||||
blogimport/source-posts
|
||||
(fn (fetch-fn)
|
||||
(map
|
||||
(fn (id) (blogimport/fetch-post fetch-fn "post-by-id" {:id id}))
|
||||
(blogimport/published-ids fetch-fn))))
|
||||
|
||||
; --- fetch an explicit id list (fallback before the enumeration query lands) ----
|
||||
(define
|
||||
blogimport/source-posts-by-ids
|
||||
(fn (fetch-fn ids)
|
||||
(map (fn (id) (blogimport/fetch-post fetch-fn "post-by-id" {:id id})) ids)))
|
||||
(fn (fetch-fn) (map blogimport/parse-row (blogimport/source-rows fetch-fn))))
|
||||
|
||||
; --- end-to-end drivers ---------------------------------------------------------
|
||||
; backfill = enumerate -> fetch -> genesis-import (idempotent). Re-runnable as the
|
||||
; backfill = enumerate+fetch -> genesis-import (idempotent). Re-runnable as the
|
||||
; one-way DB->persist sync (data-migration.md Strategy 1).
|
||||
(define
|
||||
blogimport/backfill!
|
||||
(fn (b fetch-fn at)
|
||||
(blogimport/import-all! b (blogimport/source-posts fetch-fn) at)))
|
||||
|
||||
; partial backfill: client-side filter to a subset of ids (no extra blog query).
|
||||
(define
|
||||
blogimport/backfill-ids!
|
||||
(fn (b fetch-fn ids at)
|
||||
(blogimport/import-all! b (blogimport/source-posts-by-ids fetch-fn ids) at)))
|
||||
(blogimport/import-all!
|
||||
b
|
||||
(filter (fn (p) (contains? ids (get p :id))) (blogimport/source-posts fetch-fn))
|
||||
at)))
|
||||
|
||||
; sync-verify = enumerate -> fetch -> shadow-diff the persisted streams at rest.
|
||||
; sync-verify = fetch -> shadow-diff the persisted streams at rest.
|
||||
(define
|
||||
blogimport/sync-verify
|
||||
(fn (b fetch-fn)
|
||||
|
||||
Reference in New Issue
Block a user