Eliminate raw HTML injection: convert ~kg-html/captions to native sx
Add shared/sx/html_to_sx.py (HTMLParser-based HTML→sx converter) and update lexical_to_sx.py so HTML cards, markdown cards, and captions all produce native sx expressions instead of opaque HTML strings. - ~kg-html now wraps native sx children (editor can identify the block) - New ~kg-md component for markdown card blocks - Captions are sx expressions, not escaped HTML strings - kg_cards.sx: replace (raw! caption) with direct caption rendering - sx-editor.js: htmlToSx() via DOMParser, serializeInline for captions, _childrenSx for ~kg-html/~kg-md, new kg-md edit UI - Migration script (blog/scripts/migrate_sx_html.py) to re-convert stored sx_content from lexical source Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -16,6 +16,8 @@ from typing import Callable
|
||||
|
||||
import mistune
|
||||
|
||||
from shared.sx.html_to_sx import html_to_sx
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Registry
|
||||
@@ -249,7 +251,7 @@ def _image(node: dict) -> str:
|
||||
if alt:
|
||||
parts.append(f':alt "{_esc(alt)}"')
|
||||
if caption:
|
||||
parts.append(f':caption "{_esc(caption)}"')
|
||||
parts.append(f":caption {html_to_sx(caption)}")
|
||||
if width:
|
||||
parts.append(f':width "{_esc(width)}"')
|
||||
if href:
|
||||
@@ -273,20 +275,21 @@ def _gallery(node: dict) -> str:
|
||||
if img.get("alt"):
|
||||
item_parts.append(f'"alt" "{_esc(img["alt"])}"')
|
||||
if img.get("caption"):
|
||||
item_parts.append(f'"caption" "{_esc(img["caption"])}"')
|
||||
item_parts.append(f'"caption" {html_to_sx(img["caption"])}')
|
||||
row_items.append("(dict " + " ".join(item_parts) + ")")
|
||||
rows.append("(list " + " ".join(row_items) + ")")
|
||||
|
||||
images_sx = "(list " + " ".join(rows) + ")"
|
||||
caption = node.get("caption", "")
|
||||
caption_attr = f' :caption "{_esc(caption)}"' if caption else ""
|
||||
caption_attr = f" :caption {html_to_sx(caption)}" if caption else ""
|
||||
return f"(~kg-gallery :images {images_sx}{caption_attr})"
|
||||
|
||||
|
||||
@_converter("html")
|
||||
def _html_card(node: dict) -> str:
|
||||
raw = node.get("html", "")
|
||||
return f'(~kg-html :html "{_esc(raw)}")'
|
||||
inner = html_to_sx(raw)
|
||||
return f"(~kg-html {inner})"
|
||||
|
||||
|
||||
@_converter("embed")
|
||||
@@ -295,7 +298,7 @@ def _embed(node: dict) -> str:
|
||||
caption = node.get("caption", "")
|
||||
parts = [f':html "{_esc(embed_html)}"']
|
||||
if caption:
|
||||
parts.append(f':caption "{_esc(caption)}"')
|
||||
parts.append(f":caption {html_to_sx(caption)}")
|
||||
return "(~kg-embed " + " ".join(parts) + ")"
|
||||
|
||||
|
||||
@@ -325,7 +328,7 @@ def _bookmark(node: dict) -> str:
|
||||
parts.append(f':thumbnail "{_esc(thumbnail)}"')
|
||||
caption = node.get("caption", "")
|
||||
if caption:
|
||||
parts.append(f':caption "{_esc(caption)}"')
|
||||
parts.append(f":caption {html_to_sx(caption)}")
|
||||
|
||||
return "(~kg-bookmark " + " ".join(parts) + ")"
|
||||
|
||||
@@ -390,7 +393,7 @@ def _video(node: dict) -> str:
|
||||
|
||||
parts = [f':src "{_esc(src)}"']
|
||||
if caption:
|
||||
parts.append(f':caption "{_esc(caption)}"')
|
||||
parts.append(f":caption {html_to_sx(caption)}")
|
||||
if width:
|
||||
parts.append(f':width "{_esc(width)}"')
|
||||
if thumbnail:
|
||||
@@ -425,7 +428,7 @@ def _file(node: dict) -> str:
|
||||
if size_str:
|
||||
parts.append(f':filesize "{size_str}"')
|
||||
if caption:
|
||||
parts.append(f':caption "{_esc(caption)}"')
|
||||
parts.append(f":caption {html_to_sx(caption)}")
|
||||
return "(~kg-file " + " ".join(parts) + ")"
|
||||
|
||||
|
||||
@@ -438,4 +441,5 @@ def _paywall(_node: dict) -> str:
|
||||
def _markdown(node: dict) -> str:
|
||||
md_text = node.get("markdown", "")
|
||||
rendered = mistune.html(md_text)
|
||||
return f'(~kg-html :html "{_esc(rendered)}")'
|
||||
inner = html_to_sx(rendered)
|
||||
return f"(~kg-md {inner})"
|
||||
|
||||
89
blog/scripts/migrate_sx_html.py
Normal file
89
blog/scripts/migrate_sx_html.py
Normal file
@@ -0,0 +1,89 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Re-convert sx_content from lexical JSON to eliminate ~kg-html wrappers and
|
||||
raw caption strings.
|
||||
|
||||
The updated lexical_to_sx converter now produces native sx expressions instead
|
||||
of (1) wrapping HTML/markdown cards in (~kg-html :html "...") and (2) storing
|
||||
captions as escaped HTML strings. This script re-runs the conversion on all
|
||||
posts that already have sx_content, overwriting the old output.
|
||||
|
||||
Usage:
|
||||
cd blog && python3 scripts/migrate_sx_html.py [--dry-run]
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import sys
|
||||
|
||||
from sqlalchemy import select, and_
|
||||
|
||||
|
||||
async def migrate(dry_run: bool = False) -> int:
|
||||
from shared.db.session import get_session
|
||||
from models.ghost_content import Post
|
||||
from bp.blog.ghost.lexical_to_sx import lexical_to_sx
|
||||
|
||||
converted = 0
|
||||
skipped = 0
|
||||
errors = 0
|
||||
|
||||
async with get_session() as sess:
|
||||
# All posts with lexical content (whether or not sx_content exists)
|
||||
stmt = select(Post).where(
|
||||
and_(
|
||||
Post.lexical.isnot(None),
|
||||
Post.lexical != "",
|
||||
)
|
||||
)
|
||||
result = await sess.execute(stmt)
|
||||
posts = result.scalars().all()
|
||||
|
||||
print(f"Found {len(posts)} posts with lexical content")
|
||||
|
||||
for post in posts:
|
||||
try:
|
||||
new_sx = lexical_to_sx(post.lexical)
|
||||
if post.sx_content == new_sx:
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
if dry_run:
|
||||
old_has_kg = "~kg-html" in (post.sx_content or "")
|
||||
old_has_raw = "raw! caption" in (post.sx_content or "")
|
||||
markers = []
|
||||
if old_has_kg:
|
||||
markers.append("~kg-html")
|
||||
if old_has_raw:
|
||||
markers.append("raw-caption")
|
||||
tag = f" [{', '.join(markers)}]" if markers else ""
|
||||
print(f" [DRY RUN] {post.slug}: {len(new_sx)} chars{tag}")
|
||||
else:
|
||||
post.sx_content = new_sx
|
||||
print(f" Converted: {post.slug} ({len(new_sx)} chars)")
|
||||
converted += 1
|
||||
except Exception as e:
|
||||
print(f" ERROR: {post.slug}: {e}", file=sys.stderr)
|
||||
errors += 1
|
||||
|
||||
if not dry_run and converted:
|
||||
await sess.commit()
|
||||
|
||||
print(f"\nDone: {converted} converted, {skipped} unchanged, {errors} errors")
|
||||
return converted
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Re-convert sx_content to eliminate ~kg-html and raw captions"
|
||||
)
|
||||
parser.add_argument("--dry-run", action="store_true",
|
||||
help="Preview changes without writing to database")
|
||||
args = parser.parse_args()
|
||||
|
||||
asyncio.run(migrate(dry_run=args.dry_run))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -2,7 +2,7 @@
|
||||
;; Produces same HTML structure as lexical_renderer.py so cards.css works unchanged.
|
||||
;; Used by both display pipeline and block editor.
|
||||
|
||||
;; @css kg-card kg-image-card kg-width-wide kg-width-full kg-gallery-card kg-gallery-container kg-gallery-row kg-gallery-image kg-embed-card kg-bookmark-card kg-bookmark-container kg-bookmark-content kg-bookmark-title kg-bookmark-description kg-bookmark-metadata kg-bookmark-icon kg-bookmark-author kg-bookmark-publisher kg-bookmark-thumbnail kg-callout-card kg-callout-emoji kg-callout-text kg-button-card kg-btn kg-btn-accent kg-toggle-card kg-toggle-heading kg-toggle-heading-text kg-toggle-card-icon kg-toggle-content kg-audio-card kg-audio-thumbnail kg-audio-player-container kg-audio-title kg-audio-player kg-audio-play-icon kg-audio-current-time kg-audio-time kg-audio-seek-slider kg-audio-playback-rate kg-audio-unmute-icon kg-audio-volume-slider kg-video-card kg-video-container kg-file-card kg-file-card-container kg-file-card-contents kg-file-card-title kg-file-card-filesize kg-file-card-icon kg-file-card-caption kg-align-center kg-align-left kg-callout-card-grey kg-callout-card-white kg-callout-card-blue kg-callout-card-green kg-callout-card-yellow kg-callout-card-red kg-callout-card-pink kg-callout-card-purple kg-callout-card-accent placeholder
|
||||
;; @css kg-card kg-image-card kg-width-wide kg-width-full kg-gallery-card kg-gallery-container kg-gallery-row kg-gallery-image kg-embed-card kg-bookmark-card kg-bookmark-container kg-bookmark-content kg-bookmark-title kg-bookmark-description kg-bookmark-metadata kg-bookmark-icon kg-bookmark-author kg-bookmark-publisher kg-bookmark-thumbnail kg-callout-card kg-callout-emoji kg-callout-text kg-button-card kg-btn kg-btn-accent kg-toggle-card kg-toggle-heading kg-toggle-heading-text kg-toggle-card-icon kg-toggle-content kg-audio-card kg-audio-thumbnail kg-audio-player-container kg-audio-title kg-audio-player kg-audio-play-icon kg-audio-current-time kg-audio-time kg-audio-seek-slider kg-audio-playback-rate kg-audio-unmute-icon kg-audio-volume-slider kg-video-card kg-video-container kg-file-card kg-file-card-container kg-file-card-contents kg-file-card-title kg-file-card-filesize kg-file-card-icon kg-file-card-caption kg-align-center kg-align-left kg-callout-card-grey kg-callout-card-white kg-callout-card-blue kg-callout-card-green kg-callout-card-yellow kg-callout-card-red kg-callout-card-pink kg-callout-card-purple kg-callout-card-accent kg-html-card kg-md-card placeholder
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
;; Image card
|
||||
@@ -33,10 +33,17 @@
|
||||
(when caption (figcaption caption))))
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
;; HTML card (raw HTML injection)
|
||||
;; HTML card — wraps user-pasted HTML so the editor can identify the block.
|
||||
;; Content is native sx children (no longer an opaque HTML string).
|
||||
;; ---------------------------------------------------------------------------
|
||||
(defcomp ~kg-html (&key html)
|
||||
(~rich-text :html html))
|
||||
(defcomp ~kg-html (&rest children)
|
||||
(div :class "kg-card kg-html-card" children))
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
;; Markdown card — rendered markdown content, editor can identify the block.
|
||||
;; ---------------------------------------------------------------------------
|
||||
(defcomp ~kg-md (&rest children)
|
||||
(div :class "kg-card kg-md-card" children))
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
;; Embed card
|
||||
|
||||
@@ -5,8 +5,9 @@ import sys
|
||||
import os
|
||||
import pytest
|
||||
|
||||
# The lexical_to_sx module is standalone (only depends on json).
|
||||
# Import it directly to avoid pulling in the full blog app.
|
||||
# Add project root so shared.sx.html_to_sx resolves, plus the ghost dir.
|
||||
_project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
|
||||
sys.path.insert(0, _project_root)
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "bp", "blog", "ghost"))
|
||||
from lexical_to_sx import lexical_to_sx
|
||||
|
||||
@@ -176,6 +177,13 @@ class TestCards:
|
||||
assert ':width "wide"' in result
|
||||
assert ':caption "Fig 1"' in result
|
||||
|
||||
def test_image_html_caption(self):
|
||||
result = lexical_to_sx(_doc({
|
||||
"type": "image", "src": "p.jpg", "alt": "",
|
||||
"caption": 'Photo by <a href="https://x.com">Author</a>'
|
||||
}))
|
||||
assert ':caption (<> "Photo by " (a :href "https://x.com" "Author"))' in result
|
||||
|
||||
def test_bookmark(self):
|
||||
result = lexical_to_sx(_doc({
|
||||
"type": "bookmark", "url": "https://example.com",
|
||||
@@ -214,7 +222,7 @@ class TestCards:
|
||||
result = lexical_to_sx(_doc({
|
||||
"type": "html", "html": "<div>custom</div>"
|
||||
}))
|
||||
assert "(~kg-html " in result
|
||||
assert result == '(~kg-html (div "custom"))'
|
||||
|
||||
def test_embed(self):
|
||||
result = lexical_to_sx(_doc({
|
||||
@@ -224,6 +232,14 @@ class TestCards:
|
||||
assert "(~kg-embed " in result
|
||||
assert ':caption "Video"' in result
|
||||
|
||||
def test_markdown(self):
|
||||
result = lexical_to_sx(_doc({
|
||||
"type": "markdown", "markdown": "**bold** text"
|
||||
}))
|
||||
assert result.startswith("(~kg-md ")
|
||||
assert "(p " in result
|
||||
assert "(strong " in result
|
||||
|
||||
def test_video(self):
|
||||
result = lexical_to_sx(_doc({
|
||||
"type": "video", "src": "v.mp4", "cardWidth": "wide"
|
||||
|
||||
Reference in New Issue
Block a user