feat: initialize market app with browsing, product, and scraping code
Some checks failed
Build and Deploy / build-and-deploy (push) Has been cancelled
Some checks failed
Build and Deploy / build-and-deploy (push) Has been cancelled
Split from coop monolith. Includes: - Market/browse/product blueprints - Product sync API - Suma scraping pipeline - Templates for market, browse, and product views - Dockerfile and CI workflow for independent deployment
This commit is contained in:
29
scrape/persist_snapshot/save_link_reports.py
Normal file
29
scrape/persist_snapshot/save_link_reports.py
Normal file
@@ -0,0 +1,29 @@
|
||||
# at top of persist_snapshot.py:
|
||||
from typing import List
|
||||
|
||||
from typing import Dict, List
|
||||
|
||||
from models.market import (
|
||||
LinkError,
|
||||
LinkExternal,
|
||||
)
|
||||
from db.session import get_session
|
||||
|
||||
# --- Models are unchanged, see original code ---
|
||||
|
||||
# ---------------------- Helper fns called from scraper ------------------------
|
||||
|
||||
|
||||
|
||||
async def save_link_reports(link_errors: List[Dict], link_externals: List[Dict]) -> None:
|
||||
async with get_session() as session:
|
||||
for e in link_errors:
|
||||
session.add(LinkError(
|
||||
product_slug=e.get("product"), href=e.get("href"), text=e.get("text"),
|
||||
top=e.get("top"), sub=e.get("sub"), target_slug=e.get("target_slug"), type=e.get("type"),
|
||||
))
|
||||
for e in link_externals:
|
||||
session.add(LinkExternal(
|
||||
product_slug=e.get("product"), href=e.get("href"), text=e.get("text"), host=e.get("host"),
|
||||
))
|
||||
await session.commit()
|
||||
Reference in New Issue
Block a user