9b2687b039
All checks were successful
Build and Deploy / build-and-deploy (push) Successful in 41s
- URL structure changes from /<route> to /<market_slug>/<route> - Root / shows markets listing page - app.py: url_value_preprocessor, url_defaults, hydrate_market (events app pattern) - Browse queries (db_nav, db_products_nocounts, db_products_counts) accept market_id - _productInfo reads g.market.id to scope all queries - save_nav accepts market_id, sets on new NavTop rows - API save_nav passes g.market.id - Scraper default URLs point to /suma-market/ on port 8001 Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
111 lines
3.1 KiB
Python
111 lines
3.1 KiB
Python
# at top of persist_snapshot.py:
|
|
from datetime import datetime
|
|
from sqlalchemy import (
|
|
select, tuple_
|
|
)
|
|
from typing import Dict
|
|
|
|
from models.market import (
|
|
NavTop,
|
|
NavSub,
|
|
)
|
|
from db.session import get_session
|
|
|
|
|
|
|
|
|
|
async def save_nav(nav: Dict) -> None:
|
|
async with get_session() as session:
|
|
await _save_nav(session, nav)
|
|
await session.commit()
|
|
|
|
async def _save_nav(session, nav: Dict, market_id=None) -> None:
|
|
print('===================SAVE NAV========================')
|
|
print(nav)
|
|
now = datetime.utcnow()
|
|
|
|
incoming_top_slugs = set()
|
|
incoming_sub_keys = set() # (top_slug, sub_slug)
|
|
|
|
# First pass: collect slugs
|
|
for label, data in (nav.get("cats") or {}).items():
|
|
top_slug = (data or {}).get("slug")
|
|
if not top_slug:
|
|
continue
|
|
incoming_top_slugs.add(top_slug)
|
|
|
|
for s in (data.get("subs") or []):
|
|
sub_slug = s.get("slug")
|
|
if sub_slug:
|
|
incoming_sub_keys.add((top_slug, sub_slug))
|
|
|
|
# Soft-delete stale NavSub entries
|
|
# This requires joining NavTop to access top_slug
|
|
subs_to_delete = await session.execute(
|
|
select(NavSub)
|
|
.join(NavTop, NavSub.top_id == NavTop.id)
|
|
.where(
|
|
NavSub.deleted_at.is_(None),
|
|
~tuple_(NavTop.slug, NavSub.slug).in_(incoming_sub_keys)
|
|
)
|
|
)
|
|
for sub in subs_to_delete.scalars():
|
|
sub.deleted_at = now
|
|
|
|
# Soft-delete stale NavTop entries
|
|
tops_to_delete = await session.execute(
|
|
select(NavTop)
|
|
.where(
|
|
NavTop.deleted_at.is_(None),
|
|
~NavTop.slug.in_(incoming_top_slugs)
|
|
)
|
|
)
|
|
for top in tops_to_delete.scalars():
|
|
top.deleted_at = now
|
|
|
|
await session.flush()
|
|
|
|
# Upsert NavTop and NavSub
|
|
for label, data in (nav.get("cats") or {}).items():
|
|
top_slug = (data or {}).get("slug")
|
|
if not top_slug:
|
|
continue
|
|
|
|
res = await session.execute(
|
|
select(NavTop).where(NavTop.slug == top_slug)
|
|
)
|
|
top = res.scalar_one_or_none()
|
|
|
|
if top:
|
|
top.label = label
|
|
top.deleted_at = None
|
|
if market_id is not None and top.market_id is None:
|
|
top.market_id = market_id
|
|
else:
|
|
top = NavTop(label=label, slug=top_slug, market_id=market_id)
|
|
session.add(top)
|
|
|
|
await session.flush()
|
|
|
|
for s in (data.get("subs") or []):
|
|
sub_slug = s.get("slug")
|
|
if not sub_slug:
|
|
continue
|
|
sub_label = s.get("label")
|
|
sub_href = s.get("href")
|
|
|
|
res_sub = await session.execute(
|
|
select(NavSub).where(
|
|
NavSub.slug == sub_slug,
|
|
NavSub.top_id == top.id
|
|
)
|
|
)
|
|
sub = res_sub.scalar_one_or_none()
|
|
if sub:
|
|
sub.label = sub_label
|
|
sub.href = sub_href
|
|
sub.deleted_at = None
|
|
else:
|
|
session.add(NavSub(top_id=top.id, label=sub_label, slug=sub_slug, href=sub_href))
|
|
|