Monorepo: consolidate 7 repos into one
All checks were successful
Build and Deploy / build-and-deploy (push) Successful in 1m5s
All checks were successful
Build and Deploy / build-and-deploy (push) Successful in 1m5s
Combines shared, blog, market, cart, events, federation, and account into a single repository. Eliminates submodule sync, sibling model copying at build time, and per-app CI orchestration. Changes: - Remove per-app .git, .gitmodules, .gitea, submodule shared/ dirs - Remove stale sibling model copies from each app - Update all 6 Dockerfiles for monorepo build context (root = .) - Add build directives to docker-compose.yml - Add single .gitea/workflows/ci.yml with change detection - Add .dockerignore for monorepo build context - Create __init__.py for federation and account (cross-app imports)
This commit is contained in:
110
market/scrape/persist_snapshot/save_nav.py
Normal file
110
market/scrape/persist_snapshot/save_nav.py
Normal file
@@ -0,0 +1,110 @@
|
||||
# at top of persist_snapshot.py:
|
||||
from datetime import datetime
|
||||
from sqlalchemy import (
|
||||
select, tuple_
|
||||
)
|
||||
from typing import Dict
|
||||
|
||||
from models.market import (
|
||||
NavTop,
|
||||
NavSub,
|
||||
)
|
||||
from shared.db.session import get_session
|
||||
|
||||
|
||||
|
||||
|
||||
async def save_nav(nav: Dict) -> None:
|
||||
async with get_session() as session:
|
||||
await _save_nav(session, nav)
|
||||
await session.commit()
|
||||
|
||||
async def _save_nav(session, nav: Dict, market_id=None) -> None:
|
||||
print('===================SAVE NAV========================')
|
||||
print(nav)
|
||||
now = datetime.utcnow()
|
||||
|
||||
incoming_top_slugs = set()
|
||||
incoming_sub_keys = set() # (top_slug, sub_slug)
|
||||
|
||||
# First pass: collect slugs
|
||||
for label, data in (nav.get("cats") or {}).items():
|
||||
top_slug = (data or {}).get("slug")
|
||||
if not top_slug:
|
||||
continue
|
||||
incoming_top_slugs.add(top_slug)
|
||||
|
||||
for s in (data.get("subs") or []):
|
||||
sub_slug = s.get("slug")
|
||||
if sub_slug:
|
||||
incoming_sub_keys.add((top_slug, sub_slug))
|
||||
|
||||
# Soft-delete stale NavSub entries
|
||||
# This requires joining NavTop to access top_slug
|
||||
subs_to_delete = await session.execute(
|
||||
select(NavSub)
|
||||
.join(NavTop, NavSub.top_id == NavTop.id)
|
||||
.where(
|
||||
NavSub.deleted_at.is_(None),
|
||||
~tuple_(NavTop.slug, NavSub.slug).in_(incoming_sub_keys)
|
||||
)
|
||||
)
|
||||
for sub in subs_to_delete.scalars():
|
||||
sub.deleted_at = now
|
||||
|
||||
# Soft-delete stale NavTop entries
|
||||
tops_to_delete = await session.execute(
|
||||
select(NavTop)
|
||||
.where(
|
||||
NavTop.deleted_at.is_(None),
|
||||
~NavTop.slug.in_(incoming_top_slugs)
|
||||
)
|
||||
)
|
||||
for top in tops_to_delete.scalars():
|
||||
top.deleted_at = now
|
||||
|
||||
await session.flush()
|
||||
|
||||
# Upsert NavTop and NavSub
|
||||
for label, data in (nav.get("cats") or {}).items():
|
||||
top_slug = (data or {}).get("slug")
|
||||
if not top_slug:
|
||||
continue
|
||||
|
||||
res = await session.execute(
|
||||
select(NavTop).where(NavTop.slug == top_slug)
|
||||
)
|
||||
top = res.scalar_one_or_none()
|
||||
|
||||
if top:
|
||||
top.label = label
|
||||
top.deleted_at = None
|
||||
if market_id is not None and top.market_id is None:
|
||||
top.market_id = market_id
|
||||
else:
|
||||
top = NavTop(label=label, slug=top_slug, market_id=market_id)
|
||||
session.add(top)
|
||||
|
||||
await session.flush()
|
||||
|
||||
for s in (data.get("subs") or []):
|
||||
sub_slug = s.get("slug")
|
||||
if not sub_slug:
|
||||
continue
|
||||
sub_label = s.get("label")
|
||||
sub_href = s.get("href")
|
||||
|
||||
res_sub = await session.execute(
|
||||
select(NavSub).where(
|
||||
NavSub.slug == sub_slug,
|
||||
NavSub.top_id == top.id
|
||||
)
|
||||
)
|
||||
sub = res_sub.scalar_one_or_none()
|
||||
if sub:
|
||||
sub.label = sub_label
|
||||
sub.href = sub_href
|
||||
sub.deleted_at = None
|
||||
else:
|
||||
session.add(NavSub(top_id=top.id, label=sub_label, slug=sub_slug, href=sub_href))
|
||||
|
||||
Reference in New Issue
Block a user