478636f799
Phase 1-3 of decoupling: - path_setup.py adds project root to sys.path - Market-owned models in market/models/ (market, market_place) - All imports updated: shared.infrastructure, shared.db, shared.browser, etc. - MarketPlace uses container_type/container_id instead of post_id FK Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
23 lines
711 B
Python
23 lines
711 B
Python
from urllib.parse import urljoin
|
|
from urllib.parse import urljoin
|
|
from shared.config import config
|
|
from shared.utils import log
|
|
from ...listings import scrape_products
|
|
|
|
async def capture_sub(
|
|
sub,
|
|
slug,
|
|
):
|
|
sub_slug = sub.get("slug")
|
|
if not sub_slug:
|
|
return
|
|
sub_url = urljoin(config()["base_url"], f"/{slug}/{sub_slug}")
|
|
log(f"[{slug}/{sub_slug}] page 1…")
|
|
items_s, total_pages_s = await scrape_products(sub_url, page=1)
|
|
spmax = int(total_pages_s or 1)
|
|
for p in range(2, spmax + 1):
|
|
log(f"[{slug}/{sub_slug}] page {p}…")
|
|
items_ps, _ = await scrape_products(sub_url, page=p)
|
|
items_s.extend(items_ps)
|
|
return (sub_url, items_s, total_pages_s)
|