feat: initialize market app with browsing, product, and scraping code
Some checks failed
Build and Deploy / build-and-deploy (push) Has been cancelled
Some checks failed
Build and Deploy / build-and-deploy (push) Has been cancelled
Split from coop monolith. Includes: - Market/browse/product blueprints - Product sync API - Suma scraping pipeline - Templates for market, browse, and product views - Dockerfile and CI workflow for independent deployment
This commit is contained in:
16
scrape/product/helpers/text.py
Normal file
16
scrape/product/helpers/text.py
Normal file
@@ -0,0 +1,16 @@
|
||||
|
||||
from __future__ import annotations
|
||||
import re
|
||||
from utils import normalize_text
|
||||
from config import config
|
||||
|
||||
def clean_title(t: str) -> str:
|
||||
t = normalize_text(t)
|
||||
t = re.sub(r":\s*$", "", t)
|
||||
return t
|
||||
|
||||
def is_blacklisted_heading(title: str) -> bool:
|
||||
"""Return True if heading should be skipped based on config blacklist."""
|
||||
bl = (config().get("blacklist") or {}).get("product-details") or []
|
||||
low = (title or "").strip().lower()
|
||||
return any(low == (s or "").strip().lower() for s in bl)
|
||||
Reference in New Issue
Block a user