feat: decouple market from shared_lib, add app-owned models

Phase 1-3 of decoupling:
- path_setup.py adds project root to sys.path
- Market-owned models in market/models/ (market, market_place)
- All imports updated: shared.infrastructure, shared.db, shared.browser, etc.
- MarketPlace uses container_type/container_id instead of post_id FK

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
giles
2026-02-11 12:46:32 +00:00
parent 41b4e0fe24
commit 478636f799
51 changed files with 604 additions and 93 deletions

0
__init__.py Normal file
View File

40
app.py
View File

@@ -7,11 +7,10 @@ from quart import g, abort, render_template, make_response
from jinja2 import FileSystemLoader, ChoiceLoader from jinja2 import FileSystemLoader, ChoiceLoader
from sqlalchemy import select from sqlalchemy import select
from shared.factory import create_base_app from shared.infrastructure.factory import create_base_app
from shared.cart_loader import load_cart from shared.config import config
from config import config
from suma_browser.app.bp import register_market_bp from bp import register_market_bp
async def market_context() -> dict: async def market_context() -> dict:
@@ -21,8 +20,8 @@ async def market_context() -> dict:
- menu_items: fetched from coop internal API - menu_items: fetched from coop internal API
- cart_count/cart_total: fetched from cart internal API - cart_count/cart_total: fetched from cart internal API
""" """
from shared.context import base_context from shared.infrastructure.context import base_context
from shared.internal_api import get as api_get, dictobj from shared.infrastructure.internal_api import get as api_get, dictobj
ctx = await base_context() ctx = await base_context()
@@ -44,9 +43,9 @@ async def market_context() -> dict:
def create_app() -> "Quart": def create_app() -> "Quart":
from models.market_place import MarketPlace from models.market_place import MarketPlace
from models.ghost_content import Post from blog.models.ghost_content import Post
app = create_base_app("market", context_fn=market_context, before_request_fns=[load_cart]) app = create_base_app("market", context_fn=market_context)
# App-specific templates override shared templates # App-specific templates override shared templates
app_templates = str(Path(__file__).resolve().parent / "templates") app_templates = str(Path(__file__).resolve().parent / "templates")
@@ -111,12 +110,13 @@ def create_app() -> "Quart":
}, },
} }
# Load market scoped to post # Load market scoped to post (container pattern)
market = ( market = (
await g.s.execute( await g.s.execute(
select(MarketPlace).where( select(MarketPlace).where(
MarketPlace.slug == market_slug, MarketPlace.slug == market_slug,
MarketPlace.post_id == post.id, MarketPlace.container_type == "page",
MarketPlace.container_id == post.id,
MarketPlace.deleted_at.is_(None), MarketPlace.deleted_at.is_(None),
) )
) )
@@ -135,16 +135,24 @@ def create_app() -> "Quart":
# --- Root route: market listing --- # --- Root route: market listing ---
@app.get("/") @app.get("/")
async def markets_listing(): async def markets_listing():
from sqlalchemy.orm import selectinload rows = (
markets = (
await g.s.execute( await g.s.execute(
select(MarketPlace) select(MarketPlace, Post)
.join(
Post,
(MarketPlace.container_type == "page")
& (MarketPlace.container_id == Post.id),
)
.where(MarketPlace.deleted_at.is_(None)) .where(MarketPlace.deleted_at.is_(None))
.options(selectinload(MarketPlace.post))
.order_by(MarketPlace.name) .order_by(MarketPlace.name)
) )
).scalars().all() ).all()
# Attach the joined post to each market for template access
markets = []
for market, post in rows:
market.page = post
markets.append(market)
html = await render_template( html = await render_template(
"_types/market/markets_listing.html", "_types/market/markets_listing.html",

View File

@@ -27,8 +27,8 @@ from models.market import (
ProductAllergen, ProductAllergen,
) )
from suma_browser.app.redis_cacher import clear_cache from shared.browser.app.redis_cacher import clear_cache
from suma_browser.app.csrf import csrf_exempt from shared.browser.app.csrf import csrf_exempt
products_api = Blueprint("products_api", __name__, url_prefix="/api/products") products_api = Blueprint("products_api", __name__, url_prefix="/api/products")

View File

@@ -10,7 +10,7 @@ from quart import (
make_response, make_response,
current_app, current_app,
) )
from config import config from shared.config import config
from .services.nav import category_context, get_nav from .services.nav import category_context, get_nav
from .services.blacklist.category import is_category_blocked from .services.blacklist.category import is_category_blocked
@@ -21,8 +21,8 @@ from .services import (
_current_url_without_page, _current_url_without_page,
) )
from suma_browser.app.redis_cacher import cache_page from shared.browser.app.redis_cacher import cache_page
from suma_browser.app.utils.htmx import is_htmx_request from shared.browser.app.utils.htmx import is_htmx_request
def register(): def register():
browse_bp = Blueprint("browse", __name__) browse_bp = Blueprint("browse", __name__)

View File

@@ -1,7 +1,7 @@
# suma_browser/category_blacklist.py # suma_browser/category_blacklist.py
from __future__ import annotations from __future__ import annotations
from typing import Optional from typing import Optional
from config import config from shared.config import config
def _norm(s: str) -> str: def _norm(s: str) -> str:
return (s or "").strip().lower().strip("/") return (s or "").strip().lower().strip("/")

View File

@@ -1,6 +1,6 @@
from typing import Set, Optional from typing import Set, Optional
from ..slugs import canonical_html_slug from ..slugs import canonical_html_slug
from config import config from shared.config import config
_blocked: Set[str] = set() _blocked: Set[str] = set()
_mtime: Optional[float] = None _mtime: Optional[float] = None

View File

@@ -1,5 +1,5 @@
import re import re
from config import config from shared.config import config
def _norm_title_key(t: str) -> str: def _norm_title_key(t: str) -> str:
t = (t or "").strip().lower() t = (t or "").strip().lower()

View File

@@ -1,7 +1,7 @@
from __future__ import annotations from __future__ import annotations
import os, json import os, json
from typing import List, Optional from typing import List, Optional
from config import config from shared.config import config
from .blacklist.product import is_product_blocked from .blacklist.product import is_product_blocked

View File

@@ -4,7 +4,7 @@ from typing import Dict, List, Optional
from sqlalchemy import select, and_ from sqlalchemy import select, and_
from sqlalchemy.orm import selectinload from sqlalchemy.orm import selectinload
from config import config # if unused elsewhere, you can remove this import from shared.config import config # if unused elsewhere, you can remove this import
# ORM models # ORM models
from models.market import ( from models.market import (

View File

@@ -5,7 +5,7 @@ import re
from typing import Dict, List, Tuple, Optional from typing import Dict, List, Tuple, Optional
from urllib.parse import urlparse, urljoin from urllib.parse import urlparse, urljoin
from config import config from shared.config import config
from . import db_backend as cb from . import db_backend as cb
from .blacklist.category import is_category_blocked # Reverse map: slug -> label from .blacklist.category import is_category_blocked # Reverse map: slug -> label

View File

@@ -6,11 +6,11 @@ from quart import (
g, g,
request, request,
) )
from config import config from shared.config import config
from .products import products, products_nocounts from .products import products, products_nocounts
from .blacklist.product_details import is_blacklisted_heading from .blacklist.product_details import is_blacklisted_heading
from utils import host_url from shared.utils import host_url
from sqlalchemy import select from sqlalchemy import select
@@ -163,7 +163,7 @@ def _massage_product(d):
# Re-export from canonical shared location # Re-export from canonical shared location
from shared.http_utils import vary as _vary, current_url_without_page as _current_url_without_page from shared.infrastructure.http_utils import vary as _vary, current_url_without_page as _current_url_without_page
async def _is_liked(user_id: int | None, slug: str) -> bool: async def _is_liked(user_id: int | None, slug: str) -> bool:
""" """

View File

@@ -1,6 +1,6 @@
import re import re
from urllib.parse import urljoin, urlparse from urllib.parse import urljoin, urlparse
from config import config from shared.config import config
def product_slug_from_href(href: str) -> str: def product_slug_from_href(href: str) -> str:
p = urlparse(href) p = urlparse(href)

View File

@@ -1,4 +1,4 @@
# Re-export from canonical shared location # Re-export from canonical shared location
from shared.cart_identity import CartIdentity, current_cart_identity from shared.infrastructure.cart_identity import CartIdentity, current_cart_identity
__all__ = ["CartIdentity", "current_cart_identity"] __all__ = ["CartIdentity", "current_cart_identity"]

View File

@@ -5,7 +5,7 @@ from quart import (
) )
from suma_browser.app.authz import require_admin from shared.browser.app.authz import require_admin
def register(): def register():
@@ -15,7 +15,7 @@ def register():
@bp.get("/") @bp.get("/")
@require_admin @require_admin
async def admin(): async def admin():
from suma_browser.app.utils.htmx import is_htmx_request from shared.browser.app.utils.htmx import is_htmx_request
# Determine which template to use based on request type # Determine which template to use based on request type
if not is_htmx_request(): if not is_htmx_request():

View File

@@ -2,10 +2,10 @@ from quart import request
from typing import Iterable, Optional, Union from typing import Iterable, Optional, Union
from suma_browser.app.filters.qs_base import ( from shared.browser.app.filters.qs_base import (
KEEP, _norm, make_filter_set, build_qs, KEEP, _norm, make_filter_set, build_qs,
) )
from suma_browser.app.filters.query_types import MarketQuery from shared.browser.app.filters.query_types import MarketQuery
def decode() -> MarketQuery: def decode() -> MarketQuery:

View File

@@ -15,8 +15,8 @@ from ..browse.services.slugs import canonical_html_slug
from ..browse.services.blacklist.product import is_product_blocked from ..browse.services.blacklist.product import is_product_blocked
from ..browse.services import db_backend as cb from ..browse.services import db_backend as cb
from ..browse.services import _massage_product from ..browse.services import _massage_product
from utils import host_url from shared.utils import host_url
from suma_browser.app.redis_cacher import cache_page, clear_cache from shared.browser.app.redis_cacher import cache_page, clear_cache
from ..cart.services import total from ..cart.services import total
from .services.product_operations import toggle_product_like, massage_full_product from .services.product_operations import toggle_product_like, massage_full_product
@@ -94,7 +94,7 @@ def register():
@bp.get("/") @bp.get("/")
@cache_page(tag="browse") @cache_page(tag="browse")
async def product_detail(slug: str): async def product_detail(slug: str):
from suma_browser.app.utils.htmx import is_htmx_request from shared.browser.app.utils.htmx import is_htmx_request
# Determine which template to use based on request type # Determine which template to use based on request type
if not is_htmx_request(): if not is_htmx_request():
@@ -136,11 +136,11 @@ def register():
) )
return html return html
@bp.get("/admin/") @bp.get("/admin/")
async def admin(slug: str): async def admin(slug: str):
from suma_browser.app.utils.htmx import is_htmx_request from shared.browser.app.utils.htmx import is_htmx_request
if not is_htmx_request(): if not is_htmx_request():
# Normal browser request: full page with layout # Normal browser request: full page with layout
@@ -152,8 +152,8 @@ def register():
return await make_response(html) return await make_response(html)
from suma_browser.app.bp.cart.services.identity import current_cart_identity from bp.cart.services.identity import current_cart_identity
#from suma_browser.app.bp.cart.routes import view_cart #from bp.cart.routes import view_cart
from models.market import CartItem from models.market import CartItem
from quart import request, url_for from quart import request, url_for
@@ -242,7 +242,7 @@ def register():
) )
# normal POST: go to cart page # normal POST: go to cart page
from shared.urls import cart_url from shared.infrastructure.urls import cart_url
return redirect(cart_url("/")) return redirect(cart_url("/"))

View File

@@ -13,7 +13,7 @@ def massage_full_product(product: Product) -> dict:
Convert a Product ORM model to a dictionary with all fields. Convert a Product ORM model to a dictionary with all fields.
Used for rendering product detail pages. Used for rendering product detail pages.
""" """
from suma_browser.app.bp.browse.services import _massage_product from bp.browse.services import _massage_product
gallery = [] gallery = []
if product.image: if product.image:

8
models/__init__.py Normal file
View File

@@ -0,0 +1,8 @@
from .market import (
Product, ProductLike, ProductImage, ProductSection,
NavTop, NavSub, Listing, ListingItem,
LinkError, LinkExternal, SubcategoryRedirect, ProductLog,
ProductLabel, ProductSticker, ProductAttribute, ProductNutrition, ProductAllergen,
CartItem,
)
from .market_place import MarketPlace

441
models/market.py Normal file
View File

@@ -0,0 +1,441 @@
# at top of persist_snapshot.py:
from datetime import datetime
from typing import Optional, List
from sqlalchemy.orm import Mapped, mapped_column, relationship
from typing import List, Optional
from sqlalchemy import (
String, Text, Integer, ForeignKey, DateTime, Boolean, Numeric,
UniqueConstraint, Index, func
)
from shared.db.base import Base # you already import Base in app.py
class Product(Base):
__tablename__ = "products"
id: Mapped[int] = mapped_column(primary_key=True)
slug: Mapped[str] = mapped_column(String(255), unique=True, index=True, nullable=False)
title: Mapped[Optional[str]] = mapped_column(String(512))
image: Mapped[Optional[str]] = mapped_column(Text)
description_short: Mapped[Optional[str]] = mapped_column(Text)
description_html: Mapped[Optional[str]] = mapped_column(Text)
suma_href: Mapped[Optional[str]] = mapped_column(Text)
brand: Mapped[Optional[str]] = mapped_column(String(255))
rrp: Mapped[Optional[float]] = mapped_column(Numeric(12, 2))
rrp_currency: Mapped[Optional[str]] = mapped_column(String(16))
rrp_raw: Mapped[Optional[str]] = mapped_column(String(128))
price_per_unit: Mapped[Optional[float]] = mapped_column(Numeric(12, 4))
price_per_unit_currency: Mapped[Optional[str]] = mapped_column(String(16))
price_per_unit_raw: Mapped[Optional[str]] = mapped_column(String(128))
special_price: Mapped[Optional[float]] = mapped_column(Numeric(12, 2))
special_price_currency: Mapped[Optional[str]] = mapped_column(String(16))
special_price_raw: Mapped[Optional[str]] = mapped_column(String(128))
regular_price: Mapped[Optional[float]] = mapped_column(Numeric(12, 2))
regular_price_currency: Mapped[Optional[str]] = mapped_column(String(16))
regular_price_raw: Mapped[Optional[str]] = mapped_column(String(128))
oe_list_price: Mapped[Optional[float]] = mapped_column(Numeric(12, 2))
case_size_count: Mapped[Optional[int]] = mapped_column(Integer)
case_size_item_qty: Mapped[Optional[float]] = mapped_column(Numeric(12, 3))
case_size_item_unit: Mapped[Optional[str]] = mapped_column(String(32))
case_size_raw: Mapped[Optional[str]] = mapped_column(String(128))
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True),
nullable=False,
server_default=func.now(),
)
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True),
nullable=False,
server_default=func.now(),
)
deleted_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True))
images: Mapped[List["ProductImage"]] = relationship(
back_populates="product",
cascade="all, delete-orphan",
passive_deletes=True,
)
sections: Mapped[List["ProductSection"]] = relationship(
back_populates="product",
cascade="all, delete-orphan",
passive_deletes=True,
)
labels: Mapped[List["ProductLabel"]] = relationship(
cascade="all, delete-orphan",
passive_deletes=True,
)
stickers: Mapped[List["ProductSticker"]] = relationship(
cascade="all, delete-orphan",
passive_deletes=True,
)
ean: Mapped[Optional[str]] = mapped_column(String(64))
sku: Mapped[Optional[str]] = mapped_column(String(128))
unit_size: Mapped[Optional[str]] = mapped_column(String(128))
pack_size: Mapped[Optional[str]] = mapped_column(String(128))
attributes = relationship(
"ProductAttribute",
back_populates="product",
lazy="selectin",
cascade="all, delete-orphan",
)
nutrition = relationship(
"ProductNutrition",
back_populates="product",
lazy="selectin",
cascade="all, delete-orphan",
)
allergens = relationship(
"ProductAllergen",
back_populates="product",
lazy="selectin",
cascade="all, delete-orphan",
)
likes = relationship(
"ProductLike",
back_populates="product",
cascade="all, delete-orphan",
)
cart_items: Mapped[List["CartItem"]] = relationship(
"CartItem",
back_populates="product",
cascade="all, delete-orphan",
)
# NEW: all order items that reference this product
order_items: Mapped[List["OrderItem"]] = relationship(
"OrderItem",
back_populates="product",
cascade="all, delete-orphan",
)
from sqlalchemy import Column
class ProductLike(Base):
__tablename__ = "product_likes"
id = Column(Integer, primary_key=True, autoincrement=True)
user_id = Column(Integer, ForeignKey("users.id", ondelete="CASCADE"), nullable=False)
product_slug: Mapped[str] = mapped_column(ForeignKey("products.slug", ondelete="CASCADE"))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
deleted_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True))
product: Mapped["Product"] = relationship("Product", back_populates="likes", foreign_keys=[product_slug])
user = relationship("User", back_populates="liked_products") # optional, if you want reverse access
class ProductImage(Base):
__tablename__ = "product_images"
id: Mapped[int] = mapped_column(primary_key=True)
product_id: Mapped[int] = mapped_column(ForeignKey("products.id", ondelete="CASCADE"), index=True, nullable=False)
url: Mapped[str] = mapped_column(Text, nullable=False)
position: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
kind: Mapped[str] = mapped_column(String(16), nullable=False, default="gallery")
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
deleted_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True))
product: Mapped["Product"] = relationship(back_populates="images")
__table_args__ = (
UniqueConstraint("product_id", "url", "kind", name="uq_product_images_product_url_kind"),
Index("ix_product_images_position", "position"),
)
class ProductSection(Base):
__tablename__ = "product_sections"
id: Mapped[int] = mapped_column(primary_key=True)
product_id: Mapped[int] = mapped_column(
ForeignKey("products.id", ondelete="CASCADE"),
index=True,
nullable=False,
)
title: Mapped[str] = mapped_column(String(255), nullable=False)
html: Mapped[str] = mapped_column(Text, nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
deleted_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True))
product: Mapped["Product"] = relationship(back_populates="sections")
__table_args__ = (
UniqueConstraint("product_id", "title", name="uq_product_sections_product_title"),
)
# --- Nav & listings ---
class NavTop(Base):
__tablename__ = "nav_tops"
id: Mapped[int] = mapped_column(primary_key=True)
label: Mapped[str] = mapped_column(String(255), nullable=False)
slug: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
market_id: Mapped[Optional[int]] = mapped_column(
Integer,
ForeignKey("market_places.id", ondelete="SET NULL"),
nullable=True,
index=True,
)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
deleted_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True))
listings: Mapped[List["Listing"]] = relationship(back_populates="top", cascade="all, delete-orphan")
market = relationship("MarketPlace", back_populates="nav_tops")
__table_args__ = (UniqueConstraint("label", "slug", name="uq_nav_tops_label_slug"),)
class NavSub(Base):
__tablename__ = "nav_subs"
id: Mapped[int] = mapped_column(primary_key=True)
top_id: Mapped[int] = mapped_column(ForeignKey("nav_tops.id", ondelete="CASCADE"), index=True, nullable=False)
label: Mapped[Optional[str]] = mapped_column(String(255))
slug: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
href: Mapped[Optional[str]] = mapped_column(Text)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
deleted_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True))
listings: Mapped[List["Listing"]] = relationship(back_populates="sub", cascade="all, delete-orphan")
__table_args__ = (UniqueConstraint("top_id", "slug", name="uq_nav_subs_top_slug"),)
class Listing(Base):
__tablename__ = "listings"
id: Mapped[int] = mapped_column(primary_key=True)
# Old slug-based fields (optional: remove)
# top_slug: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
# sub_slug: Mapped[Optional[str]] = mapped_column(String(255), index=True)
top_id: Mapped[int] = mapped_column(ForeignKey("nav_tops.id", ondelete="CASCADE"), index=True, nullable=False)
sub_id: Mapped[Optional[int]] = mapped_column(ForeignKey("nav_subs.id", ondelete="CASCADE"), index=True)
total_pages: Mapped[Optional[int]] = mapped_column(Integer)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
deleted_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True))
top: Mapped["NavTop"] = relationship(back_populates="listings")
sub: Mapped[Optional["NavSub"]] = relationship(back_populates="listings")
__table_args__ = (
UniqueConstraint("top_id", "sub_id", name="uq_listings_top_sub"),
)
class ListingItem(Base):
__tablename__ = "listing_items"
id: Mapped[int] = mapped_column(primary_key=True)
listing_id: Mapped[int] = mapped_column(ForeignKey("listings.id", ondelete="CASCADE"), index=True, nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
deleted_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True))
slug: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
__table_args__ = (UniqueConstraint("listing_id", "slug", name="uq_listing_items_listing_slug"),)
# --- Reports / redirects / logs ---
class LinkError(Base):
__tablename__ = "link_errors"
id: Mapped[int] = mapped_column(primary_key=True)
product_slug: Mapped[Optional[str]] = mapped_column(String(255), index=True)
href: Mapped[Optional[str]] = mapped_column(Text)
text: Mapped[Optional[str]] = mapped_column(Text)
top: Mapped[Optional[str]] = mapped_column(String(255))
sub: Mapped[Optional[str]] = mapped_column(String(255))
target_slug: Mapped[Optional[str]] = mapped_column(String(255))
type: Mapped[Optional[str]] = mapped_column(String(255))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
deleted_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True))
class LinkExternal(Base):
__tablename__ = "link_externals"
id: Mapped[int] = mapped_column(primary_key=True)
product_slug: Mapped[Optional[str]] = mapped_column(String(255), index=True)
href: Mapped[Optional[str]] = mapped_column(Text)
text: Mapped[Optional[str]] = mapped_column(Text)
host: Mapped[Optional[str]] = mapped_column(String(255))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
deleted_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True))
class SubcategoryRedirect(Base):
__tablename__ = "subcategory_redirects"
id: Mapped[int] = mapped_column(primary_key=True)
old_path: Mapped[str] = mapped_column(String(512), nullable=False, index=True)
new_path: Mapped[str] = mapped_column(String(512), nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
deleted_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True))
class ProductLog(Base):
__tablename__ = "product_logs"
id: Mapped[int] = mapped_column(primary_key=True)
slug: Mapped[Optional[str]] = mapped_column(String(255), index=True)
href_tried: Mapped[Optional[str]] = mapped_column(Text)
ok: Mapped[bool] = mapped_column(Boolean, nullable=False, server_default="false")
error_type: Mapped[Optional[str]] = mapped_column(String(255))
error_message: Mapped[Optional[str]] = mapped_column(Text)
http_status: Mapped[Optional[int]] = mapped_column(Integer)
final_url: Mapped[Optional[str]] = mapped_column(Text)
transport_error: Mapped[Optional[bool]] = mapped_column(Boolean)
title: Mapped[Optional[str]] = mapped_column(String(512))
has_description_html: Mapped[Optional[bool]] = mapped_column(Boolean)
has_description_short: Mapped[Optional[bool]] = mapped_column(Boolean)
sections_count: Mapped[Optional[int]] = mapped_column(Integer)
images_count: Mapped[Optional[int]] = mapped_column(Integer)
embedded_images_count: Mapped[Optional[int]] = mapped_column(Integer)
all_images_count: Mapped[Optional[int]] = mapped_column(Integer)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
# ...existing models...
class ProductLabel(Base):
__tablename__ = "product_labels"
id: Mapped[int] = mapped_column(primary_key=True)
product_id: Mapped[int] = mapped_column(ForeignKey("products.id", ondelete="CASCADE"), index=True, nullable=False)
name: Mapped[str] = mapped_column(String(255), nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
deleted_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True))
product: Mapped["Product"] = relationship(back_populates="labels")
__table_args__ = (UniqueConstraint("product_id", "name", name="uq_product_labels_product_name"),)
class ProductSticker(Base):
__tablename__ = "product_stickers"
id: Mapped[int] = mapped_column(primary_key=True)
product_id: Mapped[int] = mapped_column(ForeignKey("products.id", ondelete="CASCADE"), index=True, nullable=False)
name: Mapped[str] = mapped_column(String(255), nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
deleted_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True))
product: Mapped["Product"] = relationship(back_populates="stickers")
__table_args__ = (UniqueConstraint("product_id", "name", name="uq_product_stickers_product_name"),)
class ProductAttribute(Base):
__tablename__ = "product_attributes"
id: Mapped[int] = mapped_column(primary_key=True)
product_id: Mapped[int] = mapped_column(ForeignKey("products.id", ondelete="CASCADE"), index=True, nullable=False)
key: Mapped[str] = mapped_column(String(255), nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
deleted_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True))
value: Mapped[Optional[str]] = mapped_column(Text)
product = relationship("Product", back_populates="attributes")
__table_args__ = (UniqueConstraint("product_id", "key", name="uq_product_attributes_product_key"),)
class ProductNutrition(Base):
__tablename__ = "product_nutrition"
id: Mapped[int] = mapped_column(primary_key=True)
product_id: Mapped[int] = mapped_column(ForeignKey("products.id", ondelete="CASCADE"), index=True, nullable=False)
key: Mapped[str] = mapped_column(String(255), nullable=False)
value: Mapped[Optional[str]] = mapped_column(String(255))
unit: Mapped[Optional[str]] = mapped_column(String(64))
product = relationship("Product", back_populates="nutrition")
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
deleted_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True))
__table_args__ = (UniqueConstraint("product_id", "key", name="uq_product_nutrition_product_key"),)
class ProductAllergen(Base):
__tablename__ = "product_allergens"
id: Mapped[int] = mapped_column(primary_key=True)
product_id: Mapped[int] = mapped_column(ForeignKey("products.id", ondelete="CASCADE"), index=True, nullable=False)
name: Mapped[str] = mapped_column(String(255), nullable=False)
contains: Mapped[bool] = mapped_column(Boolean, nullable=False, server_default="false")
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
deleted_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True))
product: Mapped["Product"] = relationship(back_populates="allergens")
__table_args__ = (UniqueConstraint("product_id", "name", name="uq_product_allergens_product_name"),)
class CartItem(Base):
__tablename__ = "cart_items"
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
# Either a logged-in user OR an anonymous session
user_id: Mapped[int | None] = mapped_column(
ForeignKey("users.id", ondelete="CASCADE"),
nullable=True,
)
session_id: Mapped[str | None] = mapped_column(
String(128),
nullable=True,
)
# IMPORTANT: link to product *id*, not slug
product_id: Mapped[int] = mapped_column(
ForeignKey("products.id", ondelete="CASCADE"),
nullable=False,
)
quantity: Mapped[int] = mapped_column(
Integer,
nullable=False,
default=1,
server_default="1",
)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True),
nullable=False,
server_default=func.now(),
)
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True),
nullable=False,
server_default=func.now(),
)
market_place_id: Mapped[int | None] = mapped_column(
ForeignKey("market_places.id", ondelete="SET NULL"),
nullable=True,
index=True,
)
deleted_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True),
nullable=True,
)
# Relationships
market_place: Mapped["MarketPlace | None"] = relationship(
"MarketPlace",
foreign_keys=[market_place_id],
)
product: Mapped["Product"] = relationship(
"Product",
back_populates="cart_items",
)
user: Mapped["User | None"] = relationship("User", back_populates="cart_items")
__table_args__ = (
Index("ix_cart_items_user_product", "user_id", "product_id"),
Index("ix_cart_items_session_product", "session_id", "product_id"),
)

52
models/market_place.py Normal file
View File

@@ -0,0 +1,52 @@
from __future__ import annotations
from datetime import datetime, timezone
from typing import Optional, List
from sqlalchemy import (
Integer, String, Text, DateTime, ForeignKey, Index, func, text,
)
from sqlalchemy.orm import Mapped, mapped_column, relationship
from shared.db.base import Base
def utcnow() -> datetime:
return datetime.now(timezone.utc)
class MarketPlace(Base):
__tablename__ = "market_places"
id: Mapped[int] = mapped_column(Integer, primary_key=True)
container_type: Mapped[str] = mapped_column(
String(32), nullable=False, server_default=text("'page'"),
)
container_id: Mapped[int] = mapped_column(Integer, nullable=False)
name: Mapped[str] = mapped_column(String(255), nullable=False)
slug: Mapped[str] = mapped_column(String(255), nullable=False)
description: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now(),
)
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, server_default=func.now(),
)
deleted_at: Mapped[Optional[datetime]] = mapped_column(
DateTime(timezone=True), nullable=True,
)
nav_tops: Mapped[List["NavTop"]] = relationship(
"NavTop", back_populates="market",
)
__table_args__ = (
Index("ix_market_places_container", "container_type", "container_id"),
Index(
"ux_market_places_slug_active",
func.lower(slug),
unique=True,
postgresql_where=text("deleted_at IS NULL"),
),
)

View File

@@ -1,7 +1,9 @@
import sys import sys
import os import os
# Add the shared library submodule to the Python path _app_dir = os.path.dirname(os.path.abspath(__file__))
_shared = os.path.join(os.path.dirname(os.path.abspath(__file__)), "shared_lib") _project_root = os.path.dirname(_app_dir)
if _shared not in sys.path:
sys.path.insert(0, _shared) for _p in (_project_root, _app_dir):
if _p not in sys.path:
sys.path.insert(0, _p)

View File

@@ -7,9 +7,9 @@ from typing import Dict, Set
from ..http_client import configure_cookies from ..http_client import configure_cookies
from ..get_auth import login from ..get_auth import login
from config import config from shared.config import config
from utils import log from shared.utils import log
# DB: persistence helpers # DB: persistence helpers

View File

@@ -3,7 +3,7 @@ from bs4 import BeautifulSoup
from urllib.parse import urlparse, urljoin from urllib.parse import urlparse, urljoin
from ._anchor_text import _anchor_text from ._anchor_text import _anchor_text
from suma_browser.app.bp.browse.services.slugs import product_slug_from_href from bp.browse.services.slugs import product_slug_from_href
from .APP_ROOT_PLACEHOLDER import APP_ROOT_PLACEHOLDER from .APP_ROOT_PLACEHOLDER import APP_ROOT_PLACEHOLDER
def _rewrite_links_fragment( def _rewrite_links_fragment(

View File

@@ -1,6 +1,6 @@
from urllib.parse import urljoin from urllib.parse import urljoin
from config import config from shared.config import config
from utils import log from shared.utils import log
from ...listings import scrape_products from ...listings import scrape_products
async def capture_category( async def capture_category(

View File

@@ -1,7 +1,7 @@
from typing import Dict, Set from typing import Dict, Set
from .capture_category import capture_category from .capture_category import capture_category
from .capture_sub import capture_sub from .capture_sub import capture_sub
from config import config from shared.config import config
async def capture_product_slugs( async def capture_product_slugs(

View File

@@ -1,7 +1,7 @@
from urllib.parse import urljoin from urllib.parse import urljoin
from urllib.parse import urljoin from urllib.parse import urljoin
from config import config from shared.config import config
from utils import log from shared.utils import log
from ...listings import scrape_products from ...listings import scrape_products
async def capture_sub( async def capture_sub(

View File

@@ -6,12 +6,12 @@ import httpx
from ...html_utils import to_fragment from ...html_utils import to_fragment
from suma_browser.app.bp.browse.services.slugs import suma_href_from_html_slug from bp.browse.services.slugs import suma_href_from_html_slug
from config import config
from utils import log from shared.config import config
from shared.utils import log
# DB: persistence helpers # DB: persistence helpers
from ...product.product_detail import scrape_product_detail from ...product.product_detail import scrape_product_detail

View File

@@ -1,7 +1,7 @@
import asyncio import asyncio
from typing import Dict, List, Set from typing import Dict, List, Set
from config import config from shared.config import config
from utils import log from shared.utils import log
from .fetch_and_upsert_product import fetch_and_upsert_product from .fetch_and_upsert_product import fetch_and_upsert_product

View File

@@ -1,7 +1,7 @@
from typing import Dict from typing import Dict
from urllib.parse import urljoin from urllib.parse import urljoin
from config import config from shared.config import config
def rewrite_nav(nav: Dict[str, Dict], nav_redirects:Dict[str, str]): def rewrite_nav(nav: Dict[str, Dict], nav_redirects:Dict[str, str]):
if nav_redirects: if nav_redirects:

View File

@@ -2,7 +2,7 @@ from typing import Optional, Dict, Any, List
from urllib.parse import urljoin from urllib.parse import urljoin
import httpx import httpx
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from config import config from shared.config import config
class LoginFailed(Exception): class LoginFailed(Exception):
def __init__(self, message: str, *, debug: Dict[str, Any]): def __init__(self, message: str, *, debug: Dict[str, Any]):

View File

@@ -3,7 +3,7 @@ from __future__ import annotations
from typing import Optional from typing import Optional
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from urllib.parse import urljoin from urllib.parse import urljoin
from config import config from shared.config import config

View File

@@ -7,7 +7,7 @@ import secrets
from typing import Optional, Dict from typing import Optional, Dict
import httpx import httpx
from config import config from shared.config import config
_CLIENT: httpx.AsyncClient | None = None _CLIENT: httpx.AsyncClient | None = None

View File

@@ -7,8 +7,8 @@ from urllib.parse import parse_qsl, urlencode, urljoin, urlparse, urlunparse
from .http_client import fetch from .http_client import fetch
from suma_browser.app.bp.browse.services.slugs import product_slug_from_href from bp.browse.services.slugs import product_slug_from_href
from suma_browser.app.bp.browse.services.state import ( from bp.browse.services.state import (
KNOWN_PRODUCT_SLUGS, KNOWN_PRODUCT_SLUGS,
_listing_page_cache, _listing_page_cache,
_listing_page_ttl, _listing_page_ttl,
@@ -16,8 +16,8 @@ from suma_browser.app.bp.browse.services.state import (
_listing_variant_ttl, _listing_variant_ttl,
now, now,
) )
from utils import normalize_text, soup_of from shared.utils import normalize_text, soup_of
from config import config from shared.config import config
def parse_total_pages_from_text(text: str) -> Optional[int]: def parse_total_pages_from_text(text: str) -> Optional[int]:

View File

@@ -5,7 +5,7 @@ from typing import Dict, List, Tuple, Optional
from urllib.parse import urlparse, urljoin from urllib.parse import urlparse, urljoin
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from config import config from shared.config import config
from .http_client import fetch # only fetch; define soup_of locally from .http_client import fetch # only fetch; define soup_of locally
#from .. import cache_backend as cb #from .. import cache_backend as cb
#from ..blacklist.category import is_category_blocked # Reverse map: slug -> label #from ..blacklist.category import is_category_blocked # Reverse map: slug -> label

View File

@@ -17,7 +17,7 @@ from models.market import (
Listing, Listing,
ListingItem, ListingItem,
) )
from db.session import get_session from shared.db.session import get_session
# --- Models are unchanged, see original code --- # --- Models are unchanged, see original code ---

View File

@@ -5,7 +5,7 @@ from typing import Dict
from models.market import ( from models.market import (
ProductLog, ProductLog,
) )
from db.session import get_session from shared.db.session import get_session
async def log_product_result(ok: bool, payload: Dict) -> None: async def log_product_result(ok: bool, payload: Dict) -> None:

View File

@@ -7,7 +7,7 @@ from models.market import (
LinkError, LinkError,
LinkExternal, LinkExternal,
) )
from db.session import get_session from shared.db.session import get_session
# --- Models are unchanged, see original code --- # --- Models are unchanged, see original code ---

View File

@@ -9,7 +9,7 @@ from models.market import (
NavTop, NavTop,
NavSub, NavSub,
) )
from db.session import get_session from shared.db.session import get_session

View File

@@ -8,7 +8,7 @@ from sqlalchemy import (
from models.market import ( from models.market import (
SubcategoryRedirect, SubcategoryRedirect,
) )
from db.session import get_session from shared.db.session import get_session
# --- Models are unchanged, see original code --- # --- Models are unchanged, see original code ---

View File

@@ -17,7 +17,7 @@ from models.market import (
ProductNutrition, ProductNutrition,
ProductAllergen ProductAllergen
) )
from db.session import get_session from shared.db.session import get_session
from ._get import _get from ._get import _get
from .log_product_result import _log_product_result from .log_product_result import _log_product_result

View File

@@ -3,7 +3,7 @@ from __future__ import annotations
from typing import Dict, List, Union from typing import Dict, List, Union
from urllib.parse import urlparse from urllib.parse import urlparse
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from utils import normalize_text from shared.utils import normalize_text
from ..registry import extractor from ..registry import extractor
@extractor @extractor

View File

@@ -2,7 +2,7 @@
from __future__ import annotations from __future__ import annotations
from typing import Dict, List from typing import Dict, List
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from utils import normalize_text from shared.utils import normalize_text
from ...html_utils import absolutize_fragment from ...html_utils import absolutize_fragment
from ..registry import extractor from ..registry import extractor
from ..helpers.desc import ( from ..helpers.desc import (

View File

@@ -2,7 +2,7 @@
from __future__ import annotations from __future__ import annotations
from typing import Dict, Union from typing import Dict, Union
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from utils import normalize_text from shared.utils import normalize_text
from ..registry import extractor from ..registry import extractor
from ..helpers.price import parse_price, parse_case_size from ..helpers.price import parse_price, parse_case_size

View File

@@ -2,7 +2,7 @@
from __future__ import annotations from __future__ import annotations
from typing import Dict, List from typing import Dict, List
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from utils import normalize_text from shared.utils import normalize_text
from ..registry import extractor from ..registry import extractor
@extractor @extractor

View File

@@ -2,7 +2,7 @@ from __future__ import annotations
from typing import Dict, List, Optional, Tuple from typing import Dict, List, Optional, Tuple
import re import re
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from utils import normalize_text from shared.utils import normalize_text
from ..registry import extractor from ..registry import extractor
from ..helpers.desc import ( from ..helpers.desc import (
split_description_container, find_description_container, split_description_container, find_description_container,

View File

@@ -2,7 +2,7 @@
from __future__ import annotations from __future__ import annotations
from typing import Dict from typing import Dict
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from utils import normalize_text from shared.utils import normalize_text
from ..registry import extractor from ..registry import extractor
@extractor @extractor

View File

@@ -2,7 +2,7 @@
from __future__ import annotations from __future__ import annotations
from typing import Dict from typing import Dict
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from utils import normalize_text from shared.utils import normalize_text
from ..registry import extractor from ..registry import extractor
@extractor @extractor

View File

@@ -2,10 +2,10 @@
from __future__ import annotations from __future__ import annotations
from typing import Dict, List, Optional, Tuple from typing import Dict, List, Optional, Tuple
from bs4 import BeautifulSoup, NavigableString, Tag from bs4 import BeautifulSoup, NavigableString, Tag
from utils import normalize_text from shared.utils import normalize_text
from ...html_utils import absolutize_fragment from ...html_utils import absolutize_fragment
from .text import clean_title, is_blacklisted_heading from .text import clean_title, is_blacklisted_heading
from config import config from shared.config import config
def split_description_container(desc_el: Tag) -> Tuple[str, List[Dict]]: def split_description_container(desc_el: Tag) -> Tuple[str, List[Dict]]:

View File

@@ -2,7 +2,7 @@
from __future__ import annotations from __future__ import annotations
from typing import List, Optional from typing import List, Optional
from urllib.parse import urljoin, urlparse from urllib.parse import urljoin, urlparse
from config import config from shared.config import config
def first_from_srcset(val: str) -> Optional[str]: def first_from_srcset(val: str) -> Optional[str]:
if not val: if not val:

View File

@@ -1,8 +1,8 @@
from __future__ import annotations from __future__ import annotations
import re import re
from utils import normalize_text from shared.utils import normalize_text
from config import config from shared.config import config
def clean_title(t: str) -> str: def clean_title(t: str) -> str:
t = normalize_text(t) t = normalize_text(t)

View File

@@ -1,10 +1,10 @@
from __future__ import annotations from __future__ import annotations
from typing import Dict, Tuple, Union from typing import Dict, Tuple, Union
from utils import soup_of from shared.utils import soup_of
from ..http_client import fetch from ..http_client import fetch
from ..html_utils import absolutize_fragment from ..html_utils import absolutize_fragment
from suma_browser.app.bp.browse.services.slugs import product_slug_from_href from bp.browse.services.slugs import product_slug_from_href
from .registry import REGISTRY, merge_missing from .registry import REGISTRY, merge_missing
from . import extractors as _auto_register # noqa: F401 (import-time side effects) from . import extractors as _auto_register # noqa: F401 (import-time side effects)