Decouple cart/market DBs: denormalize product data, AP internal inbox, OAuth scraper auth
Remove cross-DB relationships (CartItem.product, CartItem.market_place, OrderItem.product) that break with per-service databases. Denormalize product and marketplace fields onto cart_items/order_items at write time. - Add AP internal inbox infrastructure (shared/infrastructure/internal_inbox*) for synchronous inter-service writes via HMAC-authenticated POST - Cart inbox blueprint handles Add/Remove/Update rose:CartItem activities - Market app sends AP activities to cart inbox instead of writing CartItem directly - Cart services use denormalized columns instead of cross-DB hydration/joins - Add marketplaces-by-ids data endpoint to market service - Alembic migration adds denormalized columns to cart_items and order_items - Add OAuth device flow auth to market scraper persist_api (artdag client pattern) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -76,4 +76,35 @@ def register() -> Blueprint:
|
||||
|
||||
_handlers["products-by-ids"] = _products_by_ids
|
||||
|
||||
# --- marketplaces-by-ids ---
|
||||
async def _marketplaces_by_ids():
|
||||
"""Return marketplace data for a list of IDs (comma-separated)."""
|
||||
from sqlalchemy import select
|
||||
from shared.models.market_place import MarketPlace
|
||||
|
||||
ids_raw = request.args.get("ids", "")
|
||||
try:
|
||||
ids = [int(x) for x in ids_raw.split(",") if x.strip()]
|
||||
except ValueError:
|
||||
return {"error": "ids must be comma-separated integers"}, 400
|
||||
if not ids:
|
||||
return []
|
||||
|
||||
rows = (await g.s.execute(
|
||||
select(MarketPlace).where(MarketPlace.id.in_(ids))
|
||||
)).scalars().all()
|
||||
|
||||
return [
|
||||
{
|
||||
"id": m.id,
|
||||
"name": m.name,
|
||||
"slug": m.slug,
|
||||
"container_type": m.container_type,
|
||||
"container_id": m.container_id,
|
||||
}
|
||||
for m in rows
|
||||
]
|
||||
|
||||
_handlers["marketplaces-by-ids"] = _marketplaces_by_ids
|
||||
|
||||
return bp
|
||||
|
||||
@@ -162,29 +162,25 @@ def register():
|
||||
|
||||
|
||||
from bp.cart.services.identity import current_cart_identity
|
||||
#from bp.cart.routes import view_cart
|
||||
from models.market import CartItem
|
||||
from quart import request, url_for
|
||||
from shared.infrastructure.internal_inbox_client import send_internal_activity
|
||||
from shared.infrastructure.data_client import fetch_data
|
||||
|
||||
@bp.post("/cart/")
|
||||
@clear_cache(tag="browse", tag_scope="user")
|
||||
async def cart():
|
||||
slug = g.product_slug
|
||||
# make sure product exists (we *allow* deleted_at != None later if you want)
|
||||
product_id = await g.s.scalar(
|
||||
select(Product.id).where(
|
||||
# Load product from local db_market
|
||||
product = await g.s.scalar(
|
||||
select(Product).where(
|
||||
Product.slug == slug,
|
||||
Product.deleted_at.is_(None),
|
||||
)
|
||||
)
|
||||
|
||||
product = await g.s.scalar(
|
||||
select(Product).where(Product.id == product_id)
|
||||
)
|
||||
if not product:
|
||||
return await make_response("Product not found", 404)
|
||||
|
||||
# --- NEW: read `count` from body (JSON or form), default to 1 ---
|
||||
# Read `count` from body (JSON or form), default to 1
|
||||
count = 1
|
||||
try:
|
||||
if request.is_json:
|
||||
@@ -196,68 +192,72 @@ def register():
|
||||
if "count" in form:
|
||||
count = int(form["count"])
|
||||
except (ValueError, TypeError):
|
||||
# if parsing fails, just fall back to 1
|
||||
count = 1
|
||||
# --- END NEW ---
|
||||
|
||||
ident = current_cart_identity()
|
||||
market = getattr(g, "market", None)
|
||||
|
||||
# Load cart items for current user/session
|
||||
from sqlalchemy.orm import selectinload
|
||||
cart_filters = [CartItem.deleted_at.is_(None)]
|
||||
if ident["user_id"] is not None:
|
||||
cart_filters.append(CartItem.user_id == ident["user_id"])
|
||||
else:
|
||||
cart_filters.append(CartItem.session_id == ident["session_id"])
|
||||
cart_result = await g.s.execute(
|
||||
select(CartItem)
|
||||
.where(*cart_filters)
|
||||
.order_by(CartItem.created_at.desc())
|
||||
.options(
|
||||
selectinload(CartItem.product),
|
||||
selectinload(CartItem.market_place),
|
||||
# Build AP activity with denormalized product data
|
||||
activity_type = "Add" if count > 0 else "Remove"
|
||||
activity = {
|
||||
"type": activity_type,
|
||||
"object": {
|
||||
"type": "rose:CartItem",
|
||||
"user_id": ident["user_id"],
|
||||
"session_id": ident["session_id"],
|
||||
"product_id": product.id,
|
||||
"quantity": count,
|
||||
"market_place_id": market.id if market else None,
|
||||
# Denormalized product data
|
||||
"product_title": product.title,
|
||||
"product_slug": product.slug,
|
||||
"product_image": product.image,
|
||||
"product_brand": product.brand,
|
||||
"product_regular_price": str(product.regular_price) if product.regular_price is not None else None,
|
||||
"product_special_price": str(product.special_price) if product.special_price is not None else None,
|
||||
"product_price_currency": product.regular_price_currency,
|
||||
# Denormalized marketplace data
|
||||
"market_place_name": market.name if market else None,
|
||||
"market_place_container_id": market.container_id if market else None,
|
||||
},
|
||||
}
|
||||
|
||||
await send_internal_activity("cart", activity)
|
||||
|
||||
# Fetch updated cart items from cart service for template rendering
|
||||
raw_cart = await fetch_data(
|
||||
"cart", "cart-items",
|
||||
params={
|
||||
k: v for k, v in {
|
||||
"user_id": ident["user_id"],
|
||||
"session_id": ident["session_id"],
|
||||
}.items() if v is not None
|
||||
},
|
||||
required=False,
|
||||
) or []
|
||||
|
||||
# Build minimal cart list for template (product slug + quantity)
|
||||
from types import SimpleNamespace
|
||||
g.cart = [
|
||||
SimpleNamespace(
|
||||
product_id=ci["product_id"],
|
||||
product=SimpleNamespace(slug=ci["product_slug"]),
|
||||
quantity=ci["quantity"],
|
||||
)
|
||||
)
|
||||
g.cart = list(cart_result.scalars().all())
|
||||
for ci in raw_cart
|
||||
]
|
||||
|
||||
ci = next(
|
||||
(item for item in g.cart if item.product_id == product_id),
|
||||
ci_ns = next(
|
||||
(item for item in g.cart if item.product_id == product.id),
|
||||
None,
|
||||
)
|
||||
|
||||
# --- NEW: set quantity based on `count` ---
|
||||
if ci:
|
||||
if count > 0:
|
||||
ci.quantity = count
|
||||
else:
|
||||
# count <= 0 → remove from cart entirely
|
||||
ci.quantity=0
|
||||
g.cart.remove(ci)
|
||||
await g.s.delete(ci)
|
||||
|
||||
else:
|
||||
if count > 0:
|
||||
ci = CartItem(
|
||||
user_id=ident["user_id"],
|
||||
session_id=ident["session_id"],
|
||||
product_id=product.id,
|
||||
product=product,
|
||||
quantity=count,
|
||||
market_place_id=getattr(g, "market", None) and g.market.id,
|
||||
)
|
||||
g.cart.append(ci)
|
||||
g.s.add(ci)
|
||||
# if count <= 0 and no existing item, do nothing
|
||||
# --- END NEW ---
|
||||
|
||||
# no explicit commit; your session middleware should handle it
|
||||
|
||||
# htmx response: OOB-swap mini cart + product buttons
|
||||
if request.headers.get("HX-Request") == "true":
|
||||
return await render_template(
|
||||
"_types/product/_added.html",
|
||||
cart=g.cart,
|
||||
item=ci,
|
||||
item=ci_ns,
|
||||
)
|
||||
|
||||
# normal POST: go to cart page
|
||||
|
||||
148
market/scrape/auth.py
Normal file
148
market/scrape/auth.py
Normal file
@@ -0,0 +1,148 @@
|
||||
"""OAuth device flow authentication for the market scraper.
|
||||
|
||||
Same flow as the artdag CLI client:
|
||||
1. Request device code from account server
|
||||
2. User approves in browser
|
||||
3. Poll for access token
|
||||
4. Save token to ~/.artdag/token.json (shared with artdag CLI)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
|
||||
TOKEN_DIR = Path.home() / ".artdag"
|
||||
TOKEN_FILE = TOKEN_DIR / "token.json"
|
||||
|
||||
_DEFAULT_ACCOUNT_SERVER = "https://account.rose-ash.com"
|
||||
|
||||
|
||||
def _get_account_server() -> str:
|
||||
return os.getenv("ARTDAG_ACCOUNT", _DEFAULT_ACCOUNT_SERVER)
|
||||
|
||||
|
||||
def load_token() -> dict:
|
||||
"""Load saved token from ~/.artdag/token.json."""
|
||||
if TOKEN_FILE.exists():
|
||||
try:
|
||||
with open(TOKEN_FILE) as f:
|
||||
return json.load(f)
|
||||
except (json.JSONDecodeError, IOError):
|
||||
return {}
|
||||
return {}
|
||||
|
||||
|
||||
def save_token(token_data: dict):
|
||||
"""Save token to ~/.artdag/token.json (mode 0600)."""
|
||||
TOKEN_DIR.mkdir(parents=True, exist_ok=True)
|
||||
with open(TOKEN_FILE, "w") as f:
|
||||
json.dump(token_data, f, indent=2)
|
||||
TOKEN_FILE.chmod(0o600)
|
||||
|
||||
|
||||
def get_access_token(require: bool = True) -> str | None:
|
||||
"""Return the saved access token, or None.
|
||||
|
||||
When *require* is True and no token is found, triggers interactive
|
||||
device-flow login (same as ``artdag login``).
|
||||
"""
|
||||
token = load_token().get("access_token")
|
||||
if not token and require:
|
||||
print("No saved token found — starting device-flow login...")
|
||||
token_data = login()
|
||||
token = token_data.get("access_token")
|
||||
return token
|
||||
|
||||
|
||||
def auth_headers() -> dict[str, str]:
|
||||
"""Return Authorization header dict for HTTP requests."""
|
||||
token = get_access_token(require=True)
|
||||
return {"Authorization": f"Bearer {token}"}
|
||||
|
||||
|
||||
def login() -> dict:
|
||||
"""Interactive device-flow login (blocking, prints to stdout).
|
||||
|
||||
Returns the token data dict on success. Exits on failure.
|
||||
"""
|
||||
account = _get_account_server()
|
||||
|
||||
# 1. Request device code
|
||||
try:
|
||||
with httpx.Client(timeout=10) as client:
|
||||
resp = client.post(
|
||||
f"{account}/auth/device/authorize",
|
||||
json={"client_id": "artdag"},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
except httpx.HTTPError as e:
|
||||
print(f"Login failed: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
device_code = data["device_code"]
|
||||
user_code = data["user_code"]
|
||||
verification_uri = data["verification_uri"]
|
||||
expires_in = data.get("expires_in", 900)
|
||||
interval = data.get("interval", 5)
|
||||
|
||||
print("To sign in, open this URL in your browser:")
|
||||
print(f" {verification_uri}")
|
||||
print(f" and enter code: {user_code}")
|
||||
print()
|
||||
|
||||
# Try to open browser
|
||||
try:
|
||||
import webbrowser
|
||||
webbrowser.open(verification_uri)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 2. Poll for approval
|
||||
print("Waiting for authorization", end="", flush=True)
|
||||
deadline = time.time() + expires_in
|
||||
|
||||
with httpx.Client(timeout=10) as client:
|
||||
while time.time() < deadline:
|
||||
time.sleep(interval)
|
||||
print(".", end="", flush=True)
|
||||
|
||||
try:
|
||||
resp = client.post(
|
||||
f"{account}/auth/device/token",
|
||||
json={"device_code": device_code, "client_id": "artdag"},
|
||||
)
|
||||
data = resp.json()
|
||||
except httpx.HTTPError:
|
||||
continue
|
||||
|
||||
error = data.get("error")
|
||||
if error == "authorization_pending":
|
||||
continue
|
||||
elif error == "expired_token":
|
||||
print("\nCode expired. Please try again.", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
elif error == "access_denied":
|
||||
print("\nAuthorization denied.", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
elif error:
|
||||
print(f"\nLogin failed: {error}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
# Success
|
||||
token_data = {
|
||||
"access_token": data["access_token"],
|
||||
"username": data.get("username", ""),
|
||||
"display_name": data.get("display_name", ""),
|
||||
}
|
||||
save_token(token_data)
|
||||
print(f"\nLogged in as {token_data['username'] or token_data['display_name']}")
|
||||
return token_data
|
||||
|
||||
print("\nTimed out waiting for authorization.", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
@@ -1,16 +1,16 @@
|
||||
# replace your existing upsert_product with this version
|
||||
|
||||
import os
|
||||
import httpx
|
||||
|
||||
from typing import List
|
||||
|
||||
from ..auth import auth_headers
|
||||
|
||||
async def capture_listing(
|
||||
url: str,
|
||||
items: List[str],
|
||||
total_pages: int
|
||||
):
|
||||
|
||||
|
||||
sync_url = os.getenv("CAPTURE_LISTING_URL", "http://localhost:8001/market/suma-market/api/products/listing/")
|
||||
|
||||
async with httpx.AsyncClient(timeout=httpx.Timeout(20.0, connect=10.0)) as client:
|
||||
@@ -19,7 +19,7 @@ async def capture_listing(
|
||||
"items": items,
|
||||
"total_pages": total_pages
|
||||
}
|
||||
resp = await client.post(sync_url, json=_d)
|
||||
resp = await client.post(sync_url, json=_d, headers=auth_headers())
|
||||
# Raise for non-2xx
|
||||
resp.raise_for_status()
|
||||
data = resp.json() if resp.content else {}
|
||||
|
||||
@@ -1,14 +1,14 @@
|
||||
# replace your existing upsert_product with this version
|
||||
|
||||
import os
|
||||
import httpx
|
||||
|
||||
from ..auth import auth_headers
|
||||
|
||||
|
||||
async def log_product_result(
|
||||
ok: bool,
|
||||
payload
|
||||
):
|
||||
|
||||
|
||||
sync_url = os.getenv("PRODUCT_LOG_URL", "http://localhost:8000/market/api/products/log/")
|
||||
|
||||
async with httpx.AsyncClient(timeout=httpx.Timeout(20.0, connect=10.0)) as client:
|
||||
@@ -16,7 +16,7 @@ async def log_product_result(
|
||||
"ok": ok,
|
||||
"payload": payload
|
||||
}
|
||||
resp = await client.post(sync_url, json=_d)
|
||||
resp = await client.post(sync_url, json=_d, headers=auth_headers())
|
||||
# Raise for non-2xx
|
||||
resp.raise_for_status()
|
||||
data = resp.json() if resp.content else {}
|
||||
|
||||
@@ -1,17 +1,17 @@
|
||||
# replace your existing upsert_product with this version
|
||||
|
||||
import os
|
||||
import httpx
|
||||
|
||||
from typing import Dict
|
||||
|
||||
from ..auth import auth_headers
|
||||
|
||||
async def save_nav(
|
||||
nav: Dict,
|
||||
):
|
||||
sync_url = os.getenv("SAVE_NAV_URL", "http://localhost:8001/market/suma-market/api/products/nav/")
|
||||
|
||||
async with httpx.AsyncClient(timeout=httpx.Timeout(20.0, connect=10.0)) as client:
|
||||
resp = await client.post(sync_url, json=nav)
|
||||
resp = await client.post(sync_url, json=nav, headers=auth_headers())
|
||||
# Raise for non-2xx
|
||||
resp.raise_for_status()
|
||||
data = resp.json() if resp.content else {}
|
||||
|
||||
@@ -3,11 +3,13 @@ import httpx
|
||||
|
||||
from typing import Dict
|
||||
|
||||
from ..auth import auth_headers
|
||||
|
||||
async def save_subcategory_redirects(mapping: Dict[str, str]) -> None:
|
||||
sync_url = os.getenv("SAVE_REDIRECTS", "http://localhost:8000/market/api/products/redirects/")
|
||||
|
||||
async with httpx.AsyncClient(timeout=httpx.Timeout(20.0, connect=10.0)) as client:
|
||||
resp = await client.post(sync_url, json=mapping)
|
||||
resp = await client.post(sync_url, json=mapping, headers=auth_headers())
|
||||
# Raise for non-2xx
|
||||
resp.raise_for_status()
|
||||
data = resp.json() if resp.content else {}
|
||||
|
||||
@@ -5,6 +5,8 @@ import httpx
|
||||
|
||||
from typing import Dict, List, Any
|
||||
|
||||
from ..auth import auth_headers
|
||||
|
||||
async def upsert_product(
|
||||
slug,
|
||||
href,
|
||||
@@ -30,7 +32,7 @@ async def upsert_product(
|
||||
|
||||
async def _do_call() -> Dict[str, Any]:
|
||||
async with httpx.AsyncClient(timeout=httpx.Timeout(20.0, connect=10.0)) as client:
|
||||
resp = await client.post(sync_url, json=payload)
|
||||
resp = await client.post(sync_url, json=payload, headers=auth_headers())
|
||||
resp.raise_for_status()
|
||||
# tolerate empty body
|
||||
if not resp.content:
|
||||
|
||||
Reference in New Issue
Block a user