
import asyncio
from typing import List

import httpx


from ...html_utils import to_fragment
from bp.browse.services.slugs import suma_href_from_html_slug


from shared.config import config

from shared.utils import log

# DB: persistence helpers
from ...product.product_detail import scrape_product_detail
from ._product_dict_is_cf import _product_dict_is_cf
from ._rewrite_links_fragment import _rewrite_links_fragment
from ._dedupe_preserve_order import _dedupe_preserve_order
from ._collect_html_img_srcs import _collect_html_img_srcs


async def fetch_and_upsert_product(
    upsert_product,
    log_product_result,
    sem: asyncio.Semaphore,
    slug: str,
    product_slugs,
    category_values,
    valid_subs_by_top,
    link_errors,
    link_externals,
    unknown_sub_paths
) -> bool:
  href = suma_href_from_html_slug(slug)
  try:
      async with sem:
          d = await scrape_product_detail(href)
          
      is_cf, cf_token = _product_dict_is_cf(d)
      if is_cf:
          payload = {
              "slug": slug,
              "href_tried": href,
              "error_type": "CloudflareChallengeDetected",
              "error_message": f"Detected Cloudflare interstitial via token: {cf_token}",
              "cf_token": cf_token,
          }
          await log_product_result(ok=False, payload=payload)
          log(f"  ! CF challenge detected: {slug} ({cf_token})")
          return False

      # Rewrite embedded links; collect reports
      if d.get("description_html"):
          d["description_html"] = _rewrite_links_fragment(
              d["description_html"], config()["base_url"], product_slugs, category_values,
              valid_subs_by_top, slug, link_errors, link_externals, unknown_sub_paths
          )
          d["description_html"] = to_fragment(d["description_html"])
      if d.get("sections"):
          for sec in d["sections"]:
              if isinstance(sec, dict) and sec.get("html"):
                  sec["html"] = _rewrite_links_fragment(
                      sec["html"], config()["base_url"], product_slugs, category_values,
                      valid_subs_by_top, slug, link_errors, link_externals, unknown_sub_paths
                  )
                  sec["html"] = to_fragment(sec["html"])

      # Images
      gallery = _dedupe_preserve_order(d.get("images") or [])
      embedded: List[str] = []
      if d.get("description_html"):
          embedded += _collect_html_img_srcs(d["description_html"])
      for sec in d.get("sections", []) or []:
          if isinstance(sec, dict) and sec.get("html"):
              embedded += _collect_html_img_srcs(sec["html"])
      embedded = _dedupe_preserve_order(embedded)
      all_imgs = _dedupe_preserve_order(list(gallery) + list(embedded))

      d["images"] = gallery
      d["embedded_image_urls"] = embedded
      d["all_image_urls"] = all_imgs
      await upsert_product(slug, href, d)
      # DB: upsert product + success log
      return True
  except Exception as e:
      payload = {
          "slug": slug,
          "href_tried": href,
          "error_type": e.__class__.__name__,
          "error_message": str(e),
      }
      try:
          if isinstance(e, httpx.HTTPStatusError):
              payload["http_status"] = getattr(e.response, "status_code", None)
              req = getattr(e, "request", None)
              if req is not None and getattr(req, "url", None) is not None:
                  payload["final_url"] = str(req.url)
          elif isinstance(e, httpx.TransportError):
              payload["transport_error"] = True
      except Exception:
          pass
      await log_product_result(ok=False, payload=payload)
      log(f"  ! product failed: {slug} ({e})")
      return False
