import asyncio
from typing import Dict, List, Set
from shared.config import config
from shared.utils import log
from .fetch_and_upsert_product import fetch_and_upsert_product


async def fetch_and_upsert_products(
    upsert_product,
    log_product_result,
    save_link_reports = None,
    concurrency: int=8,
    product_slugs: Set[str] = set(),
    valid_subs_by_top: Dict[str, Set[str]] = {},
    unknown_sub_paths: Set[str] = set()
):
    sem = asyncio.Semaphore(max(1, concurrency))
    link_errors: List[Dict] = []
    link_externals: List[Dict] = []
    
    category_values: Set[str] = set(config()["categories"]["allow"].values())
    to_fetch = sorted(list(product_slugs))
    log(f"Fetching {len(to_fetch)} product details (concurrency={concurrency})…")
    tasks = [asyncio.create_task(
        fetch_and_upsert_product(
            upsert_product,
            log_product_result,
            sem,
            s,
            product_slugs,
            category_values,
            valid_subs_by_top,
            link_errors,
            link_externals,
            unknown_sub_paths
        )
    ) for s in to_fetch]
    done = 0
    ok_count = 0
    for coro in asyncio.as_completed(tasks):
        ok = await coro
        done += 1
        if ok:
            ok_count += 1
        if done % 50 == 0 or done == len(tasks):
            log(f"  …{done}/{len(tasks)} saved (ok={ok_count})")
    if save_link_reports:
        await save_link_reports(link_errors, link_externals)
    