#!/usr/bin/env python3
"""Build coverage manifest for _hyperscript v0.9.90 upstream tests.

Strategy:
1. Parse every test file under /tmp/hs-upstream/test/ for `test(...)` and
   `test.describe(...)` calls (Playwright style).
2. Stack nested describes into name prefixes (the describe is the category
   marker — but our JSON keys by filename basename, so we use that).
3. Match against /root/rose-ash/spec/tests/hyperscript-upstream-tests.json.
4. Classify complexity, status, block_reason.
5. Emit manifest JSON + markdown summary.
"""
import json, os, re, sys
from collections import Counter, defaultdict
from pathlib import Path

HS_ROOT = Path('/tmp/hs-upstream')
TEST_ROOT = HS_ROOT / 'test'
OUR_JSON = Path('/root/rose-ash/spec/tests/hyperscript-upstream-tests.json')
GEN_PY  = Path('/root/rose-ash/tests/playwright/generate-sx-tests.py')
OUT_JSON = Path('/root/rose-ash/spec/tests/hyperscript-upstream-manifest.json')
OUT_MD  = Path('/root/rose-ash/spec/tests/hyperscript-upstream-manifest.md')

# ---------------------------------------------------------------------------
# Load SKIP_TEST_NAMES from generator
# ---------------------------------------------------------------------------
gen_src = GEN_PY.read_text()
m = re.search(r'SKIP_TEST_NAMES\s*=\s*\{(.*?)\n\}', gen_src, re.DOTALL)
assert m, "could not find SKIP_TEST_NAMES"
skip_block = m.group(1)
SKIP_NAMES = set()
for line in skip_block.splitlines():
    line = line.strip().rstrip(',')
    if not line or line.startswith('#'):
        continue
    # line is a quoted string literal
    try:
        # carefully parse the first string literal on the line
        # use a tiny regex because names contain both single and double quotes-escaped
        mm = re.match(r'^(["\'])(.*)\1\s*,?\s*(#.*)?$', line)
        if mm:
            raw = mm.group(2)
            # python-style unescape: mainly \\' and \\"
            raw = raw.encode('utf-8').decode('unicode_escape')
            SKIP_NAMES.add(raw)
    except Exception:
        pass
print(f"Loaded {len(SKIP_NAMES)} skip names", file=sys.stderr)

# ---------------------------------------------------------------------------
# Load our snapshot JSON
# ---------------------------------------------------------------------------
our_tests = json.load(OUR_JSON.open())
our_map = {}
def norm(s):
    return re.sub(r'\s+', ' ', s.strip()).lower()
for t in our_tests:
    key = (t['category'], norm(t['name']))
    our_map[key] = t
print(f"Our JSON: {len(our_tests)} tests, {len(our_map)} unique (cat,name)", file=sys.stderr)

# ---------------------------------------------------------------------------
# Extract upstream tests
# ---------------------------------------------------------------------------
# Pattern: test(<quoted-string>, ... or test.describe(<quoted-string>, ...
# We need to parse balanced parens to find the body snippet for classification.

SKIP_FILES = {'fixtures.js', 'global-setup.js', 'global-teardown.js',
              'entry.js', 'htmx-fixtures.js', 'playwright.config.js'}

def parse_string_literal(src, i):
    """src[i] must be quote; return (value, next_i)."""
    q = src[i]
    assert q in ('"', "'", '`'), f"not a quote at {i}: {src[i-5:i+5]!r}"
    i += 1
    out = []
    while i < len(src):
        c = src[i]
        if c == '\\':
            nxt = src[i+1] if i+1 < len(src) else ''
            # preserve common escapes
            if nxt == 'n':
                out.append('\n'); i += 2
            elif nxt == 't':
                out.append('\t'); i += 2
            elif nxt == '\\':
                out.append('\\'); i += 2
            elif nxt == q:
                out.append(q); i += 2
            else:
                out.append(nxt); i += 2
        elif c == q:
            return ''.join(out), i + 1
        else:
            out.append(c); i += 1
    raise ValueError(f"unterminated string starting at {i}")

def find_matching_paren(src, start):
    """start is index of '('; return index of matching ')'."""
    depth = 0
    i = start
    while i < len(src):
        c = src[i]
        if c in ('"', "'", '`'):
            _, i = parse_string_literal(src, i)
            continue
        if c == '/' and i+1 < len(src):
            # regex or comment
            if src[i+1] == '/':
                # line comment
                j = src.find('\n', i)
                i = len(src) if j == -1 else j + 1
                continue
            if src[i+1] == '*':
                j = src.find('*/', i)
                i = len(src) if j == -1 else j + 2
                continue
            # regex literal: heuristic — only treat as regex if preceded by an
            # operator/paren/comma/etc
            prev = ''
            k = i - 1
            while k >= 0 and src[k].isspace(): k -= 1
            prev = src[k] if k >= 0 else ''
            if prev in '(,;=!?&|:+-*/<>%^~{[' or prev == '' or (k >= 0 and src[k-5:k+1] == 'return' if k>=5 else False):
                # scan regex
                j = i + 1
                while j < len(src):
                    cc = src[j]
                    if cc == '\\':
                        j += 2; continue
                    if cc == '[':
                        while j < len(src) and src[j] != ']':
                            if src[j] == '\\': j += 2
                            else: j += 1
                        j += 1; continue
                    if cc == '/':
                        j += 1
                        while j < len(src) and src[j].isalpha(): j += 1
                        break
                    if cc == '\n':
                        break
                    j += 1
                i = j
                continue
        if c == '(':
            depth += 1; i += 1
        elif c == ')':
            depth -= 1
            if depth == 0:
                return i
            i += 1
        else:
            i += 1
    return -1

def extract_tests_from_file(path):
    src = path.read_text()
    tests = []  # list of dicts: name, body, describe_name
    # Find all test.describe( and test( calls, tracking nesting.
    # We do this by regex+scan approach: iterate over the file and at each match
    # track current describe stack (via start/end of the describe's body).

    # Simpler: find test.describe(...) blocks first, then within each find test(...)
    describes = []  # (name, start_idx_of_body_lbrace, end_idx)
    i = 0
    while i < len(src):
        m = re.search(r'test\.describe\s*\(\s*', src[i:])
        if not m:
            break
        start = i + m.start()
        paren = i + m.end() - 1
        # paren points at '(' after describe
        # parse quoted string right after
        j = paren + 1
        # skip whitespace
        while j < len(src) and src[j].isspace(): j += 1
        if j >= len(src) or src[j] not in ('"', "'", '`'):
            i = start + len(m.group(0))
            continue
        try:
            name, j = parse_string_literal(src, j)
        except Exception:
            i = start + len(m.group(0))
            continue
        # find matching close paren
        endp = find_matching_paren(src, paren)
        if endp == -1:
            i = start + len(m.group(0))
            continue
        describes.append((name, paren, endp))
        i = paren + 1

    # Now extract tests; each test is `test(` or `test(<tag>,` not `test.describe`
    for dname, dstart, dend in describes:
        region = src[dstart:dend]
        # Iterate matches of test( inside region that are not test.describe
        k = 0
        while k < len(region):
            m = re.search(r'(?<![a-zA-Z0-9_.])test\s*\(', region[k:])
            if not m:
                break
            abs_paren = dstart + k + m.end() - 1
            # Skip if this is actually test.describe (shouldn't happen due to negative lookbehind)
            # Also skip test.skip, test.only etc
            # Skip if it's test.fixme / test.describe inside nested
            check_before = region[k + m.start(): k + m.end()]
            if 'describe' in check_before:
                k += m.end(); continue
            # parse name
            j = abs_paren + 1
            while j < len(src) and src[j].isspace(): j += 1
            if j >= len(src) or src[j] not in ('"', "'", '`'):
                k += m.end(); continue
            try:
                tname, j = parse_string_literal(src, j)
            except Exception:
                k += m.end(); continue
            endp = find_matching_paren(src, abs_paren)
            if endp == -1:
                k += m.end(); continue
            body = src[abs_paren:endp+1]
            tests.append({'name': tname, 'body': body, 'describe': dname})
            # advance past this test
            k = (endp + 1) - dstart
    return tests

# ---------------------------------------------------------------------------
# Extract from all files
# ---------------------------------------------------------------------------
upstream_tests = []
for path in sorted(TEST_ROOT.rglob('*.js')):
    if path.name in SKIP_FILES:
        continue
    if 'vendor' in path.parts or 'node_modules' in path.parts or 'manual' in path.parts:
        continue
    rel = path.relative_to(HS_ROOT)
    category = path.stem  # filename without .js
    # Special cases for organization
    tests = extract_tests_from_file(path)
    for t in tests:
        upstream_tests.append({
            'category': category,
            'name': t['name'],
            'body': t['body'],
            'describe': t['describe'],
            'upstream_file': str(rel),
        })

print(f"Extracted {len(upstream_tests)} upstream tests", file=sys.stderr)

# ---------------------------------------------------------------------------
# Identify post-snapshot additions
# ---------------------------------------------------------------------------
# A test is "post-snapshot" if it appears in a file modified in the window
# AND not in our JSON. We'll compute based on the in_our_json check instead.

# ---------------------------------------------------------------------------
# Classify each test
# ---------------------------------------------------------------------------
def classify_complexity(body, name, existing):
    if existing:
        return existing.get('complexity', 'simple')
    b = body
    if 'sinon.' in b:
        return 'sinon'
    if '<script type="text/hyperscript"' in b or "<script type='text/hyperscript'" in b:
        return 'script-tag'
    if 'showModal' in b or 'HTMLDialogElement' in b or '.close()' in b and '<dialog' in b.lower():
        return 'dialog'
    if 'new Promise' in b or 'async function' in b or ' await ' in b and ('setTimeout' in b or 'resolve' in b):
        # many tests use async functions trivially; require an actual Promise / delay
        if 'new Promise' in b or 'setTimeout' in b:
            return 'promise'
    if 'evaluate' in b and 'html' not in b and 'find' not in b:
        return 'eval-only'
    if '_hyperscript.evaluate' in b or 'evaluate(' in b:
        return 'evaluate'
    if 'runCmd' in b or "_hyperscript('" in b or '_hyperscript("' in b:
        return 'run-eval'
    return 'simple'

def classify_block_reason(body, name, category):
    """Return (block_reason, unlocks_hint) or (None, None)."""
    # sinon
    if 'sinon.' in body:
        return 'needs-sinon'
    # script tag
    if '<script type="text/hyperscript"' in body or "<script type='text/hyperscript'" in body:
        return 'needs-script-tag'
    if '<script type="text/hypertemplate"' in body or "<script type='text/hypertemplate'" in body:
        return 'needs-script-tag'
    # dialog
    if 'showModal' in body or '<dialog' in body.lower():
        return 'needs-dialog-api'
    # worker
    if 'Worker(' in body or 'worker.js' in body:
        return 'needs-worker'
    # eventsource
    if 'EventSource' in body or 'eventsource' in body.lower():
        return 'needs-eventsource'
    # Server-sent events
    if 'WebSocket' in body or category == 'socket':
        return 'needs-websocket'
    # mutation observers / focus/blur bubbling / transition events
    if 'MutationObserver' in body or 'mutationobserver' in body.lower():
        return 'needs-dom-mutation-observer'
    if 'transitionend' in body or 'transition-' in body or 'viewTransition' in body or category == 'transition' or category == 'viewTransition':
        return 'needs-css-transitions'
    if '.focus()' in body or '.blur()' in body or 'focusin' in body or 'focusout' in body:
        if category == 'focus':
            return 'needs-dom-focus'
    if 'resize' in body and 'ResizeObserver' in body:
        return 'needs-resize-observer'
    if 'IntersectionObserver' in body:
        return 'needs-intersection-observer'
    # patterns in hs strings
    hs_strings = re.findall(r"_=(['\"])([^'\"]*)\1", body) + re.findall(r"['\"]on\s+[^'\"]+['\"]", body)
    # Grab any quoted hs fragment
    all_quotes = re.findall(r"['\"`]([^'\"`]{3,})['\"`]", body)
    combined = ' '.join(all_quotes)
    # positional expr `<sel/>`
    if re.search(r'<[^<>]*\/>', combined) and not '/>' in body.split('expect')[0][:1000]:
        # detect <sel/>
        if re.search(r"<[a-zA-Z#.][^<>]*\/>", combined):
            return 'needs-pattern:<sel/>'
    # template interpolation ${}
    if '${' in combined:
        return 'needs-pattern:${}'
    # {css}
    if re.search(r'\{[^{}]*[:\s][^{}]+\}', combined):
        # css-like block literal inside HS string
        if re.search(r"'\s*\{", combined) or re.search(r'"\s*\{', combined):
            pass  # too noisy
    if re.search(r'\[@[\w-]+', combined):
        return 'needs-pattern:[@attr]'
    # DOM required
    if 'html(' in body and 'find(' in body:
        return 'translation-TBD'
    return 'translation-TBD'

# Files touched in post-snapshot window
POST_SNAPSHOT_FILES = set()
import subprocess
out = subprocess.check_output(
    ['git', 'log', '--after=2026-04-09', '--before=2026-04-14',
     '--name-only', '--pretty=format:', '--', 'test/'],
    cwd=str(HS_ROOT)
).decode()
for line in out.splitlines():
    line = line.strip()
    if line.startswith('test/') and line.endswith('.js'):
        POST_SNAPSHOT_FILES.add(line)
print(f"Post-snapshot touched files: {len(POST_SNAPSHOT_FILES)}", file=sys.stderr)

OUR_CATEGORIES = set(t['category'] for t in our_tests)

# Build manifest
manifest = []
for t in upstream_tests:
    key = (t['category'], norm(t['name']))
    existing = our_map.get(key)
    in_our_json = existing is not None
    complexity = classify_complexity(t['body'], t['name'], existing)
    # Status
    if not in_our_json:
        status = 'missing'
    elif t['name'] in SKIP_NAMES:
        status = 'skip-listed'
    else:
        status = 'runnable'
    # Block reason
    if status == 'runnable':
        block_reason = None
    else:
        # First check for concrete infra gaps (sinon, worker, patterns, etc.)
        br = classify_block_reason(t['body'], t['name'], t['category'])
        if br != 'translation-TBD':
            block_reason = br
        elif status == 'missing':
            # No concrete infra gap — categorise by why it's missing
            if t['category'] not in OUR_CATEGORIES:
                block_reason = 'unscraped-category'
            elif t['upstream_file'] in POST_SNAPSHOT_FILES:
                block_reason = 'added-post-snapshot'
            else:
                block_reason = 'unscraped-in-known-category'
        else:
            block_reason = br
    manifest.append({
        'category': t['category'],
        'name': t['name'],
        'complexity': complexity,
        'status': status,
        'block_reason': block_reason,
        'upstream_file': t['upstream_file'],
        'in_our_json': in_our_json,
    })

# ---------------------------------------------------------------------------
# De-duplicate (a test could be collected twice if nested describes overlap)
# ---------------------------------------------------------------------------
seen = set()
dedup = []
for m in manifest:
    k = (m['upstream_file'], m['category'], m['name'])
    if k in seen:
        continue
    seen.add(k)
    dedup.append(m)
manifest = dedup
print(f"Final manifest: {len(manifest)} tests", file=sys.stderr)

# ---------------------------------------------------------------------------
# Write JSON
# ---------------------------------------------------------------------------
with OUT_JSON.open('w') as f:
    json.dump(manifest, f, indent=2, ensure_ascii=False)
    f.write('\n')
print(f"Wrote {OUT_JSON}", file=sys.stderr)

# ---------------------------------------------------------------------------
# Stats + markdown
# ---------------------------------------------------------------------------
total = len(manifest)
runnable = sum(1 for m in manifest if m['status'] == 'runnable')
skip_listed = sum(1 for m in manifest if m['status'] == 'skip-listed')
missing = sum(1 for m in manifest if m['status'] == 'missing')

cat_stats = defaultdict(lambda: {'total': 0, 'runnable': 0, 'skip-listed': 0, 'missing': 0})
for m in manifest:
    cat_stats[m['category']]['total'] += 1
    cat_stats[m['category']][m['status']] += 1

# Unlock tables
missing_by_reason = Counter()
example_by_reason = {}
for m in manifest:
    if m['status'] == 'missing':
        r = m['block_reason'] or 'translation-TBD'
        missing_by_reason[r] += 1
        if r not in example_by_reason:
            example_by_reason[r] = f"{m['category']} / {m['name']}"

skip_by_reason = Counter()
skip_example = {}
for m in manifest:
    if m['status'] == 'skip-listed':
        r = m['block_reason'] or 'translation-TBD'
        skip_by_reason[r] += 1
        if r not in skip_example:
            skip_example[r] = f"{m['category']} / {m['name']}"

EFFORT = {
    'needs-sinon': 'medium — build fetch-mock shim keyed by URL/response',
    'needs-script-tag': 'medium — emit <script type="text/hyperscript"> wrapper in generator',
    'needs-dialog-api': 'low — stub showModal/close on HTMLDialogElement in fixture DOM',
    'needs-worker': 'high — Web Worker host adapter',
    'needs-eventsource': 'high — EventSource mock + streaming',
    'needs-websocket': 'high — WebSocket mock server',
    'needs-dom-mutation-observer': 'medium — hook MutationObserver into event queue',
    'needs-css-transitions': 'medium — transitionend event dispatch in fixtures',
    'needs-dom-focus': 'low — focus/blur dispatch in fixture browser',
    'needs-resize-observer': 'medium — ResizeObserver mock',
    'needs-intersection-observer': 'medium — IntersectionObserver mock',
    'needs-pattern:<sel/>': 'low — parser rule for <sel/> positional expr',
    'needs-pattern:${}': 'low — template string interpolation in HS parser',
    'needs-pattern:[@attr]': 'low — attribute-ref parser rule',
    'needs-dom': 'medium — fixture DOM extensions',
    'added-post-snapshot': 'low — re-scrape upstream, bump JSON snapshot',
    'unscraped-category': 'low — extend scraper to cover these upstream files',
    'unscraped-in-known-category': 'low — re-scrape; file was walked but these cases missed',
    'translation-TBD': 'unknown — needs case-by-case generator work',
}

md = []
md.append("# _hyperscript v0.9.90 upstream test coverage manifest")
md.append("")
md.append(f"- Upstream tag: `v0.9.90` (commit `a13de2ca`, 2026-04-13)")
md.append(f"- Our snapshot JSON: `spec/tests/hyperscript-upstream-tests.json` (831 tests, scraped 2026-04-09)")
md.append(f"- Total upstream tests: **{total}**")
md.append(f"- Runnable (present + not skip-listed): **{runnable}** ({runnable*100/total:.1f}%)")
md.append(f"- Skip-listed (present but guarded in generator): **{skip_listed}** ({skip_listed*100/total:.1f}%)")
md.append(f"- Missing from our snapshot: **{missing}** ({missing*100/total:.1f}%)")
md.append("")
md.append(f"Current conformance: runnable / total = **{runnable}/{total} = {runnable*100/total:.1f}%**.")
md.append("")
md.append("## Per-category")
md.append("")
md.append("| Category | Upstream | Runnable | Skip-listed | Missing |")
md.append("|---|---:|---:|---:|---:|")
for cat in sorted(cat_stats.keys()):
    s = cat_stats[cat]
    md.append(f"| {cat} | {s['total']} | {s['runnable']} | {s['skip-listed']} | {s['missing']} |")
md.append(f"| **TOTAL** | **{total}** | **{runnable}** | **{skip_listed}** | **{missing}** |")
md.append("")
md.append("## What unlocks how many — MISSING tests by block_reason")
md.append("")
md.append("| Block reason | Missing | Example | Est. effort |")
md.append("|---|---:|---|---|")
for r, n in missing_by_reason.most_common():
    ex = example_by_reason.get(r, '')
    if len(ex) > 70: ex = ex[:67] + '...'
    md.append(f"| `{r}` | {n} | {ex} | {EFFORT.get(r, 'unknown')} |")
md.append("")
md.append("## Skip-listed tests by block_reason")
md.append("")
md.append("| Block reason | Skipped | Example | Est. effort |")
md.append("|---|---:|---|---|")
for r, n in skip_by_reason.most_common():
    ex = skip_example.get(r, '')
    if len(ex) > 70: ex = ex[:67] + '...'
    md.append(f"| `{r}` | {n} | {ex} | {EFFORT.get(r, 'unknown')} |")
md.append("")
md.append("## How this was built")
md.append("")
md.append("1. `git clone --depth 1 --branch v0.9.90 https://github.com/bigskysoftware/_hyperscript /tmp/hs-upstream`; `git fetch --unshallow` for dated history.")
md.append("2. Walked `test/` (excluding `vendor/`, `manual/`, `fixtures.js`, `global-*.js`, `entry.js`, `htmx-fixtures.js`, `playwright.config.js`).")
md.append("3. For each `.js` file, a small Python parser finds every `test.describe(...)` block, then every `test(...)` within it — balanced-paren scan that ignores strings, regex literals, line/block comments.")
md.append("4. Category = filename stem (e.g. `add.js` → `add`). Test name = the first string literal argument of `test(...)`.")
md.append("5. Matched each upstream test against `spec/tests/hyperscript-upstream-tests.json` using `(category, name-normalized)` keys (whitespace-collapsed lowercase). Copied `complexity` when found; inferred it otherwise from body content (sinon./script-tag/dialog/Promise/evaluate).")
md.append("6. `status = runnable` if present and name not in generator's `SKIP_TEST_NAMES`; `skip-listed` if present and in that set; `missing` otherwise.")
md.append("7. `block_reason` classified from body content — sinon./script tag/dialog/worker/eventsource/WebSocket/MutationObserver/transition/focus/ResizeObserver patterns or `<sel/>`/`${}`/`[@attr]` HS syntax. Missing tests in files touched between 2026-04-09 and 2026-04-14 (`git log --after --before -- test/`) are tagged `added-post-snapshot`.")
md.append("8. Regenerate: `python3 /tmp/build_manifest.py`.")
md.append("")
md.append("## Untranslated caveat")
md.append("")
md.append("The markdown reports `runnable = present + not skip-listed`. Empirical baseline is 645 pass / 109 fail / 77 skip on 831 present tests. The ~109 failures are in-scope but reveal implementation gaps; they are not statically identifiable from upstream source without running the generator. This manifest therefore does not surface an `untranslated` bucket — treat the 109 empirical fails as the lower bound of that bucket inside the `runnable` count.")
md.append("")

OUT_MD.write_text('\n'.join(md))
print(f"Wrote {OUT_MD}", file=sys.stderr)

# ---------------------------------------------------------------------------
# Print summary
# ---------------------------------------------------------------------------
print()
print(f"TOTAL upstream: {total}")
print(f"  runnable:    {runnable}")
print(f"  skip-listed: {skip_listed}")
print(f"  missing:     {missing}")
print(f"  conformance: {runnable*100/total:.1f}%")
print()
print("Top missing by block_reason:")
for r, n in missing_by_reason.most_common(8):
    print(f"  {r:35s} {n:4d}  ({example_by_reason.get(r,'')[:50]})")
print()
print("Skip-listed by block_reason:")
for r, n in skip_by_reason.most_common(8):
    print(f"  {r:35s} {n:4d}")