"""
Are brands that appear on more of the pages AI cites recommended more consistently?

For each question on each engine, takes every page cited in any of the five runs (17, 18, 19
November and replicas A and B, 2026), fetches it, and counts the share of those pages that name
each brand. Pages on the brand's own site are left out for that brand. Pages that could not be
read (blocked, not HTML, under 1,500 characters) are left out. It then compares that share with how
many of the five runs named the brand.

Brands are matched exactly as in stability.py (brands.json).

Usage: python3 coverage.py <answers.json> <pages dir> [out.json]
  pages dir: one JSON file per cited URL, named md5(url without #fragment).json, holding {"url", "text"}.
  Writes out.json and page-coverage.csv next to it.
"""
import csv
import hashlib
import json
import os
import random
import re
import sys
from collections import defaultdict
from statistics import mean

here = os.path.dirname(os.path.abspath(__file__))
answers_path, pages_dir = sys.argv[1], sys.argv[2]
out_path = sys.argv[3] if len(sys.argv) > 3 else None

# Brand matching, shared with stability.py.
_src = open(os.path.join(here, "stability.py")).read().split("# ---------------------------------------------------------------- runs")[0]
_src = _src.replace("here = os.path.dirname(os.path.abspath(__file__))", "").replace("rows = json.load(open(sys.argv[1]))", "rows = json.load(open(answers_path))")
_src = _src.replace("out_path = sys.argv[2] if len(sys.argv) > 2 else None", "")
exec(_src)  # defines rows, matchers, find_brands, clean, norm_url
random.seed(7)

DAILY = {7, 8, 9}
REPLICA = {22: "A", 23: "A", 24: "A", 25: "B", 26: "B", 27: "B"}
BLOCKED = re.compile(r"(?i)just a moment|access denied|enable javascript and cookies|captcha")

_cache = {}


def page_text(url):
    url = url.split("#")[0]
    if url not in _cache:
        f = os.path.join(pages_dir, hashlib.md5(url.encode()).hexdigest() + ".json")
        t = json.load(open(f))["text"] if os.path.exists(f) else None
        _cache[url] = t if t and len(t) >= 1500 and not BLOCKED.search(t[:3000]) else None
    return _cache[url]


def names(topic, brand, text):
    return any(rx.search(text) for b, rx in matchers[topic] if b == brand)


def own_site(brand, url):
    s = re.sub(r"[^a-z0-9]", "", brand.lower())
    return len(s) >= 4 and s in re.sub(r"[^a-z0-9]", "", url.split("/")[0])


cells = defaultdict(dict)  # (topic, prompt, engine) -> run -> {brands, urls}
for r in rows:
    if r["status"] != "ok" or r["topic"] not in matchers:
        continue
    p = int(r["project"])
    run = r["day"] if p in DAILY else REPLICA.get(p)
    if not run:
        continue
    cites = r["cites"] if isinstance(r["cites"], list) else json.loads(r["cites"] or "[]")
    cells[(r["topic"], r["prompt"], r["engine"])][run] = {
        "brands": find_brands(r["topic"], clean(r["text"] or "")),
        "urls": {norm_url(c["url"]): c["url"] for c in cites if c.get("url")},
    }

out = []
seen = [0, 0]
for (topic, prompt, engine), runs in cells.items():
    if len(runs) < 3:
        continue
    urls = {}
    for v in runs.values():
        urls.update(v["urls"])
    texts = {}
    for n, raw in urls.items():
        seen[1] += 1
        t = page_text(raw)
        if t:
            texts[n] = t
            seen[0] += 1
    if len(texts) < 3:
        continue
    for brand in {b for b, _ in matchers[topic]}:
        pages = [n for n in texts if not own_site(brand, n)]
        if not pages:
            continue
        on = sum(1 for n in pages if names(topic, brand, texts[n]))
        named = sum(1 for v in runs.values() if brand in v["brands"])
        if on == 0 and named == 0:
            continue
        out.append({"topic": topic, "prompt": prompt, "engine": engine, "brand": brand, "runs": len(runs),
                    "named": named, "pages": len(pages), "pages_naming_brand": on})

for x in out:
    x["rate"] = x["named"] / x["runs"]
    x["cov"] = x["pages_naming_brand"] / x["pages"]

BUCKETS = [(0, 0.1, "under 10%"), (0.1, 0.25, "10-25%"), (0.25, 0.5, "25-50%"), (0.5, 0.75, "50-75%"), (0.75, 1.01, "75% or more")]
res = {"pages_cited": seen[1], "pages_read": seen[0], "brand_question_engine_rows": len(out), "by_share_of_pages": {}}
for lo, hi, name in BUCKETS:
    xs = [x for x in out if lo <= x["cov"] < hi and x["pages_naming_brand"] > 0]
    res["by_share_of_pages"][name] = {
        "rows": len(xs),
        "share_of_runs_named": round(mean(x["rate"] for x in xs), 3),
        "named_in_every_run": round(mean(x["named"] == x["runs"] for x in xs), 3),
        "never_named": round(mean(x["named"] == 0 for x in xs), 3),
    }
named = [x for x in out if x["named"] > 0]
res["named_brands_on_at_least_one_page"] = round(mean(x["pages_naming_brand"] > 0 for x in named), 3)
every = [x for x in out if x["named"] == x["runs"]]
res["every_run_brands_on_half_or_more"] = round(mean(x["cov"] >= 0.5 for x in every), 3)

# Same brand, different questions and engines: is it named more where it's on more of the pages?
by_brand = defaultdict(list)
for x in out:
    by_brand[(x["topic"], x["brand"])].append(x)
pairs = []
for xs in by_brand.values():
    if len(xs) < 2:
        continue
    m = sorted(x["cov"] for x in xs)[len(xs) // 2]
    hi = [x["rate"] for x in xs if x["cov"] > m]
    lo = [x["rate"] for x in xs if x["cov"] < m]
    if hi and lo:
        pairs.append((mean(hi), mean(lo)))
boot = sorted(mean(a - b for a, b in (random.choice(pairs) for _ in pairs)) for _ in range(2000))
res["same_brand"] = {
    "brands": len(pairs),
    "named_where_on_more_pages": round(mean(a for a, _ in pairs), 3),
    "named_where_on_fewer_pages": round(mean(b for _, b in pairs), 3),
    "difference_95ci": [round(boot[50], 3), round(boot[1949], 3)],
}
res["by_engine"] = {
    e: {name: round(mean(x["rate"] for x in out if x["engine"] == e and lo <= x["cov"] < hi and x["pages_naming_brand"] > 0), 3) for lo, hi, name in BUCKETS}
    for e in ["chatgpt", "gemini", "google_aio", "google_ai_mode"]
}
print(json.dumps(res, indent=1))
if out_path:
    json.dump(res, open(out_path, "w"), indent=1)
    with open(os.path.join(os.path.dirname(out_path), "page-coverage.csv"), "w", newline="") as f:
        w = csv.writer(f)
        w.writerow(["topic", "question", "engine", "brand", "runs", "runs_naming_brand", "pages_read", "pages_naming_brand"])
        for x in sorted(out, key=lambda x: (x["topic"], x["prompt"], x["engine"], x["brand"])):
            w.writerow([x["topic"], x["prompt"], x["engine"], x["brand"], x["runs"], x["named"], x["pages"], x["pages_naming_brand"]])
