"""
How stable are AI product recommendations, and why do they change?

Compares answers to the same question on the same engine across runs:
  - the daily runs (17, 18, 19 November 2026), asked at different times of day;
  - two extra runs (replica A and B) asked at the same moment on 19 November, a good 16 hours
    after that day's daily run. A vs B is the "same question, same minute" baseline: whatever
    differs between them is randomness, not time.

Brands are matched with the same lists and rules as analyze.py (brands.json). All intervals are
95% bootstrap intervals that resample questions (the 210 questions are the unit, not the answers),
so answers to the same question never count as independent evidence.

Usage: python3 stability.py <answers.json> [out.json]
  answers.json: rows of {id, project, topic, prompt, engine, day, status, text, cites, created}.
"""
import json
import os
import random
import re
import sys
from collections import defaultdict
from statistics import mean
from urllib.parse import urlsplit

here = os.path.dirname(os.path.abspath(__file__))
rows = json.load(open(sys.argv[1]))
out_path = sys.argv[2] if len(sys.argv) > 2 else None
random.seed(7)

# ---------------------------------------------------------------- brand matching (as in analyze.py)
conf = json.load(open(os.path.join(here, "brands.json")))
always_cs = set(conf["_caseSensitive"])
words = set()
if os.path.exists("/usr/share/dict/words"):
    words = {w.strip().lower() for w in open("/usr/share/dict/words")}


def case_sensitive(brand, alias):
    if brand in always_cs:
        return True
    tokens = re.findall(r"[A-Za-z]+", alias)
    return bool(tokens) and all(t.lower() in words for t in tokens)


matchers = {}
for topic, brands in conf["categories"].items():
    matchers[topic] = [
        (brand, re.compile(r"(?<![\w.-])" + alias + r"(?![\w-])", 0 if case_sensitive(brand, alias) else re.IGNORECASE))
        for brand, aliases in brands.items()
        for alias in aliases
    ]


def clean(text):
    text = text.replace("\\", "")
    text = re.sub(r"!\[[^\]]*\]\([^)]*\)|!\[[^\]]*\]", " ", text)
    text = re.sub(r"\]\([^)]*\)", "]", text)
    return re.sub(r"https?://\S+", " ", text)


def find_brands(topic, text):
    first = {}
    for brand, rx in matchers[topic]:
        m = rx.search(text)
        if m and (brand not in first or m.start() < first[brand]):
            first[brand] = m.start()
    return [b for b, _ in sorted(first.items(), key=lambda x: x[1])]


def norm_url(u):
    s = urlsplit(u)
    host = s.netloc.lower().removeprefix("www.")
    return host + s.path.rstrip("/").lower()


# ---------------------------------------------------------------- runs
DAILY = {7: "daily", 8: "daily", 9: "daily"}
REPLICA = {22: "A", 23: "A", 24: "A", 25: "B", 26: "B", 27: "B"}

answers = {}  # (run, topic, prompt, engine) -> answer
for r in rows:
    if r["status"] != "ok" or r["topic"] not in matchers:
        continue
    p = int(r["project"])
    run = r["day"] if p in DAILY else REPLICA.get(p)
    if not run:
        continue
    cites = r["cites"] if isinstance(r["cites"], list) else json.loads(r["cites"] or "[]")
    answers[(run, r["topic"], r["prompt"], r["engine"])] = {
        "brands": find_brands(r["topic"], clean(r["text"] or "")),
        "urls": {norm_url(c["url"]) for c in cites if c.get("url")},
        "domains": {(c.get("domain") or "").lower().removeprefix("www.") for c in cites if c.get("domain")},
        "created": r.get("created"),
    }

broad = {(t, q) for (run, t, q, e) in answers if run in ("2026-11-17", "2026-11-18", "2026-11-19")}
constrained = {(t, q) for (run, t, q, e) in answers if run in ("A", "B")} - broad
ENGINES = ["chatgpt", "gemini", "google_aio", "google_ai_mode"]


# ---------------------------------------------------------------- measures
def jaccard(a, b):
    a, b = set(a), set(b)
    return len(a & b) / len(a | b) if a | b else None


def rbo(s, t, p=0.9):
    """Rank-biased overlap, extrapolated (Webber, Moffat & Zobel 2010). 1 = same ranking."""
    if not s and not t:
        return None
    if not s or not t:
        return 0.0
    if len(s) > len(t):
        s, t = t, s
    sl, ll = len(s), len(t)
    x = [0] * (ll + 1)
    seen_s, seen_t = set(), set()
    for d in range(1, ll + 1):
        if d <= sl:
            seen_s.add(s[d - 1])
        seen_t.add(t[d - 1])
        x[d] = len(seen_s & seen_t) if d <= sl else len(set(s) & seen_t)
    xs = x[sl]
    xl = x[ll]
    total = sum(x[d] / d * p**d for d in range(1, ll + 1))
    total += sum(xs * (d - sl) / (sl * d) * p**d for d in range(sl + 1, ll + 1))
    return (1 - p) / p * total + ((xl - xs) / ll + xs / sl) * p**ll


def compare(a, b):
    ba, bb = a["brands"], b["brands"]
    if not ba and not bb:
        return None
    out = {
        "jac": jaccard(ba, bb),
        "rbo": rbo(ba, bb),
        "same_set": set(ba) == set(bb),
        "lost": (len(set(ba) - set(bb)) / len(ba)) if ba else None,
        "top_kept": (ba[0] in bb) if ba else None,
        "top_same": (ba[0] == bb[0]) if ba and bb else None,
        "n": (len(ba) + len(bb)) / 2,
        "url_jac": jaccard(a["urls"], b["urls"]),
        "dom_jac": jaccard(a["domains"], b["domains"]),
    }
    # Kept by rank: was the brand at rank r in a still named in b?
    out["kept_by_rank"] = [(i + 1, x in set(bb)) for i, x in enumerate(ba)]
    return out


def pairs(run_a, run_b, questions, engine=None):
    """One comparison per question × engine where both runs answered. Keyed by question for clustering."""
    by_q = defaultdict(list)
    for t, q in questions:
        for e in ENGINES if engine is None else [engine]:
            a, b = answers.get((run_a, t, q, e)), answers.get((run_b, t, q, e))
            if a and b:
                c = compare(a, b)
                if c:
                    c["engine"] = e
                    c["topic"] = t
                    by_q[(t, q)].append(c)
    return by_q


def stat(by_q, key, fn=None, B=2000):
    """Mean of key over all pairs, with a 95% interval from resampling questions."""
    vals = {q: [fn(c) if fn else c[key] for c in cs] for q, cs in by_q.items()}
    vals = {q: [v for v in vs if v is not None] for q, vs in vals.items()}
    vals = {q: vs for q, vs in vals.items() if vs}
    qs = list(vals)
    if not qs:
        return None

    def m(sample):
        xs = [v for q in sample for v in vals[q]]
        return sum(xs) / len(xs)

    est = m(qs)
    boots = sorted(m([random.choice(qs) for _ in qs]) for _ in range(B))
    n = sum(len(v) for v in vals.values())
    return {"mean": round(est, 4), "lo": round(boots[int(0.025 * B)], 4), "hi": round(boots[int(0.975 * B)], 4), "pairs": n, "questions": len(qs)}


def diff(by_q1, by_q2, key, B=4000):
    """Paired difference of per-question means (set 1 minus set 2), 95% interval and two-sided bootstrap p."""
    q1 = {q: [c[key] for c in cs if c[key] is not None] for q, cs in by_q1.items()}
    q2 = {q: [c[key] for c in cs if c[key] is not None] for q, cs in by_q2.items()}
    qs = [q for q in q1 if q in q2 and q1[q] and q2[q]]
    d = {q: mean(q1[q]) - mean(q2[q]) for q in qs}
    est = mean(d.values())
    boots = sorted(mean(d[random.choice(qs)] for _ in qs) for _ in range(B))
    # p: how often a sign-flipped (null) resample is as extreme as what we saw.
    centred = [x - est for x in boots]
    p = sum(1 for x in centred if abs(x) >= abs(est)) / B
    return {"diff": round(est, 4), "lo": round(boots[int(0.025 * B)], 4), "hi": round(boots[int(0.975 * B)], 4), "p": round(max(p, 1 / B), 4), "questions": len(qs)}


def summary(by_q):
    return {
        "brand_overlap": stat(by_q, "jac"),
        "rbo": stat(by_q, "rbo"),
        "unchanged": stat(by_q, "same_set", lambda c: 1.0 if c["same_set"] else 0.0),
        "brands_lost": stat(by_q, "lost"),
        "top_still_named": stat(by_q, "top_kept", lambda c: None if c["top_kept"] is None else float(c["top_kept"])),
        "same_top": stat(by_q, "top_same", lambda c: None if c["top_same"] is None else float(c["top_same"])),
        "page_overlap": stat(by_q, "url_jac"),
        "site_overlap": stat(by_q, "dom_jac"),
        "brands_per_answer": stat(by_q, "n"),
    }


def gap_hours(run_a, run_b):
    ts = []
    for (r, t, q, e), a in answers.items():
        if r in (run_a, run_b) and a["created"]:
            ts.append((r, a["created"]))
    from datetime import datetime

    def med(r):
        xs = sorted(datetime.fromisoformat(c) for rr, c in ts if rr == r)
        return xs[len(xs) // 2] if xs else None

    a, b = med(run_a), med(run_b)
    return round(abs((b - a).total_seconds()) / 3600, 1) if a and b else None


res = {"answers": {}, "comparisons": {}, "engines": {}, "mechanism": {}, "rank": {}, "size": {}, "tests": {}}
for run in ["2026-11-17", "2026-11-18", "2026-11-19", "A", "B"]:
    res["answers"][run] = sum(1 for k in answers if k[0] == run)

COMPARISONS = [
    ("same moment (A vs B)", "A", "B"),
    ("day 1 vs day 2", "2026-11-17", "2026-11-18"),
    ("day 3 vs replica A", "2026-11-19", "A"),
    ("day 2 vs day 3", "2026-11-18", "2026-11-19"),
    ("day 1 vs day 3", "2026-11-17", "2026-11-19"),
]
P = {}
for name, a, b in COMPARISONS:
    P[name] = pairs(a, b, broad)
    if not P[name]:
        continue
    res["comparisons"][name] = {"hours_apart": gap_hours(a, b), **summary(P[name])}

# Drift: if answers drifted over time, two days apart would differ more than one day apart.
if P.get("day 1 vs day 3") and P.get("day 1 vs day 2"):
    res["tests"]["drift: (d1,d3) minus (d1,d2) brand overlap"] = diff(P["day 1 vs day 3"], P["day 1 vs day 2"], "jac")
    res["tests"]["drift: (d1,d3) minus (d2,d3) brand overlap"] = diff(P["day 1 vs day 3"], P["day 2 vs day 3"], "jac")
# Time vs randomness: same-moment pairs against a day apart.
if P.get("same moment (A vs B)"):
    for name in ["day 1 vs day 2", "day 2 vs day 3", "day 3 vs replica A", "day 1 vs day 3"]:
        if P.get(name):
            res["tests"][f"same moment minus {name}: brand overlap"] = diff(P["same moment (A vs B)"], P[name], "jac")
            res["tests"][f"same moment minus {name}: rbo"] = diff(P["same moment (A vs B)"], P[name], "rbo")

# Per engine, for the same-moment baseline and the one-day change.
for e in ENGINES:
    res["engines"][e] = {}
    for name, a, b in COMPARISONS:
        bq = pairs(a, b, broad, e)
        if bq:
            res["engines"][e][name] = {k: v for k, v in summary(bq).items() if k in ("brand_overlap", "unchanged", "same_top", "page_overlap")}

# Per engine: does time add change on top of same-moment randomness?
for e in ENGINES:
    same = pairs("A", "B", broad, e)
    for name, a, b in COMPARISONS[1:]:
        other = pairs(a, b, broad, e)
        if same and other:
            res["tests"][f"{e}: same moment minus {name}: brand overlap"] = diff(same, other, "jac")

# Mechanism: how much does brand overlap depend on page overlap? Pool every pair of runs.
pool = defaultdict(list)
for name, _, _ in COMPARISONS:
    for q, cs in P.get(name, {}).items():
        for c in cs:
            if c["url_jac"] is not None:
                pool[name].append(c)
buckets = [("no pages in common", lambda x: x == 0), ("under a third", lambda x: 0 < x < 1 / 3), ("a third to two thirds", lambda x: 1 / 3 <= x < 2 / 3), ("two thirds or more, not all", lambda x: 2 / 3 <= x < 1), ("exactly the same pages", lambda x: x == 1)]
allpairs = [c for cs in pool.values() for c in cs]
for label, f in buckets:
    cs = [c for c in allpairs if c["url_jac"] is not None and f(c["url_jac"])]
    if cs:
        res["mechanism"][label] = {"pairs": len(cs), "brand_overlap": round(mean(c["jac"] for c in cs), 3), "unchanged": round(mean(1.0 if c["same_set"] else 0.0 for c in cs), 3)}
xs = [(c["url_jac"], c["jac"]) for c in allpairs if c["url_jac"] is not None and c["jac"] is not None]
if len(xs) > 2:
    mx, my = mean(x for x, _ in xs), mean(y for _, y in xs)
    sxy = sum((x - mx) * (y - my) for x, y in xs)
    sxx = sum((x - mx) ** 2 for x, _ in xs)
    syy = sum((y - my) ** 2 for _, y in xs)
    res["mechanism"]["correlation page vs brand overlap"] = round(sxy / (sxx * syy) ** 0.5, 3)
    res["mechanism"]["pairs"] = len(xs)

# The same, within one question × engine: do the runs of a question that read more of the same pages
# also name more of the same brands? Each pair is compared with that question's own average, so a
# question that is simply stable can't make the link look stronger than it is.
cell = defaultdict(list)
for name, _, _ in COMPARISONS:
    for q, cs in P.get(name, {}).items():
        for c in cs:
            if c["url_jac"] is not None and c["jac"] is not None:
                cell[(q, c["engine"])].append((c["url_jac"], c["jac"]))
dx = []
for v in cell.values():
    if len(v) < 3:
        continue
    mx, my = mean(x for x, _ in v), mean(y for _, y in v)
    dx += [(x - mx, y - my) for x, y in v]
if dx:
    sxy = sum(x * y for x, y in dx)
    sxx = sum(x * x for x, _ in dx)
    syy = sum(y * y for _, y in dx)
    res["mechanism"]["within question: correlation"] = round(sxy / (sxx * syy) ** 0.5, 3)
    res["mechanism"]["within question: slope (brand overlap per unit page overlap)"] = round(sxy / sxx, 3)
    res["mechanism"]["within question: pairs"] = len(dx)
    # Pairs of the same question that read the same pages vs none of the same pages.
    same_pages, no_pages = [], []
    for v in cell.values():
        hi = [y for x, y in v if x == 1]
        lo = [y for x, y in v if x == 0]
        if hi and lo:
            same_pages.append(mean(hi))
            no_pages.append(mean(lo))
    if same_pages:
        res["mechanism"]["within question: same pages vs no pages in common"] = {"cells": len(same_pages), "same_pages": round(mean(same_pages), 3), "no_pages": round(mean(no_pages), 3)}

# Rank: probability a brand at rank r is still named in the other run.
for name in P:
    tally = defaultdict(lambda: [0, 0])
    for cs in P[name].values():
        for c in cs:
            for r, kept in c["kept_by_rank"]:
                k = str(r) if r <= 5 else "6+"
                tally[k][0] += kept
                tally[k][1] += 1
    res["rank"][name] = {k: round(v[0] / v[1], 3) for k, v in sorted(tally.items())}

# Size: does a long list look less stable only because it's long?
for name in P:
    sz = defaultdict(list)
    for cs in P[name].values():
        for c in cs:
            k = "1-3" if c["n"] <= 3 else "4-6" if c["n"] <= 6 else "7-10" if c["n"] <= 10 else "11+"
            sz[k].append(c)
    res["size"][name] = {k: {"pairs": len(v), "brand_overlap": round(mean(c["jac"] for c in v), 3), "brands_lost": round(mean(c["lost"] for c in v if c["lost"] is not None), 3)} for k, v in sorted(sz.items())}

# Broad vs constrained questions, both asked in A and B at the same moment.
if constrained:
    PC = pairs("A", "B", constrained)
    PB = P.get("same moment (A vs B)")
    if PC and PB:
        res["constrained"] = {"broad": summary(PB), "constrained": summary(PC)}
        res["constrained"]["brand_overlap difference (constrained - broad)"] = (lambda a, b: {"diff": round(a["mean"] - b["mean"], 4)})(stat(PC, "jac"), stat(PB, "jac"))
        # Shorter lists can look less stable just for being short: compare answers of the same length.
        by_size = {}
        for label, lo, hi in [("1-3 brands", 0, 3), ("4-6 brands", 3.5, 6), ("7+ brands", 6.5, 99)]:
            row = {}
            for kind, src in [("broad", PB), ("constrained", PC)]:
                cs = [c for v in src.values() for c in v if lo <= c["n"] <= hi]
                if cs:
                    row[kind] = {"pairs": len(cs), "brand_overlap": round(mean(c["jac"] for c in cs), 3), "unchanged": round(mean(1.0 if c["same_set"] else 0.0 for c in cs), 3), "page_overlap": round(mean(c["url_jac"] for c in cs if c["url_jac"] is not None), 3)}
            by_size[label] = row
        res["constrained"]["by answer length"] = by_size
        # Per engine.
        res["constrained"]["per engine"] = {
            e: {
                "broad": round(mean(c["jac"] for v in PB.values() for c in v if c["engine"] == e), 3),
                "constrained": round(mean(c["jac"] for v in PC.values() for c in v if c["engine"] == e), 3),
                "broad_pages": round(mean(c["url_jac"] for v in PB.values() for c in v if c["engine"] == e and c["url_jac"] is not None), 3),
                "constrained_pages": round(mean(c["url_jac"] for v in PC.values() for c in v if c["engine"] == e and c["url_jac"] is not None), 3),
            }
            for e in ENGINES
        }

# How many answers to know your share? Samples per question × engine across all five runs.
runs = ["2026-11-17", "2026-11-18", "2026-11-19", "A", "B"]
agree = defaultdict(lambda: [0, 0])  # true-share bucket -> [single checks that match the majority, checks]
spread = []
for t, q in broad:
    for e in ENGINES:
        samples = [answers.get((r, t, q, e)) for r in runs]
        samples = [s for s in samples if s]
        if len(samples) < 4:
            continue
        seen = defaultdict(int)
        for s in samples:
            for b in set(s["brands"]):
                seen[b] += 1
        for b, k in seen.items():
            share = k / len(samples)
            bucket = "always" if share == 1 else "most" if share >= 0.6 else "some" if share > 0.2 else "once"
            agree[bucket][0] += k
            agree[bucket][1] += 1
        spread.append((len(seen), sum(1 for k in seen.values() if k == len(samples)), len(samples), mean(len(set(s["brands"])) for s in samples)))
if spread:
    res["samples"] = {
        "question_engine_cells": len(spread),
        "avg_samples": round(mean(s[2] for s in spread), 2),
        "avg_brands_per_answer": round(mean(s[3] for s in spread), 2),
        "avg_distinct_brands_across_samples": round(mean(s[0] for s in spread), 2),
        "avg_brands_in_every_sample": round(mean(s[1] for s in spread), 2),
        "brands_by_how_often": {k: v[1] for k, v in agree.items()},
    }

print(json.dumps(res, indent=1))
if out_path:
    json.dump(res, open(out_path, "w"), indent=1)
    # Every comparison, one row per question × engine × pair of runs, so each number can be rechecked.
    import csv

    with open(os.path.join(os.path.dirname(os.path.abspath(out_path)), "stability-pairs.csv"), "w", newline="") as f:
        w = csv.writer(f)
        w.writerow(["comparison", "category", "question", "question_type", "engine", "brands_run_1", "brands_run_2", "brand_overlap", "rank_biased_overlap", "same_brands", "page_overlap"])
        sets = [(name, a, b, broad, "broad") for name, a, b in COMPARISONS] + [("same moment (A vs B)", "A", "B", constrained, "constrained")]
        for name, ra, rb, qs, kind in sets:
            for t, q in sorted(qs):
                for e in ENGINES:
                    x, y = answers.get((ra, t, q, e)), answers.get((rb, t, q, e))
                    if not x or not y:
                        continue
                    c = compare(x, y)
                    if not c:
                        continue
                    w.writerow([name, t, q, kind, e, " | ".join(x["brands"]), " | ".join(y["brands"]), round(c["jac"], 4), round(c["rbo"], 4), int(c["same_set"]), "" if c["url_jac"] is None else round(c["url_jac"], 4)])
