"""summary.py - turns raw/probe_<date>.json into summary.json (CC0, Vonk). No network."""
import json, sys, statistics as st, collections as co

src = sys.argv[1] if len(sys.argv) > 1 else "raw/probe_2026-11-01.json"
d = json.load(open(src, encoding="utf-8")); R = d["results"]

def kind(r):
    s, c, body = r.get("status"), r.get("challenge") or {}, r.get("body_head") or ""
    if s is None: return "unreachable"
    if s != 402: return "no-paywall-%dxx" % (s // 100) if s < 500 else "server-error-5xx"
    if c.get("scheme") in ("L402", "LSAT"):
        if r.get("valid_l402"): return "l402-valid"
        return "l402-incomplete"
    if c.get("scheme") == "Payment": return "mpp-instead"
    if "x402Version" in body or "eip155" in body or "USDC" in body: return "x402-instead"
    return "402-other"

for r in R: r["kind"] = kind(r)
K = co.Counter(r["kind"] for r in R)
hosts = co.defaultdict(set)
for r in R: hosts[r["host"]].add(r["kind"])
host_valid = sorted(h for h, k in hosts.items() if "l402-valid" in k)
valid = [r for r in R if r["kind"] == "l402-valid"]
sats = sorted(r["challenge"]["amount_sats"] for r in valid if r["challenge"].get("amount_sats") is not None)
nets = co.Counter(r["challenge"].get("network") for r in valid)
match = sum(1 for r in valid if r.get("listed_price_sats") is not None and r["challenge"].get("amount_sats") == r["listed_price_sats"])
listed_cmp = sum(1 for r in valid if r.get("listed_price_sats") is not None and r["challenge"].get("amount_sats") is not None)
fivexx = co.Counter(r["host"] for r in R if r["kind"] == "server-error-5xx")
health_vs = co.Counter((r.get("listed_health"), r["kind"] == "l402-valid") for r in R)
out = {
    "source_file": src, "method": "402 Index L402 listing (975 rows, 148 hosts, documented public API) -> max 3 random endpoints per host (seed 20261101) -> one unauthenticated request each, never paid",
    "listed": d["sampled_from"], "hosts_listed": d["hosts"], "probed": len(R), "hosts_probed": len(hosts),
    "kinds": dict(K.most_common()),
    "valid_l402_endpoints": len(valid), "valid_l402_share": round(len(valid) / len(R), 3),
    "hosts_with_valid_l402": len(host_valid), "hosts_with_valid_l402_list": host_valid,
    "invoice_networks": dict(nets),
    "sats": {"n": len(sats), "min": sats[0] if sats else None, "median": st.median(sats) if sats else None,
             "max": sats[-1] if sats else None, "le_10": sum(1 for s in sats if s <= 10),
             "le_100": sum(1 for s in sats if s <= 100), "gt_1000": sum(1 for s in sats if s > 1000)},
    "invoice_equals_listed_price": f"{match}/{listed_cmp}",
    "top_5xx_hosts": fivexx.most_common(5),
    "listed_health_vs_valid": {f"{h}|{v}": n for (h, v), n in sorted(health_vs.items(), key=str)},
}
json.dump(out, open("summary.json", "w", encoding="utf-8"), indent=1)
print(json.dumps({k: v for k, v in out.items() if k != "hosts_with_valid_l402_list"}, indent=1))

# --- host view + endpoint-weighted estimate (added cycle 57) ---
rows = json.load(open("raw/idx_l402_all.json", encoding="utf-8"))
import urllib.parse as up
listed_per_host = co.Counter(up.urlparse(r["url"]).hostname for r in rows)
frac = {h: sum(1 for r in R if r["host"] == h and r["kind"] == "l402-valid") / sum(1 for r in R if r["host"] == h) for h in hosts}
est = sum(frac[h] * listed_per_host[h] for h in hosts)
tunnels = [h for h in hosts if h.endswith(".lhr.life") or "ngrok" in h or "trycloudflare" in h]
out["host_view"] = {"tunnel_hosts": len(tunnels), "tunnel_hosts_with_valid": sum(1 for h in tunnels if h in host_valid),
                    "endpoint_weighted_valid_estimate": round(est), "endpoint_weighted_valid_share": round(est / len(rows), 3),
                    "biggest_hosts": [(h, listed_per_host[h], round(frac[h], 2)) for h, _ in listed_per_host.most_common(10)]}
json.dump(out, open("summary.json", "w", encoding="utf-8"), indent=1)
print(json.dumps(out["host_view"], indent=1)); print(host_valid)

# --- targeted recheck: providers that validate the body before the paywall (added cycle 57) ---
rc = json.load(open("raw/sats4ai_recheck.json", encoding="utf-8"))
rc_ok = rc["status"] == 402 and rc["challenge"]["scheme"] == "L402" and rc["challenge"]["macaroon"]
out["rechecks"] = [{"host": "sats4ai.com", "why": "empty POST body got 406 before any paywall; one retry with the minimal body from its own README",
                    "url": rc["url"], "body": rc["body"], "status": rc["status"], "valid_l402": rc_ok}]
out["hosts_with_valid_l402_incl_rechecks"] = len(host_valid) + sum(1 for r in out["rechecks"] if r["valid_l402"] and r["host"] not in host_valid)
out["fourxx_on_empty_request"] = K.get("no-paywall-4xx", 0)
out["checked"] = R[0]["at"][:10]
out["demand_visible"] = False
out["demand_note"] = "Lightning payments leave no public per-payment record, so unlike x402 on Base nobody can count paid L402 calls from outside."
json.dump(out, open("summary.json", "w", encoding="utf-8"), indent=1)
print("hosts valid incl rechecks:", out["hosts_with_valid_l402_incl_rechecks"])