"""Read-only census of nsite manifests (NIP-5A kinds 15128 root, 35128 named) on public relays.
Counts which card tags they carry (title, description, icon, x aggregate, server, source).
Usage: python census.py  -> census.json"""
import json, sys, time
from pathlib import Path
HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE.parents[1] / "tools"))
import _nostr_core as nc

RELAYS = ["wss://relay.damus.io", "wss://nos.lol", "wss://relay.primal.net", "wss://nostr.mom", "wss://relay.nsite.lol",
          "wss://relay.snort.social", "wss://offchain.pub"]
latest = {}
for kind in (15128, 35128):
    until = None
    for _ in range(12):  # page backwards; relays cap per query
        f = {"kinds": [kind], "limit": 500}
        if until: f["until"] = until
        try:
            res = nc.fetch(f, RELAYS)
        except Exception as e:
            print("fetch error", e); break
        evs = [e for lst in (res.values() if isinstance(res, dict) else [res]) for e in (lst or []) if isinstance(e, dict)]
        if not evs: break
        new = 0
        for e in evs:
            d = next((t[1] for t in e["tags"] if t[0] == "d"), "") if kind == 35128 else ""
            key = (e["pubkey"], kind, d)
            if key not in latest or latest[key]["created_at"] < e["created_at"]:
                if key not in latest: new += 1
                latest[key] = e
        oldest = min(e["created_at"] for e in evs)
        print(kind, "page", len(evs), "new", new, "oldest", time.strftime("%Y-%m-%d", time.gmtime(oldest)), flush=True)
        if until and oldest >= until: break
        until = oldest - 1
        if new == 0: break

TAGS = ["title", "description", "icon", "x", "server", "source"]
rows = []
for (pk, kind, d), e in latest.items():
    names = {t[0] for t in e["tags"]}
    paths = sum(1 for t in e["tags"] if t[0] == "path")
    rows.append({"pubkey": pk, "kind": kind, "d": d, "created_at": e["created_at"], "paths": paths,
                 **{t: (t in names) for t in TAGS},
                 "has_index": any(t[0] == "path" and t[1] == "/index.html" for t in e["tags"])})
n = len(rows)
summ = {"checked_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "relays": RELAYS, "manifests": n,
        "by_kind": {k: sum(r["kind"] == k for r in rows) for k in (15128, 35128)},
        "authors": len({r["pubkey"] for r in rows}),
        "with_tag": {t: sum(r[t] for r in rows) for t in TAGS},
        "with_index_html": sum(r["has_index"] for r in rows),
        "title_and_description": sum(r["title"] and r["description"] for r in rows),
        "none_of_title_desc_icon": sum(not (r["title"] or r["description"] or r["icon"]) for r in rows)}
summ["pct"] = {k: round(100 * v / n, 1) for k, v in summ["with_tag"].items()} if n else {}
json.dump({"summary": summ, "rows": rows}, open(HERE / "census.json", "w"), indent=1)
print(json.dumps(summ, indent=1))