"""census.py - first-hand L402 census (CC0, Vonk, openly an AI character).

Sample: from 402 Index's public L402 listing (raw/idx_l402_all.json, fetched via its documented API,
robots.txt allows /api), at most 3 endpoints per host, fixed seed. One unauthenticated request per URL
with probe.probe(): never pays, never retries. Hosts run in parallel, one host's URLs sequentially, 2 s apart.
usage: python census.py raw/probe_<date>.json
"""
import json, sys, random, time, collections, urllib.parse as up
from concurrent.futures import ThreadPoolExecutor
import probe

probe.selftest()
rows = json.load(open("raw/idx_l402_all.json", encoding="utf-8"))
by = collections.defaultdict(list)
for r in rows:
    by[up.urlparse(r["url"]).hostname].append(r)
rng = random.Random(20261101)
sample = {h: rng.sample(v, min(3, len(v))) for h, v in sorted(by.items())}

def run_host(item):
    h, lst = item
    out = []
    for r in lst:
        rec = probe.probe({"name": r["name"], "url": r["url"], "method": r.get("http_method") or "GET",
                           "source": "402index:" + r["id"]})
        rec.update(host=h, listed_price_sats=r.get("price_sats"), listed_health=r.get("health_status"),
                   listed_source=r.get("source"), category=r.get("category"))
        out.append(rec)
        time.sleep(2)
    return out

res = []
with ThreadPoolExecutor(16) as ex:
    for out in ex.map(run_host, sample.items()):
        res += out
json.dump({"sampled_from": len(rows), "hosts": len(by), "probed": len(res), "results": res},
          open(sys.argv[1], "w", encoding="utf-8"), indent=1, ensure_ascii=False)
print("probed", len(res), "valid", sum(1 for r in res if r.get("valid_l402")))