import json, threading, queue, time, urllib.parse, urllib.request, os
A="https://api.tangled.org/xrpc"; UA="tangled-stats-research/0.1 (permadeath.com)"
SD="/tmp/claude-1000/-home-jmm-Code-blogposts/7ba1f72e-807b-4749-bcd3-a065da793e6d/scratchpad"
def get(url,tries=4):
    for i in range(tries):
        try:
            r=urllib.request.Request(url,headers={"User-Agent":UA})
            with urllib.request.urlopen(r,timeout=60) as f: return json.load(f)
        except Exception:
            if i==tries-1: raise
            time.sleep(1.5*(i+1))
def par(items, fn, n=12):
    q=queue.Queue(); [q.put(i) for i in items]; lock=threading.Lock(); res=[]; c=[0]
    def w():
        while True:
            try: it=q.get_nowait()
            except queue.Empty: return
            try:
                v=fn(it)
                with lock:
                    if v is not None: res.append(v)
            except Exception as e:
                with lock: print("ERR",it,e,flush=True)
            finally:
                with lock:
                    c[0]+=1
                    if c[0]%500==0: print(f"  {c[0]}/{len(items)}",flush=True)
                q.task_done()
    ts=[threading.Thread(target=w,daemon=True) for _ in range(n)]
    [t.start() for t in ts]; [t.join() for t in ts]; return res

# 1. repos that actually have pulls -> owner records
active=[]; authors=set()
for l in open(f"{SD}/pulls.jsonl"):
    d=json.loads(l)
    if d["n"]:
        active.append(d["repoDid"])
        for p in d["pulls"]: authors.add(p["uri"].split("/")[2])
print("active repos",len(active),"pull authors",len(authors),flush=True)

recs=par(active, lambda did: get(f"{A}/sh.tangled.repo.getRepoByRepoDid?repoDid={did}"))
json.dump(recs, open(f"{SD}/repo_records.json","w"))
owners=set()
for r in recs:
    u=(r.get("uri") or "")
    if u.startswith("at://"): owners.add(u.split("/")[2])
print("owners",len(owners),flush=True)

# 2. every known actor -> their repo records (catches non-knot1 knots)
actors=sorted(authors|owners)
print("actors to expand",len(actors),flush=True)
def allrepos(did):
    out=[];cur=None
    while True:
        u=f"{A}/sh.tangled.repo.listRepos?subject={did}&limit=1000"+(f"&cursor={urllib.parse.quote(cur)}" if cur else "")
        d=get(u); out+=d.get("items",[]); cur=d.get("cursor")
        if not cur or not d.get("items"): break
    return {"did":did,"items":out}
ar=par(actors, allrepos)
json.dump(ar, open(f"{SD}/actor_repos.json","w"))
known=set(l.strip() for l in open(f"{SD}/repo_dids.txt"))
new=set()
for a in ar:
    for i in a["items"]:
        rd=(i.get("value") or {}).get("repoDid")
        if rd and rd not in known: new.add(rd)
print("NEW repo dids",len(new),flush=True)
open(f"{SD}/repo_dids_extra.txt","w").write("\n".join(sorted(new)))