import json,threading,queue,time,gzip,re,os,urllib.request
UA="tangled-stats-research/0.1 (permadeath.com)"
SD="/tmp/claude-1000/-home-jmm-Code-blogposts/7ba1f72e-807b-4749-bcd3-a065da793e6d/scratchpad"
def get(url,tries=3,timeout=90):
    for i in range(tries):
        try:
            r=urllib.request.Request(url,headers={"User-Agent":UA})
            with urllib.request.urlopen(r,timeout=timeout) as f: return f.read()
        except Exception:
            if i==tries-1: raise
            time.sleep(1.5*(i+1))
mp=json.load(open(f"{SD}/merged_pulls.json")); PM=json.load(open(f"{SD}/pds_map.json"))
seen=set()
if os.path.exists(f"{SD}/commits.jsonl"):
    for l in open(f"{SD}/commits.jsonl"):
        try: seen.add(json.loads(l)["uri"])
        except: pass
todo=[p for p in mp if (p["cid"] or p.get("inlinePatch")) and p["uri"] not in seen]
print("todo",len(todo),"of",len(mp),flush=True)
FROM=re.compile(rb"(?m)^From ([0-9a-f]{40}) ")
DATE=re.compile(rb"(?m)^Date: (.+?)\r?$")
AUTH=re.compile(rb"(?m)^From: (.+?)\r?$")
out=open(f"{SD}/commits.jsonl","a"); lock=threading.Lock(); c=[0,0]
q=queue.Queue(); [q.put(p) for p in todo]
def w():
    while True:
        try: p=q.get_nowait()
        except queue.Empty: return
        try:
            did=p["uri"].split("/")[2]; host=PM.get(did)
            inline=p.get("inlinePatch")
            if inline:
                txt=inline.encode("utf8","replace"); raw=txt
            elif host and p["cid"]:
                raw=get(f"{host}/xrpc/com.atproto.sync.getBlob?did={did}&cid={p['cid']}")
                try: txt=gzip.decompress(raw)
                except Exception: txt=raw
            else:
                txt=None
            if txt is not None:
                shas=[s.decode() for s in FROM.findall(txt)]
                n=len(shas)
                rec={"uri":p["uri"],"repoDid":p["repoDid"],"author":did,"commits":n,
                     "shas":shas,
                     "createdAt":p["createdAt"],"stateUpdatedAt":p["stateUpdatedAt"],
                     "rounds":p["rounds"],"patchBytes":len(raw),
                     "commitDates":[d.decode("utf8","replace") for d in DATE.findall(txt)][:n],
                     "commitAuthors":[a.decode("utf8","replace") for a in AUTH.findall(txt)][:n]}
                with lock:
                    out.write(json.dumps(rec)+"\n"); c[0]+=1; c[1]+=n
        except Exception as e:
            with lock: print("ERR",p["uri"][:70],str(e)[:60],flush=True)
        finally:
            with lock:
                if c[0] and c[0]%250==0: out.flush(); print(f"  {c[0]}/{len(todo)} commits={c[1]}",flush=True)
            q.task_done()
ts=[threading.Thread(target=w,daemon=True) for _ in range(10)]
[t.start() for t in ts];[t.join() for t in ts]
out.flush();out.close();print("DONE",c,flush=True)