Something went wrong. Try again.
Self-hosted file management and sharing system, supports multiple storage providers. forked from tdvorak.tngl.sh/cloudreve
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329#!/usr/bin/env python3"""Daily upstream watch across the cloudreve/* upstream repos.
Collects open issues and PRs from each upstream repository, marks the onesalready cited in our tree (inline `#NNNN` comments or commit-messagereferences), and reports the rest as untriaged. Also reports how far thevendored directories (frontend/, cli/, docs/) and the backend fork arebehind their upstream heads.
The report is written to a single pinned GitHub issue labelled`upstream-watch`; a comment is posted only when new untriaged items appear.
Requires: `gh` CLI authenticated (GH_TOKEN), run from the repo root with`upstream` remote fetched."""
import jsonimport osimport reimport subprocessimport sysfrom pathlib import Path
REPOS = [ "cloudreve/cloudreve", "cloudreve/frontend", "cloudreve/cli", "cloudreve/docs", "cloudreve/desktop",]
TRACKING_LABEL = "upstream-watch"TRACKING_TITLE = "Upstream watch"MIRROR_LABEL = "upstream-mirror"MIRROR_MARKER = re.compile(r"<!--\s*upstream-mirror:\s*(\S+)#(\d+)\s*-->")SELF_REPO = os.environ.get("GITHUB_REPOSITORY", "Dvorinka/cloudreve")PINS_FILE = Path(".github/upstream-pins.json")PENDING_MARKER = re.compile(r"<!--\s*pending:\s*(\{.*?\})\s*-->", re.S)NUM_RE = re.compile(r"#(\d{3,5})")PR_NUM_RE = re.compile(r"(?:PR|pr)\s*#(\d+)")
def gh(*args: str) -> str: proc = subprocess.run( ["gh", *args], check=False, capture_output=True, text=True ) if proc.returncode != 0: # Surface gh's stderr — CalledProcessError alone hid the API error # behind two consecutive silent CI failures. Callers catch # CalledProcessError, so print first, then raise the same type. detail = (proc.stderr or proc.stdout or "").strip() if detail: print(f"gh {' '.join(args[:2])}… failed: {detail}", file=sys.stderr) raise subprocess.CalledProcessError( proc.returncode, proc.args, output=proc.stdout, stderr=proc.stderr, ) return proc.stdout
def gh_json(*args: str): return json.loads(gh(*args))
def git(*args: str) -> str: return subprocess.run( ["git", *args], check=True, capture_output=True, text=True ).stdout
def cited_numbers() -> set[int]: """Numbers referenced anywhere in tracked sources or commit history.
The fork's convention is inline `(#NNNN)` / `upstream #NNNN` comments and `upstream PR#NN` in commit messages for ported fixes. """ text = git( "grep", "-hoE", "#[0-9]{3,5}", "--", "*.go", "*.ts", "*.tsx", "*.rs", "*.py", "*.md", ) text += git("log", "--format=%B", "-n", "3000") return {int(n) for n in NUM_RE.findall(text)} | { int(n) for n in PR_NUM_RE.findall(text) }
def open_items(repo: str) -> list[dict]: """Open issues and PRs; /issues returns both, PRs carry .pull_request.
Sorted by updated-desc so the freshest upstream activity surfaces first in the report instead of being buried under long-stale items. """ return gh_json( "api", "--paginate", f"repos/{repo}/issues?state=open&per_page=100&sort=updated&direction=desc", )
def vendored_drift() -> list[str]: if not PINS_FILE.exists(): return [] pins = json.loads(PINS_FILE.read_text()) lines = [] for dirname, pin in pins.items(): if not isinstance(pin, dict): continue repo, sha = pin["repo"], pin["sha"] try: cmp = gh_json("api", f"repos/{repo}/compare/{sha}...HEAD") except subprocess.CalledProcessError: lines.append(f"- `{dirname}/` ← `{repo}`: pin {sha[:8]} not reachable") continue count = cmp.get("total_commits", 0) subjects = [c["commit"]["message"].split("\n")[0] for c in cmp.get("commits", [])[:10]] lines.append(f"- `{dirname}/` ← `{repo}`: **{count}** new commit(s) since {sha[:8]}") lines += [f" - {s}" for s in subjects] return lines
def backend_drift() -> list[str]: try: subjects = git("log", "--format=%s", "HEAD..upstream/master").splitlines() except subprocess.CalledProcessError: return [] if not subjects: return ["- backend is up to date with `cloudreve/cloudreve`"] lines = [f"- backend is **{len(subjects)}** commit(s) behind `upstream/master`:"] lines += [f" - {s}" for s in subjects[:30]] return lines
def issue_link(repo: str, n: int, pr: bool) -> str: kind = "pull" if pr else "issues" return f"[#{n}](https://github.com/{repo}/{kind}/{n})"
def mirror_issues() -> dict[tuple[str, int], dict]: """Existing mirror issues keyed by (upstream_repo, upstream_number).
Closed mirrors are included so dedup never re-creates a triaged item. """ try: gh("label", "create", MIRROR_LABEL, "--repo", SELF_REPO, "--description", "Mirror of a single upstream issue/PR for triage", "--color", "fbca04") except subprocess.CalledProcessError: pass # label exists issues = gh_json( "issue", "list", "--repo", SELF_REPO, "--label", MIRROR_LABEL, "--state", "all", "--limit", "500", "--json", "number,body,state", ) out = {} for i in issues: m = MIRROR_MARKER.search(i.get("body") or "") if m: out[(m.group(1), int(m.group(2)))] = i return out
def sync_mirrors( per_repo: dict[str, list[dict]], cited: set[int]) -> tuple[int, int]: """Create one mirror per untriaged upstream item; close stale mirrors.
A mirror auto-closes when its upstream item is no longer open or the upstream number becomes cited in this repo's tree/history. """ existing = mirror_issues() open_upstreams = { (repo, i["number"]) for repo, items in per_repo.items() for i in items } created = closed = 0 for repo, items in per_repo.items(): for i in items: key = (repo, i["number"]) if i["number"] in cited or key in existing: continue is_pr = "pull_request" in i kind = "pull" if is_pr else "issues" body = ( f"Upstream {'PR' if is_pr else 'issue'}: " f"https://github.com/{repo}/{kind}/{i['number']}\n\n" f"_{i['title']}_\n\n" "Triage: verify applicability to this fork, port if relevant, " "then close with a verdict comment. Auto-closes when the " "upstream item closes or its number is cited in the tree.\n\n" f"<!-- upstream-mirror: {repo}#{i['number']} -->" ) gh("issue", "create", "--repo", SELF_REPO, "--title", f"[upstream {repo.split('/')[1]} " f"{'PR' if is_pr else 'issue'} #{i['number']}] " f"{i['title'][:100]}", "--label", MIRROR_LABEL, "--body", body) created += 1 for key, issue in existing.items(): if issue["state"] != "open": continue repo, n = key if key in open_upstreams and n not in cited: continue reason = ("upstream item closed" if key not in open_upstreams else "number now cited in tree") gh("issue", "comment", str(issue["number"]), "--repo", SELF_REPO, "--body", f"Auto-closed by upstream watch: {reason}.") gh("issue", "close", str(issue["number"]), "--repo", SELF_REPO) closed += 1 return created, closed
def build_report( cited: set[int], prev_pending: dict[str, list[int]], per_repo: dict[str, list[dict]],) -> tuple[str, dict[str, list[int]], list[tuple[str, int, bool]]]: out = [ "_Daily scan of open upstream issues/PRs. Items already cited in this", "repo's code or history are marked ✅; the rest are the untriaged set._", "", ] pending: dict[str, list[int]] = {} new_items: list[tuple[str, int, bool]] = [] for repo, items in per_repo.items(): untriaged = [i for i in items if i["number"] not in cited] pending[repo] = [i["number"] for i in untriaged] prev = set(prev_pending.get(repo, [])) for i in untriaged: if i["number"] not in prev: new_items.append((repo, i["number"], "pull_request" in i))
if new_items: out.append("## 🆕 New since last scan") for repo, n, is_pr in new_items: title = next( (i["title"] for i in per_repo[repo] if i["number"] == n), "" )[:110].replace("|", "\\|") tag = "PR" if is_pr else "issue" out.append(f"- {repo} {tag} {issue_link(repo, n, is_pr)} — {title}") out.append("")
for repo in REPOS: items = per_repo[repo] issues = [i for i in items if "pull_request" not in i] prs = [i for i in items if "pull_request" in i] out.append(f"### [{repo}](https://github.com/{repo}) — {len(issues)} open issues, {len(prs)} open PRs") untriaged = [i for i in items if i["number"] not in cited] pending[repo] = [i["number"] for i in untriaged] if not untriaged: out.append("All open items are cited in this repo. ✅") else: for i in untriaged[:80]: is_pr = "pull_request" in i tag = "PR" if is_pr else "issue" title = i["title"][:110].replace("|", "\\|") out.append( f"- {tag} {issue_link(repo, i['number'], is_pr)} — {title} " f"(updated {i['updated_at'][:10]})" ) if len(untriaged) > 80: out.append(f"- … and {len(untriaged) - 80} more") out.append("") out.append("### Vendored directory drift") out += vendored_drift() or ["- no pins configured"] out.append("") out.append("### Backend divergence") out += backend_drift() out.append("") out.append(f"<!-- pending: {json.dumps(pending, sort_keys=True)} -->") return "\n".join(out), pending, new_items
def previous_pending(body: str) -> dict[str, list[int]]: old = PENDING_MARKER.search(body) return json.loads(old.group(1)) if old else {}
def tracking_issue() -> tuple[int | None, str]: try: gh("label", "create", TRACKING_LABEL, "--repo", SELF_REPO, "--description", "Automated upstream issue/PR watch", "--color", "0e8a16") except subprocess.CalledProcessError: pass # label exists issues = gh_json( "issue", "list", "--repo", SELF_REPO, "--label", TRACKING_LABEL, "--state", "open", "--limit", "1", "--json", "number,body", ) if not issues: return None, "" return issues[0]["number"], issues[0].get("body") or ""
def main() -> int: number, old_body = tracking_issue() cited = cited_numbers() per_repo = {repo: open_items(repo) for repo in REPOS} report, _pending, new_items = build_report( cited, previous_pending(old_body), per_repo ) Path("upstream-report.md").write_text(report)
created, closed = sync_mirrors(per_repo, cited) print(f"mirrors: {created} created, {closed} auto-closed")
if number is None: gh("issue", "create", "--repo", SELF_REPO, "--title", TRACKING_TITLE, "--label", TRACKING_LABEL, "--body-file", "upstream-report.md") print("created tracking issue") return 0
# --body-file, not --body: the ~18KB report as a single argv failed # twice on the runner with no visible error; a file path cannot hit # argument-size or encoding edges. gh("issue", "edit", str(number), "--repo", SELF_REPO, "--body-file", "upstream-report.md")
if new_items: lines = ["New untriaged upstream items:", ""] for repo, n, is_pr in new_items: tag = "PR" if is_pr else "issue" lines.append(f"- {repo} {tag} {issue_link(repo, n, is_pr)}") gh("issue", "comment", str(number), "--repo", SELF_REPO, "--body", "\n".join(lines)) print(f"commented {len(new_items)} new items on #{number}") else: print(f"updated #{number}, no new items") return 0
if __name__ == "__main__": sys.exit(main())