Something went wrong. Try again.
unified development activity
Something went wrong. Try again.
21 kB · 582 lines
Python
at main
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583#!/usr/bin/env python3"""Collect the past year of GitHub commits on this machine.
Run it once per account. The GitHub token never leaves this machine. It walksevery branch of every selected repository and deduplicates by commit SHA,matching the service's commit definition. The uploaded payload contains only acommit SHA, its author date, the repository (a full name for publicrepositories, a salted hash for private or internal ones), whether the defaultbranch has the commit, the account label, and the optional `Change-Id:`trailer value. It has no login, commit message,URL or author address.
Private and internal repository names are hashed by default, even when--no-redact-repos is used. A --label work report hashes every repository,public or private, because a public repository can still disclose what thework account was doing. Pass --redact-all to hash public repositories on apersonal report, or --reveal-private-repos to publish private names (only forrepositories you are willing to name publicly).
The hash salt is read from --salt, then WORK_REPO_SALT, then the file named by--salt-file or TALLY_SALT_FILE, and finally ~/.config/tally/repo-salt. Thatfile is generated with mode 0600 on first use, so no salt argument is needed;keep it stable and private, because changing it changes every hash.
For --upload-url, the bearer token is read from TALLY_UPLOAD_TOKEN, thenTALLY_UPLOAD_TOKEN_FILE, then ~/.config/tally/upload-token.
The per-label state file records each repository's `pushed_at`. An unchangedrepository is skipped on the next run without listing its branches or commits.Use --full to ignore that cache.
Typical daily cron entries:
GITHUB_TOKEN=... WORK_REPO_SALT=... \ /usr/bin/python3 github_activity.py --label personal \ --s3 s3://my-tally-state/inbox/personal-activity.json
GITHUB_WORK_TOKEN=... WORK_REPO_SALT=... \ /usr/bin/python3 github_activity.py --label work \ --s3 s3://my-tally-state/inbox/work-activity.json
The S3 credential only needs s3:PutObject on those keys; the private infrarepo defines the bucket and write-only policy."""
from __future__ import annotations
import argparseimport hashlibimport jsonimport osimport reimport secretsimport subprocessimport sysimport timeimport urllib.errorimport urllib.parseimport urllib.requestfrom datetime import datetime, timedelta, timezonefrom pathlib import Path
API = "https://api.github.com"USER_AGENT = "tally-github-activity/1"
def log(message: str) -> None: print(message, file=sys.stderr)
def utc_now() -> datetime: return datetime.now(timezone.utc)
def iso(value: datetime) -> str: return value.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
class GitHub: def __init__(self, token: str) -> None: self.token = token
def _request(self, url: str) -> tuple[bytes, dict[str, str]]: request = urllib.request.Request( url, headers={ "Authorization": f"Bearer {self.token}", "Accept": "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28", "User-Agent": USER_AGENT, }, ) attempt = 0 while True: try: with urllib.request.urlopen(request, timeout=60) as response: return response.read(), dict(response.headers) except urllib.error.HTTPError as error: remaining = error.headers.get("X-RateLimit-Remaining") reset = error.headers.get("X-RateLimit-Reset") if error.code in (403, 429) and remaining == "0" and reset: wait = max(1, int(reset) - int(time.time()) + 2) log(f"rate limited; sleeping {wait}s") time.sleep(min(wait, 900)) attempt += 1 if attempt > 3: raise continue if error.code in (500, 502, 503, 504) and attempt < 3: attempt += 1 time.sleep(2**attempt) continue detail = error.read().decode("utf-8", "replace")[:300] raise RuntimeError(f"GitHub {error.code} for {url}: {detail}") from error
def json(self, url: str) -> tuple[object, dict[str, str]]: body, headers = self._request(url) return json.loads(body), headers
def paginate(self, path: str) -> list[dict]: out: list[dict] = [] url = path if path.startswith("https://") else f"{API}{path}" while url: data, headers = self.json(url) if not isinstance(data, list): raise RuntimeError(f"expected a list from {url}") out.extend(data) url = next_link(headers.get("Link", "")) return out
def login(self) -> str: data, _ = self.json(f"{API}/user") if not isinstance(data, dict) or "login" not in data: raise RuntimeError("could not read the GitHub login") return str(data["login"])
def next_link(header: str) -> str: for part in header.split(","): part = part.strip() if part.endswith('rel="next"'): start = part.find("<") + 1 end = part.find(">") if start > 0 and end > start: return part[start:end] return ""
def list_repos(github: GitHub, allowlist: list[str]) -> list[dict]: repos = github.paginate( "/user/repos?affiliation=owner,collaborator,organization_member" "&visibility=all&sort=pushed&direction=desc&per_page=100" ) by_name = {str(repo["full_name"]): repo for repo in repos}
# An allowlist should work even for a repository the account is not a # member of but the token can still read. for full_name in allowlist: if full_name in by_name: continue data, _ = github.json(f"{API}/repos/{full_name}") if isinstance(data, dict): by_name[full_name] = data
if allowlist: return [by_name[name] for name in allowlist if name in by_name] return list(by_name.values())
def list_branches(github: GitHub, repo: dict) -> list[str]: full_name = str(repo["full_name"]) branches = github.paginate(f"/repos/{full_name}/branches?per_page=100") return [str(branch["name"]) for branch in branches]
def list_commits( github: GitHub, repo: dict, login: str, since: str, branch: str) -> list[dict]: full_name = str(repo["full_name"]) quoted_login = urllib.parse.quote(login, safe="") quoted_since = urllib.parse.quote(since, safe="") quoted_branch = urllib.parse.quote(branch, safe="") path = ( f"/repos/{full_name}/commits?sha={quoted_branch}&author={quoted_login}" f"&since={quoted_since}&per_page=100" ) return github.paginate(path)
def repo_hash(salt: str, full_name: str) -> str: digest = hashlib.sha256(f"{salt}\0{full_name}".encode("utf-8")).hexdigest() return f"h_{digest[:16]}"
def repo_is_private(repo: dict) -> bool: """Treat internal/unknown visibility as private so it is hashed.""" visibility = str(repo.get("visibility") or "").lower() if visibility: return visibility != "public" return bool(repo.get("private", True))
def redacts_repo( repo: dict, label: str, redact_all: bool, reveal_private: bool,) -> bool: """A work report hashes every name, including public ones.""" return ( redact_all or label == "work" or (repo_is_private(repo) and not reveal_private) )
def load_state(path: Path) -> dict: try: return json.loads(path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): return {}
def save_state(path: Path, state: dict) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(state, indent=2, sort_keys=True) + "\n", encoding="utf-8") os.chmod(path, 0o600)
def default_salt_path() -> Path: config_home = os.environ.get("XDG_CONFIG_HOME") or str(Path.home() / ".config") return Path(config_home) / "tally" / "repo-salt"
def load_or_create_salt(path: Path) -> str: path = path.expanduser() try: value = path.read_text(encoding="utf-8").strip() except FileNotFoundError: value = "" except OSError as error: raise SystemExit(f"could not read the repository salt at {path}: {error}") from error
if value: os.chmod(path, 0o600) return value
if path.exists(): raise SystemExit( f"the repository salt file at {path} is empty; restore it or delete it " "to generate a new one" )
value = secrets.token_hex(32) path.parent.mkdir(parents=True, exist_ok=True) try: descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) except FileExistsError: # Another run created it between the read and the write. existing = path.read_text(encoding="utf-8").strip() if existing: return existing raise SystemExit(f"the repository salt file at {path} is empty") from None except OSError as error: raise SystemExit(f"could not create the repository salt at {path}: {error}") from error
with os.fdopen(descriptor, "w", encoding="utf-8") as handle: handle.write(value + "\n") os.chmod(path, 0o600) log(f"generated a repository hash salt at {path}; keep this file private") return value
def repository_salt(explicit: str | None, salt_file: str | None) -> str: if explicit: return explicit value = os.environ.get("WORK_REPO_SALT", "").strip() if value: return value path = Path(salt_file or os.environ.get("TALLY_SALT_FILE") or default_salt_path()) return load_or_create_salt(path)
def default_upload_token_path() -> Path: config_home = os.environ.get("XDG_CONFIG_HOME") or str(Path.home() / ".config") return Path(config_home) / "tally" / "upload-token"
def upload_token() -> str: value = os.environ.get("TALLY_UPLOAD_TOKEN", "").strip() if value: return value path = Path(os.environ.get("TALLY_UPLOAD_TOKEN_FILE") or default_upload_token_path()) try: value = path.expanduser().read_text(encoding="utf-8").strip() except FileNotFoundError: return "" except OSError as error: raise SystemExit(f"could not read the upload token at {path}: {error}") from error if value: os.chmod(path.expanduser(), 0o600) return value
def upload( payload_path: Path, s3: str | None, upload_url: str | None, upload_token: str | None,) -> None: if s3: subprocess.run( [ "aws", "s3", "cp", str(payload_path), s3, "--content-type", "application/json", "--no-progress", ], check=True, ) return if upload_url: data = payload_path.read_bytes() headers = {"Content-Type": "application/json"} if upload_token: headers["Authorization"] = f"Bearer {upload_token}" request = urllib.request.Request( upload_url, data=data, method="PUT", headers=headers, ) with urllib.request.urlopen(request, timeout=60) as response: if response.status >= 300: raise RuntimeError(f"upload returned HTTP {response.status}") return log(f"wrote {payload_path}; pass --s3 or --upload-url to deliver it")
def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--token-env", help="environment variable holding the token; defaults to GITHUB_TOKEN for --label personal and GITHUB_WORK_TOKEN otherwise", ) parser.add_argument("--login", help="GitHub login; defaults to the token's account") parser.add_argument( "--label", default="work", help="account label in the payload: personal or work (default: work)", ) parser.add_argument("--since-days", type=int, default=365) parser.add_argument( "--repos", default="", help="comma-separated owner/name allowlist; default is every repo the token can see", ) parser.add_argument("--salt", help="stable local salt for repository hashes") parser.add_argument( "--salt-file", help="file holding the repository salt; defaults to $TALLY_SALT_FILE or ~/.config/tally/repo-salt, created on first use", ) parser.add_argument( "--no-redact-repos", action="store_true", help="send full names for public repositories (the default for personal); has no effect on --label work", ) parser.add_argument( "--redact-all", action="store_true", help="hash every repository, including public ones; always on for --label work", ) parser.add_argument( "--reveal-private-repos", action="store_true", help="send full names for private and internal repositories too; personal reports only, and only when you are willing to name them publicly", ) parser.add_argument( "--state", help="state file; defaults to ~/.cache/tally/github-activity-<label>-state.json", ) parser.add_argument("--out", help="payload path; defaults to github-activity-<label>.json") parser.add_argument("--s3", help="e.g. s3://bucket/inbox/work-activity.json") parser.add_argument( "--upload-url", help="an HTTPS PUT endpoint; TALLY_UPLOAD_TOKEN is sent as a bearer token when set", ) parser.add_argument("--full", action="store_true", help="ignore the last-run state") parser.add_argument("--dry-run", action="store_true", help="write locally, do not upload") args = parser.parse_args() if args.redact_all and args.reveal_private_repos: parser.error("--redact-all and --reveal-private-repos are mutually exclusive")
label = re.sub(r"[^a-z0-9_-]", "-", args.label.lower()).strip("-") or "work" if label == "work" and args.reveal_private_repos: parser.error("--label work always hashes repository names") token_env = args.token_env or ( "GITHUB_TOKEN" if label == "personal" else "GITHUB_WORK_TOKEN" ) token = os.environ.get(token_env, "").strip() if not token: raise SystemExit(f"{token_env} is not set")
github = GitHub(token) login = args.login or github.login() state_path = Path( args.state or Path.home() / ".cache" / "tally" / f"github-activity-{label}-state.json" ).expanduser() state = {} if args.full else load_state(state_path)
now = utc_now() since = now - timedelta(days=max(1, args.since_days)) last_run = state.get("last_run") if last_run: try: resume = datetime.fromisoformat(str(last_run).replace("Z", "+00:00")) # Re-read two days to survive rebases and clock skew. The service # deduplicates by SHA, so overlap is free. since = max(since, resume - timedelta(days=2)) except ValueError: pass
allowlist = [name.strip() for name in args.repos.split(",") if name.strip()] repos = [repo for repo in list_repos(github, allowlist) if not repo.get("disabled")] log(f"{login}: {len(repos)} repositories, since {iso(since)}")
# Public repositories are named for a personal account. Private and # internal ones are hashed unless the caller explicitly opts into revealing # them. A work account is always hashed in full: a public repository name # can still disclose what the work account was doing. redaction = { str(repo["full_name"]): redacts_repo( repo, label, args.redact_all, args.reveal_private_repos, ) for repo in repos } salt = repository_salt(args.salt, args.salt_file) if any(redaction.values()) else ""
commits: dict[str, dict] = {} report_repos: set[str] = set() redacted_count = 0 named_count = 0 skipped_count = 0 repo_pushes = state.get("repo_pushes") if not isinstance(repo_pushes, dict): repo_pushes = {} previous_pushes = dict(repo_pushes) for index, repo in enumerate(repos, start=1): full_name = str(repo["full_name"]) redact_repo = redaction[full_name] hashed = repo_hash(salt, full_name) if redact_repo else full_name if redact_repo: redacted_count += 1 else: named_count += 1 pushed_at = str(repo.get("pushed_at") or "")
if ( not args.full and pushed_at and previous_pushes.get(hashed) == pushed_at ): report_repos.add(hashed) repo_pushes[hashed] = pushed_at skipped_count += 1 log(f" {index}/{len(repos)} {hashed}: unchanged since {pushed_at}") continue
repo_ok = True try: branches = list_branches(github, repo) except Exception as error: # keep going; one blocked repo is not fatal log(f" {index}/{len(repos)} could not list branches for {full_name}: {error}") branches = [] repo_ok = False
default = str(repo.get("default_branch") or "") if not branches: if not default: log(f" {index}/{len(repos)} {hashed}: no branches") continue branches = [default] if default and default not in branches: branches.append(default) branches.sort(key=lambda name: name != default)
added_for_repo = 0 for branch_index, branch in enumerate(branches): is_default = (branch == default) or (not default and branch_index == 0) try: records = list_commits(github, repo, login, iso(since), branch) except Exception as error: log(f" {index}/{len(repos)} {hashed}@{branch}: {error}") repo_ok = False continue
added = 0 for record in records: sha = str(record.get("sha", "")).lower() if not re.fullmatch(r"[0-9a-f]{40}|[0-9a-f]{64}", sha): continue if sha in commits: continue commit = record.get("commit") or {} author = commit.get("author") or {} authored_at = author.get("date") if not authored_at: continue try: parsed = datetime.fromisoformat(str(authored_at).replace("Z", "+00:00")) except ValueError: continue
# The Change-Id is a stable logical identity across rebases. # Export only the trailer value, never the message. change_id = "" for line in reversed(str(commit.get("message") or "").splitlines()): if line.strip().startswith("Change-Id:"): change_id = line.split(":", 1)[1].strip() break
commits[sha] = { "sha": sha, "authored_at": iso(parsed), "repo": hashed, "redacted": redact_repo, "change_id": change_id, # The default branch is read first, so a commit it has is # marked here before any side branch can claim it. "on_default": is_default, } added += 1 added_for_repo += added # The default branch is read in full. Once another branch adds # nothing new, it has joined history already seen on this repo. if not is_default and added == 0: break
report_repos.add(hashed) if repo_ok and pushed_at: repo_pushes[hashed] = pushed_at log( f" {index}/{len(repos)} {hashed}: " f"{added_for_repo} new commits across {len(branches)} branches" )
payload = { "version": 1, "label": label, "redacted": bool(redaction) and all(redaction.values()), "generated_at": iso(now), "commits": sorted(commits.values(), key=lambda item: (item["authored_at"], item["sha"])), } out_path = Path( args.out or f"github-activity-{label}.json" ).expanduser() out_path.parent.mkdir(parents=True, exist_ok=True) out_path.write_text(json.dumps(payload, separators=(",", ":")) + "\n", encoding="utf-8") log( f"wrote {out_path} with {len(payload['commits'])} commits across " f"{named_count} named and {redacted_count} hashed repositories " f"({skipped_count} unchanged)" )
if not args.dry_run: upload(out_path, args.s3, args.upload_url, upload_token()) state["last_run"] = iso(now) state["repos"] = sorted(report_repos) state["repo_pushes"] = repo_pushes save_state(state_path, state) return 0
if __name__ == "__main__": raise SystemExit(main())