diff --git a/README.md b/README.md index babf42a..2d0945f 100644 --- a/README.md +++ b/README.md @@ -131,7 +131,8 @@ exactly where this one stopped. | `TALLY_GLOBAL_REFRESH_SECS` | `86400` | how often a global repo is revisited | | `TALLY_OWNED_REFRESH_SECS` | `3600` | how often an owned repo is revisited | | `TALLY_MAX_REPOS_PER_RUN` | unset | optional cap for a run | -| `TALLY_WORK_INBOX` | unset | `s3://…`, `https://…` or local path for redacted work data | +| `TALLY_WORK_INBOX` | unset | `s3://…`, `https://…` or local path for one inbox payload | +| `TALLY_INBOXES` | unset | comma-separated inbox payloads for multiple accounts; overrides `TALLY_WORK_INBOX` | | `GITHUB_TOKEN` | unset | personal GitHub token | | `TALLY_GITHUB_LOGIN` | token's login | personal login | | `TALLY_GITHUB_OWNERS` | `permadeath-games` | extra owners to enumerate | @@ -165,39 +166,49 @@ to `author=`. Store the token in SSM SecureString and pass its parameter name to Terraform as `github_token_parameter_name`. -### Work account: upload redacted data, not a token +### Either account: upload a report instead of a token -The service never needs a work token. On the work machine: - -1. Create a fine-grained PAT with **Metadata: Read-only** and **Contents: - Read-only** on the work repositories. The token stays there. -2. Set `WORK_REPO_SALT` to a random value and keep it there. It makes the - repository hashes stable without revealing names. -3. Run `scripts/work_activity.py` daily. On the first run it collects 365 days - across every branch; later runs re-read two days of overlap and the service - deduplicates by SHA. +The service never needs the account tokens. Run the scraper once per account +on the machine that holds that token: ```bash +# personal account; full repository names are allowed here +GITHUB_TOKEN=... python3 scripts/github_activity.py \ + --label personal --login --no-redact-repos \ + --s3 s3://tally-state-ACCOUNT/inbox/personal-activity.json + +# work account; repository names are salted hashes by default GITHUB_WORK_TOKEN=... WORK_REPO_SALT=... \ - python3 scripts/work_activity.py \ + python3 scripts/github_activity.py --label work \ --s3 s3://tally-state-ACCOUNT/inbox/work-activity.json ``` -The uploaded payload contains only: +On the first run each collects 365 days across every branch; later runs +re-read two days of overlap and the service deduplicates by SHA/Change-Id. The +state file is per label, so the two runs do not overwrite each other. + +Point the service at both payloads: + +```bash +TALLY_INBOXES=s3://tally-state-ACCOUNT/inbox/personal-activity.json,s3://tally-state-ACCOUNT/inbox/work-activity.json +``` + +The payload contains only: ```json -{"version":1,"generated_at":"…","commits":[ +{"version":1,"label":"personal","login":"…","redacted":false, + "generated_at":"…","commits":[ {"sha":"<40 or 64 hex>","authored_at":"2026-10-19T15:00:00Z", - "repo":"h_ab12…","change_id":"I…"} + "repo":"owner/repo","change_id":"I…"} ]} ``` -There is no field for a repository name, commit message, URL, author name or -email. `change_id` is only the Gerrit `Change-Id:` trailer value, so work -rebases collapse to one logical change just like atgc commits. The service -reads only those fields, ignores extras, and never persists the work login. -The site shows one `work (redacted)` identity and, if hashes are present, -repository rows named `redacted-h_ab12…`. +For `--label work`, `login` is omitted and `repo` is a salted `h_ab12…` +hash. There is no field for a commit message, URL, author name or email. +`change_id` is only the Gerrit `Change-Id:` trailer value, so rebases collapse +to one logical change just like atgc commits. `--no-redact-repos` is a +deliberate opt-out: it makes the repository names public. The service reads +only the fields above and ignores extras. The work machine's credential needs one permission: diff --git a/infra/main.tf b/infra/main.tf index 901b29a..3b91d8f 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -13,7 +13,8 @@ locals { ? var.state_bucket_name : "${var.project}-state-${local.account_id}" ) - work_inbox_key = "inbox/work-activity.json" + work_inbox_key = "inbox/work-activity.json" + personal_inbox_key = "inbox/personal-activity.json" work_inbox_url = ( var.work_inbox_url != "" ? var.work_inbox_url @@ -401,10 +402,13 @@ resource "aws_iam_role_policy" "task" { Resource = ["${aws_s3_bucket.state.arn}/state/*"] }, { - Sid = "ReadWorkInbox" - Effect = "Allow" - Action = ["s3:GetObject"] - Resource = ["${aws_s3_bucket.state.arn}/${local.work_inbox_key}"] + Sid = "ReadWorkInbox" + Effect = "Allow" + Action = ["s3:GetObject"] + Resource = [ + "${aws_s3_bucket.state.arn}/${local.work_inbox_key}", + "${aws_s3_bucket.state.arn}/${local.personal_inbox_key}" + ] } ] }) @@ -484,7 +488,9 @@ resource "aws_ecs_task_definition" "tally" { { name = "TALLY_PER_HOST_RPS", value = "10" }, { name = "RUST_LOG", value = "info" }, ], - local.work_inbox_url != "" ? [ + var.inbox_urls != "" ? [ + { name = "TALLY_INBOXES", value = var.inbox_urls }, + ] : local.work_inbox_url != "" ? [ { name = "TALLY_WORK_INBOX", value = local.work_inbox_url }, ] : [], var.personal_dids != "" ? [ diff --git a/infra/outputs.tf b/infra/outputs.tf index 17070e5..34d0d2b 100644 --- a/infra/outputs.tf +++ b/infra/outputs.tf @@ -53,10 +53,13 @@ output "work_inbox_policy" { Version = "2012-10-17" Statement = [ { - Sid = "PutRedactedWorkActivity" - Effect = "Allow" - Action = ["s3:PutObject"] - Resource = ["${aws_s3_bucket.state.arn}/${local.work_inbox_key}"] + Sid = "PutRedactedWorkActivity" + Effect = "Allow" + Action = ["s3:PutObject"] + Resource = [ + "${aws_s3_bucket.state.arn}/${local.work_inbox_key}", + "${aws_s3_bucket.state.arn}/${local.personal_inbox_key}" + ] } ] }) diff --git a/infra/variables.tf b/infra/variables.tf index 5f27d54..ac27aad 100644 --- a/infra/variables.tf +++ b/infra/variables.tf @@ -146,6 +146,12 @@ variable "work_inbox_url" { description = "Optional explicit s3:// URI for the redacted work-activity payload; overrides the derived state-bucket URI." } +variable "inbox_urls" { + type = string + default = "" + description = "Optional comma-separated TALLY_INBOXES list, e.g. s3://bucket/inbox/personal-activity.json,s3://bucket/inbox/work-activity.json. Overrides work_inbox_url." +} + variable "tags" { type = map(string) default = {} diff --git a/infra/work-inbox-policy.json b/infra/work-inbox-policy.json index c2bd4ca..bc8fac8 100644 --- a/infra/work-inbox-policy.json +++ b/infra/work-inbox-policy.json @@ -5,7 +5,10 @@ "Sid": "PutRedactedWorkActivity", "Effect": "Allow", "Action": ["s3:PutObject"], - "Resource": ["arn:aws:s3:::REPLACE_STATE_BUCKET/inbox/work-activity.json"] + "Resource": [ + "arn:aws:s3:::REPLACE_STATE_BUCKET/inbox/work-activity.json", + "arn:aws:s3:::REPLACE_STATE_BUCKET/inbox/personal-activity.json" + ] } ] } diff --git a/scripts/github_activity.py b/scripts/github_activity.py new file mode 100755 index 0000000..55515af --- /dev/null +++ b/scripts/github_activity.py @@ -0,0 +1,395 @@ +#!/usr/bin/env python3 +"""Collect the past year of GitHub commits on this machine. + +Run it once per account. The GitHub token never leaves this machine. It walks +every branch of every selected repository and deduplicates by commit SHA, +matching the service's commit definition. The uploaded payload contains only a +commit SHA, its author date, the repository (a salted hash by default, or the +full name with --no-redact-repos), the account label, the optional login for +the personal account, and the optional `Change-Id:` trailer value. It has no +commit message, URL or author address. + +Typical daily cron entries: + + GITHUB_TOKEN=... WORK_REPO_SALT=... \ + /usr/bin/python3 github_activity.py --label personal \ + --s3 s3://my-tally-state/inbox/personal-activity.json + + GITHUB_WORK_TOKEN=... WORK_REPO_SALT=... \ + /usr/bin/python3 github_activity.py --label work \ + --s3 s3://my-tally-state/inbox/work-activity.json + +The S3 credential only needs s3:PutObject on those keys. See +infra/work-inbox-policy.json. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import subprocess +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +from datetime import datetime, timedelta, timezone +from pathlib import Path + +API = "https://api.github.com" +USER_AGENT = "tally-github-activity/1" + + +def log(message: str) -> None: + print(message, file=sys.stderr) + + +def utc_now() -> datetime: + return datetime.now(timezone.utc) + + +def iso(value: datetime) -> str: + return value.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +class GitHub: + def __init__(self, token: str) -> None: + self.token = token + + def _request(self, url: str) -> tuple[bytes, dict[str, str]]: + request = urllib.request.Request( + url, + headers={ + "Authorization": f"Bearer {self.token}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + "User-Agent": USER_AGENT, + }, + ) + attempt = 0 + while True: + try: + with urllib.request.urlopen(request, timeout=60) as response: + return response.read(), dict(response.headers) + except urllib.error.HTTPError as error: + remaining = error.headers.get("X-RateLimit-Remaining") + reset = error.headers.get("X-RateLimit-Reset") + if error.code in (403, 429) and remaining == "0" and reset: + wait = max(1, int(reset) - int(time.time()) + 2) + log(f"rate limited; sleeping {wait}s") + time.sleep(min(wait, 900)) + attempt += 1 + if attempt > 3: + raise + continue + if error.code in (500, 502, 503, 504) and attempt < 3: + attempt += 1 + time.sleep(2**attempt) + continue + detail = error.read().decode("utf-8", "replace")[:300] + raise RuntimeError(f"GitHub {error.code} for {url}: {detail}") from error + + def json(self, url: str) -> tuple[object, dict[str, str]]: + body, headers = self._request(url) + return json.loads(body), headers + + def paginate(self, path: str) -> list[dict]: + out: list[dict] = [] + url = path if path.startswith("https://") else f"{API}{path}" + while url: + data, headers = self.json(url) + if not isinstance(data, list): + raise RuntimeError(f"expected a list from {url}") + out.extend(data) + url = next_link(headers.get("Link", "")) + return out + + def login(self) -> str: + data, _ = self.json(f"{API}/user") + if not isinstance(data, dict) or "login" not in data: + raise RuntimeError("could not read the GitHub login") + return str(data["login"]) + + +def next_link(header: str) -> str: + for part in header.split(","): + part = part.strip() + if part.endswith('rel="next"'): + start = part.find("<") + 1 + end = part.find(">") + if start > 0 and end > start: + return part[start:end] + return "" + + +def list_repos(github: GitHub, allowlist: list[str]) -> list[dict]: + repos = github.paginate( + "/user/repos?affiliation=owner,collaborator,organization_member" + "&visibility=all&sort=pushed&direction=desc&per_page=100" + ) + by_name = {str(repo["full_name"]): repo for repo in repos} + + # An allowlist should work even for a repository the account is not a + # member of but the token can still read. + for full_name in allowlist: + if full_name in by_name: + continue + data, _ = github.json(f"{API}/repos/{full_name}") + if isinstance(data, dict): + by_name[full_name] = data + + if allowlist: + return [by_name[name] for name in allowlist if name in by_name] + return list(by_name.values()) + + +def list_branches(github: GitHub, repo: dict) -> list[str]: + full_name = str(repo["full_name"]) + branches = github.paginate(f"/repos/{full_name}/branches?per_page=100") + return [str(branch["name"]) for branch in branches] + + +def list_commits( + github: GitHub, repo: dict, login: str, since: str, branch: str +) -> list[dict]: + full_name = str(repo["full_name"]) + quoted_login = urllib.parse.quote(login, safe="") + quoted_since = urllib.parse.quote(since, safe="") + quoted_branch = urllib.parse.quote(branch, safe="") + path = ( + f"/repos/{full_name}/commits?sha={quoted_branch}&author={quoted_login}" + f"&since={quoted_since}&per_page=100" + ) + return github.paginate(path) + + +def repo_hash(salt: str, full_name: str) -> str: + digest = hashlib.sha256(f"{salt}\0{full_name}".encode("utf-8")).hexdigest() + return f"h_{digest[:16]}" + + +def load_state(path: Path) -> dict: + try: + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return {} + + +def save_state(path: Path, state: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(state, indent=2, sort_keys=True) + "\n", encoding="utf-8") + os.chmod(path, 0o600) + + +def salt_from_environment(explicit: str | None) -> str: + if explicit: + return explicit + value = os.environ.get("WORK_REPO_SALT", "").strip() + if value: + return value + raise SystemExit( + "set WORK_REPO_SALT (or --salt); it keeps repository hashes stable and must stay on this machine. " + "Pass --no-redact-repos if you want full repository names instead." + ) + + +def upload(payload_path: Path, s3: str | None, upload_url: str | None) -> None: + if s3: + subprocess.run( + [ + "aws", + "s3", + "cp", + str(payload_path), + s3, + "--content-type", + "application/json", + "--no-progress", + ], + check=True, + ) + return + if upload_url: + data = payload_path.read_bytes() + request = urllib.request.Request( + upload_url, + data=data, + method="PUT", + headers={"Content-Type": "application/json"}, + ) + with urllib.request.urlopen(request, timeout=60) as response: + if response.status >= 300: + raise RuntimeError(f"upload returned HTTP {response.status}") + return + log(f"wrote {payload_path}; pass --s3 or --upload-url to deliver it") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--token-env", default="GITHUB_WORK_TOKEN") + parser.add_argument("--login", help="GitHub login; defaults to the token's account") + parser.add_argument( + "--label", + default="work", + help="account label in the payload: personal or work (default: work)", + ) + parser.add_argument("--since-days", type=int, default=365) + parser.add_argument( + "--repos", + default="", + help="comma-separated owner/name allowlist; default is every repo the token can see", + ) + parser.add_argument("--salt", help="stable local salt for repository hashes") + parser.add_argument( + "--no-redact-repos", + action="store_true", + help="send full repository names instead of salted hashes; only for repositories you are willing to name publicly", + ) + parser.add_argument( + "--state", + help="state file; defaults to ~/.cache/tally/github-activity-