Something went wrong. Try again.
unified development activity
Something went wrong. Try again.
22 kB · 643 lines
Rust
at main
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644//! GitHub commit ingestion.//!//! There can be more than one account: the personal one (including the//! `permadeath-games` organization) and an optional work account. A work//! repository can be marked redacted. For those repositories the database//! keeps only the commit SHA, its author date and a label — no repository//! name, message, URL or author address — and the public site shows a single//! aggregate "work GitHub" series.//!//! Tokens are read from the environment and are never written to the database//! or the site.
use std::collections::{HashMap, HashSet};
use anyhow::{Context, Result, bail};use chrono::{Duration, Utc};use serde::Deserialize;
use crate::config::{Config, GithubAccountConfig};use crate::db::{Db, canonical_time, now_string};use crate::http::HttpClient;use crate::model::{Commit, CommitRef, Identity, IdentityKind};
const API: &str = "https://api.github.com";
#[derive(Debug, Clone)]pub struct Github { http: HttpClient, label: String, token: String, login: String, owners: Vec<String>, redact_all: bool, redact_repos: HashSet<String>, color: String, identity_did: String, identity_handle: String,}
#[derive(Debug, Clone, Default)]pub struct GithubReport { pub label: String, pub login: String, pub redacted: bool, pub repos: usize, pub repos_scanned: usize, pub commits_new: usize, pub errors: usize,}
#[derive(Debug, Deserialize)]struct User { login: String,}
#[derive(Debug, Deserialize)]struct Repo { id: u64, full_name: String, #[serde(default)] private: bool, #[serde(default)] disabled: bool, pushed_at: Option<String>, default_branch: Option<String>, homepage: Option<String>,}
#[derive(Debug, Deserialize)]struct Branch { name: String,}
#[derive(Debug, Deserialize)]struct ApiCommit { sha: String, #[serde(default)] html_url: String, commit: GitCommit,}
#[derive(Debug, Deserialize)]struct GitCommit { author: Option<GitPerson>, committer: Option<GitPerson>, message: String,}
#[derive(Debug, Deserialize)]struct GitPerson { name: Option<String>, email: Option<String>, date: Option<String>,}
impl Github { pub async fn connect_all(http: HttpClient, config: &Config) -> Result<Vec<Self>> { let mut accounts = Vec::new(); for account in &config.github_accounts { match Self::connect(http.clone(), account).await { Ok(client) => accounts.push(client), Err(err) => { tracing::warn!(label = %account.label, error = %err, "GitHub account could not be used"); } } } Ok(accounts) }
async fn connect(http: HttpClient, account: &GithubAccountConfig) -> Result<Self> { let login = match &account.login { Some(login) => login.clone(), None => { let user: User = http .get_json(&format!("{API}/user"), Some(&account.token)) .await .context("asking GitHub which account the token belongs to")?; user.login } };
// Account logins are needed for the API call and the author filter, // but they are not published as identity names. let (identity_did, identity_handle) = if account.label == "work" { if account.redact_all { ( "github-work:redacted".to_string(), "work GitHub".to_string(), ) } else { ("github-inbox:work".to_string(), "work GitHub".to_string()) } } else if account.label == "personal" { ("github:personal".to_string(), "personal GitHub".to_string()) } else { ( format!("github-inbox:{}", account.label), account .identity_handle .clone() .unwrap_or_else(|| format!("{} GitHub", account.label)), ) };
Ok(Self { http, label: account.label.clone(), token: account.token.clone(), login, owners: account.owners.clone(), redact_all: account.redact_all, redact_repos: account.redact_repos.iter().cloned().collect(), color: account.color.clone(), identity_did, identity_handle, }) }
pub fn identity(&self) -> Identity { Identity { did: self.identity_did.clone(), handle: self.identity_handle.clone(), kind: IdentityKind::Github, operator_did: None, color: self.color.clone(), active: true, pds: None, collections: String::new(), checked_at: Some(now_string()), } }
pub async fn crawl(&self, db: &Db, full: bool) -> Result<GithubReport> { db.upsert_identity(&self.identity())?;
let repos = self.list_repos().await?; let mut report = GithubReport { label: self.label.clone(), login: self.login.clone(), redacted: self.redact_all, repos: repos.len(), ..Default::default() };
for repo in repos { if repo.disabled { continue; } let redacted = self.redact_all || self.redact_repos.contains(&repo.full_name); let state_key = if redacted { // The numeric id is stable and does not name the repository. format!("{}:id:{}", self.label, repo.id) } else { repo.full_name.clone() };
let stored_pushed = db.github_pushed_at(&state_key)?; // A failed repository is retried even if its pushed-at stamp has // not moved; otherwise a rate-limit failure would park it until // the next real push. let had_error = db.github_last_error(&state_key)?.is_some(); let unchanged = !had_error && repo.pushed_at.is_some() && stored_pushed.as_deref() == repo.pushed_at.as_deref(); if unchanged && !full { continue; }
let since = if full { None } else { db.github_last_scanned(&state_key)? .and_then(|stamp| chrono::DateTime::parse_from_rfc3339(&stamp).ok()) .map(|stamp| { (stamp.with_timezone(&Utc) - Duration::days(2)) .to_rfc3339_opts(chrono::SecondsFormat::Secs, true) }) };
// A side-branch commit merged later keeps its old committer // date, so the default branch is read in full while any could be. let default_since = if since.is_some() && db.has_side_branch_refs("github", &self.repo_key(&repo.full_name, redacted))? { None } else { since.clone() }; match self .scan_repo_commits(&repo, redacted, since.as_deref(), default_since.as_deref()) .await { Ok(rows) => { report.commits_new += db.insert_commits(&rows)?; report.repos_scanned += 1; db.touch_github_repo( &state_key, repo.pushed_at.as_deref().unwrap_or(""), if redacted { None } else { repo.homepage.as_deref() }, None, )?; if repo.private && !redacted { tracing::debug!(repo = %repo.full_name, "indexed private GitHub repository"); } } Err(err) => { report.errors += 1; tracing::warn!( account = %self.label, repo = %repo.full_name, redacted, error = %err, "GitHub commit scan failed" ); // Do not persist the real name in an error for a redacted // repository; the message may contain it. let error_message = if redacted { "redacted repository could not be read".to_string() } else { err.to_string() }; db.touch_github_repo( &state_key, repo.pushed_at.as_deref().unwrap_or(""), if redacted { None } else { repo.homepage.as_deref() }, Some(&error_message), )?; } } }
Ok(report) }
async fn list_repos(&self) -> Result<Vec<Repo>> { let mut repos: HashMap<String, Repo> = HashMap::new(); for repo in self.list_user_repos().await? { repos.insert(repo.full_name.clone(), repo); } for owner in &self.owners { match self.list_owner_repos(owner).await { Ok(owner_repos) => { for repo in owner_repos { repos.insert(repo.full_name.clone(), repo); } } Err(err) => { tracing::warn!(owner, error = %err, "could not list GitHub owner repositories"); } } } Ok(repos.into_values().collect()) }
async fn list_user_repos(&self) -> Result<Vec<Repo>> { self.list_pages("/user/repos?affiliation=owner,collaborator,organization_member&visibility=all&sort=pushed&direction=desc") .await }
async fn list_owner_repos(&self, owner: &str) -> Result<Vec<Repo>> { // Organizations answer /orgs; plain users answer /users. Try the // organization endpoint first, since the interesting owner here is // permadeath-games. let org_path = format!("/orgs/{owner}/repos?type=all&sort=pushed&direction=desc"); match self.list_pages(&org_path).await { Ok(repos) => Ok(repos), Err(org_error) => { let user_path = format!("/users/{owner}/repos?type=owner&sort=pushed&direction=desc"); self.list_pages(&user_path).await.with_context(|| { format!("listing {owner} as an organization failed ({org_error}) and as a user also failed") }) } } }
async fn list_pages(&self, path: &str) -> Result<Vec<Repo>> { let mut out = Vec::new(); let mut page = 1u32; loop { let separator = if path.contains('?') { '&' } else { '?' }; let url = format!("{API}{path}{separator}per_page=100&page={page}"); let response = self.http.get(&url, Some(&self.token)).await?; let repos: Vec<Repo> = serde_json::from_slice(&response.body) .context("decoding GitHub repository list")?; let count = repos.len(); out.extend(repos); if count < 100 { break; } page += 1; if page > 100 { bail!("GitHub repository listing did not terminate"); } } Ok(out) }
/// Walk every branch and return the union of commits authored by this /// account, deduplicated by Git object name and marked by whether the /// default branch has them. The default branch is read first, from /// `default_since`; a later branch stops as soon as a page adds nothing /// new, which is the same shared-history rule the Tangled crawler uses. async fn scan_repo_commits( &self, repo: &Repo, redacted: bool, since: Option<&str>, default_since: Option<&str>, ) -> Result<Vec<(Commit, CommitRef)>> { let mut branches = match self.list_branches(&repo.full_name).await { Ok(branches) => branches, Err(err) => { tracing::warn!( repo = %repo.full_name, error = %err, "GitHub branch listing failed; falling back to the default branch" ); Vec::new() } }; if branches.is_empty() && let Some(default) = &repo.default_branch { branches.push(default.clone()); } if branches.is_empty() { bail!("no branches could be listed for {}", repo.full_name); }
let default = repo.default_branch.clone().unwrap_or_default(); if !default.is_empty() { if !branches.iter().any(|branch| branch == &default) { branches.push(default.clone()); } branches.sort_by_key(|branch| branch != &default); } let mut seen_branches = HashSet::new(); branches.retain(|branch| seen_branches.insert(branch.clone()));
let mut seen = HashSet::new(); let mut rows = Vec::new(); let mut first = true; for branch in branches { let is_default = if default.is_empty() { first } else { branch == default }; first = false; let from = if is_default { default_since } else { since }; let mut next = Some(self.commits_url(&repo.full_name, &branch, from)); while let Some(url) = next.take() { let (commits, link) = self.commit_page(&url).await?; let mut new_in_page = 0; for commit in commits { if seen.insert(commit.sha.clone()) { new_in_page += 1; let (commit, mut reference) = self.normalize(&repo.full_name, redacted, commit)?; reference.on_default = is_default; rows.push((commit, reference)); } } if !is_default && new_in_page == 0 { break; } next = link; } } Ok(rows) }
async fn list_branches(&self, full_name: &str) -> Result<Vec<String>> { let mut out = Vec::new(); let mut page = 1u32; loop { let url = format!("{API}/repos/{full_name}/branches?per_page=100&page={page}"); let response = self.http.get(&url, Some(&self.token)).await?; let branches: Vec<Branch> = serde_json::from_slice(&response.body).context("decoding GitHub branches")?; let count = branches.len(); out.extend(branches.into_iter().map(|branch| branch.name)); if count < 100 { break; } page += 1; if page > 100 { bail!("GitHub branch listing did not terminate"); } } Ok(out) }
fn commits_url(&self, full_name: &str, branch: &str, since: Option<&str>) -> String { let mut url = format!( "{API}/repos/{full_name}/commits?sha={}&author={}&per_page=100", urlencoding(branch), urlencoding(&self.login) ); if let Some(since) = since { url.push_str("&since="); url.push_str(&urlencoding(since)); } url }
async fn commit_page(&self, url: &str) -> Result<(Vec<ApiCommit>, Option<String>)> { let response = self.http.get(url, Some(&self.token)).await?; let commits: Vec<ApiCommit> = serde_json::from_slice(&response.body).context("decoding GitHub commits")?; let next = response .headers .get("link") .and_then(|value| value.to_str().ok()) .and_then(next_link); Ok((commits, next)) }
fn repo_key(&self, full_name: &str, redacted: bool) -> String { if redacted { format!("github:redacted:{}", self.label) } else { format!("github:{full_name}") } }
fn normalize( &self, full_name: &str, redacted: bool, commit: ApiCommit, ) -> Result<(Commit, CommitRef)> { let author = commit.commit.author.as_ref(); let committer = commit.commit.committer.as_ref(); let authored_at = author .and_then(|person| person.date.as_deref()) .and_then(canonical_time) .unwrap_or_else(now_string); let committed_at = committer .and_then(|person| person.date.as_deref()) .and_then(canonical_time) .unwrap_or_else(|| authored_at.clone()); let message = if redacted { String::new() } else { commit .commit .message .lines() .next() .unwrap_or("") .chars() .take(500) .collect::<String>() }; let change_id = if redacted { String::new() } else { crate::model::change_id_from_message(&commit.commit.message) }; let sha = commit.sha; Ok(( Commit { sha: sha.clone(), change_id, author_did: self.identity_did.clone(), author_email: if redacted { String::new() } else { author .and_then(|person| person.email.clone()) .unwrap_or_default() }, author_name: if redacted { String::new() } else { author .and_then(|person| person.name.clone()) .unwrap_or_else(|| self.login.clone()) }, authored_at, committed_at, message, }, CommitRef { sha, source: "github".to_string(), repo_key: self.repo_key(full_name, redacted), url: if redacted { String::new() } else { commit.html_url }, on_default: true, }, )) }}
fn next_link(header: &str) -> Option<String> { for part in header.split(',') { let part = part.trim(); if part.ends_with(r#"rel="next""#) { let start = part.find('<')? + 1; let end = part.find('>')?; return Some(part[start..end].to_string()); } } None}
fn urlencoding(value: &str) -> String { url::form_urlencoded::byte_serialize(value.as_bytes()).collect()}
#[cfg(test)]mod tests { use super::*;
#[test] fn parses_next_link() { let header = r#"<https://api.github.com/repositories/1/commits?page=2>; rel="next", <https://api.github.com/repositories/1/commits?page=9>; rel="last""#; assert_eq!( next_link(header).unwrap(), "https://api.github.com/repositories/1/commits?page=2" ); }
fn test_client(redact: bool) -> Github { Github { http: HttpClient::new(std::time::Duration::from_secs(1), 1.0).unwrap(), label: if redact { "work" } else { "personal" }.to_string(), token: "not-used".to_string(), login: "me".to_string(), owners: Vec::new(), redact_all: redact, redact_repos: HashSet::new(), color: if redact { "series-7" } else { "series-5" }.to_string(), identity_did: if redact { "github-work:redacted".to_string() } else { "github:personal".to_string() }, identity_handle: if redact { "work GitHub".to_string() } else { "personal GitHub".to_string() }, } }
fn sample_commit() -> ApiCommit { ApiCommit { sha: "a".repeat(40), html_url: "https://github.com/acme/secret/commit/a".to_string(), commit: GitCommit { author: Some(GitPerson { name: Some("Me".to_string()), email: Some("me@example.test".to_string()), date: Some("2026-10-19T15:00:00Z".to_string()), }), committer: None, message: "subject\n\nbody".to_string(), }, } }
#[test] fn redacted_commits_keep_only_the_object_name_and_date() { let client = test_client(true); let (commit, reference) = client .normalize("acme/secret", true, sample_commit()) .unwrap(); assert_eq!(commit.sha, "a".repeat(40)); assert_eq!(commit.author_did, "github-work:redacted"); assert!(commit.author_email.is_empty()); assert!(commit.author_name.is_empty()); assert!(commit.message.is_empty()); assert_eq!(reference.repo_key, "github:redacted:work"); assert!(reference.url.is_empty()); assert!(!reference.repo_key.contains("acme")); }
#[test] fn visible_commits_keep_their_message_and_link() { let client = test_client(false); let (commit, reference) = client .normalize("acme/public", false, sample_commit()) .unwrap(); assert_eq!(commit.message, "subject"); assert_eq!(reference.repo_key, "github:acme/public"); assert_eq!(reference.url, "https://github.com/acme/secret/commit/a"); }}