diff --git a/Cargo.lock b/Cargo.lock index 0fce9207f..9f936e52f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3033,6 +3033,7 @@ dependencies = [ "bobbin-runtime", "chrono", "deadpool-redis", + "fixedbitset", "futures-lite", "gitmirror-db", "gitmirror-git", @@ -3041,6 +3042,7 @@ dependencies = [ "gix-pack", "gix-receive-pack", "gix-transport", + "itertools 0.14.0", "jacquard-axum", "jacquard-common", "jacquard-identity", diff --git a/gitmirror/crates/gitmirror-xrpc/Cargo.toml b/gitmirror/crates/gitmirror-xrpc/Cargo.toml index 36126d70b..df8bf5b9b 100644 --- a/gitmirror/crates/gitmirror-xrpc/Cargo.toml +++ b/gitmirror/crates/gitmirror-xrpc/Cargo.toml @@ -34,6 +34,8 @@ thiserror = { workspace = true } tracing = "0.1" line-numbers = "0.4.0" rustc-hash = "2.1.2" +fixedbitset = "0.5" +itertools = { workspace = true } gix = { version = "0.84", features = ["parallel", "blob-diff", "blame", "merge", "sha1", "sha256", "revision", "tree-editor", "worktree-stream"] } gix-archive = { workspace = true } gix-pack = { workspace = true } diff --git a/gitmirror/crates/gitmirror-xrpc/src/last_commit.rs b/gitmirror/crates/gitmirror-xrpc/src/last_commit.rs index 2e7d173a0..b39286486 100644 --- a/gitmirror/crates/gitmirror-xrpc/src/last_commit.rs +++ b/gitmirror/crates/gitmirror-xrpc/src/last_commit.rs @@ -1,165 +1,219 @@ -use std::collections::{HashMap, HashSet}; +use std::collections::hash_map::Entry; +use std::collections::{BinaryHeap, HashMap}; use std::time::{Duration, Instant}; -use deadpool_redis::Pool; -use deadpool_redis::redis; +use fixedbitset::FixedBitSet; use gix::ObjectId; -use gix::bstr::ByteSlice as _; -use gix::revision::walk::Sorting; +use gix::bstr::{BStr, BString, ByteSlice as _}; +use gix::commitgraph::bloom::BloomKey; +use gix::commitgraph::file::Commit as GraphCommit; +use gix::commitgraph::{BloomFilterSettings, Graph}; +use gix::objs::tree::EntryRef; +use itertools::Itertools as _; use metrics::{counter, gauge, histogram}; use tracing::warn; -// TODO(boltless): change to something else like "/". currently empty to be v1-compatible -pub(crate) const DIR_KEY: &str = ""; +pub(crate) const BUDGET: Duration = Duration::from_secs(1); -const TTL: u64 = 30 * 24 * 60 * 60; +pub(crate) fn join(dir: impl AsRef, name: impl AsRef) -> BString { + let (dir, name) = (dir.as_ref(), name.as_ref()); + let mut out = dir.to_owned(); + if !dir.is_empty() && !name.is_empty() { + out.push(b'/'); + } + out.extend_from_slice(name); + out +} -pub(crate) const BUDGET: Duration = Duration::from_secs(5); +fn is_dir_prefix(path: &BStr, prefix: &BStr) -> bool { + path.strip_prefix(prefix.as_bytes()) + .is_some_and(|rest| rest.is_empty() || rest.starts_with_str("/")) +} -fn cache_key(head: &ObjectId, tree_path: &str, name: &str) -> String { - let joined = match (tree_path.is_empty(), name.is_empty()) { - (true, true) => String::new(), - (true, false) => name.to_owned(), - (false, true) => tree_path.to_owned(), - (false, false) => format!("{tree_path}/{name}"), - }; - format!("last_commit:{head}:{joined}") +fn within_depth(name: &BStr, depth: usize, max_depth: usize) -> bool { + depth + name.find_iter("/").count() <= max_depth } -pub(crate) async fn load( - pool: Option<&Pool>, - head: &ObjectId, - tree_path: &str, - names: &[String], -) -> HashMap { - let Some(pool) = pool else { - return HashMap::new(); - }; - if names.is_empty() { - return HashMap::new(); +struct Node { + id: ObjectId, + generation: u32, + time: i64, + tree: ObjectId, + parents: Vec, +} + +impl Node { + /// Newest generation first, then newest commit time. + fn queue_key(&self) -> (u32, i64, ObjectId) { + (self.generation, self.time, self.id) } - let keys: Vec = names - .iter() - .map(|name| cache_key(head, tree_path, name)) - .collect(); - - let cached = async { - let mut conn = pool.get().await?; - let values = redis::cmd("MGET") - .arg(&keys) - .query_async::>>(&mut conn) - .await?; - anyhow::Ok(values) +} + +fn node(repo: &gix::Repository, graph: Option<&Graph>, id: ObjectId) -> Option { + if let Some(graph) = graph + && let Some(commit) = graph.commit_by_id(id) + { + return Some(Node { + id, + generation: commit.generation(), + time: commit.committer_timestamp() as i64, + tree: commit.root_tree_id().to_owned(), + parents: commit + .iter_parents() + .map(|parent| parent.map(|pos| graph.id_at(pos).to_owned())) + .collect::>() + .ok()?, + }); } - .await; + let commit = repo.find_commit(id).ok()?; + Some(Node { + id, + generation: u32::MAX, + time: commit.time().ok()?.seconds, + tree: commit.tree_id().ok()?.detach(), + parents: commit.parent_ids().map(|id| id.detach()).collect(), + }) +} - let cached = match cached { - Ok(cached) => cached, - Err(error) => { - warn!(%error, "last-commit cache read failed"); - return HashMap::new(); - } - }; +/// A queued commit and the paths it may still be credited with. +struct Active { + node: Node, + pending: FixedBitSet, +} - names - .iter() - .zip(cached) - .filter_map(|(name, value)| { - let oid = ObjectId::from_hex(value?.as_bytes()).ok()?; - Some((name.clone(), oid)) - }) - .collect() +struct TreeDiff<'a> { + repo: &'a gix::Repository, + pathspec: Option<&'a BStr>, + /// `None` recurses without limit. + max_depth: Option, + /// Also emit the trees that are recursed into. + show_trees: bool, } -pub(crate) async fn store( - pool: Option<&Pool>, - head: &ObjectId, - tree_path: &str, - found: &HashMap, -) { - let Some(pool) = pool else { return }; - if found.is_empty() { - return; +impl TreeDiff<'_> { + /// `path` is inside the pathspec, or a directory leading to it. + fn interesting(&self, path: &BStr, is_dir: bool) -> bool { + self.pathspec + .is_none_or(|ps| is_dir_prefix(path, ps) || (is_dir && is_dir_prefix(ps, path))) } - let written = async { - let mut conn = pool.get().await?; - let mut pipe = redis::pipe(); - for (name, oid) in found { - pipe.cmd("SET") - .arg(cache_key(head, tree_path, name)) - .arg(oid.to_string()) - .arg("EX") - .arg(TTL) - .ignore(); + fn should_recurse(&self, path: &BStr) -> bool { + let Some(max_depth) = self.max_depth else { + return true; + }; + match self.pathspec { + None => within_depth(path, 1, max_depth), + Some(ps) if path.len() >= ps.len() => { + is_dir_prefix(path, ps) && within_depth(path[ps.len()..].as_bstr(), 1, max_depth) + } + Some(ps) => is_dir_prefix(ps, path), } - pipe.query_async::<()>(&mut conn).await?; - anyhow::Ok(()) } - .await; - if let Err(error) = written { - warn!(%error, "last-commit cache write failed"); + /// The paths that differ between the trees `old` and `new`; `None` is the empty tree. + fn changed_paths(&self, old: Option, new: Option) -> Vec { + let mut out = Vec::new(); + let mut stack = vec![(BString::default(), old, new)]; + while let Some((base, old, new)) = stack.pop() { + let tree = |id: Option| id.and_then(|id| self.repo.find_tree(id).ok()); + let (old, new) = (tree(old), tree(new)); + // Entries are sorted by `EntryRef`'s order, where a tree and a blob of the same + // name differ. + for pair in entries(old.as_ref()).merge_join_by(entries(new.as_ref()), Ord::cmp) { + let entry = pair.clone().reduce(|_, new| new); + let (old, new) = pair.left_and_right(); + let path = join(&base, entry.filename); + if old == new || !self.interesting(path.as_bstr(), entry.mode.is_tree()) { + continue; + } + if entry.mode.is_tree() && self.should_recurse(path.as_bstr()) { + if self.show_trees { + out.push(path.clone()); + } + let id = |side: Option| side.map(|e| e.oid.to_owned()); + stack.push((path, id(old), id(new))); + } else { + out.push(path); + } + } + } + out } } -fn commit_tree(repo: &gix::Repository, commit: ObjectId) -> Option { - repo.find_commit(commit) - .ok()? - .tree_id() - .ok() - .map(|id| id.detach()) +/// `None` and unreadable trees have no entries. +fn entries<'a>(tree: Option<&'a gix::Tree<'_>>) -> impl Iterator> { + tree.and_then(|tree| tree.decode().ok()) + .into_iter() + .flat_map(|tree| tree.entries) } -fn subtree_of( - repo: &gix::Repository, - memo: &mut HashMap>, - root: ObjectId, - tree_path: &str, -) -> Option { - if let Some(cached) = memo.get(&root) { - return *cached; - } - let found = if tree_path.is_empty() { - Some(root) - } else { - repo.find_tree(root) - .ok() - .and_then(|tree| tree.lookup_entry_by_path(tree_path).ok().flatten()) - .filter(|entry| entry.mode().is_tree()) - .map(|entry| entry.object_id()) - }; - memo.insert(root, found); - found +/// The paths to resolve; a path's index is its bit in every bitset. +struct Paths { + paths: Vec, + index: HashMap, + /// `None` without Bloom filters. + keys: Option>, } -fn ensure_entries( - repo: &gix::Repository, - memo: &mut HashMap>, - memo_entries: &mut usize, - subtree: ObjectId, -) { - if memo.contains_key(&subtree) { - return; +impl Paths { + fn new(diff: &TreeDiff, head_tree: ObjectId, bloom: Option) -> Self { + let paths = diff.changed_paths(None, Some(head_tree)); + Self { + index: paths.iter().cloned().zip(0..).collect(), + keys: bloom.map(|settings| { + paths + .iter() + .map(|path| BloomKey::from_path(path.as_bstr(), settings)) + .collect() + }), + paths, + } } - let mut entries = HashMap::new(); - if let Ok(tree) = repo.find_tree(subtree) { - for entry in tree.iter().flatten() { - entries.insert( - entry.filename().to_str_lossy().into_owned(), - entry.object_id(), - ); + + /// `false` only if the Bloom filter rules out every active path. + fn maybe_changed(&self, commit: Option>, active: &FixedBitSet) -> bool { + let (Some(keys), Some(commit)) = (&self.keys, commit) else { + return true; + }; + // `maybe_contains_all_keys` treats an empty filter as a miss. + if commit.maybe_contains_path(BStr::new("")).is_none() { + return true; } + active + .ones() + .any(|k| commit.maybe_contains_all_keys(std::slice::from_ref(&keys[k])) != Some(false)) + } +} + +/// Moves the paths unchanged from `parent_tree` out of `pending` and returns them. +fn process_parent( + diff: &TreeDiff, + paths: &Paths, + graph: Option<&Graph>, + commit: &Node, + pending: &mut FixedBitSet, + parent_i: usize, + parent_tree: ObjectId, +) -> FixedBitSet { + let mut changed = FixedBitSet::with_capacity(paths.paths.len()); + let graph_commit = || graph.and_then(|graph| graph.commit_by_id(commit.id)); + if parent_i > 0 || paths.maybe_changed(graph_commit(), pending) { + changed.extend( + diff.changed_paths(Some(parent_tree), Some(commit.tree)) + .iter() + .filter_map(|path| paths.index.get(path).copied()), + ); } - *memo_entries += entries.len(); - memo.insert(subtree, entries); + let mut same = pending.clone(); + same.difference_with(&changed); + pending.difference_with(&same); + same } struct WalkGuard { outcome: &'static str, commits: usize, - memo_trees: usize, - memo_entries: usize, } impl WalkGuard { @@ -168,8 +222,6 @@ impl WalkGuard { Self { outcome: "error", commits: 0, - memo_trees: 0, - memo_entries: 0, } } } @@ -180,95 +232,325 @@ impl Drop for WalkGuard { counter!("gitmirror_last_commit_walks_total", "outcome" => self.outcome).increment(1); histogram!("gitmirror_last_commit_commits_walked", "outcome" => self.outcome) .record(self.commits as f64); - histogram!("gitmirror_last_commit_memo_trees").record(self.memo_trees as f64); - histogram!("gitmirror_last_commit_memo_entries").record(self.memo_entries as f64); } } +/// The last commit to change each path under `pathspec` ("" for the whole tree). pub(crate) fn walk( repo: &gix::Repository, head: ObjectId, - tree_path: &str, - names: &[String], + pathspec: &str, + max_depth: Option, + show_trees: bool, deadline: Instant, -) -> HashMap { +) -> HashMap { let mut out = HashMap::new(); - if names.is_empty() { - return out; - } - let mut pending: HashSet<&str> = names.iter().map(String::as_str).collect(); let mut stats = WalkGuard::new(); - let walk = repo - .rev_walk([head]) - .sorting(Sorting::ByCommitTime(Default::default())) - .all(); - let walk = match walk { - Ok(walk) => walk, - Err(error) => { - warn!(%error, "last-commit walk could not start"); - return out; - } + let graph = repo.commit_graph_if_enabled().ok().flatten(); + let graph = graph.as_ref(); + let Some(head_node) = node(repo, graph, head) else { + warn!(%head, "last-commit walk could not start"); + return out; }; - let mut subtrees: HashMap> = HashMap::new(); - let mut entries: HashMap> = HashMap::new(); - let empty = HashMap::new(); + let pathspec = pathspec.trim_matches('/'); + let diff = TreeDiff { + repo, + pathspec: (!pathspec.is_empty()).then(|| pathspec.into()), + max_depth, + show_trees, + }; + let paths = Paths::new( + &diff, + head_node.tree, + graph.and_then(Graph::bloom_filter_settings), + ); - for info in walk { - if pending.is_empty() { - break; - } + let mut all = FixedBitSet::with_capacity(paths.paths.len()); + all.insert_range(..); + let mut queue = BinaryHeap::from([head_node.queue_key()]); + let mut active = HashMap::from([( + head, + Active { + node: head_node, + pending: all, + }, + )]); + + while let Some((_, _, id)) = queue.pop() { if Instant::now() >= deadline { stats.outcome = "deadline"; break; } - let Ok(info) = info else { break }; stats.commits += 1; - let Some(root) = commit_tree(repo, info.id) else { - continue; - }; - let current = subtree_of(repo, &mut subtrees, root, tree_path); - let parent = info - .parent_ids - .first() - .and_then(|parent| commit_tree(repo, *parent)) - .and_then(|root| subtree_of(repo, &mut subtrees, root, tree_path)); - - if current == parent { - continue; - } + let Active { + node: commit, + mut pending, + } = active.remove(&id).expect("queued commits are active"); - if pending.remove(DIR_KEY) { - out.insert(DIR_KEY.to_owned(), info.id); - } - if pending.is_empty() { - break; - } + for (i, &parent) in commit.parents.iter().enumerate() { + if pending.is_clear() { + break; + } + let Some(parent) = node(repo, graph, parent) else { + warn!(%id, %parent, "last-commit parent unavailable"); + continue; + }; - if let Some(subtree) = current { - ensure_entries(repo, &mut entries, &mut stats.memo_entries, subtree); - } - if let Some(subtree) = parent { - ensure_entries(repo, &mut entries, &mut stats.memo_entries, subtree); + let same = process_parent(&diff, &paths, graph, &commit, &mut pending, i, parent.tree); + if same.is_clear() { + continue; + } + + match active.entry(parent.id) { + Entry::Occupied(mut entry) => entry.get_mut().pending.union_with(&same), + Entry::Vacant(entry) => { + queue.push(parent.queue_key()); + entry.insert(Active { + node: parent, + pending: same, + }); + } + } } - let current = current.and_then(|s| entries.get(&s)).unwrap_or(&empty); - let parent = parent.and_then(|s| entries.get(&s)).unwrap_or(&empty); - - let touched: Vec<&str> = pending - .iter() - .copied() - .filter(|name| current.get(*name) != parent.get(*name)) - .collect(); - for name in touched { - pending.remove(name); - out.insert(name.to_owned(), info.id); + + for k in pending.ones() { + out.insert(paths.paths[k].clone(), id); } } - if pending.is_empty() { + if out.len() == paths.paths.len() { stats.outcome = "resolved"; } - stats.memo_trees = entries.len(); out } + +#[cfg(test)] +mod tests { + use super::*; + use gix::bstr::B; + use std::path::Path; + use std::process::Command; + + fn git(dir: &Path, args: &[&str]) -> String { + let out = Command::new("git") + .args(args) + .current_dir(dir) + .env("GIT_AUTHOR_NAME", "t") + .env("GIT_AUTHOR_EMAIL", "t@t") + .env("GIT_COMMITTER_NAME", "t") + .env("GIT_COMMITTER_EMAIL", "t@t") + .output() + .expect("run git"); + assert!(out.status.success(), "git {args:?} failed"); + String::from_utf8(out.stdout).unwrap().trim().to_owned() + } + + fn commit(dir: &Path, file: &str, content: &str) -> ObjectId { + let path = dir.join(file); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, content).unwrap(); + git(dir, &["add", "-A"]); + git(dir, &["commit", "-q", "-m", file]); + head(dir) + } + + fn head(dir: &Path) -> ObjectId { + ObjectId::from_hex(git(dir, &["rev-parse", "HEAD"]).as_bytes()).unwrap() + } + + /// `(pathspec, max_depth, show_trees, expected)` + type Case<'a, K> = (&'a str, Option, bool, Vec<(K, ObjectId)>); + + fn assert_walks>(dir: &Path, cases: &[Case]) { + let head = head(dir); + let deadline = Instant::now() + BUDGET; + let check = |repo: &gix::Repository| { + for (pathspec, max_depth, show_trees, expected) in cases { + let expected: HashMap = expected + .iter() + .map(|(path, id)| (path.as_ref().into(), *id)) + .collect(); + assert_eq!( + walk(repo, head, pathspec, *max_depth, *show_trees, deadline), + expected, + "{pathspec:?} {max_depth:?} {show_trees}" + ); + } + }; + + check(&gix::open(dir).unwrap()); + + git( + dir, + &[ + "-c", + "commitGraph.changedPathsVersion=2", + "commit-graph", + "write", + "--reachable", + "--changed-paths", + ], + ); + let repo = gix::open(dir).unwrap(); + assert!( + repo.commit_graph() + .unwrap() + .bloom_filter_settings() + .is_some() + ); + check(&repo); + } + + #[test] + fn walk_linear_history() { + let dir = tempfile::tempdir().unwrap(); + let p = dir.path(); + git(p, &["init", "-q", "-b", "main"]); + let a = commit(p, "src/a", "1"); + commit(p, "docs/x", "1"); + let b = commit(p, "src/lib/b", "1"); + let x2 = commit(p, "docs/x", "2"); + let c = commit(p, "src/c", "1"); + let head = commit(p, "docs/y", "1"); + + let files: [(&str, _); 5] = [ + ("src/a", a), + ("src/lib/b", b), + ("src/c", c), + ("docs/x", x2), + ("docs/y", head), + ]; + let trees = [("src", c), ("src/lib", b), ("docs", head)]; + assert_walks( + p, + &[ + ( + "src", + Some(1), + true, + vec![("src", c), ("src/a", a), ("src/lib", b), ("src/c", c)], + ), + ("", Some(0), false, vec![("src", c), ("docs", head)]), + ("src", Some(0), false, vec![("src", c)]), + ("src/a", Some(0), false, vec![("src/a", a)]), + ( + "src/lib", + Some(1), + true, + vec![("src", c), ("src/lib", b), ("src/lib/b", b)], + ), + ( + "", + Some(1), + false, + vec![ + ("src/a", a), + ("src/lib", b), + ("src/c", c), + ("docs/x", x2), + ("docs/y", head), + ], + ), + ("", None, false, files.to_vec()), + ("", None, true, [&files[..], &trees[..]].concat()), + ("nope", None, true, vec![]), + ], + ); + } + + #[test] + fn walk_credits_merges_only_when_they_differ_from_every_parent() { + let dir = tempfile::tempdir().unwrap(); + let p = dir.path(); + git(p, &["init", "-q", "-b", "main"]); + let a = commit(p, "src/a", "1"); + git(p, &["switch", "-q", "-c", "side"]); + let s = commit(p, "src/s", "1"); + let d = commit(p, "docs/x", "1"); + git(p, &["switch", "-q", "main"]); + let m = commit(p, "src/m", "1"); + git(p, &["merge", "-q", "--no-ff", "-m", "merge", "side"]); + let merge = head(p); + + assert_walks( + p, + &[ + ( + "src", + Some(1), + true, + vec![("src", merge), ("src/a", a), ("src/s", s), ("src/m", m)], + ), + ("", Some(0), false, vec![("src", merge), ("docs", d)]), + ], + ); + } + + #[test] + fn walk_non_utf8_filename() { + let dir = tempfile::tempdir().unwrap(); + let p = dir.path(); + git(p, &["init", "-q", "-b", "main"]); + let first = commit(p, "a", "1"); + + // Add a `\xff` entry with plumbing; the filesystem may reject the name. + let blob = git(p, &["hash-object", "-w", "--stdin"]); + let mut listing = git(p, &["ls-tree", "HEAD"]).into_bytes(); + listing.extend_from_slice(format!("\n100644 blob {blob}\t").as_bytes()); + listing.extend_from_slice(b"\xff\n"); + let mut mktree = Command::new("git") + .arg("mktree") + .current_dir(p) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .spawn() + .unwrap(); + std::io::Write::write_all(&mut mktree.stdin.take().unwrap(), &listing).unwrap(); + let tree = String::from_utf8(mktree.wait_with_output().unwrap().stdout).unwrap(); + let tree = tree.trim(); + let added = git( + p, + &["commit-tree", tree, "-p", &first.to_string(), "-m", "ff"], + ); + git(p, &["update-ref", "HEAD", &added]); + git(p, &["reset", "-q"]); + let added = ObjectId::from_hex(added.as_bytes()).unwrap(); + + std::fs::write(p.join("a"), "2").unwrap(); + git(p, &["add", "a"]); + git(p, &["commit", "-q", "-m", "a"]); + let last = head(p); + + assert_walks( + p, + &[( + "", + Some(0), + false, + vec![(B("a"), last), (B(b"\xff"), added)], + )], + ); + } + + #[test] + fn walk_merge_undoes_changes() { + let dir = tempfile::tempdir().unwrap(); + let p = dir.path(); + git(p, &["init", "-q", "-b", "main"]); + commit(p, "file", "A"); + let b2 = commit(p, "file", "B"); + commit(p, "file", "C"); + commit(p, "file", "D"); + git(p, &["switch", "-q", "-c", "side", &b2.to_string()]); + let b5 = commit(p, "file2", "2"); + git(p, &["switch", "-q", "main"]); + git(p, &["merge", "-q", "--no-commit", "--no-ff", "side"]); + std::fs::write(p.join("file"), "B").unwrap(); + git(p, &["add", "file"]); + git(p, &["commit", "-q", "-m", "merge"]); + + assert_walks(p, &[("", None, false, vec![("file2", b5), ("file", b2)])]); + } +} diff --git a/gitmirror/crates/gitmirror-xrpc/src/lib.rs b/gitmirror/crates/gitmirror-xrpc/src/lib.rs index b0df251fe..0ecd0a66b 100644 --- a/gitmirror/crates/gitmirror-xrpc/src/lib.rs +++ b/gitmirror/crates/gitmirror-xrpc/src/lib.rs @@ -33,6 +33,7 @@ pub struct AppState { pub(crate) knot_policy: KnotPolicy, pub(crate) clock: Arc, pub(crate) stats: Arc, + #[allow(unused)] pub(crate) redis: Option, pub(crate) max_blob_bytes: u64, pub(crate) max_archive_bytes: u64, diff --git a/gitmirror/crates/gitmirror-xrpc/src/routes/reads.rs b/gitmirror/crates/gitmirror-xrpc/src/routes/reads.rs index 9951a42ea..f20ca80ea 100644 --- a/gitmirror/crates/gitmirror-xrpc/src/routes/reads.rs +++ b/gitmirror/crates/gitmirror-xrpc/src/routes/reads.rs @@ -7,7 +7,7 @@ use axum::extract::State; use axum::http::{HeaderMap, StatusCode, header}; use axum::response::{IntoResponse as _, Response}; use gix::ObjectId; -use gix::bstr::ByteSlice as _; +use gix::bstr::{BString, ByteSlice as _}; use gix::objs::tree::EntryKind; use jacquard_axum::{ExtractXrpc, XrpcResponse}; use jacquard_common::ToSmolStr; @@ -133,8 +133,8 @@ fn hydrate(repo: &gix::Repository, oid: ObjectId) -> Result, -) -> HashMap { + found: &HashMap, +) -> HashMap { let mut commits: HashMap = HashMap::new(); found .iter() @@ -155,32 +155,26 @@ fn hydrate_all( } async fn last_commits( - state: &AppState, repo: gix::Repository, head: ObjectId, - tree_path: String, - names: Vec, -) -> Result, XrpcError> { - let pool = state.redis.clone(); - let cached = last_commit::load(pool.as_ref(), &head, &tree_path, &names).await; - let missing: Vec = names - .into_iter() - .filter(|name| !cached.contains_key(name)) - .collect(); - + pathspec: String, + max_depth: usize, + show_trees: bool, +) -> Result, XrpcError> { let deadline = Instant::now() + last_commit::BUDGET; - let path = tree_path.clone(); - let (walked, hydrated) = crate::metrics::blocking(BlockingKind::Git, move || { - let walked = last_commit::walk(&repo, head, &path, &missing, deadline); - let mut found = cached; - found.extend(walked.iter().map(|(name, oid)| (name.clone(), *oid))); - (walked, hydrate_all(&repo, &found)) + crate::metrics::blocking(BlockingKind::Git, move || { + let found = last_commit::walk( + &repo, + head, + &pathspec, + Some(max_depth), + show_trees, + deadline, + ); + hydrate_all(&repo, &found) }) .await - .map_err(|e| XrpcError::Internal(e.to_string()))?; - - last_commit::store(pool.as_ref(), &head, &tree_path, &walked).await; - Ok(hydrated) + .map_err(|e| XrpcError::Internal(e.to_string())) } pub(crate) async fn get_merge_base( State(state): State, @@ -342,10 +336,10 @@ pub(crate) async fn get_entry( .await .map_err(|e| XrpcError::Internal(e.to_string()))??; - let (dir, name) = split_path(&args.path); - output.last_commit = last_commits(&state, repo, head, dir, vec![name.clone()]) + let path = args.path.trim_matches('/').to_owned(); + output.last_commit = last_commits(repo, head, path.clone(), 0, false) .await? - .remove(&name); + .remove(path.as_bytes().as_bstr()); Ok((cache_header(cache), XrpcResponse(output))) } @@ -372,7 +366,7 @@ pub(crate) async fn get_tree( .to_owned(); let path = tree_path.clone(); - let (repo, head, mut entries) = crate::metrics::blocking(BlockingKind::Git, move || { + let (repo, head, entries) = crate::metrics::blocking(BlockingKind::Git, move || { let head = resolve_commit(&repo, &refspec)?; let entries = { let root = repo @@ -392,7 +386,7 @@ pub(crate) async fn get_tree( .map_err(|_| XrpcError::PathNotFound { path: path.clone() })? }; - let mut entries: Vec = Vec::new(); + let mut entries = Vec::new(); for entry in subtree.iter() { let entry = entry.map_err(|e| XrpcError::Internal(e.to_string()))?; let kind = entry.mode().kind(); @@ -400,13 +394,17 @@ pub(crate) async fn get_tree( EntryKind::Tree | EntryKind::Commit => 0, _ => blob_size(&repo, entry.object_id())? as i64, }; - entries.push(get_tree::TreeEntry { - name: entry.filename().to_str_lossy().as_ref().into(), - mode: mode_of(kind).as_str().into(), - size, - last_commit: None, - extra_data: Default::default(), - }); + let name = entry.filename().to_owned(); + entries.push(( + get_tree::TreeEntry { + name: name.to_str_lossy().as_ref().into(), + mode: mode_of(kind).as_str().into(), + size, + last_commit: None, + extra_data: Default::default(), + }, + name, + )); } entries }; @@ -415,19 +413,24 @@ pub(crate) async fn get_tree( .await .map_err(|e| XrpcError::Internal(e.to_string()))??; - let mut names: Vec = vec![last_commit::DIR_KEY.to_owned()]; - names.extend(entries.iter().map(|entry| entry.name.to_string())); - let mut found = last_commits(&state, repo, head, tree_path, names).await?; + // `git last-modified -t --max-depth=1 -- `; without a pathspec git counts depth from the + // top-level entries, so the root is `--max-depth=0` and reports no commit for itself. + let max_depth = if tree_path.is_empty() { 0 } else { 1 }; + let mut found = last_commits(repo, head, tree_path.clone(), max_depth, true).await?; - for entry in &mut entries { - entry.last_commit = found.remove(entry.name.as_str()); - } + let entries = entries + .into_iter() + .map(|(mut entry, name)| { + entry.last_commit = found.remove(&last_commit::join(tree_path.as_str(), name)); + entry + }) + .collect(); Ok(( cache_header(cache), XrpcResponse(get_tree::GetTreeOutput { entries, - last_commit: found.remove(last_commit::DIR_KEY), + last_commit: found.remove(tree_path.as_bytes().as_bstr()), extra_data: Default::default(), }), ))