diff --git a/knot2/crates/knot-git/src/bitmap/bitset.rs b/knot2/crates/knot-git/src/bitmap/bitset.rs index c03118775..a717bc402 100644 --- a/knot2/crates/knot-git/src/bitmap/bitset.rs +++ b/knot2/crates/knot-git/src/bitmap/bitset.rs @@ -37,6 +37,26 @@ impl Bitset { self.words[index / 64] |= 1u64 << (index % 64); } + pub(crate) fn get(&self, index: BitPosition) -> bool { + let index = index.get() as usize; + self.words[index / 64] & (1u64 << (index % 64)) != 0 + } + + pub(crate) fn union_ewah(&mut self, vector: &gix_bitmap::ewah::Vec) -> Result<(), GitError> { + let len = self.words.len() * 64; + vector + .for_each_set_bit(|index| { + (index < len).then(|| self.set(BitPosition::new(index as u32))) + }) + .ok_or_else(|| GitError::Backend("malformed ewah bitmap".to_string())) + } + + pub(crate) fn to_bools(&self, num_bits: usize) -> Vec { + (0..num_bits) + .map(|index| self.get(BitPosition::new(index as u32))) + .collect() + } + pub(crate) fn union_with(&mut self, other: &Bitset) { self.words .iter_mut() diff --git a/knot2/crates/knot-git/src/bitmap/writer.rs b/knot2/crates/knot-git/src/bitmap/writer.rs index b16f12da2..6def750c3 100644 --- a/knot2/crates/knot-git/src/bitmap/writer.rs +++ b/knot2/crates/knot-git/src/bitmap/writer.rs @@ -1,13 +1,14 @@ -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::path::Path; use gix_bitmap::ewah::Vec as Ewah; -use knot_types::Oid; +use knot_types::{Oid, UnixSeconds}; +use super::Bitset; use super::revindex::{Order, OrderTable}; use super::{BitPosition, BitmapEntryOffset, IndexPosition}; use crate::error::GitError; -use crate::objects::{Haves, Wants}; +use crate::objects::MAX_TREE_DEPTH; use crate::repo::Repo; const OPT_FULL_DAG: u16 = 0x1; @@ -100,40 +101,24 @@ fn object_category(repo: &Repo, oid: Oid) -> Result { } } -fn one_selected( - repo: &Repo, - rev: &R, - commit_pos: IndexPosition, - commit: Oid, -) -> Result { - let closure = repo.select_pack_objects(Wants::new(&[commit]), Haves::new(&[]))?; - Ok(Selected { - commit_pos, - bits: ewah(&closure_bits(rev, &closure)?)?, - }) -} - -pub(super) fn selected_entries( - repo: &Repo, - rev: &R, -) -> Result, GitError> { +pub(super) fn selected_entries(repo: &Repo, rev: &R) -> Result, GitError> { let mut seen: HashSet = HashSet::new(); - let commits: Vec<(IndexPosition, Oid)> = repo + let mut tips: Vec<(UnixSeconds, IndexPosition, Oid)> = repo .references()? .into_iter() .filter_map(|record| peel_to_commit(repo, record.target)) .filter_map(|commit| rev.index_of(commit).map(|position| (position, commit))) .filter(|(_, commit)| seen.insert(*commit)) - .collect(); - - let path = repo.path().to_owned(); - let mut entries: Vec = knot_resource::map_chunks(&commits, |batch| { - let local = Repo::open(&path)?; - batch - .iter() - .map(|(commit_pos, commit)| one_selected(&local, rev, *commit_pos, *commit)) - .collect::, GitError>>() - })?; + .map(|(position, commit)| Ok((repo.commit_time(commit.object_id())?, position, commit))) + .collect::>()?; + tips.sort_by_key(|(time, _, _)| *time); + + let mut built: HashMap = HashMap::new(); + for (_, commit_pos, commit) in tips { + let bits = ewah(&closure_bits(repo, rev, commit, &built)?.to_bools(rev.len()))?; + built.insert(commit, Selected { commit_pos, bits }); + } + let mut entries: Vec = built.into_values().collect(); entries.sort_by_key(|entry| entry.commit_pos); Ok(entries) } @@ -150,17 +135,83 @@ fn peel_to_commit(repo: &Repo, oid: Oid) -> Option { } } -fn closure_bits(rev: &impl Order, closure: &[Oid]) -> Result, GitError> { - closure - .iter() - .try_fold(vec![false; rev.len()], |mut bits, oid| { - let position = rev.index_of(*oid).ok_or_else(|| { - GitError::Backend(format!( - "closure object {oid} is absent from the pack bitmap" - )) - })?; - bits[rev.bit_at_index(position).get() as usize] = true; - Ok(bits) +fn closure_bits( + repo: &Repo, + rev: &R, + tip: Oid, + built: &HashMap, +) -> Result { + let mut bits = Bitset::zeros(rev.len()); + let mut commits = vec![tip]; + let mut trees = Vec::new(); + while let Some(commit) = commits.pop() { + let bit = bit_of(rev, commit)?; + if bits.get(bit) { + continue; + } + if let Some(ancestor) = built.get(&commit) { + bits.union_ewah(&ancestor.bits)?; + continue; + } + bits.set(bit); + let (tree, parents) = repo.commit_tree_and_parents(commit)?; + trees.push(tree); + commits.extend(parents.into_iter().map(Oid::from)); + } + trees + .into_iter() + .try_for_each(|tree| fill_tree(repo, rev, tree, &mut bits, MAX_TREE_DEPTH))?; + Ok(bits) +} + +fn fill_tree( + repo: &Repo, + rev: &R, + tree: gix::ObjectId, + bits: &mut Bitset, + nesting: usize, +) -> Result<(), GitError> { + if nesting == 0 { + return Err(GitError::DepthExceeded("tree nesting")); + } + let oid = Oid::from(tree); + if tree == gix::ObjectId::empty_tree(tree.kind()) && rev.index_of(oid).is_none() { + return Ok(()); + } + let bit = bit_of(rev, oid)?; + if bits.get(bit) { + return Ok(()); + } + bits.set(bit); + let object = repo + .git() + .find_tree(tree) + .map_err(|error| GitError::Corrupt { + oid, + message: error.to_string(), + })?; + object.iter().try_for_each(|entry| { + let entry = entry.map_err(|error| GitError::Decode(error.to_string()))?; + match entry.mode().kind() { + gix::objs::tree::EntryKind::Tree => { + fill_tree(repo, rev, entry.object_id(), bits, nesting - 1) + } + gix::objs::tree::EntryKind::Commit => Ok(()), + _ => { + bits.set(bit_of(rev, Oid::from(entry.object_id()))?); + Ok(()) + } + } + }) +} + +fn bit_of(rev: &impl Order, oid: Oid) -> Result { + rev.index_of(oid) + .map(|position| rev.bit_at_index(position)) + .ok_or_else(|| { + GitError::Backend(format!( + "closure object {oid} is absent from the pack bitmap" + )) }) } @@ -225,58 +276,3 @@ pub(super) fn install(path: &Path, bytes: &[u8]) -> Result<(), GitError> { |error| GitError::Maintenance(format!("write bitmap {}: {}", path.display(), error.source)), ) } - -#[cfg(test)] -mod tests { - use super::super::{BitPosition, IndexPosition}; - use super::*; - - struct FakeOrder { - oids: Vec, - } - - impl Order for FakeOrder { - fn len(&self) -> usize { - self.oids.len() - } - - fn index_of(&self, oid: Oid) -> Option { - self.oids - .iter() - .position(|candidate| *candidate == oid) - .map(|position| IndexPosition::new(position as u32)) - } - - fn bit_at_index(&self, position: IndexPosition) -> BitPosition { - BitPosition::new(position.get()) - } - - fn oid_at_bit(&self, bit: BitPosition) -> Oid { - self.oids[bit.get() as usize] - } - } - - fn oid(byte: u8) -> Oid { - Oid::from_hex(&format!("{byte:02x}").repeat(20)).unwrap() - } - - #[test] - fn closure_bits_sets_one_bit_per_present_object() { - let order = FakeOrder { - oids: vec![oid(1), oid(2), oid(3)], - }; - let bits = closure_bits(&order, &[oid(1), oid(3)]).unwrap(); - assert_eq!(bits, vec![true, false, true]); - } - - #[test] - fn closure_bits_errors_when_an_object_is_outside_the_pack() { - let order = FakeOrder { - oids: vec![oid(1), oid(2)], - }; - assert!( - closure_bits(&order, &[oid(1), oid(9)]).is_err(), - "an incomplete closure must fail the bitmap rather than write a partial one" - ); - } -} diff --git a/knot2/crates/knot-git/src/objects.rs b/knot2/crates/knot-git/src/objects.rs index 6c100ae0d..a7d024d3b 100644 --- a/knot2/crates/knot-git/src/objects.rs +++ b/knot2/crates/knot-git/src/objects.rs @@ -1008,7 +1008,7 @@ impl Repo { } } - fn commit_time(&self, commit: gix::ObjectId) -> Result { + pub(crate) fn commit_time(&self, commit: gix::ObjectId) -> Result { let object = self.load_object(Oid::from(commit))?; let commit = object.try_into_commit().map_err(|_| GitError::ObjectType { oid: Oid::from(commit), diff --git a/knot2/crates/knot-git/tests/bitmap.rs b/knot2/crates/knot-git/tests/bitmap.rs index a4f16c63b..0fe4d72e2 100644 --- a/knot2/crates/knot-git/tests/bitmap.rs +++ b/knot2/crates/knot-git/tests/bitmap.rs @@ -35,6 +35,13 @@ fn seed(format: &str) -> tempfile::TempDir { commit(path, "c.txt", "gamma\n", "feature work"); ok(path, &["checkout", "-q", "main"]); commit(path, "d.txt", "delta\n", "more main"); + ok(path, &["checkout", "-q", "-b", "topic"]); + commit(path, "t.txt", "topic\n", "topic work"); + ok(path, &["checkout", "-q", "main"]); + ok( + path, + &["merge", "-q", "--no-ff", "-m", "merge topic", "topic"], + ); ok(path, &["tag", "-a", "v1", "-m", "release one"]); ok(path, &["repack", "-adq"]); dir