diff --git a/knot2/crates/knot-bench/benches/coldstart.rs b/knot2/crates/knot-bench/benches/coldstart.rs index b9fccc44f..44cb3d1a3 100644 --- a/knot2/crates/knot-bench/benches/coldstart.rs +++ b/knot2/crates/knot-bench/benches/coldstart.rs @@ -36,6 +36,18 @@ fn main() { #[divan::bench(args = repo_counts())] fn boot(bencher: Bencher, repos: u64) { let registry = build_registry(RepoCount::new(repos)); + bencher + .counter(ItemsCount::new(registry.dids().len())) + .bench_local(|| { + registry.clear_registry_checkpoint(); + registry.index().rebuild().unwrap(); + }); +} + +#[divan::bench(args = repo_counts())] +fn boot_checkpointed(bencher: Bencher, repos: u64) { + let registry = build_registry(RepoCount::new(repos)); + registry.index().rebuild().unwrap(); bencher .counter(ItemsCount::new(registry.dids().len())) .bench_local(|| registry.index().rebuild().unwrap()); diff --git a/knot2/crates/knot-bench/benches/projection.rs b/knot2/crates/knot-bench/benches/projection.rs index fc51b70aa..ef58ab372 100644 --- a/knot2/crates/knot-bench/benches/projection.rs +++ b/knot2/crates/knot-bench/benches/projection.rs @@ -4,8 +4,8 @@ use knot_bench::{ GrantListCount, IssueCount, RepoCount, RepoIndex, build_collaborator_grant_list, build_issue_tracker, build_registry, }; -use knot_cob::CHECKPOINTS_PREFIX; -use knot_types::RefName; +use knot_cob::checkpoint_ref_name; +use knot_types::TypeName; #[global_allocator] static ALLOC: divan::AllocProfiler = divan::AllocProfiler::system(); @@ -53,11 +53,10 @@ fn social_boot(bencher: Bencher, issues: u32) { assert_eq!( built.checkpoints(), vec![ - RefName::new(format!( - "{CHECKPOINTS_PREFIX}{}/{}", - knot_record::ticket::TICKET_COLLECTION, - edited.oid().to_hex() - )) + checkpoint_ref_name( + &TypeName::new(knot_record::ticket::TICKET_COLLECTION).expect("ticket type name"), + edited + ) .expect("checkpoint ref name") ], "the checkpoint path oughtta apply per object: the one edited past the stride, and no other" diff --git a/knot2/crates/knot-bench/src/fixtures.rs b/knot2/crates/knot-bench/src/fixtures.rs index dacd6c63f..61a802b68 100644 --- a/knot2/crates/knot-bench/src/fixtures.rs +++ b/knot2/crates/knot-bench/src/fixtures.rs @@ -1,6 +1,6 @@ use std::path::{Path, PathBuf}; -use knot_cob::{CobError, CobHome, CobStore}; +use knot_cob::{ChangePayload, CobError, CobHome, CobStore}; use knot_cobs::{ CollaboratorsChange, Grant, MembersChange, MembersCob, Registration, RegistryChange, RepoRegistryCob, register_repo, @@ -268,6 +268,7 @@ pub struct BuiltRegistry { _dir: TempDir, meta_path: PathBuf, layout: Layout, + registry: CobId, dids: Vec, } @@ -282,6 +283,15 @@ impl BuiltRegistry { let reg = registration(index); (reg.owner, reg.rkey) } + pub fn clear_registry_checkpoint(&self) { + let meta = Repo::open(&self.meta_path).expect("openable meta path"); + let name = knot_cob::checkpoint_ref_name(&RegistryChange::type_name(), self.registry) + .expect("checkpoint ref name"); + if let Some(old) = meta.find_ref(&name).expect("able to read checkpoint ref") { + meta.update_ref(&RefUpdate::Delete { name, old }) + .expect("able to delete checkpoint ref"); + } + } } fn seed_members(meta_path: &Path, signer: &dyn knot_runtime::Signer) { @@ -351,6 +361,7 @@ pub fn build_registry(repos: RepoCount) -> BuiltRegistry { _dir: dir, meta_path, layout, + registry: registry_object, dids, } } @@ -459,14 +470,8 @@ impl BuiltTracker { }, body: ticket(knot_record::ticket::TicketState::Open), }; - let approved = knot_cob::Approved::signed_by( - &cx, - knot_cob::Materialized { - change, - record: knot_cob::Record::Bare, - }, - ) - .expect("the owner signs its own change"); + let approved = knot_cob::Approved::signed_by(&cx, knot_cob::Materialized::bare(change)) + .expect("the owner signs its own change"); store .append_approved::( &CobHome::from(&self.repo), diff --git a/knot2/crates/knot-cob/src/backend.rs b/knot2/crates/knot-cob/src/backend.rs index 0e2107887..4fa9d001b 100644 --- a/knot2/crates/knot-cob/src/backend.rs +++ b/knot2/crates/knot-cob/src/backend.rs @@ -47,10 +47,7 @@ pub fn cob_ref_name(type_name: &TypeName, object: CobId) -> Result Result { +pub fn checkpoint_ref_name(type_name: &TypeName, object: CobId) -> Result { prefixed_ref_name(CHECKPOINTS_PREFIX, type_name, object) } diff --git a/knot2/crates/knot-cob/src/lib.rs b/knot2/crates/knot-cob/src/lib.rs index 3ea809836..861b678a8 100644 --- a/knot2/crates/knot-cob/src/lib.rs +++ b/knot2/crates/knot-cob/src/lib.rs @@ -16,7 +16,8 @@ pub use object::{ }; pub use backend::{ - CHECKPOINTS_PREFIX, COBS_PREFIX, cob_ref_name, migrate_legacy_refs, parse_cob_ref, + CHECKPOINTS_PREFIX, COBS_PREFIX, checkpoint_ref_name, cob_ref_name, migrate_legacy_refs, + parse_cob_ref, }; const LEGACY_TYPE_ALIASES: [(&str, &str); 4] = [ @@ -32,7 +33,7 @@ const LEGACY_TYPE_ALIASES: [(&str, &str); 4] = [ ), ]; -pub fn legacy_type_of(modern: &TypeName) -> Option { +pub(crate) fn legacy_type_of(modern: &TypeName) -> Option { LEGACY_TYPE_ALIASES .iter() .find(|(_, modern_name)| modern.as_str() == *modern_name) @@ -46,7 +47,7 @@ pub fn modernized_type_of(legacy: &TypeName) -> Option { .and_then(|(_, modern)| TypeName::new(*modern).ok()) } -pub fn accepts_change_type(expected: &TypeName, found: &TypeName) -> bool { +pub(crate) fn accepts_change_type(expected: &TypeName, found: &TypeName) -> bool { found == expected || legacy_type_of(expected).is_some_and(|legacy| &legacy == found) } @@ -62,11 +63,33 @@ const MAX_CAS_RETRIES: usize = 16; const CHECKPOINT_GROWTH_DIVISOR: usize = 16; // don't ask -fn checkpoint_stride(snapshot_stride: SnapshotStride, size: StateSize) -> usize { - snapshot_stride - .get() - .max(size.get() / CHECKPOINT_GROWTH_DIVISOR) - .min(backend::MAX_GRAPH_CHANGES) +knot_types::scalar_newtype! { + pub(crate) struct Tail(usize); +} + +impl Tail { + fn after_append(self) -> Self { + Self(self.0.saturating_add(1)) + } + + fn reaches(self, stride: SnapshotStride) -> bool { + self.0 >= stride.get() + } +} + +fn checkpoint_stride(snapshot_stride: SnapshotStride, size: StateSize) -> SnapshotStride { + SnapshotStride::new( + snapshot_stride + .get() + .max(size.get() / CHECKPOINT_GROWTH_DIVISOR) + .clamp(1, backend::MAX_GRAPH_CHANGES), + ) +} + +struct Recovered { + state: S, + tip: ChangeId, + tail: Tail, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -192,17 +215,7 @@ impl<'r> CobStore<'r> { ) } - pub fn create_materialized( - &self, - home: &CobHome, - payload: &Materialized

, - signer: &dyn Signer, - timestamp: UnixSeconds, - ) -> Result { - self.open(home, payload, signer, timestamp) - } - - fn open( + pub fn create_materialized( &self, home: &CobHome, payload: &Materialized

, @@ -234,7 +247,9 @@ impl<'r> CobStore<'r> { { let cx = approved.context(); match E::may_create(cx) { - Decision::Allow => self.open(home, &approved.materialized, signer, timestamp), + Decision::Allow => { + self.create_materialized(home, &approved.materialized, signer, timestamp) + } Decision::Deny => Err(CobError::CreateRejected { actor: cx.actor().clone(), type_name: E::Change::type_name(), @@ -317,6 +332,12 @@ impl<'r> CobStore<'r> { } } + fn current_tip(&self, type_name: &TypeName, object: CobId) -> Result { + backend::resolve_tip(self.repo, type_name, object)? + .map(ChangeId::new) + .ok_or(CobError::NoSuchObject(object)) + } + pub fn update( &self, home: &CobHome, @@ -326,12 +347,16 @@ impl<'r> CobStore<'r> { timestamp: UnixSeconds, ) -> Result { let type_name = P::type_name(); - let expected = backend::resolve_tip(self.repo, &type_name, object)? - .map(ChangeId::new) - .ok_or(CobError::NoSuchObject(object))?; - self.append( - home, object, &type_name, expected, payload, signer, timestamp, + let expected = self.current_tip(&type_name, object)?; + self.chain( + home, + object, + &type_name, + expected, + std::iter::once((payload, &Record::Bare, timestamp)), + signer, ) + .map(|tip| tip.expect("one change chains onto one tip")) } pub fn update_materialized( &self, @@ -342,9 +367,7 @@ impl<'r> CobStore<'r> { timestamp: UnixSeconds, ) -> Result { let type_name = P::type_name(); - let expected = backend::resolve_tip(self.repo, &type_name, object)? - .map(ChangeId::new) - .ok_or(CobError::NoSuchObject(object))?; + let expected = self.current_tip(&type_name, object)?; self.append_materialized( home, object, &type_name, expected, payload, signer, timestamp, ) @@ -358,9 +381,7 @@ impl<'r> CobStore<'r> { signer: &dyn Signer, ) -> Result, CobError> { let type_name = P::type_name(); - let expected = backend::resolve_tip(self.repo, &type_name, object)? - .map(ChangeId::new) - .ok_or(CobError::NoSuchObject(object))?; + let expected = self.current_tip(&type_name, object)?; self.chain( home, object, @@ -373,28 +394,6 @@ impl<'r> CobStore<'r> { ) } - #[allow(clippy::too_many_arguments)] - fn append( - &self, - home: &CobHome, - object: CobId, - type_name: &TypeName, - expected: ChangeId, - payload: &P, - signer: &dyn Signer, - timestamp: UnixSeconds, - ) -> Result { - self.chain( - home, - object, - type_name, - expected, - std::iter::once((payload, &Record::Bare, timestamp)), - signer, - ) - .map(|tip| tip.expect("one change chains onto one tip")) - } - #[allow(clippy::too_many_arguments)] fn append_materialized( &self, @@ -660,8 +659,14 @@ impl<'r> CobStore<'r> { match decide(&state)? { None => Ok(Some(None)), Some(change) => { - match self.append( - home, object, &type_name, expected, &change, signer, timestamp, + match self.append_materialized( + home, + object, + &type_name, + expected, + &Materialized::bare(change), + signer, + timestamp, ) { Ok(id) => Ok(Some(Some(id))), Err(CobError::StaleTip { .. }) => Ok(None), @@ -707,10 +712,7 @@ impl<'r> CobStore<'r> { owner: &ActorId, ) -> Result<(), CobError> { let type_name = E::Change::type_name(); - let tip = ChangeId::new( - backend::resolve_tip(self.repo, &type_name, object)? - .ok_or(CobError::NoSuchObject(object))?, - ); + let tip = self.current_tip(&type_name, object)?; self.verify_type_from_tip(home, &type_name, object, tip, E::HISTORY, owner) } @@ -755,9 +757,7 @@ impl<'r> CobStore<'r> { history: HistoryModel, since: Option, ) -> Result { - let tip = backend::resolve_tip(self.repo, type_name, object)? - .map(ChangeId::new) - .ok_or(CobError::NoSuchObject(object))?; + let tip = self.current_tip(type_name, object)?; let collected = backend::collect(self.repo, tip, object, backend::MAX_GRAPH_CHANGES, since)?; match since { @@ -817,23 +817,6 @@ impl<'r> CobStore<'r> { }) } - pub fn update_materialized_checkpointed( - &self, - home: &CobHome, - object: CobId, - signer: &dyn Signer, - timestamp: UnixSeconds, - decide: impl Fn(&E::State) -> Result>, D>, - ) -> Result, D> - where - E: Checkpoint, - E::Change: KnotAuthored, - E::State: Serialize + DeserializeOwned, - D: From, - { - self.retrying_append::(home, object, signer, timestamp, decide) - } - fn retrying_append( &self, home: &CobHome, @@ -849,7 +832,11 @@ impl<'r> CobStore<'r> { { let type_name = E::Change::type_name(); let attempt = || -> Result>, D> { - let (state, expected, suffix) = self.checkpointed_state::(object)?; + let Recovered { + state, + tip: expected, + tail, + } = self.checkpointed_state::(object)?; let Some(materialized) = decide(&state)? else { return Ok(Some(None)); }; @@ -864,7 +851,7 @@ impl<'r> CobStore<'r> { ) { Ok(tip) => { let stride = checkpoint_stride(E::SNAPSHOT_STRIDE, E::checkpoint_size(&state)); - if suffix.saturating_add(1) >= stride { + if tail.after_append().reaches(stride) { let author = ActorId::from_secp256k1(signer.public_key().as_bytes()); let applied = E::apply(state, materialized.change, &author); if let Err(error) = self.write_checkpoint::(object, tip, &applied) { @@ -893,26 +880,32 @@ impl<'r> CobStore<'r> { E::State: Serialize + DeserializeOwned, { self.checkpointed_state::(object) - .map(|(state, tip, _)| (state, tip)) + .map(|Recovered { state, tip, .. }| (state, tip)) } - fn checkpointed_state(&self, object: CobId) -> Result<(E::State, ChangeId, usize), CobError> + fn checkpointed_state(&self, object: CobId) -> Result, CobError> where E: Checkpoint, E::State: Serialize + DeserializeOwned, { let type_name = E::Change::type_name(); - let tip = backend::resolve_tip(self.repo, &type_name, object)? - .map(ChangeId::new) - .ok_or(CobError::NoSuchObject(object))?; + let tip = self.current_tip(&type_name, object)?; match self.load_checkpoint::(object)? { - Some((checkpoint_tip, state)) if checkpoint_tip == tip => Ok((state, tip, 0)), + Some((checkpoint_tip, state)) if checkpoint_tip == tip => Ok(Recovered { + state, + tip, + tail: Tail(0), + }), Some((checkpoint_tip, state)) => { match self.changes_since::(object, Some(checkpoint_tip)) { Ok(delta) => { let applied = object::apply_changes::(state, &delta.changes, &type_name)?; - Ok((applied, tip, delta.changes.len())) + Ok(Recovered { + state: applied, + tip: delta.tip, + tail: Tail(delta.changes.len()), + }) } Err( unusable @ (CobError::DivergedTip { .. } @@ -923,28 +916,61 @@ impl<'r> CobStore<'r> { error = %unusable, "checkpoint no longer bounds this object's history, the replay starts over from its root change" ); - self.full_state::(object) - .map(|state| (state, tip, usize::MAX)) + self.recovered_state::(object) } Err(error) => Err(error), } } - None => self - .full_state::(object) - .map(|state| (state, tip, usize::MAX)), + None => self.recovered_state::(object), } } - fn full_state(&self, object: CobId) -> Result { + fn recovered_state(&self, object: CobId) -> Result, CobError> + where + E: Checkpoint, + E::State: Serialize + DeserializeOwned, + { let type_name = E::Change::type_name(); - let (graph, _tip) = backend::load_graph( + let (graph, tip) = backend::load_graph( self.repo, &type_name, object, E::HISTORY, backend::rebuild_graph_limit(), )?; - object::evaluate::(graph, &type_name).map(|(state, _history)| state) + let replay = Recovered { + tail: Tail(graph.len()), + tip, + state: object::evaluate::(graph, &type_name)?.0, + }; + let stride = checkpoint_stride(E::SNAPSHOT_STRIDE, E::checkpoint_size(&replay.state)); + if !replay.tail.reaches(stride) { + return Ok(replay); + } + match self.write_checkpoint::(object, replay.tip, &replay.state) { + Ok(()) => { + tracing::info!( + cob = type_name.as_str(), + object = %object.oid().to_hex(), + replayed = replay.tail.get(), + stride = stride.get(), + "replayed thru all state and saved a checkpoint!" + ); + Ok(Recovered { + tail: Tail(0), + ..replay + }) + } + Err(error) => { + tracing::warn!( + cob = type_name.as_str(), + object = %object.oid().to_hex(), + %error, + "failure to save a replay checkpoint" + ); + Ok(replay) + } + } } fn load_checkpoint(&self, object: CobId) -> Result, CobError> @@ -1115,6 +1141,15 @@ mod tests { } } + fn claim_unless_taken( + who: &str, + ) -> impl Fn(&BTreeSet) -> Result, NameError> + '_ { + move |state| match state.contains(who) { + true => Err(NameError::Taken), + false => Ok(Some(NameChange::Claim(who.to_string()))), + } + } + fn claim( store: &CobStore, object: CobId, @@ -1127,10 +1162,7 @@ mod tests { object, key, when, - |state| match state.contains(who) { - true => Err(NameError::Taken), - false => Ok(Some(NameChange::Claim(who.to_string()))), - }, + claim_unless_taken(who), ) } @@ -1143,6 +1175,44 @@ mod tests { } } + fn plain_claim( + store: &CobStore, + object: CobId, + key: &K256Signer, + who: &str, + when: UnixSeconds, + ) -> ChangeId { + appended(store.update_maybe::( + &cob_home(), + object, + key, + when, + claim_unless_taken(who), + )) + } + + fn plain_history(store: &CobStore, key: &K256Signer, changes: usize) -> CobId { + let object = store + .create( + &cob_home(), + &NameChange::Claim("name0000".into()), + key, + at(0), + ) + .unwrap() + .object; + (1..changes).for_each(|index| { + plain_claim( + store, + object, + key, + &format!("name{index:04}"), + at(index as i64), + ); + }); + object + } + fn fixture() -> (TempDir, Repo) { let dir = tempfile::tempdir().unwrap(); let layout = Layout::new(dir.path()); @@ -1336,24 +1406,24 @@ mod tests { fn checkpoint_stride_grows_with_state_so_large_cobs_snapshot_less_often() { assert_eq!( checkpoint_stride(SnapshotStride::new(256), StateSize::new(0)), - 256 + SnapshotStride::new(256) ); assert_eq!( checkpoint_stride( SnapshotStride::new(256), StateSize::new(256 * CHECKPOINT_GROWTH_DIVISOR) ), - 256, + SnapshotStride::new(256), "up to stride*divisor entries the fixed floor governs, so small cobs snapshot exactly as before" ); assert_eq!( checkpoint_stride(SnapshotStride::new(256), StateSize::new(1_600_000)), - 100_000, + SnapshotStride::new(100_000), "a large cob snapshots at a fixed fraction of its size, so total snapshot bytes stay linear in the change count" ); assert_eq!( checkpoint_stride(SnapshotStride::new(256), StateSize::new(4_000_000)), - backend::MAX_GRAPH_CHANGES, + SnapshotStride::new(backend::MAX_GRAPH_CHANGES), "the stride stops at the serving limit so the boot tail always replays incrementally, never overflowing into a full rebuild" ); } @@ -1391,11 +1461,13 @@ mod tests { "a full replay still counts every write after adaptive snapshotting" ); - let (_state, _tip, suffix) = store.checkpointed_state::(object).unwrap(); + let Recovered { tail, .. } = store.checkpointed_state::(object).unwrap(); let bound = checkpoint_stride(Names::SNAPSHOT_STRIDE, StateSize::new(total)); assert!( - suffix <= bound && bound < total, - "boot replays only the {suffix}-change tail since the last snapshot, bounded by the adaptive stride {bound}, never the whole {total}-change history" + tail.get() <= bound.get() && bound.get() < total, + "at boot time we'll only replay from the {:?} changes since last snapshot, bounded by stride of {:?}, not thru history of all {total}" + tail, + bound ); } @@ -1468,7 +1540,6 @@ mod tests { matches!(duplicate, Err(NameError::Taken)), "corrupt checkpoint falls back to real state instead of waving a duplicate through" ); - appended(claim(&store, object, &key, "intruder", at(101))); let healed = repo.find_ref(&snapshot).unwrap().unwrap(); let bytes = repo.read_blob(healed).unwrap(); @@ -1487,9 +1558,157 @@ mod tests { store.get::(object).unwrap().state(), "healed snapshot matches the full replay" ); + + appended(claim(&store, object, &key, "intruder", at(101))); + let Recovered { tail, .. } = store.checkpointed_state::(object).unwrap(); + assert_eq!( + tail, + Tail(1), + "healed checkpoint should only be as bounded as tail after boot" + ); }); } + #[test] + fn replay_saves_a_single_checkpoint() { + let (_dir, repo) = fixture(); + let store = CobStore::new(&repo); + let key = signer(7); + let brief_key = signer(71); + + let brief = plain_history(&store, &brief_key, Names::SNAPSHOT_STRIDE.get() - 1); + let (short, _) = store.materialize::(brief).unwrap(); + assert_eq!(short.len(), Names::SNAPSHOT_STRIDE.get() - 1); + assert!( + repo.find_ref(&backend::checkpoint_ref_name(&NameChange::type_name(), brief).unwrap()) + .unwrap() + .is_none(), + "if replay of refs count is smaller than num-to-stride, don't bother checkpointing yet" + ); + + let total = Names::SNAPSHOT_STRIDE.get() * 2; + let object = plain_history(&store, &key, total); + let snapshot = backend::checkpoint_ref_name(&NameChange::type_name(), object).unwrap(); + let (state, tip) = store.materialize::(object).unwrap(); + assert_eq!(state.len(), total, "replayed-state given back from store"); + let written = repo + .find_ref(&snapshot) + .unwrap() + .expect("replay persisted a checkpoint"); + assert_eq!( + decode_checkpoint::>(object, &repo.read_blob(written).unwrap()), + Some((tip, state)), + "checkpoint commits to tip, returning count replayable" + ); + + let Recovered { tail, .. } = store.checkpointed_state::(object).unwrap(); + assert_eq!(tail, Tail(0), "next boot will load from saved checkpoint"); + + plain_claim(&store, object, &key, "tailname", at(total as i64)); + let (moved, _new_tip) = store.materialize::(object).unwrap(); + assert_eq!( + moved.len(), + total + 1, + "even next read will replay new tail onto checkpoint" + ); + assert_eq!( + repo.find_ref(&snapshot).unwrap().unwrap(), + written, + "no touchy checkpoint if tail num smaller than stride num" + ); + } + + #[test] + fn checkpoint_then_append_will_only_write_when_bigger_than_stride_num() { + let (_dir, repo) = fixture(); + let store = CobStore::new(&repo); + let key = signer(14); + + let brief = plain_history(&store, &signer(141), 2); + store + .update_with_checkpointed::(&cob_home(), brief, &key, at(2), |_| { + Ok(NameChange::Claim("name0002".into())) + }) + .unwrap(); + assert!( + repo.find_ref(&backend::checkpoint_ref_name(&NameChange::type_name(), brief).unwrap()) + .unwrap() + .is_none(), + "don't do another checkpoint if less than stride num" + ); + + let total = Names::SNAPSHOT_STRIDE.get() * 2; + let object = plain_history(&store, &key, total); + let before = ChangeId::new( + backend::resolve_tip(&repo, &NameChange::type_name(), object) + .unwrap() + .expect("tip before appending"), + ); + store + .update_with_checkpointed::( + &cob_home(), + object, + &key, + at(total as i64), + |state| Ok(NameChange::Claim(format!("tail{}", state.len()))), + ) + .unwrap(); + let snapshot = backend::checkpoint_ref_name(&NameChange::type_name(), object).unwrap(); + let written = repo + .find_ref(&snapshot) + .unwrap() + .expect("replay should've persisted a checkpoint"); + let (tip, state) = + decode_checkpoint::>(object, &repo.read_blob(written).unwrap()) + .expect("the should checkpoint decode"); + assert_eq!(tip, before, "append doesn't cost another checkpoint write"); + assert_eq!(state.len(), total, "checkpoint matches state pre-append"); + let Recovered { tail, .. } = store.checkpointed_state::(object).unwrap(); + assert_eq!(tail, Tail(1), "next boot should only replay added change"); + } + + #[test] + fn read_that_also_reads_should_heal_a_checkpoint_which_isnt_bounding_history() { + let (_dir, repo) = fixture(); + let store = CobStore::new(&repo); + let key = signer(9); + let total = Names::SNAPSHOT_STRIDE.get() * 2; + let object = plain_history(&store, &key, total); + + let foreign = Oid::from(repo.git().write_blob(b"not a change").unwrap().detach()); + let forged = + encode_checkpoint(object, ChangeId::new(foreign), &BTreeSet::::new()).unwrap(); + let snapshot = backend::checkpoint_ref_name(&NameChange::type_name(), object).unwrap(); + let blob = Oid::from(repo.git().write_blob(&forged).unwrap().detach()); + repo.update_ref(&RefUpdate::Create { + name: snapshot.clone(), + new: blob, + }) + .unwrap(); + + let (healed_state, live_tip) = store.materialize::(object).unwrap(); + assert_eq!( + healed_state.len(), + total, + "we definitely don't serve forged checkpoint data" + ); + let replaced = repo + .find_ref(&snapshot) + .unwrap() + .expect("stale/incorrect checkpoint was nuked & replaced"); + let bytes = repo.read_blob(replaced).unwrap(); + let (tip, checkpoint_state) = decode_checkpoint::>(object, &bytes) + .expect("healed checkpoint should decode correctly"); + assert_eq!( + tip, live_tip, + "healed checkpoint is correctly bound to current tip" + ); + assert_eq!( + checkpoint_state, healed_state, + "healed checkpoint matches replay" + ); + } + #[test] fn a_checkpoint_envelope_is_bound_to_its_object_tip_and_format() { let (_dir, repo) = fixture(); @@ -2122,24 +2341,24 @@ mod tests { ); let winner = store - .append( + .append_materialized( &cob_home(), created.object, &nsid, created.tip, - &dup, + &Materialized::bare(Tag::Add("dup".into())), &key, at(2), ) .unwrap(); assert_eq!(winner, first); - let loser = store.append( + let loser = store.append_materialized( &cob_home(), created.object, &nsid, created.tip, - &dup, + &Materialized::bare(Tag::Add("dup".into())), &key, at(2), ); @@ -2671,7 +2890,7 @@ mod tests { panic!("an opened note emits a version") }; let created = CobStore::new(&repo) - .open(&cob_home(), &first, &signer(seed), at(1)) + .create_materialized(&cob_home(), &first, &signer(seed), at(1)) .unwrap(); (dir, repo, created, live) } @@ -2748,7 +2967,7 @@ mod tests { ); let bare = store - .open(&cob_home(), &erased("nel", live), &key, at(1)) + .create_materialized(&cob_home(), &erased("nel", live), &key, at(1)) .unwrap(); let invented = note_append( &store, @@ -2769,7 +2988,7 @@ mod tests { let key = signer(13); let store = CobStore::new(&repo); let (_, claimed) = note_record("kelp"); - let mismatched = store.open( + let mismatched = store.create_materialized( &cob_home(), &with_record( opened("nel", "3lubrptx57d22", "kelp"), @@ -2782,7 +3001,7 @@ mod tests { matches!(mismatched, Err(CobError::RecordMisclaimed { .. })), "named version and stored version differ, got {mismatched:?}" ); - let unwritten = store.open( + let unwritten = store.create_materialized( &cob_home(), &with_record(opened("nel", "3lubrptx57d22", "kelp"), Record::Bare), &key, @@ -2792,7 +3011,7 @@ mod tests { matches!(unwritten, Err(CobError::RecordUnwritten { .. })), "Change that names its record has to store said record, got {unwritten:?}" ); - let erasing = store.open( + let erasing = store.create_materialized( &cob_home(), &with_record( erased("nel", claimed), diff --git a/knot2/crates/knot-cobs/src/registry.rs b/knot2/crates/knot-cobs/src/registry.rs index 3dd656641..f64b2ee65 100644 --- a/knot2/crates/knot-cobs/src/registry.rs +++ b/knot2/crates/knot-cobs/src/registry.rs @@ -343,6 +343,38 @@ pub enum RegistryError { OwnerMoved { repo: RepoDid, expected: OwnerDid }, } +fn owned_record<'a>( + registry: &'a Registry, + repo: &RepoDid, + owner: &OwnerDid, +) -> Result<&'a RepoRecord, RegistryError> { + let record = registry + .record_of(repo) + .ok_or_else(|| RegistryError::NotHosted { repo: repo.clone() })?; + (record.owner == *owner) + .then_some(record) + .ok_or_else(|| RegistryError::OwnerMoved { + repo: repo.clone(), + expected: owner.clone(), + }) +} + +fn unclaimed_rkey( + registry: &Registry, + owner: &OwnerDid, + rkey: &RepoRkey, + repo: &RepoDid, +) -> Result<(), RegistryError> { + match registry.canonical_holder(owner, rkey) { + Some(holder) if holder != repo => Err(RegistryError::RkeyTaken { + owner: owner.clone(), + rkey: rkey.clone(), + existing: holder.clone(), + }), + _ => Ok(()), + } +} + pub fn register_repo( store: &CobStore, home: &CobHome, @@ -357,15 +389,12 @@ pub fn register_repo( signer, timestamp, |registry| { - if let Some(holder) = registry.canonical_holder(®istration.owner, ®istration.rkey) - && holder != ®istration.repo - { - return Err(RegistryError::RkeyTaken { - owner: registration.owner.clone(), - rkey: registration.rkey.clone(), - existing: holder.clone(), - }); - } + unclaimed_rkey( + registry, + ®istration.owner, + ®istration.rkey, + ®istration.repo, + )?; match registry .record_of(®istration.repo) .map(|record| (record, registration.diverges_from(record))) @@ -397,30 +426,11 @@ pub fn rename_repo( signer, timestamp, |registry| { - let record = - registry - .record_of(&rename.repo) - .ok_or_else(|| RegistryError::NotHosted { - repo: rename.repo.clone(), - })?; - if record.owner != rename.owner { - return Err(RegistryError::OwnerMoved { - repo: rename.repo.clone(), - expected: rename.owner.clone(), - }); - } + let record = owned_record(registry, &rename.repo, &rename.owner)?; if record.rkey == rename.rkey && record.name == rename.name { return Ok(None); } - if let Some(holder) = registry.canonical_holder(&rename.owner, &rename.rkey) - && holder != &rename.repo - { - return Err(RegistryError::RkeyTaken { - owner: rename.owner.clone(), - rkey: rename.rkey.clone(), - existing: holder.clone(), - }); - } + unclaimed_rkey(registry, &rename.owner, &rename.rkey, &rename.repo)?; Ok(Some(RegistryChange::Rename(rename.clone()))) }, ) @@ -470,18 +480,7 @@ pub fn set_repo_policy( signer, timestamp, |registry| { - let record = - registry - .record_of(&change.repo) - .ok_or_else(|| RegistryError::NotHosted { - repo: change.repo.clone(), - })?; - if record.owner != change.owner { - return Err(RegistryError::OwnerMoved { - repo: change.repo.clone(), - expected: change.owner.clone(), - }); - } + let record = owned_record(registry, &change.repo, &change.owner)?; Ok((record.policy != change.policy).then(|| RegistryChange::SetPolicy(change.clone()))) }, ) @@ -501,18 +500,7 @@ pub fn retire_repo( signer, timestamp, |registry| { - let record = - registry - .record_of(&change.repo) - .ok_or_else(|| RegistryError::NotHosted { - repo: change.repo.clone(), - })?; - if record.owner != change.owner { - return Err(RegistryError::OwnerMoved { - repo: change.repo.clone(), - expected: change.owner.clone(), - }); - } + let record = owned_record(registry, &change.repo, &change.owner)?; match record.retired_at { Some(existing) if existing >= change.at => Ok(None), _ => Ok(Some(RegistryChange::Retired(change.clone()))), diff --git a/knot2/crates/knot-index/src/lib.rs b/knot2/crates/knot-index/src/lib.rs index 1b4e853a6..ee07b2a69 100644 --- a/knot2/crates/knot-index/src/lib.rs +++ b/knot2/crates/knot-index/src/lib.rs @@ -12,12 +12,9 @@ use std::path::PathBuf; use std::sync::atomic::{AtomicU64, Ordering}; use std::time::Duration; -use knot_cob::{ChangePayload, CobStore}; +use knot_cob::{ChangePayload, CobStore, Evaluate}; pub use knot_cobs::Entry; -use knot_cobs::{ - BlocklistChange, BlocklistCob, CollaboratorsChange, CollaboratorsCob, MembersChange, - MembersCob, RegistryChange, RepoRef, RepoRegistryCob, -}; +use knot_cobs::{BlocklistCob, CollaboratorsCob, MembersCob, RepoRef, RepoRegistryCob}; use knot_git::{Layout, Repo}; use knot_resource::StructureBytes; use knot_types::{ @@ -439,18 +436,23 @@ impl Index { .is_some_and(|repo| self.unreadable.contains_sync(&repo)) } + fn sole_cob(store: &CobStore) -> Result, IndexError> { + match store.list::()?.as_slice() { + [] => Ok(None), + [object] => Ok(Some(*object)), + many => Err(IndexError::Ambiguous { + type_name: E::Change::type_name(), + count: many.len(), + }), + } + } + pub fn refresh_members(&self) -> Result<(), IndexError> { let meta = Repo::open(&self.meta_path)?; let store = CobStore::new(&meta); - match store.list::()?.as_slice() { - [] => self.members.reset(), - [object] => self.members.refresh(&self.interner, &store, *object)?, - many => { - return Err(IndexError::Ambiguous { - type_name: MembersChange::type_name(), - count: many.len(), - }); - } + match Self::sole_cob::(&store)? { + None => self.members.reset(), + Some(object) => self.members.refresh(&self.interner, &store, object)?, } self.bump_generation(); Ok(()) @@ -459,15 +461,9 @@ impl Index { pub fn refresh_blocklist(&self) -> Result<(), IndexError> { let meta = Repo::open(&self.meta_path)?; let store = CobStore::new(&meta); - match store.list::()?.as_slice() { - [] => self.blocklist.reset(), - [object] => self.blocklist.refresh(&self.interner, &store, *object)?, - many => { - return Err(IndexError::Ambiguous { - type_name: BlocklistChange::type_name(), - count: many.len(), - }); - } + match Self::sole_cob::(&store)? { + None => self.blocklist.reset(), + Some(object) => self.blocklist.refresh(&self.interner, &store, object)?, } self.bump_generation(); Ok(()) @@ -481,15 +477,9 @@ impl Index { fn registry_refresh(&self) -> Result<(), IndexError> { let meta = Repo::open(&self.meta_path)?; let store = CobStore::new(&meta); - let evacuated = match store.list::()?.as_slice() { - [] => self.registry.reset(&self.interner), - [object] => self.registry.refresh(&self.interner, &store, *object)?, - many => { - return Err(IndexError::Ambiguous { - type_name: RegistryChange::type_name(), - count: many.len(), - }); - } + let evacuated = match Self::sole_cob::(&store)? { + None => self.registry.reset(&self.interner), + Some(object) => self.registry.refresh(&self.interner, &store, object)?, }; evacuated.iter().for_each(|repo| { if let Some(key) = self.interner.repo(repo) { @@ -527,17 +517,11 @@ impl Index { ) -> Result<(), IndexError> { let git = self.layout.open(repo)?; let store = CobStore::new(&git); - match store.list::()?.as_slice() { - [] => self.collaborators.mark_repo_empty(repo_key), - [object] => { + match Self::sole_cob::(&store)? { + None => self.collaborators.mark_repo_empty(repo_key), + Some(object) => { self.collaborators - .refresh_repo(&self.interner, &store, repo_key, *object)? - } - many => { - return Err(IndexError::Ambiguous { - type_name: CollaboratorsChange::type_name(), - count: many.len(), - }); + .refresh_repo(&self.interner, &store, repo_key, object)? } } self.bump_generation(); diff --git a/knot2/crates/knot-index/tests/meta_repo.rs b/knot2/crates/knot-index/tests/meta_repo.rs index 2deab342c..fe78f310e 100644 --- a/knot2/crates/knot-index/tests/meta_repo.rs +++ b/knot2/crates/knot-index/tests/meta_repo.rs @@ -1,6 +1,8 @@ -use knot_cob::{CobHome, CobStore}; -use knot_cobs::{CollaboratorsChange, MembersChange, Registration, RegistryChange}; -use knot_git::Layout; +use knot_cob::{ChangePayload, Checkpoint, CobHome, CobStore}; +use knot_cobs::{ + CollaboratorsChange, MembersChange, Registration, RegistryChange, RepoRegistryCob, +}; +use knot_git::{Layout, Repo}; use knot_index::{Index, Resolved}; use knot_runtime::{K256Signer, SeededEntropy}; use knot_types::{KnotId, RepoDid, RepoName}; @@ -139,3 +141,34 @@ fn the_meta_repo_round_trips_every_projection_from_git() { "a rebuilt index still leaves the fostered repo out of the sweeper's candidates" ); } + +#[test] +fn when_rebuilding_we_save_checkpoint_that_can_be_used_later() { + let world = common::World::new(); + let repo = RepoDid::new("did:plc:anemone").unwrap(); + let registry = world.seed_registry(&repo); + let total = ::SNAPSHOT_STRIDE.get(); + (1..total).for_each(|index| { + world.register_extra( + &RepoDid::new(format!("did:plc:repo{index:03}")).unwrap(), + &format!("repo{index:03}"), + registry, + ); + }); + + world.index().rebuild().unwrap(); + let meta = Repo::open(&world.meta_path).unwrap(); + let checkpoint = knot_cob::checkpoint_ref_name(&RegistryChange::type_name(), registry).unwrap(); + assert!( + meta.find_ref(&checkpoint).unwrap().is_some(), + "When rebuilding, we saved a registry checkpoint" + ); + + let rebooted = world.index(); + rebooted.rebuild().unwrap(); + assert_eq!( + rebooted.hosted_repos().len(), + total, + "same exact length of checkpoint rebuild vs non-checkpoint" + ); +} diff --git a/knot2/crates/knot-xrpc/src/cob.rs b/knot2/crates/knot-xrpc/src/cob.rs index 24bf8d98c..8fe8f530a 100644 --- a/knot2/crates/knot-xrpc/src/cob.rs +++ b/knot2/crates/knot-xrpc/src/cob.rs @@ -1,4 +1,4 @@ -use knot_cob::{Checkpoint, CobHome, CobStore, Evaluate, KnotAuthored, Materialized, Record}; +use knot_cob::{Checkpoint, CobHome, CobStore, Evaluate, KnotAuthored, Materialized}; use knot_cobs::{GrantChange, GrantList}; use knot_runtime::Signer; use knot_types::{AccountDid, UnixSeconds}; @@ -31,25 +31,14 @@ where match store.list::().map_err(XrpcError::from)?.as_slice() { [] => match GrantList::empty().would_change(&change) { true => store - .create_materialized( - home, - &Materialized { - record: Record::Bare, - change: change.clone(), - }, - signer, - now, - ) + .create_materialized(home, &Materialized::bare(change.clone()), signer, now) .map(|_| ()) .map_err(XrpcError::from), false => Ok(()), }, [object] => store - .update_materialized_checkpointed::(home, *object, signer, now, |list| { - Ok(list.would_change(&change).then(|| Materialized { - record: Record::Bare, - change: change.clone(), - })) + .update_maybe_checkpointed::(home, *object, signer, now, |list| { + Ok(list.would_change(&change).then(|| change.clone())) }) .map(|_| ()), many => Err(XrpcError::internal(format!(