From bc47fdca2a044bd200e99bc58a41ad8ad09bc6e6 Mon Sep 17 00:00:00 2001 From: dawn <90008@klbr.net> Date: Sat, 20 Jun 2026 19:08:26 +0300 Subject: [PATCH] [db] refactor how repo counts are handled --- src/backfill/manager.rs | 46 +++-- src/backfill/mod.rs | 279 ++++++++++++++++-------------- src/control/repos/indexer.rs | 46 +++-- src/crawler/worker.rs | 11 +- src/db/indexer.rs | 49 ------ src/db/keys/mod.rs | 12 ++ src/db/lifecycle_counts.rs | 319 +++++++++++++++++++++++++++++++++++ src/db/migration/mod.rs | 2 + src/db/migration/v7.rs | 2 +- src/db/migration/v8.rs | 267 +++++++++++++++++++++++++++++ src/db/mod.rs | 53 ++++-- src/ingest/indexer.rs | 48 ++++-- src/ingest/relay.rs | 15 +- src/ops.rs | 18 +- 14 files changed, 906 insertions(+), 261 deletions(-) create mode 100644 src/db/lifecycle_counts.rs create mode 100644 src/db/migration/v8.rs diff --git a/src/backfill/manager.rs b/src/backfill/manager.rs index db9e5ca..ee594bb 100644 --- a/src/backfill/manager.rs +++ b/src/backfill/manager.rs @@ -1,5 +1,5 @@ use crate::db::types::TrimmedDid; -use crate::db::{self, CountDeltas, keys}; +use crate::db::{self, keys}; use crate::state::AppState; use crate::types::{GaugeState, ResyncState}; use miette::{IntoDiagnostic, Result}; @@ -11,9 +11,9 @@ use tracing::{debug, error, info}; pub fn queue_gone_backfills(state: &Arc) -> Result<()> { debug!("scanning for deactivated/takendown repos to retry..."); let mut transitions = 0usize; - let mut count_deltas = CountDeltas::default(); let mut batch = state.db.inner.batch(); + let mut lifecycle_counts = state.db.lifecycle_counts(); for guard in state.db.resync.iter() { let (key, val) = guard.into_inner().into_diagnostic()?; @@ -63,7 +63,7 @@ pub fn queue_gone_backfills(state: &Arc) -> Result<()> { crate::db::ser_repo_meta(&metadata)?, ); - count_deltas.add_gauge_diff(&GaugeState::Resync(None), &GaugeState::Pending); + lifecycle_counts.transition(&mut batch, &did, GaugeState::Pending)?; transitions += 1; } } @@ -72,10 +72,9 @@ pub fn queue_gone_backfills(state: &Arc) -> Result<()> { return Ok(()); } - let reservation = state.db.stage_count_deltas(&mut batch, &count_deltas); + let lifecycle_reservation = lifecycle_counts.stage(&mut batch); batch.commit().into_diagnostic()?; - state.db.apply_count_deltas(&count_deltas); - drop(reservation); + state.db.apply_lifecycle_counts(lifecycle_reservation); state.notify_backfill(); @@ -92,9 +91,9 @@ pub fn retry_worker(state: Arc) { let now = chrono::Utc::now().timestamp(); let mut transitions = 0usize; - let mut count_deltas = CountDeltas::default(); let mut batch = state.db.inner.batch(); + let mut lifecycle_counts = state.db.lifecycle_counts(); for guard in db.resync.iter() { let (key, value) = match guard.into_inner() { @@ -114,9 +113,7 @@ pub fn retry_worker(state: Arc) { }; match rmp_serde::from_slice::(&value) { - Ok(ResyncState::Error { - kind, next_retry, .. - }) => { + Ok(ResyncState::Error { next_retry, .. }) => { if next_retry <= now { debug!(did = %did, "retrying backfill"); @@ -144,16 +141,9 @@ pub fn retry_worker(state: Arc) { } }; - // move from resync back into pending - batch.remove(&state.db.resync, key.clone()); let old_pending = keys::pending_key(metadata.index_id); - batch.remove(&state.db.pending, old_pending); metadata.index_id = rand::random::(); - batch.insert( - &state.db.pending, - keys::pending_key(metadata.index_id), - key.clone(), - ); + let new_pending = keys::pending_key(metadata.index_id); let serialized_metadata = match crate::db::ser_repo_meta(&metadata) { Ok(s) => s, Err(e) => { @@ -161,10 +151,18 @@ pub fn retry_worker(state: Arc) { continue; } }; - batch.insert(&state.db.repo_metadata, &metadata_key, serialized_metadata); + if let Err(e) = + lifecycle_counts.transition(&mut batch, &did, GaugeState::Pending) + { + error!(did = %did, err = %e, "failed to stage lifecycle transition"); + continue; + } - count_deltas - .add_gauge_diff(&GaugeState::Resync(Some(kind)), &GaugeState::Pending); + // move from resync back into pending + batch.remove(&state.db.resync, key.clone()); + batch.remove(&state.db.pending, old_pending); + batch.insert(&state.db.pending, new_pending, key.clone()); + batch.insert(&state.db.repo_metadata, &metadata_key, serialized_metadata); transitions += 1; } } @@ -182,14 +180,14 @@ pub fn retry_worker(state: Arc) { continue; } - let reservation = state.db.stage_count_deltas(&mut batch, &count_deltas); + let lifecycle_reservation = lifecycle_counts.stage(&mut batch); if let Err(e) = batch.commit() { error!(err = %e, "failed to commit batch"); db::check_poisoned(&e); + drop(lifecycle_reservation); continue; } - state.db.apply_count_deltas(&count_deltas); - drop(reservation); + state.db.apply_lifecycle_counts(lifecycle_reservation); state.notify_backfill(); info!(count = transitions, "queued retries"); } diff --git a/src/backfill/mod.rs b/src/backfill/mod.rs index 1486522..e443125 100644 --- a/src/backfill/mod.rs +++ b/src/backfill/mod.rs @@ -207,26 +207,48 @@ async fn did_task( ) -> Result<(), BackfillError> { let db = &state.db; - match process_did(state, &http, did, verify_signatures, strategy).await { + match process_did( + state, + &http, + did, + pending_key.clone(), + verify_signatures, + strategy, + ) + .await + { Ok(Some(_repo_state)) => { - let did_key = keys::repo_key(did); + let applied = tokio::task::spawn_blocking({ + let state = state.clone(); + let did = did.clone(); + let pending_key = pending_key.clone(); + move || { + let db = &state.db; + let did_key = keys::repo_key(&did); + let mut batch = db.inner.batch(); + let mut lifecycle_counts = db.lifecycle_counts(); + let applied = lifecycle_counts.transition_pending_key( + &mut batch, + &did, + pending_key.as_ref(), + GaugeState::Synced, + )?; + batch.remove(&db.pending, pending_key.clone()); + if applied { + batch.remove(&db.resync, &did_key); + } + let lifecycle_reservation = lifecycle_counts.stage(&mut batch); + batch.commit().into_diagnostic()?; + db.apply_lifecycle_counts(lifecycle_reservation); + Ok::<_, miette::Report>(applied) + } + }) + .await + .into_diagnostic()??; - // determine old gauge state - // if it was error/suspended etc, we need to know which error kind it was to decrement correctly. - let mut batch = db.inner.batch(); - let mut count_deltas = CountDeltas::default(); - // unconditionally remove from pending - batch.remove(&db.pending, pending_key); - // remove from resync, just in case - batch.remove(&db.resync, &did_key); - count_deltas.add_gauge_diff(&GaugeState::Pending, &GaugeState::Synced); - let reservation = db.stage_count_deltas(&mut batch, &count_deltas); - - tokio::task::spawn_blocking(move || batch.commit().into_diagnostic()) - .await - .into_diagnostic()??; - db.apply_count_deltas(&count_deltas); - drop(reservation); + if !applied { + return Ok(()); + } let state = state.clone(); tokio::task::spawn_blocking(move || { @@ -250,16 +272,29 @@ async fn did_task( Ok(None) => Ok(()), Err(BackfillError::Deleted) => { warn!("orphaned pending entry, cleaning up"); - let mut batch = db.inner.batch(); - let mut count_deltas = CountDeltas::default(); - batch.remove(&db.pending, pending_key); - count_deltas.add("pending", -1); - let reservation = db.stage_count_deltas(&mut batch, &count_deltas); - tokio::task::spawn_blocking(move || batch.commit().into_diagnostic()) - .await - .into_diagnostic()??; - db.apply_count_deltas(&count_deltas); - drop(reservation); + tokio::task::spawn_blocking({ + let state = state.clone(); + let did = did.clone(); + let pending_key = pending_key.clone(); + move || { + let db = &state.db; + let mut batch = db.inner.batch(); + let mut lifecycle_counts = db.lifecycle_counts(); + lifecycle_counts.transition_pending_key( + &mut batch, + &did, + pending_key.as_ref(), + GaugeState::Synced, + )?; + batch.remove(&db.pending, pending_key); + let lifecycle_reservation = lifecycle_counts.stage(&mut batch); + batch.commit().into_diagnostic()?; + db.apply_lifecycle_counts(lifecycle_reservation); + Ok::<_, miette::Report>(()) + } + }) + .await + .into_diagnostic()??; Ok(()) } Err(e) => { @@ -291,12 +326,10 @@ async fn did_task( .transpose() })?; - let (mut retry_count, prev_kind) = match existing_state { - Some(ResyncState::Error { - kind, retry_count, .. - }) => (retry_count, Some(kind)), + let mut retry_count = match existing_state { + Some(ResyncState::Error { retry_count, .. }) => retry_count, Some(ResyncState::Gone { .. }) => return Ok(()), // should handle gone? original code didn't really? - None => (0, None), + None => 0, }; // Calculate new stats @@ -308,19 +341,13 @@ async fn did_task( retry_count, next_retry, }; - let old_gauge = prev_kind - .map(|k| GaugeState::Resync(Some(k))) - .unwrap_or(GaugeState::Pending); - let new_gauge = GaugeState::Resync(Some(error_kind)); - let mut count_deltas = CountDeltas::default(); - count_deltas.add_gauge_diff(&old_gauge, &new_gauge); - let error_string = e.to_string(); tokio::task::spawn_blocking({ let state = state.clone(); let did_key = did_key.into_static(); - let count_deltas = count_deltas.clone(); + let did = did.clone(); + let pending_key = pending_key.clone(); move || { // 3. save to resync let serialized_resync_state = @@ -340,16 +367,24 @@ async fn did_task( }; let mut batch = state.db.inner.batch(); - batch.insert(&state.db.resync, &did_key, serialized_resync_state); + let mut lifecycle_counts = state.db.lifecycle_counts(); + let applied = lifecycle_counts.transition_pending_key( + &mut batch, + &did, + pending_key.as_ref(), + GaugeState::Resync(Some(error_kind)), + )?; batch.remove(&state.db.pending, pending_key.clone()); - if let Some(state_bytes) = serialized_repo_state { - batch.insert(&state.db.repos, &did_key, state_bytes); + if applied { + batch.insert(&state.db.resync, &did_key, serialized_resync_state); + if let Some(state_bytes) = serialized_repo_state { + batch.insert(&state.db.repos, &did_key, state_bytes); + } } - let reservation = state.db.stage_count_deltas(&mut batch, &count_deltas); - batch.commit().into_diagnostic().inspect(|_| { - state.db.apply_count_deltas(&count_deltas); - drop(reservation); - }) + let lifecycle_reservation = lifecycle_counts.stage(&mut batch); + batch.commit().into_diagnostic()?; + state.db.apply_lifecycle_counts(lifecycle_reservation); + Ok::<_, miette::Report>(()) } }) .await @@ -609,7 +644,6 @@ async fn process_did_sparse( .await?; let Some((records, state)) = result else { - cleanup_discarded_repo(app_state, did).await?; return Ok(SparseBackfillResult::Discarded); }; @@ -936,10 +970,10 @@ async fn persist_sparse_backfill( let mut count_deltas = CountDeltas::default(); if delta != 0 { - count_deltas.add("records", delta); + count_deltas.add_records(delta); } if added_blocks > 0 { - count_deltas.add("blocks", added_blocks); + count_deltas.add_blocks(added_blocks); } let reservation = app_state.db.stage_count_deltas(&mut batch, &count_deltas); batch.commit().into_diagnostic()?; @@ -953,46 +987,11 @@ async fn persist_sparse_backfill( .map_err(BackfillError::from) } -async fn cleanup_discarded_repo( - app_state: &Arc, - did: &Did<'static>, -) -> Result<(), BackfillError> { - let metadata_key = keys::repo_metadata_key(did); - let metadata_bytes = app_state - .db - .repo_metadata - .get(&metadata_key) - .into_diagnostic()? - .ok_or_else(|| miette::miette!("repo metadata not found for {}", did))?; - let metadata = crate::db::deser_repo_meta(metadata_bytes.as_ref())?; - let did_key = keys::repo_key(did); - let backfill_pending_key = keys::pending_key(metadata.index_id); - let app_state = app_state.clone(); - - tokio::task::spawn_blocking(move || { - let mut batch = app_state.db.inner.batch(); - let mut count_deltas = CountDeltas::default(); - batch.remove(&app_state.db.repos, &did_key); - batch.remove(&app_state.db.repo_metadata, &metadata_key); - batch.remove(&app_state.db.pending, backfill_pending_key); - count_deltas.add("repos", -1); - count_deltas.add("pending", -1); - let reservation = app_state.db.stage_count_deltas(&mut batch, &count_deltas); - batch.commit().into_diagnostic().inspect(|_| { - app_state.db.apply_count_deltas(&count_deltas); - drop(reservation); - }) - }) - .await - .into_diagnostic()??; - - Ok(()) -} - async fn process_did( app_state: &Arc, http: &reqwest::Client, did: &Did<'static>, + pending_key: Slice, verify_signatures: bool, strategy: BackfillStrategy, ) -> Result>, BackfillError> { @@ -1115,15 +1114,22 @@ async fn process_did( if matches!(e, GetRepoError::RepoNotFound(_)) { warn!("repo not found, deleting"); let mut batch = db.inner.batch(); - let mut count_deltas = CountDeltas::default(); - if let Err(e) = crate::ops::delete_repo(&mut batch, db, did, &state) { - error!(err = %e, "failed to wipe repo during backfill"); + let mut lifecycle_counts = db.lifecycle_counts(); + let applied = lifecycle_counts.transition_pending_key( + &mut batch, + did, + pending_key.as_ref(), + GaugeState::Synced, + )?; + batch.remove(&db.pending, pending_key.clone()); + if applied { + if let Err(e) = crate::ops::delete_repo(&mut batch, db, did, &state) { + error!(err = %e, "failed to wipe repo during backfill"); + } } - count_deltas.add("pending", -1); - let reservation = db.stage_count_deltas(&mut batch, &count_deltas); + let lifecycle_reservation = lifecycle_counts.stage(&mut batch); batch.commit().into_diagnostic()?; - db.apply_count_deltas(&count_deltas); - drop(reservation); + db.apply_lifecycle_counts(lifecycle_reservation); // return None so did_task skips sending BackfillFinished (nothing to drain for a deleted repo) return Ok(None); } @@ -1147,18 +1153,41 @@ async fn process_did( let resync_bytes = rmp_serde::to_vec(&resync_state).into_diagnostic()?; let app_state_clone = app_state.clone(); - app_state - .db - .update_repo_state_async(did, move |state, (key, batch)| { - state.active = false; - state.status = status; - batch.insert(&app_state_clone.db.resync, key, resync_bytes); - Ok((true, ())) - }) - .await?; - - // return success so wrapper stops retrying - return Ok(Some(previous_state)); + let did = did.clone(); + let pending_key = pending_key.clone(); + tokio::task::spawn_blocking(move || { + let db = &app_state_clone.db; + let mut batch = db.inner.batch(); + let mut lifecycle_counts = db.lifecycle_counts(); + let applied = lifecycle_counts.transition_pending_key( + &mut batch, + &did, + pending_key.as_ref(), + GaugeState::Resync(None), + )?; + batch.remove(&db.pending, pending_key.clone()); + if applied { + Db::update_repo_state( + &mut batch, + &db.repos, + &did, + move |state, (key, batch)| { + state.active = false; + state.status = status; + batch.insert(&db.resync, key, resync_bytes); + Ok((true, ())) + }, + )?; + } + let lifecycle_reservation = lifecycle_counts.stage(&mut batch); + batch.commit().into_diagnostic()?; + db.apply_lifecycle_counts(lifecycle_reservation); + Ok::<_, miette::Report>(()) + }) + .await + .into_diagnostic()??; + + return Ok(None); } Err(e).into_diagnostic()? @@ -1484,10 +1513,10 @@ async fn process_did( let mut count_deltas = CountDeltas::default(); if delta != 0 { - count_deltas.add("records", delta); + count_deltas.add_records(delta); } if added_blocks > 0 { - count_deltas.add("blocks", added_blocks); + count_deltas.add_blocks(added_blocks); } let reservation = app_state.db.stage_count_deltas(&mut batch, &count_deltas); batch.commit().into_diagnostic()?; @@ -1500,29 +1529,33 @@ async fn process_did( .into_diagnostic()?? }; - let metadata_key = keys::repo_metadata_key(did); - let metadata_bytes = db - .repo_metadata - .get(&metadata_key) - .into_diagnostic()? - .ok_or_else(|| miette::miette!("repo metadata not found for {}", did))?; - let metadata = crate::db::deser_repo_meta(metadata_bytes.as_ref())?; - let Some(count) = result else { // signal mode: no signal-matching records found, clean up the optimistically-added repo let did_key = keys::repo_key(did); - let backfill_pending_key = keys::pending_key(metadata.index_id); + let metadata_key = keys::repo_metadata_key(did); + let backfill_pending_key = pending_key.clone(); let app_state = app_state.clone(); + let did = did.clone(); tokio::task::spawn_blocking(move || { let mut batch = app_state.db.inner.batch(); let mut count_deltas = CountDeltas::default(); - batch.remove(&app_state.db.repos, &did_key); - batch.remove(&app_state.db.repo_metadata, &metadata_key); - batch.remove(&app_state.db.pending, backfill_pending_key); - count_deltas.add("repos", -1); - count_deltas.add("pending", -1); + let mut lifecycle_counts = app_state.db.lifecycle_counts(); + let applied = lifecycle_counts.transition_pending_key( + &mut batch, + &did, + backfill_pending_key.as_ref(), + GaugeState::Synced, + )?; + batch.remove(&app_state.db.pending, backfill_pending_key.clone()); + if applied { + batch.remove(&app_state.db.repos, &did_key); + batch.remove(&app_state.db.repo_metadata, &metadata_key); + count_deltas.add_repos(-1); + } + let lifecycle_reservation = lifecycle_counts.stage(&mut batch); let reservation = app_state.db.stage_count_deltas(&mut batch, &count_deltas); batch.commit().into_diagnostic().inspect(|_| { + app_state.db.apply_lifecycle_counts(lifecycle_reservation); app_state.db.apply_count_deltas(&count_deltas); drop(reservation); }) diff --git a/src/control/repos/indexer.rs b/src/control/repos/indexer.rs index cfa976f..bc68af5 100644 --- a/src/control/repos/indexer.rs +++ b/src/control/repos/indexer.rs @@ -2,7 +2,7 @@ use futures::{FutureExt, TryFutureExt}; use rand::Rng; use super::*; -use crate::db::CountDeltas; +use crate::db::LifecycleCountBatch; impl ReposControl { /// iterates through pending repositories, returning their state. @@ -103,7 +103,7 @@ impl ReposControl { db: &Db, did: &Did<'_>, batch: &mut fjall::OwnedWriteBatch, - count_deltas: &mut CountDeltas, + lifecycle_counts: &mut LifecycleCountBatch<'_>, ) -> Result { let did_key = keys::repo_key(did); let metadata_key = keys::repo_metadata_key(did); @@ -114,7 +114,7 @@ impl ReposControl { .map(db::deser_repo_state) .transpose()?; - if let Some(repo_state) = existing { + if existing.is_some() { let metadata_bytes = db .repo_metadata .get(&metadata_key) @@ -129,8 +129,6 @@ impl ReposControl { .into_diagnostic()? .is_some(); if !is_pending { - let resync = db.resync.get(&did_key).into_diagnostic()?; - let old = db::Db::repo_gauge_state(&repo_state, resync.as_deref()); metadata.tracked = true; // insert into pending with new index_id let old_pending = keys::pending_key(metadata.index_id); @@ -143,7 +141,7 @@ impl ReposControl { &metadata_key, crate::db::ser_repo_meta(&metadata)?, ); - count_deltas.add_gauge_diff(&old, &GaugeState::Pending); + lifecycle_counts.transition(batch, did, GaugeState::Pending)?; return Ok(true); } } @@ -170,18 +168,17 @@ impl ReposControl { let db = &state.db; let mut batch = db.inner.batch(); let mut queued: Vec> = Vec::new(); - let mut count_deltas = CountDeltas::default(); + let mut lifecycle_counts = db.lifecycle_counts(); for did in dids { - if Self::_resync(db, &did, &mut batch, &mut count_deltas)? { + if Self::_resync(db, &did, &mut batch, &mut lifecycle_counts)? { queued.push(did); } } - let reservation = db.stage_count_deltas(&mut batch, &count_deltas); + let lifecycle_reservation = lifecycle_counts.stage(&mut batch); batch.commit().into_diagnostic()?; - db.apply_count_deltas(&count_deltas); - drop(reservation); + db.apply_lifecycle_counts(lifecycle_reservation); state.db.persist()?; Ok::<_, miette::Report>(queued) }) @@ -210,7 +207,8 @@ impl ReposControl { let db = &state.db; let mut batch = db.inner.batch(); let mut queued: Vec> = Vec::new(); - let mut count_deltas = CountDeltas::default(); + let mut count_deltas = crate::db::CountDeltas::default(); + let mut lifecycle_counts = db.lifecycle_counts(); for did in dids { let did_key = keys::repo_key(&did); @@ -222,7 +220,8 @@ impl ReposControl { .transpose()?; if let Some(metadata) = existing_metadata { - if !metadata.tracked && Self::_resync(db, &did, &mut batch, &mut count_deltas)? + if !metadata.tracked + && Self::_resync(db, &did, &mut batch, &mut lifecycle_counts)? { queued.push(did); } @@ -236,14 +235,16 @@ impl ReposControl { crate::db::ser_repo_meta(&metadata)?, ); batch.insert(&db.pending, keys::pending_key(metadata.index_id), &did_key); - count_deltas.add("repos", 1); - count_deltas.add_gauge_diff(&GaugeState::Synced, &GaugeState::Pending); + count_deltas.add_repos(1); + lifecycle_counts.transition(&mut batch, &did, GaugeState::Pending)?; queued.push(did); } } + let lifecycle_reservation = lifecycle_counts.stage(&mut batch); let reservation = db.stage_count_deltas(&mut batch, &count_deltas); batch.commit().into_diagnostic()?; + db.apply_lifecycle_counts(lifecycle_reservation); db.apply_count_deltas(&count_deltas); drop(reservation); state.db.persist()?; @@ -269,7 +270,7 @@ impl ReposControl { let db = &state.db; let mut batch = db.inner.batch(); let mut untracked: Vec> = Vec::new(); - let mut count_deltas = CountDeltas::default(); + let mut lifecycle_counts = db.lifecycle_counts(); for did in dids { let did_key = keys::repo_key(&did); @@ -281,7 +282,7 @@ impl ReposControl { .map(db::deser_repo_state) .transpose()?; - if let Some(repo_state) = existing { + if existing.is_some() { let metadata_bytes = db.repo_metadata.get(&metadata_key).into_diagnostic()?; let existing_metadata = metadata_bytes .map(|b| crate::db::deser_repo_meta(&b)) @@ -290,8 +291,6 @@ impl ReposControl { if let Some(mut metadata) = existing_metadata && metadata.tracked { - let resync = db.resync.get(&did_key).into_diagnostic()?; - let old = db::Db::repo_gauge_state(&repo_state, resync.as_deref()); metadata.tracked = false; batch.insert( &db.repo_metadata, @@ -300,18 +299,15 @@ impl ReposControl { ); batch.remove(&db.pending, keys::pending_key(metadata.index_id)); batch.remove(&db.resync, &did_key); - if old != GaugeState::Synced { - count_deltas.add_gauge_diff(&old, &GaugeState::Synced); - } + lifecycle_counts.transition(&mut batch, &did, GaugeState::Synced)?; untracked.push(did); } } } - let reservation = db.stage_count_deltas(&mut batch, &count_deltas); + let lifecycle_reservation = lifecycle_counts.stage(&mut batch); batch.commit().into_diagnostic()?; - db.apply_count_deltas(&count_deltas); - drop(reservation); + db.apply_lifecycle_counts(lifecycle_reservation); state.db.persist()?; Ok::<_, miette::Report>(untracked) }) diff --git a/src/crawler/worker.rs b/src/crawler/worker.rs index 8540355..5cc6525 100644 --- a/src/crawler/worker.rs +++ b/src/crawler/worker.rs @@ -145,6 +145,7 @@ impl CrawlerWorker { let mut batch = app_state.db.inner.batch(); let mut surviving = Vec::new(); let mut count_deltas = CountDeltas::default(); + let mut lifecycle_counts = app_state.db.lifecycle_counts(); for guard in guards { let did_key = keys::repo_key(&guard); let metadata_key = keys::repo_metadata_key(&guard); @@ -173,17 +174,23 @@ impl CrawlerWorker { // clear any stale retry entry, this DID is confirmed and being enqueued batch.remove(&app_state.db.crawler, keys::crawler_retry_key(&guard)); trace!(did = %*guard, "enqueuing repo"); - count_deltas.add("repos", 1); + count_deltas.add_repos(1); #[cfg(feature = "indexer")] - count_deltas.add("pending", 1); + lifecycle_counts.transition( + &mut batch, + &guard, + crate::types::GaugeState::Pending, + )?; surviving.push(guard); } if let Some(cursor) = cursor_update { batch.insert(&app_state.db.cursors, cursor.key, cursor.value); } + let lifecycle_reservation = lifecycle_counts.stage(&mut batch); let reservation = app_state.db.stage_count_deltas(&mut batch, &count_deltas); // todo: repo state overwrites here are acceptable? batch.commit().into_diagnostic()?; + app_state.db.apply_lifecycle_counts(lifecycle_reservation); app_state.db.apply_count_deltas(&count_deltas); drop(reservation); Ok(surviving) diff --git a/src/db/indexer.rs b/src/db/indexer.rs index ca20933..12a60e6 100644 --- a/src/db/indexer.rs +++ b/src/db/indexer.rs @@ -1,4 +1,3 @@ -use crate::types::{GaugeState, RepoStatus, ResyncState}; use fjall::{Keyspace, OwnedWriteBatch}; use jacquard_common::IntoStatic; use jacquard_common::types::string::Did; @@ -30,54 +29,6 @@ impl Db { Ok(None) } } - - pub(crate) async fn update_repo_state_async( - &self, - did: &Did<'_>, - f: F, - ) -> Result, T)>> - where - F: FnOnce(&mut RepoState, (&[u8], &mut fjall::OwnedWriteBatch)) -> Result<(bool, T)> - + Send - + 'static, - T: Send + 'static, - { - let mut batch = self.inner.batch(); - let repos = self.repos.clone(); - let did = did.clone().into_static(); - - tokio::task::spawn_blocking(move || { - let Some((state, t)) = Self::update_repo_state(&mut batch, &repos, &did, f)? else { - return Ok(None); - }; - batch.commit().into_diagnostic()?; - Ok(Some((state, t))) - }) - .await - .into_diagnostic()? - } - - pub(crate) fn repo_gauge_state( - repo_state: &RepoState, - resync_bytes: Option<&[u8]>, - ) -> GaugeState { - match repo_state.status { - RepoStatus::Synced => GaugeState::Synced, - RepoStatus::Error(_) - | RepoStatus::Deactivated - | RepoStatus::Takendown - | RepoStatus::Suspended - | RepoStatus::Deleted - | RepoStatus::Desynchronized - | RepoStatus::Throttled => resync_bytes - .and_then(|b| rmp_serde::from_slice::(b).ok()) - .and_then(|s| match s { - ResyncState::Error { kind, .. } => Some(GaugeState::Resync(Some(kind))), - _ => None, - }) - .unwrap_or(GaugeState::Resync(None)), - } - } } pub fn set_record_count( diff --git a/src/db/keys/mod.rs b/src/db/keys/mod.rs index f2db6c0..c2207ee 100644 --- a/src/db/keys/mod.rs +++ b/src/db/keys/mod.rs @@ -135,6 +135,18 @@ pub fn parse_count_delta_key(key: &[u8]) -> miette::Result<(u64, &str)> { pub const COUNT_DELTA_WATERMARK_KEY: &[u8] = b"w|count_delta_watermark"; +#[cfg(feature = "indexer")] +pub const COUNT_GAUGE_PREFIX: &[u8] = &[b'g', SEP]; + +#[cfg(feature = "indexer")] +pub fn count_gauge_key(did: &Did) -> Vec { + let repo = TrimmedDid::from(did); + let mut key = Vec::with_capacity(COUNT_GAUGE_PREFIX.len() + repo.len()); + key.extend_from_slice(COUNT_GAUGE_PREFIX); + repo.write_to_vec(&mut key); + key +} + pub const COUNT_COLLECTION_PREFIX: &[u8] = &[b'r', SEP]; pub fn did_collection_prefix(did: &Did) -> Vec { diff --git a/src/db/lifecycle_counts.rs b/src/db/lifecycle_counts.rs new file mode 100644 index 0000000..cc2f69a --- /dev/null +++ b/src/db/lifecycle_counts.rs @@ -0,0 +1,319 @@ +use std::collections::BTreeMap; +use std::sync::MutexGuard; + +use fjall::OwnedWriteBatch; +use jacquard_common::types::string::Did; +use miette::{IntoDiagnostic, Result, WrapErr}; + +use crate::types::{GaugeState, ResyncErrorKind, ResyncState}; + +use super::{CountDeltaReservation, CountDeltas, Db, deser_repo_meta, keys}; + +pub(crate) struct LifecycleCountBatch<'a> { + db: &'a Db, + _lock: MutexGuard<'a, ()>, + deltas: CountDeltas, + staged: BTreeMap, GaugeState>, +} + +pub(crate) struct LifecycleCountReservation<'a> { + _lock: MutexGuard<'a, ()>, + _count_reservation: Option, + deltas: CountDeltas, +} + +impl Db { + pub(crate) fn lifecycle_counts(&self) -> LifecycleCountBatch<'_> { + LifecycleCountBatch { + db: self, + _lock: self + .lifecycle_count_lock + .lock() + .expect("lifecycle count lock poisoned"), + deltas: CountDeltas::default(), + staged: BTreeMap::new(), + } + } + + pub(crate) fn apply_lifecycle_counts(&self, reservation: LifecycleCountReservation<'_>) { + self.apply_count_deltas(&reservation.deltas); + drop(reservation); + } +} + +impl<'a> LifecycleCountBatch<'a> { + pub(crate) fn transition( + &mut self, + batch: &mut OwnedWriteBatch, + did: &Did<'_>, + new: GaugeState, + ) -> Result { + let did_key = keys::repo_key(did); + let (old, had_projection) = self.current_gauge(did, &did_key)?; + + if old != new { + self.deltas.add_gauge_diff(&old, &new); + self.stage_membership(batch, did, &did_key, new); + return Ok(true); + } + + if !had_projection { + self.stage_membership(batch, did, &did_key, new); + } + + Ok(false) + } + + pub(crate) fn transition_pending_key( + &mut self, + batch: &mut OwnedWriteBatch, + did: &Did<'_>, + pending_key: &[u8], + new: GaugeState, + ) -> Result { + if !self.pending_key_matches(did, pending_key)? { + return Ok(false); + } + + self.transition(batch, did, new)?; + Ok(true) + } + + pub(crate) fn stage(self, batch: &mut OwnedWriteBatch) -> LifecycleCountReservation<'a> { + let count_reservation = self.db.stage_count_deltas(batch, &self.deltas); + LifecycleCountReservation { + _lock: self._lock, + _count_reservation: count_reservation, + deltas: self.deltas, + } + } + + fn current_gauge(&self, did: &Did<'_>, did_key: &[u8]) -> Result<(GaugeState, bool)> { + if let Some(staged) = self.staged.get(did_key) { + return Ok((*staged, true)); + } + + let projection_key = keys::count_gauge_key(did); + if let Some(bytes) = self.db.counts.get(&projection_key).into_diagnostic()? { + return decode_gauge(bytes.as_ref()).map(|gauge| (gauge, true)); + } + + if self.db.repos.get(did_key).into_diagnostic()?.is_some() { + miette::bail!("missing lifecycle count projection for existing repo {did}"); + } + + Ok((GaugeState::Synced, false)) + } + + fn pending_key_matches(&self, did: &Did<'_>, pending_key: &[u8]) -> Result { + let Some(current) = current_pending_key(self.db, did)? else { + return Ok(false); + }; + + Ok(current.as_slice() == pending_key + && self.db.pending.get(current).into_diagnostic()?.is_some()) + } + + fn stage_membership( + &mut self, + batch: &mut OwnedWriteBatch, + did: &Did<'_>, + did_key: &[u8], + new: GaugeState, + ) { + let projection_key = keys::count_gauge_key(did); + batch.insert(&self.db.counts, projection_key, encode_gauge(new)); + self.staged.insert(did_key.to_vec(), new); + } +} + +fn current_pending_key(db: &Db, did: &Did<'_>) -> Result> { + db.repo_metadata + .get(keys::repo_metadata_key(did)) + .into_diagnostic()? + .map(|bytes| { + deser_repo_meta(bytes.as_ref()) + .map(|metadata| keys::pending_key(metadata.index_id)) + .wrap_err_with(|| format!("invalid repo metadata for {did}")) + }) + .transpose() +} + +pub(super) fn gauge_from_resync(bytes: &[u8]) -> GaugeState { + rmp_serde::from_slice::(bytes) + .map(|state| match state { + ResyncState::Error { kind, .. } => GaugeState::Resync(Some(kind)), + ResyncState::Gone { .. } => GaugeState::Resync(None), + }) + .unwrap_or(GaugeState::Resync(None)) +} + +pub(super) fn encode_gauge(gauge: GaugeState) -> [u8; 1] { + match gauge { + GaugeState::Synced => [0], + GaugeState::Pending => [1], + GaugeState::Resync(None) => [2], + GaugeState::Resync(Some(ResyncErrorKind::Ratelimited)) => [3], + GaugeState::Resync(Some(ResyncErrorKind::Transport)) => [4], + GaugeState::Resync(Some(ResyncErrorKind::Generic)) => [5], + } +} + +fn decode_gauge(bytes: &[u8]) -> Result { + match bytes { + [0] => Ok(GaugeState::Synced), + [1] => Ok(GaugeState::Pending), + [2] => Ok(GaugeState::Resync(None)), + [3] => Ok(GaugeState::Resync(Some(ResyncErrorKind::Ratelimited))), + [4] => Ok(GaugeState::Resync(Some(ResyncErrorKind::Transport))), + [5] => Ok(GaugeState::Resync(Some(ResyncErrorKind::Generic))), + _ => miette::bail!("invalid lifecycle gauge projection bytes: {bytes:?}"), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::Config; + use crate::db::{ser_repo_meta, ser_repo_state}; + use crate::types::{RepoMetadata, RepoState}; + use jacquard_common::IntoStatic; + + fn test_config(path: &std::path::Path) -> Config { + Config { + database_path: path.to_path_buf(), + ..Default::default() + } + } + + fn test_did() -> Result> { + Did::new("did:plc:yk4q3id7id6p5z3bypvshc64") + .into_diagnostic() + .map(IntoStatic::into_static) + } + + fn enqueue_pending(db: &Db, did: &Did<'_>, index_id: u64) -> Result<[u8; 8]> { + let did_key = keys::repo_key(did); + let metadata_key = keys::repo_metadata_key(did); + let pending_key = keys::pending_key(index_id); + let state = RepoState::backfilling(); + let metadata = RepoMetadata::backfilling(index_id); + + let mut batch = db.inner.batch(); + batch.insert(&db.repos, &did_key, ser_repo_state(&state)?); + batch.insert(&db.repo_metadata, metadata_key, ser_repo_meta(&metadata)?); + batch.insert(&db.pending, pending_key, did_key); + let mut lifecycle = db.lifecycle_counts(); + lifecycle.transition(&mut batch, did, GaugeState::Pending)?; + let lifecycle_reservation = lifecycle.stage(&mut batch); + batch.commit().into_diagnostic()?; + db.apply_lifecycle_counts(lifecycle_reservation); + Ok(pending_key) + } + + fn complete_pending(db: &Db, did: &Did<'_>, pending_key: [u8; 8]) -> Result { + let mut batch = db.inner.batch(); + let mut lifecycle = db.lifecycle_counts(); + let applied = lifecycle.transition_pending_key( + &mut batch, + did, + pending_key.as_slice(), + GaugeState::Synced, + )?; + if applied { + batch.remove(&db.pending, pending_key); + } + let lifecycle_reservation = lifecycle.stage(&mut batch); + batch.commit().into_diagnostic()?; + db.apply_lifecycle_counts(lifecycle_reservation); + Ok(applied) + } + + #[test] + fn duplicate_pending_completion_is_idempotent() -> Result<()> { + let tmp = tempfile::tempdir().into_diagnostic()?; + let db = Db::open(&test_config(tmp.path()))?; + let did = test_did()?; + let pending_key = enqueue_pending(&db, &did, 1)?; + + assert_eq!(db.get_count_sync("pending"), 1); + assert!(complete_pending(&db, &did, pending_key)?); + assert_eq!(db.get_count_sync("pending"), 0); + + assert!(!complete_pending(&db, &did, pending_key)?); + assert_eq!(db.get_count_sync("pending"), 0); + + Ok(()) + } + + #[test] + fn stale_pending_key_cannot_complete_current_pending_state() -> Result<()> { + let tmp = tempfile::tempdir().into_diagnostic()?; + let db = Db::open(&test_config(tmp.path()))?; + let did = test_did()?; + let stale_pending = enqueue_pending(&db, &did, 1)?; + let current_pending = keys::pending_key(2); + let did_key = keys::repo_key(&did); + let metadata_key = keys::repo_metadata_key(&did); + + let mut metadata = RepoMetadata::backfilling(2); + metadata.tracked = true; + let mut batch = db.inner.batch(); + batch.remove(&db.pending, stale_pending); + batch.insert(&db.pending, current_pending, &did_key); + batch.insert(&db.repo_metadata, metadata_key, ser_repo_meta(&metadata)?); + let mut lifecycle = db.lifecycle_counts(); + lifecycle.transition(&mut batch, &did, GaugeState::Pending)?; + let lifecycle_reservation = lifecycle.stage(&mut batch); + batch.commit().into_diagnostic()?; + db.apply_lifecycle_counts(lifecycle_reservation); + + assert_eq!(db.get_count_sync("pending"), 1); + assert!(!complete_pending(&db, &did, stale_pending)?); + assert_eq!(db.get_count_sync("pending"), 1); + assert!(db.pending.get(current_pending).into_diagnostic()?.is_some()); + + Ok(()) + } + + #[test] + fn lifecycle_delta_replays_after_crash_before_apply() -> Result<()> { + let tmp = tempfile::tempdir().into_diagnostic()?; + let cfg = test_config(tmp.path()); + let did = test_did()?; + + { + let db = Db::open(&cfg)?; + let pending_key = enqueue_pending(&db, &did, 1)?; + db.persist()?; + + let mut batch = db.inner.batch(); + let mut lifecycle = db.lifecycle_counts(); + let applied = lifecycle.transition_pending_key( + &mut batch, + &did, + pending_key.as_slice(), + GaugeState::Synced, + )?; + assert!(applied); + batch.remove(&db.pending, pending_key); + let lifecycle_reservation = lifecycle.stage(&mut batch); + batch.commit().into_diagnostic()?; + db.persist()?; + drop(lifecycle_reservation); + } + + { + let db = Db::open(&cfg)?; + assert_eq!(db.get_count_sync("pending"), 0); + assert!( + db.pending + .get(keys::pending_key(1)) + .into_diagnostic()? + .is_none() + ); + } + + Ok(()) + } +} diff --git a/src/db/migration/mod.rs b/src/db/migration/mod.rs index 3ad4e58..c578825 100644 --- a/src/db/migration/mod.rs +++ b/src/db/migration/mod.rs @@ -11,6 +11,7 @@ mod v4; mod v5; mod v6; mod v7; +mod v8; type MigrationFn = fn(&Db, &mut OwnedWriteBatch) -> Result<()>; @@ -37,6 +38,7 @@ const MIGRATIONS: &[(&str, MigrationFn)] = &[ ("pds_meta_layout", v5::pds_meta_layout), ("rebuild_pds_account_counts", v6::rebuild_pds_account_counts), ("repo_state_event_clocks", v7::repo_state_event_clocks), + ("rebuild_lifecycle_counts", v8::rebuild_lifecycle_counts), ]; fn read_version(db: &Db) -> Result { diff --git a/src/db/migration/v7.rs b/src/db/migration/v7.rs index 8149950..f073c16 100644 --- a/src/db/migration/v7.rs +++ b/src/db/migration/v7.rs @@ -103,7 +103,7 @@ mod tests { .expect("db version should be set"); assert_eq!( u64::from_be_bytes(version_bytes.as_ref().try_into().into_diagnostic()?), - 7 + 8 ); Ok(()) diff --git a/src/db/migration/v8.rs b/src/db/migration/v8.rs new file mode 100644 index 0000000..634f2d3 --- /dev/null +++ b/src/db/migration/v8.rs @@ -0,0 +1,267 @@ +use fjall::OwnedWriteBatch; +use miette::Result; +#[cfg(feature = "indexer")] +use miette::{IntoDiagnostic, WrapErr}; + +use crate::db::Db; + +#[cfg(feature = "indexer")] +use crate::db::{deser_repo_meta, keys, set_ks_count}; +#[cfg(feature = "indexer")] +use crate::types::{GaugeState, ResyncErrorKind}; + +#[cfg(feature = "indexer")] +pub(crate) fn rebuild_lifecycle_counts(db: &Db, batch: &mut OwnedWriteBatch) -> Result<()> { + for guard in db.counts.prefix(keys::COUNT_GAUGE_PREFIX) { + let key = guard.key().into_diagnostic()?; + batch.remove(&db.counts, key); + } + + for guard in db.counts.prefix(keys::COUNT_DELTA_PREFIX) { + let key = guard.key().into_diagnostic()?; + let (_, name) = keys::parse_count_delta_key(&key)?; + if is_lifecycle_count(name) { + batch.remove(&db.counts, key); + } + } + + let mut pending = 0_u64; + let mut resync = 0_u64; + let mut error_ratelimited = 0_u64; + let mut error_transport = 0_u64; + let mut error_generic = 0_u64; + + for guard in db.repos.iter() { + let repo_key = guard.key().into_diagnostic()?; + let gauge = primary_lifecycle_gauge(db, repo_key.as_ref())?; + batch.insert( + &db.counts, + count_gauge_key_from_repo_key(repo_key.as_ref()), + crate::db::lifecycle_counts::encode_gauge(gauge), + ); + + match gauge { + GaugeState::Synced => {} + GaugeState::Pending => pending += 1, + GaugeState::Resync(None) => resync += 1, + GaugeState::Resync(Some(ResyncErrorKind::Ratelimited)) => { + resync += 1; + error_ratelimited += 1; + } + GaugeState::Resync(Some(ResyncErrorKind::Transport)) => { + resync += 1; + error_transport += 1; + } + GaugeState::Resync(Some(ResyncErrorKind::Generic)) => { + resync += 1; + error_generic += 1; + } + } + } + + set_ks_count(batch, db, "pending", pending); + set_ks_count(batch, db, "resync", resync); + set_ks_count(batch, db, "error_ratelimited", error_ratelimited); + set_ks_count(batch, db, "error_transport", error_transport); + set_ks_count(batch, db, "error_generic", error_generic); + + Ok(()) +} + +#[cfg(not(feature = "indexer"))] +pub(crate) fn rebuild_lifecycle_counts(_db: &Db, _batch: &mut OwnedWriteBatch) -> Result<()> { + Ok(()) +} + +#[cfg(feature = "indexer")] +fn primary_lifecycle_gauge(db: &Db, repo_key: &[u8]) -> Result { + let metadata_key = repo_metadata_key_from_repo_key(repo_key); + if let Some(metadata_bytes) = db.repo_metadata.get(&metadata_key).into_diagnostic()? { + let metadata = deser_repo_meta(metadata_bytes.as_ref()) + .wrap_err("invalid repo metadata during lifecycle count rebuild")?; + if db + .pending + .get(keys::pending_key(metadata.index_id)) + .into_diagnostic()? + .is_some() + { + return Ok(GaugeState::Pending); + } + } + + db.resync + .get(repo_key) + .into_diagnostic()? + .map(|bytes| crate::db::lifecycle_counts::gauge_from_resync(bytes.as_ref())) + .map(Ok) + .unwrap_or(Ok(GaugeState::Synced)) +} + +#[cfg(feature = "indexer")] +fn count_gauge_key_from_repo_key(repo_key: &[u8]) -> Vec { + let mut key = Vec::with_capacity(keys::COUNT_GAUGE_PREFIX.len() + repo_key.len()); + key.extend_from_slice(keys::COUNT_GAUGE_PREFIX); + key.extend_from_slice(repo_key); + key +} + +#[cfg(feature = "indexer")] +fn repo_metadata_key_from_repo_key(repo_key: &[u8]) -> Vec { + let mut key = Vec::with_capacity(keys::REPO_METADATA_PREFIX.len() + repo_key.len()); + key.extend_from_slice(keys::REPO_METADATA_PREFIX); + key.extend_from_slice(repo_key); + key +} + +#[cfg(feature = "indexer")] +fn is_lifecycle_count(name: &str) -> bool { + matches!( + name, + "pending" | "resync" | "error_ratelimited" | "error_transport" | "error_generic" + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[cfg(feature = "indexer")] + mod indexer { + use super::*; + use crate::config::Config; + use crate::db::{deser_repo_state, ser_repo_meta, ser_repo_state}; + use crate::types::{RepoMetadata, RepoState, RepoStatus, ResyncState}; + use jacquard_common::types::string::Did; + + fn test_config(path: &std::path::Path) -> Config { + Config { + database_path: path.to_path_buf(), + ..Default::default() + } + } + + fn did(value: &str) -> Result> { + use jacquard_common::IntoStatic; + Did::new(value) + .into_diagnostic() + .map(IntoStatic::into_static) + } + + fn insert_repo( + db: &Db, + batch: &mut OwnedWriteBatch, + did: &Did<'_>, + index_id: u64, + ) -> Result<()> { + batch.insert( + &db.repos, + keys::repo_key(did), + ser_repo_state(&RepoState::backfilling())?, + ); + batch.insert( + &db.repo_metadata, + keys::repo_metadata_key(did), + ser_repo_meta(&RepoMetadata::backfilling(index_id))?, + ); + Ok(()) + } + + #[test] + fn lifecycle_count_migration_rebuilds_skewed_counts_and_drops_old_deltas() -> Result<()> { + let tmp = tempfile::tempdir().into_diagnostic()?; + let cfg = test_config(tmp.path()); + + let pending_did = did("did:web:pending.example")?; + let ratelimited_did = did("did:web:ratelimited.example")?; + let gone_did = did("did:web:gone.example")?; + let synced_did = did("did:web:synced.example")?; + + { + let db = Db::open(&cfg)?; + let mut batch = db.inner.batch(); + + insert_repo(&db, &mut batch, &pending_did, 10)?; + insert_repo(&db, &mut batch, &ratelimited_did, 20)?; + insert_repo(&db, &mut batch, &gone_did, 30)?; + insert_repo(&db, &mut batch, &synced_did, 40)?; + + batch.insert( + &db.pending, + keys::pending_key(10), + keys::repo_key(&pending_did), + ); + batch.insert( + &db.resync, + keys::repo_key(&ratelimited_did), + rmp_serde::to_vec(&ResyncState::Error { + kind: ResyncErrorKind::Ratelimited, + retry_count: 3, + next_retry: 123, + }) + .into_diagnostic()?, + ); + batch.insert( + &db.resync, + keys::repo_key(&gone_did), + rmp_serde::to_vec(&ResyncState::Gone { + status: RepoStatus::Deactivated, + }) + .into_diagnostic()?, + ); + + set_ks_count(&mut batch, &db, "pending", 0); + set_ks_count(&mut batch, &db, "resync", 99); + set_ks_count(&mut batch, &db, "error_ratelimited", 42); + set_ks_count(&mut batch, &db, "error_transport", 42); + set_ks_count(&mut batch, &db, "error_generic", 42); + batch.insert( + &db.counts, + keys::count_delta_key(1, "pending"), + 10_i64.to_be_bytes(), + ); + batch.insert( + &db.counts, + keys::count_delta_key(2, "error_ratelimited"), + 10_i64.to_be_bytes(), + ); + batch.insert(&db.counts, keys::VERSIONING_KEY, 7_u64.to_be_bytes()); + batch.commit().into_diagnostic()?; + db.persist()?; + } + + let db = Db::open(&cfg)?; + + assert_eq!(db.get_count_sync("pending"), 1); + assert_eq!(db.get_count_sync("resync"), 2); + assert_eq!(db.get_count_sync("error_ratelimited"), 1); + assert_eq!(db.get_count_sync("error_transport"), 0); + assert_eq!(db.get_count_sync("error_generic"), 0); + assert_eq!(db.counts.prefix(keys::COUNT_GAUGE_PREFIX).count(), 4); + + for guard in db.counts.prefix(keys::COUNT_DELTA_PREFIX) { + let key = guard.key().into_diagnostic()?; + let (_, name) = keys::parse_count_delta_key(&key)?; + assert!(!is_lifecycle_count(name)); + } + + let version_bytes = db + .counts + .get(keys::VERSIONING_KEY) + .into_diagnostic()? + .expect("db version should be set"); + assert_eq!( + u64::from_be_bytes(version_bytes.as_ref().try_into().into_diagnostic()?), + 8 + ); + + let synced_state = db + .repos + .get(keys::repo_key(&synced_did)) + .into_diagnostic()? + .expect("synced repo should remain present"); + let _ = deser_repo_state(synced_state.as_ref())?; + + Ok(()) + } + } +} diff --git a/src/db/mod.rs b/src/db/mod.rs index 9443efb..0bcdb3d 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -30,10 +30,15 @@ pub mod compaction; pub mod ephemeral; pub mod filter; pub mod keys; +#[cfg(feature = "indexer")] +mod lifecycle_counts; pub mod migration; pub mod pds_meta; pub mod types; +#[cfg(feature = "indexer")] +pub(crate) use lifecycle_counts::LifecycleCountBatch; + use tracing::error; #[cfg(any(feature = "indexer_stream", feature = "relay"))] @@ -91,6 +96,8 @@ pub struct Db { count_delta_checkpoint_watermark: Arc, count_delta_gc_watermark: Arc, count_delta_in_flight: Arc>>, + #[cfg(feature = "indexer")] + lifecycle_count_lock: Arc>, } #[derive(Debug, Clone, Default)] @@ -99,7 +106,7 @@ pub struct CountDeltas { } impl CountDeltas { - pub(crate) fn add(&mut self, key: &str, delta: i64) { + fn add(&mut self, key: &str, delta: i64) { if delta == 0 { return; } @@ -111,12 +118,30 @@ impl CountDeltas { } } + pub(crate) fn add_repos(&mut self, delta: i64) { + self.add("repos", delta); + } + + #[cfg(feature = "indexer")] + pub(crate) fn add_records(&mut self, delta: i64) { + self.add("records", delta); + } + #[cfg(feature = "indexer")] - pub(crate) fn add_gauge_diff( - &mut self, - old: &crate::types::GaugeState, - new: &crate::types::GaugeState, - ) { + pub(crate) fn add_blocks(&mut self, delta: i64) { + self.add("blocks", delta); + } + + pub(crate) fn add_pds_account(&mut self, host: &str, delta: i64) { + self.add(&keys::pds_account_count_key(host), delta); + } + + pub(crate) fn projected_pds_account_count(&self, db: &Db, host: &str) -> u64 { + self.projected_count(db, &keys::pds_account_count_key(host)) + } + + #[cfg(feature = "indexer")] + fn add_gauge_diff(&mut self, old: &crate::types::GaugeState, new: &crate::types::GaugeState) { use crate::types::GaugeState; if old == new { @@ -171,7 +196,7 @@ impl CountDeltas { self.deltas.get(key).copied().unwrap_or(0) } - pub(crate) fn projected_count(&self, db: &Db, key: &str) -> u64 { + fn projected_count(&self, db: &Db, key: &str) -> u64 { apply_count_delta(db.get_count_sync(key), self.get(key)) } @@ -592,6 +617,8 @@ impl Db { count_delta_checkpoint_watermark: Arc::new(AtomicU64::new(0)), count_delta_gc_watermark, count_delta_in_flight: Arc::new(Mutex::new(BTreeSet::new())), + #[cfg(feature = "indexer")] + lifecycle_count_lock: Arc::new(Mutex::new(())), }; migration::run(&this)?; @@ -1204,8 +1231,8 @@ mod tests { let mut batch = db.inner.batch(); let mut deltas = CountDeltas::default(); - deltas.add("repos", 2); - deltas.add("pending", 1); + deltas.add_repos(2); + deltas.add_records(1); let reservation = db .stage_count_deltas(&mut batch, &deltas) .expect("count deltas should reserve ids"); @@ -1218,7 +1245,7 @@ mod tests { { let db = Db::open(&cfg)?; assert_eq!(db.get_count_sync("repos"), 12); - assert_eq!(db.get_count_sync("pending"), 1); + assert_eq!(db.get_count_sync("records"), 1); let checkpointed_watermark = db .checkpoint_count_deltas()? .expect("checkpoint should fold pending deltas"); @@ -1230,7 +1257,7 @@ mod tests { { let db = Db::open(&cfg)?; assert_eq!(db.get_count_sync("repos"), 12); - assert_eq!(db.get_count_sync("pending"), 1); + assert_eq!(db.get_count_sync("records"), 1); assert!(load_count_delta_watermark(&db)? >= 1); } @@ -1245,7 +1272,7 @@ mod tests { let mut batch = db.inner.batch(); let mut deltas = CountDeltas::default(); - deltas.add("repos", 1); + deltas.add_repos(1); let reservation = db .stage_count_deltas(&mut batch, &deltas) .expect("count deltas should reserve ids"); @@ -1269,7 +1296,7 @@ mod tests { let mut batch = db.inner.batch(); let mut deltas = CountDeltas::default(); - deltas.add("repos", 1); + deltas.add_repos(1); let reservation = db .stage_count_deltas(&mut batch, &deltas) .expect("count deltas should reserve ids"); diff --git a/src/ingest/indexer.rs b/src/ingest/indexer.rs index b0e0d8d..cad97b4 100644 --- a/src/ingest/indexer.rs +++ b/src/ingest/indexer.rs @@ -169,6 +169,7 @@ pub struct FirehoseWorker { struct WorkerContext<'a> { state: &'a AppState, batch: OwnedWriteBatch, + lifecycle_transitions: &'a mut Vec<(Did<'static>, GaugeState)>, added_blocks: &'a mut i64, records_delta: &'a mut i64, count_deltas: &'a mut CountDeltas, @@ -222,6 +223,7 @@ impl FirehoseWorker { let mut broadcast_events = Vec::new(); #[cfg(feature = "jetstream")] let mut jetstream_events = Vec::new(); + let mut lifecycle_transitions = Vec::new(); while let Some(msg) = rx.blocking_recv() { let batch = state.db.inner.batch(); @@ -229,6 +231,7 @@ impl FirehoseWorker { broadcast_events.clear(); #[cfg(feature = "jetstream")] jetstream_events.clear(); + lifecycle_transitions.clear(); let mut added_blocks = 0; let mut records_delta = 0; @@ -237,6 +240,7 @@ impl FirehoseWorker { let mut ctx = WorkerContext { state: &state, batch, + lifecycle_transitions: &mut lifecycle_transitions, added_blocks: &mut added_blocks, records_delta: &mut records_delta, count_deltas: &mut count_deltas, @@ -262,6 +266,7 @@ impl FirehoseWorker { let res = ops::transition_repo( &mut ctx.batch, &state.db, + ctx.lifecycle_transitions, &did, s, RepoStatus::Synced, @@ -368,7 +373,7 @@ impl FirehoseWorker { ) { Ok(RepoProcessResult::Ok(_)) => {} Ok(RepoProcessResult::Deleted) => { - ctx.count_deltas.add("repos", -1); + ctx.count_deltas.add_repos(-1); } Ok(RepoProcessResult::NeedsBackfill(Some(commit))) => { try_persist(commit); @@ -420,17 +425,32 @@ impl FirehoseWorker { let mut batch = ctx.batch; if added_blocks > 0 { - count_deltas.add("blocks", added_blocks); + count_deltas.add_blocks(added_blocks); } if records_delta != 0 { - count_deltas.add("records", records_delta); + count_deltas.add_records(records_delta); + } + let mut lifecycle_counts = state.db.lifecycle_counts(); + let mut lifecycle_failed = false; + for (did, gauge) in lifecycle_transitions.drain(..) { + if let Err(e) = lifecycle_counts.transition(&mut batch, &did, gauge) { + error!(did = %did, err = %e, "failed to stage lifecycle transition"); + lifecycle_failed = true; + break; + } + } + if lifecycle_failed { + continue; } let reservation = state.db.stage_count_deltas(&mut batch, &count_deltas); + let lifecycle_reservation = lifecycle_counts.stage(&mut batch); if let Err(e) = batch.commit() { error!(shard = id, err = %e, "failed to commit batch"); + drop(lifecycle_reservation); drop(reservation); continue; } + state.db.apply_lifecycle_counts(lifecycle_reservation); state.db.apply_count_deltas(&count_deltas); drop(reservation); #[cfg(feature = "indexer_stream")] @@ -609,6 +629,8 @@ impl FirehoseWorker { match &account.status { Some(AccountStatus::Deleted) => { debug!("account deleted, wiping data"); + ctx.lifecycle_transitions + .push((did.clone().into_static(), GaugeState::Synced)); crate::ops::delete_repo(&mut ctx.batch, db, did, &repo_state)?; return Ok(RepoProcessResult::Deleted); } @@ -616,16 +638,16 @@ impl FirehoseWorker { // status update logic is now handled in RelayWorker; // FirehoseWorker just needs to update gauges if status changed. if changed && was_active { - ctx.count_deltas - .add_gauge_diff(&GaugeState::Synced, &GaugeState::Resync(None)); + ctx.lifecycle_transitions + .push((did.clone().into_static(), GaugeState::Resync(None))); } } } } else { // if account became active, update gauges if !was_active { - ctx.count_deltas - .add_gauge_diff(&GaugeState::Resync(None), &GaugeState::Synced); + ctx.lifecycle_transitions + .push((did.clone().into_static(), GaugeState::Synced)); } } @@ -692,13 +714,10 @@ impl FirehoseWorker { ) -> Result, IngestError> { let db = &ctx.state.db; let mut batch = db.inner.batch(); - let mut count_deltas = CountDeltas::default(); + let mut lifecycle_counts = db.lifecycle_counts(); let repo_key = keys::repo_key(did); let meta_key = keys::repo_metadata_key(did); - let resync_bytes = db.resync.get(&repo_key).into_diagnostic()?; - let old_gauge = crate::db::Db::repo_gauge_state(&repo_state, resync_bytes.as_deref()); - let existing_metadata = db .repo_metadata .get(&meta_key) @@ -728,12 +747,11 @@ impl FirehoseWorker { batch.insert(&db.pending, keys::pending_key(metadata.index_id), &repo_key); batch.insert(&db.repo_metadata, &meta_key, ser_repo_meta(&metadata)?); if !was_pending { - count_deltas.add_gauge_diff(&old_gauge, &crate::types::GaugeState::Pending); + lifecycle_counts.transition(&mut batch, did, GaugeState::Pending)?; } - let reservation = db.stage_count_deltas(&mut batch, &count_deltas); + let lifecycle_reservation = lifecycle_counts.stage(&mut batch); batch.commit().into_diagnostic()?; - db.apply_count_deltas(&count_deltas); - drop(reservation); + db.apply_lifecycle_counts(lifecycle_reservation); if !was_pending { ctx.state.notify_backfill(); diff --git a/src/ingest/relay.rs b/src/ingest/relay.rs index 67a9296..cacf463 100644 --- a/src/ingest/relay.rs +++ b/src/ingest/relay.rs @@ -18,7 +18,6 @@ use tokio::runtime::Handle; use tracing::{debug, error, info, info_span, trace, warn}; use url::Url; -use crate::db::keys::pds_account_count_key; #[cfg(all(feature = "relay", feature = "jetstream"))] use crate::db::types::TrimmedDid; use crate::db::{self, CountDeltas, keys}; @@ -859,9 +858,10 @@ impl RelayWorker { } let mut update_host = |host: &str, delta| { - let count_key = pds_account_count_key(host); - ctx.count_deltas.add(&count_key, delta); - let count = ctx.count_deltas.projected_count(&ctx.state.db, &count_key); + ctx.count_deltas.add_pds_account(host, delta); + let count = ctx + .count_deltas + .projected_pds_account_count(&ctx.state.db, host); ctx.state .apply_host_limit_status(&mut ctx.batch, host, count); }; @@ -1251,7 +1251,10 @@ impl WorkerContext<'_> { } if let Some(host) = msg.firehose.host_str() { - let count = self.state.db.get_count_sync(&pds_account_count_key(host)); + let count = self + .state + .db + .get_count_sync(&keys::pds_account_count_key(host)); if self.state.is_over_account_limit(host, count) { warn!(did = %did, host, count, "account limit reached for host, dropping new account"); #[cfg(feature = "firehose-diagnostics")] @@ -1303,7 +1306,7 @@ impl WorkerContext<'_> { )); } - self.count_deltas.add("repos", 1); + self.count_deltas.add_repos(1); #[cfg(feature = "firehose-diagnostics")] { diff --git a/src/ops.rs b/src/ops.rs index de11ca4..19de2a3 100644 --- a/src/ops.rs +++ b/src/ops.rs @@ -3,6 +3,7 @@ use fjall::Slice; #[cfg(feature = "backlinks")] use jacquard_common::Data; +use jacquard_common::IntoStatic; use jacquard_common::types::did::Did; use miette::{Context, IntoDiagnostic, Result}; use std::collections::HashMap; @@ -14,7 +15,7 @@ use crate::filter::FilterConfig; use crate::ingest::stream::Commit; use crate::ingest::validation::ValidatedCommit; use crate::state::AppState; -use crate::types::{RepoState, RepoStatus, ResyncState}; +use crate::types::{GaugeState, RepoState, RepoStatus, ResyncErrorKind, ResyncState}; #[cfg(feature = "indexer_stream")] use { @@ -22,7 +23,7 @@ use { AccountEvt, BroadcastEvent, IdentityEvt, LiveRecordEvent, MarshallableEvt, StoredData, StoredEvent, }, - jacquard_common::{CowStr, IntoStatic}, + jacquard_common::CowStr, std::sync::atomic::Ordering, }; @@ -86,7 +87,6 @@ pub fn delete_repo( batch.remove(&db.pending, keys::pending_key(metadata.index_id)); } - // 1. delete from resync, and metadata // we don't delete from repos, relay uses it as a tombstone // todo: we should still delete it after some time batch.remove(&db.resync, &repo_key); @@ -130,6 +130,7 @@ pub fn delete_repo( pub fn transition_repo<'s>( batch: &mut OwnedWriteBatch, db: &Db, + lifecycle_transitions: &mut Vec<(Did<'static>, GaugeState)>, did: &Did, mut repo_state: RepoState<'s>, new_status: RepoStatus, @@ -147,11 +148,16 @@ pub fn transition_repo<'s>( // manage queues match &new_status { RepoStatus::Synced => { + lifecycle_transitions.push((did.clone().into_static(), GaugeState::Synced)); batch.remove(&db.pending, pending_key.as_slice()); // we dont have to remove from resync here because it has to transition resync -> pending first } RepoStatus::Error(msg) => { tracing::warn!("transitioning to error: {msg}"); + lifecycle_transitions.push(( + did.clone().into_static(), + GaugeState::Resync(Some(ResyncErrorKind::Generic)), + )); batch.remove(&db.pending, pending_key.as_slice()); // TODO: we need to make errors have kind instead of "message" in repo status // and then pass it to resync error kind @@ -167,6 +173,7 @@ pub fn transition_repo<'s>( ); } RepoStatus::Deactivated | RepoStatus::Takendown | RepoStatus::Suspended => { + lifecycle_transitions.push((did.clone().into_static(), GaugeState::Resync(None))); // this shouldnt be needed since a repo wont be in a pending state when it gets to any of these states // batch.remove(&db.pending, &pending_key); let resync_state = ResyncState::Gone { @@ -179,11 +186,16 @@ pub fn transition_repo<'s>( ); } RepoStatus::Deleted => { + lifecycle_transitions.push((did.clone().into_static(), GaugeState::Synced)); // terminal state: remove from queues, no resync entry needed batch.remove(&db.pending, pending_key.as_slice()); batch.remove(&db.resync, &repo_key); } RepoStatus::Desynchronized | RepoStatus::Throttled => { + lifecycle_transitions.push(( + did.clone().into_static(), + GaugeState::Resync(Some(ResyncErrorKind::Generic)), + )); // like an error: remove from pending and schedule a resync attempt batch.remove(&db.pending, pending_key.as_slice()); let resync_state = crate::types::ResyncState::Error { -- knotmirror 0.0.1