use crate::db::types::DbTid; use crate::db::{keys, ser_repo_state}; use crate::ingest::indexer::IndexerMessage; #[cfg(test)] use crate::ingest::indexer::message::shard_of; use crate::ingest::indexer::refetch::{self, Charged}; use crate::ingest::indexer::repos::on_owners; use crate::ingest::validation::is_future_rev; use crate::state::AppState; use crate::types::{BulkJobV1, GaugeState, RepoMetadata, RepoState, RepoStatus, ResyncState}; pub(crate) type ListedRepoStatus = crate::ingest::stream::AccountStatus<'static>; use jacquard_common::IntoStatic; use jacquard_common::types::did::Did; use miette::{IntoDiagnostic, Result}; use smol_str::{SmolStr, ToSmolStr}; use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; use std::time::Duration; use tokio::sync::{mpsc, oneshot}; use tracing::{debug, error, info, trace, warn}; use super::{CrawlerStats, InFlightGuard}; const WORKER_CHANNEL_CAPACITY: usize = 64; const BLOCKING_TASK_TIMEOUT: Duration = Duration::from_secs(30); #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum CapacityDecision { Proceed, EnterThrottle, StayThrottled, ReleaseThrottle, } fn capacity_decision( pending: u64, max_pending: usize, resume_pending: usize, was_throttled: bool, ) -> CapacityDecision { if pending > max_pending as u64 { if was_throttled { CapacityDecision::StayThrottled } else { CapacityDecision::EnterThrottle } } else if was_throttled && pending > resume_pending as u64 { CapacityDecision::StayThrottled } else if was_throttled { CapacityDecision::ReleaseThrottle } else { CapacityDecision::Proceed } } /// a cursor write to include atomically with the batch. pub(crate) struct CursorUpdate { pub(super) key: Vec, pub(super) value: Option>, } #[derive(Clone)] pub(crate) struct RepoListing { pub(super) did: Did, pub(super) rev: DbTid, pub(super) active: bool, pub(super) status: Option, pub(super) create_if_missing: bool, } impl RepoListing { /// a deleted account's listing, which has its owner shard wipe the repo once the listing /// changed it fn starts_a_wipe(&self) -> bool { !self.active && listed_repo_status(self.status.as_ref()) == RepoStatus::Deleted } /// a rev more than `skew_secs` ahead of our clock never counts as newer pub(crate) fn claims_newer(&self, stored: &RepoState<'_>, skew_secs: i64) -> bool { self.active && !is_future_rev(&self.rev.to_tid(), skew_secs) && stored .root .as_ref() .is_none_or(|root| self.rev.as_bytes() > root.rev.as_bytes()) } } /// how far past a page's read time a relay firehose has to have handed events on before the /// page's newer revs are judged. it covers the relay's clock running ahead of ours and the /// events still waiting in the shards' mailboxes, which is seconds at most const DISPATCH_MARGIN: Duration = Duration::from_secs(10); /// how long a page's newer revs wait for the firehose before they're skipped, so a relay that's /// down or stuck never holds a pass up. the next pass looks at them again const DISPATCH_TIMEOUT: Duration = Duration::from_secs(600); /// how long [`wait_for_firehose`] waits #[derive(Debug, Clone, Copy)] pub(crate) struct DispatchWait { pub(crate) margin: Duration, pub(crate) timeout: Duration, pub(crate) poll: Duration, } impl DispatchWait { pub(crate) const DEFAULT: Self = Self { margin: DISPATCH_MARGIN, timeout: DISPATCH_TIMEOUT, poll: Duration::from_secs(1), }; } /// waits until the relay firehose for `host` has handed on events past `read_at_ms` and the /// margin, so a listing read then can't claim a commit that's still on its way through /// hydrant. false when it didn't get there by `deadline`. with no relay firehose running /// there's nothing to wait for pub(crate) async fn wait_for_firehose( state: &AppState, host: &str, read_at_ms: i64, wait: &DispatchWait, deadline: tokio::time::Instant, ) -> bool { let target = read_at_ms.saturating_add(wait.margin.as_millis() as i64); loop { match state.firehose_dispatched(host) { None => return true, Some(dispatched) if dispatched >= target => return true, Some(_) => {} } if tokio::time::Instant::now() >= deadline { return false; } tokio::time::sleep(wait.poll).await; } } /// how many listing pages may wait for the firehose at once before the crawl waits too pub(crate) const WAITING_PAGES: usize = 32; /// the rest of a listing page: the listings that wait for the firehose, and the page's cursor struct WaitingPage { host: SmolStr, read_at_ms: i64, deadline: tokio::time::Instant, newer: Vec, cursor_updates: Vec, /// which crawl of the producer read the page, see [`WaitingPages::begin_crawl`] crawl: u64, /// whether the worker took the part of the page that went right away, none if nothing did applied: Option>, } /// the listing pages a crawl has read but not finished, in the order it read them. a task of /// its own hands each page's rest to the worker once the firehose caught up to it, so a page /// that waits doesn't stop the crawl reading the next ones, and still no page's cursor moves /// before every page ahead of it is in pub(crate) struct WaitingPages { pages: mpsc::Sender, wait: DispatchWait, crawl: AtomicU64, /// the latest crawl the worker failed a page of. that page's cursor and every later one of /// the crawl are dropped, so the crawl has to start over from the saved cursor failed: Arc, task: tokio::task::AbortHandle, } impl Drop for WaitingPages { /// a producer that's gone (its source removed or replaced) mustn't move its cursor anymore fn drop(&mut self) { self.task.abort(); } } impl WaitingPages { /// at most `bound` pages wait in line, past that [`send_page`] waits for room pub(crate) fn spawn( tx: mpsc::Sender, state: Arc, wait: DispatchWait, bound: usize, ) -> Self { let (pages, mut waiting) = mpsc::channel::(bound); let failed = Arc::new(AtomicU64::new(0)); let failed_crawl = failed.clone(); let task = tokio::spawn(async move { while let Some(page) = waiting.recv().await { let WaitingPage { host, read_at_ms, deadline, newer, cursor_updates, crawl, applied, } = page; if crawl <= failed_crawl.load(Ordering::Acquire) { continue; } // a later page reads no earlier than this one, so once this one is through the // ones behind it mostly are too, and waiting for them in turn costs little let caught_up = newer.is_empty() || wait_for_firehose(&state, &host, read_at_ms, &wait, deadline).await; let newer = if caught_up { newer } else { info!( %host, skipped = newer.len(), "the firehose didn't catch up to a listing page in time, skipping its newer revs until the next pass" ); Vec::new() }; // a shard that didn't take the page's repos would otherwise see its cursor move // past them if let Some(applied) = applied && !applied.await.unwrap_or(false) { failed_crawl.fetch_max(crawl, Ordering::AcqRel); warn!(%host, "the worker failed a listing page, crawling again from its saved cursor"); continue; } if newer.is_empty() && cursor_updates.is_empty() { continue; } let rest = CrawlerBatch { guards: Vec::new(), listings: newer, cursor_updates, applied: None, }; if tx.send(rest).await.is_err() { return; } } }); Self { pages, wait, crawl: AtomicU64::new(1), failed, task: task.abort_handle(), } } /// starts a crawl from the saved cursor, which takes it past a page the worker failed pub(crate) fn begin_crawl(&self) { self.crawl.fetch_add(1, Ordering::AcqRel); } } /// hands a listing page from `host`, read at `read_at_ms`, to the worker. everything goes right /// away but `newer`, the listings that claim a newer rev than stored, which waits in `waiting` /// for the relay firehose to catch up to the page, see [`wait_for_firehose`]. the page's cursor /// waits there too, behind every page read before it. fails once the worker failed a page of /// this crawl, whose cursor then never moves, so the crawl has to start over from the saved one pub(crate) async fn send_page( tx: &mpsc::Sender, waiting: &WaitingPages, host: &str, read_at_ms: i64, page: CrawlerBatch, newer: Vec, ) -> Result<()> { let crawl = waiting.crawl.load(Ordering::Acquire); if waiting.failed.load(Ordering::Acquire) >= crawl { miette::bail!("the worker failed a listing page of this crawl"); } let CrawlerBatch { guards, listings, cursor_updates, applied: _, } = page; let mut applied = None; if !(guards.is_empty() && listings.is_empty()) { let (done, applied_rx) = oneshot::channel(); applied = Some(applied_rx); let now = CrawlerBatch { guards, listings, cursor_updates: Vec::new(), applied: Some(done), }; if tx.send(now).await.is_err() { return Ok(()); } } let rest = WaitingPage { host: host.into(), read_at_ms, deadline: tokio::time::Instant::now() + waiting.wait.timeout, newer, cursor_updates, crawl, applied, }; let _ = waiting.pages.send(rest).await; Ok(()) } #[cfg(test)] impl RepoListing { pub(crate) fn inactive(did: Did, rev: DbTid, status: ListedRepoStatus) -> Self { Self { did, rev, active: false, status: Some(status), create_if_missing: false, } } pub(crate) fn active(did: Did, rev: DbTid) -> Self { Self { did, rev, active: true, status: None, create_if_missing: false, } } } /// a batch of confirmed repos from any crawler source ready to enqueue. /// /// `guards` hold the DIDs (via `Deref`) and keep their in-flight slots occupied /// until their repos are committed to the database. cursor updates commit only /// after every repo of the page is, so a crash can replay but never skip a /// partially applied page. pub(crate) struct CrawlerBatch { pub(super) guards: Vec, pub(super) listings: Vec, pub(super) cursor_updates: Vec, /// told whether the batch went in, for a page whose cursor comes in a later batch pub(super) applied: Option>, } pub(crate) struct CrawlerWorker { pub(super) state: Arc, pub(super) max_pending: usize, pub(super) resume_pending: usize, pub(super) stats: CrawlerStats, pub(super) rx: mpsc::Receiver, pub(super) was_throttled: bool, } impl CrawlerWorker { pub(crate) fn new( state: Arc, max_pending: usize, resume_pending: usize, stats: CrawlerStats, ) -> (Self, mpsc::Sender) { let (tx, rx) = mpsc::channel(WORKER_CHANNEL_CAPACITY); ( Self { state, max_pending, resume_pending, stats, rx, was_throttled: false, }, tx, ) } pub(crate) async fn run(mut self) { while let Some(mut batch) = self.rx.recv().await { let applied = batch.applied.take(); self.wait_for_capacity().await; let result = self.enqueue(batch).await; if let Err(e) = &result { error!(err = ?e, "crawler worker: enqueue failed"); } if let Some(applied) = applied { let _ = applied.send(result.is_ok()); } } } /// blocks until the pending queue has capacity. mirrors the hysteresis logic /// that was previously duplicated in each crawler loop: /// - above `max_pending`: hard stop, poll every 5s /// - after crossing `max_pending`: cooldown until below `resume_pending` /// - below `resume_pending`: proceed (or release throttle) async fn wait_for_capacity(&mut self) { loop { let pending = self.state.db.get_count("pending").await; match capacity_decision( pending, self.max_pending, self.resume_pending, self.was_throttled, ) { CapacityDecision::Proceed => break, CapacityDecision::EnterThrottle => { debug!( pending, max = self.max_pending, "throttling: above max pending" ); self.was_throttled = true; self.stats.set_throttled(true); tokio::time::sleep(Duration::from_secs(5)).await; } CapacityDecision::StayThrottled => { tokio::time::sleep(Duration::from_secs(5)).await; } CapacityDecision::ReleaseThrottle => { self.was_throttled = false; self.stats.set_throttled(false); info!("throttling released"); break; } } } } /// filters already-known repos, commits them to `repos` + `pending` (plus /// an optional cursor update), then releases the in-flight guards. async fn enqueue(&mut self, batch: CrawlerBatch) -> Result<()> { let CrawlerBatch { guards, listings, cursor_updates, .. } = batch; // nothing to insert but still need to commit the cursor if present. if guards.is_empty() && listings.is_empty() { if !cursor_updates.is_empty() { self.commit_cursors(cursor_updates).await?; } return Ok(()); } // every repo is written on its owner shard, so the page never holds one repo while it // waits on another. the cursor only commits once every shard answered, so a page that a // shard didn't get to is crawled again, and the listings it did apply change nothing // the second time let dids = guards.iter().map(|guard| guard.did.clone()).collect(); let ( Applied { added, changed_listings, reconciled, .. }, failed, ) = apply_on_owners(&self.state, listings, dids).await; if let Some(e) = failed { return Err(e); } if !cursor_updates.is_empty() { self.commit_cursors(cursor_updates).await?; } let count = added.len(); // release in-flight slots now that the batch is committed drop(guards); if count > 0 { self.stats.record_processed(count); } if count > 0 || reconciled > 0 { self.state.notify_backfill(); } #[cfg(feature = "indexer_stream")] if !changed_listings.is_empty() { crate::ops::announce(&self.state.db, move |events| { for listing in changed_listings { events.push_account(crate::types::AccountEvt { did: listing.did, active: listing.active, status: listing.status.map(|status| { jacquard_common::CowStr::Owned(status.as_str().to_smolstr()) }), }); } }) .await; } #[cfg(not(feature = "indexer_stream"))] drop(changed_listings); Ok(()) } async fn commit_cursors(&self, cursors: Vec) -> Result<()> { let state = self.state.clone(); tokio::time::timeout( BLOCKING_TASK_TIMEOUT, state.db.run(move |db| { let mut txn = crate::db::Txn::new(db); for cursor in cursors { stage_cursor(&mut txn.batch, db, cursor); } txn.commit() }), ) .await .map_err(|_| miette::miette!("cursor-only commit timed out"))??; Ok(()) } } /// what a page's writes came to #[derive(Default)] struct Applied { /// unknown repos it queued added: Vec, /// listings that changed whether the account is active changed_listings: Vec, /// deleted accounts whose wipe a listing started wipes: Vec, /// listings that changed their repo, a queued backfill among them reconciled: usize, } /// one write a page asks of a repo's owner shard #[derive(Clone)] enum PageWrite { Listing(RepoListing), /// an unknown repo to queue Unknown(Did), } impl PageWrite { fn did(&self) -> &Did { match self { PageWrite::Listing(listing) => &listing.did, PageWrite::Unknown(did) => did, } } } /// writes `listings` and queues the unknown `dids`, each on its owner shard, and says why /// a shard failed if one did. the others' writes stay either way async fn apply_on_owners( state: &Arc, listings: Vec, dids: Vec, ) -> (Applied, Option) { let writes = listings .into_iter() .map(PageWrite::Listing) .chain(dids.into_iter().map(PageWrite::Unknown)) .collect(); let answers = on_owners(state, writes, PageWrite::did, |state, shard, writes| { let (mut listings, mut dids) = (Vec::new(), Vec::new()); for write in writes { match write { PageWrite::Listing(listing) => listings.push(listing), PageWrite::Unknown(did) => dids.push(did), } } let applied = apply_listings(state, &listings, &dids)?; // the owner moves a deleted account's heads to history a step at a time, and with no // shard running yet the marker each wipe left resumes it at the next start if let Some(shard) = shard { for did in &applied.wipes { shard.mark(did.clone()); } } Ok(applied) }) .await; let mut applied = Applied::default(); let mut failed = None; for answer in answers { match answer.and_then(|batch| batch) { Ok(batch) => { applied.added.extend(batch.added); applied.changed_listings.extend(batch.changed_listings); applied.wipes.extend(batch.wipes); applied.reconciled += batch.reconciled; } Err(e) => failed = Some(e), } } (applied, failed) } /// writes one shard's share of a page, a txn per repo fn apply_listings(state: &AppState, listings: &[RepoListing], dids: &[Did]) -> Result { let db = &state.db; let mut applied = Applied::default(); for listing in listings { let mut txn = crate::db::Txn::new(db); let outcome = reconcile_listing(state, &mut txn, listing)?; txn.commit()?; if outcome != ReconcileOutcome::Unchanged { applied.reconciled += 1; if listing.starts_a_wipe() { applied.wipes.push(listing.did.clone()); } } if outcome.emits_account() { applied.changed_listings.push(listing.clone()); } } for did in dids { if db .filter .contains_key(crate::db::filter::exclude_key(did.as_str())?) .into_diagnostic()? { continue; } let mut txn = crate::db::Txn::new(db); if txn.read_repo(did)?.is_some() { continue; } let metadata = RepoMetadata::backfilling(rand::random()); txn.put_repo(did, &RepoState::backfilling())?; txn.batch.insert( &db.repo_metadata, keys::repo_metadata_key(did), crate::db::ser_repo_meta(&metadata)?, ); db.indexer .stage_pending_insert(&mut txn.batch, did, &keys::pending_key(metadata.index_id)); txn.batch.remove(&db.crawler, keys::crawler_retry_key(did)); trace!(did = %did, "enqueuing repo"); txn.counts.add_repos(1); txn.transition_lifecycle(did, GaugeState::Pending)?; txn.commit()?; applied.added.push(did.clone()); } Ok(applied) } fn stage_cursor(batch: &mut fjall::OwnedWriteBatch, db: &crate::db::Db, cursor: CursorUpdate) { match cursor.value { Some(value) => batch.insert(&db.cursors, cursor.key, value), None => batch.remove(&db.cursors, cursor.key), } } #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum ReconcileOutcome { Unchanged, StateOnly, AccountChanged, } impl ReconcileOutcome { fn emits_account(self) -> bool { self == Self::AccountChanged } } pub(crate) fn reconcile_listing( state: &AppState, txn: &mut crate::db::Txn<'_>, listing: &RepoListing, ) -> Result { let db = txn.db; if txn.is_excluded(&listing.did)? { return Ok(ReconcileOutcome::Unchanged); } let did_key = keys::repo_key(&listing.did); let Some(mut repo) = txn.read_repo(&listing.did)? else { if listing.active || !listing.create_if_missing { return Ok(ReconcileOutcome::Unchanged); } let status = listed_repo_status(listing.status.as_ref()); let mut repo = RepoState::backfilling(); repo.active = false; repo.status = status.clone(); if status != RepoStatus::Deleted { txn.batch.insert( &db.indexer.resync, &did_key, rmp_serde::to_vec(&ResyncState::Gone { status: status.clone(), }) .into_diagnostic()?, ); txn.transition_lifecycle(&listing.did, GaugeState::Resync(None))?; } txn.put_repo(&listing.did, &repo)?; txn.counts.add_repos(1); return Ok(ReconcileOutcome::StateOnly); }; if listing.active { let listing_is_newer = listing.claims_newer(&repo, state.rev_clock_skew_secs); if repo.active && !listing_is_newer { return Ok(ReconcileOutcome::Unchanged); } if !listing_is_newer { // the inactive status can be the pds's own answer, so a relay saying active // doesn't beat it either return ask_pds(state, txn, &mut repo, &listing.did, RepoStatus::Synced); } if listing_is_pending(db, &listing.did)? || !listing_may_queue(db, &listing.did)? { return Ok(ReconcileOutcome::Unchanged); } let was_active = repo.active; if enqueue_listing_backfill(state, txn, &mut repo, &listing.did)? == Charged::Deferred { return Ok(ReconcileOutcome::StateOnly); } txn.batch.remove(&db.indexer.resync, &did_key); txn.transition_lifecycle(&listing.did, GaugeState::Pending)?; repo.status = RepoStatus::Desynchronized; repo.active = true; repo.touch(); txn.put_repo(&listing.did, &repo)?; return Ok(if was_active { ReconcileOutcome::StateOnly } else { ReconcileOutcome::AccountChanged }); } let status = listed_repo_status(listing.status.as_ref()); if status != RepoStatus::Deleted { // a relay can take a repo down while its pds still serves it, and several relays can // disagree, so only the pds's own answer makes the account inactive return if repo.active { ask_pds(state, txn, &mut repo, &listing.did, status) } else { Ok(ReconcileOutcome::Unchanged) }; } let account_changed = repo.active || repo.status != status; if !account_changed && !listing_is_pending(db, &listing.did)? { return Ok(ReconcileOutcome::Unchanged); } txn.transition_lifecycle(&listing.did, GaugeState::Synced)?; crate::ops::start_wipe(&mut *txn, db, &listing.did, &repo)?; repo.root = None; repo.active = false; repo.status = status; repo.touch(); txn.put_repo(&listing.did, &repo)?; Ok(if account_changed { ReconcileOutcome::AccountChanged } else { ReconcileOutcome::StateOnly }) } /// queues a fetch, because only the pds's answer to it changes whether the account is active. /// the marker keeps the next hourly pass from fetching it again for the same listed status fn ask_pds( state: &AppState, txn: &mut crate::db::Txn<'_>, repo: &mut RepoState<'_>, did: &Did, status: RepoStatus, ) -> Result { let db = &state.db; let metadata_key = keys::repo_metadata_key(did); let Some(mut metadata) = db .repo_metadata .get(&metadata_key) .into_diagnostic()? .map(|bytes| crate::db::deser_repo_meta(&bytes)) .transpose()? else { return Ok(ReconcileOutcome::Unchanged); }; if !metadata.tracked || metadata.relay_inactive.as_ref() == Some(&status) { return Ok(ReconcileOutcome::Unchanged); } let pending_key = keys::pending_key(metadata.index_id); if !db.indexer.is_pending(&pending_key).into_diagnostic()? { if refetch::charge(state, txn, did, repo, &mut metadata, "listed status")? == Charged::Deferred { return Ok(ReconcileOutcome::StateOnly); } metadata.index_id = rand::random(); db.indexer .stage_pending_insert(&mut txn.batch, did, &keys::pending_key(metadata.index_id)); txn.batch.remove(&db.indexer.resync, keys::repo_key(did)); txn.transition_lifecycle(did, GaugeState::Pending)?; } metadata.relay_inactive = Some(status); txn.batch.insert( &db.repo_metadata, metadata_key, crate::db::ser_repo_meta(&metadata)?, ); Ok(ReconcileOutcome::StateOnly) } fn listing_is_pending(db: &crate::db::Db, did: &Did) -> Result { let Some(metadata_bytes) = db .repo_metadata .get(keys::repo_metadata_key(did)) .into_diagnostic()? else { return Ok(false); }; let metadata = crate::db::deser_repo_meta(&metadata_bytes)?; db.indexer .is_pending(&keys::pending_key(metadata.index_id)) .into_diagnostic() } /// an untracked repo stays that way whatever a relay lists, and one its owner shard is /// erasing or wiping would have the new backfill write into the job fn listing_may_queue(db: &crate::db::Db, did: &Did) -> Result { if matches!( crate::db::read_bulk_job(db, did)?, Some(BulkJobV1::Erase | BulkJobV1::Wipe { .. }) ) { return Ok(false); } let metadata = db .repo_metadata .get(keys::repo_metadata_key(did)) .into_diagnostic()? .map(|bytes| crate::db::deser_repo_meta(&bytes)) .transpose()?; Ok(metadata.is_none_or(|metadata| metadata.tracked)) } fn enqueue_listing_backfill( state: &AppState, txn: &mut crate::db::Txn<'_>, repo: &mut RepoState<'_>, did: &Did, ) -> Result { let db = &state.db; let metadata_key = keys::repo_metadata_key(did); let existing = db .repo_metadata .get(&metadata_key) .into_diagnostic()? .map(|bytes| crate::db::deser_repo_meta(&bytes)) .transpose()?; let mut metadata = match existing { Some(mut metadata) => { if refetch::charge(state, txn, did, repo, &mut metadata, "listing")? == Charged::Deferred { return Ok(Charged::Deferred); } db.indexer.stage_pending_remove( &mut txn.batch, did, &keys::pending_key(metadata.index_id), ); metadata } None => RepoMetadata::backfilling(rand::random()), }; metadata.index_id = rand::random(); metadata.relay_inactive = None; txn.batch.insert( &db.repo_metadata, metadata_key, crate::db::ser_repo_meta(&metadata)?, ); db.indexer .stage_pending_insert(&mut txn.batch, did, &keys::pending_key(metadata.index_id)); Ok(Charged::Allowed) } fn listed_repo_status(status: Option<&ListedRepoStatus>) -> RepoStatus { match status { Some(ListedRepoStatus::Takendown) => RepoStatus::Takendown, Some(ListedRepoStatus::Suspended) => RepoStatus::Suspended, Some(ListedRepoStatus::Deleted) => RepoStatus::Deleted, Some(ListedRepoStatus::Deactivated) => RepoStatus::Deactivated, Some(ListedRepoStatus::Desynchronized) => RepoStatus::Desynchronized, Some(ListedRepoStatus::Throttled) => RepoStatus::Throttled, Some(ListedRepoStatus::Other(status)) => RepoStatus::Error(status.to_smolstr()), None => RepoStatus::Error("inactive".into()), } } #[cfg(test)] mod tests { use super::*; use bytes::Bytes; use jacquard_common::types::string::Tid; use miette::IntoDiagnostic; #[test] fn pending_between_resume_and_max_does_not_start_throttling() { assert_eq!( capacity_decision(75, 100, 50, false), CapacityDecision::Proceed ); assert_eq!( capacity_decision(100, 100, 50, false), CapacityDecision::Proceed ); } #[test] fn pending_above_max_starts_throttling() { assert_eq!( capacity_decision(101, 100, 50, false), CapacityDecision::EnterThrottle ); } #[test] fn throttling_stays_active_until_resume_boundary() { assert_eq!( capacity_decision(100, 100, 50, true), CapacityDecision::StayThrottled ); assert_eq!( capacity_decision(51, 100, 50, true), CapacityDecision::StayThrottled ); assert_eq!( capacity_decision(50, 100, 50, true), CapacityDecision::ReleaseThrottle ); assert_eq!( capacity_decision(0, 100, 50, true), CapacityDecision::ReleaseThrottle ); } fn did() -> Did { Did::new_static("did:plc:aaaaaaaaaaaaaaaaaaaaaaaa").unwrap() } fn rev(value: u8) -> DbTid { let mut bytes = [0; 8]; bytes[7] = value; DbTid::new_from_bytes(bytes) } fn state_with_root(rev: DbTid) -> RepoState<'static> { let mut state = RepoState::synced(); state.root = Some(crate::types::Commit { version: 3, rev, data: crate::car::compute_cid(b"root"), prev: None, sig: Bytes::new(), }); state } struct TestDb(AppState); impl std::ops::Deref for TestDb { type Target = crate::db::Db; fn deref(&self) -> &Self::Target { &self.0.db } } fn open_db() -> Result<(tempfile::TempDir, TestDb)> { let tmp = tempfile::tempdir().into_diagnostic()?; let config = crate::config::Config { database_path: tmp.path().to_path_buf(), ..Default::default() }; Ok((tmp, TestDb(AppState::new(&config)?))) } fn insert_repo( db: &crate::db::Db, state: &RepoState<'_>, index_id: u64, pending: bool, ) -> Result<()> { let did = did(); let did_key = keys::repo_key(&did); db.repos .insert(&did_key, ser_repo_state(state)?) .into_diagnostic()?; db.repo_metadata .insert( keys::repo_metadata_key(&did), crate::db::ser_repo_meta(&RepoMetadata::backfilling(index_id))?, ) .into_diagnostic()?; if pending { let mut batch = db.inner.batch(); db.indexer .stage_pending_insert(&mut batch, &did, &keys::pending_key(index_id)); batch.commit().into_diagnostic()?; } Ok(()) } fn apply(db: &TestDb, listing: &RepoListing) -> Result { let mut txn = crate::db::Txn::new(db); let changed = reconcile_listing(&db.0, &mut txn, listing)?; txn.commit()?; Ok(changed) } fn read_state(db: &crate::db::Db) -> Result> { let bytes = db .repos .get(keys::repo_key(&did())) .into_diagnostic()? .expect("repo state should exist"); Ok(crate::db::deser_repo_state(&bytes)?.into_static()) } const QUICK: DispatchWait = DispatchWait { margin: Duration::ZERO, timeout: Duration::from_secs(5), poll: Duration::from_millis(10), }; /// a running crawler worker over a repo synced at `rev(1)` fn crawl_over_a_synced_repo() -> Result<(tempfile::TempDir, Arc, mpsc::Sender)> { let tmp = tempfile::tempdir().into_diagnostic()?; let config = crate::config::Config { database_path: tmp.path().to_path_buf(), ..Default::default() }; let state = Arc::new(AppState::new(&config)?); insert_repo(&state.db, &state_with_root(rev(1)), 7, false)?; let stats = CrawlerStats::new( state.clone(), Vec::new(), crate::util::throttle::Throttler::new(1), ); let (worker, tx) = CrawlerWorker::new(state.clone(), 100, 50, stats); tokio::spawn(worker.run()); Ok((tmp, state, tx)) } /// a relay firehose for `host` that has handed on events up to `dispatched_ms` fn relay_firehose(state: &AppState, host: &str, dispatched_ms: i64) -> url::Url { let url = url::Url::parse(&format!("wss://{host}/")).unwrap(); let _ = state.firehose_dispatched.insert_sync( url.clone(), std::sync::atomic::AtomicI64::new(dispatched_ms), ); url } const CURSOR: &[u8] = b"test_cursor"; /// a page that moves the cursor to `name` and leaves a key of its own named after it, with /// `guards` to queue right away fn page(name: &str, guards: Vec) -> CrawlerBatch { CrawlerBatch { guards, listings: Vec::new(), cursor_updates: vec![ CursorUpdate { key: CURSOR.to_vec(), value: Some(name.as_bytes().to_vec()), }, CursorUpdate { key: name.as_bytes().to_vec(), value: Some(Vec::new()), }, ], applied: None, } } /// a newer rev for the synced repo fn newer_rev() -> Vec { vec![RepoListing::active(did(), rev(3))] } fn committed(state: &AppState, key: &str) -> bool { state.db.cursors.contains_key(key.as_bytes()).unwrap() } /// waits for `done` to hold, which the worker makes true a little after it's handed a page async fn eventually(done: impl Fn() -> bool) { for _ in 0..500 { if done() { return; } tokio::time::sleep(Duration::from_millis(10)).await; } panic!("the worker never got there"); } #[tokio::test] async fn a_newer_rev_waits_for_the_firehose_to_catch_up_to_its_page() -> Result<()> { let (_tmp, state, tx) = crawl_over_a_synced_repo()?; let relay = relay_firehose(&state, "relay.example", 500); let waiting = WaitingPages::spawn(tx.clone(), state.clone(), QUICK, WAITING_PAGES); send_page( &tx, &waiting, "relay.example", 1_000, page("page", Vec::new()), newer_rev(), ) .await?; tokio::time::sleep(Duration::from_millis(200)).await; assert_eq!(state.db.indexer.pending_iter().count(), 0); assert!(!committed(&state, "page")); state.note_dispatched(&relay, 1_000); eventually(|| committed(&state, "page")).await; assert_eq!(state.db.indexer.pending_iter().count(), 1); Ok(()) } #[tokio::test] async fn a_listing_that_queues_a_backfill_wakes_the_backfill_worker() -> Result<()> { let (_tmp, state, tx) = crawl_over_a_synced_repo()?; tx.send(CrawlerBatch { guards: Vec::new(), listings: newer_rev(), cursor_updates: Vec::new(), applied: None, }) .await .into_diagnostic()?; tokio::time::timeout(Duration::from_secs(5), state.backfill_notify.notified()) .await .into_diagnostic()?; assert_eq!(state.db.indexer.pending_iter().count(), 1); Ok(()) } #[tokio::test] async fn a_newer_rev_the_firehose_brought_in_while_it_waited_queues_nothing() -> Result<()> { let (_tmp, state, tx) = crawl_over_a_synced_repo()?; let relay = relay_firehose(&state, "relay.example", 500); let waiting = WaitingPages::spawn(tx.clone(), state.clone(), QUICK, WAITING_PAGES); send_page( &tx, &waiting, "relay.example", 1_000, page("page", Vec::new()), newer_rev(), ) .await?; tokio::time::sleep(Duration::from_millis(200)).await; assert_eq!(state.db.indexer.pending_iter().count(), 0); // the commit the listing saw lands while the page waits insert_repo(&state.db, &state_with_root(rev(3)), 7, false)?; state.note_dispatched(&relay, 1_000); eventually(|| committed(&state, "page")).await; assert_eq!(state.db.indexer.pending_iter().count(), 0); Ok(()) } #[tokio::test] async fn a_firehose_that_never_catches_up_skips_the_newer_revs_and_moves_on() -> Result<()> { let (_tmp, state, tx) = crawl_over_a_synced_repo()?; relay_firehose(&state, "relay.example", 500); let wait = DispatchWait { timeout: Duration::from_millis(100), ..QUICK }; let waiting = WaitingPages::spawn(tx.clone(), state.clone(), wait, WAITING_PAGES); send_page( &tx, &waiting, "relay.example", 1_000, page("page", Vec::new()), newer_rev(), ) .await?; eventually(|| committed(&state, "page")).await; assert_eq!(state.db.indexer.pending_iter().count(), 0); Ok(()) } #[tokio::test] async fn pages_behind_a_waiting_one_queue_their_repos_but_keep_their_cursors_back() -> Result<()> { let (_tmp, state, tx) = crawl_over_a_synced_repo()?; let relay = relay_firehose(&state, "relay.example", 500); let waiting = WaitingPages::spawn(tx.clone(), state.clone(), QUICK, WAITING_PAGES); let in_flight = super::super::InFlight::new(); let (two, three) = ( Did::new_static("did:web:two.example").into_diagnostic()?, Did::new_static("did:web:three.example").into_diagnostic()?, ); let known = |did: &Did| state.db.repos.contains_key(keys::repo_key(did)).unwrap(); send_page( &tx, &waiting, "relay.example", 1_000, page("page 1", Vec::new()), newer_rev(), ) .await?; let page_two = page("page 2", in_flight.acquire(vec![two.clone()]).await); send_page(&tx, &waiting, "relay.example", 1_100, page_two, Vec::new()).await?; let page_three = page("page 3", in_flight.acquire(vec![three.clone()]).await); send_page( &tx, &waiting, "relay.example", 1_200, page_three, Vec::new(), ) .await?; eventually(|| known(&two) && known(&three)).await; tokio::time::sleep(Duration::from_millis(100)).await; for name in ["page 1", "page 2", "page 3"] { assert!( !committed(&state, name), "{name} moved the cursor before page 1 was in" ); } state.note_dispatched(&relay, 1_000); eventually(|| committed(&state, "page 3")).await; assert!(committed(&state, "page 1") && committed(&state, "page 2")); assert_eq!( state.db.cursors.get(CURSOR).into_diagnostic()?.as_deref(), Some(b"page 3".as_slice()) ); assert_eq!(state.db.indexer.pending_iter().count(), 3); Ok(()) } #[tokio::test] async fn a_crawl_with_its_waiting_pages_full_waits_for_room() -> Result<()> { let (_tmp, state, tx) = crawl_over_a_synced_repo()?; let relay = relay_firehose(&state, "relay.example", 500); // one page waits for the firehose and one waits behind it, the third finds no room let waiting = WaitingPages::spawn(tx.clone(), state.clone(), QUICK, 1); let crawl = tokio::spawn({ let state = state.clone(); async move { for (n, name) in ["page 1", "page 2", "page 3"].into_iter().enumerate() { let newer = if n == 0 { newer_rev() } else { Vec::new() }; send_page( &tx, &waiting, "relay.example", 1_000, page(name, Vec::new()), newer, ) .await .unwrap(); } drop(state); // dropping it would cancel the pages still waiting waiting } }); tokio::time::sleep(Duration::from_millis(200)).await; assert!(!crawl.is_finished()); state.note_dispatched(&relay, 1_000); let _waiting = tokio::time::timeout(Duration::from_secs(5), crawl) .await .into_diagnostic()? .into_diagnostic()?; eventually(|| committed(&state, "page 3")).await; Ok(()) } #[tokio::test] async fn a_host_without_a_firehose_of_its_own_waits_for_the_slowest_relay() -> Result<()> { let tmp = tempfile::tempdir().into_diagnostic()?; let state = AppState::new(&crate::config::Config { database_path: tmp.path().to_path_buf(), ..Default::default() })?; let deadline = || tokio::time::Instant::now() + Duration::from_millis(50); // no relay firehose at all, so nothing to wait for assert!(wait_for_firehose(&state, "other.example", 1_000, &QUICK, deadline()).await); relay_firehose(&state, "ahead.example", 2_000); relay_firehose(&state, "behind.example", 500); assert!(wait_for_firehose(&state, "ahead.example", 1_000, &QUICK, deadline()).await); assert!(!wait_for_firehose(&state, "other.example", 1_000, &QUICK, deadline()).await); Ok(()) } #[test] fn listings_move_the_repo_between_counted_and_not() -> Result<()> { let (_tmp, db) = open_db()?; let mut state = state_with_root(rev(2)); state.pds = Some(jacquard_common::CowStr::Borrowed("https://pds.example/")); let mut txn = crate::db::Txn::new(&db); txn.put_repo(&did(), &state)?; txn.commit()?; let accounts = || db.get_count_sync(&keys::pds_account_count_key("pds.example")); assert_eq!(accounts(), 1); let mut listing = RepoListing { did: did(), rev: rev(2), active: false, status: Some(ListedRepoStatus::Deleted), create_if_missing: false, }; assert_eq!(apply(&db, &listing)?, ReconcileOutcome::AccountChanged); assert_eq!(accounts(), 0); // the account only comes back once its owner shard finished the wipe db.crawler .remove(keys::bulk_job_key(&did())) .into_diagnostic()?; listing.active = true; listing.status = None; assert_eq!(apply(&db, &listing)?, ReconcileOutcome::AccountChanged); assert_eq!(accounts(), 1); Ok(()) } fn metadata(db: &crate::db::Db) -> Result { let bytes = db .repo_metadata .get(keys::repo_metadata_key(&did())) .into_diagnostic()? .expect("metadata should exist"); crate::db::deser_repo_meta(&bytes) } #[test] fn relay_inactive_listing_asks_the_pds_once_per_status() -> Result<()> { let (_tmp, db) = open_db()?; insert_repo(&db, &state_with_root(rev(2)), 7, false)?; let mut listing = RepoListing::inactive(did(), rev(2), ListedRepoStatus::Takendown); assert_eq!(apply(&db, &listing)?, ReconcileOutcome::StateOnly); let state = read_state(&db)?; assert!(state.active); assert_eq!(state.status, RepoStatus::Synced); let asked = metadata(&db)?; assert_eq!(asked.relay_inactive, Some(RepoStatus::Takendown)); assert!( db.indexer .is_pending(&keys::pending_key(asked.index_id)) .into_diagnostic()? ); assert!( !db.indexer .resync .contains_key(keys::repo_key(&did())) .into_diagnostic()? ); // the queued fetch is the pds's answer, so the same listing again waits for it assert_eq!(apply(&db, &listing)?, ReconcileOutcome::Unchanged); assert_eq!(db.indexer.pending_iter().count(), 1); listing.status = Some(ListedRepoStatus::Suspended); assert_eq!(apply(&db, &listing)?, ReconcileOutcome::StateOnly); assert_eq!(metadata(&db)?.relay_inactive, Some(RepoStatus::Suspended)); assert_eq!(metadata(&db)?.index_id, asked.index_id); assert_eq!(db.indexer.pending_iter().count(), 1); db.indexer.assert_pending_dids_match(); Ok(()) } #[test] fn relay_inactive_listing_leaves_a_queued_backfill_to_answer() -> Result<()> { let (_tmp, db) = open_db()?; insert_repo(&db, &RepoState::backfilling(), 7, true)?; let listing = RepoListing::inactive(did(), rev(1), ListedRepoStatus::Deactivated); assert_eq!(apply(&db, &listing)?, ReconcileOutcome::StateOnly); assert!( db.indexer .is_pending(&keys::pending_key(7)) .into_diagnostic()? ); assert_eq!(metadata(&db)?.relay_inactive, Some(RepoStatus::Deactivated)); let state = read_state(&db)?; assert!(state.active); assert_eq!(state.status, RepoStatus::Desynchronized); assert_eq!(apply(&db, &listing)?, ReconcileOutcome::Unchanged); db.indexer.assert_pending_dids_match(); Ok(()) } #[test] fn deleting_a_repo_takes_its_relay_marker_with_it() -> Result<()> { let (_tmp, db) = open_db()?; insert_repo(&db, &state_with_root(rev(2)), 7, false)?; let takendown = RepoListing::inactive(did(), rev(2), ListedRepoStatus::Takendown); assert_eq!(apply(&db, &takendown)?, ReconcileOutcome::StateOnly); assert_eq!(metadata(&db)?.relay_inactive, Some(RepoStatus::Takendown)); let deleted = RepoListing::inactive(did(), rev(2), ListedRepoStatus::Deleted); assert_eq!(apply(&db, &deleted)?, ReconcileOutcome::AccountChanged); // the repo's owner shard moves its heads to history from this marker assert!(matches!( crate::db::read_bulk_job(&db, &did())?, Some(crate::types::BulkJobV1::Wipe { .. }) )); assert!( db.repo_metadata .get(keys::repo_metadata_key(&did())) .into_diagnostic()? .is_none() ); assert_eq!(db.indexer.pending_iter().count(), 0); db.indexer.assert_pending_dids_match(); Ok(()) } #[test] fn relay_listing_does_not_relabel_an_inactive_repo() -> Result<()> { let (_tmp, db) = open_db()?; let mut state = state_with_root(rev(2)); state.active = false; state.status = RepoStatus::Deactivated; insert_repo(&db, &state, 7, false)?; let listing = RepoListing::inactive(did(), rev(2), ListedRepoStatus::Takendown); assert_eq!(apply(&db, &listing)?, ReconcileOutcome::Unchanged); assert_eq!(read_state(&db)?.status, RepoStatus::Deactivated); assert_eq!(metadata(&db)?.relay_inactive, None); Ok(()) } #[test] fn relay_active_listing_asks_the_pds_about_an_inactive_repo() -> Result<()> { let (_tmp, db) = open_db()?; let mut state = state_with_root(rev(2)); state.active = false; state.status = RepoStatus::Deactivated; insert_repo(&db, &state, 7, false)?; let mut txn = crate::db::Txn::new(&db); txn.batch.insert( &db.indexer.resync, keys::repo_key(&did()), rmp_serde::to_vec(&ResyncState::Gone { status: RepoStatus::Deactivated, }) .into_diagnostic()?, ); txn.transition_lifecycle(&did(), GaugeState::Resync(None))?; txn.commit()?; let listing = RepoListing::active(did(), rev(2)); assert_eq!(apply(&db, &listing)?, ReconcileOutcome::StateOnly); let after = read_state(&db)?; assert!(!after.active); assert_eq!(after.status, RepoStatus::Deactivated); let asked = metadata(&db)?; assert_eq!(asked.relay_inactive, Some(RepoStatus::Synced)); assert!( db.indexer .is_pending(&keys::pending_key(asked.index_id)) .into_diagnostic()? ); assert!( !db.indexer .resync .contains_key(keys::repo_key(&did())) .into_diagnostic()? ); assert_eq!(db.get_count_sync("pending"), 1); assert_eq!(db.get_count_sync("resync"), 0); // the marker keeps the same listing from asking twice assert_eq!(apply(&db, &listing)?, ReconcileOutcome::Unchanged); assert_eq!(db.indexer.pending_iter().count(), 1); db.indexer.assert_pending_dids_match(); Ok(()) } #[test] fn relay_first_persists_unknown_inactive_listing_without_backfill() -> Result<()> { let (_tmp, db) = open_db()?; let listing = RepoListing { did: did(), rev: rev(1), active: false, status: Some(ListedRepoStatus::Takendown), create_if_missing: true, }; assert_eq!(apply(&db, &listing)?, ReconcileOutcome::StateOnly); let state = read_state(&db)?; assert!(!state.active); assert_eq!(state.status, RepoStatus::Takendown); assert_eq!(db.indexer.pending_iter().count(), 0); db.indexer.assert_pending_dids_match(); assert!( db.indexer .resync .contains_key(keys::repo_key(&listing.did)) .into_diagnostic()? ); assert_eq!(apply(&db, &listing)?, ReconcileOutcome::Unchanged); Ok(()) } #[test] fn current_active_listing_does_not_reenqueue() -> Result<()> { let (_tmp, db) = open_db()?; insert_repo(&db, &state_with_root(rev(2)), 7, false)?; let listing = RepoListing { did: did(), rev: rev(2), active: true, status: None, create_if_missing: false, }; assert_eq!(apply(&db, &listing)?, ReconcileOutcome::Unchanged); assert_eq!(db.indexer.pending_iter().count(), 0); db.indexer.assert_pending_dids_match(); Ok(()) } #[test] fn inactive_listing_leaves_a_queued_check_on_an_inactive_repo() -> Result<()> { let (_tmp, db) = open_db()?; let mut state = RepoState::backfilling(); state.active = false; state.status = RepoStatus::Deactivated; insert_repo(&db, &state, 7, true)?; let listing = RepoListing::inactive(did(), rev(1), ListedRepoStatus::Deactivated); assert_eq!(apply(&db, &listing)?, ReconcileOutcome::Unchanged); assert!( db.indexer .is_pending(&keys::pending_key(7)) .into_diagnostic()? ); db.indexer.assert_pending_dids_match(); Ok(()) } #[test] fn newer_active_listing_reenqueues_missed_revision() -> Result<()> { let (_tmp, db) = open_db()?; insert_repo(&db, &state_with_root(rev(1)), 7, false)?; let listing = RepoListing { did: did(), rev: rev(2), active: true, status: None, create_if_missing: false, }; assert_eq!(apply(&db, &listing)?, ReconcileOutcome::StateOnly); assert_eq!(db.indexer.pending_iter().count(), 1); db.indexer.assert_pending_dids_match(); let state = read_state(&db)?; assert!(state.active); assert_eq!(state.status, RepoStatus::Desynchronized); Ok(()) } fn use_up_refetches(db: &TestDb) -> Result<()> { let mut metadata = metadata(db)?; metadata.refetch = Some(crate::types::Allowance { since: chrono::Utc::now().timestamp(), used: u32::MAX, }); crate::ingest::indexer::refetch::tests::put_metadata(&db.0, &did(), &metadata); Ok(()) } #[test] fn a_newer_listing_for_a_repo_out_of_refetches_waits_for_its_window() -> Result<()> { let (_tmp, db) = open_db()?; insert_repo(&db, &state_with_root(rev(1)), 7, false)?; use_up_refetches(&db)?; assert_eq!( apply(&db, &RepoListing::active(did(), rev(2)))?, ReconcileOutcome::StateOnly ); assert_eq!(db.indexer.pending_iter().count(), 0); let repo = read_state(&db)?; assert!(repo.active); assert_eq!(repo.status, RepoStatus::Throttled); assert!(metadata(&db)?.refetch_deferred); assert!( db.indexer .resync .contains_key(keys::repo_key(&did())) .into_diagnostic()? ); Ok(()) } #[test] fn a_status_check_for_a_repo_out_of_refetches_leaves_its_marker_for_the_next_pass() -> Result<()> { let (_tmp, db) = open_db()?; insert_repo(&db, &state_with_root(rev(2)), 7, false)?; use_up_refetches(&db)?; let takendown = RepoListing::inactive(did(), rev(2), ListedRepoStatus::Takendown); assert_eq!(apply(&db, &takendown)?, ReconcileOutcome::StateOnly); assert_eq!(db.indexer.pending_iter().count(), 0); assert_eq!(metadata(&db)?.relay_inactive, None); assert_eq!(read_state(&db)?.status, RepoStatus::Throttled); Ok(()) } #[test] fn a_listed_rev_from_the_future_is_not_news() -> Result<()> { let (_tmp, db) = open_db()?; insert_repo(&db, &state_with_root(rev(1)), 7, false)?; let listed_at = |rev: &str| RepoListing::active(did(), DbTid::from(&Tid::new(rev).unwrap())); assert_eq!( apply(&db, &listed_at("jzzzzzzzzzzzz"))?, ReconcileOutcome::Unchanged ); assert_eq!(db.indexer.pending_iter().count(), 0); let now = Tid::now_0(); assert_eq!( apply(&db, &listed_at(now.as_str()))?, ReconcileOutcome::StateOnly ); assert_eq!(db.indexer.pending_iter().count(), 1); Ok(()) } #[test] fn terminal_cursor_update_clears_resume_point() -> Result<()> { let (_tmp, db) = open_db()?; let key = b"crawler_cursor|relay".to_vec(); db.cursors.insert(&key, b"old").into_diagnostic()?; let mut txn = crate::db::Txn::new(&db); stage_cursor( &mut txn.batch, &db, CursorUpdate { key: key.clone(), value: None, }, ); txn.commit()?; assert!(db.cursors.get(key).into_diagnostic()?.is_none()); Ok(()) } #[test] fn cursor_batch_can_advance_one_marker_and_clear_another() -> Result<()> { let (_tmp, db) = open_db()?; let old_key = b"relay_first_cursor|relay|hosts".to_vec(); let marker_key = b"relay_first_cursor|relay|handled".to_vec(); db.cursors.insert(&old_key, b"old").into_diagnostic()?; let mut txn = crate::db::Txn::new(&db); for cursor in [ CursorUpdate { key: marker_key.clone(), value: Some(7u64.to_be_bytes().to_vec()), }, CursorUpdate { key: old_key.clone(), value: None, }, ] { stage_cursor(&mut txn.batch, &db, cursor); } txn.commit()?; assert_eq!( db.cursors.get(marker_key).into_diagnostic()?.as_deref(), Some(7u64.to_be_bytes().as_slice()) ); assert!(db.cursors.get(old_key).into_diagnostic()?.is_none()); Ok(()) } #[tokio::test] async fn a_page_a_shard_failed_on_keeps_its_cursor_back_until_it_comes_through_again() -> Result<()> { let tmp = tempfile::tempdir().into_diagnostic()?; let config = crate::config::Config { database_path: tmp.path().to_path_buf(), ..Default::default() }; let state = Arc::new(AppState::new(&config)?); let stats = CrawlerStats::new( state.clone(), Vec::new(), crate::util::throttle::Throttler::new(1), ); let (mut worker, _tx) = CrawlerWorker::new(state.clone(), 100, 50, stats); let dids: Vec = (0..40) .map(|i| Did::new_owned(format!("did:web:repo{i}.example")).unwrap()) .collect(); let in_flight = super::super::InFlight::new(); let cursor_key = b"test_cursor".to_vec(); let page = || async { CrawlerBatch { guards: in_flight.acquire(dids.clone()).await, listings: Vec::new(), cursor_updates: vec![CursorUpdate { key: cursor_key.clone(), value: Some(b"next page".to_vec()), }], applied: None, } }; let known = |did: &Did| state.db.repos.contains_key(keys::repo_key(did)).unwrap(); // one of the two shards never runs, as if it stopped partway through the page let shards = crate::ingest::indexer::FirehoseWorker::serve_shards(state.clone(), 2, Some(1)); assert!(worker.enqueue(page().await).await.is_err()); assert!( state .db .cursors .get(&cursor_key) .into_diagnostic()? .is_none() ); let on_the_gone_shard = |did: &&Did| shard_of(did, 2) == 1; assert!(dids.iter().any(|did| on_the_gone_shard(&did))); for did in &dids { assert_eq!(known(did), !on_the_gone_shard(&did), "{did}"); } // the next start crawls the same page again, and only then moves the cursor on drop(shards); let _shards = crate::ingest::indexer::FirehoseWorker::serve_shards(state.clone(), 2, None); worker.enqueue(page().await).await?; assert_eq!( state .db .cursors .get(&cursor_key) .into_diagnostic()? .as_deref(), Some(b"next page".as_slice()) ); assert!(dids.iter().all(known)); assert_eq!(state.db.indexer.pending_iter().count(), dids.len()); Ok(()) } /// a page goes in two parts, its repos right away and its cursor later, so a shard failing /// the first part must still keep the cursor back #[tokio::test] async fn a_page_whose_repos_a_shard_failed_never_moves_its_cursor() -> Result<()> { let (_tmp, state, tx) = crawl_over_a_synced_repo()?; let waiting = WaitingPages::spawn(tx.clone(), state.clone(), QUICK, WAITING_PAGES); let in_flight = super::super::InFlight::new(); let dids: Vec = (0..40) .map(|i| Did::new_owned(format!("did:web:repo{i}.example")).unwrap()) .collect(); assert!(dids.iter().any(|did| shard_of(did, 2) == 1)); // one of the two shards never runs, as if it stopped partway through the page let _shards = crate::ingest::indexer::FirehoseWorker::serve_shards(state.clone(), 2, Some(1)); let first = page("page 1", in_flight.acquire(dids.clone()).await); send_page(&tx, &waiting, "relay.example", 1_000, first, Vec::new()).await?; let second = page("page 2", Vec::new()); // the crawl may get one more page in before it hears about the failure let _ = send_page(&tx, &waiting, "relay.example", 1_100, second, Vec::new()).await; let mut refused = false; for _ in 0..500 { let third = page("page 3", Vec::new()); if send_page(&tx, &waiting, "relay.example", 1_200, third, Vec::new()) .await .is_err() { refused = true; break; } tokio::time::sleep(Duration::from_millis(10)).await; } assert!(refused, "the crawl never heard its page failed"); tokio::time::sleep(Duration::from_millis(100)).await; for name in ["page 1", "page 2", "page 3"] { assert!(!committed(&state, name), "{name} moved the cursor"); } // the crawl starts over from the saved cursor and its pages go through again waiting.begin_crawl(); send_page( &tx, &waiting, "relay.example", 1_300, page("again", Vec::new()), Vec::new(), ) .await?; eventually(|| committed(&state, "again")).await; Ok(()) } #[tokio::test] async fn pages_of_a_crawl_that_is_gone_never_move_its_cursor() -> Result<()> { let (_tmp, state, tx) = crawl_over_a_synced_repo()?; let relay = relay_firehose(&state, "relay.example", 500); let waiting = WaitingPages::spawn(tx.clone(), state.clone(), QUICK, WAITING_PAGES); send_page( &tx, &waiting, "relay.example", 1_000, page("page", Vec::new()), newer_rev(), ) .await?; // the source is removed while its page waits for the firehose drop(waiting); state.note_dispatched(&relay, 1_000); tokio::time::sleep(Duration::from_millis(300)).await; assert!(!committed(&state, "page")); Ok(()) } #[test] fn a_newer_rev_leaves_an_untracked_erased_or_wiped_repo_alone() -> Result<()> { let newer = RepoListing::active(did(), rev(3)); let queued = |db: &crate::db::Db| db.indexer.pending_iter().count(); let (_tmp, db) = open_db()?; insert_repo(&db, &state_with_root(rev(1)), 7, false)?; let mut metadata = RepoMetadata::backfilling(7); metadata.tracked = false; db.repo_metadata .insert( keys::repo_metadata_key(&did()), crate::db::ser_repo_meta(&metadata)?, ) .into_diagnostic()?; assert_eq!(apply(&db, &newer)?, ReconcileOutcome::Unchanged); assert_eq!(queued(&db), 0); // an operator erase without exclude keeps the repo row while its steps run let (_tmp, db) = open_db()?; insert_repo(&db, &state_with_root(rev(1)), 7, false)?; let mut txn = crate::db::Txn::new(&db); crate::db::start_erase(&mut txn, &did(), false)?; txn.commit()?; assert_eq!(apply(&db, &newer)?, ReconcileOutcome::Unchanged); assert_eq!(queued(&db), 0); // a deleted account's wipe, with the account listed as back meanwhile let (_tmp, db) = open_db()?; insert_repo(&db, &state_with_root(rev(1)), 7, false)?; let deleted = RepoListing::inactive(did(), rev(2), ListedRepoStatus::Deleted); assert_eq!(apply(&db, &deleted)?, ReconcileOutcome::AccountChanged); assert_eq!(apply(&db, &newer)?, ReconcileOutcome::Unchanged); assert_eq!(queued(&db), 0); Ok(()) } #[tokio::test] #[cfg(feature = "indexer_stream")] async fn changed_listing_emits_one_account_event() -> Result<()> { let tmp = tempfile::tempdir().into_diagnostic()?; let config = crate::config::Config { database_path: tmp.path().to_path_buf(), ..Default::default() }; let state = Arc::new(AppState::new(&config)?); insert_repo(&state.db, &RepoState::backfilling(), 7, true)?; let stats = CrawlerStats::new( state.clone(), Vec::new(), crate::util::throttle::Throttler::new(1), ); let (mut worker, _tx) = CrawlerWorker::new(state.clone(), 100, 50, stats); let mut events = state.db.stream.event_tx.subscribe(); let listing = RepoListing::inactive(did(), rev(1), ListedRepoStatus::Deleted); worker .enqueue(CrawlerBatch { guards: Vec::new(), listings: vec![listing.clone()], cursor_updates: Vec::new(), applied: None, }) .await?; let crate::types::BroadcastEvent::Ephemeral(event) = events.recv().await.unwrap() else { panic!("expected account event"); }; let account = event.account.expect("account payload"); assert!(!account.active); assert_eq!(account.status.as_deref(), Some("deleted")); worker .enqueue(CrawlerBatch { guards: Vec::new(), listings: vec![listing], cursor_updates: Vec::new(), applied: None, }) .await?; assert!(events.try_recv().is_err()); Ok(()) } #[tokio::test] #[cfg(feature = "indexer_stream")] async fn missed_revision_does_not_emit_account_event() -> Result<()> { let tmp = tempfile::tempdir().into_diagnostic()?; let config = crate::config::Config { database_path: tmp.path().to_path_buf(), ..Default::default() }; let state = Arc::new(AppState::new(&config)?); insert_repo(&state.db, &state_with_root(rev(1)), 7, false)?; let stats = CrawlerStats::new( state.clone(), Vec::new(), crate::util::throttle::Throttler::new(1), ); let (mut worker, _tx) = CrawlerWorker::new(state.clone(), 100, 50, stats); let mut events = state.db.stream.event_tx.subscribe(); worker .enqueue(CrawlerBatch { guards: Vec::new(), listings: vec![RepoListing { did: did(), rev: rev(2), active: true, status: None, create_if_missing: false, }], cursor_updates: Vec::new(), applied: None, }) .await?; assert!(events.try_recv().is_err()); assert_eq!(state.db.indexer.pending_iter().count(), 1); state.db.indexer.assert_pending_dids_match(); Ok(()) } }