From 095b7e2017884c87bfceef0e8edaeab80ff2f974 Mon Sep 17 00:00:00 2001 From: dawn <90008@klbr.net> Date: Thu, 8 Oct 2026 09:18:02 +0300 Subject: [PATCH] [crawler] keep reading pages while one waits for the firehose the crawl awaited each page whose newer revs waited for the firehose, so on a caught-up box any page with an account that committed in the last few seconds cost the margin and the lag before the next page was read, hours over a full pass, and during a long catch-up one page could hold the crawl for the whole timeout. a page's newer listings and its cursor now go into a bounded line of waiting pages (32), which a task of its own hands to the worker in the order the pages were read, each once the firehose caught up to it or its timeout passed. the rest of each page goes to the worker right away, the crawl only waits when the line is full, and a page's cursor still never moves before every page ahead of it is in. --- docs/concepts/relay.md | 2 +- src/crawler/list_repos/producer.rs | 17 +- src/crawler/worker.rs | 339 ++++++++++++++++++++++------- 3 files changed, 271 insertions(+), 87 deletions(-) diff --git a/docs/concepts/relay.md b/docs/concepts/relay.md index 6942f93..2dd2335 100644 --- a/docs/concepts/relay.md +++ b/docs/concepts/relay.md @@ -46,7 +46,7 @@ seeding runs from latest cursor on restart so new PDS' added to the upstream rel the crawler is configured separately from the firehose via `CRAWLER_URLS`. each source is a `[mode::]url` entry where the mode prefix is optional and defaults to `by_collection` in filter mode or `list_repos` in full-network mode. -- `list_repos`: enumerates the relay via `com.atproto.sync.listRepos`. relay account status is reconciled for repos already known to hydrant, but a relay listing an active repo as inactive only queues a fetch from its pds (once per listed status), and the account goes inactive only if the pds refuses it, so a relay-level takedown of a repo its pds still serves leaves it synced. the same goes the other way: a relay listing an inactive repo as active at the revision hydrant has only queues a fetch, and the account comes back only if the pds serves it; filtered indexes check unknown repos with `describeRepo` before enqueueing them. a listing whose revision is newer than the one hydrant stored queues a backfill only once hydrant's own firehose of that relay has caught up to when the page was read (the slowest relay firehose for a host hydrant doesn't subscribe to), since a commit still on its way through hydrant would look the same. if the firehose doesn't get there within 10 minutes, those listings are skipped until the next pass, with one log line. +- `list_repos`: enumerates the relay via `com.atproto.sync.listRepos`. relay account status is reconciled for repos already known to hydrant, but a relay listing an active repo as inactive only queues a fetch from its pds (once per listed status), and the account goes inactive only if the pds refuses it, so a relay-level takedown of a repo its pds still serves leaves it synced. the same goes the other way: a relay listing an inactive repo as active at the revision hydrant has only queues a fetch, and the account comes back only if the pds serves it; filtered indexes check unknown repos with `describeRepo` before enqueueing them. a listing whose revision is newer than the one hydrant stored queues a backfill only once hydrant's own firehose of that relay has caught up to when the page was read (the slowest relay firehose for a host hydrant doesn't subscribe to), since a commit still on its way through hydrant would look the same. the crawl keeps reading pages meanwhile, with up to 32 of them waiting, and a page's cursor only moves once it and every page before it are in. if the firehose doesn't get there within 10 minutes, those listings are skipped until the next pass, with one log line. - `relay_first`: an explicit coverage-over-moderation opt-in. it completes the relay's authoritative `listRepos` pass first, then calls the same relay's `listHosts` and crawls `listRepos` on each safe, non-banned PDS. relay-listed inactive repos are retained as status tombstones, so deep pages can only add DIDs the relay did not list and never update relay-authoritative status or revision state. this may discover inactive or other repositories omitted by the relay, which also means it deliberately bypasses relay-level repository moderation for those unknown DIDs. local PDS bans and filter excludes still win. - `by_collection`: queries `com.atproto.sync.listReposByCollection` for each configured signal. more efficient for filtered indexing since it only surfaces repos that have matching records. cursors are stored per collection. note that it won't crawl anything if no signals are specified. diff --git a/src/crawler/list_repos/producer.rs b/src/crawler/list_repos/producer.rs index 3ba9d95..28d83cd 100644 --- a/src/crawler/list_repos/producer.rs +++ b/src/crawler/list_repos/producer.rs @@ -10,7 +10,9 @@ use tokio::sync::{mpsc, watch}; use tracing::{debug, error, info, warn}; use url::Url; -use super::super::worker::{CrawlerBatch, CursorUpdate, DispatchWait, RepoListing, send_page}; +use super::super::worker::{ + CrawlerBatch, CursorUpdate, DispatchWait, RepoListing, WAITING_PAGES, WaitingPages, send_page, +}; use super::super::{CrawlerStats, InFlight, base_url}; use super::checker::SignalChecker; use crate::db::keys::crawler_cursor_key; @@ -35,8 +37,14 @@ pub(crate) struct ListReposProducer { impl ListReposProducer { pub(crate) async fn run(mut self) -> Result<()> { + let waiting = WaitingPages::spawn( + self.tx.clone(), + self.checker.state.clone(), + DispatchWait::DEFAULT, + WAITING_PAGES, + ); loop { - if let Err(e) = self.crawl().await { + if let Err(e) = self.crawl(&waiting).await { error!(err = ?e, relay = %self.url, "fatal relay crawl error, restarting in 30s"); tokio::time::sleep(Duration::from_secs(30)).await; } @@ -84,7 +92,7 @@ impl ListReposProducer { Ok(Some((key, next))) } - async fn crawl(&mut self) -> Result<()> { + async fn crawl(&mut self, waiting: &WaitingPages) -> Result<()> { let db = self.checker.state.db.clone(); let base = base_url(&self.url)?; @@ -296,10 +304,9 @@ impl ListReposProducer { }; send_page( &self.tx, - &self.checker.state, + waiting, base.host_str().unwrap_or_default(), read_at, - &DispatchWait::DEFAULT, page, newer, ) diff --git a/src/crawler/worker.rs b/src/crawler/worker.rs index c0c3f76..5348f7a 100644 --- a/src/crawler/worker.rs +++ b/src/crawler/worker.rs @@ -11,7 +11,7 @@ use jacquard_common::types::did::Did; use miette::{IntoDiagnostic, Result}; use rand::Rng; use rand::rngs::SmallRng; -use smol_str::ToSmolStr; +use smol_str::{SmolStr, ToSmolStr}; use std::sync::Arc; use std::time::Duration; use tokio::sync::mpsc; @@ -94,6 +94,7 @@ const DISPATCH_MARGIN: Duration = Duration::from_secs(10); const DISPATCH_TIMEOUT: Duration = Duration::from_secs(600); /// how long [`wait_for_firehose`] waits +#[derive(Debug, Clone, Copy)] pub(crate) struct DispatchWait { pub(crate) margin: Duration, pub(crate) timeout: Duration, @@ -110,16 +111,16 @@ impl DispatchWait { /// waits until the relay firehose for `host` has handed on events past `read_at_ms` and the /// margin, so a listing read then can't claim a commit that's still on its way through -/// hydrant. false when it didn't get there before the timeout. with no relay firehose running +/// hydrant. false when it didn't get there by `deadline`. with no relay firehose running /// there's nothing to wait for pub(crate) async fn wait_for_firehose( state: &AppState, host: &str, read_at_ms: i64, wait: &DispatchWait, + deadline: tokio::time::Instant, ) -> bool { let target = read_at_ms.saturating_add(wait.margin.as_millis() as i64); - let deadline = tokio::time::Instant::now() + wait.timeout; loop { match state.firehose_dispatched(host) { None => return true, @@ -133,53 +134,111 @@ pub(crate) async fn wait_for_firehose( } } +/// how many listing pages may wait for the firehose at once before the crawl waits too +pub(crate) const WAITING_PAGES: usize = 32; + +/// the rest of a listing page: the listings that wait for the firehose, and the page's cursor +struct WaitingPage { + host: SmolStr, + read_at_ms: i64, + deadline: tokio::time::Instant, + newer: Vec, + cursor_updates: Vec, +} + +/// the listing pages a crawl has read but not finished, in the order it read them. a task of +/// its own hands each page's rest to the worker once the firehose caught up to it, so a page +/// that waits doesn't stop the crawl reading the next ones, and still no page's cursor moves +/// before every page ahead of it is in +pub(crate) struct WaitingPages { + pages: mpsc::Sender, + wait: DispatchWait, +} + +impl WaitingPages { + /// at most `bound` pages wait in line, past that [`send_page`] waits for room + pub(crate) fn spawn( + tx: mpsc::Sender, + state: Arc, + wait: DispatchWait, + bound: usize, + ) -> Self { + let (pages, mut waiting) = mpsc::channel::(bound); + tokio::spawn(async move { + while let Some(page) = waiting.recv().await { + let WaitingPage { + host, + read_at_ms, + deadline, + newer, + cursor_updates, + } = page; + // a later page reads no earlier than this one, so once this one is through the + // ones behind it mostly are too, and waiting for them in turn costs little + let caught_up = newer.is_empty() + || wait_for_firehose(&state, &host, read_at_ms, &wait, deadline).await; + let newer = if caught_up { + newer + } else { + info!( + %host, + skipped = newer.len(), + "the firehose didn't catch up to a listing page in time, skipping its newer revs until the next pass" + ); + Vec::new() + }; + if newer.is_empty() && cursor_updates.is_empty() { + continue; + } + let rest = CrawlerBatch { + guards: Vec::new(), + listings: newer, + cursor_updates, + }; + if tx.send(rest).await.is_err() { + return; + } + } + }); + Self { pages, wait } + } +} + /// hands a listing page from `host`, read at `read_at_ms`, to the worker. everything goes right -/// away but `newer`, the listings that claim a newer rev than stored, which waits for the relay -/// firehose to catch up to the page, see [`wait_for_firehose`]. the page's cursor goes with -/// whatever goes last, so it only moves once all of the page is in +/// away but `newer`, the listings that claim a newer rev than stored, which waits in `waiting` +/// for the relay firehose to catch up to the page, see [`wait_for_firehose`]. the page's cursor +/// waits there too, behind every page read before it pub(crate) async fn send_page( tx: &mpsc::Sender, - state: &AppState, + waiting: &WaitingPages, host: &str, read_at_ms: i64, - wait: &DispatchWait, page: CrawlerBatch, newer: Vec, ) { - if newer.is_empty() { - let _ = tx.send(page).await; - return; - } let CrawlerBatch { guards, listings, cursor_updates, } = page; - let now = CrawlerBatch { - guards, - listings, - cursor_updates: Vec::new(), - }; - if tx.send(now).await.is_err() { - return; + if !(guards.is_empty() && listings.is_empty()) { + let now = CrawlerBatch { + guards, + listings, + cursor_updates: Vec::new(), + }; + if tx.send(now).await.is_err() { + return; + } } - let newer = if wait_for_firehose(state, host, read_at_ms, wait).await { - newer - } else { - info!( - host, - skipped = newer.len(), - "the firehose didn't catch up to a listing page in time, skipping its newer revs until the next pass" - ); - Vec::new() + let rest = WaitingPage { + host: host.into(), + read_at_ms, + deadline: tokio::time::Instant::now() + waiting.wait.timeout, + newer, + cursor_updates, }; - let _ = tx - .send(CrawlerBatch { - guards: Vec::new(), - listings: newer, - cursor_updates, - }) - .await; + let _ = waiting.pages.send(rest).await; } #[cfg(test)] @@ -838,7 +897,7 @@ mod tests { poll: Duration::from_millis(10), }; - /// a running crawler worker over a repo synced at `rev(1)`, and the cursor key a page moves + /// a running crawler worker over a repo synced at `rev(1)` fn crawl_over_a_synced_repo() -> Result<(tempfile::TempDir, Arc, mpsc::Sender)> { let tmp = tempfile::tempdir().into_diagnostic()?; @@ -858,7 +917,7 @@ mod tests { Ok((tmp, state, tx)) } - /// a relay firehose for `relay.example` that has handed on events up to `dispatched_ms` + /// a relay firehose for `host` that has handed on events up to `dispatched_ms` fn relay_firehose(state: &AppState, host: &str, dispatched_ms: i64) -> url::Url { let url = url::Url::parse(&format!("wss://{host}/")).unwrap(); let _ = state.firehose_dispatched.insert_sync( @@ -870,48 +929,66 @@ mod tests { const CURSOR: &[u8] = b"test_cursor"; - /// a page with nothing to send right away and a cursor, and a newer rev for the repo - fn page_with_a_newer_rev() -> (CrawlerBatch, Vec) { - let page = CrawlerBatch { - guards: Vec::new(), + /// a page that moves the cursor to `name` and leaves a key of its own named after it, with + /// `guards` to queue right away + fn page(name: &str, guards: Vec) -> CrawlerBatch { + CrawlerBatch { + guards, listings: Vec::new(), - cursor_updates: vec![CursorUpdate { - key: CURSOR.to_vec(), - value: Some(b"next page".to_vec()), - }], - }; - (page, vec![RepoListing::active(did(), rev(3))]) + cursor_updates: vec![ + CursorUpdate { + key: CURSOR.to_vec(), + value: Some(name.as_bytes().to_vec()), + }, + CursorUpdate { + key: name.as_bytes().to_vec(), + value: Some(Vec::new()), + }, + ], + } + } + + /// a newer rev for the synced repo + fn newer_rev() -> Vec { + vec![RepoListing::active(did(), rev(3))] } - /// waits for the worker to commit the page's cursor, the last thing it does with a page - async fn page_done(state: &AppState) { + fn committed(state: &AppState, key: &str) -> bool { + state.db.cursors.contains_key(key.as_bytes()).unwrap() + } + + /// waits for `done` to hold, which the worker makes true a little after it's handed a page + async fn eventually(done: impl Fn() -> bool) { for _ in 0..500 { - if state.db.cursors.contains_key(CURSOR).unwrap() { + if done() { return; } tokio::time::sleep(Duration::from_millis(10)).await; } - panic!("the page's cursor never moved"); + panic!("the worker never got there"); } #[tokio::test] async fn a_newer_rev_waits_for_the_firehose_to_catch_up_to_its_page() -> Result<()> { let (_tmp, state, tx) = crawl_over_a_synced_repo()?; let relay = relay_firehose(&state, "relay.example", 500); - let (page, newer) = page_with_a_newer_rev(); - - let sent = tokio::spawn({ - let state = state.clone(); - async move { send_page(&tx, &state, "relay.example", 1_000, &QUICK, page, newer).await } - }); + let waiting = WaitingPages::spawn(tx.clone(), state.clone(), QUICK, WAITING_PAGES); + + send_page( + &tx, + &waiting, + "relay.example", + 1_000, + page("page", Vec::new()), + newer_rev(), + ) + .await; tokio::time::sleep(Duration::from_millis(200)).await; - assert!(!sent.is_finished()); assert_eq!(state.db.indexer.pending_iter().count(), 0); - assert!(!state.db.cursors.contains_key(CURSOR).into_diagnostic()?); + assert!(!committed(&state, "page")); state.note_dispatched(&relay, 1_000); - sent.await.into_diagnostic()?; - page_done(&state).await; + eventually(|| committed(&state, "page")).await; assert_eq!(state.db.indexer.pending_iter().count(), 1); Ok(()) } @@ -920,20 +997,23 @@ mod tests { async fn a_newer_rev_the_firehose_brought_in_while_it_waited_queues_nothing() -> Result<()> { let (_tmp, state, tx) = crawl_over_a_synced_repo()?; let relay = relay_firehose(&state, "relay.example", 500); - let (page, newer) = page_with_a_newer_rev(); - - let sent = tokio::spawn({ - let state = state.clone(); - async move { send_page(&tx, &state, "relay.example", 1_000, &QUICK, page, newer).await } - }); + let waiting = WaitingPages::spawn(tx.clone(), state.clone(), QUICK, WAITING_PAGES); + + send_page( + &tx, + &waiting, + "relay.example", + 1_000, + page("page", Vec::new()), + newer_rev(), + ) + .await; tokio::time::sleep(Duration::from_millis(200)).await; - assert!(!sent.is_finished()); assert_eq!(state.db.indexer.pending_iter().count(), 0); // the commit the listing saw lands while the page waits insert_repo(&state.db, &state_with_root(rev(3)), 7, false)?; state.note_dispatched(&relay, 1_000); - sent.await.into_diagnostic()?; - page_done(&state).await; + eventually(|| committed(&state, "page")).await; assert_eq!(state.db.indexer.pending_iter().count(), 0); Ok(()) } @@ -942,18 +1022,118 @@ mod tests { async fn a_firehose_that_never_catches_up_skips_the_newer_revs_and_moves_on() -> Result<()> { let (_tmp, state, tx) = crawl_over_a_synced_repo()?; relay_firehose(&state, "relay.example", 500); - let (page, newer) = page_with_a_newer_rev(); let wait = DispatchWait { timeout: Duration::from_millis(100), ..QUICK }; - - send_page(&tx, &state, "relay.example", 1_000, &wait, page, newer).await; - page_done(&state).await; + let waiting = WaitingPages::spawn(tx.clone(), state.clone(), wait, WAITING_PAGES); + + send_page( + &tx, + &waiting, + "relay.example", + 1_000, + page("page", Vec::new()), + newer_rev(), + ) + .await; + eventually(|| committed(&state, "page")).await; assert_eq!(state.db.indexer.pending_iter().count(), 0); Ok(()) } + #[tokio::test] + async fn pages_behind_a_waiting_one_queue_their_repos_but_keep_their_cursors_back() -> Result<()> + { + let (_tmp, state, tx) = crawl_over_a_synced_repo()?; + let relay = relay_firehose(&state, "relay.example", 500); + let waiting = WaitingPages::spawn(tx.clone(), state.clone(), QUICK, WAITING_PAGES); + let in_flight = super::super::InFlight::new(); + let (two, three) = ( + Did::new_static("did:web:two.example").into_diagnostic()?, + Did::new_static("did:web:three.example").into_diagnostic()?, + ); + let known = |did: &Did| state.db.repos.contains_key(keys::repo_key(did)).unwrap(); + + send_page( + &tx, + &waiting, + "relay.example", + 1_000, + page("page 1", Vec::new()), + newer_rev(), + ) + .await; + let page_two = page("page 2", in_flight.acquire(vec![two.clone()]).await); + send_page(&tx, &waiting, "relay.example", 1_100, page_two, Vec::new()).await; + let page_three = page("page 3", in_flight.acquire(vec![three.clone()]).await); + send_page( + &tx, + &waiting, + "relay.example", + 1_200, + page_three, + Vec::new(), + ) + .await; + + eventually(|| known(&two) && known(&three)).await; + tokio::time::sleep(Duration::from_millis(100)).await; + for name in ["page 1", "page 2", "page 3"] { + assert!( + !committed(&state, name), + "{name} moved the cursor before page 1 was in" + ); + } + + state.note_dispatched(&relay, 1_000); + eventually(|| committed(&state, "page 3")).await; + assert!(committed(&state, "page 1") && committed(&state, "page 2")); + assert_eq!( + state.db.cursors.get(CURSOR).into_diagnostic()?.as_deref(), + Some(b"page 3".as_slice()) + ); + assert_eq!(state.db.indexer.pending_iter().count(), 3); + Ok(()) + } + + #[tokio::test] + async fn a_crawl_with_its_waiting_pages_full_waits_for_room() -> Result<()> { + let (_tmp, state, tx) = crawl_over_a_synced_repo()?; + let relay = relay_firehose(&state, "relay.example", 500); + // one page waits for the firehose and one waits behind it, the third finds no room + let waiting = WaitingPages::spawn(tx.clone(), state.clone(), QUICK, 1); + + let crawl = tokio::spawn({ + let state = state.clone(); + async move { + for (n, name) in ["page 1", "page 2", "page 3"].into_iter().enumerate() { + let newer = if n == 0 { newer_rev() } else { Vec::new() }; + send_page( + &tx, + &waiting, + "relay.example", + 1_000, + page(name, Vec::new()), + newer, + ) + .await; + } + drop(state); + } + }); + tokio::time::sleep(Duration::from_millis(200)).await; + assert!(!crawl.is_finished()); + + state.note_dispatched(&relay, 1_000); + tokio::time::timeout(Duration::from_secs(5), crawl) + .await + .into_diagnostic()? + .into_diagnostic()?; + eventually(|| committed(&state, "page 3")).await; + Ok(()) + } + #[tokio::test] async fn a_host_without_a_firehose_of_its_own_waits_for_the_slowest_relay() -> Result<()> { let tmp = tempfile::tempdir().into_diagnostic()?; @@ -961,17 +1141,14 @@ mod tests { database_path: tmp.path().to_path_buf(), ..Default::default() })?; - let wait = DispatchWait { - timeout: Duration::from_millis(50), - ..QUICK - }; + let deadline = || tokio::time::Instant::now() + Duration::from_millis(50); // no relay firehose at all, so nothing to wait for - assert!(wait_for_firehose(&state, "other.example", 1_000, &wait).await); + assert!(wait_for_firehose(&state, "other.example", 1_000, &QUICK, deadline()).await); relay_firehose(&state, "ahead.example", 2_000); relay_firehose(&state, "behind.example", 500); - assert!(wait_for_firehose(&state, "ahead.example", 1_000, &wait).await); - assert!(!wait_for_firehose(&state, "other.example", 1_000, &wait).await); + assert!(wait_for_firehose(&state, "ahead.example", 1_000, &QUICK, deadline()).await); + assert!(!wait_for_firehose(&state, "other.example", 1_000, &QUICK, deadline()).await); Ok(()) } -- knotmirror 0.0.1