Something went wrong. Try again.
very fast at protocol indexer with flexible filtering, xrpc queries, cursor-backed event stream, and more, built on fjall
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237use fjall::Slice;use jacquard_common::IntoStatic;use jacquard_common::types::did::Did;use miette::{IntoDiagnostic, Result};use std::sync::Arc;use tracing::{debug, error, warn};
use crate::backfill::admission::BackfillAdmission;use crate::backfill::client::ThrottledHttpClient;use crate::backfill::error::BackfillError;use crate::config::BackfillStrategy;use crate::db::{Txn as DbTxn, keys};use crate::ingest::indexer::{IndexerMessage, IndexerTx};use crate::state::AppState;use crate::types::{GaugeState, RepoState, RepoStatus, ResyncErrorKind, ResyncState};
use super::process::process_did;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]pub(super) enum TaskDisposition { Finished, Deferred,}
pub(super) async fn did_task( state: &Arc<AppState>, http: ThrottledHttpClient, buffer_tx: IndexerTx, did: &Did<'static>, pending_key: Slice, admission: BackfillAdmission, _permit: tokio::sync::OwnedSemaphorePermit, verify_signatures: bool, strategy: BackfillStrategy,) -> Result<TaskDisposition, BackfillError> { let db = &state.db;
match process_did( state, &http, did, pending_key.clone(), &admission, verify_signatures, strategy, ) .await { Ok(Some(_repo_state)) => { let applied = state .db .run({ let did = did.clone(); let pending_key = pending_key.clone(); move |db| { let did_key = keys::repo_key(&did); let mut txn = DbTxn::new(db); let applied = txn.transition_pending_key( &did, pending_key.as_ref(), GaugeState::Synced, )?; txn.batch.remove(&db.indexer.pending, pending_key.clone()); if applied { txn.batch.remove(&db.indexer.resync, &did_key); } txn.commit()?; Ok::<_, miette::Report>(applied) } }) .await?;
if !applied { return Ok(TaskDisposition::Finished); }
state .backfill_drained .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
let state = state.clone(); state .db .run(move |db| { db.inner .persist(fjall::PersistMode::Buffer) .into_diagnostic() }) .await?;
if let Err(e) = buffer_tx .send(IndexerMessage::BackfillFinished(did.clone())) .await { error!(err = %e, "failed to send BackfillFinished"); } Ok(TaskDisposition::Finished) } Ok(None) => Ok(TaskDisposition::Finished), Err(BackfillError::PdsBusy) => Ok(TaskDisposition::Deferred), Err(BackfillError::Deleted) => { warn!("orphaned pending entry, cleaning up"); state .db .run({ let did = did.clone(); let pending_key = pending_key.clone(); move |db| { let mut txn = DbTxn::new(db); txn.transition_pending_key(&did, pending_key.as_ref(), GaugeState::Synced)?; txn.batch.remove(&db.indexer.pending, pending_key); txn.commit()?; Ok::<_, miette::Report>(()) } }) .await?; state .backfill_drained .fetch_add(1, std::sync::atomic::Ordering::Relaxed); Ok(TaskDisposition::Finished) } Err(e) => { match &e { BackfillError::Ratelimited | BackfillError::PreemptivelyThrottled => { debug!("too many requests"); } BackfillError::Transport(reason) => { error!(%reason, "transport error"); } BackfillError::Generic(e) => { error!(err = %e, "failed"); } BackfillError::Deleted => unreachable!("already handled"), BackfillError::PdsBusy => unreachable!("already handled"), }
let error_kind = match &e { BackfillError::Ratelimited | BackfillError::PreemptivelyThrottled => { ResyncErrorKind::Ratelimited } BackfillError::Transport(_) => ResyncErrorKind::Transport, BackfillError::Generic(_) => ResyncErrorKind::Generic, BackfillError::Deleted => unreachable!("already handled"), BackfillError::PdsBusy => unreachable!("already handled"), };
let did_key = keys::repo_key(did);
// 1. get current retry count let did_key_clone = did_key.clone(); let existing_state = db .run(move |db| { db.indexer .resync .get(&did_key_clone) .into_diagnostic() .and_then(|b| { b.map(|b| rmp_serde::from_slice::<ResyncState>(&b).into_diagnostic()) .transpose() }) }) .await?;
let mut retry_count = match existing_state { Some(ResyncState::Error { retry_count, .. }) => retry_count, Some(ResyncState::Gone { .. }) => return Ok(TaskDisposition::Finished), // should handle gone? original code didn't really? None => 0, };
// Calculate new stats let next_retry = if matches!(e, BackfillError::PreemptivelyThrottled) { chrono::Utc::now().timestamp() + 60 } else { retry_count += 1; ResyncState::next_backoff(retry_count) };
let resync_state = ResyncState::Error { kind: error_kind, retry_count, next_retry, }; let error_string = e.to_string();
let applied = state .db .run({ let did_key = did_key.into_static(); let did = did.clone(); let pending_key = pending_key.clone(); move |db| { // 3. save to resync let serialized_resync_state = rmp_serde::to_vec(&resync_state).into_diagnostic()?;
// 4. and update the main repo state let serialized_repo_state = if let Some(state_bytes) = db.repos.get(&did_key).into_diagnostic()? { let mut state: RepoState = rmp_serde::from_slice(&state_bytes).into_diagnostic()?; state.active = true; state.status = RepoStatus::Error(error_string.into()); Some(rmp_serde::to_vec(&state).into_diagnostic()?) } else { None }; let mut txn = DbTxn::new(db); let applied = txn.transition_pending_key( &did, pending_key.as_ref(), GaugeState::Resync(Some(error_kind)), )?; txn.batch.remove(&db.indexer.pending, pending_key.clone()); if applied { txn.batch .insert(&db.indexer.resync, &did_key, serialized_resync_state); if let Some(state_bytes) = serialized_repo_state { txn.batch.insert(&db.repos, &did_key, state_bytes); } } txn.commit()?; Ok::<_, miette::Report>(applied) } }) .await?;
if applied { state .backfill_drained .fetch_add(1, std::sync::atomic::Ordering::Relaxed); }
Err(e) } }}