tangled
alpha
login
or
join now
nonbinary.computer
/
weaver
atproto blogging
24
fork
atom
overview
issues
2
pulls
pipelines
couple fixes
Orual
1 month ago
0186121d
25871687
+10
-9
2 changed files
expand all
collapse all
unified
split
crates
weaver-index
src
clickhouse.rs
parallel_tap.rs
+3
-1
crates/weaver-index/src/clickhouse.rs
···
6
7
pub use client::{Client, TableSize};
8
pub use migrations::{DbObject, MigrationResult, Migrator, ObjectType};
9
-
pub use queries::{EntryRow, HandleMappingRow, NotebookRow, ProfileCountsRow, ProfileRow, ProfileWithCounts};
0
0
10
pub use resilient_inserter::{InserterConfig, ResilientRecordInserter};
11
pub use schema::{
12
AccountRevState, FirehoseCursor, RawAccountEvent, RawEventDlq, RawIdentityEvent,
···
6
7
pub use client::{Client, TableSize};
8
pub use migrations::{DbObject, MigrationResult, Migrator, ObjectType};
9
+
pub use queries::{
10
+
EntryRow, HandleMappingRow, NotebookRow, ProfileCountsRow, ProfileRow, ProfileWithCounts,
11
+
};
12
pub use resilient_inserter::{InserterConfig, ResilientRecordInserter};
13
pub use schema::{
14
AccountRevState, FirehoseCursor, RawAccountEvent, RawEventDlq, RawIdentityEvent,
+7
-8
crates/weaver-index/src/parallel_tap.rs
···
1
-
use std::sync::atomic::{AtomicBool, Ordering};
2
use std::sync::Arc;
0
3
use std::time::{Duration, Instant};
4
5
use chrono::Utc;
···
7
use tokio::task::JoinHandle;
8
use tracing::{debug, error, info, trace, warn};
9
10
-
use crate::clickhouse::migrations::Migrator;
11
use crate::clickhouse::{
12
-
Client, InserterConfig, RawIdentityEvent, RawRecordInsert, ResilientRecordInserter,
13
};
14
use crate::config::{IndexerConfig, TapConfig};
15
use crate::error::{ClickHouseError, Result};
···
239
.compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
240
.is_ok()
241
{
242
-
info!(
243
-
worker_id,
244
-
"first live event received, scheduling backfill"
245
-
);
246
let backfill_client = client.clone();
247
tokio::spawn(async move {
248
run_backfill(backfill_client).await;
···
310
return;
311
}
312
313
-
info!(count = mvs.len(), "backfill: starting incremental MV backfill");
0
0
0
314
315
for mv in mvs {
316
info!(
···
0
1
use std::sync::Arc;
2
+
use std::sync::atomic::{AtomicBool, Ordering};
3
use std::time::{Duration, Instant};
4
5
use chrono::Utc;
···
7
use tokio::task::JoinHandle;
8
use tracing::{debug, error, info, trace, warn};
9
0
10
use crate::clickhouse::{
11
+
Client, InserterConfig, Migrator, RawIdentityEvent, RawRecordInsert, ResilientRecordInserter,
12
};
13
use crate::config::{IndexerConfig, TapConfig};
14
use crate::error::{ClickHouseError, Result};
···
238
.compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
239
.is_ok()
240
{
241
+
info!(worker_id, "first live event received, scheduling backfill");
0
0
0
242
let backfill_client = client.clone();
243
tokio::spawn(async move {
244
run_backfill(backfill_client).await;
···
306
return;
307
}
308
309
+
info!(
310
+
count = mvs.len(),
311
+
"backfill: starting incremental MV backfill"
312
+
);
313
314
for mv in mvs {
315
info!(