Constellation, Spacedust, Slingshot, UFOs: atproto crates and services for microcosm

count total samples

and nicer prints

Changed files
+13 -5
ufos
+3 -2
ufos/src/consumer.rs
··· 121 121 created_collection = true; 122 122 Default::default() 123 123 }); 124 - collection.push_front(record); 125 - collection.truncate(MAX_BATCHED_RECORDS); 124 + collection.total_seen += 1; 125 + collection.samples.push_front(record); 126 + collection.samples.truncate(MAX_BATCHED_RECORDS); 126 127 127 128 if created_collection { 128 129 self.current_batch.records.len() >= MAX_BATCHED_COLLECTIONS // full if we have collections to the max
+7 -1
ufos/src/lib.rs
··· 12 12 pub record: serde_json::Value, 13 13 } 14 14 15 + #[derive(Debug, Default)] 16 + pub struct CollectionSamples { 17 + pub total_seen: usize, 18 + pub samples: VecDeque<SetRecord>, 19 + } 20 + 15 21 #[derive(Debug)] 16 22 pub struct DeleteRecord { 17 23 pub did: Did, ··· 21 27 22 28 #[derive(Debug, Default)] 23 29 pub struct EventBatch { 24 - pub records: HashMap<Nsid, VecDeque<SetRecord>>, 30 + pub records: HashMap<Nsid, CollectionSamples>, 25 31 pub record_deletes: Vec<DeleteRecord>, 26 32 pub account_removes: Vec<Did>, 27 33 }
+3 -2
ufos/src/store.rs
··· 20 20 record_deletes, 21 21 account_removes, 22 22 } = batch; 23 - let total_records: usize = records.values().map(|v| v.len()).sum(); 23 + let total_records: usize = records.values().map(|v| v.total_seen).sum(); 24 + let total_samples: usize = records.values().map(|v| v.samples.len()).sum(); 24 25 println!( 25 - "got batch with {total_records} records in {} collections, {} record deletes, {} account removes", 26 + "got batch of {total_samples: >3} samples from {total_records: >3} records in {: >2} collections, {: >2} record deletes, {} account removes", 26 27 records.len(), 27 28 record_deletes.len(), 28 29 account_removes.len()