Fast and robust atproto CAR file processing in rust
microcosm.blue
1extern crate repo_stream;
2use repo_stream::Driver;
3
4const TINY_CAR: &'static [u8] = include_bytes!("../car-samples/tiny.car");
5const LITTLE_CAR: &'static [u8] = include_bytes!("../car-samples/little.car");
6const MIDSIZE_CAR: &'static [u8] = include_bytes!("../car-samples/midsize.car");
7
8async fn test_car(bytes: &[u8], expected_records: usize, expected_sum: usize) {
9 let mut driver = match Driver::load_car(bytes, |block| block.len(), 10 /* MiB */)
10 .await
11 .unwrap()
12 {
13 Driver::Memory(_commit, mem_driver) => mem_driver,
14 Driver::Disk(_) => panic!("too big"),
15 };
16
17 let mut records = 0;
18 let mut sum = 0;
19 let mut found_bsky_profile = false;
20 let mut prev_rkey = "".to_string();
21
22 while let Some(pairs) = driver.next_chunk(256).await.unwrap() {
23 for (rkey, size) in pairs {
24 records += 1;
25 sum += size;
26 if rkey == "app.bsky.actor.profile/self" {
27 found_bsky_profile = true;
28 }
29 assert!(rkey > prev_rkey, "rkeys are streamed in order");
30 prev_rkey = rkey;
31 }
32 }
33
34 assert_eq!(records, expected_records);
35 assert_eq!(sum, expected_sum);
36 assert!(found_bsky_profile);
37}
38
39#[tokio::test]
40async fn test_tiny_car() {
41 test_car(TINY_CAR, 8, 2071).await
42}
43
44#[tokio::test]
45async fn test_little_car() {
46 test_car(LITTLE_CAR, 278, 246960).await
47}
48
49#[tokio::test]
50async fn test_midsize_car() {
51 test_car(MIDSIZE_CAR, 11585, 3741393).await
52}