public api cleanup

benches/huge-car.rs

···
 extern crate repo_stream;
+use repo_stream::Driver;
 use std::path::{Path, PathBuf};
 
 use criterion::{Criterion, criterion_group, criterion_main};
···
 
     let mb = 2_usize.pow(20);
 
-    let mut driver = match repo_stream::drive::load_car(reader, |block| block.len(), 1024 * mb)
+    let mut driver = match Driver::load_car(reader, |block| block.len(), 1024 * mb)
         .await
         .unwrap()
     {
-        repo_stream::drive::Vehicle::Lil(_, mem_driver) => mem_driver,
-        repo_stream::drive::Vehicle::Big(_) => panic!("not doing disk for benchmark"),
+        Driver::Lil(_, mem_driver) => mem_driver,
+        Driver::Big(_) => panic!("not doing disk for benchmark"),
     };
 
     let mut n = 0;

benches/non-huge-cars.rs

···
 extern crate repo_stream;
+use repo_stream::Driver;
 
 use criterion::{Criterion, criterion_group, criterion_main};
 
···
 }
 
 async fn drive_car(bytes: &[u8]) -> usize {
-    let mut driver =
-        match repo_stream::drive::load_car(bytes, |block| block.len(), 32 * 2_usize.pow(20))
-            .await
-            .unwrap()
-        {
-            repo_stream::drive::Vehicle::Lil(_, mem_driver) => mem_driver,
-            repo_stream::drive::Vehicle::Big(_) => panic!("not benching big cars here"),
-        };
+    let mut driver = match Driver::load_car(bytes, |block| block.len(), 32 * 2_usize.pow(20))
+        .await
+        .unwrap()
+    {
+        Driver::Lil(_, mem_driver) => mem_driver,
+        Driver::Big(_) => panic!("not benching big cars here"),
+    };
 
     let mut n = 0;
     while let Some(pairs) = driver.next_chunk(256).await.unwrap() {

examples/disk-read-file/main.rs

···
 extern crate repo_stream;
 use clap::Parser;
+use repo_stream::{Driver, noop};
 use std::path::PathBuf;
 
 type Result<T> = std::result::Result<T, Box<dyn std::error::Error>>;
···
 
     let limit_mb = 32;
 
-    let driver = match repo_stream::drive::load_car(reader, |block| block, 10 * mb).await? {
-        repo_stream::drive::Vehicle::Lil(_, _) => panic!("try this on a bigger car"),
-        repo_stream::drive::Vehicle::Big(big_stuff) => {
+    let driver = match Driver::load_car(reader, noop, 10 * mb).await? {
+        Driver::Lil(_, _) => panic!("try this on a bigger car"),
+        Driver::Big(big_stuff) => {
             let disk_store = repo_stream::disk::SqliteStore::new(tmpfile.clone(), limit_mb).await?;
             let (commit, driver) = big_stuff.finish_loading(disk_store).await?;
             log::warn!("big: {:?}", commit);

examples/read-file/main.rs

···
 extern crate repo_stream;
 use clap::Parser;
+use repo_stream::Driver;
 use std::path::PathBuf;
 
 type Result<T> = std::result::Result<T, Box<dyn std::error::Error>>;
···
     let reader = tokio::io::BufReader::new(reader);
 
     let (commit, mut driver) =
-        match repo_stream::drive::load_car(reader, |block| block.len(), 1024 * 1024).await? {
-            repo_stream::drive::Vehicle::Lil(commit, mem_driver) => (commit, mem_driver),
-            repo_stream::drive::Vehicle::Big(_) => panic!("can't handle big cars yet"),
+        match Driver::load_car(reader, |block| block.len(), 16 * 1024 * 1024).await? {
+            Driver::Lil(commit, mem_driver) => (commit, mem_driver),
+            Driver::Big(_) => panic!("can't handle big cars yet"),
         };
 
     log::info!("got commit: {commit:?}");

src/drive.rs

···
     }
 }
 
-pub enum Vehicle<R: AsyncRead + Unpin, T: Processable> {
+pub enum Driver<R: AsyncRead + Unpin, T: Processable> {
     Lil(Commit, MemDriver<T>),
     Big(BigCar<R, T>),
 }
 
-pub async fn load_car<R: AsyncRead + Unpin, T: Processable>(
-    reader: R,
-    process: fn(Vec<u8>) -> T,
-    max_size: usize,
-) -> Result<Vehicle<R, T>, DriveError> {
-    let mut mem_blocks = HashMap::new();
+impl<R: AsyncRead + Unpin, T: Processable> Driver<R, T> {
+    pub async fn load_car(
+        reader: R,
+        process: fn(Vec<u8>) -> T,
+        max_size: usize,
+    ) -> Result<Driver<R, T>, DriveError> {
+        let mut mem_blocks = HashMap::new();
 
-    let mut car = CarReader::new(reader).await?;
+        let mut car = CarReader::new(reader).await?;
 
-    let root = *car
-        .header()
-        .roots()
-        .first()
-        .ok_or(DriveError::MissingRoot)?;
-    log::debug!("root: {root:?}");
+        let root = *car
+            .header()
+            .roots()
+            .first()
+            .ok_or(DriveError::MissingRoot)?;
+        log::debug!("root: {root:?}");
 
-    let mut commit = None;
+        let mut commit = None;
 
-    // try to load all the blocks into memory
-    let mut mem_size = 0;
-    while let Some((cid, data)) = car.next_block().await? {
-        // the root commit is a Special Third Kind of block that we need to make
-        // sure not to optimistically send to the processing function
-        if cid == root {
-            let c: Commit = serde_ipld_dagcbor::from_slice(&data)?;
-            commit = Some(c);
-            continue;
-        }
+        // try to load all the blocks into memory
+        let mut mem_size = 0;
+        while let Some((cid, data)) = car.next_block().await? {
+            // the root commit is a Special Third Kind of block that we need to make
+            // sure not to optimistically send to the processing function
+            if cid == root {
+                let c: Commit = serde_ipld_dagcbor::from_slice(&data)?;
+                commit = Some(c);
+                continue;
+            }
 
-        // remaining possible types: node, record, other. optimistically process
-        let maybe_processed = if Node::could_be(&data) {
-            MaybeProcessedBlock::Raw(data)
-        } else {
-            MaybeProcessedBlock::Processed(process(data))
-        };
+            // remaining possible types: node, record, other. optimistically process
+            let maybe_processed = if Node::could_be(&data) {
+                MaybeProcessedBlock::Raw(data)
+            } else {
+                MaybeProcessedBlock::Processed(process(data))
+            };
 
-        // stash (maybe processed) blocks in memory as long as we have room
-        mem_size += std::mem::size_of::<Cid>() + maybe_processed.get_size();
-        mem_blocks.insert(cid, maybe_processed);
-        if mem_size >= max_size {
-            return Ok(Vehicle::Big(BigCar {
-                car,
-                root,
-                process,
-                max_size,
-                mem_blocks,
-                commit,
-            }));
+            // stash (maybe processed) blocks in memory as long as we have room
+            mem_size += std::mem::size_of::<Cid>() + maybe_processed.get_size();
+            mem_blocks.insert(cid, maybe_processed);
+            if mem_size >= max_size {
+                return Ok(Driver::Big(BigCar {
+                    car,
+                    root,
+                    process,
+                    max_size,
+                    mem_blocks,
+                    commit,
+                }));
+            }
         }
-    }
 
-    // all blocks loaded and we fit in memory! hopefully we found the commit...
-    let commit = commit.ok_or(DriveError::MissingCommit)?;
+        // all blocks loaded and we fit in memory! hopefully we found the commit...
+        let commit = commit.ok_or(DriveError::MissingCommit)?;
 
-    let walker = Walker::new(commit.data);
+        let walker = Walker::new(commit.data);
 
-    Ok(Vehicle::Lil(
-        commit,
-        MemDriver {
-            blocks: mem_blocks,
-            walker,
-            process,
-        },
-    ))
+        Ok(Driver::Lil(
+            commit,
+            MemDriver {
+                blocks: mem_blocks,
+                walker,
+                process,
+            },
+        ))
+    }
 }
 
 /// a paritally memory-loaded car file that needs disk spillover to continue

src/lib.rs

···
 pub mod mst;
 pub mod process;
 pub mod walk;
+
+pub use disk::SqliteStore;
+pub use drive::{DriveError, Driver};
+pub use process::{Processable, noop};

src/process.rs

···
     fn get_size(&self) -> usize;
 }
 
+#[inline]
+pub fn noop(block: Vec<u8>) -> Vec<u8> {
+    block
+}
+
 impl Processable for u8 {
     fn get_size(&self) -> usize {
         0

tests/non-huge-cars.rs

···
 extern crate repo_stream;
+use repo_stream::Driver;
 
 const TINY_CAR: &'static [u8] = include_bytes!("../car-samples/tiny.car");
 const LITTLE_CAR: &'static [u8] = include_bytes!("../car-samples/little.car");
···
 async fn test_car(bytes: &[u8], expected_records: usize, expected_sum: usize) {
     let mb = 2_usize.pow(20);
 
-    let mut driver = match repo_stream::drive::load_car(bytes, |block| block.len(), 10 * mb)
+    let mut driver = match Driver::load_car(bytes, |block| block.len(), 10 * mb)
         .await
         .unwrap()
     {
-        repo_stream::drive::Vehicle::Lil(_commit, mem_driver) => mem_driver,
-        repo_stream::drive::Vehicle::Big(_) => panic!("too big"),
+        Driver::Lil(_commit, mem_driver) => mem_driver,
+        Driver::Big(_) => panic!("too big"),
     };
 
     let mut records = 0;