add builder-style constructors for driver + store

examples/disk-read-file/main.rs

···
 
 extern crate repo_stream;
 use clap::Parser;
-use repo_stream::{DiskStore, Driver, process::noop};
+use repo_stream::{DiskBuilder, Driver, DriverBuilder};
 use std::path::PathBuf;
 
 #[derive(Debug, Parser)]
···
     let reader = tokio::fs::File::open(car).await?;
     let reader = tokio::io::BufReader::new(reader);
 
-    // configure how much memory can be used before spilling to disk.
-    // real memory usage may differ somewhat.
-    let in_mem_limit = 10; // MiB
-
-    // configure how much memory sqlite is allowed to use when dumping to disk
-    let db_cache_mb = 32; // MiB
-
     log::info!("hello! reading the car...");
 
     // in this example we only bother handling CARs that are too big for memory
     // `noop` helper means: do no block processing, store the raw blocks
-    let driver = match Driver::load_car(reader, noop, in_mem_limit).await? {
+    let driver = match DriverBuilder::new()
+        .with_mem_limit_mb(10) // how much memory can be used before disk spill
+        .load_car(reader)
+        .await?
+    {
         Driver::Memory(_, _) => panic!("try this on a bigger car"),
         Driver::Disk(big_stuff) => {
             // we reach here if the repo was too big and needs to be spilled to
             // disk to continue
 
             // set up a disk store we can spill to
-            let disk_store = DiskStore::new(tmpfile.clone(), db_cache_mb).await?;
+            let disk_store = DiskBuilder::new().open(tmpfile).await?;
 
             // do the spilling, get back a (similar) driver
             let (commit, driver) = big_stuff.finish_loading(disk_store).await?;

examples/read-file/main.rs

···
 
 extern crate repo_stream;
 use clap::Parser;
-use repo_stream::Driver;
+use repo_stream::{Driver, DriverBuilder};
 use std::path::PathBuf;
 
 type Result<T> = std::result::Result<T, Box<dyn std::error::Error>>;
···
     let reader = tokio::fs::File::open(file).await?;
     let reader = tokio::io::BufReader::new(reader);
 
-    let (commit, mut driver) =
-        match Driver::load_car(reader, |block| block.len(), 16 /* MiB */).await? {
-            Driver::Memory(commit, mem_driver) => (commit, mem_driver),
-            Driver::Disk(_) => panic!("this example doesn't handle big CARs"),
-        };
+    let (commit, mut driver) = match DriverBuilder::new()
+        .with_block_processor(|block| block.len())
+        .load_car(reader)
+        .await?
+    {
+        Driver::Memory(commit, mem_driver) => (commit, mem_driver),
+        Driver::Disk(_) => panic!("this example doesn't handle big CARs"),
+    };
 
     log::info!("got commit: {commit:?}");
 

src/disk.rs

···
 to be the best behaved in terms of both on-disk space usage and memory usage.
 
 ```no_run
-# use repo_stream::{DiskStore, DiskError};
+# use repo_stream::{DiskBuilder, DiskError};
 # #[tokio::main]
 # async fn main() -> Result<(), DiskError> {
-let db_cache_size = 32; // MiB
-let store = DiskStore::new("/some/path.db".into(), db_cache_size).await?;
+let store = DiskBuilder::new()
+    .with_cache_size_mb(32)
+    .with_max_stored_mb(1024) // errors when >1GiB of processed blocks are inserted
+    .open("/some/path.db".into()).await?;
 # Ok(())
 # }
 ```
···
     /// A tokio blocking task failed to join
     #[error("Failed to join a tokio blocking task: {0}")]
     JoinError(#[from] tokio::task::JoinError),
+    /// The total size of stored blocks exceeded the allowed size
+    ///
+    /// If you need to process *really* big CARs, you can configure a higher
+    /// limit.
+    #[error("Maximum disk size reached")]
+    MaxSizeExceeded,
     #[error("this error was replaced, seeing this is a bug.")]
     #[doc(hidden)]
     Stolen,
···
     }
 }
 
+/// Builder-style disk store setup
+pub struct DiskBuilder {
+    /// Database in-memory cache allowance
+    ///
+    /// Default: 32 MiB
+    pub cache_size_mb: usize,
+    /// Database stored block size limit
+    ///
+    /// Default: 10 GiB
+    ///
+    /// Note: actual size on disk may be more, but should approximately scale
+    /// with this limit
+    pub max_stored_mb: usize,
+}
+
+impl Default for DiskBuilder {
+    fn default() -> Self {
+        Self {
+            cache_size_mb: 32,
+            max_stored_mb: 10 * 1024, // 10 GiB
+        }
+    }
+}
+
+impl DiskBuilder {
+    /// Begin configuring the storage with defaults
+    pub fn new() -> Self {
+        Default::default()
+    }
+    /// Set the in-memory cache allowance for the database
+    ///
+    /// Default: 32 MiB
+    pub fn with_cache_size_mb(mut self, size: usize) -> Self {
+        self.cache_size_mb = size;
+        self
+    }
+    /// Set the approximate stored block size limit
+    ///
+    /// Default: 10 GiB
+    pub fn with_max_stored_mb(mut self, max: usize) -> Self {
+        self.max_stored_mb = max;
+        self
+    }
+    /// Open and initialize the actual disk storage
+    pub async fn open(self, path: PathBuf) -> Result<DiskStore, DiskError> {
+        DiskStore::new(path, self.cache_size_mb, self.max_stored_mb).await
+    }
+}
+
 /// On-disk block storage
 pub struct DiskStore {
     conn: rusqlite::Connection,
+    max_stored: usize,
+    stored: usize,
 }
 
 impl DiskStore {
     /// Initialize a new disk store
-    pub async fn new(path: PathBuf, cache_mb: usize) -> Result<Self, DiskError> {
+    pub async fn new(
+        path: PathBuf,
+        cache_mb: usize,
+        max_stored_mb: usize,
+    ) -> Result<Self, DiskError> {
+        let max_stored = max_stored_mb * 2_usize.pow(20);
         let conn = tokio::task::spawn_blocking(move || {
             let conn = rusqlite::Connection::open(path)?;
 
···
         })
         .await??;
 
-        Ok(Self { conn })
+        Ok(Self {
+            conn,
+            max_stored,
+            stored: 0,
+        })
     }
     pub(crate) fn get_writer(&'_ mut self) -> Result<SqliteWriter<'_>, DiskError> {
         let tx = self.conn.transaction()?;
-        Ok(SqliteWriter { tx })
+        Ok(SqliteWriter {
+            tx,
+            stored: &mut self.stored,
+            max: self.max_stored,
+        })
     }
     pub(crate) fn get_reader<'conn>(&'conn self) -> Result<SqliteReader<'conn>, DiskError> {
         let select_stmt = self.conn.prepare("SELECT val FROM blocks WHERE key = ?1")?;
···
 
 pub(crate) struct SqliteWriter<'conn> {
     tx: rusqlite::Transaction<'conn>,
+    stored: &'conn mut usize,
+    max: usize,
 }
 
 impl SqliteWriter<'_> {
···
             .map_err(DiskError::DbError)?;
         for pair in kv {
             let (k, v) = pair?;
+            *self.stored += v.len();
+            if *self.stored > self.max {
+                return Err(DiskError::MaxSizeExceeded.into());
+            }
             insert_stmt.execute((k, v)).map_err(DiskError::DbError)?;
         }
         Ok(())

src/drive.rs

···
     Disk(NeedDisk<R, T>),
 }
 
+/// Builder-style driver setup
+pub struct DriverBuilder {
+    pub mem_limit_mb: usize,
+}
+
+impl Default for DriverBuilder {
+    fn default() -> Self {
+        Self { mem_limit_mb: 16 }
+    }
+}
+
+impl DriverBuilder {
+    /// Begin configuring the driver with defaults
+    pub fn new() -> Self {
+        Default::default()
+    }
+    /// Set the in-memory size limit, in MiB
+    ///
+    /// Default: 16 MiB
+    pub fn with_mem_limit_mb(self, new_limit: usize) -> Self {
+        Self {
+            mem_limit_mb: new_limit,
+        }
+    }
+    /// Set the block processor
+    ///
+    /// Default: noop, raw blocks will be emitted
+    pub fn with_block_processor<T: Processable>(
+        self,
+        p: fn(Vec<u8>) -> T,
+    ) -> DriverBuilderWithProcessor<T> {
+        DriverBuilderWithProcessor {
+            mem_limit_mb: self.mem_limit_mb,
+            block_processor: p,
+        }
+    }
+    /// Begin processing an atproto MST from a CAR file
+    pub async fn load_car<R: AsyncRead + Unpin>(
+        self,
+        reader: R,
+    ) -> Result<Driver<R, Vec<u8>>, DriveError> {
+        Driver::load_car(reader, crate::process::noop, self.mem_limit_mb).await
+    }
+}
+
+/// Builder-style driver intermediate step
+///
+/// start from `DriverBuilder`
+pub struct DriverBuilderWithProcessor<T: Processable> {
+    pub mem_limit_mb: usize,
+    pub block_processor: fn(Vec<u8>) -> T,
+}
+
+impl<T: Processable> DriverBuilderWithProcessor<T> {
+    /// Set the in-memory size limit, in MiB
+    ///
+    /// Default: 16 MiB
+    pub fn with_mem_limit_mb(mut self, new_limit: usize) -> Self {
+        self.mem_limit_mb = new_limit;
+        self
+    }
+    /// Begin processing an atproto MST from a CAR file
+    pub async fn load_car<R: AsyncRead + Unpin>(
+        self,
+        reader: R,
+    ) -> Result<Driver<R, T>, DriveError> {
+        Driver::load_car(reader, self.block_processor, self.mem_limit_mb).await
+    }
+}
+
 impl<R: AsyncRead + Unpin, T: Processable> Driver<R, T> {
     /// Begin processing an atproto MST from a CAR file
     ///
     /// Blocks will be loaded, processed, and buffered in memory. If the entire
-    /// processed size is under the `max_size_mb` limit, a `Driver::Memory` will
-    /// be returned along with a `Commit` ready for validation.
+    /// processed size is under the `mem_limit_mb` limit, a `Driver::Memory`
+    /// will be returned along with a `Commit` ready for validation.
     ///
-    /// If the `max_size_mb` limit is reached before loading all blocks, the
+    /// If the `mem_limit_mb` limit is reached before loading all blocks, the
     /// partial state will be returned as `Driver::Disk(needed)`, which can be
     /// resumed by providing a `SqliteStorage` for on-disk block storage.
     pub async fn load_car(
         reader: R,
         process: fn(Vec<u8>) -> T,
-        max_size_mb: usize,
+        mem_limit_mb: usize,
     ) -> Result<Driver<R, T>, DriveError> {
-        let max_size = max_size_mb * 2_usize.pow(20);
+        let max_size = mem_limit_mb * 2_usize.pow(20);
         let mut mem_blocks = HashMap::new();
 
         let mut car = CarReader::new(reader).await?;

src/lib.rs

···
 `iroh_car` additionally applies a block size limit of `2MiB`.
 
 ```
-use repo_stream::{Driver, DiskStore};
+use repo_stream::{Driver, DriverBuilder, DiskBuilder};
 
 # #[tokio::main]
 # async fn main() -> Result<(), Box<dyn std::error::Error>> {
 # let reader = include_bytes!("../car-samples/tiny.car").as_slice();
 let mut total_size = 0;
-let process = |rec: Vec<u8>| rec.len(); // block processing: just extract the size
-let in_mem_limit = 10; /* MiB */
-let db_cache_size = 32; /* MiB */
 
-match Driver::load_car(reader, process, in_mem_limit).await? {
+match DriverBuilder::new()
+    .with_mem_limit_mb(10)
+    .with_block_processor(|rec| rec.len()) // block processing: just extract the raw record size
+    .load_car(reader)
+    .await?
+{
 
     // if all blocks fit within memory
     Driver::Memory(_commit, mut driver) => {
···
     // if the CAR was too big for in-memory processing
     Driver::Disk(paused) => {
         // set up a disk store we can spill to
-        let store = DiskStore::new("some/path.db".into(), db_cache_size).await?;
+        let store = DiskBuilder::new().open("some/path.db".into()).await?;
         // do the spilling, get back a (similar) driver
         let (_commit, mut driver) = paused.finish_loading(store).await?;
 
···
 pub mod drive;
 pub mod process;
 
-pub use disk::{DiskError, DiskStore};
-pub use drive::{DriveError, Driver};
+pub use disk::{DiskBuilder, DiskError, DiskStore};
+pub use drive::{DriveError, Driver, DriverBuilder};
 pub use mst::Commit;
 pub use process::Processable;