sizing and profile a lil

Cargo.toml

···
 inherits = "release"
 debug = true
 
+[profile.release]
+debug = true
+
 [[bench]]
 name = "non-huge-cars"
 harness = false

examples/disk-read-file/main.rs

···
 extern crate repo_stream;
 use clap::Parser;
-use repo_stream::disk::RedbStore;
 use repo_stream::drive::Processable;
 use serde::{Deserialize, Serialize};
 use std::path::PathBuf;
···
 #[derive(Clone, Serialize, Deserialize)]
 struct S(usize);
 
-impl Processable for S {}
+impl Processable for S {
+    fn get_size(&self) -> usize {
+        0 // no additional space taken, just its stack size (newtype is free)
+    }
+}
 
 #[tokio::main]
 async fn main() -> Result<()> {
···
     let reader = tokio::fs::File::open(car).await?;
     let reader = tokio::io::BufReader::new(reader);
 
+    // let kb = 2_usize.pow(10);
+    let mb = 2_usize.pow(20);
+
     let mut driver =
-        match repo_stream::drive::load_car(reader, |block| S(block.len()), 1024).await? {
+        match repo_stream::drive::load_car(reader, |block| S(block.len()), 16 * mb).await? {
             repo_stream::drive::Vehicle::Lil(_, _) => panic!("try this on a bigger car"),
             repo_stream::drive::Vehicle::Big(big_stuff) => {
-                let disk_store = RedbStore::new(tmpfile);
+                // let disk_store = repo_stream::disk::SqliteStore::new(tmpfile);
+                let disk_store = repo_stream::disk::RedbStore::new(tmpfile);
                 let (commit, driver) = big_stuff.finish_loading(disk_store).await?;
                 log::warn!("big: {:?}", commit);
                 driver

src/disk.rs

···
 
             conn.pragma_update(None, "journal_mode", "WAL")?;
             conn.pragma_update(None, "synchronous", "OFF")?;
-            conn.pragma_update(None, "cache_size", (-32 * 2_i64.pow(10)).to_string())?;
+            conn.pragma_update(None, "cache_size", (-4 * 2_i64.pow(10)).to_string())?;
             conn.execute(
                 "CREATE TABLE blocks (
                     key  BLOB PRIMARY KEY NOT NULL,
···
     type Access = RedbAccess;
     async fn get_access(&mut self) -> Result<RedbAccess, redb::Error> {
         let path = self.path.clone();
+        let kb = 2_usize.pow(10);
         let db = tokio::task::spawn_blocking(move || {
-            let db = redb::Database::create(path)?;
+            let db = redb::Database::builder()
+                .set_cache_size(16 * kb)
+                .create(path)?;
             Ok::<_, Self::StorageError>(db)
         })
         .await
···
         Ok(rv)
     }
 }
+
+///// TODO: that other single file db thing to try

src/drive.rs

···
 //     DiskDriveError(#[from] DiskDriveError<E>),
 // }
 
-pub trait Processable: Clone + Serialize + DeserializeOwned {}
+pub trait Processable: Clone + Serialize + DeserializeOwned {
+    /// the additional size taken up (not including its mem::size_of)
+    fn get_size(&self) -> usize;
+}
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub enum MaybeProcessedBlock<T> {
···
     Processed(T),
 }
 
-impl<T: Processable> Processable for MaybeProcessedBlock<T> {}
+impl<T: Processable> Processable for MaybeProcessedBlock<T> {
+    /// TODO this is probably a little broken
+    fn get_size(&self) -> usize {
+        use std::{cmp::max, mem::size_of};
+
+        // enum is always as big as its biggest member?
+        let base_size = max(size_of::<Vec<u8>>(), size_of::<T>());
+
+        let extra = match self {
+            Self::Raw(bytes) => bytes.len(),
+            Self::Processed(t) => t.get_size(),
+        };
+
+        base_size + extra
+    }
+}
 
 pub enum Vehicle<R: AsyncRead + Unpin, T: Processable> {
     Lil(Commit, MemDriver<T>),
···
     let mut commit = None;
 
     // try to load all the blocks into memory
+    let mut mem_size = 0;
     while let Some((cid, data)) = car.next_block().await? {
         // the root commit is a Special Third Kind of block that we need to make
         // sure not to optimistically send to the processing function
···
         };
 
         // stash (maybe processed) blocks in memory as long as we have room
+        mem_size += std::mem::size_of::<Cid>() + maybe_processed.get_size();
         mem_blocks.insert(cid, maybe_processed);
-        if mem_blocks.len() >= max_size {
+        if mem_size >= max_size {
             return Ok(Vehicle::Big(BigCar {
                 car,
                 root,
···
         // dump the rest to disk (in chunks)
         loop {
             let mut chunk = vec![];
+            let mut mem_size = 0;
             loop {
                 let Some((cid, data)) = self.car.next_block().await? else {
                     break;
···
                 } else {
                     MaybeProcessedBlock::Processed((self.process)(&data))
                 };
+                mem_size += std::mem::size_of::<Cid>() + maybe_processed.get_size();
                 chunk.push((cid, maybe_processed));
-                if chunk.len() >= self.max_size {
-                    // eventually this won't be .len()
+                if mem_size >= self.max_size {
                     break;
                 }
             }