maybe processed helper + example comments

examples/disk-read-file/main.rs

···
 use repo_stream::{Driver, noop};
 use std::path::PathBuf;
 
-type Result<T> = std::result::Result<T, Box<dyn std::error::Error>>;
-
 #[derive(Debug, Parser)]
 struct Args {
     #[arg()]
···
 }
 
 #[tokio::main]
-async fn main() -> Result<()> {
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
     env_logger::init();
 
     let Args { car, tmpfile } = Args::parse();
+
+    // repo-stream takes an AsyncRead as input. wrapping a filesystem read in
+    // BufReader can provide a really significant performance win.
     let reader = tokio::fs::File::open(car).await?;
     let reader = tokio::io::BufReader::new(reader);
 
-    // let kb = 2_usize.pow(10);
-    let mb = 2_usize.pow(20);
+    // configure how much memory can be used before spilling to disk.
+    // real memory usage may differ somewhat.
+    let in_mem_limit = 10 * 2_usize.pow(20);
 
-    let limit_mb = 32;
+    // configure how much memory sqlite is allowed to use when dumping to disk
+    let db_cache_mb = 32;
 
-    let driver = match Driver::load_car(reader, noop, 10 * mb).await? {
+    log::info!("hello! reading the car...");
+
+    // in this example we only bother handling CARs that are too big for memory
+    // `noop` helper means: do no block processing, store the raw blocks
+    let driver = match Driver::load_car(reader, noop, in_mem_limit).await? {
         Driver::Lil(_, _) => panic!("try this on a bigger car"),
         Driver::Big(big_stuff) => {
-            let disk_store = repo_stream::disk::SqliteStore::new(tmpfile.clone(), limit_mb).await?;
+            // we reach here if the repo was too big and needs to be spilled to
+            // disk to continue
+
+            // set up a disk store we can spill to
+            let disk_store =
+                repo_stream::disk::SqliteStore::new(tmpfile.clone(), db_cache_mb).await?;
+
+            // do the spilling, get back a (similar) driver
             let (commit, driver) = big_stuff.finish_loading(disk_store).await?;
-            log::warn!("big: {:?}", commit);
+
+            // at this point you might want to fetch the account's signing key
+            // via the DID from the commit, and then verify the signature.
+            log::warn!("big's comit: {:?}", commit);
+
+            // pop the driver back out to get some code indentation relief
             driver
         }
     };
 
+    // collect some random stats about the blocks
     let mut n = 0;
     let mut zeros = 0;
-    let mut rx = driver.to_channel(512);
 
-    log::debug!("walking...");
+    log::info!("walking...");
+
+    // this example uses the disk driver's channel mode: the tree walking is
+    // spawned onto a blocking thread, and we get chunks of rkey+blocks back
+    let (mut rx, join) = driver.to_channel(512);
     while let Some(r) = rx.recv().await {
         let pairs = r?;
+
+        // keep a count of the total number of blocks seen
         n += pairs.len();
+
         for (_, block) in pairs {
+            // for each block, count how many bytes are equal to '0'
+            // (this is just an example, you probably want to do something more
+            // interesting)
             zeros += block.into_iter().filter(|&b| b == b'0').count()
         }
     }
-    log::debug!("done walking!");
+
+    log::info!("arrived! joining rx...");
 
-    // log::info!("now is the time to check mem...");
-    // tokio::time::sleep(std::time::Duration::from_secs(22)).await;
-    log::info!("bye! n={n} zeros={zeros}");
+    join.await?.reset_store().await?;
 
-    std::fs::remove_file(tmpfile).unwrap(); // need to also remove -shm -wal
+    log::info!("done. n={n} zeros={zeros}");
 
     Ok(())
 }

src/disk.rs

···
         let select_stmt = self.conn.prepare("SELECT val FROM blocks WHERE key = ?1")?;
         Ok(SqliteReader { select_stmt })
     }
+    pub fn reset(&mut self) -> Result<(), rusqlite::Error> {
+        self.conn.execute("DROP TABLE blocks", ())?;
+        Ok(())
+    }
 }
 
 pub struct SqliteWriter<'conn> {

src/drive.rs

···
     }
 }
 
+impl<T> MaybeProcessedBlock<T> {
+    fn maybe(process: fn(Vec<u8>) -> T, data: Vec<u8>) -> Self {
+        if Node::could_be(&data) {
+            MaybeProcessedBlock::Raw(data)
+        } else {
+            MaybeProcessedBlock::Processed(process(data))
+        }
+    }
+}
+
 pub enum Driver<R: AsyncRead + Unpin, T: Processable> {
     Lil(Commit, MemDriver<T>),
     Big(BigCar<R, T>),
···
             }
 
             // remaining possible types: node, record, other. optimistically process
-            let maybe_processed = if Node::could_be(&data) {
-                MaybeProcessedBlock::Raw(data)
-            } else {
-                MaybeProcessedBlock::Processed(process(data))
-            };
+            let maybe_processed = MaybeProcessedBlock::maybe(process, data);
 
             // stash (maybe processed) blocks in memory as long as we have room
             mem_size += std::mem::size_of::<Cid>() + maybe_processed.get_size();
···
             match self.walker.step(&mut self.blocks, self.process)? {
                 Step::Missing(cid) => return Err(DriveError::MissingBlock(cid)),
                 Step::Finish => break,
-                Step::Step { rkey, data } => {
+                Step::Found { rkey, data } => {
                     out.push((rkey, data));
                     continue;
                 }
···
                 }
                 // remaining possible types: node, record, other. optimistically process
                 // TODO: get the actual in-memory size to compute disk spill
-                let maybe_processed = if Node::could_be(&data) {
-                    MaybeProcessedBlock::Raw(data)
-                } else {
-                    MaybeProcessedBlock::Processed((self.process)(data))
-                };
+                let maybe_processed = MaybeProcessedBlock::maybe(self.process, data);
                 mem_size += std::mem::size_of::<Cid>() + maybe_processed.get_size();
                 chunk.push((cid, maybe_processed));
                 if mem_size >= self.max_size {
···
                 match self.walker.disk_step(&mut reader, self.process)? {
                     Step::Missing(cid) => return Err(DriveError::MissingBlock(cid)),
                     Step::Finish => break,
-                    Step::Step { rkey, data } => {
+                    Step::Found { rkey, data } => {
                         out.push((rkey, data));
                         continue;
                     }
···
     }
 
     fn read_tx_blocking(
-        mut self,
+        &mut self,
         n: usize,
         tx: mpsc::Sender<Result<BlockChunk<T>, DriveError>>,
     ) -> Result<(), mpsc::error::SendError<Result<BlockChunk<T>, DriveError>>> {
···
                         return tx.blocking_send(Err(DriveError::MissingBlock(cid)));
                     }
                     Step::Finish => return Ok(()),
-                    Step::Step { rkey, data } => {
+                    Step::Found { rkey, data } => {
                         out.push((rkey, data));
                         continue;
                     }
···
         Ok(())
     }
 
-    pub fn to_channel(self, n: usize) -> mpsc::Receiver<Result<BlockChunk<T>, DriveError>> {
+    pub fn to_channel(
+        mut self,
+        n: usize,
+    ) -> (
+        mpsc::Receiver<Result<BlockChunk<T>, DriveError>>,
+        tokio::task::JoinHandle<Self>,
+    ) {
         let (tx, rx) = mpsc::channel::<Result<BlockChunk<T>, DriveError>>(1);
 
         // sketch: this worker is going to be allowed to execute without a join handle
-        tokio::task::spawn_blocking(move || {
+        let chan_task = tokio::task::spawn_blocking(move || {
             if let Err(mpsc::error::SendError(_)) = self.read_tx_blocking(n, tx) {
                 log::debug!("big car reader exited early due to dropped receiver channel");
             }
+            self
         });
 
-        rx
+        (rx, chan_task)
+    }
+
+    pub async fn reset_store(mut self) -> Result<SqliteStore, DriveError> {
+        tokio::task::spawn_blocking(move || {
+            self.store.reset()?;
+            Ok(self.store)
+        })
+        .await?
     }
 }

src/lib.rs

···
 //!
 //! For now see the [examples](https://tangled.org/@microcosm.blue/repo-stream/tree/main/examples)
 
+mod mst;
+mod walk;
+
 pub mod disk;
 pub mod drive;
-pub mod mst;
 pub mod process;
-pub mod walk;
 
 pub use disk::SqliteStore;
 pub use drive::{DriveError, Driver};

src/walk.rs

···
     /// Reached the end of the MST! yay!
     Finish,
     /// A record was found!
-    Step { rkey: String, data: T },
+    Found { rkey: String, data: T },
 }
 
 #[derive(Debug, Clone, PartialEq)]
···
                     }
                     self.prev = rkey.clone();
 
-                    return Ok(Step::Step { rkey, data });
+                    return Ok(Step::Found { rkey, data });
                 }
             }
         }
···
                     }
                     self.prev = rkey.clone();
 
-                    return Ok(Step::Step { rkey, data });
+                    return Ok(Step::Found { rkey, data });
                 }
             }
         }