channels

seems like it's good on the write side

unclear if there's a win on reads

examples/disk-read-file/main.rs

···
 
     let limit_mb = 32;
 
-    let mut driver =
-        match repo_stream::drive::load_car(reader, |block| S(block.len()), 10 * mb).await? {
-            repo_stream::drive::Vehicle::Lil(_, _) => panic!("try this on a bigger car"),
-            repo_stream::drive::Vehicle::Big(big_stuff) => {
-                let disk_store = repo_stream::disk::SqliteStore::new(tmpfile.clone(), limit_mb);
-                let (commit, driver) = big_stuff.finish_loading(disk_store).await?;
-                log::warn!("big: {:?}", commit);
-                driver
-            }
-        };
-
-    println!("hello!");
+    let driver = match repo_stream::drive::load_car(reader, |block| S(block.len()), 10 * mb).await?
+    {
+        repo_stream::drive::Vehicle::Lil(_, _) => panic!("try this on a bigger car"),
+        repo_stream::drive::Vehicle::Big(big_stuff) => {
+            let disk_store = repo_stream::disk::SqliteStore::new(tmpfile.clone(), limit_mb);
+            let (commit, driver) = big_stuff.finish_loading(disk_store).await?;
+            log::warn!("big: {:?}", commit);
+            driver
+        }
+    };
 
     let mut n = 0;
-    loop {
-        let (d, p) = driver.next_chunk(1024).await?;
-        driver = d;
-        let Some(pairs) = p else {
-            break;
-        };
+    let (mut rx, worker) = driver.rx(512).await?;
+
+    log::debug!("walking...");
+    while let Some(pairs) = rx.recv().await {
         n += pairs.len();
-        // log::info!("got {rkey:?}");
     }
+    log::debug!("done walking! joining...");
+
+    worker.await.unwrap().unwrap();
+
+    log::debug!("joined.");
+
     // log::info!("now is the time to check mem...");
     // tokio::time::sleep(std::time::Duration::from_secs(22)).await;
-    drop(driver);
     log::info!("bye! {n}");
 
     std::fs::remove_file(tmpfile).unwrap(); // need to also remove -shm -wal

src/drive.rs

···
             Ok((self, Some(out)))
         }
     }
+
+    pub async fn rx(
+        mut self,
+        n: usize,
+    ) -> Result<
+        (
+            tokio::sync::mpsc::Receiver<Vec<(String, T)>>,
+            tokio::task::JoinHandle<Result<(), DiskDriveError>>,
+        ),
+        DiskDriveError,
+    > {
+        let (tx, rx) = tokio::sync::mpsc::channel::<Vec<(String, T)>>(1);
+
+        // sketch: this worker is going to be allowed to execute without a join handle
+        // ...should we return the join handle here so the caller at least knows about it?
+        // yes probably for error handling?? (orrr put errors in the channel)
+        let worker = tokio::task::spawn_blocking(move || {
+            let mut reader = self.access.get_reader()?;
+
+            loop {
+                let mut out = Vec::with_capacity(n);
+
+                for _ in 0..n {
+                    // walk as far as we can until we run out of blocks or find a record
+                    match self.walker.disk_step(&mut reader, self.process)? {
+                        Step::Missing(cid) => return Err(DiskDriveError::MissingBlock(cid)),
+                        Step::Finish => break,
+                        Step::Step { rkey, data } => {
+                            out.push((rkey, data));
+                            continue;
+                        }
+                    };
+                }
+
+                if out.is_empty() {
+                    break;
+                }
+                tx.blocking_send(out).unwrap();
+            }
+
+            drop(reader); // cannot outlive access
+            Ok(())
+        }); // await later
+
+        Ok((rx, worker))
+    }
 }
 
 /// The core driver between the block stream and MST walker