back to eager processing

eh

examples/disk-read-file/main.rs

···
     // let stream = Box::pin(reader.stream());
     let stream = std::pin::pin!(reader.stream());
 
-    let (commit, v) = repo_stream::disk_drive::Vehicle::init(root, stream, redb_store, |block| {
-        Ok::<_, Infallible>(block.len())
-    })
-    .await?;
+    let (commit, v) =
+        repo_stream::disk_drive::Vehicle::init(root, stream, redb_store, |block| block.len())
+            .await?;
     let mut record_stream = std::pin::pin!(v.stream());
 
     log::info!("got commit: {commit:?}");

src/disk_drive.rs

···
 use std::collections::VecDeque;
 use std::error::Error;
 
-use crate::disk_walk::{Trip, Walker, RkeyError};
-use crate::mst::Commit;
+use crate::disk_walk::{RkeyError, Trip, Walker};
+use crate::mst::{Commit, Node};
 
 use ipld_core::cid::Cid;
-use serde::{Serialize, de::DeserializeOwned};
+use serde::{Deserialize, Serialize, de::DeserializeOwned};
 
 /// Errors that can happen while consuming and emitting blocks and records
 #[derive(Debug, thiserror::Error)]
-pub enum DriveError<E: Error> {
+pub enum DriveError {
     #[error("Failed to initialize CarReader: {0}")]
     CarReader(#[from] iroh_car::Error),
     #[error("Car block stream error: {0}")]
···
     WalkingProblem(#[from] WalkError),
     #[error("whatever: {0}")]
     Boooooo(String),
-    #[error("processing error: {0}")]
-    ProcessingError(E),
+    #[error("Error while encoding: {0}")]
+    EncodingError(#[from] bincode::error::EncodeError),
+    #[error("Error while decoding: {0}")]
+    DecodingError(#[from] bincode::error::DecodeError),
 }
 
 /// Limited subset of errors that can happen while walking
···
     #[error("Could not find block: {0}")]
     MissingBlock(Cid),
 }
+
+#[derive(Serialize, Deserialize)]
+pub enum MaybeProcessedBlock<T: Serialize> {
+    Raw(Vec<u8>),
+    Processed(T),
+}
+
+pub type Records = Vec<(String, Vec<u8>)>;
 
 /// Storage backend for caching large-repo blocks
 ///
 /// Since
-pub trait BlockStore<MPB: Serialize + DeserializeOwned> {
-    fn put_batch(&self, blocks: Vec<(Cid, MPB)>) -> impl Future<Output = Result<(), BlockStoreError>>; // unwraps for now
+pub trait BlockStore {
+    fn put_batch(
+        &self,
+        blocks: Vec<(Cid, Vec<u8>)>,
+    ) -> impl Future<Output = Result<(), BlockStoreError>>; // unwraps for now
     fn walk_batch(
         &self,
         walker: Walker,
         n: usize,
-    ) -> impl Future<Output = Result<(Walker, Vec<(String, MPB)>), BlockStoreError>>; // boo string error for now because
+    ) -> impl Future<Output = Result<(Walker, Records), BlockStoreError>>; // boo string error for now because
 }
 
 type CarBlock<E> = Result<(Cid, Vec<u8>), E>;
 
 /// The core driver between the block stream and MST walker
-pub struct Vehicle<SE, S, T, BS, P, PE>
+pub struct Vehicle<SE, S, T, BS, P>
 where
     SE: Error + 'static,
     S: Stream<Item = CarBlock<SE>>,
     T: Clone + Serialize + DeserializeOwned,
-    BS: BlockStore<Vec<u8>>,
-    P: Fn(&[u8]) -> Result<T, PE>,
-    PE: Error,
+    BS: BlockStore,
+    P: Fn(&[u8]) -> T,
 {
     #[allow(dead_code)]
     block_stream: Option<S>,
···
     out_cache: VecDeque<(String, T)>,
 }
 
-impl<SE, S, T, BS, P, PE> Vehicle<SE, S, T, BS, P, PE>
+impl<SE, S, T, BS, P> Vehicle<SE, S, T, BS, P>
 where
     SE: Error + 'static,
     S: Stream<Item = CarBlock<SE>> + Unpin + Send,
     T: Clone + Serialize + DeserializeOwned + Send,
-    BS: BlockStore<Vec<u8>> + Send,
-    P: Fn(&[u8]) -> Result<T, PE> + Send,
-    PE: Error,
+    BS: BlockStore + Send,
+    P: Fn(&[u8]) -> T,
 {
     /// Set up the stream
     ///
···
         block_stream: S,
         block_store: BS,
         process: P,
-    ) -> Result<(Commit, Self), DriveError<PE>> {
+    ) -> Result<(Commit, Self), DriveError> {
         let mut commit = None;
 
         log::warn!("init: load blocks");
 
-        let mut chunked = block_stream.try_chunks(4096);
+        let mut chunked = block_stream.try_chunks(256);
 
         // go ahead and put all blocks in the block store
         while let Some(chunk) = chunked
···
                         .map_err(|e| DriveError::BadCommit(e.into()))?;
                     commit = Some(c);
                 } else {
-                    to_insert.push((cid, data));
+                    let wrapped = if Node::could_be(&data) {
+                        MaybeProcessedBlock::Raw(data)
+                    } else {
+                        MaybeProcessedBlock::Processed(process(&data))
+                    };
+                    let bytes =
+                        bincode::serde::encode_to_vec(wrapped, bincode::config::standard())?;
+
+                    to_insert.push((cid, bytes));
                 }
             }
             block_store
···
         Ok((commit, me))
     }
 
-    async fn load_chunk(&mut self, n: usize) -> Result<(), DriveError<PE>> {
+    async fn load_chunk(&mut self, n: usize) -> Result<(), DriveError> {
         let walker = std::mem::take(&mut self.walker);
         let (walker, batch) = self
             .block_store
             .walk_batch(walker, n)
             .await
-            .map_err(|e| DriveError::Boooooo(format!("booo! {e}")))?; // TODO
+            .map_err(|e| DriveError::Boooooo(format!("booo! (here right?) {e}")))?; // TODO
         self.walker = walker;
 
         let processed = batch
             .into_iter()
-            .map(|(k, raw)| (self.process)(&raw).map(|t| (k, t)))
-            .collect::<Result<Vec<_>, _>>()
-            .map_err(DriveError::ProcessingError)?;
+            .map(|(k, encoded)| {
+                let (decoded, n): (MaybeProcessedBlock<T>, usize) =
+                    bincode::serde::decode_from_slice(&encoded, bincode::config::standard())?;
+                assert_eq!(n, encoded.len());
+                let processed = match decoded {
+                    MaybeProcessedBlock::Processed(t) => t,
+                    MaybeProcessedBlock::Raw(block) => (self.process)(&block),
+                };
+                Ok((k, processed))
+            })
+            .collect::<Result<Vec<_>, DriveError>>()?;
 
         self.out_cache.extend(processed);
         Ok(())
···
     /// (but non-zero), even if it's not the last chunk.
     ///
     /// an empty vec will be returned to signal the end.
-    pub async fn next_chunk(&mut self, n: usize) -> Result<Vec<(String, T)>, DriveError<PE>> {
+    pub async fn next_chunk(&mut self, n: usize) -> Result<Vec<(String, T)>, DriveError> {
         if self.out_cache.is_empty() {
             self.load_chunk(n).await?;
         }
···
     }
 
     /// Manually step through the record outputs
-    pub async fn next_record(&mut self) -> Result<Option<(String, T)>, DriveError<PE>> {
+    pub async fn next_record(&mut self) -> Result<Option<(String, T)>, DriveError> {
         if self.out_cache.is_empty() {
-            self.load_chunk(64).await?; // TODO
+            self.load_chunk(128).await?; // TODO
         }
         Ok(self.out_cache.pop_front())
     }
 
     /// Convert to a futures::stream of record outputs
-    pub fn stream(self) -> impl Stream<Item = Result<(String, T), DriveError<PE>>> {
+    pub fn stream(self) -> impl Stream<Item = Result<(String, T), DriveError>> {
         futures::stream::try_unfold(self, |mut this| async move {
             let maybe_record = this.next_record().await?;
             Ok(maybe_record.map(|b| (b, this)))

src/disk_redb.rs

···
-use crate::disk_drive::{BlockStore, BlockStoreError};
+use crate::disk_drive::{BlockStore, BlockStoreError, MaybeProcessedBlock, Records};
 use crate::disk_walk::{Need, Walker};
 use ipld_core::cid::Cid;
 use redb::{Database, Durability, Error, ReadableDatabase, TableDefinition};
···
     }
 }
 
-impl BlockStore<Vec<u8>> for RedbStore {
+impl BlockStore for RedbStore {
     async fn put_batch(&self, blocks: Vec<(Cid, Vec<u8>)>) -> Result<(), BlockStoreError> {
         let db = self.db.clone();
         tokio::task::spawn_blocking(move || -> Result<(), BlockStoreError> {
···
         &self,
         mut walker: Walker,
         n: usize,
-    ) -> Result<(Walker, Vec<(String, Vec<u8>)>), BlockStoreError> {
+    ) -> Result<(Walker, Records), BlockStoreError> {
         let db = self.db.clone();
         tokio::task::spawn_blocking(move || -> Result<_, BlockStoreError> {
             let tx = db.begin_read()?;
···
                 let block = res.value();
 
                 match need {
-                    Need::Node(_) => walker
-                        .handle_node(block)?,
+                    Need::Node(_) => {
+                        let (mpb, n) =
+                            bincode::serde::decode_from_slice(block, bincode::config::standard())
+                                .unwrap();
+                        assert_eq!(n, block.len());
+                        // DANGER: we're throwing in unit () as a placeholder here and assuming bincode will still work since Raw is the first variant
+                        let MaybeProcessedBlock::Raw(bytes): MaybeProcessedBlock<()> = mpb else {
+                            panic!("should have not been processed"); // tODO
+                        };
+                        walker.handle_node(&bytes)?
+                    }
                     Need::Record { rkey, .. } => {
                         out.push((rkey, block.to_vec()));
                         if out.len() >= n {

src/disk_walk.rs

···
     EntryRkeyNotUtf8(#[from] std::string::FromUtf8Error),
     #[error("Encountered an rkey out of order while walking the MST")]
     RkeyOutOfOrder,
-    #[error("Failed to decode commit block: {0}")]
-    BlockDecodeError(#[from] serde_ipld_dagcbor::DecodeError<Infallible>),
+    #[error("Failed to decode node block: {0}")]
+    NodeDecodeError(#[from] serde_ipld_dagcbor::DecodeError<Infallible>),
 }
 
 /// Walker outputs