redb disk version

Cargo.lock

···
 ]
 
 [[package]]
+name = "bincode"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740"
+dependencies = [
+ "bincode_derive",
+ "serde",
+ "unty",
+]
+
+[[package]]
+name = "bincode_derive"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bf95709a440f45e986983918d0e8a1f30a9b1df04918fc828670606804ac3c09"
+dependencies = [
+ "virtue",
+]
+
+[[package]]
 name = "bitflags"
 version = "2.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 ]
 
 [[package]]
+name = "fallible-iterator"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649"
+
+[[package]]
+name = "fallible-streaming-iterator"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
+
+[[package]]
+name = "foldhash"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
+
+[[package]]
 name = "futures"
 version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 ]
 
 [[package]]
+name = "hashbrown"
+version = "0.15.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
+dependencies = [
+ "foldhash",
+]
+
+[[package]]
+name = "hashlink"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1"
+dependencies = [
+ "hashbrown",
+]
+
+[[package]]
 name = "heck"
 version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 checksum = "58f929b4d672ea937a23a1ab494143d968337a5f47e56d0815df1e0890ddf174"
 
 [[package]]
+name = "libsqlite3-sys"
+version = "0.35.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "133c182a6a2c87864fe97778797e46c7e999672690dc9fa3ee8e241aa4a9c13f"
+dependencies = [
+ "pkg-config",
+ "vcpkg",
+]
+
+[[package]]
 name = "lock_api"
 version = "0.4.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
 
 [[package]]
+name = "pkg-config"
+version = "0.3.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"
+
+[[package]]
 name = "plotters"
 version = "0.3.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 ]
 
 [[package]]
+name = "redb"
+version = "3.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ae323eb086579a3769daa2c753bb96deb95993c534711e0dbe881b5192906a06"
+dependencies = [
+ "libc",
+]
+
+[[package]]
 name = "redox_syscall"
 version = "0.5.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 name = "repo-stream"
 version = "0.1.1"
 dependencies = [
+ "bincode",
  "clap",
  "criterion",
  "env_logger",
···
  "iroh-car",
  "log",
  "multibase",
+ "redb",
+ "rusqlite",
  "serde",
  "serde_bytes",
  "serde_ipld_dagcbor",
···
 ]
 
 [[package]]
+name = "rusqlite"
+version = "0.37.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "165ca6e57b20e1351573e3729b958bc62f0e48025386970b6e4d29e7a7e71f3f"
+dependencies = [
+ "bitflags",
+ "fallible-iterator",
+ "fallible-streaming-iterator",
+ "hashlink",
+ "libsqlite3-sys",
+ "smallvec",
+]
+
+[[package]]
 name = "rustc-demangle"
 version = "0.1.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 checksum = "eb066959b24b5196ae73cb057f45598450d2c5f71460e98c49b738086eff9c06"
 
 [[package]]
+name = "unty"
+version = "0.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae"
+
+[[package]]
 name = "utf8parse"
 version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
+
+[[package]]
+name = "vcpkg"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
+
+[[package]]
+name = "virtue"
+version = "0.0.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1"
 
 [[package]]
 name = "walkdir"

Cargo.toml

···
 repository = "https://tangled.org/@microcosm.blue/repo-stream"
 
 [dependencies]
+bincode = { version = "2.0.1", features = ["serde"] }
 futures = "0.3.31"
 futures-core = "0.3.31"
 ipld-core = { version = "0.4.2", features = ["serde"] }
 iroh-car = "0.5.1"
 log = "0.4.28"
 multibase = "0.9.2"
+redb = "3.1.0"
+rusqlite = "0.37.0"
 serde = { version = "1.0.228", features = ["derive"] }
 serde_bytes = "0.11.19"
 serde_ipld_dagcbor = "0.6.4"

examples/disk-read-file/main.rs

···
+extern crate repo_stream;
+use clap::Parser;
+use futures::TryStreamExt;
+use iroh_car::CarReader;
+use std::convert::Infallible;
+use std::path::PathBuf;
+
+type Result<T> = std::result::Result<T, Box<dyn std::error::Error>>;
+
+#[derive(Debug, Parser)]
+struct Args {
+    #[arg()]
+    car: PathBuf,
+    #[arg()]
+    tmpfile: PathBuf,
+}
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    env_logger::init();
+
+    let Args { car, tmpfile } = Args::parse();
+    let reader = tokio::fs::File::open(car).await?;
+    let reader = tokio::io::BufReader::new(reader);
+
+    println!("hello!");
+
+    let reader = CarReader::new(reader).await?;
+
+    let redb_store = repo_stream::disk_redb::RedbStore::new(tmpfile)?;
+
+    let root = reader
+        .header()
+        .roots()
+        .first()
+        .ok_or("missing root")?
+        .clone();
+    log::debug!("root: {root:?}");
+
+    // let stream = Box::pin(reader.stream());
+    let stream = std::pin::pin!(reader.stream());
+
+    let (commit, v) = repo_stream::disk_drive::Vehicle::init(root, stream, redb_store, |block| {
+        Ok::<_, Infallible>(block.len())
+    })
+    .await?;
+    let mut record_stream = std::pin::pin!(v.stream());
+
+    log::info!("got commit: {commit:?}");
+
+    while let Some((rkey, _rec)) = record_stream.try_next().await? {
+        log::info!("got {rkey:?}");
+    }
+    log::info!("bye!");
+
+    Ok(())
+}

src/disk_drive.rs

···
+use futures::Stream;
+use futures::TryStreamExt;
+use std::error::Error;
+
+use crate::disk_walk::{Step, Trip, Walker};
+use crate::mst::Commit;
+use crate::mst::Node;
+
+use ipld_core::cid::Cid;
+use serde::{Deserialize, Serialize, de::DeserializeOwned};
+
+/// Errors that can happen while consuming and emitting blocks and records
+#[derive(Debug, thiserror::Error)]
+pub enum DriveError {
+    #[error("Failed to initialize CarReader: {0}")]
+    CarReader(#[from] iroh_car::Error),
+    #[error("Car block stream error: {0}")]
+    CarBlockError(Box<dyn Error>),
+    #[error("Failed to decode commit block: {0}")]
+    BadCommit(Box<dyn Error>),
+    #[error("The Commit block reference by the root was not found")]
+    MissingCommit,
+    #[error("The MST block {0} could not be found")]
+    MissingBlock(Cid),
+    #[error("Failed to walk the mst tree: {0}")]
+    Tripped(#[from] Trip),
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum MaybeProcessedBlock<T: Clone + Serialize> {
+    /// A block that's *probably* a Node (but we can't know yet)
+    ///
+    /// It *can be* a record that suspiciously looks a lot like a node, so we
+    /// cannot eagerly turn it into a Node. We only know for sure what it is
+    /// when we actually walk down the MST
+    Raw(Vec<u8>),
+    /// A processed record from a block that was definitely not a Node
+    ///
+    /// If we _never_ needed this block, then we may have wasted a bit of effort
+    /// trying to process it. Oh well.
+    ///
+    /// Processing has to be fallible because the CAR can have totally-unused
+    /// blocks, which can just be garbage. since we're eagerly trying to process
+    /// record blocks without knowing for sure that they *are* records, we
+    /// discard any definitely-not-nodes that fail processing and keep their
+    /// error in the buffer for them. if we later try to retreive them as a
+    /// record, then we can surface the error.
+    ///
+    /// The error type is `String` because we don't really want to put
+    /// any constraints like `Serialize` on the error type, and `Error`
+    /// at least requires `Display`. It's a compromise.
+    ProcessedOk(T),
+    Unprocessable(String),
+}
+
+pub trait BlockStore<MPB: Serialize + DeserializeOwned> {
+    fn put(&self, key: Cid, value: MPB); // unwraps for now
+    fn get(&self, key: Cid) -> Option<MPB>;
+}
+
+type CarBlock<E> = Result<(Cid, Vec<u8>), E>;
+
+/// The core driver between the block stream and MST walker
+pub struct Vehicle<SE, S, T, BS, P, PE>
+where
+    SE: Error + 'static,
+    S: Stream<Item = CarBlock<SE>>,
+    T: Clone + Serialize + DeserializeOwned,
+    BS: BlockStore<MaybeProcessedBlock<T>>,
+    P: Fn(&[u8]) -> Result<T, PE>,
+    PE: Error,
+{
+    block_stream: S,
+    block_store: BS,
+    walker: Walker,
+    process: P,
+}
+
+impl<SE, S, T, BS, P, PE> Vehicle<SE, S, T, BS, P, PE>
+where
+    SE: Error + 'static,
+    S: Stream<Item = CarBlock<SE>> + Unpin,
+    T: Clone + Serialize + DeserializeOwned,
+    BS: BlockStore<MaybeProcessedBlock<T>>,
+    P: Fn(&[u8]) -> Result<T, PE>,
+    PE: Error,
+{
+    /// Set up the stream
+    ///
+    /// This will eagerly consume blocks until the `Commit` object is found.
+    /// *Usually* the it's the first block, but there is no guarantee.
+    ///
+    /// ### Parameters
+    ///
+    /// `root`: CID of the commit object that is the root of the MST
+    ///
+    /// `block_stream`: Input stream of raw CAR blocks
+    ///
+    /// `process`: record-transforming callback:
+    ///
+    /// For tasks where records can be quickly processed into a *smaller*
+    /// useful representation, you can do that eagerly as blocks come in by
+    /// passing the processor as a callback here. This can reduce overall
+    /// memory usage.
+    pub async fn init(
+        root: Cid,
+        mut block_stream: S,
+        block_store: BS,
+        process: P,
+    ) -> Result<(Commit, Self), DriveError> {
+        let mut commit = None;
+
+        while let Some((cid, data)) = block_stream
+            .try_next()
+            .await
+            .map_err(|e| DriveError::CarBlockError(e.into()))?
+        {
+            if cid == root {
+                let c: Commit = serde_ipld_dagcbor::from_slice(&data)
+                    .map_err(|e| DriveError::BadCommit(e.into()))?;
+                commit = Some(c);
+                break;
+            } else {
+                block_store.put(
+                    cid,
+                    if Node::could_be(&data) {
+                        MaybeProcessedBlock::Raw(data)
+                    } else {
+                        match process(&data) {
+                            Ok(t) => MaybeProcessedBlock::ProcessedOk(t),
+                            Err(e) => MaybeProcessedBlock::Unprocessable(e.to_string()),
+                        }
+                    },
+                );
+            }
+        }
+
+        // we either broke out or read all the blocks without finding the commit...
+        let commit = commit.ok_or(DriveError::MissingCommit)?;
+
+        let walker = Walker::new(commit.data);
+
+        let me = Self {
+            block_stream,
+            block_store,
+            walker,
+            process,
+        };
+        Ok((commit, me))
+    }
+
+    async fn drive_until(&mut self, cid_needed: Cid) -> Result<(), DriveError> {
+        while let Some((cid, data)) = self
+            .block_stream
+            .try_next()
+            .await
+            .map_err(|e| DriveError::CarBlockError(e.into()))?
+        {
+            self.block_store.put(
+                cid,
+                if Node::could_be(&data) {
+                    MaybeProcessedBlock::Raw(data)
+                } else {
+                    match (self.process)(&data) {
+                        Ok(t) => MaybeProcessedBlock::ProcessedOk(t),
+                        Err(e) => MaybeProcessedBlock::Unprocessable(e.to_string()),
+                    }
+                },
+            );
+            if cid == cid_needed {
+                return Ok(());
+            }
+        }
+
+        // if we never found the block
+        Err(DriveError::MissingBlock(cid_needed))
+    }
+
+    /// Manually step through the record outputs
+    pub async fn next_record(&mut self) -> Result<Option<(String, T)>, DriveError> {
+        loop {
+            // walk as far as we can until we run out of blocks or find a record
+            let cid_needed = match self.walker.step(&mut self.block_store, &self.process)? {
+                Step::Rest(cid) => cid,
+                Step::Finish => return Ok(None),
+                Step::Step { rkey, data } => return Ok(Some((rkey, data))),
+            };
+
+            // load blocks until we reach that cid
+            self.drive_until(cid_needed).await?;
+        }
+    }
+
+    /// Convert to a futures::stream of record outputs
+    pub fn stream(self) -> impl Stream<Item = Result<(String, T), DriveError>> {
+        futures::stream::try_unfold(self, |mut this| async move {
+            let maybe_record = this.next_record().await?;
+            Ok(maybe_record.map(|b| (b, this)))
+        })
+    }
+}

src/disk_redb.rs

···
+use crate::disk_drive::BlockStore;
+use ipld_core::cid::Cid;
+use redb::{Database, Error, ReadableDatabase, TableDefinition};
+use serde::{Serialize, de::DeserializeOwned};
+use std::path::Path;
+
+const TABLE: TableDefinition<&[u8], &[u8]> = TableDefinition::new("blocks");
+
+pub struct RedbStore {
+    db: Database,
+}
+
+impl RedbStore {
+    pub fn new(path: impl AsRef<Path>) -> Result<Self, Error> {
+        let db = Database::create(path)?;
+        Ok(Self { db })
+    }
+}
+
+// TODO: clean up on drop
+
+impl<MPB: Serialize + DeserializeOwned> BlockStore<MPB> for RedbStore {
+    fn put(&self, c: Cid, t: MPB) {
+        let key_bytes = c.to_bytes();
+        let val_bytes = bincode::serde::encode_to_vec(t, bincode::config::standard()).unwrap();
+
+        let mut tx = self.db.begin_write().unwrap();
+        tx.set_durability(redb::Durability::None).unwrap();
+        {
+            let mut table = tx.open_table(TABLE).unwrap();
+            table.insert(&*key_bytes, &*val_bytes).unwrap();
+        }
+        tx.commit().unwrap();
+    }
+    fn get(&self, c: Cid) -> Option<MPB> {
+        let key_bytes = c.to_bytes();
+        let tx = self.db.begin_read().unwrap();
+        let table = match tx.open_table(TABLE) {
+            Ok(t) => t,
+            Err(redb::TableError::TableDoesNotExist(_)) => return None,
+            e => e.unwrap(),
+        };
+        let maybe_val_bytes = table.get(&*key_bytes).unwrap()?;
+        let (t, n): (MPB, usize) =
+            bincode::serde::decode_from_slice(maybe_val_bytes.value(), bincode::config::standard())
+                .unwrap();
+        assert_eq!(maybe_val_bytes.value().len(), n);
+        Some(t)
+    }
+}

src/disk_walk.rs

···
+//! Depth-first MST traversal
+
+use crate::disk_drive::{BlockStore, MaybeProcessedBlock};
+use crate::mst::Node;
+
+use ipld_core::cid::Cid;
+use serde::{Serialize, de::DeserializeOwned};
+use std::error::Error;
+
+/// Errors that can happen while walking
+#[derive(Debug, thiserror::Error)]
+pub enum Trip {
+    #[error("empty mst nodes are not allowed")]
+    NodeEmpty,
+    #[error("Failed to decode commit block: {0}")]
+    BadCommit(Box<dyn std::error::Error>),
+    #[error("Action node error: {0}")]
+    RkeyError(#[from] RkeyError),
+    #[error("Process failed: {0}")]
+    ProcessFailed(String),
+    #[error("Encountered an rkey out of order while walking the MST")]
+    RkeyOutOfOrder,
+}
+
+/// Errors from invalid Rkeys
+#[derive(Debug, thiserror::Error)]
+pub enum RkeyError {
+    #[error("Failed to compute an rkey due to invalid prefix_len")]
+    EntryPrefixOutOfbounds,
+    #[error("RKey was not utf-8")]
+    EntryRkeyNotUtf8(#[from] std::string::FromUtf8Error),
+}
+
+/// Walker outputs
+#[derive(Debug)]
+pub enum Step<T: Serialize + DeserializeOwned> {
+    /// We need a CID but it's not in the block store
+    ///
+    /// Give the needed CID to the driver so it can load blocks until it's found
+    Rest(Cid),
+    /// Reached the end of the MST! yay!
+    Finish,
+    /// A record was found!
+    Step { rkey: String, data: T },
+}
+
+#[derive(Debug, Clone, PartialEq)]
+enum Need {
+    Node(Cid),
+    Record { rkey: String, cid: Cid },
+}
+
+fn push_from_node(stack: &mut Vec<Need>, node: &Node) -> Result<(), RkeyError> {
+    let mut entries = Vec::with_capacity(node.entries.len());
+
+    let mut prefix = vec![];
+    for entry in &node.entries {
+        let mut rkey = vec![];
+        let pre_checked = prefix
+            .get(..entry.prefix_len)
+            .ok_or(RkeyError::EntryPrefixOutOfbounds)?;
+        rkey.extend_from_slice(pre_checked);
+        rkey.extend_from_slice(&entry.keysuffix);
+        prefix = rkey.clone();
+
+        entries.push(Need::Record {
+            rkey: String::from_utf8(rkey)?,
+            cid: entry.value,
+        });
+        if let Some(ref tree) = entry.tree {
+            entries.push(Need::Node(*tree));
+        }
+    }
+
+    entries.reverse();
+    stack.append(&mut entries);
+
+    if let Some(tree) = node.left {
+        stack.push(Need::Node(tree));
+    }
+    Ok(())
+}
+
+/// Traverser of an atproto MST
+///
+/// Walks the tree from left-to-right in depth-first order
+#[derive(Debug)]
+pub struct Walker {
+    stack: Vec<Need>,
+    prev: String,
+}
+
+impl Walker {
+    pub fn new(tree_root_cid: Cid) -> Self {
+        Self {
+            stack: vec![Need::Node(tree_root_cid)],
+            prev: "".to_string(),
+        }
+    }
+
+    /// Advance through nodes until we find a record or can't go further
+    pub fn step<T: Clone + Serialize + DeserializeOwned, E: Error>(
+        &mut self,
+        block_store: &mut impl BlockStore<MaybeProcessedBlock<T>>,
+        process: impl Fn(&[u8]) -> Result<T, E>,
+    ) -> Result<Step<T>, Trip> {
+        loop {
+            let Some(mut need) = self.stack.last() else {
+                log::trace!("tried to walk but we're actually done.");
+                return Ok(Step::Finish);
+            };
+
+            match &mut need {
+                Need::Node(cid) => {
+                    log::trace!("need node {cid:?}");
+                    let Some(mpb) = block_store.get(*cid) else {
+                        log::trace!("node not found, resting");
+                        return Ok(Step::Rest(*cid));
+                    };
+
+                    let MaybeProcessedBlock::<T>::Raw(block) = mpb else {
+                        return Err(Trip::BadCommit("failed commit fingerprint".into()));
+                    };
+                    let node = serde_ipld_dagcbor::from_slice::<Node>(&block)
+                        .map_err(|e| Trip::BadCommit(e.into()))?;
+
+                    // found node, make sure we remember
+                    self.stack.pop();
+
+                    // queue up work on the found node next
+                    push_from_node(&mut self.stack, &node)?;
+                }
+                Need::Record { rkey, cid } => {
+                    log::trace!("need record {cid:?}");
+                    let Some(mpb) = block_store.get(*cid) else {
+                        log::trace!("record block not found, resting");
+                        return Ok(Step::Rest(*cid));
+                    };
+                    let rkey = rkey.clone();
+                    let data = match mpb {
+                        MaybeProcessedBlock::Raw(data) => match process(&data) {
+                            Ok(t) => Ok(t),
+                            Err(e) => Err(Trip::ProcessFailed(e.to_string())),
+                        },
+                        MaybeProcessedBlock::ProcessedOk(t) => Ok(t.clone()),
+                        MaybeProcessedBlock::Unprocessable(s) => {
+                            return Err(Trip::ProcessFailed(s.clone()));
+                        }
+                    };
+
+                    // found node, make sure we remember
+                    self.stack.pop();
+
+                    log::trace!("emitting a block as a step. depth={}", self.stack.len());
+
+                    let data = data.map_err(|e| Trip::ProcessFailed(e.to_string()))?;
+
+                    // rkeys *must* be in order or else the tree is invalid (or
+                    // we have a bug)
+                    if rkey <= self.prev {
+                        return Err(Trip::RkeyOutOfOrder);
+                    }
+                    self.prev = rkey.clone();
+
+                    return Ok(Step::Step { rkey, data });
+                }
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    // use crate::mst::Entry;
+
+    fn cid1() -> Cid {
+        "bafyreihixenvk3ahqbytas4hk4a26w43bh6eo3w6usjqtxkpzsvi655a3m"
+            .parse()
+            .unwrap()
+    }
+    //     fn cid2() -> Cid {
+    //         "QmY7Yh4UquoXHLPFo2XbhXkhBvFoPwmQUSa92pxnxjQuPU"
+    //             .parse()
+    //             .unwrap()
+    //     }
+    //     fn cid3() -> Cid {
+    //         "bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi"
+    //             .parse()
+    //             .unwrap()
+    //     }
+    //     fn cid4() -> Cid {
+    //         "QmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR"
+    //             .parse()
+    //             .unwrap()
+    //     }
+    //     fn cid5() -> Cid {
+    //         "QmSnuWmxptJZdLJpKRarxBMS2Ju2oANVrgbr2xWbie9b2D"
+    //             .parse()
+    //             .unwrap()
+    //     }
+    //     fn cid6() -> Cid {
+    //         "QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm"
+    //             .parse()
+    //             .unwrap()
+    //     }
+    //     fn cid7() -> Cid {
+    //         "bafybeiaysi4s6lnjev27ln5icwm6tueaw2vdykrtjkwiphwekaywqhcjze"
+    //             .parse()
+    //             .unwrap()
+    //     }
+    //     fn cid8() -> Cid {
+    //         "bafyreif3tfdpr5n4jdrbielmcapwvbpcthepfkwq2vwonmlhirbjmotedi"
+    //             .parse()
+    //             .unwrap()
+    //     }
+    //     fn cid9() -> Cid {
+    //         "bafyreicnokmhmrnlp2wjhyk2haep4tqxiptwfrp2rrs7rzq7uk766chqvq"
+    //             .parse()
+    //             .unwrap()
+    //     }
+
+    #[test]
+    fn test_next_from_node_empty() {
+        let node = Node {
+            left: None,
+            entries: vec![],
+        };
+        let mut stack = vec![];
+        push_from_node(&mut stack, &node).unwrap();
+        assert_eq!(stack.last(), None);
+    }
+
+    #[test]
+    fn test_needs_from_node_just_left() {
+        let node = Node {
+            left: Some(cid1()),
+            entries: vec![],
+        };
+        let mut stack = vec![];
+        push_from_node(&mut stack, &node).unwrap();
+        assert_eq!(stack.last(), Some(Need::Node(cid1())).as_ref());
+    }
+
+    //     #[test]
+    //     fn test_needs_from_node_just_one_record() {
+    //         let node = Node {
+    //             left: None,
+    //             entries: vec![Entry {
+    //                 keysuffix: "asdf".into(),
+    //                 prefix_len: 0,
+    //                 value: cid1(),
+    //                 tree: None,
+    //             }],
+    //         };
+    //         assert_eq!(
+    //             needs_from_node(node).unwrap(),
+    //             vec![Need::Record {
+    //                 rkey: "asdf".into(),
+    //                 cid: cid1(),
+    //             },]
+    //         );
+    //     }
+
+    //     #[test]
+    //     fn test_needs_from_node_two_records() {
+    //         let node = Node {
+    //             left: None,
+    //             entries: vec![
+    //                 Entry {
+    //                     keysuffix: "asdf".into(),
+    //                     prefix_len: 0,
+    //                     value: cid1(),
+    //                     tree: None,
+    //                 },
+    //                 Entry {
+    //                     keysuffix: "gh".into(),
+    //                     prefix_len: 2,
+    //                     value: cid2(),
+    //                     tree: None,
+    //                 },
+    //             ],
+    //         };
+    //         assert_eq!(
+    //             needs_from_node(node).unwrap(),
+    //             vec![
+    //                 Need::Record {
+    //                     rkey: "asdf".into(),
+    //                     cid: cid1(),
+    //                 },
+    //                 Need::Record {
+    //                     rkey: "asgh".into(),
+    //                     cid: cid2(),
+    //                 },
+    //             ]
+    //         );
+    //     }
+
+    //     #[test]
+    //     fn test_needs_from_node_with_both() {
+    //         let node = Node {
+    //             left: None,
+    //             entries: vec![Entry {
+    //                 keysuffix: "asdf".into(),
+    //                 prefix_len: 0,
+    //                 value: cid1(),
+    //                 tree: Some(cid2()),
+    //             }],
+    //         };
+    //         assert_eq!(
+    //             needs_from_node(node).unwrap(),
+    //             vec![
+    //                 Need::Record {
+    //                     rkey: "asdf".into(),
+    //                     cid: cid1(),
+    //                 },
+    //                 Need::Node(cid2()),
+    //             ]
+    //         );
+    //     }
+
+    //     #[test]
+    //     fn test_needs_from_node_left_and_record() {
+    //         let node = Node {
+    //             left: Some(cid1()),
+    //             entries: vec![Entry {
+    //                 keysuffix: "asdf".into(),
+    //                 prefix_len: 0,
+    //                 value: cid2(),
+    //                 tree: None,
+    //             }],
+    //         };
+    //         assert_eq!(
+    //             needs_from_node(node).unwrap(),
+    //             vec![
+    //                 Need::Node(cid1()),
+    //                 Need::Record {
+    //                     rkey: "asdf".into(),
+    //                     cid: cid2(),
+    //                 },
+    //             ]
+    //         );
+    //     }
+
+    //     #[test]
+    //     fn test_needs_from_full_node() {
+    //         let node = Node {
+    //             left: Some(cid1()),
+    //             entries: vec![
+    //                 Entry {
+    //                     keysuffix: "asdf".into(),
+    //                     prefix_len: 0,
+    //                     value: cid2(),
+    //                     tree: Some(cid3()),
+    //                 },
+    //                 Entry {
+    //                     keysuffix: "ghi".into(),
+    //                     prefix_len: 1,
+    //                     value: cid4(),
+    //                     tree: Some(cid5()),
+    //                 },
+    //                 Entry {
+    //                     keysuffix: "jkl".into(),
+    //                     prefix_len: 2,
+    //                     value: cid6(),
+    //                     tree: Some(cid7()),
+    //                 },
+    //                 Entry {
+    //                     keysuffix: "mno".into(),
+    //                     prefix_len: 4,
+    //                     value: cid8(),
+    //                     tree: Some(cid9()),
+    //                 },
+    //             ],
+    //         };
+    //         assert_eq!(
+    //             needs_from_node(node).unwrap(),
+    //             vec![
+    //                 Need::Node(cid1()),
+    //                 Need::Record {
+    //                     rkey: "asdf".into(),
+    //                     cid: cid2(),
+    //                 },
+    //                 Need::Node(cid3()),
+    //                 Need::Record {
+    //                     rkey: "aghi".into(),
+    //                     cid: cid4(),
+    //                 },
+    //                 Need::Node(cid5()),
+    //                 Need::Record {
+    //                     rkey: "agjkl".into(),
+    //                     cid: cid6(),
+    //                 },
+    //                 Need::Node(cid7()),
+    //                 Need::Record {
+    //                     rkey: "agjkmno".into(),
+    //                     cid: cid8(),
+    //                 },
+    //                 Need::Node(cid9()),
+    //             ]
+    //         );
+    //     }
+}

src/lib.rs

···
 //!
 //! For now see the [examples](https://tangled.org/@microcosm.blue/repo-stream/tree/main/examples)
 
+pub mod disk_drive;
+pub mod disk_redb;
+pub mod disk_walk;
 pub mod drive;
 pub mod mst;
 pub mod walk;