docs etc

Cargo.lock

···
 
 [[package]]
 name = "repo-stream"
-version = "0.1.0"
+version = "0.1.1"
 dependencies = [
  "clap",
  "criterion",

Cargo.toml

···
 [package]
 name = "repo-stream"
-version = "0.1.0"
+version = "0.1.1"
 edition = "2024"
 license = "MIT OR Apache-2.0"
 description = "Fast and robust atproto CAR file processing in rust"
+repository = "https://tangled.org/@microcosm.blue/repo-stream"
 
 [dependencies]
 futures = "0.3.31"

src/drive.rs

···
+//! Consume an MST block stream, producing an ordered stream of records
+
 use futures::{Stream, TryStreamExt};
 use ipld_core::cid::Cid;
 use std::collections::HashMap;
···
 use crate::mst::{Commit, Node};
 use crate::walk::{Step, Trip, Walker};
 
+/// Errors that can happen while consuming and emitting blocks and records
 #[derive(Debug, thiserror::Error)]
 pub enum DriveError<E: Error> {
     #[error("Failed to initialize CarReader: {0}")]
···
     MissingBlock(Cid),
     #[error("Failed to walk the mst tree: {0}")]
     Tripped(#[from] Trip<E>),
-    #[error("Encountered an rkey out of order while walking the MST")]
-    RkeyOutOfOrder,
 }
 
 type CarBlock<E> = Result<(Cid, Vec<u8>), E>;
 
+/// Newtype because i'll mix up strings somewhere if i don't
 #[derive(Debug)]
 pub struct Rkey(pub String);
 
···
     Processed(Result<T, E>),
 }
 
+/// The core driver between the block stream and MST walker
 pub struct Vehicle<SE, S, T, P, PE>
 where
     S: Stream<Item = CarBlock<SE>>,
···
     blocks: HashMap<Cid, MaybeProcessedBlock<T, PE>>,
     walker: Walker,
     process: P,
-    prev_rkey: String,
 }
 
 impl<SE, S, T: Clone, P, PE> Vehicle<SE, S, T, P, PE>
···
     P: Fn(&[u8]) -> Result<T, PE>,
     PE: Error,
 {
+    /// Set up the stream
+    ///
+    /// This will eagerly consume blocks until the `Commit` object is found.
+    /// *Usually* the it's the first block, but there is no guarantee.
+    ///
+    /// ### Parameters
+    ///
+    /// `root`: CID of the commit object that is the root of the MST
+    ///
+    /// `block_stream`: Input stream of raw CAR blocks
+    ///
+    /// `process`: record-transforming callback:
+    ///
+    /// For tasks where records can be quickly processed into a *smaller*
+    /// useful representation, you can do that eagerly as blocks come in by
+    /// passing the processor as a callback here. This can reduce overall
+    /// memory usage.
     pub async fn init(
         root: Cid,
         mut block_stream: S,
···
             blocks,
             walker,
             process,
-            prev_rkey: "".to_string(),
         };
         Ok((commit, me))
     }
···
         Err(DriveError::MissingBlock(cid_needed))
     }
 
-    pub async fn next_record(&mut self) -> Result<Option<(Rkey, T)>, DriveError<PE>> {
+    /// Manually step through the record outputs
+    pub async fn next_record(&mut self) -> Result<Option<(String, T)>, DriveError<PE>> {
         loop {
             // walk as far as we can until we run out of blocks or find a record
-            let cid_needed = match self.walker.walk(&mut self.blocks, &self.process)? {
+            let cid_needed = match self.walker.step(&mut self.blocks, &self.process)? {
                 Step::Rest(cid) => cid,
                 Step::Finish => return Ok(None),
-                Step::Step { rkey, data } => {
-                    if rkey <= self.prev_rkey {
-                        return Err(DriveError::RkeyOutOfOrder);
-                    }
-                    return Ok(Some((Rkey(rkey), data)));
-                }
+                Step::Step { rkey, data } => return Ok(Some((rkey, data))),
             };
 
             // load blocks until we reach that cid
···
         }
     }
 
-    pub fn stream(self) -> impl Stream<Item = Result<(Rkey, T), DriveError<PE>>> {
+    /// Convert to a futures::stream of record outputs
+    pub fn stream(self) -> impl Stream<Item = Result<(String, T), DriveError<PE>>> {
         futures::stream::try_unfold(self, |mut this| async move {
             let maybe_record = this.next_record().await?;
             Ok(maybe_record.map(|b| (b, this)))

src/lib.rs

···
+//! rep-stream
+//!
+//! For now see the [examples](https://tangled.org/@microcosm.blue/repo-stream/tree/main/examples)
+
 pub mod drive;
 pub mod mst;
 pub mod walk;

src/walk.rs

···
 use std::collections::HashMap;
 use std::error::Error;
 
+/// Errors that can happen while walking
 #[derive(Debug, thiserror::Error)]
 pub enum Trip<E: Error> {
     #[error("empty mst nodes are not allowed")]
···
     #[error("Failed to decode commit block: {0}")]
     BadCommit(Box<dyn std::error::Error>),
     #[error("Action node error: {0}")]
-    ActionNode(#[from] ActionNodeError),
+    RkeyError(#[from] RkeyError),
     #[error("Process failed: {0}")]
     ProcessFailed(E),
+    #[error("Encountered an rkey out of order while walking the MST")]
+    RkeyOutOfOrder,
 }
 
+/// Errors from invalid Rkeys
 #[derive(Debug, thiserror::Error)]
-pub enum ActionNodeError {
+pub enum RkeyError {
     #[error("Failed to compute an rkey due to invalid prefix_len")]
     EntryPrefixOutOfbounds,
     #[error("RKey was not utf-8")]
     EntryRkeyNotUtf8(#[from] std::string::FromUtf8Error),
 }
 
+/// Walker outputs
 #[derive(Debug)]
 pub enum Step<T> {
+    /// We need a CID but it's not in the block store
+    ///
+    /// Give the needed CID to the driver so it can load blocks until it's found
     Rest(Cid),
+    /// Reached the end of the MST! yay!
     Finish,
+    /// A record was found!
     Step { rkey: String, data: T },
 }
 
···
     Record { rkey: String, cid: Cid },
 }
 
-fn push_from_node(stack: &mut Vec<Need>, node: &Node) -> Result<(), ActionNodeError> {
+fn push_from_node(stack: &mut Vec<Need>, node: &Node) -> Result<(), RkeyError> {
     let mut entries = Vec::with_capacity(node.entries.len());
 
     let mut prefix = vec![];
···
         let mut rkey = vec![];
         let pre_checked = prefix
             .get(..entry.prefix_len)
-            .ok_or(ActionNodeError::EntryPrefixOutOfbounds)?;
+            .ok_or(RkeyError::EntryPrefixOutOfbounds)?;
         rkey.extend_from_slice(pre_checked);
         rkey.extend_from_slice(&entry.keysuffix);
         prefix = rkey.clone();
···
     Ok(())
 }
 
+/// Traverser of an atproto MST
+///
+/// Walks the tree from left-to-right in depth-first order
 #[derive(Debug)]
 pub struct Walker {
     stack: Vec<Need>,
+    prev: String,
 }
 
 impl Walker {
     pub fn new(tree_root_cid: Cid) -> Self {
         Self {
             stack: vec![Need::Node(tree_root_cid)],
+            prev: "".to_string(),
         }
     }
 
-    pub fn walk<T: Clone, E: Error>(
+    /// Advance through nodes until we find a record or can't go further
+    pub fn step<T: Clone, E: Error>(
         &mut self,
         blocks: &mut HashMap<Cid, MaybeProcessedBlock<T, E>>,
         process: impl Fn(&[u8]) -> Result<T, E>,
···
 
                     log::trace!("emitting a block as a step. depth={}", self.stack.len());
                     let data = data.map_err(Trip::ProcessFailed)?;
+
+                    // rkeys *must* be in order or else the tree is invalid (or
+                    // we have a bug)
+                    if rkey <= self.prev {
+                        return Err(Trip::RkeyOutOfOrder);
+                    }
+                    self.prev = rkey.clone();
+
                     return Ok(Step::Step { rkey, data });
                 }
             }

tests/non-huge-cars.rs

···
     while let Some((rkey, size)) = record_stream.try_next().await.unwrap() {
         records += 1;
         sum += size;
-        if rkey.0 == "app.bsky.actor.profile/self" {
+        if rkey == "app.bsky.actor.profile/self" {
             found_bsky_profile = true;
         }
-        assert!(rkey.0 > prev_rkey, "rkeys are streamed in order");
-        prev_rkey = rkey.0;
+        assert!(rkey > prev_rkey, "rkeys are streamed in order");
+        prev_rkey = rkey;
     }
     assert_eq!(records, expected_records);
     assert_eq!(sum, expected_sum);