testing improvements

crates/jacquard-repo/src/car/writer.rs

···
     blocks: BTreeMap<IpldCid, Bytes>,
 ) -> Result<()> {
     let path = path.as_ref();
-    let file = File::create(path)
-        .await
-        .map_err(|e| RepoError::io(e).with_context(format!("creating CAR file: {}", path.display())))?;
+    let file = File::create(path).await.map_err(|e| {
+        RepoError::io(e).with_context(format!("creating CAR file: {}", path.display()))
+    })?;
 
     let header = iroh_car::CarHeader::new_v1(roots);
     let mut writer = CarWriter::new(header, file);
···
             .map_err(|e| RepoError::car(e).with_context(format!("writing block {}", cid)))?;
     }
 
-    writer.finish().await.map_err(|e| RepoError::car(e).with_context("finalizing CAR file"))?;
+    writer
+        .finish()
+        .await
+        .map_err(|e| RepoError::car(e).with_context("finalizing CAR file"))?;
 
     Ok(())
 }
···
             .map_err(|e| RepoError::car(e).with_context(format!("writing block {}", cid)))?;
     }
 
-    writer.finish().await.map_err(|e| RepoError::car(e).with_context("finalizing CAR bytes"))?;
+    writer
+        .finish()
+        .await
+        .map_err(|e| RepoError::car(e).with_context("finalizing CAR bytes"))?;
 
-    buffer.flush().await.map_err(|e| RepoError::io(e).with_context("flushing CAR buffer"))?;
+    buffer
+        .flush()
+        .await
+        .map_err(|e| RepoError::io(e).with_context("flushing CAR buffer"))?;
 
     Ok(buffer)
 }
···
 /// - All record blocks (from storage)
 ///
 /// Uses streaming to avoid loading all blocks into memory.
+///
+/// Should write in the correct order for [streaming car processing](https://github.com/bluesky-social/proposals/blob/main/0006-sync-iteration/README.md#streaming-car-processing) from sync v1.1
 pub async fn export_repo_car<S: BlockStore + Sync + 'static>(
     path: impl AsRef<Path>,
     commit_cid: IpldCid,
     mst: &Mst<S>,
 ) -> Result<()> {
     let path = path.as_ref();
-    let file = File::create(path)
-        .await
-        .map_err(|e| RepoError::io(e).with_context(format!("creating CAR export file: {}", path.display())))?;
+    let file = File::create(path).await.map_err(|e| {
+        RepoError::io(e).with_context(format!("creating CAR export file: {}", path.display()))
+    })?;
 
     let header = iroh_car::CarHeader::new_v1(vec![commit_cid]);
     let mut writer = CarWriter::new(header, file);
···
     mst.write_blocks_to_car(&mut writer).await?;
 
     // Finish writing
-    writer.finish().await.map_err(|e| RepoError::car(e).with_context("finalizing CAR export"))?;
+    writer
+        .finish()
+        .await
+        .map_err(|e| RepoError::car(e).with_context("finalizing CAR export"))?;
 
     Ok(())
 }

crates/jacquard-repo/src/commit/firehose.rs

···
 
     /// DEPRECATED: Unused
     pub rebase: bool,
+
+    /// Debug: block sources for validation analysis
+    #[cfg(debug_assertions)]
+    pub block_sources: BTreeMap<IpldCid, String>,
+
+    #[cfg(debug_assertions)]
+    pub excluded_blocks: BTreeMap<IpldCid, Vec<u8>>, // blocks we skipped
+
+    #[cfg(debug_assertions)]
+    pub excluded_metadata: BTreeMap<IpldCid, Vec<String>>, // context about excluded blocks
 }
 
 /// A repository operation (mutation of a single record)
···
             blobs: self.blobs.into_iter().map(|b| b.into_static()).collect(),
             too_big: self.too_big,
             rebase: self.rebase,
+            #[cfg(debug_assertions)]
+            block_sources: self.block_sources,
+            #[cfg(debug_assertions)]
+            excluded_blocks: self.excluded_blocks,
+            #[cfg(debug_assertions)]
+            excluded_metadata: self.excluded_metadata,
         }
     }
 }
···
 use crate::mst::{Mst, VerifiedWriteOp};
 use crate::storage::{BlockStore, LayeredBlockStore, MemoryBlockStore};
 use cid::Cid as IpldCid;
+use std::collections::BTreeMap;
 use std::sync::Arc;
 
 impl<'a> FirehoseCommit<'a> {
···
     /// **Inductive property:** Can validate without any external state besides the blocks
     /// in this message. The `prev_data` field provides the starting MST root, and operations
     /// include `prev` CIDs for validation. All necessary blocks must be in the CAR bytes.
+    ///
+    /// Note: Because this uses the same merkle search tree struct as the repository itself,
+    /// this is far from the most efficient possible validation function possible. The repo
+    /// tree struct carries extra information. However,
+    /// it has the virtue of making everything self-validating.
     pub async fn validate_v1_1(&self, pubkey: &PublicKey<'_>) -> Result<IpldCid> {
         // 1. Require prev_data for v1.1
         let prev_data_cid: IpldCid = self
···
 
         // 2. Parse CAR blocks from the firehose message into temporary storage
         let parsed = parse_car_bytes(&self.blocks).await?;
+
+        #[cfg(debug_assertions)]
+        let provided_blocks = parsed
+            .blocks
+            .keys()
+            .cloned()
+            .collect::<std::collections::HashSet<_>>();
+        #[cfg(debug_assertions)]
+        let accessed_blocks = Arc::new(std::sync::RwLock::new(std::collections::HashSet::new()));
+
+        #[cfg(debug_assertions)]
+        let missing_blocks = Arc::new(std::sync::RwLock::new(Vec::new()));
+
+        #[cfg(debug_assertions)]
+        let block_categories = self.block_sources.clone();
+
+        #[cfg(debug_assertions)]
+        let excluded_blocks_ref = self.excluded_blocks.clone();
+
         let temp_storage = Arc::new(MemoryBlockStore::new_from_blocks(parsed.blocks));
 
+        #[cfg(debug_assertions)]
+        let tracking_storage = {
+            use crate::storage::BlockStore;
+            #[derive(Clone)]
+            struct TrackingStorage {
+                inner: Arc<MemoryBlockStore>,
+                accessed: Arc<std::sync::RwLock<std::collections::HashSet<IpldCid>>>,
+                missing: Arc<std::sync::RwLock<Vec<IpldCid>>>,
+                excluded: BTreeMap<IpldCid, Vec<u8>>,
+            }
+            impl BlockStore for TrackingStorage {
+                async fn get(&self, cid: &IpldCid) -> Result<Option<Bytes>> {
+                    self.accessed.write().unwrap().insert(*cid);
+                    let result = self.inner.get(cid).await?;
+
+                    if result.is_none() {
+                        // Check if this block was excluded
+                        if let Some(excluded_block) = self.excluded.get(cid) {
+                            self.missing.write().unwrap().push(*cid);
+                            eprintln!(
+                                "[MISS] Block {} was EXCLUDED but needed during validation",
+                                cid
+                            );
+                            // Return the excluded block so validation can continue
+                            return Ok(Some(Bytes::copy_from_slice(&excluded_block)));
+                        } else {
+                            self.missing.write().unwrap().push(*cid);
+                            eprintln!(
+                                "[MISS] Block {} not found (never seen during commit creation)",
+                                cid
+                            );
+                        }
+                    }
+
+                    Ok(result)
+                }
+                async fn put(&self, data: &[u8]) -> Result<IpldCid> {
+                    self.inner.put(data).await
+                }
+                async fn has(&self, cid: &IpldCid) -> Result<bool> {
+                    self.inner.has(cid).await
+                }
+                async fn put_many(
+                    &self,
+                    blocks: impl IntoIterator<Item = (IpldCid, Bytes)> + Send,
+                ) -> Result<()> {
+                    self.inner.put_many(blocks).await
+                }
+                async fn get_many(&self, cids: &[IpldCid]) -> Result<Vec<Option<Bytes>>> {
+                    self.inner.get_many(cids).await
+                }
+                async fn apply_commit(&self, commit: crate::repo::CommitData) -> Result<()> {
+                    self.inner.apply_commit(commit).await
+                }
+            }
+            Arc::new(TrackingStorage {
+                inner: temp_storage.clone(),
+                accessed: accessed_blocks.clone(),
+                missing: missing_blocks.clone(),
+                excluded: excluded_blocks_ref,
+            })
+        };
+
+        #[cfg(not(debug_assertions))]
+        let tracking_storage = temp_storage.clone();
+
         // 3. Extract and verify commit object from temporary storage
         let commit_cid: IpldCid = self
             .commit
             .to_ipld()
             .map_err(|e| RepoError::invalid_cid_conversion(e, "commit CID"))?;
-        let commit_bytes = temp_storage
+        let commit_bytes = tracking_storage
             .get(&commit_cid)
             .await?
             .ok_or_else(|| RepoError::not_found("commit block", &commit_cid))?;
···
 
         // 5. Load new MST from commit.data (claimed result)
         let expected_root = *commit.data();
-        let mut new_mst = Mst::load(temp_storage, expected_root, None);
+
+        let mut new_mst = Mst::load(tracking_storage, expected_root, None);
 
         let verified_ops = self
             .ops
···
             )));
         }
 
+        #[cfg(debug_assertions)]
+        {
+            let accessed = accessed_blocks.read().unwrap();
+            let missing = missing_blocks.read().unwrap();
+            let unused: Vec<_> = provided_blocks.difference(&*accessed).copied().collect();
+
+            println!("[validation stats]");
+            println!("  provided: {} blocks", provided_blocks.len());
+            println!("  accessed: {} blocks", accessed.len());
+
+            if !missing.is_empty() {
+                println!(
+                    "  MISSING: {} blocks NEEDED but not provided!",
+                    missing.len()
+                );
+
+                // Show operation breakdown for this commit
+                let mut op_counts: BTreeMap<&str, usize> = BTreeMap::new();
+                for op in &self.ops {
+                    *op_counts.entry(op.action.as_ref()).or_insert(0) += 1;
+                }
+                println!("  operations in this commit:");
+                for (action, count) in op_counts {
+                    println!("    {}: {}", action, count);
+                }
+
+                println!("  missing block CIDs:");
+                for cid in missing.iter() {
+                    if self.excluded_blocks.contains_key(cid) {
+                        println!("    {} (was excluded)", cid);
+                        if let Some(metadata) = self.excluded_metadata.get(cid) {
+                            for context in metadata {
+                                println!("      -> {}", context);
+                            }
+                        }
+                    } else {
+                        println!("    {} (never seen)", cid);
+                    }
+                }
+            }
+
+            if !unused.is_empty() {
+                println!(
+                    "  UNUSED: {} blocks ({}%)",
+                    unused.len(),
+                    (unused.len() * 100) / provided_blocks.len()
+                );
+
+                // Show breakdown by category
+                let mut category_stats: BTreeMap<&str, (usize, usize)> = BTreeMap::new();
+                for (cid, category) in &block_categories {
+                    let stats = category_stats.entry(category).or_insert((0, 0));
+                    stats.0 += 1; // total provided
+                    if accessed.contains(cid) {
+                        stats.1 += 1; // accessed
+                    }
+                }
+
+                println!("\n  breakdown by category:");
+                for (category, (total, accessed_count)) in category_stats {
+                    let unused_count = total - accessed_count;
+                    let unused_pct = if total > 0 {
+                        (unused_count * 100) / total
+                    } else {
+                        0
+                    };
+                    println!(
+                        "    {}: {} total, {} accessed, {} unused ({}%)",
+                        category, total, accessed_count, unused_count, unused_pct
+                    );
+                }
+            } else {
+                println!("  ✓ all blocks were used");
+            }
+        }
+
         Ok(expected_root)
     }
 }
···
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::commit::{Commit, SigningKey as _};
+    use crate::Repository;
+    use crate::commit::Commit;
     use crate::mst::{Mst, RecordWriteOp};
     use crate::storage::MemoryBlockStore;
-    use crate::{CommitData, Repository};
     use jacquard_common::types::crypto::{KeyCodec, PublicKey};
     use jacquard_common::types::recordkey::Rkey;
     use jacquard_common::types::string::{Nsid, RecordKey};
···
         let parsed = parse_car_bytes(&firehose_commit.blocks).await.unwrap();
         let commit_cid: IpldCid = firehose_commit.commit.to_ipld().unwrap();
 
-        let mut blocks_without_commit: BTreeMap<IpldCid, bytes::Bytes> = parsed
+        let blocks_without_commit: BTreeMap<IpldCid, bytes::Bytes> = parsed
             .blocks
             .into_iter()
             .filter(|(cid, _)| cid != &commit_cid)
···
             .insert(fake_commit_cid, bytes::Bytes::from(fake_commit_cbor));
         commit_data.cid = fake_commit_cid;
 
-        let mut firehose_commit = commit_data
+        let firehose_commit = commit_data
             .to_firehose_commit(&did, 1, Datetime::now(), repo_ops, vec![])
             .await
             .unwrap();

crates/jacquard-repo/src/mst/cursor.rs

···
 use cid::Cid as IpldCid;
 use smol_str::SmolStr;
 
+#[cfg(debug_assertions)]
+use std::collections::HashSet;
+#[cfg(debug_assertions)]
+use std::sync::{Arc, RwLock};
+
 /// Position within an MST traversal
 #[derive(Debug, Clone)]
 pub enum CursorPosition<S: BlockStore> {
···
 
     /// Current position in traversal
     current: CursorPosition<S>,
+
+    /// Track CIDs accessed during traversal (debug only)
+    #[cfg(debug_assertions)]
+    accessed_cids: Option<Arc<RwLock<HashSet<IpldCid>>>>,
 }
 
 impl<S: BlockStore + Sync + 'static> MstCursor<S> {
···
         Self {
             path: Vec::new(),
             current: CursorPosition::Tree { mst: root },
+            #[cfg(debug_assertions)]
+            accessed_cids: None,
+        }
+    }
+
+    /// Create new cursor with dirty tracking enabled
+    ///
+    /// Records all CIDs accessed during traversal in the provided set.
+    #[cfg(debug_assertions)]
+    pub fn new_with_tracking(root: Mst<S>, tracking: Arc<RwLock<HashSet<IpldCid>>>) -> Self {
+        Self {
+            path: Vec::new(),
+            current: CursorPosition::Tree { mst: root },
+            accessed_cids: Some(tracking),
         }
     }
 
···
     /// If at the root level (before stepping in), returns root's layer + 1.
     pub async fn layer(&self) -> Result<usize> {
         if let Some((walking_node, _, _)) = self.path.last() {
+            // Track CID access
+            #[cfg(debug_assertions)]
+            if let Some(ref tracking) = self.accessed_cids {
+                if let Ok(cid) = walking_node.get_pointer().await {
+                    tracking.write().unwrap().insert(cid);
+                }
+            }
+
             // We're inside a node - return its layer
             walking_node.get_layer().await
         } else {
···
             // is one layer higher than being "inside" the root
             match &self.current {
                 CursorPosition::Tree { mst } => {
+                    // Track CID access
+                    #[cfg(debug_assertions)]
+                    if let Some(ref tracking) = self.accessed_cids {
+                        if let Ok(cid) = mst.get_pointer().await {
+                            tracking.write().unwrap().insert(cid);
+                        }
+                    }
+
                     let root_layer = mst.get_layer().await?;
                     Ok(root_layer + 1)
                 }
···
 
     /// Descend into a tree node
     async fn step_into(&mut self, mst: Mst<S>) -> Result<()> {
+        // Track CID access
+        #[cfg(debug_assertions)]
+        if let Some(ref tracking) = self.accessed_cids {
+            if let Ok(cid) = mst.get_pointer().await {
+                tracking.write().unwrap().insert(cid);
+            }
+        }
+
         let entries = mst.get_entries().await?;
 
         if entries.is_empty() {

crates/jacquard-repo/src/mst/diff.rs

···
 use std::future::Future;
 use std::pin::Pin;
 
+#[cfg(debug_assertions)]
+use std::collections::HashSet;
+#[cfg(debug_assertions)]
+use std::sync::{Arc, RwLock};
+
 use super::cursor::{CursorPosition, MstCursor};
 use super::tree::Mst;
 use super::util::serialize_node_data;
···
     /// When modifying a tree, old MST nodes along changed paths become unreachable.
     /// This tracks those nodes for garbage collection.
     pub removed_mst_blocks: Vec<IpldCid>,
+
+    /// CIDs accessed from old tree during diff (debug only)
+    ///
+    /// Tracks all blocks touched when walking the old tree during diff.
+    /// This is the precise set of blocks needed for validation.
+    #[cfg(debug_assertions)]
+    pub old_tree_accessed: Vec<IpldCid>,
+
+    /// CIDs accessed from new tree during diff (debug only)
+    #[cfg(debug_assertions)]
+    pub new_tree_accessed: Vec<IpldCid>,
 }
 
 use super::tree::VerifiedWriteOp;
···
             removed_cids: Vec::new(),
             new_mst_blocks: BTreeMap::new(),
             removed_mst_blocks: Vec::new(),
+            #[cfg(debug_assertions)]
+            old_tree_accessed: Vec::new(),
+            #[cfg(debug_assertions)]
+            new_tree_accessed: Vec::new(),
         }
     }
 
···
             return Ok(());
         }
 
-        // CIDs differ - use cursors to walk both trees
+        // CIDs differ - use cursors to walk both trees with tracking
+        #[cfg(debug_assertions)]
+        let old_tracking = Arc::new(RwLock::new(HashSet::new()));
+        #[cfg(debug_assertions)]
+        let new_tracking = Arc::new(RwLock::new(HashSet::new()));
+
+        #[cfg(debug_assertions)]
+        let mut old_cursor = MstCursor::new_with_tracking(old.clone(), old_tracking.clone());
+        #[cfg(debug_assertions)]
+        let mut new_cursor = MstCursor::new_with_tracking(new.clone(), new_tracking.clone());
+
+        #[cfg(not(debug_assertions))]
         let mut old_cursor = MstCursor::new(old.clone());
+        #[cfg(not(debug_assertions))]
         let mut new_cursor = MstCursor::new(new.clone());
 
         // Don't advance yet - let loop handle roots like any other tree comparison
···
                     }
                 }
             }
+        }
+
+        // Collect tracking data
+        #[cfg(debug_assertions)]
+        {
+            diff.old_tree_accessed = old_tracking.read().unwrap().iter().copied().collect();
+            diff.new_tree_accessed = new_tracking.read().unwrap().iter().copied().collect();
         }
 
         Ok(())

crates/jacquard-repo/src/repo.rs

···
 
     /// CIDs of blocks to delete
     pub deleted_cids: Vec<IpldCid>,
+
+    /// Debug: block sources for validation analysis
+    #[cfg(debug_assertions)]
+    pub block_sources: BTreeMap<IpldCid, String>,
+    #[cfg(debug_assertions)]
+    pub excluded_blocks: BTreeMap<IpldCid, Bytes>, // blocks we skipped
+    #[cfg(debug_assertions)]
+    pub excluded_metadata: BTreeMap<IpldCid, Vec<String>>, // context about excluded blocks
 }
 
 impl CommitData {
···
             blobs,
             too_big: false,
             rebase: false,
+            #[cfg(debug_assertions)]
+            block_sources: self.block_sources.clone(),
+            #[cfg(debug_assertions)]
+            excluded_blocks: self
+                .excluded_blocks
+                .iter()
+                .map(|(cid, b)| (cid.clone(), b.to_vec()))
+                .collect(),
+            #[cfg(debug_assertions)]
+            excluded_metadata: self.excluded_metadata.clone(),
         })
     }
 }
···
             blocks: blocks.clone(),
             relevant_blocks: blocks,
             deleted_cids: Vec::new(),
+            #[cfg(debug_assertions)]
+            block_sources: BTreeMap::new(),
+            #[cfg(debug_assertions)]
+            excluded_blocks: BTreeMap::new(),
+            #[cfg(debug_assertions)]
+            excluded_metadata: BTreeMap::new(),
         })
     }
 
···
     // But these bulk operations would benefit significantly from cursor's skip_subtree()
     // to avoid traversing unrelated branches when searching lexicographically-organized data.
 
-    /// Apply record write operations with inline data
-    ///
-    /// Serializes record data to DAG-CBOR, computes CIDs, stores data blocks,
-    /// then applies write operations to the MST. Returns the diff for inspection.
-    ///
-    /// For creating commits with operations, use `create_commit()` instead.
-    pub async fn apply_record_writes(&mut self, ops: &[RecordWriteOp<'_>]) -> Result<MstDiff> {
-        use smol_str::format_smolstr;
-
-        let mut updated_tree = self.mst.clone();
-
-        for op in ops {
-            updated_tree = match op {
-                RecordWriteOp::Create {
-                    collection,
-                    rkey,
-                    record,
-                } => {
-                    let key = format_smolstr!("{}/{}", collection.as_ref(), rkey.as_ref());
-
-                    // Serialize record to DAG-CBOR
-                    let cbor = serde_ipld_dagcbor::to_vec(record).map_err(|e| {
-                        RepoError::serialization(e).with_context(format!(
-                            "serializing record data for {}/{}",
-                            collection.as_ref(),
-                            rkey.as_ref()
-                        ))
-                    })?;
-
-                    // Compute CID and store data
-                    let cid = self.storage.put(&cbor).await?;
-
-                    updated_tree.add(key.as_str(), cid).await?
-                }
-                RecordWriteOp::Update {
-                    collection,
-                    rkey,
-                    record,
-                    prev,
-                } => {
-                    let key = format_smolstr!("{}/{}", collection.as_ref(), rkey.as_ref());
-
-                    // Serialize record to DAG-CBOR
-                    let cbor = serde_ipld_dagcbor::to_vec(record).map_err(|e| {
-                        RepoError::serialization(e).with_context(format!(
-                            "serializing record data for {}/{}",
-                            collection.as_ref(),
-                            rkey.as_ref()
-                        ))
-                    })?;
-
-                    // Compute CID and store data
-                    let cid = self.storage.put(&cbor).await?;
-
-                    // Validate prev if provided
-                    if let Some(prev_cid) = prev {
-                        if &cid != prev_cid {
-                            return Err(RepoError::cid_mismatch(format!(
-                                "Update prev CID mismatch for key {}: expected {}, got {}",
-                                key, prev_cid, cid
-                            )));
-                        }
-                    }
-
-                    updated_tree.add(key.as_str(), cid).await?
-                }
-                RecordWriteOp::Delete {
-                    collection,
-                    rkey,
-                    prev,
-                } => {
-                    let key = format_smolstr!("{}/{}", collection.as_ref(), rkey.as_ref());
-
-                    // Check exists
-                    let current = self
-                        .mst
-                        .get(key.as_str())
-                        .await?
-                        .ok_or_else(|| RepoError::not_found("record", key.as_str()))?;
-
-                    // Validate prev if provided
-                    if let Some(prev_cid) = prev {
-                        if &current != prev_cid {
-                            return Err(RepoError::cid_mismatch(format!(
-                                "Delete prev CID mismatch for key {}: expected {}, got {}",
-                                key, prev_cid, current
-                            )));
-                        }
-                    }
-
-                    updated_tree.delete(key.as_str()).await?
-                }
-            };
-        }
-
-        // Compute diff before updating
-        let diff = self.mst.diff(&updated_tree).await?;
-
-        println!("Repo before:\n{}", self);
-        // Update mst
-        self.mst = updated_tree;
-
-        println!("Repo after:\n{}", self);
-        Ok(diff)
-    }
-
     /// Create a commit from record write operations
     ///
     /// Applies write operations, creates signed commit, and collects blocks:
···
         let mut blocks = diff.new_mst_blocks;
         let mut relevant_blocks = BTreeMap::new();
 
-        // Add the previous MST root block (needed to load prev_data in validation)
-        if let Some(prev_root_block) = self.storage.get(&prev_data).await? {
-            relevant_blocks.insert(prev_data, prev_root_block);
-        }
+        #[cfg(debug_assertions)]
+        let mut block_sources: BTreeMap<IpldCid, &str> = BTreeMap::new();
+
+        // // Add the previous MST root block (needed to load prev_data in validation)
+        // if let Some(prev_root_block) = self.storage.get(&prev_data).await? {
+        //     #[cfg(debug_assertions)]
+        //     block_sources.insert(prev_data, "prev_root");
+        //     relevant_blocks.insert(prev_data, prev_root_block);
+        // }
 
-        // Walk paths in both old and new trees for each operation
+        let mut new_tree_cids = std::collections::HashSet::new();
         for op in ops {
             let key = format_smolstr!("{}/{}", op.collection().as_ref(), op.rkey().as_ref());
-
             updated_tree
                 .blocks_for_path(&key, &mut relevant_blocks)
                 .await?;
 
-            self.mst.blocks_for_path(&key, &mut relevant_blocks).await?;
+            let new_path_cids = updated_tree.cids_for_path(&key).await?;
+            for cid in &new_path_cids {
+                new_tree_cids.insert(*cid);
+                #[cfg(debug_assertions)]
+                block_sources.insert(*cid, "new_tree_path");
+            }
+        }
+        let mut old_path_blocks = BTreeMap::new();
+        let mut old_path_cids = std::collections::HashSet::new();
+
+        // Step 2: Walk OLD tree, only add blocks NOT in new tree (changed nodes)
+        for op in ops {
+            let key = format_smolstr!("{}/{}", op.collection().as_ref(), op.rkey().as_ref());
+
+            self.mst.blocks_for_path(&key, &mut old_path_blocks).await?;
+            for cid in updated_tree.cids_for_path(&key).await? {
+                old_path_cids.insert(cid);
+            }
+        }
+
+        let mut excluded_blocks = BTreeMap::new();
+        #[cfg(debug_assertions)]
+        let mut excluded_metadata: BTreeMap<IpldCid, Vec<String>> = BTreeMap::new();
+
+        // Re-walk old tree paths to collect metadata about excluded blocks
+        #[cfg(debug_assertions)]
+        for (op_idx, op) in ops.iter().enumerate() {
+            let key = format_smolstr!("{}/{}", op.collection().as_ref(), op.rkey().as_ref());
+            let old_path_cids = self.mst.cids_for_path(&key).await?;
+
+            for (depth, cid) in old_path_cids.iter().enumerate() {
+                if !new_tree_cids.contains(cid) {
+                    let metadata = format!("op#{} ({}) path depth {}", op_idx, key, depth);
+                    excluded_metadata.entry(*cid).or_insert_with(Vec::new).push(metadata);
+                }
+            }
         }
 
-        // Add new leaf blocks to both collections (single iteration)
-        for (cid, block) in &leaf_blocks {
-            if diff.new_leaf_cids.contains(cid) {
-                blocks.insert(*cid, block.clone());
-                relevant_blocks.insert(*cid, block.clone());
+        for (cid, block) in old_path_blocks.into_iter() {
+            // Only include if this block CHANGED (different CID in new tree)
+            if !new_tree_cids.contains(&cid) {
+                //relevant_blocks.insert(cid, block);
+                excluded_blocks.insert(cid, block);
+                #[cfg(debug_assertions)]
+                block_sources.insert(cid, "old_tree_changed");
             }
         }
 
+        // // Add new leaf blocks to both collections (single iteration)
+        // for (cid, block) in &leaf_blocks {
+        //     if diff.new_leaf_cids.contains(cid) {
+        //         blocks.insert(*cid, block.clone());
+        //         #[cfg(debug_assertions)]
+        //         block_sources.insert(*cid, "new_leaf");
+        //         relevant_blocks.insert(*cid, block.clone());
+        //     }
+        // }
+
+        // For DELETE operations, we need the deleted record blocks for inversion
+        // (when inverting a delete, we insert the prev CID back)
+        // for cid in &deleted_cids {
+        //     if let Some(block) = self.storage.get(cid).await? {
+        //         #[cfg(debug_assertions)]
+        //         block_sources.insert(*cid, "deleted_leaf");
+        //         relevant_blocks.insert(*cid, block);
+        //     }
+        // }
+
         // Step 6: Create and sign commit
         let rev = Ticker::new().next(Some(self.commit.rev.clone()));
         let commit = Commit::new_unsigned(did.clone().into_static(), data, rev.clone(), prev)
···
 
         // Step 7: Add commit block to both collections
         blocks.insert(commit_cid, commit_bytes.clone());
+        #[cfg(debug_assertions)]
+        block_sources.insert(commit_cid, "commit");
         relevant_blocks.insert(commit_cid, commit_bytes);
 
+        #[cfg(debug_assertions)]
+        {
+            let mut by_source: BTreeMap<&str, usize> = BTreeMap::new();
+            for source in block_sources.values() {
+                *by_source.entry(source).or_insert(0) += 1;
+            }
+            println!("[commit creation] relevant_blocks by source:");
+            for (source, count) in by_source {
+                println!("  {}: {}", source, count);
+            }
+            println!("  TOTAL: {}", relevant_blocks.len());
+        }
+
         // Step 8: Update internal MST state
         self.mst = updated_tree;
 
···
                 blocks,
                 relevant_blocks,
                 deleted_cids,
+                #[cfg(debug_assertions)]
+                block_sources: block_sources
+                    .into_iter()
+                    .map(|(k, v)| (k, v.to_string()))
+                    .collect(),
+                #[cfg(debug_assertions)]
+                excluded_blocks,
+                #[cfg(debug_assertions)]
+                excluded_metadata,
             },
         ))
     }

crates/jacquard-repo/tests/firehose_stress.rs crates/jacquard-repo/tests/large_proof_tests.rs

···
 
     firehose_commit.validate_v1_1(&pubkey).await.unwrap();
 
-    for batch_num in 1..=2000 {
+    for batch_num in 1..=5000 {
         let batch_size = rng.gen_range(1..=20);
         let ops = generate_random_ops(&mut rng, &mut tracker, batch_size);
         let record_writes = test_ops_to_record_writes(ops, &collection);