reduced block over-provision, empirical but not mathematical solution

crates/jacquard-repo/src/commit/firehose.rs

···
 
     /// DEPRECATED: Unused
     pub rebase: bool,
-
-    /// Debug: block sources for validation analysis
-    #[cfg(debug_assertions)]
-    pub block_sources: BTreeMap<IpldCid, String>,
-
-    #[cfg(debug_assertions)]
-    pub excluded_blocks: BTreeMap<IpldCid, Vec<u8>>, // blocks we skipped
-
-    #[cfg(debug_assertions)]
-    pub excluded_metadata: BTreeMap<IpldCid, Vec<String>>, // context about excluded blocks
 }
 
 /// A repository operation (mutation of a single record)
···
             blobs: self.blobs.into_iter().map(|b| b.into_static()).collect(),
             too_big: self.too_big,
             rebase: self.rebase,
-            #[cfg(debug_assertions)]
-            block_sources: self.block_sources,
-            #[cfg(debug_assertions)]
-            excluded_blocks: self.excluded_blocks,
-            #[cfg(debug_assertions)]
-            excluded_metadata: self.excluded_metadata,
         }
     }
 }
···
 use crate::mst::{Mst, VerifiedWriteOp};
 use crate::storage::{BlockStore, LayeredBlockStore, MemoryBlockStore};
 use cid::Cid as IpldCid;
-use std::collections::BTreeMap;
 use std::sync::Arc;
 
 impl<'a> FirehoseCommit<'a> {
···
         // 2. Parse CAR blocks from the firehose message into temporary storage
         let parsed = parse_car_bytes(&self.blocks).await?;
 
-        #[cfg(debug_assertions)]
-        let provided_blocks = parsed
-            .blocks
-            .keys()
-            .cloned()
-            .collect::<std::collections::HashSet<_>>();
-        #[cfg(debug_assertions)]
-        let accessed_blocks = Arc::new(std::sync::RwLock::new(std::collections::HashSet::new()));
-
-        #[cfg(debug_assertions)]
-        let missing_blocks = Arc::new(std::sync::RwLock::new(Vec::new()));
-
-        #[cfg(debug_assertions)]
-        let block_categories = self.block_sources.clone();
-
-        #[cfg(debug_assertions)]
-        let excluded_blocks_ref = self.excluded_blocks.clone();
-
         let temp_storage = Arc::new(MemoryBlockStore::new_from_blocks(parsed.blocks));
 
-        #[cfg(debug_assertions)]
-        let tracking_storage = {
-            use crate::storage::BlockStore;
-            #[derive(Clone)]
-            struct TrackingStorage {
-                inner: Arc<MemoryBlockStore>,
-                accessed: Arc<std::sync::RwLock<std::collections::HashSet<IpldCid>>>,
-                missing: Arc<std::sync::RwLock<Vec<IpldCid>>>,
-                excluded: BTreeMap<IpldCid, Vec<u8>>,
-            }
-            impl BlockStore for TrackingStorage {
-                async fn get(&self, cid: &IpldCid) -> Result<Option<Bytes>> {
-                    self.accessed.write().unwrap().insert(*cid);
-                    let result = self.inner.get(cid).await?;
-
-                    if result.is_none() {
-                        // Check if this block was excluded
-                        if let Some(excluded_block) = self.excluded.get(cid) {
-                            self.missing.write().unwrap().push(*cid);
-                            eprintln!(
-                                "[MISS] Block {} was EXCLUDED but needed during validation",
-                                cid
-                            );
-                            // Return the excluded block so validation can continue
-                            return Ok(Some(Bytes::copy_from_slice(&excluded_block)));
-                        } else {
-                            self.missing.write().unwrap().push(*cid);
-                            eprintln!(
-                                "[MISS] Block {} not found (never seen during commit creation)",
-                                cid
-                            );
-                        }
-                    }
-
-                    Ok(result)
-                }
-                async fn put(&self, data: &[u8]) -> Result<IpldCid> {
-                    self.inner.put(data).await
-                }
-                async fn has(&self, cid: &IpldCid) -> Result<bool> {
-                    self.inner.has(cid).await
-                }
-                async fn put_many(
-                    &self,
-                    blocks: impl IntoIterator<Item = (IpldCid, Bytes)> + Send,
-                ) -> Result<()> {
-                    self.inner.put_many(blocks).await
-                }
-                async fn get_many(&self, cids: &[IpldCid]) -> Result<Vec<Option<Bytes>>> {
-                    self.inner.get_many(cids).await
-                }
-                async fn apply_commit(&self, commit: crate::repo::CommitData) -> Result<()> {
-                    self.inner.apply_commit(commit).await
-                }
-            }
-            Arc::new(TrackingStorage {
-                inner: temp_storage.clone(),
-                accessed: accessed_blocks.clone(),
-                missing: missing_blocks.clone(),
-                excluded: excluded_blocks_ref,
-            })
-        };
-
-        #[cfg(not(debug_assertions))]
-        let tracking_storage = temp_storage.clone();
-
         // 3. Extract and verify commit object from temporary storage
         let commit_cid: IpldCid = self
             .commit
             .to_ipld()
             .map_err(|e| RepoError::invalid_cid_conversion(e, "commit CID"))?;
-        let commit_bytes = tracking_storage
+        let commit_bytes = temp_storage
             .get(&commit_cid)
             .await?
             .ok_or_else(|| RepoError::not_found("commit block", &commit_cid))?;
···
         // 5. Load new MST from commit.data (claimed result)
         let expected_root = *commit.data();
 
-        let mut new_mst = Mst::load(tracking_storage, expected_root, None);
+        let mut new_mst = Mst::load(temp_storage, expected_root, None);
 
         let verified_ops = self
             .ops
···
                 "MST root mismatch: expected {}, got {}",
                 prev_data_cid, computed_root
             )));
-        }
-
-        #[cfg(debug_assertions)]
-        {
-            let accessed = accessed_blocks.read().unwrap();
-            let missing = missing_blocks.read().unwrap();
-            let unused: Vec<_> = provided_blocks.difference(&*accessed).copied().collect();
-
-            println!("[validation stats]");
-            println!("  provided: {} blocks", provided_blocks.len());
-            println!("  accessed: {} blocks", accessed.len());
-
-            if !missing.is_empty() {
-                println!(
-                    "  MISSING: {} blocks NEEDED but not provided!",
-                    missing.len()
-                );
-
-                // Show operation breakdown for this commit
-                let mut op_counts: BTreeMap<&str, usize> = BTreeMap::new();
-                for op in &self.ops {
-                    *op_counts.entry(op.action.as_ref()).or_insert(0) += 1;
-                }
-                println!("  operations in this commit:");
-                for (action, count) in op_counts {
-                    println!("    {}: {}", action, count);
-                }
-
-                println!("  missing block CIDs:");
-                for cid in missing.iter() {
-                    if self.excluded_blocks.contains_key(cid) {
-                        println!("    {} (was excluded)", cid);
-                        if let Some(metadata) = self.excluded_metadata.get(cid) {
-                            for context in metadata {
-                                println!("      -> {}", context);
-                            }
-                        }
-                    } else {
-                        println!("    {} (never seen)", cid);
-                    }
-                }
-            }
-
-            if !unused.is_empty() {
-                println!(
-                    "  UNUSED: {} blocks ({}%)",
-                    unused.len(),
-                    (unused.len() * 100) / provided_blocks.len()
-                );
-
-                // Show breakdown by category
-                let mut category_stats: BTreeMap<&str, (usize, usize)> = BTreeMap::new();
-                for (cid, category) in &block_categories {
-                    let stats = category_stats.entry(category).or_insert((0, 0));
-                    stats.0 += 1; // total provided
-                    if accessed.contains(cid) {
-                        stats.1 += 1; // accessed
-                    }
-                }
-
-                println!("\n  breakdown by category:");
-                for (category, (total, accessed_count)) in category_stats {
-                    let unused_count = total - accessed_count;
-                    let unused_pct = if total > 0 {
-                        (unused_count * 100) / total
-                    } else {
-                        0
-                    };
-                    println!(
-                        "    {}: {} total, {} accessed, {} unused ({}%)",
-                        category, total, accessed_count, unused_count, unused_pct
-                    );
-                }
-            } else {
-                println!("  ✓ all blocks were used");
-            }
         }
 
         Ok(expected_root)

crates/jacquard-repo/src/mst/cursor.rs

···
 use cid::Cid as IpldCid;
 use smol_str::SmolStr;
 
-#[cfg(debug_assertions)]
-use std::collections::HashSet;
-#[cfg(debug_assertions)]
-use std::sync::{Arc, RwLock};
-
 /// Position within an MST traversal
 #[derive(Debug, Clone)]
 pub enum CursorPosition<S: BlockStore> {
···
 
     /// Current position in traversal
     current: CursorPosition<S>,
-
-    /// Track CIDs accessed during traversal (debug only)
-    #[cfg(debug_assertions)]
-    accessed_cids: Option<Arc<RwLock<HashSet<IpldCid>>>>,
 }
 
 impl<S: BlockStore + Sync + 'static> MstCursor<S> {
···
         Self {
             path: Vec::new(),
             current: CursorPosition::Tree { mst: root },
-            #[cfg(debug_assertions)]
-            accessed_cids: None,
-        }
-    }
-
-    /// Create new cursor with dirty tracking enabled
-    ///
-    /// Records all CIDs accessed during traversal in the provided set.
-    #[cfg(debug_assertions)]
-    pub fn new_with_tracking(root: Mst<S>, tracking: Arc<RwLock<HashSet<IpldCid>>>) -> Self {
-        Self {
-            path: Vec::new(),
-            current: CursorPosition::Tree { mst: root },
-            accessed_cids: Some(tracking),
         }
     }
 
···
     /// If at the root level (before stepping in), returns root's layer + 1.
     pub async fn layer(&self) -> Result<usize> {
         if let Some((walking_node, _, _)) = self.path.last() {
-            // Track CID access
-            #[cfg(debug_assertions)]
-            if let Some(ref tracking) = self.accessed_cids {
-                if let Ok(cid) = walking_node.get_pointer().await {
-                    tracking.write().unwrap().insert(cid);
-                }
-            }
-
             // We're inside a node - return its layer
             walking_node.get_layer().await
         } else {
···
             // is one layer higher than being "inside" the root
             match &self.current {
                 CursorPosition::Tree { mst } => {
-                    // Track CID access
-                    #[cfg(debug_assertions)]
-                    if let Some(ref tracking) = self.accessed_cids {
-                        if let Ok(cid) = mst.get_pointer().await {
-                            tracking.write().unwrap().insert(cid);
-                        }
-                    }
-
                     let root_layer = mst.get_layer().await?;
                     Ok(root_layer + 1)
                 }
···
 
     /// Descend into a tree node
     async fn step_into(&mut self, mst: Mst<S>) -> Result<()> {
-        // Track CID access
-        #[cfg(debug_assertions)]
-        if let Some(ref tracking) = self.accessed_cids {
-            if let Ok(cid) = mst.get_pointer().await {
-                tracking.write().unwrap().insert(cid);
-            }
-        }
-
         let entries = mst.get_entries().await?;
 
         if entries.is_empty() {

crates/jacquard-repo/src/mst/diff.rs

···
 use std::future::Future;
 use std::pin::Pin;
 
-#[cfg(debug_assertions)]
-use std::collections::HashSet;
-#[cfg(debug_assertions)]
-use std::sync::{Arc, RwLock};
-
 use super::cursor::{CursorPosition, MstCursor};
 use super::tree::Mst;
 use super::util::serialize_node_data;
···
     /// When modifying a tree, old MST nodes along changed paths become unreachable.
     /// This tracks those nodes for garbage collection.
     pub removed_mst_blocks: Vec<IpldCid>,
-
-    /// CIDs accessed from old tree during diff (debug only)
-    ///
-    /// Tracks all blocks touched when walking the old tree during diff.
-    /// This is the precise set of blocks needed for validation.
-    #[cfg(debug_assertions)]
-    pub old_tree_accessed: Vec<IpldCid>,
-
-    /// CIDs accessed from new tree during diff (debug only)
-    #[cfg(debug_assertions)]
-    pub new_tree_accessed: Vec<IpldCid>,
 }
 
 use super::tree::VerifiedWriteOp;
···
             removed_cids: Vec::new(),
             new_mst_blocks: BTreeMap::new(),
             removed_mst_blocks: Vec::new(),
-            #[cfg(debug_assertions)]
-            old_tree_accessed: Vec::new(),
-            #[cfg(debug_assertions)]
-            new_tree_accessed: Vec::new(),
         }
     }
 
···
             return Ok(());
         }
 
-        // CIDs differ - use cursors to walk both trees with tracking
-        #[cfg(debug_assertions)]
-        let old_tracking = Arc::new(RwLock::new(HashSet::new()));
-        #[cfg(debug_assertions)]
-        let new_tracking = Arc::new(RwLock::new(HashSet::new()));
-
-        #[cfg(debug_assertions)]
-        let mut old_cursor = MstCursor::new_with_tracking(old.clone(), old_tracking.clone());
-        #[cfg(debug_assertions)]
-        let mut new_cursor = MstCursor::new_with_tracking(new.clone(), new_tracking.clone());
-
-        #[cfg(not(debug_assertions))]
+        // CIDs differ - use cursors to walk both trees
         let mut old_cursor = MstCursor::new(old.clone());
-        #[cfg(not(debug_assertions))]
         let mut new_cursor = MstCursor::new(new.clone());
 
         // Don't advance yet - let loop handle roots like any other tree comparison
···
                     }
                 }
             }
-        }
-
-        // Collect tracking data
-        #[cfg(debug_assertions)]
-        {
-            diff.old_tree_accessed = old_tracking.read().unwrap().iter().copied().collect();
-            diff.new_tree_accessed = new_tracking.read().unwrap().iter().copied().collect();
         }
 
         Ok(())

crates/jacquard-repo/src/repo.rs

···
 use crate::commit::firehose::{FirehoseCommit, RepoOp};
 use crate::commit::{Commit, SigningKey};
 use crate::error::{RepoError, Result};
-use crate::mst::{Mst, MstDiff, RecordWriteOp};
+use crate::mst::{Mst, RecordWriteOp};
 use crate::storage::BlockStore;
 use bytes::Bytes;
 use cid::Cid as IpldCid;
···
 
     /// CIDs of blocks to delete
     pub deleted_cids: Vec<IpldCid>,
-
-    /// Debug: block sources for validation analysis
-    #[cfg(debug_assertions)]
-    pub block_sources: BTreeMap<IpldCid, String>,
-    #[cfg(debug_assertions)]
-    pub excluded_blocks: BTreeMap<IpldCid, Bytes>, // blocks we skipped
-    #[cfg(debug_assertions)]
-    pub excluded_metadata: BTreeMap<IpldCid, Vec<String>>, // context about excluded blocks
 }
 
 impl CommitData {
···
             blobs,
             too_big: false,
             rebase: false,
-            #[cfg(debug_assertions)]
-            block_sources: self.block_sources.clone(),
-            #[cfg(debug_assertions)]
-            excluded_blocks: self
-                .excluded_blocks
-                .iter()
-                .map(|(cid, b)| (cid.clone(), b.to_vec()))
-                .collect(),
-            #[cfg(debug_assertions)]
-            excluded_metadata: self.excluded_metadata.clone(),
         })
     }
 }
···
             blocks: blocks.clone(),
             relevant_blocks: blocks,
             deleted_cids: Vec::new(),
-            #[cfg(debug_assertions)]
-            block_sources: BTreeMap::new(),
-            #[cfg(debug_assertions)]
-            excluded_blocks: BTreeMap::new(),
-            #[cfg(debug_assertions)]
-            excluded_metadata: BTreeMap::new(),
         })
     }
 
···
             .into_iter()
             .map(|op| op.into_static())
             .collect();
-        let deleted_cids = diff.removed_cids;
 
-        // Step 4: Build blocks and relevant_blocks collections
+        // Step 4: Build blocks and relevant_blocks collections using diff tracking
         let mut blocks = diff.new_mst_blocks;
         let mut relevant_blocks = BTreeMap::new();
 
-        #[cfg(debug_assertions)]
-        let mut block_sources: BTreeMap<IpldCid, &str> = BTreeMap::new();
-
-        // // Add the previous MST root block (needed to load prev_data in validation)
-        // if let Some(prev_root_block) = self.storage.get(&prev_data).await? {
-        //     #[cfg(debug_assertions)]
-        //     block_sources.insert(prev_data, "prev_root");
-        //     relevant_blocks.insert(prev_data, prev_root_block);
-        // }
-
-        let mut new_tree_cids = std::collections::HashSet::new();
         for op in ops {
             let key = format_smolstr!("{}/{}", op.collection().as_ref(), op.rkey().as_ref());
             updated_tree
                 .blocks_for_path(&key, &mut relevant_blocks)
                 .await?;
 
-            let new_path_cids = updated_tree.cids_for_path(&key).await?;
-            for cid in &new_path_cids {
-                new_tree_cids.insert(*cid);
-                #[cfg(debug_assertions)]
-                block_sources.insert(*cid, "new_tree_path");
-            }
-        }
-        let mut old_path_blocks = BTreeMap::new();
-        let mut old_path_cids = std::collections::HashSet::new();
-
-        // Step 2: Walk OLD tree, only add blocks NOT in new tree (changed nodes)
-        for op in ops {
-            let key = format_smolstr!("{}/{}", op.collection().as_ref(), op.rkey().as_ref());
-
-            self.mst.blocks_for_path(&key, &mut old_path_blocks).await?;
-            for cid in updated_tree.cids_for_path(&key).await? {
-                old_path_cids.insert(cid);
-            }
-        }
-
-        let mut excluded_blocks = BTreeMap::new();
-        #[cfg(debug_assertions)]
-        let mut excluded_metadata: BTreeMap<IpldCid, Vec<String>> = BTreeMap::new();
-
-        // Re-walk old tree paths to collect metadata about excluded blocks
-        #[cfg(debug_assertions)]
-        for (op_idx, op) in ops.iter().enumerate() {
-            let key = format_smolstr!("{}/{}", op.collection().as_ref(), op.rkey().as_ref());
-            let old_path_cids = self.mst.cids_for_path(&key).await?;
-
-            for (depth, cid) in old_path_cids.iter().enumerate() {
-                if !new_tree_cids.contains(cid) {
-                    let metadata = format!("op#{} ({}) path depth {}", op_idx, key, depth);
-                    excluded_metadata.entry(*cid).or_insert_with(Vec::new).push(metadata);
-                }
-            }
-        }
-
-        for (cid, block) in old_path_blocks.into_iter() {
-            // Only include if this block CHANGED (different CID in new tree)
-            if !new_tree_cids.contains(&cid) {
-                //relevant_blocks.insert(cid, block);
-                excluded_blocks.insert(cid, block);
-                #[cfg(debug_assertions)]
-                block_sources.insert(cid, "old_tree_changed");
+            // For CREATE ops in multi-op commits, include old tree paths.
+            // Empirically necessary: tree restructuring from multiple creates
+            // can access old MST nodes during inversion (reason TBD).
+            if let RecordWriteOp::Create { .. } = op
+                && ops.len() > 1
+            {
+                self.mst.blocks_for_path(&key, &mut relevant_blocks).await?;
             }
         }
 
-        // // Add new leaf blocks to both collections (single iteration)
-        // for (cid, block) in &leaf_blocks {
-        //     if diff.new_leaf_cids.contains(cid) {
-        //         blocks.insert(*cid, block.clone());
-        //         #[cfg(debug_assertions)]
-        //         block_sources.insert(*cid, "new_leaf");
-        //         relevant_blocks.insert(*cid, block.clone());
-        //     }
-        // }
+        let deleted_cids = diff.removed_cids;
 
-        // For DELETE operations, we need the deleted record blocks for inversion
-        // (when inverting a delete, we insert the prev CID back)
-        // for cid in &deleted_cids {
-        //     if let Some(block) = self.storage.get(cid).await? {
-        //         #[cfg(debug_assertions)]
-        //         block_sources.insert(*cid, "deleted_leaf");
-        //         relevant_blocks.insert(*cid, block);
-        //     }
-        // }
-
-        // Step 6: Create and sign commit
+        // Step 5: Create and sign commit
         let rev = Ticker::new().next(Some(self.commit.rev.clone()));
         let commit = Commit::new_unsigned(did.clone().into_static(), data, rev.clone(), prev)
             .sign(signing_key)?;
···
         let commit_cid = crate::mst::util::compute_cid(&commit_cbor)?;
         let commit_bytes = bytes::Bytes::from(commit_cbor);
 
-        // Step 7: Add commit block to both collections
+        // Step 6: Add commit block to both collections
         blocks.insert(commit_cid, commit_bytes.clone());
-        #[cfg(debug_assertions)]
-        block_sources.insert(commit_cid, "commit");
         relevant_blocks.insert(commit_cid, commit_bytes);
 
-        #[cfg(debug_assertions)]
-        {
-            let mut by_source: BTreeMap<&str, usize> = BTreeMap::new();
-            for source in block_sources.values() {
-                *by_source.entry(source).or_insert(0) += 1;
-            }
-            println!("[commit creation] relevant_blocks by source:");
-            for (source, count) in by_source {
-                println!("  {}: {}", source, count);
-            }
-            println!("  TOTAL: {}", relevant_blocks.len());
-        }
-
-        // Step 8: Update internal MST state
+        // Step 7: Update internal MST state
         self.mst = updated_tree;
 
         Ok((
···
                 blocks,
                 relevant_blocks,
                 deleted_cids,
-                #[cfg(debug_assertions)]
-                block_sources: block_sources
-                    .into_iter()
-                    .map(|(k, v)| (k, v.to_string()))
-                    .collect(),
-                #[cfg(debug_assertions)]
-                excluded_blocks,
-                #[cfg(debug_assertions)]
-                excluded_metadata,
             },
         ))
     }