try candystore (ehhhhhhh.)

Cargo.lock

···
 checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43"
 
 [[package]]
+name = "bytemuck"
+version = "1.24.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fbdf580320f38b612e485521afda1ee26d10cc9884efaaa750d383e13e3c5f4"
+dependencies = [
+ "bytemuck_derive",
+]
+
+[[package]]
+name = "bytemuck_derive"
+version = "1.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.106",
+]
+
+[[package]]
 name = "bytes"
 version = "1.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a"
+
+[[package]]
+name = "candystore"
+version = "0.5.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e015e4215c0e855880a745ed0d9be7f2d8f49d0426006ccbc66b7b0e10a1bd1a"
+dependencies = [
+ "anyhow",
+ "bytemuck",
+ "crossbeam-channel",
+ "databuf",
+ "fslock",
+ "libc",
+ "memmap",
+ "parking_lot",
+ "rand",
+ "simd-itertools",
+ "siphasher",
+ "uuid",
+]
 
 [[package]]
 name = "cast"
···
 ]
 
 [[package]]
+name = "crossbeam-channel"
+version = "0.5.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2"
+dependencies = [
+ "crossbeam-utils",
+]
+
+[[package]]
 name = "crossbeam-deque"
 version = "0.8.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 ]
 
 [[package]]
+name = "databuf"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9e1ad1d99bee317a8dac0b7cd86896c5a5f24307009292985dabbf3e412c8b9d"
+dependencies = [
+ "databuf-derive",
+]
+
+[[package]]
+name = "databuf-derive"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "04040c9fc8fcb4084222a26c99faf5b3014772a6115e076b7a50fe49bf25d0ea"
+dependencies = [
+ "databuf_derive_impl",
+]
+
+[[package]]
+name = "databuf_derive_impl"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "daf656eb071fe87d23716f933788a35a8ad6baa6fdbf66a67a261dbd3f9dc81a"
+dependencies = [
+ "quote2",
+ "syn 2.0.106",
+]
+
+[[package]]
 name = "digest"
 version = "0.10.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
+
+[[package]]
+name = "fslock"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "04412b8935272e3a9bae6f48c7bfff74c2911f60525404edfdd28e49884c3bfb"
+dependencies = [
+ "libc",
+ "winapi",
+]
 
 [[package]]
 name = "futures"
···
 checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273"
 
 [[package]]
+name = "memmap"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b"
+dependencies = [
+ "libc",
+ "winapi",
+]
+
+[[package]]
 name = "miniz_oxide"
 version = "0.8.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 ]
 
 [[package]]
+name = "multiversion"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7edb7f0ff51249dfda9ab96b5823695e15a052dc15074c9dbf3d118afaf2c201"
+dependencies = [
+ "multiversion-macros",
+ "target-features",
+]
+
+[[package]]
+name = "multiversion-macros"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b093064383341eb3271f42e381cb8f10a01459478446953953c75d24bd339fc0"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.106",
+ "target-features",
+]
+
+[[package]]
 name = "num-traits"
 version = "0.2.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 ]
 
 [[package]]
+name = "ppv-lite86"
+version = "0.2.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9"
+dependencies = [
+ "zerocopy",
+]
+
+[[package]]
 name = "proc-macro2"
 version = "1.0.101"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 dependencies = [
  "proc-macro2",
 ]
+
+[[package]]
+name = "quote2"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "970573b86f7e5795c8c6c50c56ef602368593f0687188da27fd489a59e253630"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "quote2-macros",
+]
+
+[[package]]
+name = "quote2-macros"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5f4b89c37b2d870a28629ad20da669bb0e7d7214878d0d5111b304aa466e1977"
 
 [[package]]
 name = "r-efi"
···
 checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
 
 [[package]]
+name = "rand"
+version = "0.9.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1"
+dependencies = [
+ "rand_chacha",
+ "rand_core",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
+dependencies = [
+ "ppv-lite86",
+ "rand_core",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38"
+dependencies = [
+ "getrandom",
+]
+
+[[package]]
 name = "rayon"
 version = "1.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 version = "0.2.2"
 dependencies = [
  "bincode",
+ "candystore",
  "clap",
  "criterion",
  "env_logger",
···
 ]
 
 [[package]]
+name = "simd-itertools"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a037ed5ba0cb7102a5b720453b642c5b2cf39960edd2ceace91af8ec3743082a"
+dependencies = [
+ "multiversion",
+]
+
+[[package]]
+name = "siphasher"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d"
+
+[[package]]
 name = "slab"
 version = "0.4.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
  "quote",
  "unicode-ident",
 ]
+
+[[package]]
+name = "target-features"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c1bbb9f3c5c463a01705937a24fdabc5047929ac764b2d5b9cf681c1f5041ed5"
 
 [[package]]
 name = "tempfile"
···
 checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
 
 [[package]]
+name = "uuid"
+version = "1.19.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2e054861b4bd027cd373e18e8d8d8e6548085000e41290d95ce0c373a654b4a"
+dependencies = [
+ "js-sys",
+ "wasm-bindgen",
+]
+
+[[package]]
 name = "vcpkg"
 version = "0.2.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 ]
 
 [[package]]
+name = "winapi"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
+dependencies = [
+ "winapi-i686-pc-windows-gnu",
+ "winapi-x86_64-pc-windows-gnu",
+]
+
+[[package]]
+name = "winapi-i686-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+
+[[package]]
 name = "winapi-util"
 version = "0.1.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 dependencies = [
  "windows-sys 0.60.2",
 ]
+
+[[package]]
+name = "winapi-x86_64-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 
 [[package]]
 name = "windows-link"

Cargo.toml

···
 
 [dependencies]
 bincode = { version = "2.0.1", features = ["serde"] }
+candystore = "0.5.6"
 futures = "0.3.31"
 futures-core = "0.3.31"
 ipld-core = { version = "0.4.2", features = ["serde"] }

examples/disk-read-file/main.rs

···
             // disk to continue
 
             // set up a disk store we can spill to
-            let disk_store = DiskBuilder::new().open(tmpfile).await?;
+            let disk_store = DiskBuilder::new().open(tmpfile, Some(big_stuff.keys_hint())).await?;
 
             // do the spilling, get back a (similar) driver
             let (commit, driver) = big_stuff.finish_loading(disk_store).await?;

src/disk.rs

···
 # #[tokio::main]
 # async fn main() -> Result<(), DiskError> {
 let store = DiskBuilder::new()
-    .with_cache_size_mb(32)
     .with_max_stored_mb(1024) // errors when >1GiB of processed blocks are inserted
     .open("/some/path.db".into()).await?;
 # Ok(())
···
 */
 
 use crate::drive::DriveError;
-use rusqlite::OptionalExtension;
+use candystore::{CandyError, CandyStore, Config};
 use std::path::PathBuf;
 
 #[derive(Debug, thiserror::Error)]
···
     /// (The wrapped err should probably be obscured to remove public-facing
     /// sqlite bits)
     #[error(transparent)]
-    DbError(#[from] rusqlite::Error),
+    DbError(#[from] CandyError),
+    /// Unfortunately candystore uses anyhow::Result for it's open call
+    #[error("Failed on a db call, see logs")]
+    DbGarbageError,
     /// A tokio blocking task failed to join
     #[error("Failed to join a tokio blocking task: {0}")]
     JoinError(#[from] tokio::task::JoinError),
···
     /// limit.
     #[error("Maximum disk size reached")]
     MaxSizeExceeded,
-    #[error("this error was replaced, seeing this is a bug.")]
-    #[doc(hidden)]
-    Stolen,
-}
-
-impl DiskError {
-    /// hack for ownership challenges with the disk driver
-    pub(crate) fn steal(&mut self) -> Self {
-        let mut swapped = DiskError::Stolen;
-        std::mem::swap(self, &mut swapped);
-        swapped
-    }
 }
 
 /// Builder-style disk store setup
 #[derive(Debug, Clone)]
 pub struct DiskBuilder {
-    /// Database in-memory cache allowance
-    ///
-    /// Default: 32 MiB
-    pub cache_size_mb: usize,
     /// Database stored block size limit
     ///
     /// Default: 10 GiB
···
 impl Default for DiskBuilder {
     fn default() -> Self {
         Self {
-            cache_size_mb: 32,
             max_stored_mb: 10 * 1024, // 10 GiB
         }
     }
···
     pub fn new() -> Self {
         Default::default()
     }
-    /// Set the in-memory cache allowance for the database
-    ///
-    /// Default: 32 MiB
-    pub fn with_cache_size_mb(mut self, size: usize) -> Self {
-        self.cache_size_mb = size;
-        self
-    }
     /// Set the approximate stored block size limit
     ///
     /// Default: 10 GiB
···
         self
     }
     /// Open and initialize the actual disk storage
-    pub async fn open(&self, path: PathBuf) -> Result<DiskStore, DiskError> {
-        DiskStore::new(path, self.cache_size_mb, self.max_stored_mb).await
+    pub async fn open(&self, path: PathBuf, keys_hint: Option<usize>) -> Result<DiskStore, DiskError> {
+        DiskStore::new(path, self.max_stored_mb, keys_hint).await
     }
 }
 
 /// On-disk block storage
 pub struct DiskStore {
-    conn: rusqlite::Connection,
+    db: CandyStore,
     max_stored: usize,
     stored: usize,
 }
 
 impl DiskStore {
     /// Initialize a new disk store
-    pub async fn new(
-        path: PathBuf,
-        cache_mb: usize,
-        max_stored_mb: usize,
-    ) -> Result<Self, DiskError> {
+    pub async fn new(path: PathBuf, max_stored_mb: usize, keys_hint: Option<usize>) -> Result<Self, DiskError> {
         let max_stored = max_stored_mb * 2_usize.pow(20);
-        let conn = tokio::task::spawn_blocking(move || {
-            let conn = rusqlite::Connection::open(path)?;
-
-            let sqlite_one_mb = -(2_i64.pow(10)); // negative is kibibytes for sqlite cache_size
-
-            // conn.pragma_update(None, "journal_mode", "OFF")?;
-            // conn.pragma_update(None, "journal_mode", "MEMORY")?;
-            conn.pragma_update(None, "journal_mode", "WAL")?;
-            // conn.pragma_update(None, "wal_autocheckpoint", "0")?; // this lets things get a bit big on disk
-            conn.pragma_update(None, "synchronous", "OFF")?;
-            conn.pragma_update(
-                None,
-                "cache_size",
-                (cache_mb as i64 * sqlite_one_mb).to_string(),
-            )?;
-            Self::reset_tables(&conn)?;
+        let db = tokio::task::spawn_blocking(move || {
+            let mut conf = Config::default();
+            // conf.max_shard_size = 256 * 1024 * 1024;
+            // conf.min_compaction_threashold = 32 * 1024 * 1024;
+            // conf.expected_number_of_keys = 1_200_000;
+            if let Some(hint) = keys_hint {
+                conf.expected_number_of_keys = hint;
+            }
+            conf.num_compaction_threads = 1;
+            let db = CandyStore::open(path, conf).map_err(|e| {
+                log::error!("{e:?}");
+                DiskError::DbGarbageError
+            })?;
 
-            Ok::<_, DiskError>(conn)
+            Ok::<_, DiskError>(db)
         })
         .await??;
 
         Ok(Self {
-            conn,
+            db,
             max_stored,
             stored: 0,
         })
     }
-    pub(crate) fn get_writer(&'_ mut self) -> Result<SqliteWriter<'_>, DiskError> {
-        let tx = self.conn.transaction()?;
-        Ok(SqliteWriter {
-            tx,
-            stored: &mut self.stored,
-            max: self.max_stored,
-        })
-    }
-    pub(crate) fn get_reader<'conn>(&'conn self) -> Result<SqliteReader<'conn>, DiskError> {
-        let select_stmt = self.conn.prepare("SELECT val FROM blocks WHERE key = ?1")?;
-        Ok(SqliteReader { select_stmt })
-    }
+
     /// Drop and recreate the kv table
     pub async fn reset(self) -> Result<Self, DiskError> {
         tokio::task::spawn_blocking(move || {
-            Self::reset_tables(&self.conn)?;
+            Self::reset_tables(&self.db)?;
             Ok(self)
         })
         .await?
     }
-    fn reset_tables(conn: &rusqlite::Connection) -> Result<(), DiskError> {
-        conn.execute("DROP TABLE IF EXISTS blocks", ())?;
-        conn.execute(
-            "CREATE TABLE blocks (
-                key  BLOB PRIMARY KEY NOT NULL,
-                val  BLOB NOT NULL
-            ) WITHOUT ROWID",
-            (),
-        )?;
+    fn reset_tables(db: &CandyStore) -> Result<(), DiskError> {
+        db.clear().map_err(|e| {
+            log::error!("{e:?}");
+            DiskError::DbGarbageError
+        })?;
+
         Ok(())
     }
-}
 
-pub(crate) struct SqliteWriter<'conn> {
-    tx: rusqlite::Transaction<'conn>,
-    stored: &'conn mut usize,
-    max: usize,
-}
-
-impl SqliteWriter<'_> {
     pub(crate) fn put_many(
         &mut self,
         kv: impl Iterator<Item = Result<(Vec<u8>, Vec<u8>), DriveError>>,
     ) -> Result<(), DriveError> {
-        let mut insert_stmt = self
-            .tx
-            .prepare_cached("INSERT INTO blocks (key, val) VALUES (?1, ?2)")
-            .map_err(DiskError::DbError)?;
         for pair in kv {
             let (k, v) = pair?;
-            *self.stored += v.len();
-            if *self.stored > self.max {
+            self.stored += v.len();
+            if self.stored > self.max_stored {
                 return Err(DiskError::MaxSizeExceeded.into());
             }
-            insert_stmt.execute((k, v)).map_err(DiskError::DbError)?;
+            self.db.owned_set(k, &v).map_err(|e| {
+                log::error!("{e:?}");
+                DiskError::DbGarbageError
+            })?;
         }
         Ok(())
     }
-    pub fn commit(self) -> Result<(), DiskError> {
-        self.tx.commit()?;
-        Ok(())
-    }
-}
-
-pub(crate) struct SqliteReader<'conn> {
-    select_stmt: rusqlite::Statement<'conn>,
-}
-
-impl SqliteReader<'_> {
-    pub(crate) fn get(&mut self, key: Vec<u8>) -> rusqlite::Result<Option<Vec<u8>>> {
-        self.select_stmt
-            .query_one((&key,), |row| row.get(0))
-            .optional()
+    pub(crate) fn get(&mut self, key: Vec<u8>) -> Result<Option<Vec<u8>>, DiskError> {
+        self.db.owned_get(key).map_err(|e| {
+            log::error!("{e:?}");
+            DiskError::DbGarbageError
+        })
     }
 }

src/drive.rs

···
     pub commit: Option<Commit>,
 }
 
+impl<R: AsyncRead + Unpin, T: Processable> NeedDisk<R, T> {
+    pub fn keys_hint(&self) -> usize {
+        self.mem_blocks.len() * 10
+    }
+}
+
 fn encode(v: impl Serialize) -> Result<Vec<u8>, bincode::error::EncodeError> {
     bincode::serde::encode_to_vec(v, bincode::config::standard())
 }
···
         // move store in and back out so we can manage lifetimes
         // dump mem blocks into the store
         store = tokio::task::spawn(async move {
-            let mut writer = store.get_writer()?;
-
             let kvs = self
                 .mem_blocks
                 .into_iter()
                 .map(|(k, v)| Ok(encode(v).map(|v| (k.to_bytes(), v))?));
 
-            writer.put_many(kvs)?;
-            writer.commit()?;
+            store.put_many(kvs)?;
             Ok::<_, DriveError>(store)
         })
         .await??;
···
         let (tx, mut rx) = mpsc::channel::<Vec<(Cid, MaybeProcessedBlock<T>)>>(1);
 
         let store_worker = tokio::task::spawn_blocking(move || {
-            let mut writer = store.get_writer()?;
-
             while let Some(chunk) = rx.blocking_recv() {
                 let kvs = chunk
                     .into_iter()
                     .map(|(k, v)| Ok(encode(v).map(|v| (k.to_bytes(), v))?));
-                writer.put_many(kvs)?;
+                store.put_many(kvs)?;
             }
 
-            writer.commit()?;
             Ok::<_, DriveError>(store)
         }); // await later
 
···
         // comes out again.
         let (state, res) = tokio::task::spawn_blocking(
             move || -> (BigState, Result<BlockChunk<T>, DriveError>) {
-                let mut reader_res = state.store.get_reader();
-                let reader: &mut _ = match reader_res {
-                    Ok(ref mut r) => r,
-                    Err(ref mut e) => {
-                        // unfortunately we can't return the error directly because
-                        // (for some reason) it's attached to the lifetime of the
-                        // reader?
-                        // hack a mem::swap so we can get it out :/
-                        let e_swapped = e.steal();
-                        // the pain: `state` *has to* outlive the reader
-                        drop(reader_res);
-                        return (state, Err(e_swapped.into()));
-                    }
-                };
-
                 let mut out = Vec::with_capacity(n);
 
                 for _ in 0..n {
                     // walk as far as we can until we run out of blocks or find a record
-                    let step = match state.walker.disk_step(reader, process) {
+                    let step = match state.walker.disk_step(&mut state.store, process) {
                         Ok(s) => s,
                         Err(e) => {
-                            // the pain: `state` *has to* outlive the reader
-                            drop(reader_res);
                             return (state, Err(e.into()));
                         }
                     };
                     match step {
                         Step::Missing(cid) => {
-                            // the pain: `state` *has to* outlive the reader
-                            drop(reader_res);
                             return (state, Err(DriveError::MissingBlock(cid)));
                         }
                         Step::Finish => break,
                         Step::Found { rkey, data } => out.push((rkey, data)),
                     };
                 }
-
-                // `state` *has to* outlive the reader
-                drop(reader_res);
 
                 (state, Ok::<_, DriveError>(out))
             },
···
         tx: mpsc::Sender<Result<BlockChunk<T>, DriveError>>,
     ) -> Result<(), mpsc::error::SendError<Result<BlockChunk<T>, DriveError>>> {
         let BigState { store, walker } = self.state.as_mut().expect("valid state");
-        let mut reader = match store.get_reader() {
-            Ok(r) => r,
-            Err(e) => return tx.blocking_send(Err(e.into())),
-        };
 
         loop {
             let mut out: BlockChunk<T> = Vec::with_capacity(n);
···
             for _ in 0..n {
                 // walk as far as we can until we run out of blocks or find a record
 
-                let step = match walker.disk_step(&mut reader, self.process) {
+                let step = match walker.disk_step(store, self.process) {
                     Ok(s) => s,
                     Err(e) => return tx.blocking_send(Err(e.into())),
                 };

src/walk.rs

···
 //! Depth-first MST traversal
 
-use crate::disk::SqliteReader;
+use crate::DiskError;
+use crate::DiskStore;
 use crate::drive::{DecodeError, MaybeProcessedBlock};
 use crate::mst::Node;
 use crate::process::Processable;
···
     #[error("Action node error: {0}")]
     MstError(#[from] MstError),
     #[error("storage error: {0}")]
-    StorageError(#[from] rusqlite::Error),
+    StorageError(#[from] DiskError),
     #[error("Decode error: {0}")]
     DecodeError(#[from] DecodeError),
 }
···
     /// blocking!!!!!!
     pub fn disk_step<T: Processable>(
         &mut self,
-        reader: &mut SqliteReader,
+        db: &mut DiskStore,
         process: impl Fn(Vec<u8>) -> T,
     ) -> Result<Step<T>, WalkError> {
         loop {
···
                 &mut Need::Node { depth, cid } => {
                     let cid_bytes = cid.to_bytes();
                     log::trace!("need node {cid:?}");
-                    let Some(block_bytes) = reader.get(cid_bytes)? else {
+                    let Some(block_bytes) = db.get(cid_bytes)? else {
                         log::trace!("node not found, resting");
                         return Ok(Step::Missing(cid));
                     };
···
                 Need::Record { rkey, cid } => {
                     log::trace!("need record {cid:?}");
                     let cid_bytes = cid.to_bytes();
-                    let Some(data_bytes) = reader.get(cid_bytes)? else {
+                    let Some(data_bytes) = db.get(cid_bytes)? else {
                         log::trace!("record block not found, resting");
                         return Ok(Step::Missing(*cid));
                     };