throw in sqlite

just slightly slower than redb atm

Cargo.lock

···
 ]
 
 [[package]]
+name = "errno"
+version = "0.3.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
+dependencies = [
+ "libc",
+ "windows-sys 0.60.2",
+]
+
+[[package]]
 name = "fallible-iterator"
 version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 version = "0.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
+
+[[package]]
+name = "fastrand"
+version = "2.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
 
 [[package]]
 name = "foldhash"
···
 ]
 
 [[package]]
+name = "getrandom"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "r-efi",
+ "wasi 0.14.7+wasi-0.2.4",
+]
+
+[[package]]
 name = "gimli"
 version = "0.32.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 ]
 
 [[package]]
+name = "linux-raw-sys"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039"
+
+[[package]]
 name = "lock_api"
 version = "0.4.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c"
 dependencies = [
  "libc",
- "wasi",
+ "wasi 0.11.1+wasi-snapshot-preview1",
  "windows-sys 0.59.0",
 ]
 
···
 ]
 
 [[package]]
+name = "r-efi"
+version = "5.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
+
+[[package]]
 name = "rayon"
 version = "1.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
  "serde",
  "serde_bytes",
  "serde_ipld_dagcbor",
+ "tempfile",
  "thiserror 2.0.17",
  "tokio",
 ]
···
 checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace"
 
 [[package]]
+name = "rustix"
+version = "1.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e"
+dependencies = [
+ "bitflags",
+ "errno",
+ "libc",
+ "linux-raw-sys",
+ "windows-sys 0.60.2",
+]
+
+[[package]]
 name = "rustversion"
 version = "1.0.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 ]
 
 [[package]]
+name = "tempfile"
+version = "3.23.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16"
+dependencies = [
+ "fastrand",
+ "getrandom",
+ "once_cell",
+ "rustix",
+ "windows-sys 0.60.2",
+]
+
+[[package]]
 name = "thiserror"
 version = "1.0.69"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
 
 [[package]]
+name = "wasi"
+version = "0.14.7+wasi-0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c"
+dependencies = [
+ "wasip2",
+]
+
+[[package]]
+name = "wasip2"
+version = "1.0.1+wasi-0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7"
+dependencies = [
+ "wit-bindgen",
+]
+
+[[package]]
 name = "wasm-bindgen"
 version = "0.2.104"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 version = "0.53.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650"
+
+[[package]]
+name = "wit-bindgen"
+version = "0.46.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59"
 
 [[package]]
 name = "zerocopy"

Cargo.toml

···
 criterion = { version = "0.7.0", features = ["async_tokio"] }
 env_logger = "0.11.8"
 multibase = "0.9.2"
+tempfile = "3.23.0"
 tokio = { version = "1.47.1", features = ["full"] }
 
 [profile.profiling]

src/disk_drive.rs

···
     P: Fn(&[u8]) -> Result<T, PE>,
     PE: Error,
 {
+    #[allow(dead_code)]
     block_stream: S,
     block_store: BS,
     walker: Walker,
···
     ) -> Result<(Commit, Self), DriveError> {
         let mut commit = None;
 
+        log::warn!("init: load blocks");
+
+        // go ahead and put all blocks in the block store
         while let Some((cid, data)) = block_stream
             .try_next()
             .await
···
                 let c: Commit = serde_ipld_dagcbor::from_slice(&data)
                     .map_err(|e| DriveError::BadCommit(e.into()))?;
                 commit = Some(c);
-                break;
             } else {
                 block_store.put(
                     cid,
···
                 );
             }
         }
+
+        log::warn!("init: got commit?");
 
         // we either broke out or read all the blocks without finding the commit...
         let commit = commit.ok_or(DriveError::MissingCommit)?;
 
         let walker = Walker::new(commit.data);
 
+        log::warn!("init: wrapping up");
+
         let me = Self {
             block_stream,
             block_store,
···
         Ok((commit, me))
     }
 
-    async fn drive_until(&mut self, cid_needed: Cid) -> Result<(), DriveError> {
-        while let Some((cid, data)) = self
-            .block_stream
-            .try_next()
-            .await
-            .map_err(|e| DriveError::CarBlockError(e.into()))?
-        {
-            self.block_store.put(
-                cid,
-                if Node::could_be(&data) {
-                    MaybeProcessedBlock::Raw(data)
-                } else {
-                    match (self.process)(&data) {
-                        Ok(t) => MaybeProcessedBlock::ProcessedOk(t),
-                        Err(e) => MaybeProcessedBlock::Unprocessable(e.to_string()),
-                    }
-                },
-            );
-            if cid == cid_needed {
-                return Ok(());
-            }
-        }
-
-        // if we never found the block
-        Err(DriveError::MissingBlock(cid_needed))
-    }
-
     /// Manually step through the record outputs
     pub async fn next_record(&mut self) -> Result<Option<(String, T)>, DriveError> {
-        loop {
-            // walk as far as we can until we run out of blocks or find a record
-            let cid_needed = match self.walker.step(&mut self.block_store, &self.process)? {
-                Step::Rest(cid) => cid,
-                Step::Finish => return Ok(None),
-                Step::Step { rkey, data } => return Ok(Some((rkey, data))),
-            };
-
-            // load blocks until we reach that cid
-            self.drive_until(cid_needed).await?;
+        match self.walker.step(&mut self.block_store, &self.process)? {
+            Step::Rest(cid) => Err(DriveError::MissingBlock(cid)),
+            Step::Finish => Ok(None),
+            Step::Step { rkey, data } => Ok(Some((rkey, data))),
         }
     }
 

src/disk_redb.rs

···
 
 impl RedbStore {
     pub fn new(path: impl AsRef<Path>) -> Result<Self, Error> {
+        log::warn!("redb new");
         let db = Database::create(path)?;
+        log::warn!("db created");
         Ok(Self { db })
     }
 }
···
     fn get(&self, c: Cid) -> Option<MPB> {
         let key_bytes = c.to_bytes();
         let tx = self.db.begin_read().unwrap();
-        let table = match tx.open_table(TABLE) {
-            Ok(t) => t,
-            Err(redb::TableError::TableDoesNotExist(_)) => return None,
-            e => e.unwrap(),
-        };
+        let table = tx.open_table(TABLE).unwrap();
         let maybe_val_bytes = table.get(&*key_bytes).unwrap()?;
         let (t, n): (MPB, usize) =
             bincode::serde::decode_from_slice(maybe_val_bytes.value(), bincode::config::standard())

src/disk_sqlite.rs

···
+use crate::disk_drive::BlockStore;
+use ipld_core::cid::Cid;
+use rusqlite::{Connection, OptionalExtension, Result};
+use serde::{Serialize, de::DeserializeOwned};
+use std::path::Path;
+
+pub struct SqliteStore {
+    conn: Connection,
+}
+
+impl SqliteStore {
+    pub fn new(path: impl AsRef<Path>) -> Result<Self> {
+        let conn = Connection::open(path)?;
+        conn.pragma_update(None, "journal_mode", "WAL")?;
+        conn.pragma_update(None, "synchronous", "OFF")?;
+        conn.pragma_update(None, "cache_size", (-32 * 2_i64.pow(10)).to_string())?;
+        conn.execute(
+            "CREATE TABLE blocks (
+                key  BLOB PRIMARY KEY NOT NULL,
+                val  BLOB NOT NULL
+            ) WITHOUT ROWID",
+            (),
+        )?;
+
+        Ok(Self { conn })
+    }
+}
+
+impl Drop for SqliteStore {
+    fn drop(&mut self) {
+        self.conn.execute("DROP TABLE blocks", ()).unwrap();
+    }
+}
+
+impl<MPB: Serialize + DeserializeOwned> BlockStore<MPB> for SqliteStore {
+    fn put(&self, c: Cid, t: MPB) {
+        let key_bytes = c.to_bytes();
+        let val_bytes = bincode::serde::encode_to_vec(t, bincode::config::standard()).unwrap();
+
+        self.conn
+            .execute(
+                "INSERT INTO blocks (key, val) VALUES (?1, ?2)",
+                (&key_bytes, &val_bytes),
+            )
+            .unwrap();
+    }
+    fn get(&self, c: Cid) -> Option<MPB> {
+        let key_bytes = c.to_bytes();
+
+        let val_bytes: Vec<u8> = self
+            .conn
+            .query_one(
+                "SELECT val FROM blocks WHERE key = ?1",
+                (&key_bytes,),
+                |row| row.get(0),
+            )
+            .optional()
+            .unwrap()?;
+
+        let (t, n): (MPB, usize) =
+            bincode::serde::decode_from_slice(&val_bytes, bincode::config::standard()).unwrap();
+        assert_eq!(val_bytes.len(), n);
+        Some(t)
+    }
+}

src/lib.rs

···
 
 pub mod disk_drive;
 pub mod disk_redb;
+pub mod disk_sqlite;
 pub mod disk_walk;
 pub mod drive;
 pub mod mst;