sqlite -> fjall 2 for ~2x speedup

Cargo.lock

···
 checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43"
 
 [[package]]
-name = "byteorder-lite"
-version = "0.1.0"
+name = "byteorder"
+version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495"
+checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
 
 [[package]]
 name = "bytes"
···
 
 [[package]]
 name = "byteview"
-version = "0.10.0"
+version = "0.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dda4398f387cc6395a3e93b3867cd9abda914c97a0b344d1eefb2e5c51785fca"
+checksum = "6236364b88b9b6d0bc181ba374cf1ab55ba3ef97a1cb6f8cddad48a273767fb5"
 
 [[package]]
 name = "cast"
···
 ]
 
 [[package]]
+name = "double-ended-peekable"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c0d05e1c0dbad51b52c38bda7adceef61b9efc2baf04acfe8726a8c4630a6f57"
+
+[[package]]
 name = "either"
 version = "1.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 ]
 
 [[package]]
-name = "fallible-iterator"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649"
-
-[[package]]
-name = "fallible-streaming-iterator"
-version = "0.1.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
-
-[[package]]
 name = "fastrand"
 version = "2.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 
 [[package]]
 name = "fjall"
-version = "3.0.0"
+version = "2.11.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4986f550347ed1666561f36e8bf1be3c97df72850ecef0140129da6e2d0aa911"
+checksum = "0b25ad44cd4360a0448a9b5a0a6f1c7a621101cca4578706d43c9a821418aebc"
 dependencies = [
- "byteorder-lite",
+ "byteorder",
  "byteview",
  "dashmap",
- "flume",
  "log",
  "lsm-tree",
- "lz4_flex",
+ "path-absolutize",
+ "std-semaphore",
  "tempfile",
  "xxhash-rust",
 ]
 
 [[package]]
-name = "flume"
-version = "0.12.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5e139bc46ca777eb5efaf62df0ab8cc5fd400866427e56c68b22e414e53bd3be"
-dependencies = [
- "spin",
-]
-
-[[package]]
-name = "foldhash"
-version = "0.1.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
-
-[[package]]
 name = "futures"
 version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7"
 
 [[package]]
+name = "guardian"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "17e2ac29387b1aa07a1e448f7bb4f35b500787971e965b02842b900afa5c8f6f"
+
+[[package]]
 name = "half"
 version = "2.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 
 [[package]]
 name = "hashbrown"
-version = "0.15.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
-dependencies = [
- "foldhash",
-]
-
-[[package]]
-name = "hashbrown"
 version = "0.16.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
 
 [[package]]
-name = "hashlink"
-version = "0.10.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1"
-dependencies = [
- "hashbrown 0.15.5",
-]
-
-[[package]]
 name = "heck"
 version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 checksum = "58f929b4d672ea937a23a1ab494143d968337a5f47e56d0815df1e0890ddf174"
 
 [[package]]
-name = "libsqlite3-sys"
-version = "0.35.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "133c182a6a2c87864fe97778797e46c7e999672690dc9fa3ee8e241aa4a9c13f"
-dependencies = [
- "pkg-config",
- "vcpkg",
-]
-
-[[package]]
 name = "linux-raw-sys"
 version = "0.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 
 [[package]]
 name = "lsm-tree"
-version = "3.0.0"
+version = "2.10.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3a206e87e8bc38114045060ec1fc6bc4e4559748a37e9622b910d80e48863e87"
+checksum = "799399117a2bfb37660e08be33f470958babb98386b04185288d829df362ea15"
 dependencies = [
- "byteorder-lite",
- "byteview",
+ "byteorder",
  "crossbeam-skiplist",
+ "double-ended-peekable",
  "enum_dispatch",
+ "guardian",
  "interval-heap",
  "log",
- "lz4_flex",
+ "path-absolutize",
  "quick_cache",
  "rustc-hash",
  "self_cell",
- "sfa",
  "tempfile",
+ "value-log",
  "varint-rs",
  "xxhash-rust",
-]
-
-[[package]]
-name = "lz4_flex"
-version = "0.11.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08ab2867e3eeeca90e844d1940eab391c9dc5228783db2ed999acbc0a9ed375a"
-dependencies = [
- "twox-hash",
 ]
 
 [[package]]
···
 ]
 
 [[package]]
+name = "path-absolutize"
+version = "3.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e4af381fe79fa195b4909485d99f73a80792331df0625188e707854f0b3383f5"
+dependencies = [
+ "path-dedot",
+]
+
+[[package]]
+name = "path-dedot"
+version = "3.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "07ba0ad7e047712414213ff67533e6dd477af0a4e1d14fb52343e53d30ea9397"
+dependencies = [
+ "once_cell",
+]
+
+[[package]]
 name = "pin-project-lite"
 version = "0.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
-
-[[package]]
-name = "pkg-config"
-version = "0.3.32"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"
 
 [[package]]
 name = "plotters"
···
  "iroh-car",
  "log",
  "multibase",
- "rusqlite",
  "serde",
  "serde_bytes",
  "serde_ipld_dagcbor",
···
  "tempfile",
  "thiserror 2.0.17",
  "tokio",
-]
-
-[[package]]
-name = "rusqlite"
-version = "0.37.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "165ca6e57b20e1351573e3729b958bc62f0e48025386970b6e4d29e7a7e71f3f"
-dependencies = [
- "bitflags",
- "fallible-iterator",
- "fallible-streaming-iterator",
- "hashlink",
- "libsqlite3-sys",
- "smallvec",
 ]
 
 [[package]]
···
 ]
 
 [[package]]
-name = "sfa"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1296838937cab56cd6c4eeeb8718ec777383700c33f060e2869867bd01d1175"
-dependencies = [
- "byteorder-lite",
- "log",
- "xxhash-rust",
-]
-
-[[package]]
 name = "sha2"
 version = "0.10.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 ]
 
 [[package]]
-name = "spin"
-version = "0.9.8"
+name = "std-semaphore"
+version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
-dependencies = [
- "lock_api",
-]
+checksum = "33ae9eec00137a8eed469fb4148acd9fc6ac8c3f9b110f52cd34698c8b5bfa0e"
 
 [[package]]
 name = "strsim"
···
 ]
 
 [[package]]
-name = "twox-hash"
-version = "2.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c"
-
-[[package]]
 name = "typenum"
 version = "1.19.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
 
 [[package]]
+name = "value-log"
+version = "1.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "62fc7c4ce161f049607ecea654dca3f2d727da5371ae85e2e4f14ce2b98ed67c"
+dependencies = [
+ "byteorder",
+ "byteview",
+ "interval-heap",
+ "log",
+ "path-absolutize",
+ "rustc-hash",
+ "tempfile",
+ "varint-rs",
+ "xxhash-rust",
+]
+
+[[package]]
 name = "varint-rs"
 version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8f54a172d0620933a27a4360d3db3e2ae0dd6cceae9730751a036bbf182c4b23"
-
-[[package]]
-name = "vcpkg"
-version = "0.2.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
 
 [[package]]
 name = "version_check"

Cargo.toml

···
 
 [dependencies]
 bincode = { version = "2.0.1", features = ["serde"] }
-fjall = "3.0.0"
+fjall = { version = "2.11.2", default-features = false }
 futures = "0.3.31"
 futures-core = "0.3.31"
 ipld-core = { version = "0.4.2", features = ["serde"] }
 iroh-car = "0.5.1"
 log = "0.4.28"
 multibase = "0.9.2"
-rusqlite = "0.37.0"
 serde = { version = "1.0.228", features = ["derive"] }
 serde_bytes = "0.11.19"
 serde_ipld_dagcbor = "0.6.4"

examples/disk-read-file/main.rs

···
     // in this example we only bother handling CARs that are too big for memory
     // `noop` helper means: do no block processing, store the raw blocks
     let driver = match DriverBuilder::new()
-        .with_mem_limit_mb(10) // how much memory can be used before disk spill
+        .with_mem_limit_mb(32) // how much memory can be used before disk spill
         .load_car(reader)
         .await?
     {
···
     }
 
     log::info!("arrived! ({:?}) joining rx...", t0.elapsed());
+
+    let driver = join.await?;
+
+    driver.reset_store().await?;
 
     log::info!("done. n={n} zeros={zeros}");
 

src/disk.rs

···
 */
 
 use crate::drive::DriveError;
-use fjall::{Database, Keyspace, KeyspaceCreateOptions, Error as FjallError};
+use fjall::{Config, Error as FjallError, Keyspace, Partition, PartitionCreateOptions};
 use std::path::PathBuf;
 
 #[derive(Debug, thiserror::Error)]
···
     /// limit.
     #[error("Maximum disk size reached")]
     MaxSizeExceeded,
-    #[error("this error was replaced, seeing this is a bug.")]
-    #[doc(hidden)]
-    Stolen,
-}
-
-impl DiskError {
-    /// hack for ownership challenges with the disk driver
-    pub(crate) fn steal(&mut self) -> Self {
-        let mut swapped = DiskError::Stolen;
-        std::mem::swap(self, &mut swapped);
-        swapped
-    }
 }
 
 /// Builder-style disk store setup
···
 /// On-disk block storage
 pub struct DiskStore {
     #[allow(unused)]
-    db: Database,
-    ks: Keyspace,
+    db: Keyspace,
+    partition: Partition,
     max_stored: usize,
     stored: usize,
 }
···
         max_stored_mb: usize,
     ) -> Result<Self, DiskError> {
         let max_stored = max_stored_mb * 2_usize.pow(20);
-        let (db, ks) = tokio::task::spawn_blocking(move || {
-            let db = Database::builder(path)
+        let (db, partition) = tokio::task::spawn_blocking(move || {
+            let db = Config::new(path)
                 // .manual_journal_persist(true)
-                // .worker_threads(1)
-                // .cache_size(cache_mb as u64 * 2_u64.pow(20))
-                // .temporary(true)
+                // .flush_workers(1)
+                // .compaction_workers(1)
+                .cache_size(cache_mb as u64 * 2_u64.pow(20))
+                .temporary(true)
                 .open()?;
-            let ks = db.keyspace("z", ||
-                KeyspaceCreateOptions::default()
-                    // .expect_point_read_hits(true)
-                    // .manual_journal_persist(true)
-            )?;
+            let partition = Self::get_partition(&db)?;
 
-            // Self::reset_tables(&ks)?;
-
-            Ok::<_, DiskError>((db, ks))
+            Ok::<_, DiskError>((db, partition))
         })
         .await??;
 
         Ok(Self {
             db,
-            ks,
+            partition,
             max_stored,
             stored: 0,
         })
     }
-    pub(crate) fn get_writer(&'_ mut self) -> Result<SqliteWriter<'_>, DiskError> {
-        Ok(SqliteWriter {
-            ks: self.ks.clone(),
-            stored: &mut self.stored,
-            max: self.max_stored,
-        })
-    }
-    pub(crate) fn get_reader(&self) -> Result<SqliteReader, DiskError> {
-        Ok(SqliteReader {
-            ks: self.ks.clone(),
-        })
-    }
+
     /// Drop and recreate the kv table
-    pub async fn reset(self) -> Result<Self, DiskError> {
+    pub async fn reset(mut self) -> Result<Self, DiskError> {
         tokio::task::spawn_blocking(move || {
-            Self::reset_tables(&self.ks)?;
+            let partition = self.partition;
+            Self::reset_partition(&self.db, partition)?;
+            self.partition = Self::get_partition(&self.db)?;
             Ok(self)
         })
         .await?
     }
-    fn reset_tables(ks: &Keyspace) -> Result<(), DiskError> {
-        ks.clear()?;
-        Ok(())
-    }
-}
 
-pub(crate) struct SqliteWriter<'a> {
-    ks: Keyspace,
-    stored: &'a mut usize,
-    max: usize,
-}
-
-impl SqliteWriter<'_> {
+    fn get_partition(db: &Keyspace) -> Result<Partition, FjallError> {
+        db.open_partition("z", PartitionCreateOptions::default())
+    }
+    fn reset_partition(keyspace: &Keyspace, partition: Partition) -> Result<Partition, DiskError> {
+        keyspace.delete_partition(partition)?;
+        let partition = Self::get_partition(keyspace)?;
+        Ok(partition)
+    }
     pub(crate) fn put_many(
         &mut self,
         kv: impl Iterator<Item = Result<(Vec<u8>, Vec<u8>), DriveError>>,
     ) -> Result<(), DriveError> {
+        let mut batch = self.db.batch();
         for pair in kv {
             let (k, v) = pair?;
-            *self.stored += v.len();
-            if *self.stored > self.max {
+            self.stored += v.len();
+            if self.stored > self.max_stored {
                 return Err(DiskError::MaxSizeExceeded.into());
             }
-            self.ks.insert(k, v).map_err(DiskError::DbError)?;
+            batch.insert(&self.partition, k, v);
         }
+        batch.commit().map_err(DiskError::DbError)?;
         Ok(())
     }
-}
 
-pub(crate) struct SqliteReader {
-    ks: Keyspace,
-}
-
-impl SqliteReader {
-    pub(crate) fn get(&mut self, key: Vec<u8>) -> Result<Option<Vec<u8>>, FjallError> {
-        let rv = self
-            .ks
-            .get(&key)?
-            .map(|v| v.as_ref().into());
-        Ok(rv)
+    #[inline]
+    pub(crate) fn get(&mut self, key: &[u8]) -> Result<Option<fjall::Slice>, FjallError> {
+        self.partition.get(key)
     }
 }

src/drive.rs

···
         // move store in and back out so we can manage lifetimes
         // dump mem blocks into the store
         store = tokio::task::spawn(async move {
-            let mut writer = store.get_writer()?;
-
             let kvs = self
                 .mem_blocks
                 .into_iter()
                 .map(|(k, v)| Ok(encode(v).map(|v| (k.to_bytes(), v))?));
 
-            writer.put_many(kvs)?;
+            store.put_many(kvs)?;
             Ok::<_, DriveError>(store)
         })
         .await??;
···
         let (tx, mut rx) = mpsc::channel::<Vec<(Cid, MaybeProcessedBlock<T>)>>(1);
 
         let store_worker = tokio::task::spawn_blocking(move || {
-            let mut writer = store.get_writer()?;
-
             while let Some(chunk) = rx.blocking_recv() {
                 let kvs = chunk
                     .into_iter()
                     .map(|(k, v)| Ok(encode(v).map(|v| (k.to_bytes(), v))?));
-                writer.put_many(kvs)?;
+                store.put_many(kvs)?;
             }
             Ok::<_, DriveError>(store)
         }); // await later
···
         // comes out again.
         let (state, res) = tokio::task::spawn_blocking(
             move || -> (BigState, Result<BlockChunk<T>, DriveError>) {
-                let mut reader_res = state.store.get_reader();
-                let reader: &mut _ = match reader_res {
-                    Ok(ref mut r) => r,
-                    Err(ref mut e) => {
-                        // unfortunately we can't return the error directly because
-                        // (for some reason) it's attached to the lifetime of the
-                        // reader?
-                        // hack a mem::swap so we can get it out :/
-                        let e_swapped = e.steal();
-                        // the pain: `state` *has to* outlive the reader
-                        drop(reader_res);
-                        return (state, Err(e_swapped.into()));
-                    }
-                };
-
                 let mut out = Vec::with_capacity(n);
 
                 for _ in 0..n {
                     // walk as far as we can until we run out of blocks or find a record
-                    let step = match state.walker.disk_step(reader, process) {
+                    let step = match state.walker.disk_step(&mut state.store, process) {
                         Ok(s) => s,
                         Err(e) => {
-                            // the pain: `state` *has to* outlive the reader
-                            drop(reader_res);
                             return (state, Err(e.into()));
                         }
                     };
                     match step {
                         Step::Missing(cid) => {
-                            // the pain: `state` *has to* outlive the reader
-                            drop(reader_res);
                             return (state, Err(DriveError::MissingBlock(cid)));
                         }
                         Step::Finish => break,
···
                     };
                 }
 
-                // `state` *has to* outlive the reader
-                drop(reader_res);
-
                 (state, Ok::<_, DriveError>(out))
             },
         )
···
         tx: mpsc::Sender<Result<BlockChunk<T>, DriveError>>,
     ) -> Result<(), mpsc::error::SendError<Result<BlockChunk<T>, DriveError>>> {
         let BigState { store, walker } = self.state.as_mut().expect("valid state");
-        let mut reader = match store.get_reader() {
-            Ok(r) => r,
-            Err(e) => return tx.blocking_send(Err(e.into())),
-        };
 
         loop {
             let mut out: BlockChunk<T> = Vec::with_capacity(n);
···
             for _ in 0..n {
                 // walk as far as we can until we run out of blocks or find a record
 
-                let step = match walker.disk_step(&mut reader, self.process) {
+                let step = match walker.disk_step(store, self.process) {
                     Ok(s) => s,
                     Err(e) => return tx.blocking_send(Err(e.into())),
                 };

src/walk.rs

···
 //! Depth-first MST traversal
 
-use crate::disk::SqliteReader;
+use crate::disk::DiskStore;
 use crate::drive::{DecodeError, MaybeProcessedBlock};
 use crate::mst::Node;
 use crate::process::Processable;
···
     /// blocking!!!!!!
     pub fn disk_step<T: Processable>(
         &mut self,
-        reader: &mut SqliteReader,
+        reader: &mut DiskStore,
         process: impl Fn(Vec<u8>) -> T,
     ) -> Result<Step<T>, WalkError> {
         loop {
···
                 &mut Need::Node { depth, cid } => {
                     let cid_bytes = cid.to_bytes();
                     log::trace!("need node {cid:?}");
-                    let Some(block_bytes) = reader.get(cid_bytes)? else {
+                    let Some(block_bytes) = reader.get(&cid_bytes)? else {
                         log::trace!("node not found, resting");
                         return Ok(Step::Missing(cid));
                     };
···
                 Need::Record { rkey, cid } => {
                     log::trace!("need record {cid:?}");
                     let cid_bytes = cid.to_bytes();
-                    let Some(data_bytes) = reader.get(cid_bytes)? else {
+                    let Some(data_bytes) = reader.get(&cid_bytes)? else {
                         log::trace!("record block not found, resting");
                         return Ok(Step::Missing(*cid));
                     };