fjall v3

pretty good speedup!

Cargo.lock

···
 checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43"
 
 [[package]]
-name = "byteorder"
-version = "1.5.0"
+name = "byteorder-lite"
+version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
+checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495"
 
 [[package]]
 name = "bytes"
···
 
 [[package]]
 name = "byteview"
-version = "0.6.1"
+version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6236364b88b9b6d0bc181ba374cf1ab55ba3ef97a1cb6f8cddad48a273767fb5"
+checksum = "dda4398f387cc6395a3e93b3867cd9abda914c97a0b344d1eefb2e5c51785fca"
 
 [[package]]
 name = "cast"
···
 ]
 
 [[package]]
-name = "double-ended-peekable"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c0d05e1c0dbad51b52c38bda7adceef61b9efc2baf04acfe8726a8c4630a6f57"
-
-[[package]]
 name = "either"
 version = "1.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 
 [[package]]
 name = "fjall"
-version = "2.11.2"
+version = "3.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b25ad44cd4360a0448a9b5a0a6f1c7a621101cca4578706d43c9a821418aebc"
+checksum = "4986f550347ed1666561f36e8bf1be3c97df72850ecef0140129da6e2d0aa911"
 dependencies = [
- "byteorder",
+ "byteorder-lite",
  "byteview",
  "dashmap",
+ "flume",
  "log",
  "lsm-tree",
- "path-absolutize",
- "std-semaphore",
  "tempfile",
  "xxhash-rust",
 ]
 
 [[package]]
+name = "flume"
+version = "0.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5e139bc46ca777eb5efaf62df0ab8cc5fd400866427e56c68b22e414e53bd3be"
+dependencies = [
+ "spin",
+]
+
+[[package]]
 name = "futures"
 version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 version = "0.32.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7"
-
-[[package]]
-name = "guardian"
-version = "1.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "17e2ac29387b1aa07a1e448f7bb4f35b500787971e965b02842b900afa5c8f6f"
 
 [[package]]
 name = "half"
···
 
 [[package]]
 name = "lsm-tree"
-version = "2.10.4"
+version = "3.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "799399117a2bfb37660e08be33f470958babb98386b04185288d829df362ea15"
+checksum = "3a206e87e8bc38114045060ec1fc6bc4e4559748a37e9622b910d80e48863e87"
 dependencies = [
- "byteorder",
+ "byteorder-lite",
+ "byteview",
  "crossbeam-skiplist",
- "double-ended-peekable",
  "enum_dispatch",
- "guardian",
  "interval-heap",
  "log",
- "path-absolutize",
  "quick_cache",
  "rustc-hash",
  "self_cell",
+ "sfa",
  "tempfile",
- "value-log",
  "varint-rs",
  "xxhash-rust",
 ]
···
 ]
 
 [[package]]
-name = "path-absolutize"
-version = "3.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e4af381fe79fa195b4909485d99f73a80792331df0625188e707854f0b3383f5"
-dependencies = [
- "path-dedot",
-]
-
-[[package]]
-name = "path-dedot"
-version = "3.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "07ba0ad7e047712414213ff67533e6dd477af0a4e1d14fb52343e53d30ea9397"
-dependencies = [
- "once_cell",
-]
-
-[[package]]
 name = "pin-project-lite"
 version = "0.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 ]
 
 [[package]]
+name = "sfa"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1296838937cab56cd6c4eeeb8718ec777383700c33f060e2869867bd01d1175"
+dependencies = [
+ "byteorder-lite",
+ "log",
+ "xxhash-rust",
+]
+
+[[package]]
 name = "sha2"
 version = "0.10.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 ]
 
 [[package]]
-name = "std-semaphore"
-version = "0.1.0"
+name = "spin"
+version = "0.9.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "33ae9eec00137a8eed469fb4148acd9fc6ac8c3f9b110f52cd34698c8b5bfa0e"
+checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
+dependencies = [
+ "lock_api",
+]
 
 [[package]]
 name = "strsim"
···
 version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
-
-[[package]]
-name = "value-log"
-version = "1.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62fc7c4ce161f049607ecea654dca3f2d727da5371ae85e2e4f14ce2b98ed67c"
-dependencies = [
- "byteorder",
- "byteview",
- "interval-heap",
- "log",
- "path-absolutize",
- "rustc-hash",
- "tempfile",
- "varint-rs",
- "xxhash-rust",
-]
 
 [[package]]
 name = "varint-rs"

Cargo.toml

···
 
 [dependencies]
 bincode = { version = "2.0.1", features = ["serde"] }
-fjall = { version = "2.11.2", default-features = false }
+fjall = { version = "3.0.0", default-features = false }
 futures = "0.3.31"
 futures-core = "0.3.31"
 ipld-core = { version = "0.4.2", features = ["serde"] }

examples/disk-read-file/main.rs

···
 
     log::info!("arrived! ({:?}) joining rx...", t0.elapsed());
 
-    let driver = join.await?;
-
-    driver.reset_store().await?;
+    join.await?;
 
     log::info!("done. n={n} zeros={zeros}");
 

readme.md

···
                     total_size += size;
                 }
             }
-
-            // clean up the disk store (drop tables etc)
-            driver.reset_store().await?;
         }
     };
     println!("sum of size of all records: {total_size}");

src/disk.rs

···
 */
 
 use crate::drive::DriveError;
-use fjall::{Config, Error as FjallError, Keyspace, Partition, PartitionCreateOptions};
+use fjall::{Database, Error as FjallError, Keyspace, KeyspaceCreateOptions};
 use std::path::PathBuf;
 
 #[derive(Debug, thiserror::Error)]
···
 /// On-disk block storage
 pub struct DiskStore {
     #[allow(unused)]
-    db: Keyspace,
-    partition: Partition,
+    db: Database,
+    partition: Keyspace,
     max_stored: usize,
     stored: usize,
 }
···
     ) -> Result<Self, DiskError> {
         let max_stored = max_stored_mb * 2_usize.pow(20);
         let (db, partition) = tokio::task::spawn_blocking(move || {
-            let db = Config::new(path)
+            let db = Database::builder(path)
                 // .manual_journal_persist(true)
                 // .flush_workers(1)
                 // .compaction_workers(1)
                 .cache_size(cache_mb as u64 * 2_u64.pow(20))
                 .temporary(true)
                 .open()?;
-            let partition = Self::get_partition(&db)?;
+            let partition = db.keyspace("z", KeyspaceCreateOptions::default)?;
 
             Ok::<_, DiskError>((db, partition))
         })
···
         })
     }
 
-    /// Drop and recreate the kv table
-    pub async fn reset(mut self) -> Result<Self, DiskError> {
-        tokio::task::spawn_blocking(move || {
-            let partition = self.partition;
-            Self::reset_partition(&self.db, partition)?;
-            self.partition = Self::get_partition(&self.db)?;
-            Ok(self)
-        })
-        .await?
-    }
-
-    fn get_partition(db: &Keyspace) -> Result<Partition, FjallError> {
-        db.open_partition("z", PartitionCreateOptions::default())
-    }
-    fn reset_partition(keyspace: &Keyspace, partition: Partition) -> Result<Partition, DiskError> {
-        keyspace.delete_partition(partition)?;
-        let partition = Self::get_partition(keyspace)?;
-        Ok(partition)
-    }
     pub(crate) fn put_many(
         &mut self,
         kv: impl Iterator<Item = Result<(Vec<u8>, Vec<u8>), DriveError>>,

src/drive.rs

···
     ///         println!("{rkey}: size={}", record.len());
     ///     }
     /// }
-    /// let store = disk_driver.reset_store().await?;
     /// # Ok(())
     /// # }
     /// ```
···
     ///     }
     ///
     /// }
-    /// let store = join.await?.reset_store().await?;
     /// # Ok(())
     /// # }
     /// ```
···
         });
 
         (rx, chan_task)
-    }
-
-    /// Reset the disk storage so it can be reused. You must call this.
-    ///
-    /// Ideally we'd put this in an `impl Drop`, but since it makes blocking
-    /// calls, that would be risky in an async context. For now you just have to
-    /// carefully make sure you call it.
-    ///
-    /// The sqlite store is returned, so it can be reused for another
-    /// `DiskDriver`.
-    pub async fn reset_store(mut self) -> Result<DiskStore, DriveError> {
-        let BigState { store, .. } = self.state.take().expect("valid state");
-        Ok(store.reset().await?)
     }
 }

src/lib.rs

···
                 total_size += size;
             }
         }
-
-        // clean up the disk store (drop tables etc)
-        driver.reset_store().await?;
     }
 };
 println!("sum of size of all records: {total_size}");