cleanup: store and processable for usize

benches/huge-car.rs

···
 extern crate repo_stream;
-use repo_stream::drive::Processable;
-use serde::{Deserialize, Serialize};
 use std::path::{Path, PathBuf};
 
 use criterion::{Criterion, criterion_group, criterion_main};
-
-#[derive(Clone, Serialize, Deserialize)]
-struct S(usize);
-
-impl Processable for S {
-    fn get_size(&self) -> usize {
-        0 // no additional space taken, just its stack size (newtype is free)
-    }
-}
 
 pub fn criterion_benchmark(c: &mut Criterion) {
     let rt = tokio::runtime::Builder::new_multi_thread()
···
 
     let mb = 2_usize.pow(20);
 
-    let mut driver = match repo_stream::drive::load_car(reader, |block| S(block.len()), 1024 * mb)
+    let mut driver = match repo_stream::drive::load_car(reader, |block| block.len(), 1024 * mb)
         .await
         .unwrap()
     {

benches/non-huge-cars.rs

···
 extern crate repo_stream;
 
 use criterion::{Criterion, criterion_group, criterion_main};
-use repo_stream::drive::Processable;
-use serde::{Deserialize, Serialize};
 
 const TINY_CAR: &'static [u8] = include_bytes!("../car-samples/tiny.car");
 const LITTLE_CAR: &'static [u8] = include_bytes!("../car-samples/little.car");
 const MIDSIZE_CAR: &'static [u8] = include_bytes!("../car-samples/midsize.car");
-
-#[derive(Clone, Serialize, Deserialize)]
-struct S(usize);
-
-impl Processable for S {
-    fn get_size(&self) -> usize {
-        0 // no additional space taken, just its stack size (newtype is free)
-    }
-}
 
 pub fn criterion_benchmark(c: &mut Criterion) {
     let rt = tokio::runtime::Builder::new_multi_thread()
···
 
 async fn drive_car(bytes: &[u8]) -> usize {
     let mut driver =
-        match repo_stream::drive::load_car(bytes, |block| S(block.len()), 32 * 2_usize.pow(20))
+        match repo_stream::drive::load_car(bytes, |block| block.len(), 32 * 2_usize.pow(20))
             .await
             .unwrap()
         {

examples/disk-read-file/main.rs

···
 extern crate repo_stream;
 use clap::Parser;
-use repo_stream::drive::Processable;
-use serde::{Deserialize, Serialize};
 use std::path::PathBuf;
 
 type Result<T> = std::result::Result<T, Box<dyn std::error::Error>>;
···
     tmpfile: PathBuf,
 }
 
-#[derive(Clone, Serialize, Deserialize)]
-struct S(usize);
-
-impl Processable for S {
-    fn get_size(&self) -> usize {
-        0 // no additional space taken, just its stack size (newtype is free)
-    }
-}
-
 #[tokio::main]
 async fn main() -> Result<()> {
     env_logger::init();
···
 
     let limit_mb = 32;
 
-    let driver = match repo_stream::drive::load_car(reader, |block| S(block.len()), 10 * mb).await?
-    {
+    let driver = match repo_stream::drive::load_car(reader, |block| block.len(), 10 * mb).await? {
         repo_stream::drive::Vehicle::Lil(_, _) => panic!("try this on a bigger car"),
         repo_stream::drive::Vehicle::Big(big_stuff) => {
-            let disk_store = repo_stream::disk::SqliteStore::new(tmpfile.clone(), limit_mb);
+            let disk_store = repo_stream::disk::SqliteStore::new(tmpfile.clone(), limit_mb).await?;
             let (commit, driver) = big_stuff.finish_loading(disk_store).await?;
             log::warn!("big: {:?}", commit);
             driver

examples/read-file/main.rs

···
 extern crate repo_stream;
 use clap::Parser;
-use repo_stream::drive::Processable;
-use serde::{Deserialize, Serialize};
 use std::path::PathBuf;
 
 type Result<T> = std::result::Result<T, Box<dyn std::error::Error>>;
···
     file: PathBuf,
 }
 
-#[derive(Clone, Serialize, Deserialize)]
-struct S(usize);
-
-impl Processable for S {
-    fn get_size(&self) -> usize {
-        0 // no additional space taken, just its stack size (newtype is free)
-    }
-}
-
 #[tokio::main]
 async fn main() -> Result<()> {
     env_logger::init();
···
     let reader = tokio::io::BufReader::new(reader);
 
     let (commit, mut driver) =
-        match repo_stream::drive::load_car(reader, |block| S(block.len()), 1024 * 1024).await? {
+        match repo_stream::drive::load_car(reader, |block| block.len(), 1024 * 1024).await? {
             repo_stream::drive::Vehicle::Lil(commit, mem_driver) => (commit, mem_driver),
             repo_stream::drive::Vehicle::Big(_) => panic!("can't handle big cars yet"),
         };

src/disk.rs

···
 use std::path::PathBuf;
 
 pub struct SqliteStore {
-    path: PathBuf,
-    limit_mb: usize,
+    conn: rusqlite::Connection,
 }
 
 impl SqliteStore {
-    pub fn new(path: PathBuf, limit_mb: usize) -> Self {
-        Self { path, limit_mb }
-    }
-}
-
-impl SqliteStore {
-    pub async fn get_access(&mut self) -> Result<SqliteAccess, rusqlite::Error> {
-        let path = self.path.clone();
-        let limit_mb = self.limit_mb;
+    pub async fn new(path: PathBuf, cache_mb: usize) -> Result<Self, rusqlite::Error> {
         let conn = tokio::task::spawn_blocking(move || {
             let conn = rusqlite::Connection::open(path)?;
 
-            let sq_mb = -(2_i64.pow(10)); // negative is kibibytes for sqlite cache_size
+            let sqlite_one_mb = -(2_i64.pow(10)); // negative is kibibytes for sqlite cache_size
 
             // conn.pragma_update(None, "journal_mode", "OFF")?;
             // conn.pragma_update(None, "journal_mode", "MEMORY")?;
             conn.pragma_update(None, "journal_mode", "WAL")?;
             // conn.pragma_update(None, "wal_autocheckpoint", "0")?; // this lets things get a bit big on disk
             conn.pragma_update(None, "synchronous", "OFF")?;
-            conn.pragma_update(None, "cache_size", (limit_mb as i64 * sq_mb).to_string())?;
+            conn.pragma_update(
+                None,
+                "cache_size",
+                (cache_mb as i64 * sqlite_one_mb).to_string(),
+            )?;
             conn.execute(
                 "CREATE TABLE blocks (
                     key  BLOB PRIMARY KEY NOT NULL,
···
         .await
         .expect("join error")?;
 
-        Ok(SqliteAccess { conn })
+        Ok(Self { conn })
     }
-}
-
-pub struct SqliteAccess {
-    conn: rusqlite::Connection,
-}
-
-impl SqliteAccess {
     pub fn get_writer(&'_ mut self) -> Result<SqliteWriter<'_>, rusqlite::Error> {
         let tx = self.conn.transaction()?;
         // let insert_stmt = tx.prepare("INSERT INTO blocks (key, val) VALUES (?1, ?2)")?;

src/drive.rs

···
 //! Consume an MST block stream, producing an ordered stream of records
 
-use crate::disk::{SqliteAccess, SqliteStore};
+use crate::disk::SqliteStore;
+use crate::process::Processable;
 use ipld_core::cid::Cid;
 use iroh_car::CarReader;
-use serde::de::DeserializeOwned;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::convert::Infallible;
···
     BincodeDecodeError(#[from] bincode::error::DecodeError),
     #[error("extra bytes remained after decoding")]
     ExtraGarbage,
-}
-
-pub trait Processable: Clone + Serialize + DeserializeOwned {
-    /// the additional size taken up (not including its mem::size_of)
-    fn get_size(&self) -> usize;
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
···
         mut self,
         mut store: SqliteStore,
     ) -> Result<(Commit, BigCarReady<T>), DriveError> {
-        // set up access for real
-        let mut access = store.get_access().await?;
-
-        // move access in and back out so we can manage lifetimes
+        // move store in and back out so we can manage lifetimes
         // dump mem blocks into the store
-        access = tokio::task::spawn(async move {
-            let mut writer = access.get_writer()?;
+        store = tokio::task::spawn(async move {
+            let mut writer = store.get_writer()?;
 
             let kvs = self
                 .mem_blocks
···
 
             writer.put_many(kvs)?;
             writer.commit()?;
-            Ok::<_, DriveError>(access)
+            Ok::<_, DriveError>(store)
         })
         .await??;
 
         let (tx, mut rx) = tokio::sync::mpsc::channel::<Vec<(Cid, MaybeProcessedBlock<T>)>>(2);
 
-        let access_worker = tokio::task::spawn_blocking(move || {
-            let mut writer = access.get_writer()?;
+        let store_worker = tokio::task::spawn_blocking(move || {
+            let mut writer = store.get_writer()?;
 
             while let Some(chunk) = rx.blocking_recv() {
                 let kvs = chunk
···
             }
 
             writer.commit()?;
-            Ok::<_, DriveError>(access)
+            Ok::<_, DriveError>(store)
         }); // await later
 
         // dump the rest to disk (in chunks)
···
         drop(tx);
         log::debug!("done. waiting for worker to finish...");
 
-        access = access_worker.await??;
+        store = store_worker.await??;
 
         log::debug!("worker finished.");
 
···
             commit,
             BigCarReady {
                 process: self.process,
-                access,
+                store,
                 walker,
             },
         ))
···
 
 pub struct BigCarReady<T: Clone> {
     process: fn(Vec<u8>) -> T,
-    access: SqliteAccess,
+    store: SqliteStore,
     walker: Walker,
 }
 
···
     ) -> Result<(Self, Option<Vec<(String, T)>>), DriveError> {
         let mut out = Vec::with_capacity(n);
         (self, out) = tokio::task::spawn_blocking(move || {
-            let access = self.access;
-            let mut reader = access.get_reader()?;
+            let store = self.store;
+            let mut reader = store.get_reader()?;
 
             for _ in 0..n {
                 // walk as far as we can until we run out of blocks or find a record
···
                 };
             }
 
-            drop(reader); // cannot outlive access
-            self.access = access;
+            drop(reader); // cannot outlive store
+            self.store = store;
             Ok::<_, DriveError>((self, out))
         })
         .await??;
···
         // ...should we return the join handle here so the caller at least knows about it?
         // yes probably for error handling?? (orrr put errors in the channel)
         let worker = tokio::task::spawn_blocking(move || {
-            let mut reader = self.access.get_reader()?;
+            let mut reader = self.store.get_reader()?;
 
             loop {
                 let mut out = Vec::with_capacity(n);
···
                     .map_err(|_| DriveError::ChannelSendError)?;
             }
 
-            drop(reader); // cannot outlive access
+            drop(reader); // cannot outlive store
             Ok(())
         }); // await later
 

src/lib.rs

···
 pub mod disk;
 pub mod drive;
 pub mod mst;
+pub mod process;
 pub mod walk;

src/process.rs

···
+use serde::{Serialize, de::DeserializeOwned};
+
+pub trait Processable: Clone + Serialize + DeserializeOwned {
+    /// the additional size taken up (not including its mem::size_of)
+    fn get_size(&self) -> usize;
+}
+
+impl Processable for usize {
+    fn get_size(&self) -> usize {
+        0 // no additional space taken, just its stack size (newtype is free)
+    }
+}

src/walk.rs

···
 //! Depth-first MST traversal
 
 use crate::disk::SqliteReader;
-use crate::drive::{DecodeError, MaybeProcessedBlock, Processable};
+use crate::drive::{DecodeError, MaybeProcessedBlock};
 use crate::mst::Node;
+use crate::process::Processable;
 use ipld_core::cid::Cid;
 use sha2::{Digest, Sha256};
 use std::collections::HashMap;

tests/non-huge-cars.rs

···
 extern crate repo_stream;
-use repo_stream::drive::Processable;
-use serde::{Deserialize, Serialize};
 
 const TINY_CAR: &'static [u8] = include_bytes!("../car-samples/tiny.car");
 const LITTLE_CAR: &'static [u8] = include_bytes!("../car-samples/little.car");
 const MIDSIZE_CAR: &'static [u8] = include_bytes!("../car-samples/midsize.car");
 
-#[derive(Clone, Serialize, Deserialize)]
-struct S(usize);
-
-impl Processable for S {
-    fn get_size(&self) -> usize {
-        0 // no additional space taken, just its stack size (newtype is free)
-    }
-}
-
 async fn test_car(bytes: &[u8], expected_records: usize, expected_sum: usize) {
     let mb = 2_usize.pow(20);
 
-    let mut driver = match repo_stream::drive::load_car(bytes, |block| S(block.len()), 10 * mb)
+    let mut driver = match repo_stream::drive::load_car(bytes, |block| block.len(), 10 * mb)
         .await
         .unwrap()
     {
···
     let mut prev_rkey = "".to_string();
 
     while let Some(pairs) = driver.next_chunk(256).await.unwrap() {
-        for (rkey, S(size)) in pairs {
+        for (rkey, size) in pairs {
             records += 1;
             sum += size;
             if rkey == "app.bsky.actor.profile/self" {