Fast and robust atproto CAR file processing in rust

some settings that might not make sense

Changed files
+11 -2
src
+11 -2
src/disk.rs
··· 18 */ 19 20 use crate::drive::DriveError; 21 - use fjall::{Database, Error as FjallError, Keyspace, KeyspaceCreateOptions}; 22 use std::path::PathBuf; 23 24 #[derive(Debug, thiserror::Error)] ··· 112 // .manual_journal_persist(true) 113 // .flush_workers(1) 114 // .compaction_workers(1) 115 .cache_size(cache_mb as u64 * 2_u64.pow(20)) 116 .temporary(true) 117 .open()?; 118 - let partition = db.keyspace("z", KeyspaceCreateOptions::default)?; 119 120 Ok::<_, DiskError>((db, partition)) 121 })
··· 18 */ 19 20 use crate::drive::DriveError; 21 + use fjall::config::{CompressionPolicy, PinningPolicy, RestartIntervalPolicy}; 22 + use fjall::{CompressionType, Database, Error as FjallError, Keyspace, KeyspaceCreateOptions}; 23 use std::path::PathBuf; 24 25 #[derive(Debug, thiserror::Error)] ··· 113 // .manual_journal_persist(true) 114 // .flush_workers(1) 115 // .compaction_workers(1) 116 + .journal_compression(CompressionType::None) 117 .cache_size(cache_mb as u64 * 2_u64.pow(20)) 118 .temporary(true) 119 .open()?; 120 + let opts = KeyspaceCreateOptions::default() 121 + .data_block_restart_interval_policy(RestartIntervalPolicy::all(8)) 122 + .filter_block_pinning_policy(PinningPolicy::disabled()) 123 + .expect_point_read_hits(true) 124 + .data_block_compression_policy(CompressionPolicy::disabled()) 125 + .manual_journal_persist(true) 126 + .max_memtable_size(32 * 2_u64.pow(20)); 127 + let partition = db.keyspace("z", || opts)?; 128 129 Ok::<_, DiskError>((db, partition)) 130 })