Fast and robust atproto CAR file processing in rust

some settings that might not make sense

Changed files
+11 -2
src
+11 -2
src/disk.rs
··· 18 18 */ 19 19 20 20 use crate::drive::DriveError; 21 - use fjall::{Database, Error as FjallError, Keyspace, KeyspaceCreateOptions}; 21 + use fjall::config::{CompressionPolicy, PinningPolicy, RestartIntervalPolicy}; 22 + use fjall::{CompressionType, Database, Error as FjallError, Keyspace, KeyspaceCreateOptions}; 22 23 use std::path::PathBuf; 23 24 24 25 #[derive(Debug, thiserror::Error)] ··· 112 113 // .manual_journal_persist(true) 113 114 // .flush_workers(1) 114 115 // .compaction_workers(1) 116 + .journal_compression(CompressionType::None) 115 117 .cache_size(cache_mb as u64 * 2_u64.pow(20)) 116 118 .temporary(true) 117 119 .open()?; 118 - let partition = db.keyspace("z", KeyspaceCreateOptions::default)?; 120 + let opts = KeyspaceCreateOptions::default() 121 + .data_block_restart_interval_policy(RestartIntervalPolicy::all(8)) 122 + .filter_block_pinning_policy(PinningPolicy::disabled()) 123 + .expect_point_read_hits(true) 124 + .data_block_compression_policy(CompressionPolicy::disabled()) 125 + .manual_journal_persist(true) 126 + .max_memtable_size(32 * 2_u64.pow(20)); 127 + let partition = db.keyspace("z", || opts)?; 119 128 120 129 Ok::<_, DiskError>((db, partition)) 121 130 })