nekomimi.pet / microcosm-rs
forked from microcosm.blue/microcosm-rs
Constellation, Spacedust, Slingshot, UFOs: atproto crates and services for microcosm
fork atom

Merge pull request #36 from at-microcosm/ufos/fjall-weak-delete

use git-main fjall

authored by bad-example.com and committed by GitHub d1f2891a fbb2f92d

options
unified split
Changed files
+26 -20
Cargo.lock
ufos
Cargo.toml
src
storage_fjall.rs
+15 -16
Cargo.lock
··· 1220 checksum = "18e4fdb82bd54a12e42fb58a800dcae6b9e13982238ce2296dc3570b92148e1f" 1221 dependencies = [ 1222 "data-encoding", 1223 - "syn 2.0.103", 1224 ] 1225 1226 [[package]] ··· 1569 checksum = "976dd42dc7e85965fe702eb8164f21f450704bdde31faefd6471dba214cb594e" 1570 dependencies = [ 1571 "libc", 1572 - "windows-sys 0.59.0", 1573 ] 1574 1575 [[package]] ··· 1611 1612 [[package]] 1613 name = "fjall" 1614 - version = "2.8.0" 1615 - source = "registry+https://github.com/rust-lang/crates.io-index" 1616 - checksum = "26b2ced3483989a62b3533c9f99054d73b527c6c0045cf22b00fe87956f1a46f" 1617 dependencies = [ 1618 "byteorder", 1619 "byteview", ··· 2595 dependencies = [ 2596 "hermit-abi", 2597 "libc", 2598 - "windows-sys 0.59.0", 2599 ] 2600 2601 [[package]] ··· 2800 checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" 2801 dependencies = [ 2802 "cfg-if", 2803 - "windows-targets 0.52.6", 2804 ] 2805 2806 [[package]] ··· 2931 2932 [[package]] 2933 name = "lsm-tree" 2934 - version = "2.8.0" 2935 source = "registry+https://github.com/rust-lang/crates.io-index" 2936 - checksum = "d0a63a5e98a38b51765274137d8aedfbd848da5f4d016867e186b673fcc06a8c" 2937 dependencies = [ 2938 "byteorder", 2939 "crossbeam-skiplist", ··· 4054 "once_cell", 4055 "socket2 0.5.9", 4056 "tracing", 4057 - "windows-sys 0.59.0", 4058 ] 4059 4060 [[package]] ··· 4436 "errno", 4437 "libc", 4438 "linux-raw-sys 0.4.15", 4439 - "windows-sys 0.59.0", 4440 ] 4441 4442 [[package]] ··· 4449 "errno", 4450 "libc", 4451 "linux-raw-sys 0.9.4", 4452 - "windows-sys 0.59.0", 4453 ] 4454 4455 [[package]] ··· 5188 "getrandom 0.3.3", 5189 "once_cell", 5190 "rustix 1.0.5", 5191 - "windows-sys 0.59.0", 5192 ] 5193 5194 [[package]] ··· 5821 5822 [[package]] 5823 name = "value-log" 5824 - version = "1.8.0" 5825 source = "registry+https://github.com/rust-lang/crates.io-index" 5826 - checksum = "fd29b17c041f94e0885179637289815cd038f0c9fc19c4549d5a97017404fb7d" 5827 dependencies = [ 5828 "byteorder", 5829 "byteview", ··· 6073 source = "registry+https://github.com/rust-lang/crates.io-index" 6074 checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" 6075 dependencies = [ 6076 - "windows-sys 0.59.0", 6077 ] 6078 6079 [[package]]
··· 1220 checksum = "18e4fdb82bd54a12e42fb58a800dcae6b9e13982238ce2296dc3570b92148e1f" 1221 dependencies = [ 1222 "data-encoding", 1223 + "syn 1.0.109", 1224 ] 1225 1226 [[package]] ··· 1569 checksum = "976dd42dc7e85965fe702eb8164f21f450704bdde31faefd6471dba214cb594e" 1570 dependencies = [ 1571 "libc", 1572 + "windows-sys 0.52.0", 1573 ] 1574 1575 [[package]] ··· 1611 1612 [[package]] 1613 name = "fjall" 1614 + version = "2.11.2" 1615 + source = "git+https://github.com/fjall-rs/fjall.git#42d811f7c8cc9004407d520d37d2a1d8d246c03d" 1616 dependencies = [ 1617 "byteorder", 1618 "byteview", ··· 2594 dependencies = [ 2595 "hermit-abi", 2596 "libc", 2597 + "windows-sys 0.52.0", 2598 ] 2599 2600 [[package]] ··· 2799 checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" 2800 dependencies = [ 2801 "cfg-if", 2802 + "windows-targets 0.48.5", 2803 ] 2804 2805 [[package]] ··· 2930 2931 [[package]] 2932 name = "lsm-tree" 2933 + version = "2.10.2" 2934 source = "registry+https://github.com/rust-lang/crates.io-index" 2935 + checksum = "55b6d7475a8dd22e749186968daacf8e2a77932b061b1bd263157987bbfc0c6c" 2936 dependencies = [ 2937 "byteorder", 2938 "crossbeam-skiplist", ··· 4053 "once_cell", 4054 "socket2 0.5.9", 4055 "tracing", 4056 + "windows-sys 0.52.0", 4057 ] 4058 4059 [[package]] ··· 4435 "errno", 4436 "libc", 4437 "linux-raw-sys 0.4.15", 4438 + "windows-sys 0.52.0", 4439 ] 4440 4441 [[package]] ··· 4448 "errno", 4449 "libc", 4450 "linux-raw-sys 0.9.4", 4451 + "windows-sys 0.52.0", 4452 ] 4453 4454 [[package]] ··· 5187 "getrandom 0.3.3", 5188 "once_cell", 5189 "rustix 1.0.5", 5190 + "windows-sys 0.52.0", 5191 ] 5192 5193 [[package]] ··· 5820 5821 [[package]] 5822 name = "value-log" 5823 + version = "1.9.0" 5824 source = "registry+https://github.com/rust-lang/crates.io-index" 5825 + checksum = "62fc7c4ce161f049607ecea654dca3f2d727da5371ae85e2e4f14ce2b98ed67c" 5826 dependencies = [ 5827 "byteorder", 5828 "byteview", ··· 6072 source = "registry+https://github.com/rust-lang/crates.io-index" 6073 checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" 6074 dependencies = [ 6075 + "windows-sys 0.48.0", 6076 ] 6077 6078 [[package]]
+1 -1
ufos/Cargo.toml
··· 13 clap = { version = "4.5.31", features = ["derive"] } 14 dropshot = "0.16.0" 15 env_logger = "0.11.7" 16 - fjall = { version = "2.8.0", features = ["lz4"] } 17 getrandom = "0.3.3" 18 http = "1.3.1" 19 jetstream = { path = "../jetstream", features = ["metrics"] }
··· 13 clap = { version = "4.5.31", features = ["derive"] } 14 dropshot = "0.16.0" 15 env_logger = "0.11.7" 16 + fjall = { git = "https://github.com/fjall-rs/fjall.git", features = ["lz4"] } 17 getrandom = "0.3.3" 18 http = "1.3.1" 19 jetstream = { path = "../jetstream", features = ["metrics"] }
+10 -3
ufos/src/storage_fjall.rs
··· 1222 AllTimeRecordsKey::new(new_creates_count.into(), &nsid).to_db_bytes()?, 1223 ), 1224 }; 1225 - batch.remove(&self.rollups, &old_k); // TODO: when fjall gets weak delete, this will hopefully work way better 1226 batch.insert(&self.rollups, &new_k, ""); 1227 } 1228 ··· 1246 AllTimeDidsKey::new(new_dids_estimate.into(), &nsid).to_db_bytes()?, 1247 ), 1248 }; 1249 - batch.remove(&self.rollups, &old_k); // TODO: when fjall gets weak delete, this will hopefully work way better 1250 batch.insert(&self.rollups, &new_k, ""); 1251 } 1252 ··· 1619 histogram!("storage_trim_dirty_nsids").record(completed.len() as f64); 1620 histogram!("storage_trim_duration").record(dt.as_micros() as f64); 1621 counter!("storage_trim_removed", "dangling" => "true").increment(total_danglers as u64); 1622 - counter!("storage_trim_removed", "dangling" => "false").increment((total_deleted - total_danglers) as u64); 1623 for c in completed { 1624 dirty_nsids.remove(&c); 1625 }
··· 1222 AllTimeRecordsKey::new(new_creates_count.into(), &nsid).to_db_bytes()?, 1223 ), 1224 }; 1225 + // remove_weak is allowed here because the secondary ranking index only ever inserts once at a key 1226 + batch.remove_weak(&self.rollups, &old_k); 1227 batch.insert(&self.rollups, &new_k, ""); 1228 } 1229 ··· 1247 AllTimeDidsKey::new(new_dids_estimate.into(), &nsid).to_db_bytes()?, 1248 ), 1249 }; 1250 + // remove_weak is allowed here because the secondary ranking index only ever inserts once at a key 1251 + batch.remove_weak(&self.rollups, &old_k); 1252 batch.insert(&self.rollups, &new_k, ""); 1253 } 1254 ··· 1621 histogram!("storage_trim_dirty_nsids").record(completed.len() as f64); 1622 histogram!("storage_trim_duration").record(dt.as_micros() as f64); 1623 counter!("storage_trim_removed", "dangling" => "true").increment(total_danglers as u64); 1624 + if total_deleted >= total_danglers { 1625 + counter!("storage_trim_removed", "dangling" => "false").increment((total_deleted - total_danglers) as u64); 1626 + } else { 1627 + // TODO: probably think through what's happening here 1628 + log::warn!("weird trim case: more danglers than deleted? metric will be missing for dangling=false. deleted={total_deleted} danglers={total_danglers}"); 1629 + } 1630 for c in completed { 1631 dirty_nsids.remove(&c); 1632 }