feat(server): implement delta-of-deltas encoding and varint and chunking

server/Cargo.lock

···
 ]
 
 [[package]]
+name = "atomic-time"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9622f5c6fb50377516c70f65159e70b25465409760c6bd6d4e581318bf704e83"
+dependencies = [
+ "once_cell",
+ "portable-atomic",
+]
+
+[[package]]
 name = "atomic-waker"
 version = "1.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e"
 
 [[package]]
-name = "overload"
-version = "0.1.1"
+name = "ordered-varint"
+version = "2.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
+checksum = "e9cc9f18ab4bad1e01726bda1259feb8f11e5e76308708a966b4c0136e9db34c"
 
 [[package]]
-name = "papaya"
-version = "0.2.3"
+name = "overload"
+version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f92dd0b07c53a0a0c764db2ace8c541dc47320dad97c2200c2a637ab9dd2328f"
-dependencies = [
- "equivalent",
- "seize",
-]
+checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
 
 [[package]]
 name = "parking_lot"
···
 version = "0.3.32"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"
+
+[[package]]
+name = "portable-atomic"
+version = "1.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483"
 
 [[package]]
 name = "proc-macro2"
···
 ]
 
 [[package]]
+name = "scc"
+version = "2.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22b2d775fb28f245817589471dd49c5edf64237f4a19d10ce9a92ff4651a27f4"
+dependencies = [
+ "sdd",
+]
+
+[[package]]
 name = "schannel"
 version = "0.1.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
 
 [[package]]
+name = "sdd"
+version = "3.0.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "490dcfcbfef26be6800d11870ff2df8774fa6e86d047e3e8c8a76b25655e41ca"
+
+[[package]]
 name = "security-framework"
 version = "3.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 dependencies = [
  "core-foundation-sys",
  "libc",
-]
-
-[[package]]
-name = "seize"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e4b8d813387d566f627f3ea1b914c068aac94c40ae27ec43f5f33bde65abefe7"
-dependencies = [
- "libc",
- "windows-sys 0.52.0",
 ]
 
 [[package]]
···
 dependencies = [
  "anyhow",
  "async-trait",
+ "atomic-time",
  "axum",
  "axum-tws",
  "fjall",
  "futures-util",
- "papaya",
+ "ordered-varint",
  "pingora-limits",
  "rkyv",
  "rustls",
+ "scc",
  "serde",
  "serde_json",
  "smol_str",

server/Cargo.toml

···
 fjall = { version = "2", default-features = false, features = ["miniz", "lz4"] }
 rkyv = {version = "0.8", features = ["unaligned"]}
 smol_str = { version = "0.3", features = ["serde"] }
-papaya = "0.2"
 serde = { version = "1", features = ["derive"] }
 serde_json = "1.0.141"
+scc = "2.3.4"
+atomic-time = "0.1.5"
+ordered-varint = "2.0.0"
 
 [target.'cfg(not(target_env = "msvc"))'.dependencies]
 tikv-jemallocator = "0.6"

server/src/api.rs

···
 use crate::{
     db::Db,
     error::{AppError, AppResult},
+    utils::time_now,
 };
 
 struct LatencyMillis(u128);
···
     let maybe_hits = db
         .get_hits(
             &params.nsid,
-            params.to.unwrap_or(0)
-                ..params.from.unwrap_or(
-                    std::time::SystemTime::now()
-                        .duration_since(UNIX_EPOCH)
-                        .expect("oops")
-                        .as_micros() as u64,
-                ),
-        )?
+            params.to.unwrap_or(0)..params.from.unwrap_or(time_now()),
+        )
         .take(MAX_HITS);
     let mut hits = Vec::with_capacity(maybe_hits.size_hint().0);
 
     for maybe_hit in maybe_hits {
-        let (timestamp, hit) = maybe_hit?;
+        let hit = maybe_hit?;
+        let hit_data = hit.access();
+
         hits.push(Hit {
-            timestamp,
-            deleted: hit.deleted,
+            timestamp: hit.timestamp,
+            deleted: hit_data.deleted,
         });
     }
 

server/src/db.rs

···
-use std::{
-    ops::{Bound, Deref, RangeBounds},
-    path::Path,
-    time::Duration,
-};
-
-use fjall::{Config, Keyspace, Partition, PartitionCreateOptions};
-use pingora_limits::rate::Rate;
-use rkyv::{Archive, Deserialize, Serialize, rancor::Error};
-use smol_str::SmolStr;
-use tokio::sync::broadcast;
-
-use crate::{
-    error::{AppError, AppResult},
-    jetstream::JetstreamEvent,
-};
-
-#[derive(Clone, Debug, Default, Archive, Deserialize, Serialize, PartialEq)]
-#[rkyv(compare(PartialEq), derive(Debug))]
-pub struct NsidCounts {
-    pub count: u128,
-    pub deleted_count: u128,
-    pub last_seen: u64,
-}
-
-#[derive(Debug, Default, Archive, Deserialize, Serialize, PartialEq)]
-#[rkyv(compare(PartialEq), derive(Debug))]
-pub struct NsidHit {
-    pub deleted: bool,
-}
-
-pub struct EventRecord {
-    pub nsid: SmolStr,
-    pub timestamp: u64,
-    pub deleted: bool,
-}
-
-impl EventRecord {
-    pub fn from_jetstream(event: JetstreamEvent) -> Option<Self> {
-        match event {
-            JetstreamEvent::Commit {
-                time_us, commit, ..
-            } => Some(Self {
-                nsid: commit.collection.into(),
-                timestamp: time_us,
-                deleted: false,
-            }),
-            JetstreamEvent::Delete {
-                time_us, commit, ..
-            } => Some(Self {
-                nsid: commit.collection.into(),
-                timestamp: time_us,
-                deleted: true,
-            }),
-            _ => None,
-        }
-    }
-}
-
-// counts is nsid -> NsidCounts
-// hits is tree per nsid: timestamp -> NsidHit
-pub struct Db {
-    inner: Keyspace,
-    hits: papaya::HashMap<SmolStr, Partition>,
-    counts: Partition,
-    event_broadcaster: broadcast::Sender<(SmolStr, NsidCounts)>,
-    eps: Rate,
-}
-
-impl Db {
-    pub fn new(path: impl AsRef<Path>) -> AppResult<Self> {
-        tracing::info!("opening db...");
-        let ks = Config::new(path)
-            .cache_size(8 * 1024 * 1024) // from talna
-            .open()?;
-        Ok(Self {
-            hits: Default::default(),
-            counts: ks.open_partition(
-                "_counts",
-                PartitionCreateOptions::default().compression(fjall::CompressionType::None),
-            )?,
-            inner: ks,
-            event_broadcaster: broadcast::channel(1000).0,
-            eps: Rate::new(Duration::from_secs(1)),
-        })
-    }
-
-    pub fn eps(&self) -> usize {
-        self.eps.rate(&()) as usize
-    }
-
-    pub fn new_listener(&self) -> broadcast::Receiver<(SmolStr, NsidCounts)> {
-        self.event_broadcaster.subscribe()
-    }
-
-    #[inline(always)]
-    fn run_in_nsid_tree<T>(
-        &self,
-        nsid: &str,
-        f: impl FnOnce(&Partition) -> AppResult<T>,
-    ) -> AppResult<T> {
-        f(self.hits.pin().get_or_insert_with(SmolStr::new(nsid), || {
-            let opts = PartitionCreateOptions::default()
-                .compression(fjall::CompressionType::Miniz(9))
-                .compaction_strategy(fjall::compaction::Strategy::Fifo(fjall::compaction::Fifo {
-                    limit: 5 * 1024 * 1024 * 1024,        // 5 gb
-                    ttl_seconds: Some(60 * 60 * 24 * 30), // 30 days
-                }));
-            self.inner.open_partition(nsid, opts).unwrap()
-        }))
-    }
-
-    pub fn record_event(&self, e: EventRecord) -> AppResult<()> {
-        let EventRecord {
-            nsid,
-            timestamp,
-            deleted,
-        } = e;
-
-        self.insert_event(&nsid, timestamp, deleted)?;
-        // increment count
-        let mut counts = self.get_count(&nsid)?;
-        counts.last_seen = timestamp;
-        if deleted {
-            counts.deleted_count += 1;
-        } else {
-            counts.count += 1;
-        }
-        self.insert_count(&nsid, counts.clone())?;
-        if self.event_broadcaster.receiver_count() > 0 {
-            let _ = self.event_broadcaster.send((SmolStr::new(&nsid), counts));
-        }
-        self.eps.observe(&(), 1);
-        Ok(())
-    }
-
-    #[inline(always)]
-    fn insert_event(&self, nsid: &str, timestamp: u64, deleted: bool) -> AppResult<()> {
-        self.run_in_nsid_tree(nsid, |tree| {
-            tree.insert(
-                timestamp.to_be_bytes(),
-                unsafe { rkyv::to_bytes::<Error>(&NsidHit { deleted }).unwrap_unchecked() }
-                    .as_slice(),
-            )
-            .map_err(AppError::from)
-        })
-    }
-
-    #[inline(always)]
-    fn insert_count(&self, nsid: &str, counts: NsidCounts) -> AppResult<()> {
-        self.counts
-            .insert(
-                nsid,
-                unsafe { rkyv::to_bytes::<Error>(&counts).unwrap_unchecked() }.as_slice(),
-            )
-            .map_err(AppError::from)
-    }
-
-    pub fn get_count(&self, nsid: &str) -> AppResult<NsidCounts> {
-        let Some(raw) = self.counts.get(nsid)? else {
-            return Ok(NsidCounts::default());
-        };
-        Ok(unsafe { rkyv::from_bytes_unchecked::<_, Error>(&raw).unwrap_unchecked() })
-    }
-
-    pub fn get_counts(&self) -> impl Iterator<Item = AppResult<(SmolStr, NsidCounts)>> {
-        self.counts.iter().map(|res| {
-            res.map_err(AppError::from).map(|(key, val)| {
-                (
-                    SmolStr::new(unsafe { str::from_utf8_unchecked(&key) }),
-                    unsafe { rkyv::from_bytes_unchecked::<_, Error>(&val).unwrap_unchecked() },
-                )
-            })
-        })
-    }
-
-    pub fn get_nsids(&self) -> impl Iterator<Item = impl Deref<Target = str>> {
-        self.inner
-            .list_partitions()
-            .into_iter()
-            .filter(|k| k.deref() != "_counts")
-    }
-
-    pub fn get_hits(
-        &self,
-        nsid: &str,
-        range: impl RangeBounds<u64>,
-    ) -> AppResult<Box<dyn Iterator<Item = AppResult<(u64, NsidHit)>>>> {
-        let start = range.start_bound().cloned().map(u64::to_be_bytes);
-        let end = range.end_bound().cloned().map(u64::to_be_bytes);
-
-        let _guard = self.hits.guard();
-        let Some(tree) = self.hits.get(nsid, &_guard) else {
-            return Ok(Box::new(std::iter::empty()));
-        };
-
-        Ok(Box::new(tree.range(TimestampRange { start, end }).map(
-            |res| {
-                res.map_err(AppError::from).map(|(key, val)| {
-                    (
-                        u64::from_be_bytes(key.as_ref().try_into().unwrap()),
-                        unsafe { rkyv::from_bytes_unchecked::<_, Error>(&val).unwrap_unchecked() },
-                    )
-                })
-            },
-        )))
-    }
-
-    pub fn tracking_since(&self) -> AppResult<u64> {
-        let _guard = self.hits.guard();
-        // HACK: we should actually store when we started tracking but im lazy
-        // should be accurate enough
-        let Some(tree) = self.hits.get("app.bsky.feed.like", &_guard) else {
-            return Ok(0);
-        };
-
-        let Some((timestamp_raw, _)) = tree.first_key_value()? else {
-            return Ok(0);
-        };
-
-        Ok(u64::from_be_bytes(
-            timestamp_raw.as_ref().try_into().unwrap(),
-        ))
-    }
-}
-
-type TimestampRepr = [u8; 8];
-
-struct TimestampRange {
-    start: Bound<TimestampRepr>,
-    end: Bound<TimestampRepr>,
-}
-
-impl RangeBounds<TimestampRepr> for TimestampRange {
-    #[inline(always)]
-    fn start_bound(&self) -> Bound<&TimestampRepr> {
-        self.start.as_ref()
-    }
-
-    #[inline(always)]
-    fn end_bound(&self) -> Bound<&TimestampRepr> {
-        self.end.as_ref()
-    }
-}

server/src/db/block.rs

···
+use ordered_varint::Variable;
+use rkyv::{
+    Archive, Serialize, api::high::HighSerializer, rancor, ser::allocator::ArenaHandle,
+    util::AlignedVec,
+};
+use std::{
+    io::{self, Read, Write},
+    marker::PhantomData,
+};
+
+use crate::error::AppResult;
+
+pub struct Item<T> {
+    pub timestamp: u64,
+    data: AlignedVec,
+    phantom: PhantomData<T>,
+}
+
+impl<T: Archive> Item<T> {
+    pub fn access(&self) -> &T::Archived {
+        unsafe { rkyv::access_unchecked::<T::Archived>(&self.data) }
+    }
+}
+
+impl<T: for<'a> Serialize<HighSerializer<AlignedVec, ArenaHandle<'a>, rancor::Error>>> Item<T> {
+    pub fn new(timestamp: u64, data: &T) -> Self {
+        Item {
+            timestamp,
+            data: unsafe { rkyv::to_bytes(data).unwrap_unchecked() },
+            phantom: PhantomData,
+        }
+    }
+}
+
+pub struct ItemEncoder<W: Write, T> {
+    writer: W,
+    prev_timestamp: u64,
+    prev_delta: i64,
+    _item: PhantomData<T>,
+}
+
+impl<W: Write, T> ItemEncoder<W, T> {
+    pub fn new(writer: W) -> Self {
+        ItemEncoder {
+            writer,
+            prev_timestamp: 0,
+            prev_delta: 0,
+            _item: PhantomData,
+        }
+    }
+
+    pub fn encode(&mut self, item: &Item<T>) -> AppResult<()> {
+        if self.prev_timestamp == 0 {
+            // self.writer.write_varint(item.timestamp)?;
+            self.prev_timestamp = item.timestamp;
+            self.write_data(&item.data)?;
+            return Ok(());
+        }
+
+        let delta = (item.timestamp as i128 - self.prev_timestamp as i128) as i64;
+
+        self.writer.write_varint(delta - self.prev_delta)?;
+        self.prev_timestamp = item.timestamp;
+        self.prev_delta = delta;
+
+        self.write_data(&item.data)?;
+
+        Ok(())
+    }
+
+    fn write_data(&mut self, data: &[u8]) -> AppResult<()> {
+        self.writer.write_varint(data.len())?;
+        self.writer.write_all(data)?;
+        Ok(())
+    }
+
+    pub fn finish(mut self) -> AppResult<W> {
+        self.writer.flush()?;
+        Ok(self.writer)
+    }
+}
+
+pub struct ItemDecoder<R, T> {
+    reader: R,
+    current_timestamp: u64,
+    current_delta: i64,
+    first_item: bool,
+    _item: PhantomData<T>,
+}
+
+impl<R: Read, T: Archive> ItemDecoder<R, T> {
+    pub fn new(reader: R, start_timestamp: u64) -> AppResult<Self> {
+        Ok(ItemDecoder {
+            reader,
+            current_timestamp: start_timestamp,
+            current_delta: 0,
+            first_item: true,
+            _item: PhantomData,
+        })
+    }
+
+    pub fn decode(&mut self) -> AppResult<Option<Item<T>>> {
+        if self.first_item {
+            // read the first timestamp
+            // let timestamp = match self.reader.read_varint::<u64>() {
+            //     Ok(timestamp) => timestamp,
+            //     Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => return Ok(None),
+            //     Err(e) => return Err(e.into()),
+            // };
+            // self.current_timestamp = timestamp;
+
+            let Some(data_raw) = self.read_item()? else {
+                return Ok(None);
+            };
+            self.first_item = false;
+            return Ok(Some(Item {
+                timestamp: self.current_timestamp,
+                data: data_raw,
+                phantom: PhantomData,
+            }));
+        }
+
+        let Some(_delta) = self.read_timestamp()? else {
+            return Ok(None);
+        };
+
+        // read data
+        let data_raw = match self.read_item()? {
+            Some(data_raw) => data_raw,
+            None => {
+                return Err(io::Error::new(
+                    io::ErrorKind::UnexpectedEof,
+                    "expected data after delta",
+                )
+                .into());
+            }
+        };
+
+        Ok(Some(Item {
+            timestamp: self.current_timestamp,
+            data: data_raw,
+            phantom: PhantomData,
+        }))
+    }
+
+    // [10, 11, 12, 14] -> [1, 1, 2] -> [0, 1]
+    fn read_timestamp(&mut self) -> AppResult<Option<u64>> {
+        let delta = match self.reader.read_varint::<i64>() {
+            Ok(delta) => delta,
+            Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => return Ok(None),
+            Err(e) => return Err(e.into()),
+        };
+        self.current_delta += delta;
+        self.current_timestamp =
+            (self.current_timestamp as i128 + self.current_delta as i128) as u64;
+        Ok(Some(self.current_timestamp))
+    }
+
+    fn read_item(&mut self) -> AppResult<Option<AlignedVec>> {
+        let data_len = match self.reader.read_varint::<usize>() {
+            Ok(data_len) => data_len,
+            Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => return Ok(None),
+            Err(e) => return Err(e.into()),
+        };
+        let mut data_raw = AlignedVec::with_capacity(data_len);
+        for _ in 0..data_len {
+            data_raw.push(0);
+        }
+        self.reader.read_exact(data_raw.as_mut_slice())?;
+        Ok(Some(data_raw))
+    }
+}
+
+impl<R: Read, T: Archive> Iterator for ItemDecoder<R, T> {
+    type Item = AppResult<Item<T>>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        self.decode().transpose()
+    }
+}
+
+pub trait WriteVariableExt: Write {
+    fn write_varint(&mut self, value: impl Variable) -> io::Result<usize> {
+        value.encode_variable(self)
+    }
+}
+impl<W: Write> WriteVariableExt for W {}
+
+pub trait ReadVariableExt: Read {
+    fn read_varint<T: Variable>(&mut self) -> io::Result<T> {
+        T::decode_variable(self)
+    }
+}
+impl<R: Read> ReadVariableExt for R {}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use rkyv::{Archive, Deserialize, Serialize};
+    use std::io::Cursor;
+
+    #[derive(Archive, Deserialize, Serialize, Debug, PartialEq)]
+    #[rkyv(compare(PartialEq))]
+    struct TestData {
+        id: u32,
+        value: String,
+    }
+
+    #[test]
+    fn test_encoder_decoder_single_item() {
+        let data = TestData {
+            id: 123,
+            value: "test".to_string(),
+        };
+
+        let item = Item::new(1000, &data);
+
+        // encode
+        let mut buffer = Vec::new();
+        let mut encoder = ItemEncoder::new(&mut buffer);
+        encoder.encode(&item).unwrap();
+        encoder.finish().unwrap();
+
+        // decode
+        let cursor = Cursor::new(buffer);
+        let mut decoder = ItemDecoder::<_, TestData>::new(cursor, 1000).unwrap();
+
+        let decoded_item = decoder.decode().unwrap().unwrap();
+        assert_eq!(decoded_item.timestamp, 1000);
+
+        let decoded_data = decoded_item.access();
+        assert_eq!(decoded_data.id, 123);
+        assert_eq!(decoded_data.value.as_str(), "test");
+    }
+
+    #[test]
+    fn test_encoder_decoder_multiple_items() {
+        let items = vec![
+            Item::new(
+                1000,
+                &TestData {
+                    id: 1,
+                    value: "first".to_string(),
+                },
+            ),
+            Item::new(
+                1010,
+                &TestData {
+                    id: 2,
+                    value: "second".to_string(),
+                },
+            ),
+            Item::new(
+                1015,
+                &TestData {
+                    id: 3,
+                    value: "third".to_string(),
+                },
+            ),
+            Item::new(
+                1025,
+                &TestData {
+                    id: 4,
+                    value: "fourth".to_string(),
+                },
+            ),
+        ];
+
+        // encode
+        let mut buffer = Vec::new();
+        let mut encoder = ItemEncoder::new(&mut buffer);
+
+        for item in &items {
+            encoder.encode(item).unwrap();
+        }
+        encoder.finish().unwrap();
+
+        // decode
+        let cursor = Cursor::new(buffer);
+        let mut decoder = ItemDecoder::<_, TestData>::new(cursor, 1000).unwrap();
+
+        let mut decoded_items = Vec::new();
+        while let Some(item) = decoder.decode().unwrap() {
+            decoded_items.push(item);
+        }
+
+        assert_eq!(decoded_items.len(), 4);
+
+        for (original, decoded) in items.iter().zip(decoded_items.iter()) {
+            assert_eq!(original.timestamp, decoded.timestamp);
+            assert_eq!(original.access().id, decoded.access().id);
+            assert_eq!(
+                original.access().value.as_str(),
+                decoded.access().value.as_str()
+            );
+        }
+    }
+
+    #[test]
+    fn test_encoder_decoder_with_iterator() {
+        let items = vec![
+            Item::new(
+                2000,
+                &TestData {
+                    id: 10,
+                    value: "a".to_string(),
+                },
+            ),
+            Item::new(
+                2005,
+                &TestData {
+                    id: 20,
+                    value: "b".to_string(),
+                },
+            ),
+            Item::new(
+                2012,
+                &TestData {
+                    id: 30,
+                    value: "c".to_string(),
+                },
+            ),
+        ];
+
+        // encode
+        let mut buffer = Vec::new();
+        let mut encoder = ItemEncoder::new(&mut buffer);
+
+        for item in &items {
+            encoder.encode(item).unwrap();
+        }
+        encoder.finish().unwrap();
+
+        // decode
+        let cursor = Cursor::new(buffer);
+        let decoder = ItemDecoder::<_, TestData>::new(cursor, 2000).unwrap();
+
+        let decoded_items: Result<Vec<_>, _> = decoder.collect();
+        let decoded_items = decoded_items.unwrap();
+
+        assert_eq!(decoded_items.len(), 3);
+        assert_eq!(decoded_items[0].timestamp, 2000);
+        assert_eq!(decoded_items[1].timestamp, 2005);
+        assert_eq!(decoded_items[2].timestamp, 2012);
+
+        assert_eq!(decoded_items[0].access().id, 10);
+        assert_eq!(decoded_items[1].access().id, 20);
+        assert_eq!(decoded_items[2].access().id, 30);
+    }
+
+    #[test]
+    fn test_delta_compression() {
+        let items = vec![
+            Item::new(
+                1000,
+                &TestData {
+                    id: 1,
+                    value: "a".to_string(),
+                },
+            ),
+            Item::new(
+                1010,
+                &TestData {
+                    id: 2,
+                    value: "b".to_string(),
+                },
+            ), // delta = 10
+            Item::new(
+                1020,
+                &TestData {
+                    id: 3,
+                    value: "c".to_string(),
+                },
+            ), // delta = 10, delta-of-delta = 0
+            Item::new(
+                1025,
+                &TestData {
+                    id: 4,
+                    value: "d".to_string(),
+                },
+            ), // delta = 5, delta-of-delta = -5
+        ];
+
+        let mut buffer = Vec::new();
+        let mut encoder = ItemEncoder::new(&mut buffer);
+
+        for item in &items {
+            encoder.encode(item).unwrap();
+        }
+        encoder.finish().unwrap();
+
+        // decode and verify
+        let cursor = Cursor::new(buffer);
+        let decoder = ItemDecoder::<_, TestData>::new(cursor, 1000).unwrap();
+
+        let decoded_items: Result<Vec<_>, _> = decoder.collect();
+        let decoded_items = decoded_items.unwrap();
+
+        for (original, decoded) in items.iter().zip(decoded_items.iter()) {
+            assert_eq!(original.timestamp, decoded.timestamp);
+            assert_eq!(original.access().id, decoded.access().id);
+        }
+    }
+
+    #[test]
+    fn test_empty_decode() {
+        let buffer = Vec::new();
+        let cursor = Cursor::new(buffer);
+        let mut decoder = ItemDecoder::<_, TestData>::new(cursor, 1000).unwrap();
+
+        let result = decoder.decode().unwrap();
+        assert!(result.is_none());
+    }
+
+    #[test]
+    fn test_backwards_timestamp() {
+        let items = vec![
+            Item::new(
+                1000,
+                &TestData {
+                    id: 1,
+                    value: "first".to_string(),
+                },
+            ),
+            Item::new(
+                900,
+                &TestData {
+                    id: 2,
+                    value: "second".to_string(),
+                },
+            ),
+        ];
+
+        let mut buffer = Vec::new();
+        let mut encoder = ItemEncoder::new(&mut buffer);
+
+        for item in &items {
+            encoder.encode(item).unwrap();
+        }
+        encoder.finish().unwrap();
+
+        let cursor = Cursor::new(buffer);
+        let decoder = ItemDecoder::<_, TestData>::new(cursor, 1000).unwrap();
+
+        let decoded_items: Result<Vec<_>, _> = decoder.collect();
+        let decoded_items = decoded_items.unwrap();
+
+        assert_eq!(decoded_items.len(), 2);
+        assert_eq!(decoded_items[0].timestamp, 1000);
+        assert_eq!(decoded_items[1].timestamp, 900);
+    }
+
+    #[test]
+    fn test_different_data_sizes() {
+        let small_data = TestData {
+            id: 1,
+            value: "x".to_string(),
+        };
+        let large_data = TestData {
+            id: 2,
+            value: "a".repeat(1000),
+        };
+
+        let items = vec![Item::new(1000, &small_data), Item::new(1001, &large_data)];
+
+        let mut buffer = Vec::new();
+        let mut encoder = ItemEncoder::new(&mut buffer);
+
+        for item in &items {
+            encoder.encode(item).unwrap();
+        }
+        encoder.finish().unwrap();
+
+        let cursor = Cursor::new(buffer);
+        let decoder = ItemDecoder::<_, TestData>::new(cursor, 1000).unwrap();
+
+        let decoded_items: Result<Vec<_>, _> = decoder.collect();
+        let decoded_items = decoded_items.unwrap();
+
+        assert_eq!(decoded_items.len(), 2);
+        assert_eq!(decoded_items[0].access().value.as_str(), "x");
+        assert_eq!(decoded_items[1].access().value.len(), 1000);
+        assert_eq!(decoded_items[1].access().value.as_str(), "a".repeat(1000));
+    }
+}

server/src/db/mod.rs

···
+use std::{
+    io::Cursor,
+    ops::{Bound, Deref, RangeBounds},
+    path::Path,
+    sync::{
+        Arc,
+        atomic::{AtomicUsize, Ordering as AtomicOrdering},
+    },
+    time::{Duration, Instant},
+};
+
+use atomic_time::AtomicInstant;
+use fjall::{Config, Keyspace, Partition, PartitionCreateOptions, Slice};
+use ordered_varint::Variable;
+use pingora_limits::rate::Rate;
+use rkyv::{Archive, Deserialize, Serialize, rancor::Error};
+use smol_str::SmolStr;
+use tokio::sync::broadcast;
+
+use crate::{
+    db::block::{ReadVariableExt, WriteVariableExt},
+    error::{AppError, AppResult},
+    jetstream::JetstreamEvent,
+    utils::time_now,
+};
+
+mod block;
+
+#[derive(Clone, Debug, Default, Archive, Deserialize, Serialize, PartialEq)]
+#[rkyv(compare(PartialEq), derive(Debug))]
+pub struct NsidCounts {
+    pub count: u128,
+    pub deleted_count: u128,
+    pub last_seen: u64,
+}
+
+#[derive(Debug, Default, Archive, Deserialize, Serialize, PartialEq)]
+#[rkyv(compare(PartialEq), derive(Debug))]
+pub struct NsidHit {
+    pub deleted: bool,
+}
+
+#[derive(Clone)]
+pub struct EventRecord {
+    pub nsid: SmolStr,
+    pub timestamp: u64, // seconds
+    pub deleted: bool,
+}
+
+impl EventRecord {
+    pub fn from_jetstream(event: JetstreamEvent) -> Option<Self> {
+        match event {
+            JetstreamEvent::Commit {
+                time_us, commit, ..
+            } => Some(Self {
+                nsid: commit.collection.into(),
+                timestamp: time_us / 1_000_000,
+                deleted: false,
+            }),
+            JetstreamEvent::Delete {
+                time_us, commit, ..
+            } => Some(Self {
+                nsid: commit.collection.into(),
+                timestamp: time_us / 1_000_000,
+                deleted: true,
+            }),
+            _ => None,
+        }
+    }
+}
+
+// counts is nsid -> NsidCounts
+// hits is tree per nsid: timestamp -> NsidHit
+pub struct DbOld {
+    inner: Keyspace,
+    hits: scc::HashIndex<SmolStr, Partition>,
+    counts: Partition,
+    event_broadcaster: broadcast::Sender<(SmolStr, NsidCounts)>,
+    eps: Rate,
+}
+
+impl DbOld {
+    pub fn new(path: impl AsRef<Path>) -> AppResult<Self> {
+        tracing::info!("opening db...");
+        let ks = Config::new(path)
+            .cache_size(8 * 1024 * 1024) // from talna
+            .open()?;
+        Ok(Self {
+            hits: Default::default(),
+            counts: ks.open_partition(
+                "_counts",
+                PartitionCreateOptions::default().compression(fjall::CompressionType::None),
+            )?,
+            inner: ks,
+            event_broadcaster: broadcast::channel(1000).0,
+            eps: Rate::new(Duration::from_secs(1)),
+        })
+    }
+
+    pub fn eps(&self) -> usize {
+        self.eps.rate(&()) as usize
+    }
+
+    pub fn new_listener(&self) -> broadcast::Receiver<(SmolStr, NsidCounts)> {
+        self.event_broadcaster.subscribe()
+    }
+
+    #[inline(always)]
+    fn run_in_nsid_tree<T>(
+        &self,
+        nsid: &str,
+        f: impl FnOnce(&Partition) -> AppResult<T>,
+    ) -> AppResult<T> {
+        f(self
+            .hits
+            .entry(SmolStr::new(nsid))
+            .or_insert_with(|| {
+                let opts = PartitionCreateOptions::default()
+                    .compression(fjall::CompressionType::Miniz(9))
+                    .compaction_strategy(fjall::compaction::Strategy::Fifo(
+                        fjall::compaction::Fifo {
+                            limit: 5 * 1024 * 1024 * 1024,        // 5 gb
+                            ttl_seconds: Some(60 * 60 * 24 * 30), // 30 days
+                        },
+                    ));
+                self.inner.open_partition(nsid, opts).unwrap()
+            })
+            .get())
+    }
+
+    pub fn record_event(&self, e: EventRecord) -> AppResult<()> {
+        let EventRecord {
+            nsid,
+            timestamp,
+            deleted,
+        } = e;
+
+        self.insert_event(&nsid, timestamp, deleted)?;
+        // increment count
+        let mut counts = self.get_count(&nsid)?;
+        counts.last_seen = timestamp;
+        if deleted {
+            counts.deleted_count += 1;
+        } else {
+            counts.count += 1;
+        }
+        self.insert_count(&nsid, counts.clone())?;
+        if self.event_broadcaster.receiver_count() > 0 {
+            let _ = self.event_broadcaster.send((SmolStr::new(&nsid), counts));
+        }
+        self.eps.observe(&(), 1);
+        Ok(())
+    }
+
+    #[inline(always)]
+    fn insert_event(&self, nsid: &str, timestamp: u64, deleted: bool) -> AppResult<()> {
+        self.run_in_nsid_tree(nsid, |tree| {
+            tree.insert(
+                timestamp.to_be_bytes(),
+                unsafe { rkyv::to_bytes::<Error>(&NsidHit { deleted }).unwrap_unchecked() }
+                    .as_slice(),
+            )
+            .map_err(AppError::from)
+        })
+    }
+
+    #[inline(always)]
+    fn insert_count(&self, nsid: &str, counts: NsidCounts) -> AppResult<()> {
+        self.counts
+            .insert(
+                nsid,
+                unsafe { rkyv::to_bytes::<Error>(&counts).unwrap_unchecked() }.as_slice(),
+            )
+            .map_err(AppError::from)
+    }
+
+    pub fn get_count(&self, nsid: &str) -> AppResult<NsidCounts> {
+        let Some(raw) = self.counts.get(nsid)? else {
+            return Ok(NsidCounts::default());
+        };
+        Ok(unsafe { rkyv::from_bytes_unchecked::<_, Error>(&raw).unwrap_unchecked() })
+    }
+
+    pub fn get_counts(&self) -> impl Iterator<Item = AppResult<(SmolStr, NsidCounts)>> {
+        self.counts.iter().map(|res| {
+            res.map_err(AppError::from).map(|(key, val)| {
+                (
+                    SmolStr::new(unsafe { str::from_utf8_unchecked(&key) }),
+                    unsafe { rkyv::from_bytes_unchecked::<_, Error>(&val).unwrap_unchecked() },
+                )
+            })
+        })
+    }
+
+    pub fn get_nsids(&self) -> impl Iterator<Item = impl Deref<Target = str>> {
+        self.inner
+            .list_partitions()
+            .into_iter()
+            .filter(|k| k.deref() != "_counts")
+    }
+
+    pub fn get_hits(
+        &self,
+        nsid: &str,
+        range: impl RangeBounds<u64>,
+    ) -> AppResult<Box<dyn Iterator<Item = AppResult<(u64, NsidHit)>>>> {
+        let start = range.start_bound().cloned().map(u64::to_be_bytes);
+        let end = range.end_bound().cloned().map(u64::to_be_bytes);
+
+        let _guard = scc::ebr::Guard::new();
+        let Some(tree) = self.hits.peek(nsid, &_guard) else {
+            return Ok(Box::new(std::iter::empty()));
+        };
+
+        Ok(Box::new(tree.range(TimestampRangeOld { start, end }).map(
+            |res| {
+                res.map_err(AppError::from).map(|(key, val)| {
+                    (
+                        u64::from_be_bytes(key.as_ref().try_into().unwrap()),
+                        unsafe { rkyv::from_bytes_unchecked::<_, Error>(&val).unwrap_unchecked() },
+                    )
+                })
+            },
+        )))
+    }
+
+    pub fn tracking_since(&self) -> AppResult<u64> {
+        let _guard = scc::ebr::Guard::new();
+        // HACK: we should actually store when we started tracking but im lazy
+        // should be accurate enough
+        let Some(tree) = self.hits.peek("app.bsky.feed.like", &_guard) else {
+            return Ok(0);
+        };
+        let Some((timestamp_raw, _)) = tree.first_key_value()? else {
+            return Ok(0);
+        };
+        drop(_guard);
+
+        Ok(u64::from_be_bytes(
+            timestamp_raw.as_ref().try_into().unwrap(),
+        ))
+    }
+}
+
+type ItemDecoder = block::ItemDecoder<Cursor<Slice>, NsidHit>;
+type ItemEncoder = block::ItemEncoder<Vec<u8>, NsidHit>;
+type Item = block::Item<NsidHit>;
+
+pub struct LexiconHandle {
+    tree: Partition,
+    buf: Arc<scc::Queue<EventRecord>>,
+    buf_len: AtomicUsize,
+    last_insert: AtomicInstant,
+    eps: Rate,
+    block_size: AtomicUsize,
+}
+
+impl LexiconHandle {
+    fn new(keyspace: &Keyspace, nsid: &str) -> Self {
+        let opts = PartitionCreateOptions::default().compression(fjall::CompressionType::Miniz(9));
+        Self {
+            tree: keyspace.open_partition(nsid, opts).unwrap(),
+            buf: Default::default(),
+            buf_len: AtomicUsize::new(0),
+            last_insert: AtomicInstant::now(),
+            eps: Rate::new(Duration::from_secs(5)),
+            block_size: AtomicUsize::new(1000),
+        }
+    }
+
+    fn item_count(&self) -> usize {
+        self.buf_len.load(AtomicOrdering::Acquire)
+    }
+
+    fn last_insert(&self) -> Instant {
+        self.last_insert.load(AtomicOrdering::Acquire)
+    }
+
+    fn suggested_block_size(&self) -> usize {
+        self.block_size.load(AtomicOrdering::Relaxed)
+    }
+
+    fn insert(&self, event: EventRecord) {
+        self.buf.push(event);
+        self.buf_len.fetch_add(1, AtomicOrdering::Release);
+        self.last_insert
+            .store(Instant::now(), AtomicOrdering::Release);
+        self.eps.observe(&(), 1);
+        let rate = self.eps.rate(&()) as usize;
+        if rate != 0 {
+            self.block_size.store(rate * 60, AtomicOrdering::Relaxed);
+        }
+    }
+
+    fn sync(&self) -> AppResult<()> {
+        let mut writer = ItemEncoder::new(Vec::with_capacity(
+            size_of::<u64>() + self.item_count() * size_of::<(u64, NsidHit)>(),
+        ));
+        let mut start_timestamp = None;
+        let mut end_timestamp = None;
+        while let Some(event) = self.buf.pop() {
+            let item = Item::new(
+                event.timestamp,
+                &NsidHit {
+                    deleted: event.deleted,
+                },
+            );
+            writer.encode(&item)?;
+            if start_timestamp.is_none() {
+                start_timestamp = Some(event.timestamp);
+            }
+            end_timestamp = Some(event.timestamp);
+        }
+        if let (Some(start_timestamp), Some(end_timestamp)) = (start_timestamp, end_timestamp) {
+            self.buf_len.store(0, AtomicOrdering::Release);
+            let value = writer.finish()?;
+            let mut key = Vec::with_capacity(size_of::<u64>() * 2);
+            key.write_varint(start_timestamp)?;
+            key.write_varint(end_timestamp)?;
+            self.tree.insert(key, value)?;
+        }
+        Ok(())
+    }
+}
+
+type BoxedIter<T> = Box<dyn Iterator<Item = T>>;
+
+// counts is nsid -> NsidCounts
+// hits is tree per nsid: varint start time + varint end time -> block of hits
+pub struct Db {
+    inner: Keyspace,
+    hits: scc::HashIndex<SmolStr, Arc<LexiconHandle>>,
+    counts: Partition,
+    event_broadcaster: broadcast::Sender<(SmolStr, NsidCounts)>,
+    eps: Rate,
+    min_block_size: usize,
+    max_last_activity: Duration,
+}
+
+impl Db {
+    pub fn new(path: impl AsRef<Path>) -> AppResult<Self> {
+        tracing::info!("opening db...");
+        let ks = Config::new(path)
+            .cache_size(8 * 1024 * 1024) // from talna
+            .open()?;
+        Ok(Self {
+            hits: Default::default(),
+            counts: ks.open_partition(
+                "_counts",
+                PartitionCreateOptions::default().compression(fjall::CompressionType::None),
+            )?,
+            inner: ks,
+            event_broadcaster: broadcast::channel(1000).0,
+            eps: Rate::new(Duration::from_secs(1)),
+            min_block_size: 512,
+            max_last_activity: Duration::from_secs(10),
+        })
+    }
+
+    pub fn sync(&self, all: bool) -> AppResult<()> {
+        let _guard = scc::ebr::Guard::new();
+        for (nsid, tree) in self.hits.iter(&_guard) {
+            let count = tree.item_count();
+            let is_max_block_size = count > self.min_block_size.max(tree.suggested_block_size());
+            let is_too_old = tree.last_insert().elapsed() > self.max_last_activity;
+            if count > 0 && (all || is_max_block_size || is_too_old) {
+                tracing::info!("syncing {count} of {nsid} to db");
+                tree.sync()?;
+            }
+        }
+        Ok(())
+    }
+
+    #[inline(always)]
+    pub fn eps(&self) -> usize {
+        self.eps.rate(&()) as usize
+    }
+
+    #[inline(always)]
+    pub fn new_listener(&self) -> broadcast::Receiver<(SmolStr, NsidCounts)> {
+        self.event_broadcaster.subscribe()
+    }
+
+    #[inline(always)]
+    fn maybe_run_in_nsid_tree<T>(
+        &self,
+        nsid: &str,
+        f: impl FnOnce(&LexiconHandle) -> T,
+    ) -> Option<T> {
+        let _guard = scc::ebr::Guard::new();
+        let handle = match self.hits.peek(nsid, &_guard) {
+            Some(handle) => handle.clone(),
+            None => {
+                if self.inner.partition_exists(nsid) {
+                    let handle = Arc::new(LexiconHandle::new(&self.inner, nsid));
+                    let _ = self.hits.insert(SmolStr::new(nsid), handle.clone());
+                    handle
+                } else {
+                    return None;
+                }
+            }
+        };
+        Some(f(&handle))
+    }
+
+    #[inline(always)]
+    fn run_in_nsid_tree<T>(
+        &self,
+        nsid: SmolStr,
+        f: impl FnOnce(&LexiconHandle) -> AppResult<T>,
+    ) -> AppResult<T> {
+        f(self
+            .hits
+            .entry(nsid.clone())
+            .or_insert_with(move || Arc::new(LexiconHandle::new(&self.inner, &nsid)))
+            .get())
+    }
+
+    pub fn record_event(&self, e: EventRecord) -> AppResult<()> {
+        let EventRecord {
+            nsid,
+            timestamp,
+            deleted,
+        } = e.clone();
+
+        // insert event
+        self.run_in_nsid_tree(nsid.clone(), move |tree| Ok(tree.insert(e)))?;
+        // increment count
+        let mut counts = self.get_count(&nsid)?;
+        counts.last_seen = timestamp;
+        if deleted {
+            counts.deleted_count += 1;
+        } else {
+            counts.count += 1;
+        }
+        self.insert_count(&nsid, counts.clone())?;
+        if self.event_broadcaster.receiver_count() > 0 {
+            let _ = self.event_broadcaster.send((SmolStr::new(&nsid), counts));
+        }
+        self.eps.observe(&(), 1);
+        Ok(())
+    }
+
+    #[inline(always)]
+    fn insert_count(&self, nsid: &str, counts: NsidCounts) -> AppResult<()> {
+        self.counts
+            .insert(
+                nsid,
+                unsafe { rkyv::to_bytes::<Error>(&counts).unwrap_unchecked() }.as_slice(),
+            )
+            .map_err(AppError::from)
+    }
+
+    pub fn get_count(&self, nsid: &str) -> AppResult<NsidCounts> {
+        let Some(raw) = self.counts.get(nsid)? else {
+            return Ok(NsidCounts::default());
+        };
+        Ok(unsafe { rkyv::from_bytes_unchecked::<_, Error>(&raw).unwrap_unchecked() })
+    }
+
+    pub fn get_counts(&self) -> impl Iterator<Item = AppResult<(SmolStr, NsidCounts)>> {
+        self.counts.iter().map(|res| {
+            res.map_err(AppError::from).map(|(key, val)| {
+                (
+                    SmolStr::new(unsafe { str::from_utf8_unchecked(&key) }),
+                    unsafe { rkyv::from_bytes_unchecked::<_, Error>(&val).unwrap_unchecked() },
+                )
+            })
+        })
+    }
+
+    pub fn get_nsids(&self) -> impl Iterator<Item = impl Deref<Target = str>> {
+        self.inner
+            .list_partitions()
+            .into_iter()
+            .filter(|k| k.deref() != "_counts")
+    }
+
+    pub fn get_hits_debug(&self, nsid: &str) -> BoxedIter<AppResult<(Slice, Slice)>> {
+        self.maybe_run_in_nsid_tree(nsid, |handle| -> BoxedIter<AppResult<(Slice, Slice)>> {
+            Box::new(
+                handle
+                    .tree
+                    .iter()
+                    .rev()
+                    .map(|res| res.map_err(AppError::from)),
+            )
+        })
+        .unwrap_or_else(|| Box::new(std::iter::empty()))
+    }
+
+    pub fn get_hits(
+        &self,
+        nsid: &str,
+        range: impl RangeBounds<u64> + std::fmt::Debug,
+    ) -> BoxedIter<AppResult<Item>> {
+        let start = range
+            .start_bound()
+            .cloned()
+            .map(|t| unsafe { t.to_variable_vec().unwrap_unchecked() });
+        let end = range
+            .end_bound()
+            .cloned()
+            .map(|t| unsafe { t.to_variable_vec().unwrap_unchecked() });
+        let limit = match range.end_bound().cloned() {
+            Bound::Included(end) => end,
+            Bound::Excluded(end) => end.saturating_sub(1),
+            Bound::Unbounded => u64::MAX,
+        };
+
+        self.maybe_run_in_nsid_tree(nsid, move |handle| -> BoxedIter<AppResult<Item>> {
+            let map_block = move |(key, val)| {
+                let mut key_reader = Cursor::new(key);
+                let start_timestamp = key_reader.read_varint::<u64>()?;
+                let items =
+                    ItemDecoder::new(Cursor::new(val), start_timestamp)?.take_while(move |item| {
+                        item.as_ref().map_or(true, |item| item.timestamp <= limit)
+                    });
+                Ok(items)
+            };
+
+            Box::new(
+                handle
+                    .tree
+                    .range(TimestampRange { start, end })
+                    .map(move |res| res.map_err(AppError::from).and_then(map_block))
+                    .flatten()
+                    .flatten(),
+            )
+        })
+        .unwrap_or_else(|| Box::new(std::iter::empty()))
+    }
+
+    pub fn tracking_since(&self) -> AppResult<u64> {
+        let _guard = scc::ebr::Guard::new();
+        // HACK: we should actually store when we started tracking but im lazy
+        // should be accurate enough
+        let Some(handle) = self.hits.peek("app.bsky.feed.like", &_guard) else {
+            return Ok(0);
+        };
+        let Some((timestamps_raw, _)) = handle.tree.first_key_value()? else {
+            return Ok(0);
+        };
+        drop(_guard);
+
+        let mut timestamp_reader = Cursor::new(timestamps_raw);
+        timestamp_reader
+            .read_varint::<u64>()
+            .map_err(AppError::from)
+    }
+}
+
+type TimestampRepr = Vec<u8>;
+
+struct TimestampRange {
+    start: Bound<TimestampRepr>,
+    end: Bound<TimestampRepr>,
+}
+
+impl RangeBounds<TimestampRepr> for TimestampRange {
+    #[inline(always)]
+    fn start_bound(&self) -> Bound<&TimestampRepr> {
+        self.start.as_ref()
+    }
+
+    #[inline(always)]
+    fn end_bound(&self) -> Bound<&TimestampRepr> {
+        self.end.as_ref()
+    }
+}
+
+type TimestampReprOld = [u8; 8];
+
+struct TimestampRangeOld {
+    start: Bound<TimestampReprOld>,
+    end: Bound<TimestampReprOld>,
+}
+
+impl RangeBounds<TimestampReprOld> for TimestampRangeOld {
+    #[inline(always)]
+    fn start_bound(&self) -> Bound<&TimestampReprOld> {
+        self.start.as_ref()
+    }
+
+    #[inline(always)]
+    fn end_bound(&self) -> Bound<&TimestampReprOld> {
+        self.end.as_ref()
+    }
+}

server/src/main.rs

···
 
 use crate::{
     api::serve,
-    db::{Db, EventRecord},
+    db::{Db, DbOld, EventRecord},
     error::AppError,
     jetstream::JetstreamClient,
 };
···
 mod db;
 mod error;
 mod jetstream;
+mod utils;
 
 #[cfg(not(target_env = "msvc"))]
 #[global_allocator]
···
         .compact()
         .init();
 
-    if std::env::args()
-        .nth(1)
-        .map_or(false, |arg| arg == "migrate")
-    {
-        migrate_to_miniz();
-        return;
+    match std::env::args().nth(1).as_deref() {
+        Some("migrate") => {
+            migrate();
+            return;
+        }
+        Some("debug") => {
+            debug();
+            return;
+        }
+        _ => {}
     }
 
     let db = Arc::new(Db::new(".fjall_data").expect("couldnt create db"));
···
         };
 
     let cancel_token = CancellationToken::new();
-    let (event_tx, mut event_rx) = tokio::sync::mpsc::channel(1000);
 
     let consume_events = tokio::spawn({
         let consume_cancel = cancel_token.child_token();
+        let db = db.clone();
         async move {
             jetstream.connect().await?;
             loop {
···
                             let Some(record) = EventRecord::from_jetstream(event) else {
                                 continue;
                             };
-                            let _ = event_tx.send(record).await;
+                            let db = db.clone();
+                            tokio::task::spawn_blocking(move || {
+                                if let Err(err) = db.record_event(record) {
+                                    tracing::error!("failed to record event: {}", err);
+                                }
+                            });
                         }
                         Err(err) => return Err(err),
                     },
···
         }
     });
 
-    let ingest_events = std::thread::spawn({
+    std::thread::spawn({
         let db = db.clone();
         move || {
-            tracing::info!("starting ingest events thread...");
-            while let Some(e) = event_rx.blocking_recv() {
-                if let Err(e) = db.record_event(e) {
-                    tracing::error!("failed to record event: {}", e);
+            loop {
+                match db.sync(false) {
+                    Ok(_) => (),
+                    Err(e) => tracing::error!("failed to sync db: {}", e),
                 }
+                std::thread::sleep(std::time::Duration::from_secs(1));
             }
         }
     });
 
     tokio::select! {
-        res = serve(db, cancel_token.child_token()) => {
+        res = serve(db.clone(), cancel_token.child_token()) => {
             if let Err(e) = res {
                 tracing::error!("serve failed: {}", e);
             }
···
     }
 
     tracing::info!("shutting down...");
-    ingest_events
-        .join()
-        .expect("failed to join ingest events thread");
+    db.sync(true).expect("couldnt sync db");
+}
+
+fn debug() {
+    let db = Db::new(".fjall_data").expect("couldnt create db");
+    for nsid in db.get_nsids() {
+        let nsid = nsid.deref();
+        for hit in db.get_hits(nsid, ..) {
+            let hit = hit.expect("cant read event");
+            println!("{nsid} {}", hit.timestamp);
+        }
+    }
 }
 
-fn migrate_to_miniz() {
-    let from = Db::new(".fjall_data").expect("couldnt create db");
-    let to = Db::new(".fjall_data_miniz").expect("couldnt create db");
+fn migrate() {
+    let from = DbOld::new(".fjall_data").expect("couldnt create db");
+    let to = Db::new(".fjall_data_migrated").expect("couldnt create db");
 
     let mut total_count = 0_u64;
     for nsid in from.get_nsids() {

server/src/utils.rs

···
+use std::time::UNIX_EPOCH;
+
+pub fn time_now() -> u64 {
+    std::time::SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .expect("oops")
+        .as_micros() as u64
+}