bucket sums, more samples, export json

Cargo.lock

···
  "ratelimit",
  "rocksdb",
  "serde",
+ "serde_json",
  "serde_with",
  "tempfile",
  "tinyjson",
···
 
 [[package]]
 name = "serde_json"
-version = "1.0.138"
+version = "1.0.139"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d434192e7da787e94a6ea7e9670b26a036d0ca41e0b7efb2676dd32bae872949"
+checksum = "44f86c3acccc9c65b153fe1b85a3be07fe5515274ec9f0653b4a0875731c72a6"
 dependencies = [
  "itoa",
  "memchr",

constellation/Cargo.toml

···
 ratelimit = "0.10.0"
 rocksdb = { version = "0.23.0", optional = true }
 serde = { version = "1.0.215", features = ["derive"] }
+serde_json = "1.0.139"
 serde_with = { version = "3.12.0", features = ["hex"] }
 tinyjson = "2.5.1"
 tokio-util = "0.7.13"

constellation/readme.md

···
   - [ ] read ops (api)
   - [ ] expose internal stats?
 - [ ] figure out what's the right thing to do if merge op fails. happened on startup after an unclean reboot.
-
+- [x] backups!
+  - [x] manual backup on startup
+  - [x] background task to create backups on an interval
 
 cache
 - [ ] set api response headers

constellation/src/bin/rocks-target-stats.rs constellation/src/bin/rocks-link-stats.rs

···
 use bincode::config::Options;
 use clap::Parser;
+use serde::Serialize;
 use std::collections::HashMap;
 use std::path::PathBuf;
 
···
 
 type LinkType = String;
 
-#[derive(Debug, Eq, Hash, PartialEq)]
-struct SourceLink(Collection, RPath, LinkType);
+#[derive(Debug, Eq, Hash, PartialEq, Serialize)]
+struct SourceLink(Collection, RPath, LinkType, Option<Collection>); // last is target collection, if it's an at-uri link with a collection
 
-#[derive(Debug, Default)]
-struct Buckets([u64; 23]);
+#[derive(Debug, Serialize)]
+struct SourceSample {
+    did: String,
+    rkey: String,
+}
+
+#[derive(Debug, Default, Serialize)]
+struct Bucket {
+    count: u64,
+    sum: u64,
+    sample: Option<SourceSample>,
+}
 
-const BUCKETS: Buckets = Buckets([
-    1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 16, 32, 64, 128, 256, 512, 1024, 4096, 16384, 65535, 262144,
-    1048576,
-]);
+#[derive(Debug, Default, Serialize)]
+struct Buckets([Bucket; 23]);
+
+const BUCKETS: [u64; 23] = [
+    1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 16, 32, 64, 128, 256, 512, 1024, 4096, 16_384, 65_535,
+    262_144, 1_048_576,
+];
 
 // b1, b2, b3, b4, b5, b6, b7, b8, b9, b10, b12, b16, b32, b64, b128, b256, b512, b1024, b4096, b16384, b65535, b262144, bmax
 
···
 
 const REPORT_INTERVAL: usize = 50_000;
 
-type Stats = HashMap<SourceLink, (String, String, Buckets)>;
+type Stats = HashMap<SourceLink, Buckets>;
+
+#[derive(Debug, Serialize)]
+struct Printable {
+    collection: String,
+    path: String,
+    link_type: String,
+    target_collection: Option<String>,
+    buckets: Buckets,
+}
 
 #[derive(Debug, Default)]
 struct ErrStats {
···
                 err_stats.failed_to_parse_target_as_link += 1;
                 continue;
             };
-            SourceLink(collection, rpath, parsed.name().into())
+            SourceLink(
+                collection,
+                rpath,
+                parsed.name().into(),
+                parsed.at_uri_collection().map(Collection),
+            )
         };
 
         let Ok(Some(links_raw)) = db.get_cf(&target_links_cf, &target_id) else {
···
         }
 
         let mut bucket = 0;
-        for edge in BUCKETS.0 {
+        for edge in BUCKETS {
             if n <= edge || bucket == 22 {
                 break;
             }
             bucket += 1;
         }
 
-        stats
-            .entry(source)
-            .or_insert_with(|| {
-                let (DidId(did_id), RKey(k)) = &linkers.0[(n - 1) as usize];
-                if let Ok(Some(did_bytes)) = db.get_cf(&did_ids_cf, did_id.to_be_bytes()) {
-                    if let Ok(Did(did)) = _bincode_opts().deserialize(&did_bytes) {
-                        return (did, k.clone(), Default::default());
-                    }
+        let b = &mut stats.entry(source).or_default().0[bucket];
+        b.count += 1;
+        b.sum += n;
+        if b.sample.is_none() {
+            let (DidId(did_id), RKey(k)) = &linkers.0[(n - 1) as usize];
+            if let Ok(Some(did_bytes)) = db.get_cf(&did_ids_cf, did_id.to_be_bytes()) {
+                if let Ok(Did(did)) = _bincode_opts().deserialize(&did_bytes) {
+                    b.sample = Some(SourceSample {
+                        did,
+                        rkey: k.clone(),
+                    });
+                } else {
+                    err_stats.failed_to_get_sample += 1;
                 }
+            } else {
                 err_stats.failed_to_get_sample += 1;
-                ("".into(), "".into(), Default::default())
-            })
-            .2
-             .0[bucket] += 1;
+            }
+        }
 
-        // if i >= 400_000 { break }
+        if i >= 40_000 {
+            break;
+        }
     }
 
     eprintln!(
···
     );
     eprintln!("{err_stats:?}");
 
-    for (SourceLink(Collection(c), RPath(p), t), (d, r, Buckets(b))) in stats {
-        let sample_at_uri = if !(d.is_empty() || r.is_empty()) {
-            format!("at://{d}/{c}/{r}")
-        } else {
-            "".into()
-        };
-        println!(
-            "{c:?}, {p:?}, {t:?}, {sample_at_uri:?}, {}",
-            b.map(|n| n.to_string()).join(", ")
-        );
+    let itemified = stats
+        .into_iter()
+        .map(
+            |(
+                SourceLink(Collection(collection), RPath(path), link_type, target_collection),
+                buckets,
+            )| Printable {
+                collection,
+                path,
+                link_type,
+                target_collection: target_collection.map(|Collection(c)| c),
+                buckets,
+            },
+        )
+        .collect::<Vec<_>>();
+
+    match serde_json::to_string(&itemified) {
+        Ok(s) => println!("{s}"),
+        Err(e) => eprintln!("failed to serialize results: {e:?}"),
     }
 
     eprintln!("bye.");

constellation/src/storage/rocks_store.rs

···
 }
 fn get_db_read_opts() -> Options {
     let mut opts = Options::default();
-    opts.optimize_for_point_lookup(128); // mb
+    opts.optimize_for_point_lookup(16_384); // mb (run this on big machines)
     opts
 }
 

links/src/at_uri.rs

···
     // there's a more normalization to do still. ugh.
 }
 
+pub fn at_uri_collection(at_uri: &str) -> Option<String> {
+    let (proto, rest) = at_uri.split_at_checked(5)?;
+    if !proto.eq_ignore_ascii_case("at://") {
+        return None;
+    }
+    let (_did, rest) = rest.split_once('/')?;
+    if let Some((collection, _path_rest)) = rest.split_once('/') {
+        return Some(collection.to_string());
+    }
+    if let Some((collection, _query_rest)) = rest.split_once('?') {
+        return Some(collection.to_string());
+    }
+    if let Some((collection, _hash_rest)) = rest.split_once('#') {
+        return Some(collection.to_string());
+    }
+    Some(rest.to_string())
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
···
         ] {
             assert_eq!(
                 parse_at_uri(case),
+                expected.map(|s| s.to_string()),
+                "{detail}"
+            );
+        }
+    }
+
+    #[test]
+    fn test_at_uri_collection() {
+        for (case, expected, detail) in vec![
+            ("", None, "empty"),
+            ("at://did:plc:vc7f4oafdgxsihk4cry2xpze", None, "did only"),
+            (
+                "at://did:plc:vc7f4oafdgxsihk4cry2xpze/collec.tion",
+                Some("collec.tion"),
+                "no path (weird)",
+            ),
+            (
+                "at://did:plc:vc7f4oafdgxsihk4cry2xpze/collec.tion/path",
+                Some("collec.tion"),
+                "normal at-uri",
+            ),
+            (
+                "at://did:plc:vc7f4oafdgxsihk4cry2xpze/collec.tion?query",
+                Some("collec.tion"),
+                "colleciton with query",
+            ),
+            (
+                "at://did:plc:vc7f4oafdgxsihk4cry2xpze/collec.tion#hash",
+                Some("collec.tion"),
+                "colleciton with hash",
+            ),
+            (
+                "at://did:plc:vc7f4oafdgxsihk4cry2xpze/collec.tion/path?query#hash",
+                Some("collec.tion"),
+                "colleciton with everything",
+            ),
+            (
+                "at://did:web:example.com/collec.tion/path",
+                Some("collec.tion"),
+                "did:web",
+            ),
+            (
+                "at://did:web:example.com/col.lec.tio.ns.so.long.going.on.and.on",
+                Some("col.lec.tio.ns.so.long.going.on.and.on"),
+                "long collection",
+            ),
+        ] {
+            assert_eq!(
+                at_uri_collection(case),
                 expected.map(|s| s.to_string()),
                 "{detail}"
             );

links/src/lib.rs

···
             Link::Did(_) => "did",
         }
     }
+    pub fn at_uri_collection(&self) -> Option<String> {
+        if let Link::AtUri(at_uri) = self {
+            at_uri::at_uri_collection(at_uri)
+        } else {
+            None
+        }
+    }
 }
 
 #[derive(Debug, PartialEq)]
···
             parse_any_link("did:plc:44ybard66vv44zksje25o7dz"),
             Some(Link::Did("did:plc:44ybard66vv44zksje25o7dz".into()))
         )
+    }
+
+    #[test]
+    fn test_at_uri_collection() {
+        assert_eq!(
+            parse_any_link("https://example.com")
+                .unwrap()
+                .at_uri_collection(),
+            None
+        );
+        assert_eq!(
+            parse_any_link("did:web:bad-example.com")
+                .unwrap()
+                .at_uri_collection(),
+            None
+        );
+        assert_eq!(
+            parse_any_link("at://did:web:bad-example.com/my.collection/3jwdwj2ctlk26")
+                .unwrap()
+                .at_uri_collection(),
+            Some("my.collection".into())
+        );
     }
 }