backfill data for the user from the car file

theres no other functionality except the basic car backfill lol

Cargo.lock

···
 ]
 
 [[package]]
+name = "anyhow"
+version = "1.0.100"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61"
+
+[[package]]
 name = "ascii"
 version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
  "multihash",
  "serde",
  "serde_bytes",
- "unsigned-varint",
+ "unsigned-varint 0.8.0",
 ]
 
 [[package]]
···
 version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2f421161cb492475f1661ddc9815a745a1c894592070661180fdec3d4872e9c3"
+
+[[package]]
+name = "cordyceps"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "688d7fbb8092b8de775ef2536f36c8c31f2bc4006ece2e8d8ad2d17d00ce0a2a"
+dependencies = [
+ "loom",
+ "tracing",
+]
 
 [[package]]
 name = "core-foundation"
···
 ]
 
 [[package]]
+name = "curve25519-dalek"
+version = "4.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be"
+dependencies = [
+ "cfg-if",
+ "cpufeatures",
+ "curve25519-dalek-derive",
+ "digest",
+ "fiat-crypto",
+ "rustc_version",
+ "subtle",
+ "zeroize",
+]
+
+[[package]]
+name = "curve25519-dalek-derive"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.109",
+]
+
+[[package]]
 name = "darling"
 version = "0.21.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 ]
 
 [[package]]
+name = "derive_more"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4a9b99b9cbbe49445b21764dc0625032a89b145a2642e67603e1c936f5458d05"
+dependencies = [
+ "derive_more-impl",
+]
+
+[[package]]
+name = "derive_more-impl"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.109",
+ "unicode-xid",
+]
+
+[[package]]
+name = "diatomic-waker"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ab03c107fafeb3ee9f5925686dbb7a73bc76e3932abb0d2b365cb64b169cf04c"
+
+[[package]]
 name = "digest"
 version = "0.10.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 ]
 
 [[package]]
+name = "ed25519"
+version = "2.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53"
+dependencies = [
+ "pkcs8",
+ "signature",
+]
+
+[[package]]
+name = "ed25519-dalek"
+version = "2.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9"
+dependencies = [
+ "curve25519-dalek",
+ "ed25519",
+ "rand_core 0.6.4",
+ "serde",
+ "sha2",
+ "subtle",
+ "zeroize",
+]
+
+[[package]]
 name = "either"
 version = "1.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 ]
 
 [[package]]
+name = "fiat-crypto"
+version = "0.2.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d"
+
+[[package]]
 name = "filetime"
 version = "0.2.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 dependencies = [
  "futures-core",
  "futures-sink",
- "spin",
+ "spin 0.9.8",
 ]
 
 [[package]]
···
 ]
 
 [[package]]
+name = "futures"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876"
+dependencies = [
+ "futures-channel",
+ "futures-core",
+ "futures-executor",
+ "futures-io",
+ "futures-sink",
+ "futures-task",
+ "futures-util",
+]
+
+[[package]]
+name = "futures-buffered"
+version = "0.2.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a8e0e1f38ec07ba4abbde21eed377082f17ccb988be9d988a5adbf4bafc118fd"
+dependencies = [
+ "cordyceps",
+ "diatomic-waker",
+ "futures-core",
+ "pin-project-lite",
+ "spin 0.10.0",
+]
+
+[[package]]
 name = "futures-channel"
 version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6"
 
 [[package]]
+name = "futures-lite"
+version = "2.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad"
+dependencies = [
+ "fastrand",
+ "futures-core",
+ "futures-io",
+ "parking",
+ "pin-project-lite",
+]
+
+[[package]]
 name = "futures-macro"
 version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81"
 dependencies = [
+ "futures-channel",
  "futures-core",
  "futures-io",
  "futures-macro",
···
  "pin-project-lite",
  "pin-utils",
  "slab",
+]
+
+[[package]]
+name = "generator"
+version = "0.8.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "605183a538e3e2a9c1038635cc5c2d194e2ee8fd0d1b66b8349fad7dbacce5a2"
+dependencies = [
+ "cc",
+ "cfg-if",
+ "libc",
+ "log",
+ "rustversion",
+ "windows",
 ]
 
 [[package]]
···
  "js-sys",
  "log",
  "wasm-bindgen",
- "windows-core",
+ "windows-core 0.62.2",
 ]
 
 [[package]]
···
 name = "index"
 version = "0.1.0"
 dependencies = [
+ "base64 0.22.1",
  "env_logger",
+ "ipld-core",
  "jacquard",
+ "jacquard-repo",
  "log",
+ "serde",
+ "serde_ipld_dagcbor",
+ "serde_json",
  "sqlx",
  "thiserror 2.0.17",
  "tokio",
···
 ]
 
 [[package]]
+name = "iroh-car"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cb7f8cd4cb9aa083fba8b52e921764252d0b4dcb1cd6d120b809dbfe1106e81a"
+dependencies = [
+ "anyhow",
+ "cid",
+ "futures",
+ "serde",
+ "serde_ipld_dagcbor",
+ "thiserror 1.0.69",
+ "tokio",
+ "unsigned-varint 0.7.2",
+]
+
+[[package]]
 name = "is_terminal_polyfill"
 version = "1.70.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
  "bytes",
  "chrono",
  "cid",
+ "ed25519-dalek",
  "getrandom 0.2.16",
  "getrandom 0.3.4",
  "http",
···
 ]
 
 [[package]]
+name = "jacquard-repo"
+version = "0.9.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2ccdadfea11df142fbfb11cf7e5c1f7b8c0548758dedb0ce0eac182777f91f18"
+dependencies = [
+ "bytes",
+ "cid",
+ "ed25519-dalek",
+ "iroh-car",
+ "jacquard-common",
+ "jacquard-derive",
+ "k256",
+ "miette",
+ "multihash",
+ "n0-future",
+ "p256",
+ "serde",
+ "serde_bytes",
+ "serde_ipld_dagcbor",
+ "sha2",
+ "smol_str",
+ "thiserror 2.0.17",
+ "tokio",
+ "trait-variant",
+]
+
+[[package]]
 name = "jiff"
 version = "0.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
  "cfg-if",
  "ecdsa",
  "elliptic-curve",
+ "once_cell",
  "sha2",
+ "signature",
 ]
 
 [[package]]
···
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
 dependencies = [
- "spin",
+ "spin 0.9.8",
 ]
 
 [[package]]
···
 checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432"
 
 [[package]]
+name = "loom"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca"
+dependencies = [
+ "cfg-if",
+ "generator",
+ "scoped-tls",
+ "tracing",
+ "tracing-subscriber",
+]
+
+[[package]]
 name = "lru-cache"
 version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
  "proc-macro2",
  "quote",
  "syn 1.0.109",
+]
+
+[[package]]
+name = "matchers"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9"
+dependencies = [
+ "regex-automata",
 ]
 
 [[package]]
···
 dependencies = [
  "core2",
  "serde",
- "unsigned-varint",
+ "unsigned-varint 0.8.0",
 ]
 
 [[package]]
···
 ]
 
 [[package]]
+name = "n0-future"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7bb0e5d99e681ab3c938842b96fcb41bf8a7bb4bfdb11ccbd653a7e83e06c794"
+dependencies = [
+ "cfg_aliases",
+ "derive_more",
+ "futures-buffered",
+ "futures-lite",
+ "futures-util",
+ "js-sys",
+ "pin-project",
+ "send_wrapper",
+ "tokio",
+ "tokio-util",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "web-time",
+]
+
+[[package]]
 name = "ndk-context"
 version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 dependencies = [
  "memchr",
  "minimal-lexical",
+]
+
+[[package]]
+name = "nu-ansi-term"
+version = "0.50.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
+dependencies = [
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
···
 ]
 
 [[package]]
+name = "pin-project"
+version = "1.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a"
+dependencies = [
+ "pin-project-internal",
+]
+
+[[package]]
+name = "pin-project-internal"
+version = "1.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.109",
+]
+
+[[package]]
 name = "pin-project-lite"
 version = "0.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d"
 
 [[package]]
+name = "rustc_version"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92"
+dependencies = [
+ "semver",
+]
+
+[[package]]
 name = "rustix"
 version = "1.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 ]
 
 [[package]]
+name = "scoped-tls"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294"
+
+[[package]]
 name = "scopeguard"
 version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 ]
 
 [[package]]
+name = "send_wrapper"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cd0b0ec5f1c1ca621c432a25813d8d60c88abe6d3e08a3eb9cf37d97a0fe3d73"
+
+[[package]]
 name = "serde"
 version = "1.0.228"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 ]
 
 [[package]]
+name = "sharded-slab"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6"
+dependencies = [
+ "lazy_static",
+]
+
+[[package]]
 name = "shlex"
 version = "1.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 ]
 
 [[package]]
+name = "spin"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591"
+
+[[package]]
 name = "spki"
 version = "0.7.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 ]
 
 [[package]]
+name = "thread_local"
+version = "1.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
 name = "threadpool"
 version = "1.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
  "bytes",
  "futures-core",
  "futures-sink",
+ "futures-util",
  "pin-project-lite",
  "tokio",
 ]
···
 checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678"
 dependencies = [
  "once_cell",
+ "valuable",
+]
+
+[[package]]
+name = "tracing-log"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3"
+dependencies = [
+ "log",
+ "once_cell",
+ "tracing-core",
+]
+
+[[package]]
+name = "tracing-subscriber"
+version = "0.3.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2054a14f5307d601f88daf0553e1cbf472acc4f2c51afab632431cdcd72124d5"
+dependencies = [
+ "matchers",
+ "nu-ansi-term",
+ "once_cell",
+ "regex-automata",
+ "sharded-slab",
+ "smallvec",
+ "thread_local",
+ "tracing",
+ "tracing-core",
+ "tracing-log",
 ]
 
 [[package]]
···
 checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af"
 
 [[package]]
+name = "unicode-xid"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
+
+[[package]]
+name = "unsigned-varint"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6889a77d49f1f013504cec6bf97a2c730394adedaeb1deb5ea08949a50541105"
+
+[[package]]
 name = "unsigned-varint"
 version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
 version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
+
+[[package]]
+name = "valuable"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"
 
 [[package]]
 name = "vcpkg"
···
 ]
 
 [[package]]
+name = "windows"
+version = "0.61.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893"
+dependencies = [
+ "windows-collections",
+ "windows-core 0.61.2",
+ "windows-future",
+ "windows-link 0.1.3",
+ "windows-numerics",
+]
+
+[[package]]
+name = "windows-collections"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8"
+dependencies = [
+ "windows-core 0.61.2",
+]
+
+[[package]]
+name = "windows-core"
+version = "0.61.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3"
+dependencies = [
+ "windows-implement",
+ "windows-interface",
+ "windows-link 0.1.3",
+ "windows-result 0.3.4",
+ "windows-strings 0.4.2",
+]
+
+[[package]]
 name = "windows-core"
 version = "0.62.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
···
  "windows-link 0.2.1",
  "windows-result 0.4.1",
  "windows-strings 0.5.1",
+]
+
+[[package]]
+name = "windows-future"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e"
+dependencies = [
+ "windows-core 0.61.2",
+ "windows-link 0.1.3",
+ "windows-threading",
 ]
 
 [[package]]
···
 version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
+
+[[package]]
+name = "windows-numerics"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1"
+dependencies = [
+ "windows-core 0.61.2",
+ "windows-link 0.1.3",
+]
 
 [[package]]
 name = "windows-registry"
···
  "windows_x86_64_gnu 0.53.1",
  "windows_x86_64_gnullvm 0.53.1",
  "windows_x86_64_msvc 0.53.1",
+]
+
+[[package]]
+name = "windows-threading"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6"
+dependencies = [
+ "windows-link 0.1.3",
 ]
 
 [[package]]

Cargo.toml

···
 edition = "2024"
 
 [dependencies]
+base64 = "0.22.1"
 env_logger = "0.11.8"
+ipld-core = "0.4.2"
 jacquard = "0.9.0"
+jacquard-repo = "0.9.2"
 log = "0.4.28"
+serde = "1.0.228"
+serde_ipld_dagcbor = "0.6.4"
+serde_json = { version = "1.0.145", features = ["raw_value"] }
 sqlx = { version = "0.8.6", features = ["runtime-tokio", "postgres"] }
 thiserror = "2.0.17"
 tokio = "1.48.0"

src/backfill/load_car.rs

···
+use jacquard::api::com_atproto;
+use jacquard::client::Agent;
+use jacquard::types::{did::Did, tid::Tid};
+use jacquard::url::Url;
+use jacquard::xrpc::XrpcExt;
+use jacquard_repo::commit::Commit;
+use jacquard_repo::{BlockStore, MemoryBlockStore, Mst};
+use std::sync::Arc;
+use thiserror::Error;
+
+#[derive(Error, Debug)]
+pub enum Error {
+    #[error("Error loading car file: {}", .0)]
+    ClientError(#[from] jacquard::error::ClientError),
+    #[error("Error loading car file: {}", .0)]
+    RepoError(#[from] jacquard_repo::RepoError),
+    #[error("Missing root block from car file (malformed car file)")]
+    MissingRoot,
+}
+
+pub struct Car {
+    pub storage: MemoryBlockStore,
+    pub mst: Mst<MemoryBlockStore>,
+    pub rev: Tid,
+}
+
+impl PartialEq<Tid> for Car {
+    fn eq(&self, other: &Tid) -> bool {
+        &self.rev == other
+    }
+}
+
+use std::cmp::Ordering;
+impl PartialOrd<Tid> for Car {
+    fn partial_cmp(&self, other: &Tid) -> Option<Ordering> {
+        return match self.rev.compare_to(other) {
+            1 => Some(Ordering::Greater),
+            0 => Some(Ordering::Equal),
+            -1 => Some(Ordering::Less),
+            _ => None,
+        };
+    }
+}
+
+pub async fn load_car(user: Did<'_>, pds: Url) -> Result<Car, Error> {
+    let agent = Agent::unauthenticated();
+    let req = com_atproto::sync::get_repo::GetRepo::new()
+        .did(user)
+        .build();
+    let res = agent.xrpc(pds).send(&req).await?;
+
+    let car = res.buffer();
+    let car = jacquard_repo::car::parse_car_bytes(&car).await?;
+
+    let storage = jacquard_repo::storage::MemoryBlockStore::new_from_blocks(car.blocks);
+
+    let root = storage
+        .get(&car.root)
+        .await?
+        .ok_or_else(|| Error::MissingRoot)?;
+    let root = Commit::from_cbor(&root)?;
+    let rev = root.rev().to_owned();
+    let root = root.data();
+
+    let mst = Mst::load(Arc::new(storage.clone()), *root, None);
+
+    Ok(Car { storage, mst, rev })
+}

src/backfill/mod.rs

···
+use std::{cmp::Ordering, str::FromStr};
+
+use jacquard::{types::tid::Tid, url::Url};
+use sqlx::{Pool, Postgres, query};
+use thiserror::Error;
+
+use crate::{
+    backfill::{load_car::load_car, parse_car::parse_car},
+    config,
+};
+
+pub mod load_car;
+pub mod parse_car;
+
+const DB_MAX_REQ: usize = 65535;
+
+#[derive(Error, Debug)]
+pub enum Error {
+    #[error("Error parsing TID: {}", .0)]
+    TidParse(#[from] jacquard::types::string::AtStrError),
+    #[error("{}", .0)]
+    GetCarError(#[from] crate::backfill::load_car::Error),
+    #[error(
+        "The database claims to be more up to date than the PDS.
+Most likely either the PDS or repo is broken, or the database has been corrupted.
+Check your PDS repo is working and/or drop the database."
+    )]
+    DbTidTooLow,
+    #[error("Database error: {}", .0)]
+    DbError(#[from] sqlx::Error),
+    #[error("{}", .0)]
+    ParseCarError(#[from] crate::backfill::parse_car::Error),
+}
+
+/// backfill works as follows (https://docs.bsky.app/docs/advanced-guides/backfill)
+///
+/// 1. resolve did -> pds
+/// 2. stream com.atproto.sync.subscribeRepos to a buffer
+/// 3. get a car file from com.atproto.sync.getRepo (diff if a rev is stored in database)
+/// 4. apply car file diff to database (incl rev)
+/// 5. start playing events from buffer
+///       1. drop all events from other users
+///       2. drop all events with a lower rev than current rev
+///       3. apply event & update rev
+///       4. (non blocking) get blobs if missing
+///       5. (non blocking) parse for strongref and store strongrefs
+///       6. (non blocking) trigger garbage collection of blobs and strongref
+/// 6. once buffer is empty, parse events live
+pub async fn backfill(pds: &str, conn: &Pool<Postgres>) -> Result<(), Error> {
+    let db_rev = if let Some(rev) = query!(
+        "SELECT (rev) FROM meta WHERE did = $1",
+        config::USER.to_string()
+    )
+    .fetch_one(conn)
+    .await
+    .ok()
+    .and_then(|x| x.rev)
+    {
+        Tid::from_str(&rev)?
+    } else {
+        Tid::from_time(0, 0)
+    };
+
+    let pds = Url::from_str(&format!("https://{pds}/")).unwrap();
+    let car = load_car(config::USER.clone(), pds).await?;
+
+    match car.partial_cmp(&db_rev) {
+        Some(val) => match val {
+            // car rev newer than db rev
+            // continue on; every other branch diverges
+            Ordering::Greater => {}
+            // revisions are the same so we can skip backfill
+            Ordering::Equal => return Ok(()),
+            // db rev newer than car rev
+            // this means the db or car file is borked
+            // panic out and let the user deal with things
+            Ordering::Less => return Err(Error::DbTidTooLow),
+            //             panic!(
+            //                 r"The database claims to be more up to date than the PDS.
+            // Most likely either the PDS or repo is broken, or the database has been corrupted.
+            // Check your PDS repo is working and/or drop the database."
+            //             ),
+        },
+        // cant compare rev so assume all is ok and continue
+        None => {}
+    };
+
+    // erase all old records and return if it fails
+    // we dont use diffs bc theyre complex and the overhead is minimal rn
+    // only real overhead is network latency which would be ~= anyway
+    let _ = query!("DELETE FROM records").execute(conn).await?;
+
+    let data = parse_car(&car).await?;
+    let mut data = data.chunks(DB_MAX_REQ / 4);
+
+    while let Some(data) = data.next() {
+        let mut query = sqlx::QueryBuilder::new("INSERT INTO records(collection, rkey, record) ");
+        query.push_values(
+            data,
+            |mut b: sqlx::query_builder::Separated<'_, '_, Postgres, &'static str>, data| {
+                b.push_bind(data.0.0.clone())
+                    .push_bind(data.0.1.clone())
+                    .push_bind(data.1.clone());
+            },
+        );
+
+        match query.build().execute(conn).await {
+            Err(err) => {
+                // couldnt backfill so go nuclear
+                // this is program startup so its prolly safe lol
+                println!("Got error \"{}\"\nDeleting records and exiting...", err);
+                let _ = query!("DELETE FROM records").execute(conn).await?;
+                panic!()
+            }
+            _ => {}
+        };
+    }
+
+    match query!(
+        "UPDATE meta SET rev = $1 WHERE did = $2",
+        car.rev.to_string(),
+        config::USER.to_string()
+    )
+    .execute(conn)
+    .await
+    {
+        Err(err) => {
+            // couldnt save tid so go nuclear
+            // this is program startup so its prolly safe lol
+            println!("Got error \"{}\"\nDeleting records and exiting...", err);
+            let _ = query!("DELETE FROM records").execute(conn).await?;
+            panic!()
+        }
+        _ => {}
+    };
+
+    Ok(())
+}

src/backfill/parse_car.rs

···
+use std::iter::zip;
+
+use ipld_core::{cid::CidGeneric, ipld::Ipld};
+use jacquard::smol_str::SmolStr;
+use jacquard_repo::BlockStore;
+use serde_json::Value;
+use thiserror::Error;
+
+use crate::{backfill::load_car::Car, utils::ipld_json::ipld_to_json_value};
+
+#[derive(Debug, Error)]
+pub enum Error {
+    #[error("Error getting records from car file: {}", .0)]
+    RepoError(#[from] jacquard_repo::RepoError),
+    #[error("Missing CID from car file")]
+    MissingCid,
+    #[error("Could not decode record: {}", .0)]
+    DecodeError(#[from] serde_ipld_dagcbor::DecodeError<std::convert::Infallible>),
+    #[error("Could not convert into json: {}", .0)]
+    IpldToJsonError(#[from] crate::utils::ipld_json::Error),
+    #[error("Could not break {} into a collection and rkey", .0)]
+    MalformedRecordKey(SmolStr),
+}
+
+pub type AccountData = Vec<((String, String), Value)>;
+
+pub async fn parse_car(car: &Car) -> Result<AccountData, Error> {
+    let (keys, records): (Vec<SmolStr>, Vec<CidGeneric<64>>) =
+        car.mst.leaves().await?.into_iter().unzip();
+
+    // convert keys into (collection, rkey)
+    let keys = keys
+        .into_iter()
+        .map(|x| {
+            let mut parts = x.split('/');
+            let collection = parts.next();
+            let rkey = parts.next();
+            if parts.next().is_none()
+                && let Some(collection) = collection
+                && let Some(rkey) = rkey
+            {
+                Ok::<_, Error>((collection.to_string(), rkey.to_string()))
+            } else {
+                Err(Error::MalformedRecordKey(x))
+            }
+        })
+        .collect::<Result<Vec<_>, _>>()?;
+
+    // convert records into Value
+    let records = &records[..];
+    let records = car
+        .storage
+        .get_many(records)
+        .await?
+        .into_iter()
+        .collect::<Option<Vec<_>>>()
+        .ok_or_else(|| Error::MissingCid)?
+        .into_iter()
+        .map(|x| {
+            let data = serde_ipld_dagcbor::from_slice::<Ipld>(&x)?;
+            let value = ipld_to_json_value(&data)?;
+            Ok::<_, Error>(value)
+        })
+        .collect::<Result<Vec<_>, _>>()?;
+
+    let data = zip(keys, records).collect::<Vec<((_, _), _)>>();
+
+    Ok(data)
+}

src/main.rs

···
 use sqlx::{Pool, Postgres};
 
+use crate::backfill::backfill;
+
+mod backfill;
 mod config;
 mod db;
 mod utils;
···
     let pds = match utils::resolver::resolve(&config::USER).await {
         Ok(val) => val,
         Err(err) => panic!("{}", err),
+    };
+
+    let backfilled = backfill(&pds, &conn).await;
+    if let Err(err) = backfilled {
+        println!("{}", err);
+        return Err(());
     };
 
     println!("Completed sucessfully!");

src/utils/ipld_json.rs

···
+use base64::{Engine, prelude::BASE64_STANDARD_NO_PAD};
+use ipld_core::{cid::multibase::Base, ipld::Ipld};
+use log::warn;
+use serde_json::{Map, Number, Value, json};
+use thiserror::Error;
+
+#[derive(Error, Debug)]
+pub enum Error {
+    #[error("CID error: {}", .0)]
+    CidError(#[from] ipld_core::cid::Error),
+    #[error("Number too big: {0} > {1} || {0} < {2}", .val, u64::MAX, i64::MIN)]
+    IntInvalidSize { val: i128 },
+    #[error("{} was NaN or Infinity", .0)]
+    FloatInvalidSize(f64),
+}
+
+/// Convert any decoded IPLD data into serde_json Value
+///
+/// This can be used for `sqlx` queries or decoded into plain JSON  
+///
+/// We can't use bog standard `to_string` or `to_json` functions
+/// due to the fact that ATProto DAG-CBOR does not map straight
+/// to ATProto JSON (see: `Ipld::Bytes` and `Ipld::Link` in the function definition.)
+///
+/// To get an atproto JSON representation in string format call
+/// `Value::to_string(&self)` on the Value from this function
+pub fn ipld_to_json_value(data: &Ipld) -> Result<Value, Error> {
+    Ok(match data {
+        Ipld::Null => Value::Null,
+        Ipld::Bool(bool) => Value::Bool(*bool),
+        Ipld::Integer(int) => Value::Number(
+            Number::from_i128(*int).ok_or_else(|| Error::IntInvalidSize { val: *int })?,
+        ),
+        Ipld::Float(float) => {
+            warn!("Got float in IPLD data: {}", float);
+            Value::Number(Number::from_f64(*float).ok_or_else(|| Error::FloatInvalidSize(*float))?)
+        }
+        Ipld::String(str) => Value::String(str.clone()),
+        Ipld::Bytes(items) => json!({ "$bytes": BASE64_STANDARD_NO_PAD.encode(items) }),
+        Ipld::List(iplds) => Value::Array(
+            iplds
+                .into_iter()
+                .map(|x| ipld_to_json_value(x))
+                .collect::<Result<Vec<_>, _>>()?,
+        ),
+        Ipld::Map(map) => Value::Object(
+            map.into_iter()
+                .map(|(k, v)| Ok::<_, Error>((k.clone(), ipld_to_json_value(v)?)))
+                .collect::<Result<Map<String, Value>, _>>()?,
+        ),
+        Ipld::Link(cid) => json!({"$link": 
+            cid.to_string_of_base(Base::Base32Lower)? }),
+    })
+}

src/utils/mod.rs

···
+pub mod ipld_json;
 pub mod resolver;