store cid in db

this should prepare us for backfill later and allow strongrefs etc bc thats just like. sane lol

migrations/0002_cid-for-records.sql

···
+-- Add migration script here
+
+ALTER TABLE records ADD cid TEXT;
+
+UPDATE records SET cid = '' WHERE cid IS NULL;
+
+ALTER TABLE records ALTER COLUMN cid SET NOT NULL;

src/backfill/mod.rs

···
 
 use std::str::FromStr;
 
+use ipld_core::cid::multibase::Base;
 use jacquard::url::Url;
 use sqlx::{Pool, Postgres, query};
 use thiserror::Error;
···
 pub mod load_car;
 pub mod parse_car;
 
-const DB_MAX_REQ: usize = 65535;
-
 #[derive(Error, Debug)]
 pub enum Error {
     #[error("Error parsing TID: {}", .0)]
···
     Db(#[from] sqlx::Error),
     #[error("{}", .0)]
     ParseCar(#[from] crate::backfill::parse_car::Error),
+    #[error("Error processing cid: {}", .0)]
+    Cid(#[from] ipld_core::cid::Error),
 }
 
 pub async fn backfill(
···
     // only real overhead is network latency which would be ~= anyway
     let _ = query!("DELETE FROM records").execute(conn).await?;
 
-    let data = parse_car(&car).await?;
-    let data = data.chunks(DB_MAX_REQ / 4);
+    let data = parse_car(&car)
+        .await?
+        .into_iter()
+        .map(|(collection, rkey, cid, value)| {
+            Ok::<_, Error>((
+                collection,
+                rkey,
+                cid.to_string_of_base(Base::Base32Lower)?,
+                value,
+            ))
+        })
+        .collect::<Result<Vec<_>, _>>()?;
+    let data = data.chunks(config::DB_MAX_REQ / 4);
 
     if let Some(time) = time {
         println!("Parsed car file ({:?})", time.elapsed());
···
     let time = time.map(|_| std::time::Instant::now());
 
     for data in data {
-        let mut query = sqlx::QueryBuilder::new("INSERT INTO records(collection, rkey, record) ");
+        let mut query =
+            sqlx::QueryBuilder::new("INSERT INTO records(collection, rkey, cid, record) ");
         query.push_values(
-            data,
+            data.to_owned(),
             |mut b: sqlx::query_builder::Separated<'_, '_, Postgres, &'static str>, data| {
-                b.push_bind(data.0.0.clone())
-                    .push_bind(data.0.1.clone())
-                    .push_bind(data.1.clone());
+                b.push_bind(data.0)
+                    .push_bind(data.1)
+                    .push_bind(data.2)
+                    .push_bind(data.3);
             },
         );
 

src/backfill/parse_car.rs

···
     IpldToJson(#[from] crate::utils::ipld_json::Error),
     #[error("Could not break {} into a collection and rkey", .0)]
     MalformedRecordKey(SmolStr),
+    #[error("Could not generate cid for commit: {}", .0)]
+    Cid(#[from] ipld_core::cid::Error),
 }
 
-pub type AccountData = Vec<((String, String), Value)>;
+pub type AccountData = Vec<(String, String, CidGeneric<64>, Value)>;
 
 pub async fn parse_car(car: &Car) -> Result<AccountData, Error> {
-    let (keys, records): (Vec<SmolStr>, Vec<CidGeneric<64>>) =
+    let (keys, record_cids): (Vec<SmolStr>, Vec<CidGeneric<64>>) =
         car.mst.leaves().await?.into_iter().unzip();
 
     // convert keys into (collection, rkey)
···
         .collect::<Result<Vec<_>, _>>()?;
 
     // convert records into Value
-    let records = &records[..];
+    let record_cids = &record_cids[..];
     let records = car
         .storage
-        .get_many(records)
+        .get_many(record_cids)
         .await?
         .into_iter()
         .collect::<Option<Vec<_>>>()
-        .ok_or_else(|| Error::MissingCid)?
-        .into_iter()
-        .map(|x| {
-            let data = serde_ipld_dagcbor::from_slice::<Ipld>(&x)?;
+        .ok_or_else(|| Error::MissingCid)?;
+
+    let records = zip(records, record_cids)
+        .map(|(bytes, cid)| {
+            let data = serde_ipld_dagcbor::from_slice::<Ipld>(&bytes)?;
             let value = ipld_to_json_value(&data)?;
-            Ok::<_, Error>(value)
+            Ok::<(CidGeneric<64>, Value), Error>((cid.to_owned(), value))
         })
         .collect::<Result<Vec<_>, _>>()?;
 
-    let data = zip(keys, records).collect::<Vec<((_, _), _)>>();
+    let data = zip(keys, records)
+        .map(|((collection, rkey), (cid, record))| (collection, rkey, cid, record))
+        .collect();
 
     Ok(data)
 }

src/config.rs

···
 use std::env;
 use std::sync::LazyLock;
 
+pub const DB_MAX_REQ: usize = 65535;
+
 // this should be loaded before the program starts any threads
 // if this panics threads that access it will be poisoned
 pub static USER: LazyLock<Did<'static>> = LazyLock::new(|| {