correct threadgate backfilling

needed to do a reorder of COPY to make sure everything was in before calculating gates

consumer/src/backfill/mod.rs

···
     follows: Vec<(String, String, DateTime<Utc>)>,
     list_items: Vec<(String, records::AppBskyGraphListItem)>,
     verifications: Vec<(String, Cid, records::AppBskyGraphVerification)>,
+    threadgates: Vec<(String, Cid, records::AppBskyFeedThreadgate)>, // not COPY'd but needs to be kept until last.
     records: Vec<(String, Cid)>,
 }
 
 impl CopyStore {
     async fn submit(self, t: &mut Transaction<'_>, did: &str) -> Result<(), tokio_postgres::Error> {
         db::copy::copy_likes(t, did, self.likes).await?;
-        db::copy::copy_posts(t, did, self.posts).await?;
         db::copy::copy_reposts(t, did, self.reposts).await?;
         db::copy::copy_blocks(t, did, self.blocks).await?;
         db::copy::copy_follows(t, did, self.follows).await?;
         db::copy::copy_list_items(t, self.list_items).await?;
         db::copy::copy_verification(t, did, self.verifications).await?;
+        db::copy::copy_posts(t, did, self.posts).await?;
+        for (at_uri, cid, record) in self.threadgates {
+            db::threadgate_enforce_backfill(t, did, &record).await?;
+            db::threadgate_upsert(t, &at_uri, cid, record).await?;
+        }
         db::copy::copy_records(t, did, self.records).await?;
 
         Ok(())

consumer/src/backfill/repo.rs

···
 };
 use crate::indexer::records;
 use crate::indexer::types::{AggregateDeltaStore, RecordTypes};
+use crate::utils::at_uri_is_by;
 use crate::{db, indexer};
 use deadpool_postgres::Transaction;
 use ipld_core::cid::Cid;
···
             copies
                 .reposts
                 .push((rkey.to_string(), rec.subject, rec.via, rec.created_at));
+        }
+        RecordTypes::AppBskyFeedThreadgate(record) => {
+            if !at_uri_is_by(&record.post, did) {
+                tracing::warn!("tried to create a threadgate on a post we don't control!");
+                return Ok(());
+            }
+
+            copies.push_record(&at_uri, cid);
+            copies.threadgates.push((at_uri, cid, record));
         }
         RecordTypes::AppBskyGraphBlock(rec) => {
             copies.push_record(&at_uri, cid);

consumer/src/db/gates.rs

···
 use super::{PgExecResult, PgResult};
 use crate::indexer::records::{
-    THREADGATE_RULE_FOLLOWER, THREADGATE_RULE_FOLLOWING, THREADGATE_RULE_LIST,
-    THREADGATE_RULE_MENTION,
+    AppBskyFeedThreadgate, ThreadgateRule, THREADGATE_RULE_FOLLOWER, THREADGATE_RULE_FOLLOWING,
+    THREADGATE_RULE_LIST, THREADGATE_RULE_MENTION,
 };
 use chrono::prelude::*;
 use chrono::{DateTime, Utc};
···
     )
     .await
 }
+
+// variant of post_enforce_threadgate that runs when backfilling to clean up any posts already in DB
+pub async fn threadgate_enforce_backfill<C: GenericClient>(
+    conn: &mut C,
+    root_author: &str,
+    threadgate: &AppBskyFeedThreadgate,
+) -> PgExecResult {
+    // pull out allow - if it's None we can skip this gate.
+    let Some(allow) = threadgate.allow.as_ref() else {
+        return Ok(0);
+    };
+
+    let root = &threadgate.post;
+
+    if allow.is_empty() {
+        // blind update everything
+        return conn.execute(
+            "UPDATE posts SET violates_threadgate=TRUE WHERE root_uri=$1 AND did != $2 AND created_at >= $3",
+            &[&root, &root_author, &threadgate.created_at],
+        ).await;
+    }
+
+    // pull authors with our root_uri where the author is not the root author and are dated after created_at
+    // this is mutable because we'll remove ALLOWED dids
+    let mut dids: HashSet<String> = conn
+        .query(
+            "SELECT DISTINCT did FROM posts WHERE root_uri=$1 AND did != $2 AND created_at >= $3",
+            &[&root, &root_author, &threadgate.created_at],
+        )
+        .await?
+        .into_iter()
+        .map(|row| row.get(0))
+        .collect();
+
+    // this will be empty if there are no replies.
+    if dids.is_empty() {
+        return Ok(0);
+    }
+
+    let allowed_lists = allow
+        .iter()
+        .filter_map(|rule| match rule {
+            ThreadgateRule::List { list } => Some(list),
+            _ => None,
+        })
+        .collect::<Vec<_>>();
+
+    let allow: HashSet<_> = HashSet::from_iter(allow.into_iter().map(|v| v.as_str()));
+
+    if allow.contains(THREADGATE_RULE_FOLLOWER) && !dids.is_empty() {
+        let current_dids: Vec<_> = dids.iter().collect();
+
+        let res = conn.query(
+            "SELECT subject FROM profile_states WHERE did=$1 AND subject=ANY($2) AND followed IS NOT NULL",
+            &[&root_author, &current_dids]
+        ).await?;
+
+        dids = &dids - &HashSet::from_iter(res.into_iter().map(|r| r.get(0)));
+    }
+
+    if allow.contains(THREADGATE_RULE_FOLLOWING) && !dids.is_empty() {
+        let current_dids: Vec<_> = dids.iter().collect();
+
+        let res = conn.query(
+            "SELECT subject FROM profile_states WHERE did=$1 AND subject=ANY($2) AND following IS NOT NULL",
+            &[&root_author, &current_dids]
+        ).await?;
+
+        dids = &dids - &HashSet::from_iter(res.into_iter().map(|r| r.get(0)));
+    }
+
+    if allow.contains(THREADGATE_RULE_MENTION) && !dids.is_empty() {
+        let mentions: Vec<String> = conn
+            .query_opt("SELECT mentions FROM posts WHERE at_uri=$1", &[&root])
+            .await?
+            .map(|r| r.get(0))
+            .unwrap_or_default();
+
+        dids = &dids - &HashSet::from_iter(mentions);
+    }
+
+    if allow.contains(THREADGATE_RULE_LIST) && !dids.is_empty() {
+        let current_dids: Vec<_> = dids.iter().collect();
+
+        let res = conn
+            .query(
+                "SELECT subject FROM list_items WHERE list_uri = ANY($1) AND subject = ANY($2)",
+                &[&allowed_lists, &current_dids],
+            )
+            .await?;
+
+        dids = &dids - &HashSet::from_iter(res.into_iter().map(|r| r.get(0)));
+    }
+
+    let dids = dids.into_iter().collect::<Vec<_>>();
+
+    conn.execute(
+        "UPDATE posts SET violates_threadgate=TRUE WHERE root_uri = $1 AND did = ANY($2) AND created_at >= $3",
+        &[&threadgate.post, &dids, &threadgate.created_at]
+    ).await
+}