skyscheduler / src / utils / bsky / bskyPrune.ts

import isEmpty from 'just-is-empty';
import split from 'just-split';
import type { AllContext } from '../../types';
import { getAllPostedPosts, getAllPostedPostsOfUser } from '../db/data';
import { getPostRecords } from './bskyApi';

// This looks for a bunch of posts that are posted and determines if the posts
// are still on the network or not. If they are not, then this prunes the posts from
// the database. This call is quite expensive and should only be ran on a weekly
// cron job.
export const pruneBskyPosts = async (c: AllContext, userId?: string) => {
  const allPostedPosts = (userId !== undefined) ? await getAllPostedPostsOfUser(c, userId) : await getAllPostedPosts(c);
  let removePostIds: string[] = [];
  let postedGroups = split(allPostedPosts, 25);
  while (!isEmpty(postedGroups)) {
    const currentGroup = postedGroups.pop();
    // This technically shouldn't be possible because we do a check in the while loop beforehand.
    if (currentGroup === undefined) {
      console.error("current group was somehow undefined...");
      break;
    }

    // Create a map and an array so we can do lookups for reconciliation later
    // to figure out what posts can be removed.
    let urisOnly: string[] = [];
    const postMap: Map<string, string> = new Map();
    currentGroup.forEach((itm) => {
      if (itm.uri === null) {
        console.log(`Skipping a "posted" post that does not have an uri: ${itm.id}`);
        return;
      }
      postMap.set(itm.uri, itm.id);
      urisOnly.push(itm.uri);
    });

    try {
      // Records from the public bsky api service
      const bskyLookupResults = await getPostRecords(urisOnly);
      if (bskyLookupResults !== null) {
        // There is a discrepancy, we need to find which record it is.
        if (bskyLookupResults.length !== urisOnly.length) {
          // Go through the entire lookup results
          while (!isEmpty(bskyLookupResults)) {
            // remove the object at the end so we can try to figure out what's left
            const currentRecord = bskyLookupResults.pop();
            if (currentRecord === undefined)
              break;

            // if this record exists, remove it from our post map
            // we will reconcile outside of this loop to delete the posts.
            postMap.delete(currentRecord.uri);
          }
          // Add all the deleted keys to the main array.
          postMap.forEach((value, key) => {
            removePostIds.push(value);
          });
        }
      }
    } catch(err) {
      console.error(`encountered error trying to prune: ${err}`);
      // Remove all the posts in the current post group.
      continue;
    }
  }
  return removePostIds;
};