Dynamic post loading (#2)

Dynamically load the posts so that you can scroll a chronologically sorted timeline infinitely

Reviewed-on: https://git.witchcraft.systems/scientific-witchery/pds-dash/pulls/2
Co-authored-by: ari <[e-mail]>
Co-committed-by: ari <[e-mail]>

config.ts

···
   static readonly PDS_URL: string = "https://pds.witchcraft.systems";
 
   /**
-   * The base URL of the frontend service for linking to replies
+   * The base URL of the frontend service for linking to replies/quotes/accounts etc.
    * @default "https://deer.social"
    */
   static readonly FRONTEND_URL: string = "https://deer.social";
 
   /**
-   * Maximum number of posts to show in the feed (across all users)
-   * @default 100
+   * Maximum number of posts to fetch from the PDS per request
+   * Should be around 20 for about 10 users on the pds
+   * The more users you have, the lower the number should be
+   * since sorting is slow and is done on the frontend
+   * @default 20
    */
-  static readonly MAX_POSTS: number = 100;
+  static readonly MAX_POSTS: number = 20;
 
   /**
    * Footer text for the dashboard
···
   static readonly FOOTER_TEXT: string =
     "Astrally projected from <a href='https://witchcraft.systems' target='_blank'>witchcraft.systems</a>";
 
-    /**
-     * Whether to show the posts that are in the future
-     * @default false
-     */
-    static readonly SHOW_FUTURE_POSTS: boolean = false;
+  /**
+   * Whether to show the posts that are in the future
+   * @default false
+   */
+  static readonly SHOW_FUTURE_POSTS: boolean = false;
 }

deno.lock

···
     "npm:@tsconfig/svelte@^5.0.4": "5.0.4",
     "npm:moment@^2.30.1": "2.30.1",
     "npm:svelte-check@^4.1.5": "4.1.6_svelte@5.28.1__acorn@8.14.1_typescript@5.7.3",
+    "npm:svelte-infinite-loading@^1.4.0": "1.4.0",
     "npm:svelte@^5.23.1": "5.28.1_acorn@8.14.1",
     "npm:typescript@~5.7.2": "5.7.3",
     "npm:vite@^6.3.1": "6.3.2_picomatch@4.0.2"
···
         "typescript"
       ]
     },
+    "svelte-infinite-loading@1.4.0": {
+      "integrity": "sha512-Jo+f/yr/HmZQuIiiKKzAHVFXdAUWHW2RBbrcQTil8JVk1sCm/riy7KTJVzjBgQvHasrFQYKF84zvtc9/Y4lFYg=="
+    },
     "svelte@5.28.1_acorn@8.14.1": {
       "integrity": "sha512-iOa9WmfNG95lSOSJdMhdjJ4Afok7IRAQYXpbnxhd5EINnXseG0GVa9j6WPght4eX78XfFez45Fi+uRglGKPV/Q==",
       "dependencies": [
···
         "npm:@tsconfig/svelte@^5.0.4",
         "npm:moment@^2.30.1",
         "npm:svelte-check@^4.1.5",
+        "npm:svelte-infinite-loading@^1.4.0",
         "npm:svelte@^5.23.1",
         "npm:typescript@~5.7.2",
         "npm:vite@^6.3.1"

package.json

···
     "@atcute/bluesky": "^2.0.2",
     "@atcute/client": "^3.0.1",
     "@atcute/identity-resolver": "^0.1.2",
-    "moment": "^2.30.1"
+    "moment": "^2.30.1",
+    "svelte-infinite-loading": "^1.4.0"
   },
   "devDependencies": {
     "@sveltejs/vite-plugin-svelte": "^5.0.3",

src/App.svelte

···
 <script lang="ts">
   import PostComponent from "./lib/PostComponent.svelte";
   import AccountComponent from "./lib/AccountComponent.svelte";
-  import { fetchAllPosts, Post, getAllMetadataFromPds } from "./lib/pdsfetch";
+  import InfiniteLoading from "svelte-infinite-loading";
+  import { getNextPosts, Post, getAllMetadataFromPds } from "./lib/pdsfetch";
   import { Config } from "../config";
-  const postsPromise = fetchAllPosts();
   const accountsPromise = getAllMetadataFromPds();
+  import { onMount } from "svelte";
+
+  let posts: Post[] = [];
+
+  onMount(() => {
+    // Fetch initial posts
+    getNextPosts().then((initialPosts) => {
+      posts = initialPosts;
+    });
+  });
+  // Infinite loading function
+  const onInfinite = ({ detail: { loaded, complete } } : { detail : { loaded : () => void, complete : () => void}}) => {
+    getNextPosts().then((newPosts) => {
+      if (newPosts.length > 0) {
+        posts = [...posts, ...newPosts];
+        loaded();
+      } else {
+        complete();
+      }
+    });
+  };
 </script>
 
 <main>
···
       <p>Error: {error.message}</p>
     {/await}
 
-    {#await postsPromise}
-      <p>Loading...</p>
-    {:then postsData}
-      <div id="Feed">
-        <div id="spacer"></div>
-        {#each postsData as postObject}
-          <PostComponent post={postObject as Post} />
-        {/each}
-        <div id="spacer"></div>
-      </div>
-    {/await}
+    <div id="Feed">
+      <div id="spacer"></div>
+      {#each posts as postObject}
+        <PostComponent post={postObject as Post} />
+      {/each}
+      <InfiniteLoading on:infinite={onInfinite} 
+        distance={0}
+      />
+      <div id="spacer"></div>
+    </div>
   </div>
 </main>
 

src/lib/PostComponent.svelte

···
           <div id="carouselControls">
             <button
               id="prevBtn"
-              on:click={prevImage}
+              onclick={prevImage}
               disabled={currentImageIndex === 0}>←</button
             >
             <div id="carouselIndicators">
···
             </div>
             <button
               id="nextBtn"
-              on:click={nextImage}
+              onclick={nextImage}
               disabled={currentImageIndex === post.imagesCid.length - 1}
               >→</button
             >

src/lib/pdsfetch.ts

···
 // import { AppBskyActorDefs } from "@atcute/client/lexicons";
 
 interface AccountMetadata {
-  did: string;
+  did: At.Did;
   displayName: string;
   handle: string;
   avatarCid: string | null;
+  currentCursor?: string;
 }
+
+let accountsMetadata: AccountMetadata[] = [];
+
 interface atUriObject {
   repo: string;
   collection: string;
···
 
   constructor(
     record: ComAtprotoRepoListRecords.Record,
-    account: AccountMetadata
+    account: AccountMetadata,
   ) {
     this.postCid = record.cid;
     this.recordName = processAtUri(record.uri).rkey;
···
     switch (post.embed?.$type) {
       case "app.bsky.embed.images":
         this.imagesCid = post.embed.images.map(
-          (imageRecord: any) => imageRecord.image.ref.$link
+          (imageRecord: any) => imageRecord.image.ref.$link,
         );
         break;
       case "app.bsky.embed.video":
···
         switch (post.embed.media.$type) {
           case "app.bsky.embed.images":
             this.imagesCid = post.embed.media.images.map(
-              (imageRecord) => imageRecord.image.ref.$link
+              (imageRecord) => imageRecord.image.ref.$link,
             );
 
             break;
···
   return data.repos.map((repo: any) => repo.did) as At.Did[];
 };
 const getAccountMetadata = async (
-  did: `did:${string}:${string}`
-): Promise<AccountMetadata> => {
+  did: `did:${string}:${string}`,
+) => {
   // gonna assume self exists in the app.bsky.actor.profile
   try {
     const { data } = await rpc.get("com.atproto.repo.getRecord", {
···
     return account;
   } catch (e) {
     console.error(`Error fetching metadata for ${did}:`, e);
-    return {
-      did: "error",
-      displayName: "",
-      avatarCid: null,
-      handle: "error",
-    };
+    return null;
   }
 };
 
···
   const metadata = await Promise.all(
     dids.map(async (repo: `did:${string}:${string}`) => {
       return await getAccountMetadata(repo);
-    })
+    }),
   );
-  return metadata.filter((account) => account.did !== "error");
-};
-
-const fetchPosts = async (did: string) => {
-  try {
-    const { data } = await rpc.get("com.atproto.repo.listRecords", {
-      params: {
-        repo: did as At.Identifier,
-        collection: "app.bsky.feed.post",
-        limit: Config.MAX_POSTS,
-      },
-    });
-    return {
-      records: data.records as ComAtprotoRepoListRecords.Record[],
-      did: did,
-      error: false,
-    };
-  } catch (e) {
-    console.error(`Error fetching posts for ${did}:`, e);
-    return {
-      records: [],
-      did: did,
-      error: true,
-    };
-  }
+  return metadata.filter((account) => account !== null) as AccountMetadata[];
 };
 
 const identityResolve = async (did: At.Did) => {
···
 
   if (did.startsWith("did:plc:") || did.startsWith("did:web:")) {
     const doc = await resolver.resolve(
-      did as `did:plc:${string}` | `did:web:${string}`
+      did as `did:plc:${string}` | `did:web:${string}`,
     );
     return doc;
   } else {
···
   }
 };
 
-const fetchAllPosts = async () => {
-  const users: AccountMetadata[] = await getAllMetadataFromPds();
-  const postRecords = await Promise.all(
-    users.map(
-      async (metadata: AccountMetadata) => await fetchPosts(metadata.did)
-    )
-  );
-  const validPostRecords = postRecords.filter((record) => !record.error);
-  const posts: Post[] = validPostRecords.flatMap((userFetch) =>
-    userFetch.records.map((record) => {
-      const user = users.find(
-        (user: AccountMetadata) => user.did == userFetch.did
+interface PostsAcc {
+  posts: ComAtprotoRepoListRecords.Record[];
+  account: AccountMetadata;
+}
+const getCutoffDate = (postAccounts: PostsAcc[]) => {
+  const now = Date.now();
+  let cutoffDate: Date | null = null;
+  postAccounts.forEach((postAcc) => {
+    const latestPost = new Date(
+      (postAcc.posts[postAcc.posts.length - 1].value as AppBskyFeedPost.Record)
+        .createdAt,
+    );
+    if (!cutoffDate) {
+      cutoffDate = latestPost;
+    } else {
+      if (latestPost > cutoffDate) {
+        cutoffDate = latestPost;
+      }
+    }
+  });
+  if (cutoffDate) {
+    return cutoffDate;
+  } else {
+    return new Date(now);
+  }
+};
+
+const filterPostsByDate = (posts: PostsAcc[], cutoffDate: Date) => {
+  // filter posts for each account that are older than the cutoff date and save the cursor of the last post included
+  const filteredPosts: PostsAcc[] = posts.map((postAcc) => {
+    const filtered = postAcc.posts.filter((post) => {
+      const postDate = new Date(
+        (post.value as AppBskyFeedPost.Record).createdAt,
+      );
+      return postDate >= cutoffDate;
+    });
+    if (filtered.length > 0) {
+      postAcc.account.currentCursor = processAtUri(filtered[filtered.length - 1].uri).rkey;
+    }
+    return {
+      posts: filtered,
+      account: postAcc.account,
+    };
+  });
+  return filteredPosts;
+};
+// nightmare function. However it works so I am not touching it
+const getNextPosts = async () => {
+  if (!accountsMetadata.length) {
+    accountsMetadata = await getAllMetadataFromPds();
+  }
+
+  const postsAcc: PostsAcc[] = await Promise.all(
+    accountsMetadata.map(async (account) => {
+      const posts = await fetchPostsForUser(
+        account.did,
+        account.currentCursor || null,
       );
-      if (!user) {
-        throw new Error(`User with DID ${userFetch.did} not found`);
+      if (posts) {
+        return {
+          posts: posts,
+          account: account,
+        };
+      } else {
+        return {
+          posts: [],
+          account: account,
+        };
       }
-      return new Post(record, user);
-    })
+    }),
+  );
+  const recordsFiltered = postsAcc.filter((postAcc) =>
+    postAcc.posts.length > 0
   );
-
-  posts.sort((a, b) => b.timestamp - a.timestamp);
-  
-  if(!Config.SHOW_FUTURE_POSTS) {
-    // Filter out posts that are in the future
+  const cutoffDate = getCutoffDate(recordsFiltered);
+  const recordsCutoff = filterPostsByDate(recordsFiltered, cutoffDate);
+  // update the accountMetadata with the new cursor
+  accountsMetadata = accountsMetadata.map((account) => {
+    const postAcc = recordsCutoff.find(
+      (postAcc) => postAcc.account.did == account.did,
+    );
+    if (postAcc) {
+      account.currentCursor = postAcc.account.currentCursor;
+    }
+    return account;
+  }
+  );
+  // throw the records in a big single array
+  let records = recordsCutoff.flatMap((postAcc) => postAcc.posts);
+  // sort the records by timestamp
+  records = records.sort((a, b) => {
+    const aDate = new Date(
+      (a.value as AppBskyFeedPost.Record).createdAt,
+    ).getTime();
+    const bDate = new Date(
+      (b.value as AppBskyFeedPost.Record).createdAt,
+    ).getTime();
+    return bDate - aDate;
+  });
+  // filter out posts that are in the future
+  if (!Config.SHOW_FUTURE_POSTS) {
     const now = Date.now();
-    const filteredPosts = posts.filter((post) => post.timestamp <= now);
-    return filteredPosts.slice(0, Config.MAX_POSTS);
+    records = records.filter((post) => {
+      const postDate = new Date(
+        (post.value as AppBskyFeedPost.Record).createdAt,
+      ).getTime();
+      return postDate <= now;
+    });
   }
 
-  return posts.slice(0, Config.MAX_POSTS);
+  const newPosts = records.map((record) => {
+    const account = accountsMetadata.find(
+      (account) => account.did == processAtUri(record.uri).repo,
+    );
+    if (!account) {
+      throw new Error(
+        `Account with DID ${processAtUri(record.uri).repo} not found`,
+      );
+    }
+    return new Post(record, account);
+  });
+  return newPosts;
 };
-export { fetchAllPosts, getAllMetadataFromPds, Post };
+
+const fetchPostsForUser = async (did: At.Did, cursor: string | null) => {
+  try {
+    const { data } = await rpc.get("com.atproto.repo.listRecords", {
+      params: {
+        repo: did as At.Identifier,
+        collection: "app.bsky.feed.post",
+        limit: Config.MAX_POSTS,
+        cursor: cursor || undefined,
+      },
+    });
+    return data.records as ComAtprotoRepoListRecords.Record[];
+  } catch (e) {
+    console.error(`Error fetching posts for ${did}:`, e);
+    return null;
+  }
+};
+
+export { getAllMetadataFromPds, getNextPosts, Post };
 export type { AccountMetadata };