my website at ewancroft.uk

refactor: add DRY pagination utility for AT Protocol

- Create reusable fetchAllRecords utility with cursor-based pagination
- Add fetchAllUserRecords convenience function
- Refactor archive page to use new pagination utilities
- Ensures all posts are fetched (not limited to 100)
- Add proper TypeScript types (AtProtoRecord interface)

ewancroft.uk b02e0a2d 05e92482

verified
Changed files
+105 -45
src
lib
components
layout
main
services
atproto
routes
archive
+3 -1
src/lib/components/layout/main/card/LinkCard.svelte
··· 75 75 {/if} 76 76 77 77 <!-- 👇 Title is always below the badges --> 78 - <h3 class="overflow-wrap-anywhere font-semibold wrap-break-word text-ink-900 dark:text-ink-50"> 78 + <h3 79 + class="overflow-wrap-anywhere font-semibold wrap-break-word text-ink-900 dark:text-ink-50" 80 + > 79 81 {title} 80 82 </h3> 81 83
+92
src/lib/services/atproto/pagination/fetchAllRecords.ts
··· 1 + import { withFallback } from '$lib/services/atproto/agents'; 2 + import { PUBLIC_ATPROTO_DID } from '$env/static/public'; 3 + 4 + /** 5 + * Configuration for fetching paginated records 6 + */ 7 + export interface FetchRecordsConfig { 8 + /** The repository DID to fetch from */ 9 + repo: string; 10 + /** The AT Protocol collection to fetch from */ 11 + collection: string; 12 + /** Number of records to fetch per page (max 100) */ 13 + limit?: number; 14 + /** Optional fetch function for SSR */ 15 + fetchFn?: typeof fetch; 16 + } 17 + 18 + /** 19 + * Type for AT Protocol record response 20 + */ 21 + export interface AtProtoRecord<T = any> { 22 + uri: string; 23 + value: T; 24 + cid?: string; 25 + } 26 + 27 + /** 28 + * Generic function to fetch all records from an AT Protocol collection with automatic pagination. 29 + * 30 + * @param config - Configuration object for the fetch operation 31 + * @returns Promise resolving to array of all records 32 + * 33 + * @example 34 + * ```ts 35 + * const posts = await fetchAllRecords({ 36 + * repo: PUBLIC_ATPROTO_DID, 37 + * collection: 'com.whtwnd.blog.entry', 38 + * fetchFn: fetch 39 + * }); 40 + * ``` 41 + */ 42 + export async function fetchAllRecords<T = any>( 43 + config: FetchRecordsConfig 44 + ): Promise<AtProtoRecord<T>[]> { 45 + const { repo, collection, limit = 100, fetchFn } = config; 46 + const allRecords: AtProtoRecord<T>[] = []; 47 + 48 + let cursor: string | undefined; 49 + 50 + try { 51 + do { 52 + const records = await withFallback( 53 + repo, 54 + async (agent) => { 55 + const response = await agent.com.atproto.repo.listRecords({ 56 + repo, 57 + collection, 58 + limit, 59 + cursor 60 + }); 61 + cursor = response.data.cursor; 62 + return response.data.records; 63 + }, 64 + true, 65 + fetchFn 66 + ); 67 + 68 + allRecords.push(...(records as AtProtoRecord<T>[])); 69 + } while (cursor); 70 + } catch (error) { 71 + console.warn(`Failed to fetch records from ${collection}:`, error); 72 + throw error; 73 + } 74 + 75 + return allRecords; 76 + } 77 + 78 + /** 79 + * Convenience function to fetch all records from the configured user's repository 80 + */ 81 + export async function fetchAllUserRecords<T = any>( 82 + collection: string, 83 + fetchFn?: typeof fetch, 84 + limit?: number 85 + ): Promise<AtProtoRecord<T>[]> { 86 + return fetchAllRecords<T>({ 87 + repo: PUBLIC_ATPROTO_DID, 88 + collection, 89 + limit, 90 + fetchFn 91 + }); 92 + }
+2
src/lib/services/atproto/pagination/index.ts
··· 1 + export { fetchAllRecords, fetchAllUserRecords } from './fetchAllRecords'; 2 + export type { FetchRecordsConfig, AtProtoRecord } from './fetchAllRecords';
+8 -44
src/routes/archive/+page.ts
··· 1 1 import type { PageLoad } from './$types'; 2 2 import { createSiteMeta, type SiteMetadata } from '$lib/helper/siteMeta'; 3 - import { withFallback } from '$lib/services/atproto/agents'; 3 + import { fetchAllUserRecords } from '$lib/services/atproto/pagination'; 4 4 import { PUBLIC_ATPROTO_DID } from '$env/static/public'; 5 5 import type { BlogPost } from '$lib/services/atproto'; 6 6 7 7 /** 8 - * Fetches ALL blog posts from WhiteWind and Leaflet (no limit) 8 + * Fetches ALL blog posts from WhiteWind and Leaflet with proper pagination 9 9 */ 10 10 async function fetchAllBlogPosts(fetchFn?: typeof fetch): Promise<BlogPost[]> { 11 11 const posts: BlogPost[] = []; 12 12 13 - // Fetch WhiteWind posts 13 + // Fetch WhiteWind posts with pagination 14 14 try { 15 - const whiteWindRecords = await withFallback( 16 - PUBLIC_ATPROTO_DID, 17 - async (agent) => { 18 - const response = await agent.com.atproto.repo.listRecords({ 19 - repo: PUBLIC_ATPROTO_DID, 20 - collection: 'com.whtwnd.blog.entry', 21 - limit: 100 22 - }); 23 - return response.data.records; 24 - }, 25 - true, 26 - fetchFn 27 - ); 15 + const whiteWindRecords = await fetchAllUserRecords('com.whtwnd.blog.entry', fetchFn); 28 16 29 17 for (const record of whiteWindRecords) { 30 18 const value = record.value as any; ··· 46 34 console.warn('Failed to fetch WhiteWind posts:', error); 47 35 } 48 36 49 - // Fetch Leaflet publications and documents 37 + // Fetch Leaflet publications and documents with pagination 50 38 try { 51 - // Get all publications first 52 - const publicationsRecords = await withFallback( 53 - PUBLIC_ATPROTO_DID, 54 - async (agent) => { 55 - const response = await agent.com.atproto.repo.listRecords({ 56 - repo: PUBLIC_ATPROTO_DID, 57 - collection: 'pub.leaflet.publication', 58 - limit: 100 59 - }); 60 - return response.data.records; 61 - }, 62 - true, 63 - fetchFn 64 - ); 39 + // Fetch all publications 40 + const publicationsRecords = await fetchAllUserRecords('pub.leaflet.publication', fetchFn); 65 41 66 42 const publicationsMap = new Map<string, { name: string; basePath?: string }>(); 67 43 for (const pubRecord of publicationsRecords) { ··· 73 49 } 74 50 75 51 // Fetch all Leaflet documents 76 - const leafletDocsRecords = await withFallback( 77 - PUBLIC_ATPROTO_DID, 78 - async (agent) => { 79 - const response = await agent.com.atproto.repo.listRecords({ 80 - repo: PUBLIC_ATPROTO_DID, 81 - collection: 'pub.leaflet.document', 82 - limit: 100 83 - }); 84 - return response.data.records; 85 - }, 86 - true, 87 - fetchFn 88 - ); 52 + const leafletDocsRecords = await fetchAllUserRecords('pub.leaflet.document', fetchFn); 89 53 90 54 for (const record of leafletDocsRecords) { 91 55 const value = record.value as any;