Sifa professional network API (Fastify, AT Protocol, Jetstream) sifa.id/

feat(heatmap): warn to GlitchTip when pagination truncates history (#159)

When the heatmap hits maxPages without reaching the requested time
window, log a warning and send it to GlitchTip via Sentry.captureMessage.
This gives visibility into users whose activity exceeds our pagination
limits, so we can adjust before they notice incomplete heatmaps.

authored by

Guido X Jansen and committed by
GitHub
85df5fec 1366d8bb

+30 -10
+30 -10
src/routes/activity-heatmap.ts
··· 1 1 import type { FastifyInstance } from 'fastify'; 2 2 import type { NodeOAuthClient } from '@atproto/oauth-client-node'; 3 3 import { Agent } from '@atproto/api'; 4 + import * as Sentry from '@sentry/node'; 4 5 import type { Database } from '../db/index.js'; 5 6 import type { ValkeyClient } from '../cache/index.js'; 6 7 import { ··· 84 85 /** 85 86 * Fetch all PDS records for a given collection since a given date. 86 87 */ 88 + interface PdsFetchResult { 89 + items: ActivityItem[]; 90 + truncated: boolean; 91 + pagesUsed: number; 92 + } 93 + 87 94 async function fetchAllPdsItems( 88 95 pdsHost: string, 89 96 did: string, ··· 91 98 entry: AppRegistryEntry, 92 99 since: Date, 93 100 maxPages: number, 94 - ): Promise<ActivityItem[]> { 101 + ): Promise<PdsFetchResult> { 95 102 const agent = new Agent(`https://${pdsHost}`); 96 - const allItems: ActivityItem[] = []; 103 + const items: ActivityItem[] = []; 97 104 let cursor: string | undefined; 105 + let pagesUsed = 0; 98 106 99 107 for (let page = 0; page < maxPages; page++) { 108 + pagesUsed++; 100 109 const params: { repo: string; collection: string; limit: number; cursor?: string } = { 101 110 repo: did, 102 111 collection, ··· 115 124 reachedEnd = true; 116 125 break; 117 126 } 118 - allItems.push({ 127 + items.push({ 119 128 uri: rec.uri, 120 129 collection, 121 130 rkey: rec.uri.split('/').pop() ?? '', ··· 127 136 }); 128 137 } 129 138 130 - if (reachedEnd || !res.data.cursor || res.data.records.length < 100) break; 139 + if (reachedEnd || !res.data.cursor || res.data.records.length < 100) { 140 + return { items, truncated: false, pagesUsed }; 141 + } 131 142 cursor = res.data.cursor; 132 143 } 133 144 134 - return allItems; 145 + return { items, truncated: true, pagesUsed }; 135 146 } 136 147 137 148 export function registerHeatmapRoutes( ··· 186 197 const targetStats = stats.slice(0, MAX_APPS_FOR_HEATMAP); 187 198 const fetchPromises = targetStats.map((stat) => { 188 199 const entry = registry.find((e) => e.id === stat.appId); 189 - if (!entry) return Promise.resolve([] as ActivityItem[]); 200 + const emptyResult: PdsFetchResult = { items: [], truncated: false, pagesUsed: 0 }; 201 + if (!entry) return Promise.resolve(emptyResult); 190 202 191 203 // For the heatmap, always fetch from PDS via listRecords. 192 204 // The AppView's getAuthorFeed mixes in reposts that get filtered, 193 205 // wasting pages. PDS listRecords returns only the user's own records. 194 - if (!pdsHost) return Promise.resolve([] as ActivityItem[]); 206 + if (!pdsHost) return Promise.resolve(emptyResult); 195 207 196 208 const collection = entry.id === 'bluesky' ? 'app.bsky.feed.post' : getCollectionForApp(entry); 197 209 const pdsEntry = ··· 202 214 return fetchAllPdsItems(pdsHost, did, collection, pdsEntry, since, maxPages).catch( 203 215 (err: unknown) => { 204 216 request.log.warn({ err, appId: entry.id }, 'Failed to fetch PDS items for heatmap'); 205 - return [] as ActivityItem[]; 217 + return { items: [] as ActivityItem[], truncated: false, pagesUsed: 0 }; 206 218 }, 207 219 ); 208 220 }); 209 221 210 222 const results = await Promise.all(fetchPromises); 211 223 const allItems: ActivityItem[] = []; 212 - for (const items of results) { 213 - allItems.push(...items); 224 + for (const result of results) { 225 + allItems.push(...result.items); 226 + if (result.truncated) { 227 + const msg = `Heatmap truncated for ${did}: ${result.items.length} items in ${result.pagesUsed} pages did not reach ${since.toISOString()}`; 228 + request.log.warn(msg); 229 + Sentry.captureMessage(msg, { 230 + level: 'warning', 231 + extra: { did, daysParam, pagesUsed: result.pagesUsed, itemCount: result.items.length }, 232 + }); 233 + } 214 234 } 215 235 216 236 const days = aggregateByDay(allItems);