import { z } from "zod"; import { upstream } from "./xrpc"; const endpoint = "https://jetstream.us-west.bsky.network/xrpc/network.bsky.jetstream.subscribeEvents"; const idleMs = 1500; // A backlog scan from an old cursor can be silent for longer. const backlogIdleMs = 4000; const windowMs = 5000; const maxMessages = 200; const overlapMs = 30_000; const staleMs = 12 * 60 * 60_000; const recentMs = overlapMs + 60_000; const poisonMs = 60 * 60_000; export interface JetstreamFilter { collections: string[]; dids: string[]; } const message = z.object({ payload: z.object({ seq: z.int(), did: z.string(), collection: z.string(), rkey: z.string(), operation: z.string(), }), }); export type JetstreamEvent = z.infer["payload"]; // The cursor is a seq, or a time in µs when the stream was caught up. export interface JetstreamState { cursor: number; savedAt: number; failSeq?: number; failSince?: number; } export function subscribeUrl(filter: JetstreamFilter, cursor: number): URL { const url = new URL(endpoint); for (const collection of filter.collections) { url.searchParams.append("collections", collection); } url.searchParams.append("kinds", "commit"); for (const did of filter.dids) { url.searchParams.append("dids", did); } url.searchParams.append("cursor", String(cursor)); return url; } export function parseEvent(data: string): JetstreamEvent | undefined { try { return message.safeParse(JSON.parse(data)).data?.payload; } catch { return undefined; } } // Jetstream sends no end of backlog marker, so a quiet stream counts as caught up. async function read( url: URL, quietMs: number, ): Promise<{ messages: string[]; idle: boolean; closed: string | undefined }> { const response = await upstream(url, 503, { headers: { Upgrade: "websocket", "Sec-WebSocket-Protocol": "xrpc.v1.json" }, }); const socket = response.webSocket; if (socket === null) { throw new Error(`HTTP ${response.status}`); } socket.accept(); return await new Promise((resolve) => { const messages: string[] = []; let done = false; let idleTimer: number | null = null; let closed: string | undefined; const finish = (idle: boolean): void => { if (done) { return; } done = true; clearTimeout(idleTimer); clearTimeout(windowTimer); try { socket.close(1000); } catch { // The server already closed the socket. } resolve({ messages, idle, closed }); }; const windowTimer = setTimeout(() => finish(false), windowMs); const wait = (): void => { clearTimeout(idleTimer); idleTimer = setTimeout(() => finish(true), quietMs); }; socket.addEventListener("message", (event) => { if (done) { return; } // The JSON subprotocol sends only text frames. const data: unknown = event.data; messages.push(typeof data === "string" ? data : ""); if (messages.length >= maxMessages) { finish(false); } else { wait(); } }); socket.addEventListener("close", ({ code, reason }) => { closed = `${code} ${reason}`; finish(false); }); socket.addEventListener("error", () => { closed = "error"; finish(false); }); wait(); }); } // Reads the backlog since the cursor and handles its events in order. // A handler that throws stops the batch, and the next tick retries that event. export async function catchUp( state: JetstreamState | undefined, filter: JetstreamFilter, handle: (event: JetstreamEvent) => Promise, now = Date.now(), ): Promise { const tail = (now - overlapMs) * 1000; let start = state; if (start === undefined || start.savedAt < now - staleMs) { if (start !== undefined) { console.error("jetstream cursor is stale, events since it are lost", start); } start = { cursor: tail, savedAt: now }; } // Seq cursors are far below any time cursor. const recent = start.cursor >= (now - recentMs) * 1000; let batch; try { batch = await read(subscribeUrl(filter, start.cursor), recent ? idleMs : backlogIdleMs); } catch (error) { console.warn("jetstream unavailable", { error: String(error) }); return start; } const events = batch.messages.flatMap((data) => parseEvent(data) ?? []); if (events.length === 0 && !batch.idle) { console.warn("jetstream sent no events", { closed: batch.closed }); return start; } let last: number | undefined; for (const event of events) { try { // oxlint-disable-next-line no-await-in-loop await handle(event); } catch (error) { const failSince = start.failSeq === event.seq ? (start.failSince ?? now) : now; if (now - failSince <= poisonMs) { console.warn("jetstream event failed", { ...event, error: String(error) }); return { cursor: event.seq, savedAt: now, failSeq: event.seq, failSince }; } console.error("jetstream event skipped after an hour", { ...event, error: String(error), }); } last = event.seq; } if (batch.idle) { if (!recent) { console.warn("jetstream backlog ended after a quiet period", { from: start.cursor }); } return { cursor: tail, savedAt: now }; } return { cursor: last === undefined ? start.cursor : last + 1, savedAt: now }; }