Something went wrong. Try again.
Tangled Spindle built on Cloudflare Workers
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179import { z } from "zod";import { upstream } from "./xrpc";
const endpoint = "https://jetstream.us-west.bsky.network/xrpc/network.bsky.jetstream.subscribeEvents";const idleMs = 1500;// A backlog scan from an old cursor can be silent for longer.const backlogIdleMs = 4000;const windowMs = 5000;const maxMessages = 200;const overlapMs = 30_000;const staleMs = 12 * 60 * 60_000;const recentMs = overlapMs + 60_000;const poisonMs = 60 * 60_000;
export interface JetstreamFilter { collections: string[]; dids: string[];}
const message = z.object({ payload: z.object({ seq: z.int(), did: z.string(), collection: z.string(), rkey: z.string(), operation: z.string(), }),});export type JetstreamEvent = z.infer<typeof message>["payload"];
// The cursor is a seq, or a time in µs when the stream was caught up.export interface JetstreamState { cursor: number; savedAt: number; failSeq?: number; failSince?: number;}
export function subscribeUrl(filter: JetstreamFilter, cursor: number): URL { const url = new URL(endpoint); for (const collection of filter.collections) { url.searchParams.append("collections", collection); } url.searchParams.append("kinds", "commit"); for (const did of filter.dids) { url.searchParams.append("dids", did); } url.searchParams.append("cursor", String(cursor)); return url;}
export function parseEvent(data: string): JetstreamEvent | undefined { try { return message.safeParse(JSON.parse(data)).data?.payload; } catch { return undefined; }}
// Jetstream sends no end of backlog marker, so a quiet stream counts as caught up.async function read( url: URL, quietMs: number,): Promise<{ messages: string[]; idle: boolean; closed: string | undefined }> { const response = await upstream(url, 503, { headers: { Upgrade: "websocket", "Sec-WebSocket-Protocol": "xrpc.v1.json" }, }); const socket = response.webSocket; if (socket === null) { throw new Error(`HTTP ${response.status}`); } socket.accept(); return await new Promise((resolve) => { const messages: string[] = []; let done = false; let idleTimer: number | null = null; let closed: string | undefined; const finish = (idle: boolean): void => { if (done) { return; } done = true; clearTimeout(idleTimer); clearTimeout(windowTimer); try { socket.close(1000); } catch { // The server already closed the socket. } resolve({ messages, idle, closed }); }; const windowTimer = setTimeout(() => finish(false), windowMs); const wait = (): void => { clearTimeout(idleTimer); idleTimer = setTimeout(() => finish(true), quietMs); }; socket.addEventListener("message", (event) => { if (done) { return; } // The JSON subprotocol sends only text frames. const data: unknown = event.data; messages.push(typeof data === "string" ? data : ""); if (messages.length >= maxMessages) { finish(false); } else { wait(); } }); socket.addEventListener("close", ({ code, reason }) => { closed = `${code} ${reason}`; finish(false); }); socket.addEventListener("error", () => { closed = "error"; finish(false); }); wait(); });}
// Reads the backlog since the cursor and handles its events in order.// A handler that throws stops the batch, and the next tick retries that event.export async function catchUp( state: JetstreamState | undefined, filter: JetstreamFilter, handle: (event: JetstreamEvent) => Promise<void>, now = Date.now(),): Promise<JetstreamState> { const tail = (now - overlapMs) * 1000; let start = state; if (start === undefined || start.savedAt < now - staleMs) { if (start !== undefined) { console.error("jetstream cursor is stale, events since it are lost", start); } start = { cursor: tail, savedAt: now }; } // Seq cursors are far below any time cursor. const recent = start.cursor >= (now - recentMs) * 1000; let batch; try { batch = await read(subscribeUrl(filter, start.cursor), recent ? idleMs : backlogIdleMs); } catch (error) { console.warn("jetstream unavailable", { error: String(error) }); return start; } const events = batch.messages.flatMap((data) => parseEvent(data) ?? []); if (events.length === 0 && !batch.idle) { console.warn("jetstream sent no events", { closed: batch.closed }); return start; } let last: number | undefined; for (const event of events) { try { // oxlint-disable-next-line no-await-in-loop await handle(event); } catch (error) { const failSince = start.failSeq === event.seq ? (start.failSince ?? now) : now; if (now - failSince <= poisonMs) { console.warn("jetstream event failed", { ...event, error: String(error) }); return { cursor: event.seq, savedAt: now, failSeq: event.seq, failSince }; } console.error("jetstream event skipped after an hour", { ...event, error: String(error), }); } last = event.seq; } if (batch.idle) { if (!recent) { console.warn("jetstream backlog ended after a quiet period", { from: start.cursor }); } return { cursor: tail, savedAt: now }; } return { cursor: last === undefined ? start.cursor : last + 1, savedAt: now };}