///
import assert from 'node:assert/strict';
import { readFileSync } from 'node:fs';
import { test } from 'node:test';
import { IndexModel } from '../src/index/model.ts';
import { ingest, type IngestDeps, type WebSocketLike } from '../src/index/ingest.ts';
import { sqliteSql } from './sqlite-sql.ts';
import { upstream, type Upstream } from '../src/upstream.ts';
type FakeFrame = { type: 'message'; data: unknown } | { type: 'close' } | { type: 'error' };
/** A fake live tail that plays back a fixed script of frames, one per microtask, and records
* the URL it was opened with. After the script runs out it stays open and silent. */
function fakeSocket(frames: FakeFrame[], urls: string[]): (url: string) => Promise {
return async (url) => {
urls.push(url);
const listeners: Record void)[]> = {};
let index = 0;
const playNext = () => {
if (index >= frames.length) return;
const frame = frames[index++]!;
const data = frame.type === 'message' ? frame.data : undefined;
for (const listener of listeners[frame.type] ?? []) listener({ data });
if (frame.type !== 'close' && frame.type !== 'error') queueMicrotask(playNext);
};
setTimeout(playNext, 0);
return {
addEventListener(type: string, listener: (event: { data: unknown }) => void) {
(listeners[type] ??= []).push(listener as never);
},
close() {
for (const listener of listeners.close ?? []) listener({ data: undefined });
},
};
};
}
const CAPTURED = readFileSync(`${import.meta.dirname}/fixtures/jetstream/project-create.json`, 'utf8');
/** The captured project-create frame (see jetstream.test.ts), re-timed and re-keyed. */
function commitFrame(seq: number, timeIso: string): FakeFrame {
const frame = JSON.parse(CAPTURED);
Object.assign(frame.payload, { seq, time: timeIso, witnessedAt: timeIso, rkey: `3la${seq}`, cid: `bafy${seq}`, did: 'did:plc:alice' });
frame.payload.record.createdAt = timeIso;
return { type: 'message', data: JSON.stringify(frame) };
}
const NOW = Date.parse('2026-11-06T00:00:10.000Z');
const us = (iso: string) => Date.parse(iso) * 1000;
const noRepos = upstream((async () => Response.json({ repos: [] })) as typeof fetch);
function baseDeps(overrides: Partial = {}): { model: IndexModel; urls: string[]; deps: IngestDeps } {
const model = new IndexModel(sqliteSql());
const urls: string[] = [];
const deps: IngestDeps = {
jetstreamUrl: 'wss://jetstream.us-east.bsky.network',
connect: fakeSocket([], urls),
index: model,
backfill: {
relayUrl: 'https://relay1.us-east.bsky.network',
fetch: noRepos,
resolvePds: async () => {
throw new Error('not reached: no repos to resolve');
},
},
now: () => NOW,
drainBudgetMs: 20,
caughtUpSkewMs: 5000,
...overrides,
};
return { model, urls, deps };
}
function cursorParam(url: string | undefined): string | null {
return url ? new URL(url).searchParams.get('cursor') : null;
}
test('drains the live tail until an event lands within the caught-up skew, then closes', async () => {
const urls: string[] = [];
const { model, deps } = baseDeps({
connect: fakeSocket(
[
commitFrame(1, '2026-11-05T12:00:00.000Z'),
commitFrame(2, '2026-11-06T00:00:08.000Z'),
commitFrame(3, '2026-11-06T00:00:09.000Z'),
],
urls,
),
});
model.setCursor(us('2026-11-05T11:00:00.000Z'));
await ingest(deps);
assert.deepStrictEqual(model.recent().projects.map((p) => p.uri), [
'at://did:plc:alice/dev.nibling.project/3la2',
'at://did:plc:alice/dev.nibling.project/3la1',
]);
assert.equal(model.cursor(), us('2026-11-06T00:00:08.000Z'));
assert.equal(cursorParam(urls[0]), String(us('2026-11-05T11:00:00.000Z')));
});
test('a backfill records its start as the cursor and replays the live tail from it', async () => {
let clock = NOW;
const urls: string[] = [];
const { model, deps } = baseDeps({
now: () => clock,
backfill: {
relayUrl: 'https://relay1.us-east.bsky.network',
fetch: upstream((async () => {
clock += 30_000;
return Response.json({ repos: [] });
}) as typeof fetch),
resolvePds: async () => 'https://example.pds',
},
connect: fakeSocket([{ type: 'close' }], urls),
});
await ingest(deps);
assert.equal(model.cursor(), NOW * 1000);
assert.equal(cursorParam(urls[0]), String(NOW * 1000));
});
test('a backfill prunes rows the repo no longer lists', async () => {
const { model, deps } = baseDeps({
backfill: {
relayUrl: 'https://relay1.us-east.bsky.network',
fetch: upstream((async (input: string | URL | Request) => {
const url = new URL(typeof input === 'string' ? input : input.toString());
if (url.pathname === '/xrpc/com.atproto.sync.listReposByCollection') {
return Response.json({ repos: url.searchParams.get('collection') === 'dev.nibling.project' ? [{ did: 'did:plc:alice' }] : [] });
}
return Response.json({
records: [
{
uri: 'at://did:plc:alice/dev.nibling.project/kept',
cid: 'bafy-kept',
value: { $type: 'dev.nibling.project', name: 'Kept', files: [], createdAt: '2026-11-01T00:00:00.000Z' },
},
],
});
}) as typeof fetch),
resolvePds: async () => 'https://alice.pds.example',
},
});
for (const rkey of ['kept', 'deleted-while-down']) {
model.apply({
action: 'put',
collection: 'dev.nibling.project',
uri: `at://did:plc:alice/dev.nibling.project/${rkey}`,
cid: 'bafy-old',
did: 'did:plc:alice',
record: { name: rkey, createdAt: '2026-10-01T00:00:00.000Z' },
indexedAt: '2026-10-01T00:00:00.000Z',
});
}
model.apply({
action: 'put',
collection: 'dev.nibling.project',
uri: 'at://did:plc:bob/dev.nibling.project/other-repo',
cid: 'bafy-bob',
did: 'did:plc:bob',
record: { name: 'Bob', createdAt: '2026-10-01T00:00:00.000Z' },
indexedAt: '2026-10-01T00:00:00.000Z',
});
await ingest(deps);
assert.deepStrictEqual(model.recent().projects.map((p) => p.uri).sort(), [
'at://did:plc:alice/dev.nibling.project/kept',
'at://did:plc:bob/dev.nibling.project/other-repo',
]);
});
test('a quiet drain moves the cursor up to when the socket opened', async () => {
let clock = NOW;
const { model, deps } = baseDeps({ now: () => clock });
model.setCursor((NOW - 60_000) * 1000);
const run = ingest(deps);
await new Promise((r) => setTimeout(r, 5));
clock += 6000;
await run;
assert.equal(model.cursor(), (NOW - 5000) * 1000);
});
test('a quiet drain moves the cursor at most maxQuietAdvanceMs past where it started', async () => {
let clock = NOW;
const { model, deps } = baseDeps({ now: () => clock, maxQuietAdvanceMs: 10_000 });
const start = (NOW - 3_600_000) * 1000;
model.setCursor(start);
const run = ingest(deps);
await new Promise((r) => setTimeout(r, 5));
clock += 6000;
await run;
assert.equal(model.cursor(), start + 10_000 * 1000);
});
test('a drain that errors does not move the cursor', async () => {
const { model, deps } = baseDeps({ connect: fakeSocket([{ type: 'error' }], []) });
model.setCursor((NOW - 60_000) * 1000);
await ingest(deps);
assert.equal(model.cursor(), (NOW - 60_000) * 1000);
});
test('skips backfill when the stored cursor is still inside the replay window', async () => {
let backfillRan = false;
const { model, deps } = baseDeps({
backfill: {
relayUrl: 'https://relay1.us-east.bsky.network',
fetch: upstream((async () => {
backfillRan = true;
return Response.json({ repos: [] });
}) as typeof fetch),
resolvePds: async () => 'https://example.pds',
},
connect: fakeSocket([{ type: 'close' }], []),
});
model.setCursor(us('2026-11-06T00:00:00.000Z'));
await ingest(deps);
assert.equal(backfillRan, false);
});
test('backfills when the stored cursor is older than the replay window', async () => {
let backfillRan = false;
const { model, deps } = baseDeps({
backfill: {
relayUrl: 'https://relay1.us-east.bsky.network',
fetch: upstream((async () => {
backfillRan = true;
return Response.json({ repos: [] });
}) as typeof fetch),
resolvePds: async () => 'https://example.pds',
},
connect: fakeSocket([{ type: 'close' }], []),
});
model.setCursor(us('2026-11-04T00:00:00.000Z'));
await ingest(deps);
assert.equal(backfillRan, true);
assert.equal(model.cursor(), NOW * 1000);
});