Something went wrong. Try again.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290import fs from 'fs';import { spawn, ChildProcess } from 'child_process';import { CONFIG } from '../config';import { TapFilters, ServiceStatus } from '../types';import { getDb } from './db';import { IService, ServiceHealth } from './service-manager';
let tapProcess: ChildProcess | null = null;let tapReady = false;let tapStartInProgress = false;let tapFilters = loadTapFilters();
// Exponential backoff for restart loops. Resets to 0 when the child reports// "listening" (i.e., the process started successfully) or when it exits cleanly// (code === 0). Prevents a fast-crash loop from hammering CPU and logs.let tapRestartAttempts = 0;const TAP_RESTART_BASE_MS = 1000;const TAP_RESTART_MAX_MS = 60_000;
export function getTapState() { return { process: tapProcess, ready: tapReady, filters: tapFilters };}
function normalizeFilters(filters: any[]): string[] { if (!Array.isArray(filters)) return []; const cleaned = filters .map((f) => String(f).trim()) .filter(Boolean); return Array.from(new Set(cleaned));}
export function loadTapFilters(): string[] { try { if (fs.existsSync(CONFIG.TAP.FILTERS_PATH)) { const raw = fs.readFileSync(CONFIG.TAP.FILTERS_PATH, 'utf8'); const data = JSON.parse(raw); if (Array.isArray(data.filters)) { return normalizeFilters(data.filters); } } } catch (error: any) { console.warn('[Tap] Failed to load filters:', error.message); }
if (CONFIG.TAP.COLLECTION_FILTERS) { return normalizeFilters(CONFIG.TAP.COLLECTION_FILTERS.split(',')); }
return [];}
export function saveTapFilters(filters: string[]): string[] { const normalized = normalizeFilters(filters); tapFilters = normalized; // Atomic write: write-to-tmp + rename is atomic on POSIX, so concurrent // writers can't interleave and leave a partially-written file. const body = JSON.stringify( { filters: normalized, updatedAt: new Date().toISOString() }, null, 2, ); const target = CONFIG.TAP.FILTERS_PATH; const tmp = `${target}.${process.pid}.${Date.now()}.tmp`; try { fs.writeFileSync(tmp, body); fs.renameSync(tmp, target); } catch (error: any) { console.error('[Tap] Failed to save filters:', error.message); // Best-effort cleanup of the tmp file if the rename didn't happen. try { fs.unlinkSync(tmp); } catch {} } return normalized;}
export async function checkTapHealth(): Promise<boolean> { try { const response = await fetch(`${CONFIG.TAP.URL}/health`, { signal: AbortSignal.timeout(2000) }); return response.ok; } catch { return false; }}
export async function getTapStats() { try { const [repoCount, recordCount, outboxBuffer, cursors] = await Promise.all([ fetch(`${CONFIG.TAP.URL}/stats/repo-count`).then(r => r.json() as Promise<{ repo_count: number }>).catch(() => ({ repo_count: 0 })), fetch(`${CONFIG.TAP.URL}/stats/record-count`).then(r => r.json() as Promise<{ record_count: number }>).catch(() => ({ record_count: 0 })), fetch(`${CONFIG.TAP.URL}/stats/outbox-buffer`).then(r => r.json() as Promise<{ buffer_count: number }>).catch(() => ({ buffer_count: 0 })), fetch(`${CONFIG.TAP.URL}/stats/cursors`).then(r => r.json()).catch(() => ({})) ]);
return { repoCount: repoCount.repo_count || 0, recordCount: recordCount.record_count || 0, outboxBuffer: outboxBuffer.buffer_count || 0, cursors }; } catch (error) { return null; }}
export async function startTap() { if (tapProcess || tapStartInProgress) { return; }
tapStartInProgress = true; const existingHealthy = await checkTapHealth(); if (existingHealthy) { tapReady = true; tapStartInProgress = false; console.log('[Substrate] Tap already running; skipping spawn.'); return; }
console.log('[Substrate] Starting Tap service...');
const env: NodeJS.ProcessEnv = { ...process.env, TAP_DATABASE_URL: `sqlite://${CONFIG.TAP.DB_PATH}`, TAP_BIND: process.env.TAP_BIND || ':2480', TAP_RELAY_URL: CONFIG.TAP.URL === 'http://localhost:2480' ? 'https://relay1.us-east.bsky.network' : process.env.TAP_RELAY_URL, // Fallback if using defaults // Child env keeps TAP's native env vars as the source of truth. TAP_DISABLE_ACKS: process.env.TAP_DISABLE_ACKS || 'true', TAP_LOG_LEVEL: process.env.TAP_LOG_LEVEL || 'info', };
// Ensure CONFIG values are passed if env vars weren't set explicitly if (!env.TAP_RELAY_URL) env.TAP_RELAY_URL = 'https://relay1.us-east.bsky.network';
if (tapFilters.length > 0) { env.TAP_COLLECTION_FILTERS = tapFilters.join(','); } else { delete env.TAP_COLLECTION_FILTERS; }
tapProcess = spawn('tap', ['run'], { env });
tapProcess.stdout?.on('data', (data) => { const text = data.toString().trim(); console.log(`[Tap] ${text}`); if (text.includes('listening')) { tapReady = true; // Reset the backoff counter once the process is healthy. tapRestartAttempts = 0; } });
tapProcess.stderr?.on('data', (data) => { console.error(`[Tap Error] ${data.toString().trim()}`); });
tapProcess.on('close', (code) => { console.log(`[Tap] Process exited with code ${code}`); tapReady = false; tapProcess = null; // Clean exit → reset; crash → exponential backoff. if (code === 0) { tapRestartAttempts = 0; } else { tapRestartAttempts++; } const delay = Math.min( TAP_RESTART_BASE_MS * Math.pow(2, Math.max(0, tapRestartAttempts - 1)), TAP_RESTART_MAX_MS, ); console.warn(`[Tap] Restart scheduled in ${delay}ms (attempt ${tapRestartAttempts})`); setTimeout(startTap, delay); });
tapStartInProgress = false;}
export async function waitForTapReady(maxWaitMs = 15000) { const start = Date.now(); while (Date.now() - start < maxWaitMs) { const ok = await checkTapHealth(); if (ok) return true; await new Promise((resolve) => setTimeout(resolve, 500)); } return false;}
// Re-enqueues every tracked repo by toggling remove/add on the Tap relay, forcing it to// re-emit the full record history for each DID. Used after filter changes.async function backfillTrackedRepos() { try { const db = getDb(); const repos = db.prepare('SELECT did FROM repos').all().map((r: any) => r.did);
for (const did of repos) { await fetch(`${CONFIG.TAP.URL}/repos/remove`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ dids: [did] }) }).catch(() => null);
await fetch(`${CONFIG.TAP.URL}/repos/add`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ dids: [did] }) }).catch(() => null); } } catch (error: any) { console.error('[Tap] Backfill failed:', error.message); }}
// Nuclear backfill: wipes repo_records rows for tracked DIDs before re-enqueueing.// Use when the local record cache has drifted and a soft backfill won't reconcile it.async function hardBackfillTrackedRepos() { let repos: string[] = []; try { const db = getDb({ readonly: false }); repos = db.prepare('SELECT did FROM repos').all().map((r: any) => r.did);
if (repos.length === 0) { return; }
const placeholders = repos.map(() => '?').join(','); db.prepare(`DELETE FROM repo_records WHERE did IN (${placeholders})`).run(...repos);
// Log the DID set so an operator can recover if backfill fails partway. // backfillTrackedRepos itself per-DID-catches, so partial failures are silent. console.log(`[Tap] Hard-backfill wiped ${repos.length} repo(s); re-enqueueing. DIDs: ${JSON.stringify(repos)}`);
await backfillTrackedRepos(); } catch (error: any) { console.error( '[Tap] Hard backfill failed:', error.message, repos.length > 0 ? `(affected DIDs: ${JSON.stringify(repos)})` : '', ); }}
export async function restartTap(options = { backfill: false, hardBackfill: false }) { if (tapProcess) { tapProcess.kill(); tapProcess = null; tapReady = false; }
await startTap();
if (options.backfill || options.hardBackfill) { const ready = await waitForTapReady(); if (ready) { if (options.hardBackfill) { await hardBackfillTrackedRepos(); } else { await backfillTrackedRepos(); } } }}
export class TapService implements IService { readonly name = 'tap'; readonly type = 'core' as const;
async start(): Promise<void> { await startTap(); }
async stop(): Promise<void> { if (tapProcess) { tapProcess.kill(); tapProcess = null; tapReady = false; } }
async health(): Promise<ServiceHealth> { const healthy = await checkTapHealth(); if (!healthy) return { status: 'stopped' }; const stats = await getTapStats(); return { status: 'running', details: stats }; }}