// Crash on uncaught errors — Docker will restart us. // Continuing after an uncaught exception leaves Node in an unreliable state. process.on('uncaughtException', (error) => { console.error('[Substrate] Uncaught exception — exiting:', error); process.exit(1); }); process.on('unhandledRejection', (reason: any) => { console.error('[Substrate] Unhandled rejection — exiting:', reason); process.exit(1); }); import http from 'http'; import { getRequestListener } from '@hono/node-server'; import { WebSocketServer, WebSocket } from 'ws'; import app, { safeEqual, isAllowedOrigin } from './app'; import { CONFIG } from './config'; import { serviceManager } from './services/service-manager'; import { TapService } from './services/tap'; import { TerminalManager } from './services/terminal'; import { ContainerManager } from './services/container-manager'; import { setContainerManager } from './routes/containers'; import { setWorkspaceContainerManager } from './routes/workspace'; import { EmbeddingsService } from './services/embeddings'; import { setEmbeddingsService } from './routes/embeddings'; import { LlmService } from './services/llm'; import { setLlmService } from './routes/llm'; import { workspaceWatcher } from './services/workspace-watcher'; import { createLspBridge } from './services/lsp-bridge'; import { closeDbs } from './services/db'; import { log } from './services/logger'; import { errMessage } from './services/errors'; // Register services with the ServiceManager const tapService = new TapService(); const terminalManager = new TerminalManager(); const containerManager = new ContainerManager(); const embeddingsService = new EmbeddingsService(); const llmService = new LlmService(); serviceManager.register(tapService); serviceManager.register(terminalManager); serviceManager.register(containerManager); serviceManager.register(embeddingsService); serviceManager.register(llmService); setContainerManager(containerManager); setWorkspaceContainerManager(containerManager); setEmbeddingsService(embeddingsService); setLlmService(llmService); // Create HTTP Server using Hono's request listener const server = http.createServer(getRequestListener(app.fetch)); // Disable Nagle's algorithm — flush every write immediately. // Critical for SSE streaming where each chunk is small. server.on('connection', (socket) => { socket.setNoDelay(true); }); const WS_MAX_PAYLOAD_BYTES = 512 * 1024; const WS_AUTH_MAX_BYTES = 4096; const WS_TERMINAL_MAX_BYTES = 16 * 1024; const WS_LSP_MAX_BYTES = 256 * 1024; // WebSocket server for terminal and event streaming const wss = new WebSocketServer({ server, path: '/ws', maxPayload: WS_MAX_PAYLOAD_BYTES }); // WebSocket connection caps — prevent event/terminal endpoints being used to // exhaust sockets. Terminal sessions also have their own cap inside TerminalManager. const MAX_WS_PER_IP = 10; const MAX_WS_TOTAL = 200; const SUPPORTED_WS_TYPES = new Set(['events', 'terminal', 'container-terminal', 'lsp']); const wsByIp = new Map(); const wsByType = new Map(); let wsTotal = 0; function incWs(ip: string): boolean { if (wsTotal >= MAX_WS_TOTAL) return false; if ((wsByIp.get(ip) ?? 0) >= MAX_WS_PER_IP) return false; wsByIp.set(ip, (wsByIp.get(ip) ?? 0) + 1); wsTotal++; return true; } function decWs(ip: string): void { wsTotal = Math.max(0, wsTotal - 1); const n = (wsByIp.get(ip) ?? 1) - 1; if (n <= 0) wsByIp.delete(ip); else wsByIp.set(ip, n); } function incWsType(type: string): number { const next = (wsByType.get(type) ?? 0) + 1; wsByType.set(type, next); return next; } function decWsType(type: string): number { const next = (wsByType.get(type) ?? 1) - 1; if (next <= 0) { wsByType.delete(type); return 0; } wsByType.set(type, next); return next; } function wsMessageSize(msg: Buffer | string | ArrayBuffer | Buffer[]): number { if (typeof msg === 'string') return Buffer.byteLength(msg); if (Buffer.isBuffer(msg)) return msg.length; if (Array.isArray(msg)) return msg.reduce((n, b) => n + b.length, 0); return msg.byteLength; } function installWsMessageLimit(ws: WebSocket, maxBytes: number, label: string): void { ws.on('message', (msg) => { if (wsMessageSize(msg as any) > maxBytes) { ws.send(JSON.stringify({ type: 'error', message: `${label} message too large` })); ws.close(1009, 'Message too large'); } }); } // Derive the Tap WebSocket URL from CONFIG.TAP.URL so a reconfigured port is // honoured. Accepts http(s):// and rewrites the scheme to ws(s)://. function tapWsUrl(): string { const base = CONFIG.TAP.URL.replace(/^http(s?):\/\//, 'ws$1://'); return `${base.replace(/\/$/, '')}/channel`; } // Dispatches an already-authenticated WS connection to the right backend: event stream, // host terminal, container terminal, or LSP bridge. Called after the auth handshake. function handleAuthenticatedWs(ws: WebSocket, type: string, url: URL) { if (type === 'events') { // Proxy Tap's WebSocket events. On upstream close we attempt one reconnect // after a short delay; if that also fails the client connection is closed // so they can reconnect on their own schedule. let clientOpen = true; let tapWs: WebSocket | null = null; let reconnectAttempted = false; const connectUpstream = () => { if (!clientOpen) return; tapWs = new WebSocket(tapWsUrl()); tapWs.on('open', () => { if (clientOpen) ws.send(JSON.stringify({ type: 'connected', service: 'tap' })); }); tapWs.on('message', (data) => { if (clientOpen) ws.send(data.toString()); }); tapWs.on('error', () => { if (clientOpen) ws.send(JSON.stringify({ type: 'error', message: 'Event stream error' })); }); tapWs.on('close', () => { if (!clientOpen) return; if (reconnectAttempted) { ws.send(JSON.stringify({ type: 'error', message: 'Event stream lost' })); ws.close(1011, 'Upstream closed'); return; } reconnectAttempted = true; setTimeout(connectUpstream, 2000); }); }; ws.on('close', () => { clientOpen = false; tapWs?.close(); }); connectUpstream(); } else if (type === 'terminal') { terminalManager.createSession(ws); } else if (type === 'container-terminal') { installWsMessageLimit(ws, WS_TERMINAL_MAX_BYTES, 'Terminal'); const containerParam = url.searchParams.get('containerId'); const useWorkspace = url.searchParams.get('workspace') === '1'; if (!containerParam && !useWorkspace) { ws.send(JSON.stringify({ type: 'error', message: 'containerId or workspace=1 parameter required' })); ws.close(1008, 'Missing terminal target'); return; } const resolveContainer = async (): Promise => { if (containerParam) return containerParam; const info = await containerManager.getOrCreateWorkspace(); return info.id; }; resolveContainer() .then((containerId) => containerManager.createTerminalSession(containerId, ws)) .catch((error: any) => { const message = error?.statusCode && error.statusCode < 500 ? errMessage(error) : 'Terminal session failed'; ws.send(JSON.stringify({ type: 'error', message })); ws.close(error?.statusCode === 403 ? 1008 : 1011, 'Terminal session error'); }); } else if (type === 'lsp') { installWsMessageLimit(ws, WS_LSP_MAX_BYTES, 'LSP'); const language = url.searchParams.get('lang'); const containerParam = url.searchParams.get('containerId'); const useWorkspace = url.searchParams.get('workspace') === '1'; if (!language || (!containerParam && !useWorkspace)) { ws.send(JSON.stringify({ type: 'error', message: 'lang and (containerId or workspace=1) are required' })); ws.close(1008, 'Missing LSP parameters'); return; } const resolveContainer = async (): Promise => { if (!CONFIG.CONTAINERS.ENABLED) { throw Object.assign(new Error('Containers are disabled'), { statusCode: 403 }); } if (containerParam) { const info = await containerManager.inspect(containerParam); if (info.status !== 'running') { throw Object.assign(new Error('Container is not running'), { statusCode: 400 }); } return containerParam; } const info = await containerManager.getOrCreateWorkspace(); return info.id; }; resolveContainer() .then((containerId) => createLspBridge({ containerId, language, ws })) .catch((error: any) => { const message = error?.statusCode && error.statusCode < 500 ? errMessage(error) : 'LSP bridge failed'; try { ws.send(JSON.stringify({ type: 'error', message })); } catch { /* ignore */ } try { ws.close(error?.statusCode === 403 ? 1008 : 1011, 'LSP bridge failed'); } catch { /* ignore */ } }); } } wss.on('connection', (ws: WebSocket, req) => { // Honour proxy headers only behind a trusted reverse proxy; otherwise // the client can spoof them and bypass per-IP WS caps. const fwd = (req.headers['x-forwarded-for'] as string | undefined)?.split(',')[0]?.trim(); const real = req.headers['x-real-ip'] as string | undefined; const trusted = CONFIG.TRUST_PROXY ? (fwd || real) : undefined; const ip: string = trusted || req.socket.remoteAddress || '0.0.0.0'; let authenticated = false; let wsType: string | null = null; if (!incWs(ip)) { console.warn(`[WS] Connection cap hit for ${ip} (total=${wsTotal})`); ws.close(1013, 'Too many connections'); return; } ws.on('close', (code, reasonBuffer) => { const reason = reasonBuffer.toString() || 'none'; let remainingType = 0; if (authenticated && wsType) { remainingType = decWsType(wsType); } decWs(ip); if (authenticated && wsType) { console.log( `[WS] ${wsType} disconnected from ${ip} ` + `(code=${code}, reason=${reason}, typeOpen=${remainingType}, total=${wsTotal})`, ); } }); // Validate Origin header to prevent cross-site WebSocket hijacking const origin = req.headers.origin; if (origin && !isAllowedOrigin(origin)) { ws.close(1008, 'Origin not allowed'); return; } const url = new URL(req.url!, `http://${req.headers.host}`); const type = url.searchParams.get('type') || 'events'; const queryKey = url.searchParams.get('key'); if (!SUPPORTED_WS_TYPES.has(type)) { ws.send(JSON.stringify({ type: 'error', message: `Unsupported WebSocket type: ${type}` })); ws.close(1008, 'Unsupported WebSocket type'); return; } // Refuse legacy query-string auth outright. Raw API keys in WebSocket URLs // leak to browser history, tunnel/CDN logs, and crash reports. if (queryKey) { ws.send(JSON.stringify({ type: 'error', message: 'Legacy WebSocket query auth is no longer supported. Authenticate with the first message.', })); ws.close(1008, 'Legacy auth disabled'); return; } // Preferred auth: first message must be { type: 'auth', key: '...' } const authTimeout = setTimeout(() => { ws.send(JSON.stringify({ type: 'error', message: 'Authentication timeout' })); ws.close(1008, 'Auth timeout'); }, 5000); ws.once('message', (msg) => { clearTimeout(authTimeout); if (wsMessageSize(msg as any) > WS_AUTH_MAX_BYTES) { ws.send(JSON.stringify({ type: 'error', message: 'Authentication message too large' })); ws.close(1009, 'Auth message too large'); return; } try { const parsed = JSON.parse(msg.toString()); if (parsed.type === 'auth' && safeEqual(parsed.key, CONFIG.API_KEY)) { authenticated = true; wsType = type; const typeOpen = incWsType(type); console.log(`[WS] Authenticated ${type} connected from ${ip} (typeOpen=${typeOpen}, total=${wsTotal})`); handleAuthenticatedWs(ws, type, url); } else { ws.send(JSON.stringify({ type: 'error', message: 'Unauthorized' })); ws.close(1008, 'Unauthorized'); } } catch (err) { log.warn({ ip, err: errMessage(err) }, 'WS auth message parse failed'); ws.send(JSON.stringify({ type: 'error', message: 'Invalid auth message' })); ws.close(1008, 'Invalid auth'); } }); }); // Start Server server.listen(CONFIG.PORT, () => { log.info({ port: CONFIG.PORT }, 'API server listening'); if (CONFIG.WORKSPACE.ENABLED && CONFIG.CONTAINERS.ENABLED) { workspaceWatcher.start(); } else if (CONFIG.WORKSPACE.ENABLED) { log.warn('workspace enabled without containers; workspace bootstrap skipped'); } tapService.start().catch((error) => log.error({ service: 'tap', err: errMessage(error) }, 'service start failed'), ); containerManager.start().catch((error) => log.error({ service: 'containers', err: errMessage(error) }, 'service start failed'), ); if (CONFIG.EMBEDDINGS.ENABLED) { embeddingsService.start().catch((error) => log.error({ service: 'embeddings', err: errMessage(error) }, 'service start failed'), ); } if (CONFIG.LLM.ENABLED) { llmService.start().catch((error) => log.error({ service: 'llm', err: errMessage(error) }, 'service start failed'), ); } }); // Graceful shutdown process.on('SIGTERM', async () => { console.log('[Substrate] Shutting down...'); for (const service of serviceManager.getAll()) { await service.stop().catch(() => {}); } closeDbs(); server.close(); process.exit(0); });