Something went wrong. Try again.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619// LLM provider service: persists provider config, proxies chat completions so// provider API keys never leave the server. Supports client-side single-call// streaming and server-side multi-turn agent loops.
import fs from 'fs';import http from 'http';import https from 'https';import crypto from 'crypto';import { CONFIG } from '../config';import type { IService, ServiceHealth } from './service-manager';import { assertProviderBaseUrlAllowed, resolveProviderBaseUrl, validateProviderBaseUrl,} from '../utils/llm-provider-security';import { sanitizeUpstreamError } from '../utils/secret-redact';
// Cap provider error bodies before logging or returning diagnostics.const LIST_MODELS_MAX_BYTES = 256 * 1024;const CHAT_ERROR_MAX_BYTES = 256 * 1024;
// API keys are wrapped with AES-256-GCM keyed from a per-record HKDF of// SUBSTRATE_API_KEY — rotating the substrate secret intentionally invalidates// persisted keys. Plaintext `apiKey` fields are migrated on next save.
const ENC_VERSION = 1;
function deriveKey(secret: string, salt: Buffer): Buffer { return Buffer.from( crypto.hkdfSync('sha256', Buffer.from(secret, 'utf8'), salt, Buffer.from('aetheros-llm-v1'), 32), );}
function encryptApiKey(plaintext: string, secret: string): string { const salt = crypto.randomBytes(16); const iv = crypto.randomBytes(12); const key = deriveKey(secret, salt); const cipher = crypto.createCipheriv('aes-256-gcm', key, iv); const ct = Buffer.concat([cipher.update(plaintext, 'utf8'), cipher.final()]); const tag = cipher.getAuthTag(); return [ `v${ENC_VERSION}`, salt.toString('base64'), iv.toString('base64'), ct.toString('base64'), tag.toString('base64'), ].join(':');}
function decryptApiKey(blob: string, secret: string): string | null { const parts = blob.split(':'); if (parts.length !== 5 || parts[0] !== `v${ENC_VERSION}`) return null; try { const salt = Buffer.from(parts[1], 'base64'); const iv = Buffer.from(parts[2], 'base64'); const ct = Buffer.from(parts[3], 'base64'); const tag = Buffer.from(parts[4], 'base64'); const key = deriveKey(secret, salt); const decipher = crypto.createDecipheriv('aes-256-gcm', key, iv); decipher.setAuthTag(tag); const pt = Buffer.concat([decipher.update(ct), decipher.final()]); return pt.toString('utf8'); } catch { return null; }}
// Types
export interface LlmProviderConfig { id: string; name: string; type: 'local' | 'hosted'; baseUrl: string; apiKey?: string; // stored server-side only, never sent to frontend defaultModel: string; enabled: boolean;}
export interface LlmConfig { defaultProviderId: string; providers: LlmProviderConfig[];}
export interface ChatRequest { messages: any[]; model?: string; // override model for this call providerId?: string; // override provider for this call systemPrompt?: string; tools?: any[]; stream?: boolean; // default true}
// Built-in provider templates
const BUILT_IN_TEMPLATES: Omit<LlmProviderConfig, 'apiKey' | 'enabled'>[] = [ { id: 'ollama', name: 'Ollama', type: 'local', baseUrl: 'http://localhost:11434/v1', defaultModel: '' }, { id: 'lmstudio', name: 'LM Studio', type: 'local', baseUrl: 'http://localhost:1234/v1', defaultModel: '' }, { id: 'openai-compatible', name: 'OpenAI Compatible', type: 'local', baseUrl: 'http://localhost:8080/v1', defaultModel: '' }, { id: 'anthropic', name: 'Anthropic', type: 'hosted', baseUrl: 'https://api.anthropic.com', defaultModel: 'claude-sonnet-4-20250514' }, { id: 'openai', name: 'OpenAI', type: 'hosted', baseUrl: 'https://api.openai.com/v1', defaultModel: 'gpt-4o' }, { id: 'letta', name: 'Letta', type: 'hosted', baseUrl: 'https://api.letta.com/v1', defaultModel: '' },];
// Helpers
function normalizeProviderType(type: unknown): LlmProviderConfig['type'] { if (type === 'local' || type === 'hosted') { return type; } throw new Error('type must be either "local" or "hosted"');}
// Service
export class LlmService implements IService { readonly name = 'llm'; readonly type = 'core' as const;
private config: LlmConfig = { defaultProviderId: '', providers: [] }; private initialized = false;
// Keep-alive agents — reuse TCP connections across sequential LLM requests // (e.g. tool-call round-trips) instead of opening a new connection each time. private httpAgent = new http.Agent({ keepAlive: true, maxSockets: 10 }); private httpsAgent = new https.Agent({ keepAlive: true, maxSockets: 10 });
async start(): Promise<void> { this.loadConfig(); this.initialized = true; console.log(`[LLM] Service started — ${this.config.providers.filter(p => p.enabled).length} provider(s) enabled`); }
async stop(): Promise<void> { this.initialized = false; this.httpAgent.destroy(); this.httpsAgent.destroy(); }
async health(): Promise<ServiceHealth> { if (!CONFIG.LLM.ENABLED) return { status: 'stopped' }; if (!this.initialized) return { status: 'error', details: 'Not initialized' }; const active = this.getActiveProvider(); return { status: 'running', details: { activeProvider: active?.id ?? null, activeModel: active?.defaultModel ?? null, providerCount: this.config.providers.length, enabledCount: this.config.providers.filter(p => p.enabled).length, }, }; }
// Config persistence
private loadConfig(): void { const path = CONFIG.LLM.CONFIG_PATH;
if (fs.existsSync(path)) { try { const raw = fs.readFileSync(path, 'utf-8'); const parsed = JSON.parse(raw) as LlmConfig & { providers: (LlmProviderConfig & { apiKeyEnc?: string })[] };
// Decrypt any encrypted keys; accept legacy plaintext `apiKey` for // migration (first save will re-encrypt). let migrated = false; let decryptFailures = 0; for (const p of parsed.providers || []) { const withEnc = p as LlmProviderConfig & { apiKeyEnc?: string }; try { p.type = normalizeProviderType(p.type); p.baseUrl = validateProviderBaseUrl(p.baseUrl, p.type); } catch (error) { console.warn(`[LLM] Disabling provider "${p.id}" with invalid baseUrl/type: ${(error as Error).message}`); p.enabled = false; } if (withEnc.apiKeyEnc) { const pt = decryptApiKey(withEnc.apiKeyEnc, CONFIG.API_KEY); if (pt !== null) { p.apiKey = pt; } else { decryptFailures++; p.apiKey = undefined; } delete withEnc.apiKeyEnc; } else if (typeof p.apiKey === 'string' && p.apiKey.length > 0) { // Legacy plaintext — accept and schedule re-encryption on save. migrated = true; } }
this.config = parsed; console.log(`[LLM] Loaded config from ${path}`); if (decryptFailures > 0) { console.warn(`[LLM] ${decryptFailures} API key(s) could not be decrypted — SUBSTRATE_API_KEY may have rotated. Affected providers will need their keys re-entered.`); } if (migrated) { console.log('[LLM] Migrating plaintext API keys to encrypted-at-rest format'); this.saveConfig(); } return; } catch (err: any) { console.warn(`[LLM] Failed to parse config, using defaults: ${err.message}`); } }
// Bootstrap from environment variables this.config = this.buildDefaultConfig(); this.saveConfig(); }
private buildDefaultConfig(): LlmConfig { const providers: LlmProviderConfig[] = BUILT_IN_TEMPLATES.map(t => ({ ...t, enabled: false, apiKey: undefined, }));
// If env vars specify a default provider, configure it if (CONFIG.LLM.DEFAULT_PROVIDER) { const provider = providers.find(p => p.id === CONFIG.LLM.DEFAULT_PROVIDER); if (provider) { provider.enabled = true; if (CONFIG.LLM.DEFAULT_BASE_URL) { provider.baseUrl = validateProviderBaseUrl(CONFIG.LLM.DEFAULT_BASE_URL, provider.type); } if (CONFIG.LLM.DEFAULT_MODEL) provider.defaultModel = CONFIG.LLM.DEFAULT_MODEL; if (CONFIG.LLM.DEFAULT_API_KEY) provider.apiKey = CONFIG.LLM.DEFAULT_API_KEY; } }
return { defaultProviderId: CONFIG.LLM.DEFAULT_PROVIDER || '', providers, }; }
private saveConfig(): void { const path = CONFIG.LLM.CONFIG_PATH; // Serialize with `apiKey` replaced by `apiKeyEnc` — no plaintext credentials // on disk. In-memory this.config keeps the plaintext for use in proxy calls. const serialized: LlmConfig = { ...this.config, providers: this.config.providers.map((p) => { const { apiKey, ...rest } = p; if (typeof apiKey === 'string' && apiKey.length > 0) { return { ...rest, apiKeyEnc: encryptApiKey(apiKey, CONFIG.API_KEY) } as LlmProviderConfig & { apiKeyEnc: string }; } return rest as LlmProviderConfig; }), }; try { // Write-to-tmp + rename so a crash mid-write can't corrupt the config. const tmp = `${path}.${process.pid}.${Date.now()}.tmp`; fs.writeFileSync(tmp, JSON.stringify(serialized, null, 2), 'utf-8'); fs.renameSync(tmp, path); } catch (err: any) { console.error(`[LLM] Failed to save config: ${err.message}`); } }
// Provider management
getConfig(): Omit<LlmConfig, 'providers'> & { providers: Omit<LlmProviderConfig, 'apiKey'>[] } { return { ...this.config, providers: this.config.providers.map(({ apiKey, ...rest }) => ({ ...rest, hasApiKey: !!apiKey, })) as any, }; }
getActiveProvider(): LlmProviderConfig | null { if (this.config.defaultProviderId) { const p = this.config.providers.find( p => p.id === this.config.defaultProviderId && p.enabled, ); if (p) return p; } // Fall back to first enabled return this.config.providers.find(p => p.enabled) ?? null; }
getProvider(id: string): LlmProviderConfig | undefined { return this.config.providers.find(p => p.id === id); }
setDefaultProvider(id: string): void { this.config.defaultProviderId = id; this.saveConfig(); }
async upsertProvider(update: Partial<LlmProviderConfig> & { id: string }): Promise<LlmProviderConfig> { const idx = this.config.providers.findIndex(p => p.id === update.id); const providerType = normalizeProviderType(update.type ?? this.config.providers[idx]?.type ?? 'local');
const nextProvider: LlmProviderConfig = idx >= 0 ? { ...this.config.providers[idx], ...update, type: providerType, apiKey: update.apiKey !== undefined ? update.apiKey : this.config.providers[idx].apiKey, } : { id: update.id, name: update.name || update.id, type: providerType, baseUrl: update.baseUrl || '', defaultModel: update.defaultModel || '', enabled: update.enabled ?? true, apiKey: update.apiKey, };
if (!nextProvider.baseUrl) { throw new Error('baseUrl is required'); } nextProvider.baseUrl = validateProviderBaseUrl(nextProvider.baseUrl, nextProvider.type); await assertProviderBaseUrlAllowed(nextProvider.baseUrl, nextProvider.type);
if (idx >= 0) { this.config.providers[idx] = nextProvider; } else { this.config.providers.push(nextProvider); } this.saveConfig(); return nextProvider; }
removeProvider(id: string): boolean { const before = this.config.providers.length; this.config.providers = this.config.providers.filter(p => p.id !== id); if (this.config.defaultProviderId === id) { this.config.defaultProviderId = ''; } this.saveConfig(); return this.config.providers.length < before; }
// Chat proxy
// Proxy chat completions as OpenAI-compatible SSE. async proxyChatStream( req: ChatRequest, onData: (chunk: Buffer) => void, onEnd: () => void, onError: (err: Error) => void, ): Promise<void> { const provider = req.providerId ? this.config.providers.find(p => p.id === req.providerId) : this.getActiveProvider();
if (!provider) { onError(new Error('No LLM provider configured')); return; }
if (!provider.enabled) { onError(new Error(`Provider "${provider.id}" is not enabled`)); return; } try { await assertProviderBaseUrlAllowed(provider.baseUrl, provider.type); } catch (error) { onError(error as Error); return; }
const model = req.model || provider.defaultModel; if (!model) { onError(new Error(`No model specified and provider "${provider.id}" has no default model`)); return; }
// Build OpenAI-compatible request body const body: any = { model, messages: req.messages, stream: req.stream !== false, };
if (req.systemPrompt) { body.messages = [ { role: 'system', content: req.systemPrompt }, ...body.messages, ]; }
if (req.tools && req.tools.length > 0) { body.tools = req.tools.map((t: any) => ({ type: 'function', function: { name: t.name, description: t.description, parameters: t.parameters || t.inputSchema, }, })); }
const url = new URL(`${resolveProviderBaseUrl(provider.baseUrl, provider.type)}/chat/completions`); const isHttps = url.protocol === 'https:'; const transport = isHttps ? https : http;
const headers: Record<string, string> = { 'Content-Type': 'application/json', 'Accept': 'text/event-stream', };
if (provider.apiKey) { // Anthropic uses a different header; all others use Bearer if (provider.id === 'anthropic') { headers['x-api-key'] = provider.apiKey; headers['anthropic-version'] = '2023-06-01'; } else { headers['Authorization'] = `Bearer ${provider.apiKey}`; } }
const payload = JSON.stringify(body); headers['Content-Length'] = Buffer.byteLength(payload).toString();
const proxyReq = transport.request( { hostname: url.hostname, port: url.port || (isHttps ? 443 : 80), path: url.pathname + url.search, method: 'POST', headers, timeout: CONFIG.LLM.PROXY_TIMEOUT_MS, agent: isHttps ? this.httpsAgent : this.httpAgent, }, (proxyRes) => { if (proxyRes.statusCode && proxyRes.statusCode >= 400) { const chunks: Buffer[] = []; let total = 0; let truncated = false; let settled = false; const finalize = () => { if (settled) return; settled = true; const body = Buffer.concat(chunks).toString(); const safeBody = sanitizeUpstreamError(body, 500); console.warn( `[LLM] chat(${provider.id}) failed: HTTP ${proxyRes.statusCode}: ${safeBody}${truncated ? ' [truncated]' : ''}`, ); onError(new Error(`LLM provider returned HTTP ${proxyRes.statusCode}`)); }; proxyRes.on('data', (c: Buffer) => { if (truncated) return; const remaining = CHAT_ERROR_MAX_BYTES - total; if (c.length <= remaining) { chunks.push(c); total += c.length; return; } if (remaining > 0) chunks.push(c.subarray(0, remaining)); total += remaining; truncated = true; proxyRes.destroy(); finalize(); }); proxyRes.on('end', finalize); proxyRes.on('error', () => { if (truncated) return; onError(new Error('LLM provider error response stream failed')); }); return; }
proxyRes.on('data', onData); proxyRes.on('end', onEnd); proxyRes.on('error', onError); }, );
proxyReq.on('error', onError); proxyReq.on('timeout', () => { proxyReq.destroy(); onError(new Error('LLM proxy request timed out')); });
proxyReq.write(payload); proxyReq.end(); }
// Model listing
async listModels( providerId?: string, ): Promise<{ provider: string; models: any[]; error?: string; status?: number }> { const provider = providerId ? this.config.providers.find(p => p.id === providerId) : this.getActiveProvider();
if (!provider) { const error = providerId ? `No provider configured with id "${providerId}"` : 'No active LLM provider configured'; console.warn(`[LLM] listModels: ${error}`); return { provider: '', models: [], error }; }
try { await assertProviderBaseUrlAllowed(provider.baseUrl, provider.type); const url = new URL(`${resolveProviderBaseUrl(provider.baseUrl, provider.type)}/models`); const isHttps = url.protocol === 'https:'; const transport = isHttps ? https : http;
const headers: Record<string, string> = { 'Accept': 'application/json' }; if (provider.apiKey) { if (provider.id === 'anthropic') { headers['x-api-key'] = provider.apiKey; headers['anthropic-version'] = '2023-06-01'; } else { headers['Authorization'] = `Bearer ${provider.apiKey}`; } }
return new Promise((resolve) => { let settled = false; const settle = (value: { provider: string; models: any[]; error?: string; status?: number }): void => { if (settled) return; settled = true; resolve(value); };
const req = transport.request( { hostname: url.hostname, port: url.port || (isHttps ? 443 : 80), path: url.pathname + url.search, method: 'GET', headers, timeout: 10_000, agent: isHttps ? this.httpsAgent : this.httpAgent, }, (res) => { const chunks: Buffer[] = []; let total = 0; let truncated = false;
const finalize = () => { const body = Buffer.concat(chunks).toString(); const status = res.statusCode ?? 0; if (status >= 400) { // Log the (redacted) upstream body for diagnostics, but return a // generic message to the caller — providers that echo submitted // credentials in 401/403 bodies must not leak them downstream. const safeBody = sanitizeUpstreamError(body, 200); console.warn( `[LLM] listModels(${provider.id}) failed: HTTP ${status}: ${safeBody}${truncated ? ' [truncated]' : ''}`, ); settle({ provider: provider.id, models: [], error: `Provider returned HTTP ${status}`, status, }); return; } try { const data = JSON.parse(body); const models = data.data || data.models || []; settle({ provider: provider.id, models }); } catch (err) { const error = `Failed to parse /models response as JSON: ${(err as Error).message}`; console.warn(`[LLM] listModels(${provider.id}) parse error: ${error}`); settle({ provider: provider.id, models: [], error, status }); } };
res.on('data', (c: Buffer) => { if (truncated) return; const remaining = LIST_MODELS_MAX_BYTES - total; if (c.length <= remaining) { chunks.push(c); total += c.length; return; } if (remaining > 0) chunks.push(c.subarray(0, remaining)); total += remaining; truncated = true; // Stop consuming and finalize after the cap is reached. res.destroy(); finalize(); }); res.on('end', finalize); res.on('error', () => { if (truncated) return; // we've already finalised settle({ provider: provider.id, models: [], error: 'Provider /models response stream error', }); }); }, ); req.on('error', (err) => { const error = `Request error: ${err.message}`; console.warn(`[LLM] listModels(${provider.id}) network error: ${error}`); settle({ provider: provider.id, models: [], error }); }); req.on('timeout', () => { req.destroy(); const error = 'Provider /models request timed out after 10s'; console.warn(`[LLM] listModels(${provider.id}): ${error}`); settle({ provider: provider.id, models: [], error }); }); req.end(); }); } catch (err) { const error = `Invalid provider URL "${provider.baseUrl}": ${(err as Error).message}`; console.warn(`[LLM] listModels(${provider.id}): ${error}`); return { provider: provider.id, models: [], error }; } }}