import type { WebConfig } from "#lib/server/config.ts"; import { pushMetrics, Registry, type Labels } from "#lib/server/otlp.ts"; const PUSH_INTERVAL_MS = 15_000; const LAG_INTERVAL_MS = 500; const SECONDS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 15, 30, 60]; const LAG_SECONDS = [0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5]; export const registry = new Registry(); const requests = registry.counter("svfe.http.requests", "{request}"); const requestSeconds = registry.histogram("svfe.http.request.duration", "s", SECONDS); const upstreams = registry.counter("svfe.upstream.requests", "{request}"); const upstreamSeconds = registry.histogram("svfe.upstream.duration", "s", SECONDS); const loopLag = registry.histogram("svfe.event_loop.lag", "s", LAG_SECONDS); const sheds = registry.counter("svfe.http.shed", "{request}"); let inFlight = 0; let busySince: number | undefined; registry.gauge("svfe.http.in_flight", "{request}", () => inFlight); const statusClass = (status: number) => `${Math.floor(status / 100)}xx`; export const requestStarted = () => { if (inFlight++ === 0) busySince = performance.now(); }; export const requestFinished = (routeClass: string, status: number, startedAt: number) => { if (--inFlight === 0) busySince = undefined; const labels = { route_class: routeClass, status_class: statusClass(status) }; requests.add(labels); requestSeconds.record(labels, (performance.now() - startedAt) / 1000); }; export const recordShed = (gate: string) => sheds.add({ gate }); type UpstreamOutcome = "ok" | "error" | "timeout"; // anything that isn't ours is "other", knots and pdses would make endless labels export const upstreamHost = (url: URL, ours: ReadonlySet) => ours.has(url.origin) ? url.host : "other"; export const recordUpstream = ( host: string, outcome: UpstreamOutcome, status: number | undefined, startedAt: number ) => { const labels = { host, outcome, status_class: status === undefined ? "none" : statusClass(status) }; upstreams.add(labels); upstreamSeconds.record({ host }, (performance.now() - startedAt) / 1000); }; // a tick that comes late after idle is cloud run's cpu throttle, not a slow loop. lag only counts // when requests were in flight for the whole tick before it export const startLagSampler = (intervalMs = LAG_INTERVAL_MS) => { let last = performance.now(); const timer = setInterval(() => { const now = performance.now(); const wholeTick = busySince !== undefined && busySince <= last; if (wholeTick) loopLag.record({}, Math.max(0, now - last - intervalMs) / 1000); last = now; }, intervalMs); timer.unref?.(); return () => clearInterval(timer); }; export const resourceAttributes = (raw: string): Labels => Object.fromEntries( raw .split(",") .map((pair) => pair.split("=")) .filter(([name, value]) => name?.trim() && value !== undefined) .map(([name, ...value]) => [name.trim(), decodeURIComponent(value.join("=").trim())]) ); export const startMetrics = (config: WebConfig) => { if (!config.otlpEndpoint) return undefined; const stopLag = startLagSampler(); const push = pushMetrics({ registry, endpoint: config.otlpEndpoint, auth: config.otlpAuth, resource: { ...resourceAttributes(config.otlpResourceAttributes), "service.name": config.otlpServiceName, "service.instance.id": crypto.randomUUID().replaceAll("-", "").slice(0, 16) }, intervalMs: PUSH_INTERVAL_MS }); return () => { stopLag(); push.stop(); }; };