// Rate limits for the service: token buckets, kept in memory (one service process). // // Unauthenticated requests are limited per client address; authenticated ones per owner DID or per // runner. Model calls spend the service's own model credentials, so they get a tighter bucket and a // cap on concurrent streams per owner. The defaults sit well above what languid's own clients do (a // CLI sign-in polls every 2s, `languid run` polls every 500ms, pi makes a few calls per step). import type http from 'node:http' import net from 'node:net' export interface Bucket { /** Requests allowed at once. */ burst: number /** Sustained requests per minute. */ perMinute: number } export const DEFAULT_LIMITS = { /** * Per client address: starting a sign-in or a sign-up. Sized for a room on one Wi-Fi address: a * hundred people arriving within a couple of minutes, each taking two (sign-up, then sign-in). * Passwords are checked by the PDS, not here, and a sign-up needs an invite code. */ signIn: { burst: 200, perMinute: 60 }, /** Per client address: the other sign-in steps (the CLI's poll, the browser's redirects and approval). As signIn: a room shares one address. */ oauth: { burst: 200, perMinute: 200 }, /** Per client address: computer registration with an invite code. As signIn: a room shares one address. */ register: { burst: 40, perMinute: 20 }, /** Per client address: pages for browsers that aren't signed in (signed-in ones count against their owner). As signIn: a room shares one address. */ pages: { burst: 300, perMinute: 300 }, /** Per client address: requests with a missing or wrong token. When it runs out, the address gets 429s until it refills. */ authFailures: { burst: 30, perMinute: 6 }, /** Per owner DID: every XRPC call. */ owner: { burst: 300, perMinute: 600 }, /** Per owner DID: model calls and runs, which spend the service's model credentials. */ model: { burst: 60, perMinute: 60 }, /** Per owner DID: creating agents, each a permanent account and DID. */ createAgent: { burst: 10, perMinute: 2 }, /** Per runner: polls and results. */ runner: { burst: 300, perMinute: 600 }, } satisfies Record export type LimitName = keyof typeof DEFAULT_LIMITS export interface LimitOptions { buckets?: Partial> /** Concurrent model streams per owner. */ modelStreams?: number /** * A header that holds the client's address, set by a proxy in front of the service (Fly sets * `fly-client-ip`). Only set this behind such a proxy: otherwise clients choose their own address. */ clientIpHeader?: string } const MAX_KEYS = 50_000 export class RateLimiter { private buckets: Record private state = new Map() private streams = new Map() /** Who waits for a stream, per owner, oldest first (waitStream). */ private waiters = new Map void) | false) => void)[]>() private sweeper: NodeJS.Timeout readonly modelStreams: number private clientIpHeader?: string private now: () => number constructor(opts: LimitOptions = {}, now: () => number = Date.now) { this.buckets = { ...DEFAULT_LIMITS, ...opts.buckets } this.modelStreams = opts.modelStreams ?? 8 this.clientIpHeader = opts.clientIpHeader?.toLowerCase() this.now = now this.sweeper = setInterval(() => this.sweep(), 60_000) this.sweeper.unref() } close() { clearInterval(this.sweeper) } private level(name: LimitName, key: string) { const b = this.buckets[name] const k = `${name}\0${key}` const t = this.now() const s = this.state.get(k) const tokens = s ? Math.min(b.burst, s.tokens + ((t - s.at) * b.perMinute) / 60_000) : b.burst return { b, k, t, tokens } } /** Take one request from `key`'s bucket. Returns 0 if allowed, else the seconds until one is. */ take(name: LimitName, key: string): number { const { b, k, t, tokens } = this.level(name, key) if (tokens < 1) return Math.max(1, Math.ceil(((1 - tokens) * 60) / b.perMinute)) if (!this.state.has(k) && this.state.size >= MAX_KEYS) this.sweep() this.state.delete(k) // re-insert, so the Map stays in last-used order for eviction this.state.set(k, { tokens: tokens - 1, at: t }) return 0 } /** Like take, without taking: the seconds until `key` has a request left, or 0. */ wait(name: LimitName, key: string): number { const { b, tokens } = this.level(name, key) return tokens < 1 ? Math.max(1, Math.ceil(((1 - tokens) * 60) / b.perMinute)) : 0 } /** Start a model stream for `owner`; false if it already has the maximum running, or others wait for one. Call the result to end it. */ startStream(owner: string): (() => void) | false { const n = this.streams.get(owner) ?? 0 if (n >= this.modelStreams || this.waiters.get(owner)?.length) return false this.streams.set(owner, n + 1) return this.ender(owner) } /** * Wait for a model stream for `owner`, first come first served: tasks and their wakeups queue here * instead of being refused. A freed stream goes straight to the longest waiter, so a new chat * can't take it first. Resolves false if `signal` aborts first. */ waitStream(owner: string, signal?: AbortSignal): Promise<(() => void) | false> { const now = this.startStream(owner) if (now) return Promise.resolve(now) if (signal?.aborted) return Promise.resolve(false) return new Promise((resolve) => { const queue = this.waiters.get(owner) ?? [] const waiter = (end: (() => void) | false) => { signal?.removeEventListener('abort', onAbort) resolve(end) } const onAbort = () => { const q = this.waiters.get(owner) ?? [] const i = q.indexOf(waiter) if (i >= 0) q.splice(i, 1) if (!q.length) this.waiters.delete(owner) waiter(false) } signal?.addEventListener('abort', onAbort, { once: true }) queue.push(waiter) this.waiters.set(owner, queue) }) } /** How many wait for a stream of `owner`'s. */ waiting(owner: string): number { return this.waiters.get(owner)?.length ?? 0 } /** The function that ends one of `owner`'s streams: it passes the stream to the next waiter, or gives it back. */ private ender(owner: string): () => void { let ended = false return () => { if (ended) return ended = true const q = this.waiters.get(owner) const next = q?.shift() if (q && !q.length) this.waiters.delete(owner) // Handed over: the count stays the same. if (next) return next(this.ender(owner)) const left = (this.streams.get(owner) ?? 1) - 1 if (left > 0) this.streams.set(owner, left) else this.streams.delete(owner) } } /** Drop buckets that have refilled (they'd start full anyway); past MAX_KEYS, the least recently used. */ private sweep() { const t = this.now() for (const [k, s] of this.state) { const b = this.buckets[k.slice(0, k.indexOf('\0')) as LimitName] if (s.tokens + ((t - s.at) * b.perMinute) / 60_000 >= b.burst) this.state.delete(k) } for (const k of this.state.keys()) { if (this.state.size < MAX_KEYS) break this.state.delete(k) } } /** The key for a request's client: its address, with IPv6 cut to its /64 (one host usually has a whole /64). */ client(req: http.IncomingMessage): string { const header = this.clientIpHeader ? req.headers[this.clientIpHeader] : undefined const joined = Array.isArray(header) ? header.join(',') : header let fromHeader: string | undefined if (joined) { const parts = joined .split(',') .map((s) => s.trim()) .filter(Boolean) const picked = this.clientIpHeader === 'x-forwarded-for' ? parts.at(-1) : parts[0] if (picked && net.isIP(picked)) fromHeader = picked } const raw = fromHeader || req.socket.remoteAddress || 'unknown' const ip = raw.startsWith('::ffff:') && net.isIPv4(raw.slice(7)) ? raw.slice(7) : raw if (!net.isIPv6(ip)) return ip return expand6(ip).slice(0, 4).join(':') + '::/64' } } /** * Whether a socket address is this host (loopback) or on a tailnet (Tailscale's 100.64.0.0/10 and * fd7a:115c:a1e0::/48). Callers pass the socket's own address, never one taken from a header. */ export function hostOrTailnet(address: string | undefined): boolean { if (!address) return false const ip = address.startsWith('::ffff:') && net.isIPv4(address.slice(7)) ? address.slice(7) : address if (net.isIPv4(ip)) { const [a, b] = ip.split('.').map(Number) return a === 127 || (a === 100 && b >= 64 && b <= 127) } if (!net.isIPv6(ip)) return false const g = expand6(ip) return g.join(':') === '0:0:0:0:0:0:0:1' || (g[0] === 'fd7a' && g[1] === '115c' && g[2] === 'a1e0') } /** The eight groups of an IPv6 address. */ function expand6(ip: string): string[] { const [head, tail] = ip.split('%')[0].split('::') const h = head ? head.split(':') : [] const t = tail !== undefined && tail ? tail.split(':') : [] const groups = tail === undefined ? h : [...h, ...Array(8 - h.length - t.length).fill('0'), ...t] return groups.map((g) => g.toLowerCase().replace(/^0+(?=.)/, '')) }