// Running many eval trials at once, as many as the machine and the model provider allow. // // A trial is mostly waiting on a model, so the limit is rarely this machine: it's the provider's // rate limit. The pool starts at a size the machine can take, and adapts the way TCP does: a model // call that's rate-limited halves the limit, and each trial that finishes without one adds one back, // up to the machine's ceiling. import os from 'node:os' /** * The most trials this machine should run at once. A trial is mostly waiting on the model; what it * costs here is its harness and, in the factory, its commands: about 256 MB and a quarter of a core * at worst. Total memory, not free: macOS counts its file cache as used, so free memory is always low. */ export function machineCeiling(): number { const byMemory = Math.floor((os.totalmem() * 0.5) / (256 * 1024 * 1024)) const byCpu = os.availableParallelism() * 4 return Math.max(1, Math.min(64, byMemory, byCpu)) } export interface PoolOptions { /** A fixed number of trials at once, or 'auto' (adapt between 1 and the machine's ceiling). */ concurrency?: number | 'auto' /** Where 'auto' starts. Default: 4, or the ceiling if lower. */ start?: number ceiling?: number onLimit?: (limit: number, why: string) => void } /** A concurrency limit that halves on backoff() and grows by one on each clean finish, when it's 'auto'. */ export class Limiter { limit: number readonly max: number readonly adaptive: boolean private running = 0 private waiting: (() => void)[] = [] private onLimit?: PoolOptions['onLimit'] /** When the limit last halved: rate limits arriving together count once. */ private lastBackoff = 0 constructor(opts: PoolOptions = {}) { this.adaptive = (opts.concurrency ?? 'auto') === 'auto' this.max = this.adaptive ? (opts.ceiling ?? machineCeiling()) : Math.max(1, Number(opts.concurrency)) this.limit = this.adaptive ? Math.min(this.max, opts.start ?? 4) : this.max this.onLimit = opts.onLimit } async acquire(): Promise { while (this.running >= this.limit) await new Promise((r) => this.waiting.push(r)) this.running++ } release(clean: boolean) { this.running-- if (clean && this.adaptive && this.limit < this.max) { this.limit++ this.onLimit?.(this.limit, 'a trial finished without being rate-limited') } // Wake as many waiters as there's now room for. for (let free = this.limit - this.running; free > 0 && this.waiting.length; free--) this.waiting.shift()!() } backoff(why: string) { if (!this.adaptive || Date.now() - this.lastBackoff < 5_000) return this.lastBackoff = Date.now() const next = Math.max(1, Math.floor(this.limit / 2)) if (next === this.limit) return this.limit = next this.onLimit?.(next, why) } } /** Whether a model error is the provider asking us to slow down. */ export const isRateLimit = (message: string) => /rate.?limit|too many requests|\b429\b|overload|quota/i.test(message) /** * Run every task, up to the limiter's number at once, and return their results in order. A task * gets a `slowDown` to call when it's rate-limited; it's then not counted as finishing cleanly. * Every task runs even if one throws; the first error is thrown once they've all settled. */ export async function runPool(items: T[], task: (item: T, slowDown: (why: string) => void) => Promise, limiter = new Limiter()): Promise { const out: R[] = new Array(items.length) const settled = await Promise.allSettled( items.map(async (item, i) => { await limiter.acquire() let clean = true try { out[i] = await task(item, (why) => ((clean = false), limiter.backoff(why))) } catch (e) { clean = false throw e } finally { limiter.release(clean) } }), ) const failed = settled.find((r) => r.status === 'rejected') if (failed) throw (failed as PromiseRejectedResult).reason return out }