Something went wrong. Try again.
Open source agent hosting on the AT Protocol
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101// Running many eval trials at once, as many as the machine and the model provider allow.//// A trial is mostly waiting on a model, so the limit is rarely this machine: it's the provider's// rate limit. The pool starts at a size the machine can take, and adapts the way TCP does: a model// call that's rate-limited halves the limit, and each trial that finishes without one adds one back,// up to the machine's ceiling.
import os from 'node:os'
/** * The most trials this machine should run at once. A trial is mostly waiting on the model; what it * costs here is its harness and, in the factory, its commands: about 256 MB and a quarter of a core * at worst. Total memory, not free: macOS counts its file cache as used, so free memory is always low. */export function machineCeiling(): number { const byMemory = Math.floor((os.totalmem() * 0.5) / (256 * 1024 * 1024)) const byCpu = os.availableParallelism() * 4 return Math.max(1, Math.min(64, byMemory, byCpu))}
export interface PoolOptions { /** A fixed number of trials at once, or 'auto' (adapt between 1 and the machine's ceiling). */ concurrency?: number | 'auto' /** Where 'auto' starts. Default: 4, or the ceiling if lower. */ start?: number ceiling?: number onLimit?: (limit: number, why: string) => void}
/** A concurrency limit that halves on backoff() and grows by one on each clean finish, when it's 'auto'. */export class Limiter { limit: number readonly max: number readonly adaptive: boolean private running = 0 private waiting: (() => void)[] = [] private onLimit?: PoolOptions['onLimit'] /** When the limit last halved: rate limits arriving together count once. */ private lastBackoff = 0
constructor(opts: PoolOptions = {}) { this.adaptive = (opts.concurrency ?? 'auto') === 'auto' this.max = this.adaptive ? (opts.ceiling ?? machineCeiling()) : Math.max(1, Number(opts.concurrency)) this.limit = this.adaptive ? Math.min(this.max, opts.start ?? 4) : this.max this.onLimit = opts.onLimit }
async acquire(): Promise<void> { while (this.running >= this.limit) await new Promise<void>((r) => this.waiting.push(r)) this.running++ }
release(clean: boolean) { this.running-- if (clean && this.adaptive && this.limit < this.max) { this.limit++ this.onLimit?.(this.limit, 'a trial finished without being rate-limited') } // Wake as many waiters as there's now room for. for (let free = this.limit - this.running; free > 0 && this.waiting.length; free--) this.waiting.shift()!() }
backoff(why: string) { if (!this.adaptive || Date.now() - this.lastBackoff < 5_000) return this.lastBackoff = Date.now() const next = Math.max(1, Math.floor(this.limit / 2)) if (next === this.limit) return this.limit = next this.onLimit?.(next, why) }}
/** Whether a model error is the provider asking us to slow down. */export const isRateLimit = (message: string) => /rate.?limit|too many requests|\b429\b|overload|quota/i.test(message)
/** * Run every task, up to the limiter's number at once, and return their results in order. A task * gets a `slowDown` to call when it's rate-limited; it's then not counted as finishing cleanly. * Every task runs even if one throws; the first error is thrown once they've all settled. */export async function runPool<T, R>(items: T[], task: (item: T, slowDown: (why: string) => void) => Promise<R>, limiter = new Limiter()): Promise<R[]> { const out: R[] = new Array(items.length) const settled = await Promise.allSettled( items.map(async (item, i) => { await limiter.acquire() let clean = true try { out[i] = await task(item, (why) => ((clean = false), limiter.backoff(why))) } catch (e) { clean = false throw e } finally { limiter.release(clean) } }), ) const failed = settled.find((r) => r.status === 'rejected') if (failed) throw (failed as PromiseRejectedResult).reason return out}