// Run eval cases on the dev stack, within one standing dev agent (docs/evals.md). // // just eval xai/grok-4.7 # evals/cases, 3 trials each // node scripts/eval.ts --model xai/grok-4.7 --trials 5 --judge openai-codex/gpt-6-luna evals/cases/pin-block.json // // The first run makes an owner and an agent named `evals` on the dev PDS and keeps them, with // their state in .dev/evals; later runs reuse them, so no accounts or DIDs are made per run. The // case files are put in the agent's space as the owner's (keyed by file name) and run there, each // trial in a scratch copy of the agent; the run and its results are the owner's too // (`LANGUID_HOME=.dev/evals node src/cli.ts eval list evals`). Exits 1 unless every trial passed. // // Hosted agents run the same thing with `languid eval run `. import crypto from 'node:crypto' import path from 'node:path' import { parseArgs } from 'node:util' const { values, positionals } = parseArgs({ allowPositionals: true, options: { model: { type: 'string' }, trials: { type: 'string', default: '3' }, judge: { type: 'string' }, untrusted: { type: 'boolean', default: false }, }, }) if (!values.model) { console.error('usage: node scripts/eval.ts --model [--trials n] [--judge provider/model] [--untrusted] [case.json | dir (default: evals/cases)]...') process.exit(2) } const root = path.resolve(import.meta.dirname, '..') process.env.LANGUID_HOME ??= path.join(root, '.dev', 'evals') const { loadState } = await import('../src/state.ts') const { createAgent, ensureOwner, agentSession, handleDomain, installationAuthority } = await import('../src/manager.ts') const { SpaceRepo } = await import('../src/atproto/space.ts') const { createModels } = await import('../src/models.ts') const { NSID } = await import('../src/records.ts') const { agentCases, evalAgent, readCaseFiles } = await import('../src/evals.ts') const files = readCaseFiles(positionals.length ? positionals : [path.join(root, 'evals/cases')]) const state = loadState() if (!state.owner) await ensureOwner(state, `evals${crypto.randomBytes(3).toString('hex')}${await handleDomain(state.pds)}`) const entry = state.agents.evals ?? (await createAgent(state, 'evals', { model: { provider: 'radius', model: 'cheap' } })) let session try { session = await agentSession(state, entry) } catch (e: any) { console.error(`can't sign in as the dev eval agent (${e.message}); if the dev PDS was reset, remove ${process.env.LANGUID_HOME} and run again`) process.exit(2) } const owner = await installationAuthority(state) const home = new SpaceRepo(owner, entry.space) for (const f of files) await home.put(NSID.evalCase, f.rkey, f.value) const cases = await agentCases({ owner, agent: session }, entry, { only: files.map((f) => f.rkey), authors: ['owner'] }) const trials = Number(values.trials) const out = await evalAgent(state, entry, cases, { authority: owner, agent: session, writer: owner }, { models: await createModels(), trials, model: values.model, judge: values.judge, untrustedScripts: values.untrusted, onTrial: (k, trial, o) => { const failed = o.checks.filter((x) => !x.advisory && x.verdict !== 'pass' && x.verdict !== 'skipped') const why = o.status !== 'completed' ? `${o.status}${o.error ? `: ${o.error.slice(0, 200)}` : ''}` : failed.map((x) => `${x.id} ${x.verdict}: ${x.reason ?? ''}`).join('; ') console.log(`${o.passed ? 'PASS' : 'FAIL'} ${k.rkey} trial ${trial}/${trials} ${Math.round(o.usage.durationMs / 1000)}s${o.score !== undefined ? ` score ${o.score}` : ''}${why ? ` ${why}` : ''}`) }, }) const s = out.run.summary console.log(`\n${values.model}: pass^${trials} ${s.passAll / 10}% pass@${trials} ${s.passAny / 10}% mean ${s.mean / 10}% over ${s.cases} case(s)`) console.log(out.uri) process.exit(out.run.status === 'completed' && s.passAll === 1000 ? 0 : 1)