Something went wrong. Try again.
Open source agent hosting on the AT Protocol
Something went wrong. Try again.
12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273// Run eval cases on the dev stack, within one standing dev agent (docs/evals.md).//// just eval xai/grok-4.7 # evals/cases, 3 trials each// node scripts/eval.ts --model xai/grok-4.7 --trials 5 --judge openai-codex/gpt-6-luna evals/cases/pin-block.json//// The first run makes an owner and an agent named `evals` on the dev PDS and keeps them, with// their state in .dev/evals; later runs reuse them, so no accounts or DIDs are made per run. The// case files are put in the agent's space as the owner's (keyed by file name) and run there, each// trial in a scratch copy of the agent; the run and its results are the owner's too// (`LANGUID_HOME=.dev/evals node src/cli.ts eval list evals`). Exits 1 unless every trial passed.//// Hosted agents run the same thing with `languid eval run <agent>`.
import crypto from 'node:crypto'import path from 'node:path'import { parseArgs } from 'node:util'
const { values, positionals } = parseArgs({ allowPositionals: true, options: { model: { type: 'string' }, trials: { type: 'string', default: '3' }, judge: { type: 'string' }, untrusted: { type: 'boolean', default: false }, },})if (!values.model) { console.error('usage: node scripts/eval.ts --model <provider/model> [--trials n] [--judge provider/model] [--untrusted] [case.json | dir (default: evals/cases)]...') process.exit(2)}const root = path.resolve(import.meta.dirname, '..')process.env.LANGUID_HOME ??= path.join(root, '.dev', 'evals')const { loadState } = await import('../src/state.ts')const { createAgent, ensureOwner, agentSession, handleDomain, installationAuthority } = await import('../src/manager.ts')const { SpaceRepo } = await import('../src/atproto/space.ts')const { createModels } = await import('../src/models.ts')const { NSID } = await import('../src/records.ts')const { agentCases, evalAgent, readCaseFiles } = await import('../src/evals.ts')
const files = readCaseFiles(positionals.length ? positionals : [path.join(root, 'evals/cases')])const state = loadState()if (!state.owner) await ensureOwner(state, `evals${crypto.randomBytes(3).toString('hex')}${await handleDomain(state.pds)}`)const entry = state.agents.evals ?? (await createAgent(state, 'evals', { model: { provider: 'radius', model: 'cheap' } }))let sessiontry { session = await agentSession(state, entry)} catch (e: any) { console.error(`can't sign in as the dev eval agent (${e.message}); if the dev PDS was reset, remove ${process.env.LANGUID_HOME} and run again`) process.exit(2)}const owner = await installationAuthority(state)const home = new SpaceRepo(owner, entry.space)for (const f of files) await home.put(NSID.evalCase, f.rkey, f.value)const cases = await agentCases({ owner, agent: session }, entry, { only: files.map((f) => f.rkey), authors: ['owner'] })
const trials = Number(values.trials)const out = await evalAgent(state, entry, cases, { authority: owner, agent: session, writer: owner }, { models: await createModels(), trials, model: values.model, judge: values.judge, untrustedScripts: values.untrusted, onTrial: (k, trial, o) => { const failed = o.checks.filter((x) => !x.advisory && x.verdict !== 'pass' && x.verdict !== 'skipped') const why = o.status !== 'completed' ? `${o.status}${o.error ? `: ${o.error.slice(0, 200)}` : ''}` : failed.map((x) => `${x.id} ${x.verdict}: ${x.reason ?? ''}`).join('; ') console.log(`${o.passed ? 'PASS' : 'FAIL'} ${k.rkey} trial ${trial}/${trials} ${Math.round(o.usage.durationMs / 1000)}s${o.score !== undefined ? ` score ${o.score}` : ''}${why ? ` ${why}` : ''}`) },})const s = out.run.summaryconsole.log(`\n${values.model}: pass^${trials} ${s.passAll / 10}% pass@${trials} ${s.passAny / 10}% mean ${s.mean / 10}% over ${s.cases} case(s)`)console.log(out.uri)process.exit(out.run.status === 'completed' && s.passAll === 1000 ? 0 : 1)