Something went wrong. Try again.
Open source agent hosting on the AT Protocol
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169// Check a model against the full languid loop: real pi (with pi-languid) -> languid service ->// runner and PDS. The model must list a directory through the runner, save the names to a// block, and then recall them in a resumed session from records alone.//// node scripts/model-check.ts xai/grok-4.7 [more/models ...]// node scripts/model-check.ts --headless xai/grok-4.7 # the Harness (languid run) instead of pi// node scripts/model-check.ts --service xai/grok-4.7 # a service-side run (ai.languid.agent.run)// node scripts/model-check.ts --local ollama/lfm3:latest # pi calls the model itself (--languid-local);// # the agent's own model stays radius/cheap//// Needs the dev stack (./dev/stack up). Uses a throwaway owner/agent and deletes them after.// pi runs with your normal pi config (so /login credentials such as subscriptions work), but// its sessions go to a temp dir, not your pi history. Prints one JSON line per model.
import { spawn } from 'node:child_process'import crypto from 'node:crypto'import fs from 'node:fs'import os from 'node:os'import path from 'node:path'
const headless = process.argv.includes('--headless')const viaService = process.argv.includes('--service')const local = process.argv.includes('--local')const models = process.argv.slice(2).filter((a) => a !== '--headless' && a !== '--service' && a !== '--local')const mode = headless ? 'headless' : viaService ? 'service' : local ? 'local' : 'pi'/** In --local mode, the agent's configured model: pi must not use it, so it needn't be callable. */const AGENT_MODEL = { provider: 'radius', model: 'cheap' }if (!models.length) { console.error('usage: node scripts/model-check.ts <provider/model> [...]') process.exit(2)}const root = path.resolve(import.meta.dirname, '..')const piBin = path.join(root, 'packages/pi-languid/node_modules/.bin/pi')
async function check(spec: string) { const [provider, ...rest] = spec.split('/') const model = rest.join('/') const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'languid-mc-')) process.env.LANGUID_HOME = path.join(tmp, 'languid') const { loadState, saveState } = await import('../src/state.ts') const { createAgent, ensureOwner, agentSession, deleteAccount } = await import('../src/manager.ts') const { AgentStore } = await import('../src/store.ts') const { connect, register } = await import('../src/runner.ts') const { inProcessRunner } = await import('../src/hub.ts') const { LanguidClient } = await import('../src/client.ts') const { createService } = await import('../src/server.ts')
const state = loadState() const sfx = crypto.randomBytes(3).toString('hex') await ensureOwner(state, `mcowner${sfx}.test`) const name = `mc${sfx}` let entry try { entry = await createAgent(state, name, { model: local ? AGENT_MODEL : { provider, model } }) } catch (e) { await deleteAccount(state.pds, state.owner!.did) // don't leave the throwaway owner behind fs.rmSync(tmp, { recursive: true, force: true }) throw e } const svcToken = crypto.randomBytes(16).toString('hex') const svc = createService({ host: '127.0.0.1', port: 0, token: svcToken }) state.server = { url: `http://127.0.0.1:${((await svc.listen()).address() as any).port}`, token: svcToken } saveState(state) const session = await agentSession(state, entry) // A computer "box" (this one) joined to the service, its runner polling in this process. const stopRunner = new AbortController() const { code } = await LanguidClient.fromState(state, name).inviteComputer([name], 5) const joined = await register(state.server.url, code, 'box') void connect({ serviceUrl: state.server.url, token: joined.token, signal: stopRunner.signal, log: () => {} }) const dir = path.join(tmp, 'listme') fs.mkdirSync(dir) fs.writeFileSync(path.join(dir, 'alpha.txt'), 'a') fs.writeFileSync(path.join(dir, 'beta.md'), 'b')
const pi = (prompt: string, extra: string[] = []) => new Promise<string>((resolve, reject) => { // stdin must be closed: pi reads piped stdin as part of the prompt. const localFlag = local ? ['--languid-local', spec] : [] const child = spawn(piBin, ['-e', path.join(root, 'packages/pi-languid'), '--languid', name, '--session-dir', path.join(tmp, 'sessions'), ...localFlag, ...extra, '--print', prompt], { cwd: root, env: process.env, stdio: ['ignore', 'pipe', 'pipe'], }) let out = '' let err = '' child.stdout.on('data', (d) => (out += d)) child.stderr.on('data', (d) => (err += d)) const timer = setTimeout(() => child.kill(), 180_000) child.on('close', (code) => { clearTimeout(timer) code === 0 ? resolve(out.trim()) : reject(new Error(`pi exited ${code}: ${err.trim().slice(0, 300)}`)) }) })
// Headless: the Harness with pi's ModelRuntime (pi's auth.json, env keys), in this process. let headlessRun: ((prompt: string) => Promise<string>) | undefined if (headless) { const { Harness } = await import('../src/harness.ts') const { createModels } = await import('../src/models.ts') let text = '' // No one to ask: deny every tool that needs approval (bash, edit, write). The check only needs ls. const h = new Harness(state, entry, session, await createModels(), { onTextDelta: (d) => (text += d), approve: async () => false }) h.runners = inProcessRunner('box') await h.load() const thread = (await h.newThread('model-check')).rkey headlessRun = async (prompt) => { text = '' await h.prompt(thread, prompt) return text.trim() } } // Service: ai.languid.agent.run, writes denied (approveWrites defaults to false). if (viaService) { const client = LanguidClient.fromState(state, name) let first = true headlessRun = async (prompt) => { const r = await client.runToCompletion(prompt, { continue: !first, pollMs: 200 }) first = false if (r.status !== 'done') throw new Error(r.error ?? 'run failed') return r.events.filter((e) => e.type === 'text').map((e: any) => e.text).join('\n') } } const ask = (prompt: string, extra: string[] = []) => (headlessRun ? headlessRun(prompt) : pi(prompt, extra))
const t0 = Date.now() try { const first = await ask(`List the directory ${dir} with ls, then save the file names to your notes block. Reply in one sentence.`) const second = await ask('Which files did you see? Answer from memory, no tools.', ['--continue']) const s = new AgentStore(state, entry, session) await s.load() const notes = s.blocks.get('notes')!.value.value const tokens = s.messages.reduce( (u, m) => ({ in: u.in + (m.value.usage?.input ?? 0) + (m.value.usage?.cacheRead ?? 0), out: u.out + (m.value.usage?.output ?? 0) }), { in: 0, out: 0 }, ) // In --local mode every assistant turn must have come from the local model, not the service's. const answeredBy = [...new Set(s.messages.filter((m) => m.value.role === 'assistant').map((m) => `${m.value.provider}/${m.value.model}`))] const ok = (!local || (answeredBy.length === 1 && answeredBy[0] === spec)) && s.calls.some((c) => c.value.tool === 'ls' && c.value.assignedComputer === 'box') && /alpha/.test(notes) && /beta/.test(notes) && /alpha/.test(second) && /beta/.test(second) return { model: spec, mode, ok, secs: Math.round((Date.now() - t0) / 1000), answeredBy, tools: s.calls.map((c) => c.value.tool), tokens, first, second } } catch (e: any) { return { model: spec, ok: false, error: e.message } } finally { stopRunner.abort() await svc.close() for (const did of [entry.agent.did, ...Object.values(state.authorities ?? {}).map((a) => a.did), state.owner!.did]) { for (let i = 0; i < 3; i++) { try { await deleteAccount(state.pds, did) break } catch { await new Promise((r) => setTimeout(r, 500)) } } } fs.rmSync(tmp, { recursive: true, force: true }) }}
let failed = falsefor (const m of models) { const r = await check(m) failed ||= !r.ok console.log(JSON.stringify(r))}process.exit(failed ? 1 : 0)