Something went wrong. Try again.
Parametric learning in your code harness.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123// The instructions optimizer is itself a machine. Its input is a report on// another machine: what it is for, its tools, its rubric, its current// instructions, and how it failed on graded runs. Its output is revised// instructions. Whether a proposal is kept is decided by comparing the two// versions on the same cases, never by the optimizer.
import type { Comparison } from "./compare.ts";import { check, defineEvaluator } from "./evaluator.ts";import { defineMachine, type Machine } from "./machine.ts";import { defineRubric } from "./rubric.ts";import type { Run } from "./types.ts";
const rubric = defineRubric({ proposal: { description: "Returns complete revised instructions between <instructions> and </instructions>, different from the current instructions.", },});
/** The revised instructions in an optimizer reply, or null if there are none. */export function parseProposal(output: string): string | null { const match = output.match(/<instructions>\s*([\s\S]*?)\s*<\/instructions>/); const text = match?.[1]?.trim(); return text ? text : null;}
export const optimizer = defineMachine({ name: "optimizer", intent: "Improve another machine's instructions from its graded failures.", instructions: [ 'You improve the instructions of another AI system, called "the machine".', "You get the machine's purpose, its tools, the rubric it is graded on, its current instructions, and the cases where it failed, with what it did and why the graders failed it.", "Work out the cause of each failure: what in the instructions was missing, unclear, or misleading. Then write complete revised instructions that fix those causes.", "Keep everything that already works, and do not break the cases that passed.", "Prefer clear general rules over lists of specific test inputs. Examples are fine when they illustrate a rule.", "If inherited instructions are given, the machine also follows them and you cannot change them: revise only the current instructions, and do not repeat the inherited ones.", "The machine only sees its instructions and the user's message: do not mention the rubric, graders, tests, or this process.", "Reply with one or two sentences on the causes, then the complete revised instructions between <instructions> and </instructions>, and nothing after them.", ].join("\n"), model: "set-at-use", rubric, evaluator: defineEvaluator(rubric, { proposal: check("proposal", (run) => { const proposal = parseProposal(run.output); if (!proposal) return { score: 0, reason: "no <instructions> block" }; const current = run.input.match(/<current_instructions>\s*([\s\S]*?)\s*<\/current_instructions>/)?.[1]?.trim(); if (proposal === current) return { score: 0, reason: "proposal is identical to the current instructions" }; return { score: 1, reason: `${proposal.length} characters` }; }), }),});
/** * The optimizer's input: everything it needs to improve `machine`, built from * graded runs grouped by case (all trials of one input together). */export function failureReport(machine: Machine, runsByCase: Run[][]): string { const clip = (text: string, n = 600) => (text.length > n ? `${text.slice(0, n)}…` : text); const tools = machine.tools.map((t) => `- ${t.name}: ${t.description}`).join("\n") || "(none)"; const rubricText = Object.entries(machine.rubric).map(([id, c]) => `- ${id}: ${c.description}`).join("\n"); const failing: string[] = []; const passing: string[] = []; for (const runs of runsByCase) { if (runs.length === 0) continue; const failed = runs.filter((r) => (r.scorecard?.total ?? 0) < 1); if (failed.length === 0) { // For a conversation, how it opened; its last message alone says little. passing.push(`- ${runs[0].history.find((m) => m.role === "user")?.content ?? runs[0].input}`); continue; } const example = failed[0]; const did = example.trajectory .filter((m) => m.role === "assistant" && m.toolCallId) .map((m) => `called ${m.name} ${clip(JSON.stringify(m.toolInput ?? {}), 200)}`); const reasons = Object.entries(example.scorecard?.criteria ?? {}) .filter(([, c]) => c.score < 1) .map(([id, c]) => `${id}: ${c.reason}`); // A conversation's earlier turns, so the optimizer sees how it got to the last one. const earlier = example.history.map((m) => (m.role === "tool" ? `(tool ${m.name} → ${clip(m.content, 200)})` : m.toolCallId ? `(called ${m.name})` : `${m.role === "user" ? "User" : "Machine"}: ${clip(m.content, 400)}`)); failing.push( [ `<case failed="${failed.length}/${runs.length}">`, ...earlier, `User: ${example.input}`, `The machine ${did.length ? did.join("; ") : "called no tools"}, then replied: ${clip(example.output) || "(nothing)"}`, `Why it failed: ${reasons.join("; ") || "unknown"}`, "</case>", ].join("\n"), ); } return [ `<purpose>\n${machine.intent}\n</purpose>`, `<tools>\n${tools}\n</tools>`, `<rubric>\n${rubricText}\n</rubric>`, ...(machine.parent ? [`<inherited_instructions>\n${machine.parent.instructions}\n</inherited_instructions>`] : []), `<current_instructions>\n${machine.ownInstructions}\n</current_instructions>`, `<failures>\n${failing.join("\n") || "(none)"}\n</failures>`, `<passing_cases>\n${passing.join("\n") || "(none)"}\n</passing_cases>`, ].join("\n\n");}
export type Decision = { keep: boolean; reason: string };
/** * Keep a proposal only if it scores clearly higher on the cases the optimizer * learned from (by at least `minZ` standard errors, default 2) and no lower on * held-out cases it never saw. A smaller win is indistinguishable from trial * noise; without the held-out check, a rewrite can win by memorizing its * failures. */export function decide(train: Comparison, holdout?: Comparison, options: { minZ?: number } = {}): Decision { const minZ = options.minZ ?? 2; const fmt = (c: Comparison) => `${c.diff >= 0 ? "+" : ""}${c.diff.toFixed(3)} ± ${c.stderr.toFixed(3)}`; if (train.diff <= 1e-9) return { keep: false, reason: `it did not score higher on the cases it learned from (${fmt(train)})` }; if (train.diff < minZ * train.stderr) { return { keep: false, reason: `its gain on the cases it learned from (${fmt(train)}) is under ${minZ} standard errors, within trial noise; run more trials or cases` }; } if (!holdout) return { keep: true, reason: `it scores higher on the cases it learned from (${fmt(train)}); there is no held-out set to check it against` }; if (holdout.diff < -1e-9) { return { keep: false, reason: `it scores higher on the cases it learned from (${fmt(train)}) but lower on held-out cases (${fmt(holdout)}): it likely memorized its failures` }; } return { keep: true, reason: `it scores higher on the cases it learned from (${fmt(train)}) and no lower on held-out cases (${fmt(holdout)})` };}