#!/usr/bin/env bun
// machine: run, use interactively, and grade the machines in a workspace directory.
// Machines live at <workspace>/machines/<name>/machine.ts; runs and sessions
// are stored under <workspace>/.machine/.

import { readFile, writeFile } from "node:fs/promises";
import path from "node:path";
import { parseArgs } from "node:util";
import { pathToFileURL } from "node:url";
import { agreement, contamination } from "./agree.ts";
import { compareTrials, summarize, trialStats, type Comparison, type TrialStats } from "./compare.ts";
import { judge, type Grader } from "./evaluator.ts";
import { jev } from "./graders/jev.ts";
import type { Harness } from "./harness.ts";
import { DEFAULT_TINKER_JUDGE, defaultHarness, lazyHarness } from "./harnesses/select.ts";
import { INSTRUCTIONS_FILE, withEvaluation, withInstructions, withModel, type Machine } from "./machine.ts";
import { decide, failureReport, optimizer, parseProposal } from "./optimizer.ts";
import { Budget, FREE, isFreeModel, tinkerPrice } from "./pricing.ts";
import { checkoutRevision, descendants, git, loadMachineFrom, machineChangedSince } from "./revision.ts";
import { exportRuns } from "./export.ts";
import { converse } from "./converse.ts";
import { hasCaseFile, loadCases, type Case } from "./cases.ts";
import { comparePaired, mapLimit, runMatrix } from "./trials.ts";
import { CommitRefused, formatReport, machine as sdkMachine } from "./sdk.ts";
import { formatSpecialization } from "./specialize.ts";
import { findVersion, versionFile } from "./versions.ts";
import { interactive } from "./interactive.ts";
import { fileEvents } from "./events.ts";
import { serveDashboard } from "./dashboard.ts";
import { nameSegments } from "./store.ts";
import type { Run } from "./types.ts";
import { createWorkspace, type Workspace } from "./workspace.ts";

const USAGE = `Usage:
  machine dashboard                    Live view of runs, sessions, and training in this workspace (--port, default 4242;
                                       --judge-model and --budget for grading from the page, default $1 per request;
                                       --host 0.0.0.0 to reach it from other machines, with a printed token)
  machine <machine>                    Use the machine interactively in Pi (--continue resumes, --eval grades each exchange)
  machine run <machine> [input...]     Run once and print the reply; without input, open it in Pi
  machine runs <machine>               List recent runs, newest first
  machine eval <machine> <run-id>      Grade a stored run against the machine's rubric
  machine suite <machine>              Run and grade every input in machines/<machine>/cases.json
  machine optimize <machine>           Rewrite the machine's instructions.md from its failures; keep a winning rewrite
  machine agree <machine> <criterion>  Measure another grader against the machine's own on stored runs
  machine compare <machine>            Compare the working tree's machine with a git revision of it
  machine tree <machine>               Show a machine, its children, and what each inherits
  machine export <machine>             Write graded runs as chat JSONL for fine-tuning
  machine train <machine>              Fine-tune the live model on good runs; report the candidate against the live
                                       version on holdout.json and leave it pending (never changes the machine)
  machine specialize <machine> --focus <area>
                                       An author model writes a child of the machine for the focus (new criteria with
                                       graders, instructions) and rehearsal cases (cases.json, holdout.json), validated
  machine propose <machine> --model <id> Measure another model against the live version on holdout.json, as a candidate
  machine commit <machine> [candidate] Make a pending candidate (default: the newest) the live version, if it passed
                                       the promotion gate (--force to override)
  machine revert <machine>             Make the previous live version live again
  machine versions <machine>           List the machine's versions; the first call records the current one as v1
  machine converse <machine> [case]    Run one conversation with the machine's counterpart and print it as it happens
                                       (case: a number in --cases, default 1, or the text of a new case)

Options:
  --harness tinker|pi|openrouter|opencode|fx|openai|echo
                                       Agent loop (default: $MACHINE_HARNESS, else pi for openrouter/ models
                                       and tinker otherwise)
  --model <id>                         Override the machine's model
  --judge-model <id>                   Model for judge graders (default: $MACHINE_JUDGE_MODEL; on tinker,
                                       thinkingmachines/Inkling-Small; otherwise the run's model)
  --budget <usd>|none                  Stop spending at this many dollars (default: 1 on tinker and pi, none elsewhere)
  --effort <level>                     Reasoning effort for runs: none, low, medium, high, xhigh, or 0-0.99 (default: the model's)
  --judge-effort <level>               Reasoning effort for judges (default: medium on tinker, else the model's)
  --price-as <model>                   Price a fine-tuned tinker:// checkpoint as this base model
  --concurrency <n>                    Runs at once in suite, compare, and optimize (default: 4)
  --base-url <url>                     Endpoint for --harness openai ($OPENAI_BASE_URL)
  --eval                               Grade each run after it finishes
  --workspace <dir>                    Workspace directory (default: current directory)
  --name <name>                        Workspace name (default: $MACHINE_WORKSPACE or local)
  --limit <n>                          Number of runs to list (default: 20; agree uses all stored runs)
  --json                               Print runs as JSON
  --rounds <n>                         In optimize, rewrite rounds (default: 1)
  --optimizer-model <id>               In optimize, the optimizer's model (default: the machine's)
  --with jev|judge                     In agree, the grader to measure (default: jev)
  --question <text>                    In agree, ask Jev this instead of the criterion's description
  --ref <git-ref>                      In compare, the revision to compare against (default: HEAD)
  --holdout <file>|none                In optimize, held-out cases (default: holdout.json if present)
  --cases <file>                       In suite, compare, and converse, a case file in the machine's directory (default:
                                       cases.json); in export, only runs made for these cases
  --trials <n>                         In suite and compare, run each case n times (default: 1)
  --tree                               In suite and export, include the machine's descendants
  --min-z <z>                          In optimize, standard errors a rewrite must win by (default: 2)
  --min-score <x>                      In export, the lowest scorecard total to include (default: 1)
  --best <k>                           In export, keep only each case's best k runs (rejection sampling)
  --since <time>                       In export, only runs started at or after this ISO time
  --per-turn                           In export, one example per assistant message, and every turn of a conversation
  --out <file>                         In export, the JSONL file to write (default: stdout)
  --from <machine>                     In export, write that machine's good runs as this machine's training data
                                       (its instructions, its transformReasoning), e.g. a parent's runs for a child
  --no-reasoning                       In export, leave the model's reasoning out (default: included, through the machine's transformReasoning)
  --samples <n>                        In train, new runs per case of cases.json before training (default: 0, stored runs)
  --best-of <k>                        In train, at most k good runs per case (default: 2)
  --epochs <n>, --lr <x>               In train, SFT epochs (default: 2) and learning rate (default: the cookbook's)
  --batch-size <n>                     In train, examples per training step (default: 8)
  --any-model                          In train, also train on good runs by other models (distillation)
  --force                              In commit, commit a candidate that did not pass the gate; in specialize, replace
                                       an existing child directory
  --focus <area>                       In specialize, what the child specializes in (required)
  --author-model <id>                  In specialize, the model that writes the child (default: thinkingmachines/Inkling);
                                       --model is the child's model (default: the parent's)
  --slug <name>, --train-cases <n>, --holdout-cases <n>
                                       In specialize, the child's directory name (default: the author's choice) and
                                       how many cases to keep (default: 20 and 8)
  --reuse-cases                        In specialize, rewrite only the child (with --slug), keeping its cases
  -v, --verbose                        In suite and compare, show the reason for each imperfect score

In train, --trials is runs per held-out case for each version (default: 2), --min-score the lowest score to
train on, --min-z the margin the candidate must win by, and --budget caps sampling, grading, training, and the
comparison together. Training runs scripts/train_sft.py with $MACHINE_PYTHON (a Python with tinker-cookbook).`;

async function main(argv: string[]): Promise<number> {
  const { values, positionals } = parseArgs({
    args: argv,
    allowPositionals: true,
    options: {
      harness: { type: "string" },
      model: { type: "string" },
      "judge-model": { type: "string" },
      "base-url": { type: "string" },
      eval: { type: "boolean", default: false },
      workspace: { type: "string", default: process.cwd() },
      name: { type: "string" },
      limit: { type: "string" },
      trials: { type: "string" },
      ref: { type: "string", default: "HEAD" },
      with: { type: "string", default: "jev" },
      rounds: { type: "string", default: "1" },
      cases: { type: "string" },
      holdout: { type: "string" },
      "optimizer-model": { type: "string" },
      question: { type: "string" },
      budget: { type: "string" },
      "price-as": { type: "string" },
      effort: { type: "string" },
      "judge-effort": { type: "string" },
      concurrency: { type: "string", default: "4" },
      tree: { type: "boolean", default: false },
      "min-z": { type: "string", default: "2" },
      "min-score": { type: "string", default: "1" },
      best: { type: "string" },
      since: { type: "string" },
      "per-turn": { type: "boolean", default: false },
      out: { type: "string" },
      "no-reasoning": { type: "boolean", default: false },
      from: { type: "string" },
      json: { type: "boolean", default: false },
      continue: { type: "boolean", short: "c", default: false },
      port: { type: "string" },
      host: { type: "string" },
      samples: { type: "string" },
      "best-of": { type: "string" },
      epochs: { type: "string" },
      lr: { type: "string" },
      "any-model": { type: "boolean", default: false },
      force: { type: "boolean", default: false },
      focus: { type: "string" },
      "author-model": { type: "string" },
      slug: { type: "string" },
      "train-cases": { type: "string" },
      "holdout-cases": { type: "string" },
      "reuse-cases": { type: "boolean", default: false },
      "batch-size": { type: "string" },
      verbose: { type: "boolean", short: "v", default: false },
      help: { type: "boolean", short: "h", default: false },
    },
  });
  if (positionals[0] === "dashboard") {
    const workspaceDir = path.resolve(values.workspace);
    const port = Number(values.port ?? 4242);
    if (!Number.isInteger(port) || port < 0 || port > 65535) throw new Error("--port must be a port number.");
    const gradeBudgetUsd = values.budget === undefined || values.budget === "none" ? undefined : Number(values.budget);
    if (gradeBudgetUsd !== undefined && !(gradeBudgetUsd >= 0)) throw new Error("--budget must be a dollar amount.");
    const host = values.host ?? "127.0.0.1";
    const server = await serveDashboard({ workspaceDir, port, host, judgeModel: values["judge-model"], gradeBudgetUsd });
    if (server.token) {
      // Reachable from other machines: print an address they can use, with the token.
      const { networkInterfaces } = await import("node:os");
      const lan = Object.values(networkInterfaces()).flat().filter((a) => a && a.family === "IPv4" && !a.internal).map((a) => a!.address);
      const hosts = host === "0.0.0.0" ? lan : [host];
      console.log(`machine dashboard on ${host}:${server.port}. Open once with the token; your browser keeps it:`);
      for (const h of hosts) console.log(`  http://${h}:${server.port}/?token=${server.token}`);
      console.log("Anyone with this link can read runs and spend on grading. Ctrl+C to stop.");
    } else {
      console.log(`machine dashboard: ${server.url}  (Ctrl+C to stop)`);
    }
    await new Promise(() => {});
  }
  // `machine <name>` opens the machine; anything else is `machine <command> <name> ...`.
  const interactiveMode = positionals.length === 1 && !COMMANDS.has(positionals[0]);
  const [command, machineName, ...rest] = interactiveMode ? ["interactive", positionals[0]] : positionals;
  if (values.help || !command || !machineName) {
    console.log(USAGE);
    return values.help ? 0 : 1;
  }

  const workspaceDir = path.resolve(values.workspace);
  if (VERSION_COMMANDS.has(command)) return versionCommand(command, machineName, rest, values, workspaceDir);
  const machine = await loadMachineFrom(workspaceDir, machineName);
  // Runs of the definition on disk record its version when it is in the machine's history.
  const version = values.model ? undefined : await recordedVersion(workspaceDir, machine);
  const kind = values.harness ?? process.env.MACHINE_HARNESS ?? defaultHarness(values.model ?? machine.model);
  // A conversation on Pi is Pi's own terminal UI, recorded by the machine like `machine <name>`.
  if (command === "interactive" || (command === "run" && rest.length === 0)) {
    const workspace = createWorkspace({
      name: values.name ?? process.env.MACHINE_WORKSPACE ?? "local",
      root: path.join(workspaceDir, ".machine"),
      harness: lazyHarness(values.harness ?? process.env.MACHINE_HARNESS ?? "pi", { priceAs: values["price-as"] }),
      judgeModel: values["judge-model"] ?? process.env.MACHINE_JUDGE_MODEL ?? DEFAULT_TINKER_JUDGE,
      judgeEffort: effortValue(values["judge-effort"]),
      events: fileEvents(path.join(workspaceDir, ".machine", "events.jsonl")),
    });
    return interactive(machine, workspace, {
      model: values.model,
      version,
      continue: values.continue,
      evaluate: values.eval,
      stateDir: path.join(workspaceDir, ".machine", "pi"),
      price: (model) => (isFreeModel(model) ? FREE : tinkerPrice(model.replace(/^tinker\//, ""), values["price-as"])),
    });
  }
  const harness = lazyHarness(kind, { baseURL: values["base-url"], priceAs: values["price-as"] });
  const budget = makeBudget(values.budget, kind);
  const workspace = createWorkspace({
    name: values.name ?? process.env.MACHINE_WORKSPACE ?? "local",
    root: path.join(workspaceDir, ".machine"),
    harness,
    judgeModel: values["judge-model"] ?? process.env.MACHINE_JUDGE_MODEL ?? (kind === "tinker" || kind === "pi" ? DEFAULT_TINKER_JUDGE : undefined),
    budget,
    events: fileEvents(path.join(workspaceDir, ".machine", "events.jsonl")),
    reasoningEffort: effortValue(values.effort),
    judgeEffort: effortValue(values["judge-effort"] ?? (kind === "tinker" ? "medium" : undefined)),
  });
  const runOptions = { model: values.model, evaluate: values.eval, version };
  try {
    return await dispatch(command, rest, machine, workspace, runOptions, values, workspaceDir, harness);
  } finally {
    await harness.close?.();
    if (budget && (budget.tokens.input || budget.tokens.output)) console.error(`budget: ${budget.summary()}`);
  }
}

async function dispatch(
  command: string,
  rest: string[],
  machine: Machine,
  workspace: Workspace,
  runOptions: { model?: string; evaluate?: boolean; version?: string },
  values: {
    json: boolean;
    limit?: string;
    verbose: boolean;
    trials?: string;
    ref: string;
    with: string;
    question?: string;
    "judge-model"?: string;
    "judge-effort"?: string;
    rounds: string;
    "optimizer-model"?: string;
    cases?: string;
    holdout?: string;
    concurrency: string;
    tree: boolean;
    "min-z": string;
    "min-score": string;
    best?: string;
    since?: string;
    "per-turn": boolean;
    out?: string;
    "no-reasoning": boolean;
    from?: string;
  },
  workspaceDir: string,
  harness: Harness,
): Promise<number> {

  switch (command) {
    case "run": {
      const input = rest.join(" ").trim();
      if (!input) throw new Error("run needs an input.");
      printRun(await workspace.run(machine, input, runOptions), values.json);
      return 0;
    }
    case "runs": {
      const runs = await workspace.store.listRuns(machine.name, { limit: Number(values.limit ?? 20) });
      if (values.json) console.log(JSON.stringify(runs, null, 2));
      else for (const run of runs) console.log(`${run.id}  ${scoreLabel(run)}  ${run.conversation ? `[turn ${run.conversation.turn}${run.conversation.final ? ", last" : ""}] ` : ""}${preview(run.input, 60)}`);
      return 0;
    }
    case "suite": {
      const trials = trialCount(values.trials ?? "1");
      const concurrency = positiveInt(values.concurrency, "--concurrency");
      const machines = values.tree ? [machine, ...(await descendants(workspaceDir, machine.name))] : [machine];
      const results: Array<{ machine: Machine; runsByCase: Run[][] }> = [];
      for (const m of machines) {
        let cases: Case[];
        try {
          cases = await loadCases(workspaceDir, m, values.cases);
        } catch (error) {
          if (values.tree) {
            console.log(`${m.name}: skipped (${error instanceof Error ? error.message : error})`);
            continue;
          }
          throw error;
        }
        if (values.tree) console.log(`\n== ${m.name}`);
        const [runsByCase] = await runMatrix(workspace, [{ machine: m, model: runOptions.model, version: m === machine ? runOptions.version : undefined }], cases, trials, concurrency, (i, [runs]) => {
          console.log(`${String(i + 1).padStart(3)}. ${runs.length ? statLabel(trialStats(runs)) : "error"}  ${preview(cases[i].input, 50)}`);
          if (values.verbose) printFailures(runs, "");
        });
        printSummary(cases.length, trials, runsByCase);
        results.push({ machine: m, runsByCase });
      }
      if (values.tree && results.length > 1) printTreeSummary(machine, results);
      return 0;
    }
    case "compare": {
      if (rest.length > 0) throw new Error("compare takes one machine; choose the revision to compare against with --ref.");
      const ref = values.ref;
      const checkout = await checkoutRevision(workspaceDir, ref);
      try {
        const old = await loadMachineFrom(checkout.workspaceDir, machine.name);
        // Grade both versions with today's rubric and evaluator, so the comparison
        // measures the change in behavior and nothing else.
        const baseline = withEvaluation(old, machine.rubric, machine.evaluator);
        if (!(await machineChangedSince(workspaceDir, machine.name, checkout.commit))) {
          console.error(`note: machines/${machine.name} has no changes since ${ref} (${checkout.short}); both sides run the same machine, so differences are noise.`);
        }
        if (JSON.stringify(old.rubric) !== JSON.stringify(machine.rubric)) {
          console.error(`note: the rubric changed since ${ref}; both sides are graded with the current one.`);
        }
        const cases = await loadCases(workspaceDir, machine, values.cases);
        const trials = trialCount(values.trials ?? "1");
        const head = await git(workspaceDir, ["rev-parse", "--short", "HEAD"]);
        const labelA = `${ref} (${checkout.short})`;
        const labelB = "working tree";
        const revisionA = checkout.short;
        const revisionB = `${head.stdout.trim()}+working-tree`;
        const width = Math.max(labelA.length, labelB.length, 14);
        console.log(`${"".padStart(4)} ${labelA.padStart(width)} ${labelB.padStart(width)}`);
        // Each case's trials of both versions are scheduled together, so both see the same conditions (rate limits, provider load).
        const [runsA, runsB] = await runMatrix(
          workspace,
          [
            { machine: baseline, model: runOptions.model, revision: revisionA },
            { machine, model: runOptions.model, revision: revisionB, version: runOptions.version },
          ],
          cases,
          trials,
          positiveInt(values.concurrency, "--concurrency"),
          (i, [a, b]) => {
            const cell = (runs: Run[]) => (runs.length ? statLabel(trialStats(runs)) : "error");
            const mark = a.length && b.length ? (trialStats(b).mean > trialStats(a).mean ? "+" : trialStats(b).mean < trialStats(a).mean ? "-" : " ") : "?";
            console.log(`${String(i + 1).padStart(3)}. ${cell(a).padStart(width)} ${cell(b).padStart(width)} ${mark} ${preview(cases[i].input, 45)}`);
            if (values.verbose) {
              printFailures(a, `${labelA} `);
              printFailures(b, `${labelB} `);
            }
          },
        );
        printComparison(labelA, labelB, comparePaired(cases, runsA, runsB), trials);
      } finally {
        await checkout.close();
      }
      return 0;
    }
    case "optimize": {
      await runOptimize(machine, workspace, workspaceDir, runOptions, values);
      return 0;
    }
    case "tree": {
      printTree(machine, await descendants(workspaceDir, machine.name));
      return 0;
    }
    case "export": {
      // --from <machine>: that machine's good runs, written as this machine's training data.
      const source = values.from ? await loadMachineFrom(workspaceDir, values.from) : undefined;
      if (source && values.tree) throw new Error("export takes --from or --tree, not both.");
      const machines = source ? [source] : values.tree ? [machine, ...(await descendants(workspaceDir, machine.name))] : [machine];
      const minScore = Number(values["min-score"]);
      if (!(minScore >= 0 && minScore <= 1)) throw new Error("--min-score must be between 0 and 1.");
      const bestOf = values.best === undefined ? undefined : positiveInt(values.best, "--best");
      if (values.since && Number.isNaN(Date.parse(values.since))) throw new Error("--since must be an ISO time, e.g. 2026-10-11T18:00:00Z.");
      const cases = values.cases ? (await loadCases(workspaceDir, machine, values.cases)).map((c) => c.input) : undefined;
      const { lines, counts } = await exportRuns(workspace.store, machines, {
        minScore,
        reasoning: !values["no-reasoning"],
        as: source ? machine : undefined,
        bestOf,
        since: values.since ? new Date(values.since).toISOString() : undefined,
        cases,
        perTurn: values["per-turn"],
      });
      const text = lines.map((l) => JSON.stringify(l)).join("\n") + (lines.length ? "\n" : "");
      if (values.out) await writeFile(values.out, text, { mode: 0o600 });
      else process.stdout.write(text);
      for (const [name, c] of Object.entries(counts)) console.error(`${name}: ${c.kept} of ${c.graded} graded runs at or above ${minScore}`);
      console.error(`${lines.length} examples${values.out ? ` written to ${values.out}` : ""}. They hold prompts and outputs: inspect before training or sharing.`);
      return 0;
    }
    case "agree": {
      const criterionId = rest[0];
      if (!criterionId || !(criterionId in machine.rubric)) {
        throw new Error(`agree needs a criterion of ${machine.name}: ${Object.keys(machine.rubric).join(", ")}.`);
      }
      await runAgreement(machine, criterionId, workspace, harness, values);
      return 0;
    }
    case "converse": {
      if (!machine.counterpart) throw new Error(`${machine.name} has no counterpart; give its machine.ts a counterpart to converse with.`);
      const arg = rest.join(" ").trim();
      const cases = await loadCases(workspaceDir, machine, values.cases);
      const index = arg === "" ? 1 : /^\d+$/.test(arg) ? Number(arg) : undefined;
      if (index !== undefined && !(index >= 1 && index <= cases.length)) throw new Error(`Case ${index} is out of range: there are ${cases.length}.`);
      const caseInput = index === undefined ? arg : cases[index - 1].input;
      const other = machine.counterpart.machine.name;
      console.log(`${machine.name} with ${other}${index === undefined ? "" : `, case ${index}`}: ${preview(caseInput, 100)}\n`);
      const result = await converse(workspace, machine, caseInput, {
        ...runOptions,
        evaluateCounterpart: runOptions.evaluate,
        onTurn: (side, run) => printTurn(side === "agent" ? machine.name : other, run),
      });
      console.log(`${result.ended === "done" ? "Done" : "Stopped at the turn limit"} after ${result.agentRuns.length} turns · conversation ${result.id}`);
      for (const run of [result.run, ...(runOptions.evaluate ? [result.counterpartRuns.at(-1)!] : [])]) {
        if (!run.scorecard) continue;
        console.log(`\n${run.machine} ${run.id}: score ${run.scorecard.total}`);
        for (const [id, c] of Object.entries(run.scorecard.criteria)) console.log(`  ${id.padEnd(20)} ${c.score.toFixed(2)}  ${c.reason}`);
      }
      return 0;
    }
    case "eval": {
      const id = rest[0];
      if (!id) throw new Error("eval needs a run id.");
      const run = await workspace.store.getRun(machine.name, id);
      if (!run) throw new Error(`No run ${id} for ${machine.name}.`);
      printRun(await workspace.evaluate(machine, run), values.json);
      return 0;
    }
    default:
      console.log(USAGE);
      return 1;
  }
}

/**
 * Rewrite a machine's instructions from its failures. Each round runs the
 * suite, gives the failures to the optimizer machine, runs the suite with the
 * proposal, and keeps the proposal only if it scores higher. A kept proposal
 * is written to instructions.md and left uncommitted for review.
 */
async function runOptimize(
  machine: Machine,
  workspace: Workspace,
  workspaceDir: string,
  runOptions: { model?: string },
  values: { trials?: string; rounds: string; "optimizer-model"?: string; verbose: boolean; holdout?: string; concurrency: string; "min-z": string },
) {
  const concurrency = positiveInt(values.concurrency, "--concurrency");
  const minZ = Number(values["min-z"]);
  if (!(minZ >= 0)) throw new Error("--min-z must be a non-negative number.");
  const runCases = async (m: Machine, inputs: Case[], revision: string) =>
    (await runMatrix(workspace, [{ machine: m, model: runOptions.model, revision }], inputs, trials, concurrency))[0];
  const file = path.join(workspaceDir, "machines", ...nameSegments(machine.name, "machine name"), INSTRUCTIONS_FILE);
  let onDisk: string;
  try {
    onDisk = (await readFile(file, "utf8")).trim();
  } catch {
    throw new Error(`${machine.name}'s instructions are not an optimization target: move them to ${file} and load them with instructionsFrom(import.meta.url).`);
  }
  if (onDisk !== machine.ownInstructions) throw new Error(`${machine.name}'s instructions do not come from ${file}.`);

  const cases = await loadCases(workspaceDir, machine);
  const trials = trialCount(values.trials ?? "1");
  const rounds = trialCount(values.rounds);
  // Held-out cases judge each proposal but are never shown to the optimizer.
  const machineDir = path.dirname(file);
  const holdoutFile =
    values.holdout === "none" ? undefined : (values.holdout ?? ((await hasCaseFile(workspaceDir, machine, "holdout.json")) ? "holdout.json" : undefined));
  if (holdoutFile && path.resolve(machineDir, holdoutFile) === path.join(machineDir, "cases.json")) {
    throw new Error("The held-out file must not be cases.json, which the optimizer learns from.");
  }
  const holdout = holdoutFile ? await loadCases(workspaceDir, machine, holdoutFile) : undefined;
  if (holdout) console.log(`held-out: ${holdout.length} cases from ${holdoutFile}, never shown to the optimizer`);
  else console.log("warning: no held-out set (machines/<name>/holdout.json); a rewrite may win by memorizing its failures");
  const head = (await git(workspaceDir, ["rev-parse", "--short", "HEAD"])).stdout.trim() || "unversioned";
  const optimizerMachine = withModel(optimizer, values["optimizer-model"] ?? runOptions.model ?? machine.model);

  let current = machine;
  console.log(`baseline: ${cases.length} cases × ${trials} trials`);
  let currentRuns = await runCases(current, cases, `${head}+working-tree`);
  let currentHoldout: Run[][] | undefined;
  let kept = 0;
  for (let round = 1; round <= rounds; round++) {
    const before = summarize(currentRuns);
    console.log(`\nround ${round}: current instructions score ${before.mean.toFixed(3)} (pass rate ${pct(before.passRate)})`);
    if (before.passRate === 1) {
      console.log("  every run passed; nothing to learn from.");
      break;
    }
    const optRun = await workspace.run(optimizerMachine, failureReport(current, currentRuns), { evaluate: true });
    const proposal = parseProposal(optRun.output);
    if (!proposal || optRun.scorecard?.total !== 1) {
      console.log(`  the optimizer made no usable proposal (${optRun.scorecard?.criteria.proposal?.reason ?? "no reply"}); run ${optRun.id}`);
      continue;
    }
    const diagnosis = optRun.output.slice(0, optRun.output.indexOf("<instructions>")).trim();
    if (diagnosis) console.log(`  optimizer: ${diagnosis.replace(/\s+/g, " ")}`);
    console.log(`  proposal:\n${proposal.replace(/^/gm, "    | ")}`);
    const candidate = withInstructions(current, proposal);
    const candidateRuns = await runCases(candidate, cases, `optimizer:${optRun.id}`);
    const c = comparePaired(cases, currentRuns, candidateRuns);
    console.log(`  proposal scores ${c.b.mean.toFixed(3)} vs ${c.a.mean.toFixed(3)} (${signed(c.diff)} ± ${c.stderr.toFixed(3)}); better on ${c.better} cases, worse on ${c.worse}`);
    if (values.verbose) {
      for (const x of c.cases) if (x.diff !== 0) console.log(`    ${signed(x.diff)}  ${preview(x.input, 60)}`);
    }
    let h: Comparison | undefined;
    let candidateHoldout: Run[][] | undefined;
    if (c.diff > 0 && holdout) {
      currentHoldout ??= await runCases(current, holdout, `${head}+working-tree`);
      candidateHoldout = await runCases(candidate, holdout, `optimizer:${optRun.id}`);
      h = comparePaired(holdout, currentHoldout, candidateHoldout);
      console.log(`  held out: ${h.b.mean.toFixed(3)} vs ${h.a.mean.toFixed(3)} (${signed(h.diff)} ± ${h.stderr.toFixed(3)}); better on ${h.better} cases, worse on ${h.worse}`);
      if (values.verbose) {
        for (const x of h.cases) if (x.diff !== 0) console.log(`    ${signed(x.diff)}  ${preview(x.input, 60)} (held out)`);
      }
    }
    const decision = decide(c, h, { minZ });
    if (decision.keep) {
      current = candidate;
      currentRuns = candidateRuns;
      currentHoldout = candidateHoldout;
      kept++;
      await writeFile(file, `${proposal}\n`);
      console.log(`  kept: ${decision.reason}; wrote ${path.relative(workspaceDir, file)}`);
    } else {
      console.log(`  discarded: ${decision.reason}`);
    }
  }
  console.log(
    kept
      ? `\n${kept} rewrite${kept === 1 ? "" : "s"} kept. The change is uncommitted: review with git diff, commit to accept, or git checkout -- ${path.relative(workspaceDir, file)} to reject.`
      : "\nNo rewrite was kept; instructions.md is unchanged.",
  );
}

/**
 * Grade a machine's stored runs with a candidate grader and measure it against
 * the machine's own grader for one criterion. Stored scores from that grader
 * are reused, so a judge reference is not paid for twice.
 */
async function runAgreement(
  machine: Machine,
  criterionId: string,
  workspace: Workspace,
  harness: Harness,
  values: { limit?: string; with: string; question?: string; "judge-model"?: string; "judge-effort"?: string; verbose: boolean },
) {
  const criterion = machine.rubric[criterionId];
  const reference = machine.evaluator[criterionId];
  // A conversation is graded on its last turn; earlier turns are not runs to grade on their own.
  const runs = (await workspace.store.listRuns(machine.name, { limit: Number(values.limit ?? 10_000) })).filter(
    (r) => r.harness !== "echo" && (!r.conversation || r.conversation.final),
  );
  if (runs.length === 0) throw new Error(`No stored runs for ${machine.name}; run its suite first.`);
  let candidate: Grader;
  if (values.with === "jev") candidate = jev().noul({ soft: true, question: values.question });
  else if (values.with === "judge") candidate = judge({ model: values["judge-model"], reasoningEffort: effortValue(values["judge-effort"]) });
  else throw new Error(`--with must be jev or judge, not "${values.with}".`);
  if (candidate.name === reference.name) throw new Error(`${criterionId} is already graded by ${reference.name}.`);

  const grade = async (grader: Grader, run: Run) => {
    workspace.budget?.check(`grading run ${run.id} with ${grader.name}`);
    const score = await grader.grade({ run, criterionId, criterion, harness, judgeModel: values["judge-model"] });
    workspace.budget?.charge(score.usage, `Grading run ${run.id} with ${grader.name}`);
    return score.score;
  };
  let reused = 0;
  const pairs = await mapLimit(runs, 8, async (run) => {
    const stored = run.scorecard?.criteria[criterionId];
    const ref = stored && stored.grader === reference.name ? (reused++, stored.score) : await grade(reference, run);
    return { run, ref: ref >= 0.5, p: await grade(candidate, run) };
  });

  const refLabels = pairs.map((x) => x.ref);
  const passRate = refLabels.filter(Boolean).length / pairs.length;
  console.log(`${machine.name} · ${criterionId} · ${candidate.name} against ${reference.name} · ${pairs.length} stored runs (${reused} reference grades reused)`);
  console.log(`reference passes ${pct(passRate)} of runs\n`);
  console.log("threshold  agree  kappa  false-pass  false-fail  precision  bits/label  bad kept at 10% / 50% pass rate");
  const thresholds = values.with === "jev" ? [0.5, 0.7, 0.9] : [0.5];
  for (const t of thresholds) {
    const a = agreement(refLabels, pairs.map((x) => x.p >= t));
    const at = (p: number) => pct(contamination(p, a.falsePassRate, a.falseFailRate));
    console.log(
      `${t.toFixed(1).padStart(9)}  ${pct(a.agreement).padStart(5)}  ${a.kappa.toFixed(2).padStart(5)}  ${`${pct(a.falsePassRate)} (${a.negatives - a.trueFails}/${a.negatives})`.padStart(10)}  ${`${pct(a.falseFailRate)} (${a.positives - a.truePasses}/${a.positives})`.padStart(10)}  ${pct(a.precision).padStart(9)}  ${Math.max(0, a.bitsPerLabel).toFixed(2).padStart(10)}  ${at(0.1)} / ${at(0.5)}`,
    );
  }
  const unsure = pairs.filter((x) => x.p > 0.3 && x.p < 0.7).length;
  if (unsure) {
    console.log(`\n${unsure} of ${pairs.length} runs scored between 0.3 and 0.7: the candidate is unsure there, and its scores can cross 0.5 from one call to the next. Escalate those to a stronger grader.`);
  }
  if (refLabels.every(Boolean) || refLabels.every((x) => !x)) {
    console.log(`\nnote: the reference ${refLabels[0] ? "passed" : "failed"} every run, so only one error rate can be measured; add runs of the other kind.`);
  }
  const disagreements = new Map<string, number>();
  for (const x of pairs) {
    if ((x.p >= 0.5) === x.ref) continue;
    const key = `${x.ref ? "false fail" : "false pass"}: ${preview(x.run.input, 70)}`;
    disagreements.set(key, (disagreements.get(key) ?? 0) + 1);
  }
  if (disagreements.size) {
    console.log("\nDisagreements at 0.5:");
    for (const [k, v] of [...disagreements].sort((a, b) => b[1] - a[1]).slice(0, values.verbose ? 50 : 10)) console.log(`  ${v}× ${k}`);
  }
}


function printComparison(nameA: string, nameB: string, c: Comparison, trials: number) {
  console.log(`\n${c.cases.length} cases × ${trials} trial${trials === 1 ? "" : "s"} per machine`);
  console.log(`  ${nameA.padEnd(14)} mean ${c.a.mean.toFixed(3)} · pass rate ${pct(c.a.passRate)}`);
  console.log(`  ${nameB.padEnd(14)} mean ${c.b.mean.toFixed(3)} · pass rate ${pct(c.b.passRate)}`);
  console.log(`  difference     ${signed(c.diff)} ± ${c.stderr.toFixed(3)} (standard error over cases)`);
  console.log(`  ${nameB} better on ${c.better} cases, worse on ${c.worse}, tied on ${c.tied}`);
  for (const id of Object.keys(c.a.criteria)) {
    console.log(`  ${id.padEnd(14)} ${c.a.criteria[id].toFixed(3)} → ${c.b.criteria[id].toFixed(3)}  ${signed(c.b.criteria[id] - c.a.criteria[id])}`);
  }
}

function printFailures(runs: Run[], label: string) {
  for (const [t, run] of runs.entries()) {
    for (const [id, c] of Object.entries(run.scorecard!.criteria)) {
      if (c.score < 1) console.log(`       ${label}trial ${t + 1} ${id}: ${c.reason}`);
    }
  }
}

function statLabel(s: TrialStats): string {
  return s.trials === 1 ? s.mean.toFixed(2) : `${s.mean.toFixed(2)} (${Math.round(s.passRate * s.trials)}/${s.trials})`;
}

function pct(x: number): string {
  return `${Math.round(x * 100)}%`;
}

function trialCount(value: string): number {
  const n = Number(value);
  if (!Number.isInteger(n) || n < 1 || n > 100) throw new Error("--trials must be a whole number from 1 to 100.");
  return n;
}

function signed(x: number): string {
  return `${x >= 0 ? "+" : ""}${x.toFixed(3)}`;
}



const COMMANDS = new Set(["dashboard", "run", "runs", "eval", "suite", "optimize", "agree", "compare", "tree", "export", "converse", "train", "propose", "commit", "revert", "versions", "specialize"]);

function makeBudget(value: string | undefined, kind: string): Budget | undefined {
  if (value === "none") return undefined;
  if (value === undefined) return kind === "tinker" || kind === "pi" ? new Budget(1) : undefined;
  const usd = Number(value);
  if (!Number.isFinite(usd) || usd < 0) throw new Error("--budget must be a dollar amount or none.");
  return new Budget(usd);
}

/** "none" … "xhigh" pass through; a number in [0, 0.99] is sent as a number. */
function effortValue(value: string | undefined): string | number | undefined {
  if (value === undefined || value === "default") return undefined;
  if (["none", "minimal", "low", "medium", "high", "xhigh"].includes(value)) return value;
  const n = Number(value);
  if (Number.isFinite(n) && n >= 0 && n <= 0.99) return n;
  throw new Error(`Effort must be none, minimal, low, medium, high, xhigh, default, or a number from 0 to 0.99; got "${value}".`);
}

function positiveInt(value: string, flag: string): number {
  const n = Number(value);
  if (!Number.isInteger(n) || n < 1 || n > 64) throw new Error(`${flag} must be a whole number from 1 to 64.`);
  return n;
}

function printSummary(caseCount: number, trials: number, runsByCase: Run[][]) {
  if (runsByCase.flat().length === 0) {
    console.log("\nno successful runs");
    return;
  }
  const summary = summarize(runsByCase);
  console.log(`\n${caseCount} cases × ${trials} trial${trials === 1 ? "" : "s"} · mean ${summary.mean.toFixed(3)} · pass rate ${pct(summary.passRate)}`);
  for (const [id, score] of Object.entries(summary.criteria)) console.log(`  ${id.padEnd(16)} ${score.toFixed(3)}`);
}

/**
 * Criteria shared down a tree, scored across every machine that inherits
 * them: how well the family does at what all of it must do, next to what
 * each child adds.
 */
function printTreeSummary(root: Machine, results: Array<{ machine: Machine; runsByCase: Run[][] }>) {
  console.log(`\n== ${root.name} tree: inherited criteria across ${results.length} machines`);
  for (const id of Object.keys(root.rubric)) {
    const scores = results.flatMap((r) => r.runsByCase.flat().map((run) => run.scorecard?.criteria[id]?.score)).filter((x): x is number => x !== undefined);
    if (scores.length) console.log(`  ${id.padEnd(16)} ${(scores.reduce((a, b) => a + b, 0) / scores.length).toFixed(3)}  (${scores.length} runs)`);
  }
}

function printTree(root: Machine, children: Machine[]) {
  const own = (m: Machine) => Object.keys(m.rubric).filter((id) => !m.parent || !(id in m.parent.rubric));
  const depth = (m: Machine) => m.name.split("/").length - root.name.split("/").length;
  for (const m of [root, ...children]) {
    const pad = "  ".repeat(depth(m));
    console.log(`${pad}${m.name} · ${m.model}`);
    console.log(`${pad}  ${m.intent}`);
    if (m.parent) console.log(`${pad}  inherits: ${Object.keys(m.parent.rubric).join(", ")}`);
    console.log(`${pad}  ${m.parent ? "adds" : "criteria"}: ${own(m).join(", ")}`);
  }
}

function printRun(run: Run, json: boolean) {
  if (json) {
    console.log(JSON.stringify(run, null, 2));
    return;
  }
  console.log(run.output);
  console.error(`\n${run.id} · ${run.harness} · ${run.model}`);
  if (run.scorecard) {
    console.error(`score ${run.scorecard.total}`);
    for (const [id, c] of Object.entries(run.scorecard.criteria)) {
      console.error(`  ${id.padEnd(12)} ${c.score.toFixed(2)}  (${c.grader}) ${c.reason}`);
    }
  }
}

/** One turn of a conversation: who spoke, their tool calls with results, and what they said. */
function printTurn(speaker: string, run: Run) {
  console.log(`── ${speaker}`);
  for (const m of run.trajectory) {
    if (m.role === "tool") console.log(`   [${m.name} →] ${m.content.replace(/\n/g, "\n   ")}`);
  }
  console.log(`${run.output.trim() || "(no message)"}\n`);
}

function scoreLabel(run: Run): string {
  return run.scorecard ? `score ${run.scorecard.total.toFixed(2)}` : "unscored  ";
}

function preview(text: string, limit: number): string {
  const flat = text.replace(/\s+/g, " ").trim();
  return flat.length > limit ? `${flat.slice(0, limit - 1)}…` : flat;
}

const VERSION_COMMANDS = new Set(["train", "propose", "commit", "revert", "versions", "specialize"]);

/** The recorded version this definition matches, if the machine keeps a history. Read-only. */
async function recordedVersion(workspaceDir: string, machine: Machine): Promise<string | undefined> {
  try {
    const file = versionFile(path.join(workspaceDir, "machines", ...nameSegments(machine.name, "machine name"), "versions.json"));
    return findVersion(await file.read(machine.name), machine)?.id;
  } catch {
    return undefined;
  }
}

/** train, commit, revert, and versions: through the SDK. */
async function versionCommand(
  command: string,
  name: string,
  rest: string[],
  values: {
    harness?: string;
    model?: string;
    "judge-model"?: string;
    "judge-effort"?: string;
    effort?: string;
    budget?: string;
    name?: string;
    concurrency: string;
    trials?: string;
    "min-score": string;
    "min-z": string;
    cases?: string;
    holdout?: string;
    samples?: string;
    "best-of"?: string;
    epochs?: string;
    lr?: string;
    "any-model": boolean;
    force: boolean;
    json: boolean;
    focus?: string;
    "author-model"?: string;
    slug?: string;
    "train-cases"?: string;
    "holdout-cases"?: string;
    "reuse-cases": boolean;
    "batch-size"?: string;
  },
  workspaceDir: string,
): Promise<number> {
  const m = await sdkMachine.load(name, {
    workspace: workspaceDir,
    workspaceName: values.name,
    harness: values.harness,
    judgeModel: values["judge-model"],
    judgeEffort: effortValue(values["judge-effort"]),
    reasoningEffort: effortValue(values.effort),
    concurrency: positiveInt(values.concurrency, "--concurrency"),
    log: (line) => console.error(line),
  });
  try {
    switch (command) {
      case "versions": {
        const { live, versions, liveOrder } = await m.versions();
        if (values.json) {
          console.log(JSON.stringify({ live: live.id, liveOrder, versions }, null, 2));
          return 0;
        }
        for (const v of versions) {
          const mark = v.id === live.id ? "*" : " ";
          const source = v.source.kind === "train" ? `trained (candidate ${v.source.candidate})` : v.source.kind === "optimize" ? `optimized (candidate ${v.source.candidate})` : v.source.kind === "propose" ? `proposed (candidate ${v.source.candidate})` : v.source.kind;
          console.log(`${mark} ${v.id.padEnd(4)} ${v.createdAt.slice(0, 16).replace("T", " ")}  ${v.model}  ${source}${v.note ? ` · ${v.note}` : ""}`);
        }
        console.log(`\nlive: ${live.id}${liveOrder.length > 1 ? ` (became live in order ${liveOrder.join(" → ")})` : ""}`);
        const pending = await m.candidates({ status: "pending" });
        if (pending.length) console.log(`pending candidates:\n${pending.map((c) => `  ${c.id}  ${c.report.decision.keep ? "passed" : "held  "}  ${c.report.kind}  held out ${c.report.holdout.live.mean.toFixed(3)} → ${c.report.holdout.candidate.mean.toFixed(3)}`).join("\n")}`);
        return 0;
      }
      case "train": {
        if (values.budget === undefined || values.budget === "none") throw new Error("train needs --budget <usd>: it samples, grades, trains, and compares on paid endpoints.");
        const budget = Number(values.budget);
        if (!(budget >= 0)) throw new Error("--budget must be a dollar amount.");
        const definition = m.definition;
        const candidate = await m.train({
          budget,
          samples: values.samples === undefined ? 0 : Number(values.samples),
          cases: values.cases ? await loadCases(workspaceDir, definition, values.cases) : undefined,
          holdout: values.holdout ? await loadCases(workspaceDir, definition, values.holdout) : undefined,
          trials: values.trials === undefined ? undefined : trialCount(values.trials),
          minScore: Number(values["min-score"]),
          minZ: Number(values["min-z"]),
          bestOf: values["best-of"] === undefined ? undefined : positiveInt(values["best-of"], "--best-of"),
          epochs: values.epochs === undefined ? undefined : positiveInt(values.epochs, "--epochs"),
          learningRate: values.lr === undefined ? undefined : Number(values.lr),
          batchSize: values["batch-size"] === undefined ? undefined : positiveInt(values["batch-size"], "--batch-size"),
          anyModel: values["any-model"],
        });
        console.log(formatReport(candidate.report));
        console.log(
          candidate.report.decision.keep
            ? `\nCandidate ${candidate.id} is pending. Make it live: machine commit ${name}`
            : `\nCandidate ${candidate.id} is pending but did not pass the gate (machine commit ${name} --force overrides).`,
        );
        return 0;
      }
      case "specialize": {
        if (!values.focus?.trim()) throw new Error('specialize needs --focus "<area>".');
        if (values.budget === undefined || values.budget === "none") throw new Error("specialize needs --budget <usd>: the author model is paid.");
        const result = await m.specialize({
          focus: values.focus.trim(),
          budget: Number(values.budget),
          model: values.model,
          authorModel: values["author-model"],
          slug: values.slug,
          trainCases: values["train-cases"] === undefined ? undefined : positiveInt(values["train-cases"], "--train-cases"),
          holdoutCases: values["holdout-cases"] === undefined ? undefined : positiveInt(values["holdout-cases"], "--holdout-cases"),
          ...(values.effort === undefined ? {} : { authorEffort: effortValue(values.effort) }),
          overwrite: values.force,
          reuseCases: values["reuse-cases"],
        });
        console.log(formatSpecialization(result));
        if (values.json) console.log(JSON.stringify(result, null, 2));
        return 0;
      }
      case "propose": {
        if (!values.model) throw new Error("propose needs --model <id>.");
        if (values.budget === undefined || values.budget === "none") throw new Error("propose needs --budget <usd>.");
        const definition = m.definition;
        const candidate = await m.propose(
          { model: values.model },
          {
            budget: Number(values.budget),
            holdout: values.holdout ? await loadCases(workspaceDir, definition, values.holdout) : undefined,
            trials: values.trials === undefined ? undefined : trialCount(values.trials),
            minZ: Number(values["min-z"]),
          },
        );
        console.log(formatReport(candidate.report));
        console.log(candidate.report.decision.keep ? `\nCandidate ${candidate.id} is pending. Make it live: machine commit ${name}` : `\nCandidate ${candidate.id} is pending but did not pass the gate.`);
        return 0;
      }
      case "commit": {
        const candidate = await m.candidate(rest[0] ?? "latest");
        const from = await m.version();
        try {
          const entry = await candidate.commit({ force: values.force });
          const h = candidate.report.holdout;
          console.log(`${name}: ${from.id} → ${entry.id} (${entry.model})`);
          console.log(`  held out: mean ${h.live.mean.toFixed(3)} → ${h.candidate.mean.toFixed(3)}, pass rate ${Math.round(h.live.passRate * 100)}% → ${Math.round(h.candidate.passRate * 100)}%`);
          console.log(`  ${candidate.report.decision.keep ? "passed" : "FORCED past"} the gate: ${candidate.report.decision.reason}`);
          console.log(`  undo with: machine revert ${name}`);
          return 0;
        } catch (error) {
          if (error instanceof CommitRefused) {
            const h = error.report.holdout;
            console.error(`${name}: not committed; ${from.id} stays live (${from.model})`);
            console.error(`  candidate ${candidate.id} (${error.report.candidate.model})`);
            console.error(`  held out: mean ${h.live.mean.toFixed(3)} → ${h.candidate.mean.toFixed(3)} (${h.diff >= 0 ? "+" : ""}${h.diff.toFixed(3)} ± ${h.stderr.toFixed(3)}), pass rate ${Math.round(h.live.passRate * 100)}% → ${Math.round(h.candidate.passRate * 100)}%`);
            console.error(`  ${error.message.replace(/^Not committed: /, "").replace(/ Force it with .*$/, "")}`);
            console.error(`  machine commit ${name} --force commits it anyway.`);
            return 1;
          }
          throw error;
        }
      }
      case "revert": {
        const from = await m.version();
        const to = await m.revert();
        console.log(`${name}: ${from.id} → ${to.id} (${to.model})`);
        return 0;
      }
      default:
        return 1;
    }
  } finally {
    await m.close();
  }
}

main(process.argv.slice(2)).then(
  (code) => process.exit(code),
  (error) => {
    console.error(`machine: ${error instanceof Error ? error.message : error}`);
    process.exit(1);
  },
);