// `machine dashboard`: a live local view of a workspace. It reads what is // already on disk (machine definitions, stored runs, training logs under // .machine/training/) and follows .machine/events.jsonl, where every machine // process reports what it is doing. Served on 127.0.0.1 only: runs hold // prompts and outputs. import { randomBytes, timingSafeEqual } from "node:crypto"; import { readdir, readFile, stat } from "node:fs/promises"; import { createServer, type ServerResponse } from "node:http"; import path from "node:path"; import { fileEvents, readEventsFrom, type MachineEvent } from "./events.ts"; import type { Harness } from "./harness.ts"; import { chatCompletionsHarness, TINKER_OAI_BASE_URL } from "./harnesses/chat-completions.ts"; import { piHarness } from "./harnesses/pi.ts"; import type { Machine } from "./machine.ts"; import { Budget, FREE, isFreeModel, TINKER_PRICES, TINKER_TRAIN_PRICES, tinkerPrice, withPricing } from "./pricing.ts"; import { loadMachineFrom } from "./revision.ts"; import { createWorkspace } from "./workspace.ts"; import type { Run } from "./types.ts"; export type MachineInfo = { name: string; intent: string; model: string; parent?: string; criteria: Array<{ id: string; description: string; weight: number; grader: string; inherited: boolean }>; tools: string[]; }; export type RunSummary = { id: string; machine: string; model: string; harness: string; input: string; output: string; total?: number; criteria?: Record; costUsd?: number; startedAt: string; finishedAt: string; session?: string; revision?: string; version?: string; stopReason?: string; feedback?: string[]; steps: number; toolCalls: number; reasoning: boolean; }; export type TrainingInfo = { name: string; model?: string; learningRate?: number; epochs?: number; loraRank?: number; steps: Array<{ step: number; epoch?: number; train?: number; test?: number; tokens?: number }>; checkpoints: Array<{ name: string; sampler?: string; tokens?: number }>; tokens: number; costUsd?: number; updatedAt?: string; examples?: number; }; const clip = (text: string, n: number) => { const flat = (text ?? "").replace(/\s+/g, " ").trim(); return flat.length > n ? `${flat.slice(0, n - 1)}…` : flat; }; /** Every machine defined under machines/, parents before children. Machines that fail to load are skipped. */ export async function listMachines(workspaceDir: string): Promise { const root = path.join(workspaceDir, "machines"); const names: string[] = []; async function walk(dir: string, prefix: string[]) { let entries; try { entries = await readdir(dir, { withFileTypes: true }); } catch { return; } for (const e of entries.filter((e) => e.isDirectory()).sort((a, b) => a.name.localeCompare(b.name))) { const here = [...prefix, e.name]; try { await stat(path.join(dir, e.name, "machine.ts")); names.push(here.join("/")); } catch {} await walk(path.join(dir, e.name), here); } } await walk(root, []); const out: MachineInfo[] = []; for (const name of names) { let m: Machine; try { m = await loadMachineFrom(workspaceDir, name); } catch { continue; } out.push({ name: m.name, intent: m.intent, model: m.model, parent: m.parent?.name, criteria: Object.entries(m.rubric).map(([id, c]) => ({ id, description: c.description, weight: c.weight ?? 1, grader: m.evaluator[id]?.name ?? "", inherited: Boolean(m.parent && id in m.parent.rubric), })), tools: m.tools.map((t) => t.name), }); } return out; } /** Stored runs, summarized, newest first. Parsed files are cached by modification time. */ export function runReader(storeDir: string) { const cache = new Map(); async function files(): Promise { const out: string[] = []; async function walk(dir: string) { let entries; try { entries = await readdir(dir, { withFileTypes: true }); } catch { return; } for (const e of entries) { const p = path.join(dir, e.name); if (e.isDirectory()) await walk(p); else if (e.name.endsWith(".json")) out.push(p); } } await walk(path.join(storeDir, "runs")); return out; } return { async summaries(): Promise { const list = await files(); const out: RunSummary[] = []; await Promise.all( list.map(async (file) => { try { const mtime = (await stat(file)).mtimeMs; const hit = cache.get(file); if (hit && hit.mtime === mtime) return void out.push(hit.summary); const run = JSON.parse(await readFile(file, "utf8")) as Run; const summary = summarize(run); cache.set(file, { mtime, summary }); out.push(summary); } catch {} }), ); return out.sort((a, b) => (a.startedAt < b.startedAt ? 1 : a.startedAt > b.startedAt ? -1 : b.id.localeCompare(a.id))); }, async run(machine: string, id: string): Promise { if (!/^[a-z0-9][a-z0-9._/-]*$/.test(machine) || machine.includes("..") || !/^[a-z0-9][a-z0-9._-]*$/.test(id)) return undefined; try { return JSON.parse(await readFile(path.join(storeDir, "runs", ...machine.split("/"), `${id}.json`), "utf8")); } catch { return undefined; } }, }; } export function summarize(run: Run): RunSummary { const assistant = run.trajectory.filter((m) => m.role === "assistant"); return { id: run.id, machine: run.machine, model: run.model, harness: run.harness, input: clip(run.input, 160), output: clip(run.output, 200), total: run.scorecard?.total, criteria: run.scorecard ? Object.fromEntries(Object.entries(run.scorecard.criteria).map(([k, c]) => [k, c.score])) : undefined, costUsd: (run.usage?.costUsd ?? 0) + (run.scorecard?.usage?.costUsd ?? 0) || undefined, startedAt: run.startedAt, finishedAt: run.finishedAt, session: run.session, revision: run.revision, version: run.version, stopReason: run.stopReason, feedback: run.feedback?.map((f) => (f.note ? `${f.kind}: ${f.note}` : f.kind)), steps: assistant.length, toolCalls: assistant.filter((m) => m.toolCallId).length, reasoning: assistant.some((m) => m.reasoning), }; } /** Training runs under .machine/training// in tinker-cookbook's log format. */ export async function listTraining(storeDir: string): Promise { const dir = path.join(storeDir, "training"); let names: string[]; try { names = (await readdir(dir, { withFileTypes: true })).filter((e) => e.isDirectory()).map((e) => e.name).sort(); } catch { return []; } const jsonl = async (file: string) => { try { return (await readFile(file, "utf8")).split("\n").filter(Boolean).flatMap((l) => { try { return [JSON.parse(l)]; } catch { return []; } }); } catch { return []; } }; const out: TrainingInfo[] = []; for (const name of names) { const base = path.join(dir, name); let config: Record = {}; try { config = JSON.parse(await readFile(path.join(base, "config.json"), "utf8")); } catch {} const metrics = await jsonl(path.join(base, "metrics.jsonl")); const checkpoints = await jsonl(path.join(base, "checkpoints.jsonl")); let examples: number | undefined; try { examples = (await readFile(path.join(base, "data.jsonl"), "utf8")).split("\n").filter(Boolean).length; } catch {} let updatedAt: string | undefined; try { updatedAt = (await stat(path.join(base, "metrics.jsonl"))).mtime.toISOString(); } catch {} const tokens = metrics.reduce((s, r) => s + (r.num_tokens ?? 0), 0); const model = config.model_name; const price = model ? TINKER_TRAIN_PRICES[model] : undefined; out.push({ name, model, learningRate: config.learning_rate, epochs: config.num_epochs, loraRank: config.lora_rank, steps: metrics .filter((r) => typeof r.step === "number") .map((r) => ({ step: r.step, epoch: r.epoch, train: r.train_mean_nll, test: r["test/nll"], tokens: r.num_tokens })), checkpoints: checkpoints.map((c) => ({ name: c.name, sampler: c.sampler_path, tokens: c.elapsed_tokens })), tokens, costUsd: price ? (tokens * price) / 1_000_000 : undefined, updatedAt, examples, }); } return out; } /** * The harness a judge model runs on: Tinker's endpoint for Tinker models, * Pi for anything else (OpenRouter's free Inkling, OpenAI, and so on). */ export function judgeHarness(model: string): Harness { if (model in TINKER_PRICES || model.startsWith("tinker://")) { const apiKey = process.env.TINKER_API_KEY; if (!apiKey) throw new Error(`Judging with ${model} needs TINKER_API_KEY.`); return chatCompletionsHarness({ name: "tinker", baseURL: TINKER_OAI_BASE_URL, apiKey, maxTokens: 8192, price: (m) => tinkerPrice(m) }); } return withPricing(piHarness(), (m) => (isFreeModel(m) ? FREE : tinkerPrice(m.replace(/^tinker\//, "")))); } export type DashboardOptions = { workspaceDir: string; port?: number; host?: string; /** Judge for grading from the dashboard. Defaults to $MACHINE_JUDGE_MODEL, else Inkling-Small on Tinker. */ judgeModel?: string; /** Spending limit for one grading request (one run, or all of a machine's ungraded runs). Default $1. */ gradeBudgetUsd?: number; /** * Required on every request when set: as ?token= once (then kept in a cookie) or the cookie * itself. Defaults to a random token whenever the host is not loopback, since runs hold prompts * and the page can spend money on grading. */ token?: string; }; const isLoopback = (host: string) => host === "127.0.0.1" || host === "localhost" || host === "::1"; function sameSecret(a: string, b: string): boolean { const x = Buffer.from(a); const y = Buffer.from(b); return x.length === y.length && timingSafeEqual(x, y); } /** Start the dashboard. Resolves once it is listening. */ export async function serveDashboard(options: DashboardOptions) { const storeDir = path.join(options.workspaceDir, ".machine"); const eventsFile = path.join(storeDir, "events.jsonl"); const runs = runReader(storeDir); const page = await readFile(new URL("./dashboard.html", import.meta.url), "utf8"); const clients = new Set(); let offset = 0; let recent: MachineEvent[] = []; // Catch up on the log once, keeping the last few hundred events for new viewers. ({ events: recent, offset } = await readEventsFrom(eventsFile, 0)); recent = recent.slice(-300); let trainingStamp = ""; const tick = async () => { const { events, offset: next } = await readEventsFrom(eventsFile, offset); offset = next; if (events.length) { recent = [...recent, ...events].slice(-300); for (const e of events) broadcast("event", e); } const stamp = (await listTraining(storeDir)).map((t) => `${t.name}:${t.steps.length}:${t.checkpoints.length}`).join("|"); if (stamp !== trainingStamp) { if (trainingStamp) broadcast("training", {}); trainingStamp = stamp; } }; const timer = setInterval(() => void tick().catch(() => {}), 1000); function broadcast(kind: string, data: unknown) { const line = `event: ${kind}\ndata: ${JSON.stringify(data)}\n\n`; for (const res of clients) res.write(line); } const host = options.host ?? "127.0.0.1"; const token = options.token ?? (isLoopback(host) ? undefined : randomBytes(18).toString("base64url")); const judgeModel = options.judgeModel ?? process.env.MACHINE_JUDGE_MODEL ?? "thinkingmachines/Inkling-Small"; const judgeKey = judgeModel.startsWith("openrouter/") ? "OPENROUTER_API_KEY" : "TINKER_API_KEY"; if (!process.env[judgeKey]) console.error(`machine dashboard: ${judgeKey} is not set, so grading with ${judgeModel} will fail.`); const gradeBudget = options.gradeBudgetUsd ?? 1; /** Grade stored runs with each machine's own rubric, under one budget per request. */ async function grade(machineName: string, ids: string[]) { const machine = await loadMachineFrom(options.workspaceDir, machineName); const harness = judgeHarness(judgeModel); const budget = new Budget(gradeBudget); const workspace = createWorkspace({ name: "dashboard", root: storeDir, harness, judgeModel, judgeEffort: judgeModel in TINKER_PRICES ? "medium" : undefined, budget, events: fileEvents(eventsFile), }); const graded: Array<{ id: string; total?: number; error?: string }> = []; try { for (let i = 0; i < ids.length; i += 4) { await Promise.all( ids.slice(i, i + 4).map(async (id) => { const run = await runs.run(machineName, id); if (!run) return void graded.push({ id, error: "not found" }); try { const done = await workspace.evaluate(machine, run); graded.push({ id, total: done.scorecard?.total }); } catch (error) { graded.push({ id, error: error instanceof Error ? error.message : String(error) }); } }), ); if (budget.spentUsd >= budget.limitUsd) break; } } finally { await harness.close?.(); } return { judge: judgeModel, graded, spentUsd: budget.spentUsd, limitUsd: budget.limitUsd }; } const json = (res: ServerResponse, value: unknown, status = 200) => { res.writeHead(status, { "content-type": "application/json; charset=utf-8", "cache-control": "no-store" }); res.end(JSON.stringify(value)); }; const server = createServer(async (req, res) => { try { const url = new URL(req.url ?? "/", "http://localhost"); if (token) { const given = url.searchParams.get("token"); const cookie = /(?:^|;\s*)machine_token=([^;]+)/.exec(req.headers.cookie ?? "")?.[1]; if (given && sameSecret(given, token)) { // Keep it in a cookie and drop it from the address bar. res.writeHead(303, { location: url.pathname === "/" ? "/" : url.pathname, "set-cookie": `machine_token=${token}; HttpOnly; SameSite=Strict; Path=/; Max-Age=2592000` }); return void res.end(); } if (!cookie || !sameSecret(cookie, token)) { res.writeHead(401, { "content-type": "text/plain; charset=utf-8" }); return void res.end("This dashboard needs its link with ?token=..., printed when it started."); } } if (req.method === "POST" && url.pathname === "/api/grade") { // Grading spends money. Only this page may ask: a custom header forces a CORS preflight, // which this server never approves, so other sites in the browser cannot trigger it. if (req.headers["x-machine-dashboard"] !== "1") return json(res, { error: "forbidden" }, 403); const machine = url.searchParams.get("machine") ?? ""; const id = url.searchParams.get("id"); let ids: string[]; if (id) ids = [id]; else { const all = await runs.summaries(); ids = all.filter((r) => r.machine === machine && r.total === undefined && r.harness !== "echo").map((r) => r.id); } if (!ids.length) return json(res, { judge: judgeModel, graded: [], spentUsd: 0, limitUsd: gradeBudget }); return json(res, await grade(machine, ids)); } if (req.method !== "GET") return void res.writeHead(405).end(); switch (url.pathname) { case "/": res.writeHead(200, { "content-type": "text/html; charset=utf-8", "cache-control": "no-store" }); return void res.end(page); case "/api/machines": return json(res, await listMachines(options.workspaceDir)); case "/api/runs": { const machine = url.searchParams.get("machine"); const limit = Math.min(Number(url.searchParams.get("limit") ?? 200) || 200, 5000); const all = await runs.summaries(); return json(res, (machine ? all.filter((r) => r.machine === machine) : all).slice(0, limit)); } case "/api/config": return json(res, { judgeModel, gradeBudgetUsd: gradeBudget }); case "/api/stats": { const all = await runs.summaries(); return json(res, stats(all)); } case "/api/run": { const run = await runs.run(url.searchParams.get("machine") ?? "", url.searchParams.get("id") ?? ""); return run ? json(res, run) : json(res, { error: "not found" }, 404); } case "/api/training": return json(res, await listTraining(storeDir)); case "/api/events": { res.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-store", connection: "keep-alive" }); res.write(`event: backlog\ndata: ${JSON.stringify(recent)}\n\n`); clients.add(res); const ping = setInterval(() => res.write(": ping\n\n"), 15_000); req.on("close", () => { clearInterval(ping); clients.delete(res); }); return; } default: res.writeHead(404).end(); } } catch (error) { json(res, { error: error instanceof Error ? error.message : String(error) }, 500); } }); await new Promise((resolve, reject) => { server.once("error", reject); server.listen(options.port ?? 4242, host, resolve); }); const address = server.address(); const port = typeof address === "object" && address ? address.port : options.port; return { url: `http://${host}:${port}`, port: port as number, token, close: () => new Promise((resolve) => { clearInterval(timer); for (const res of clients) res.end(); server.close(() => resolve()); }), }; } /** Per-machine totals over stored runs: counts, recent quality, spend, and models used. */ export function stats(all: RunSummary[]) { const byMachine: Record }> = {}; for (const r of all) { const m = (byMachine[r.machine] ??= { runs: 0, graded: 0, costUsd: 0, models: {} }); m.runs++; m.costUsd += r.costUsd ?? 0; m.models[r.model] = (m.models[r.model] ?? 0) + 1; if (!m.lastAt || r.startedAt > m.lastAt) m.lastAt = r.startedAt; } // Quality over each machine's 50 most recent graded runs (summaries arrive newest first). // Echo-harness dry runs test plumbing, not the machine, so they are left out. for (const [name, m] of Object.entries(byMachine)) { const real = all.filter((r) => r.machine === name && r.total !== undefined && r.harness !== "echo"); const graded = real.slice(0, 50); m.graded = real.length; if (graded.length) { m.mean = graded.reduce((s, r) => s + r.total!, 0) / graded.length; m.passRate = graded.filter((r) => r.total === 1).length / graded.length; } } const totalCost = all.reduce((s, r) => s + (r.costUsd ?? 0), 0); const day = new Date(Date.now() - 86_400_000).toISOString(); return { runs: all.length, runsToday: all.filter((r) => r.startedAt >= day).length, graded: all.filter((r) => r.total !== undefined && r.harness !== "echo").length, costUsd: totalCost, byMachine, }; }