Something went wrong. Try again.
Parametric learning in your code harness.
Something went wrong. Try again.
TypeScript
at letta-code
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745import { execFile as execFileCb } from "node:child_process";import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";import { mkdir, writeFile } from "node:fs/promises";import os from "node:os";import path from "node:path";import { fileURLToPath } from "node:url";import { promisify } from "node:util";import { countJsonlRows, flagOn, parseOptions, readJsonl, rel, shellSplit } from "../lib/cli.ts";import { appendTaggedExample, formatTaggedExampleList } from "../lib/examples.ts";import { forkObservedRun, formatMachineDataStatus } from "../lib/fork.ts";import { lastTaggedTurnFromLetta, lastUserTurnLettaMessages, lettaMessageType, recordsFromLettaMessages, type LettaHistoryMessage,} from "../lib/letta-trajectory.ts";import { pythonBin } from "../lib/python.ts";import { ensureMachineDataDir, machineDataFile, readMachineSettings, writeMachineSettings,} from "../lib/storage.ts";import { applyAssistantRewrite, extractMessageText, previewText, summarizeTrajectory, trajectoryToMessages, type TaggedExampleTag,} from "../lib/trajectory.ts";
const execFile = promisify(execFileCb);const PACKAGE_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");const DEFAULT_SFT_MODEL = "thinkingmachines/Inkling-Small";const DEFAULT_FT_MODEL = "thinkingmachines/Inkling-Small:peft:262144";const LANE_STATE_FILE = path.join(os.homedir(), ".letta", "mods", "machine-lane.json");
type ReportLevel = "info" | "success" | "warning" | "error";
function resolveTaggedExamplePath(cwd: string, tag: TaggedExampleTag, out?: string, trace = false): string { if (out) return path.resolve(cwd, out); ensureMachineDataDir(cwd); if (tag === "observed") return machineDataFile(cwd, "observed"); if (tag === "bad") return machineDataFile(cwd, "rejected"); return machineDataFile(cwd, trace ? "traces" : "train");}
function currentModelHandle(model?: { provider?: string | null; id?: string | null } | null): string | undefined { if (!model?.id) return undefined; return model.provider ? `${model.provider}/${model.id}` : model.id;}
function isTinkerHandle(handle?: string): boolean { return Boolean(handle && (handle.startsWith("tinker/") || handle.startsWith("thinkingmachines/")));}
function readLaneState(): { workModel?: string } { if (!existsSync(LANE_STATE_FILE)) return {}; try { return JSON.parse(readFileSync(LANE_STATE_FILE, "utf8")) as { workModel?: string }; } catch { return {}; }}
function writeLaneState(state: { workModel?: string }): void { mkdirSync(path.dirname(LANE_STATE_FILE), { recursive: true }); writeFileSync(LANE_STATE_FILE, `${JSON.stringify(state, null, 2)}\n`);}
function validateDataset(filePath: string): string { if (!existsSync(filePath)) return `File not found: ${filePath}`; const lines = readFileSync(filePath, "utf8").split(/\n/).filter((line) => line.trim()); let ok = 0; let assistantMessages = 0; let userMessages = 0; const issues: string[] = []; const samples: string[] = []; lines.forEach((line, idx) => { try { const obj = JSON.parse(line) as { messages?: unknown }; if (!Array.isArray(obj.messages)) { issues.push(`line ${idx + 1}: missing messages[]`); return; } if (obj.messages.length < 2) issues.push(`line ${idx + 1}: fewer than 2 messages`); for (const [j, msg] of obj.messages.entries()) { if (!msg || typeof msg !== "object") { issues.push(`line ${idx + 1} message ${j}: not an object`); continue; } const message = msg as { role?: string; content?: unknown }; if (!["system", "user", "assistant", "tool"].includes(String(message.role))) { issues.push(`line ${idx + 1} message ${j}: unexpected role ${JSON.stringify(message.role)}`); } if (message.role === "assistant") assistantMessages++; if (message.role === "user") userMessages++; if (!(typeof message.content === "string" || Array.isArray(message.content))) { issues.push(`line ${idx + 1} message ${j}: content should be string or content-part array`); } } ok++; if (samples.length < 3) { const preview = obj.messages .map((message) => { const item = message as { role?: string; content?: unknown }; return `${item.role}: ${typeof item.content === "string" ? item.content : JSON.stringify(item.content)}`; }) .join("\n") .slice(0, 1200); samples.push(`### Example ${idx + 1}\n\n\`\`\`text\n${preview}\n\`\`\``); } } catch (error) { issues.push(`line ${idx + 1}: invalid JSON (${error instanceof Error ? error.message : String(error)})`); } }); const warnings: string[] = []; if (assistantMessages === 0) warnings.push("No assistant messages found; SFT will have nothing obvious to train on."); if (userMessages === 0) warnings.push("No user messages found; expected chat JSONL may be malformed."); if (lines.length < 20) warnings.push("Very small dataset; use this only for smoke tests."); return [ `# Dataset validation: ${filePath}`, `- JSONL rows: ${lines.length}`, `- Valid rows: ${ok}`, `- User messages: ${userMessages}`, `- Assistant messages: ${assistantMessages}`, issues.length ? `\n## Issues\n${issues.slice(0, 50).map((item) => `- ${item}`).join("\n")}` : "\n## Issues\nNone found by the lightweight validator.", warnings.length ? `\n## Warnings\n${warnings.map((item) => `- ${item}`).join("\n")}` : "", `\n## Samples\n${samples.join("\n\n")}`, ].filter(Boolean).join("\n");}
function renderTemplate(name: string, replacements: Record<string, string>): string { let text = readFileSync(path.join(PACKAGE_ROOT, "templates", name), "utf8"); for (const [key, value] of Object.entries(replacements)) text = text.replaceAll(key, value); const leftover = text.match(/__[A-Z_]+__/g); if (leftover?.length) { throw new Error(`unsubstituted template placeholders in ${name}: ${[...new Set(leftover)].join(", ")}`); } return text;}
async function writeFiles(cwd: string, files: Array<{ rel: string; content: string }>, force: boolean): Promise<string[]> { const written: string[] = []; for (const file of files) { const target = path.join(cwd, file.rel); if (existsSync(target) && !force) throw new Error(`${target} already exists. Re-run with --force to overwrite.`); await mkdir(path.dirname(target), { recursive: true }); await writeFile(target, file.content); written.push(file.rel); } return written;}
function compactMetrics(row: Record<string, unknown>, limit = 28): string { const interesting = Object.entries(row) .filter(([key, value]) => typeof value === "number" && (key.includes("loss") || key.includes("score") || key.includes("reward") || key.includes("kl") || key.includes("nll") || key.startsWith("progress/") || key.startsWith("time/total"))) .slice(0, limit); if (interesting.length === 0) return JSON.stringify(row, null, 2).slice(0, 2000); return interesting .map(([key, value]) => `- ${key}: ${typeof value === "number" ? Number(value.toFixed(6)) : value}`) .join("\n");}
function latestMetrics(metricsPath: string): string | undefined { if (!existsSync(metricsPath)) return undefined; try { const last = readJsonl(metricsPath).slice(-1)[0]; return last ? compactMetrics(last) : undefined; } catch (error) { return `Could not parse ${metricsPath}: ${error instanceof Error ? error.message : String(error)}`; }}
type EvalSummary = { target?: string; renderer?: string; effort?: number | null; num_examples?: number; num_correct?: number; accuracy?: number; results?: Array<{ index: number; correct: boolean; expected: string; output: string }>;};
function formatEvalSummary(filePath: string): string { const result = JSON.parse(readFileSync(filePath, "utf8")) as EvalSummary; const accuracy = typeof result.accuracy === "number" ? `${(result.accuracy * 100).toFixed(1)}%` : "n/a"; const examples = result.num_examples ?? result.results?.length ?? 0; const correct = result.num_correct ?? result.results?.filter((row) => row.correct).length ?? 0; return [ `- File: \`${filePath}\``, `- Target: \`${result.target ?? "unknown"}\``, `- Renderer: \`${result.renderer ?? "unknown"}\``, result.effort !== null && result.effort !== undefined ? `- Inkling effort: ${result.effort}` : "", `- Accuracy: ${accuracy} (${correct}/${examples})`, ].filter(Boolean).join("\n");}
function compareEvalSummaries(baselinePath: string, candidatePath: string): string { const baseline = JSON.parse(readFileSync(baselinePath, "utf8")) as EvalSummary; const candidate = JSON.parse(readFileSync(candidatePath, "utf8")) as EvalSummary; const bAcc = baseline.accuracy ?? 0; const cAcc = candidate.accuracy ?? 0; return [ `# Eval comparison`, `- Baseline: \`${baselinePath}\` — ${(bAcc * 100).toFixed(1)}% (${baseline.num_correct}/${baseline.num_examples})`, `- Candidate: \`${candidatePath}\` — ${(cAcc * 100).toFixed(1)}% (${candidate.num_correct}/${candidate.num_examples})`, `- Delta: ${cAcc - bAcc >= 0 ? "+" : ""}${((cAcc - bAcc) * 100).toFixed(1)} points`, ].join("\n");}
export default function activate(letta) { const disposers: Array<() => void> = [];
const report = (title: string, body: string, _level: ReportLevel = "info") => ({ type: "output" as const, output: `# ${title}\n\n${body}`, });
async function loadHistory(ctx): Promise<LettaHistoryMessage[]> { const history = await ctx.conversation.getHistory({ limit: 500, order: "asc", includeErrors: true, }); return Array.isArray(history) ? history as LettaHistoryMessage[] : []; }
async function saveTaggedTurn(tag: TaggedExampleTag, rawArgs: string, ctx) { const { positional, options } = parseOptions(shellSplit(rawArgs)); const traceFlag = flagOn(options, "trace"); const trajectoryFlag = flagOn(options, "trajectory"); const reasoningFlag = flagOn(options, "reasoning"); const note = String(options.note ?? [traceFlag.leftover, trajectoryFlag.leftover, reasoningFlag.leftover, ...positional].filter(Boolean).join(" ")).trim(); const replacement = String(options.as ?? options.rewrite ?? "").trim(); const trace = traceFlag.on || trajectoryFlag.on; const includeReasoning = reasoningFlag.on; const messages = await loadHistory(ctx); const turn = lastTaggedTurnFromLetta(messages); if (!turn) { return report(`Save ${tag} example`, "No user/assistant turn found to tag. Chat first, then run `/good` or `/bad`.", "warning"); } const records = applyAssistantRewrite( recordsFromLettaMessages(lastUserTurnLettaMessages(messages)).filter((record) => includeReasoning || record.role !== "reasoning"), replacement, ); const assistant = replacement || turn.assistant; const chatMessages = trace ? trajectoryToMessages(records) : [ { role: "user", content: turn.user }, { role: "assistant", content: assistant }, ]; const outFile = resolveTaggedExamplePath(ctx.cwd, tag, typeof options.out === "string" ? options.out : undefined, trace); try { const saved = await appendTaggedExample({ tag, messages: chatMessages, records: trace ? records : undefined, note: note || undefined, outFile, source: ctx.conversation.id ?? undefined, userId: turn.userId, assistantId: turn.assistantId, kind: trace ? "trace" : "final", }); return report(`Saved ${tag} ${trace ? "trace" : "example"}`, [ `Appended the last turn to \`${rel(ctx.cwd, outFile)}\` (${saved.count} row${saved.count === 1 ? "" : "s"}).`, saved.row.id ? `id: \`${String(saved.row.id)}\`` : "", trace ? "Reconstructed the full user→tool-call→result→answer trajectory." : "Saved the final assistant answer only.", replacement ? "Used the rewritten assistant text you provided." : "", note ? `Note: ${note}` : "", "", trace ? `## Trajectory\n${summarizeTrajectory(records)}` : `## User\n${previewText(turn.user, 600)}\n\n## Assistant\n${previewText(assistant, 600)}`, "", tag === "good" ? (trace ? "`messages` is Tinker chat JSONL including tool calls. Train with `TrainOnWhat.ALL_ASSISTANT_MESSAGES` so results stay in context but are not the target. `trajectory` keeps the Letta-style records." : "This file is Tinker chat JSONL. Inspect it, then run `/train validate`.") : "Rejected rows stay out of SFT. Pair them later for DPO, or rewrite and `/good --as \"...\"`.", ].filter(Boolean).join("\n"), "success"); } catch (error) { return report(`Save ${tag} example failed`, error instanceof Error ? error.message : String(error), "error"); } }
function listTaggedExamples(rawArgs: string, ctx) { const { options } = parseOptions(shellSplit(rawArgs)); const files = [ typeof options.out === "string" ? path.resolve(ctx.cwd, options.out) : undefined, machineDataFile(ctx.cwd, "train"), machineDataFile(ctx.cwd, "traces"), machineDataFile(ctx.cwd, "observed"), machineDataFile(ctx.cwd, "rejected"), ].filter((file, idx, all): file is string => Boolean(file) && all.indexOf(file) === idx); return report("Tagged examples", formatTaggedExampleList(ctx.cwd, files)); }
async function switchLane(lane: "work" | "student", rawArgs: string, ctx) { const current = currentModelHandle(ctx.model); const state = readLaneState(); if (lane === "work") { const target = rawArgs.trim() || state.workModel; if (!target) { return report("Work model", "No daily-driver model remembered yet. Stay on Grok (or whatever you use), then `/machine student`. The next `/machine work` will jump back.", "warning"); } if (current && !isTinkerHandle(current)) writeLaneState({ workModel: current }); try { await ctx.conversation.updateLlmConfig({ model: target }); return report("Work model", `Now using \`${target}\` for execution.`, "success"); } catch (error) { return report("Work model", `Could not select \`${target}\`: ${error instanceof Error ? error.message : String(error)}`, "error"); } }
if (current && !isTinkerHandle(current)) writeLaneState({ workModel: current }); const target = rawArgs.trim() || `tinker/${DEFAULT_FT_MODEL}`; try { await ctx.conversation.updateLlmConfig({ model: target }); const remembered = readLaneState().workModel; return report("Student model", [ `Now using \`${target}\`.`, remembered ? `Daily driver remembered as \`${remembered}\`. \`/machine work\` jumps back.` : "`/machine work` will remember the next non-Tinker model you use.", "Tinker chat is a local provider (`tinker`). It needs `TINKER_API_KEY` and `/reload` after installing `~/.letta/mods/tinker.ts`.", ].join("\n"), "success"); } catch (error) { return report("Student model", [ `Could not select \`${target}\`: ${error instanceof Error ? error.message : String(error)}`, "Install the local Tinker provider (`mods/tinker-provider.ts`) and set `TINKER_API_KEY`, or pass an explicit handle:", "```text", "/machine student tinker/Qwen/Qwen3.8-27B:peft:262144", "/machine student tinker/thinkingmachines/Inkling-Small:peft:262144", "```", ].join("\n"), "error"); } }
function setAutoTrace(args: string, ctx) { const token = args.trim().toLowerCase(); let autoTrace = readMachineSettings(ctx.cwd).autoTrace; if (token === "on" || token === "true" || token === "1") autoTrace = true; else if (token === "off" || token === "false" || token === "0") autoTrace = false; else if (token === "status") { return report("Auto trace", [ autoTrace ? "On for this project." : "Off for this project (default).", `Observed traces: \`${machineDataFile(ctx.cwd, "observed")}\``, ].join("\n")); } else if (!token) { autoTrace = !autoTrace; } else { return report("Auto trace", "Usage: `/machine auto`, `/machine auto on`, `/machine auto off`, `/machine auto status`.", "warning"); } writeMachineSettings(ctx.cwd, { autoTrace }); return report("Auto trace", autoTrace ? [ `On. Machine will save completed spans as unlabeled observations in \`${machineDataFile(ctx.cwd, "observed")}\`.`, "Observed trajectories include tool output. Machine does not select them for training by default; inspect them before using or copying them.", ].join("\n") : "Off. Machine will not capture compaction or headless spans automatically.", "success"); }
function showDataStatus(ctx) { return report("Machine data", formatMachineDataStatus(ctx.cwd)); }
async function saveAutoTrace(options: { cwd: string; messages: LettaHistoryMessage[]; source?: string; kind: "auto-compact" | "auto-headless"; trigger: string; note: string; }) { const records = recordsFromLettaMessages(options.messages).filter((record) => record.role !== "reasoning"); if (records.length === 0) return undefined; const outFile = resolveTaggedExamplePath(options.cwd, "observed", undefined, true); return appendTaggedExample({ tag: "observed", messages: trajectoryToMessages(records), records, note: options.note, outFile, source: options.source, userId: options.messages.find((message) => lettaMessageType(message) === "user_message")?.id, assistantId: [...options.messages].reverse().find((message) => lettaMessageType(message) === "assistant_message")?.id, kind: options.kind, trigger: options.trigger, }); }
if (letta.capabilities?.commands) { disposers.push(letta.commands.register({ id: "good", description: "Save the last turn as a good training example", args: "[note] [--trace] [--as text] [--reasoning] [--out path]", runWhenBusy: true, async run(ctx) { return saveTaggedTurn("good", ctx.args ?? "", ctx); }, })); disposers.push(letta.commands.register({ id: "bad", description: "Save the last turn as a rejected / bad example", args: "[note] [--trace] [--as text] [--reasoning] [--out path]", runWhenBusy: true, async run(ctx) { return saveTaggedTurn("bad", ctx.args ?? "", ctx); }, })); disposers.push(letta.commands.register({ id: "tagged", description: "List recent judged examples and automatic observations", args: "[--out path]", runWhenBusy: true, run(ctx) { return listTaggedExamples(ctx.args ?? "", ctx); }, })); disposers.push(letta.commands.register({ id: "machine", description: "Parametric learning: auto-trace, fork observed runs, or see /good /bad /tagged /train", args: "[auto|data|fork|correction|work|student|help]", runWhenBusy: true, async run(ctx) { const [subcommand, ...rest] = shellSplit(ctx.args ?? ""); if (!subcommand || subcommand === "help" || subcommand === "--help") { return report("machine", [ "```text", "/good [--trace] save last turn", "/bad save rejected turn", "/tagged list saved examples", "/machine auto configure compact + headless observation", "/machine data show the project data store", "/machine fork [note] retrigger the latest observed headless run", "/machine correction --id <id> prompt", "/machine work switch back to the daily driver", "/machine student switch to Inkling / Tinker", "/train validate|sft|smoke|eval", "```", ].join("\n")); } if (subcommand === "auto") return setAutoTrace(rest.join(" "), ctx); if (subcommand === "data") return showDataStatus(ctx); if (subcommand === "fork") { const { positional, options } = parseOptions(rest); const result = await forkObservedRun({ cwd: ctx.cwd, correction: positional.join(" "), spawn: true, inheritStdio: false, id: typeof options.id === "string" ? options.id : undefined, mode: "fork", }); return report(result.title, result.body, result.level); } if (subcommand === "correction" || subcommand === "correct") { const { positional, options } = parseOptions(rest); const id = typeof options.id === "string" ? options.id : ""; const correction = positional.join(" ").trim(); if (!id || !correction) { return report("machine correction", "Usage: `/machine correction --id <id> prompt`", "warning"); } const result = await forkObservedRun({ cwd: ctx.cwd, correction, spawn: true, inheritStdio: false, id, mode: "correction", }); return report(result.title, result.body, result.level); } if (subcommand === "work" || subcommand === "student" || subcommand === "ft" || subcommand === "model") { return switchLane(subcommand === "work" ? "work" : "student", rest.join(" "), ctx); } return report("machine", `Unknown subcommand: ${subcommand}. Run \`/machine help\`.`, "warning"); }, })); disposers.push(letta.commands.register({ id: "train", description: "Validate data, scaffold SFT, smoke-test, and evaluate Tinker runs", args: "[validate|sft|smoke|eval] [args]", runWhenBusy: true, async run(ctx) { const [subcommandRaw, ...rest] = shellSplit(ctx.args ?? ""); const subcommand = subcommandRaw ?? "help"; if (subcommand === "help" || subcommand === "--help" || subcommand === "-h") { return report("machine train", [ "```text", "/train validate [jsonl]", "/train sft [jsonl]", "/train smoke --yes", "/train eval init", "/train eval baseline --yes", "/train eval checkpoint tinker://... --yes", "/train eval compare eval_results/baseline.json eval_results/checkpoint.json", "```", "", `Defaults to \`${machineDataFile(ctx.cwd, "train")}\` and \`${DEFAULT_SFT_MODEL}\`.`, ].join("\n")); }
if (subcommand === "validate") { const { positional, options } = parseOptions(rest); const dataFile = positional[0] ? path.resolve(ctx.cwd, positional[0]) : machineDataFile(ctx.cwd, "train"); const lightweight = validateDataset(dataFile); const script = path.join(PACKAGE_ROOT, "scripts", "validate-dataset.py"); if (options.quick === true || !existsSync(script)) return report("Dataset validation", lightweight); try { const { stdout } = await execFile(pythonBin(ctx.cwd, PACKAGE_ROOT), [ script, dataFile, String(options.model ?? DEFAULT_SFT_MODEL), String(options.examples ?? 200), String(options["max-length"] ?? 32768), ], { timeout: 240_000, maxBuffer: 12 * 1024 * 1024 }); return report("Dataset validation", `${stdout.trim()}\n\n---\n\n${lightweight}`); } catch (error) { return report("Dataset validation", `${lightweight}\n\n## Python-backed validation failed\n${error instanceof Error ? error.message : String(error)}`, "warning"); } }
if (subcommand === "sft") { const { positional, options } = parseOptions(rest); const dataFileArg = positional[0]; const dataFile = dataFileArg ? path.resolve(ctx.cwd, dataFileArg) : machineDataFile(ctx.cwd, "train"); if (!existsSync(dataFile)) return report("SFT scaffold", `Data file not found: ${dataFile}`, "error"); const model = String(options.model ?? DEFAULT_SFT_MODEL); const maxSteps = String(options.steps ?? options.max_steps ?? "20"); const batchSize = String(options["batch-size"] ?? "8"); const learningRate = String(options.lr ?? options["learning-rate"] ?? "2e-4"); const numEpochs = String(options.epochs ?? "1"); const testSize = String(options["test-size"] ?? "0"); const maxLength = String(options["max-length"] ?? "32768"); const logPath = String(options.log ?? `logs/sft-${new Date().toISOString().replace(/[:.]/g, "-").slice(0, 19)}`); try { const written = await writeFiles(ctx.cwd, [ { rel: "train_sft.py", content: renderTemplate("train_sft.py", { "__MODEL__": JSON.stringify(model), "__MAX_LENGTH__": maxLength, "__BATCH_SIZE__": batchSize, "__DATA_FILE__": JSON.stringify(dataFile), "__TEST_SIZE__": testSize, "__LOG_PATH__": JSON.stringify(logPath), "__LEARNING_RATE__": learningRate, "__NUM_EPOCHS__": numEpochs, "__MAX_STEPS__": maxSteps, }), }, { rel: "tinker.yaml", content: `task: sft\ndata: ${dataFile}\nmodel: ${model}\nlog_path: ${logPath}\nnum_epochs: ${numEpochs}\nmax_steps: ${maxSteps}\nbatch_size: ${batchSize}\nlearning_rate: ${learningRate}\n`, }, ], options.force === true); return report("SFT scaffold created", [ `Wrote ${written.map((item) => `\`${item}\``).join(", ")} from \`${rel(ctx.cwd, dataFile)}\`.`, "", "Next:", "```text", dataFileArg ? `/train validate ${dataFileArg}` : "/train validate", "/train smoke --yes", "```", ].join("\n"), "success"); } catch (error) { return report("SFT scaffold", error instanceof Error ? error.message : String(error), "warning"); } }
if (subcommand === "smoke") { const { positional, options } = parseOptions(rest); const script = path.resolve(ctx.cwd, positional[0] ?? "train_sft.py"); if (!existsSync(script)) return report("Smoke test", `Script not found: ${script}. Run \`/train sft\` first.`, "error"); if (options.yes !== true) return report("Smoke test", "This may create a real Tinker training client. Re-run with `/train smoke --yes`.", "warning"); try { const { stdout, stderr } = await execFile(pythonBin(ctx.cwd, PACKAGE_ROOT), [script, "max_steps=2"], { cwd: ctx.cwd, timeout: Number(options.timeout ?? 1_800_000), maxBuffer: 10 * 1024 * 1024, }); const combined = `${stdout}\n${stderr}`.trim(); const logMatch = combined.match(/log_path='([^']+)'|log_path=([^,\s)]+)/); const logDir = logMatch ? path.resolve(ctx.cwd, (logMatch[1] ?? logMatch[2] ?? "").replace(/^['"]|['"]$/g, "")) : undefined; const metrics = logDir ? latestMetrics(path.join(logDir, "metrics.jsonl")) : undefined; return report("Smoke test passed", [ "The 2-step command completed.", metrics ? `\n## Latest metrics\n${metrics}` : "", `\n## Output tail\n\`\`\`text\n${combined.split(/\n/).slice(-80).join("\n")}\n\`\`\``, ].filter(Boolean).join("\n"), "success"); } catch (error: unknown) { const err = error as { message?: string; stdout?: string; stderr?: string }; return report("Smoke test failed", [ err?.message ?? String(error), err?.stdout ? `\n## stdout\n\`\`\`text\n${String(err.stdout).split(/\n/).slice(-80).join("\n")}\n\`\`\`` : "", err?.stderr ? `\n## stderr\n\`\`\`text\n${String(err.stderr).split(/\n/).slice(-80).join("\n")}\n\`\`\`` : "", ].filter(Boolean).join("\n"), "error"); } }
if (subcommand === "eval") { const [actionRaw, ...evalRest] = rest; const action = actionRaw ?? "help"; const { positional, options } = parseOptions(evalRest); if (action === "help" || action === "--help") { return report("machine eval", [ "- `/train eval init` — create `eval.py` and `data/eval.jsonl`.", "- `/train eval baseline --yes` — evaluate the base model.", "- `/train eval checkpoint tinker://... --yes` — evaluate a sampler checkpoint.", "- `/train eval compare a.json b.json` — compare results.", ].join("\n")); } if (action === "init") { try { const written = await writeFiles(ctx.cwd, [ { rel: "eval.py", content: readFileSync(path.join(PACKAGE_ROOT, "templates", "eval.py"), "utf8") }, { rel: path.join("data", "eval.jsonl"), content: [ JSON.stringify({ messages: [{ role: "user", content: "What is --as?" }], expected: "rewrite", match: "contains" }), JSON.stringify({ messages: [{ role: "user", content: "Where do tagged traces go?" }], expected: "traces.jsonl", match: "contains" }), ].join("\n") + "\n", }, ], options.force === true); return report("Eval initialized", `Wrote ${written.map((item) => `\`${item}\``).join(", ")}.`, "success"); } catch (error) { return report("Eval init", error instanceof Error ? error.message : String(error), "warning"); } } if (action === "baseline" || action === "checkpoint") { const script = path.resolve(ctx.cwd, String(options.script ?? "eval.py")); if (!existsSync(script)) return report("Eval", `Missing \`${script}\`. Run \`/train eval init\` first.`, "warning"); const model = String(options.model ?? options["base-model"] ?? DEFAULT_SFT_MODEL); const data = path.resolve(ctx.cwd, String(options.data ?? "data/eval.jsonl")); if (!existsSync(data)) return report("Eval", `Eval data not found: ${data}`, "error"); const checkpoint = action === "checkpoint" ? positional[0] : undefined; if (action === "checkpoint" && !checkpoint) { return report("Eval checkpoint", "Usage: `/train eval checkpoint tinker://.../sampler_weights/...`", "warning"); } if (options.yes !== true) return report("Eval", "This will sample from Tinker. Re-run with `--yes`.", "warning"); const out = path.resolve(ctx.cwd, String(options.out ?? (action === "baseline" ? "eval_results/baseline.json" : `eval_results/${String(checkpoint).split("/").slice(-1)[0] || "checkpoint"}.json`))); const pythonArgs = action === "baseline" ? [script, "--base-model", model, "--data", data, "--out", out] : [script, "--model-path", String(checkpoint), "--renderer-model", model, "--data", data, "--out", out]; if (options.effort) pythonArgs.push("--effort", String(options.effort)); if (options.limit) pythonArgs.push("--limit", String(options.limit)); try { const { stdout, stderr } = await execFile(pythonBin(ctx.cwd, PACKAGE_ROOT), pythonArgs, { cwd: ctx.cwd, timeout: Number(options.timeout ?? 1_800_000), maxBuffer: 12 * 1024 * 1024, }); return report("Eval completed", [ formatEvalSummary(out), `\n## Output tail\n\`\`\`text\n${`${stdout}\n${stderr}`.trim().split(/\n/).slice(-60).join("\n")}\n\`\`\``, ].join("\n"), "success"); } catch (error: unknown) { const err = error as { message?: string; stdout?: string; stderr?: string }; return report("Eval failed", [ err?.message ?? String(error), err?.stdout ? `\n## stdout\n\`\`\`text\n${String(err.stdout).split(/\n/).slice(-80).join("\n")}\n\`\`\`` : "", err?.stderr ? `\n## stderr\n\`\`\`text\n${String(err.stderr).split(/\n/).slice(-80).join("\n")}\n\`\`\`` : "", ].filter(Boolean).join("\n"), "error"); } } if (action === "compare") { const baseline = positional[0] ? path.resolve(ctx.cwd, positional[0]) : path.resolve(ctx.cwd, "eval_results/baseline.json"); const candidate = positional[1] ? path.resolve(ctx.cwd, positional[1]) : ""; if (!candidate) return report("Eval compare", "Usage: `/train eval compare eval_results/baseline.json eval_results/checkpoint.json`", "warning"); try { return report("Eval comparison", compareEvalSummaries(baseline, candidate)); } catch (error) { return report("Eval comparison", `Could not compare eval results: ${error instanceof Error ? error.message : String(error)}`, "error"); } } return report("Eval", `Unknown eval action: ${action}. Run \`/train eval help\`.`, "warning"); }
return report("Unknown train command", `Unknown subcommand: ${subcommand}\n\nRun \`/train help\`.`, "warning"); }, })); }
if (letta.capabilities?.events?.compact) { disposers.push(letta.events.on("compact_start", async (event, ctx) => { if (!readMachineSettings(ctx.cwd).autoTrace) return; try { const messages = await loadHistory(ctx); const saved = await saveAutoTrace({ cwd: ctx.cwd, messages, source: ctx.conversation.id ?? undefined, kind: "auto-compact", trigger: event.trigger, note: `auto compact (${event.trigger})`, }); if (letta.capabilities?.ui?.panels) { letta.ui.notify(saved ? `machine: compact observation ${saved.count} row${saved.count === 1 ? "" : "s"}` : "machine: compaction fired with no span to save"); } } catch (error) { letta.diagnostics?.report?.({ message: `machine auto compact failed: ${error instanceof Error ? error.message : String(error)}`, severity: "warning", }); } })); }
if (letta.capabilities?.events?.lifecycle) { disposers.push(letta.events.on("conversation_close", async (event, ctx) => { if (!readMachineSettings(ctx.cwd).autoTrace) return; if (event.reason !== "quit") return; try { const messages = lastUserTurnLettaMessages(await loadHistory(ctx)); const userText = extractMessageText(messages.find((message) => lettaMessageType(message) === "user_message")?.content); if (!userText || userText.startsWith("/")) return; const saved = await saveAutoTrace({ cwd: ctx.cwd, messages, source: ctx.conversation.id ?? undefined, kind: "auto-headless", trigger: event.reason, note: "auto headless (quit)", }); if (saved) { const idNote = saved.row.id ? ` id=${String(saved.row.id)}` : ""; process.stderr.write(`machine: wrote headless observation to ${machineDataFile(ctx.cwd, "observed")} (${saved.count} rows)${idNote}\n`); } } catch (error) { process.stderr.write(`machine: failed to write headless trace: ${error instanceof Error ? error.message : String(error)}\n`); } })); }
return () => { for (const dispose of disposers.reverse()) dispose(); };}