Something went wrong. Try again.
Parametric learning in your code harness.
Something went wrong. Try again.
JavaScript
12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064#!/usr/bin/env node// Unit tests for the pure helpers in lib/. These run without the pi binary,// so they catch regressions close to the source instead of only through the// end-to-end integration test.import { mkdtempSync, writeFileSync, readFileSync } from "node:fs";import { execFileSync } from "node:child_process";import os from "node:os";import path from "node:path";import { countJsonlRows, flagOn, parseOptions, readJsonl, rel, shellSplit } from "../lib/cli.ts";import { applyAssistantRewrite, extractMessageText, isoTimestamp, lastTaggedTurn, lastUserTurnEntries, recordsFromSessionEntries, injectApprovedReasoning, messagesContainApprovedThinking, trajectoryToMessages,} from "../lib/trajectory.ts";import { formatMachineStatus, joinMachineStatusLine, resolveLastTurnStatus } from "../lib/statusline.ts";import { countPoolTags, isTrainableRow } from "../lib/storage.ts";import { approveCorrectionContribution, buildTeacherContributionPrompt, cancelPendingCorrectionRetry, correctionStateLabel, createCorrectionRecord, formatCorrectionDetail, interpretCorrectionDiagnosis, parseCorrectionRecord, REASONING_SCRUB_MAX_TOKENS, rejectScopeFlag, resolveRetryTeacher, shouldRequestTeacherInterpretation, TEACHER_INTERPRETATION_MAX_TOKENS,} from "../lib/correction.ts";import { buildReasoningScrubPrompt, buildTeacherReasoningWrapper, conservativeArbitraryContribution, findLeakedSecretSpan, interpretDiagnosisAsContribution, parseScrubbedReasoning, parseTeacherContributionResponse, validateSelfContainedReasoning, validateTrainableHygiene,} from "../lib/reasoning-machine.ts";import { buildThinkingValidationManifest, validateThinkingDatasetForModel,} from "../lib/thinking-renderer.ts";import { validatePoolJsonl } from "../lib/validate.ts";import { pythonBin } from "../lib/python.ts";import { checkpointPathsFromLog, formatBytes, projectCheckpointPlan } from "../lib/tinker-gc.ts";import { builtinTinkerChatModels, DEFAULT_MACHINE_MODEL, isTinkerNativeId, listTinkerChatModels } from "../lib/tinker-catalog.ts";
let checks = 0;function assert(condition, message) { if (!condition) throw new Error(message); checks++;}function eq(actual, expected, message) { const a = JSON.stringify(actual); const b = JSON.stringify(expected); assert(a === b, `${message}: expected ${b}, got ${a}`);}
// --- shellSplit ---eq(shellSplit(`--as "Go to Settings → Security."`), ["--as", "Go to Settings → Security."], "double-quoted value should keep spaces");eq(shellSplit("--note 'too terse'"), ["--note", "too terse"], "single-quoted value should keep spaces");eq(shellSplit("a\\ b c"), ["a b", "c"], "backslash should escape spaces");eq(shellSplit("it's 'fine'"), ["its fine"], "apostrophes open quoting, matching POSIX shell behavior");eq(shellSplit(""), [], "empty input should split to nothing");eq(shellSplit(" "), [], "whitespace-only input should split to nothing");
// --- parseOptions ---eq(parseOptions(["file.jsonl"]), { positional: ["file.jsonl"], options: {} }, "plain tokens are positional");eq(parseOptions(["--force"]), { positional: [], options: { force: true } }, "lone flag becomes true");eq(parseOptions(["--note", "too", "terse"]), { positional: ["terse"], options: { note: "too" } }, "flag consumes the next non-option token");eq(parseOptions(["--steps=20", "--next-flag"]), { positional: [], options: { steps: "20", "next-flag": true } }, "equals form and flag-followed-by-flag");eq(parseOptions(["--trace", "why", "--reasoning"]), { positional: [], options: { trace: "why", reasoning: true } }, "a flag consumes the next non-option token, even as a stray value");
// --- flagOn ---eq(flagOn({ trace: true }, "trace"), { on: true }, "true value is on");eq(flagOn({ trace: "false" }, "trace"), { on: false }, "string false is off");eq(flagOn({ trace: "why" }, "trace"), { on: true, leftover: "why" }, "stray value is returned as leftover");eq(flagOn({}, "trace"), { on: false }, "missing value is off");
// --- rel ---eq(rel("/a/b", "/a/b/c/train.jsonl"), "c/train.jsonl", "paths under cwd are relative");eq(rel("/a/b", "/elsewhere/train.jsonl"), "/elsewhere/train.jsonl", "paths outside cwd are absolute");
// --- readJsonl / countJsonlRows ---const tmp = mkdtempSync(path.join(os.tmpdir(), "machine-unit-"));const jsonl = path.join(tmp, "rows.jsonl");writeFileSync(jsonl, [ JSON.stringify({ n: 1 }), "", JSON.stringify({ n: 2 }), "not json {", "null", "123", JSON.stringify({ n: 3 }), "",].join("\n") + "\n");eq(readJsonl(jsonl).map((r) => r.n), [1, 2, 3], "corrupt and non-object lines are skipped, not fatal");eq(readJsonl(path.join(tmp, "missing.jsonl")), [], "missing files read as empty");assert(countJsonlRows(jsonl) === 6, "row counts should not require parseable JSON");
// --- extractMessageText ---eq(extractMessageText("plain"), "plain", "string content passes through");eq(extractMessageText([{ type: "text", text: "a" }, { type: "thinking", thinking: "b" }, { type: "text", text: "c" }]), "a\nc", "only text blocks are joined");eq(extractMessageText([{ type: "toolCall", name: "bash" }]), "", "non-text blocks contribute nothing");eq(extractMessageText(undefined), "", "missing content is empty");eq(extractMessageText({ text: "nested" }), "nested", "object text fields unwrap");
// --- isoTimestamp ---eq(isoTimestamp("2026-01-01T00:00:01.000Z"), "2026-01-01T00:00:01.000Z", "ISO strings normalize");eq(isoTimestamp(2), "1970-01-01T00:00:00.002Z", "numeric epochs (milliseconds) normalize");eq(isoTimestamp("garbage", "fallback"), "fallback", "unparseable values fall back");assert(isoTimestamp(undefined).endsWith("Z"), "missing timestamps default to now");
// --- session fixtures (same shape as real session JSONL) ---const session = [ { type: "session", id: "s", timestamp: "2026-01-01T00:00:00.000Z" }, { type: "message", id: "u1", timestamp: "2026-01-01T00:00:01.000Z", message: { role: "user", content: [{ type: "text", text: "How do I reset my password?" }], timestamp: 1 } }, { type: "message", id: "a1", timestamp: "2026-01-01T00:00:02.000Z", message: { role: "assistant", content: [{ type: "thinking", thinking: "Check docs." }, { type: "text", text: "Checking." }, { type: "toolCall", id: "call_1", name: "bash", arguments: { command: "which julia" } }], timestamp: 2 } }, { type: "message", id: "t1", timestamp: "2026-01-01T00:00:03.000Z", message: { role: "toolResult", toolCallId: "call_1", content: [{ type: "text", text: "/usr/bin/julia" }], isError: false, timestamp: 3 } }, { type: "message", id: "a2", timestamp: "2026-01-01T00:00:04.000Z", message: { role: "assistant", content: [{ type: "text", text: "Julia is installed." }], timestamp: 4 } },];
// --- lastTaggedTurn / lastUserTurnEntries ---eq(lastTaggedTurn(session), { userId: "u1", assistantId: "a2", user: "How do I reset my password?", assistant: "Julia is installed." }, "last user/assistant pair is found");eq(lastTaggedTurn([{ type: "message", id: "u1", message: { role: "user", content: "hi" } }]), undefined, "no pair without an assistant");eq(lastUserTurnEntries(session).map((e) => e.id), ["u1", "a1", "t1", "a2"], "the turn span starts at the last user message");
// --- recordsFromSessionEntries ---const records = recordsFromSessionEntries(session);eq(records.map((r) => [r.role, r.content === null ? "tool_calls" : typeof r.content === "string" ? "text" : "?"]), [ ["user", "text"], ["reasoning", "text"], ["assistant", "text"], ["assistant", "tool_calls"], ["tool", "text"], ["assistant", "text"],], "records preserve the full trajectory shape");const toolCallRecord = records.find((r) => r.role === "assistant" && r.content === null);eq(toolCallRecord.tool_calls, [{ id: "call_1", name: "bash", args: JSON.stringify({ command: "which julia" }) }], "tool call arguments are serialized to a JSON string");const toolRecord = records.find((r) => r.role === "tool");eq(toolRecord, { role: "tool", tool_call_id: "call_1", content: "/usr/bin/julia", timestamp: "2026-01-01T00:00:03.000Z" }, "tool results keep their call id and text");const failing = recordsFromSessionEntries([ { type: "message", id: "t2", timestamp: "2026-01-01T00:00:05.000Z", message: { role: "toolResult", toolCallId: "call_1", content: [{ type: "text", text: "boom" }], isError: true, timestamp: 5 } },]);assert(failing[0].content.startsWith("Error:"), "error tool results are prefixed");eq(recordsFromSessionEntries([{ type: "message", id: "u9", message: { role: "user", content: [{ type: "text", text: "/machine data" }], timestamp: 9 } }]).length, 1, "slash commands are still user records");
// --- applyAssistantRewrite ---const rewritten = applyAssistantRewrite(records, "Rewritten answer.");assert(rewritten.at(-1).content === "Rewritten answer.", "the last assistant text is replaced");assert(rewritten.length === records.length && records.at(-1).content === "Julia is installed.", "the input records are not mutated");eq(applyAssistantRewrite(records, ""), records, "empty rewrites are no-ops");
// --- trajectoryToMessages ---const messages = trajectoryToMessages(records);const toolCallMessage = messages.find((m) => Array.isArray(m.tool_calls));eq(toolCallMessage, { role: "assistant", content: "", tool_calls: [{ id: "call_1", type: "function", function: { name: "bash", arguments: JSON.stringify({ command: "which julia" }) } }],}, "OpenAI-style tool_calls are emitted");eq(messages.find((m) => m.role === "tool"), { role: "tool", content: "/usr/bin/julia", tool_call_id: "call_1" }, "tool results become role:tool messages");assert(!messages.some((m) => m.role === "reasoning"), "observed reasoning is dropped from chat messages");eq(messages[0], { role: "user", content: "How do I reset my password?" }, "the opening user message is preserved");
const approvedReasoning = "Give the shortest answer that fully explains the relevant distinction.";const withThinking = trajectoryToMessages(records, { thinking: approvedReasoning });const firstAssistant = withThinking.find((m) => m.role === "assistant");assert(Array.isArray(firstAssistant.content), "approved reasoning uses structured content");eq(firstAssistant.content[0], { type: "thinking", thinking: approvedReasoning }, "thinking is injected on the first assistant target");assert(withThinking.some((m) => m.tool_calls?.[0]?.function?.name === "bash"), "later tool-call assistants remain in the trajectory");assert(!withThinking.some((m) => JSON.stringify(m).includes("Check docs.")), "observed Pi reasoning never enters saved messages");
const greetingRecords = [ { role: "user", content: "Good morning.", timestamp: "2026-01-01T00:00:01.000Z" }, { role: "reasoning", content: "Based on the retry instructions, keep it brief.", timestamp: "2026-01-01T00:00:02.000Z" }, { role: "assistant", content: "Good morning.", timestamp: "2026-01-01T00:00:02.000Z" },];const greetingThinking = "This is a casual greeting rather than a substantive request. Match its brevity and wait for the actual request.";const greetingMessages = trajectoryToMessages(greetingRecords, { thinking: greetingThinking });eq(greetingMessages, [ { role: "user", content: "Good morning." }, { role: "assistant", content: [ { type: "thinking", thinking: greetingThinking }, { type: "text", text: "Good morning." }, ], },], "greeting fixture saves approved thinking plus visible answer only");assert(!JSON.stringify(greetingMessages).includes("retry instructions"), "observed retry-instruction reasoning is omitted");
const injected = injectApprovedReasoning([ { role: "user", content: "Write a Turing model" }, { role: "assistant", content: "", tool_calls: [{ id: "call_1", type: "function", function: { name: "bash", arguments: "{}" } }] },], "Check whether Julia is installed before drafting code.");assert(Array.isArray(injected[1].content), "tool-call-first assistant keeps structured content");eq(injected[1].content[0].type, "thinking", "tool-call-first injects thinking before tool calls");assert(injected[1].tool_calls?.length === 1, "tool-call-first preserves tool_calls");assert(messagesContainApprovedThinking(injected), "messagesContainApprovedThinking detects thinking parts");
// --- SFT scaffold contract ---// The /train sft scaffold is rendered by string substitution:// extensions/train.ts passes a fixed replacement set to templates/train_sft.py.// These checks pin the template and host together so a new placeholder cannot// ship without the host substituting it (the original num_epochs bug).const repoRoot = path.resolve(import.meta.dirname, "..");const templatePath = path.join(repoRoot, "templates", "train_sft.py");const template = readFileSync(templatePath, "utf8");const templatePlaceholders = [...new Set([...template.matchAll(/__[A-Z_]+__/g)].map((m) => m[0]))].sort();
function renderKeysFor(hostFile) { const source = readFileSync(path.join(repoRoot, hostFile), "utf8"); const match = source.match(/renderTemplate\(\s*"train_sft\.py"\s*,\s*\{([\s\S]*?)\}\s*\)/); assert(match, `${hostFile} should call renderTemplate("train_sft.py", {...})`); return [...new Set([...match[1].matchAll(/__[A-Z_]+__/g)].map((m) => m[0]))].sort();}
eq(templatePlaceholders.length >= 8, true, "train_sft template should have its expected placeholders");eq(renderKeysFor("extensions/train.ts"), templatePlaceholders, "Pi /train sft substitutes exactly the template placeholders");assert(template.includes("good_conversation_path"), "SFT template must filter the pool to good rows");assert(template.includes("expand_assistant_targets"), "SFT template expands one prefix per assistant target");assert(template.includes("TrainOnWhat.LAST_ASSISTANT_MESSAGE"), "SFT template trains LAST assistant targets");assert(!template.includes("TrainOnWhat.ALL_ASSISTANT_MESSAGES"), "SFT template must not use ALL_ASSISTANT_MESSAGES");assert(readFileSync(path.join(repoRoot, "extensions", "train.ts"), "utf8").includes("num_epochs: ${numEpochs}"), "Pi tinker.yaml should record num_epochs");
// Smoke-render the template the way the host does and require a clean result.const replacements = Object.fromEntries(templatePlaceholders.map((key) => [key, { __PACKAGE_ROOT__: '"/tmp/machine"', __MODEL__: '"m"', __MAX_LENGTH__: "32768", __BATCH_SIZE__: "8", __DATA_FILE__: '"d.jsonl"', __TEST_SIZE__: "0", __LOG_PATH__: '"logs/x"', __LEARNING_RATE__: "2e-4", __NUM_EPOCHS__: "10", __MAX_STEPS__: "60",}[key]]));let smokeRendered = template;for (const [key, value] of Object.entries(replacements)) smokeRendered = smokeRendered.replaceAll(key, value);eq(/[A-Z]__[A-Z_]+__|__[A-Z_]+__/.test(smokeRendered), false, "rendered scaffold should contain no unsubstituted placeholders");assert(smokeRendered.includes('"num_epochs": 10,'), "rendered scaffold should carry the requested epoch count");
// --- statusline ---function statusText(input) { return joinMachineStatusLine(formatMachineStatus(input), (_tone, text) => text);}const noTurn = { state: "none" };eq(statusText({ autoTrace: false, student: false, lastTurn: noTurn, good: 0 }), undefined, "empty state hides the footer");eq(statusText({ autoTrace: false, student: false, lastTurn: noTurn, good: 3 }), "⚙ 3 good", "quiet good count when nothing else is pending");eq(statusText({ autoTrace: true, student: false, lastTurn: noTurn, good: 3 }), "⚙ auto · 3 good", "auto headline plus quiet count");eq(statusText({ autoTrace: true, student: false, lastTurn: { state: "untagged" }, good: 3 }), "⚙ last turn untagged · 3 good", "untagged last turn wins over auto");eq(statusText({ autoTrace: false, student: false, lastTurn: { state: "good" }, good: 4 }), "⚙ last: good · 4 good", "just-tagged good trace");eq(statusText({ autoTrace: false, student: false, lastTurn: { state: "rejected" }, good: 3 }), "⚙ last: correction · 3 good", "just-tagged correction");eq(statusText({ autoTrace: true, student: true, lastTurn: noTurn, good: 3 }), "⚙ student · 3 good", "student lane wins over auto");eq(statusText({ autoTrace: false, student: true, lastTurn: noTurn, good: 0 }), "⚙ student", "student lane alone is still worth showing");eq(statusText({ autoTrace: true, student: false, lastTurn: noTurn, good: 0 }), "⚙ auto", "auto alone is still worth showing");eq(statusText({ autoTrace: false, student: false, lastTurn: { state: "untagged" }, good: 0 }), "⚙ last turn untagged", "untagged with an empty pool");eq(resolveLastTurnStatus({ hasTurn: false }), { state: "none" }, "no turn means no last-turn headline");eq(resolveLastTurnStatus({ hasTurn: true }), { state: "untagged" }, "a live turn without a judgment is untagged");eq(resolveLastTurnStatus({ hasTurn: true, label: "good" }), { state: "none" }, "a historical good label is not a just-tagged headline");eq(resolveLastTurnStatus({ hasTurn: true, assistantId: "a2", lastSave: { assistantId: "a2", tag: "good" } }), { state: "good" }, "a matching save is a just-tagged good trace");eq(resolveLastTurnStatus({ hasTurn: true, assistantId: "a2", lastSave: { assistantId: "a2", tag: "bad" } }), { state: "rejected" }, "a matching save is a just-tagged rejection");eq(resolveLastTurnStatus({ hasTurn: true, assistantId: "a3", lastSave: { assistantId: "a2", tag: "good" } }), { state: "untagged" }, "a save for an older turn does not cover a new one");eq(resolveLastTurnStatus({ hasTurn: true, assistantId: "a3", label: "good", lastSave: { assistantId: "a2", tag: "good" } }), { state: "none" }, "a labeled historical turn stays quiet");eq(countPoolTags([{ tag: "good" }, { tag: "bad" }, { tag: "observed" }, {}, null]), { good: 3, bad: 1, observed: 1, other: 0, total: 5 }, "untagged rows count as good");assert(isTrainableRow({ tag: "good", messages: [{ role: "user" }] }), "good rows with messages are trainable");assert(!isTrainableRow({ tag: "observed", messages: [{ role: "user" }] }), "observed rows are not trainable");assert(!isTrainableRow({ tag: "good" }), "good rows without messages are not trainable");assert(!isTrainableRow(null), "null is not trainable");const mixedPool = [ JSON.stringify({ tag: "good", messages: [{ role: "user", content: "hi" }, { role: "assistant", content: "hello" }] }), JSON.stringify({ tag: "bad", messages: [{ role: "user", content: "x" }] }), JSON.stringify({ tag: "observed", messages: [{ role: "user", content: "y" }, { role: "assistant", content: "z" }] }), JSON.stringify({ tag: "good" }), JSON.stringify({ tag: "good", messages: [{ role: "unknown_role", content: "hi" }] }), "123", "null",].join("\n");const validated = validatePoolJsonl(mixedPool);eq(validated.skipped, 2, "bad and observed rows are skipped, not validated");assert(validated.issues.some((issue) => issue.includes("missing messages[]")), "good rows without messages are reported");assert(validated.issues.some((issue) => issue.includes("not a JSON object")), "non-object JSON rows are reported as issues");assert(validated.issues.some((issue) => issue.includes("unexpected role")), "unexpected role messages are reported as issues");eq(validated.ok, 1, "only structurally valid good rows count as valid");eq(validated.trainable, 2, "good rows with non-empty messages are counted as trainable");
// --- tinker catalog ---assert(isTinkerNativeId("Qwen/Qwen3.8-27B:peft:262144"), "Qwen ids are Tinker-native");assert(isTinkerNativeId("thinkingmachines/Inkling-Small"), "Inkling ids are Tinker-native");assert(isTinkerNativeId("tinker://abc/sampler_weights/final"), "checkpoint paths are Tinker-native");assert(!isTinkerNativeId("zai/glm-5.3"), "non-Tinker providers stay prefixed");const builtinIds = builtinTinkerChatModels().map((model) => model.id);assert(builtinIds.includes("Qwen/Qwen3.8-27B"), "catalog includes Qwen3.8-27B");assert(builtinIds.includes("Qwen/Qwen3.8-27B:peft:262144"), "catalog includes Qwen3.8-27B 256K");assert(builtinTinkerChatModels().find((model) => model.id === "Qwen/Qwen3.8-27B")?.input.includes("image"), "Qwen3.8-27B is VL");eq(formatBytes(314.6e9), "314.6 GB", "hosted checkpoint sizes render in GB");eq(formatBytes(875017796), "875.0 MB", "mid sizes render in MB");const ckptLog = mkdtempSync(path.join(os.tmpdir(), "machine-ckpt-"));writeFileSync(path.join(ckptLog, "checkpoints.jsonl"), [ JSON.stringify({ sampler_path: "tinker://abc/sampler_weights/final", state_path: "tinker://abc/weights/final" }), JSON.stringify({ sampler_path: "tinker://abc/sampler_weights/final" }),].join("\n") + "\n");eq(checkpointPathsFromLog(ckptLog), ["tinker://abc/sampler_weights/final", "tinker://abc/weights/final"], "log dir yields unique tinker paths");eq(projectCheckpointPlan(undefined, { sampler: "tinker://new/sampler_weights/final", state: "tinker://new/weights/final" }), { keep: "tinker://new/sampler_weights/final", drop: ["tinker://new/weights/final"] }, "first run keeps sampler, drops training weights");eq(projectCheckpointPlan({ sampler: "tinker://old/sampler_weights/final" }, { sampler: "tinker://new/sampler_weights/final", state: "tinker://new/weights/final" }), { keep: "tinker://new/sampler_weights/final", previous: "tinker://old/sampler_weights/final", drop: ["tinker://new/weights/final"] }, "next run keeps the previous sampler as rollback");eq(projectCheckpointPlan({ sampler: "tinker://b/sampler_weights/final", previous: "tinker://a/sampler_weights/final" }, { sampler: "tinker://c/sampler_weights/final", state: "tinker://c/weights/final" }), { keep: "tinker://c/sampler_weights/final", previous: "tinker://b/sampler_weights/final", drop: ["tinker://c/weights/final", "tinker://a/sampler_weights/final"] }, "third run drops the oldest sampler");assert(DEFAULT_MACHINE_MODEL === "thinkingmachines/Inkling-Small", "train and student share Inkling-Small");assert(builtinIds.includes("thinkingmachines/Inkling-Small:peft:262144"), "catalog includes Inkling-Small 256K");const withCheckpoint = listTinkerChatModels([{ id: "tinker://ckpt", name: "my-sft", contextWindow: 32768, maxTokens: 4096 }]);assert(withCheckpoint.some((model) => model.id === "tinker://ckpt"), "registered checkpoints join the chat catalog");
// --- reasoning contributor slice ---const terse = interpretDiagnosisAsContribution("too terse");assert(terse, "too terse should match deterministic fallback");assert(terse.problem.toLowerCase().includes("omitted") || terse.problem.toLowerCase().includes("explanation"), "too terse diagnoses missing explanation");assert(terse.reasoning.toLowerCase().includes("explain"), "too terse proposes explanatory reasoning");assert(!/\bbe\s+terse\b/.test(terse.reasoning.toLowerCase()), "too terse must not literalize as be terse");const greeting = interpretDiagnosisAsContribution("greeting too long");assert(greeting?.reasoning.toLowerCase().includes("casual greeting"), "greeting diagnosis proposes casual-greeting reasoning");const wrong = interpretDiagnosisAsContribution("incorrect answer");assert(wrong?.reasoning.toLowerCase().includes("verify"), "factual error proposes verification reasoning");assert(!wrong?.reasoning.toLowerCase().includes("prior answer"), "factual fallback avoids prior-answer wording");assert(interpretDiagnosisAsContribution("custom tone issue") === undefined, "arbitrary diagnosis has no deterministic fallback");const arbitrary = conservativeArbitraryContribution();assert(!arbitrary.reasoning.toLowerCase().includes("prior answer"), "conservative arbitrary fallback stays task-facing");assert(!arbitrary.reasoning.toLowerCase().includes("failed because"), "conservative arbitrary fallback does not interpolate diagnosis");// --- leak detection is provenance, not vocabulary ---// A pasted teacher wrapper is caught with no per-record context, because the// wrapper boilerplate is a constant Machine generates.assert( !validateSelfContainedReasoning(buildTeacherReasoningWrapper("Explain the distinction, then stop.")).ok, "a pasted teacher wrapper is caught without any secrets supplied",);assert(validateSelfContainedReasoning("Match the greeting brevity and wait for the substantive request.").ok, "task-facing reasoning passes");
// Diagnosis wording becomes a leak when the reasoning attributes it to the// correction rather than using it as standalone task content.const echoDiagnosis = "you were too terse and skipped the explanation";assert( !validateSelfContainedReasoning( "I was too terse and skipped the explanation, so explain more.", "reasoning", { diagnosis: echoDiagnosis }, ).ok, "reasoning that attributes diagnosis wording to the correction is rejected",);assert( validateSelfContainedReasoning( "Name both options, then state which one applies here.", "reasoning", { diagnosis: echoDiagnosis }, ).ok, "reasoning about the task passes even when a diagnosis exists",);
// A diagnosis can supply facts the corrected trajectory needs. Matching those// facts is clean until the text attributes them to feedback or an earlier run.const taskFactDiagnosis = "Use the account lock before writing the transaction record";assert( validateSelfContainedReasoning( "Use the account lock before writing the transaction record.", "reasoning", { diagnosis: taskFactDiagnosis }, ).ok, "standalone task facts from the diagnosis remain trainable",);assert( validateSelfContainedReasoning( "The user asked to use the account lock before writing the transaction record.", "reasoning", { diagnosis: taskFactDiagnosis }, ).ok, "ordinary task attribution is not mistaken for correction attribution",);assert( !validateSelfContainedReasoning( "You said to use the account lock before writing the transaction record.", "reasoning", { diagnosis: taskFactDiagnosis }, ).ok, "diagnosis wording coupled to feedback attribution is rejected",);for (const attributed of [ "Per your feedback, use the account lock before writing the transaction record.", "You told me to use the account lock before writing the transaction record.", "Following your correction, use the account lock before writing the transaction record.", "After your feedback, use the account lock before writing the transaction record.",]) { assert( !validateSelfContainedReasoning(attributed, "reasoning", { diagnosis: taskFactDiagnosis }).ok, `correction attribution is rejected: ${attributed}`, );}
// Regression: ordinary software vocabulary is not evidence of a leak. The// previous word list blocked every one of these and offered no override.for (const ordinary of [ "Add a retry with backoff around the flaky fetch.", "The CLI should prompt for confirmation before deleting the directory.", "Decode the instruction opcode before dispatching to the handler.", "Apply gamma correction after the tone map, not before.", "Attempt the fast parse first, then fall back to the tolerant parser.",]) { assert( validateSelfContainedReasoning(ordinary, "reasoning", { diagnosis: "too terse", userPrompt: "make the fetch resilient" }).ok, `ordinary coding reasoning must not be blocked: ${ordinary}`, );}
// Words the user introduced are theirs to reuse.assert( findLeakedSecretSpan("Add the retry the user asked for.", { diagnosis: "add a retry around the flaky fetch", userPrompt: "add a retry around the flaky fetch" }) === undefined, "wording the user's own prompt introduced is exempt",);// A one-word diagnosis cannot be told apart from ordinary vocabulary.assert( findLeakedSecretSpan("This approach is wrong for large inputs.", { diagnosis: "wrong" }) === undefined, "a single-word diagnosis is not treated as a signature",);eq(correctionStateLabel("linked"), "linked to good trace", "linked label");eq(rejectScopeFlag({}), undefined, "absent scope flag is fine");assert(rejectScopeFlag({ scope: "project" })?.includes("Unsupported flag `--scope=project`"), "scope flag rejected");assert(rejectScopeFlag({ scope: true })?.includes("Unsupported flag `--scope`"), "bare scope flag rejected");assert(TEACHER_INTERPRETATION_MAX_TOKENS >= 256 && TEACHER_INTERPRETATION_MAX_TOKENS <= 300, "teacher interpretation stays near 256-300 tokens");const teacherPrompt = buildTeacherContributionPrompt({ userPrompt: "How do I reset my password?", rejectedAssistant: "Delete your account.", diagnosis: "sounds dismissive and skips the real steps",});assert(teacherPrompt.systemPrompt.includes("JSON"), "teacher system prompt requires JSON");assert(teacherPrompt.userText.includes("How do I reset my password?"), "teacher prompt includes the original user prompt");assert(teacherPrompt.systemPrompt.includes("self-contained"), "teacher prompt requires self-contained reasoning");const validContribution = '{"problem":"The answer was dismissive.","behavior":"Be concrete","applicability":"Reset questions","reasoning":"Give the concrete reset steps without sarcasm.","verifier":"Steps are present."}';eq(parseTeacherContributionResponse(validContribution)?.reasoning, "Give the concrete reset steps without sarcasm.", "plain JSON teacher responses parse");eq(parseTeacherContributionResponse('```json\n{"problem":"Skipped steps.","behavior":"Reset","applicability":"Reset","reasoning":"List the reset path.","verifier":"Has steps."}\n```')?.reasoning, "List the reset path.", "fenced JSON teacher responses parse");eq(parseTeacherContributionResponse("not json"), undefined, "malformed teacher output is rejected");// The parser no longer discards a proposal for its wording; the scrub pass// repairs it. Silently dropping valid proposals made the fallback look better// than it was.eq( parseTeacherContributionResponse('{"behavior":"x","applicability":"y","reasoning":"Based on the retry instructions, be nice.","verifier":"z"}')?.reasoning, "Based on the retry instructions, be nice.", "teacher proposals are parsed even when wording needs a rewrite",);assert( !validateSelfContainedReasoning("Based on the retry instructions, be nice.").ok, "a paraphrased apparatus reference is still caught by the offline backstop",);assert( !validateSelfContainedReasoning("As you pointed out, provide the concrete steps.").ok, "feedback attribution without copied diagnosis wording is still rejected",);assert( validateSelfContainedReasoning( "Explain when the phrase 'as you pointed out' is appropriate.", "reasoning", { userPrompt: "Explain when the phrase 'as you pointed out' is appropriate." }, ).ok, "apparatus-like wording from the task remains available",);eq(shouldRequestTeacherInterpretation({ hasUI: true, reasoningOverride: false, stay: false, retry: false, decline: false }), true, "interactive requests teacher");eq(shouldRequestTeacherInterpretation({ hasUI: true, reasoningOverride: true, stay: false, retry: false, decline: false }), false, "reasoning override bypasses teacher");eq(shouldRequestTeacherInterpretation({ hasUI: false, reasoningOverride: false, stay: true, retry: false, decline: false }), false, "headless --stay uses fallback");eq(shouldRequestTeacherInterpretation({ hasUI: false, reasoningOverride: false, stay: false, retry: true, decline: false }), true, "headless --retry requests teacher");const mocked = await interpretCorrectionDiagnosis({ diagnosis: "sounds dismissive and skips the real steps", userPrompt: "How do I reset my password?", rejectedAssistant: "Delete your account.", requestTeacher: true, teacher: { provider: "fake", id: "teacher-1" }, complete: async ({ systemPrompt, userText, maxTokens, teacher }) => { assert(systemPrompt.includes("JSON"), "injected complete receives system prompt"); assert(userText.includes("sounds dismissive"), "injected complete receives arbitrary diagnosis"); assert(maxTokens === TEACHER_INTERPRETATION_MAX_TOKENS, "injected complete receives token ceiling"); assert(teacher.id === "teacher-1", "injected complete receives teacher ref"); return '{"problem":"The answer was dismissive and omitted steps.","behavior":"Reset steps","applicability":"Password reset","reasoning":"Provide the concrete Settings path for resetting a password.","verifier":"Steps are present."}'; },});eq(mocked.interpretation.source, "teacher", "successful mock teacher is recorded as teacher provenance");assert(mocked.proposal.reasoning.toLowerCase().includes("settings") || mocked.proposal.reasoning.toLowerCase().includes("password"), "mock teacher proposal is used");const badTeacher = await interpretCorrectionDiagnosis({ diagnosis: "too terse", userPrompt: "q", rejectedAssistant: "a", requestTeacher: true, teacher: { provider: "fake", id: "teacher-1" }, complete: async () => "definitely not json",});eq(badTeacher.interpretation.source, "deterministic", "invalid teacher JSON falls back");eq(badTeacher.interpretation.fallbackReason, "invalid-teacher-response", "fallback reason is recorded");
// --- reasoning scrub: repair copied wording instead of rejecting it ---assert(REASONING_SCRUB_MAX_TOKENS > 0 && REASONING_SCRUB_MAX_TOKENS <= TEACHER_INTERPRETATION_MAX_TOKENS, "scrub request stays bounded");const scrubPrompt = buildReasoningScrubPrompt({ reasoning: "You said I was too terse, so explain more.", userPrompt: "What is the difference between a mutex and a semaphore?", leakedSpan: "i was too terse",});assert(scrubPrompt.systemPrompt.includes("JSON"), "scrub prompt requires JSON");assert(scrubPrompt.userText.includes("mutex"), "scrub prompt carries the user request");eq(parseScrubbedReasoning('{"reasoning":"Name both primitives, then say which fits."}'), "Name both primitives, then say which fits.", "scrub JSON parses");eq(parseScrubbedReasoning("not json"), undefined, "malformed scrub output is rejected");
const leakyDiagnosis = "you were too terse and skipped the explanation";let scrubCalls = 0;const scrubbed = await interpretCorrectionDiagnosis({ diagnosis: leakyDiagnosis, userPrompt: "What is the difference between a mutex and a semaphore?", rejectedAssistant: "They differ.", requestTeacher: true, teacher: { provider: "fake", id: "teacher-1" }, complete: async ({ maxTokens }) => { scrubCalls++; if (scrubCalls === 1) { // First call is the interpretation; it parrots the diagnosis. return '{"problem":"p","behavior":"b","applicability":"a","reasoning":"You were too terse and skipped the explanation.","verifier":"v"}'; } eq(maxTokens, REASONING_SCRUB_MAX_TOKENS, "scrub call uses the scrub ceiling"); return '{"reasoning":"Define both primitives, then state which one fits this case."}'; },});eq(scrubCalls, 2, "a leaked proposal triggers exactly one repair call");eq(scrubbed.proposal.reasoning, "Define both primitives, then state which one fits this case.", "scrubbed reasoning replaces the copied wording");eq(scrubbed.interpretation.scrub?.ok, true, "successful scrub is recorded");assert(scrubbed.interpretation.scrub?.leakedSpan, "scrub evidence names the copied span");
// A failed rewrite keeps the proposal and its task facts visible for explicit// editing instead of silently replacing it with canned reasoning.let failingCalls = 0;const scrubFailed = await interpretCorrectionDiagnosis({ diagnosis: leakyDiagnosis, userPrompt: "What is the difference between a mutex and a semaphore?", rejectedAssistant: "They differ.", requestTeacher: true, teacher: { provider: "fake", id: "teacher-1" }, complete: async () => { failingCalls++; if (failingCalls === 1) { return '{"problem":"p","behavior":"b","applicability":"a","reasoning":"You were too terse and skipped the explanation.","verifier":"v"}'; } throw new Error("scrub request failed"); },});eq(scrubFailed.interpretation.scrub?.attempted, true, "a failed rewrite is recorded as attempted");eq(scrubFailed.interpretation.scrub?.ok, false, "a failed rewrite is not marked clean");assert( findLeakedSecretSpan(scrubFailed.proposal.reasoning, { diagnosis: leakyDiagnosis }) !== undefined, "a failed rewrite remains blocked pending explicit editing",);assert( scrubFailed.proposal.reasoning.toLowerCase().includes("too terse"), "a failed rewrite retains the original proposal rather than hiding fact loss",);
let factRewriteCalls = 0;const factRichScrubFailure = await interpretCorrectionDiagnosis({ diagnosis: taskFactDiagnosis, userPrompt: "How should the write run?", rejectedAssistant: "Write immediately.", requestTeacher: true, teacher: { provider: "fake", id: "teacher-1" }, complete: async () => { factRewriteCalls++; if (factRewriteCalls === 1) { return JSON.stringify({ problem: "p", behavior: "b", applicability: "a", reasoning: "Per your feedback, use the account lock before writing the transaction record.", verifier: "v", }); } throw new Error("scrub request failed"); },});assert( factRichScrubFailure.proposal.reasoning.includes("account lock before writing the transaction record"), "failed scrub retains diagnosis-supplied task facts for review",);assert( findLeakedSecretSpan(factRichScrubFailure.proposal.reasoning, { diagnosis: taskFactDiagnosis }) !== undefined, "retained fact-rich proposal cannot pass approval without editing",);
// Reasoning the author supplied is theirs: no rewrite, no diagnosis check.const authoredOverride = await interpretCorrectionDiagnosis({ diagnosis: leakyDiagnosis, userPrompt: "What is the difference between a mutex and a semaphore?", rejectedAssistant: "They differ.", reasoningOverride: "Explain both primitives before choosing one.", requestTeacher: true, teacher: { provider: "fake", id: "teacher-1" }, complete: async () => { throw new Error("an override must not call the teacher"); },});eq(authoredOverride.interpretation.source, "user", "author-supplied reasoning is recorded as the user's");eq(authoredOverride.proposal.reasoning, "Explain both primitives before choosing one.", "author-supplied reasoning is kept verbatim");const authoredTaskPhrase = await interpretCorrectionDiagnosis({ diagnosis: "custom issue", userPrompt: "Compare the attempts.", rejectedAssistant: "They match.", reasoningOverride: "Compare the previous attempt with the current output.", requestTeacher: false,});eq( authoredTaskPhrase.proposal.reasoning, "Compare the previous attempt with the current output.", "explicitly authored reasoning may use an apparatus-like task phrase",);let rejectedAuthoredAttribution = false;try { await interpretCorrectionDiagnosis({ diagnosis: taskFactDiagnosis, userPrompt: "How should the write run?", rejectedAssistant: "Write immediately.", reasoningOverride: "Per your feedback, use the account lock before writing the transaction record.", requestTeacher: false, });} catch { rejectedAuthoredAttribution = true;}assert(rejectedAuthoredAttribution, "explicit override still rejects correction attribution absent from the task");assert( (() => { try { approveCorrectionContribution( createCorrectionRecord({ diagnosis: "custom issue", proposal: arbitrary }), buildTeacherReasoningWrapper("Explain the distinction."), ); return false; } catch { return true; } })(), "explicit approval still rejects pasted hidden-wrapper boilerplate",);
// A clean proposal must not trigger a repair call at all.let cleanCalls = 0;const cleanProposal = await interpretCorrectionDiagnosis({ diagnosis: "too terse", userPrompt: "What is the difference between a mutex and a semaphore?", rejectedAssistant: "They differ.", requestTeacher: true, teacher: { provider: "fake", id: "teacher-1" }, complete: async () => { cleanCalls++; return '{"problem":"p","behavior":"b","applicability":"a","reasoning":"Define both primitives, then state which one fits this case.","verifier":"v"}'; },});eq(cleanCalls, 1, "a clean proposal costs one request, not two");eq(cleanProposal.interpretation.scrub, undefined, "clean proposals record no scrub evidence");assert(badTeacher.proposal.reasoning.toLowerCase().includes("explain"), "fallback still proposes explanation for too terse");const override = await interpretCorrectionDiagnosis({ diagnosis: "too terse", userPrompt: "q", rejectedAssistant: "a", reasoningOverride: "Always include one short reason before the answer.", requestTeacher: true, teacher: { provider: "fake", id: "teacher-1" }, complete: async () => { throw new Error("should not be called"); },});eq(override.interpretation.source, "user", "reasoning override bypasses teacher call");eq(override.proposal.reasoning, "Always include one short reason before the answer.", "override supplies the proposal reasoning");const record = createCorrectionRecord({ diagnosis: "too terse", proposal: terse, interpretation: mocked.interpretation, now: "2026-01-01T00:00:00.000Z",});eq(record.state, "proposed", "new corrections start proposed");eq(record.interpretation?.source, "teacher", "ledger keeps interpretation provenance");assert(record.approvedReasoning === undefined, "proposed corrections must not carry approved reasoning yet");assert(record.approvedContribution === undefined, "proposed corrections must not carry approved contribution yet");const legacy = parseCorrectionRecord({ id: "corr_legacy", diagnosis: "too terse", problem: "x", proposedBehavior: "y", behavior: "y", scope: "turn", state: "linked", behaviorEdits: [], createdAt: "2026-01-01T00:00:00.000Z", updatedAt: "2026-01-01T00:00:00.000Z",});assert(legacy?.state === "linked", "legacy linked rows still parse as linked");assert(legacy?.approvedContribution === undefined, "legacy rows without reviewed reasoning must not fabricate approved contribution");assert(legacy?.approvedReasoning === undefined, "legacy rows without reviewed reasoning must not fabricate approved reasoning");assert(legacy?.evidenceClass === undefined, "legacy rows without approval must not claim reason_first");const legacyHidden = parseCorrectionRecord({ id: "corr_legacy_hidden", diagnosis: "too terse", problem: "x", proposedBehavior: "Be terse", behavior: "Be terse", scope: "turn", state: "retained", hiddenInstruction: "Based on the retry instructions, be brief.", behaviorEdits: [], createdAt: "2026-01-01T00:00:00.000Z", updatedAt: "2026-01-01T00:00:00.000Z",});assert(legacyHidden?.state === "recorded", "legacy retained maps to recorded, not linked/pending");assert(legacyHidden?.approvedReasoning === undefined, "legacy hidden-instruction rows do not become approved reasoning");assert(legacyHidden?.approvedContribution === undefined, "legacy hidden-instruction rows cannot become pending ThinkingParts");// A legacy hidden instruction must still be caught if it reaches trainable text.assert( !validateTrainableHygiene({ messages: [ { role: "user", content: "How do I reset my password?" }, { role: "assistant", content: `${legacyHidden?.teacherWrapper ?? ""} Open Settings.` }, ], userPrompt: "How do I reset my password?", hiddenTeacherWrapper: legacyHidden?.teacherWrapper, }).ok, "legacy hidden wrappers are blocked from trainable text",);const legacyDetail = formatCorrectionDetail(legacyHidden);assert(legacyDetail.includes("unapproved / no target"), "detail must not claim reason_first without approval");const proposedDetail = formatCorrectionDetail(record);assert(proposedDetail.includes("unapproved / no target"), "proposed rows show unapproved evidence");const workMissing = resolveRetryTeacher({ workModel: { provider: "openai", id: "gpt-missing" }, currentModel: { provider: "anthropic", id: "claude-ok" }, findModel: (provider, id) => (provider === "anthropic" && id === "claude-ok" ? { id } : undefined),});assert(workMissing.ok && workMissing.teacher.model === "claude-ok", "missing work model falls back to proven current non-Tinker");const workOnlyMissing = resolveRetryTeacher({ workModel: { provider: "openai", id: "gpt-missing" }, currentModel: { provider: "tinker", id: "thinkingmachines/Inkling-Small" }, findModel: () => undefined,});assert(!workOnlyMissing.ok, "missing work model with only Tinker current refuses retry");const workFound = resolveRetryTeacher({ workModel: { provider: "openai", id: "gpt-ok" }, currentModel: { provider: "anthropic", id: "claude-ok" }, findModel: (provider, id) => (provider === "openai" && id === "gpt-ok" ? { id } : undefined),});assert(workFound.ok && workFound.teacher.model === "gpt-ok", "records the work model actually selected");const cancelled = cancelPendingCorrectionRetry({ ...record, state: "pending", approvedReasoning: "x" });eq(cancelled.state, "recorded", "cancelled retry leaves correction recorded, not pending");const approved = approveCorrectionContribution(record, "Explain the distinction, then stop.");assert(approved.approvedReasoning === "Explain the distinction, then stop.", "approval sets approved reasoning");const approvedDetail = formatCorrectionDetail({ ...approved, sourceTraceDigest: "abc", targetTraceDigest: "def", retryTeacher: { provider: "openai", model: "gpt-ok" },});assert(approvedDetail.includes("reason_first"), "approved rows show evidence class");assert(approvedDetail.includes("Retry teacher: openai/gpt-ok"), "detail shows retry teacher");assert(approvedDetail.includes("Source digest: abc"), "detail shows source digest");assert(approvedDetail.includes("Target digest: def"), "detail shows target digest");const wrapper = buildTeacherReasoningWrapper(approved.approvedReasoning);assert(wrapper.includes(approved.approvedReasoning), "teacher wrapper carries approved reasoning");assert(wrapper.toLowerCase().includes("do not mention corrections"), "teacher wrapper forbids visible leakage");const cleanMessages = [ { role: "user", content: "How do I reset my password?" }, { role: "assistant", content: [{ type: "thinking", thinking: approved.approvedReasoning }, { type: "text", text: "Go to Settings → Security → Reset password." }] },];// The reasoning embedded in a wrapper is trainable on purpose, so it must be// exempt whether or not the caller passes it explicitly.assert(validateTrainableHygiene({ messages: cleanMessages, userPrompt: "How do I reset my password?", hiddenTeacherWrapper: wrapper }).ok, "clean traces pass hygiene");assert( validateTrainableHygiene({ messages: cleanMessages, userPrompt: "How do I reset my password?", hiddenTeacherWrapper: wrapper, approvedReasoning: approved.approvedReasoning, diagnosis: "too terse", }).ok, "approved reasoning stays clean when diagnosis and reasoning are both supplied",);const correctedFactMessages = [ { role: "user", content: "How should the write run?" }, { role: "assistant", content: [ { type: "thinking", thinking: "Acquire synchronization before performing the write." }, { type: "text", text: `${taskFactDiagnosis}.` }, ], tool_calls: [{ id: "call_lock", type: "function", function: { name: "write_record", arguments: JSON.stringify({ plan: taskFactDiagnosis }) }, }], },];assert( validateTrainableHygiene({ messages: correctedFactMessages, userPrompt: "How should the write run?", diagnosis: taskFactDiagnosis, approvedReasoning: "Acquire synchronization before performing the write.", }).ok, "diagnosis-supplied task facts survive in thinking, visible text, and tool arguments",);const attributedFactMessages = [ { role: "user", content: "How should the write run?" }, { role: "assistant", content: [ { type: "thinking", thinking: `The correction says ${taskFactDiagnosis.toLowerCase()}.` }, { type: "text", text: `The correction says ${taskFactDiagnosis.toLowerCase()}.` }, ], tool_calls: [{ id: "call_leaky_lock", type: "function", function: { name: "write_record", arguments: JSON.stringify({ plan: `The correction says ${taskFactDiagnosis.toLowerCase()}.` }), }, }], },];const attributedFactHygiene = validateTrainableHygiene({ messages: attributedFactMessages, userPrompt: "How should the write run?", diagnosis: taskFactDiagnosis, approvedReasoning: taskFactDiagnosis,});assert(!attributedFactHygiene.ok, "approved task facts do not exempt correction attribution");assert(attributedFactHygiene.violations.length === 3, "thinking, visible text, and tool arguments all reject attribution");const leakyMessages = [ { role: "user", content: "Good morning." }, { role: "assistant", content: [{ type: "text", text: "Good morning. Based on the retry instructions." }] },];const leaky = validateTrainableHygiene({ messages: leakyMessages, userPrompt: "Good morning." });assert(!leaky.ok, "visible retry-instruction leakage fails hygiene");assert(leaky.violations[0].field.includes("assistant"), "hygiene reports the trainable field");
const thinkingJsonl = path.join(tmp, "thinking-pool.jsonl");writeFileSync(thinkingJsonl, `${JSON.stringify({ tag: "good", messages: [ { role: "user", content: "Good morning." }, { role: "assistant", content: [{ type: "thinking", thinking: greeting?.reasoning ?? "This is a casual greeting rather than a substantive request." }, { type: "text", text: "Good morning." }] }, ],})}\n`);const plainJsonl = path.join(tmp, "plain-pool.jsonl");writeFileSync(plainJsonl, `${JSON.stringify({ tag: "good", messages: [{ role: "user", content: "hi" }, { role: "assistant", content: "hello" }] })}\n`);
// Renderer execution checks need tinker-cookbook and live in `thinking-test`.// What stays here runs on a stdlib python3, so CI is hermetic: a plain dataset// short-circuits before any renderer import.const plainGate = validateThinkingDatasetForModel({ repoRoot, cwd: repoRoot, dataFile: plainJsonl, model: "Qwen/Qwen3.5-9B-Base", maxExamples: 0 });assert(plainGate.ok && !plainGate.hasThinkingDataset, "plain legacy dataset stays scaffoldable on non-thinking renderer");
const manifestResult = { ok: true, hasThinkingDataset: true, model: "thinkingmachines/Inkling-Small", renderer: "tml_v0", checkedThinkingRows: 1,};const manifestA = buildThinkingValidationManifest(thinkingJsonl, manifestResult);const manifestAgain = buildThinkingValidationManifest(thinkingJsonl, manifestResult);assert(manifestA.dataDigest.length === 64, "manifest includes dataDigest");eq(manifestA.digest, manifestAgain.digest, "identical content/model/result yields stable digest across timestamps");assert(manifestA.digest !== buildThinkingValidationManifest(thinkingJsonl, { ...manifestResult, model: "Qwen/Qwen3-8B", renderer: "qwen3",}).digest, "changed model changes digest");assert(manifestA.dataDigest !== buildThinkingValidationManifest(plainJsonl, { ok: true, hasThinkingDataset: false, model: "thinkingmachines/Inkling-Small", renderer: "tml_v0",}).dataDigest, "changed content changes dataDigest");// Path is a non-address field: same bytes under a copied path keep the digest.const thinkingCopy = path.join(tmp, "thinking-copy.jsonl");writeFileSync(thinkingCopy, readFileSync(thinkingJsonl));eq( buildThinkingValidationManifest(thinkingCopy, manifestResult).digest, manifestA.digest, "identical bytes under a different path keep the content-addressed digest",);
const importBypassProbe = path.join(tmp, "import_bypass_probe.py");writeFileSync(importBypassProbe, `import builtins, json, sysfrom pathlib import Pathsys.path.insert(0, ${JSON.stringify(path.join(repoRoot, "scripts"))})real_import = builtins.__import__def blocked_import(name, *args, **kwargs): if name == "tinker_cookbook" or name.startswith("tinker_cookbook."): raise ImportError("blocked for regression") return real_import(name, *args, **kwargs)builtins.__import__ = blocked_importfrom thinking_renderer_validation import validate_dataset_thinking_renderer, scan_dataset_thinkingscan = scan_dataset_thinking(${JSON.stringify(thinkingJsonl)})assert scan["hasThinkingDataset"], "scan detects thinking before imports"result = validate_dataset_thinking_renderer(${JSON.stringify(thinkingJsonl)}, "thinkingmachines/Inkling-Small", None)print(json.dumps(result))assert result.get("hasThinkingDataset") is Trueassert result.get("ok") is Falseassert result.get("blocked") is Trueassert result.get("stage") == "import"`);execFileSync(pythonBin(repoRoot, repoRoot), [importBypassProbe], { stdio: "pipe" });
// Executing the generated script needs the real cookbook; that check lives in// `thinking-test`. The equivalent fail-closed path with deps *absent* is proven// below with a stdlib-only stand-in.const missingDepTrain = path.join(tmp, "train_sft_missing_dep.py");writeFileSync(missingDepTrain, `import json, sysfrom pathlib import PathPACKAGE_ROOT = Path(${JSON.stringify(repoRoot)})sys.path.insert(0, str(PACKAGE_ROOT / "scripts"))import builtinsreal_import = builtins.__import__def blocked_import(name, *args, **kwargs): if name == "tinker_cookbook" or name.startswith("tinker_cookbook."): raise ImportError("blocked for missing-dep regression") return real_import(name, *args, **kwargs)builtins.__import__ = blocked_importfrom thinking_renderer_validation import validate_dataset_thinking_renderer
def assert_thinking_renderer_ready(model_name, data_file): result = validate_dataset_thinking_renderer(data_file, model_name, None) if result.get("hasThinkingDataset") and not result.get("ok"): print(json.dumps({"ok": False, "stage": "thinking-renderer", "blocked": True, "reason": result.get("reason"), "hasThinkingDataset": True})) raise SystemExit(1) print(json.dumps({"unexpected": result})) raise SystemExit(2)
assert_thinking_renderer_ready("thinkingmachines/Inkling-Small", ${JSON.stringify(thinkingJsonl)})print("CLIENT_INIT_REACHED")`);try { execFileSync(pythonBin(repoRoot, repoRoot), [missingDepTrain], { stdio: "pipe" }); assert(false, "missing renderer deps must block before client init");} catch (error) { const output = `${error.stdout ?? ""}${error.stderr ?? ""}`; assert(!output.includes("CLIENT_INIT_REACHED"), "missing deps never reach client init"); assert(/thinking-renderer|blocked/i.test(output), "missing-dep generated gate emits fail-closed receipt");}
const expansionProbe = path.join(tmp, "expansion_probe.py");writeFileSync(expansionProbe, `import json, syssys.path.insert(0, ${JSON.stringify(path.join(repoRoot, "scripts"))})from assistant_targets import expand_assistant_targets, MalformedConversationError, split_source_groups
answer_only = [ {"role": "user", "content": "hi"}, {"role": "assistant", "content": "hello"},]thinking_answer = [ {"role": "user", "content": "hi"}, {"role": "assistant", "content": [{"type": "thinking", "thinking": "Be brief."}, {"type": "text", "text": "hello"}]},]tool_trace = [ {"role": "user", "content": "check"}, {"role": "assistant", "content": [{"type": "thinking", "thinking": "Use bash."}], "tool_calls": [{"id": "1", "type": "function", "function": {"name": "bash", "arguments": "{}"}}]}, {"role": "tool", "content": "/bin/bash", "tool_call_id": "1"}, {"role": "assistant", "content": "done"},]a = expand_assistant_targets(answer_only)b = expand_assistant_targets(thinking_answer)c = expand_assistant_targets(tool_trace)assert len(a) == 1 and len(a[0]) == 2assert len(b) == 1 and any(p.get("type") == "thinking" for p in b[0][-1]["content"])assert len(c) == 2assert c[0][-1].get("tool_calls") and any(p.get("type") == "thinking" for p in c[0][-1]["content"])assert c[1][-1]["content"] == "done"assert any(m.get("role") == "tool" for m in c[1])assert c[1][1].get("tool_calls")try: expand_assistant_targets([{"role": "tool", "content": "orphan"}]) raise SystemExit("expected malformed orphan")except MalformedConversationError: passtry: expand_assistant_targets([ {"role": "user", "content": "x"}, {"role": "assistant", "tool_calls": [ {"id": "a", "type": "function", "function": {"name": "bash", "arguments": "{}"}}, {"id": "a", "type": "function", "function": {"name": "bash", "arguments": "{}"}}, ]}, ]) raise SystemExit("expected duplicate id")except MalformedConversationError as e: assert "duplicate" in str(e)try: expand_assistant_targets([ {"role": "user", "content": "x"}, {"role": "assistant", "tool_calls": [{"id": "a", "type": "function", "function": {"name": "bash", "arguments": "{}"}}]}, {"role": "tool", "content": "ok", "tool_call_id": "missing"}, ]) raise SystemExit("expected unknown id")except MalformedConversationError as e: assert "does not match" in str(e)try: expand_assistant_targets([ {"role": "user", "content": "x"}, {"role": "assistant", "tool_calls": [{"id": "a", "type": "function", "function": {"name": "bash", "arguments": "{}"}}]}, {"role": "tool", "content": "ok"}, ]) raise SystemExit("expected missing tool_call_id")except MalformedConversationError as e: assert "tool_call_id" in str(e)try: expand_assistant_targets([ {"role": "user", "content": "x"}, {"role": "assistant", "tool_calls": [ {"id": "a", "type": "function", "function": {"name": "bash", "arguments": "{}"}}, {"id": "b", "type": "function", "function": {"name": "bash", "arguments": "{}"}}, ]}, {"role": "tool", "content": "ok", "tool_call_id": "a"}, ]) raise SystemExit("expected leftover id")except MalformedConversationError as e: assert "unanswered" in str(e)
groups = [ {"source_id": 10, "targets": ["t10a", "t10b"]}, {"source_id": 20, "targets": ["t20a", "t20b"]}, {"source_id": 30, "targets": ["t30a"]},]train, test, train_ids, test_ids = split_source_groups(groups, test_size=1, shuffle_seed=0)assert set(train_ids).isdisjoint(set(test_ids)), "source ids must not leak across split"assert all(sid in train_ids for sid in set(train_ids)), "train ids consistent"for sid in set(train_ids): assert sid not in test_idsfor sid in set(test_ids): # every target from this source is on the test side assert train_ids.count(sid) == 0 assert test_ids.count(sid) == sum(1 for g in groups if g["source_id"] == sid for _ in g["targets"])print(json.dumps({ "ok": True, "answerOnly": len(a), "thinkingAnswer": len(b), "toolTargets": len(c), "trainSources": sorted(set(train_ids)), "testSources": sorted(set(test_ids)),}))`);const expansionOut = JSON.parse(execFileSync(pythonBin(repoRoot, repoRoot), [expansionProbe], { encoding: "utf8" }).trim());assert(expansionOut.ok && expansionOut.toolTargets === 2, "assistant-target expansion shapes are correct");assert(Array.isArray(expansionOut.trainSources) && Array.isArray(expansionOut.testSources), "source-group split returns both sides");assert(expansionOut.trainSources.every((id) => !expansionOut.testSources.includes(id)), "all targets from one source stay on one side");
console.log(`unit tests passed (${checks} checks)`);