#!/usr/bin/env node import { execFileSync } from "node:child_process"; import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, statSync, writeFileSync } from "node:fs"; import os from "node:os"; import path from "node:path"; import { machineDataFile, machineProjectDataDir, readMachineSettings } from "../lib/storage.ts"; const repo = process.cwd(); const tmp = mkdtempSync(path.join(os.tmpdir(), "machine-integration-")); process.env.MACHINE_DATA_DIR = path.join(tmp, "machine-state"); mkdirSync(machineProjectDataDir(tmp), { recursive: true, mode: 0o755 }); writeFileSync( machineDataFile(tmp, "train"), `${JSON.stringify({ messages: [{ role: "user", content: "seed" }, { role: "assistant", content: "seed" }], tag: "good", kind: "final" })}\n`, { mode: 0o644 }, ); writeFileSync( machineDataFile(tmp, "rejected"), `${JSON.stringify({ messages: [{ role: "user", content: "seed" }, { role: "assistant", content: "seed" }], tag: "bad", kind: "final" })}\n`, { mode: 0o644 }, ); if (process.platform !== "win32") { chmodSync(machineProjectDataDir(tmp), 0o755); chmodSync(machineDataFile(tmp, "train"), 0o644); chmodSync(machineDataFile(tmp, "rejected"), 0o644); } function run(cmd, args, options = {}) { return execFileSync(cmd, args, { cwd: options.cwd ?? repo, encoding: "utf8", stdio: options.stdio ?? "pipe", env: { ...process.env, ...(options.env ?? {}) }, }); } function pi(args, options = {}) { const sessionFlags = args.includes("--session") ? [] : ["--no-session"]; const extensionRoot = options.extensionRoot ?? repo; return run("pi", ["--no-extensions", "-e", extensionRoot, ...sessionFlags, ...args], options); } function assert(condition, message) { if (!condition) throw new Error(message); } run("pi", ["--version"]); const taggedSession = path.join(tmp, "tagged-session.jsonl"); writeFileSync( taggedSession, [ JSON.stringify({ type: "session", version: 3, id: "machine-tag-test", timestamp: "2026-01-01T00:00:00.000Z", cwd: tmp }), JSON.stringify({ type: "message", id: "user0001", parentId: null, timestamp: "2026-01-01T00:00:01.000Z", message: { role: "user", content: [{ type: "text", text: "How do I reset my password?" }], timestamp: 1 } }), JSON.stringify({ type: "message", id: "asst0001", parentId: "user0001", timestamp: "2026-01-01T00:00:02.000Z", message: { role: "assistant", content: [{ type: "text", text: "Delete your account." }], timestamp: 2 } }), ].join("\n") + "\n", ); pi(["--session", taggedSession, "-p", "/bad too terse"], { cwd: tmp }); pi(["--session", taggedSession, "-p", "/good --as \"Go to Settings → Security → Reset password.\""], { cwd: tmp }); const goodRows = readFileSync(machineDataFile(tmp, "train"), "utf8").trim().split("\n").map((line) => JSON.parse(line)); const badRows = readFileSync(machineDataFile(tmp, "rejected"), "utf8").trim().split("\n").map((line) => JSON.parse(line)); assert(goodRows.some((row) => row.tag === "good" && String(row.messages[1]?.content).includes("Reset password")), "/good did not append a rewritten training example"); assert(badRows.some((row) => row.tag === "bad" && row.note === "too terse"), "/bad did not append a rejected example with a note"); assert(goodRows.some((row) => row.tag === "good" && typeof row.id === "string" && /^[0-9a-f]{12}$/.test(row.id)), "/good rows need a stable id"); assert(badRows.some((row) => row.tag === "bad" && typeof row.id === "string" && /^[0-9a-f]{12}$/.test(row.id)), "/bad rows need a stable id"); assert(!existsSync(path.join(tmp, "data")), "judged examples should not create a project-local data directory"); if (process.platform !== "win32") { assert((statSync(machineDataFile(tmp, "train")).mode & 0o777) === 0o600, "training data should be owner-only"); assert((statSync(machineDataFile(tmp, "rejected")).mode & 0o777) === 0o600, "rejected data should be owner-only"); } const tracedSession = path.join(tmp, "traced-session.jsonl"); writeFileSync( tracedSession, [ JSON.stringify({ type: "session", version: 3, id: "machine-trace-test", timestamp: "2026-01-01T00:00:00.000Z", cwd: tmp }), JSON.stringify({ type: "message", id: "user0002", parentId: null, timestamp: "2026-01-01T00:00:01.000Z", message: { role: "user", content: [{ type: "text", text: "Write a Turing model" }], timestamp: 1 } }), JSON.stringify({ type: "message", id: "asst0002", parentId: "user0002", timestamp: "2026-01-01T00:00:02.000Z", message: { role: "assistant", content: [{ type: "thinking", thinking: "I should check Julia." }, { type: "text", text: "Checking Julia." }, { type: "toolCall", id: "call_julia", name: "bash", arguments: { command: "which julia" } }], timestamp: 2 } }), JSON.stringify({ type: "message", id: "tool0002", parentId: "asst0002", timestamp: "2026-01-01T00:00:03.000Z", message: { role: "toolResult", toolCallId: "call_julia", toolName: "bash", content: [{ type: "text", text: "/usr/bin/julia" }], isError: false, timestamp: 3 } }), JSON.stringify({ type: "message", id: "asst0003", parentId: "tool0002", timestamp: "2026-01-01T00:00:04.000Z", message: { role: "assistant", content: [{ type: "text", text: "Julia is installed." }], timestamp: 4 } }), ].join("\n") + "\n", ); pi(["--session", tracedSession, "-p", "/good --trace"], { cwd: tmp }); const traceRows = readFileSync(machineDataFile(tmp, "traces"), "utf8").trim().split("\n").map((line) => JSON.parse(line)); const trace = traceRows.find((row) => row.kind === "trace"); assert(trace, "/good --trace did not write the private traces.jsonl"); const toolCall = trace.messages?.find((message) => message.role === "assistant" && Array.isArray(message.tool_calls))?.tool_calls?.[0]; assert(toolCall?.function?.name === "bash" && typeof toolCall.function.arguments === "string", "/good --trace did not emit OpenAI-style tool_calls"); assert(trace.messages?.some((message) => message.role === "tool" && message.content.includes("julia")), "/good --trace did not keep tool results"); assert(!trace.trajectory?.some((record) => record.role === "reasoning"), "/good --trace should omit reasoning by default"); assert(typeof trace.source === "string" && !trace.source.includes(path.sep), "trace provenance should not expose an absolute session path"); assert(typeof trace.id === "string" && /^[0-9a-f]{12}$/.test(trace.id), "trace rows need a stable id"); assert(existsSync(machineDataFile(tmp, "train")), "expected private train.jsonl"); pi(["-p", "/tagged"], { cwd: tmp }); pi(["-p", "/machine auto status"], { cwd: tmp }); assert(readMachineSettings(tmp).autoTrace === false, "automatic observation should default off"); pi(["--session", tracedSession, "-p", "/tagged"], { cwd: tmp }); assert(!existsSync(machineDataFile(tmp, "observed")), "disabled automatic observation should write nothing"); pi(["-p", "/machine auto on"], { cwd: tmp }); assert(readMachineSettings(tmp).autoTrace === true, "/machine auto on should persist per project"); pi(["-p", "/machine auto status"], { cwd: tmp }); pi(["--session", tracedSession, "-p", "/tagged"], { cwd: tmp }); const observedRows = readFileSync(machineDataFile(tmp, "observed"), "utf8").trim().split("\n").map((line) => JSON.parse(line)); assert(observedRows.length === 1, "enabled headless observation should append one row"); assert(observedRows[0]?.tag === "observed" && observedRows[0]?.kind === "auto-headless", "automatic headless capture should be observed, not good"); assert(observedRows[0]?.messages?.some((message) => message.role === "tool"), "automatic observation should retain tool results"); assert(typeof observedRows[0]?.id === "string" && /^[0-9a-f]{12}$/.test(observedRows[0].id), "observed rows need a stable id"); const cliObserved = run(process.execPath, [path.join(repo, "scripts", "machine-cli.mjs"), "observed"], { cwd: tmp }); assert(cliObserved.includes(observedRows[0].id), "machine observed should print row ids"); pi(["-p", "/machine correction"], { cwd: tmp }); assert(readFileSync(machineDataFile(tmp, "traces"), "utf8").trim().split("\n").length === traceRows.length, "automatic observation should not enter explicitly good traces"); const slashSession = path.join(tmp, "slash-session.jsonl"); writeFileSync( slashSession, [ JSON.stringify({ type: "session", version: 3, id: "machine-slash-test", timestamp: "2026-01-01T00:00:00.000Z", cwd: tmp }), JSON.stringify({ type: "message", id: "slash0001", parentId: null, timestamp: "2026-01-01T00:00:01.000Z", message: { role: "user", content: [{ type: "text", text: "/machine data" }], timestamp: 1 } }), JSON.stringify({ type: "message", id: "slash0002", parentId: "slash0001", timestamp: "2026-01-01T00:00:02.000Z", message: { role: "assistant", content: [{ type: "text", text: "Machine data." }], timestamp: 2 } }), ].join("\n") + "\n", ); pi(["--session", slashSession, "-p", "/tagged"], { cwd: tmp }); assert(readFileSync(machineDataFile(tmp, "observed"), "utf8").trim().split("\n").length === 1, "slash-only sessions should not become observations"); pi(["-p", "/machine auto off"], { cwd: tmp }); assert(readMachineSettings(tmp).autoTrace === false, "/machine auto off should persist per project"); pi(["-p", "/machine data"], { cwd: tmp }); pi(["-p", "/machine help"], { cwd: tmp }); const legacyData = path.join(tmp, "data", "train.jsonl"); mkdirSync(path.dirname(legacyData), { recursive: true }); writeFileSync(legacyData, "legacy project-local data\n"); pi(["-p", "/train validate --quick"], { cwd: tmp }); pi(["-p", "/train sft --force"], { cwd: tmp }); assert(readFileSync(legacyData, "utf8") === "legacy project-local data\n", "Machine should not migrate or select legacy data implicitly"); assert(existsSync(path.join(tmp, "train_sft.py")), "/train sft did not write train_sft.py"); assert(readFileSync(path.join(tmp, "train_sft.py"), "utf8").includes("machine_sft"), "generated SFT script missing machine recipe name"); assert(readFileSync(path.join(tmp, "tinker.yaml"), "utf8").includes(machineDataFile(tmp, "train")), "tinker.yaml did not point at the private train.jsonl"); assert(machineProjectDataDir(tmp).startsWith(process.env.MACHINE_DATA_DIR), "Machine data should use the configured private root"); pi(["-p", "/train eval init --force"], { cwd: tmp }); assert(existsSync(path.join(tmp, "eval.py")), "/train eval init did not write eval.py"); assert(existsSync(path.join(tmp, "data", "eval.jsonl")), "/train eval init did not write data/eval.jsonl"); const packDir = path.join(tmp, "pack"); const unpackDir = path.join(tmp, "unpacked"); const packedWork = path.join(tmp, "packed-work"); mkdirSync(packDir, { recursive: true }); mkdirSync(unpackDir, { recursive: true }); mkdirSync(packedWork, { recursive: true }); const archiveName = run("npm", ["pack", "--silent", "--pack-destination", packDir], { cwd: repo }).trim().split(/\n/).slice(-1)[0]; assert(archiveName, "npm pack did not return an archive name"); run("tar", ["-xzf", path.join(packDir, archiveName), "-C", unpackDir]); const packedRoot = path.join(unpackDir, "package"); assert(existsSync(path.join(packedRoot, "lib", "storage.ts")), "packed package is missing lib/storage.ts"); pi(["-p", "/machine data"], { cwd: packedWork, extensionRoot: packedRoot }); console.log(`local integration test passed (${tmp})`);