import assert from "node:assert/strict"; import { execFileSync } from "node:child_process"; import { mkdtemp, rm } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { after, test } from "node:test"; import coder, { ranWrittenCode, runTool, writeFileTool } from "../machines/coder/machine.ts"; import { localSandboxes, type Message, type Run } from "../src/index.ts"; const dirs: string[] = []; after(() => Promise.all(dirs.map((d) => rm(d, { recursive: true, force: true })))); async function sandbox(isolation: "auto" | "none") { const root = await mkdtemp(path.join(os.homedir(), ".machine-coder-test-")); dirs.push(root); return localSandboxes(root, { isolation }).create("r"); } let hasBwrap = false; try { execFileSync("bwrap", ["--ro-bind", "/", "/", "--unshare-all", "--", "true"], { stdio: "ignore" }); hasBwrap = true; } catch {} (hasBwrap ? test : test.skip)("write_file and run produce a real hello world inside the isolated sandbox", async () => { const sb = await sandbox("auto"); const ctx = { sandbox: sb }; assert.match(String(await writeFileTool.execute({ path: "hello.py", content: "print(\"Hello, world!\")\n" }, ctx)), /Wrote hello.py/); const result = await runTool.execute({ command: "python3", args: ["hello.py"] }, ctx); assert.deepEqual(result, { exitCode: 0, timedOut: false, stdout: "Hello, world!\n", stderr: "" }); const failed = (await runTool.execute({ command: "python3", args: ["-c", "raise SystemExit(4)"] }, ctx)) as { exitCode: number }; assert.equal(failed.exitCode, 4); await sb.dispose(); }); test("run refuses to execute code when the sandbox is not isolated", async () => { const sb = await sandbox("none"); await assert.rejects(runTool.execute({ command: "python3", args: ["-c", "print(1)"] }, { sandbox: sb }), /not isolated/); await sb.dispose(); }); function run(steps: Array<[string, unknown, string]>): Run { const trajectory: Message[] = [{ role: "user", content: "hello world" }]; steps.forEach(([name, input, result], i) => { trajectory.push({ role: "assistant", content: "", name, toolCallId: `c${i}`, toolInput: input }); trajectory.push({ role: "tool", content: result, name, toolCallId: `c${i}` }); }); return { id: "r", machine: "coder", workspace: "w", input: "hello world", history: [], trajectory, output: "", model: "m", harness: "h", startedAt: "", finishedAt: "" }; } const ok = JSON.stringify({ exitCode: 0, stdout: "Hello, world!\n" }); const bad = JSON.stringify({ exitCode: 1, stderr: "SyntaxError" }); test("ran_code needs a written file that was run and exited 0", () => { assert.equal(ranWrittenCode(run([["write_file", { path: "hello.py" }, "Wrote hello.py (23 bytes)."], ["run", { command: "python3", args: ["hello.py"] }, ok]])).score, 1); assert.equal(ranWrittenCode(run([["write_file", { path: "./hello.py" }, "Wrote ./hello.py (23 bytes)."], ["run", { command: "python3", args: ["hello.py"] }, ok]])).score, 1, "paths are normalized"); assert.equal(ranWrittenCode(run([["write_file", { path: "hello.py" }, "Wrote hello.py"], ["run", { command: "python3", args: ["hello.py"] }, bad], ["write_file", { path: "hello.py" }, "Wrote hello.py"], ["run", { command: "python3", args: ["hello.py"] }, ok]])).score, 1, "a fix-and-rerun counts"); assert.match(ranWrittenCode(run([])).reason, /no file was written/); assert.match(ranWrittenCode(run([["write_file", { path: "hello.py" }, "Wrote hello.py"]])).reason, /never ran/); assert.match(ranWrittenCode(run([["write_file", { path: "hello.py" }, "Wrote hello.py"], ["run", { command: "echo", args: ["Hello, world!"] }, ok]])).reason, /none of them used a written file/); assert.match(ranWrittenCode(run([["write_file", { path: "hello.py" }, "Wrote hello.py"], ["run", { command: "python3", args: ["hello.py"] }, bad]])).reason, /never exited 0/); assert.equal(ranWrittenCode(run([["write_file", { path: "../x.py" }, "Error: leaves the working directory"], ["run", { command: "python3", args: ["../x.py"] }, ok]])).score, 0, "a failed write does not count"); }); test("coder declares its two tools", () => { assert.deepEqual(coder.tools.map((t) => t.name), ["write_file", "run"]); assert.deepEqual(Object.keys(coder.rubric), ["ran_code", "does_task"]); });