import { decodeFirst } from "@atcute/cbor"; import { runDurableObjectAlarm, runInDurableObject } from "cloudflare:test"; import { env, exports } from "cloudflare:workers"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { Runner, parsePoll, pollCommand, readPoll, runScript } from "../src/runner"; import type { Box, Poll, RunnerParams } from "../src/runner"; import { logCap } from "../src/logs"; import { didDoc, getRecordRoute, knot, offline, knotService, record, repoDid as repo, routeFetch, } from "./helpers"; describe("runScript", () => { it("runs each step in order with its environment and stops at a failure", () => { const script = runScript({ environment: { A: "job", B: "it's" }, steps: [ { name: "test", command: "--help", environment: { A: "x y" } }, { name: "lint\n\u009bx", command: "false", environment: {} }, ], }); expect(script).toBe( [ "exec > /treadle/run.log 2>&1 < /dev/null", 'd() { echo "treadle: $*" >> /treadle/debug.log 2>/dev/null || true; }', "trap 'd script exit $?' EXIT", "echo $$ > /treadle/pid", "mkdir -p /treadle/log", "if [ -e /etc/cloudflare/certs/cloudflare-containers-ca.crt ]; then mkdir -p /usr/local/share/ca-certificates && cp /etc/cloudflare/certs/cloudflare-containers-ca.crt /usr/local/share/ca-certificates/ && if command -v update-ca-certificates > /dev/null; then update-ca-certificates > /dev/null; fi; fi", "cd /workspace || exit 1", "d step 1 'test' start; t=$SECONDS", "env A='x y' B='it'\\''s' bash -eo pipefail -c -- '--help' > /treadle/log/1 2>&1 < /dev/null", "rc=$?; echo $rc > /treadle/log/1.tmp && mv /treadle/log/1.tmp /treadle/log/1.exit", "d step 1 exit $rc after $((SECONDS - t))s", '[ "$rc" = 0 ] || exit 1', "d step 2 'lint x' start; t=$SECONDS", "env A='job' B='it'\\''s' bash -eo pipefail -c -- 'false' > /treadle/log/2 2>&1 < /dev/null", "rc=$?; echo $rc > /treadle/log/2.tmp && mv /treadle/log/2.tmp /treadle/log/2.exit", "d step 2 exit $rc after $((SECONDS - t))s", '[ "$rc" = 0 ] || exit 1', "", ].join("\n"), ); }); }); describe("pollCommand", () => { it("reads liveness before the exit file, and a bounded chunk of the log", () => { const [timeout, seconds, shell, flag, command = ""] = pollCommand(2, 100, 4096); expect([timeout, seconds, shell, flag]).toEqual(["timeout", "10", "bash", "-c"]); expect(command).toContain("f=/treadle/log/2"); expect(command).toContain("tail -c +101 $f 2>/dev/null | head -c 4096"); expect(command.indexOf("a=1")).toBeLessThan(command.indexOf("$f.exit")); expect(command.indexOf("$f.exit")).toBeLessThan(command.indexOf("stat -c %s $f")); }); }); describe("parsePoll", () => { const encoder = new TextEncoder(); it("reads a running step", () => { expect(parsePoll(encoder.encode("1 - 10 3 1\nabc"))).toEqual({ started: true, exit: undefined, size: 10, chunk: encoder.encode("abc"), alive: true, }); }); it("reads a finished step", () => { expect(parsePoll(encoder.encode("1 2 0 0 0\n"))).toMatchObject({ started: true, exit: 2, alive: false, }); }); it("reads a step that has not started", () => { expect(parsePoll(encoder.encode("0 - 0 0 1\n"))).toMatchObject({ started: false, exit: undefined, alive: true, }); }); it("keeps newlines and bytes of the chunk", () => { const chunk = Uint8Array.of(10, 0xff, 10); expect(parsePoll(Uint8Array.of(...encoder.encode("1 - 3 3 1\n"), ...chunk)).chunk).toEqual( chunk, ); }); it.each([ ["no output", ""], ["no newline", "1 - 10 0 1"], ["a short header", "1 - 10\n"], ["a bad exit code", "1 x 0 0 1\n"], ["a short chunk", "1 - 10 5 1\nabc"], ])("rejects %s", (_, output) => { expect(() => parsePoll(encoder.encode(output))).toThrow("unreadable"); }); }); describe("readPoll", () => { const encoder = new TextEncoder(); const fresh = { step: 1, offset: 0, started: false, capped: false }; const poll = (chunk: string, size: number, exit?: number): Poll => ({ started: true, exit, size, chunk: encoder.encode(chunk), alive: exit === undefined, }); it("starts a step and keeps a partial line for the next poll", () => { const { cursor, events } = readPoll(fresh, poll("a\nb", 3)); expect(cursor).toEqual({ step: 1, offset: 2, started: true, capped: false }); expect(events).toMatchObject([ { kind: "start" }, { kind: "lines", lines: [{ text: "a" }] }, ]); }); it("waits for a step that has not started", () => { const notStarted = { ...poll("", 0), started: false }; expect(readPoll(fresh, notStarted)).toEqual({ cursor: fresh, events: [] }); }); it("ends a step only after its whole log is read", () => { const started = { ...fresh, started: true }; const first = readPoll(started, poll("a\n", 5, 1)); expect(first.events).toMatchObject([{ kind: "lines" }]); const second = readPoll(first.cursor, poll("b\nc", 5, 1)); expect(second.events).toEqual([ { kind: "lines", lines: [ { text: "b", time: undefined }, { text: "c", time: undefined }, ], }, { kind: "end", exit: 1 }, ]); expect(second.cursor.offset).toBe(5); }); it("reads a full chunk without a newline as one line", () => { const started = { ...fresh, started: true }; const { cursor, events } = readPoll(started, poll("abc", 10)); expect(cursor.offset).toBe(3); expect(events).toMatchObject([{ kind: "lines", lines: [{ text: "abc" }] }]); }); it("leaves a secret at the chunk edge for the next poll", () => { const started = { ...fresh, started: true }; const secrets = [encoder.encode("s3cret"), encoder.encode("xyz")]; const first = readPoll(started, poll("abc s3c", 20), secrets); expect(first.cursor.offset).toBe(4); expect(first.events).toMatchObject([{ kind: "lines", lines: [{ text: "abc " }] }]); expect(readPoll(started, poll("abc s3q", 20), secrets).cursor.offset).toBe(7); }); it("stops reading a log at the cap", () => { const near = { ...fresh, started: true, offset: logCap - 2 }; const capped = readPoll(near, poll("x\n", logCap + 100)); expect(capped.cursor.capped).toBe(true); expect(capped.events.at(-1)).toEqual({ kind: "capped" }); expect(readPoll(capped.cursor, poll("", logCap + 100, 0)).events).toEqual([ { kind: "end", exit: 0 }, ]); }); }); const gzip = Uint8Array.of(0x1f, 0x8b, 1, 2, 3); const sha = "c0ffee".padEnd(40, "0"); const raw = "engine: cloudflare\ncache: true\nsteps:\n - {name: test, command: 'echo hi'}\n"; const uncached = raw.replace("cache: true\n", ""); const injectedWorkflow = "engine: cloudflare\nsecrets: [{name: API_TOKEN, hosts: [api.example.com, '*.example.org']}]\nsteps: [{name: test, command: 'true'}]\n"; const mixedWorkflow = injectedWorkflow.replace("secrets: [", "secrets: [ENV_TOKEN, "); interface Exec { cmd: string[]; options: ContainerExecOptions | undefined; stdin: Uint8Array | undefined; } class FakeContainer implements Box { running = false; starts: ContainerStartupOptions[] = []; execs: Exec[] = []; destroyed = 0; monitorError: string | undefined; stopped = false; checkoutError: Error | undefined; checkoutHangs = false; launchExit = 0; checkoutExit = 0; polls: string[] = []; snapshots = 0; snapshotError: Error | undefined; snapshotHangs = false; badSnapshot = false; pollHangs = false; ignoresSignals = false; signals: number[] = []; destroyCalls = 0; #waiting: (() => void)[] = []; startError: Error | undefined; // The number of containers started before each intercept. intercepts: { host: string; proxy: Fetcher; starts: number }[] = []; interceptError: Error | undefined; interceptOutboundHttps(host: string, proxy: Fetcher): Promise { if (this.interceptError !== undefined) { return Promise.reject(this.interceptError); } this.intercepts.push({ host, proxy, starts: this.starts.length }); return Promise.resolve(); } start(options: ContainerStartupOptions): void { if (this.startError !== undefined) { throw this.startError; } this.running = true; this.starts.push(options); } monitor(): Promise { if (this.monitorError !== undefined) { return Promise.reject(new Error(this.monitorError)); } return this.stopped || !this.running ? Promise.resolve() : new Promise((resolve) => { this.#waiting.push(resolve); }); } destroyError: Error | undefined; destroy(): Promise { this.destroyCalls++; if (this.destroyError !== undefined) { return Promise.reject(this.destroyError); } this.#stop(); return Promise.resolve(); } signal(signo: number): void { this.signals.push(signo); if (!this.ignoresSignals) { this.#stop(); } } #stop(): void { this.running = false; this.destroyed++; for (const stop of this.#waiting.splice(0)) { stop(); } } inactivityTimeouts: number[] = []; setInactivityTimeout(durationMs: number): Promise { this.inactivityTimeouts.push(durationMs); return Promise.resolve(); } async snapshotContainer(): Promise { while (this.snapshotHangs) { // oxlint-disable-next-line no-await-in-loop await scheduler.wait(10); } if (this.snapshotError !== undefined) { throw this.snapshotError; } this.snapshots++; return { id: `snap-${this.snapshots}`, size: 1000 }; } async exec(cmd: string[], options?: ContainerExecOptions): Promise { const stdin = options?.stdin instanceof ReadableStream ? new Uint8Array(await new Response(options.stdin).arrayBuffer()) : undefined; this.execs.push({ cmd, options, stdin }); if (cmd.join(" ").includes("tar -xz")) { if (this.checkoutError !== undefined) { throw this.checkoutError; } if (this.badSnapshot && this.starts.at(-1)?.containerSnapshot !== undefined) { throw new Error("snapshot not found"); } if (this.checkoutHangs) { await new Promise((_resolve, reject) => { this.#waiting.push(() => { reject(new Error("container is not running")); }); }); } return process(new Uint8Array(), this.checkoutExit); } if (cmd[0] === "timeout") { if (this.pollHangs) { await new Promise((resolve) => { this.#waiting.push(resolve); }); } return process(new TextEncoder().encode(this.polls.shift() ?? "0 - 0 0 1\n"), 0); } return process(new Uint8Array(), this.launchExit); } } function process(stdout: Uint8Array, exitCode: number): ExecProcess { const output = { stdout: stdout.buffer instanceof ArrayBuffer ? stdout.buffer : new ArrayBuffer(0), stderr: new ArrayBuffer(0), exitCode, }; return { stdin: null, stdout: null, stderr: null, pid: 1, isPty: false, exitCode: Promise.resolve(exitCode), output: () => Promise.resolve(output), kill: () => {}, resize: () => {}, }; } async function edit(stub: DurableObjectStub, patch: Record) { await runInDurableObject(stub, (_instance, state) => { const key = `run/${state.storage.kv.get("current")}`; state.storage.kv.put(key, { ...state.storage.kv.get(key), ...patch }); }); } // A finished run sets its alarm only to delete it after the retention. async function retentionAlarm(stub: DurableObjectStub) { return await runInDurableObject(stub, async (_instance, state) => { const alarm = (await state.storage.getAlarm()) ?? 0; return alarm > Date.now() + 29 * 86_400_000 && alarm <= Date.now() + 30 * 86_400_000; }); } async function pipelineFrames(stub: DurableObjectStub) { return await runInDurableObject(stub, (_instance, state) => Object.fromEntries( state.storage.sql .exec<{ pipeline: string; n: number }>( "SELECT pipeline, count(*) AS n FROM frames GROUP BY pipeline", ) .toArray() .map(({ pipeline, n }) => [pipeline, n]), ), ); } // A container that ignores SIGTERM gets the grace period before destroy(). async function stuckCancel(stub: DurableObjectStub) { vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); try { const cancel = stub.cancel(); await vi.waitFor(() => { expect(vi.getTimerCount()).toBe(1); }); await runInDurableObject(stub, () => vi.advanceTimersByTimeAsync(30_000)); await cancel; } finally { vi.useRealTimers(); } } async function start(stub: DurableObjectStub, params: RunnerParams) { return await runInDurableObject( stub, async (instance) => (await instance.start(params)) ?? "started", ); } function streamed(response: Response): Promise { const ws = response.webSocket; if (ws === null) { throw new Error("no WebSocket"); } return new Promise((resolve) => { let n = 0; ws.addEventListener("message", () => { n++; }); ws.addEventListener("close", () => { resolve(n); }); ws.accept(); }); } async function frames(stub: DurableObjectStub) { return await runInDurableObject(stub, (_instance, state) => state.storage.sql .exec<{ frame: ArrayBuffer }>("SELECT frame FROM frames ORDER BY id") .toArray() .map(({ frame }) => { const [, rest]: [unknown, Uint8Array] = decodeFirst(new Uint8Array(frame)); const body: [unknown, Uint8Array] = decodeFirst(rest); return body[0]; }), ); } describe("Runner", () => { let fake: FakeContainer; let archives: URL[]; let archiveSignals: AbortSignal[]; let archiveRedirects: string[]; let archiveStatus = 200; beforeEach(() => { fake = new FakeContainer(); archives = []; archiveSignals = []; archiveRedirects = []; archiveStatus = 200; routeFetch({ [`GET https://plc.directory/${repo}`]: () => Response.json(didDoc(repo, undefined, [knotService])), [`GET https://plc.directory/${env.OWNER_DID}`]: offline, [`GET ${knot}/xrpc/org.tangled.git.getArchive`]: (url, request) => { archives.push(url); archiveSignals.push(request.signal); archiveRedirects.push(request.redirect); return new Response(gzip, { status: archiveStatus }); }, [getRecordRoute(knot, repo, "org.tangled.git.refs", "self")]: () => record({ $type: "org.tangled.git.refs", defaultBranch: "main", refs: [] }), [`GET ${knot}/xrpc/org.tangled.git.listCommits`]: (url) => { expect(url.searchParams.get("ref")).toBe("refs/heads/main"); return Response.json({ commits: [{ hash: sha }] }); }, }); }); async function runner( name: string, workflow = raw, trusted = false, params: Partial = {}, ) { const stub = exports.Runner.getByName(name); await runInDurableObject(stub, async (instance) => { instance.container = fake; await instance.start({ pipeline: name, workflow: "ci.yml", repo, commit: "abc123", branch: "feature", raw: workflow, trusted, ...params, }); }); return stub; } async function succeed(stub: DurableObjectStub) { await runDurableObjectAlarm(stub); fake.polls.push("1 0 0 0 0\n"); await runDurableObjectAlarm(stub); } const mainHead = { branch: "main", commit: sha }; const saved = { id: "snap-old", image: "cloudflare/debian-trixie", tier: "public" as const }; it("saves a clean snapshot after a trusted run at the default branch head", async () => { const stub = await runner("snapshot", raw, true, mainHead); await succeed(stub); expect(await stub.status("snapshot")).toMatchObject({ status: "success", snapshot: { id: "snap-1", image: "cloudflare/debian-trixie", tier: "public", }, }); const wipe = fake.execs.at(-1)?.cmd.join(" ") ?? ""; expect(wipe).toContain("rm -rf /treadle /workspace && find /tmp -mindepth 1 -delete"); expect(fake.destroyed).toBe(1); expect( await runInDurableObject(stub, (_instance, state) => state.storage.kv.get("reported")), ).toBe("snapshot success snap-1"); expect((await frames(stub)).slice(-3)).toMatchObject([ { step: 2, status: "start", content: "Save cache", kind: "system" }, { step: 2, content: "treadle: saved snapshot snap-1 (1000 bytes)\n" }, { step: 2, status: "end", outcome: "success" }, ]); }); it.each([ ["an untrusted run", raw, false, mainHead], ["another branch", raw, true, { branch: "feature", commit: sha }], ["a commit behind the head", raw, true, { ...mainHead, commit: "a".repeat(40) }], ["an untrusted run without cache", uncached, false, { ...mainHead, cached: true }], [ "another branch without cache", uncached, true, { branch: "feature", commit: sha, cached: true }, ], [ "a commit behind the head without cache", uncached, true, { ...mainHead, commit: "a".repeat(40), cached: true }, ], ["a workflow without cache or snapshots", uncached, true, mainHead], ])("saves and discards no snapshot after %s", async (_, workflow, trusted, params) => { const stub = await runner(`no-snapshot-${_}`, workflow, trusted, params); await succeed(stub); const status = await stub.status(`no-snapshot-${_}`); expect(status).toMatchObject({ status: "success" }); expect(status?.snapshot).toBeUndefined(); expect(status?.discardCache).toBeUndefined(); expect(fake.snapshots).toBe(0); }); it("discards the snapshots after a trusted run without cache at the default branch head", async () => { const stub = await runner("discard", uncached, true, { ...mainHead, cached: true }); await succeed(stub); expect(await stub.status("discard")).toMatchObject({ status: "success", discardCache: true, }); expect(fake.snapshots).toBe(0); expect( await runInDurableObject(stub, (_instance, state) => state.storage.kv.get("reported")), ).toBe("discard success discard"); }); it("saves no snapshot after a failed run", async () => { const stub = await runner("failed-snapshot", raw, true, mainHead); await runDurableObjectAlarm(stub); fake.polls.push("1 1 0 0 0\n"); await runDurableObjectAlarm(stub); expect(await stub.status("failed-snapshot")).toMatchObject({ status: "failed" }); expect(fake.snapshots).toBe(0); }); it("saves the trusted snapshot after a run with secrets or from a trusted snapshot", async () => { await exports.Secrets.getByName(repo).add("API_TOKEN", "s3cret", "did:plc:owner"); const secret = await runner( "tier-secret", `cache: true\n${secretWorkflow}`, true, mainHead, ); await succeed(secret); expect(await secret.status("tier-secret")).toMatchObject({ snapshot: { tier: "trusted" } }); const restored = await runner("tier-restored", raw, true, { ...mainHead, restore: { ...saved, tier: "trusted" }, }); await succeed(restored); expect(await restored.status("tier-restored")).toMatchObject({ snapshot: { tier: "trusted" }, }); }); it("saves the trusted snapshot when secrets are injected", async () => { await exports.Secrets.getByName(repo).add("API_TOKEN", "s3cret", "did:plc:owner"); const stub = await runner( "tier-injected", `cache: true\n${injectedWorkflow}`, true, mainHead, ); await succeed(stub); expect(await stub.status("tier-injected")).toMatchObject({ snapshot: { tier: "trusted" } }); }); it("saves the public snapshot after a trusted snapshot did not start", async () => { fake.badSnapshot = true; const stub = await runner("tier-fallback", raw, true, { ...mainHead, restore: { ...saved, tier: "trusted" }, }); await runDurableObjectAlarm(stub); await succeed(stub); expect(await stub.status("tier-fallback")).toMatchObject({ snapshot: { tier: "public" } }); }); it("ends the build and the snapshot step green when the snapshot fails", async () => { fake.snapshotError = new Error("disk too large"); const stub = await runner("snapshot-error", raw, true, mainHead); await succeed(stub); expect(await stub.status("snapshot-error")).toMatchObject({ status: "success" }); expect((await stub.status("snapshot-error"))?.snapshot).toBeUndefined(); expect((await frames(stub)).slice(-2)).toMatchObject([ { step: 2, content: "treadle: no snapshot: disk too large\n" }, { step: 2, status: "end", outcome: "success" }, ]); expect(fake.destroyed).toBe(1); }); it("reports the success before the snapshot ends", async () => { fake.snapshotHangs = true; const stub = await runner("snapshot-slow", raw, true, mainHead); await runDurableObjectAlarm(stub); fake.polls.push("1 0 0 0 0\n"); const alarm = runDurableObjectAlarm(stub); await vi.waitFor(async () => { expect( await runInDurableObject(stub, (_instance, state) => state.storage.kv.get("reported"), ), ).toBe("snapshot-slow success "); }); expect(fake.destroyed).toBe(0); fake.snapshotHangs = false; await alarm; expect(fake.destroyed).toBe(1); }); it("saves the snapshot when a viewer connects during the snapshot step", async () => { fake.snapshotHangs = true; const stub = await runner("snapshot-viewer", raw, true, mainHead); await runDurableObjectAlarm(stub); fake.polls.push("1 0 0 0 0\n"); const alarm = runDurableObjectAlarm(stub); await vi.waitFor(async () => { expect( await runInDurableObject(stub, (_instance, state) => state.storage.kv.get("snapshotStep"), ), ).toBe(2); }); await runInDurableObject(stub, (instance) => { instance.fetch(new Request("https://runner/?pipeline=snapshot-viewer")); }); fake.snapshotHangs = false; await alarm; expect(await stub.status("snapshot-viewer")).toMatchObject({ snapshot: { id: "snap-1" } }); expect(fake.destroyed).toBe(1); }); it("ends the snapshot step after a restart during the snapshot", async () => { const stub = await runner("snapshot-restart", raw, true, mainHead); await succeed(stub); fake.running = true; await runInDurableObject(stub, async (_instance, state) => { state.storage.kv.put("snapshotStep", 2); await state.storage.setAlarm(Date.now()); }); await runDurableObjectAlarm(stub); expect(fake.destroyed).toBe(2); expect((await frames(stub)).slice(-2)).toMatchObject([ { step: 2, content: "treadle: no snapshot: the Runner restarted\n" }, { step: 2, status: "end", content: "Save cache", outcome: "success" }, ]); }); it("starts from a snapshot of the same image", async () => { const stub = await runner("restore", raw, false, { restore: saved }); await runDurableObjectAlarm(stub); expect(fake.starts).toMatchObject([{ containerSnapshot: { id: "snap-old" } }]); expect(fake.starts[0]).not.toHaveProperty("image"); expect(await frames(stub)).toMatchObject([ { step: 0, status: "start" }, { step: 0, content: "treadle: started from snapshot snap-old\n" }, { step: 0, status: "end", outcome: "success" }, ]); }); it.each([ ["another image", raw, { ...saved, image: "cloudflare/debian-bookworm" }], ["the trusted tier in an untrusted run", raw, { ...saved, tier: "trusted" as const }], ["a workflow without cache", uncached, saved], ])("starts from the image with a snapshot of %s", async (_, workflow, snapshot) => { const stub = await runner(`ignore-${_}`, workflow, false, { restore: snapshot }); await runDurableObjectAlarm(stub); expect(fake.starts).toMatchObject([{ image: "cloudflare/debian-trixie" }]); expect(fake.starts[0]).not.toHaveProperty("containerSnapshot"); }); it("starts from the image when the snapshot does not start", async () => { fake.badSnapshot = true; const stub = await runner("expired", raw, false, { restore: saved }); await runDurableObjectAlarm(stub); expect(await stub.status("expired")).toMatchObject({ status: "started" }); expect(fake.destroyed).toBe(1); await runDurableObjectAlarm(stub); expect(await stub.status("expired")).toMatchObject({ status: "running" }); expect(fake.starts).toMatchObject([ { containerSnapshot: { id: "snap-old" } }, { image: "cloudflare/debian-trixie" }, ]); expect(await frames(stub)).toMatchObject([ { step: 0, status: "start" }, { step: 0, content: "treadle: snapshot snap-old did not start (snapshot not found)\n", }, { step: 0, status: "end", outcome: "success" }, ]); }); it("checks out the commit, runs each step and reports success", async () => { const stub = await runner( "success", "engine: cloudflare\nsteps:\n - {name: one, command: 'echo hi'}\n - {name: two, command: 'true'}\n", ); await runDurableObjectAlarm(stub); expect(fake.starts).toMatchObject([ { image: "cloudflare/debian-trixie", entrypoint: [ "bash", "-c", "trap 'exit 143' TERM; mkdir -p /treadle && touch /treadle/debug.log && { tail -F /treadle/debug.log & wait $!; }", ], enableInternet: true, instance: "standard-1", }, ]); expect(archives.map((url) => Object.fromEntries(url.searchParams))).toEqual([ { repo, ref: "abc123", format: "tar.gz", prefix: "src" }, ]); expect(archiveRedirects).toEqual(["manual"]); const [checkout, launch] = fake.execs; expect(checkout?.stdin).toEqual(gzip); expect(checkout?.cmd.join(" ")).toContain("tar -xz --strip-components=1 -C /workspace"); expect(checkout?.cmd.join(" ")).toContain( 'echo "treadle: checkout exit $rc" >> /treadle/debug.log 2>/dev/null; exit $rc', ); expect(new TextDecoder().decode(launch?.stdin)).toContain( "bash -eo pipefail -c -- 'echo hi'", ); expect(launch?.options?.env).toMatchObject({ CI: "true", TREADLE_COMMIT: "abc123" }); expect(await stub.status("success")).toMatchObject({ status: "running" }); fake.polls.push("1 0 3 3 0\nhi\n", "1 0 0 0 0\n"); await runDurableObjectAlarm(stub); expect(await stub.status("success")).toMatchObject({ status: "running" }); await runDurableObjectAlarm(stub); expect(await stub.status("success")).toMatchObject({ status: "success" }); expect(fake.execs.filter(({ cmd }) => cmd[0] === "timeout").length).toBe(2); expect(fake.destroyed).toBe(1); expect(await frames(stub)).toMatchObject([ { step: 0, status: "start" }, { step: 0, status: "end" }, { step: 1, status: "start", content: "one" }, { step: 1, content: "hi\n" }, { step: 1, status: "end" }, { step: 2, status: "start", content: "two" }, { step: 2, status: "end" }, ]); expect(await retentionAlarm(stub)).toBe(true); }); it("keeps the inactivity timeout of a 6h run within 6h", async () => { const stub = await runner("long", `${raw}treadle:\n cloudflare:\n timeout: 6h\n`); await runDurableObjectAlarm(stub); expect(await stub.status("long")).toMatchObject({ status: "running" }); expect(fake.inactivityTimeouts).toEqual([6 * 3_600_000]); }); it("starts the run only once", async () => { const stub = await runner("once"); await runDurableObjectAlarm(stub); await runner("once", "engine: cloudflare\nsteps: [{name: x, command: 'false'}]\n"); expect(await stub.status("once")).toMatchObject({ status: "running" }); }); it.each([ ["another engine", "engine: nixery\n", "ci.yml is not an engine: cloudflare workflow"], ["an invalid workflow", "engine: cloudflare\n", "steps must be a non-empty list"], ])("fails a run of %s", async (_, workflow, error) => { const stub = await runner(`refused-${error}`, workflow); await runDurableObjectAlarm(stub); expect(await stub.status(`refused-${error}`)).toMatchObject({ status: "failed", error }); expect(fake.starts).toEqual([]); }); it("fails a step that exits with an error", async () => { const stub = await runner("failure"); await runDurableObjectAlarm(stub); fake.polls.push("1 2 0 0 0\n"); await runDurableObjectAlarm(stub); expect(await stub.status("failure")).toMatchObject({ status: "failed", error: "test failed", }); expect(fake.destroyed).toBe(1); }); it("fails with the monitor's reason when the container does not start", async () => { fake.monitorError = "invalid instance type huge"; fake.checkoutHangs = true; const stub = await runner("boot"); await runDurableObjectAlarm(stub); expect(await stub.status("boot")).toMatchObject({ status: "failed", error: "invalid instance type huge", }); }); it("fails with the error of a start that throws", async () => { fake.startError = new TypeError("Invalid container instance type."); const stub = await runner("start"); await runDurableObjectAlarm(stub); expect(await stub.status("start")).toMatchObject({ status: "failed", error: "Invalid container instance type.", }); }); it("fails when the knot refuses the archive", async () => { archiveStatus = 404; const stub = await runner("missing"); await runDurableObjectAlarm(stub); expect(await stub.status("missing")).toMatchObject({ status: "failed", error: "archive unavailable: HTTP 404", }); }); it("destroys the container again after a failed destroy", async () => { const stub = await runner("destroy"); await runDurableObjectAlarm(stub); fake.ignoresSignals = true; fake.destroyError = new Error("busy"); await stuckCancel(stub); expect(fake.running).toBe(true); fake.ignoresSignals = false; fake.destroyError = undefined; expect(await runDurableObjectAlarm(stub)).toBe(true); expect(fake.running).toBe(false); expect(await retentionAlarm(stub)).toBe(true); }); it("fails when the container stops before the checkout ends", async () => { fake.stopped = true; fake.checkoutHangs = true; const stub = await runner("stopped"); await runDurableObjectAlarm(stub); expect(await stub.status("stopped")).toMatchObject({ status: "failed", error: "the container stopped", }); expect(archiveSignals.map((signal) => signal.aborted)).toEqual([true]); }); it("ends the archive download and clears its timers after each phase", async () => { vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); try { const stub = await runner("deadline", raw, true, mainHead); await runDurableObjectAlarm(stub); expect(await stub.status("deadline")).toMatchObject({ status: "running" }); expect(archiveSignals.map((signal) => signal.aborted)).toEqual([true]); expect(vi.getTimerCount()).toBe(0); fake.polls.push("1 0 0 0 0\n"); await runDurableObjectAlarm(stub); expect(await stub.status("deadline")).toMatchObject({ status: "success", snapshot: { id: "snap-1" }, }); expect(vi.getTimerCount()).toBe(0); } finally { vi.useRealTimers(); } }); it("fails with the exec error when the monitor has no reason", async () => { fake.checkoutError = new Error("exec failed"); const stub = await runner("exec"); await runDurableObjectAlarm(stub); expect(await stub.status("exec")).toMatchObject({ status: "failed", error: "exec failed" }); expect(archiveSignals.map((signal) => signal.aborted)).toEqual([true]); }); it("fails when the checkout fails", async () => { fake.checkoutExit = 2; const stub = await runner("checkout"); await runDurableObjectAlarm(stub); expect(await stub.status("checkout")).toMatchObject({ status: "failed", error: "checkout failed", }); }); it("fails when the steps do not start", async () => { fake.launchExit = 1; const stub = await runner("launch"); await runDurableObjectAlarm(stub); expect(await stub.status("launch")).toMatchObject({ status: "failed", error: "the steps did not start", }); }); it("tries the checkout again when the knot has no archive", async () => { archiveStatus = 429; const stub = await runner("archive"); await runDurableObjectAlarm(stub); expect(await stub.status("archive")).toMatchObject({ status: "started" }); archiveStatus = 200; await runDurableObjectAlarm(stub); expect(await stub.status("archive")).toMatchObject({ status: "running" }); expect(fake.starts.length).toBe(1); }); it("fails a run whose step script stopped after a grace period", async () => { const stub = await runner("lost"); await runDurableObjectAlarm(stub); fake.polls.push("1 - 0 0 0\n", "1 - 0 0 0\n"); await runDurableObjectAlarm(stub); expect(await stub.status("lost")).toMatchObject({ status: "running" }); await edit(stub, { launchedAt: 0 }); await runDurableObjectAlarm(stub); expect(await stub.status("lost")).toMatchObject({ status: "failed", error: "the step script stopped", }); }); it("polls again after unreadable poll output", async () => { const stub = await runner("garbled"); await runDurableObjectAlarm(stub); fake.polls.push("garbage", "1 0 0 0 0\n"); await runDurableObjectAlarm(stub); expect(await stub.status("garbled")).toMatchObject({ status: "running" }); await runDurableObjectAlarm(stub); expect(await stub.status("garbled")).toMatchObject({ status: "success" }); }); it("fails a run whose container stopped", async () => { const stub = await runner("gone"); await runDurableObjectAlarm(stub); fake.running = false; await runDurableObjectAlarm(stub); expect(await stub.status("gone")).toMatchObject({ status: "failed", error: "the container stopped", }); }); it("times out a run at its deadline and stops its container with SIGTERM", async () => { const stub = await runner("timeout"); await runDurableObjectAlarm(stub); await edit(stub, { deadline: 0 }); await runDurableObjectAlarm(stub); expect(await stub.status("timeout")).toMatchObject({ status: "timeout", error: "the run timed out", }); expect(fake.signals).toEqual([15]); expect(fake.destroyed).toBe(1); expect(fake.destroyCalls).toBe(0); }); it("destroys a container that ignores SIGTERM after the grace period", async () => { const stub = await runner("sigterm"); await runDurableObjectAlarm(stub); fake.ignoresSignals = true; await stuckCancel(stub); expect(fake.signals).toEqual([15]); expect(fake.destroyCalls).toBe(1); expect(fake.running).toBe(false); }); it("fails a run whose checkout does not end", async () => { vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); try { fake.checkoutHangs = true; const stub = await runner("checkout-stalled"); const alarm = runDurableObjectAlarm(stub); await vi.waitFor(() => { expect(fake.execs.length).toBe(1); }); await runInDurableObject(stub, () => vi.advanceTimersByTimeAsync(10 * 60_000 + 1000)); await alarm; expect(await stub.status("checkout-stalled")).toMatchObject({ status: "failed", error: "the container did not answer", }); expect(archiveSignals.map((signal) => signal.aborted)).toEqual([true]); expect(fake.running).toBe(false); } finally { vi.useRealTimers(); } }); it("fails a run whose poll does not answer, and stops its container", async () => { vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); try { const stub = await runner("stalled"); await runDurableObjectAlarm(stub); fake.pollHangs = true; const alarm = runDurableObjectAlarm(stub); await vi.waitFor(() => { expect(vi.getTimerCount()).toBe(1); }); await runInDurableObject(stub, () => vi.advanceTimersByTimeAsync(60_000)); await alarm; expect(await stub.status("stalled")).toMatchObject({ status: "failed", error: "the container did not answer", }); expect(fake.signals).toEqual([15]); expect(fake.running).toBe(false); } finally { vi.useRealTimers(); } }); it("cancels a run", async () => { const stub = await runner("cancel"); await runDurableObjectAlarm(stub); await stub.cancel(); expect(await stub.status("cancel")).toMatchObject({ status: "cancelled" }); expect(fake.destroyed).toBe(1); await stub.cancel(); await runDurableObjectAlarm(stub); expect(await stub.status("cancel")).toMatchObject({ status: "cancelled" }); expect(fake.destroyed).toBe(1); }); it("times out a run that the Store times out", async () => { const stub = await runner("store-timeout"); await runDurableObjectAlarm(stub); await runInDurableObject(stub, (_instance, state) => state.storage.deleteAlarm()); await stub.cancel("store-timeout", "timeout"); expect(await stub.status("store-timeout")).toMatchObject({ status: "timeout", error: "the run timed out", }); expect(await runDurableObjectAlarm(stub)).toBe(true); }); it("keeps a cancel that arrives while the container boots", async () => { fake.checkoutHangs = true; const stub = await runner("cancel-boot"); const alarm = runDurableObjectAlarm(stub); await vi.waitFor(() => { expect(fake.execs.length).toBe(1); }); await stub.cancel(); await alarm; expect(fake.execs.length).toBe(1); expect(await stub.status("cancel-boot")).toMatchObject({ status: "cancelled" }); }); const secretWorkflow = "engine: cloudflare\nsecrets: [API_TOKEN]\nsteps: [{name: test, command: 'true'}]\n"; it("injects secrets through intercepts for each container of a trusted run", async () => { await exports.Secrets.getByName(repo).add("API_TOKEN", "s3cret", "did:plc:owner"); fake.badSnapshot = true; const stub = await runner("injected", `cache: true\n${injectedWorkflow}`, true, { restore: saved, }); await runDurableObjectAlarm(stub); await runDurableObjectAlarm(stub); expect(await stub.status("injected")).toMatchObject({ status: "running" }); expect(fake.intercepts.map(({ host, starts }) => [host, starts])).toEqual([ ["api.example.com", 0], ["*.example.org", 0], ["api.example.com", 1], ["*.example.org", 1], ]); const launchEnv = fake.execs.at(-1)?.options?.env ?? {}; const placeholder = launchEnv["API_TOKEN"] ?? ""; expect(placeholder).toMatch(/^[0-9a-f]{64}$/); expect(launchEnv["NODE_EXTRA_CA_CERTS"]).toBe( "/etc/cloudflare/certs/cloudflare-containers-ca.crt", ); expect(JSON.stringify([fake.starts, fake.execs])).not.toContain("s3cret"); await runInDurableObject(stub, (instance) => { instance.mask = undefined; }); fake.interceptError = new Error("unavailable"); await runDurableObjectAlarm(stub); fake.interceptError = undefined; fake.polls.push("1 - 7 7 1\ns3cret\n"); await runDurableObjectAlarm(stub); expect(fake.intercepts.slice(4).map(({ host }) => host)).toEqual([ "api.example.com", "*.example.org", ]); expect(JSON.stringify(await frames(stub))).not.toContain("s3cret"); const seen: (string | null)[] = []; routeFetch({ "GET https://api.example.com/user": (_url, request) => { seen.push(request.headers.get("Authorization")); return new Response("ok"); }, }); await runInDurableObject(stub, async () => { await fake.intercepts.at(-1)?.proxy.fetch("https://api.example.com/user", { headers: { Authorization: `Bearer ${placeholder}` }, }); }); expect(seen).toEqual(["Bearer s3cret"]); }); it("gives a trusted run its env secrets and placeholders for the injected ones", async () => { const secrets = exports.Secrets.getByName(repo); await secrets.add("API_TOKEN", "s3cret", "did:plc:owner"); await secrets.add("ENV_TOKEN", "plain", "did:plc:owner"); const stub = await runner("mixed", mixedWorkflow, true); await runDurableObjectAlarm(stub); const launchEnv = fake.execs.at(-1)?.options?.env; expect(launchEnv).toMatchObject({ ENV_TOKEN: "plain" }); expect(launchEnv?.["API_TOKEN"]).toMatch(/^[0-9a-f]{64}$/); }); it("gives an untrusted run no intercepts and no placeholders", async () => { await exports.Secrets.getByName(repo).add("API_TOKEN", "s3cret", "did:plc:owner"); const stub = await runner("injected-untrusted", injectedWorkflow); await runDurableObjectAlarm(stub); expect(await stub.status("injected-untrusted")).toMatchObject({ status: "running" }); expect(fake.intercepts).toEqual([]); const launchEnv = fake.execs.at(-1)?.options?.env; expect(launchEnv).not.toHaveProperty("API_TOKEN"); expect(launchEnv).not.toHaveProperty("NODE_EXTRA_CA_CERTS"); }); it("gives a trusted run its secrets only in the step environment", async () => { await exports.Secrets.getByName(repo).add("API_TOKEN", "s3cret", "did:plc:owner"); const stub = await runner("secret", secretWorkflow, true); await runDurableObjectAlarm(stub); const launch = fake.execs.at(-1); expect(launch?.options?.env).toMatchObject({ API_TOKEN: "s3cret", CI: "true" }); expect(new TextDecoder().decode(launch?.stdin)).not.toContain("s3cret"); await runDurableObjectAlarm(stub); const others = fake.execs.filter((exec) => exec !== launch); expect(others.map((exec) => exec.cmd[0])).toEqual(["bash", "timeout"]); expect(JSON.stringify(others.map((exec) => exec.options))).not.toContain("s3cret"); }); it("fails a run whose secret was removed before a restart", async () => { const secrets = exports.Secrets.getByName(repo); await secrets.add("REMOVED", "gone", "did:plc:owner"); const stub = await runner( "removed", "engine: cloudflare\nsecrets: [REMOVED]\nsteps: [{name: test, command: 'true'}]\n", true, ); await runDurableObjectAlarm(stub); await secrets.remove("REMOVED"); await runInDurableObject(stub, (instance) => { instance.mask = undefined; }); fake.polls.push("1 - 5 5 1\ngone\n"); await runDurableObjectAlarm(stub); expect(await stub.status("removed")).toMatchObject({ status: "failed", error: "secrets are not set: REMOVED", }); expect(JSON.stringify(await frames(stub))).not.toContain("gone"); }); it("fails a run whose secret was replaced before a restart", async () => { const secrets = exports.Secrets.getByName(repo); await secrets.add("ROTATED", "old", "did:plc:owner"); const stub = await runner( "replaced", "engine: cloudflare\nsecrets: [ROTATED]\nsteps: [{name: test, command: 'true'}]\n", true, ); await runDurableObjectAlarm(stub); await runInDurableObject(secrets, (_instance, state) => { state.storage.sql.exec( "UPDATE secrets SET value = 'new', created_at = '2099-01-01T00:00:00.000Z' WHERE key = 'ROTATED'", ); }); await runInDurableObject(stub, (instance) => { instance.mask = undefined; }); fake.polls.push("1 - 4 4 1\nold\n"); await runDurableObjectAlarm(stub); expect(await stub.status("replaced")).toMatchObject({ status: "failed", error: "secrets changed: ROTATED", }); expect(JSON.stringify(await frames(stub))).not.toContain("old"); }); it("masks secrets in the stored log, also after a restart", async () => { await exports.Secrets.getByName(repo).add("API_TOKEN", "s3cret", "did:plc:owner"); const stub = await runner("masked", secretWorkflow, true); await runDurableObjectAlarm(stub); fake.polls.push("1 - 22 22 1\ntoken s3cret czNjcmV0\n"); await runDurableObjectAlarm(stub); await runInDurableObject(stub, (instance) => { instance.mask = undefined; }); fake.polls.push("1 0 39 17 0\nagain s3\u001b[0mcret\n"); await runDurableObjectAlarm(stub); expect(await stub.status("masked")).toMatchObject({ status: "success" }); expect(await frames(stub)).toMatchObject([ {}, {}, { step: 1, status: "start" }, { step: 1, content: "token *** ***\n" }, { step: 1, content: "again ***\n" }, { step: 1, status: "end" }, ]); await runInDurableObject(stub, (_instance, state) => { const stored = JSON.stringify([ ...state.storage.kv.list(), state.storage.sql.exec("SELECT * FROM frames").toArray(), ]); expect(stored).not.toContain("s3cret"); }); }); it("runs an untrusted run without its secrets", async () => { await exports.Secrets.getByName(repo).add("API_TOKEN", "s3cret", "did:plc:owner"); const stub = await runner("untrusted", secretWorkflow); await runDurableObjectAlarm(stub); expect(await stub.status("untrusted")).toMatchObject({ status: "running" }); expect(fake.execs.at(-1)?.options?.env).not.toHaveProperty("API_TOKEN"); }); it("fails a trusted run that names a secret that is not set", async () => { const stub = await runner( "unset", "engine: cloudflare\nsecrets: [MISSING, OTHER]\nsteps: [{name: test, command: 'true'}]\n", true, ); await runDurableObjectAlarm(stub); expect(await stub.status("unset")).toMatchObject({ status: "failed", error: "secrets are not set: MISSING, OTHER", }); expect(fake.starts).toEqual([]); }); it("installs the apt dependencies in a system step before the steps", async () => { const stub = await runner( "apt", "engine: cloudflare\ndependencies: [jq, 'tzdata=1:2025b~rc1']\nsteps: [{name: test, command: 'true'}]\n", ); await runDurableObjectAlarm(stub); const script = new TextDecoder().decode(fake.execs.at(-1)?.stdin); expect(script).toContain( "env DEBIAN_FRONTEND='noninteractive' bash -eo pipefail -c -- 'apt-get update && ", ); expect(script).toContain("-c -- 'true' > /treadle/log/2 "); fake.polls.push("1 0 8 8 1\nSetting\n", "1 0 0 0 0\n"); await runDurableObjectAlarm(stub); await runDurableObjectAlarm(stub); expect(await stub.status("apt")).toMatchObject({ status: "success" }); expect(await frames(stub)).toMatchObject([ { step: 0, status: "start", kind: "system" }, { step: 0, status: "end" }, { step: 1, status: "start", content: "Install dependencies", command: "apt-get update && apt-get install -y --no-install-recommends 'jq' 'tzdata=1:2025b~rc1'", kind: "system", }, { step: 1, content: "Setting\n" }, { step: 1, status: "end", kind: "system", outcome: "success" }, { step: 2, status: "start", content: "test", kind: "user" }, { step: 2, status: "end" }, ]); }); it("fails a run whose dependencies do not install", async () => { const stub = await runner( "apt-failure", "engine: cloudflare\ndependencies: [nope]\nsteps: [{name: test, command: 'true'}]\n", ); await runDurableObjectAlarm(stub); fake.polls.push("1 100 0 0 0\n"); await runDurableObjectAlarm(stub); expect(await stub.status("apt-failure")).toMatchObject({ status: "failed", error: "Install dependencies failed", }); }); const next = (pipeline: string): RunnerParams => ({ pipeline, workflow: "ci.yml", repo, branch: "feature", commit: "def456", raw, trusted: false, }); // Runs pipeline name to success, then starts and boots pipeline `${name}-2`. async function twoRuns(name: string) { const stub = await runner(name); await runDurableObjectAlarm(stub); expect(await start(stub, next(`${name}-2`))).toBe("busy"); fake.polls.push("1 0 0 0 0\n"); await runDurableObjectAlarm(stub); await stub.start(next(`${name}-2`)); await runDurableObjectAlarm(stub); return stub; } it("runs the runs of many pipelines one after another, each in a new container", async () => { const stub = await twoRuns("serial"); await stub.start(next("serial-2")); expect(fake.starts.length).toBe(2); expect(await stub.status("serial")).toMatchObject({ status: "success" }); expect(await stub.status("serial-2")).toMatchObject({ status: "running" }); }); it("forgets the secret mask of the last run", async () => { const stub = await runner("mask"); await runDurableObjectAlarm(stub); await stub.cancel(); await stub.start(next("mask-2")); expect(await runInDurableObject(stub, (instance) => instance.mask)).toBeUndefined(); }); it("keeps the state and the log of each run apart", async () => { const stub = await twoRuns("apart"); await stub.cancel("apart"); expect(await stub.status("apart-2")).toMatchObject({ status: "running" }); await stub.cancel("apart-2"); expect(await stub.status("apart")).toMatchObject({ status: "success" }); expect(await pipelineFrames(stub)).toEqual({ apart: 4, "apart-2": 2 }); const served = await runInDurableObject(stub, (instance) => Promise.all([ streamed(instance.fetch(new Request("https://runner/?pipeline=apart"))), streamed(instance.fetch(new Request("https://runner/?pipeline=apart-2"))), ]), ); expect(served).toEqual([4, 2]); }); it("streams the frames of its own run to a live viewer and closes at the end", async () => { const stub = await runner("live"); const watch = async (pipeline: string) => { const response = await stub.fetch(`https://runner/?pipeline=${pipeline}`, { headers: { Upgrade: "websocket" }, }); const ws = response.webSocket; if (ws === null) { throw new Error("no WebSocket"); } const received: unknown[] = []; ws.binaryType = "arraybuffer"; ws.addEventListener("message", (event) => { if (event.data instanceof ArrayBuffer) { const [, rest]: [unknown, Uint8Array] = decodeFirst(new Uint8Array(event.data)); received.push(decodeFirst(rest)[0]); } }); const closed = new Promise((resolve) => { ws.addEventListener("close", (event) => { resolve(event.code); }); }); ws.accept(); return { received, closed }; }; const viewer = await watch("live"); const other = await watch("other"); await runDurableObjectAlarm(stub); fake.polls.push("1 0 3 3 0\nhi\n"); await runDurableObjectAlarm(stub); expect(await viewer.closed).toBe(1000); expect(viewer.received).toHaveLength(5); expect(viewer.received).toEqual(await frames(stub)); expect(other.received).toEqual([]); }); it("refuses a new run while the container of the last run still runs", async () => { const stub = await runner("busy"); await runDurableObjectAlarm(stub); fake.ignoresSignals = true; fake.destroyError = new Error("busy"); await stuckCancel(stub); expect(await start(stub, next("busy-2"))).toBe("busy"); }); it("refuses a new run while an alarm of the last run runs", async () => { fake.checkoutHangs = true; const stub = await runner("in-flight"); const alarm = runDurableObjectAlarm(stub); await vi.waitFor(() => { expect(fake.execs.length).toBe(1); }); fake.running = false; await stub.cancel(); expect(await start(stub, next("in-flight-2"))).toBe("busy"); await runInDurableObject(stub, () => fake.destroy()); await alarm; expect(await start(stub, next("in-flight-2"))).toBe("started"); expect(await stub.status("in-flight-2")).toMatchObject({ status: "started" }); }); it("deletes the runs that finished more than LOG_RETENTION_DAYS ago", async () => { const stub = await runner("expire"); await runDurableObjectAlarm(stub); await stub.cancel(); await edit(stub, { finishedAt: new Date(Date.now() - 31 * 86_400_000).toISOString() }); await stub.start(next("expire-2")); await runDurableObjectAlarm(stub); await stub.cancel("expire-2"); await runDurableObjectAlarm(stub); expect(await stub.status("expire")).toBeUndefined(); expect(await stub.status("expire-2")).toMatchObject({ status: "cancelled" }); expect(await pipelineFrames(stub)).toEqual({ "expire-2": 2 }); }); // Converts the storage of the current run to the form of older code, and runs fn on a Runner // that starts on it. async function asLegacy( stub: DurableObjectStub, fn: (legacy: Runner) => Promise, ): Promise { return await runInDurableObject(stub, async (_instance, state) => { const { sql, kv } = state.storage; const pipeline = kv.get("current") ?? ""; const old = sql.exec<{ frame: ArrayBuffer }>("SELECT frame FROM frames").toArray(); sql.exec("DROP TABLE frames"); sql.exec("CREATE TABLE frames (id INTEGER PRIMARY KEY, frame BLOB NOT NULL)"); for (const { frame } of old) { sql.exec("INSERT INTO frames (frame) VALUES (?)", frame); } kv.put("run", kv.get(`run/${pipeline}`)); kv.put("reported", "running "); for (const key of ["current", `run/${pipeline}`, "schemaVersion"]) { kv.delete(key); } const legacy = new Runner(state, env); legacy.container = fake; return await fn(legacy); }); } it("finishes, serves and cancels the runs that older code stored", async () => { const stub = await runner("legacy"); await runDurableObjectAlarm(stub); fake.polls.push("1 0 3 3 0\nhi\n"); const finished = await asLegacy(stub, async (legacy) => { await legacy.alarm(); return { status: legacy.status("legacy"), served: await streamed( legacy.fetch(new Request("https://runner/?pipeline=legacy")), ), }; }); expect(finished).toMatchObject({ status: { status: "success" }, served: 5 }); expect( await runInDurableObject(stub, (_instance, state) => ({ run: state.storage.kv.get("run"), current: state.storage.kv.get("current"), schemaVersion: state.storage.kv.get("schemaVersion"), })), ).toEqual({ run: undefined, current: "legacy", schemaVersion: 1 }); expect(await pipelineFrames(stub)).toEqual({ legacy: 5 }); expect( await runInDurableObject(stub, (_instance, state) => state.storage.kv.get("reported")), ).toBe("legacy success "); const cancelled = await runner("legacy-cancel"); await runDurableObjectAlarm(cancelled); const status = await asLegacy(cancelled, async (legacy) => { await legacy.cancel("legacy-cancel"); return legacy.status("legacy-cancel"); }); expect(status).toMatchObject({ status: "cancelled" }); expect(fake.destroyed).toBe(2); }); it("knows no run before start", async () => { expect(await exports.Runner.getByName("none").status("none")).toBeUndefined(); }); });