From bf3f8273b894e514b9a49dc3bed251399fef239d Mon Sep 17 00:00:00 2001 From: Gregory Anders Date: Sat, 7 Nov 2026 12:24:04 -0600 Subject: [PATCH] Restore and save container snapshots for engine: cloudflare --- src/notify.ts | 2 +- src/runner.ts | 174 +++++++++++++++++++++++++++++++---- src/store.ts | 57 +++++++++++- test/runner.test.ts | 218 +++++++++++++++++++++++++++++++++++++++++++- test/store.test.ts | 51 ++++++++++- 5 files changed, 478 insertions(+), 24 deletions(-) diff --git a/src/notify.ts b/src/notify.ts index 690449f..4f7e659 100644 --- a/src/notify.ts +++ b/src/notify.ts @@ -85,7 +85,7 @@ async function refBranch(knot: string, repo: string, ref: string | undefined): P return ref === undefined || ref === "" ? await defaultBranch(knot, repo) : branchFromRef(ref); } -async function defaultBranch(knot: string, repo: string): Promise { +export async function defaultBranch(knot: string, repo: string): Promise { const refs = await getRecord(knot, repo, "org.tangled.git.refs", "self", 500); const branch = refs?.["defaultBranch"]; return typeof branch === "string" && branch !== "" ? branchFromRef(branch) : "HEAD"; diff --git a/src/runner.ts b/src/runner.ts index 52036ba..fce1427 100644 --- a/src/runner.ts +++ b/src/runner.ts @@ -1,9 +1,9 @@ import { DurableObject } from "cloudflare:workers"; import type { RunStatus } from "./buildkite"; -import type { RunnerState } from "./store"; +import type { RunnerState, Snapshot } from "./store"; import { controlFrame, dataFrame, logLines } from "./logs"; import type { cleanLine } from "./logs"; -import { resolveKnot } from "./notify"; +import { defaultBranch, headCommit, resolveKnot } from "./notify"; import { secretMask } from "./secrets"; import { parseWorkflow } from "./workflow"; import type { CloudflareJob, Step } from "./workflow"; @@ -162,6 +162,9 @@ export interface RunnerParams { commit: string; raw: string; trusted: boolean; + // Missing from older Stores. + branch?: string; + restore?: Snapshot; } type RunState = Omit & @@ -179,6 +182,17 @@ const pollMs = 3000; const retryMs = 10_000; const graceMs = 30_000; const chunkBytes = 256 * 1024; +const snapshotMs = 5 * 60_000; +// Kills what the steps left running, then removes the logs, the checkout and temporary files, so +// that no snapshot holds them. +const wipeCommand = [ + "timeout", + "120", + "bash", + "-c", + 'p=$(cat /treadle/pid 2>/dev/null); [ -n "$p" ] && kill -KILL -- "-$p" 2>/dev/null; ' + + "rm -rf /treadle /workspace && find /tmp -mindepth 1 -delete", +]; const checkoutCommand = "[ -e /treadle/started ] && exit 0; rm -rf /workspace && mkdir -p /workspace /treadle/log && tar -xz --strip-components=1 -C /workspace; " + 'rc=$?; echo "treadle: checkout exit $rc" >> /treadle/debug.log 2>/dev/null; exit $rc'; @@ -187,11 +201,22 @@ const launchCommand = export type Box = Pick< Container, - "running" | "start" | "monitor" | "destroy" | "exec" | "setInactivityTimeout" + | "running" + | "start" + | "monitor" + | "destroy" + | "exec" + | "setInactivityTimeout" + | "snapshotContainer" >; -function state({ status, startedAt, finishedAt, error }: RunState): RunnerState { - return { status, startedAt, finishedAt, error }; +function state({ status, startedAt, finishedAt, error, snapshot }: RunState): RunnerState { + return { status, startedAt, finishedAt, error, snapshot }; +} + +function withoutRestore(run: RunState): RunState { + const { restore: _, ...rest } = run; + return rest; } function message(error: unknown): string { @@ -270,9 +295,27 @@ export class Runner extends DurableObject { if (run === undefined) { return; } - if (run.phase === "done" && this.container?.running === true) { + if (run.phase === "done") { + // The Runner restarted during the snapshot. + const step = this.ctx.storage.kv.get("snapshotStep"); + if (step !== undefined) { + this.#line(run, step, "no snapshot: the Runner restarted"); + this.#emit( + controlFrame( + run.workflow, + snapshotStep(step), + "end", + new Date().toISOString(), + "failed", + ), + ); + this.ctx.storage.kv.delete("snapshotStep"); + for (const ws of this.ctx.getWebSockets()) { + ws.close(1000, "log stream complete"); + } + } await this.#destroy(run); - } else if (run.phase !== "done") { + } else { try { run = await this.#tick(run); } catch (error) { @@ -338,7 +381,16 @@ export class Runner extends DurableObject { // start() throws only for some invalid options. The first exec waits until the container // runs, and monitor() gives the reason when it does not. - async #boot(run: RunState, container: Box): Promise { + async #boot(current: RunState, container: Box): Promise { + // Older Runner code stored the params of a newer Store without these checks. + const { restore } = current; + const run = + restore === undefined || + (current.job.cache && + restore.image === image && + (restore.tier === "public" || current.trusted)) + ? current + : this.#save(withoutRestore(current)); const { values: secrets, unset } = await this.#secrets(run); if (unset.length > 0) { return await this.#finish(run, "failed", `secrets are not set: ${unset.join(", ")}`); @@ -348,7 +400,9 @@ export class Runner extends DurableObject { const instance = run.job.instance as NonNullable; try { container.start({ - image, + ...(run.restore === undefined + ? { image } + : { containerSnapshot: { id: run.restore.id } }), entrypoint: [ "bash", "-c", @@ -358,7 +412,7 @@ export class Runner extends DurableObject { instance, }); } catch (error) { - return await this.#finish(run, "failed", message(error)); + return await this.#bootFailed(run, message(error)); } } const monitor = container.monitor().then( @@ -387,7 +441,7 @@ export class Runner extends DurableObject { } throw new Error(error); } - this.#emit(controlFrame(run.workflow, checkoutStep, "start", new Date().toISOString())); + this.#startCheckout(run); let exit: number; try { const checkout = await Promise.race([ @@ -400,13 +454,15 @@ export class Runner extends DurableObject { exit = await checkout.exitCode; } catch (error) { const reason = await Promise.race([monitor, scheduler.wait(1000).then(() => null)]); - this.#emit(this.#end(run, checkoutStep.step, "failed")); - return await this.#finish(run, "failed", reason ?? message(error)); + return await this.#bootFailed(run, reason ?? message(error)); } - this.#emit(this.#end(run, checkoutStep.step, exit === 0 ? "success" : "failed")); if (exit !== 0) { - return await this.#finish(run, "failed", "checkout failed"); + return await this.#bootFailed(run, "checkout failed"); + } + if (run.restore !== undefined) { + this.#line(run, 0, `started from snapshot ${run.restore.id}`); } + this.#emit(this.#end(run, checkoutStep.step, "success")); await container.setInactivityTimeout(run.deadline - Date.now() + 10 * 60_000); const launch = await container.exec(["bash", "-c", launchCommand], { stdin: new Blob([runScript(run.job)]).stream(), @@ -521,14 +577,95 @@ export class Runner extends DurableObject { ); } + #startCheckout(run: RunState): void { + const started = this.ctx.storage.sql.exec("SELECT 1 FROM frames LIMIT 1").toArray(); + if (started.length === 0) { + this.#emit(controlFrame(run.workflow, checkoutStep, "start", new Date().toISOString())); + } + } + + // An expired snapshot does not start, so the run starts again from the image. + async #bootFailed(run: RunState, error: string): Promise { + if (run.restore === undefined) { + if (this.ctx.storage.sql.exec("SELECT 1 FROM frames LIMIT 1").toArray().length > 0) { + this.#emit(this.#end(run, checkoutStep.step, "failed")); + } + return await this.#finish(run, "failed", error); + } + this.#startCheckout(run); + this.#line(run, 0, `snapshot ${run.restore.id} did not start (${error})`); + await this.#destroy(run); + return this.#save(withoutRestore(run)); + } + + #line(run: RunState, step: number, text: string): void { + this.#emit(dataFrame(run.workflow, step, `treadle: ${text}\n`, new Date().toISOString())); + } + + // Only a trusted run at the head of the default branch saves a snapshot. The Store already + // has the success, so a slow snapshot does not hold it back. + async #saveSnapshot(run: RunState, container: Box): Promise { + if (!run.trusted || !run.job.cache || run.branch === undefined) { + return run; + } + await this.#report(run); + try { + const knot = await resolveKnot(run.repo); + if ( + run.branch !== (await defaultBranch(knot, run.repo)) || + (await headCommit(knot, run.repo, `refs/heads/${run.branch}`))?.toLowerCase() !== + run.commit.toLowerCase() + ) { + return run; + } + } catch (error) { + console.warn("branch head unavailable", { repo: run.repo, error: message(error) }); + return run; + } + const info = snapshotStep(run.job.steps.length + 1); + this.#emit(controlFrame(run.workflow, info, "start", new Date().toISOString())); + this.ctx.storage.kv.put("snapshotStep", info.step); + let saved: Snapshot | undefined; + try { + const wipe = await container.exec(wipeCommand); + if ((await wipe.exitCode) !== 0) { + throw new Error("the cleanup failed"); + } + const snapshot = await Promise.race([ + container.snapshotContainer(), + scheduler + .wait(snapshotMs) + .then(() => Promise.reject(new Error("the snapshot timed out"))), + ]); + // Secrets or a trusted snapshot may have left files that untrusted runs must not read. + const tier = + (run.job.secrets ?? []).length > 0 || run.restore?.tier === "trusted" + ? "trusted" + : "public"; + saved = { id: snapshot.id, image, tier }; + this.#line(run, info.step, `saved snapshot ${snapshot.id} (${snapshot.size} bytes)`); + } catch (error) { + this.#line(run, info.step, `no snapshot: ${message(error)}`); + } + const outcome = saved === undefined ? "failed" : "success"; + this.#emit(controlFrame(run.workflow, info, "end", new Date().toISOString(), outcome)); + this.ctx.storage.kv.delete("snapshotStep"); + const next = { ...run, snapshot: saved }; + this.ctx.storage.kv.put("run", next); + return next; + } + async #finish(run: RunState, status: RunStatus, error?: string): Promise { - const done = this.#save({ + let done = this.#save({ ...run, phase: "done", status, error, finishedAt: new Date().toISOString(), }); + if (done.status === "success" && this.container?.running === true) { + done = await this.#saveSnapshot(done, this.container); + } await this.#destroy(run); for (const ws of this.ctx.getWebSockets()) { ws.close(1000, "log stream complete"); @@ -551,7 +688,7 @@ export class Runner extends DurableObject { // Returns false when the Store must hear about the run again. async #report(run: RunState): Promise { - const key = `${run.status} ${run.error ?? ""}`; + const key = `${run.status} ${run.error ?? ""}${run.snapshot === undefined ? "" : ` ${run.snapshot.id}`}`; if (this.ctx.storage.kv.get("reported") === key) { return true; } @@ -591,5 +728,8 @@ const noJob: CloudflareJob = { secrets: [], apt: [], }; +function snapshotStep(step: number) { + return { step, name: "Save snapshot", command: undefined, kind: "system" as const }; +} const checkoutStep = { step: 0, name: "Checkout", command: undefined, kind: "system" as const }; const capMarker = "[treadle: log longer than 2 MiB]"; diff --git a/src/store.ts b/src/store.ts index 84d4816..b74c2cd 100644 --- a/src/store.ts +++ b/src/store.ts @@ -90,11 +90,15 @@ export interface PipelineFilter { limit: number; } +// Only trusted runs restore the trusted tier. The public tier holds nothing of a run with secrets. +export type Snapshot = { id: string; image: string; tier: "trusted" | "public" }; + export type RunnerState = { status: RunStatus; startedAt?: string | undefined; finishedAt?: string | undefined; error?: string | undefined; + snapshot?: Snapshot | undefined; }; const terminal = new Set(["success", "failed", "timeout", "cancelled"]); @@ -135,6 +139,8 @@ type QueuedRun = RunKey & { trusted: number | null; repo: string; commit_sha: string; + branch: string; + snapshot: string | null; }; // Runs of engine: cloudflare hold a Runner from dispatch until they finish. @@ -217,6 +223,17 @@ const migrations = [ ALTER TABLE workflow_runs ADD COLUMN engine TEXT NOT NULL DEFAULT 'buildkite'; ALTER TABLE workflow_runs ADD COLUMN trusted INTEGER; `, + ` + CREATE TABLE snapshots ( + repo TEXT NOT NULL, + workflow TEXT NOT NULL, + tier TEXT NOT NULL, + snapshot_id TEXT NOT NULL, + image TEXT NOT NULL, + pipeline TEXT NOT NULL, + PRIMARY KEY (repo, workflow, tier) + ); + `, ]; // Production databases created before versioning hold the first schema at version 0. @@ -512,6 +529,34 @@ export class Store extends DurableObject { ts, ); } + const { snapshot } = state; + if (snapshot !== undefined) { + const { rowsWritten } = this.ctx.storage.sql.exec( + `INSERT INTO snapshots (repo, workflow, tier, snapshot_id, image, pipeline) + SELECT p.repo, r.workflow, ?, ?, ?, r.pipeline + FROM workflow_runs r JOIN pipelines p ON p.rkey = r.pipeline + WHERE r.pipeline = ? AND r.workflow = ? AND r.trusted = 1 AND r.status = 'success' + ON CONFLICT DO UPDATE SET snapshot_id = excluded.snapshot_id, image = excluded.image, + pipeline = excluded.pipeline + WHERE excluded.pipeline > snapshots.pipeline`, + snapshot.tier, + snapshot.id, + snapshot.image, + pipeline, + workflow, + ); + // A trusted run that did not restore the trusted snapshot found it expired or of another + // image, so trusted runs restore the newer public one. + if (snapshot.tier === "public" && rowsWritten > 0) { + this.ctx.storage.sql.exec( + `DELETE FROM snapshots WHERE tier = 'trusted' AND workflow = ? AND pipeline < ? + AND repo = (SELECT repo FROM pipelines WHERE rkey = ?)`, + workflow, + pipeline, + pipeline, + ); + } + } return done; } @@ -769,7 +814,12 @@ export class Store extends DurableObject { .one(); const queued = sql .exec( - `SELECT r.pipeline, r.workflow, r.raw, r.trusted, p.repo, p.commit_sha, + `SELECT r.pipeline, r.workflow, r.raw, r.trusted, p.repo, p.commit_sha, p.branch, + (SELECT json_object('id', s.snapshot_id, 'image', s.image, 'tier', s.tier) + FROM snapshots s + WHERE s.repo = p.repo AND s.workflow = r.workflow + AND (s.tier = 'public' OR r.trusted = 1) + ORDER BY s.tier = 'trusted' DESC LIMIT 1) AS snapshot, ${windowStart} AS created_at FROM workflow_runs r JOIN pipelines p ON p.rkey = r.pipeline WHERE r.status = 'pending' AND r.engine = 'cloudflare' @@ -789,6 +839,11 @@ export class Store extends DurableObject { commit: run.commit_sha, raw: run.raw, trusted: run.trusted === 1, + branch: run.branch, + ...(run.snapshot === null + ? {} + : // oxlint-disable-next-line typescript/no-unsafe-type-assertion -- built by the query + { restore: JSON.parse(run.snapshot) as Snapshot }), }); } catch (error) { const reason = error instanceof Error ? error.message : String(error); diff --git a/test/runner.test.ts b/test/runner.test.ts index 50fbab3..8fd679f 100644 --- a/test/runner.test.ts +++ b/test/runner.test.ts @@ -3,7 +3,7 @@ import { runDurableObjectAlarm, runInDurableObject } from "cloudflare:test"; import { exports } from "cloudflare:workers"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { logCap, parsePoll, pollCommand, readPoll, runScript } from "../src/runner"; -import type { Box, Poll, Runner } from "../src/runner"; +import type { Box, Poll, Runner, RunnerParams } from "../src/runner"; describe("runScript", () => { it("runs each step in order with its environment and stops at a failure", () => { @@ -158,6 +158,7 @@ describe("readPoll", () => { }); const gzip = Uint8Array.of(0x1f, 0x8b, 1, 2, 3); +const sha = "c0ffee".padEnd(40, "0"); const repo = "did:plc:rrrrrrrrrrrrrrrrrrrrrrrr"; const raw = "engine: cloudflare\nsteps:\n - {name: test, command: 'echo hi'}\n"; @@ -179,6 +180,10 @@ class FakeContainer implements Box { launchExit = 0; checkoutExit = 0; polls: string[] = []; + snapshots = 0; + snapshotError: Error | undefined; + snapshotHangs = false; + badSnapshot = false; #waiting: (() => void)[] = []; startError: Error | undefined; @@ -220,6 +225,18 @@ class FakeContainer implements Box { return Promise.resolve(); } + async snapshotContainer(): Promise { + while (this.snapshotHangs) { + // oxlint-disable-next-line no-await-in-loop + await scheduler.wait(10); + } + if (this.snapshotError !== undefined) { + throw this.snapshotError; + } + this.snapshots++; + return { id: `snap-${this.snapshots}`, size: 1000 }; + } + async exec(cmd: string[], options?: ContainerExecOptions): Promise { const stdin = options?.stdin instanceof ReadableStream @@ -230,6 +247,9 @@ class FakeContainer implements Box { if (this.checkoutError !== undefined) { throw this.checkoutError; } + if (this.badSnapshot && this.starts.at(-1)?.containerSnapshot !== undefined) { + throw new Error("snapshot not found"); + } if (this.checkoutHangs) { await new Promise((_resolve, reject) => { this.#waiting.push(() => { @@ -315,6 +335,21 @@ describe("Runner", () => { archives.push(url); return new Response(gzip, { status: archiveStatus }); } + if ( + url.pathname === "/xrpc/com.atproto.repo.getRecord" && + url.searchParams.get("collection") === "org.tangled.git.refs" + ) { + return Response.json({ + uri: "at://x", + value: { $type: "org.tangled.git.refs", defaultBranch: "main", refs: [] }, + }); + } + if ( + url.pathname === "/xrpc/org.tangled.git.listCommits" && + url.searchParams.get("ref") === "refs/heads/main" + ) { + return Response.json({ commits: [{ hash: sha }] }); + } return new Response("unexpected", { status: 599 }); } @@ -322,7 +357,12 @@ describe("Runner", () => { vi.restoreAllMocks(); }); - async function runner(name: string, workflow = raw, trusted = false) { + async function runner( + name: string, + workflow = raw, + trusted = false, + params: Partial = {}, + ) { const stub = exports.Runner.getByName(name); await runInDurableObject(stub, async (instance) => { instance.container = fake; @@ -333,11 +373,185 @@ describe("Runner", () => { commit: "abc123", raw: workflow, trusted, + ...params, }); }); return stub; } + async function succeed(stub: DurableObjectStub) { + await runDurableObjectAlarm(stub); + fake.polls.push("1 0 0 0 0\n"); + await runDurableObjectAlarm(stub); + } + + const mainHead = { branch: "main", commit: sha }; + const saved = { id: "snap-old", image: "cloudflare/debian-trixie", tier: "public" as const }; + + it("saves a clean snapshot after a trusted run at the default branch head", async () => { + const stub = await runner("snapshot", raw, true, mainHead); + await succeed(stub); + expect(await stub.status()).toMatchObject({ + status: "success", + snapshot: { + id: "snap-1", + image: "cloudflare/debian-trixie", + tier: "public", + }, + }); + const wipe = fake.execs.at(-1)?.cmd.join(" ") ?? ""; + expect(wipe).toContain("rm -rf /treadle /workspace && find /tmp -mindepth 1 -delete"); + expect(fake.destroyed).toBe(1); + expect( + await runInDurableObject(stub, (_instance, state) => state.storage.kv.get("reported")), + ).toBe("success snap-1"); + expect((await frames(stub)).slice(-3)).toMatchObject([ + { step: 2, status: "start", content: "Save snapshot", kind: "system" }, + { step: 2, content: "treadle: saved snapshot snap-1 (1000 bytes)\n" }, + { step: 2, status: "end", outcome: "success" }, + ]); + }); + + it.each([ + ["an untrusted run", raw, false, mainHead], + ["another branch", raw, true, { branch: "feature", commit: sha }], + ["a commit behind the head", raw, true, { ...mainHead, commit: "a".repeat(40) }], + ["a run from an older Store", raw, true, { commit: sha }], + ["cache: false", `${raw}treadle: {cloudflare: {cache: false}}\n`, true, mainHead], + ])("saves no snapshot after %s", async (_, workflow, trusted, params) => { + const stub = await runner(`no-snapshot-${_}`, workflow, trusted, params); + await succeed(stub); + expect(await stub.status()).toMatchObject({ status: "success" }); + expect((await stub.status())?.snapshot).toBeUndefined(); + expect(fake.snapshots).toBe(0); + }); + + it("saves no snapshot after a failed run", async () => { + const stub = await runner("failed-snapshot", raw, true, mainHead); + await runDurableObjectAlarm(stub); + fake.polls.push("1 1 0 0 0\n"); + await runDurableObjectAlarm(stub); + expect(await stub.status()).toMatchObject({ status: "failed" }); + expect(fake.snapshots).toBe(0); + }); + + it("saves the trusted snapshot after a run with secrets or from a trusted snapshot", async () => { + await exports.Secrets.getByName(repo).add("API_TOKEN", "s3cret", "did:plc:owner"); + const secret = await runner("tier-secret", secretWorkflow, true, mainHead); + await succeed(secret); + expect(await secret.status()).toMatchObject({ snapshot: { tier: "trusted" } }); + const restored = await runner("tier-restored", raw, true, { + ...mainHead, + restore: { ...saved, tier: "trusted" }, + }); + await succeed(restored); + expect(await restored.status()).toMatchObject({ snapshot: { tier: "trusted" } }); + }); + + it("saves the public snapshot after a trusted snapshot did not start", async () => { + fake.badSnapshot = true; + const stub = await runner("tier-fallback", raw, true, { + ...mainHead, + restore: { ...saved, tier: "trusted" }, + }); + await runDurableObjectAlarm(stub); + await succeed(stub); + expect(await stub.status()).toMatchObject({ snapshot: { tier: "public" } }); + }); + + it("keeps the success when the snapshot fails", async () => { + fake.snapshotError = new Error("disk too large"); + const stub = await runner("snapshot-error", raw, true, mainHead); + await succeed(stub); + expect(await stub.status()).toMatchObject({ status: "success" }); + expect((await stub.status())?.snapshot).toBeUndefined(); + expect((await frames(stub)).slice(-2)).toMatchObject([ + { step: 2, content: "treadle: no snapshot: disk too large\n" }, + { step: 2, status: "end", outcome: "failed" }, + ]); + expect(fake.destroyed).toBe(1); + }); + + it("reports the success before the snapshot ends", async () => { + fake.snapshotHangs = true; + const stub = await runner("snapshot-slow", raw, true, mainHead); + await runDurableObjectAlarm(stub); + fake.polls.push("1 0 0 0 0\n"); + const alarm = runDurableObjectAlarm(stub); + await vi.waitFor(async () => { + expect( + await runInDurableObject(stub, (_instance, state) => + state.storage.kv.get("reported"), + ), + ).toBe("success "); + }); + expect(fake.destroyed).toBe(0); + fake.snapshotHangs = false; + await alarm; + expect(fake.destroyed).toBe(1); + }); + + it("ends the snapshot step after a restart during the snapshot", async () => { + const stub = await runner("snapshot-restart", raw, true, mainHead); + await succeed(stub); + fake.running = true; + await runInDurableObject(stub, async (_instance, state) => { + state.storage.kv.put("snapshotStep", 2); + await state.storage.setAlarm(Date.now()); + }); + await runDurableObjectAlarm(stub); + expect(fake.destroyed).toBe(2); + expect((await frames(stub)).slice(-2)).toMatchObject([ + { step: 2, content: "treadle: no snapshot: the Runner restarted\n" }, + { step: 2, status: "end", content: "Save snapshot", outcome: "failed" }, + ]); + }); + + it("starts from a snapshot of the same image", async () => { + const stub = await runner("restore", raw, false, { restore: saved }); + await runDurableObjectAlarm(stub); + expect(fake.starts).toMatchObject([{ containerSnapshot: { id: "snap-old" } }]); + expect(fake.starts[0]).not.toHaveProperty("image"); + expect(await frames(stub)).toMatchObject([ + { step: 0, status: "start" }, + { step: 0, content: "treadle: started from snapshot snap-old\n" }, + { step: 0, status: "end", outcome: "success" }, + ]); + }); + + it.each([ + ["another image", raw, { ...saved, image: "cloudflare/debian-bookworm" }], + ["the trusted tier in an untrusted run", raw, { ...saved, tier: "trusted" as const }], + ["cache: false", `${raw}treadle: {cloudflare: {cache: false}}\n`, saved], + ])("starts from the image with a snapshot of %s", async (_, workflow, snapshot) => { + const stub = await runner(`ignore-${_}`, workflow, false, { restore: snapshot }); + await runDurableObjectAlarm(stub); + expect(fake.starts).toMatchObject([{ image: "cloudflare/debian-trixie" }]); + expect(fake.starts[0]).not.toHaveProperty("containerSnapshot"); + }); + + it("starts from the image when the snapshot does not start", async () => { + fake.badSnapshot = true; + const stub = await runner("expired", raw, false, { restore: saved }); + await runDurableObjectAlarm(stub); + expect(await stub.status()).toMatchObject({ status: "started" }); + expect(fake.destroyed).toBe(1); + await runDurableObjectAlarm(stub); + expect(await stub.status()).toMatchObject({ status: "running" }); + expect(fake.starts).toMatchObject([ + { containerSnapshot: { id: "snap-old" } }, + { image: "cloudflare/debian-trixie" }, + ]); + expect(await frames(stub)).toMatchObject([ + { step: 0, status: "start" }, + { + step: 0, + content: "treadle: snapshot snap-old did not start (snapshot not found)\n", + }, + { step: 0, status: "end", outcome: "success" }, + ]); + }); + it("checks out the commit, runs each step and reports success", async () => { const stub = await runner( "success", diff --git a/test/store.test.ts b/test/store.test.ts index 816aca7..015532b 100644 --- a/test/store.test.ts +++ b/test/store.test.ts @@ -58,7 +58,7 @@ describe("migrate", () => { (_instance, state) => { const { sql, kv } = state.storage; sql.exec( - "DROP TABLE workflow_runs; DROP TABLE pipelines; DROP TABLE service_tokens; DROP TABLE IF EXISTS pulls", + "DROP TABLE workflow_runs; DROP TABLE pipelines; DROP TABLE service_tokens; DROP TABLE IF EXISTS pulls; DROP TABLE IF EXISTS snapshots", ); kv.delete("schemaVersion"); sql.exec(firstSchema); @@ -89,9 +89,9 @@ describe("migrate", () => { }, ); expect(rows).toEqual({ - version: 5, + version: 6, indexes: [{ name: "pipelines_repo" }, { name: "workflow_runs_build" }], - tables: ["pipelines", "pulls", "service_tokens", "workflow_runs"], + tables: ["pipelines", "pulls", "service_tokens", "snapshots", "workflow_runs"], pipelines: [{ rkey: "3aaaaaaaaaaa2", released_at: null, message: null }], runs: [ { @@ -310,6 +310,7 @@ describe("Cloudflare runs", () => { commit: "abc", raw: "engine: cloudflare\nsteps: [{name: a, command: b}]\n", trusted: true, + branch: "main", }); expect(runners.params.get(`${rkeys[1]}/ci.yml`)?.trusted).toBe(false); expect(await status(rkeys[10] ?? "")).toBe("pending"); @@ -321,6 +322,50 @@ describe("Cloudflare runs", () => { expect(await status(rkeys[10] ?? "")).toBe("cancelled"); }); + it("keeps the newest snapshot of each tier from trusted runs and gives it out", async () => { + const { store, runners, create } = await setup("runners-snapshots"); + const [older, newer, untrusted] = [ + await create(true), + await create(true), + await create(false), + ]; + await runDurableObjectAlarm(store); + const image = "cloudflare/debian-trixie"; + const report = (rkey: string, id: string, tier: "public" | "trusted", ts: number) => + store.updateRunner( + rkey, + "ci.yml", + { status: "success", snapshot: { id, image, tier } }, + ts, + ); + await report(newer, "public-new", "public", 1); + await report(older, "public-old", "public", 2); + await report(untrusted, "untrusted", "public", 3); + const [trusted, publicRun] = [await create(true), await create(false)]; + await runDurableObjectAlarm(store); + const given = (rkey: string) => runners.params.get(`${rkey}/ci.yml`)?.restore; + expect(given(trusted)).toEqual({ id: "public-new", image, tier: "public" }); + expect(given(publicRun)).toEqual({ id: "public-new", image, tier: "public" }); + expect(given(older)).toBeUndefined(); + + await report(trusted, "trusted-1", "trusted", 4); + const [second, third] = [await create(true), await create(false)]; + await runInDurableObject(store, (_instance, state) => { + state.storage.kv.put("buildWorkAt", 0); + }); + await runDurableObjectAlarm(store); + expect(given(second)).toEqual({ id: "trusted-1", image, tier: "trusted" }); + expect(given(third)).toEqual({ id: "public-new", image, tier: "public" }); + + await report(second, "public-2", "public", 5); + const fourth = await create(true); + await runInDurableObject(store, (_instance, state) => { + state.storage.kv.put("buildWorkAt", 0); + }); + await runDurableObjectAlarm(store); + expect(given(fourth)).toEqual({ id: "public-2", image, tier: "public" }); + }); + it("tries a start again after an error", async () => { const { store, runners, create, status } = await setup("runners-retry"); runners.startError = new Error("overloaded"); -- knotmirror 0.0.1