import { DurableObject } from "cloudflare:workers"; import type { RunStatus } from "./buildkite"; import type { RunnerState, Snapshot } from "./store"; import { capMarker, controlFrame, dataFrame, frameLine, logCap, logLines } from "./logs"; import type { cleanLine } from "./logs"; import { defaultBranch, headCommit, resolveKnot } from "./notify"; import { secretForms, secretMask } from "./secrets"; import { parseWorkflow } from "./workflow"; import type { CloudflareJob, Step } from "./workflow"; import { errorMessage, xrpcUrl } from "./xrpc"; function quote(word: string): string { return `'${word.replaceAll("'", "'\\''")}'`; } // Container logs keep only what the entrypoint prints, so it follows this debug log. The lines // never carry commands, environment values or step output. const debug = 'd() { echo "treadle: $*" >> /treadle/debug.log 2>/dev/null || true; }'; // The CA of the intercepts. The ca-certificates package also trusts it once installed. const caFile = "/etc/cloudflare/certs/cloudflare-containers-ca.crt"; const caStore = "/usr/local/share/ca-certificates"; // Steps write their output and exit codes to files, so a restarted Runner can read them again. // The workflow environment applies to the steps only, so that it cannot change this script. export function runScript(job: Pick): string { const lines = [ "exec > /treadle/run.log 2>&1 < /dev/null", debug, "trap 'd script exit $?' EXIT", "echo $$ > /treadle/pid", "mkdir -p /treadle/log", `if [ -e ${caFile} ]; then mkdir -p ${caStore} && cp ${caFile} ${caStore}/ && if command -v update-ca-certificates > /dev/null; then update-ca-certificates > /dev/null; fi; fi`, "cd /workspace || exit 1", ]; job.steps.forEach((step, i) => { const log = `/treadle/log/${i + 1}`; const env = Object.entries({ ...job.environment, ...step.environment }).map( ([k, v]) => ` ${k}=${quote(v)}`, ); const name = quote(step.name.replaceAll(/[\p{Cc}\u2028\u2029]/gu, " ")); lines.push( `d step ${i + 1} ${name} start; t=$SECONDS`, `env${env.join("")} bash -eo pipefail -c -- ${quote(step.command)} > ${log} 2>&1 < /dev/null`, `rc=$?; echo $rc > ${log}.tmp && mv ${log}.tmp ${log}.exit`, `d step ${i + 1} exit $rc after $((SECONDS - t))s`, '[ "$rc" = 0 ] || exit 1', ); }); return `${lines.join("\n")}\n`; } // Prints " ", a newline, then up to limit new bytes // of the log. Liveness is read before the exit file, so "-" with 0 means the script died. The // exit file is read before the size, so a finished step is done once its offset reaches the size. // A step may replace these files with FIFOs, so the poll is time-bound. export function pollCommand(step: number, offset: number, limit: number): string[] { return [ "timeout", "10", "bash", "-c", [ `f=/treadle/log/${step}`, "p=$(cat /treadle/pid 2>/dev/null)", "st=$(sed 's/.*) //' /proc/$p/stat 2>/dev/null | cut -c1)", '[ -n "$p" ] && [ -n "$st" ] && [ "$st" != Z ] && a=1 || a=0', "e=$(cat $f.exit 2>/dev/null || echo -)", "if [ -e $f ]; then s=1; z=$(stat -c %s $f); else s=0; z=0; fi", "c=$(mktemp)", `tail -c +${offset + 1} $f 2>/dev/null | head -c ${limit} > $c`, 'echo "$s $e $z $(stat -c %s $c) $a"', "cat $c; rm -f $c", ].join("\n"), ]; } export interface Poll { started: boolean; exit: number | undefined; size: number; chunk: Uint8Array; alive: boolean; } export function parsePoll(output: Uint8Array): Poll { const newline = output.indexOf(10); const header = new TextDecoder().decode(output.subarray(0, newline)); const match = newline === -1 ? null : /^([01]) (-|\d+) (\d+) (\d+) ([01])$/.exec(header); const chunk = output.subarray(newline + 1); if (match === null || Number(match[4]) !== chunk.length) { throw new Error(`unreadable poll output: ${header.slice(0, 100)}`); } return { started: match[1] === "1", exit: match[2] === "-" ? undefined : Number(match[2]), size: Number(match[3]), chunk, alive: match[5] === "1", }; } export interface Cursor { step: number; offset: number; started: boolean; capped: boolean; } export type StepEvent = | { kind: "start" } | { kind: "lines"; lines: ReturnType[] } | { kind: "capped" } | { kind: "end"; exit: number }; // Leaves a possible secret at the end of a chunk for the next poll, so the mask sees it whole. function edgeCut(chunk: Uint8Array, secrets: Uint8Array[]): number { let cut = chunk.length; for (const secret of secrets) { for (let start = Math.max(1, chunk.length - secret.length + 1); start < cut; start++) { if (secret.subarray(0, chunk.length - start).every((b, i) => chunk[start + i] === b)) { cut = start; break; } } } return cut; } // Turns one poll of the current step into log events. The step ends once its whole log is read. export function readPoll( current: Cursor, poll: Poll, secrets: Uint8Array[] = [], ): { cursor: Cursor; events: StepEvent[] } { const cursor = { ...current }; const events: StepEvent[] = []; if (poll.started && !cursor.started) { cursor.started = true; events.push({ kind: "start" }); } if (!cursor.capped) { const final = poll.exit !== undefined && cursor.offset + poll.chunk.length >= poll.size; const pending = cursor.offset + poll.chunk.length < poll.size; let { lines, used } = logLines(poll.chunk, final); // A full chunk without a newline would otherwise never move the cursor. if (used === 0 && pending) { ({ lines, used } = logLines( poll.chunk.subarray(0, edgeCut(poll.chunk, secrets)), true, )); } cursor.offset += used; if (lines.length > 0) { events.push({ kind: "lines", lines }); } if (cursor.offset >= logCap) { cursor.capped = true; events.push({ kind: "capped" }); } } if (poll.exit !== undefined && (cursor.capped || cursor.offset >= poll.size)) { events.push({ kind: "end", exit: poll.exit }); } return { cursor, events }; } // Installs the apt packages as the first step, so it logs, fails and survives a restart as steps do. function withDependencies(job: CloudflareJob | undefined): CloudflareJob | undefined { if (job === undefined || job.apt.length === 0) { return job; } const install: Step = { name: "Install dependencies", command: `apt-get update && apt-get install -y --no-install-recommends ${job.apt.map(quote).join(" ")}`, environment: { DEBIAN_FRONTEND: "noninteractive" }, kind: "system", }; return { ...job, steps: [install, ...job.steps] }; } export interface RunnerParams { pipeline: string; workflow: string; repo: string; commit: string; raw: string; trusted: boolean; branch: string; restore?: Snapshot; // A trusted run of a workflow that has saved snapshots. Older Stores leave it out. cached?: boolean; } type RunState = Omit & RunnerState & { job: CloudflareJob; phase: "boot" | "steps" | "done"; deadline: number; cursor: Cursor; launchedAt: number | undefined; // When each secret was set at launch. Runs stored by older code have none. secretsAt?: Record; // The container gets these in place of the injected secrets. placeholders?: Record; }; const image = "cloudflare/debian-trixie"; const pollMs = 3000; const retryMs = 10_000; const graceMs = 30_000; const chunkBytes = 256 * 1024; const snapshotMs = 5 * 60_000; const execMs = 60_000; const stopMs = 30_000; const maxInactivityMs = 6 * 3_600_000; class Stalled extends Error {} // A step runs as root and can replace the tools that an exec uses, so no exec waits forever. async function within( promise: Promise, ms: number, message = "the container did not answer", ): Promise { let timer: ReturnType | null = null; try { return await Promise.race([ promise, new Promise((_resolve, reject) => { timer = setTimeout(() => { reject(new Stalled(message)); }, ms); }), ]); } finally { clearTimeout(timer); } } // Kills what the steps left running, then removes the logs, the checkout and temporary files, so // that no snapshot holds them. const wipeCommand = [ "timeout", "120", "bash", "-c", 'p=$(cat /treadle/pid 2>/dev/null); [ -n "$p" ] && kill -KILL -- "-$p" 2>/dev/null; ' + "rm -rf /treadle /workspace && find /tmp -mindepth 1 -delete", ]; const checkoutCommand = "[ -e /treadle/started ] && exit 0; rm -rf /workspace && mkdir -p /workspace /treadle/log && tar -xz --strip-components=1 -C /workspace; " + 'rc=$?; echo "treadle: checkout exit $rc" >> /treadle/debug.log 2>/dev/null; exit $rc'; const launchCommand = "mkdir /treadle/started 2>/dev/null || exit 0; cat > /treadle/run.sh && { setsid bash /treadle/run.sh > /dev/null 2>&1 < /dev/null & }"; export type Box = Pick< Container, | "running" | "start" | "monitor" | "destroy" | "exec" | "setInactivityTimeout" | "snapshotContainer" | "signal" | "interceptOutboundHttps" >; function state({ status, startedAt, finishedAt, error, snapshot, discardCache, }: RunState): RunnerState { return { status, startedAt, finishedAt, error, snapshot, discardCache }; } function withoutRestore(run: RunState): RunState { const { restore: _, ...rest } = run; return rest; } // Runs one engine: cloudflare workflow run in its own Container. The alarm drives every phase and // reads all progress from files in the container, so a restarted Runner continues the run. export class Runner extends DurableObject { container: Box | undefined = this.ctx.container; // Secret values stay in memory only, so a restarted Runner reads them again. mask: ((text: string) => string) | undefined; secrets: Uint8Array[] = []; // An alarm in flight may still start a container or set its alarm for the last run. #alarming = false; constructor(ctx: DurableObjectState, env: Env) { super(ctx, env); const { sql, kv } = ctx.storage; sql.exec("CREATE TABLE IF NOT EXISTS frames (id INTEGER PRIMARY KEY, frame BLOB NOT NULL)"); // Older code kept one run in "run". if (kv.get("schemaVersion") === undefined) { ctx.storage.transactionSync(() => { sql.exec(`ALTER TABLE frames ADD COLUMN pipeline TEXT; CREATE INDEX frames_pipeline ON frames (pipeline, id)`); const legacy = kv.get("run"); if (legacy !== undefined) { sql.exec("UPDATE frames SET pipeline = ?", legacy.pipeline); kv.put(`run/${legacy.pipeline}`, legacy); kv.put("current", legacy.pipeline); kv.delete("run"); } kv.put("schemaVersion", 1); }); } } // Returns "busy" while the last run is not done or still stopping, so the Store asks again. async start(params: RunnerParams): Promise<"busy" | undefined> { if (this.#run(params.pipeline) !== undefined) { return undefined; } const last = this.#run(); if ( (last !== undefined && last.phase !== "done") || this.container?.running === true || this.#alarming ) { return "busy"; } this.mask = undefined; const { raw, ...rest } = params; let job: CloudflareJob | undefined; let error = `${params.workflow} is not an engine: cloudflare workflow`; try { job = withDependencies(parseWorkflow(params.workflow, raw).cloudflare); } catch (e) { error = errorMessage(e); } const now = Date.now(); const placeholders = Object.fromEntries( (params.trusted ? (job?.injected ?? []) : []).map(({ name }) => [ name, crypto.getRandomValues(new Uint8Array(32)).toHex(), ]), ); const fresh = { ...rest, placeholders, cursor: { step: 1, offset: 0, started: false, capped: false }, launchedAt: undefined, }; // A run that cannot start fails here, so the Store only retries a start that did not arrive. this.#save( job === undefined ? { ...fresh, job: noJob, status: "failed", error, finishedAt: new Date(now).toISOString(), phase: "done", deadline: now, } : { ...fresh, job, status: "started", phase: "boot", deadline: now + job.timeoutMs, }, ); this.ctx.storage.kv.put("current", params.pipeline); await this.ctx.storage.setAlarm(now); return undefined; } status(pipeline: string): RunnerState | undefined { const run = this.#run(pipeline); return run === undefined ? undefined : state(run); } // Returns the log of a run as text, as the log stream shows it. readLog(pipeline: string, step?: number, tail = Infinity, timestamps = false): string[] { const lines = []; for (const { frame } of this.ctx.storage.sql.exec<{ frame: ArrayBuffer }>( "SELECT frame FROM frames WHERE pipeline = ? ORDER BY id", pipeline, )) { const line = frameLine(new Uint8Array(frame)); if (line !== undefined && (step === undefined || line.step === step)) { lines.push(timestamps ? `${line.time} ${line.text}` : line.text); } } return lines.slice(-tail); } // The alarm reports the end, also when the alarms had stopped. async cancel(pipeline?: string, status: "cancelled" | "timeout" = "cancelled"): Promise { const run = this.#run(pipeline); if (run !== undefined && run.phase !== "done") { await this.#finish(run, status, status === "timeout" ? "the run timed out" : undefined); if ((await this.ctx.storage.getAlarm()) === null) { await this.ctx.storage.setAlarm(Date.now()); } } } override async alarm(): Promise { this.#alarming = true; try { await this.#alarm(); } finally { this.#alarming = false; } } async #alarm(): Promise { if (this.#run()?.phase === "done") { this.#expire(); } let run = this.#run(); if (run === undefined) { return; } if (run.phase === "done") { // The Runner restarted during the snapshot. const step = this.ctx.storage.kv.get("snapshotStep"); if (step !== undefined) { this.#line(run, step, "no snapshot: the Runner restarted"); this.#emit( run, controlFrame( run.workflow, snapshotStep(step), "end", new Date().toISOString(), "success", ), ); this.ctx.storage.kv.delete("snapshotStep"); for (const ws of this.ctx.getWebSockets(run.pipeline)) { ws.close(1000, "log stream complete"); } } await this.#destroy(run); } else { try { run = await this.#tick(run); } catch (error) { console.warn("runner tick failed", { pipeline: run.pipeline, workflow: run.workflow, error: errorMessage(error), }); // A cancel may have finished the run meanwhile. run = this.#run(run.pipeline) ?? run; if (error instanceof Stalled) { run = await this.#finish(run, "failed", error.message); } } } const reported = await this.#report(run); if (run.phase !== "done") { await this.ctx.storage.setAlarm(Date.now() + (reported ? pollMs : retryMs)); } else if (!reported || this.container?.running === true) { await this.ctx.storage.setAlarm(Date.now() + retryMs); } else { await this.ctx.storage.setAlarm( (Date.parse(run.finishedAt ?? "") || Date.now()) + this.#retentionMs(), ); } } #retentionMs(): number { return (Number(this.env.LOG_RETENTION_DAYS) || 30) * 86_400_000; } #expire(): void { const { kv, sql } = this.ctx.storage; const cutoff = Date.now() - this.#retentionMs(); for (const [key, run] of kv.list({ prefix: "run/" })) { if (run.phase === "done" && Date.parse(run.finishedAt ?? "") < cutoff) { kv.delete(key); sql.exec("DELETE FROM frames WHERE pipeline = ?", run.pipeline); } } } override fetch(request: Request): Response { const params = new URL(request.url).searchParams; const pipeline = params.get("pipeline") ?? ""; const { 0: client, 1: server } = new WebSocketPair(); this.ctx.acceptWebSocket(server, [pipeline]); for (const { frame } of this.ctx.storage.sql.exec<{ frame: ArrayBuffer }>( "SELECT frame FROM frames WHERE pipeline = ? ORDER BY id", pipeline, )) { server.send(frame); } if (this.#run(pipeline)?.phase === "done" || params.has("done")) { server.close(1000, "log stream complete"); } return new Response(null, { status: 101, webSocket: client }); } #run(pipeline = this.ctx.storage.kv.get("current")): RunState | undefined { return pipeline === undefined ? undefined : this.ctx.storage.kv.get(`run/${pipeline}`); } // A cancel can finish the run while an alarm awaits, so a finished run is never overwritten. #save(run: RunState): RunState { const stored = this.#run(run.pipeline); if (stored?.phase === "done") { return stored; } this.ctx.storage.kv.put(`run/${run.pipeline}`, run); return run; } async #tick(run: RunState): Promise { const container = this.container; if (Date.now() >= run.deadline) { return await this.#finish(run, "timeout", "the run timed out"); } if (container === undefined) { return await this.#finish(run, "failed", "no container is available"); } if (run.phase === "boot") { return await this.#boot(run, container); } if (!container.running) { return await this.#finish(run, "failed", "the container stopped"); } return await this.#poll(run, container); } // start() throws only for some invalid options. The first exec waits until the container // runs, and monitor() gives the reason when it does not. async #boot(current: RunState, container: Box): Promise { // A snapshot of another image, or a trusted one in an untrusted run, starts from the image. const { restore } = current; const run = restore === undefined || (current.job.cache && restore.image === image && (restore.tier === "public" || current.trusted)) ? current : this.#save(withoutRestore(current)); const { values: secrets, unset, setAt } = await this.#secrets(run); if (unset.length > 0) { return await this.#finish(run, "failed", `secrets are not set: ${unset.join(", ")}`); } await this.#intercept(run, container, secrets); if (!container.running) { // oxlint-disable-next-line typescript/no-unsafe-type-assertion -- the runtime checks the instance const instance = run.job.instance as NonNullable; try { container.start({ ...(run.restore === undefined ? { image } : { containerSnapshot: { id: run.restore.id } }), // bash stays PID 1 of the container, so it reaps orphaned step processes. entrypoint: [ "bash", "-c", "trap 'exit 143' TERM; mkdir -p /treadle && touch /treadle/debug.log && { tail -F /treadle/debug.log & wait $!; }", ], enableInternet: true, instance, }); } catch (error) { return await this.#bootFailed(run, errorMessage(error)); } } const monitor = container.monitor().then( () => "the container stopped", (error: unknown) => errorMessage(error), ); const knot = await resolveKnot(run.repo); const archiveAbort = new AbortController(); const bootBy = Date.now() + 10 * 60_000; const deadline = setTimeout(() => { archiveAbort.abort(new DOMException("The operation timed out.", "TimeoutError")); }, 5 * 60_000); let exit: number; try { const archive = await fetch( xrpcUrl(knot, "org.tangled.git.getArchive", { repo: run.repo, ref: run.commit, format: "tar.gz", prefix: "src", }), { redirect: "manual", signal: archiveAbort.signal }, ); if (!archive.ok || archive.body === null) { await archive.body?.cancel(); const error = `archive unavailable: HTTP ${archive.status}`; if ( archive.status >= 400 && archive.status < 500 && ![408, 429].includes(archive.status) ) { return await this.#finish(run, "failed", error); } throw new Error(error); } this.#startCheckout(run); try { exit = await within( Promise.race([ container .exec(["bash", "-c", checkoutCommand], { stdin: archive.body, stderr: "combined", }) .then((checkout) => checkout.exitCode), monitor.then((reason) => Promise.reject(new Error(reason))), ]), bootBy - Date.now(), ); } catch (error) { archiveAbort.abort(); const reason = await within(monitor, 1000).catch(() => null); return await this.#bootFailed(run, reason ?? errorMessage(error)); } } finally { clearTimeout(deadline); // Ends an archive download that the checkout left unread. archiveAbort.abort(); } if (exit !== 0) { return await this.#bootFailed(run, "checkout failed"); } if (run.restore !== undefined) { this.#line(run, 0, `started from snapshot ${run.restore.id}`); } this.#emit(run, this.#end(run, checkoutStep.step, "success")); await container.setInactivityTimeout( Math.min(run.deadline - Date.now() + 10 * 60_000, maxInactivityMs), ); const launch = container.exec(["bash", "-c", launchCommand], { stdin: new Blob([runScript(run.job)]).stream(), env: { ...Object.fromEntries( Object.entries(secrets).filter(([name]) => run.job.secrets.includes(name)), ), ...run.placeholders, ...(Object.keys(run.placeholders ?? {}).length > 0 ? { NODE_EXTRA_CA_CERTS: caFile } : {}), PATH: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", HOME: "/root", CI: "true", TREADLE_REPO: run.repo, TREADLE_COMMIT: run.commit, TREADLE_PIPELINE: run.pipeline, TREADLE_WORKFLOW: run.workflow, }, }); if ( (await within( launch.then((process) => process.exitCode), execMs, )) !== 0 ) { return await this.#finish(run, "failed", "the steps did not start"); } return this.#save({ ...run, phase: "steps", status: "running", startedAt: run.startedAt ?? new Date().toISOString(), launchedAt: Date.now(), secretsAt: setAt, }); } async #poll(current: RunState, container: Box): Promise { if (this.mask === undefined) { const { values, unset, setAt } = await this.#secrets(current); const { secretsAt } = current; const changed = secretsAt === undefined ? [] : Object.keys(setAt).filter((name) => setAt[name] !== secretsAt[name]); if (unset.length > 0) { return await this.#finish( current, "failed", `secrets are not set: ${unset.join(", ")}`, ); } if (changed.length > 0) { return await this.#finish( current, "failed", `secrets changed: ${changed.join(", ")}`, ); } await this.#intercept(current, container, values).catch((error: unknown) => { this.mask = undefined; throw error; }); } const mask = this.mask ?? ((text: string) => text); let run = current; const { cursor } = run; const output = await within( container .exec(pollCommand(cursor.step, cursor.offset, cursor.capped ? 0 : chunkBytes)) .then((process) => process.output()), execMs, ); const poll = parsePoll(new Uint8Array(output.stdout)); const read = readPoll(cursor, poll, this.secrets); const step = run.job.steps[cursor.step - 1]; const info = { step: cursor.step, name: step?.name ?? "", command: step?.command, kind: step?.kind ?? ("user" as const), }; let last = new Date().toISOString(); let exit: number | undefined; for (const event of read.events) { if (event.kind === "start") { this.#emit(run, controlFrame(run.workflow, info, "start", last)); } else if (event.kind === "lines") { for (const line of event.lines) { last = line.time ?? last; this.#emit( run, dataFrame(run.workflow, cursor.step, `${mask(line.text)}\n`, last), ); } } else if (event.kind === "capped") { this.#emit(run, dataFrame(run.workflow, cursor.step, `${capMarker}\n`, last)); } else { const outcome = event.exit === 0 ? "success" : "failed"; this.#emit(run, controlFrame(run.workflow, info, "end", last, outcome)); exit = event.exit; } } if (exit !== undefined && exit !== 0) { return await this.#finish(run, "failed", `${info.name} failed`); } if (exit === 0) { return cursor.step >= run.job.steps.length ? await this.#finish(run, "success") : this.#save({ ...run, cursor: { step: cursor.step + 1, offset: 0, started: false, capped: false }, }); } run = this.#save({ ...run, cursor: read.cursor }); if ( poll.exit === undefined && !poll.alive && Date.now() - (run.launchedAt ?? 0) > graceMs ) { return await this.#finish(run, "failed", "the step script stopped"); } return run; } // Only trusted runs get secrets, as upstream. A secret removed or replaced during the run // fails it, because the mask would miss the old value. The times are read before the values, // so a secret replaced in between reads as changed. async #secrets(run: RunState): Promise<{ values: Record; unset: string[]; setAt: Record; }> { const names = run.trusted ? [...run.job.secrets, ...(run.job.injected ?? []).map(({ name }) => name)] : []; let values: Record = {}; let setAt: Record = {}; if (names.length > 0) { const secrets = this.ctx.exports.Secrets.getByName(run.repo); setAt = Object.fromEntries( (await secrets.list()) .filter(({ key }) => names.includes(key)) .map(({ key, createdAt }) => [key, createdAt]), ); values = await secrets.values(names); } this.mask = secretMask(Object.values(values)); this.secrets = secretForms(Object.values(values)).map((form) => new TextEncoder().encode(form), ); return { values, setAt, unset: names.filter((name) => !(name in values)) }; } // Intercepts last as long as the container, so each new container needs them again. async #intercept(run: RunState, container: Box, values: Record): Promise { const secrets = (run.job.injected ?? []).flatMap(({ name, hosts, headers }) => { const placeholder = run.placeholders?.[name]; const value = values[name]; return placeholder === undefined || value === undefined ? [] : [{ placeholder, value, hosts, headers }]; }); const proxy = this.ctx.exports.SecretProxy({ props: { secrets } }); const hosts = new Set(secrets.flatMap((secret) => secret.hosts)); await Promise.all([...hosts].map((host) => container.interceptOutboundHttps(host, proxy))); } #end(run: RunState, step: number, outcome: string): Uint8Array { return controlFrame( run.workflow, { ...checkoutStep, step }, "end", new Date().toISOString(), outcome, ); } #startCheckout(run: RunState): void { if (!this.#started(run)) { this.#emit( run, controlFrame(run.workflow, checkoutStep, "start", new Date().toISOString()), ); } } // An expired snapshot does not start, so the run starts again from the image. async #bootFailed(run: RunState, error: string): Promise { if (run.restore === undefined) { if (this.#started(run)) { this.#emit(run, this.#end(run, checkoutStep.step, "failed")); } return await this.#finish(run, "failed", error); } this.#startCheckout(run); this.#line(run, 0, `snapshot ${run.restore.id} did not start (${error})`); await this.#destroy(run); return this.#save(withoutRestore(run)); } #started(run: RunState): boolean { return ( this.ctx.storage.sql .exec("SELECT 1 FROM frames WHERE pipeline = ? LIMIT 1", run.pipeline) .toArray().length > 0 ); } #line(run: RunState, step: number, text: string): void { this.#emit( run, dataFrame(run.workflow, step, `treadle: ${text}\n`, new Date().toISOString()), ); } // Only a trusted run at the head of the default branch changes the cache of its workflow. async #atHead(run: RunState): Promise { try { const knot = await resolveKnot(run.repo); return ( run.branch === (await defaultBranch(knot, run.repo)) && (await headCommit(knot, run.repo, `refs/heads/${run.branch}`))?.toLowerCase() === run.commit.toLowerCase() ); } catch (error) { console.warn("branch head unavailable", { repo: run.repo, error: errorMessage(error) }); return false; } } async #discardCache(run: RunState): Promise { if (!run.trusted || run.job.cache || run.cached !== true) { return run; } await this.#report(run); if (!(await this.#atHead(run))) { return run; } const next = { ...run, discardCache: true as const }; this.ctx.storage.kv.put(`run/${run.pipeline}`, next); return next; } // The Store already has the success, so a slow snapshot does not hold it back. async #saveSnapshot(run: RunState, container: Box): Promise { if (!run.trusted || !run.job.cache) { return run; } await this.#report(run); if (!(await this.#atHead(run))) { return run; } const info = snapshotStep(run.job.steps.length + 1); this.#emit(run, controlFrame(run.workflow, info, "start", new Date().toISOString())); this.ctx.storage.kv.put("snapshotStep", info.step); let saved: Snapshot | undefined; try { const wipe = container.exec(wipeCommand).then((process) => process.exitCode); if ((await within(wipe, 150_000)) !== 0) { throw new Error("the cleanup failed"); } const snapshot = await within( container.snapshotContainer(), snapshotMs, "the snapshot timed out", ); // Secrets, responses from secret hosts, or a trusted snapshot may have left files that // untrusted runs must not read. const tier = run.job.secrets.length > 0 || (run.job.injected?.length ?? 0) > 0 || run.restore?.tier === "trusted" ? "trusted" : "public"; saved = { id: snapshot.id, image, tier }; this.#line(run, info.step, `saved snapshot ${snapshot.id} (${snapshot.size} bytes)`); } catch (error) { this.#line(run, info.step, `no snapshot: ${errorMessage(error)}`); } // A missing snapshot only makes the next run slower, so it never fails the build. this.#emit( run, controlFrame(run.workflow, info, "end", new Date().toISOString(), "success"), ); this.ctx.storage.kv.delete("snapshotStep"); const next = { ...run, snapshot: saved }; this.ctx.storage.kv.put(`run/${run.pipeline}`, next); return next; } async #finish(run: RunState, status: RunStatus, error?: string): Promise { let done = this.#save({ ...run, phase: "done", status, error, finishedAt: new Date().toISOString(), }); if (done.status === "success" && !done.job.cache) { done = await this.#discardCache(done); } else if (done.status === "success" && this.container?.running === true) { done = await this.#saveSnapshot(done, this.container); } await this.#destroy(run); for (const ws of this.ctx.getWebSockets(run.pipeline)) { ws.close(1000, "log stream complete"); } return done; } // The alarm tries again while the container runs. async #destroy(run: RunState): Promise { const container = this.container; if (container?.running !== true) { return; } container.signal(15); await within(container.monitor(), stopMs).catch(() => {}); if (this.container?.running === true) { await container.destroy().catch((error: unknown) => { console.warn("container not destroyed", { pipeline: run.pipeline, workflow: run.workflow, error: errorMessage(error), }); }); } } // Returns false when the Store must hear about the run again. async #report(run: RunState): Promise { const key = `${run.pipeline} ${run.status} ${run.error ?? ""}${run.snapshot === undefined ? "" : ` ${run.snapshot.id}`}${run.discardCache === true ? " discard" : ""}`; if (this.ctx.storage.kv.get("reported") === key) { return true; } try { await this.ctx.exports.Store.getByName("main").updateRunner( run.pipeline, run.workflow, state(run), Math.floor(Date.now() / 1000), ); } catch (error) { console.warn("runner status not reported", { pipeline: run.pipeline, workflow: run.workflow, error: errorMessage(error), }); return false; } this.ctx.storage.kv.put("reported", key); return true; } #emit(run: RunState, frame: Uint8Array): void { this.ctx.storage.sql.exec( "INSERT INTO frames (pipeline, frame) VALUES (?, ?)", run.pipeline, frame, ); // A socket that the Runner closed stays listed until the viewer answers the close. for (const ws of this.ctx.getWebSockets(run.pipeline)) { if (ws.readyState === WebSocket.OPEN) { ws.send(frame); } } } } const noJob: CloudflareJob = { environment: {}, steps: [], instance: "standard-1", timeoutMs: 0, cache: false, secrets: [], apt: [], }; function snapshotStep(step: number) { return { step, name: "Save cache", command: undefined, kind: "system" as const }; } const checkoutStep = { step: 0, name: "Checkout", command: undefined, kind: "system" as const };