import { DurableObject } from "cloudflare:workers"; import { cancelBuild, createBuild, createBuildBody, getBuild, isTerminal } from "./buildkite"; import type { BuildState, Commit, RunStatus } from "./buildkite"; import { repoNames } from "./identity"; import { catchUp } from "./jetstream"; import { errorMessage, isDid, isObject, isPairList, string } from "./xrpc"; import type { JetstreamState } from "./jetstream"; import { admitter, handlePatch, handleTicket, patchCollection, ticketCollection } from "./pulls"; import type { Action, PullStore } from "./pulls"; import type { RunnerParams } from "./runner"; const tidAlphabet = "234567abcdefghijklmnopqrstuvwxyz"; const minuteMs = 60_000; const giveUpMs = 3 * 24 * 60 * minuteMs; const hourMs = 60 * minuteMs; const checkMs = 10 * minuteMs; const maxBackoffMs = hourMs; const tickMs = 10_000; const declaredMs = 10 * minuteMs; const maxRunners = 10; const maxHeldPerSender = 10; const heldMs = 14 * 24 * hourMs; // The longest workflow timeout, and time for the Runner to end the run. const maxRunMs = 6 * hourMs + 30 * minuteMs; export function tid(micros: number, clockId: number): string { let n = (BigInt(micros) << 10n) | BigInt(clockId & 0x3ff); let out = ""; for (let i = 0; i < 13; i++) { out = tidAlphabet.charAt(Number(n & 31n)) + out; n >>= 5n; } return out; } interface NewRun { engine: "buildkite" | "cloudflare"; trusted: boolean; workflow: string; raw: string; triggers: string[]; org: string; pipeline: string; cleanCheckout: boolean | undefined; // Older Stores ignore it and start the run at once. after?: string[]; error?: string; } export interface NewPipeline extends Partial { repo: string; trigger: object; pushKey: string | null; commit: string; branch: string; runs: NewRun[]; // The runs wait for Approve. held?: boolean; sender?: string; } export type OpenBuild = { workflow: string; status: string; bk_org: string; bk_pipeline: string; build_number: number; }; export type BuildUpdate = BuildState & { ts: number; pipeline: string | undefined; workflow: string | undefined; slug: string | undefined; commit: string | undefined; }; type RunKey = { pipeline: string; workflow: string; }; // A webhook of the same second may still be out of order, so it must not lower the rank. const statusRank = new Map([ ["running", 1], ["blocked", 1], ["success", 2], ["failed", 2], ["timeout", 2], ["cancelled", 2], ]); function rank(status: string): number { return statusRank.get(status) ?? 0; } export interface PipelineFilter { commits: string[]; triggers: string[]; cursor: string | undefined; limit: number; } // Only trusted runs restore the trusted tier. The public tier holds no secret value. export type Snapshot = { id: string; image: string; tier: "trusted" | "public" }; export type RunnerState = { status: RunStatus; startedAt?: string | undefined; finishedAt?: string | undefined; error?: string | undefined; snapshot?: Snapshot | undefined; // The run turned the cache off, so the saved snapshots of its workflow are deleted. discardCache?: true | undefined; }; const terminalSql = "('success', 'failed', 'timeout', 'cancelled')"; export type RunRow = { pipeline: string; workflow: string; engine: string; trusted: number | null; bk_org: string; bk_pipeline: string; build_number: number | null; triggers: string; status: string; error: string | null; started_at: string | null; finished_at: string | null; runner: string | null; }; export type PipelineRow = { rkey: string; repo: string; trigger: string; commit_sha: string; created_at: string; // Older Stores leave it out. unapproved?: number; runs: RunRow[]; }; export interface RunnerStub { // Older Runners return undefined. start(params: RunnerParams): Promise<"busy" | void>; status(pipeline: string): Promise; // Older Runners take only the pipeline. cancel(pipeline: string, status?: "cancelled" | "timeout"): Promise; } type RunnerRun = RunKey & { runner: string }; type QueuedRun = RunnerRun & { raw: string; trusted: number | null; repo: string; commit_sha: string; branch: string; snapshot: string | null; cached: number; }; // Runs of engine: cloudflare hold a Runner from dispatch until they finish. const dispatched = `r.engine = 'cloudflare' AND r.build_uuid IS NOT NULL AND r.status NOT IN ${terminalSql}`; type PendingRun = { pipeline: string; workflow: string; bk_org: string; bk_pipeline: string; clean_checkout: number | null; attempts: number; repo: string; trigger: string; commit_sha: string; branch: string; created_at: string; message: string | null; author_name: string | null; author_email: string | null; }; const migrations = [ ` CREATE TABLE IF NOT EXISTS pipelines ( rkey TEXT PRIMARY KEY, repo TEXT NOT NULL, trigger TEXT NOT NULL, push_key TEXT UNIQUE, commit_sha TEXT NOT NULL, branch TEXT NOT NULL, created_at TEXT NOT NULL ); CREATE TABLE IF NOT EXISTS workflow_runs ( pipeline TEXT NOT NULL REFERENCES pipelines (rkey), workflow TEXT NOT NULL, raw TEXT NOT NULL, bk_org TEXT NOT NULL, bk_pipeline TEXT NOT NULL, clean_checkout INTEGER, build_number INTEGER, build_uuid TEXT, build_url TEXT, status TEXT NOT NULL, attempts INTEGER NOT NULL DEFAULT 0, error TEXT, PRIMARY KEY (pipeline, workflow) ); CREATE TABLE IF NOT EXISTS service_tokens ( issuer TEXT NOT NULL, jti TEXT NOT NULL, expires_at INTEGER NOT NULL, PRIMARY KEY (issuer, jti) ); `, ` ALTER TABLE workflow_runs ADD COLUMN started_at TEXT; ALTER TABLE workflow_runs ADD COLUMN finished_at TEXT; ALTER TABLE workflow_runs ADD COLUMN event_at INTEGER; ALTER TABLE workflow_runs ADD COLUMN triggers TEXT NOT NULL DEFAULT '[]'; CREATE INDEX pipelines_repo ON pipelines (repo, rkey); CREATE INDEX workflow_runs_build ON workflow_runs (build_uuid); `, ` CREATE TABLE pulls ( repo TEXT NOT NULL, ticket TEXT NOT NULL, patch TEXT NOT NULL, PRIMARY KEY (repo, ticket) ); ALTER TABLE pipelines ADD COLUMN released_at TEXT; `, ` ALTER TABLE pipelines ADD COLUMN message TEXT; ALTER TABLE pipelines ADD COLUMN author_name TEXT; ALTER TABLE pipelines ADD COLUMN author_email TEXT; `, ` ALTER TABLE workflow_runs ADD COLUMN engine TEXT NOT NULL DEFAULT 'buildkite'; ALTER TABLE workflow_runs ADD COLUMN trusted INTEGER; `, ` CREATE TABLE snapshots ( repo TEXT NOT NULL, workflow TEXT NOT NULL, tier TEXT NOT NULL, snapshot_id TEXT NOT NULL, image TEXT NOT NULL, pipeline TEXT NOT NULL, PRIMARY KEY (repo, workflow, tier) ); `, ` ALTER TABLE workflow_runs ADD COLUMN runner TEXT; UPDATE workflow_runs SET runner = pipeline || '/' || workflow WHERE engine = 'cloudflare'; `, ` ALTER TABLE workflow_runs ADD COLUMN dispatched_at TEXT; UPDATE workflow_runs SET dispatched_at = coalesce(started_at, strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) WHERE engine = 'cloudflare' AND build_uuid IS NOT NULL AND status NOT IN ${terminalSql}; `, ` ALTER TABLE pipelines ADD COLUMN held INTEGER; UPDATE pipelines SET held = 1 WHERE released_at IS NULL AND json_extract(trigger, '$."$type"') = 'org.tangled.event.pullRequest' AND EXISTS (SELECT 1 FROM workflow_runs r WHERE r.pipeline = pipelines.rkey AND r.status IN ('blocked', 'cancelled') AND r.build_number IS NULL AND r.started_at IS NULL); `, ` ALTER TABLE pipelines ADD COLUMN sender TEXT; CREATE INDEX pipelines_unapproved ON pipelines (created_at) WHERE held = 1 AND released_at IS NULL; `, ` ALTER TABLE workflow_runs ADD COLUMN after_workflows TEXT; `, ` ALTER TABLE snapshots ADD COLUMN workflow_hash TEXT; `, ]; const unapprovedSql = "held = 1 AND released_at IS NULL"; // The workflows of the same pipeline that run r waits for and that have not succeeded. const unmetSql = `SELECT a.value AS workflow, d.status FROM json_each(r.after_workflows) a LEFT JOIN workflow_runs d ON d.pipeline = r.pipeline AND d.workflow = a.value WHERE d.status IS NOT 'success'`; const ready = `NOT EXISTS (${unmetSql})`; // Runs of one repo, ref and workflow share a Runner, which runs them one at a time. export function runnerName( repo: string, trigger: object, branch: string, workflow: string, ): string { const pull = isObject(trigger) && trigger["$type"] === "org.tangled.event.pullRequest" ? string(trigger["pull"]) : undefined; const ref = pull === undefined ? branch : `pull/${pull.slice(pull.lastIndexOf("/") + 1)}`; return `${repo}/${ref}/${workflow}`; } // The hash covers the snapshot too, so a row that older code overwrote keeps a stale hash that // matches no file. async function workflowHash(snapshot: string, raw: string): Promise { return new Uint8Array( await crypto.subtle.digest("SHA-256", new TextEncoder().encode(`${snapshot}\n${raw}`)), ).toHex(); } // Production databases created before versioning hold the first schema at version 0. export function migrate(storage: DurableObjectStorage): void { const version = storage.kv.get("schemaVersion") ?? 0; for (let i = version; i < migrations.length; i++) { storage.transactionSync(() => { storage.sql.exec(migrations[i] ?? ""); storage.kv.put("schemaVersion", i + 1); }); } } // Approve restarts the start and check window of a held pipeline. const windowStart = "coalesce(p.released_at, p.created_at)"; // Builds that the alarm still checks with Buildkite. The parameter is the oldest start. const watched = `r.status IN ('started', 'running', 'blocked') AND r.build_number IS NOT NULL AND ${windowStart} > ?`; function watchedSince(now: number): string { return new Date(now - giveUpMs).toISOString(); } // The alarm watches pull requests, and a new deployment has no pipeline to set it. export async function ensureAlarm(storage: DurableObjectStorage): Promise { const alarm = await storage.getAlarm(); if (alarm === null || alarm > Date.now() + tickMs) { await storage.setAlarm(Date.now()); } } function backoff(attempts: number): number { return Math.min(2 ** attempts * minuteMs, maxBackoffMs); } export class Store extends DurableObject { runner = (name: string): RunnerStub => this.ctx.exports.Runner.getByName(name); #lastMicros = 0; readonly #clockId = Math.floor(Math.random() * 1024); constructor(ctx: DurableObjectState, env: Env) { super(ctx, env); migrate(ctx.storage); void ctx.blockConcurrencyWhile(() => ensureAlarm(ctx.storage)); } admitToken(issuer: string, jti: string, expiresAt: number): boolean { const sql = this.ctx.storage.sql; sql.exec("DELETE FROM service_tokens WHERE expires_at < ?", Date.now() / 1000 - 60); const inserted = sql.exec( "INSERT INTO service_tokens (issuer, jti, expires_at) VALUES (?, ?, ?) ON CONFLICT DO NOTHING", issuer, jti, expiresAt, ); return inserted.rowsWritten > 0; } pushPipeline(pushKey: string): string | undefined { return this.ctx.storage.sql .exec<{ rkey: string }>("SELECT rkey FROM pipelines WHERE push_key = ?", pushKey) .toArray()[0]?.rkey; } // Returns the existing pipeline instead when one exists for pushKey. async createPipeline(input: NewPipeline): Promise { const { rkey, created } = this.#insert(input, input.held === true ? "blocked" : "pending"); if (created) { await this.ctx.storage.setAlarm(Date.now()); } return rkey; } #insert(input: NewPipeline, status: "pending" | "blocked"): { rkey: string; created: boolean } { const sql = this.ctx.storage.sql; return this.ctx.storage.transactionSync(() => { const existing = input.pushKey === null ? undefined : this.pushPipeline(input.pushKey); if (existing !== undefined) { return { rkey: existing, created: false }; } const micros = Math.max(Date.now() * 1000, this.#lastMicros + 1); this.#lastMicros = micros; const newKey = tid(micros, this.#clockId); const now = new Date().toISOString(); sql.exec( `INSERT INTO pipelines (rkey, repo, trigger, push_key, commit_sha, branch, created_at, message, author_name, author_email, held, sender) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, newKey, input.repo, JSON.stringify(input.trigger), input.pushKey, input.commit, input.branch, now, input.message ?? null, input.author?.name ?? null, input.author?.email ?? null, status === "blocked" ? 1 : null, input.sender ?? null, ); for (const run of input.runs) { sql.exec( `INSERT INTO workflow_runs (pipeline, workflow, raw, triggers, bk_org, bk_pipeline, clean_checkout, status, engine, trusted, runner, error, finished_at, after_workflows) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, newKey, run.workflow, run.raw, JSON.stringify(run.triggers), run.org, run.pipeline, run.cleanCheckout === undefined ? null : Number(run.cleanCheckout), run.error === undefined ? status : "failed", run.engine, Number(run.trusted), run.engine === "cloudflare" ? runnerName(input.repo, input.trigger, input.branch, run.workflow) : null, run.error ?? null, run.error === undefined ? null : now, run.after === undefined ? null : JSON.stringify(run.after), ); } if (status === "pending") { this.ctx.storage.kv.put("buildWorkAt", Date.now()); } else if (input.sender !== undefined) { this.#deleteUnapproved( `SELECT rkey FROM pipelines WHERE repo = ? AND sender = ? AND ${unapprovedSql} ORDER BY rkey DESC LIMIT -1 OFFSET ?`, input.repo, input.sender, maxHeldPerSender, ); } return { rkey: newKey, created: true }; }); } #deleteUnapproved(select: string, ...params: (string | number)[]): void { const { sql } = this.ctx.storage; sql.exec(`DELETE FROM workflow_runs WHERE pipeline IN (${select})`, ...params); sql.exec(`DELETE FROM pipelines WHERE rkey IN (${select})`, ...params); } // Cancels the queued runs and the Runners, and returns the open runs with a Buildkite build. async cancelRuns(repo: string, rkey: string, workflows: string[]): Promise { this.#cancel(repo, rkey, workflows, true); const runners = this.ctx.storage.sql .exec( `SELECT r.pipeline, r.workflow, r.runner FROM workflow_runs r JOIN pipelines p ON p.rkey = r.pipeline WHERE p.repo = ? AND p.rkey = ? AND r.workflow IN (SELECT value FROM json_each(?)) AND ${dispatched}`, repo, rkey, JSON.stringify(workflows), ) .toArray(); // A run whose Runner does not answer stays open, so it can be cancelled again. for (const run of runners) { try { // oxlint-disable-next-line no-await-in-loop await this.runner(run.runner).cancel(run.pipeline); this.#cancel(repo, rkey, [run.workflow], false); } catch (error) { console.warn("runner not cancelled", { ...run, error: String(error) }); } } return this.ctx.storage.sql .exec( `SELECT workflow, status, bk_org, bk_pipeline, build_number FROM workflow_runs WHERE pipeline = (SELECT rkey FROM pipelines WHERE repo = ? AND rkey = ?) AND workflow IN (SELECT value FROM json_each(?)) AND build_number IS NOT NULL AND status NOT IN ${terminalSql}`, repo, rkey, JSON.stringify(workflows), ) .toArray(); } markCancelled(repo: string, rkey: string, workflows: string[]): void { this.#cancel(repo, rkey, workflows, false); } // A webhook older than the cancel does not reopen the run. #cancel(repo: string, rkey: string, workflows: string[], unstarted: boolean): void { const now = new Date(); this.ctx.storage.sql.exec( `UPDATE workflow_runs SET status = 'cancelled', finished_at = ?, event_at = ? WHERE pipeline = (SELECT rkey FROM pipelines WHERE repo = ? AND rkey = ?) AND workflow IN (SELECT value FROM json_each(?)) AND status NOT IN ${terminalSql} AND (? = 0 OR (build_number IS NULL AND status IN ('pending', 'blocked')))`, now.toISOString(), Math.floor(now.getTime() / 1000), repo, rkey, JSON.stringify(workflows), Number(unstarted), ); this.#cancelUnmet(); } // Moves the runs that wait for Approve without a Buildkite build to pending. async releaseBlocked(repo: string, rkey: string): Promise { const now = Date.now(); const released = this.ctx.storage.transactionSync(() => { const { rowsWritten } = this.ctx.storage.sql.exec( `UPDATE workflow_runs SET status = 'pending', attempts = 0 WHERE pipeline = (SELECT rkey FROM pipelines WHERE repo = ? AND rkey = ?) AND status = 'blocked' AND build_uuid IS NULL AND build_number IS NULL`, repo, rkey, ); if (rowsWritten > 0) { this.ctx.storage.sql.exec( "UPDATE pipelines SET released_at = ? WHERE rkey = ?", new Date(now).toISOString(), rkey, ); this.ctx.storage.kv.put("buildWorkAt", now); } return rowsWritten; }); if (released > 0) { await this.ctx.storage.setAlarm(now); } return released; } #action(repo: string, ticket: string, patch: string): Action { const first = this.ctx.storage.sql .exec<{ patch: string }>( `INSERT INTO pulls (repo, ticket, patch) VALUES (?, ?, ?) ON CONFLICT DO UPDATE SET patch = patch RETURNING patch`, repo, ticket, patch, ) .one().patch; return first === patch ? "opened" : "synchronize"; } readonly #pulls: PullStore = { pushPipeline: (pushKey) => this.pushPipeline(pushKey), action: (repo, ticket, patch) => this.#action(repo, ticket, patch), insert: (pipeline, status) => { this.#insert(pipeline, status); }, webhooks: (repo) => this.ctx.exports.Webhooks.getByName(repo), }; async updateBuild(update: BuildUpdate): Promise { const run = this.ctx.storage.sql .exec( `SELECT pipeline, workflow FROM workflow_runs WHERE build_uuid = ? UNION ALL SELECT r.pipeline, r.workflow FROM workflow_runs r JOIN pipelines p ON p.rkey = r.pipeline WHERE r.build_uuid IS NULL AND r.status IN ('pending', 'started') AND r.engine = 'buildkite' AND r.pipeline = ? AND r.workflow = ? AND r.bk_pipeline = ? AND p.commit_sha = ?`, update.id, update.pipeline ?? null, update.workflow ?? null, update.slug ?? null, update.commit ?? null, ) .toArray()[0]; if (run !== undefined && this.#apply(run, update, update.ts)) { await this.ctx.storage.setAlarm(Date.now()); } } // Reports from a Runner. A finished run frees a slot, so pending runs may start. async updateRunner( pipeline: string, workflow: string, state: RunnerState, ts: number, ): Promise { if (await this.#applyRunner({ pipeline, workflow }, state, ts)) { this.ctx.storage.kv.put("buildWorkAt", Date.now()); await this.ctx.storage.setAlarm(Date.now()); } } // Returns whether the run has finished. async #applyRunner( { pipeline, workflow }: RunKey, state: RunnerState, ts: number, ): Promise { const { snapshot } = state; const hash = snapshot === undefined ? null : await workflowHash( snapshot.id, this.ctx.storage.sql .exec<{ raw: string }>( "SELECT raw FROM workflow_runs WHERE pipeline = ? AND workflow = ?", pipeline, workflow, ) .toArray()[0]?.raw ?? "", ); const done = isTerminal(state.status); this.#apply( { pipeline, workflow }, { id: undefined, number: undefined, url: undefined, status: state.status, startedAt: state.startedAt, finishedAt: state.finishedAt ?? (done ? new Date().toISOString() : undefined), }, ts, ); if (state.error !== undefined) { this.ctx.storage.sql.exec( "UPDATE workflow_runs SET error = ? WHERE pipeline = ? AND workflow = ? AND event_at = ?", state.error, pipeline, workflow, ts, ); } if (snapshot !== undefined) { const { rowsWritten } = this.ctx.storage.sql.exec( `INSERT INTO snapshots (repo, workflow, tier, snapshot_id, image, pipeline, workflow_hash) SELECT p.repo, r.workflow, ?, ?, ?, r.pipeline, ? FROM workflow_runs r JOIN pipelines p ON p.rkey = r.pipeline WHERE r.pipeline = ? AND r.workflow = ? AND r.trusted = 1 AND r.status = 'success' ON CONFLICT DO UPDATE SET snapshot_id = excluded.snapshot_id, image = excluded.image, pipeline = excluded.pipeline, workflow_hash = excluded.workflow_hash WHERE excluded.pipeline > snapshots.pipeline`, snapshot.tier, snapshot.id, snapshot.image, hash, pipeline, workflow, ); // A trusted run that did not restore the trusted snapshot found it expired or of another // image, so trusted runs restore the newer public one. if (snapshot.tier === "public" && rowsWritten > 0) { this.ctx.storage.sql.exec( `DELETE FROM snapshots WHERE tier = 'trusted' AND workflow = ? AND pipeline < ? AND repo = (SELECT repo FROM pipelines WHERE rkey = ?)`, workflow, pipeline, pipeline, ); } } if (state.discardCache === true) { this.ctx.storage.sql.exec( `DELETE FROM snapshots WHERE workflow = ? AND pipeline < ? AND repo = (SELECT p.repo FROM workflow_runs r JOIN pipelines p ON p.rkey = r.pipeline WHERE r.pipeline = ? AND r.workflow = ? AND r.trusted = 1 AND r.status = 'success')`, workflow, pipeline, pipeline, workflow, ); } return done; } // Returns whether the run ended and other runs wait for it, which makes build work due. #apply( key: RunKey, state: Omit & { id: string | undefined }, ts: number, ): boolean { const sql = this.ctx.storage.sql; const run = sql .exec<{ status: string; event_at: number | null }>( "SELECT status, event_at FROM workflow_runs WHERE pipeline = ? AND workflow = ?", key.pipeline, key.workflow, ) .toArray()[0]; if ( run === undefined || (run.event_at !== null && (ts < run.event_at || (ts === run.event_at && rank(state.status) < rank(run.status)))) ) { return false; } sql.exec( `UPDATE workflow_runs SET status = ?, build_uuid = coalesce(?, build_uuid, runner), build_number = coalesce(?, build_number), build_url = coalesce(?, build_url), started_at = coalesce(?, started_at), finished_at = ?, event_at = ? WHERE pipeline = ? AND workflow = ?`, state.status, state.id ?? null, state.number ?? null, state.url ?? null, state.startedAt ?? null, state.finishedAt ?? null, ts, key.pipeline, key.workflow, ); const waited = isTerminal(state.status) && sql .exec( `SELECT 1 FROM workflow_runs r, json_each(r.after_workflows) a WHERE r.pipeline = ? AND r.status = 'pending' AND a.value = ?`, key.pipeline, key.workflow, ) .toArray().length > 0; if (waited) { this.ctx.storage.kv.put("buildWorkAt", Date.now()); } return waited; } queryPipelines( repo: string, filter: PipelineFilter, ): { total: number; pipelines: PipelineRow[] } { const sql = this.ctx.storage.sql; const where = `repo = ?1 AND (json_array_length(?2) = 0 OR commit_sha IN (SELECT value FROM json_each(?2))) AND (json_array_length(?3) = 0 OR json_extract(trigger, '$."$type"') IN (SELECT value FROM json_each(?3)))`; const params = [repo, JSON.stringify(filter.commits), JSON.stringify(filter.triggers)]; const { total } = sql .exec<{ total: number }>( `SELECT count(*) AS total FROM pipelines WHERE ${where}`, ...params, ) .one(); const rows = sql .exec>( `SELECT rkey, repo, trigger, commit_sha, created_at FROM pipelines WHERE ${where} AND (?4 IS NULL OR rkey < ?4) ORDER BY rkey DESC LIMIT ?5`, ...params, filter.cursor ?? null, filter.limit, ) .toArray(); return { total, pipelines: this.#withRuns(rows) }; } listRepos(): string[] { return this.ctx.storage.sql .exec<{ repo: string }>("SELECT DISTINCT repo FROM pipelines ORDER BY repo") .toArray() .map((row) => row.repo); } getPipeline(repo: string, rkey: string): PipelineRow | undefined { const rows = this.ctx.storage.sql .exec>( `SELECT rkey, repo, trigger, commit_sha, created_at, coalesce(held, 0) AND released_at IS NULL AS unapproved FROM pipelines WHERE repo = ? AND rkey = ?`, repo, rkey, ) .toArray(); return this.#withRuns(rows)[0]; } #withRuns(pipelines: Omit[]): PipelineRow[] { const runs = this.ctx.storage.sql .exec( `SELECT pipeline, workflow, engine, trusted, bk_org, bk_pipeline, build_number, triggers, status, error, started_at, finished_at, runner FROM workflow_runs WHERE pipeline IN (SELECT value FROM json_each(?)) ORDER BY workflow`, JSON.stringify(pipelines.map((p) => p.rkey)), ) .toArray(); return pipelines.map((p) => ({ ...p, runs: runs.filter((run) => run.pipeline === p.rkey), })); } override async alarm(): Promise { const { kv } = this.ctx.storage; const now = Date.now(); this.#deleteUnapproved( `SELECT rkey FROM pipelines WHERE ${unapprovedSql} AND created_at <= ?`, new Date(now - heldMs).toISOString(), ); // Older code kept no work time, so a missing one is due. const workAt = kv.get("buildWorkAt"); if (now >= (workAt ?? 0)) { const next = await this.#buildWork(now); // A pipeline created during the work has already set an earlier time. if (kv.get("buildWorkAt") === workAt) { if (next === undefined) { kv.delete("buildWorkAt"); } else { kv.put("buildWorkAt", next); } } } else { await this.#startRunners(true); } const dids = await this.#declaredRepos(now); // A trusted pull request sets the work time to now, so the next alarm starts it. if (dids.length > 0) { const admit = admitter(this.env); const state = await catchUp( kv.get("jetstream"), { collections: [patchCollection, ticketCollection], dids }, (event) => event.collection === ticketCollection ? handleTicket(this.env, this.#pulls, event, admit) : handlePatch(this.env, this.#pulls, event, admit), ); kv.put("jetstream", state); } const nextAlarm = Math.min( kv.get("buildWorkAt") ?? Infinity, dids.length > 0 ? Date.now() + tickMs : Infinity, ); if (nextAlarm === Infinity) { return; } const current = await this.ctx.storage.getAlarm(); if (current === null || current > nextAlarm) { await this.ctx.storage.setAlarm(nextAlarm); } } // Answers true when the owner's declarations are unavailable, so the caller checks the repo itself. async declares(repo: string): Promise { const { kv } = this.ctx.storage; if (kv.get<{ dids: string[] }>("declaredRepos")?.dids.includes(repo) === true) { return true; } try { const dids = [...(await repoNames(this.env.OWNER_DID)).keys()].filter(isDid); kv.put("declaredRepos", { dids, at: Date.now() }); return dids.includes(repo); } catch (error) { console.warn("declared repos unavailable", errorMessage(error)); return true; } } async #declaredRepos(now: number): Promise { const { kv } = this.ctx.storage; const cached = kv.get<{ dids: string[]; at: number }>("declaredRepos"); if (cached !== undefined && now - cached.at < declaredMs) { return cached.dids; } try { const dids = [...(await repoNames(this.env.OWNER_DID)).keys()].filter(isDid); kv.put("declaredRepos", { dids, at: now }); return dids; } catch (error) { console.warn("declared repos unavailable", errorMessage(error)); const dids = cached?.dids ?? this.listRepos(); kv.put("declaredRepos", { dids, at: now - declaredMs + minuteMs }); return dids; } } // Starts pending runs and checks open builds, and returns when to do so again. async #buildWork(now: number): Promise { const pending = this.ctx.storage.sql .exec( `SELECT r.pipeline, r.workflow, r.bk_org, r.bk_pipeline, r.clean_checkout, r.attempts, p.repo, p.trigger, p.commit_sha, p.branch, ${windowStart} AS created_at, p.message, p.author_name, p.author_email FROM workflow_runs r JOIN pipelines p ON p.rkey = r.pipeline WHERE r.status = 'pending' AND r.engine = 'buildkite' AND ${ready}`, ) .toArray(); // One at a time: each request's timeout starts before it waits for a free connection. for (const run of pending) { // oxlint-disable-next-line no-await-in-loop await this.#start(run, now); } await this.#reconcile(now); await this.#checkRunners(now); await this.#startRunners(); const { attempts, open } = this.ctx.storage.sql .exec<{ attempts: number | null; open: number }>( `SELECT min(CASE WHEN r.status = 'pending' AND (r.engine = 'buildkite' OR r.attempts > 0) AND ${ready} THEN r.attempts END) AS attempts, count(CASE WHEN ${watched} OR ${dispatched} THEN 1 END) AS open FROM workflow_runs r JOIN pipelines p ON p.rkey = r.pipeline`, watchedSince(now), ) .one(); if (attempts === null && open === 0) { return undefined; } return ( now + Math.min( attempts === null ? checkMs : backoff(attempts), open === 0 ? Infinity : checkMs, ) ); } // Cancels the pending runs that wait for a run that ended without success, until none is left, // because each cancel can end a run that others wait for. #cancelUnmet(): void { const now = new Date(); let cancelled; do { cancelled = this.ctx.storage.sql.exec( `UPDATE workflow_runs AS r SET status = 'cancelled', finished_at = ?, event_at = ?, error = (SELECT u.workflow || ' did not succeed' FROM (${unmetSql}) u WHERE u.status IS NULL OR u.status IN ${terminalSql} LIMIT 1) WHERE r.status = 'pending' AND EXISTS (SELECT 1 FROM (${unmetSql}) u WHERE u.status IS NULL OR u.status IN ${terminalSql})`, now.toISOString(), Math.floor(now.getTime() / 1000), ).rowsWritten; } while (cancelled > 0); } // Asks Buildkite about builds whose webhooks may have been lost. async #reconcile(now: number): Promise { const runs = this.ctx.storage.sql .exec< RunKey & { event_at: number | null; bk_org: string; bk_pipeline: string; build_number: number; created_at: string; } >( `SELECT r.pipeline, r.workflow, r.event_at, r.bk_org, r.bk_pipeline, r.build_number, ${windowStart} AS created_at FROM workflow_runs r JOIN pipelines p ON p.rkey = r.pipeline WHERE ${watched}`, watchedSince(now), ) .toArray() .filter( (run) => (run.event_at === null ? Date.parse(run.created_at) : run.event_at * 1000) < now - checkMs, ); for (const run of runs) { // A webhook that arrives during the request is newer than the answer. const ts = Math.floor(Date.now() / 1000); try { // oxlint-disable-next-line no-await-in-loop const state = await getBuild( this.env.BUILDKITE_TOKEN, run.bk_org, run.bk_pipeline, run.build_number, ); if (state !== undefined) { this.#apply(run, state, ts); } } catch (error) { console.warn("buildkite build status unavailable", { ...run, error: String(error), }); } } } // Asks the Runners of quiet runs, in case a report was lost. A run that its Runner did not end // in time frees its slot. async #checkRunners(now: number): Promise { const expired = this.ctx.storage.sql .exec( `SELECT r.pipeline, r.workflow, r.runner FROM workflow_runs r WHERE ${dispatched} AND r.dispatched_at < ?`, new Date(now - maxRunMs).toISOString(), ) .toArray(); for (const run of expired) { // oxlint-disable-next-line no-await-in-loop await this.runner(run.runner) .cancel(run.pipeline, "timeout") .catch((error: unknown) => { console.warn("runner not cancelled", { ...run, error: String(error) }); }); const open = this.ctx.storage.sql .exec( `SELECT 1 FROM workflow_runs r WHERE r.pipeline = ? AND r.workflow = ? AND ${dispatched}`, run.pipeline, run.workflow, ) .toArray(); if (open.length === 0) { continue; } // oxlint-disable-next-line no-await-in-loop await this.#applyRunner( run, { status: "timeout", error: "the run timed out" }, Math.floor(Date.now() / 1000), ); } const runs = this.ctx.storage.sql .exec( `SELECT r.pipeline, r.workflow, r.runner FROM workflow_runs r WHERE ${dispatched} AND coalesce(r.event_at, 0) * 1000 < ?`, now - checkMs, ) .toArray(); for (const run of runs) { const ts = Math.floor(Date.now() / 1000); try { // oxlint-disable-next-line no-await-in-loop const state = await this.runner(run.runner).status(run.pipeline); // oxlint-disable-next-line no-await-in-loop await this.#applyRunner( run, state ?? { status: "failed", error: "the Runner lost the run" }, ts, ); } catch (error) { console.warn("runner status unavailable", { ...run, error: String(error) }); } } } // Starts queued runs, oldest first, while Runners are free. A Runner runs one run at a time. A // newer push or pull request run cancels the queued runs of its Runner that are not more // trusted. A busy Runner costs no attempt, and the next tick asks it again. A start that failed // waits for its backoff instead. async #startRunners(unfailed = false): Promise { const sql = this.ctx.storage.sql; const now = new Date(); sql.exec( `UPDATE workflow_runs AS o SET status = 'cancelled', finished_at = ?, event_at = ? WHERE o.engine = 'cloudflare' AND o.status = 'pending' AND EXISTS (SELECT 1 FROM workflow_runs n JOIN pipelines p ON p.rkey = n.pipeline WHERE n.runner = o.runner AND n.status = 'pending' AND n.pipeline > o.pipeline AND p.push_key IS NOT NULL AND n.trusted >= o.trusted)`, now.toISOString(), Math.floor(now.getTime() / 1000), ); this.#cancelUnmet(); const { busy } = sql .exec<{ busy: number }>( `SELECT count(*) AS busy FROM workflow_runs r WHERE ${dispatched}`, ) .one(); const queued = sql .exec( `SELECT r.pipeline, r.workflow, r.runner, r.raw, r.trusted, p.repo, p.commit_sha, p.branch, (SELECT json_object('id', s.snapshot_id, 'image', s.image, 'tier', s.tier, 'hash', s.workflow_hash) FROM snapshots s WHERE s.repo = p.repo AND s.workflow = r.workflow AND (s.tier = 'public' OR r.trusted = 1) ORDER BY s.tier = 'trusted' DESC LIMIT 1) AS snapshot, r.trusted = 1 AND EXISTS (SELECT 1 FROM snapshots s WHERE s.repo = p.repo AND s.workflow = r.workflow) AS cached, ${windowStart} AS created_at FROM workflow_runs r JOIN pipelines p ON p.rkey = r.pipeline WHERE r.status = 'pending' AND r.engine = 'cloudflare' AND ${ready} AND r.runner NOT IN (SELECT r.runner FROM workflow_runs r WHERE ${dispatched}) AND (? = 0 OR r.attempts = 0) ORDER BY created_at, r.pipeline, r.workflow LIMIT ?`, Number(unfailed), Math.max(0, maxRunners - busy), ) .toArray(); for (const run of queued) { const runner = this.runner(run.runner); // oxlint-disable-next-line typescript/no-unsafe-type-assertion -- built by the query const saved = JSON.parse(run.snapshot ?? "null") as | (Snapshot & { hash: string | null }) | null; // A snapshot of another workflow file, or one that older code saved without a hash, is // not restored. const restore = // oxlint-disable-next-line no-await-in-loop saved === null || saved.hash !== (await workflowHash(saved.id, run.raw)) ? undefined : { id: saved.id, image: saved.image, tier: saved.tier }; try { // oxlint-disable-next-line no-await-in-loop const result = await runner.start({ pipeline: run.pipeline, workflow: run.workflow, repo: run.repo, commit: run.commit_sha, raw: run.raw, trusted: run.trusted === 1, branch: run.branch, ...(restore === undefined ? {} : { restore }), ...(run.cached === 1 ? { cached: true } : {}), }); if (result === "busy") { if (Date.now() - Date.parse(run.created_at) > giveUpMs) { this.#fail(run, "gave up: the Runner is busy"); } continue; } } catch (error) { const reason = errorMessage(error); console.warn("runner did not start", { ...run, raw: undefined, error: reason }); if (Date.now() - Date.parse(run.created_at) > giveUpMs) { this.#fail( { pipeline: run.pipeline, workflow: run.workflow }, `gave up: ${reason}`, ); } else { sql.exec( "UPDATE workflow_runs SET attempts = attempts + 1 WHERE pipeline = ? AND workflow = ? AND status = 'pending'", run.pipeline, run.workflow, ); } continue; } const { status } = sql .exec<{ status: string }>( `UPDATE workflow_runs SET build_uuid = ?, dispatched_at = ?, status = CASE WHEN status = 'pending' THEN 'started' ELSE status END WHERE pipeline = ? AND workflow = ? RETURNING status`, run.runner, new Date().toISOString(), run.pipeline, run.workflow, ) .one(); // The run may have been cancelled while its Runner started. if (status === "cancelled") { // oxlint-disable-next-line no-await-in-loop await runner.cancel(run.pipeline).catch((error: unknown) => { console.warn("runner not cancelled", { ...run, raw: undefined, error: String(error), }); }); } } } async #start(run: PendingRun, now: number): Promise { const sql = this.ctx.storage.sql; const current = sql .exec<{ status: string }>( "SELECT status FROM workflow_runs WHERE pipeline = ? AND workflow = ?", run.pipeline, run.workflow, ) .toArray()[0]; // A webhook may have claimed the run while an earlier build request waited. if (current?.status !== "pending") { return; } const trigger: unknown = JSON.parse(run.trigger); const body = createBuildBody({ commit: run.commit_sha, branch: run.branch, repo: run.repo, pipeline: run.pipeline, workflow: run.workflow, cleanCheckout: run.clean_checkout === null ? undefined : run.clean_checkout !== 0, ...(run.message === null ? {} : { message: run.message }), ...(run.author_name === null || run.author_email === null ? {} : { author: { name: run.author_name, email: run.author_email } }), inputs: isObject(trigger) && isPairList(trigger["inputs"]) ? trigger["inputs"] : [], baseBranch: isObject(trigger) && trigger["$type"] === "org.tangled.event.pullRequest" ? string(trigger["targetBranch"]) : undefined, }); let result; try { result = await createBuild(this.env.BUILDKITE_TOKEN, run.bk_org, run.bk_pipeline, body); } catch (error) { const reason = errorMessage(error); console.warn("buildkite build did not start", { ...run, error: reason }); if (now - Date.parse(run.created_at) > giveUpMs) { this.#fail(run, `gave up: ${reason}`); return; } sql.exec( "UPDATE workflow_runs SET attempts = attempts + 1 WHERE pipeline = ? AND workflow = ? AND status = 'pending'", run.pipeline, run.workflow, ); return; } if (typeof result === "string") { console.error("buildkite refused build", { ...run, error: result }); this.#fail(run, result); return; } const build = [ result.number ?? null, result.id ?? null, result.web_url ?? null, run.pipeline, run.workflow, ]; const started = sql.exec( `UPDATE workflow_runs SET status = 'started', build_number = ?, build_uuid = ?, build_url = ? WHERE pipeline = ? AND workflow = ? AND status = 'pending'`, ...build, ).rowsWritten; // The run may have been cancelled while Buildkite created its build. const cancelled = started === 0 && sql.exec( `UPDATE workflow_runs SET build_number = ?, build_uuid = ?, build_url = ? WHERE pipeline = ? AND workflow = ? AND status = 'cancelled' AND build_uuid IS NULL`, ...build, ).rowsWritten > 0; if (!cancelled || result.number === undefined) { return; } await cancelBuild( this.env.BUILDKITE_TOKEN, run.bk_org, run.bk_pipeline, result.number, ).catch((error: unknown) => { console.warn("buildkite build not cancelled", { ...run, error: String(error) }); }); } #fail(run: RunKey, error: string): void { this.ctx.storage.sql.exec( "UPDATE workflow_runs SET status = 'failed', error = ? WHERE pipeline = ? AND workflow = ? AND status = 'pending'", error, run.pipeline, run.workflow, ); this.#cancelUnmet(); } }