import { runDurableObjectAlarm, runInDurableObject } from "cloudflare:test"; import { env, exports } from "cloudflare:workers"; import { beforeEach, describe, expect, it, vi } from "vitest"; import type { RunStatus } from "../src/buildkite"; import type { RunnerParams } from "../src/runner"; import { pipelineUrl } from "../src/buildkite"; import { migrate, runnerName, tid } from "../src/store"; import type { RunnerState, RunnerStub } from "../src/store"; import { declarationsRoute, didDoc, jetstreamSocket, jetstreamUrl, offline, pdsService, repoDid, routeFetch, seed, } from "./helpers"; import type { Route } from "./helpers"; async function tick(store: DurableObjectStub) { await runInDurableObject(store, async (_instance, state) => { await state.storage.setAlarm(Date.now()); }); await runDurableObjectAlarm(store); } describe("tid", () => { it("encodes a sortable 13-character record key", () => { const a = tid(1_700_000_000_000_000, 7); const b = tid(1_700_000_000_000_001, 0); expect(a).toMatch(/^[234567abcdefghij][234567abcdefghijklmnopqrstuvwxyz]{12}$/); expect(a < b).toBe(true); }); }); const firstSchema = ` CREATE TABLE pipelines ( rkey TEXT PRIMARY KEY, repo TEXT NOT NULL, trigger TEXT NOT NULL, push_key TEXT UNIQUE, commit_sha TEXT NOT NULL, branch TEXT NOT NULL, created_at TEXT NOT NULL ); CREATE TABLE workflow_runs ( pipeline TEXT NOT NULL REFERENCES pipelines (rkey), workflow TEXT NOT NULL, raw TEXT NOT NULL, bk_org TEXT NOT NULL, bk_pipeline TEXT NOT NULL, clean_checkout INTEGER, build_number INTEGER, build_uuid TEXT, build_url TEXT, status TEXT NOT NULL, attempts INTEGER NOT NULL DEFAULT 0, error TEXT, PRIMARY KEY (pipeline, workflow) ); CREATE TABLE service_tokens ( issuer TEXT NOT NULL, jti TEXT NOT NULL, expires_at INTEGER NOT NULL, PRIMARY KEY (issuer, jti) ); INSERT INTO pipelines VALUES ('3aaaaaaaaaaa2', 'did:plc:r', '{}', 'k', 'abc', 'main', '2026-11-06T00:00:00Z'); INSERT INTO workflow_runs (pipeline, workflow, raw, bk_org, bk_pipeline, build_uuid, status) VALUES ('3aaaaaaaaaaa2', 'ci.yml', '', 'gpanders', 'treadle', 'uuid-1', 'started'); `; describe("runnerName", () => { const repo = "did:plc:r"; const pull = { $type: "org.tangled.event.pullRequest", pull: `at://${repo}/org.tangled.track.ticket/3ticket`, }; it.each([ ["a push", { $type: "org.tangled.event.push" }, "main", `${repo}/main/ci.yml`], ["a pull request from a branch", pull, "feature", `${repo}/pull/3ticket/ci.yml`], ["a pull request from a fork", pull, "pull/3ticket", `${repo}/pull/3ticket/ci.yml`], ])("names the Runner of %s by repo, ref and workflow", (_, trigger, branch, name) => { expect(runnerName(repo, trigger, branch, "ci.yml")).toBe(name); }); }); describe("migrate", () => { it("upgrades the first schema in place and keeps its rows", async () => { const rows = await runInDurableObject( exports.Store.getByName("legacy"), (_instance, state) => { const { sql, kv } = state.storage; sql.exec( "DROP TABLE workflow_runs; DROP TABLE pipelines; DROP TABLE service_tokens; DROP TABLE IF EXISTS pulls; DROP TABLE IF EXISTS snapshots", ); kv.delete("schemaVersion"); sql.exec(firstSchema); migrate(state.storage); migrate(state.storage); return { version: kv.get("schemaVersion"), indexes: sql .exec( "SELECT name FROM sqlite_master WHERE type = 'index' AND sql IS NOT NULL ORDER BY name", ) .toArray(), tables: sql .exec( "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE '_cf%' ORDER BY name", ) .toArray() .map((row) => row["name"]), pipelines: sql .exec("SELECT rkey, released_at, message FROM pipelines") .toArray(), runs: sql .exec( "SELECT workflow, status, build_uuid, triggers, started_at, finished_at, event_at, engine, trusted, runner FROM workflow_runs", ) .toArray(), }; }, ); expect(rows).toEqual({ version: 12, indexes: [ { name: "pipelines_repo" }, { name: "pipelines_unapproved" }, { name: "workflow_runs_build" }, ], tables: ["pipelines", "pulls", "service_tokens", "snapshots", "workflow_runs"], pipelines: [{ rkey: "3aaaaaaaaaaa2", released_at: null, message: null }], runs: [ { workflow: "ci.yml", status: "started", build_uuid: "uuid-1", triggers: "[]", started_at: null, finished_at: null, event_at: null, engine: "buildkite", trusted: null, runner: null, }, ], }); }); }); describe("Cloudflare runs", () => { beforeEach(() => { routeFetch({ [`GET https://plc.directory/${env.OWNER_DID}`]: offline, [`GET ${jetstreamUrl}`]: offline, }); }); const repo = "did:plc:cloudflare"; const cloudflare = exports.Store.getByName("cloudflare"); const pipeline = { repo, trigger: { $type: "org.tangled.event.push" }, pushKey: null, commit: "abc", branch: "main", }; const run = { workflow: "ci.yml", raw: "engine: cloudflare\nsteps: [{name: a, command: b}]\n", triggers: [], org: "gpanders", pipeline: "treadle", cleanCheckout: undefined, }; const job = (workflow: string, after?: string[]) => ({ ...run, workflow, engine: "cloudflare" as const, trusted: false, ...(after === undefined ? {} : { after }), }); it("stores the engine and the trust of each run", async () => { const rkey = await cloudflare.createPipeline({ ...pipeline, runs: [ { ...run, engine: "cloudflare", trusted: true }, { ...run, workflow: "other.yml", engine: "buildkite", trusted: false }, ], }); const rows = await runInDurableObject(cloudflare, (_instance, state) => state.storage.sql .exec( "SELECT workflow, engine, trusted, runner FROM workflow_runs WHERE pipeline = ? ORDER BY workflow", rkey, ) .toArray(), ); expect(rows).toEqual([ { workflow: "ci.yml", engine: "cloudflare", trusted: 1, runner: `${repo}/main/ci.yml` }, { workflow: "other.yml", engine: "buildkite", trusted: 0, runner: null }, ]); const stored = await cloudflare.getPipeline(repo, rkey); expect(stored?.runs.map((r) => r.engine)).toEqual(["cloudflare", "buildkite"]); }); it("applies Runner updates in order and wakes the alarm when a run ends", async () => { const rkey = await cloudflare.createPipeline({ ...pipeline, runs: [{ ...run, engine: "cloudflare", trusted: false }], }); const update = (status: RunStatus, ts: number) => cloudflare.updateRunner( rkey, "ci.yml", { status, startedAt: "2026-11-07T00:00:00.000Z" }, ts, ); await update("running", 10); await update("timeout", 20); await update("running", 15); const row = await runInDurableObject(cloudflare, async (instance, state) => { state.storage.kv.delete("buildWorkAt"); await instance.updateRunner(rkey, "ci.yml", { status: "timeout" }, 20); return { ...state.storage.sql .exec<{ finished_at: string | null }>( "SELECT status, build_uuid, started_at, finished_at, event_at FROM workflow_runs WHERE pipeline = ?", rkey, ) .one(), workAt: state.storage.kv.get("buildWorkAt"), }; }); expect(row).toMatchObject({ status: "timeout", build_uuid: `${repo}/main/ci.yml`, started_at: "2026-11-07T00:00:00.000Z", event_at: 20, }); expect(row.finished_at).toEqual(expect.any(String)); expect(row.workAt).toBeLessThanOrEqual(Date.now()); expect(await cloudflare.cancelRuns(repo, rkey, ["ci.yml"])).toEqual([]); }); it("never starts a Cloudflare run as a Buildkite build", async () => { const alone = exports.Store.getByName("cloudflare-alone"); await runInDurableObject(alone, (object) => { object.runner = () => ({ start: () => Promise.resolve(), status: () => Promise.resolve({ status: "started" as const }), cancel: () => Promise.resolve(), }); }); const fetch = vi.spyOn(globalThis, "fetch"); const rkey = await alone.createPipeline({ ...pipeline, runs: [{ ...run, pipeline: "alone", engine: "cloudflare", trusted: false }], }); await alone.updateBuild({ id: "build-1", number: 1, url: undefined, status: "running", startedAt: undefined, finishedAt: undefined, ts: 1, pipeline: rkey, workflow: "ci.yml", slug: "alone", commit: "abc", }); await runDurableObjectAlarm(alone); const urls = fetch.mock.calls.map(([input]) => new Request(input).url); fetch.mockRestore(); // Alarms of the other Stores in this file may call Buildkite at any time. expect(urls.filter((url) => url.includes("/pipelines/alone/"))).toEqual([]); const stored = await alone.getPipeline(repo, rkey); expect(stored?.runs[0]?.build_number).toBeNull(); }); describe("Runners", () => { class FakeRunners { started: string[] = []; params = new Map(); cancelled: string[] = []; states = new Map(); startError: Error | undefined; busy = false; cancelError: Error | undefined; onStart: ((params: RunnerParams) => Promise) | undefined; get = (name: string): RunnerStub => ({ start: async (params) => { if (this.startError !== undefined) { throw this.startError; } if (!this.busy) { await this.onStart?.(params); this.started.push(name); this.params.set(name, params); } return this.busy ? "busy" : undefined; }, status: () => Promise.resolve(this.states.get(name)), cancel: (_pipeline, status) => { if (this.cancelError !== undefined) { return Promise.reject(this.cancelError); } this.cancelled.push(status === undefined ? name : `${name} ${status}`); return Promise.resolve(); }, }); } async function setup(storeName: string) { const store = exports.Store.getByName(storeName); const runners = new FakeRunners(); const instance = await runInDurableObject(store, (object) => { object.runner = runners.get; return object; }); const names = new Map(); const create = async ( trusted = false, branch = `b${names.size}`, pushKey: string | null = `push ${names.size}`, raw = run.raw, ) => { const rkey = await store.createPipeline({ ...pipeline, branch, pushKey, runs: [{ ...run, raw, engine: "cloudflare", trusted }], }); names.set(rkey, `${repo}/${branch}/ci.yml`); return rkey; }; const name = (rkey = "") => names.get(rkey) ?? ""; const status = async (rkey: string) => (await store.getPipeline(repo, rkey))?.runs[0]?.status; return { store, instance, runners, create, name, status }; } it("stores a run with an error as failed and never starts it", async () => { const { store, runners } = await setup("runners-invalid"); const rkey = await store.createPipeline({ ...pipeline, runs: [{ ...run, engine: "cloudflare", trusted: false, error: "bad file" }], }); await tick(store); expect(runners.started).toEqual([]); const [stored] = (await store.getPipeline(repo, rkey))?.runs ?? []; expect(stored).toMatchObject({ status: "failed", error: "bad file" }); expect(stored?.finished_at).toEqual(expect.any(String)); }); it("starts queued runs oldest first, at most ten at once", async () => { const { store, runners, create, name, status } = await setup("runners-queue"); const rkeys = []; for (let i = 0; i < 12; i++) { // oxlint-disable-next-line no-await-in-loop rkeys.push(await create(i === 0)); } await runDurableObjectAlarm(store); expect(runners.started).toEqual(rkeys.slice(0, 10).map(name)); expect(runners.params.get(name(rkeys[0]))).toEqual({ pipeline: rkeys[0], workflow: "ci.yml", repo, commit: "abc", raw: "engine: cloudflare\nsteps: [{name: a, command: b}]\n", trusted: true, branch: "b0", }); expect(runners.params.get(name(rkeys[1]))?.trusted).toBe(false); expect(await status(rkeys[10] ?? "")).toBe("pending"); await store.cancelRuns(repo, rkeys[10] ?? "", ["ci.yml"]); await store.updateRunner(rkeys[0] ?? "", "ci.yml", { status: "success" }, 1); await runDurableObjectAlarm(store); expect(runners.started.slice(10)).toEqual([name(rkeys[11])]); expect(await status(rkeys[10] ?? "")).toBe("cancelled"); }); it("keeps the newest snapshot of each tier from trusted runs and gives it out", async () => { const { store, runners, create, name } = await setup("runners-snapshots"); const [older, newer, untrusted] = [ await create(true), await create(true), await create(false), ]; await runDurableObjectAlarm(store); const image = "cloudflare/debian-trixie"; const report = (rkey: string, id: string, tier: "public" | "trusted", ts: number) => store.updateRunner( rkey, "ci.yml", { status: "success", snapshot: { id, image, tier } }, ts, ); await report(newer, "public-new", "public", 1); await report(older, "public-old", "public", 2); await report(untrusted, "untrusted", "public", 3); const [trusted, publicRun] = [await create(true), await create(false)]; await runDurableObjectAlarm(store); const given = (rkey: string) => runners.params.get(name(rkey))?.restore; expect(given(trusted)).toEqual({ id: "public-new", image, tier: "public" }); expect(given(publicRun)).toEqual({ id: "public-new", image, tier: "public" }); expect(given(older)).toBeUndefined(); await report(trusted, "trusted-1", "trusted", 4); const [second, third] = [await create(true), await create(false)]; await runInDurableObject(store, (_instance, state) => { state.storage.kv.put("buildWorkAt", 0); }); await runDurableObjectAlarm(store); expect(given(second)).toEqual({ id: "trusted-1", image, tier: "trusted" }); expect(given(third)).toEqual({ id: "public-new", image, tier: "public" }); await report(second, "public-2", "public", 5); const fourth = await create(true); await runInDurableObject(store, (_instance, state) => { state.storage.kv.put("buildWorkAt", 0); }); await runDurableObjectAlarm(store); expect(given(fourth)).toEqual({ id: "public-2", image, tier: "public" }); }); it("gives a snapshot only to runs of the same workflow file", async () => { const { store, runners, create, name } = await setup("runners-snapshot-file"); const saver = await create(true); await runDurableObjectAlarm(store); const snapshot = { id: "s", image: "cloudflare/debian-trixie", tier: "public" as const, }; await store.updateRunner(saver, "ci.yml", { status: "success", snapshot }, 1); const [same, edited] = [ await create(false), await create(true, undefined, undefined, `${run.raw}# edited\n`), ]; await runDurableObjectAlarm(store); const given = (rkey: string) => runners.params.get(name(rkey))?.restore; expect(given(same)).toEqual(snapshot); expect(given(edited)).toBeUndefined(); expect(runners.params.get(name(edited))?.cached).toBe(true); expect(runners.params.get(name(same))?.cached).toBeUndefined(); const newer = { ...snapshot, id: "s2" }; await store.updateRunner(edited, "ci.yml", { status: "success", snapshot: newer }, 2); const [old, current] = [ await create(false), await create(false, undefined, undefined, `${run.raw}# edited\n`), ]; await runInDurableObject(store, (_instance, state) => { state.storage.kv.put("buildWorkAt", 0); }); await runDurableObjectAlarm(store); expect(given(old)).toBeUndefined(); expect(given(current)).toEqual(newer); for (const update of [ "UPDATE snapshots SET snapshot_id = 'overwritten'", "UPDATE snapshots SET workflow_hash = NULL", ]) { // oxlint-disable-next-line no-await-in-loop await runInDurableObject(store, (_instance, state) => { state.storage.sql.exec(update); state.storage.kv.put("buildWorkAt", 0); }); // oxlint-disable-next-line no-await-in-loop const legacy = await create(false, undefined, undefined, `${run.raw}# edited\n`); // oxlint-disable-next-line no-await-in-loop await runDurableObjectAlarm(store); expect(runners.started).toContain(name(legacy)); expect(given(legacy)).toBeUndefined(); } }); it("discards the snapshots of a workflow after a trusted run without cache", async () => { const { store, create } = await setup("runners-discard"); const [older, untrusted, failed, discarder, newer] = [ await create(true), await create(false), await create(true), await create(true), await create(true), ]; await runDurableObjectAlarm(store); const image = "cloudflare/debian-trixie"; await store.updateRunner( older, "ci.yml", { status: "success", snapshot: { id: "old", image, tier: "public" } }, 1, ); await runInDurableObject(store, (_instance, state) => { state.storage.sql.exec( `INSERT INTO snapshots (repo, workflow, tier, snapshot_id, image, pipeline) VALUES ('did:plc:other', 'ci.yml', 'public', 'other', ?, '2')`, image, ); }); const rows = () => runInDurableObject(store, async (_instance, state) => { await state.storage.deleteAlarm(); return state.storage.sql .exec("SELECT snapshot_id FROM snapshots ORDER BY snapshot_id") .toArray() .map((row) => row["snapshot_id"]); }); await store.updateRunner( untrusted, "ci.yml", { status: "success", discardCache: true }, 2, ); await store.updateRunner(failed, "ci.yml", { status: "failed", discardCache: true }, 2); expect(await rows()).toEqual(["old", "other"]); await store.updateRunner( newer, "ci.yml", { status: "success", snapshot: { id: "new", image, tier: "trusted" } }, 3, ); await store.updateRunner( discarder, "ci.yml", { status: "success", discardCache: true }, 4, ); expect(await rows()).toEqual(["new", "other"]); }); it("tries a start again after an error", async () => { const { store, runners, create, status } = await setup("runners-retry"); runners.startError = new Error("overloaded"); const rkey = await create(); await runDurableObjectAlarm(store); await runDurableObjectAlarm(store); expect(await status(rkey)).toBe("pending"); expect( await runInDurableObject( store, (_instance, state) => state.storage.sql .exec<{ attempts: number }>("SELECT attempts FROM workflow_runs") .one().attempts, ), ).toBe(1); runners.startError = undefined; await runInDurableObject(store, (_instance, state) => { state.storage.kv.put("buildWorkAt", 0); }); await runDurableObjectAlarm(store); expect(await status(rkey)).toBe("started"); }); it("asks a busy Runner again at the next tick without backoff", async () => { const { store, runners, create, status } = await setup("runners-busy"); runners.busy = true; const rkey = await create(); await runDurableObjectAlarm(store); const row = await runInDurableObject(store, async (_instance, state) => ({ ...state.storage.sql .exec( "SELECT status, attempts, build_uuid FROM workflow_runs WHERE pipeline = ?", rkey, ) .one(), alarm: (await state.storage.getAlarm()) ?? Infinity, })); expect(row).toMatchObject({ status: "pending", attempts: 0, build_uuid: null }); expect(row.alarm).toBeLessThanOrEqual(Date.now() + 10_000); runners.busy = false; await runDurableObjectAlarm(store); expect(await status(rkey)).toBe("started"); }); it.each([ ["an error", new Error("overloaded"), false, "gave up: overloaded"], ["a busy Runner", undefined, true, "gave up: the Runner is busy"], ])("gives up a start after three days of %s", async (_, error, busy, reason) => { const { store, runners, create } = await setup(`runners-refused-${busy}`); runners.startError = error; runners.busy = busy; const rkey = await create(); await runInDurableObject(store, (_instance, state) => { state.storage.sql.exec("UPDATE pipelines SET created_at = '2026-01-01T00:00:00Z'"); state.storage.kv.put("buildWorkAt", 0); }); await runDurableObjectAlarm(store); expect((await store.getPipeline(repo, rkey))?.runs[0]).toMatchObject({ status: "failed", error: reason, }); }); it("cancels a Runner whose run was cancelled while it started", async () => { const { store, instance, runners, create, name, status } = await setup("runners-race"); runners.onStart = (params) => instance.cancelRuns(repo, params.pipeline, ["ci.yml"]); const rkey = await create(); await runDurableObjectAlarm(store); expect(await status(rkey)).toBe("cancelled"); expect(runners.cancelled).toEqual([name(rkey)]); }); it("checks quiet runs with their Runner and frees the slot of a lost run", async () => { const { store, runners, create, name } = await setup("runners-check"); const rkeys = []; for (let i = 0; i < 11; i++) { // oxlint-disable-next-line no-await-in-loop rkeys.push(await create()); } const [done = "", lost = ""] = rkeys; await runDurableObjectAlarm(store); for (const rkey of rkeys.slice(0, 10)) { runners.states.set(name(rkey), { status: "running" }); } runners.states.set(name(lost), undefined); await runInDurableObject(store, (_instance, state) => { state.storage.kv.put("buildWorkAt", 0); }); await runDurableObjectAlarm(store); expect(runners.started.at(-1)).toBe(name(rkeys[10])); runners.states.set(name(done), { status: "success", finishedAt: "x" }); await runInDurableObject(store, (_instance, state) => { state.storage.sql.exec("UPDATE workflow_runs SET event_at = 0"); state.storage.kv.put("buildWorkAt", 0); }); await runDurableObjectAlarm(store); expect((await store.getPipeline(repo, done))?.runs[0]).toMatchObject({ status: "success", }); expect((await store.getPipeline(repo, lost))?.runs[0]).toMatchObject({ status: "failed", error: "the Runner lost the run", }); }); it("times out a run that its Runner has not ended after the longest timeout", async () => { const { store, runners, create, name, status } = await setup("runners-stuck"); const [stuck, fresh] = [await create(), await create()]; await runDurableObjectAlarm(store); runners.states.set(name(stuck), { status: "running" }); runners.states.set(name(fresh), { status: "running" }); await runInDurableObject(store, (_instance, state) => { state.storage.sql.exec( "UPDATE workflow_runs SET dispatched_at = ? WHERE pipeline = ?", new Date(Date.now() - 7 * 3_600_000).toISOString(), stuck, ); state.storage.kv.put("buildWorkAt", 0); }); await runDurableObjectAlarm(store); expect((await store.getPipeline(repo, stuck))?.runs[0]).toMatchObject({ status: "timeout", error: "the run timed out", }); expect(runners.cancelled).toEqual([`${name(stuck)} timeout`]); expect(await status(fresh)).toBe("running"); }); it("cancels a started run through its Runner", async () => { const { store, runners, create, name, status } = await setup("runners-cancel"); const rkey = await create(); await runDurableObjectAlarm(store); runners.cancelError = new Error("unreachable"); expect(await store.cancelRuns(repo, rkey, ["ci.yml"])).toEqual([]); expect(await status(rkey)).toBe("started"); runners.cancelError = undefined; expect(await store.cancelRuns(repo, rkey, ["ci.yml"])).toEqual([]); expect(runners.cancelled).toEqual([name(rkey)]); expect(await status(rkey)).toBe("cancelled"); }); it("runs one run at a time for each Runner name and starts only the newest waiting run", async () => { const { store, runners, create, name, status } = await setup("runners-serial"); const first = await create(false, "main"); await runDurableObjectAlarm(store); runners.states.set(name(first), { status: "running" }); const skipped = await create(false, "main"); const newest = await create(false, "main"); const other = await create(false, "dev"); const held = await create(false, "main"); await runInDurableObject(store, (_instance, state) => { state.storage.sql.exec( "UPDATE workflow_runs SET status = 'blocked' WHERE pipeline = ?", held, ); state.storage.kv.put("buildWorkAt", 0); }); await runDurableObjectAlarm(store); expect(runners.started).toEqual([name(first), name(other)]); expect(await status(skipped)).toBe("cancelled"); expect(await status(newest)).toBe("pending"); expect(await status(held)).toBe("blocked"); const ts = Math.floor(Date.now() / 1000) + 1; await store.updateRunner(first, "ci.yml", { status: "success" }, ts); await runDurableObjectAlarm(store); expect(runners.started).toEqual([name(first), name(other), name(newest)]); expect(runners.params.get(name(newest))?.pipeline).toBe(newest); expect(await status(newest)).toBe("started"); }); it("keeps a waiting run that a manual or less trusted run follows", async () => { const { store, runners, create, name, status } = await setup("runners-supersede"); const first = await create(true, "main"); await runDurableObjectAlarm(store); runners.states.set(name(first), { status: "running" }); const push = await create(true, "main"); const manual = await create(true, "main", null); const untrusted = await create(false, "main"); await runDurableObjectAlarm(store); expect([await status(push), await status(manual), await status(untrusted)]).toEqual([ "pending", "pending", "pending", ]); const newest = await create(true, "main"); await runDurableObjectAlarm(store); expect([await status(push), await status(manual), await status(untrusted)]).toEqual([ "cancelled", "cancelled", "cancelled", ]); expect(await status(newest)).toBe("pending"); }); it("starts a run of either engine only after the runs that it waits for succeed", async () => { const posted: string[] = []; const builds = (slug: string): Route => () => { posted.push(slug); return Response.json({ id: `build-${slug}`, number: 1 }, { status: 201 }); }; routeFetch({ [`GET https://plc.directory/${env.OWNER_DID}`]: offline, [`GET ${jetstreamUrl}`]: offline, [`POST ${pipelineUrl("gpanders", "after-check")}/builds`]: builds("after-check"), [`POST ${pipelineUrl("gpanders", "after-publish")}/builds`]: builds("after-publish"), }); const { store, runners } = await setup("runners-after"); const buildkite = { ...run, engine: "buildkite" as const, trusted: false }; await store.createPipeline({ ...pipeline, runs: [ { ...buildkite, workflow: "check.yml", pipeline: "after-check" }, { ...run, workflow: "deploy.yml", engine: "cloudflare", trusted: true, after: ["check.yml"], }, { ...buildkite, workflow: "publish.yml", pipeline: "after-publish", after: ["check.yml"], }, ], }); await runDurableObjectAlarm(store); expect(posted).toEqual(["after-check"]); expect(runners.started).toEqual([]); const workAt = await runInDurableObject(store, (_instance, state) => state.storage.kv.get("buildWorkAt"), ); expect(workAt).toBeGreaterThan(Date.now() + 5 * 60_000); await store.updateBuild({ id: "build-after-check", number: 1, url: undefined, status: "success", startedAt: undefined, finishedAt: undefined, ts: 1, pipeline: undefined, workflow: undefined, slug: undefined, commit: undefined, }); const alarm = await runInDurableObject(store, (_instance, state) => state.storage.getAlarm(), ); expect(alarm).toBeLessThanOrEqual(Date.now()); await runDurableObjectAlarm(store); expect(posted).toEqual(["after-check", "after-publish"]); expect(runners.started).toEqual([`${repo}/main/deploy.yml`]); }); it.each(["failed", "timeout", "cancelled"] as const)( "cancels the runs that wait for a %s run, and holds waiting runs for Approve", async (ended) => { const { store, runners } = await setup(`runners-after-${ended}`); const runs = [job("check.yml"), job("deploy.yml", ["check.yml"])]; const failed = await store.createPipeline({ ...pipeline, runs: [...runs, job("notify.yml", ["deploy.yml"])], }); const held = await store.createPipeline({ ...pipeline, branch: "pr", held: true, runs, }); await runDurableObjectAlarm(store); expect(runners.started).toEqual([`${repo}/main/check.yml`]); await store.updateRunner(failed, "check.yml", { status: ended }, 1); await runDurableObjectAlarm(store); expect((await store.getPipeline(repo, failed))?.runs).toMatchObject([ { workflow: "check.yml", status: ended }, { workflow: "deploy.yml", status: "cancelled", error: "check.yml did not succeed", }, { workflow: "notify.yml", status: "cancelled", error: "deploy.yml did not succeed", }, ]); expect(await store.releaseBlocked(repo, held)).toBe(2); await runDurableObjectAlarm(store); expect(runners.started).toEqual([`${repo}/main/check.yml`, `${repo}/pr/check.yml`]); await store.cancelRuns(repo, held, ["check.yml", "deploy.yml"]); expect((await store.getPipeline(repo, held))?.runs.map((r) => r.status)).toEqual([ "cancelled", "cancelled", ]); }, ); it("addresses the runs of older code by their stored Runner name", async () => { const { store, runners, create, status } = await setup("runners-legacy"); const [started, checked, queued] = [await create(), await create(), await create()]; const held = await store.createPipeline({ ...pipeline, trigger: { $type: "org.tangled.event.pullRequest" }, runs: [{ ...run, engine: "cloudflare", trusted: false }], }); await runInDurableObject(store, (_instance, state) => { const { sql, kv } = state.storage; sql.exec("UPDATE workflow_runs SET status = 'cancelled' WHERE pipeline = ?", held); sql.exec( `UPDATE workflow_runs SET status = 'started', build_uuid = pipeline || '/' || workflow, event_at = CASE WHEN pipeline = ? THEN 0 ELSE unixepoch() END, started_at = CASE WHEN pipeline = ? THEN '2026-01-01T00:00:00.000Z' END WHERE pipeline IN (?, ?)`, checked, started, started, checked, ); sql.exec(`DROP INDEX pipelines_unapproved; ALTER TABLE workflow_runs DROP COLUMN runner; ALTER TABLE workflow_runs DROP COLUMN dispatched_at; ALTER TABLE workflow_runs DROP COLUMN after_workflows; ALTER TABLE pipelines DROP COLUMN held; ALTER TABLE pipelines DROP COLUMN sender; ALTER TABLE snapshots DROP COLUMN workflow_hash`); kv.put("schemaVersion", 6); migrate(state.storage); kv.put("buildWorkAt", 0); }); runners.states.set(`${checked}/ci.yml`, { status: "success" }); await runDurableObjectAlarm(store); expect(runners.started).toEqual([`${queued}/ci.yml`]); expect(await status(checked)).toBe("success"); expect(runners.cancelled).toEqual([`${started}/ci.yml timeout`]); expect(await status(started)).toBe("timeout"); expect( await runInDurableObject(store, (_instance, state) => state.storage.sql .exec<{ rkey: string }>("SELECT rkey FROM pipelines WHERE held = 1") .toArray(), ), ).toEqual([{ rkey: held }]); }); }); }); describe("Jetstream filter", () => { const owner = env.OWNER_DID; const declarations = declarationsRoute(owner); const ownerDoc = `GET https://plc.directory/${owner}`; let filtered: string[][]; let listed: number; let routes: Record; beforeEach(() => { filtered = []; listed = 0; routes = { [ownerDoc]: () => Response.json(didDoc(owner, undefined, [pdsService])), [declarations]: () => { listed += 1; return Response.json({ records: [ { uri: `at://${owner}/org.tangled.repo.declaration/treadle`, value: { repo: repoDid }, }, { uri: `at://${owner}/org.tangled.repo.declaration/bad`, value: {} }, ], }); }, [`GET ${jetstreamUrl}`]: (url) => { filtered.push(url.searchParams.getAll("dids")); return jetstreamSocket([]); }, }; routeFetch(routes); }); it("watches the repos that the owner declares, and lists them again after 10 minutes", async () => { const store = exports.Store.getByName("filter-declared"); await tick(store); await tick(store); expect(filtered).toEqual([[repoDid], [repoDid]]); expect(listed).toBe(1); await runInDurableObject(store, (_instance, state) => { const cached = state.storage.kv.get<{ at: number }>("declaredRepos"); state.storage.kv.put("declaredRepos", { ...cached, at: Date.now() - 600_000 }); }); await tick(store); expect(listed).toBe(2); }); it("reads every page of declarations", async () => { routes[declarations] = () => Response.json({ cursor: "next", records: [ { uri: `at://${owner}/org.tangled.repo.declaration/treadle`, value: { repo: repoDid }, }, ], }); routes[`${declarations}&cursor=next`] = () => Response.json({ records: [ { uri: `at://${owner}/org.tangled.repo.declaration/jig`, value: { repo: "did:plc:jig" }, }, ], }); await tick(exports.Store.getByName("filter-pages")); expect(filtered).toEqual([[repoDid, "did:plc:jig"]]); }); it("watches the repos with pipelines until the PDS first answers", async () => { vi.spyOn(console, "warn").mockReturnValue(); const store = exports.Store.getByName("filter-unlisted"); await runInDurableObject(store, (_instance, state) => { seed(state.storage, [ { rkey: "3aaaaaaaaaaa2", repo: "did:plc:p", trigger: {}, commit: "abc", createdAt: "2026-11-06T00:00:00Z", runs: [], }, ]); }); routes[declarations] = () => new Response(null, { status: 500 }); await tick(store); expect(filtered).toEqual([["did:plc:p"]]); }); it("keeps the last list for a minute when the PDS fails", async () => { const warn = vi.spyOn(console, "warn").mockReturnValue(); const store = exports.Store.getByName("filter-fallback"); await runInDurableObject(store, (_instance, state) => { seed(state.storage, [ { rkey: "3aaaaaaaaaaa2", repo: "did:plc:p", trigger: {}, commit: "abc", createdAt: "2026-11-06T00:00:00Z", runs: [], }, ]); state.storage.kv.put("declaredRepos", { dids: ["did:plc:cached"], at: 0 }); }); routes[declarations] = () => { listed += 1; return new Response(null, { status: 500 }); }; await tick(store); await tick(store); expect(filtered).toEqual([["did:plc:cached"], ["did:plc:cached"]]); expect(listed).toBe(1); expect(warn).toHaveBeenCalledWith("declared repos unavailable", expect.any(String)); await runInDurableObject(store, (_instance, state) => { const cached = state.storage.kv.get<{ at: number }>("declaredRepos"); state.storage.kv.put("declaredRepos", { ...cached, at: (cached?.at ?? 0) - 60_000 }); }); await tick(store); expect(listed).toBe(2); }); });