Something went wrong. Try again.
Tangled Spindle built on Cloudflare Workers
Something went wrong. Try again.
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024import { runDurableObjectAlarm, runInDurableObject } from "cloudflare:test";import { env, exports } from "cloudflare:workers";import { beforeEach, describe, expect, it, vi } from "vitest";import type { RunStatus } from "../src/buildkite";import type { RunnerParams } from "../src/runner";import { pipelineUrl } from "../src/buildkite";import { migrate, runnerName, tid } from "../src/store";import type { RunnerState, RunnerStub } from "../src/store";import { declarationsRoute, didDoc, jetstreamSocket, jetstreamUrl, offline, pdsService, repoDid, routeFetch, seed,} from "./helpers";import type { Route } from "./helpers";
async function tick(store: DurableObjectStub) { await runInDurableObject(store, async (_instance, state) => { await state.storage.setAlarm(Date.now()); }); await runDurableObjectAlarm(store);}
describe("tid", () => { it("encodes a sortable 13-character record key", () => { const a = tid(1_700_000_000_000_000, 7); const b = tid(1_700_000_000_000_001, 0); expect(a).toMatch(/^[234567abcdefghij][234567abcdefghijklmnopqrstuvwxyz]{12}$/); expect(a < b).toBe(true); });});
const firstSchema = ` CREATE TABLE pipelines ( rkey TEXT PRIMARY KEY, repo TEXT NOT NULL, trigger TEXT NOT NULL, push_key TEXT UNIQUE, commit_sha TEXT NOT NULL, branch TEXT NOT NULL, created_at TEXT NOT NULL ); CREATE TABLE workflow_runs ( pipeline TEXT NOT NULL REFERENCES pipelines (rkey), workflow TEXT NOT NULL, raw TEXT NOT NULL, bk_org TEXT NOT NULL, bk_pipeline TEXT NOT NULL, clean_checkout INTEGER, build_number INTEGER, build_uuid TEXT, build_url TEXT, status TEXT NOT NULL, attempts INTEGER NOT NULL DEFAULT 0, error TEXT, PRIMARY KEY (pipeline, workflow) ); CREATE TABLE service_tokens ( issuer TEXT NOT NULL, jti TEXT NOT NULL, expires_at INTEGER NOT NULL, PRIMARY KEY (issuer, jti) ); INSERT INTO pipelines VALUES ('3aaaaaaaaaaa2', 'did:plc:r', '{}', 'k', 'abc', 'main', '2026-11-06T00:00:00Z'); INSERT INTO workflow_runs (pipeline, workflow, raw, bk_org, bk_pipeline, build_uuid, status) VALUES ('3aaaaaaaaaaa2', 'ci.yml', '', 'gpanders', 'treadle', 'uuid-1', 'started');`;
describe("runnerName", () => { const repo = "did:plc:r"; const pull = { $type: "org.tangled.event.pullRequest", pull: `at://${repo}/org.tangled.track.ticket/3ticket`, }; it.each([ ["a push", { $type: "org.tangled.event.push" }, "main", `${repo}/main/ci.yml`], ["a pull request from a branch", pull, "feature", `${repo}/pull/3ticket/ci.yml`], ["a pull request from a fork", pull, "pull/3ticket", `${repo}/pull/3ticket/ci.yml`], ])("names the Runner of %s by repo, ref and workflow", (_, trigger, branch, name) => { expect(runnerName(repo, trigger, branch, "ci.yml")).toBe(name); });});
describe("migrate", () => { it("upgrades the first schema in place and keeps its rows", async () => { const rows = await runInDurableObject( exports.Store.getByName("legacy"), (_instance, state) => { const { sql, kv } = state.storage; sql.exec( "DROP TABLE workflow_runs; DROP TABLE pipelines; DROP TABLE service_tokens; DROP TABLE IF EXISTS pulls; DROP TABLE IF EXISTS snapshots", ); kv.delete("schemaVersion"); sql.exec(firstSchema); migrate(state.storage); migrate(state.storage); return { version: kv.get("schemaVersion"), indexes: sql .exec( "SELECT name FROM sqlite_master WHERE type = 'index' AND sql IS NOT NULL ORDER BY name", ) .toArray(), tables: sql .exec( "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE '_cf%' ORDER BY name", ) .toArray() .map((row) => row["name"]), pipelines: sql .exec("SELECT rkey, released_at, message FROM pipelines") .toArray(), runs: sql .exec( "SELECT workflow, status, build_uuid, triggers, started_at, finished_at, event_at, engine, trusted, runner FROM workflow_runs", ) .toArray(), }; }, ); expect(rows).toEqual({ version: 12, indexes: [ { name: "pipelines_repo" }, { name: "pipelines_unapproved" }, { name: "workflow_runs_build" }, ], tables: ["pipelines", "pulls", "service_tokens", "snapshots", "workflow_runs"], pipelines: [{ rkey: "3aaaaaaaaaaa2", released_at: null, message: null }], runs: [ { workflow: "ci.yml", status: "started", build_uuid: "uuid-1", triggers: "[]", started_at: null, finished_at: null, event_at: null, engine: "buildkite", trusted: null, runner: null, }, ], }); });});
describe("Cloudflare runs", () => { beforeEach(() => { routeFetch({ [`GET https://plc.directory/${env.OWNER_DID}`]: offline, [`GET ${jetstreamUrl}`]: offline, }); });
const repo = "did:plc:cloudflare"; const cloudflare = exports.Store.getByName("cloudflare"); const pipeline = { repo, trigger: { $type: "org.tangled.event.push" }, pushKey: null, commit: "abc", branch: "main", }; const run = { workflow: "ci.yml", raw: "engine: cloudflare\nsteps: [{name: a, command: b}]\n", triggers: [], org: "gpanders", pipeline: "treadle", cleanCheckout: undefined, }; const job = (workflow: string, after?: string[]) => ({ ...run, workflow, engine: "cloudflare" as const, trusted: false, ...(after === undefined ? {} : { after }), });
it("stores the engine and the trust of each run", async () => { const rkey = await cloudflare.createPipeline({ ...pipeline, runs: [ { ...run, engine: "cloudflare", trusted: true }, { ...run, workflow: "other.yml", engine: "buildkite", trusted: false }, ], }); const rows = await runInDurableObject(cloudflare, (_instance, state) => state.storage.sql .exec( "SELECT workflow, engine, trusted, runner FROM workflow_runs WHERE pipeline = ? ORDER BY workflow", rkey, ) .toArray(), ); expect(rows).toEqual([ { workflow: "ci.yml", engine: "cloudflare", trusted: 1, runner: `${repo}/main/ci.yml` }, { workflow: "other.yml", engine: "buildkite", trusted: 0, runner: null }, ]); const stored = await cloudflare.getPipeline(repo, rkey); expect(stored?.runs.map((r) => r.engine)).toEqual(["cloudflare", "buildkite"]); });
it("applies Runner updates in order and wakes the alarm when a run ends", async () => { const rkey = await cloudflare.createPipeline({ ...pipeline, runs: [{ ...run, engine: "cloudflare", trusted: false }], }); const update = (status: RunStatus, ts: number) => cloudflare.updateRunner( rkey, "ci.yml", { status, startedAt: "2026-11-07T00:00:00.000Z" }, ts, ); await update("running", 10); await update("timeout", 20); await update("running", 15); const row = await runInDurableObject(cloudflare, async (instance, state) => { state.storage.kv.delete("buildWorkAt"); await instance.updateRunner(rkey, "ci.yml", { status: "timeout" }, 20); return { ...state.storage.sql .exec<{ finished_at: string | null }>( "SELECT status, build_uuid, started_at, finished_at, event_at FROM workflow_runs WHERE pipeline = ?", rkey, ) .one(), workAt: state.storage.kv.get<number>("buildWorkAt"), }; }); expect(row).toMatchObject({ status: "timeout", build_uuid: `${repo}/main/ci.yml`, started_at: "2026-11-07T00:00:00.000Z", event_at: 20, }); expect(row.finished_at).toEqual(expect.any(String)); expect(row.workAt).toBeLessThanOrEqual(Date.now()); expect(await cloudflare.cancelRuns(repo, rkey, ["ci.yml"])).toEqual([]); });
it("never starts a Cloudflare run as a Buildkite build", async () => { const alone = exports.Store.getByName("cloudflare-alone"); await runInDurableObject(alone, (object) => { object.runner = () => ({ start: () => Promise.resolve(), status: () => Promise.resolve({ status: "started" as const }), cancel: () => Promise.resolve(), }); }); const fetch = vi.spyOn(globalThis, "fetch"); const rkey = await alone.createPipeline({ ...pipeline, runs: [{ ...run, pipeline: "alone", engine: "cloudflare", trusted: false }], }); await alone.updateBuild({ id: "build-1", number: 1, url: undefined, status: "running", startedAt: undefined, finishedAt: undefined, ts: 1, pipeline: rkey, workflow: "ci.yml", slug: "alone", commit: "abc", }); await runDurableObjectAlarm(alone); const urls = fetch.mock.calls.map(([input]) => new Request(input).url); fetch.mockRestore(); // Alarms of the other Stores in this file may call Buildkite at any time. expect(urls.filter((url) => url.includes("/pipelines/alone/"))).toEqual([]); const stored = await alone.getPipeline(repo, rkey); expect(stored?.runs[0]?.build_number).toBeNull(); });
describe("Runners", () => { class FakeRunners { started: string[] = []; params = new Map<string, RunnerParams>(); cancelled: string[] = []; states = new Map<string, RunnerState | undefined>(); startError: Error | undefined; busy = false; cancelError: Error | undefined; onStart: ((params: RunnerParams) => Promise<unknown>) | undefined;
get = (name: string): RunnerStub => ({ start: async (params) => { if (this.startError !== undefined) { throw this.startError; } if (!this.busy) { await this.onStart?.(params); this.started.push(name); this.params.set(name, params); } return this.busy ? "busy" : undefined; }, status: () => Promise.resolve(this.states.get(name)), cancel: (_pipeline, status) => { if (this.cancelError !== undefined) { return Promise.reject(this.cancelError); } this.cancelled.push(status === undefined ? name : `${name} ${status}`); return Promise.resolve(); }, }); }
async function setup(storeName: string) { const store = exports.Store.getByName(storeName); const runners = new FakeRunners(); const instance = await runInDurableObject(store, (object) => { object.runner = runners.get; return object; }); const names = new Map<string, string>(); const create = async ( trusted = false, branch = `b${names.size}`, pushKey: string | null = `push ${names.size}`, raw = run.raw, ) => { const rkey = await store.createPipeline({ ...pipeline, branch, pushKey, runs: [{ ...run, raw, engine: "cloudflare", trusted }], }); names.set(rkey, `${repo}/${branch}/ci.yml`); return rkey; }; const name = (rkey = "") => names.get(rkey) ?? ""; const status = async (rkey: string) => (await store.getPipeline(repo, rkey))?.runs[0]?.status; return { store, instance, runners, create, name, status }; }
it("stores a run with an error as failed and never starts it", async () => { const { store, runners } = await setup("runners-invalid"); const rkey = await store.createPipeline({ ...pipeline, runs: [{ ...run, engine: "cloudflare", trusted: false, error: "bad file" }], }); await tick(store); expect(runners.started).toEqual([]); const [stored] = (await store.getPipeline(repo, rkey))?.runs ?? []; expect(stored).toMatchObject({ status: "failed", error: "bad file" }); expect(stored?.finished_at).toEqual(expect.any(String)); });
it("starts queued runs oldest first, at most ten at once", async () => { const { store, runners, create, name, status } = await setup("runners-queue"); const rkeys = []; for (let i = 0; i < 12; i++) { // oxlint-disable-next-line no-await-in-loop rkeys.push(await create(i === 0)); } await runDurableObjectAlarm(store); expect(runners.started).toEqual(rkeys.slice(0, 10).map(name)); expect(runners.params.get(name(rkeys[0]))).toEqual({ pipeline: rkeys[0], workflow: "ci.yml", repo, commit: "abc", raw: "engine: cloudflare\nsteps: [{name: a, command: b}]\n", trusted: true, branch: "b0", }); expect(runners.params.get(name(rkeys[1]))?.trusted).toBe(false); expect(await status(rkeys[10] ?? "")).toBe("pending");
await store.cancelRuns(repo, rkeys[10] ?? "", ["ci.yml"]); await store.updateRunner(rkeys[0] ?? "", "ci.yml", { status: "success" }, 1); await runDurableObjectAlarm(store); expect(runners.started.slice(10)).toEqual([name(rkeys[11])]); expect(await status(rkeys[10] ?? "")).toBe("cancelled"); });
it("keeps the newest snapshot of each tier from trusted runs and gives it out", async () => { const { store, runners, create, name } = await setup("runners-snapshots"); const [older, newer, untrusted] = [ await create(true), await create(true), await create(false), ]; await runDurableObjectAlarm(store); const image = "cloudflare/debian-trixie"; const report = (rkey: string, id: string, tier: "public" | "trusted", ts: number) => store.updateRunner( rkey, "ci.yml", { status: "success", snapshot: { id, image, tier } }, ts, ); await report(newer, "public-new", "public", 1); await report(older, "public-old", "public", 2); await report(untrusted, "untrusted", "public", 3); const [trusted, publicRun] = [await create(true), await create(false)]; await runDurableObjectAlarm(store); const given = (rkey: string) => runners.params.get(name(rkey))?.restore; expect(given(trusted)).toEqual({ id: "public-new", image, tier: "public" }); expect(given(publicRun)).toEqual({ id: "public-new", image, tier: "public" }); expect(given(older)).toBeUndefined();
await report(trusted, "trusted-1", "trusted", 4); const [second, third] = [await create(true), await create(false)]; await runInDurableObject(store, (_instance, state) => { state.storage.kv.put("buildWorkAt", 0); }); await runDurableObjectAlarm(store); expect(given(second)).toEqual({ id: "trusted-1", image, tier: "trusted" }); expect(given(third)).toEqual({ id: "public-new", image, tier: "public" });
await report(second, "public-2", "public", 5); const fourth = await create(true); await runInDurableObject(store, (_instance, state) => { state.storage.kv.put("buildWorkAt", 0); }); await runDurableObjectAlarm(store); expect(given(fourth)).toEqual({ id: "public-2", image, tier: "public" }); });
it("gives a snapshot only to runs of the same workflow file", async () => { const { store, runners, create, name } = await setup("runners-snapshot-file"); const saver = await create(true); await runDurableObjectAlarm(store); const snapshot = { id: "s", image: "cloudflare/debian-trixie", tier: "public" as const, }; await store.updateRunner(saver, "ci.yml", { status: "success", snapshot }, 1); const [same, edited] = [ await create(false), await create(true, undefined, undefined, `${run.raw}# edited\n`), ]; await runDurableObjectAlarm(store); const given = (rkey: string) => runners.params.get(name(rkey))?.restore; expect(given(same)).toEqual(snapshot); expect(given(edited)).toBeUndefined(); expect(runners.params.get(name(edited))?.cached).toBe(true); expect(runners.params.get(name(same))?.cached).toBeUndefined();
const newer = { ...snapshot, id: "s2" }; await store.updateRunner(edited, "ci.yml", { status: "success", snapshot: newer }, 2); const [old, current] = [ await create(false), await create(false, undefined, undefined, `${run.raw}# edited\n`), ]; await runInDurableObject(store, (_instance, state) => { state.storage.kv.put("buildWorkAt", 0); }); await runDurableObjectAlarm(store); expect(given(old)).toBeUndefined(); expect(given(current)).toEqual(newer);
for (const update of [ "UPDATE snapshots SET snapshot_id = 'overwritten'", "UPDATE snapshots SET workflow_hash = NULL", ]) { // oxlint-disable-next-line no-await-in-loop await runInDurableObject(store, (_instance, state) => { state.storage.sql.exec(update); state.storage.kv.put("buildWorkAt", 0); }); // oxlint-disable-next-line no-await-in-loop const legacy = await create(false, undefined, undefined, `${run.raw}# edited\n`); // oxlint-disable-next-line no-await-in-loop await runDurableObjectAlarm(store); expect(runners.started).toContain(name(legacy)); expect(given(legacy)).toBeUndefined(); } });
it("discards the snapshots of a workflow after a trusted run without cache", async () => { const { store, create } = await setup("runners-discard"); const [older, untrusted, failed, discarder, newer] = [ await create(true), await create(false), await create(true), await create(true), await create(true), ]; await runDurableObjectAlarm(store); const image = "cloudflare/debian-trixie"; await store.updateRunner( older, "ci.yml", { status: "success", snapshot: { id: "old", image, tier: "public" } }, 1, ); await runInDurableObject(store, (_instance, state) => { state.storage.sql.exec( `INSERT INTO snapshots (repo, workflow, tier, snapshot_id, image, pipeline) VALUES ('did:plc:other', 'ci.yml', 'public', 'other', ?, '2')`, image, ); }); const rows = () => runInDurableObject(store, async (_instance, state) => { await state.storage.deleteAlarm(); return state.storage.sql .exec("SELECT snapshot_id FROM snapshots ORDER BY snapshot_id") .toArray() .map((row) => row["snapshot_id"]); }); await store.updateRunner( untrusted, "ci.yml", { status: "success", discardCache: true }, 2, ); await store.updateRunner(failed, "ci.yml", { status: "failed", discardCache: true }, 2); expect(await rows()).toEqual(["old", "other"]);
await store.updateRunner( newer, "ci.yml", { status: "success", snapshot: { id: "new", image, tier: "trusted" } }, 3, ); await store.updateRunner( discarder, "ci.yml", { status: "success", discardCache: true }, 4, ); expect(await rows()).toEqual(["new", "other"]); });
it("tries a start again after an error", async () => { const { store, runners, create, status } = await setup("runners-retry"); runners.startError = new Error("overloaded"); const rkey = await create(); await runDurableObjectAlarm(store); await runDurableObjectAlarm(store); expect(await status(rkey)).toBe("pending"); expect( await runInDurableObject( store, (_instance, state) => state.storage.sql .exec<{ attempts: number }>("SELECT attempts FROM workflow_runs") .one().attempts, ), ).toBe(1); runners.startError = undefined; await runInDurableObject(store, (_instance, state) => { state.storage.kv.put("buildWorkAt", 0); }); await runDurableObjectAlarm(store); expect(await status(rkey)).toBe("started"); });
it("asks a busy Runner again at the next tick without backoff", async () => { const { store, runners, create, status } = await setup("runners-busy"); runners.busy = true; const rkey = await create(); await runDurableObjectAlarm(store); const row = await runInDurableObject(store, async (_instance, state) => ({ ...state.storage.sql .exec( "SELECT status, attempts, build_uuid FROM workflow_runs WHERE pipeline = ?", rkey, ) .one(), alarm: (await state.storage.getAlarm()) ?? Infinity, })); expect(row).toMatchObject({ status: "pending", attempts: 0, build_uuid: null }); expect(row.alarm).toBeLessThanOrEqual(Date.now() + 10_000); runners.busy = false; await runDurableObjectAlarm(store); expect(await status(rkey)).toBe("started"); });
it.each([ ["an error", new Error("overloaded"), false, "gave up: overloaded"], ["a busy Runner", undefined, true, "gave up: the Runner is busy"], ])("gives up a start after three days of %s", async (_, error, busy, reason) => { const { store, runners, create } = await setup(`runners-refused-${busy}`); runners.startError = error; runners.busy = busy; const rkey = await create(); await runInDurableObject(store, (_instance, state) => { state.storage.sql.exec("UPDATE pipelines SET created_at = '2026-01-01T00:00:00Z'"); state.storage.kv.put("buildWorkAt", 0); }); await runDurableObjectAlarm(store); expect((await store.getPipeline(repo, rkey))?.runs[0]).toMatchObject({ status: "failed", error: reason, }); });
it("cancels a Runner whose run was cancelled while it started", async () => { const { store, instance, runners, create, name, status } = await setup("runners-race"); runners.onStart = (params) => instance.cancelRuns(repo, params.pipeline, ["ci.yml"]); const rkey = await create(); await runDurableObjectAlarm(store); expect(await status(rkey)).toBe("cancelled"); expect(runners.cancelled).toEqual([name(rkey)]); });
it("checks quiet runs with their Runner and frees the slot of a lost run", async () => { const { store, runners, create, name } = await setup("runners-check"); const rkeys = []; for (let i = 0; i < 11; i++) { // oxlint-disable-next-line no-await-in-loop rkeys.push(await create()); } const [done = "", lost = ""] = rkeys; await runDurableObjectAlarm(store); for (const rkey of rkeys.slice(0, 10)) { runners.states.set(name(rkey), { status: "running" }); } runners.states.set(name(lost), undefined); await runInDurableObject(store, (_instance, state) => { state.storage.kv.put("buildWorkAt", 0); }); await runDurableObjectAlarm(store); expect(runners.started.at(-1)).toBe(name(rkeys[10])); runners.states.set(name(done), { status: "success", finishedAt: "x" }); await runInDurableObject(store, (_instance, state) => { state.storage.sql.exec("UPDATE workflow_runs SET event_at = 0"); state.storage.kv.put("buildWorkAt", 0); }); await runDurableObjectAlarm(store); expect((await store.getPipeline(repo, done))?.runs[0]).toMatchObject({ status: "success", }); expect((await store.getPipeline(repo, lost))?.runs[0]).toMatchObject({ status: "failed", error: "the Runner lost the run", }); });
it("times out a run that its Runner has not ended after the longest timeout", async () => { const { store, runners, create, name, status } = await setup("runners-stuck"); const [stuck, fresh] = [await create(), await create()]; await runDurableObjectAlarm(store); runners.states.set(name(stuck), { status: "running" }); runners.states.set(name(fresh), { status: "running" }); await runInDurableObject(store, (_instance, state) => { state.storage.sql.exec( "UPDATE workflow_runs SET dispatched_at = ? WHERE pipeline = ?", new Date(Date.now() - 7 * 3_600_000).toISOString(), stuck, ); state.storage.kv.put("buildWorkAt", 0); }); await runDurableObjectAlarm(store); expect((await store.getPipeline(repo, stuck))?.runs[0]).toMatchObject({ status: "timeout", error: "the run timed out", }); expect(runners.cancelled).toEqual([`${name(stuck)} timeout`]); expect(await status(fresh)).toBe("running"); });
it("cancels a started run through its Runner", async () => { const { store, runners, create, name, status } = await setup("runners-cancel"); const rkey = await create(); await runDurableObjectAlarm(store); runners.cancelError = new Error("unreachable"); expect(await store.cancelRuns(repo, rkey, ["ci.yml"])).toEqual([]); expect(await status(rkey)).toBe("started"); runners.cancelError = undefined; expect(await store.cancelRuns(repo, rkey, ["ci.yml"])).toEqual([]); expect(runners.cancelled).toEqual([name(rkey)]); expect(await status(rkey)).toBe("cancelled"); });
it("runs one run at a time for each Runner name and starts only the newest waiting run", async () => { const { store, runners, create, name, status } = await setup("runners-serial"); const first = await create(false, "main"); await runDurableObjectAlarm(store); runners.states.set(name(first), { status: "running" }); const skipped = await create(false, "main"); const newest = await create(false, "main"); const other = await create(false, "dev"); const held = await create(false, "main"); await runInDurableObject(store, (_instance, state) => { state.storage.sql.exec( "UPDATE workflow_runs SET status = 'blocked' WHERE pipeline = ?", held, ); state.storage.kv.put("buildWorkAt", 0); }); await runDurableObjectAlarm(store); expect(runners.started).toEqual([name(first), name(other)]); expect(await status(skipped)).toBe("cancelled"); expect(await status(newest)).toBe("pending"); expect(await status(held)).toBe("blocked");
const ts = Math.floor(Date.now() / 1000) + 1; await store.updateRunner(first, "ci.yml", { status: "success" }, ts); await runDurableObjectAlarm(store); expect(runners.started).toEqual([name(first), name(other), name(newest)]); expect(runners.params.get(name(newest))?.pipeline).toBe(newest); expect(await status(newest)).toBe("started"); });
it("keeps a waiting run that a manual or less trusted run follows", async () => { const { store, runners, create, name, status } = await setup("runners-supersede"); const first = await create(true, "main"); await runDurableObjectAlarm(store); runners.states.set(name(first), { status: "running" }); const push = await create(true, "main"); const manual = await create(true, "main", null); const untrusted = await create(false, "main"); await runDurableObjectAlarm(store); expect([await status(push), await status(manual), await status(untrusted)]).toEqual([ "pending", "pending", "pending", ]);
const newest = await create(true, "main"); await runDurableObjectAlarm(store); expect([await status(push), await status(manual), await status(untrusted)]).toEqual([ "cancelled", "cancelled", "cancelled", ]); expect(await status(newest)).toBe("pending"); });
it("starts a run of either engine only after the runs that it waits for succeed", async () => { const posted: string[] = []; const builds = (slug: string): Route => () => { posted.push(slug); return Response.json({ id: `build-${slug}`, number: 1 }, { status: 201 }); }; routeFetch({ [`GET https://plc.directory/${env.OWNER_DID}`]: offline, [`GET ${jetstreamUrl}`]: offline, [`POST ${pipelineUrl("gpanders", "after-check")}/builds`]: builds("after-check"), [`POST ${pipelineUrl("gpanders", "after-publish")}/builds`]: builds("after-publish"), }); const { store, runners } = await setup("runners-after"); const buildkite = { ...run, engine: "buildkite" as const, trusted: false }; await store.createPipeline({ ...pipeline, runs: [ { ...buildkite, workflow: "check.yml", pipeline: "after-check" }, { ...run, workflow: "deploy.yml", engine: "cloudflare", trusted: true, after: ["check.yml"], }, { ...buildkite, workflow: "publish.yml", pipeline: "after-publish", after: ["check.yml"], }, ], }); await runDurableObjectAlarm(store); expect(posted).toEqual(["after-check"]); expect(runners.started).toEqual([]); const workAt = await runInDurableObject(store, (_instance, state) => state.storage.kv.get<number>("buildWorkAt"), ); expect(workAt).toBeGreaterThan(Date.now() + 5 * 60_000);
await store.updateBuild({ id: "build-after-check", number: 1, url: undefined, status: "success", startedAt: undefined, finishedAt: undefined, ts: 1, pipeline: undefined, workflow: undefined, slug: undefined, commit: undefined, }); const alarm = await runInDurableObject(store, (_instance, state) => state.storage.getAlarm(), ); expect(alarm).toBeLessThanOrEqual(Date.now()); await runDurableObjectAlarm(store); expect(posted).toEqual(["after-check", "after-publish"]); expect(runners.started).toEqual([`${repo}/main/deploy.yml`]); });
it.each(["failed", "timeout", "cancelled"] as const)( "cancels the runs that wait for a %s run, and holds waiting runs for Approve", async (ended) => { const { store, runners } = await setup(`runners-after-${ended}`); const runs = [job("check.yml"), job("deploy.yml", ["check.yml"])]; const failed = await store.createPipeline({ ...pipeline, runs: [...runs, job("notify.yml", ["deploy.yml"])], }); const held = await store.createPipeline({ ...pipeline, branch: "pr", held: true, runs, }); await runDurableObjectAlarm(store); expect(runners.started).toEqual([`${repo}/main/check.yml`]);
await store.updateRunner(failed, "check.yml", { status: ended }, 1); await runDurableObjectAlarm(store); expect((await store.getPipeline(repo, failed))?.runs).toMatchObject([ { workflow: "check.yml", status: ended }, { workflow: "deploy.yml", status: "cancelled", error: "check.yml did not succeed", }, { workflow: "notify.yml", status: "cancelled", error: "deploy.yml did not succeed", }, ]);
expect(await store.releaseBlocked(repo, held)).toBe(2); await runDurableObjectAlarm(store); expect(runners.started).toEqual([`${repo}/main/check.yml`, `${repo}/pr/check.yml`]); await store.cancelRuns(repo, held, ["check.yml", "deploy.yml"]); expect((await store.getPipeline(repo, held))?.runs.map((r) => r.status)).toEqual([ "cancelled", "cancelled", ]); }, );
it("addresses the runs of older code by their stored Runner name", async () => { const { store, runners, create, status } = await setup("runners-legacy"); const [started, checked, queued] = [await create(), await create(), await create()]; const held = await store.createPipeline({ ...pipeline, trigger: { $type: "org.tangled.event.pullRequest" }, runs: [{ ...run, engine: "cloudflare", trusted: false }], }); await runInDurableObject(store, (_instance, state) => { const { sql, kv } = state.storage; sql.exec("UPDATE workflow_runs SET status = 'cancelled' WHERE pipeline = ?", held); sql.exec( `UPDATE workflow_runs SET status = 'started', build_uuid = pipeline || '/' || workflow, event_at = CASE WHEN pipeline = ? THEN 0 ELSE unixepoch() END, started_at = CASE WHEN pipeline = ? THEN '2026-01-01T00:00:00.000Z' END WHERE pipeline IN (?, ?)`, checked, started, started, checked, ); sql.exec(`DROP INDEX pipelines_unapproved; ALTER TABLE workflow_runs DROP COLUMN runner; ALTER TABLE workflow_runs DROP COLUMN dispatched_at; ALTER TABLE workflow_runs DROP COLUMN after_workflows; ALTER TABLE pipelines DROP COLUMN held; ALTER TABLE pipelines DROP COLUMN sender; ALTER TABLE snapshots DROP COLUMN workflow_hash`); kv.put("schemaVersion", 6); migrate(state.storage); kv.put("buildWorkAt", 0); }); runners.states.set(`${checked}/ci.yml`, { status: "success" }); await runDurableObjectAlarm(store); expect(runners.started).toEqual([`${queued}/ci.yml`]); expect(await status(checked)).toBe("success"); expect(runners.cancelled).toEqual([`${started}/ci.yml timeout`]); expect(await status(started)).toBe("timeout"); expect( await runInDurableObject(store, (_instance, state) => state.storage.sql .exec<{ rkey: string }>("SELECT rkey FROM pipelines WHERE held = 1") .toArray(), ), ).toEqual([{ rkey: held }]); }); });});
describe("Jetstream filter", () => { const owner = env.OWNER_DID; const declarations = declarationsRoute(owner); const ownerDoc = `GET https://plc.directory/${owner}`; let filtered: string[][]; let listed: number; let routes: Record<string, Route>;
beforeEach(() => { filtered = []; listed = 0; routes = { [ownerDoc]: () => Response.json(didDoc(owner, undefined, [pdsService])), [declarations]: () => { listed += 1; return Response.json({ records: [ { uri: `at://${owner}/org.tangled.repo.declaration/treadle`, value: { repo: repoDid }, }, { uri: `at://${owner}/org.tangled.repo.declaration/bad`, value: {} }, ], }); }, [`GET ${jetstreamUrl}`]: (url) => { filtered.push(url.searchParams.getAll("dids")); return jetstreamSocket([]); }, }; routeFetch(routes); });
it("watches the repos that the owner declares, and lists them again after 10 minutes", async () => { const store = exports.Store.getByName("filter-declared"); await tick(store); await tick(store); expect(filtered).toEqual([[repoDid], [repoDid]]); expect(listed).toBe(1);
await runInDurableObject(store, (_instance, state) => { const cached = state.storage.kv.get<{ at: number }>("declaredRepos"); state.storage.kv.put("declaredRepos", { ...cached, at: Date.now() - 600_000 }); }); await tick(store); expect(listed).toBe(2); });
it("reads every page of declarations", async () => { routes[declarations] = () => Response.json({ cursor: "next", records: [ { uri: `at://${owner}/org.tangled.repo.declaration/treadle`, value: { repo: repoDid }, }, ], }); routes[`${declarations}&cursor=next`] = () => Response.json({ records: [ { uri: `at://${owner}/org.tangled.repo.declaration/jig`, value: { repo: "did:plc:jig" }, }, ], }); await tick(exports.Store.getByName("filter-pages")); expect(filtered).toEqual([[repoDid, "did:plc:jig"]]); });
it("watches the repos with pipelines until the PDS first answers", async () => { vi.spyOn(console, "warn").mockReturnValue(); const store = exports.Store.getByName("filter-unlisted"); await runInDurableObject(store, (_instance, state) => { seed(state.storage, [ { rkey: "3aaaaaaaaaaa2", repo: "did:plc:p", trigger: {}, commit: "abc", createdAt: "2026-11-06T00:00:00Z", runs: [], }, ]); }); routes[declarations] = () => new Response(null, { status: 500 }); await tick(store); expect(filtered).toEqual([["did:plc:p"]]); });
it("keeps the last list for a minute when the PDS fails", async () => { const warn = vi.spyOn(console, "warn").mockReturnValue(); const store = exports.Store.getByName("filter-fallback"); await runInDurableObject(store, (_instance, state) => { seed(state.storage, [ { rkey: "3aaaaaaaaaaa2", repo: "did:plc:p", trigger: {}, commit: "abc", createdAt: "2026-11-06T00:00:00Z", runs: [], }, ]); state.storage.kv.put("declaredRepos", { dids: ["did:plc:cached"], at: 0 }); }); routes[declarations] = () => { listed += 1; return new Response(null, { status: 500 }); }; await tick(store); await tick(store); expect(filtered).toEqual([["did:plc:cached"], ["did:plc:cached"]]); expect(listed).toBe(1); expect(warn).toHaveBeenCalledWith("declared repos unavailable", expect.any(String));
await runInDurableObject(store, (_instance, state) => { const cached = state.storage.kv.get<{ at: number }>("declaredRepos"); state.storage.kv.put("declaredRepos", { ...cached, at: (cached?.at ?? 0) - 60_000 }); }); await tick(store); expect(listed).toBe(2); });});