import { createExecutionContext, runDurableObjectAlarm, runInDurableObject } from "cloudflare:test"; import { env, exports } from "cloudflare:workers"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { ensureAlarm } from "../src/store"; import { buildkiteWebhook } from "../src/webhook"; import { jetstreamSocket, jetstreamUrl, offline, seed, signature, repoDid as repo, } from "./helpers"; import type { SeedRun } from "./helpers"; const store = exports.Store.getByName("main"); const rkey = "3aaaaaaaaaaa2"; const commit = "a".repeat(40); const maxBody = 1024 * 1024; const hourMs = 60 * 60_000; const checkMs = 10 * 60_000; const buildUrl = "https://api.buildkite.com/v2/organizations/gpanders/pipelines/treadle/builds/7"; type RunRow = { workflow: string; status: string; attempts: number; error: string | null; build_uuid: string | null; build_number: number | null; build_url: string | null; started_at: string | null; finished_at: string | null; event_at: number | null; }; async function reset(seeded: SeedRun[], createdAt = new Date().toISOString()): Promise { await runInDurableObject(store, async (_instance, state) => { seed(state.storage, [ { rkey, repo, trigger: { $type: "org.tangled.event.push" }, commit, createdAt, runs: seeded, }, ]); state.storage.kv.delete("buildWorkAt"); await state.storage.deleteAlarm(); }); } beforeEach(async () => { await reset([{ workflow: "ci.yml", status: "started", buildUuid: "uuid-1" }]); }); afterEach(async () => { await runInDurableObject(store, (_instance, state) => state.storage.deleteAlarm()); }); function runs(): Promise { return runInDurableObject(store, (_instance, state) => state.storage.sql .exec( `SELECT workflow, status, attempts, error, build_uuid, build_number, build_url, started_at, finished_at, event_at FROM workflow_runs ORDER BY workflow`, ) .toArray(), ); } async function run(): Promise { return (await runs())[0]; } function alarmTime(): Promise { return runInDurableObject(store, (_instance, state) => state.storage.getAlarm()); } function workAt(): Promise { return runInDurableObject( store, (_instance, state) => state.storage.kv.get("buildWorkAt") ?? null, ); } type Fetch = (input: RequestInfo | URL, init?: RequestInit) => Promise; // Answers the Jetstream stream of each alarm, fails the owner's DID document, and passes the // other requests to the handler. function mockFetch(handler: Fetch = () => Promise.reject(new Error("unexpected fetch"))) { const fetches = vi.fn(handler); vi.spyOn(globalThis, "fetch").mockImplementation((input, init) => { const { url } = new Request(input, init); if (url.startsWith(jetstreamUrl)) { return Promise.resolve(jetstreamSocket([])); } return url === `https://plc.directory/${env.OWNER_DID}` ? offline() : fetches(input, init); }); return fetches; } function now(): number { return Math.floor(Date.now() / 1000); } interface DeliveryOptions { event?: string; timestamp?: number; secret?: string; header?: string | null; raw?: string; slug?: string; prefix?: string; } async function deliver(payload: object, options: DeliveryOptions = {}): Promise { const event = options.event ?? "build.running"; const body = options.raw ?? JSON.stringify({ event, build: payload, pipeline: { slug: options.slug ?? "treadle" } }); const header = options.header === undefined ? await signature( body, options.timestamp ?? now(), options.secret ?? env.BUILDKITE_WEBHOOK_SECRET ?? "", ) : options.header; const headers: Record = { "Content-Type": "application/json", "X-Buildkite-Event": event, }; if (header !== null) { headers["X-Buildkite-Signature"] = `${options.prefix ?? ""}${header}`; } return await exports.default.fetch("https://treadle.example.com/webhooks/buildkite", { method: "POST", headers, body, }); } function build(state: string, extra: object = {}): object { return { id: "uuid-1", number: 7, state, commit, web_url: "https://buildkite.com/b/7", ...extra, }; } async function alarm(): Promise { await runInDurableObject(store, (_instance, state) => state.storage.setAlarm(Date.now() + 60_000), ); expect(await runDurableObjectAlarm(store)).toBe(true); } describe("verification", () => { it.each([-290, 0, 290])("accepts a delivery signed %i s from now", async (skew) => { const res = await deliver(build("running"), { timestamp: now() + skew }); expect(res.status).toBe(200); expect(await run()).toMatchObject({ status: "running" }); }); it.each<[string, () => DeliveryOptions]>([ ["no signature", () => ({ header: null })], ["another secret", () => ({ secret: "wrong" })], ["a malformed header", () => ({ header: "signature=abc" })], ["a non-hex signature", () => ({ header: `timestamp=${now()},signature=zz` })], ["a prefixed header", () => ({ prefix: "x" })], ["a timestamp 310 s old", () => ({ timestamp: now() - 310 })], ["a timestamp 310 s ahead", () => ({ timestamp: now() + 310 })], ])("rejects %s", async (_, options) => { const res = await deliver(build("running"), options()); expect(res.status).toBe(401); expect(await run()).toMatchObject({ status: "started" }); }); it("rejects every delivery when no secret is set", async () => { const { BUILDKITE_WEBHOOK_SECRET: _, ...unset } = env; const body = JSON.stringify({ event: "build.running", build: build("running") }); const request = new Request("https://treadle.example.com/webhooks/buildkite", { method: "POST", headers: { "Content-Length": String(body.length), "X-Buildkite-Event": "build.running", "X-Buildkite-Signature": await signature(body, now(), "secret"), }, body, }); const res = await buildkiteWebhook(request, unset, createExecutionContext()); expect(res.status).toBe(401); }); it("checks the signature header before the body size", async () => { const res = await deliver({}, { header: null, raw: "x".repeat(maxBody + 1) }); expect(res.status).toBe(401); }); it("rejects a signed body that is not JSON", async () => { const res = await deliver({}, { raw: "{" }); expect(res.status).toBe(400); }); it.each([ ["exactly 1 MiB", 0, 200], ["one byte over 1 MiB", 1, 413], ])("answers a body of %s", async (_, extra, status) => { const prefix = '{"pad":"'; const raw = `${prefix}${"a".repeat(maxBody - prefix.length - 2 + extra)}"}`; const res = await deliver({}, { raw }); expect(res.status).toBe(status); }); it("rejects a body without a length", async () => { const bytes = new TextEncoder().encode("{}"); const res = await exports.default.fetch("https://treadle.example.com/webhooks/buildkite", { method: "POST", headers: { "X-Buildkite-Signature": await signature("{}", now(), "x") }, body: new ReadableStream({ start(controller) { controller.enqueue(bytes); controller.close(); }, }), }); expect(res.status).toBe(413); }); }); describe("build state", () => { it.each<[string, object, DeliveryOptions, string]>([ ["a running build", build("running"), {}, "running"], [ "a finished build at a block step", build("passed", { blocked: true }), { event: "build.finished" }, "blocked", ], ["a job event", build("passed"), { event: "job.finished" }, "started"], ["a state treadle ignores", build("waiting"), {}, "started"], ["a build it did not start", build("passed", { id: "uuid-other" }), {}, "started"], ])("maps %s to %s", async (_, payload, options, status) => { expect((await deliver(payload, options)).status).toBe(200); expect(await run()).toMatchObject({ status }); }); it("records the build times and ignores a time that is not a date", async () => { const t = now(); await deliver(build("running", { started_at: "soon" }), { timestamp: t - 3 }); expect(await run()).toMatchObject({ status: "running", started_at: null }); await deliver(build("running", { started_at: "2026-11-06T10:00:05Z" }), { timestamp: t - 2, }); await deliver( build("passed", { started_at: "2026-11-06T10:00:05Z", finished_at: "2026-11-06T10:10:00.5Z", }), { timestamp: t - 1 }, ); expect(await run()).toMatchObject({ status: "success", started_at: "2026-11-06T10:00:05.000Z", finished_at: "2026-11-06T10:10:00.500Z", event_at: t - 1, }); }); }); describe("ordering", () => { it.each([ ["a stale failed after passed", "passed", "failed", "success"], ["a stale scheduled after running", "running", "scheduled", "running"], ["a stale running after passed", "passed", "running", "success"], ])("drops %s", async (_, first, late, status) => { const t = now(); await deliver(build(first), { timestamp: t }); await deliver(build(late), { timestamp: t - 1 }); expect(await run()).toMatchObject({ status, event_at: t }); }); it.each([ ["a lower rank", "passed", "running", "success"], ["the same rank", "passed", "failed", "failed"], ["a higher rank", "running", "passed", "success"], ])("applies an event of the same second only for %s or higher", async (_, a, b, status) => { const t = now(); await deliver(build(a), { timestamp: t }); await deliver(build(b), { timestamp: t }); expect(await run()).toMatchObject({ status }); }); it("follows a finished build that runs again after a job retry", async () => { const t = now(); const startedAt = "2026-11-06T10:00:05Z"; await deliver( build("failed", { started_at: startedAt, finished_at: "2026-11-06T10:01:00Z" }), { event: "build.finished", timestamp: t - 2 }, ); await deliver(build("running", { started_at: startedAt }), { timestamp: t - 1 }); expect(await run()).toMatchObject({ status: "running", finished_at: null }); await deliver( build("passed", { started_at: startedAt, finished_at: "2026-11-06T10:06:00Z" }), { event: "build.finished", timestamp: t }, ); expect(await run()).toMatchObject({ status: "success", started_at: "2026-11-06T10:00:05.000Z", finished_at: "2026-11-06T10:06:00.000Z", }); }); it("moves a running build to blocked and back", async () => { const t = now(); await deliver(build("running"), { timestamp: t - 2 }); await deliver(build("passed", { blocked: true, finished_at: "2026-11-06T10:01:00Z" }), { event: "build.finished", timestamp: t - 1, }); expect(await run()).toMatchObject({ status: "blocked", finished_at: null }); await deliver(build("running"), { timestamp: t }); expect(await run()).toMatchObject({ status: "running" }); }); it("keeps build details that a later event omits", async () => { const t = now(); await deliver(build("running", { started_at: "2026-11-06T10:00:05Z" }), { timestamp: t - 1, }); await deliver({ id: "uuid-1", state: "failing", commit }, { timestamp: t }); expect(await run()).toMatchObject({ status: "running", build_number: 7, build_url: "https://buildkite.com/b/7", started_at: "2026-11-06T10:00:05.000Z", }); }); }); describe("meta_data fallback", () => { const meta = { "treadle:pipeline": rkey, "treadle:workflow": "ci.yml" }; it.each(["pending", "started"])( "finds a %s run that has no build yet and records the build", async (status) => { await reset([{ workflow: "ci.yml", status }]); await deliver(build("scheduled", { id: "uuid-9", meta_data: meta })); expect(await run()).toMatchObject({ status: "started", build_uuid: "uuid-9", build_number: 7, build_url: "https://buildkite.com/b/7", }); }, ); it.each<[string, SeedRun, object, DeliveryOptions]>([ [ "a run of another build", { workflow: "ci.yml", status: "started", buildUuid: "uuid-1" }, {}, {}, ], ["a failed run", { workflow: "ci.yml", status: "failed" }, {}, {}], [ "a build of another commit", { workflow: "ci.yml", status: "started" }, { commit: "b".repeat(40) }, {}, ], [ "a build of another pipeline", { workflow: "ci.yml", status: "started" }, {}, { slug: "other" }, ], ])("does not claim %s", async (_, seeded, extra, options) => { await reset([seeded]); await deliver(build("passed", { id: "uuid-9", meta_data: meta, ...extra }), options); expect(await run()).toMatchObject({ status: seeded.status, build_uuid: seeded.buildUuid ?? null, }); }); it.each([ ["starts", () => Response.json({ id: "uuid-9", number: 7 }, { status: 201 })], ["refuses", () => Response.json({ message: "No pipeline" }, { status: 404 })], ["fails", () => new Response("unavailable", { status: 500 })], ])("keeps a webhook that arrives before Buildkite %s the build", async (_, answer) => { await reset([{ workflow: "ci.yml", status: "pending" }]); await runInDurableObject(store, (_instance, state) => state.storage.setAlarm(Date.now() + 60_000), ); const fetches = mockFetch(async () => { const res = await deliver(build("running", { id: "uuid-9", meta_data: meta })); expect(res.status).toBe(200); return answer(); }); expect(await runDurableObjectAlarm(store)).toBe(true); expect(fetches).toHaveBeenCalledTimes(1); expect(await run()).toMatchObject({ status: "running", build_uuid: "uuid-9", attempts: 0, error: null, }); }); it("does not start a run that a webhook claimed during the alarm", async () => { await reset([ { workflow: "a.yml", status: "pending" }, { workflow: "b.yml", status: "pending" }, ]); await runInDurableObject(store, (_instance, state) => state.storage.setAlarm(Date.now() + 60_000), ); const fetches = mockFetch(async (input, init) => { const request = new Request(input, init); const body: { meta_data: Record } = await request.json(); const other = body.meta_data["treadle:workflow"] === "a.yml" ? "b.yml" : "a.yml"; await deliver( build("scheduled", { id: `uuid-${other}`, meta_data: { "treadle:pipeline": rkey, "treadle:workflow": other }, }), ); return Response.json({ id: "uuid-first", number: 1 }, { status: 201 }); }); expect(await runDurableObjectAlarm(store)).toBe(true); expect(fetches).toHaveBeenCalledTimes(1); expect((await runs()).map((row) => row.status)).toEqual(["started", "started"]); }); }); describe("reconciliation", () => { const stale = new Date(Date.now() - 2 * hourMs).toISOString(); it("asks Buildkite about a build without news for ten minutes", async () => { await reset( [{ workflow: "ci.yml", status: "started", buildUuid: "uuid-1", buildNumber: 7 }], stale, ); const fetches = mockFetch((input, init) => { const request = new Request(input, init); expect(`${request.method} ${request.url}`).toBe(`GET ${buildUrl}`); expect(request.headers.get("authorization")).toBe("Bearer test-token"); return Promise.resolve( Response.json(build("passed", { finished_at: "2026-11-06T10:01:00Z" })), ); }); const before = now(); await alarm(); expect(fetches).toHaveBeenCalledTimes(1); expect(await run()).toMatchObject({ status: "success", finished_at: "2026-11-06T10:01:00.000Z", }); expect((await run())?.event_at).toBeGreaterThanOrEqual(before); expect(await workAt()).toBeNull(); }); it("does not ask Buildkite before the next check is due", async () => { await reset( [{ workflow: "ci.yml", status: "started", buildUuid: "uuid-1", buildNumber: 7 }], stale, ); const at = Date.now() + checkMs; await runInDurableObject(store, (_instance, state) => { state.storage.kv.put("buildWorkAt", at); }); const fetches = mockFetch(); await alarm(); expect(fetches).not.toHaveBeenCalled(); expect(await workAt()).toBe(at); }); it("keeps a webhook that arrives while it asks Buildkite", async () => { await reset( [{ workflow: "ci.yml", status: "running", buildUuid: "uuid-1", buildNumber: 7 }], stale, ); mockFetch(async () => { await deliver(build("passed"), { event: "build.finished", timestamp: now() + 1 }); return Response.json(build("running")); }); await alarm(); expect(await run()).toMatchObject({ status: "success" }); }); it("stops asking about a build from more than three days ago", async () => { await reset( [{ workflow: "ci.yml", status: "running", buildNumber: 7 }], new Date(Date.now() - 73 * hourMs).toISOString(), ); const fetches = mockFetch(); await alarm(); expect(fetches).not.toHaveBeenCalled(); expect(await workAt()).toBeNull(); }); it.each<[string, SeedRun, string]>([ [ "a build with recent news", { workflow: "ci.yml", status: "running", buildNumber: 7, eventAt: Math.floor(Date.now() / 1000) - 60, }, stale, ], [ "a recent build without news", { workflow: "ci.yml", status: "started", buildNumber: 7 }, new Date().toISOString(), ], ])("waits for %s and checks again in ten minutes", async (_, seeded, createdAt) => { await reset([seeded], createdAt); const fetches = mockFetch(); await alarm(); expect(fetches).not.toHaveBeenCalled(); const delay = ((await workAt()) ?? 0) - Date.now(); expect(delay).toBeGreaterThan(checkMs - 5_000); expect(delay).toBeLessThanOrEqual(checkMs); }); it("checks an open build before a pending run's long backoff", async () => { await reset( [ { workflow: "ci.yml", status: "started", buildNumber: 7 }, { workflow: "lint.yml", status: "pending", attempts: 5 }, ], new Date().toISOString(), ); mockFetch(() => Promise.resolve(new Response("unavailable", { status: 502 }))); await alarm(); const delay = ((await workAt()) ?? 0) - Date.now(); expect(delay).toBeGreaterThan(checkMs - 5_000); expect(delay).toBeLessThanOrEqual(checkMs); }); it("keeps a run when Buildkite does not answer", async () => { await reset([{ workflow: "ci.yml", status: "running", buildNumber: 7 }], stale); mockFetch(() => Promise.resolve(new Response("unavailable", { status: 502 }))); await alarm(); expect(await run()).toMatchObject({ status: "running", event_at: null }); expect(await workAt()).toBeGreaterThan(Date.now() + checkMs - 5_000); }); it("schedules an alarm at startup without pipelines", async () => { await runInDurableObject(store, async (_instance, state) => { state.storage.sql.exec("DELETE FROM workflow_runs; DELETE FROM pipelines"); await state.storage.deleteAlarm(); await ensureAlarm(state.storage); }); expect(await alarmTime()).toBeLessThanOrEqual(Date.now() + 10_000); }); it("moves a later alarm to now at startup", async () => { await runInDurableObject(store, async (_instance, state) => { await state.storage.setAlarm(Date.now() + hourMs); await ensureAlarm(state.storage); }); expect(await alarmTime()).toBeLessThanOrEqual(Date.now() + 10_000); }); it("keeps an alarm that is already set at startup", async () => { const at = Date.now() + 5_000; await runInDurableObject(store, async (_instance, state) => { await state.storage.setAlarm(at); await ensureAlarm(state.storage); }); expect(await alarmTime()).toBe(at); }); });