Something went wrong. Try again.
Tangled Spindle built on Cloudflare Workers
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583import { createExecutionContext, runDurableObjectAlarm, runInDurableObject } from "cloudflare:test";import { env, exports } from "cloudflare:workers";import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";import { ensureAlarm } from "../src/store";import { buildkiteWebhook } from "../src/webhook";import { jetstreamSocket, jetstreamUrl, offline, seed, signature, repoDid as repo,} from "./helpers";import type { SeedRun } from "./helpers";
const store = exports.Store.getByName("main");const rkey = "3aaaaaaaaaaa2";const commit = "a".repeat(40);const maxBody = 1024 * 1024;const hourMs = 60 * 60_000;const checkMs = 10 * 60_000;const buildUrl = "https://api.buildkite.com/v2/organizations/gpanders/pipelines/treadle/builds/7";
type RunRow = { workflow: string; status: string; attempts: number; error: string | null; build_uuid: string | null; build_number: number | null; build_url: string | null; started_at: string | null; finished_at: string | null; event_at: number | null;};
async function reset(seeded: SeedRun[], createdAt = new Date().toISOString()): Promise<void> { await runInDurableObject(store, async (_instance, state) => { seed(state.storage, [ { rkey, repo, trigger: { $type: "org.tangled.event.push" }, commit, createdAt, runs: seeded, }, ]); state.storage.kv.delete("buildWorkAt"); await state.storage.deleteAlarm(); });}
beforeEach(async () => { await reset([{ workflow: "ci.yml", status: "started", buildUuid: "uuid-1" }]);});
afterEach(async () => { await runInDurableObject(store, (_instance, state) => state.storage.deleteAlarm());});
function runs(): Promise<RunRow[]> { return runInDurableObject(store, (_instance, state) => state.storage.sql .exec<RunRow>( `SELECT workflow, status, attempts, error, build_uuid, build_number, build_url, started_at, finished_at, event_at FROM workflow_runs ORDER BY workflow`, ) .toArray(), );}
async function run(): Promise<RunRow | undefined> { return (await runs())[0];}
function alarmTime(): Promise<number | null> { return runInDurableObject(store, (_instance, state) => state.storage.getAlarm());}
function workAt(): Promise<number | null> { return runInDurableObject( store, (_instance, state) => state.storage.kv.get<number>("buildWorkAt") ?? null, );}
type Fetch = (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
// Answers the Jetstream stream of each alarm, fails the owner's DID document, and passes the// other requests to the handler.function mockFetch(handler: Fetch = () => Promise.reject(new Error("unexpected fetch"))) { const fetches = vi.fn(handler); vi.spyOn(globalThis, "fetch").mockImplementation((input, init) => { const { url } = new Request(input, init); if (url.startsWith(jetstreamUrl)) { return Promise.resolve(jetstreamSocket([])); } return url === `https://plc.directory/${env.OWNER_DID}` ? offline() : fetches(input, init); }); return fetches;}
function now(): number { return Math.floor(Date.now() / 1000);}
interface DeliveryOptions { event?: string; timestamp?: number; secret?: string; header?: string | null; raw?: string; slug?: string; prefix?: string;}
async function deliver(payload: object, options: DeliveryOptions = {}): Promise<Response> { const event = options.event ?? "build.running"; const body = options.raw ?? JSON.stringify({ event, build: payload, pipeline: { slug: options.slug ?? "treadle" } }); const header = options.header === undefined ? await signature( body, options.timestamp ?? now(), options.secret ?? env.BUILDKITE_WEBHOOK_SECRET ?? "", ) : options.header; const headers: Record<string, string> = { "Content-Type": "application/json", "X-Buildkite-Event": event, }; if (header !== null) { headers["X-Buildkite-Signature"] = `${options.prefix ?? ""}${header}`; } return await exports.default.fetch("https://treadle.example.com/webhooks/buildkite", { method: "POST", headers, body, });}
function build(state: string, extra: object = {}): object { return { id: "uuid-1", number: 7, state, commit, web_url: "https://buildkite.com/b/7", ...extra, };}
async function alarm(): Promise<void> { await runInDurableObject(store, (_instance, state) => state.storage.setAlarm(Date.now() + 60_000), ); expect(await runDurableObjectAlarm(store)).toBe(true);}describe("verification", () => { it.each([-290, 0, 290])("accepts a delivery signed %i s from now", async (skew) => { const res = await deliver(build("running"), { timestamp: now() + skew }); expect(res.status).toBe(200); expect(await run()).toMatchObject({ status: "running" }); });
it.each<[string, () => DeliveryOptions]>([ ["no signature", () => ({ header: null })], ["another secret", () => ({ secret: "wrong" })], ["a malformed header", () => ({ header: "signature=abc" })], ["a non-hex signature", () => ({ header: `timestamp=${now()},signature=zz` })], ["a prefixed header", () => ({ prefix: "x" })], ["a timestamp 310 s old", () => ({ timestamp: now() - 310 })], ["a timestamp 310 s ahead", () => ({ timestamp: now() + 310 })], ])("rejects %s", async (_, options) => { const res = await deliver(build("running"), options()); expect(res.status).toBe(401); expect(await run()).toMatchObject({ status: "started" }); });
it("rejects every delivery when no secret is set", async () => { const { BUILDKITE_WEBHOOK_SECRET: _, ...unset } = env; const body = JSON.stringify({ event: "build.running", build: build("running") }); const request = new Request("https://treadle.example.com/webhooks/buildkite", { method: "POST", headers: { "Content-Length": String(body.length), "X-Buildkite-Event": "build.running", "X-Buildkite-Signature": await signature(body, now(), "secret"), }, body, }); const res = await buildkiteWebhook(request, unset, createExecutionContext()); expect(res.status).toBe(401); });
it("checks the signature header before the body size", async () => { const res = await deliver({}, { header: null, raw: "x".repeat(maxBody + 1) }); expect(res.status).toBe(401); });
it("rejects a signed body that is not JSON", async () => { const res = await deliver({}, { raw: "{" }); expect(res.status).toBe(400); });
it.each([ ["exactly 1 MiB", 0, 200], ["one byte over 1 MiB", 1, 413], ])("answers a body of %s", async (_, extra, status) => { const prefix = '{"pad":"'; const raw = `${prefix}${"a".repeat(maxBody - prefix.length - 2 + extra)}"}`; const res = await deliver({}, { raw }); expect(res.status).toBe(status); });
it("rejects a body without a length", async () => { const bytes = new TextEncoder().encode("{}"); const res = await exports.default.fetch("https://treadle.example.com/webhooks/buildkite", { method: "POST", headers: { "X-Buildkite-Signature": await signature("{}", now(), "x") }, body: new ReadableStream({ start(controller) { controller.enqueue(bytes); controller.close(); }, }), }); expect(res.status).toBe(413); });});
describe("build state", () => { it.each<[string, object, DeliveryOptions, string]>([ ["a running build", build("running"), {}, "running"], [ "a finished build at a block step", build("passed", { blocked: true }), { event: "build.finished" }, "blocked", ], ["a job event", build("passed"), { event: "job.finished" }, "started"], ["a state treadle ignores", build("waiting"), {}, "started"], ["a build it did not start", build("passed", { id: "uuid-other" }), {}, "started"], ])("maps %s to %s", async (_, payload, options, status) => { expect((await deliver(payload, options)).status).toBe(200); expect(await run()).toMatchObject({ status }); });
it("records the build times and ignores a time that is not a date", async () => { const t = now(); await deliver(build("running", { started_at: "soon" }), { timestamp: t - 3 }); expect(await run()).toMatchObject({ status: "running", started_at: null }); await deliver(build("running", { started_at: "2026-11-06T10:00:05Z" }), { timestamp: t - 2, }); await deliver( build("passed", { started_at: "2026-11-06T10:00:05Z", finished_at: "2026-11-06T10:10:00.5Z", }), { timestamp: t - 1 }, ); expect(await run()).toMatchObject({ status: "success", started_at: "2026-11-06T10:00:05.000Z", finished_at: "2026-11-06T10:10:00.500Z", event_at: t - 1, }); });});
describe("ordering", () => { it.each([ ["a stale failed after passed", "passed", "failed", "success"], ["a stale scheduled after running", "running", "scheduled", "running"], ["a stale running after passed", "passed", "running", "success"], ])("drops %s", async (_, first, late, status) => { const t = now(); await deliver(build(first), { timestamp: t }); await deliver(build(late), { timestamp: t - 1 }); expect(await run()).toMatchObject({ status, event_at: t }); });
it.each([ ["a lower rank", "passed", "running", "success"], ["the same rank", "passed", "failed", "failed"], ["a higher rank", "running", "passed", "success"], ])("applies an event of the same second only for %s or higher", async (_, a, b, status) => { const t = now(); await deliver(build(a), { timestamp: t }); await deliver(build(b), { timestamp: t }); expect(await run()).toMatchObject({ status }); });
it("follows a finished build that runs again after a job retry", async () => { const t = now(); const startedAt = "2026-11-06T10:00:05Z"; await deliver( build("failed", { started_at: startedAt, finished_at: "2026-11-06T10:01:00Z" }), { event: "build.finished", timestamp: t - 2 }, ); await deliver(build("running", { started_at: startedAt }), { timestamp: t - 1 }); expect(await run()).toMatchObject({ status: "running", finished_at: null }); await deliver( build("passed", { started_at: startedAt, finished_at: "2026-11-06T10:06:00Z" }), { event: "build.finished", timestamp: t }, ); expect(await run()).toMatchObject({ status: "success", started_at: "2026-11-06T10:00:05.000Z", finished_at: "2026-11-06T10:06:00.000Z", }); });
it("moves a running build to blocked and back", async () => { const t = now(); await deliver(build("running"), { timestamp: t - 2 }); await deliver(build("passed", { blocked: true, finished_at: "2026-11-06T10:01:00Z" }), { event: "build.finished", timestamp: t - 1, }); expect(await run()).toMatchObject({ status: "blocked", finished_at: null }); await deliver(build("running"), { timestamp: t }); expect(await run()).toMatchObject({ status: "running" }); });
it("keeps build details that a later event omits", async () => { const t = now(); await deliver(build("running", { started_at: "2026-11-06T10:00:05Z" }), { timestamp: t - 1, }); await deliver({ id: "uuid-1", state: "failing", commit }, { timestamp: t }); expect(await run()).toMatchObject({ status: "running", build_number: 7, build_url: "https://buildkite.com/b/7", started_at: "2026-11-06T10:00:05.000Z", }); });});
describe("meta_data fallback", () => { const meta = { "treadle:pipeline": rkey, "treadle:workflow": "ci.yml" };
it.each(["pending", "started"])( "finds a %s run that has no build yet and records the build", async (status) => { await reset([{ workflow: "ci.yml", status }]); await deliver(build("scheduled", { id: "uuid-9", meta_data: meta })); expect(await run()).toMatchObject({ status: "started", build_uuid: "uuid-9", build_number: 7, build_url: "https://buildkite.com/b/7", }); }, );
it.each<[string, SeedRun, object, DeliveryOptions]>([ [ "a run of another build", { workflow: "ci.yml", status: "started", buildUuid: "uuid-1" }, {}, {}, ], ["a failed run", { workflow: "ci.yml", status: "failed" }, {}, {}], [ "a build of another commit", { workflow: "ci.yml", status: "started" }, { commit: "b".repeat(40) }, {}, ], [ "a build of another pipeline", { workflow: "ci.yml", status: "started" }, {}, { slug: "other" }, ], ])("does not claim %s", async (_, seeded, extra, options) => { await reset([seeded]); await deliver(build("passed", { id: "uuid-9", meta_data: meta, ...extra }), options); expect(await run()).toMatchObject({ status: seeded.status, build_uuid: seeded.buildUuid ?? null, }); });
it.each([ ["starts", () => Response.json({ id: "uuid-9", number: 7 }, { status: 201 })], ["refuses", () => Response.json({ message: "No pipeline" }, { status: 404 })], ["fails", () => new Response("unavailable", { status: 500 })], ])("keeps a webhook that arrives before Buildkite %s the build", async (_, answer) => { await reset([{ workflow: "ci.yml", status: "pending" }]); await runInDurableObject(store, (_instance, state) => state.storage.setAlarm(Date.now() + 60_000), ); const fetches = mockFetch(async () => { const res = await deliver(build("running", { id: "uuid-9", meta_data: meta })); expect(res.status).toBe(200); return answer(); }); expect(await runDurableObjectAlarm(store)).toBe(true); expect(fetches).toHaveBeenCalledTimes(1); expect(await run()).toMatchObject({ status: "running", build_uuid: "uuid-9", attempts: 0, error: null, }); });
it("does not start a run that a webhook claimed during the alarm", async () => { await reset([ { workflow: "a.yml", status: "pending" }, { workflow: "b.yml", status: "pending" }, ]); await runInDurableObject(store, (_instance, state) => state.storage.setAlarm(Date.now() + 60_000), ); const fetches = mockFetch(async (input, init) => { const request = new Request(input, init); const body: { meta_data: Record<string, string> } = await request.json(); const other = body.meta_data["treadle:workflow"] === "a.yml" ? "b.yml" : "a.yml"; await deliver( build("scheduled", { id: `uuid-${other}`, meta_data: { "treadle:pipeline": rkey, "treadle:workflow": other }, }), ); return Response.json({ id: "uuid-first", number: 1 }, { status: 201 }); }); expect(await runDurableObjectAlarm(store)).toBe(true); expect(fetches).toHaveBeenCalledTimes(1); expect((await runs()).map((row) => row.status)).toEqual(["started", "started"]); });});
describe("reconciliation", () => { const stale = new Date(Date.now() - 2 * hourMs).toISOString();
it("asks Buildkite about a build without news for ten minutes", async () => { await reset( [{ workflow: "ci.yml", status: "started", buildUuid: "uuid-1", buildNumber: 7 }], stale, ); const fetches = mockFetch((input, init) => { const request = new Request(input, init); expect(`${request.method} ${request.url}`).toBe(`GET ${buildUrl}`); expect(request.headers.get("authorization")).toBe("Bearer test-token"); return Promise.resolve( Response.json(build("passed", { finished_at: "2026-11-06T10:01:00Z" })), ); }); const before = now(); await alarm(); expect(fetches).toHaveBeenCalledTimes(1); expect(await run()).toMatchObject({ status: "success", finished_at: "2026-11-06T10:01:00.000Z", }); expect((await run())?.event_at).toBeGreaterThanOrEqual(before); expect(await workAt()).toBeNull(); });
it("does not ask Buildkite before the next check is due", async () => { await reset( [{ workflow: "ci.yml", status: "started", buildUuid: "uuid-1", buildNumber: 7 }], stale, ); const at = Date.now() + checkMs; await runInDurableObject(store, (_instance, state) => { state.storage.kv.put("buildWorkAt", at); }); const fetches = mockFetch(); await alarm(); expect(fetches).not.toHaveBeenCalled(); expect(await workAt()).toBe(at); });
it("keeps a webhook that arrives while it asks Buildkite", async () => { await reset( [{ workflow: "ci.yml", status: "running", buildUuid: "uuid-1", buildNumber: 7 }], stale, ); mockFetch(async () => { await deliver(build("passed"), { event: "build.finished", timestamp: now() + 1 }); return Response.json(build("running")); }); await alarm(); expect(await run()).toMatchObject({ status: "success" }); });
it("stops asking about a build from more than three days ago", async () => { await reset( [{ workflow: "ci.yml", status: "running", buildNumber: 7 }], new Date(Date.now() - 73 * hourMs).toISOString(), ); const fetches = mockFetch(); await alarm(); expect(fetches).not.toHaveBeenCalled(); expect(await workAt()).toBeNull(); });
it.each<[string, SeedRun, string]>([ [ "a build with recent news", { workflow: "ci.yml", status: "running", buildNumber: 7, eventAt: Math.floor(Date.now() / 1000) - 60, }, stale, ], [ "a recent build without news", { workflow: "ci.yml", status: "started", buildNumber: 7 }, new Date().toISOString(), ], ])("waits for %s and checks again in ten minutes", async (_, seeded, createdAt) => { await reset([seeded], createdAt); const fetches = mockFetch(); await alarm(); expect(fetches).not.toHaveBeenCalled(); const delay = ((await workAt()) ?? 0) - Date.now(); expect(delay).toBeGreaterThan(checkMs - 5_000); expect(delay).toBeLessThanOrEqual(checkMs); });
it("checks an open build before a pending run's long backoff", async () => { await reset( [ { workflow: "ci.yml", status: "started", buildNumber: 7 }, { workflow: "lint.yml", status: "pending", attempts: 5 }, ], new Date().toISOString(), ); mockFetch(() => Promise.resolve(new Response("unavailable", { status: 502 }))); await alarm(); const delay = ((await workAt()) ?? 0) - Date.now(); expect(delay).toBeGreaterThan(checkMs - 5_000); expect(delay).toBeLessThanOrEqual(checkMs); });
it("keeps a run when Buildkite does not answer", async () => { await reset([{ workflow: "ci.yml", status: "running", buildNumber: 7 }], stale); mockFetch(() => Promise.resolve(new Response("unavailable", { status: 502 }))); await alarm(); expect(await run()).toMatchObject({ status: "running", event_at: null }); expect(await workAt()).toBeGreaterThan(Date.now() + checkMs - 5_000); });
it("schedules an alarm at startup without pipelines", async () => { await runInDurableObject(store, async (_instance, state) => { state.storage.sql.exec("DELETE FROM workflow_runs; DELETE FROM pipelines"); await state.storage.deleteAlarm(); await ensureAlarm(state.storage); }); expect(await alarmTime()).toBeLessThanOrEqual(Date.now() + 10_000); });
it("moves a later alarm to now at startup", async () => { await runInDurableObject(store, async (_instance, state) => { await state.storage.setAlarm(Date.now() + hourMs); await ensureAlarm(state.storage); }); expect(await alarmTime()).toBeLessThanOrEqual(Date.now() + 10_000); });
it("keeps an alarm that is already set at startup", async () => { const at = Date.now() + 5_000; await runInDurableObject(store, async (_instance, state) => { await state.storage.setAlarm(at); await ensureAlarm(state.storage); }); expect(await alarmTime()).toBe(at); });});