import { runDurableObjectAlarm, runInDurableObject } from "cloudflare:test"; import { env, exports } from "cloudflare:workers"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { assertOutput, commitMessage, declarationsRoute, didDoc, getRecordRoute, jetstreamSocket, jetstreamUrl, keypair, offline, pdsService, record, repoDid, routeFetch, seed, serviceToken, sha, spindle, } from "./helpers"; import type { Keypair, Route, TokenOptions } from "./helpers"; import { parseDocument } from "yaml"; import { newRuns, trustedWorkflows } from "../src/notify"; import { invalidWorkflow, parseWorkflow } from "../src/workflow"; const knotDid = "did:web:knot.example.com"; const owner = env.OWNER_DID; const method = "org.tangled.event.notifyEvent"; const store = exports.Store.getByName("main"); const knotKey = keypair("secp256k1"); const ownerKey = keypair("p256"); const repoKey = keypair(); const strangerKey = keypair(); const workflowFiles: Record = { "ci.yml": [ "engine: buildkite", "when:", " - event: push", " branch: main", "treadle:", " buildkite:", " pipeline: treadle", " clean_checkout: true", ].join("\n"), "lint.yaml": "engine: buildkite\nwhen:\n - event: pull_request\n branch: main\ntreadle:\n buildkite:\n pipeline: lint\n org: other\n", "always.yml": "engine: buildkite\n", "noengine.yml": "treadle:\n buildkite:\n pipeline: always\n", "nix.yml": "engine: nixery\ntreadle:\n buildkite:\n pipeline: always\n", "plain.yml": "engine: nixery\n", }; const buildkiteBase = "https://api.buildkite.com/v2/organizations"; const pipelineUrls = { treadle: `${buildkiteBase}/gpanders/pipelines/treadle/builds`, lint: `${buildkiteBase}/other/pipelines/lint/builds`, always: `${buildkiteBase}/gpanders/pipelines/my-repo/builds`, }; const treadleRoute = `POST ${pipelineUrls.treadle}`; const alwaysRoute = `POST ${pipelineUrls.always}`; const knotDocRoute = "GET https://knot.example.com/.well-known/did.json"; const repoDocRoute = `GET https://plc.directory/${repoDid}`; const ownerDocRoute = `GET https://plc.directory/${owner}`; const treeRoute = "GET https://knot.example.com/xrpc/org.tangled.git.getTree"; const blobRoute = "GET https://knot.example.com/xrpc/org.tangled.git.getBlob"; const commitsRoute = "GET https://knot.example.com/xrpc/org.tangled.git.listCommits"; const commitAuthor = { name: "Ada", email: "ada@example.com" }; const commitInfo = { message: "Fix the build\n\nDetails", author: commitAuthor }; const manifestRoute = getRecordRoute( "https://knot.example.com", repoDid, "org.tangled.repo.manifest", "self", ); const refsRoute = getRecordRoute( "https://knot.example.com", repoDid, "org.tangled.git.refs", "self", ); const pdsRoute = getRecordRoute( "https://pds.example.com", owner, "org.tangled.repo.declaration", "my.repo", ); interface BuildRequest { url: string; authorization: string | null; body: unknown; } let routes: Record; let sentBuilds: BuildRequest[]; let commits: Set; let manifest: Record; let hookPosts: { event: string | null; agent: string | null; body: unknown }[]; const hookUrl = "https://hooks.example.com/tangled"; const hooks = exports.Webhooks.getByName(repoDid); const repository = { name: "My.Repo", full_name: `${owner}/my.repo`, description: "", fork: false, html_url: `https://knot.example.com/${owner}/my.repo`, clone_url: `https://knot.example.com/${owner}/my.repo`, ssh_url: `ssh://git@knot.example.com/${owner}/my.repo`, created_at: "2026-11-06T00:00:00Z", updated_at: "2026-11-06T00:00:00Z", owner: { did: owner }, }; function pullBody(action: string, extra: object): object { return { action, pull_request: { number: 0, title: "Fix", body: "", state: "open", target_branch: "main", round_number: 0, owner: { did: owner }, created_at: "2026-11-07T00:00:00Z", ...extra, }, repository, sender: { did: owner }, }; } async function addHook(events: string[]): Promise { await hooks.create({ url: hookUrl, secret: "", active: true, events }); } async function drainHooks(): Promise { await runDurableObjectAlarm(hooks); await vi.waitFor(async () => { const due = await runInDurableObject( hooks, (_instance, state) => state.storage.sql.exec<{ n: number }>("SELECT count(*) AS n FROM jobs").one().n, ); expect(due).toBe(0); }); } function xrpcError(error: string, status = 400): Response { return Response.json({ error, message: "upstream detail" }, { status }); } function failing(status = 502): () => Response { return () => new Response("unavailable", { status }); } function created(): () => Response { let number = 0; return () => { number++; return Response.json( { id: `uuid-${number}`, number, web_url: `https://buildkite.com/b/${number}` }, { status: 201 }, ); }; } function workflowRef(url: URL): Response | undefined { const ref = url.searchParams.get("ref"); return ref !== null && commits.has(ref) ? undefined : xrpcError("RefNotFound"); } beforeEach(async () => { await runInDurableObject(store, async (_instance, state) => { state.storage.sql.exec( "DELETE FROM workflow_runs; DELETE FROM pipelines; DELETE FROM service_tokens; DELETE FROM pulls", ); state.storage.kv.delete("buildWorkAt"); state.storage.kv.delete("jetstream"); state.storage.kv.delete("declaredRepos"); await state.storage.deleteAlarm(); }); await runInDurableObject(hooks, async (_instance, state) => { state.storage.sql.exec( "DELETE FROM hooks; DELETE FROM jobs; DELETE FROM deliveries; DELETE FROM pushes; DELETE FROM tickets; DELETE FROM patches", ); await state.storage.deleteAlarm(); }); hookPosts = []; sentBuilds = []; commits = new Set(["refs/heads/main", "refs/heads/feature", "refs/heads/trunk"]); manifest = { $type: "org.tangled.repo.manifest", declaration: { owner, name: "My.Repo" }, spindle, createdAt: "2026-11-06T00:00:00Z", }; const start = created(); routes = { [knotDocRoute]: () => Response.json(didDoc(knotDid, knotKey)), "GET https://stranger.example.com/.well-known/did.json": () => Response.json(didDoc("did:web:stranger.example.com", strangerKey)), [repoDocRoute]: () => Response.json( didDoc(repoDid, repoKey, [ { id: "#tangled_knot", type: "TangledKnot", serviceEndpoint: `https://knot.example.com/${repoDid}`, }, ]), ), [ownerDocRoute]: () => Response.json(didDoc(owner, ownerKey, [pdsService])), [declarationsRoute(owner)]: () => Response.json({ records: [ { uri: `at://${owner}/org.tangled.repo.declaration/treadle`, value: { repo: repoDid }, }, ], }), [manifestRoute]: () => record(manifest), [refsRoute]: () => record({ $type: "org.tangled.git.refs", defaultBranch: "trunk", refs: [] }), [pdsRoute]: () => record({ $type: "org.tangled.repo.declaration", repo: repoDid, createdAt: "x" }), [treeRoute]: (url) => workflowRef(url) ?? Response.json({ entries: [ ...Object.entries(workflowFiles).map(([name, file]) => ({ name, mode: "0100644", size: file.length, })), { name: "README.md", mode: "0100644", size: 1 }, { name: "nested", mode: "0040000", size: 0 }, ], }), [blobRoute]: (url) => { const file = workflowFiles[ url.searchParams.get("path")?.replace(".tangled/workflows/", "") ?? "" ]; return ( workflowRef(url) ?? (file === undefined ? xrpcError("PathNotFound") : new Response(file)) ); }, [commitsRoute]: (url) => { const ref = url.searchParams.get("ref") ?? ""; expect(url.searchParams.get("limit")).toBe("1"); const signature = { ...commitAuthor, when: "2026-11-06T00:00:00Z" }; return ( workflowRef(url) ?? Response.json({ commits: [ { hash: ref.startsWith("refs/") ? sha(0xfff) : ref, author: signature, committer: signature, message: "Fix the build\n\nDetails\n", tree: sha(0xeee), }, ], }) ); }, [treadleRoute]: start, [`POST ${pipelineUrls.lint}`]: start, [alwaysRoute]: start, [`GET ${jetstreamUrl}`]: () => jetstreamSocket([]), [`POST ${hookUrl}`]: async (_url, request) => { hookPosts.push({ event: request.headers.get("x-tangled-event"), agent: request.headers.get("user-agent"), body: await request.json(), }); return new Response(null); }, [`GET ${pipelineUrls.always}/1`]: () => Response.json({ id: "uuid-1", number: 1, state: "running" }), }; routeFetch(routes, async (request) => { expect(request.redirect).toBe("manual"); if (new URL(request.url).origin === "https://api.buildkite.com") { const text = await request.clone().text(); sentBuilds.push({ url: request.url, authorization: request.headers.get("authorization"), body: text === "" ? undefined : (JSON.parse(text) as unknown), }); } }); }); afterEach(async () => { await runInDurableObject(store, (_instance, state) => state.storage.deleteAlarm()); }); function token({ key = knotKey, iss = knotDid, lxm = method, ...rest }: Partial & { key?: Keypair } = {}): string { return serviceToken(key, { iss, lxm, ...rest }); } function ownerToken(lxm = method): string { return token({ key: ownerKey, iss: owner, lxm }); } function strangerToken(lxm = method): string { return token({ key: strangerKey, iss: "did:web:stranger.example.com", lxm }); } function pushEvent(newSha: string, overrides: object = {}): object { commits.add(newSha); return { repo: repoDid, event: { $type: "org.tangled.event.push", repo: repoDid, ref: "refs/heads/main", oldSha: sha(0x1), newSha, actor: owner, ...overrides, }, }; } function manualEvent(commit: string, overrides: object = {}): object { commits.add(commit); return { repo: repoDid, event: { $type: "org.tangled.ci.trigger.manual", repo: repoDid, commit, ...overrides }, }; } function notify(body: object, jwt: string | null = token(), ip?: string): Promise { const headers: Record = { "Content-Type": "application/json" }; if (ip !== undefined) { headers["CF-Connecting-IP"] = ip; } if (jwt !== null) { headers["Authorization"] = `Bearer ${jwt}`; } return exports.default.fetch(`https://treadle.example.com/xrpc/${method}`, { method: "POST", headers, body: JSON.stringify(body), }); } async function pipelines(res: Response): Promise { expect(res.status).toBe(200); const body: { ci?: { pipelines: string[] } } = await res.json(); return body.ci?.pipelines ?? []; } async function pipeline(res: Response): Promise { const [rkey, ...rest] = await pipelines(res); expect(rest).toEqual([]); expect(rkey).toMatch(/^[a-z2-7]{13}$/); return rkey ?? ""; } async function expectError(res: Response, status: number, error: string, message: string) { expect(res.status).toBe(status); const body: { error: string; message: string } = await res.json(); expect(body.error).toBe(error); expect(body.message).toContain(message); expect(body.message).not.toContain("upstream detail"); } function alarmTime(): Promise { return runInDurableObject(store, (_instance, state) => state.storage.getAlarm()); } function workAt(): Promise { return runInDurableObject( store, (_instance, state) => state.storage.kv.get("buildWorkAt") ?? null, ); } // Lets the next alarm run the build work at once. function buildDue(): Promise { return runInDurableObject(store, (_instance, state) => { state.storage.kv.put("buildWorkAt", 0); }); } // Started builds keep checking for lost webhooks. async function expectCheckAlarm(): Promise { const delay = ((await workAt()) ?? 0) - Date.now(); expect(delay).toBeGreaterThan(10 * 60_000 - 5_000); expect(delay).toBeLessThanOrEqual(10 * 60_000); } function pipelineCount(): Promise { return runInDurableObject( store, (_instance, state) => state.storage.sql.exec<{ n: number }>("SELECT count(*) AS n FROM pipelines").one().n, ); } function tokenCount(): Promise { return runInDurableObject( store, (_instance, state) => state.storage.sql.exec<{ n: number }>("SELECT count(*) AS n FROM service_tokens").one() .n, ); } type RunRow = { workflow: string; status: string; attempts: number; error: string | null; }; function runs(rkey: string): Promise { return runInDurableObject(store, (_instance, state) => state.storage.sql .exec( "SELECT workflow, status, attempts, error FROM workflow_runs WHERE pipeline = ? ORDER BY workflow", rkey, ) .toArray(), ); } function openRuns(): Promise { return runInDurableObject( store, (_instance, state) => state.storage.sql .exec<{ n: number }>( `SELECT count(*) AS n FROM workflow_runs WHERE status = 'pending' OR (status = 'started' AND build_number IS NOT NULL)`, ) .one().n, ); } // The runtime fires an alarm set for now by itself, so tests wait for its effect. async function settled(rkey: string, expected: Partial[]): Promise { return await vi.waitFor(async () => { const rows = await runs(rkey); expect(rows).toMatchObject(expected); if ((await openRuns()) > 0) { expect(await alarmTime()).toBeGreaterThan(Date.now()); } return rows; }); } const started = { status: "started" }; function addWorkflowFiles(): void { for (let i = Object.keys(workflowFiles).length; i <= 50; i++) { workflowFiles[`extra${i}.yml`] = "engine: buildkite\n"; } } const overLimits: [string, () => void][] = [ ["more than 50 workflow files", addWorkflowFiles], [ "a workflow file over 256 KiB", () => (workflowFiles["big.yml"] = `engine: buildkite\n#${"x".repeat(256 * 1024)}`), ], ]; async function expectTooLarge(rkey: string): Promise { const [run, ...rest] = await runs(rkey); expect(rest).toEqual([]); expect(run).toMatchObject({ workflow: ".tangled/workflows", status: "failed" }); expect(run?.error).toContain("at most 50 workflow files of 256 KiB"); expect(sentBuilds).toEqual([]); } async function withWorkflowFiles(setup: () => void, run: () => Promise): Promise { const saved = { ...workflowFiles }; setup(); try { await run(); } finally { for (const name of Object.keys(workflowFiles)) { delete workflowFiles[name]; } Object.assign(workflowFiles, saved); } } describe("push", () => { it.each(overLimits)("fails one run for %s", async (_, setup) => { await withWorkflowFiles(setup, async () => { await expectTooLarge(await pipeline(await notify(pushEvent(sha(0x106))))); }); }); it("ignores a push to a repo that the owner does not declare", async () => { const other = "did:plc:aaaaaaaaaaaaaaaaaaaaaaaa"; const res = await notify({ ...pushEvent(sha(0x105), { repo: other }), repo: other }); expect(await res.json()).toEqual({}); expect(await tokenCount()).toBe(0); }); it("finds a repo that the owner declares after the last lookup", async () => { const other = "did:plc:aaaaaaaaaaaaaaaaaaaaaaaa"; expect(await store.declares(other)).toBe(false); routes[declarationsRoute(owner)] = () => Response.json({ records: [ { uri: `at://${owner}/org.tangled.repo.declaration/x`, value: { repo: other } }, ], }); expect(await store.declares(other)).toBe(true); }); it("refuses a manual run of a repo that the owner does not declare", async () => { const other = "did:plc:aaaaaaaaaaaaaaaaaaaaaaaa"; const body = { ...manualEvent(sha(0x105), { repo: other }), repo: other }; await expectError(await notify(body, ownerToken()), 404, "RepoNotFound", "not served"); expect(await tokenCount()).toBe(0); }); it("records a pipeline and starts its builds from the alarm", async () => { const rkey = await pipeline(await notify(pushEvent(sha(0x100)))); await settled(rkey, [ { workflow: "always.yml", ...started }, { workflow: "ci.yml", ...started }, ]); expect(sentBuilds.map((build) => build.url).toSorted()).toEqual([ pipelineUrls.always, pipelineUrls.treadle, ]); expect(sentBuilds.map((build) => build.authorization)).toEqual([ "Bearer test-token", "Bearer test-token", ]); expect(sentBuilds.find((build) => build.url === pipelineUrls.treadle)?.body).toMatchObject({ commit: sha(0x100), branch: "main", env: { TREADLE_REPO: repoDid, TREADLE_PIPELINE: rkey, TREADLE_WORKFLOW: "ci.yml" }, clean_checkout: true, ...commitInfo, }); await expectCheckAlarm(); }); it.each([ ["fails", failing(500)], ["is unreachable", offline], ["finds no commit", () => Response.json({ commits: [] })], [ "names another commit", () => Response.json({ commits: [{ hash: sha(0x1), message: "x" }] }), ], [ "finds an empty message", () => Response.json({ commits: [{ hash: sha(0x110), message: " \n" }] }), ], ])("falls back to the workflow name when the commit lookup %s", async (_, answer) => { routes[commitsRoute] = answer; const rkey = await pipeline(await notify(pushEvent(sha(0x110)))); await settled(rkey, [started, started]); const body = sentBuilds.find((build) => build.url === pipelineUrls.treadle)?.body; expect(body).toMatchObject({ message: "tangled: ci.yml" }); expect(body).not.toHaveProperty("author"); }); it("sends the message without an incomplete author", async () => { routes[commitsRoute] = () => Response.json({ commits: [{ hash: sha(0x112), message: "Fix", author: { name: "Ada" } }], }); const rkey = await pipeline(await notify(pushEvent(sha(0x112)))); await settled(rkey, [started, started]); const body = sentBuilds.find((build) => build.url === pipelineUrls.treadle)?.body; expect(body).toMatchObject({ message: "Fix" }); expect(body).not.toHaveProperty("author"); }); it("skips only the workflows without a pipeline when the repo name gives no slug", async () => { manifest = { ...manifest, declaration: { owner, name: "__" } }; routes[ getRecordRoute("https://pds.example.com", owner, "org.tangled.repo.declaration", "__") ] = () => record({ $type: "org.tangled.repo.declaration", repo: repoDid, createdAt: "x" }); const rkey = await pipeline(await notify(pushEvent(sha(0x111)))); await settled(rkey, [{ workflow: "ci.yml", ...started }]); expect(sentBuilds.map((build) => build.url)).toEqual([pipelineUrls.treadle]); }); it("starts builds before it reads Jetstream", async () => { const connects: number[] = []; routes[`GET ${jetstreamUrl}`] = () => { connects.push(sentBuilds.length); return jetstreamSocket([]); }; const rkey = await pipeline(await notify(pushEvent(sha(0x114)))); await settled(rkey, [started, started]); expect(connects[0]).toBe(2); }); it("returns the same pipeline when the push is delivered again", async () => { const rkey = await pipeline(await notify(pushEvent(sha(0x101)))); await settled(rkey, [started, started]); routes[pdsRoute] = failing(500); expect(await pipeline(await notify(pushEvent(sha(0x101))))).toBe(rkey); await expectCheckAlarm(); expect(sentBuilds).toHaveLength(2); }); it("creates one pipeline for concurrent deliveries", async () => { const responses = await Promise.all([ notify(pushEvent(sha(0x102))), notify(pushEvent(sha(0x102))), ]); const [first, second] = await Promise.all(responses.map(pipeline)); expect(second).toBe(first); await settled(first ?? "", [started, started]); expect(sentBuilds).toHaveLength(2); expect(await pipelineCount()).toBe(1); }); it("retries only the builds that failed to start", async () => { routes[treadleRoute] = failing(); const rkey = await pipeline(await notify(pushEvent(sha(0x103)))); await settled(rkey, [started, { status: "pending", attempts: 1 }]); expect(sentBuilds).toHaveLength(2); routes[treadleRoute] = created(); await buildDue(); expect(await runDurableObjectAlarm(store)).toBe(true); const retried = sentBuilds.slice(2); expect(retried.map((build) => build.url)).toEqual([pipelineUrls.treadle]); expect(retried[0]?.body).toMatchObject({ env: { TREADLE_PIPELINE: rkey } }); expect(await runs(rkey)).toMatchObject([ { workflow: "always.yml", status: "started", attempts: 0 }, { workflow: "ci.yml", status: "started", attempts: 1 }, ]); }); it("does not retry a backed-off build before its time", async () => { routes[treadleRoute] = failing(); const rkey = await pipeline(await notify(pushEvent(sha(0x113)))); await settled(rkey, [started, { status: "pending", attempts: 1 }]); const alarm = (await alarmTime()) ?? 0; expect(alarm).toBeLessThanOrEqual(Date.now() + 10_000); expect(await workAt()).toBeGreaterThan(alarm); expect(await runDurableObjectAlarm(store)).toBe(true); expect(sentBuilds).toHaveLength(2); expect(await runs(rkey)).toMatchObject([started, { status: "pending", attempts: 1 }]); }); it("starts a new push without waiting for a backoff", async () => { routes[treadleRoute] = failing(); const first = await pipeline(await notify(pushEvent(sha(0x110)))); await settled(first, [started, { status: "pending", attempts: 1 }]); expect(await workAt()).toBeGreaterThan(Date.now() + 60_000); routes[treadleRoute] = created(); const second = await pipeline(await notify(pushEvent(sha(0x111)))); await settled(second, [started, started]); await settled(first, [started, started]); }); it("backs off by the fewest attempts of the pending runs", async () => { routes[treadleRoute] = failing(); routes[alwaysRoute] = failing(); const rkey = await pipeline(await notify(pushEvent(sha(0x112)))); await settled(rkey, [ { status: "pending", attempts: 1 }, { status: "pending", attempts: 1 }, ]); await runInDurableObject(store, (_instance, state) => { state.storage.sql.exec( "UPDATE workflow_runs SET attempts = 3 WHERE workflow = 'ci.yml'", ); }); await buildDue(); expect(await runDurableObjectAlarm(store)).toBe(true); expect(await runs(rkey)).toMatchObject([{ attempts: 2 }, { attempts: 4 }]); const delay = ((await workAt()) ?? 0) - Date.now(); expect(delay).toBeGreaterThan(4 * 60_000 - 5_000); expect(delay).toBeLessThanOrEqual(4 * 60_000); }); it.each([ ["429", failing(429), "pending", 1], ["500", failing(500), "pending", 1], ["a network error", offline, "pending", 1], ["a 201 without JSON", () => new Response("nope", { status: 201 }), "started", 0], ["a 201 with a null body", () => Response.json(null, { status: 201 }), "started", 0], ["a 201 with a string body", () => Response.json("x", { status: 201 }), "started", 0], [ "a 201 with a broken body", () => new Response( new ReadableStream({ start(controller) { controller.error(new Error("reset")); }, }), { status: 201 }, ), "started", 0, ], ["404", () => Response.json({ message: "No pipeline" }, { status: 404 }), "failed", 0], ])("handles a Buildkite answer of %s", async (_, answer, status, attempts) => { routes[treadleRoute] = answer; const rkey = await pipeline(await notify(pushEvent(sha(0x104)))); await settled(rkey, [started, { workflow: "ci.yml", status, attempts }]); if (status !== "pending") { await expectCheckAlarm(); } }); it.each([ ["more than 3 days", 3 * 24 * 60 + 1, "failed", 1, /^gave up: /], ["less than 3 days", 3 * 24 * 60 - 1, "pending", 2, null], ])( "retries a build that has not started for %s", async (_, minutes, status, attempts, error) => { routes[treadleRoute] = failing(503); const rkey = await pipeline(await notify(pushEvent(sha(0x105)))); await settled(rkey, [started, { status: "pending", attempts: 1 }]); await runInDurableObject(store, (_instance, state) => { state.storage.sql.exec( "UPDATE pipelines SET created_at = ?", new Date(Date.now() - minutes * 60_000).toISOString(), ); }); await buildDue(); expect(await runDurableObjectAlarm(store)).toBe(true); const [, ci] = await runs(rkey); expect(ci).toMatchObject({ status, attempts }); if (error === null) { expect(ci?.error).toBeNull(); } else { expect(ci?.error).toMatch(error); } if (status === "pending") { expect(await workAt()).toBeLessThan(Date.now() + 60 * 60_000 - 5_000); } else { expect(await workAt()).toBeNull(); } }, ); it("starts only unconditional workflows for other branches", async () => { const rkey = await pipeline(await notify(pushEvent(sha(0x106), { ref: "refs/heads/dev" }))); await settled(rkey, [{ workflow: "always.yml", ...started }]); expect(sentBuilds.map((build) => build.url)).toEqual([pipelineUrls.always]); }); it.each([ ["a ref deletion", { newSha: "0".repeat(40) }], ["a non-branch ref", { ref: "refs/notes/commits" }], ["a skip-ci push option", { pushOptions: ["skip-ci"] }], ["a ci-skip push option", { pushOptions: ["ci-skip"] }], ])("runs nothing for %s", async (_, overrides) => { expect(await pipelines(await notify(pushEvent(sha(0x107), overrides)))).toEqual([]); expect(await pipelineCount()).toBe(0); }); it.each([ [ "names another spindle", () => (manifest = { ...manifest, spindle: "did:web:other.example.com" }), ], [ "has another owner", () => (manifest = { ...manifest, declaration: { owner: "did:plc:aaaaaaaaaaaaaaaaaaaaaaaa", name: "x" }, }), ], [ "is not declared by its owner", () => (routes[pdsRoute] = () => xrpcError("RecordNotFound")), ], [ "is declared for another repo", () => (routes[pdsRoute] = () => record({ repo: "did:plc:bbbbbbbbbbbbbbbbbbbbbbbb", createdAt: "x" })), ], ["has no workflow directory", () => (routes[treeRoute] = () => xrpcError("PathNotFound"))], [ "has only workflows the knot refuses to read", () => (routes[blobRoute] = () => xrpcError("BlobTooLarge", 413)), ], ])("ignores a repo that %s", async (_, setup) => { setup(); expect(await pipelines(await notify(pushEvent(sha(0x108))))).toEqual([]); expect(await pipelineCount()).toBe(0); }); const knotXrpc = "https://knot.example.com/xrpc"; it.each<[string, () => void, number, string, string]>([ ["does not have the pushed commit", () => commits.clear(), 422, "PushRefused", sha(0x109)], [ "is unknown to its knot", () => (routes[treeRoute] = () => xrpcError("RepoNotFound")), 422, "PushRefused", "RepoNotFound", ], [ "fails to list workflows", () => (routes[treeRoute] = failing(500)), 500, "InternalError", `${knotXrpc}/org.tangled.git.getTree: HTTP 500`, ], [ "cannot reach its knot to list workflows", () => (routes[treeRoute] = offline), 500, "InternalError", `${knotXrpc}/org.tangled.git.getTree: TypeError`, ], [ "lists workflows without entries", () => (routes[treeRoute] = () => Response.json({})), 500, "InternalError", `${knotXrpc}/org.tangled.git.getTree: response has no entries`, ], [ "fails to read a workflow", () => (routes[blobRoute] = failing(500)), 500, "InternalError", `${knotXrpc}/org.tangled.git.getBlob: HTTP 500`, ], [ "cannot reach its knot to read a workflow", () => (routes[blobRoute] = offline), 500, "InternalError", `${knotXrpc}/org.tangled.git.getBlob: TypeError`, ], [ "times out workflow reads", () => (routes[blobRoute] = failing(408)), 500, "InternalError", `${knotXrpc}/org.tangled.git.getBlob: HTTP 408`, ], [ "rate-limits workflow reads", () => (routes[blobRoute] = () => xrpcError("RateLimitExceeded", 429)), 500, "InternalError", `${knotXrpc}/org.tangled.git.getBlob: HTTP 429`, ], [ "fails to read the manifest", () => (routes[manifestRoute] = failing(502)), 500, "InternalError", `${knotXrpc}/com.atproto.repo.getRecord: HTTP 502`, ], [ "answers a manifest without a value", () => (routes[manifestRoute] = () => Response.json({ uri: "at://x" })), 500, "InternalError", `${knotXrpc}/com.atproto.repo.getRecord: response has no record value`, ], [ "has an owner whose PDS fails", () => (routes[pdsRoute] = failing(500)), 503, "UpstreamUnavailable", "https://pds.example.com/xrpc/com.atproto.repo.getRecord: HTTP 500", ], [ "has an owner without a PDS", () => (routes[ownerDocRoute] = () => Response.json(didDoc(owner, ownerKey))), 503, "UpstreamUnavailable", `${owner} has no PDS`, ], [ "cannot be resolved", () => (routes[repoDocRoute] = failing(500)), 503, "UpstreamUnavailable", `https://plc.directory/${repoDid}: HTTP 500`, ], [ "resolves to another DID document", () => (routes[repoDocRoute] = () => Response.json(didDoc("did:plc:dddddddddddddddddddddddd"))), 503, "UpstreamUnavailable", `https://plc.directory/${repoDid}: no DID document`, ], [ "declares no knot", () => (routes[repoDocRoute] = () => Response.json(didDoc(repoDid, repoKey))), 503, "RepoKnotUnavailable", "declares no knot", ], ])("refuses a repo that %s", async (_, setup, status, error, detail) => { const event = pushEvent(sha(0x109)); setup(); await expectError(await notify(event), status, error, detail); expect(await pipelineCount()).toBe(0); }); it("rejects a malformed SHA for good", async () => { const res = await notify(pushEvent("abc")); await expectError(res, 422, "PushRefused", "hex object IDs"); }); it("rejects an event for another repo", async () => { const res = await notify( pushEvent(sha(0x10a), { repo: "did:plc:cccccccccccccccccccccccc" }), ); await expectError(res, 400, "InvalidRequest", "different repo"); }); it("refuses a knot that does not host the repo", async () => { const res = await notify(pushEvent(sha(0x10b)), strangerToken()); await expectError(res, 403, "Auth", "does not host"); }); it("refuses a replayed token", async () => { const jwt = token(); const rkey = await pipeline(await notify(pushEvent(sha(0x10c)), jwt)); await settled(rkey, [started, started]); const res = await notify(pushEvent(sha(0x10d)), jwt); await expectError(res, 403, "Auth", "already used"); }); it.each([ ["without a jti", { jti: null }], ["that lives too long", { lifetime: 600 }], ])("refuses a token %s", async (_, options) => { const res = await notify(pushEvent(sha(0x10e)), token(options)); await expectError(res, 403, "Auth", "no jti or lives too long"); }); }); describe("push webhooks", () => { it("sends a push webhook before skip-ci, once for each push", async () => { await addHook(["push"]); const event = pushEvent(sha(0x120), { pushOptions: ["skip-ci"] }); expect(await pipelines(await notify(event))).toEqual([]); await drainHooks(); expect(hookPosts).toEqual([ { event: "push", agent: "Tangled-Hook/0000000", body: { ref: "refs/heads/main", before: sha(0x1), after: sha(0x120), repository, pusher: { did: owner }, }, }, ]); expect(await pipelines(await notify(event))).toEqual([]); await drainHooks(); expect(hookPosts).toHaveLength(1); }); it("sends a push webhook for a tag", async () => { await addHook(["push"]); expect((await notify(pushEvent(sha(0x122), { ref: "refs/tags/v1" }))).status).toBe(200); await drainHooks(); expect(hookPosts).toMatchObject([{ event: "push", body: { ref: "refs/tags/v1" } }]); }); it.each<[string, object, object]>([ [ "its description, website and source", { description: "A repo", website: "https://example.com/my.repo", source: { $type: "org.tangled.repo.defs#sourceExternal", url: "https://x.y/z" }, }, { description: "A repo", website: "https://example.com/my.repo", fork: true }, ], [ "only valid values", { createdAt: "2026", description: 7, website: "javascript:alert(1)" }, { created_at: "", updated_at: "", description: "", fork: false }, ], ])("fills the push webhook repository from the manifest with %s", async (_, fields, filled) => { await addHook(["push"]); manifest = { ...manifest, ...fields }; expect((await notify(pushEvent(sha(0x123)))).status).toBe(200); await drainHooks(); expect(hookPosts).toHaveLength(1); expect(hookPosts[0]?.body).toHaveProperty("repository", { ...repository, ...filled }); }); it.each<[string, object, () => void]>([ ["a deleted ref", { newSha: "0".repeat(40) }, () => {}], ["a ref outside heads and tags", { ref: "refs/notes/x" }, () => {}], [ "a repo it does not serve", {}, () => (manifest = { ...manifest, spindle: "did:web:other.example.com" }), ], ])("sends no push webhook for %s", async (_, overrides, setup) => { await addHook(["push"]); setup(); expect((await notify(pushEvent(sha(0x121), overrides))).status).toBe(200); await drainHooks(); expect(hookPosts).toEqual([]); }); }); describe("service auth", () => { it("requires a token", async () => { const res = await notify(pushEvent(sha(0x200)), null); await expectError(res, 401, "AuthRequired", "token is required"); }); it.each([ ["a bad signature", () => token({ key: strangerKey }), "signature verification failed"], [ "an issuer without an atproto key", () => { routes[knotDocRoute] = () => Response.json(didDoc(knotDid)); return token(); }, "signature verification failed", ], ["another audience", () => token({ aud: "did:web:other.example.com" }), "another audience"], ["another method", () => token({ lxm: "org.tangled.ci.retryPipeline" }), "not bound to"], ["an expiry in the past", () => token({ lifetime: -60 }), "expired"], ])("rejects a token with %s", async (_, jwt, message) => { const res = await notify(pushEvent(sha(0x201)), jwt()); await expectError(res, 401, "InvalidToken", message); }); it("rejects an issuer with a query", async () => { const res = await notify(pushEvent(sha(0x204)), token({ iss: `${repoDid}?x` })); await expectError(res, 400, "InvalidRequest", "unsupported DID"); }); it("rejects a did:web issuer with a port", async () => { const res = await notify( pushEvent(sha(0x203)), token({ iss: "did:web:knot.example.com%3A8443" }), ); await expectError(res, 401, "InvalidToken", "token issuer"); }); it("limits issuer lookups to 120 a minute per client IP", async () => { let lookups = 0; routes[knotDocRoute] = () => { lookups++; return Response.json(didDoc(knotDid, knotKey)); }; const jwt = token({ key: strangerKey }); const allowed = await Promise.all( Array.from({ length: 120 }, () => notify({}, jwt, "203.0.113.30")), ); expect(allowed.map((res) => res.status)).toEqual(Array(120).fill(401)); const limited = await notify({}, jwt, "203.0.113.30"); await expectError(limited, 429, "RateLimitExceeded", "too many"); expect(lookups).toBe(120); expect((await notify({}, jwt, "203.0.113.31")).status).toBe(401); }); it("answers 503 when the issuer cannot be resolved", async () => { routes[knotDocRoute] = offline; const res = await notify(pushEvent(sha(0x202))); await expectError(res, 503, "UpstreamUnavailable", knotDocRoute.slice("GET ".length)); }); }); async function manualBuilds(res: Response, count: number): Promise { const rkey = await pipeline(res); await settled( rkey, Array.from({ length: count }, () => started), ); return sentBuilds; } describe("manual", () => { it.each(overLimits)("fails one run of the named workflows for %s", async (_, setup) => { await withWorkflowFiles(setup, async () => { const event = manualEvent(sha(0x30d), { workflows: ["ci.yml"] }); await expectTooLarge(await pipeline(await notify(event, ownerToken()))); }); }); it("runs the named workflows on the given commit", async () => { const builds = await manualBuilds( await notify( manualEvent(sha(0x300), { ref: "refs/heads/feature", workflows: ["lint.yaml"] }), ownerToken(), ), 1, ); expect(builds.map((build) => build.url)).toEqual([pipelineUrls.lint]); expect(builds[0]?.body).toMatchObject({ commit: sha(0x300), branch: "feature", ...commitInfo, }); }); it("also runs the workflows that a named workflow waits for", async () => { await withWorkflowFiles( () => (workflowFiles["lint.yaml"] += " after: [always]\n"), async () => { const res = await notify( manualEvent(sha(0x30e), { workflows: ["lint.yaml"] }), ownerToken(), ); await settled(await pipeline(res), [ { workflow: "always.yml", ...started }, { workflow: "lint.yaml", status: "pending" }, ]); expect(sentBuilds.map((build) => build.url)).toEqual([pipelineUrls.always]); }, ); }); it("passes inputs to the builds", async () => { const inputs = [ { key: "target", value: "prod" }, { key: "_dry_run2", value: "1" }, ]; const builds = await manualBuilds( await notify(manualEvent(sha(0x30c), { workflows: ["ci.yml"], inputs }), ownerToken()), 1, ); expect(builds[0]?.body).toMatchObject({ env: { TANGLED_INPUT_TARGET: "prod", TANGLED_INPUT__DRY_RUN2: "1" }, }); }); it.each([ ["an input name with a dash", { inputs: [{ key: "a-b", value: "x" }] }, "input"], ["an empty input name", { inputs: [{ key: "", value: "x" }] }, "input"], [ "an input name that starts with a digit", { inputs: [{ key: "1a", value: "x" }] }, "input", ], ["an input without a value", { inputs: [{ key: "a" }] }, "input"], ["an inputs map", { inputs: { a: "b" } }, "input"], ["a ref that does not exist", { ref: "refs/heads/gone" }, "refs/heads/gone"], ])("rejects %s", async (_, overrides, message) => { const res = await notify(manualEvent(sha(0x30d), overrides), ownerToken()); await expectError(res, 400, "InvalidRequest", message); expect(sentBuilds).toEqual([]); }); it("refuses a replayed token", async () => { const jwt = ownerToken(); await pipeline(await notify(manualEvent(sha(0x30a), { workflows: ["ci.yml"] }), jwt)); const res = await notify(manualEvent(sha(0x30a), { workflows: ["ci.yml"] }), jwt); await expectError(res, 403, "Auth", "already used"); }); it("refuses a token without a jti", async () => { const jwt = token({ key: ownerKey, iss: owner, jti: null }); const res = await notify(manualEvent(sha(0x30b)), jwt); await expectError(res, 403, "Auth", "no jti or lives too long"); }); it.each([ ["without a filter", {}], ["with an empty filter", { workflows: [] }], ])("runs every workflow %s and ignores when", async (_, overrides) => { const builds = await manualBuilds( await notify(manualEvent(sha(0x301), overrides), ownerToken()), 3, ); expect(builds.map((build) => build.url).toSorted()).toEqual([ pipelineUrls.always, pipelineUrls.treadle, pipelineUrls.lint, ]); }); it.each([ [ "valid fields", { ref: "refs/heads/main", workflows: ["ci.yml"], inputs: [{ key: "a", value: "b" }] }, { ref: "refs/heads/main", workflows: ["ci.yml"], inputs: [{ key: "a", value: "b" }] }, ], ["fields of the wrong shape", { ref: 5, workflows: ["ci.yml", 3], extra: true }, {}], ])("stores a manual trigger with %s", async (_, overrides, fields) => { const rkey = await pipeline(await notify(manualEvent(sha(0x309), overrides), ownerToken())); const stored = await runInDurableObject(store, (_instance, state) => ({ trigger: state.storage.sql .exec<{ trigger: string }>("SELECT trigger FROM pipelines WHERE rkey = ?", rkey) .one().trigger, triggers: state.storage.sql .exec<{ triggers: string }>( "SELECT triggers FROM workflow_runs WHERE pipeline = ? AND workflow = 'ci.yml'", rkey, ) .one().triggers, })); expect(JSON.parse(stored.trigger)).toEqual({ $type: "org.tangled.ci.trigger.manual", repo: repoDid, commit: sha(0x309), ...fields, }); expect(JSON.parse(stored.triggers)).toEqual(["org.tangled.event.push"]); await vi.waitFor(async () => expect(await alarmTime()).toBeGreaterThan(Date.now())); }); it.each([ ["no ref", {}, undefined, "trunk"], ["an empty ref", { ref: "" }, undefined, "trunk"], ["no refs record", {}, () => xrpcError("RecordNotFound"), "HEAD"], [ "an empty default branch", {}, () => record({ $type: "org.tangled.git.refs", defaultBranch: "", refs: [] }), "HEAD", ], ])("builds the default branch for %s", async (_, event, refs, branch) => { if (refs !== undefined) { routes[refsRoute] = refs; } const builds = await manualBuilds( await notify( manualEvent(sha(0x303), { workflows: ["ci.yml"], ...event }), ownerToken(), ), 1, ); expect(builds[0]?.body).toMatchObject({ commit: sha(0x303), branch }); }); it("fails when the knot cannot name the default branch", async () => { routes[refsRoute] = failing(500); const res = await notify(manualEvent(sha(0x304), { workflows: ["ci.yml"] }), ownerToken()); await expectError( res, 500, "InternalError", "https://knot.example.com/xrpc/com.atproto.repo.getRecord: HTTP 500", ); }); it("rejects a filter that names no workflow", async () => { const res = await notify( manualEvent(sha(0x306), { workflows: ["nope.yml"] }), ownerToken(), ); await expectError(res, 400, "InvalidRequest", "no workflows matched"); }); }); it("rejects other events", async () => { const res = await notify({ repo: repoDid, event: { $type: "org.tangled.event.pullRequest" } }); await expectError(res, 400, "InvalidRequest", "not a push or manual trigger"); }); describe("trustedWorkflows", () => { const knot = "https://knot.example.com"; const job = "engine: cloudflare\nsteps: [{name: a, command: b}]\n"; const ci = parseWorkflow("ci.yml", job); const main: Record = {}; const blobRefs: string[] = []; beforeEach(() => { routes[blobRoute] = (url) => { blobRefs.push(url.searchParams.get("ref") ?? ""); const file = main[url.searchParams.get("path") ?? ""]; return ( workflowRef(url) ?? (file === undefined ? xrpcError("PathNotFound") : new Response(file)) ); }; }); afterEach(() => { for (const key of Object.keys(main)) { delete main[key]; } blobRefs.length = 0; }); function trusted( event: "push" | "manual" | "pull_request", branch: string | null, commit = sha(0xfff), workflows = [ci], ): Promise { return trustedWorkflows(knot, repoDid, event, branch, commit, workflows).then((set) => [ ...set, ]); } it("trusts a push to the default branch without a trusted key", async () => { expect(await trusted("push", "trunk")).toEqual(["ci.yml"]); expect(blobRefs).toEqual(["refs/heads/trunk"]); expect(await trusted("push", "feature")).toEqual([]); }); it("reads trusted branches from the default-branch file only", async () => { main[".tangled/workflows/ci.yml"] = `${job}trusted: {branch: ['release/*']}\n`; const own = parseWorkflow("ci.yml", `${job}trusted: {branch: [feature]}\n`); expect(await trusted("push", "release/1")).toEqual(["ci.yml"]); expect(await trusted("push", "feature", sha(0xfff), [own])).toEqual([]); expect(await trusted("push", "trunk")).toEqual([]); }); it("does not trust a pull request, a tag or a non-Cloudflare workflow", async () => { main[".tangled/workflows/ci.yml"] = `${job}trusted: {branch: ['**']}\n`; expect(await trusted("pull_request", "trunk")).toEqual([]); expect(await trusted("push", null)).toEqual([]); expect( await trusted("push", "trunk", sha(0xfff), [ parseWorkflow("ci.yml", "engine: buildkite"), ]), ).toEqual([]); expect(blobRefs).toEqual([]); }); it("trusts a manual run only at the branch head", async () => { expect(await trusted("manual", "trunk")).toEqual(["ci.yml"]); expect(await trusted("manual", "trunk", sha(0xfff).toUpperCase())).toEqual(["ci.yml"]); expect(await trusted("manual", "trunk", sha(0x123))).toEqual([]); main[".tangled/workflows/ci.yml"] = `${job}trusted: {branch: ['release/*']}\n`; commits.add("refs/heads/release/1"); expect(await trusted("manual", "release/1")).toEqual(["ci.yml"]); expect(await trusted("manual", "release/1", sha(0x123))).toEqual([]); }); it.each([ ["the workflow file", blobRoute], ["the default branch", refsRoute], ["the branch head", commitsRoute], ])("does not trust a run when the knot fails to read %s", async (_, route) => { routes[route] = failing(); expect(await trusted("manual", "trunk")).toEqual([]); }); it("does not trust a run without a default branch", async () => { routes[refsRoute] = () => record({ $type: "org.tangled.git.refs", refs: [] }); expect(await trusted("push", "HEAD")).toEqual([]); }); }); async function runnerTrust(rkey: string): Promise { return await vi.waitFor(async () => { const { runner } = await runInDurableObject( exports.Store.getByName("main"), (_instance, state) => state.storage.sql .exec<{ runner: string }>( "SELECT runner FROM workflow_runs WHERE pipeline = ? AND workflow = 'cf.yml'", rkey, ) .one(), ); const trusted = await runInDurableObject( exports.Runner.getByName(runner), (_instance, state) => state.storage.kv.get<{ trusted: boolean }>(`run/${rkey}`)?.trusted, ); expect(trusted).toBeTypeOf("boolean"); return trusted === true; }); } describe("engine: cloudflare", () => { const job = "engine: cloudflare\nsteps: [{name: a, command: b}]\n"; beforeEach(() => { workflowFiles["cf.yml"] = job; }); afterEach(() => { delete workflowFiles["cf.yml"]; }); it.each([ [ "a push to the default branch", () => notify(pushEvent(sha(0x500), { ref: "refs/heads/trunk" })), true, ], ["a tag push", () => notify(pushEvent(sha(0x502), { ref: "refs/tags/v1" })), false], [ "a manual run at the branch head", () => notify(manualEvent(sha(0xfff), { ref: "refs/heads/trunk" }), ownerToken()), true, ], ])("hands the trust of %s to the Runner", async (_, event, trusted) => { const [rkey = ""] = await pipelines(await event()); expect(await runnerTrust(rkey)).toBe(trusted); }); it("keeps the stored trust on a retry", async () => { const [rkey = ""] = await pipelines( await notify(pushEvent(sha(0x504), { ref: "refs/heads/trunk" })), ); expect(await runnerTrust(rkey)).toBe(true); workflowFiles["cf.yml"] = `${job}trusted: {branch: [other]}\n`; const again = await retryKey( await call(retry, { repo: repoDid, pipeline: rkey }, ownerToken(retry)), ); expect(await runnerTrust(again)).toBe(true); const [other = ""] = await pipelines(await notify(pushEvent(sha(0x505)))); expect(await runnerTrust(other)).toBe(false); workflowFiles["cf.yml"] = `${job}trusted: {branch: [main]}\n`; const kept = await retryKey( await call(retry, { repo: repoDid, pipeline: other }, ownerToken(retry)), ); expect(await runnerTrust(kept)).toBe(false); }); }); const invalidFiles: Record = { "broken.yml": "when: [\n", "nosteps.yml": "engine: cloudflare\nwhen: [{event: push, branch: main}]\n", "badwhen.yml": "engine: buildkite\nwhen: 5\n", "unknown.yml": "engine: nixery\nwhen: 5\n", }; function failedRun(workflow: string, error: string): Partial { return { workflow, status: "failed", error }; } const brokenRun = failedRun( "broken.yml", parseDocument(invalidFiles["broken.yml"] ?? "").errors[0]?.message ?? "", ); const badWhenRun = failedRun("badwhen.yml", "when must be a list"); const noStepsRun = failedRun("nosteps.yml", "steps must be a non-empty list"); const waitsFor = (after: string) => `engine: buildkite\ntreadle: {after: [${after}]}\n`; describe("invalid workflows", () => { beforeEach(() => { Object.assign(workflowFiles, invalidFiles); }); afterEach(() => { for (const name of Object.keys(invalidFiles)) { delete workflowFiles[name]; } }); it("fails the runs of a push that match by the readable when", async () => { const rkey = await pipeline(await notify(pushEvent(sha(0x700)))); await settled(rkey, [ { workflow: "always.yml", ...started }, badWhenRun, brokenRun, { workflow: "ci.yml", ...started }, noStepsRun, ]); expect(sentBuilds.map((build) => build.url).toSorted()).toEqual([ pipelineUrls.always, pipelineUrls.treadle, ]); const res = await exports.default.fetch( `https://treadle.example.com/xrpc/org.tangled.ci.getPipeline?repo=${repoDid}&pipeline=${rkey}`, ); const body: { workflows: { id: string }[] } = await res.json(); assertOutput("org.tangled.ci.getPipeline", body); const run = body.workflows.find((w) => w.id === "nosteps.yml"); expect(run).toMatchObject({ status: "failed", error: "steps must be a non-empty list" }); expect(run).toHaveProperty("finishedAt"); }); it("never trusts an invalid workflow or builds it on Buildkite", () => { const workflow = invalidWorkflow("badwhen.yml", invalidFiles["badwhen.yml"] ?? "", "bad"); expect( newRuns( env, workflow === undefined ? [] : [workflow], "repo", new Set(["badwhen.yml"]), ), ).toMatchObject([{ engine: "cloudflare", trusted: false, error: "bad" }]); }); it("fails a run with an unknown or cyclic treadle.after", () => { const workflows = [ parseWorkflow("a.yml", waitsFor("b.yml")), parseWorkflow("b.yml", waitsFor("a.yml")), parseWorkflow("c.yml", waitsFor("a.yml")), parseWorkflow("d.yml", waitsFor("gone.yml")), parseWorkflow("e.yml", waitsFor("e.yml")), ]; const cycle = "treadle.after makes a cycle: a.yml → b.yml → a.yml"; const result = newRuns(env, workflows, "repo"); expect(result).toMatchObject([ { workflow: "a.yml", error: cycle }, { workflow: "b.yml", error: cycle }, { workflow: "c.yml", engine: "buildkite", after: ["a.yml"] }, { workflow: "d.yml", error: "treadle.after: gone.yml is not a workflow of this pipeline", }, { workflow: "e.yml", error: "treadle.after makes a cycle: e.yml → e.yml" }, ]); expect(result[2]).not.toHaveProperty("error"); }); it("fails the runs without a readable when for every push", async () => { const rkey = await pipeline(await notify(pushEvent(sha(0x701), { ref: "refs/heads/dev" }))); await settled(rkey, [{ workflow: "always.yml", ...started }, badWhenRun, brokenRun]); }); it("fails the runs of a manual run", async () => { const res = await notify( manualEvent(sha(0x702), { workflows: ["broken.yml", "nosteps.yml", "unknown.yml"] }), ownerToken(), ); await settled(await pipeline(res), [brokenRun, noStepsRun]); expect(sentBuilds).toEqual([]); }); it("reads the file again on a retry", async () => { await seedPipeline({ "nosteps.yml": "failed" }); workflowFiles["nosteps.yml"] = "engine: cloudflare\nsteps: [{name: a}]\n"; const rkey = await retryKey( await call(retry, { repo: repoDid, pipeline: "3old" }, ownerToken(retry)), ); await settled(rkey, [failedRun("nosteps.yml", "steps[0].command must be a string")]); }); }); const retry = "org.tangled.ci.retryPipeline"; const approve = "org.tangled.ci.approvePipeline"; const otherRepo = "did:plc:cccccccccccccccccccccccc"; const pushTrigger = { $type: "org.tangled.event.push", repo: repoDid, ref: "refs/heads/release", oldSha: sha(0x1), newSha: sha(0x400), actor: "did:plc:pppppppppppppppppppppppp", }; function call(nsid: string, body: object, jwt: string): Promise { return exports.default.fetch(`https://treadle.example.com/xrpc/${nsid}`, { method: "POST", headers: { "Content-Type": "application/json", Authorization: `Bearer ${jwt}` }, body: JSON.stringify(body), }); } async function seedPipeline( statuses: Record, trigger: object = pushTrigger, buildNumbers: Record = {}, ): Promise { commits.add(sha(0x400)); await runInDurableObject(store, (_instance, state) => { seed(state.storage, [ { rkey: "3old", repo: repoDid, trigger, commit: sha(0x400), createdAt: new Date().toISOString(), runs: Object.entries(statuses).map(([workflow, status]) => ({ workflow, status, buildNumber: buildNumbers[workflow], })), }, ]); }); } function storedTrigger(rkey: string): Promise { return runInDurableObject( store, (_instance, state) => JSON.parse( state.storage.sql .exec<{ trigger: string }>("SELECT trigger FROM pipelines WHERE rkey = ?", rkey) .one().trigger, ) as unknown, ); } async function retryKey(res: Response): Promise { expect(res.status).toBe(200); const body: { pipeline: string } = await res.json(); assertOutput(retry, body); expect(body.pipeline).not.toBe("3old"); return body.pipeline; } describe("retryPipeline", () => { it.each([ ["no filter", {}], ["an empty filter", { workflows: [] }], ])("starts the stored workflows again as a new pipeline with %s", async (_, filter) => { await seedPipeline({ "ci.yml": "running", "always.yml": "success" }); const rkey = await retryKey( await call(retry, { repo: repoDid, pipeline: "3old", ...filter }, ownerToken(retry)), ); await settled(rkey, [ { workflow: "always.yml", ...started }, { workflow: "ci.yml", ...started }, ]); expect(sentBuilds.map((build) => build.url).toSorted()).toEqual([ pipelineUrls.always, pipelineUrls.treadle, ]); for (const build of sentBuilds) { expect(build.body).toMatchObject({ commit: sha(0x400), branch: "release", env: { TREADLE_PIPELINE: rkey }, ...commitInfo, }); } expect(await storedTrigger(rkey)).toEqual({ ...pushTrigger, actor: owner }); expect((await runs("3old")).map((run) => run.status)).toEqual(["success", "running"]); }); it("fails one run when the workflow files are over the limits", async () => { await seedPipeline({ "ci.yml": "failed" }); await withWorkflowFiles(addWorkflowFiles, async () => { const res = await call(retry, { repo: repoDid, pipeline: "3old" }, ownerToken(retry)); await expectTooLarge(await retryKey(res)); }); }); it("starts only the named workflows", async () => { await seedPipeline({ "ci.yml": "failed", "always.yml": "success" }); const rkey = await retryKey( await call( retry, { repo: repoDid, pipeline: "3old", workflows: ["ci.yml"] }, ownerToken(retry), ), ); await settled(rkey, [{ workflow: "ci.yml", ...started }]); expect(sentBuilds.map((build) => build.url)).toEqual([pipelineUrls.treadle]); }); it("starts a named workflow again with the workflows that it waits for", async () => { await seedPipeline({ "ci.yml": "failed", "always.yml": "success" }); await withWorkflowFiles( () => (workflowFiles["ci.yml"] += "\n after: [always]"), async () => { const rkey = await retryKey( await call( retry, { repo: repoDid, pipeline: "3old", workflows: ["ci.yml"] }, ownerToken(retry), ), ); await settled(rkey, [ { workflow: "always.yml", ...started }, { workflow: "ci.yml", status: "pending" }, ]); expect(sentBuilds.map((build) => build.url)).toEqual([pipelineUrls.always]); }, ); }); it.each([ ["a workflow outside the pipeline", { workflows: ["lint.yaml"] }, "has no workflow lint"], ["a workflow that is gone", { workflows: ["gone.yml"] }, "no workflows matched"], ["a malformed filter", { workflows: "ci.yml" }, "list of names"], ["no pipeline", { pipeline: "" }, "pipeline is required"], ["no repo", { repo: "nope" }, "repo DID"], ])("rejects %s", async (_, body, message) => { await seedPipeline({ "ci.yml": "failed", "gone.yml": "failed" }); const res = await call( retry, { repo: repoDid, pipeline: "3old", ...body }, ownerToken(retry), ); await expectError(res, 400, "InvalidRequest", message); }); it("builds the default branch for a manual trigger without a ref", async () => { await seedPipeline( { "lint.yaml": "failed" }, { $type: "org.tangled.ci.trigger.manual", repo: repoDid, commit: sha(0x400) }, ); const rkey = await retryKey( await call(retry, { repo: repoDid, pipeline: "3old" }, ownerToken(retry)), ); await settled(rkey, [started]); expect(sentBuilds[0]?.body).toMatchObject({ commit: sha(0x400), branch: "trunk" }); }); const user = "did:plc:uuuuuuuuuuuuuuuuuuuuuuuu"; it.each([ ["the user of a repo token", repoKey, repoDid, { sub: user }, user], ["a repo token without a user", repoKey, repoDid, undefined, repoDid], ["a repo token with a malformed user", repoKey, repoDid, { sub: "nope" }, repoDid], ["the owner even with a user", ownerKey, owner, { sub: user }, owner], ])("names %s as the actor", async (_, key, iss, act, actor) => { await seedPipeline({ "ci.yml": "failed" }); const jwt = token({ key, iss, lxm: retry, ...(act === undefined ? {} : { act }) }); const rkey = await retryKey(await call(retry, { repo: repoDid, pipeline: "3old" }, jwt)); expect(await storedTrigger(rkey)).toMatchObject({ actor }); await settled(rkey, [started]); }); it.each([ [ "a fork", { sourceRepo: "did:plc:ffffffffffffffffffffffff", ref: "refs/heads/pull/3tik" }, "pull/3tik", ], ["a branch of the repo", { sourceBranch: "feature", ref: "refs/heads/feature" }, "feature"], ])("builds a pull request from %s on its own branch", async (_, source, branch) => { await seedPipeline( { "lint.yaml": "failed" }, { $type: "org.tangled.event.pullRequest", action: "opened", pull: `at://${repoDid}/org.tangled.track.ticket/3tik`, targetBranch: "main", sourceSha: sha(0x400), ...source, }, ); const rkey = await retryKey( await call(retry, { repo: repoDid, pipeline: "3old" }, ownerToken(retry)), ); await settled(rkey, [started]); expect(sentBuilds[0]?.body).toMatchObject({ branch, pull_request_base_branch: "main" }); }); it("refuses a token that was already used", async () => { await seedPipeline({ "ci.yml": "failed" }); const jwt = ownerToken(retry); await retryKey(await call(retry, { repo: repoDid, pipeline: "3old" }, jwt)); const res = await call(retry, { repo: repoDid, pipeline: "3old" }, jwt); await expectError(res, 403, "Auth", "already used"); }); it("refuses a blocked pipeline", async () => { await seedPipeline({ "ci.yml": "blocked", "always.yml": "success" }, pushTrigger, { "ci.yml": 7, }); const res = await call(retry, { repo: repoDid, pipeline: "3old" }, ownerToken(retry)); await expectError(res, 400, "PipelineNotRetryable", "blocked"); }); it.each([ ["an unknown pipeline", { repo: repoDid, pipeline: "3nope" }], ["a pipeline of another repo", { repo: otherRepo, pipeline: "3old" }], ])("reports %s", async (_, body) => { await seedPipeline({ "ci.yml": "failed" }); const res = await call(retry, body, ownerToken(retry)); await expectError(res, 404, "PipelineNotFound", body.pipeline); }); }); function buildUrl(n: number): string { return `GET ${pipelineUrls.treadle}/${n}`; } function unblockUrl(id: string): string { return `${buildkiteBase}/gpanders/jobs/${id}/unblock`; } function unblocks(): string[] { return sentBuilds.filter((b) => b.url.endsWith("/unblock")).map((b) => b.url); } describe("approvePipeline", () => { beforeEach(() => { routes[buildUrl(7)] = () => Response.json({ id: "uuid-7", number: 7, state: "blocked", jobs: [ { id: "script", type: "script", state: "passed" }, { id: "gate", type: "manual", state: "blocked", unblockable: true }, { id: "locked", type: "manual", state: "blocked", unblockable: false }, { id: "done", type: "manual", state: "unblocked", unblockable: false }, { id: "trap", type: "waiter", state: "blocked", unblockable: true }, ], }); routes[buildUrl(8)] = () => Response.json({ id: "uuid-8", number: 8, state: "blocked", jobs: [{ id: "second", type: "manual", state: "blocked", unblockable: true }], }); for (const id of ["gate", "second"]) { routes[`PUT ${unblockUrl(id)}`] = () => Response.json({ id, state: "unblocked" }); } }); it("unblocks the block steps of every blocked run", async () => { await seedPipeline( { "ci.yml": "blocked", "always.yml": "blocked", "lint.yaml": "running" }, pushTrigger, { "ci.yml": 7, "always.yml": 8, "lint.yaml": 9 }, ); const res = await call(approve, { repo: repoDid, pipeline: "3old" }, ownerToken(approve)); expect(res.status).toBe(200); expect(await res.text()).toBe(""); expect(unblocks().toSorted()).toEqual([unblockUrl("gate"), unblockUrl("second")]); for (const build of sentBuilds.filter((b) => b.url.endsWith("/unblock"))) { expect(build).toMatchObject({ authorization: "Bearer test-token", body: {} }); } expect((await runs("3old")).map((run) => run.status)).toEqual([ "blocked", "blocked", "running", ]); }); it("refuses a pipeline without a blocked run", async () => { await seedPipeline({ "ci.yml": "running", "always.yml": "failed" }); const res = await call(approve, { repo: repoDid, pipeline: "3old" }, ownerToken(approve)); await expectError(res, 400, "PipelineNotBlocked", "3old"); expect(sentBuilds).toEqual([]); }); it("fails when Buildkite does not answer", async () => { routes[buildUrl(7)] = failing(); await seedPipeline({ "ci.yml": "blocked" }, pushTrigger, { "ci.yml": 7 }); const res = await call(approve, { repo: repoDid, pipeline: "3old" }, ownerToken(approve)); await expectError(res, 503, "UpstreamUnavailable", "HTTP 502"); }); it("fails when Buildkite answers without jobs", async () => { routes[buildUrl(7)] = () => new Response("not json"); await seedPipeline({ "ci.yml": "blocked" }, pushTrigger, { "ci.yml": 7 }); const res = await call(approve, { repo: repoDid, pipeline: "3old" }, ownerToken(approve)); await expectError(res, 503, "UpstreamUnavailable", "no jobs"); }); it("fails when Buildkite refuses the unblock", async () => { routes[`PUT ${unblockUrl("gate")}`] = failing(422); await seedPipeline({ "ci.yml": "blocked" }, pushTrigger, { "ci.yml": 7 }); const res = await call(approve, { repo: repoDid, pipeline: "3old" }, ownerToken(approve)); await expectError(res, 503, "UpstreamUnavailable", "HTTP 422"); }); it("reports a pipeline of another repo", async () => { await seedPipeline({ "ci.yml": "blocked" }, pushTrigger, { "ci.yml": 7 }); const res = await call(approve, { repo: otherRepo, pipeline: "3old" }, ownerToken(approve)); await expectError(res, 404, "PipelineNotFound", "3old"); }); }); const cancel = "org.tangled.ci.cancelPipeline"; function cancelUrl(n: number): string { return `${pipelineUrls.treadle}/${n}/cancel`; } function cancels(): string[] { return sentBuilds.filter((b) => b.url.endsWith("/cancel")).map((b) => b.url); } describe("cancelPipeline", () => { beforeEach(() => { for (const n of [7, 8]) { routes[`PUT ${cancelUrl(n)}`] = () => Response.json({ number: n, state: "canceling" }); } }); it("cancels the builds and the runs that have not started, and skips finished runs", async () => { await seedPipeline( { "always.yml": "pending", "ci.yml": "running", "lint.yaml": "success", "nix.yml": "blocked", "other.yml": "started", }, pushTrigger, { "ci.yml": 7, "lint.yaml": 9, "other.yml": 8 }, ); const res = await call(cancel, { repo: repoDid, pipeline: "3old" }, ownerToken(cancel)); expect(res.status).toBe(200); expect(await res.text()).toBe(""); expect(cancels().toSorted()).toEqual([cancelUrl(7), cancelUrl(8)]); expect(sentBuilds.map((b) => b.authorization)).toEqual([ "Bearer test-token", "Bearer test-token", ]); expect((await runs("3old")).map((run) => [run.workflow, run.status])).toEqual([ ["always.yml", "cancelled"], ["ci.yml", "cancelled"], ["lint.yaml", "success"], ["nix.yml", "cancelled"], ["other.yml", "cancelled"], ]); await runInDurableObject(store, async (_instance, state) => { state.storage.kv.put("buildWorkAt", Date.now()); await state.storage.setAlarm(Date.now()); }); await runDurableObjectAlarm(store); expect(sentBuilds.filter((b) => !b.url.endsWith("/cancel"))).toEqual([]); }); it.each([ ["keeps a run cancelled when an older webhook arrives", false, "running", -1, "cancelled"], ["keeps a run that finishes during the cancel", true, "success", 0, "success"], ])("%s", async (_, during, status, age, expected) => { await seedPipeline({ "ci.yml": "running" }, pushTrigger, { "ci.yml": 7 }); await runInDurableObject(store, (_instance, state) => { state.storage.sql.exec("UPDATE workflow_runs SET build_uuid = 'uuid-7'"); }); const webhook = (): Promise => exports.Store.getByName("main").updateBuild({ id: "uuid-7", number: 7, url: undefined, status: status === "success" ? "success" : "running", startedAt: undefined, finishedAt: undefined, ts: Math.floor(Date.now() / 1000) + age, pipeline: undefined, workflow: undefined, slug: undefined, commit: undefined, }); if (during) { routes[`PUT ${cancelUrl(7)}`] = async () => { await webhook(); return Response.json({ number: 7, state: "canceling" }); }; } expect( (await call(cancel, { repo: repoDid, pipeline: "3old" }, ownerToken(cancel))).status, ).toBe(200); if (!during) { await webhook(); } expect((await runs("3old")).map((run) => run.status)).toEqual([expected]); }); it("cancels a build that Buildkite creates during the cancel", async () => { routes[`PUT ${cancelUrl(1)}`] = () => Response.json({ number: 1, state: "canceling" }); const start = created(); routes[treadleRoute] = async () => { const body = { repo: repoDid, pipeline: "3old" }; expect((await call(cancel, body, ownerToken(cancel))).status).toBe(200); return start(); }; await seedPipeline({ "ci.yml": "pending" }); await runInDurableObject(store, async (_instance, state) => { state.storage.kv.put("buildWorkAt", Date.now()); await state.storage.setAlarm(Date.now()); }); await runDurableObjectAlarm(store); expect(cancels()).toEqual([cancelUrl(1)]); expect(await runs("3old")).toMatchObject([{ status: "cancelled" }]); const build = await runInDurableObject(store, (_instance, state) => state.storage.sql.exec("SELECT build_uuid, build_number FROM workflow_runs").one(), ); expect(build).toEqual({ build_uuid: "uuid-1", build_number: 1 }); }); it("cancels a build that waits at a block step", async () => { routes[`PUT ${cancelUrl(7)}`] = () => Response.json({ message: "cannot cancel" }, { status: 422 }); await seedPipeline({ "ci.yml": "blocked" }, pushTrigger, { "ci.yml": 7 }); expect( (await call(cancel, { repo: repoDid, pipeline: "3old" }, ownerToken(cancel))).status, ).toBe(200); expect(cancels()).toEqual([cancelUrl(7)]); expect((await runs("3old")).map((run) => run.status)).toEqual(["cancelled"]); }); it("cancels only the named workflows", async () => { await seedPipeline({ "ci.yml": "running", "always.yml": "running" }, pushTrigger, { "ci.yml": 7, "always.yml": 8, }); const body = { repo: repoDid, pipeline: "3old", workflows: ["ci.yml"] }; expect((await call(cancel, body, ownerToken(cancel))).status).toBe(200); expect(cancels()).toEqual([cancelUrl(7)]); expect((await runs("3old")).map((run) => run.status)).toEqual(["running", "cancelled"]); }); it("leaves a build that Buildkite already finished to its webhook", async () => { routes[`PUT ${cancelUrl(7)}`] = () => Response.json({ message: "finished" }, { status: 422 }); await seedPipeline({ "ci.yml": "running" }, pushTrigger, { "ci.yml": 7 }); expect( (await call(cancel, { repo: repoDid, pipeline: "3old" }, ownerToken(cancel))).status, ).toBe(200); expect((await runs("3old")).map((run) => run.status)).toEqual(["running"]); }); it("keeps the cancels that succeeded when Buildkite fails for a build", async () => { routes[`PUT ${cancelUrl(8)}`] = failing(500); await seedPipeline( { "always.yml": "running", "ci.yml": "running", "lint.yaml": "pending" }, pushTrigger, { "always.yml": 7, "ci.yml": 8 }, ); const res = await call(cancel, { repo: repoDid, pipeline: "3old" }, ownerToken(cancel)); await expectError(res, 503, "UpstreamUnavailable", "HTTP 500"); expect((await runs("3old")).map((run) => run.status)).toEqual([ "cancelled", "running", "cancelled", ]); }); it.each([ [ "a workflow outside the pipeline", { workflows: ["gone.yml"] }, 400, "InvalidRequest", "has no workflow gone.yml", ], ["a malformed filter", { workflows: "ci.yml" }, 400, "InvalidRequest", "list of names"], ["a pipeline of another repo", { repo: otherRepo }, 404, "PipelineNotFound", "3old"], ["no pipeline", { pipeline: "" }, 400, "InvalidRequest", "pipeline is required"], ])("rejects %s", async (_, overrides, status, error, message) => { await seedPipeline({ "ci.yml": "running" }, pushTrigger, { "ci.yml": 7 }); const body = { repo: repoDid, pipeline: "3old", ...overrides }; await expectError(await call(cancel, body, ownerToken(cancel)), status, error, message); expect(sentBuilds).toEqual([]); }); }); const addSecret = "org.tangled.secret.addSecret"; const removeSecret = "org.tangled.secret.removeSecret"; const guarded: [string, () => Promise][] = [ [method, () => Promise.resolve(manualEvent(sha(0x310), { workflows: ["ci.yml"] }))], [ retry, async () => { await seedPipeline({ "ci.yml": "failed" }); return { repo: repoDid, pipeline: "3old" }; }, ], [ approve, async () => { routes[buildUrl(7)] = () => Response.json({ id: "uuid-7", number: 7, jobs: [{ id: "gate", type: "manual", state: "blocked", unblockable: true }], }); routes[`PUT ${unblockUrl("gate")}`] = () => Response.json({ id: "gate" }); await seedPipeline({ "ci.yml": "blocked" }, pushTrigger, { "ci.yml": 7 }); return { repo: repoDid, pipeline: "3old" }; }, ], [ cancel, async () => { routes[`PUT ${cancelUrl(7)}`] = () => Response.json({ number: 7 }); await seedPipeline({ "ci.yml": "running" }, pushTrigger, { "ci.yml": 7 }); return { repo: repoDid, pipeline: "3old" }; }, ], [ addSecret, async () => { await exports.Secrets.getByName(repoDid).remove("ACCESS"); return { repo: repoDid, key: "ACCESS", value: "x" }; }, ], [ removeSecret, async () => { await exports.Secrets.getByName(repoDid).add("ACCESS", "x", owner); return { repo: repoDid, key: "ACCESS" }; }, ], ]; describe.each(guarded)("access to %s", (nsid, setup) => { it("accepts a token from the repo itself", async () => { const body = await setup(); const res = await call(nsid, body, token({ key: repoKey, iss: repoDid, lxm: nsid })); expect(res.status).toBe(200); }); it("refuses a caller who does not own the repo", async () => { const res = await call(nsid, await setup(), strangerToken(nsid)); await expectError(res, 403, "AccessControl", "cannot run CI"); expect(sentBuilds).toEqual([]); }); it("refuses a token for another method", async () => { const res = await call(nsid, await setup(), ownerToken(nsid === method ? retry : method)); await expectError(res, 401, "InvalidToken", "not bound to"); expect(sentBuilds).toEqual([]); }); it("refuses a replayed token", async () => { const jwt = ownerToken(nsid); expect((await call(nsid, await setup(), jwt)).status).toBe(200); await expectError(await call(nsid, await setup(), jwt), 403, "Auth", "already used"); }); it("reports a repo this spindle does not serve", async () => { const body = await setup(); manifest = { ...manifest, spindle: "did:web:other.example.com" }; const res = await call(nsid, body, ownerToken(nsid)); await expectError(res, 404, "RepoNotFound", "not served"); expect(sentBuilds).toEqual([]); }); }); const getDefinition = "org.tangled.ci.getWorkflowDefinition"; const listDefinitions = "org.tangled.ci.listWorkflowDefinitions"; const heads: Record = { HEAD: sha(0x600), "refs/heads/feature": sha(0x601) }; function definitions(nsid: string, params: Record): Promise { const query = new URLSearchParams({ repo: repoDid, ...params }); return exports.default.fetch(`https://treadle.example.com/xrpc/${nsid}?${query.toString()}`); } function definition(id: string, commit: string, triggers: string[]): object { return { id, name: id, source: { $type: "org.tangled.ci.defs#fileSource", repo: repoDid, commit, path: `.tangled/workflows/${id}`, }, triggers, }; } describe("workflow definitions", () => { beforeEach(() => { commits.add(sha(0x600)); commits.add(sha(0x601)); routes[commitsRoute] = (url) => { const hash = heads[url.searchParams.get("ref") ?? ""]; return hash === undefined ? xrpcError("RefNotFound") : Response.json({ commits: [{ hash, message: "x" }] }); }; }); it("lists the Buildkite workflows at HEAD", async () => { const res = await definitions(listDefinitions, {}); expect(res.status).toBe(200); const body: unknown = await res.json(); assertOutput(listDefinitions, body); expect(body).toEqual({ workflows: [ definition("ci.yml", sha(0x600), ["org.tangled.event.push"]), definition("lint.yaml", sha(0x600), ["org.tangled.event.pullRequest"]), definition("always.yml", sha(0x600), []), ], }); }); it("lists the workflows at a ref", async () => { const res = await definitions(listDefinitions, { ref: "refs/heads/feature" }); const body: { workflows: { source: { commit: string } }[] } = await res.json(); expect(body.workflows.map((w) => w.source.commit)).toEqual([ sha(0x601), sha(0x601), sha(0x601), ]); }); it("gets one workflow at HEAD", async () => { const res = await definitions(getDefinition, { workflow: "lint.yaml" }); expect(res.status).toBe(200); const body: unknown = await res.json(); assertOutput(getDefinition, body); expect(body).toEqual({ workflow: definition("lint.yaml", sha(0x600), ["org.tangled.event.pullRequest"]), }); }); it.each([ ["an unknown ref", listDefinitions, { ref: "nope" }, 404, "RefNotFound"], ["a long ref", listDefinitions, { ref: "r".repeat(257) }, 400, "InvalidRequest"], ["an unknown workflow", getDefinition, { workflow: "gone.yml" }, 404, "WorkflowNotFound"], [ "a workflow of another engine", getDefinition, { workflow: "nix.yml" }, 404, "WorkflowNotFound", ], ["no workflow", getDefinition, {}, 400, "InvalidRequest"], [ "a long workflow name", getDefinition, { workflow: "w".repeat(129) }, 400, "InvalidRequest", ], ["a repo that is not a DID", listDefinitions, { repo: "nope" }, 400, "InvalidRequest"], ])("rejects %s", async (_, nsid, params, status, error) => { const res = await definitions(nsid, params); expect(res.status).toBe(status); expect(await res.json()).toMatchObject({ error }); }); it("limits each client IP to 20 requests a minute", async () => { const init = { headers: { "CF-Connecting-IP": "203.0.113.20" } }; const url = `https://treadle.example.com/xrpc/${listDefinitions}?repo=nope`; const allowed = await Promise.all( Array.from({ length: 20 }, () => exports.default.fetch(url, init)), ); expect(allowed.map((res) => res.status)).toEqual(Array(20).fill(400)); const query = new URLSearchParams({ repo: repoDid, workflow: "ci.yml" }); const res = await exports.default.fetch( `https://treadle.example.com/xrpc/${getDefinition}?${query.toString()}`, init, ); await expectError(res, 429, "RateLimitExceeded", "too many requests"); const other = { headers: { "CF-Connecting-IP": "203.0.113.21" } }; expect((await exports.default.fetch(url, other)).status).toBe(400); }); it("reports no workflows for a repo without commits", async () => { routes[commitsRoute] = () => xrpcError("RefNotFound"); expect(await (await definitions(listDefinitions, {})).json()).toEqual({ workflows: [] }); const res = await definitions(getDefinition, { workflow: "ci.yml" }); await expectError(res, 404, "WorkflowNotFound", "ci.yml"); }); it.each([listDefinitions, getDefinition])( "reports a repo this spindle does not serve to %s", async (nsid) => { manifest = { ...manifest, spindle: "did:web:other.example.com" }; const res = await definitions(nsid, { workflow: "ci.yml" }); await expectError(res, 404, "RepoNotFound", "not served"); }, ); }); const knotBase = "https://knot.example.com"; const versionsRoute = `GET ${knotBase}/xrpc/org.tangled.repo.listRecordVersions`; const forkDid = "did:plc:ffffffffffffffffffffffff"; const contributor = "did:plc:kkkkkkkkkkkkkkkkkkkkkkkk"; const ticketUri = `at://${repoDid}/org.tangled.track.ticket/3tik`; async function tick(): Promise { await runInDurableObject(store, (_instance, state) => state.storage.setAlarm(Date.now() + 60_000), ); expect(await runDurableObjectAlarm(store)).toBe(true); } function pullPipelines(): Promise<{ rkey: string; trigger: unknown; branch: string }[]> { return runInDurableObject(store, (_instance, state) => state.storage.sql .exec<{ rkey: string; trigger: string; branch: string }>( "SELECT rkey, trigger, branch FROM pipelines WHERE push_key LIKE 'pull %' ORDER BY rkey", ) .toArray() .map(({ rkey, trigger, branch }) => ({ rkey, trigger: JSON.parse(trigger) as unknown, branch, })), ); } function agePipelines(days: number): Promise { return runInDurableObject(store, (_instance, state) => { state.storage.sql.exec( "UPDATE pipelines SET created_at = ?", new Date(Date.now() - days * 24 * 60 * 60_000).toISOString(), ); }); } function jetstreamState(): Promise { return runInDurableObject(store, (_instance, state) => state.storage.kv.get("jetstream")); } describe("pull requests", () => { let patches: Record>; let ticket: Record; let author: string; let reads: string[]; let streams: URL[]; function patch(rkey: string, head: string, overrides: object = {}): void { commits.add(head); patches[rkey] = { $type: "org.tangled.review.patch", ticket: "3tik", base: { oid: sha(0x1) }, head: { oid: head }, source: { repo: repoDid, branch: "feature" }, createdAt: "2026-11-07T00:00:00Z", "x-tngl-editor": owner, ...overrides, }; } function stream(...rkeys: string[]): void { routes[`GET ${jetstreamUrl}`] = (url) => { streams.push(url); return jetstreamSocket(rkeys.map((rkey, i) => commitMessage(100 + i, repoDid, rkey))); }; } beforeEach(async () => { patches = {}; ticket = { $type: "org.tangled.track.ticket", title: { text: "Fix" }, state: "open", targetBranch: "main", createdAt: "2026-11-07T00:00:00Z", }; author = owner; reads = []; streams = []; await seedPipeline({ "ci.yml": "success" }); routes[`GET ${knotBase}/xrpc/com.atproto.repo.getRecord`] = (url) => { const { collection, rkey } = Object.fromEntries(url.searchParams); reads.push(`${collection} ${rkey}`); expect(url.searchParams.get("repo")).toBe(repoDid); const value = collection === "org.tangled.track.ticket" && rkey === "3tik" ? ticket : patches[rkey ?? ""]; return value === undefined ? xrpcError("RecordNotFound") : record(value); }; routes[versionsRoute] = (url) => { reads.push("versions"); expect(Object.fromEntries(url.searchParams)).toEqual({ uri: ticketUri, limit: "1", order: "asc", }); return Response.json({ uri: ticketUri, versions: [{ changeId: "c1", editor: author, createdAt: "x", cid: "bafyt" }], }); }; }); it("starts an owner's pull request from the Jetstream tick", async () => { patch("bafy1", sha(0x501)); stream("bafy1"); await tick(); expect(streams[0]?.searchParams.getAll("dids")).toEqual([repoDid]); expect(streams[0]?.searchParams.getAll("collections")).toEqual([ "org.tangled.review.patch", "org.tangled.track.ticket", ]); const [pr, ...rest] = await pullPipelines(); expect(rest).toEqual([]); expect(pr).toMatchObject({ branch: "feature", trigger: { $type: "org.tangled.event.pullRequest", action: "opened", pull: ticketUri, sourceBranch: "feature", targetBranch: "main", sourceSha: sha(0x501), }, }); const rkey = pr?.rkey ?? ""; // The build work of the next alarm, which is due at once, starts the builds. await settled(rkey, [started, started]); expect(await runs(rkey)).toMatchObject([ { workflow: "always.yml", ...started }, { workflow: "lint.yaml", ...started }, ]); expect(sentBuilds.map((build) => build.url).toSorted()).toEqual([ pipelineUrls.always, pipelineUrls.lint, ]); for (const build of sentBuilds) { expect(build.body).toMatchObject({ commit: sha(0x501), branch: "feature", pull_request_base_branch: "main", ...commitInfo, }); } const res = await exports.default.fetch( `https://treadle.example.com/xrpc/org.tangled.ci.getPipeline?repo=${repoDid}&pipeline=${rkey}`, ); assertOutput("org.tangled.ci.getPipeline", await res.json()); }); it.each([ ["another sender", () => patch("bafy1", sha(0x502), { "x-tngl-editor": contributor }), []], ["another ticket author", () => (author = contributor), ["versions"]], ])("holds a pull request from %s until Approve", async (_, setup, extra) => { patch("bafy1", sha(0x502)); setup(); stream("bafy1"); await tick(); const [pr] = await pullPipelines(); expect(await runs(pr?.rkey ?? "")).toMatchObject([ { workflow: "always.yml", status: "blocked" }, { workflow: "lint.yaml", status: "blocked" }, ]); expect(sentBuilds).toEqual([]); expect(reads).toEqual([ "org.tangled.review.patch bafy1", "org.tangled.track.ticket 3tik", ...extra, ]); }); it("keeps the 10 newest held pull requests of each sender", async () => { const rkeys = Array.from({ length: 11 }, (_, i) => `bafy${i + 10}`); rkeys.forEach((rkey, i) => patch(rkey, sha(0x520 + i), { "x-tngl-editor": contributor })); patch("bafyother", sha(0x540), { "x-tngl-editor": "did:plc:oooooooooooooooooooooooo" }); stream(...rkeys, "bafyother"); await tick(); const kept = [...rkeys.slice(1).map((_, i) => sha(0x521 + i)), sha(0x540)]; const prs = await pullPipelines(); expect(prs).toHaveLength(kept.length); expect(prs).toMatchObject(kept.map((head) => ({ trigger: { sourceSha: head } }))); const orphans = await runInDurableObject( store, (_instance, state) => state.storage.sql .exec<{ n: number }>( "SELECT count(*) AS n FROM workflow_runs WHERE pipeline NOT IN (SELECT rkey FROM pipelines)", ) .one().n, ); expect(orphans).toBe(0); }); it("deletes a pull request that waits 14 days for Approve", async () => { patch("bafy1", sha(0x541), { "x-tngl-editor": contributor }); patch("bafy2", sha(0x542), { "x-tngl-editor": contributor }); stream("bafy1", "bafy2"); await tick(); const [, approved] = await pullPipelines(); const body = { repo: repoDid, pipeline: approved?.rkey }; expect((await call(approve, body, ownerToken(approve))).status).toBe(200); stream(); await agePipelines(13.9); await tick(); expect(await pullPipelines()).toHaveLength(2); await agePipelines(14); await tick(); expect(await pullPipelines()).toMatchObject([{ rkey: approved?.rkey }]); expect(await pipelineCount()).toBe(2); expect(await runs("3old")).toHaveLength(1); }); it("holds the retry of a pull request until Approve", async () => { patch("bafy1", sha(0x502), { "x-tngl-editor": contributor }); stream("bafy1"); await tick(); const [pr] = await pullPipelines(); const held = { repo: repoDid, pipeline: pr?.rkey }; expect((await call(cancel, held, ownerToken(cancel))).status).toBe(200); const again = await retryKey(await call(retry, held, ownerToken(retry))); await settled(again, [{ status: "blocked" }, { status: "blocked" }]); expect(sentBuilds).toEqual([]); const approved = { repo: repoDid, pipeline: again }; expect((await call(approve, approved, ownerToken(approve))).status).toBe(200); await settled(again, [started, started]); const third = await retryKey(await call(retry, approved, ownerToken(retry))); await settled(third, [started, started]); }); it("fails an invalid workflow of a held pull request without Approve", async () => { workflowFiles["broken.yml"] = invalidFiles["broken.yml"] ?? ""; try { patch("bafy1", sha(0x509), { "x-tngl-editor": contributor }); stream("bafy1"); await tick(); const [pr] = await pullPipelines(); expect(await runs(pr?.rkey ?? "")).toMatchObject([ { workflow: "always.yml", status: "blocked" }, brokenRun, { workflow: "lint.yaml", status: "blocked" }, ]); expect(sentBuilds).toEqual([]); } finally { delete workflowFiles["broken.yml"]; } }); it("builds a pull request on the Buildkite pipeline of the default-branch file", async () => { const trunk: Record = { ...workflowFiles, "lint.yaml": `${workflowFiles["lint.yaml"]} clean_checkout: true\n`, }; delete trunk["always.yml"]; const own = { ...workflowFiles }; workflowFiles["lint.yaml"] = "engine: buildkite\nwhen:\n - event: pull_request\n branch: main\ntreadle:\n buildkite:\n pipeline: deploy\n org: evil\n"; workflowFiles["always.yml"] = "engine: buildkite\ntreadle:\n buildkite:\n pipeline: deploy\n"; const headBlob = routes[blobRoute]; routes[blobRoute] = (url, request) => { if (url.searchParams.get("ref") !== "refs/heads/trunk") { return headBlob?.(url, request) ?? xrpcError("PathNotFound"); } const file = trunk[url.searchParams.get("path")?.replace(".tangled/workflows/", "") ?? ""]; return file === undefined ? xrpcError("PathNotFound") : new Response(file); }; try { patch("bafy1", sha(0x50a)); stream("bafy1"); await tick(); const [pr] = await pullPipelines(); await settled(pr?.rkey ?? "", [started, started]); const again = await retryKey( await call(retry, { repo: repoDid, pipeline: pr?.rkey }, ownerToken(retry)), ); await settled(again, [started, started]); expect(sentBuilds.map((build) => build.url).toSorted()).toEqual([ pipelineUrls.always, pipelineUrls.always, pipelineUrls.lint, pipelineUrls.lint, ]); for (const build of sentBuilds) { expect(build.body).not.toHaveProperty("clean_checkout"); } } finally { Object.assign(workflowFiles, own); } }); it("builds a fork's pull request on a pull branch", async () => { patch("bafy1", sha(0x503), { source: { repo: forkDid, branch: "feature" } }); stream("bafy1"); await tick(); const [pr] = await pullPipelines(); expect(pr).toMatchObject({ branch: "pull/3tik", trigger: { sourceRepo: forkDid } }); expect(pr?.trigger).not.toHaveProperty("sourceBranch"); await settled(pr?.rkey ?? "", [started, started]); expect(sentBuilds[0]?.body).toMatchObject({ branch: "pull/3tik" }); }); it("creates one pipeline for a replayed patch", async () => { patch("bafy1", sha(0x504)); stream("bafy1"); await tick(); reads = []; await tick(); expect(await pullPipelines()).toHaveLength(1); expect(reads).toEqual([]); }); it("marks later patches of a ticket as synchronize", async () => { patch("bafy1", sha(0x505)); commits.delete(sha(0x505)); patch("bafy2", sha(0x506)); stream("bafy1", "bafy2"); await tick(); const prs = await pullPipelines(); expect(prs.map((pr) => pr.trigger)).toMatchObject([{ action: "synchronize" }]); }); it("reads no pull request of a repo it does not serve", async () => { manifest = { ...manifest, spindle: "did:web:other.example.com" }; patch("bafy1", sha(0x507)); stream("bafy1"); await tick(); expect(reads).toEqual([]); expect(await pullPipelines()).toEqual([]); }); it.each([ ["a closed ticket", () => (ticket = { ...ticket, state: "closed" })], ["an issue", () => (ticket = { ...ticket, targetBranch: undefined })], ["a missing patch", () => delete patches["bafy1"]], ["no matching workflow", () => (ticket = { ...ticket, targetBranch: "dev" })], ...overLimits, ])("skips %s", async (_, setup) => { const prepare = (): void => { delete workflowFiles["always.yml"]; patch("bafy1", sha(0x508)); setup(); }; await withWorkflowFiles(prepare, async () => { stream("bafy1"); await tick(); expect(await pullPipelines()).toEqual([]); const state = await jetstreamState(); expect(state).not.toHaveProperty("failSeq"); }); }); it.each([ [ "the knot fails", () => (routes[`GET ${knotBase}/xrpc/com.atproto.repo.getRecord`] = failing()), ], [ "the ticket history is warming", () => (routes[versionsRoute] = () => xrpcError("ProjectionWarming")), ], ])("retries the patch when %s", async (_, setup) => { patch("bafy1", sha(0x509)); const saved = { ...routes }; setup(); stream("bafy1"); await tick(); expect(await pullPipelines()).toEqual([]); expect(await jetstreamState()).toMatchObject({ cursor: 100, failSeq: 100 }); Object.assign(routes, saved, { [`GET ${jetstreamUrl}`]: routes[`GET ${jetstreamUrl}`] ?? failing(), }); await tick(); expect(await pullPipelines()).toHaveLength(1); expect(streams[1]?.searchParams.get("cursor")).toBe("100"); }); it("sends a webhook for each new patch, once", async () => { await addHook(["pull_request:created", "pull_request:resubmitted"]); patch("bafy1", sha(0x50a)); stream("bafy1"); await tick(); await tick(); await drainHooks(); const source = { branch: "feature", repo: repoDid, sha: sha(0x50a) }; expect(hookPosts).toEqual([ { event: "pull_request:created", agent: "Tangled-Hook/pull_request", body: pullBody("created", { source, round_number: 1 }), }, ]); patch("bafy2", sha(0x50b), { source: undefined, "x-tngl-editor": contributor }); stream("bafy1", "bafy2"); await tick(); await drainHooks(); expect(hookPosts[1]).toEqual({ event: "pull_request:resubmitted", agent: "Tangled-Hook/pull_request", body: { ...pullBody("resubmitted", { source: { branch: "", sha: sha(0x50b) }, round_number: 2, }), sender: { did: contributor }, }, }); }); it("sends a webhook for a patch that matches no workflow", async () => { await addHook(["pull_request:created"]); ticket = { ...ticket, targetBranch: "dev" }; delete workflowFiles["always.yml"]; try { patch("bafy1", sha(0x50c)); stream("bafy1"); await tick(); await tick(); } finally { workflowFiles["always.yml"] = "engine: buildkite\n"; } await drainHooks(); expect(await pullPipelines()).toEqual([]); expect(hookPosts.map((post) => post.event)).toEqual(["pull_request:created"]); }); it("sends merged, closed and reopened webhooks when the ticket state changes", async () => { await addHook(["pull_request:merged", "pull_request:closed", "pull_request:reopened"]); routes[`GET ${jetstreamUrl}`] = () => jetstreamSocket([ commitMessage(200, repoDid, "3tik", "update", "org.tangled.track.ticket"), ]); const change = async (state: string): Promise => { ticket = { ...ticket, state, "x-tngl-editor": contributor }; await tick(); await tick(); }; await change("open"); await change("closed"); await change("open"); await change("completed"); await change("completed"); await drainHooks(); expect(hookPosts).toEqual([ { event: "pull_request:closed", agent: "Tangled-Hook/pull_request", body: { ...pullBody("closed", { state: "closed" }), sender: { did: contributor } }, }, { event: "pull_request:reopened", agent: "Tangled-Hook/pull_request", body: { ...pullBody("reopened", {}), sender: { did: contributor } }, }, { event: "pull_request:merged", agent: "Tangled-Hook/pull_request", body: { ...pullBody("merged", { state: "merged" }), sender: { did: contributor } }, }, ]); }); it("tracks the ticket state while no webhook subscribes", async () => { routes[`GET ${jetstreamUrl}`] = () => jetstreamSocket([ commitMessage(200, repoDid, "3tik", "update", "org.tangled.track.ticket"), ]); ticket = { ...ticket, state: "open" }; await tick(); ticket = { ...ticket, state: "closed" }; await tick(); await addHook(["pull_request:closed", "pull_request:reopened"]); await tick(); routes[versionsRoute] = () => new Response("down", { status: 502 }); ticket = { ...ticket, state: "open" }; await tick(); await drainHooks(); expect(hookPosts).toHaveLength(1); expect(hookPosts).toMatchObject([ { event: "pull_request:reopened", body: { action: "reopened", pull_request: { owner: { did: "" } } }, }, ]); }); it("reads no ticket of a repo it does not serve", async () => { manifest = { ...manifest, spindle: "did:web:other.example.com" }; routes[`GET ${jetstreamUrl}`] = () => jetstreamSocket([ commitMessage(200, repoDid, "3tik", "update", "org.tangled.track.ticket"), ]); await tick(); expect(reads).toEqual([]); }); it("approves held runs that have no build yet", async () => { await seedPipeline({ "ci.yml": "blocked", "always.yml": "blocked" }, pushTrigger, { "ci.yml": 7, }); routes[buildUrl(7)] = () => Response.json({ id: "uuid-7", number: 7, jobs: [{ id: "gate", type: "manual", state: "blocked", unblockable: true }], }); routes[`PUT ${unblockUrl("gate")}`] = () => Response.json({ id: "gate" }); await runInDurableObject(store, (_instance, state) => { state.storage.sql.exec( "UPDATE pipelines SET created_at = ?", new Date(Date.now() - 4 * 24 * 60 * 60_000).toISOString(), ); }); const res = await call(approve, { repo: repoDid, pipeline: "3old" }, ownerToken(approve)); expect(res.status).toBe(200); expect(unblocks()).toEqual([unblockUrl("gate")]); await settled("3old", [{ workflow: "always.yml", ...started }, { status: "blocked" }]); expect(sentBuilds.filter((b) => b.url === pipelineUrls.treadle)).toHaveLength(1); }); it("releases only the held runs", async () => { await seedPipeline({ "always.yml": "blocked", "ci.yml": "failed", "lint.yaml": "pending" }); await runInDurableObject(store, (_instance, state) => { state.storage.sql.exec( "UPDATE workflow_runs SET attempts = 2 WHERE status != 'failed'", ); }); const res = await call(approve, { repo: repoDid, pipeline: "3old" }, ownerToken(approve)); expect(res.status).toBe(200); await settled("3old", [ { workflow: "always.yml", ...started, attempts: 0 }, { workflow: "ci.yml", status: "failed" }, { workflow: "lint.yaml", ...started, attempts: 2 }, ]); }); it("counts the start window of approved runs from the approval", async () => { routes[treadleRoute] = failing(503); await seedPipeline({ "always.yml": "blocked" }); await runInDurableObject(store, (_instance, state) => { state.storage.sql.exec( "UPDATE pipelines SET created_at = ?", new Date(Date.now() - 4 * 24 * 60 * 60_000).toISOString(), ); }); const res = await call(approve, { repo: repoDid, pipeline: "3old" }, ownerToken(approve)); expect(res.status).toBe(200); await settled("3old", [{ workflow: "always.yml", status: "pending", attempts: 1 }]); }); }); it("opens no Jetstream stream when the owner declares no repos", async () => { let opened = false; routes[declarationsRoute(owner)] = () => Response.json({ records: [] }); routes[`GET ${jetstreamUrl}`] = () => { opened = true; return jetstreamSocket([]); }; await runInDurableObject(store, (_instance, state) => state.storage.setAlarm(Date.now() + 60_000), ); expect(await runDurableObjectAlarm(store)).toBe(true); expect(opened).toBe(false); expect(await alarmTime()).toBeNull(); });