Something went wrong. Try again.
Tangled Spindle built on Cloudflare Workers
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703import { DurableObject } from "cloudflare:workers";import { authenticate } from "./auth";import { authorize, claimToken, servedRepo } from "./notify";import { errorMessage, fetchWithin, invalidRequest, isDid, isObject, XrpcError } from "./xrpc";
const eventNames = new Set([ "push", "repository:renamed", "pull_request:created", "pull_request:resubmitted", "pull_request:merged", "pull_request:closed", "pull_request:reopened",]);const maxAttempts = 3;const maxResponseBytes = 10 * 1024;const kept = 50;const pushTtlMs = 7 * 24 * 60 * 60_000;const batch = 20;
export interface Hook { url: string; secret: string; active: boolean; events: string[];}
export interface Repository { name: string; full_name: string; description: string; fork: boolean; html_url: string; clone_url: string; ssh_url: string; website?: string; created_at: string; updated_at: string; owner: { did: string };}
// The parts of a pull_request payload that come from the ticket.export interface Pull { title: string; body: string; targetBranch: string; author: string; createdAt: string; sender: string; repository: Repository;}
export interface PatchHead { rkey: string; head: string; branch: string; repo: string;}
type Job = { id: string; hook: number; event: string; body: string; agent: string; attempts: number;};
interface Attempt { code: number | null; response: Uint8Array; retry: boolean;}
export const ticketEvents: Record<string, { state: string; event: string }> = { open: { state: "open", event: "pull_request:reopened" }, closed: { state: "closed", event: "pull_request:closed" }, completed: { state: "merged", event: "pull_request:merged" },};
export class Webhooks extends DurableObject<Env> { constructor(ctx: DurableObjectState, env: Env) { super(ctx, env); ctx.storage.sql.exec( `CREATE TABLE IF NOT EXISTS hooks ( id INTEGER PRIMARY KEY AUTOINCREMENT, url TEXT NOT NULL, secret TEXT NOT NULL, active INTEGER NOT NULL, events TEXT NOT NULL, created_at TEXT NOT NULL, updated_at TEXT NOT NULL ); CREATE TABLE IF NOT EXISTS jobs ( id TEXT PRIMARY KEY, hook INTEGER NOT NULL, event TEXT NOT NULL, body TEXT NOT NULL, agent TEXT NOT NULL, attempts INTEGER NOT NULL DEFAULT 0, due INTEGER NOT NULL ); CREATE TABLE IF NOT EXISTS deliveries ( seq INTEGER PRIMARY KEY AUTOINCREMENT, id TEXT NOT NULL, hook INTEGER NOT NULL, event TEXT NOT NULL, url TEXT NOT NULL, body TEXT NOT NULL, code INTEGER, response BLOB NOT NULL, success INTEGER NOT NULL, created_at TEXT NOT NULL ); CREATE TABLE IF NOT EXISTS pushes (key TEXT PRIMARY KEY, at INTEGER NOT NULL); CREATE TABLE IF NOT EXISTS tickets (rkey TEXT PRIMARY KEY, state TEXT NOT NULL); CREATE TABLE IF NOT EXISTS patches ( rkey TEXT PRIMARY KEY, ticket TEXT NOT NULL, head TEXT NOT NULL, branch TEXT NOT NULL, repo TEXT NOT NULL )`, ); }
create(hook: Hook): string { const now = new Date().toISOString(); return String( this.ctx.storage.sql .exec<{ id: number }>( `INSERT INTO hooks (url, secret, active, events, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?) RETURNING id`, hook.url, hook.secret, Number(hook.active), JSON.stringify(hook.events), now, now, ) .one().id, ); }
update(id: number, changes: { [K in keyof Hook]?: Hook[K] | undefined }): boolean { return ( this.ctx.storage.sql.exec( `UPDATE hooks SET url = coalesce(?, url), secret = coalesce(?, secret), active = coalesce(?, active), events = coalesce(?, events), updated_at = ? WHERE id = ?`, changes.url ?? null, changes.secret ?? null, changes.active === undefined ? null : Number(changes.active), changes.events === undefined ? null : JSON.stringify(changes.events), new Date().toISOString(), id, ).rowsWritten > 0 ); }
remove(id: number): boolean { const sql = this.ctx.storage.sql; sql.exec("DELETE FROM jobs WHERE hook = ?", id); sql.exec("DELETE FROM deliveries WHERE hook = ?", id); return sql.exec("DELETE FROM hooks WHERE id = ?", id).rowsWritten > 0; }
list(): object[] { return this.ctx.storage.sql .exec<{ id: number; url: string; active: number; events: string; createdAt: string; updatedAt: string; }>( "SELECT id, url, active, events, created_at AS createdAt, updated_at AS updatedAt FROM hooks ORDER BY id", ) .toArray() .map((row) => ({ id: String(row.id), url: row.url, active: row.active === 1, events: JSON.parse(row.events) as unknown, createdAt: row.createdAt, updatedAt: row.updatedAt, })); }
deliveries(ids: number[], limit: number): object[] { return this.ctx.storage.sql .exec<{ id: string; hook: number; event: string; url: string; body: string; code: number | null; response: ArrayBuffer; success: number; createdAt: string; }>( `SELECT id, hook, event, url, body, code, response, success, created_at AS createdAt FROM (SELECT *, row_number() OVER (PARTITION BY hook ORDER BY seq DESC) AS n FROM deliveries WHERE hook IN (SELECT value FROM json_each(?))) WHERE n <= ? ORDER BY hook, seq DESC`, JSON.stringify(ids), limit, ) .toArray() .map((row) => { const delivery: Record<string, unknown> = { id: row.id, webhook: String(row.hook), event: row.event, url: row.url, success: row.success === 1, createdAt: row.createdAt, }; if (row.body !== "") { delivery["requestBody"] = bytes(new TextEncoder().encode(row.body)); } if (row.code !== null) { delivery["responseCode"] = row.code; } if (row.response.byteLength > 0) { delivery["responseBody"] = bytes(new Uint8Array(row.response)); } return delivery; }); }
async retry(id: number, delivery: string): Promise<string | undefined> { const sql = this.ctx.storage.sql; if (sql.exec("SELECT 1 FROM hooks WHERE id = ?", id).toArray().length === 0) { return "WebhookNotFound"; } const [stored] = sql .exec<{ event: string; body: string }>( "SELECT event, body FROM deliveries WHERE hook = ? AND id = ?", id, delivery, ) .toArray(); if (stored === undefined) { return "DeliveryNotFound"; } this.#enqueue(id, stored.event, stored.body, "Tangled-Hook/retry"); await this.ctx.storage.setAlarm(Date.now()); return undefined; }
subscribed(wanted: string[]): boolean { return this.#subscribers(wanted).length > 0; }
async firePush(key: string, payload: object, agent: string): Promise<void> { const sql = this.ctx.storage.sql; const now = Date.now(); sql.exec("DELETE FROM pushes WHERE at < ?", now - pushTtlMs); if ( sql.exec("INSERT INTO pushes VALUES (?, ?) ON CONFLICT DO NOTHING", key, now) .rowsWritten > 0 ) { await this.#fire("push", payload, agent); } }
async patch(ticket: string, patch: PatchHead, pull?: Pull): Promise<void> { const sql = this.ctx.storage.sql; sql.exec("INSERT INTO tickets VALUES (?, 'open') ON CONFLICT DO NOTHING", ticket); const added = sql.exec( "INSERT INTO patches VALUES (?, ?, ?, ?, ?) ON CONFLICT DO NOTHING", patch.rkey, ticket, patch.head, patch.branch, patch.repo, ).rowsWritten; if (added > 0 && pull !== undefined) { const rounds = sql .exec<{ n: number }>("SELECT count(*) AS n FROM patches WHERE ticket = ?", ticket) .one().n; const event = rounds === 1 ? "pull_request:created" : "pull_request:resubmitted"; await this.#firePull(event, ticket, "open", pull); } }
async ticket(ticket: string, state: string, pull?: Pull): Promise<void> { const previous = this.ctx.storage.sql .exec<{ state: string }>("SELECT state FROM tickets WHERE rkey = ?", ticket) .toArray()[0]?.state; this.ctx.storage.sql.exec( "INSERT INTO tickets VALUES (?, ?) ON CONFLICT DO UPDATE SET state = excluded.state", ticket, state, ); const change = ticketEvents[state]; if ( previous === undefined || previous === state || pull === undefined || change === undefined ) { return; } await this.#firePull(change.event, ticket, change.state, pull); }
async #firePull(event: string, ticket: string, state: string, pull: Pull): Promise<void> { const sql = this.ctx.storage.sql; const rounds = sql .exec<{ n: number }>("SELECT count(*) AS n FROM patches WHERE ticket = ?", ticket) .one().n; const head = sql .exec<{ head: string; branch: string; repo: string; }>( "SELECT head, branch, repo FROM patches WHERE ticket = ? ORDER BY rowid DESC LIMIT 1", ticket, ) .toArray()[0]; const payload = { action: event.slice("pull_request:".length), pull_request: { number: 0, title: pull.title, body: pull.body, state, target_branch: pull.targetBranch, ...(head === undefined ? {} : { source: { branch: head.branch, ...(head.repo === "" ? {} : { repo: head.repo }), sha: head.head, }, }), round_number: rounds, owner: { did: pull.author }, created_at: pull.createdAt, }, repository: pull.repository, sender: { did: pull.sender }, }; await this.#fire(event, payload, "Tangled-Hook/pull_request"); }
#subscribers(wanted: string[]): number[] { return this.ctx.storage.sql .exec<{ id: number }>( `SELECT DISTINCT h.id FROM hooks h, json_each(h.events) e WHERE h.active = 1 AND e.value IN (SELECT value FROM json_each(?))`, JSON.stringify(wanted), ) .toArray() .map(({ id }) => id); }
async #fire(event: string, payload: object, agent: string): Promise<void> { const hooks = this.#subscribers([event]); if (hooks.length === 0) { return; } const body = JSON.stringify(payload); for (const hook of hooks) { this.#enqueue(hook, event, body, agent); } await this.ctx.storage.setAlarm(Date.now()); }
#enqueue(hook: number, event: string, body: string, agent: string): void { this.ctx.storage.sql.exec( "INSERT INTO jobs (id, hook, event, body, agent, due) VALUES (?, ?, ?, ?, ?, ?)", crypto.randomUUID(), hook, event, body, agent, Date.now(), ); }
override async alarm(): Promise<void> { const sql = this.ctx.storage.sql; const jobs = sql .exec<Job>( "SELECT id, hook, event, body, agent, attempts FROM jobs WHERE due <= ? ORDER BY due LIMIT ?", Date.now(), batch, ) .toArray(); await Promise.all(jobs.map((job) => this.#send(job))); const next = sql.exec<{ due: number | null }>("SELECT min(due) AS due FROM jobs").one().due; if (next !== null) { await this.ctx.storage.setAlarm(Math.max(next, Date.now())); } }
async #send(job: Job): Promise<void> { const sql = this.ctx.storage.sql; const [hook] = sql .exec<{ url: string; secret: string }>( "SELECT url, secret FROM hooks WHERE id = ?", job.hook, ) .toArray(); if (hook === undefined) { sql.exec("DELETE FROM jobs WHERE id = ?", job.id); return; } // Counted first, so a crash during the request still uses up an attempt. const attempts = job.attempts + 1; sql.exec("UPDATE jobs SET attempts = ? WHERE id = ?", attempts, job.id); const result = job.attempts >= maxAttempts ? { code: null, response: encode("delivery interrupted"), retry: false } : await post(hook, job); if (result.retry && attempts < maxAttempts) { sql.exec( "UPDATE jobs SET due = ? WHERE id = ?", Date.now() + 1000 * 2 ** (attempts - 1), job.id, ); return; } this.ctx.storage.transactionSync(() => { sql.exec( `INSERT INTO deliveries (id, hook, event, url, body, code, response, success, created_at) SELECT ?, ?, ?, ?, ?, ?, ?, ?, ? WHERE EXISTS (SELECT 1 FROM hooks WHERE id = ?)`, job.id, job.hook, job.event, hook.url, job.body, result.code, result.response, Number(result.code !== null && result.code >= 200 && result.code < 300), new Date().toISOString(), job.hook, ); sql.exec("DELETE FROM jobs WHERE id = ?", job.id); sql.exec( `DELETE FROM deliveries WHERE hook = ? AND seq NOT IN (SELECT seq FROM deliveries WHERE hook = ? ORDER BY seq DESC LIMIT ?)`, job.hook, job.hook, kept, ); }); }}
function bytes(data: Uint8Array): { $bytes: string } { return { $bytes: data.toBase64({ omitPadding: true }) };}
function encode(text: string): Uint8Array { return new TextEncoder().encode(text);}
async function sign(secret: string, body: string): Promise<string> { const key = await crypto.subtle.importKey( "raw", new TextEncoder().encode(secret), { name: "HMAC", hash: "SHA-256" }, false, ["sign"], ); const mac = await crypto.subtle.sign("HMAC", key, new TextEncoder().encode(body)); return `sha256=${new Uint8Array(mac).toHex()}`;}
async function readHead(response: Response): Promise<Uint8Array> { const reader = response.body?.getReader(); if (reader === undefined) { return new Uint8Array(); } const head = new Uint8Array(maxResponseBytes); let size = 0; try { while (size < maxResponseBytes) { // oxlint-disable-next-line no-await-in-loop const chunk = await reader.read(); if (chunk.done || !(chunk.value instanceof Uint8Array)) { break; } const part = chunk.value.subarray(0, maxResponseBytes - size); head.set(part, size); size += part.length; } } catch { // The receiver already answered, so a broken body does not fail the delivery. } await reader.cancel().catch(() => {}); return head.slice(0, size);}
async function post(hook: { url: string; secret: string }, job: Job): Promise<Attempt> { let repo = ""; try { const parsed: unknown = JSON.parse(job.body); const named = isObject(parsed) && isObject(parsed["repository"]) ? parsed["repository"] : {}; repo = typeof named["full_name"] === "string" ? named["full_name"] : ""; } catch { // A stored body is always JSON. } const headers: Record<string, string> = { "Content-Type": "application/json", "User-Agent": job.agent, "X-Tangled-Event": job.event, "X-Tangled-Hook-ID": String(job.hook), "X-Tangled-Delivery": job.id, "X-Tangled-Repo": repo, }; if (hook.secret !== "") { headers["X-Tangled-Signature-256"] = await sign(hook.secret, job.body); } try { return await fetchWithin( hook.url, { method: "POST", headers, body: job.body, redirect: "manual" }, 10_000, async (response) => { if (response.status >= 500) { await response.body?.cancel(); return { code: response.status, response: encode(`server error: ${response.status}`), retry: true, }; } return { code: response.status, response: await readHead(response), retry: false }; }, ); } catch (error) { return { code: null, response: encode(errorMessage(error)), retry: true }; }}
function isPublicIp(host: string): boolean { const v4 = /^(\d+)\.(\d+)\.(\d+)\.(\d+)$/.exec(host); if (v4 !== null) { const [a = 0, b = 0, c = 0] = v4.slice(1).map(Number); return !( a === 0 || a === 10 || a === 127 || a >= 224 || (a === 100 && b >= 64 && b < 128) || (a === 169 && b === 254) || (a === 172 && b >= 16 && b < 32) || (a === 192 && b === 168) || (a === 192 && b === 88 && c === 99) || (a === 192 && b === 0 && (c === 0 || c === 2)) || (a === 198 && (b === 18 || b === 19 || (b === 51 && c === 100))) || (a === 203 && b === 0 && c === 113) ); } if (host.startsWith("[")) { // Only global unicast, without the documentation, Teredo and 6to4 ranges. const [a = 0, b = 0] = host .slice(1) .split(":") .map((group) => Number.parseInt(group || "0", 16)); return ( a >= 0x2000 && a <= 0x3fff && a !== 0x2002 && !(a === 0x2001 && (b === 0 || b === 0xdb8)) ); } const name = host.replace(/\.+$/, ""); return name !== "localhost" && !name.endsWith(".localhost");}
function hookUrl(value: unknown): string { const url = typeof value === "string" ? value.trim() : ""; const parsed = URL.parse(url); if ( parsed === null || (parsed.protocol !== "http:" && parsed.protocol !== "https:") || parsed.hostname === "" || !isPublicIp(parsed.hostname) ) { throw invalidRequest("url must be an http or https URL with a public host"); } return url;}
function hookEvents(value: unknown): string[] { if (!Array.isArray(value) || !value.every((item) => typeof item === "string")) { throw invalidRequest("events must be a list of event names"); } const names = [...new Set(value.map((item) => item.trim()))]; const unknown = names.find((name) => !eventNames.has(name)); if (unknown !== undefined) { throw new XrpcError(400, "InvalidEvent", `unknown webhook event "${unknown}"`); } if (names.length === 0) { throw new XrpcError(400, "NoEventsSelected", "at least one event must be specified"); } return names;}
function hookId(value: unknown): number { if (typeof value !== "string" || !/^\d{1,15}$/.test(value)) { throw invalidRequest("invalid webhook id"); } return Number(value);}
function optionalString(value: unknown, name: string): string | undefined { if (value !== undefined && typeof value !== "string") { throw invalidRequest(`${name} must be a string`); } return value;}
function optionalBoolean(value: unknown, name: string): boolean | undefined { if (value !== undefined && typeof value !== "boolean") { throw invalidRequest(`${name} must be a boolean`); } return value;}
export async function webhookXrpc( method: string, request: Request, env: Env, exports: Cloudflare.Exports,): Promise<Response> { const claims = await authenticate(request, env, exports, method); const params = new URL(request.url).searchParams; const query = method.startsWith("org.tangled.webhook.list"); const input: unknown = query ? Object.fromEntries(params) : await request.json().catch(() => null); if (!isObject(input) || !isDid(input["repo"])) { throw invalidRequest("repo must be a DID"); } const repo = input["repo"]; authorize({ claims, repo, env }); if (!query) { await claimToken({ claims, store: exports.Store.getByName("main") }); } await servedRepo({ env, repo }); const hooks = exports.Webhooks.getByName(repo); switch (method) { case "org.tangled.webhook.listWebhooks": { return Response.json({ webhooks: await hooks.list() }); } case "org.tangled.webhook.listWebhookDeliveries": { const ids = params.getAll("webhooks"); if (ids.length === 0 || ids.length > 50) { throw invalidRequest("webhooks must name 1 to 50 webhook ids"); } const limit = Number(params.get("limit") ?? 50); if (!Number.isInteger(limit) || limit < 1 || limit > 100) { throw invalidRequest("limit must be an integer from 1 to 100"); } return Response.json({ deliveries: await hooks.deliveries(ids.map(hookId), limit) }); } case "org.tangled.webhook.createWebhook": { const hook = { url: hookUrl(input["url"]), events: hookEvents(input["events"]), secret: optionalString(input["secret"], "secret")?.trim() ?? "", active: optionalBoolean(input["active"], "active") ?? true, }; return Response.json({ id: await hooks.create(hook) }); } case "org.tangled.webhook.updateWebhook": { const id = hookId(input["webhook"]); const url = optionalString(input["url"], "url"); const events = input["events"]; const changes = { ...(url === undefined || url.trim() === "" ? {} : { url: hookUrl(url) }), ...(events === undefined || (Array.isArray(events) && events.length === 0) ? {} : { events: hookEvents(events) }), secret: optionalString(input["secret"], "secret")?.trim(), active: optionalBoolean(input["active"], "active"), }; if (!(await hooks.update(id, changes))) { throw new XrpcError(404, "WebhookNotFound", "webhook not found"); } return new Response(null); } case "org.tangled.webhook.deleteWebhook": { if (!(await hooks.remove(hookId(input["webhook"])))) { throw new XrpcError(404, "WebhookNotFound", "webhook not found"); } return new Response(null); } case "org.tangled.webhook.retryWebhookDelivery": { const id = hookId(input["webhook"]); const delivery = input["delivery"]; if (typeof delivery !== "string") { throw invalidRequest("delivery must be a delivery id"); } const error = await hooks.retry(id, delivery); if (error !== undefined) { throw new XrpcError( 404, error, `${error === "WebhookNotFound" ? "webhook" : "delivery"} not found`, ); } return new Response(null); } default: throw new XrpcError(501, "MethodNotImplemented", `${method} is not implemented`); }}