Something went wrong. Try again.
Tangled Spindle built on Cloudflare Workers
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670import { DurableObject } from "cloudflare:workers";import { authorization, buildState, isTerminal, pipelineUrl, readBuild, time } from "./buildkite";import type { RunStatus } from "./buildkite";import { decode, decodeFirst, encode } from "@atcute/cbor";import { findPipeline, findRun } from "./notify";import { repoParam } from "./pipelines";import { buffered, invalidRequest, isObject, rateLimit, string, upstream } from "./xrpc";
export const logCap = 2 * 1024 * 1024;const pollMs = 5_000;const checkMs = 60_000;const storeRetryMs = 10_000;const minRemaining = 15;export const capMarker = "[treadle: log longer than 2 MiB]";// oxlint-disable-next-line no-control-regexconst timestamps = /\u001b_bk;t=(\d+)\u0007/g;// oxlint-disable-next-line no-control-regexexport const csi = /\u001b\[[0-?]*[ -/]*[@-~]/g;const decoder = new TextDecoder();
export function cleanLine(line: string): { text: string; time: string | undefined } { const ms = Number(line.matchAll(timestamps).next().value?.[1]); const date = new Date(ms); const text = line.replaceAll(timestamps, "").replace(/\r$/, ""); return { text: text.slice(text.lastIndexOf("\r") + 1), time: Number.isNaN(date.getTime()) ? undefined : date.toISOString(), };}
// Splits at newline bytes and keeps a partial last line unless the log is final.export function logLines( bytes: Uint8Array, final: boolean,): { lines: ReturnType<typeof cleanLine>[]; used: number } { const lines = []; let used = 0; for (let end = bytes.indexOf(10); end !== -1; end = bytes.indexOf(10, used)) { lines.push(cleanLine(decoder.decode(bytes.subarray(used, end)))); used = end + 1; } if (final && used < bytes.length) { lines.push(cleanLine(decoder.decode(bytes.subarray(used)))); used = bytes.length; } return { lines, used };}
export interface StepInfo { step: number; name: string; command: string | undefined; kind?: "system" | "user";}
function encodeFrame(header: object, body: object): Uint8Array { const head = encode(header); const rest = encode(body); const bytes = new Uint8Array(head.length + rest.length); bytes.set(head); bytes.set(rest, head.length); return bytes;}
export function controlFrame( workflow: string, { step, name, command, kind = "user" }: StepInfo, status: "start" | "end", at: string, outcome?: string,): Uint8Array { return encodeFrame( { op: 1, t: "#control" }, { time: at, workflow, step, content: name, command, status, kind, outcome }, );}
export function dataFrame(workflow: string, step: number, content: string, at: string): Uint8Array { return encodeFrame( { op: 1, t: "#data" }, { time: at, workflow, step, content, stream: "stdout" }, );}
export function frameLine( frame: Uint8Array,): { step: number; text: string; time: string } | undefined { const first: [unknown, Uint8Array] = decodeFirst(frame); const [header, rest] = first; const body: unknown = decode(rest); if (!isObject(header) || !isObject(body) || typeof body["step"] !== "number") { return undefined; } const { step, content, status, outcome } = body; const text = typeof content === "string" ? content : ""; const at = typeof body["time"] === "string" ? body["time"] : ""; if (header["t"] === "#data") { return { step, time: at, text: text.replace(/\n$/, "") }; } const end = status === "end" ? ` (${typeof outcome === "string" ? outcome : "done"})` : ""; return { step, time: at, text: `--- step ${step}: ${text}${end}` };}
interface Job { id: string; name: string; command: string | undefined; startedAt: string | undefined; finishedAt: string | undefined; outcome: string | undefined;}
const outcomes = new Map([ ["passed", "success"], ["failed", "failed"], ["timed_out", "failed"], ["canceled", "failed"],]);
function parseJob(value: unknown): Job | undefined { if (!isObject(value) || value["type"] !== "script") { return undefined; } const id = string(value["id"]); if (id === undefined) { return undefined; } const command = string(value["command"]); return { id, name: string(value["name"]) ?? command ?? id, command, startedAt: time(value["started_at"]), finishedAt: time(value["finished_at"]), outcome: value["retried"] === true ? undefined : outcomes.get(String(value["state"])), };}
function scriptJobs(build: Record<string, unknown>): Job[] { const jobs: unknown[] = Array.isArray(build["jobs"]) ? build["jobs"] : []; return jobs.map(parseJob).filter((job) => job !== undefined);}
type Target = { repo: string; pipeline: string; workflow: string };type Run = { org: string; slug: string; number: number | null; status: string };type Cursor = { step: number; name: string; command: string | undefined; offset: number; started: string | undefined; end: { time: string; outcome: string | undefined } | undefined; capped: boolean;};type Log = { start: number; bytes: Uint8Array };
class Rejected extends Error {}
function httpError(url: string, status: number): Error { const message = `${url}: HTTP ${status}`; return status >= 400 && status < 500 && status !== 429 ? new Rejected(message) : new Error(message);}
export async function subscribePipelineLogs( request: Request, env: Env, ctx: ExecutionContext,): Promise<Response> { await rateLimit(request, env.LOG_LIMIT); const params = new URL(request.url).searchParams; const repo = repoParam(params.get("repo")); const workflows = params.getAll("workflows"); const [workflow] = workflows; if (workflows.length !== 1 || workflow === undefined) { throw invalidRequest("workflows must name exactly one workflow"); } if (request.headers.get("Upgrade")?.toLowerCase() !== "websocket") { throw invalidRequest("expected a WebSocket upgrade"); } const pipeline = await findPipeline( ctx.exports.Store.getByName("main"), repo, params.get("pipeline"), ); const run = findRun(pipeline, workflow); if (run.runner !== null) { // A run can end before it reaches a Runner, for example when it is cancelled in the queue. const done = isTerminal(run.status) ? "&done" : ""; return await ctx.exports.Runner.getByName(run.runner).fetch( `https://runner/?pipeline=${encodeURIComponent(pipeline.rkey)}${done}`, { headers: { Upgrade: "websocket" } }, ); } const url = new URL("https://logs/"); url.search = new URLSearchParams({ repo, pipeline: pipeline.rkey, workflow }).toString(); return await ctx.exports.Logs.getByName(`${pipeline.rkey}/${workflow}`).fetch(url, { headers: request.headers, });}
// Streams the job logs of one workflow run. A socket waits for its replay, then gets// broadcasts while it is ready, and is done after it is closed.export class Logs extends DurableObject<Env> { #queue: Promise<unknown> = Promise.resolve();
#serial<T>(task: () => Promise<T>): Promise<T> { const result = this.#queue.then(task); this.#queue = result.catch(() => null); return result; }
override async fetch(request: Request): Promise<Response> { const params = new URL(request.url).searchParams; const target = { repo: params.get("repo") ?? "", pipeline: params.get("pipeline") ?? "", workflow: params.get("workflow") ?? "", }; const { 0: client, 1: server } = new WebSocketPair(); this.ctx.acceptWebSocket(server); server.serializeAttachment("waiting"); this.ctx.storage.kv.put("target", target); await this.#wake(Date.now()); return new Response(null, { status: 101, webSocket: client }); }
async update(number: number, status: RunStatus | undefined, job: unknown): Promise<void> { await this.#serial(async () => { const run = this.ctx.storage.kv.get<Run>("run"); if (this.ctx.getWebSockets().length === 0 || run?.number !== number) { return; } const parsed = parseJob(job); if (parsed !== undefined) { await this.#sync(run, [parsed]); } // Only a successful check stores a terminal status, so a failed one is retried. if (status !== undefined && isTerminal(status)) { try { await this.#check(run); } catch (error) { this.ctx.storage.kv.delete("checkedAt"); throw error; } } else if (status !== undefined) { this.ctx.storage.kv.put("run", { ...run, status }); await this.#wake(Date.now() + pollMs); } }); }
override async alarm(): Promise<void> { const next = await this.#serial(() => this.#tick()).catch((error: unknown) => { console.warn("log stream check failed", { error: String(error) }); return Date.now() + pollMs; }); if (next !== undefined) { await this.#wake(Math.max(next, this.ctx.storage.kv.get<number>("limitedUntil") ?? 0)); } }
// Sends one replay to every waiting socket. Buildkite rejections close them with an error. async #join(current: Run): Promise<Run> { let run = current; const waiting = this.ctx .getWebSockets() .filter((ws) => ws.deserializeAttachment() === "waiting"); if (waiting.length === 0) { return run; } let jobs: [Cursor, Log | undefined][]; try { [run, jobs] = await this.#replay(run); } catch (error) { if (!(error instanceof Rejected)) { throw error; } console.warn("log replay rejected", { error: String(error) }); for (const ws of waiting) { ws.serializeAttachment("done"); ws.send(encodeFrame({ op: -1 }, { error: "UpstreamUnavailable" })); ws.close(1011, "log replay failed"); } return run; } // Frames are built one job at a time, because a capped log has many lines. for (const [cursor, log] of jobs) { const frames = this.#replayJob(cursor, log); for (const ws of waiting) { for (const frame of frames) { ws.send(frame); } } } for (const ws of waiting) { if (isTerminal(run.status)) { this.#close(ws); } else { ws.serializeAttachment("ready"); } } return run; }
// Returns the next alarm time, or undefined when the stream is done. async #tick(): Promise<number | undefined> { const kv = this.ctx.storage.kv; const now = Date.now(); if (this.ctx.getWebSockets().length === 0) { return undefined; } const limited = kv.get<number>("limitedUntil") ?? 0; if (limited > now) { return limited; } let run = kv.get<Run>("run"); if (typeof run?.number !== "number") { run = await this.#refresh(); if (run !== undefined && isTerminal(run.status) && run.number === null) { for (const ws of this.ctx.getWebSockets()) { this.#close(ws); } await this.#finish(); return undefined; } if (typeof run?.number !== "number") { return now + storeRetryMs; } } // Replays run here, in order with polls and within the rate limit. try { run = await this.#join(run); } catch (error) { console.warn("log replay failed", { error: String(error) }); } if (isTerminal(run.status)) { if (this.ctx.getWebSockets().some((ws) => ws.deserializeAttachment() === "waiting")) { return now + pollMs; } await this.#finish(); return undefined; } if (now - (kv.get<number>("checkedAt") ?? 0) >= checkMs) { run = await this.#check(run); if (run === undefined) { return undefined; } } if (run.status !== "blocked" && now - (kv.get<number>("polledAt") ?? 0) >= pollMs) { kv.put("polledAt", now); await this.#poll(run); } return run.status === "blocked" ? (kv.get<number>("checkedAt") ?? now) + checkMs : Date.now() + pollMs; }
// Stores the run that the Store has, or returns undefined so that the alarm asks again. async #refresh(): Promise<Run | undefined> { const target = this.ctx.storage.kv.get<Target>("target"); if (target === undefined) { return undefined; } const store = this.ctx.exports.Store.getByName("main"); const pipeline = await store.getPipeline(target.repo, target.pipeline); const row = pipeline?.runs.find((run) => run.workflow === target.workflow); if (row === undefined) { return undefined; } const run = { org: row.bk_org, slug: row.bk_pipeline, number: row.build_number, status: row.status, }; this.ctx.storage.kv.put("run", run); return run; }
// Returns the updated run, or undefined when the stream is done. async #check(run: Run): Promise<Run | undefined> { this.ctx.storage.kv.put("checkedAt", Date.now()); const build = await this.#getBuild(run); await this.#sync(run, scriptJobs(build)); // Jobs of a canceling build still run and write logs. const status = build["state"] === "canceling" ? "running" : buildState(build)?.status; const updated = { ...run, status: status ?? run.status }; this.ctx.storage.kv.put("run", updated); if (isTerminal(updated.status)) { await this.#finish(); return undefined; } return updated; }
async #sync(run: Run, jobs: Job[]): Promise<void> { const kv = this.ctx.storage.kv; for (const job of jobs) { const key = `job:${job.id}`; const cursor = kv.get<Cursor>(key) ?? this.#cursor(job); if (cursor.started === undefined && job.startedAt !== undefined) { cursor.started = job.startedAt; this.#broadcast(this.#control(cursor, "start", cursor.started)); } kv.put(key, cursor); if ( cursor.started !== undefined && cursor.end === undefined && job.finishedAt !== undefined ) { // oxlint-disable-next-line no-await-in-loop await this.#pull(run, job.id, cursor, true); cursor.end = { time: job.finishedAt, outcome: job.outcome }; this.#broadcast(this.#control(cursor, "end", job.finishedAt, job.outcome)); kv.put(key, cursor); } } }
#cursor(job: Job): Cursor { const step = this.ctx.storage.kv.get<number>("steps") ?? 0; this.ctx.storage.kv.put("steps", step + 1); return { step, name: job.name, command: job.command, offset: 0, started: undefined, end: undefined, capped: false, }; }
async #poll(run: Run): Promise<void> { const kv = this.ctx.storage.kv; for (const [key, cursor] of Array.from(kv.list<Cursor>({ prefix: "job:" }))) { if (cursor.started === undefined || cursor.end !== undefined) { continue; } if ((kv.get<number>("limitedUntil") ?? 0) > Date.now()) { return; } // oxlint-disable-next-line no-await-in-loop await this.#pull(run, key.slice("job:".length), cursor, false); kv.put(key, cursor); } }
async #pull(run: Run, id: string, cursor: Cursor, final: boolean): Promise<void> { if (cursor.capped) { return; } const log = cursor.offset < logCap ? await this.#log(run, id, logCap - cursor.offset) : null; if (log === undefined) { return; } if (log === null || log.start > cursor.offset) { this.#broadcast(this.#data(cursor.step, `${capMarker}\n`, new Date().toISOString())); cursor.capped = true; return; } const { lines, used } = logLines(log.bytes.subarray(cursor.offset - log.start), final); for (const frame of this.#lines(cursor.step, lines)) { this.#broadcast(frame); } cursor.offset += used; }
// Fetches the logs of every started job, so that a failure comes before any frame. async #replay(current: Run): Promise<[Run, [Cursor, Log | undefined][]]> { let run = current; let cursors: [string, Cursor][]; const build = isTerminal(run.status) ? await this.#getBuild(run) : undefined; if (build === undefined || build["state"] === "canceling") { if (build !== undefined) { run = { ...run, status: "running" }; this.ctx.storage.kv.put("run", run); } cursors = Array.from( this.ctx.storage.kv.list<Cursor>({ prefix: "job:" }), ([key, cursor]): [string, Cursor] => [key.slice("job:".length), cursor], ).toSorted((a, b) => a[1].step - b[1].step); } else { cursors = scriptJobs(build).map((job, step): [string, Cursor] => [ job.id, { step, name: job.name, command: job.command, offset: Infinity, started: job.startedAt, end: job.finishedAt === undefined ? undefined : { time: job.finishedAt, outcome: job.outcome }, capped: false, }, ]); } const jobs: [Cursor, Log | undefined][] = []; for (const [id, cursor] of cursors) { if (cursor.started !== undefined) { // oxlint-disable-next-line no-await-in-loop const log = cursor.offset > 0 ? await this.#log(run, id, logCap) : undefined; jobs.push([cursor, log]); } } return [run, jobs]; }
#replayJob(cursor: Cursor, log: Log | undefined): Uint8Array[] { return [ this.#control(cursor, "start", cursor.started ?? ""), ...(log === undefined ? [] : this.#replayLog(cursor.step, log, cursor.offset)), ...(cursor.end === undefined ? [] : [this.#control(cursor, "end", cursor.end.time, cursor.end.outcome)]), ]; }
// Returns the log frames before upTo. A log that starts after byte 0 begins mid-line. #replayLog(step: number, log: Log, upTo: number): Uint8Array[] { let bytes = log.bytes.subarray(0, Math.max(0, upTo - log.start)); if (log.start <= 0) { return this.#lines(step, logLines(bytes, true).lines); } const newline = bytes.indexOf(10); bytes = bytes.subarray(newline === -1 ? bytes.length : newline + 1); return [ this.#data(step, `${capMarker}\n`, new Date().toISOString()), ...this.#lines(step, logLines(bytes, true).lines), ]; }
// Viewers that still wait for a replay keep the target and get one from the next alarm. async #finish(): Promise<void> { const target = this.ctx.storage.kv.get<Target>("target"); const limited = this.ctx.storage.kv.get<number>("limitedUntil"); let waiting = false; for (const ws of this.ctx.getWebSockets()) { const state: unknown = ws.deserializeAttachment(); if (state === "ready") { this.#close(ws); } waiting ||= state === "waiting"; } await this.ctx.storage.deleteAll(); if (waiting && target !== undefined) { this.ctx.storage.kv.put("target", target); if (limited !== undefined) { this.ctx.storage.kv.put("limitedUntil", limited); } await this.#wake(Date.now()); } }
#close(ws: WebSocket): void { if (ws.deserializeAttachment() !== "done") { ws.serializeAttachment("done"); ws.close(1000, "log stream complete"); } }
async #wake(at: number): Promise<void> { const current = await this.ctx.storage.getAlarm(); if (current === null || current > at) { await this.ctx.storage.setAlarm(at); } }
async #getBuild(run: Run): Promise<Record<string, unknown>> { const url = `${pipelineUrl(run.org, run.slug)}/builds/${run.number}`; const response = await this.#get(url, {}); const body: unknown = response.ok ? await response.json() : undefined; if (!isObject(body)) { throw httpError(url, response.status); } return body; }
// Returns the last n bytes of a job log, or undefined for an empty log. async #log(run: Run, id: string, n: number): Promise<Log | undefined> { const url = `${pipelineUrl(run.org, run.slug)}/builds/${run.number}/jobs/${encodeURIComponent(id)}/log`; // A full response must fit in the requested suffix, or it may be a whole huge log. const fits = (r: Response): boolean => Number(r.headers.get("Content-Length") ?? Infinity) <= n; const response = await this.#get( url, { Accept: "text/plain", Range: `bytes=-${n}` }, async (answer) => { if (answer.status === 206 || fits(answer)) { return await buffered(answer, n); } await answer.body?.cancel(); return new Response(null, answer); }, ); if (response.status === 416) { return undefined; } if (!response.ok) { throw httpError(url, response.status); } const range = /^bytes (\d+)-\d+\/\d+$/.exec(response.headers.get("Content-Range") ?? ""); let start = range?.[1]; if (response.status !== 206) { start = fits(response) ? "0" : undefined; } if (start === undefined) { throw new Rejected(`${url}: HTTP ${response.status} without a usable range`); } return { start: Number(start), bytes: new Uint8Array(await response.arrayBuffer()) }; }
// Buildkite allows 50 requests a minute per user, and build starts share them. async #get(url: string, headers: Record<string, string>, read = readBuild): Promise<Response> { const response = await upstream( url, 503, { headers: { Authorization: authorization(this.env.BUILDKITE_TOKEN), ...headers } }, { read }, ); const remaining = response.headers.get("RateLimit-User-Remaining"); if (response.status === 429 || (remaining !== null && Number(remaining) < minRemaining)) { const reset = Number(response.headers.get("RateLimit-User-Reset")); this.ctx.storage.kv.put( "limitedUntil", Date.now() + (Number.isFinite(reset) && reset > 0 ? reset : 60) * 1000, ); } return response; }
#lines(step: number, lines: ReturnType<typeof cleanLine>[]): Uint8Array[] { let last: string | undefined; return lines.map((line) => { last = line.time ?? last ?? new Date().toISOString(); return this.#data(step, `${line.text}\n`, last); }); }
#workflow(): string { return this.ctx.storage.kv.get<Target>("target")?.workflow ?? ""; }
#control( cursor: Pick<Cursor, "step" | "name" | "command">, status: "start" | "end", at: string, outcome?: string, ): Uint8Array { return controlFrame(this.#workflow(), cursor, status, at, outcome); }
#data(step: number, content: string, at: string): Uint8Array { return dataFrame(this.#workflow(), step, content, at); }
#broadcast(frame: Uint8Array): void { for (const ws of this.ctx.getWebSockets()) { if (ws.deserializeAttachment() === "ready") { ws.send(frame); } } }}