import { DurableObject } from "cloudflare:workers"; import { authorization, buildState, isTerminal, pipelineUrl, readBuild, time } from "./buildkite"; import type { RunStatus } from "./buildkite"; import { decode, decodeFirst, encode } from "@atcute/cbor"; import { findPipeline, findRun } from "./notify"; import { repoParam } from "./pipelines"; import { buffered, invalidRequest, isObject, rateLimit, string, upstream } from "./xrpc"; export const logCap = 2 * 1024 * 1024; const pollMs = 5_000; const checkMs = 60_000; const storeRetryMs = 10_000; const minRemaining = 15; export const capMarker = "[treadle: log longer than 2 MiB]"; // oxlint-disable-next-line no-control-regex const timestamps = /\u001b_bk;t=(\d+)\u0007/g; // oxlint-disable-next-line no-control-regex export const csi = /\u001b\[[0-?]*[ -/]*[@-~]/g; const decoder = new TextDecoder(); export function cleanLine(line: string): { text: string; time: string | undefined } { const ms = Number(line.matchAll(timestamps).next().value?.[1]); const date = new Date(ms); const text = line.replaceAll(timestamps, "").replace(/\r$/, ""); return { text: text.slice(text.lastIndexOf("\r") + 1), time: Number.isNaN(date.getTime()) ? undefined : date.toISOString(), }; } // Splits at newline bytes and keeps a partial last line unless the log is final. export function logLines( bytes: Uint8Array, final: boolean, ): { lines: ReturnType[]; used: number } { const lines = []; let used = 0; for (let end = bytes.indexOf(10); end !== -1; end = bytes.indexOf(10, used)) { lines.push(cleanLine(decoder.decode(bytes.subarray(used, end)))); used = end + 1; } if (final && used < bytes.length) { lines.push(cleanLine(decoder.decode(bytes.subarray(used)))); used = bytes.length; } return { lines, used }; } export interface StepInfo { step: number; name: string; command: string | undefined; kind?: "system" | "user"; } function encodeFrame(header: object, body: object): Uint8Array { const head = encode(header); const rest = encode(body); const bytes = new Uint8Array(head.length + rest.length); bytes.set(head); bytes.set(rest, head.length); return bytes; } export function controlFrame( workflow: string, { step, name, command, kind = "user" }: StepInfo, status: "start" | "end", at: string, outcome?: string, ): Uint8Array { return encodeFrame( { op: 1, t: "#control" }, { time: at, workflow, step, content: name, command, status, kind, outcome }, ); } export function dataFrame(workflow: string, step: number, content: string, at: string): Uint8Array { return encodeFrame( { op: 1, t: "#data" }, { time: at, workflow, step, content, stream: "stdout" }, ); } export function frameLine( frame: Uint8Array, ): { step: number; text: string; time: string } | undefined { const first: [unknown, Uint8Array] = decodeFirst(frame); const [header, rest] = first; const body: unknown = decode(rest); if (!isObject(header) || !isObject(body) || typeof body["step"] !== "number") { return undefined; } const { step, content, status, outcome } = body; const text = typeof content === "string" ? content : ""; const at = typeof body["time"] === "string" ? body["time"] : ""; if (header["t"] === "#data") { return { step, time: at, text: text.replace(/\n$/, "") }; } const end = status === "end" ? ` (${typeof outcome === "string" ? outcome : "done"})` : ""; return { step, time: at, text: `--- step ${step}: ${text}${end}` }; } interface Job { id: string; name: string; command: string | undefined; startedAt: string | undefined; finishedAt: string | undefined; outcome: string | undefined; } const outcomes = new Map([ ["passed", "success"], ["failed", "failed"], ["timed_out", "failed"], ["canceled", "failed"], ]); function parseJob(value: unknown): Job | undefined { if (!isObject(value) || value["type"] !== "script") { return undefined; } const id = string(value["id"]); if (id === undefined) { return undefined; } const command = string(value["command"]); return { id, name: string(value["name"]) ?? command ?? id, command, startedAt: time(value["started_at"]), finishedAt: time(value["finished_at"]), outcome: value["retried"] === true ? undefined : outcomes.get(String(value["state"])), }; } function scriptJobs(build: Record): Job[] { const jobs: unknown[] = Array.isArray(build["jobs"]) ? build["jobs"] : []; return jobs.map(parseJob).filter((job) => job !== undefined); } type Target = { repo: string; pipeline: string; workflow: string }; type Run = { org: string; slug: string; number: number | null; status: string }; type Cursor = { step: number; name: string; command: string | undefined; offset: number; started: string | undefined; end: { time: string; outcome: string | undefined } | undefined; capped: boolean; }; type Log = { start: number; bytes: Uint8Array }; class Rejected extends Error {} function httpError(url: string, status: number): Error { const message = `${url}: HTTP ${status}`; return status >= 400 && status < 500 && status !== 429 ? new Rejected(message) : new Error(message); } export async function subscribePipelineLogs( request: Request, env: Env, ctx: ExecutionContext, ): Promise { await rateLimit(request, env.LOG_LIMIT); const params = new URL(request.url).searchParams; const repo = repoParam(params.get("repo")); const workflows = params.getAll("workflows"); const [workflow] = workflows; if (workflows.length !== 1 || workflow === undefined) { throw invalidRequest("workflows must name exactly one workflow"); } if (request.headers.get("Upgrade")?.toLowerCase() !== "websocket") { throw invalidRequest("expected a WebSocket upgrade"); } const pipeline = await findPipeline( ctx.exports.Store.getByName("main"), repo, params.get("pipeline"), ); const run = findRun(pipeline, workflow); if (run.runner !== null) { // A run can end before it reaches a Runner, for example when it is cancelled in the queue. const done = isTerminal(run.status) ? "&done" : ""; return await ctx.exports.Runner.getByName(run.runner).fetch( `https://runner/?pipeline=${encodeURIComponent(pipeline.rkey)}${done}`, { headers: { Upgrade: "websocket" } }, ); } const url = new URL("https://logs/"); url.search = new URLSearchParams({ repo, pipeline: pipeline.rkey, workflow }).toString(); return await ctx.exports.Logs.getByName(`${pipeline.rkey}/${workflow}`).fetch(url, { headers: request.headers, }); } // Streams the job logs of one workflow run. A socket waits for its replay, then gets // broadcasts while it is ready, and is done after it is closed. export class Logs extends DurableObject { #queue: Promise = Promise.resolve(); #serial(task: () => Promise): Promise { const result = this.#queue.then(task); this.#queue = result.catch(() => null); return result; } override async fetch(request: Request): Promise { const params = new URL(request.url).searchParams; const target = { repo: params.get("repo") ?? "", pipeline: params.get("pipeline") ?? "", workflow: params.get("workflow") ?? "", }; const { 0: client, 1: server } = new WebSocketPair(); this.ctx.acceptWebSocket(server); server.serializeAttachment("waiting"); this.ctx.storage.kv.put("target", target); await this.#wake(Date.now()); return new Response(null, { status: 101, webSocket: client }); } async update(number: number, status: RunStatus | undefined, job: unknown): Promise { await this.#serial(async () => { const run = this.ctx.storage.kv.get("run"); if (this.ctx.getWebSockets().length === 0 || run?.number !== number) { return; } const parsed = parseJob(job); if (parsed !== undefined) { await this.#sync(run, [parsed]); } // Only a successful check stores a terminal status, so a failed one is retried. if (status !== undefined && isTerminal(status)) { try { await this.#check(run); } catch (error) { this.ctx.storage.kv.delete("checkedAt"); throw error; } } else if (status !== undefined) { this.ctx.storage.kv.put("run", { ...run, status }); await this.#wake(Date.now() + pollMs); } }); } override async alarm(): Promise { const next = await this.#serial(() => this.#tick()).catch((error: unknown) => { console.warn("log stream check failed", { error: String(error) }); return Date.now() + pollMs; }); if (next !== undefined) { await this.#wake(Math.max(next, this.ctx.storage.kv.get("limitedUntil") ?? 0)); } } // Sends one replay to every waiting socket. Buildkite rejections close them with an error. async #join(current: Run): Promise { let run = current; const waiting = this.ctx .getWebSockets() .filter((ws) => ws.deserializeAttachment() === "waiting"); if (waiting.length === 0) { return run; } let jobs: [Cursor, Log | undefined][]; try { [run, jobs] = await this.#replay(run); } catch (error) { if (!(error instanceof Rejected)) { throw error; } console.warn("log replay rejected", { error: String(error) }); for (const ws of waiting) { ws.serializeAttachment("done"); ws.send(encodeFrame({ op: -1 }, { error: "UpstreamUnavailable" })); ws.close(1011, "log replay failed"); } return run; } // Frames are built one job at a time, because a capped log has many lines. for (const [cursor, log] of jobs) { const frames = this.#replayJob(cursor, log); for (const ws of waiting) { for (const frame of frames) { ws.send(frame); } } } for (const ws of waiting) { if (isTerminal(run.status)) { this.#close(ws); } else { ws.serializeAttachment("ready"); } } return run; } // Returns the next alarm time, or undefined when the stream is done. async #tick(): Promise { const kv = this.ctx.storage.kv; const now = Date.now(); if (this.ctx.getWebSockets().length === 0) { return undefined; } const limited = kv.get("limitedUntil") ?? 0; if (limited > now) { return limited; } let run = kv.get("run"); if (typeof run?.number !== "number") { run = await this.#refresh(); if (run !== undefined && isTerminal(run.status) && run.number === null) { for (const ws of this.ctx.getWebSockets()) { this.#close(ws); } await this.#finish(); return undefined; } if (typeof run?.number !== "number") { return now + storeRetryMs; } } // Replays run here, in order with polls and within the rate limit. try { run = await this.#join(run); } catch (error) { console.warn("log replay failed", { error: String(error) }); } if (isTerminal(run.status)) { if (this.ctx.getWebSockets().some((ws) => ws.deserializeAttachment() === "waiting")) { return now + pollMs; } await this.#finish(); return undefined; } if (now - (kv.get("checkedAt") ?? 0) >= checkMs) { run = await this.#check(run); if (run === undefined) { return undefined; } } if (run.status !== "blocked" && now - (kv.get("polledAt") ?? 0) >= pollMs) { kv.put("polledAt", now); await this.#poll(run); } return run.status === "blocked" ? (kv.get("checkedAt") ?? now) + checkMs : Date.now() + pollMs; } // Stores the run that the Store has, or returns undefined so that the alarm asks again. async #refresh(): Promise { const target = this.ctx.storage.kv.get("target"); if (target === undefined) { return undefined; } const store = this.ctx.exports.Store.getByName("main"); const pipeline = await store.getPipeline(target.repo, target.pipeline); const row = pipeline?.runs.find((run) => run.workflow === target.workflow); if (row === undefined) { return undefined; } const run = { org: row.bk_org, slug: row.bk_pipeline, number: row.build_number, status: row.status, }; this.ctx.storage.kv.put("run", run); return run; } // Returns the updated run, or undefined when the stream is done. async #check(run: Run): Promise { this.ctx.storage.kv.put("checkedAt", Date.now()); const build = await this.#getBuild(run); await this.#sync(run, scriptJobs(build)); // Jobs of a canceling build still run and write logs. const status = build["state"] === "canceling" ? "running" : buildState(build)?.status; const updated = { ...run, status: status ?? run.status }; this.ctx.storage.kv.put("run", updated); if (isTerminal(updated.status)) { await this.#finish(); return undefined; } return updated; } async #sync(run: Run, jobs: Job[]): Promise { const kv = this.ctx.storage.kv; for (const job of jobs) { const key = `job:${job.id}`; const cursor = kv.get(key) ?? this.#cursor(job); if (cursor.started === undefined && job.startedAt !== undefined) { cursor.started = job.startedAt; this.#broadcast(this.#control(cursor, "start", cursor.started)); } kv.put(key, cursor); if ( cursor.started !== undefined && cursor.end === undefined && job.finishedAt !== undefined ) { // oxlint-disable-next-line no-await-in-loop await this.#pull(run, job.id, cursor, true); cursor.end = { time: job.finishedAt, outcome: job.outcome }; this.#broadcast(this.#control(cursor, "end", job.finishedAt, job.outcome)); kv.put(key, cursor); } } } #cursor(job: Job): Cursor { const step = this.ctx.storage.kv.get("steps") ?? 0; this.ctx.storage.kv.put("steps", step + 1); return { step, name: job.name, command: job.command, offset: 0, started: undefined, end: undefined, capped: false, }; } async #poll(run: Run): Promise { const kv = this.ctx.storage.kv; for (const [key, cursor] of Array.from(kv.list({ prefix: "job:" }))) { if (cursor.started === undefined || cursor.end !== undefined) { continue; } if ((kv.get("limitedUntil") ?? 0) > Date.now()) { return; } // oxlint-disable-next-line no-await-in-loop await this.#pull(run, key.slice("job:".length), cursor, false); kv.put(key, cursor); } } async #pull(run: Run, id: string, cursor: Cursor, final: boolean): Promise { if (cursor.capped) { return; } const log = cursor.offset < logCap ? await this.#log(run, id, logCap - cursor.offset) : null; if (log === undefined) { return; } if (log === null || log.start > cursor.offset) { this.#broadcast(this.#data(cursor.step, `${capMarker}\n`, new Date().toISOString())); cursor.capped = true; return; } const { lines, used } = logLines(log.bytes.subarray(cursor.offset - log.start), final); for (const frame of this.#lines(cursor.step, lines)) { this.#broadcast(frame); } cursor.offset += used; } // Fetches the logs of every started job, so that a failure comes before any frame. async #replay(current: Run): Promise<[Run, [Cursor, Log | undefined][]]> { let run = current; let cursors: [string, Cursor][]; const build = isTerminal(run.status) ? await this.#getBuild(run) : undefined; if (build === undefined || build["state"] === "canceling") { if (build !== undefined) { run = { ...run, status: "running" }; this.ctx.storage.kv.put("run", run); } cursors = Array.from( this.ctx.storage.kv.list({ prefix: "job:" }), ([key, cursor]): [string, Cursor] => [key.slice("job:".length), cursor], ).toSorted((a, b) => a[1].step - b[1].step); } else { cursors = scriptJobs(build).map((job, step): [string, Cursor] => [ job.id, { step, name: job.name, command: job.command, offset: Infinity, started: job.startedAt, end: job.finishedAt === undefined ? undefined : { time: job.finishedAt, outcome: job.outcome }, capped: false, }, ]); } const jobs: [Cursor, Log | undefined][] = []; for (const [id, cursor] of cursors) { if (cursor.started !== undefined) { // oxlint-disable-next-line no-await-in-loop const log = cursor.offset > 0 ? await this.#log(run, id, logCap) : undefined; jobs.push([cursor, log]); } } return [run, jobs]; } #replayJob(cursor: Cursor, log: Log | undefined): Uint8Array[] { return [ this.#control(cursor, "start", cursor.started ?? ""), ...(log === undefined ? [] : this.#replayLog(cursor.step, log, cursor.offset)), ...(cursor.end === undefined ? [] : [this.#control(cursor, "end", cursor.end.time, cursor.end.outcome)]), ]; } // Returns the log frames before upTo. A log that starts after byte 0 begins mid-line. #replayLog(step: number, log: Log, upTo: number): Uint8Array[] { let bytes = log.bytes.subarray(0, Math.max(0, upTo - log.start)); if (log.start <= 0) { return this.#lines(step, logLines(bytes, true).lines); } const newline = bytes.indexOf(10); bytes = bytes.subarray(newline === -1 ? bytes.length : newline + 1); return [ this.#data(step, `${capMarker}\n`, new Date().toISOString()), ...this.#lines(step, logLines(bytes, true).lines), ]; } // Viewers that still wait for a replay keep the target and get one from the next alarm. async #finish(): Promise { const target = this.ctx.storage.kv.get("target"); const limited = this.ctx.storage.kv.get("limitedUntil"); let waiting = false; for (const ws of this.ctx.getWebSockets()) { const state: unknown = ws.deserializeAttachment(); if (state === "ready") { this.#close(ws); } waiting ||= state === "waiting"; } await this.ctx.storage.deleteAll(); if (waiting && target !== undefined) { this.ctx.storage.kv.put("target", target); if (limited !== undefined) { this.ctx.storage.kv.put("limitedUntil", limited); } await this.#wake(Date.now()); } } #close(ws: WebSocket): void { if (ws.deserializeAttachment() !== "done") { ws.serializeAttachment("done"); ws.close(1000, "log stream complete"); } } async #wake(at: number): Promise { const current = await this.ctx.storage.getAlarm(); if (current === null || current > at) { await this.ctx.storage.setAlarm(at); } } async #getBuild(run: Run): Promise> { const url = `${pipelineUrl(run.org, run.slug)}/builds/${run.number}`; const response = await this.#get(url, {}); const body: unknown = response.ok ? await response.json() : undefined; if (!isObject(body)) { throw httpError(url, response.status); } return body; } // Returns the last n bytes of a job log, or undefined for an empty log. async #log(run: Run, id: string, n: number): Promise { const url = `${pipelineUrl(run.org, run.slug)}/builds/${run.number}/jobs/${encodeURIComponent(id)}/log`; // A full response must fit in the requested suffix, or it may be a whole huge log. const fits = (r: Response): boolean => Number(r.headers.get("Content-Length") ?? Infinity) <= n; const response = await this.#get( url, { Accept: "text/plain", Range: `bytes=-${n}` }, async (answer) => { if (answer.status === 206 || fits(answer)) { return await buffered(answer, n); } await answer.body?.cancel(); return new Response(null, answer); }, ); if (response.status === 416) { return undefined; } if (!response.ok) { throw httpError(url, response.status); } const range = /^bytes (\d+)-\d+\/\d+$/.exec(response.headers.get("Content-Range") ?? ""); let start = range?.[1]; if (response.status !== 206) { start = fits(response) ? "0" : undefined; } if (start === undefined) { throw new Rejected(`${url}: HTTP ${response.status} without a usable range`); } return { start: Number(start), bytes: new Uint8Array(await response.arrayBuffer()) }; } // Buildkite allows 50 requests a minute per user, and build starts share them. async #get(url: string, headers: Record, read = readBuild): Promise { const response = await upstream( url, 503, { headers: { Authorization: authorization(this.env.BUILDKITE_TOKEN), ...headers } }, { read }, ); const remaining = response.headers.get("RateLimit-User-Remaining"); if (response.status === 429 || (remaining !== null && Number(remaining) < minRemaining)) { const reset = Number(response.headers.get("RateLimit-User-Reset")); this.ctx.storage.kv.put( "limitedUntil", Date.now() + (Number.isFinite(reset) && reset > 0 ? reset : 60) * 1000, ); } return response; } #lines(step: number, lines: ReturnType[]): Uint8Array[] { let last: string | undefined; return lines.map((line) => { last = line.time ?? last ?? new Date().toISOString(); return this.#data(step, `${line.text}\n`, last); }); } #workflow(): string { return this.ctx.storage.kv.get("target")?.workflow ?? ""; } #control( cursor: Pick, status: "start" | "end", at: string, outcome?: string, ): Uint8Array { return controlFrame(this.#workflow(), cursor, status, at, outcome); } #data(step: number, content: string, at: string): Uint8Array { return dataFrame(this.#workflow(), step, content, at); } #broadcast(frame: Uint8Array): void { for (const ws of this.ctx.getWebSockets()) { if (ws.deserializeAttachment() === "ready") { ws.send(frame); } } } }