import { isMap, isScalar, parse, parseDocument } from "yaml"; import type { Document } from "yaml"; import { z } from "zod"; import { isObject, string } from "./xrpc"; export interface BuildkiteTarget { pipeline?: string | undefined; org?: string | undefined; cleanCheckout?: boolean | undefined; } export interface Step { name: string; command: string; environment: Record; kind?: "system"; } export interface CloudflareJob { environment: Record; steps: Step[]; instance: Instance; timeoutMs: number; cache: boolean; secrets: string[]; // Runs stored by older code have none. injected?: InjectedSecret[]; apt: string[]; } export interface Workflow { name: string; raw: string; engine: string | undefined; when: Constraint[]; buildkite: BuildkiteTarget; cloudflare: CloudflareJob | undefined; after?: string[]; error?: string; } export const variableName = /^[A-Za-z_][A-Za-z0-9_]*$/; const hourMs = 60 * 60_000; const maxTimeoutMs = 6 * hourMs; // A Debian package name with an optional version, which the shell cannot interpret. const aptPackage = /^[a-z0-9][a-z0-9+.-]+(?:=[0-9][A-Za-z0-9.+~:-]*)?$/; const strings = z .union([z.string(), z.array(z.string())], "must be a string or a list of strings") .nullish() .transform((list) => [list ?? []].flat()); const constraints = z .array(z.object({ event: strings, branch: strings, tag: strings, types: strings })) .nullish() .transform((list) => list ?? []); type Constraint = z.infer[number]; const buildkiteTarget = z .object({ pipeline: z.string().min(1, "must not be empty").optional(), org: z.string().optional(), clean_checkout: z.boolean().optional(), }) .transform(({ clean_checkout: cleanCheckout, ...target }): BuildkiteTarget => ({ ...target, cleanCheckout, })); const workflowFile = z.object({ engine: z.string().optional(), when: constraints, treadle: z.object({ buildkite: buildkiteTarget.nullish(), after: strings }).nullish(), }); const trustedFile = z.object({ trusted: z.object({ branch: z.array(z.string()) }).optional() }); // Only the shape is checked, because account limits can be raised. The runtime checks the rest. const instanceError = "must be an instance type or a custom instance"; const positive = z.int().positive(); const instance = z.union( [ z.string().min(1, instanceError), z .object({ vcpu: positive, memory_mib: positive, disk_mb: positive }) .transform(({ vcpu, memory_mib: memoryMib, disk_mb: diskMb }) => ({ vcpu, memoryMib, diskMb, })), ], instanceError, ); export type Instance = z.infer; // A duration such as 45m or 1h30m. const durationError = "must be a duration of at most 6h"; const duration = z .string(durationError) .transform((value) => { const [, h = 0, m = 0, s = 0] = /^(?:(\d+)h)?(?:(\d+)m)?(?:(\d+)s)?$/.exec(value) ?? []; return ((Number(h) * 60 + Number(m)) * 60 + Number(s)) * 1000; }) .refine((ms) => ms > 0 && ms <= maxTimeoutMs, durationError); const patternList = (pattern: RegExp, kind: string) => strings.pipe( z.array( z.string().regex(pattern, { error: (issue) => `is not a ${kind}: ${String(issue.input)}`, }), ), ); // Exact hostnames or *. globs, as Cloudflare Containers matches them on port 443. const hostname = /^(?:\*\.)?(?:[a-z0-9-]+\.)+(?=[a-z0-9-]*[a-z])[a-z0-9-]+$/; // Cloudflare Containers intercepts at most 64 hostnames for each container. const maxHosts = 64; const headerName = /^[A-Za-z0-9!#$%&'*+.^_`|~-]+$/; const injectedSecret = z.strictObject({ name: z.string().regex(variableName, { error: (issue) => `is not a variable name: ${String(issue.input)}`, }), hosts: patternList(hostname, "hostname").refine( (hosts) => hosts.length > 0, "must name a host", ), headers: patternList(headerName, "header name").transform((headers) => headers.length > 0 ? headers : ["Authorization"], ), }); export type InjectedSecret = z.output; // Picks the schema by the type of the value, so that an error names the nested field. const secret = z.unknown().transform((value, ctx) => { if (typeof value !== "string" && !isObject(value)) { ctx.addIssue({ code: "custom", message: "must be a secret name or a map" }); return z.NEVER; } const result = ( typeof value === "string" ? injectedSecret.shape.name : injectedSecret ).safeParse(value, { error: errorMap, }); if (result.success) { return result.data; } for (const issue of result.error.issues) { ctx.addIssue({ code: "custom", message: issue.message, path: issuePath(issue) }); } return z.NEVER; }); const cloudflareFile = trustedFile.extend({ environment: z.unknown().optional(), steps: z .array( z.object({ name: z.string(), command: z.string(), environment: z.unknown().optional(), }), "must be a non-empty list", ) .min(1), treadle: z .object({ cloudflare: z.strictObject({ timeout: duration.optional() }).nullish() }) .nullish(), runs_on: instance.default("standard-1"), cache: z.boolean().default(false), secrets: z.preprocess( (value) => (typeof value === "string" ? [value] : (value ?? [])), z.array(secret), ), dependencies: patternList(aptPackage, "Debian package name"), }); const typeErrors: Record = { object: "must be a map", array: "must be a list", string: "must be a string", boolean: "must be true or false", }; function errorMap(issue: z.core.$ZodRawIssue): string | undefined { return issue.code === "unrecognized_keys" ? "is not a setting" : typeErrors[issue.code === "invalid_type" ? issue.expected : ""]; } function issuePath(issue: z.core.$ZodIssue): PropertyKey[] { return issue.code === "unrecognized_keys" ? [...issue.path, ...issue.keys.slice(0, 1)] : issue.path; } // Throws the first issue as one line that names its field. function parsed(schema: T, value: unknown): z.output { const result = schema.safeParse(value, { error: errorMap }); if (result.success) { return result.data; } const [issue] = result.error.issues; const path = issue === undefined ? [] : issuePath(issue); throw new Error(`${z.core.toDotPath(path) || "workflow"} ${issue?.message ?? "is invalid"}`); } // Keeps a number or boolean as written, as upstream does, so 3.10 stays "3.10". function environment( document: Document, path: (string | number)[], js: unknown, ): Record { const field = z.core.toDotPath(path); const node: unknown = document.getIn(path, true); if (node === undefined && js !== undefined) { throw new Error(`${field} must not be behind an alias`); } if (node === undefined || (isScalar(node) && node.value === null)) { return {}; } if (!isMap(node)) { throw new Error(`${field} must be a map`); } return Object.fromEntries( node.items.map(({ key, value }) => { const name = isScalar(key) && typeof key.value === "string" ? key.value : ""; if (!variableName.test(name)) { throw new Error(`${field}.${name} is not a variable name`); } if (!isScalar(value) || !["string", "number", "boolean"].includes(typeof value.value)) { throw new Error(`${field}.${name} must be a scalar`); } return [name, typeof value.value === "string" ? value.value : String(value.source)]; }), ); } function cloudflareJob(doc: unknown, document: Document): CloudflareJob { const file = parsed(cloudflareFile, doc); const names = file.secrets.map((item) => (typeof item === "string" ? item : item.name)); const twice = names.find((name, i) => names.indexOf(name) !== i); if (twice !== undefined) { throw new Error(`secrets lists ${twice} twice`); } const injected = file.secrets.filter((item) => typeof item !== "string"); if (new Set(injected.flatMap((item) => item.hosts)).size > maxHosts) { throw new Error(`secrets list more than ${maxHosts} hosts`); } return { environment: environment(document, ["environment"], file.environment), steps: file.steps.map(({ name, command, environment: env }, i) => ({ name, command, environment: environment(document, ["steps", i, "environment"], env), })), instance: file.runs_on, timeoutMs: file.treadle?.cloudflare?.timeout ?? hourMs, cache: file.cache, secrets: file.secrets.filter((item) => typeof item === "string"), injected, apt: file.dependencies, }; } // Reads trusted: only from the workflow file of the default branch, because the built commit // may come from a pull request. Pull request runs are never trusted, whatever their branch. export function isTrusted( event: "push" | "manual" | "pull_request", defaultBranchRaw: string | undefined, defaultBranch: string, branch: string, ): boolean { if (event === "pull_request") { return false; } try { const doc: unknown = defaultBranchRaw === undefined ? {} : (parse(defaultBranchRaw) ?? {}); const branches = trustedFile.parse(doc).trusted?.branch ?? [defaultBranch]; return matchesAny(branches, branch); } catch { return false; } } export function parseWorkflow(name: string, raw: string): Workflow { const document = parseDocument(raw); const [error] = document.errors; if (error !== undefined) { throw error; } const doc: unknown = document.toJS() ?? {}; const { engine, when, treadle } = parsed(workflowFile, doc); return { name, raw, engine, when, buildkite: treadle?.buildkite ?? {}, cloudflare: engine === "cloudflare" ? cloudflareJob(doc, document) : undefined, after: treadle?.after ?? [], }; } // Returns the error of each workflow that waits for a workflow outside the list or for itself. export function afterErrors(workflows: Workflow[]): Map { const after = new Map(workflows.map((workflow) => [workflow.name, workflow.after ?? []])); const errors = new Map(); const done = new Set(); const visit = (name: string, path: string[]): void => { const start = path.indexOf(name); if (start !== -1) { const cycle = [...path.slice(start), name].join(" → "); for (const member of path.slice(start)) { errors.set(member, `treadle.after makes a cycle: ${cycle}`); } return; } if (done.has(name)) { return; } for (const prerequisite of after.get(name) ?? []) { if (after.has(prerequisite)) { visit(prerequisite, [...path, name]); } else { errors.set( name, `treadle.after: ${prerequisite} is not a workflow of this pipeline`, ); } } done.add(name); }; for (const name of after.keys()) { visit(name, []); } return errors; } // Gives each name in treadle.after the file name of its workflow, which may drop .yml or .yaml. export function resolveAfter(workflows: Workflow[]): Workflow[] { const names = new Set(workflows.map((workflow) => workflow.name)); const resolve = (name: string): string => [name, `${name}.yml`, `${name}.yaml`].find((file) => names.has(file)) ?? name; return workflows.map((workflow) => workflow.after === undefined ? workflow : { ...workflow, after: workflow.after.map(resolve) }, ); } // Adds the workflows that the selected workflows wait for. export function withPrerequisites( workflows: Workflow[], selected: (workflow: Workflow) => boolean, ): Workflow[] { const names = new Set(workflows.filter(selected).map((workflow) => workflow.name)); for (const name of names) { for (const prerequisite of workflows.find((w) => w.name === name)?.after ?? []) { names.add(prerequisite); } } return workflows.filter((workflow) => names.has(workflow.name)); } export function invalidWorkflow(name: string, raw: string, error: string): Workflow | undefined { let doc: unknown; let when: Constraint[] = []; try { doc = parse(raw); when = constraints.parse(isObject(doc) ? doc["when"] : undefined); } catch { when = []; } const engine = isObject(doc) ? string(doc["engine"]) : undefined; if (isObject(doc) && engine !== "buildkite" && engine !== "cloudflare") { return undefined; } return { name, raw, engine, when, buildkite: {}, cloudflare: undefined, error }; } // Buildkite pipeline slugs allow only lowercase letters, digits, and dashes. export function pipelineSlug(repoName: string): string { return repoName .toLowerCase() .replaceAll(/[^a-z0-9]+/g, "-") .replaceAll(/^-|-$/g, ""); } // `**` spans directories only when it fills a whole path segment. export function globMatch(pattern: string, value: string): boolean { const re = pattern.replace( /\*\*\/|\*\*|\*|\?|\[(!?)([^\]]*)\]|[.+^${}()|[\]\\]/gu, (token, negate: string | undefined, chars: string | undefined, offset: number) => { const segmentStart = offset === 0 || pattern.charAt(offset - 1) === "/"; if (token === "**/") { return segmentStart ? "(?:.*/)?" : "[^/]*/"; } if (token === "**") { return segmentStart && offset + 2 === pattern.length ? ".*" : "[^/]*"; } if (token === "*") { return "[^/]*"; } if (token === "?") { return "[^/]"; } if (chars !== undefined) { return `[${negate === "!" ? "^" : ""}${chars}]`; } return `\\${token}`; }, ); try { return new RegExp(`^${re}$`, "u").test(value); } catch { return false; } } function matchesAny(patterns: string[], value: string): boolean { return patterns.some((pattern) => globMatch(pattern, value)); } function constraintMatchesPush(constraint: Constraint, ref: string): boolean { if (!constraint.event.includes("push")) { return false; } if (ref.startsWith("refs/heads/")) { return matchesAny(constraint.branch, ref.slice("refs/heads/".length)); } if (ref.startsWith("refs/tags/")) { return matchesAny(constraint.tag, ref.slice("refs/tags/".length)); } return false; } export function matchesPush(workflow: Workflow, ref: string): boolean { return workflow.when.length === 0 || workflow.when.some((c) => constraintMatchesPush(c, ref)); } const defaultPullTypes = ["opened", "reopened", "synchronize"]; export function matchesPullRequest( workflow: Workflow, targetBranch: string, action: string, ): boolean { return ( workflow.when.length === 0 || workflow.when.some( (c) => c.event.includes("pull_request") && matchesAny(c.branch, targetBranch) && (c.types.length > 0 ? c.types : defaultPullTypes).includes(action), ) ); } const triggerNsids = new Map([ ["push", "org.tangled.event.push"], ["pull_request", "org.tangled.event.pullRequest"], ["manual", "org.tangled.ci.trigger.manual"], ]); export function triggers(workflow: Workflow): string[] { const events = workflow.when.flatMap((constraint) => constraint.event); return [...new Set(events.map((event) => triggerNsids.get(event) ?? event))]; }