Something went wrong. Try again.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174/** * The self-check: it runs the fixture scripts (fixtures/commands/scripts) through the real `aite run` against a real Chrome with the real extension, * and compares each result with what the script's first lines say it must do. Its own job is to be able to fail: a script that should fail and passes is * a failure, a script that fails for another reason than the one written down is a failure, and "the extension never loaded" is never a skip. */
export type Expectation = | { kind: "pass" } | { kind: "fail"; reason: string; verb: string; file?: string } | { kind: "refused"; fragment: string };
/** Reads the `# expect:` line of a fixture script. */export function parseExpectation(text: string): Expectation | { error: string } { const lines = text.split("\n").filter((l) => /^#\s*expect:/.test(l)); if (lines.length === 0) return { error: "no `# expect:` line (a fixture must say what it is supposed to do)" }; if (lines.length > 1) return { error: "more than one `# expect:` line" }; const body = lines[0]!.replace(/^#\s*expect:\s*/, "").trim(); const bad = (why: string): { error: string } => ({ error: `\`# expect: ${body}\`: ${why}. Use \`pass\`, \`fail REASON COMMAND [in FILE]\` or \`refused "TEXT"\`` }); if (body === "pass") return { kind: "pass" }; const refused = /^refused\s+"([^"]+)"$/.exec(body); if (refused) return { kind: "refused", fragment: refused[1]! }; const fail = /^fail\s+([a-z-]+)\s+([A-Za-z]+)(?:\s+in\s+(\S+))?$/.exec(body); if (fail) return { kind: "fail", reason: fail[1]!, verb: fail[2]!, ...(fail[3] !== undefined ? { file: fail[3] } : {}) }; return bad("not understood");}
interface RunJsonLike { chrome?: string; extension?: string; tests?: Array<{ status?: string; steps?: Array<{ commands?: Array<{ command?: string; status?: string; reason?: string; file?: string }> }> }>;}
export interface Outcome { exitCode: number; output: string; /** The run's run.json, when the run got far enough to write one. */ json?: RunJsonLike;}
const firstFailed = (json: RunJsonLike | undefined) => json?.tests?.flatMap((t) => t.steps ?? []).flatMap((s) => s.commands ?? []).find((c) => c.status === "failed");const loaded = (json: RunJsonLike | undefined): boolean => json?.chrome !== undefined && json.chrome !== "unknown" && json.extension !== undefined && json.extension !== "unknown";
/** Whether a run did what the fixture said it must. The reason it did not is part of the answer. */export function judge(expectation: Expectation, outcome: Outcome): { ok: true } | { ok: false; why: string } { const no = (why: string) => ({ ok: false as const, why }); const { exitCode, json } = outcome; switch (expectation.kind) { case "pass": if (exitCode !== 0) return no(`expected a pass, got exit ${exitCode}: ${outcome.output.split("\n").filter((l) => /failed|error|refused|did not/i.test(l)).slice(0, 2).join(" | ") || outcome.output.slice(0, 160)}`); if (!json) return no("expected a pass, but there is no run.json to show the run happened"); if (!loaded(json)) return no("the report names no Chrome or extension, so the extension was not shown to be loaded"); if (firstFailed(json)) return no("exit 0, but run.json has a failed command (the exit code and the report disagree)"); return { ok: true }; case "fail": { if (exitCode !== 1) return no(`expected fail ${expectation.reason} ${expectation.verb}, got exit ${exitCode}`); if (!loaded(json)) return no("the report names no Chrome or extension, so the extension was not shown to be loaded"); const failed = firstFailed(json); if (!failed) return no(`expected fail ${expectation.reason} ${expectation.verb}, but no command failed in run.json`); if (failed.reason !== expectation.reason || failed.command !== expectation.verb) return no(`expected fail ${expectation.reason} ${expectation.verb}, got fail ${failed.reason} ${failed.command}`); if (expectation.file !== undefined && failed.file !== expectation.file) return no(`expected the failure in ${expectation.file}, got ${failed.file ?? "the main script"}`); return { ok: true }; } case "refused": if (exitCode !== 2) return no(`expected the script to be refused (exit 2) with "${expectation.fragment}", got exit ${exitCode}`); return outcome.output.includes(expectation.fragment) ? { ok: true } : no(`refused, but the message does not contain "${expectation.fragment}": ${outcome.output.slice(0, 160)}`); }}
/** * Whether the Chrome and extension that answered are the ones this check is meant for. Chrome reports only its major version to the extension * (the user agent is "reduced" to 155.0.0.0), so the major is compared with the pinned version. */export function checkIdentity(seen: { chrome?: string; extension?: string }, expected: { chromePinned: string; extensionVersion: string }): string | undefined { if (!seen.chrome || seen.chrome === "unknown") return "the report names no Chrome"; if (!seen.extension || seen.extension === "unknown") return "the report names no extension"; const major = (v: string) => v.split(".")[0]; if (major(seen.chrome) !== major(expected.chromePinned)) return `Chrome reports version ${seen.chrome}, but ${expected.chromePinned} is pinned`; if (seen.extension !== expected.extensionVersion) return `the extension reports version ${seen.extension}, but this build is ${expected.extensionVersion}`; return undefined;}
export interface SelfCheckScript { /** The file name without .aite. */ name: string; text: string;}
export interface ScriptResult { name: string; expectation: string; ok: boolean; flaky: boolean; passedRuns: number; totalRuns: number; problems: string[]; ms: number;}
export interface SelfCheckResult { /** 0 every run was as its fixture says; 1 something was not; 3 the check could not run (never a skip). */ exitCode: 0 | 1 | 3; scripts: ScriptResult[]; report: string;}
export interface SelfCheckDeps { /** Shows the extension is there: runs a small script and reads which Chrome and extension answered. Throws if it cannot. */ canary(): Promise<{ chrome: string; extension: string }>; run(script: SelfCheckScript, attempt: number): Promise<Outcome>;}
const describe = (e: Expectation): string => (e.kind === "pass" ? "pass" : e.kind === "fail" ? `fail ${e.reason} ${e.verb}${e.file ? ` in ${e.file}` : ""}` : `refused "${e.fragment}"`);
export async function runSelfCheck(scripts: SelfCheckScript[], options: { repeat: number; parallel: number }, deps: SelfCheckDeps): Promise<SelfCheckResult> { const stopped = (why: string): SelfCheckResult => ({ exitCode: 3, scripts: [], report: `SELF-CHECK COULD NOT RUN\n${why}\nThis is a failure, not a skip: nothing was proven.` }); if (scripts.length === 0) return { exitCode: 1, scripts: [], report: "SELF-CHECK FAILED\nno fixture scripts were found, so nothing was checked" };
let identity: { chrome: string; extension: string }; try { identity = await deps.canary(); } catch (err) { return stopped(`the canary could not show a Chrome with the extension loaded: ${err instanceof Error ? err.message : String(err)}`); }
const results = new Map<string, ScriptResult>(); const attempts = scripts.flatMap((s) => Array.from({ length: options.repeat }, (_, attempt) => ({ s, attempt }))); let next = 0; let environment: string | undefined;
async function worker(): Promise<void> { while (next < attempts.length && environment === undefined) { const { s, attempt } = attempts[next++]!; const parsed = parseExpectation(s.text); const entry = results.get(s.name) ?? { name: s.name, expectation: "error" in parsed ? "?" : describe(parsed), ok: true, flaky: false, passedRuns: 0, totalRuns: 0, problems: [], ms: 0 }; results.set(s.name, entry); entry.totalRuns++; if ("error" in parsed) { if (!entry.problems.includes(parsed.error)) entry.problems.push(parsed.error); continue; } const t0 = Date.now(); try { const outcome = await deps.run(s, attempt); entry.ms += Date.now() - t0; if (outcome.exitCode === 3) { environment = `${s.name}: ${outcome.output.split("\n").slice(0, 3).join(" ")}`; return; } const verdict = judge(parsed, outcome); if (verdict.ok) entry.passedRuns++; else entry.problems.push(`run ${attempt + 1}: ${verdict.why}`); } catch (err) { entry.ms += Date.now() - t0; entry.problems.push(`run ${attempt + 1}: ${err instanceof Error ? err.message : String(err)}`); } } } await Promise.all(Array.from({ length: Math.max(1, options.parallel) }, () => worker()));
if (environment !== undefined) return stopped(`a run ended because the environment failed: ${environment}`);
const list = scripts.map((s) => results.get(s.name)!).map((r) => ({ ...r, ok: r.problems.length === 0 && r.passedRuns === r.totalRuns, flaky: r.passedRuns > 0 && r.passedRuns < r.totalRuns })); const bad = list.filter((r) => !r.ok); const runs = list.reduce((n, r) => n + r.totalRuns, 0); const lines = [ bad.length === 0 ? "SELF-CHECK PASSED" : "SELF-CHECK FAILED", `${list.length} script${list.length === 1 ? "" : "s"}, ${runs} run${runs === 1 ? "" : "s"}, Chrome ${identity.chrome}, extension ${identity.extension}`, ...list.map((r) => ` ${r.ok ? "ok " : "FAIL"} ${r.name} (${r.expectation}) ${r.passedRuns}/${r.totalRuns}${r.flaky ? ` flaky (${r.passedRuns} of ${r.totalRuns})` : ""}`), ]; for (const r of bad) for (const p of [...new Set(r.problems)].slice(0, 3)) lines.push(` ${r.name}: ${p}`); return { exitCode: bad.length === 0 ? 0 : 1, scripts: list, report: lines.join("\n") };}