Something went wrong. Try again.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782import type { Settled } from "../bridge/dispatcher";import type { Command, Script, Target } from "./parser";import type { Diagnosis } from "./diagnosis/types";import { formatDiagnosis } from "./diagnosis/format";import { printCommand } from "./printer";import { DRIFTED_MIN, REPAIRED_MIN } from "../shared/repairScore";import { isElementDescriptor, type ElementDescriptor } from "../shared/descriptor";import type { LockStore, LockSummary } from "./lockfile";
export interface EngineDeps { request(body: { cmd: string; [key: string]: unknown }, options?: { timeoutMs?: number }): Promise<Settled>; now(): number; /** A fixed pause, for the `sleep` command. */ sleep(ms: number): Promise<void>; /** Where to put a picture (a path under the run folder, and the base64 PNG). Without it no screenshots are taken. */ saveScreenshot?(path: string, base64Png: string): Promise<void>; /** `failed`: the picture that ends each step is only taken when the step failed (a `screenshot` command still takes its own). Default `all`. A picture can show a balance or a name, so a device that runs other people's tests takes as few as it can. */ stepScreenshots?: "all" | "failed"; /** Runs `call NAME ARGS` (a TypeScript check from scripts/checks). Without it a `call` fails and says so. */ runCheck?(call: CheckCall): Promise<CheckOutcome>; /** Explains a failed command by looking down the stack (docs/design/failure-diagnosis.md). Without it a failure is reported as the script saw it. */ diagnose?(failure: FailureToDiagnose): Promise<Diagnosis>; /** The lock file in memory: the first time a target is found its descriptor is kept (docs/design/data-formats.md). Without it nothing is remembered. */ lock?: LockStore; /** * Ends the run: when it is aborted the script stops where it is (no further request, however long the thing it was waiting for takes) and runScript rejects with * RunAborted. The extension host uses this for the person's pause, the central system's stop, and the job's time budget. */ signal?: AbortSignal; /** * Runs once the start page is open, before any step (#70): the host looks at the page and says whether the profile is signed in as the account the job's profile label * stands for. A failure ends the run there, as a failed `start` with the reason `wrong-session`, so nothing a script does happens on the wrong account. */ afterStart?(tab: number | undefined): Promise<{ ok: true } | { ok: false; message: string; expected: string; found: string }>; /** An unattended run: a target the lock file does not know is refused, and nothing is written (the lock store is frozen too). */ freeze?: true;}
/** What the diagnosis is told about the command that failed. */export interface FailureToDiagnose { /** The profile the failed command ran on (several-profile scripts); the observer only watches the default one. */ profile?: string | undefined; verb: string; reason: string; line: number; file?: string; detail: Record<string, unknown>; /** When the command began and when it gave up, on the run's clock. */ startedAt: number; endedAt: number; stepIndex: number; commandIndex: number;}
export type CheckOutcome = { ok: true } | { ok: false; reason: "mismatch" | "internal" | "timeout" | "pre-state"; message: string; expected?: unknown; found?: unknown };
/** What a `call` hands to the runner: the check's name and arguments, the variables set so far, and the tab and time limit of the run. */export interface CheckCall { /** The profile of the command that called it (several-profile scripts); absent in a script with one. */ profile?: string | undefined; name: string; args: string[]; vars: Record<string, string>; timeoutMs: number; tab: number | undefined;}
export interface Failure { /** Line in the script; 0 when there is none. */ line: number; verb: string; /** The command as written, for the report. */ command: string; /** The script it was written in, when that is not the one being run. */ file?: string; reason: string; detail: Record<string, unknown>;}
/** Something a person should look at that is not a failure. `code` is stable; `message` is for people. */export interface Warning { code: "sleep" | "no-assertion" | "screenshot" | "tab-already-closed" | "diagnosis" | "outline"; message: string; line?: number; /** The script the line belongs to, when it is an included one. */ file?: string;}
/** A target found by repair or by its remembered description rather than by what the script says (docs/design/locator-repair.md). */export interface RepairRecord { /** `repair`: scored against every element on the page; `descriptor`: found by a remembered field such as its data-testid. */ via: "repair" | "descriptor"; /** The target as the script wrote it. */ source: string; score: number; runnerUp: number; /** Kinds of thing that changed: text, name, role, tag, classes, attributes, surroundings, position. */ changed: string[]; /** The same, as words: `text "Send" -> "Send now"`. */ changes: string[]; old: ElementDescriptor; now: ElementDescriptor;}
export interface CommandResult { /** The profile that did this command; only set in a script that uses several. */ profile?: string; line: number; file?: string; verb: string; /** The command as written, for the report. */ source: string; status: "passed" | "passed-after-retry" | "repaired" | "failed" | "skipped"; /** Times the command was sent again: by the extension after the page navigated under it, or by the bridge after a reconnect. */ retries: number; durationMs: number; warnings: Warning[]; /** Path of the picture a `screenshot` command saved, relative to the run folder. */ screenshot?: string; /** For a failed command: where in the stack the fault is, with the evidence. */ diagnosis?: Diagnosis; /** Targets this command found by repair. */ repairs?: RepairRecord[]; reason?: string; expected?: unknown; found?: unknown; detail?: Record<string, unknown>;}
export interface StepResult { name: string; line: number; status: "passed" | "passed-after-retry" | "repaired" | "failed" | "skipped"; durationMs: number; retries: number; commands: CommandResult[]; /** The warnings of the step's commands, and the step's own. */ warnings: Warning[]; failure?: Failure; /** Path of the picture taken after the step, relative to the run folder. */ screenshot?: string;}
export interface RunResult { name: string; /** True when no command failed. */ ok: boolean; status: "passed" | "failed"; steps: StepResult[]; durationMs: number; retries: number; /** The test's own warnings (steps carry theirs). */ warnings: Warning[]; /** All warnings of the run, the test's and the steps'. */ warningCount: number; /** What the run did with the lock file; absent when the run had none. */ lock?: LockSummary; /** Targets found by repair (a pass that a person should glance at). */ repairs: number; /** The profiles the test used, the default first; only set when it used more than one. */ profiles?: string[];}
const DEFAULT_TIMEOUT_MS = 10_000;/** The page is told to wait `timeoutMs`; the bridge waits a little longer so the page's own answer arrives first. */const BRIDGE_SLACK_MS = 5_000;const SHOT_TIMEOUT_MS = 10_000;const pad = (n: number, width: number) => String(n).padStart(width, "0");
const BARE_KEYS = new Set(["role", "index"]);export const describeTarget = (target: Record<string, string>): string => Object.entries(target).map(([k, v]) => (BARE_KEYS.has(k) ? `${k}:${v}` : `${k}:${JSON.stringify(v)}`)).join(" ");/** What typed text is replaced with in anything that is written down. */const MASK = "********";/** A failure's text is bounded: a page can make `found` as long as it likes, and run.json, the report and the archive carry it. */const MAX_DETAIL_TEXT = 500;const clampText = (v: unknown): unknown => (typeof v === "string" && v.length > MAX_DETAIL_TEXT ? `${v.slice(0, MAX_DETAIL_TEXT)}... (${v.length - MAX_DETAIL_TEXT} more characters)` : v);function clampDetail(d: Record<string, unknown>): Record<string, unknown> { const out: Record<string, unknown> = {}; for (const [k, v] of Object.entries(d)) out[k] = typeof v === "string" ? clampText(v) : Array.isArray(v) ? v.slice(0, 20).map(clampText) : v !== null && typeof v === "object" ? JSON.parse(JSON.stringify(v, (_k, x) => clampText(x))) : v; return out;}
/** Words that say a field holds a secret. Whole words only: "pin" in "spinner" or "pinterest" is not one. */const SECRET_WORDS = /\b(?:pass(?:word|wd|phrase|code)?|pwd|secret|pin|otp|cvv|cvc|token|mnemonic|seed|private[ -]?key)\b/i;const looksSecret = (words: string): boolean => SECRET_WORDS.test(words);
export const describeCommand = (c: Command): string => { switch (c.verb) { // What is typed into a field that names a secret is never written down, found or not; a password input found by the page is masked as well (see the type item). case "type": return looksSecret(Object.values(c.target).join(" ")) ? printCommand({ ...c, text: MASK }) : printCommand(c); case "set": return looksSecret(c.name) ? `set ${c.name} ${JSON.stringify(MASK)}` : `set ${c.name} ${JSON.stringify(c.value)}`; case "click": return `click ${describeTarget(c.target)}`; case "sleep": return `sleep ${c.ms}ms`; case "verify": return c.kind === "text" && c.match === undefined && c.within === undefined && c.timeoutMs === undefined ? `verify text:${JSON.stringify(c.value)}` : printCommand(c); default: return printCommand(c); }};
/** The run was ended from outside (see EngineDeps.signal). Not a failure of the test: there is no result. */export class RunAborted extends Error { override name = "RunAborted"; constructor() { super("the run was aborted"); }}
class StepFailed extends Error { constructor(readonly failure: Failure) { super(failure.reason); }}
/** A retry count from an answer: a whole number of 1 or more, otherwise nothing. */const retriesOf = (answer: Record<string, unknown>): number => (typeof answer.retries === "number" && Number.isInteger(answer.retries) && answer.retries > 0 ? answer.retries : 0);
const NO_VERIFY: Warning = { code: "no-assertion", message: "this test has no verify command, so it cannot fail on what the page shows" };const NO_COMMANDS: Warning = { code: "no-assertion", message: "this step has no commands, so it checks nothing" };
type Shot = { screenshot?: string; warning?: Warning };
/** One thing a step does: a command of the script, or the opening of the start page. */interface Item { /** The profile that does it (several-profile scripts). */ profile?: string | undefined; line: number; file?: string; verb: string; source: string; /** The command with its typed text hidden, for when the page turns out to be a password field. */ masked?: string; run(acc: Acc): Promise<void>;}
/** What a command collects while it runs. */interface Acc { retries: number; warnings: Warning[]; repairs: RepairRecord[]; screenshot?: string; /** The field typed into is a password input: what was typed is not written down. */ secret?: boolean;}
/** What a failed step was about, to draw around it on the picture: the candidate that could not be trusted, the matches that could not be told apart, or the element an action failed on. */function outlineItems(failure: Failure, resolvedXpath: string | undefined): Array<{ xpath: string; label: string; tone: "wanted" | "candidate" }> { const d = failure.detail; const xpathOf = (v: unknown): string | undefined => (typeof v === "object" && v !== null && typeof (v as { xpath?: unknown }).xpath === "string" ? (v as { xpath: string }).xpath : undefined); if (failure.reason === "drifted") { const xpath = xpathOf(d.candidate); return xpath === undefined ? [] : [{ xpath, label: `possible match${typeof d.score === "number" ? ` ${d.score.toFixed(2)}` : ""}`, tone: "candidate" }]; } if (failure.reason === "ambiguous" && Array.isArray(d.candidates)) { return d.candidates.flatMap((c: unknown, i: number) => { const scored = typeof c === "object" && c !== null && "descriptor" in c; const xpath = xpathOf(scored ? (c as { descriptor: unknown }).descriptor : c); const score = scored ? (c as { score?: unknown }).score : undefined; return xpath === undefined ? [] : [{ xpath, label: `${i + 1}${typeof score === "number" ? ` (${score.toFixed(2)})` : ""}`, tone: "candidate" as const }]; }); } const xpath = xpathOf(d.descriptor) ?? resolvedXpath; return xpath === undefined ? [] : [{ xpath, label: "wanted", tone: "wanted" }];}
/** The repair an answer reports, or nothing when it does not say enough to record. */function repairOf(answer: Record<string, unknown>, source: string, now: ElementDescriptor): RepairRecord | undefined { const r = answer.repair; if (typeof r !== "object" || r === null) return undefined; const d = r as Record<string, unknown>; if (!isElementDescriptor(d.old) || typeof d.score !== "number") return undefined; const strings = (v: unknown): string[] => (Array.isArray(v) ? v.filter((x): x is string => typeof x === "string") : []); return { via: answer.via === "descriptor" ? "descriptor" : "repair", source, score: d.score, runnerUp: typeof d.runnerUp === "number" ? d.runnerUp : 0, changed: strings(d.changed), changes: strings(d.changes), old: d.old, now };}
/** * Runs a parsed script against the extension, one command at a time. The first failing command stops the test: the rest * of its step and every later step are reported as skipped, so a run never carries on from a state it did not expect. * The engine never runs a failed command again. Retries are only what the extension or the bridge report having done * (a page command sent again after the page navigated, or after a reconnect), and they are counted, never hidden. */export async function runScript(script: Script, rawDeps: EngineDeps): Promise<RunResult> { const signal = rawDeps.signal; const abortedNow = new Promise<never>((_, reject) => { if (signal === undefined) return; if (signal.aborted) reject(new RunAborted()); else signal.addEventListener("abort", () => reject(new RunAborted()), { once: true }); }); abortedNow.catch(() => undefined); // it is raced where it matters; an abort nobody is waiting on is not an unhandled rejection const stopIfAborted = (): void => { if (signal?.aborted === true) throw new RunAborted(); }; /** Every request and every pause goes through here: none starts after an abort, and none that is waiting keeps the run waiting. */ const deps: EngineDeps = { ...rawDeps, request: async (body, opts) => { stopIfAborted(); const answer = await Promise.race([rawDeps.request(body, opts), abortedNow]); stopIfAborted(); return answer; }, sleep: async (ms) => { stopIfAborted(); await Promise.race([rawDeps.sleep(ms), abortedNow]); stopIfAborted(); }, }; stopIfAborted(); const started = deps.now(); const defaultTimeout = script.timeoutMs ?? DEFAULT_TIMEOUT_MS; /** * How long a page load may take. The script's `timeout` is how long to wait for elements and text, and scripts keep it short to fail fast; how fast a * page loads is another matter (a loaded machine took more than 700 ms to open a page and failed a check that was about something else). So a load * gets at least the default 10 s, more if the script asks for more, and exactly what the command says when the command has its own `timeout:`. */ const loadTimeout = (own: number | undefined): number => own ?? Math.max(defaultTimeout, DEFAULT_TIMEOUT_MS); /** * Which profile the command being run is on. Each profile has a Chrome of its own, so tabs are numbered per profile: for each, the tab commands act on * now and the tab `start` opened (switchTab and closeTab move the first; `start` goes back to the second). */ let active: string | undefined; const tabStates = new Map<string, { tab?: number | undefined; testTab?: number | undefined }>(); const st = () => { const key = active ?? ""; let s = tabStates.get(key); if (s === undefined) tabStates.set(key, (s = {})); return s; }; /** True when the script uses more than one profile: only then do requests and results say which profile they are for. */ const multi = script.alsoNeeds !== undefined && script.alsoNeeds.length > 0; const which = (): { profile: string } | Record<string, never> => (multi && active !== undefined ? { profile: active } : {}); const steps: StepResult[] = []; const vars = new Map<string, string>(); /** The name of the step being run, which with the target as written is the lock file's key. */ let stepName = ""; /** The last element a command of this step found, for the outline on the step's picture if the step fails. */ let resolvedXpath: string | undefined;
const send = async (body: { cmd: string; [key: string]: unknown }, timeoutMs: number, who: Omit<Failure, "reason" | "detail">, acc: { retries: number }, options: { newTab?: true; tab?: number } = {}): Promise<Record<string, unknown>> => { const useTab = options.tab ?? (options.newTab ? undefined : st().tab); const answer = (await deps.request({ ...body, ...which(), ...(useTab !== undefined ? { tab: useTab } : {}), timeoutMs }, { timeoutMs: timeoutMs + BRIDGE_SLACK_MS })) as Record<string, unknown>; acc.retries += retriesOf(answer); if (answer.ok === true) return answer; const reason = typeof answer.reason === "string" ? answer.reason : "internal"; const detail = typeof answer.detail === "object" && answer.detail !== null ? (answer.detail as Record<string, unknown>) : {}; throw new StepFailed({ ...who, reason, detail: answer.reason === undefined ? { message: "the extension gave an answer with no result" } : detail }); };
/** Replaces ${NAME} with the variable's value. A name that is not set cannot come from a parsed script (E007), so it is an error of the run. */ const fill = (text: string, who: Omit<Failure, "reason" | "detail">): string => text.replace(/\$\{([A-Za-z_][A-Za-z0-9_]*)\}/g, (_, name: string) => { const value = vars.get(name); if (value === undefined) throw new StepFailed({ ...who, reason: "internal", detail: { message: `the variable "${name}" is not set` } }); return value; });
const fillTarget = (t: Target, who: Omit<Failure, "reason" | "detail">): Target => Object.fromEntries(Object.entries(t).map(([k, v]) => [k, fill(v, who)]));
/** Finds the one element a target names, as a descriptor to act on. With a remembered descriptor the page may find it by repair; that is recorded. */ const resolve = async (target: Target, timeoutMs: number, who: Omit<Failure, "reason" | "detail">, acc: Acc): Promise<unknown> => { // the same words on another profile's page can be another element: its own entry const source = describeTarget(target) + (multi && active !== undefined && active !== script.needsSession ? ` on:${active}` : ""); const stored = deps.lock?.find(stepName, source); if (deps.freeze && stored === undefined) { throw new StepFailed({ ...who, reason: "unlocked", detail: { message: `no remembered description of ${source} in the lock file: run once interactively first (aite run without --freeze), look at the result, and commit the lock file`, target: source } }); } const resolved = await send({ cmd: "resolve", target: fillTarget(target, who), ...(stored !== undefined ? { descriptor: stored } : {}) }, timeoutMs, who, acc); if (typeof resolved.descriptor !== "object" || resolved.descriptor === null) throw new StepFailed({ ...who, reason: "internal", detail: { message: "the page did not describe the element it found" } }); const found = resolved.descriptor; if (typeof (found as { xpath?: unknown }).xpath === "string") resolvedXpath = (found as { xpath: string }).xpath; if (isElementDescriptor(found)) { if (resolved.repaired === true) { const r = repairOf(resolved, source, found); if (r) { acc.repairs.push(r); deps.lock?.repair(stepName, source, found); } } else deps.lock?.capture(stepName, source, found); } return found; };
const navigate = async (url: string, timeoutMs: number, who: Omit<Failure, "reason" | "detail">, acc: Acc, own: boolean): Promise<void> => { // `start` goes to the tab it opened before, if that is still the run's; otherwise a new one. `goto` moves the tab the run is on. const options = own ? (st().testTab !== undefined ? { tab: st().testTab as number } : { newTab: true as const }) : {}; const answer = await send({ cmd: "navigate", url }, timeoutMs, who, acc, options); if (typeof answer.tab === "number") { st().tab = answer.tab; if (own || st().testTab === undefined) st().testTab = answer.tab; } };
/** Saves a picture of the page, or says why not. */ const snap = async (path: string): Promise<Warning | undefined> => { const warn = (message: string): Warning => ({ code: "screenshot", message: `could not take a screenshot: ${message}` }); if (!deps.saveScreenshot) return warn("this run has nowhere to save pictures"); try { const answer = (await deps.request({ cmd: "screenshot", ...which(), ...(st().tab !== undefined ? { tab: st().tab as number } : {}), timeoutMs: SHOT_TIMEOUT_MS }, { timeoutMs: SHOT_TIMEOUT_MS + BRIDGE_SLACK_MS })) as Record<string, unknown>; if (answer.ok !== true) { const detail = (typeof answer.detail === "object" && answer.detail !== null ? answer.detail : {}) as Record<string, unknown>; return warn(String(detail.message ?? detail.why ?? answer.reason ?? "no reason given")); } if (typeof answer.png !== "string") return warn("the answer had no picture"); await deps.saveScreenshot(path, answer.png); return undefined; } catch (err) { return warn(err instanceof Error ? err.message : String(err)); } };
const itemOf = (c: Command): Item => { const profile = c.on ?? script.needsSession; const source = describeCommand(c); const who = { line: c.line, verb: c.verb, command: source, ...(c.file !== undefined ? { file: c.file } : {}) }; const timeoutMs = c.timeoutMs ?? defaultTimeout; const base = { line: c.line, verb: c.verb, source, ...(c.file !== undefined ? { file: c.file } : {}), ...(multi && profile !== undefined ? { profile } : {}) }; const within = "within" in c && c.within !== undefined ? { within: fill(c.within, who) } : {}; switch (c.verb) { case "start": return { ...base, run: (acc) => navigate(fill(c.url, who), loadTimeout(c.timeoutMs), who, acc, true) }; case "goto": return { ...base, run: (acc) => navigate(fill(c.url, who), loadTimeout(c.timeoutMs), who, acc, false) }; case "click": case "doubleClick": case "check": case "uncheck": return { ...base, run: async (acc) => { const descriptor = await resolve(c.target, timeoutMs, who, acc); await send({ cmd: "act", action: c.verb, descriptor }, timeoutMs, who, acc); }, }; case "type": return { ...base, masked: printCommand({ ...c, text: MASK }), run: async (acc) => { const descriptor = await resolve(c.target, timeoutMs, who, acc); if (isElementDescriptor(descriptor) && descriptor.attrs.type === "password") acc.secret = true; await send({ cmd: "act", action: "type", descriptor, text: fill(c.text, who) }, timeoutMs, who, acc); }, }; case "select": return { ...base, run: async (acc) => { const descriptor = await resolve(c.target, timeoutMs, who, acc); await send({ cmd: "act", action: "select", descriptor, option: fill(c.option, who) }, timeoutMs, who, acc); }, }; case "press": return { ...base, run: async (acc) => void (await send({ cmd: "act", action: "press", key: c.key }, timeoutMs, who, acc)) }; case "drag": return { ...base, run: async (acc) => { const descriptor = await resolve(c.target, timeoutMs, who, acc); const to = await resolve(c.to, timeoutMs, who, acc); await send({ cmd: "act", action: "drag", descriptor, to }, timeoutMs, who, acc); }, }; case "capture": return { ...base, run: async (acc) => { const read = await send({ cmd: "read", target: fillTarget(c.target, who) }, timeoutMs, who, acc); vars.set(c.name, typeof read.text === "string" ? read.text : ""); }, }; case "set": return { ...base, run: async () => void vars.set(c.name, fill(c.value, who)) }; case "sleep": return { ...base, run: async (acc) => { acc.warnings.push({ code: "sleep", line: c.line, ...(c.file !== undefined ? { file: c.file } : {}), message: `sleep pauses the run for ${c.ms} ms whether or not the page is ready; wait for a condition instead if you can` }); await deps.sleep(c.ms); }, }; case "verify": case "verifyNot": return { ...base, run: async (acc) => { const negate = c.verb === "verifyNot" ? { negate: true } : {}; let fields: Record<string, unknown>; switch (c.kind) { case "exists": fields = { kind: "exists", target: fillTarget(c.target, who) }; break; case "text": fields = { kind: "text", value: fill(c.value, who), ...(c.match !== undefined ? { match: c.match } : {}) }; break; case "targetText": fields = { kind: "targetText", target: fillTarget(c.target, who), value: fill(c.value, who), ...(c.match !== undefined ? { match: c.match } : {}) }; break; case "value": fields = { kind: "value", target: fillTarget(c.target, who), value: fill(c.value, who) }; break; case "state": fields = { kind: "state", target: fillTarget(c.target, who), state: c.state }; break; case "count": fields = { kind: "count", target: fillTarget(c.target, who), count: c.count }; break; case "url": fields = { kind: "url", contains: fill(c.contains, who) }; break; case "title": fields = { kind: "title", value: fill(c.value, who) }; break; case "attr": fields = { kind: "attr", target: fillTarget(c.target, who), attr: c.attr, op: c.op, value: fill(c.value, who) }; break; case "download": fields = { kind: "download", name: fill(c.name, who), ...(c.nameMatch !== undefined ? { nameMatch: c.nameMatch } : {}), ...(c.sizeMin !== undefined ? { sizeMin: c.sizeMin } : {}) }; break; default: { const from = vars.get(c.from); if (from === undefined) throw new StepFailed({ ...who, reason: "internal", detail: { message: `the variable "${c.from}" is not set (capture or set it first)` } }); fields = { kind: "numberChange", target: fillTarget(c.target, who), from, equals: c.equals }; } } await send({ cmd: "verify", ...fields, ...within, ...negate }, timeoutMs, who, acc); }, }; case "wait": return { ...base, run: async (acc) => { const fields: Record<string, unknown> = c.kind === "url" ? { kind: "url", contains: fill(c.contains, who) } : c.kind === "networkIdle" ? { kind: "networkIdle" } : { kind: c.kind, target: fillTarget(c.target, who) }; await send({ cmd: "wait", ...fields, ...within }, timeoutMs, who, acc); }, }; case "switchTab": return { ...base, run: async (acc) => { const match = c.tab.kind === "new" ? { new: true } : { urlContains: fill(c.tab.contains, who) }; const answer = await send({ cmd: "switchTab", match }, timeoutMs, who, acc); if (typeof answer.tab === "number") st().tab = answer.tab; }, }; case "closeTab": return { ...base, run: async (acc) => { const answer = await send({ cmd: "closeTab" }, timeoutMs, who, acc); if (answer.closed === st().testTab) st().testTab = undefined; st().tab = typeof answer.tab === "number" ? answer.tab : undefined; if (answer.alreadyClosed === true) acc.warnings.push({ code: "tab-already-closed", line: c.line, ...(c.file !== undefined ? { file: c.file } : {}), message: "the tab had already closed itself, so there was nothing to close" }); }, }; case "screenshot": return { ...base, run: async (acc) => { const slug = fill(c.name, who).toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 40) || "shot"; const path = `shots/${pad(steps.length, 3)}-${slug}.png`; const warning = await snap(path); if (warning) acc.warnings.push({ ...warning, line: c.line, ...(c.file !== undefined ? { file: c.file } : {}) }); else acc.screenshot = path; }, }; case "call": return { ...base, run: async () => { if (!deps.runCheck) throw new StepFailed({ ...who, reason: "internal", detail: { message: "this run cannot run checks (scripts/checks)" } }); const outcome = await deps.runCheck({ name: c.name, args: c.args.map((a) => fill(a, who)), vars: Object.fromEntries(vars), timeoutMs, tab: st().tab, ...(multi && active !== undefined ? { profile: active } : {}) }); if (!outcome.ok) throw new StepFailed({ ...who, reason: outcome.reason, detail: { message: outcome.message, ...(outcome.expected !== undefined ? { expected: outcome.expected } : {}), ...(outcome.found !== undefined ? { found: outcome.found } : {}) } }); }, }; case "run": return { ...base, run: async () => { throw new StepFailed({ ...who, reason: "internal", detail: { message: "run was not expanded before the run started" } }); } }; } };
/** A picture of the page after a step. Failing to take one is a warning, never a failed step. */ const takeScreenshot = async (index: number, commandsRun: number, outline?: Array<{ xpath: string; label: string; tone: string }>): Promise<Shot & { warnings?: Warning[] }> => { const path = `shots/${pad(index, 3)}-${pad(commandsRun, 2)}.png`; const extra: Warning[] = []; const ask = async (body: { cmd: string; [key: string]: unknown }): Promise<Record<string, unknown> | undefined> => { try { return (await deps.request({ ...body, ...which(), ...(st().tab !== undefined ? { tab: st().tab as number } : {}), timeoutMs: SHOT_TIMEOUT_MS }, { timeoutMs: SHOT_TIMEOUT_MS + BRIDGE_SLACK_MS })) as Record<string, unknown>; } catch (err) { extra.push({ code: "outline", message: `could not draw or remove the outline: ${err instanceof Error ? err.message : String(err)}` }); return undefined; } }; const drawing = outline !== undefined && outline.length > 0; if (drawing) { const drawn = await ask({ cmd: "outline", items: outline }); if (drawn !== undefined && drawn.ok !== true) { const detail = (typeof drawn.detail === "object" && drawn.detail !== null ? drawn.detail : {}) as Record<string, unknown>; extra.push({ code: "outline", message: `could not draw the outline: ${String(detail.message ?? drawn.reason ?? "no reason given")}` }); } } let warning: Warning | undefined; try { warning = await snap(path); } finally { if (drawing) { const cleared = await ask({ cmd: "outline", clear: true }); if (cleared !== undefined && typeof cleared.remaining === "number" && cleared.remaining > 0) extra.push({ code: "outline", message: "an outline was left on the page after it was cleared" }); } } return { ...(warning ? { warning } : { screenshot: path }), ...(extra.length > 0 ? { warnings: extra } : {}) }; };
const skipped = (item: { line: number; file?: string; verb: string; source: string; profile?: string | undefined }): CommandResult => { const { profile, ...rest } = item; return { ...rest, ...(profile !== undefined ? { profile } : {}), status: "skipped", retries: 0, durationMs: 0, warnings: [] }; };
/** Runs the items in order and stops at the first failure; the rest are skipped. Returns whether the step passed. */ const runStep = async (name: string, line: number, items: Item[]): Promise<boolean> => { const t0 = deps.now(); const index = steps.length; stepName = name; resolvedXpath = undefined; const commands: CommandResult[] = []; let failure: Failure | undefined; let lastProfile: string | undefined = items[0]?.profile; for (const item of items) { stopIfAborted(); if (failure) { commands.push(skipped(item)); continue; } const acc: Acc = { retries: 0, warnings: [], repairs: [] }; const c0 = deps.now(); active = item.profile; lastProfile = item.profile; const result = { line: item.line, verb: item.verb, source: item.source, ...(item.file !== undefined ? { file: item.file } : {}), ...(item.profile !== undefined ? { profile: item.profile } : {}) }; try { await item.run(acc); if (acc.secret && item.masked !== undefined) result.source = item.masked; commands.push({ ...result, status: acc.repairs.length > 0 ? "repaired" : acc.retries > 0 ? "passed-after-retry" : "passed", retries: acc.retries, durationMs: deps.now() - c0, warnings: acc.warnings, ...(acc.repairs.length > 0 ? { repairs: acc.repairs } : {}), ...(acc.screenshot !== undefined ? { screenshot: acc.screenshot } : {}) }); } catch (err) { if (!(err instanceof StepFailed)) throw err; failure = acc.secret && item.masked !== undefined ? { ...err.failure, command: item.masked } : err.failure; if (acc.secret && item.masked !== undefined) result.source = item.masked; const d = clampDetail(failure.detail); const endedAt = deps.now(); const failed: CommandResult = { ...result, status: "failed", retries: acc.retries, durationMs: endedAt - c0, warnings: acc.warnings, reason: failure.reason, detail: d, ...(d.expected !== undefined ? { expected: d.expected } : {}), ...(d.found !== undefined ? { found: d.found } : {}), }; if (deps.diagnose) { try { failed.diagnosis = await deps.diagnose({ ...(item.profile !== undefined ? { profile: item.profile } : {}), verb: item.verb, reason: failure.reason, line: item.line, ...(item.file !== undefined ? { file: item.file } : {}), detail: d, startedAt: c0, endedAt, stepIndex: index, commandIndex: commands.length }); } catch (e) { failed.warnings.push({ code: "diagnosis", line: item.line, message: `could not explain the failure: ${e instanceof Error ? e.message : String(e)}` }); } } commands.push(failed); } } const durationMs = deps.now() - t0; const ran = commands.filter((c) => c.status !== "skipped").length; const warnings: Warning[] = [...commands.flatMap((c) => c.warnings), ...(items.length === 0 ? [NO_COMMANDS] : [])]; active = lastProfile; // the picture is of the profile that did the last thing in the step const shot: Shot & { warnings?: Warning[] } = deps.saveScreenshot && (deps.stepScreenshots !== "failed" || failure) ? await takeScreenshot(index, ran, failure ? outlineItems(failure, resolvedXpath) : undefined) : {}; if (shot.warning) warnings.push(shot.warning); if (shot.warnings) warnings.push(...shot.warnings); const retries = commands.reduce((n, c) => n + c.retries, 0); const status = failure ? "failed" : commands.some((c) => c.status === "repaired") ? "repaired" : retries > 0 ? "passed-after-retry" : "passed"; steps.push({ name, line, status, durationMs, retries, commands, warnings, ...(failure ? { failure } : {}), ...(shot.screenshot !== undefined ? { screenshot: shot.screenshot } : {}) }); return failure === undefined; };
let ok = true; const skipStep = (name: string, line: number, items: Item[]) => steps.push({ name, line, status: "skipped", durationMs: 0, retries: 0, commands: items.map(skipped), warnings: [] });
if (script.start !== undefined) { const url = script.start; const startLine = script.startLine ?? 0; const who = { line: startLine, verb: "start", command: `start ${url}` }; ok = await runStep(`open ${url}`, startLine, [{ ...(multi && script.needsSession !== undefined ? { profile: script.needsSession } : {}), line: startLine, verb: "start", source: who.command, run: async (acc) => { await navigate(url, loadTimeout(undefined), who, acc, true); const verdict = await deps.afterStart?.(st().tab); if (verdict !== undefined && !verdict.ok) throw new StepFailed({ ...who, reason: "wrong-session", detail: { message: verdict.message, expected: verdict.expected, found: verdict.found } }); } }]); } for (const step of script.steps) { const items = step.commands.map(itemOf); if (!ok) skipStep(step.name, step.line, items); else ok = await runStep(step.name, step.line, items); }
const hasVerify = script.steps.some((s) => s.commands.some((c) => c.verb === "verify")); const warnings = hasVerify ? [] : [NO_VERIFY]; const retries = steps.reduce((n, s) => n + s.retries, 0); const warningCount = warnings.length + steps.reduce((n, s) => n + s.warnings.length, 0); const repairs = steps.reduce((n, st) => n + st.commands.reduce((m, c) => m + (c.repairs?.length ?? 0), 0), 0); return { name: script.name, ok, status: ok ? "passed" : "failed", steps, durationMs: deps.now() - started, retries, warnings, warningCount, repairs, ...(deps.lock ? { lock: deps.lock.summary() } : {}), ...(multi ? { profiles: [script.needsSession as string, ...(script.alsoNeeds ?? []).map((n) => n.name)] } : {}) };}
/** Escapes anything outside printable ASCII so a report is safe in any terminal or log. */const ascii = (s: string): string => s.replace(/[^\x20-\x7e]/g, (c) => `\\u${c.charCodeAt(0).toString(16).padStart(4, "0")}`);const quote = (s: unknown): string => ascii(JSON.stringify(String(s)));
/** What the script saw when a command failed, in lines, for the report and run.json. */export const explainFailure = (reason: string, detail: Record<string, unknown>): string[] => explain({ line: 0, verb: "", command: "", reason, detail });
function explain(f: Failure): string[] { const d = f.detail; switch (f.reason) { case "mismatch": { const lines = [...(typeof d.message === "string" ? [ascii(d.message).slice(0, 300)] : []), `expected ${quote(d.expected)}`, `found ${quote(d.found ?? "")}`]; const nearby = Array.isArray(d.nearby) ? d.nearby : []; if (nearby.length > 0) lines.push(`also on the page: ${nearby.map(quote).join(", ")}`); if (typeof d.resolved === "string") lines.push(`resolved to ${quote(d.resolved)}`); if (typeof d.why === "string") lines.push(ascii(d.why).slice(0, 300)); return lines; } case "not-found": return [ `no element matched ${typeof d.target === "object" && d.target !== null ? ascii(describeTarget(d.target as Record<string, string>)) : "the target"}${typeof d.waitedMs === "number" ? ` after ${d.waitedMs} ms` : ""}`, ...(d.repairTried === true && typeof d.bestScore === "number" ? [`looked for something like it on the page: best score ${d.bestScore.toFixed(2)} (needs ${DRIFTED_MIN.toFixed(2)})`] : []), ...(typeof d.note === "string" ? [ascii(d.note)] : []), ]; case "drifted": { const c = d.candidate as Record<string, unknown> | undefined; const words = c !== undefined ? String(c.text || c.name || "") : ""; return [ `possible match, please confirm: <${ascii(String(c?.tag ?? "?"))}> ${quote(words)} (score ${typeof d.score === "number" ? d.score.toFixed(2) : "?"}, needs ${REPAIRED_MIN.toFixed(2)})`, ...(Array.isArray(d.changes) ? (d.changes as unknown[]).map((x) => ` ${ascii(String(x))}`) : []), ...(typeof d.note === "string" ? [ascii(d.note)] : []), ]; } case "ambiguous": { const c = Array.isArray(d.candidates) ? (d.candidates as Array<Record<string, unknown>>) : []; return [ `${c.length} elements matched; make the target more specific or add index:`, ...c.map((x) => { const inner = typeof x.descriptor === "object" && x.descriptor !== null ? (x.descriptor as Record<string, unknown>) : x; return ` ${ascii(String(inner.xpath ?? "?"))} ${quote(inner.text ?? "")}${typeof x.score === "number" ? ` (score ${x.score.toFixed(2)})` : ""}`; }), ]; } case "not-visible": return [ typeof d.coveredBy === "string" ? `another element is on top of it: ${ascii(d.coveredBy)}` : typeof d.why === "string" ? ascii(d.why) : "the element is on the page but not visible", ]; case "not-enabled": return ["the element is disabled"]; case "timeout": return [ `timed out${typeof d.waitedMs === "number" ? ` after ${d.waitedMs} ms` : ""}${typeof d.url === "string" ? ` loading ${ascii(d.url)}` : ""}`, ...(typeof d.inflight === "number" ? [`${d.inflight} request${d.inflight === 1 ? " was" : "s were"} still open`] : []), ...(typeof d.stillThere === "number" && d.stillThere > 0 ? [`${d.stillThere} matching element${d.stillThere === 1 ? " is" : "s are"} still there`] : []), ...(typeof d.why === "string" ? [ascii(d.why)] : []), ]; default: { const text = typeof d.why === "string" ? d.why : typeof d.message === "string" ? d.message : Object.keys(d).length > 0 ? JSON.stringify(d) : ""; return text ? [ascii(text).slice(0, 300)] : []; } }}
export function formatReport(result: RunResult, file: string): string { const out = [`test ${quote(result.name)} (${ascii(file)})`]; const where = (line: number | undefined, from?: string) => (line !== undefined && line > 0 ? `${ascii(from ?? file)}:${line}: ` : ""); for (const s of result.steps) { const tag = s.status === "passed" ? "ok " : s.status === "passed-after-retry" ? "retry" : s.status === "repaired" ? "repair" : s.status === "failed" ? "FAIL " : "skip "; out.push(` ${tag} ${ascii(s.name)}${s.status === "skipped" ? "" : ` (${s.durationMs} ms)`}`); if (s.status === "passed-after-retry") out.push(` passed only after ${s.retries} retr${s.retries === 1 ? "y" : "ies"}`); if (result.profiles !== undefined) { const used = [...new Set(s.commands.flatMap((c) => (c.profile !== undefined && c.status !== "skipped" ? [c.profile] : [])))]; for (const profile of used) out.push(` as ${ascii(profile)}: ${s.commands.filter((c) => c.profile === profile && c.status !== "skipped").map((c) => c.verb).join(", ")}`); } for (const r of s.commands.flatMap((c) => c.repairs ?? [])) { const kinds = r.changed.length > 0 ? ` (${r.changed.join(", ")} changed)` : ""; const how = r.via === "descriptor" ? "found by its remembered description" : `score ${r.score.toFixed(2)}, next best ${r.runnerUp.toFixed(2)}`; out.push(` locator repaired${kinds}: ${ascii(r.source)} (${how})`); for (const line of r.changes) out.push(` ${ascii(line)}`); } if (s.failure) { const f = s.failure; const failedOn = s.commands.find((c) => c.status === "failed")?.profile; out.push(` ${ascii(f.file ?? file)}${f.line > 0 ? `:${f.line}` : ""}: ${ascii(f.command)} failed (${f.reason})${failedOn !== undefined ? ` as ${ascii(failedOn)}` : ""}`); const diagnosis = s.commands.find((c) => c.status === "failed")?.diagnosis; if (diagnosis) { for (const line of formatDiagnosis(diagnosis)) out.push(` ${ascii(line)}`); out.push(" what the script saw:"); for (const line of explain(f)) out.push(` ${line}`); } else for (const line of explain(f)) out.push(` ${line}`); } if (s.screenshot) out.push(` screenshot: ${s.screenshot}`); for (const w of s.warnings) out.push(` warning: ${where(w.line, w.file)}${ascii(w.message)}`); } for (const w of result.warnings) out.push(` warning: ${ascii(w.message)}`); const count = (...status: StepResult["status"][]) => result.steps.filter((s) => status.includes(s.status)).length; const extras = [result.retries > 0 ? `${count("passed-after-retry")} passed after a retry` : "", result.repairs > 0 ? `${result.repairs} repaired` : "", result.warningCount > 0 ? `${result.warningCount} warning${result.warningCount === 1 ? "" : "s"}` : ""].filter(Boolean); out.push(`${count("passed", "passed-after-retry", "repaired")} passed, ${count("failed")} failed, ${count("skipped")} skipped${extras.length > 0 ? `, ${extras.join(", ")}` : ""} (${(result.durationMs / 1000).toFixed(1)} s)`); return out.join("\n");}