Something went wrong. Try again.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175import { describe, expect, it } from "vitest";import { checkIdentity, judge, parseExpectation, runSelfCheck, type Outcome, type SelfCheckScript } from "../../tools/selfCheck";
const okJson = { chrome: "155.0.8059.39", extension: "0.0.0", tests: [{ status: "passed", steps: [{ commands: [{ command: "click", status: "passed" }] }] }] };const failJson = (reason: string, command = "click", file?: string) => ({ chrome: "155.0.8059.39", extension: "0.0.0", tests: [{ status: "failed", steps: [{ commands: [{ command: "verify", status: "passed" }, { command, status: "failed", reason, ...(file ? { file } : {}) }, { command: "click", status: "skipped" }] }] }],});
describe("parseExpectation", () => { it("reads pass, fail with reason and command, fail in a file, and refused with a fragment", () => { expect(parseExpectation("# expect: pass\ntest ...")).toEqual({ kind: "pass" }); expect(parseExpectation("# expect: fail not-visible click\n")).toEqual({ kind: "fail", reason: "not-visible", verb: "click" }); expect(parseExpectation("# expect: fail not-found click in shared/broken.aite\n")).toEqual({ kind: "fail", reason: "not-found", verb: "click", file: "shared/broken.aite" }); expect(parseExpectation('# expect: refused "unknown key"\n')).toEqual({ kind: "refused", fragment: "unknown key" }); }); it("finds the line anywhere in the first lines, and refuses a script that has none (a fixture must say what it should do)", () => { expect(parseExpectation("# a note\n# expect: pass\n")).toEqual({ kind: "pass" }); expect(parseExpectation('test "t"\n')).toEqual({ error: expect.stringContaining("no `# expect:` line") }); }); it.each(["# expect: maybe", "# expect: fail", "# expect: fail not-visible", "# expect: refused", "# expect: fail not-visible click banana", "# expect: pass extra words"])("%s is a mistake in the fixture", (line) => { expect(parseExpectation(line + "\n")).toEqual({ error: expect.stringContaining("expect") }); }); it("two expect lines are a mistake", () => { expect(parseExpectation("# expect: pass\n# expect: pass\n")).toEqual({ error: expect.stringContaining("more than one") }); });});
describe("judge", () => { it("pass: exit 0 and a passed test", () => { expect(judge({ kind: "pass" }, { exitCode: 0, output: "", json: okJson })).toEqual({ ok: true }); }); it("pass fails when the run failed, was refused, or has no run.json to show the extension was there", () => { expect(judge({ kind: "pass" }, { exitCode: 1, output: "x", json: failJson("mismatch") })).toMatchObject({ ok: false, why: expect.stringContaining("expected a pass") }); expect(judge({ kind: "pass" }, { exitCode: 2, output: "bad script" })).toMatchObject({ ok: false }); expect(judge({ kind: "pass" }, { exitCode: 0, output: "" })).toMatchObject({ ok: false, why: expect.stringContaining("run.json") }); }); it("pass does not accept exit 0 with a failed command inside (a report and an exit code that disagree)", () => { expect(judge({ kind: "pass" }, { exitCode: 0, output: "", json: failJson("mismatch") })).toMatchObject({ ok: false }); }); it("fail: exit 1 and the FIRST failed command has the expected reason and command", () => { expect(judge({ kind: "fail", reason: "not-visible", verb: "click" }, { exitCode: 1, output: "", json: failJson("not-visible") })).toEqual({ ok: true }); }); it.each([ ["the wrong reason", { kind: "fail" as const, reason: "not-found", verb: "click" }, failJson("not-visible")], ["the wrong command", { kind: "fail" as const, reason: "not-visible", verb: "type" }, failJson("not-visible")], ["the wrong file", { kind: "fail" as const, reason: "not-visible", verb: "click", file: "shared/x.aite" }, failJson("not-visible", "click", "shared/y.aite")], ])("fail is not satisfied by %s", (_n, expectation, json) => { const r = judge(expectation, { exitCode: 1, output: "", json }); expect(r).toMatchObject({ ok: false }); expect(r.ok === false && r.why).toContain("expected"); }); it("fail is satisfied by the right file", () => { expect(judge({ kind: "fail", reason: "not-found", verb: "click", file: "shared/broken.aite" }, { exitCode: 1, output: "", json: failJson("not-found", "click", "shared/broken.aite") })).toEqual({ ok: true }); }); it("fail is not satisfied by a pass, a refusal, or an environment failure", () => { const e = { kind: "fail" as const, reason: "mismatch", verb: "verify" }; expect(judge(e, { exitCode: 0, output: "", json: okJson })).toMatchObject({ ok: false }); expect(judge(e, { exitCode: 2, output: "x" })).toMatchObject({ ok: false }); expect(judge(e, { exitCode: 3, output: "x" })).toMatchObject({ ok: false }); }); it("refused: exit 2 and the output has the fragment", () => { expect(judge({ kind: "refused", fragment: "unknown key" }, { exitCode: 2, output: 't.aite:5: unknown key "Banana" (...)' })).toEqual({ ok: true }); expect(judge({ kind: "refused", fragment: "unknown key" }, { exitCode: 2, output: "something else" })).toMatchObject({ ok: false }); expect(judge({ kind: "refused", fragment: "unknown key" }, { exitCode: 1, output: "unknown key", json: failJson("mismatch") })).toMatchObject({ ok: false }); });});
const script = (name: string, expect = "pass"): SelfCheckScript => ({ name, text: `# expect: ${expect}\n` });const good: Outcome = { exitCode: 0, output: "", json: okJson };
describe("runSelfCheck", () => { const options = { repeat: 1, parallel: 2 }; it("runs every script the given number of times and passes when every run is as expected", async () => { const calls: string[] = []; const r = await runSelfCheck([script("a"), script("b")], { ...options, repeat: 3 }, { canary: async () => ({ chrome: "155.0", extension: "0.0.0" }), run: async (s) => (calls.push(s.name), good) }); expect(r.exitCode).toBe(0); expect(calls.sort()).toEqual(["a", "a", "a", "b", "b", "b"]); expect(r.scripts.map((s) => [s.name, s.ok, s.passedRuns, s.totalRuns])).toEqual([["a", true, 3, 3], ["b", true, 3, 3]]); }); it("exit 1 when a script does not behave as its fixture says, and the report names it and says why", async () => { const r = await runSelfCheck([script("a"), script("b", "fail mismatch verify")], options, { canary: async () => ({ chrome: "155.0", extension: "0.0.0" }), run: async () => good }); expect(r.exitCode).toBe(1); expect(r.report).toContain("b"); expect(r.report).toMatch(/expected fail mismatch verify/); expect(r.scripts.find((s) => s.name === "a")!.ok).toBe(true); }); it("a script that passes sometimes is flaky, which is a failure and is called flaky", async () => { let n = 0; const r = await runSelfCheck([script("a")], { ...options, repeat: 4 }, { canary: async () => ({ chrome: "155.0", extension: "0.0.0" }), run: async () => (++n % 4 === 0 ? { exitCode: 1, output: "", json: failJson("timeout", "wait") } : good) }); expect(r.exitCode).toBe(1); expect(r.scripts[0]).toMatchObject({ flaky: true }); expect(r.report).toContain("flaky (3 of 4)"); }); it("a script that always fails is wrong, not flaky", async () => { const r = await runSelfCheck([script("a")], { ...options, repeat: 3 }, { canary: async () => ({ chrome: "155.0", extension: "0.0.0" }), run: async () => ({ exitCode: 1, output: "", json: failJson("mismatch", "verify") }) }); expect(r.scripts[0]).toMatchObject({ flaky: false, ok: false }); }); it("never runs more at once than asked", async () => { let now = 0; let peak = 0; const r = await runSelfCheck(Array.from({ length: 9 }, (_, i) => script("s" + i)), { repeat: 1, parallel: 3 }, { canary: async () => ({ chrome: "155.0", extension: "0.0.0" }), run: async () => { peak = Math.max(peak, ++now); await new Promise((r) => setTimeout(r, 5)); now--; return good; }, }); expect(r.exitCode).toBe(0); expect(peak).toBe(3); }); it("LOUD: if the canary cannot show the extension is there, nothing else runs and the exit code is 3", async () => { const calls: string[] = []; const r = await runSelfCheck([script("a")], options, { canary: async () => { throw new Error("the extension did not connect within 40000 ms"); }, run: async (s) => (calls.push(s.name), good) }); expect(r.exitCode).toBe(3); expect(calls).toEqual([]); expect(r.report).toContain("SELF-CHECK COULD NOT RUN"); expect(r.report).toContain("the extension did not connect"); expect(r.report).not.toMatch(/skipped/i); }); it("LOUD: a run that ends with the environment failing (exit 3) stops the whole check with exit 3, not a pass", async () => { const r = await runSelfCheck([script("a"), script("b")], { repeat: 1, parallel: 1 }, { canary: async () => ({ chrome: "155.0", extension: "0.0.0" }), run: async () => ({ exitCode: 3, output: "Chrome was stopped by SIGKILL" }) }); expect(r.exitCode).toBe(3); expect(r.report).toContain("Chrome was stopped by SIGKILL"); }); it("a run whose report shows no extension (chrome/extension unknown) cannot count as a pass", async () => { const r = await runSelfCheck([script("a")], options, { canary: async () => ({ chrome: "155.0", extension: "0.0.0" }), run: async () => ({ exitCode: 0, output: "", json: { ...okJson, extension: "unknown" } }) }); expect(r.exitCode).toBe(1); expect(r.report).toContain("extension"); }); it("a fixture with a mistake in its expectation fails the check and says which file", async () => { const r = await runSelfCheck([{ name: "bad.fail", text: "# expect: banana\n" }], options, { canary: async () => ({ chrome: "155.0", extension: "0.0.0" }), run: async () => good }); expect(r.exitCode).toBe(1); expect(r.report).toContain("bad.fail"); }); it("a script that throws is a failure of that script, with the message, and the others still run", async () => { const r = await runSelfCheck([script("a"), script("b")], { repeat: 1, parallel: 1 }, { canary: async () => ({ chrome: "155.0", extension: "0.0.0" }), run: async (s) => { if (s.name === "a") throw new Error("boom"); return good; } }); expect(r.exitCode).toBe(1); expect(r.report).toContain("boom"); expect(r.scripts.find((s) => s.name === "b")!.ok).toBe(true); }); it("no scripts at all is a failure (an empty check proves nothing)", async () => { const r = await runSelfCheck([], options, { canary: async () => ({ chrome: "155.0", extension: "0.0.0" }), run: async () => good }); expect(r.exitCode).toBe(1); expect(r.report).toContain("no fixture scripts"); }); it("the report says how many scripts, how many runs, and which Chrome and extension it ran against", async () => { const r = await runSelfCheck([script("a")], { repeat: 2, parallel: 1 }, { canary: async () => ({ chrome: "155.0.8059.39", extension: "0.0.0" }), run: async () => good }); expect(r.report).toContain("1 script"); expect(r.report).toContain("2 runs"); expect(r.report).toContain("Chrome 155.0.8059.39"); expect(r.report).toContain("extension 0.0.0"); });});
describe("checkIdentity", () => { const expected = { chromePinned: "155.0.8059.39", extensionVersion: "0.0.0" }; it("accepts the pinned major (Chrome reports 155.0.0.0) and this build's extension", () => { expect(checkIdentity({ chrome: "155.0.0.0", extension: "0.0.0" }, expected)).toBeUndefined(); expect(checkIdentity({ chrome: "155.0.8059.39", extension: "0.0.0" }, expected)).toBeUndefined(); }); it.each([ [{ chrome: "154.0.0.0", extension: "0.0.0" }, "154.0.0.0, but 155.0.8059.39 is pinned"], [{ chrome: "155.0.0.0", extension: "9.9.9" }, "9.9.9, but this build is 0.0.0"], [{ extension: "0.0.0" }, "names no Chrome"], [{ chrome: "unknown", extension: "0.0.0" }, "names no Chrome"], [{ chrome: "155.0.0.0" }, "names no extension"], [{ chrome: "155.0.0.0", extension: "unknown" }, "names no extension"], ])("%j is refused: %s", (seen, why) => { expect(checkIdentity(seen, expected)).toContain(why); });});