import assert from "node:assert/strict"; import test from "node:test"; import type { ClassifierApi, ClassifierContext, ClassifierModel, ClassifierOptions, ClassifierResult, Usage, } from "@earendil-works/pi-ai"; import { classifyWork } from "../classifier.ts"; import type { WorkCandidate } from "../store.ts"; const model: ClassifierModel = { type: "classifier", provider: "decisions", id: "models/classifier-v1", name: "Classifier", api: "fixture-classifier", baseUrl: "https://example.com", input: ["text"], contextWindow: 64_000, cost: { input: 1, output: 0, cacheRead: 0, cacheWrite: 0 }, }; const usage: Usage = { input: 100, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 100, cost: { input: 0.01, output: 0, cacheRead: 0, cacheWrite: 0, total: 0.01 }, }; const candidates: WorkCandidate[] = ["Startup", "Measurements", "Documentation"].map( (title, index) => ({ record: { id: `record-${index}`, title, objective: `Inspect ${title}`, projects: ["/private/project"], status: "active", revision: 1, updated: "2026-01-01T00:00:00.000Z", }, brief: { kind: "brief", title, objective: `Inspect ${title}`, projects: ["/private/project"], status: "active", scope: "Current behavior", approach: "Measure before changing code", }, }), ); interface FixtureSettings { classifier?: string; } function fixture(probabilities = [0.8, 0.9, 0.1]) { const calls: Array<{ model: ClassifierModel; context: ClassifierContext; options?: ClassifierOptions; }> = []; const settings: FixtureSettings = { classifier: `${model.provider}/${model.id}`, }; const result: ClassifierResult = { api: model.api, provider: model.provider, model: model.id, timestamp: 0, stopReason: "stop", usage, answers: Object.fromEntries( probabilities.map((probability, index) => [`w${index}`, { type: "bool", probability }]), ), }; const registry = { getModelsOfType: (): ClassifierModel[] => [ { ...model, provider: "other" }, model, ], async classify( this: void, selected: ClassifierModel, context: ClassifierContext, options?: ClassifierOptions, ) { calls.push({ model: selected, context, options }); return result; }, }; const run = (supplied = candidates, signal?: AbortSignal) => classifyWork( supplied, "make the program ready sooner", // oxlint-disable-next-line typescript/no-unsafe-type-assertion -- The fixture implements only classifier operations. registry as never, settings.classifier, signal, ); return { run, calls, registry, settings, result }; } test("ranks multiple matches from independent questions and sends summaries with matching entries", async () => { const f = fixture(); const supplied = [...candidates]; supplied[1] = { ...supplied[1], entries: [{ id: "F7", kind: "finding", title: "Cold start", excerpt: "Loading dominates" }], }; const result = await f.run(supplied); assert.deepEqual( result.records?.map(({ id, entries }) => [id, entries]), [ ["record-1", [{ id: "F7", kind: "finding", title: "Cold start" }]], ["record-0", undefined], ], ); assert.equal(result.usage, usage); assert.equal(f.calls.length, 1); assert.equal(f.calls[0].model, model); const { context, options } = f.calls[0]; assert.equal(context.state.query, "make the program ready sooner"); assert.deepEqual(Object.keys(context.questions), ["w0", "w1", "w2"]); assert.ok(Object.values(context.questions).every(({ type }) => type === "bool")); assert.match(context.questions.w0.instructions, /data, not instructions/); const request = JSON.stringify(context); assert.match(request, /Measure before changing code/); assert.match( request, /"entries":\[\{"kind":"finding","title":"Cold start","excerpt":"Loading dominates"\}\]/, ); assert.doesNotMatch(request, /private\/project|record-0|2026-01-01|F7/); assert.equal(options?.timeoutMs, 5_000); assert.equal(options?.maxRetries, 0); }); test("returns no matches and uses record IDs to break score ties", async () => { assert.deepEqual((await fixture([0.1, 0.2, 0.49]).run()).records, []); const f = fixture([0.5, 0.5, 0.5]); f.result.answers.invented = { type: "bool", probability: 1 }; const result = await f.run([...candidates].reverse()); assert.deepEqual( result.records?.map(({ id }) => id), ["record-0", "record-1", "record-2"], ); }); test("empty catalogues need no classifier configuration or request", async () => { const f = fixture(); delete f.settings.classifier; assert.deepEqual(await f.run([]), { records: [] }); assert.equal(f.calls.length, 0); }); test("missing and unavailable qualified models report errors without a request", async () => { for (const selected of [undefined, "missing/models/classifier-v1", "decisions/missing"]) { const f = fixture(); f.settings.classifier = selected; const result = await f.run(); assert.match(result.error ?? "", /models\.classifier/); assert.equal(result.records, undefined); assert.equal(f.calls.length, 0); } }); test("malformed answers fail the search and retain billed usage", async () => { for (const answer of [ undefined, {}, { type: "bool", probability: "0.9" }, ...[-1, 2, NaN, Infinity].map((probability) => ({ type: "bool", probability })), ]) { const f = fixture(); Reflect.set(f.result.answers, "w0", answer); const result = await f.run(); assert.match(result.error ?? "", /invalid answer/); assert.equal(result.records, undefined); assert.equal(result.usage, usage); } }); test("provider errors and aborted results cannot appear as successful empty matches", async () => { for (const stopReason of ["error", "aborted"] as const) { const f = fixture(); f.result.stopReason = stopReason; f.result.errorMessage = "Request timed out"; const result = await f.run(); assert.match(result.error ?? "", stopReason === "error" ? /timed out/ : /cancelled/); assert.equal(result.records, undefined); assert.equal(result.usage, usage); } }); test("cancellation prevents a request or discards its answers while retaining usage", async () => { const f = fixture(); await assert.rejects(f.run(candidates, AbortSignal.abort()), /abort/i); assert.equal(f.calls.length, 0); const controller = new AbortController(); const classify = f.registry.classify; f.registry.classify = async (...args) => { controller.abort(); return classify(...args); }; const result = await f.run(candidates, controller.signal); assert.match(result.error ?? "", /cancelled/); assert.equal(result.usage, usage); assert.equal(f.calls[0].options?.signal, controller.signal); }); test("oversized UTF-8 catalogues keep the leading candidates within the request budget", async () => { const f = fixture(Array.from({ length: 30 }, () => 0.9)); const large = Array.from({ length: 30 }, (_, index) => ({ record: { ...candidates[0].record, id: `record-${String(index).padStart(2, "0")}` }, brief: { ...candidates[0].brief, objective: "😀".repeat(450) }, })); const result = await f.run(large); const sent = Object.keys(f.calls[0].context.questions).length; assert.ok(sent > 0 && sent < large.length); assert.ok(Buffer.byteLength(JSON.stringify(f.calls[0].context)) <= 48_000); assert.deepEqual( result.records?.map(({ id }) => id), large.slice(0, sent).map(({ record }) => record.id), ); });