chadfowler.com
Reference implementation for the Phoenix Architecture. Work in progress.
aicoding.leaflet.pub/
ai
coding
crazy
1import { readFileSync } from 'node:fs';
2import { resolve } from 'node:path';
3import { parseSpec } from '../src/spec-parser.js';
4import { extractCanonicalNodes } from '../src/canonicalizer.js';
5import { GOLD_SPECS } from '../tests/eval/gold-standard.js';
6
7const ROOT = resolve(import.meta.dirname, '..');
8
9for (const s of GOLD_SPECS) {
10 const text = readFileSync(resolve(ROOT, s.path), 'utf8');
11 const clauses = parseSpec(text, s.docId);
12 const nodes = extractCanonicalNodes(clauses);
13 let found = 0, correct = 0;
14 const misses: string[] = [];
15 for (const g of s.expectedNodes) {
16 const n = nodes.find(n => n.statement.toLowerCase().includes(g.statement.toLowerCase()));
17 if (n) {
18 found++;
19 if (n.type === g.type) correct++;
20 else misses.push(` MISS "${g.statement}" gold=${g.type} got=${n.type}`);
21 } else {
22 misses.push(` GONE "${g.statement}"`);
23 }
24 }
25 if (misses.length > 0) {
26 console.log(`=== ${s.name} (${correct}/${found} correct) ===`);
27 misses.forEach(m => console.log(m));
28 }
29}