chadfowler.com
Reference implementation for the Phoenix Architecture. Work in progress.
aicoding.leaflet.pub/
ai
coding
crazy
1/**
2 * Shadow Pipeline — runs old and new canonicalization pipelines in parallel,
3 * compares output, and classifies the upgrade.
4 */
5
6import type { CanonicalNode } from './models/canonical.js';
7import type { ShadowDiffMetrics, ShadowResult, PipelineConfig } from './models/pipeline.js';
8import { UpgradeClassification } from './models/pipeline.js';
9
10/**
11 * Compare two sets of canonical nodes produced by different pipeline versions.
12 */
13export function computeShadowDiff(
14 oldNodes: CanonicalNode[],
15 newNodes: CanonicalNode[],
16): ShadowDiffMetrics {
17 const oldIds = new Set(oldNodes.map(n => n.canon_id));
18 const newIds = new Set(newNodes.map(n => n.canon_id));
19
20 const addedNodes = newNodes.filter(n => !oldIds.has(n.canon_id));
21 const removedNodes = oldNodes.filter(n => !newIds.has(n.canon_id));
22 const keptNodes = newNodes.filter(n => oldIds.has(n.canon_id));
23
24 const totalNodes = Math.max(oldNodes.length, 1);
25 const nodeChangePct = ((addedNodes.length + removedNodes.length) / totalNodes) * 100;
26
27 // Edge changes
28 const oldEdges = new Set(oldNodes.flatMap(n => n.linked_canon_ids.map(l => `${n.canon_id}->${l}`)));
29 const newEdges = new Set(newNodes.flatMap(n => n.linked_canon_ids.map(l => `${n.canon_id}->${l}`)));
30 const edgeAdded = [...newEdges].filter(e => !oldEdges.has(e)).length;
31 const edgeRemoved = [...oldEdges].filter(e => !newEdges.has(e)).length;
32 const totalEdges = Math.max(oldEdges.size, 1);
33 const edgeChangePct = ((edgeAdded + edgeRemoved) / totalEdges) * 100;
34
35 // Orphan nodes (new nodes with no links and no source)
36 const orphanNodes = addedNodes.filter(
37 n => n.linked_canon_ids.length === 0 && n.source_clause_ids.length === 0
38 ).length;
39
40 // Risk escalations: nodes that changed type (approximate by statement match)
41 const oldByStmt = new Map(oldNodes.map(n => [n.statement, n]));
42 let riskEscalations = 0;
43 for (const nn of newNodes) {
44 const old = oldByStmt.get(nn.statement);
45 if (old && old.type !== nn.type) riskEscalations++;
46 }
47
48 // Semantic statement drift: how many statements are completely new
49 const oldStmts = new Set(oldNodes.map(n => n.statement));
50 const driftCount = newNodes.filter(n => !oldStmts.has(n.statement)).length;
51 const semanticDrift = (driftCount / Math.max(newNodes.length, 1)) * 100;
52
53 return {
54 node_change_pct: Math.round(nodeChangePct * 100) / 100,
55 edge_change_pct: Math.round(edgeChangePct * 100) / 100,
56 risk_escalations: riskEscalations,
57 orphan_nodes: orphanNodes,
58 out_of_scope_growth: addedNodes.length - removedNodes.length,
59 semantic_stmt_drift: Math.round(semanticDrift * 100) / 100,
60 };
61}
62
63/**
64 * Classify a shadow diff as SAFE, COMPACTION_EVENT, or REJECT.
65 */
66export function classifyShadowDiff(metrics: ShadowDiffMetrics): {
67 classification: UpgradeClassification;
68 reason: string;
69} {
70 if (metrics.orphan_nodes > 0) {
71 return { classification: UpgradeClassification.REJECT, reason: `${metrics.orphan_nodes} orphan nodes detected` };
72 }
73 if (metrics.semantic_stmt_drift > 50) {
74 return { classification: UpgradeClassification.REJECT, reason: `Semantic drift too high: ${metrics.semantic_stmt_drift}%` };
75 }
76 if (metrics.node_change_pct <= 3 && metrics.risk_escalations === 0) {
77 return { classification: UpgradeClassification.SAFE, reason: `Node change ${metrics.node_change_pct}% ≤ 3%, no risk escalations` };
78 }
79 if (metrics.node_change_pct <= 25 && metrics.orphan_nodes === 0) {
80 return { classification: UpgradeClassification.COMPACTION_EVENT, reason: `Node change ${metrics.node_change_pct}% ≤ 25%, no orphans` };
81 }
82 return { classification: UpgradeClassification.REJECT, reason: `Excessive churn: ${metrics.node_change_pct}% node change` };
83}
84
85/**
86 * Run a full shadow comparison.
87 */
88export function runShadowPipeline(
89 oldPipeline: PipelineConfig,
90 newPipeline: PipelineConfig,
91 oldNodes: CanonicalNode[],
92 newNodes: CanonicalNode[],
93): ShadowResult {
94 const metrics = computeShadowDiff(oldNodes, newNodes);
95 const { classification, reason } = classifyShadowDiff(metrics);
96 return { old_pipeline: oldPipeline, new_pipeline: newPipeline, metrics, classification, reason };
97}