refactor(mst): flesh it out even more!

mise.toml

···
 bun = "1.2.13"
 node = "22"
 pnpm = "10"
+python = "3.11"
+uv = "latest"

packages/utilities/mst/.gitignore

···
+/mst-test-suite/.venv/
+
+/mst-test-suite/cars/**/*.car
+/mst-test-suite/tests/**/*.json

packages/utilities/mst/lib/errors.ts

···
 		super(`missing block in store; cid=${cid}` + (def ? `; type=${def}` : ``));
 	}
 }
-
-/**
- * thrown when a block's decoded object doesn't match the expected type
- */
-export class UnexpectedObjectError extends Error {
-	constructor(
-		public cid: string,
-		public def: string,
-	) {
-		super(`unexpected object in store; cid=${cid}; expected=${def}`);
-	}
-}

packages/utilities/mst/lib/node-wrangler.ts

···
 import { NodeStore } from './node-store.js';
 
 /**
- * array helper: replaces element at index with a new value
+ * replaces element at index with a new value
  */
 const replaceAt = <T>(arr: readonly T[], index: number, value: T): readonly T[] => {
-	return [...arr.slice(0, index), value, ...arr.slice(index + 1)];
+	return arr.with(index, value);
 };
 
 /**
- * array helper: inserts element at index
+ * inserts element at index
  */
 const insertAt = <T>(arr: readonly T[], index: number, value: T): readonly T[] => {
-	return [...arr.slice(0, index), value, ...arr.slice(index)];
+	return arr.toSpliced(index, 0, value);
 };
 
 /**
- * array helper: removes element at index
+ * removes element at index
  */
 const removeAt = <T>(arr: readonly T[], index: number): readonly T[] => {
-	return [...arr.slice(0, index), ...arr.slice(index + 1)];
+	return arr.toSpliced(index, 1);
 };
 
 /**
···
  * the external APIs take a CID (the MST root) and return a CID (the new root),
  * while storing any newly created nodes in the NodeStore.
  *
- * neither method should ever fail - deleting a node that doesn't exist is a nop,
- * and adding the same node twice with the same value is also a nop. Callers
+ * neither method should ever fail - deleting a node that doesn't exist is a noop,
+ * and adding the same node twice with the same value is also a nop. callers
  * can detect these cases by seeing if the initial and final CIDs changed.
  */
 export class NodeWrangler {

packages/utilities/mst/lib/node.ts

···
 					t: subtrees[idx + 1],
 					v: values[idx],
 				});
+
+				prevKey = key;
 			}
 
 			const n: NodeData = {

packages/utilities/mst/lib/stores.ts

···
-import * as CBOR from '@atcute/cbor';
-
 import { deleteMany, setMany, type BlockMap } from './blockmap.js';
-import { MissingBlockError, UnexpectedObjectError } from './errors.js';
 
 /**
  * a read-only interface for retrieving blocks by their CID
···
  * all writes go to the upper store only
  */
 export class OverlayBlockStore implements BlockStore {
-	/** the writable upper layer store */
+	/** writable upper layer store */
 	upper: BlockStore;
-	/** the read-only lower layer store */
+	/** read-only lower layer store */
 	lower: ReadonlyBlockStore;
 
 	/**
···
 }
 
 /**
- * reads and decodes a block, validating it matches the expected type
- * @param store block store to read from
- * @param cid CID of the block to read
- * @param def schema definition with name and validation function
- * @returns the decoded and validated object
- * @throws {MissingBlockError} if block is not found
- * @throws {UnexpectedObjectError} if block doesn't match expected type
+ * a read-only block store wrapper that tracks all get() accesses
+ * useful for collecting proof nodes during MST operations
  */
-export const readObject = async <T>(store: ReadonlyBlockStore, cid: string, def: CheckDef<T>): Promise<T> => {
-	const bytes = await store.get(cid);
-	if (bytes === null) {
-		throw new MissingBlockError(cid, def.name);
-	}
+export class LoggingBlockStore implements ReadonlyBlockStore {
+	/** block store being proxied */
+	readonly wrapped: ReadonlyBlockStore;
+	/** set of CIDs that were accessed via get() or getMany() */
+	readonly accessed = new Set<string>();
 
-	const decoded = CBOR.decode(bytes);
-	if (!def.check(decoded)) {
-		throw new UnexpectedObjectError(cid, def.name);
+	/**
+	 * creates a new logging block store wrapper
+	 * @param store the block store to wrap
+	 */
+	constructor(store: ReadonlyBlockStore) {
+		this.wrapped = store;
 	}
 
-	return decoded;
-};
+	async get(cid: string): Promise<Uint8Array<ArrayBuffer> | null> {
+		this.accessed.add(cid);
 
-/**
- * reads and decodes a block without type validation
- * @param store block store to read from
- * @param cid CID of the block to read
- * @returns the decoded object
- * @throws {MissingBlockError} if block is not found
- */
-export const readRecord = async (store: ReadonlyBlockStore, cid: string): Promise<unknown> => {
-	const bytes = await store.get(cid);
-	if (bytes === null) {
-		throw new MissingBlockError(cid, undefined);
+		return this.wrapped.get(cid);
 	}
 
-	const decoded = CBOR.decode(bytes);
+	async getMany(cids: string[]): Promise<{ found: BlockMap; missing: string[] }> {
+		const accessed = this.accessed;
 
-	return decoded;
-};
+		for (const cid of cids) {
+			accessed.add(cid);
+		}
 
-/**
- * defines a type validator for use with readObject
- * combines a human-readable type name with a type guard function
- */
-export interface CheckDef<T> {
-	/** human-readable name of the expected type */
-	name: string;
-	/** type guard function to validate the decoded value */
-	check: (value: unknown) => value is T;
+		return this.wrapped.getMany(cids);
+	}
+
+	async has(cid: string): Promise<boolean> {
+		// has() doesn't count as an access for proof purposes
+		return this.wrapped.has(cid);
+	}
 }

packages/utilities/mst/lib/test-suite.test.ts

···
-import { readFileSync, readdirSync, statSync } from 'node:fs';
-import { join } from 'node:path';
-import { describe, expect, it } from 'vitest';
+import { beforeAll, describe, expect, it } from 'vitest';
+import * as v from 'valibot';
+
+import * as fs from 'node:fs/promises';
+import * as path from 'node:path';
 
 import { fromUint8Array } from '@atcute/car/v4/car-reader';
 import * as CID from '@atcute/cid';
 
+import { setMany } from './blockmap.js';
 import { DeltaType, mstDiff, recordDiff } from './diff.js';
 import { NodeStore } from './node-store.js';
-import { MemoryBlockStore } from './stores.js';
+import { NodeWrangler } from './node-wrangler.js';
+import { buildExclusionProof, buildInclusionProof } from './proof.js';
+import {
+	LoggingBlockStore,
+	MemoryBlockStore,
+	OverlayBlockStore,
+	ReadonlyMemoryBlockStore,
+} from './stores.js';
 
-interface MstDiffTestCase {
-	$type: 'mst-diff';
-	description: string;
-	inputs: {
-		mst_a: string;
-		mst_b: string;
-	};
-	results: {
-		created_nodes: string[];
-		deleted_nodes: string[];
-		record_ops: Array<{
-			rpath: string;
-			old_value: string | null;
-			new_value: string | null;
-		}>;
-		proof_nodes: string[];
-		inductive_proof_nodes: string[];
-		firehose_cids: string | string[];
-	};
-}
+const mstDiffTestCaseSchema = v.object({
+	$type: v.literal('mst-diff'),
+	description: v.string(),
+	inputs: v.object({
+		mst_a: v.string(),
+		mst_b: v.string(),
+	}),
+	results: v.object({
+		created_nodes: v.array(v.string()),
+		deleted_nodes: v.array(v.string()),
+		record_ops: v.array(
+			v.object({
+				rpath: v.string(),
+				old_value: v.nullable(v.string()),
+				new_value: v.nullable(v.string()),
+			}),
+		),
+		proof_nodes: v.array(v.string()),
+		inductive_proof_nodes: v.array(v.string()),
+	}),
+});
+
+type MstDiffTestCase = v.InferOutput<typeof mstDiffTestCaseSchema>;
+
+const testSuiteRoot = path.join(__dirname, '../mst-test-suite');
 
 /**
  * Load a CAR file into a MemoryBlockStore and extract the root CID
  */
-const loadCar = (carPath: string): { store: MemoryBlockStore; root: string } => {
-	const testSuiteRoot = join(__dirname, '..', '.research', 'mst-test-suite');
-	const fullPath = join(testSuiteRoot, carPath);
-	const carBytes = readFileSync(fullPath);
+const loadCar = async (relname: string): Promise<{ store: ReadonlyMemoryBlockStore; root: string }> => {
+	const filename = path.join(testSuiteRoot, relname);
+	const bytes = await fs.readFile(filename);
 
-	const car = fromUint8Array(carBytes);
+	const car = fromUint8Array(bytes);
 	const store = new MemoryBlockStore();
 
-	// Load all blocks from CAR into the store
 	for (const entry of car) {
 		const cidStr = CID.toCidLink(entry.cid).$link;
-		store.blocks.set(cidStr, entry.bytes);
+		store.blocks.set(cidStr, entry.bytes as Uint8Array<ArrayBuffer>);
 	}
 
-	// Extract root CID from CAR header
 	if (car.roots.length !== 1) {
-		throw new Error(`Expected exactly 1 root in CAR, got ${car.roots.length}`);
+		throw new Error(`expected exactly 1 root in CAR, got ${car.roots.length}`);
 	}
 
 	const root = car.roots[0].$link;
 	return { store, root };
 };
 
-/**
- * Recursively find all .json test files in a directory
- */
-const findTestFiles = (dir: string): string[] => {
-	const results: string[] = [];
-	const entries = readdirSync(dir);
-
-	for (const entry of entries) {
-		const fullPath = join(dir, entry);
-		const stat = statSync(fullPath);
+const testCases = await (async () => {
+	const testsDir = path.join(testSuiteRoot, 'tests');
 
-		if (stat.isDirectory()) {
-			results.push(...findTestFiles(fullPath));
-		} else if (entry.endsWith('.json')) {
-			results.push(fullPath);
-		}
-	}
+	const testCases: Array<{ path: string; description: string; testCase: MstDiffTestCase }> = [];
 
-	return results;
-};
-
-/**
- * Load all test cases from the test suite
- */
-const loadTestCases = (): Array<{ path: string; testCase: MstDiffTestCase }> => {
-	const testSuiteRoot = join(__dirname, '..', '.research', 'mst-test-suite');
-	const testsDir = join(testSuiteRoot, 'tests');
-	const testFiles = findTestFiles(testsDir);
+	for await (const name of fs.glob('**/*.json', { cwd: testsDir })) {
+		const filename = path.join(testsDir, name);
 
-	const testCases: Array<{ path: string; testCase: MstDiffTestCase }> = [];
+		const raw = await fs.readFile(filename, 'utf-8');
+		const json = JSON.parse(raw);
 
-	for (const filePath of testFiles) {
-		const content = readFileSync(filePath, 'utf-8');
-		const testCase = JSON.parse(content) as MstDiffTestCase;
+		const testCase = v.parse(mstDiffTestCaseSchema, json);
 
-		if (testCase.$type === 'mst-diff') {
-			testCases.push({ path: filePath, testCase });
-		}
+		testCases.push({
+			path: filename,
+			description: testCase.description.replace(`procedurally generated MST diff test case `, ``),
+			testCase,
+		});
 	}
 
 	return testCases;
-};
+})();
 
 describe('MST Test Suite', () => {
-	const allTestCases = loadTestCases();
+	describe.each(testCases)('$description', ({ testCase }) => {
+		let storeA: ReadonlyMemoryBlockStore;
+		let rootA: string;
 
-	// Run all test cases
-	const testCases = allTestCases;
+		let storeB: ReadonlyMemoryBlockStore;
+		let rootB: string;
 
-	it(`should have loaded test cases (${testCases.length} total)`, () => {
-		expect(testCases.length).toBeGreaterThan(1000); // Should have 16k+ tests
-	});
+		beforeAll(async () => {
+			({ store: storeA, root: rootA } = await loadCar(testCase.inputs.mst_a));
+			({ store: storeB, root: rootB } = await loadCar(testCase.inputs.mst_b));
+		});
 
-	describe.each(testCases)('$testCase.description', ({ testCase }) => {
-		it('should compute correct mstDiff', async () => {
-			// Load both CARs
-			const { store: storeA, root: rootA } = loadCar(testCase.inputs.mst_a);
-			const { store: storeB, root: rootB } = loadCar(testCase.inputs.mst_b);
-
-			// Create NodeStores (combine both block stores for access to all blocks)
-			// We need an overlay approach since diff needs to read from both trees
+		it('computes the correct mstDiff', async () => {
 			const combinedStore = new MemoryBlockStore();
-			for (const [cid, bytes] of storeA.blocks) {
-				combinedStore.blocks.set(cid, bytes);
-			}
-			for (const [cid, bytes] of storeB.blocks) {
-				combinedStore.blocks.set(cid, bytes);
-			}
+			setMany(combinedStore.blocks, storeA.blocks);
+			setMany(combinedStore.blocks, storeB.blocks);
 
 			const nodeStore = new NodeStore(combinedStore);
 
-			// Run mstDiff
 			const [createdNodes, deletedNodes] = await mstDiff(nodeStore, rootA, rootB);
 
-			// Compare created_nodes (as sets, order doesn't matter)
 			const expectedCreated = new Set(testCase.results.created_nodes);
 			expect(createdNodes).toEqual(expectedCreated);
 
-			// Compare deleted_nodes (as sets, order doesn't matter)
 			const expectedDeleted = new Set(testCase.results.deleted_nodes);
 			expect(deletedNodes).toEqual(expectedDeleted);
 		});
 
-		it('should compute correct recordDiff', async () => {
-			// Load both CARs
-			const { store: storeA, root: rootA } = loadCar(testCase.inputs.mst_a);
-			const { store: storeB, root: rootB } = loadCar(testCase.inputs.mst_b);
-
-			// Create combined NodeStore
+		it('computes the correct recordDiff', async () => {
 			const combinedStore = new MemoryBlockStore();
-			for (const [cid, bytes] of storeA.blocks) {
-				combinedStore.blocks.set(cid, bytes);
-			}
-			for (const [cid, bytes] of storeB.blocks) {
-				combinedStore.blocks.set(cid, bytes);
-			}
+			setMany(combinedStore.blocks, storeA.blocks);
+			setMany(combinedStore.blocks, storeB.blocks);
 
 			const nodeStore = new NodeStore(combinedStore);
 
-			// Run mstDiff and recordDiff
 			const [createdNodes, deletedNodes] = await mstDiff(nodeStore, rootA, rootB);
 
-			const deltas = [];
-			for await (const delta of recordDiff(nodeStore, createdNodes, deletedNodes)) {
-				deltas.push(delta);
-			}
+			const deltas = await Array.fromAsync(recordDiff(nodeStore, createdNodes, deletedNodes));
+			deltas.sort((a, b) => +(a.path > b.path) - +(a.path < b.path));
 
-			// Sort both actual and expected by rpath for comparison
-			const sortedDeltas = deltas.sort((a, b) => a.path.localeCompare(b.path));
-			const sortedExpected = [...testCase.results.record_ops].sort((a, b) => a.rpath.localeCompare(b.rpath));
+			const expectance = testCase.results.record_ops.toSorted(
+				(a, b) => +(a.rpath > b.rpath) - +(a.rpath < b.rpath),
+			);
 
-			expect(sortedDeltas.length).toBe(sortedExpected.length);
+			expect(deltas.length).toBe(expectance.length);
 
-			for (let i = 0; i < sortedDeltas.length; i++) {
-				const actual = sortedDeltas[i];
-				const expected = sortedExpected[i];
+			for (let idx = 0, len = deltas.length; idx < len; idx++) {
+				const actual = deltas[idx];
+				const expected = expectance[idx];
 
 				expect(actual.path).toBe(expected.rpath);
 				expect(actual.priorValue?.$link ?? null).toBe(expected.old_value);
···
 					expect(actual.deltaType).toBe(DeltaType.UPDATED);
 				}
 			}
+		});
+
+		it('computes the correct proof_nodes', async () => {
+			// create combined store
+			const combinedStore = new MemoryBlockStore();
+			setMany(combinedStore.blocks, storeA.blocks);
+			setMany(combinedStore.blocks, storeB.blocks);
+
+			const nodeStore = new NodeStore(combinedStore);
+
+			// collect proof nodes for all record operations
+			const proofNodes = new Set<string>();
+
+			for (const op of testCase.results.record_ops) {
+				let proof: Set<string>;
+
+				if (op.old_value === null) {
+					// CREATED: inclusion proof for new record in rootB
+					proof = await buildInclusionProof(nodeStore, rootB, op.rpath);
+				} else if (op.new_value === null) {
+					// DELETED: exclusion proof in rootB
+					proof = await buildExclusionProof(nodeStore, rootB, op.rpath);
+				} else {
+					// UPDATED: inclusion proof for updated record in rootB
+					proof = await buildInclusionProof(nodeStore, rootB, op.rpath);
+				}
+
+				// add all proof nodes to the set
+				for (const cid of proof) {
+					proofNodes.add(cid);
+				}
+			}
+
+			// compare against expected proof_nodes (as sets, order doesn't matter)
+			const expectedProofNodes = new Set(testCase.results.proof_nodes);
+			expect(proofNodes).toEqual(expectedProofNodes);
+		});
+
+		it('computes the correct inductive_proof_nodes', async () => {
+			// create combined store
+			const combinedStore = new MemoryBlockStore();
+			setMany(combinedStore.blocks, storeA.blocks);
+			setMany(combinedStore.blocks, storeB.blocks);
+
+			// inductive proofs: nodes that get READ when applying ops in REVERSE order
+			// this is used for MST operation inversion (verifying B→A instead of A→B)
+
+			const loggingStore = new LoggingBlockStore(combinedStore);
+
+			const overlayStore = new OverlayBlockStore(new MemoryBlockStore(), loggingStore);
+			const nodeStore = new NodeStore(overlayStore);
+			const wrangler = new NodeWrangler(nodeStore);
+
+			// start from rootB and apply operations in REVERSE order
+			let currentRoot = rootB;
+			const reversedOps = testCase.results.record_ops.toReversed();
+
+			for (const op of reversedOps) {
+				if (op.old_value === null) {
+					// was CREATE, reverse it with DELETE
+					currentRoot = await wrangler.deleteRecord(currentRoot, op.rpath);
+				} else {
+					// was UPDATE or DELETE, reverse with PUT of old value
+					currentRoot = await wrangler.putRecord(currentRoot, op.rpath, { $link: op.old_value });
+				}
+			}
+
+			// after reversing all operations, we should end up back at rootA
+			expect(currentRoot).toBe(rootA);
+
+			// the blocks that were accessed (read) are the inductive proof nodes
+			const inductiveProofNodes = loggingStore.accessed;
+			const expectedInductiveProofNodes = new Set(testCase.results.inductive_proof_nodes);
+			expect(inductiveProofNodes).toEqual(expectedInductiveProofNodes);
 		});
 	});
 });

packages/utilities/mst/mst-test-suite/pyproject.toml

···
+[project]
+name = "mst-test-suite"
+version = "0.1.0"
+dependencies = [
+    "atmst>=0.0.6",
+    "cbrrr>=1.0.1",
+]

packages/utilities/mst/mst-test-suite/scripts/generate_exhaustive_cars.py

···
+from typing import BinaryIO, Optional
+import json
+
+from atmst.mst.node import MSTNode
+from atmst.mst.node_store import NodeStore
+from atmst.mst.node_wrangler import NodeWrangler
+from atmst.mst.node_walker import NodeWalker
+from atmst.mst.diff import very_slow_mst_diff, record_diff
+from atmst.blockstore import MemoryBlockStore, OverlayBlockStore, BlockStore
+from atmst.blockstore.car_file import encode_varint
+from atmst.mst import proof
+import cbrrr
+from cbrrr import CID
+
+
+class LoggingBlockStoreWrapper(BlockStore):
+    def __init__(self, bs: BlockStore):
+        self.bs = bs
+        self.gets = set()
+
+    def put_block(self, key: bytes, value: bytes) -> None:
+        self.bs.put_block(key, value)
+
+    def get_block(self, key: bytes) -> bytes:
+        self.gets.add(key)
+        return self.bs.get_block(key)
+
+    def del_block(self, key: bytes) -> None:
+        self.bs.del_block(key)
+
+
+"""
+class LoggingNodeStore(NodeStore):
+	def __init__(self, bs):
+		self.read_cids = set()
+		self.stored_cids = set()
+		super().__init__(bs)
+
+	def get_node(self, cid: Optional[CID]) -> MSTNode:
+		if cid is None:
+			self.read_cids.add(MSTNode.empty_root().cid)
+		else:
+			self.read_cids.add(cid)
+		return super().get_node(cid)
+
+	def stored_node(self, node: MSTNode) -> MSTNode:
+		self.stored_cids.add(node.cid)
+		return super().stored_node(node)
+"""
+
+
+class CarWriter:
+    def __init__(self, stream: BinaryIO, root: cbrrr.CID) -> None:
+        self.stream = stream
+        header_bytes = cbrrr.encode_dag_cbor({"version": 1, "roots": [root]})
+        stream.write(encode_varint(len(header_bytes)))
+        stream.write(header_bytes)
+
+    def write_block(self, cid: cbrrr.CID, value: bytes):
+        cid_bytes = bytes(cid)
+        self.stream.write(encode_varint(len(cid_bytes) + len(value)))
+        self.stream.write(cid_bytes)
+        self.stream.write(value)
+
+
+keys = []
+key_heights = [
+    0,
+    1,
+    0,
+    2,
+    0,
+    1,
+    0,
+]  # if all these keys are added to a MST, it'll form a perfect binary tree.
+i = 0
+for height in key_heights:
+    while True:
+        key = f"k/{i:02d}"
+        i += 1
+        if MSTNode.key_height(key) == height:
+            keys.append(key)
+            break
+
+vals = [
+    CID.cidv1_dag_cbor_sha256_32_from(
+        cbrrr.encode_dag_cbor({"$type": "mst-test-data", "value_for": k})
+    )
+    for k in keys
+]
+
+val_for_key = dict(zip(keys, vals))
+
+print(keys)
+print(vals)
+
+# we can reuse these
+bs = MemoryBlockStore()
+ns = NodeStore(bs)
+wrangler = NodeWrangler(ns)
+
+roots = []
+
+for i in range(2 ** len(keys)):
+    filename = f"./cars/exhaustive/exhaustive_{i:03d}.car"
+    root = ns.get_node(None).cid
+    for j in range(len(keys)):
+        if (i >> j) & 1:
+            # filename += f"_{keys[j]}h{key_heights[j]}"
+            root = wrangler.put_record(root, keys[j], vals[j])
+    # filename += ".car"
+    print(i, filename)
+
+    car_blocks = []
+    for node in NodeWalker(ns, root).iter_nodes():
+        car_blocks.append((node.cid, node.serialised))
+
+    assert len(set(cid for cid, val in car_blocks)) == len(car_blocks)  # no dupes
+
+    with open(filename, "wb") as carfile:
+        car = CarWriter(carfile, root)
+        for cid, val in sorted(car_blocks, key=lambda x: bytes(x[0])):
+            car.write_block(cid, val)
+
+    roots.append(root)
+
+# collecting these stats just for the sake of curiosity
+# identical_proof_and_creation_count = 0
+# proof_superset_of_creation_count = 0
+# creation_superset_of_proof_count = 0
+inversion_needs_extra_blocks = 0
+clusion_proof_nodes_not_in_inversion_proof = 0
+
+# generate exhaustive test cases
+for ai, root_a in enumerate(roots):
+    for bi, root_b in enumerate(roots):
+        filename = f"./tests/diff/exhaustive/exhaustive_{ai:03d}_{bi:03d}.json"
+        print(filename)
+        car_a = f"./cars/exhaustive/exhaustive_{ai:03d}.car"
+        car_b = f"./cars/exhaustive/exhaustive_{bi:03d}.car"
+        created_nodes, deleted_nodes = very_slow_mst_diff(ns, root_a, root_b)
+        record_ops = []
+        proof_nodes = set()
+        no_deletions = True
+        for delta in record_diff(ns, created_nodes, deleted_nodes):
+            record_ops.append(
+                {
+                    "rpath": delta.path,
+                    "old_value": None
+                    if delta.prior_value is None
+                    else delta.prior_value.encode(),
+                    "new_value": None
+                    if delta.later_value is None
+                    else delta.later_value.encode(),
+                }
+            )
+            if delta.later_value is None:  # deletion
+                proof_nodes.update(proof.build_exclusion_proof(ns, root_b, delta.path))
+                no_deletions = False
+            else:  # update or create
+                proof_nodes.update(proof.build_inclusion_proof(ns, root_b, delta.path))
+
+        if no_deletions:  # commits with no deletions are more well-behaved
+            assert proof_nodes.issubset(created_nodes)
+
+        # my inductive-proof-generation logic is ops order sensitive, so we do the sort beforehand
+        # TODO: maybe "deletes first" or similar produces smaller proofs on average?
+        record_ops.sort(key=lambda x: x["rpath"])
+
+        # figure out which blocks are required for inductive proofs.
+        # the idea here is that we use an overlay blockstore and log every "get" that has to fall thru to the lower layer.
+        # those gets are therefore the blocks required for a stateless consumer to verify the proof.
+        upper = MemoryBlockStore()
+        lbs = LoggingBlockStoreWrapper(bs)
+        lns = NodeStore(OverlayBlockStore(upper, lbs))
+        lnw = NodeWrangler(lns)
+        proof_root = root_b
+        for op in record_ops[
+            ::-1
+        ]:  # while the order does not effect the final root CID, it does affect the set of CIDs that fall thru
+            if op["old_value"] is None:
+                proof_root = lnw.del_record(proof_root, op["rpath"])
+            else:
+                proof_root = lnw.put_record(
+                    proof_root, op["rpath"], val_for_key[op["rpath"]]
+                )
+        assert proof_root == root_a  # we're back to where we started
+        inductive_proof_nodes = set(CID(cid) for cid in lbs.gets)
+
+        if inductive_proof_nodes - (created_nodes | proof_nodes):
+            # print(delta)
+            inversion_needs_extra_blocks += 1
+
+        if proof_nodes - inductive_proof_nodes:
+            clusion_proof_nodes_not_in_inversion_proof += 1
+
+        # if proof_nodes == created_nodes:
+        # identical_proof_and_creation_count += 1
+        # if proof_nodes.issuperset(created_nodes):
+        # proof_superset_of_creation_count += 1
+        # if created_nodes.issuperset(proof_nodes):
+        # creation_superset_of_proof_count += 1
+
+        testcase = {
+            "$type": "mst-diff",
+            "description": f"procedurally generated MST diff test case between MST {ai} and {bi}",
+            "inputs": {"mst_a": car_a, "mst_b": car_b},
+            "results": {
+                "created_nodes": sorted([cid.encode() for cid in created_nodes]),
+                "deleted_nodes": sorted([cid.encode() for cid in deleted_nodes]),
+                "record_ops": record_ops,  # these were sorted earlier
+                "proof_nodes": sorted([cid.encode() for cid in proof_nodes]),
+                "inductive_proof_nodes": sorted(
+                    [cid.encode() for cid in inductive_proof_nodes]
+                ),
+                "firehose_cids": "TODO",
+            },
+        }
+        with open(filename, "w") as jsonfile:
+            json.dump(testcase, jsonfile, indent="\t")
+
+# print("identical_proof_and_creation_count", identical_proof_and_creation_count / (len(roots)**2)) # 0.75
+# print("proof_superset_of_creation_count", proof_superset_of_creation_count / (len(roots)**2)) # 0.84
+# print("creation_superset_of_proof_count", creation_superset_of_proof_count / (len(roots)**2)) # 0.91
+print(
+    "inversion_needs_extra_blocks", inversion_needs_extra_blocks / (len(roots) ** 2)
+)  # 0.04
+print(clusion_proof_nodes_not_in_inversion_proof)

packages/utilities/mst/mst-test-suite/uv.lock

···
+version = 1
+revision = 3
+requires-python = ">=3.11"
+
+[[package]]
+name = "atmst"
+version = "0.0.6"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "cbrrr" },
+    { name = "lru-dict" },
+    { name = "more-itertools" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/47/7a/2cca04368b664d372473504615e37150466ecc796dff018504d8daf5de6d/atmst-0.0.6.tar.gz", hash = "sha256:bdc3ada3f234e28dada73f50cd40359534f99208436e831fe035f6fc7c7b188e", size = 18577, upload-time = "2024-12-21T11:37:20.15Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/f1/ec/d743d809cadfaae230ebed08c00f63a68f1bfe82042f74ea98c51965ed8f/atmst-0.0.6-py3-none-any.whl", hash = "sha256:e63801225f31b602a3aacfe73360561fa5f488adb1a56d64e0746d462981ecff", size = 19744, upload-time = "2024-12-21T11:37:17.804Z" },
+]
+
+[[package]]
+name = "cbrrr"
+version = "1.0.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/a3/2e/321b68b2b12c99864f0872feb8ce80f03483b59c113730f7cb4465a49e0e/cbrrr-1.0.1.tar.gz", hash = "sha256:2dc5f78a71b67849e1b364053819053f03cd2032797ce6adbe1d02e8d602698c", size = 17503, upload-time = "2024-11-16T22:59:41.164Z" }
+
+[[package]]
+name = "lru-dict"
+version = "1.3.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/96/e3/42c87871920602a3c8300915bd0292f76eccc66c38f782397acbf8a62088/lru-dict-1.3.0.tar.gz", hash = "sha256:54fd1966d6bd1fcde781596cb86068214edeebff1db13a2cea11079e3fd07b6b", size = 13123, upload-time = "2023-11-06T01:40:12.951Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/a8/c9/6fac0cb67160f0efa3cc76a6a7d04d5e21a516eeb991ebba08f4f8f01ec5/lru_dict-1.3.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:20c595764695d20bdc3ab9b582e0cc99814da183544afb83783a36d6741a0dac", size = 17750, upload-time = "2023-11-06T01:38:52.667Z" },
+    { url = "https://files.pythonhosted.org/packages/61/14/f90dee4bc547ae266dbeffd4e11611234bb6af511dea48f3bc8dac1de478/lru_dict-1.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d9b30a8f50c3fa72a494eca6be5810a1b5c89e4f0fda89374f0d1c5ad8d37d51", size = 11055, upload-time = "2023-11-06T01:38:53.798Z" },
+    { url = "https://files.pythonhosted.org/packages/4e/63/a0ae20525f9d52f62ac0def47935f8a2b3b6fcd2c145218b9a27fc1fb910/lru_dict-1.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9710737584650a4251b9a566cbb1a86f83437adb209c9ba43a4e756d12faf0d7", size = 11330, upload-time = "2023-11-06T01:38:54.847Z" },
+    { url = "https://files.pythonhosted.org/packages/e9/c6/8c2b81b61e5206910c81b712500736227289aefe4ccfb36137aa21807003/lru_dict-1.3.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b84c321ae34f2f40aae80e18b6fa08b31c90095792ab64bb99d2e385143effaa", size = 31793, upload-time = "2023-11-06T01:38:56.163Z" },
+    { url = "https://files.pythonhosted.org/packages/f9/d7/af9733f94df67a2e9e31ef47d4c41aff1836024f135cdbda4743eb628452/lru_dict-1.3.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:eed24272b4121b7c22f234daed99899817d81d671b3ed030c876ac88bc9dc890", size = 33090, upload-time = "2023-11-06T01:38:57.091Z" },
+    { url = "https://files.pythonhosted.org/packages/5b/6e/5b09b069a70028bcf05dbdc57a301fbe8b3bafecf916f2ed5a3065c79a71/lru_dict-1.3.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9bd13af06dab7c6ee92284fd02ed9a5613a07d5c1b41948dc8886e7207f86dfd", size = 29795, upload-time = "2023-11-06T01:38:58.278Z" },
+    { url = "https://files.pythonhosted.org/packages/21/92/4690daefc2602f7c3429ecf54572d37a9e3c372d370344d2185daa4d5ecc/lru_dict-1.3.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a1efc59bfba6aac33684d87b9e02813b0e2445b2f1c444dae2a0b396ad0ed60c", size = 31586, upload-time = "2023-11-06T01:38:59.363Z" },
+    { url = "https://files.pythonhosted.org/packages/3c/67/0a29a91087196b02f278d8765120ee4e7486f1f72a4c505fd1cd3109e627/lru_dict-1.3.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:cfaf75ac574447afcf8ad998789071af11d2bcf6f947643231f692948839bd98", size = 36662, upload-time = "2023-11-06T01:39:00.795Z" },
+    { url = "https://files.pythonhosted.org/packages/36/54/8d56c514cd2333b652bd44c8f1962ab986cbe68e8ad7258c9e0f360cddb6/lru_dict-1.3.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c95f8751e2abd6f778da0399c8e0239321d560dbc58cb063827123137d213242", size = 35118, upload-time = "2023-11-06T01:39:01.883Z" },
+    { url = "https://files.pythonhosted.org/packages/f5/9a/c7a175d10d503b86974cb07141ca175947145dd1c7370fcda86fbbcaf326/lru_dict-1.3.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:abd0c284b26b5c4ee806ca4f33ab5e16b4bf4d5ec9e093e75a6f6287acdde78e", size = 38198, upload-time = "2023-11-06T01:39:03.306Z" },
+    { url = "https://files.pythonhosted.org/packages/fd/59/2e5086c8e8a05a7282a824a2a37e3c45cd5714e7b83d8bc0267cb3bb5b4f/lru_dict-1.3.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:2a47740652b25900ac5ce52667b2eade28d8b5fdca0ccd3323459df710e8210a", size = 36542, upload-time = "2023-11-06T01:39:04.751Z" },
+    { url = "https://files.pythonhosted.org/packages/12/52/80d0a06e5f45fe7c278dd662da6ea5b39f2ff003248f448189932f6b71c2/lru_dict-1.3.0-cp311-cp311-win32.whl", hash = "sha256:a690c23fc353681ed8042d9fe8f48f0fb79a57b9a45daea2f0be1eef8a1a4aa4", size = 12533, upload-time = "2023-11-06T01:39:05.838Z" },
+    { url = "https://files.pythonhosted.org/packages/ce/fe/1f12f33513310860ec6d722709ec4ad8256d9dcc3385f6ae2a244e6e66f5/lru_dict-1.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:efd3f4e0385d18f20f7ea6b08af2574c1bfaa5cb590102ef1bee781bdfba84bc", size = 13651, upload-time = "2023-11-06T01:39:06.871Z" },
+    { url = "https://files.pythonhosted.org/packages/fc/5c/385f080747eb3083af87d8e4c9068f3c4cab89035f6982134889940dafd8/lru_dict-1.3.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:c279068f68af3b46a5d649855e1fb87f5705fe1f744a529d82b2885c0e1fc69d", size = 17174, upload-time = "2023-11-06T01:39:07.923Z" },
+    { url = "https://files.pythonhosted.org/packages/3c/de/5ef2ed75ce55d7059d1b96177ba04fa7ee1f35564f97bdfcd28fccfbe9d2/lru_dict-1.3.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:350e2233cfee9f326a0d7a08e309372d87186565e43a691b120006285a0ac549", size = 10742, upload-time = "2023-11-06T01:39:08.871Z" },
+    { url = "https://files.pythonhosted.org/packages/ca/05/f69a6abb0062d2cf2ce0aaf0284b105b97d1da024ca6d3d0730e6151242e/lru_dict-1.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4eafb188a84483b3231259bf19030859f070321b00326dcb8e8c6cbf7db4b12f", size = 11079, upload-time = "2023-11-06T01:39:09.766Z" },
+    { url = "https://files.pythonhosted.org/packages/ea/59/cf891143abe58a455b8eaa9175f0e80f624a146a2bf9a1ca842ee0ef930a/lru_dict-1.3.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:73593791047e36b37fdc0b67b76aeed439fcea80959c7d46201240f9ec3b2563", size = 32469, upload-time = "2023-11-06T01:39:11.091Z" },
+    { url = "https://files.pythonhosted.org/packages/59/88/d5976e9f70107ce11e45d93c6f0c2d5eaa1fc30bb3c8f57525eda4510dff/lru_dict-1.3.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1958cb70b9542773d6241974646e5410e41ef32e5c9e437d44040d59bd80daf2", size = 33496, upload-time = "2023-11-06T01:39:12.463Z" },
+    { url = "https://files.pythonhosted.org/packages/6c/f8/94d6e910d54fc1fa05c0ee1cd608c39401866a18cf5e5aff238449b33c11/lru_dict-1.3.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bc1cd3ed2cee78a47f11f3b70be053903bda197a873fd146e25c60c8e5a32cd6", size = 29914, upload-time = "2023-11-06T01:39:13.395Z" },
+    { url = "https://files.pythonhosted.org/packages/ca/b9/9db79780c8a3cfd66bba6847773061e5cf8a3746950273b9985d47bbfe53/lru_dict-1.3.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82eb230d48eaebd6977a92ddaa6d788f14cf4f4bcf5bbffa4ddfd60d051aa9d4", size = 32241, upload-time = "2023-11-06T01:39:14.612Z" },
+    { url = "https://files.pythonhosted.org/packages/9b/b6/08a623019daec22a40c4d6d2c40851dfa3d129a53b2f9469db8eb13666c1/lru_dict-1.3.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:5ad659cbc349d0c9ba8e536b5f40f96a70c360f43323c29f4257f340d891531c", size = 37320, upload-time = "2023-11-06T01:39:15.875Z" },
+    { url = "https://files.pythonhosted.org/packages/70/0b/d3717159c26155ff77679cee1b077d22e1008bf45f19921e193319cd8e46/lru_dict-1.3.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:ba490b8972531d153ac0d4e421f60d793d71a2f4adbe2f7740b3c55dce0a12f1", size = 35054, upload-time = "2023-11-06T01:39:17.063Z" },
+    { url = "https://files.pythonhosted.org/packages/04/74/f2ae00de7c27984a19b88d2b09ac877031c525b01199d7841ec8fa657fd6/lru_dict-1.3.0-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:c0131351b8a7226c69f1eba5814cbc9d1d8daaf0fdec1ae3f30508e3de5262d4", size = 38613, upload-time = "2023-11-06T01:39:18.136Z" },
+    { url = "https://files.pythonhosted.org/packages/5a/0b/e30236aafe31b4247aa9ae61ba8aac6dde75c3ea0e47a8fb7eef53f6d5ce/lru_dict-1.3.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:0e88dba16695f17f41701269fa046197a3fd7b34a8dba744c8749303ddaa18df", size = 37143, upload-time = "2023-11-06T01:39:19.571Z" },
+    { url = "https://files.pythonhosted.org/packages/1c/28/b59bcebb8d76ba8147a784a8be7eab6a4ad3395b9236e73740ff675a5a52/lru_dict-1.3.0-cp312-cp312-win32.whl", hash = "sha256:6ffaf595e625b388babc8e7d79b40f26c7485f61f16efe76764e32dce9ea17fc", size = 12653, upload-time = "2023-11-06T01:39:20.574Z" },
+    { url = "https://files.pythonhosted.org/packages/bd/18/06d9710cb0a0d3634f8501e4bdcc07abe64a32e404d82895a6a36fab97f6/lru_dict-1.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:cf9da32ef2582434842ab6ba6e67290debfae72771255a8e8ab16f3e006de0aa", size = 13811, upload-time = "2023-11-06T01:39:21.599Z" },
+]
+
+[[package]]
+name = "more-itertools"
+version = "10.8.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/ea/5d/38b681d3fce7a266dd9ab73c66959406d565b3e85f21d5e66e1181d93721/more_itertools-10.8.0.tar.gz", hash = "sha256:f638ddf8a1a0d134181275fb5d58b086ead7c6a72429ad725c67503f13ba30bd", size = 137431, upload-time = "2025-09-02T15:23:11.018Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/a4/8e/469e5a4a2f5855992e425f3cb33804cc07bf18d48f2db061aec61ce50270/more_itertools-10.8.0-py3-none-any.whl", hash = "sha256:52d4362373dcf7c52546bc4af9a86ee7c4579df9a8dc268be0a2f949d376cc9b", size = 69667, upload-time = "2025-09-02T15:23:09.635Z" },
+]
+
+[[package]]
+name = "mst-test-suite"
+version = "0.1.0"
+source = { virtual = "." }
+dependencies = [
+    { name = "atmst" },
+    { name = "cbrrr" },
+]
+
+[package.metadata]
+requires-dist = [
+    { name = "atmst", specifier = ">=0.0.6" },
+    { name = "cbrrr", specifier = ">=1.0.1" },
+]

packages/utilities/mst/package.json

···
 	"type": "module",
 	"name": "@atcute/mst",
 	"version": "0.1.0",
-	"description": "atproto mst manipulation utilities",
+	"description": "atproto MST manipulation utilities",
 	"keywords": [
 		"atproto",
 		"repo",
-		"mst",
-		"dasl",
-		"car"
+		"mst"
 	],
 	"license": "0BSD",
 	"repository": {
···
 	},
 	"sideEffects": false,
 	"scripts": {
+		"generate-tests": "cd mst-test-suite && mise exec -- uv run python scripts/generate_exhaustive_cars.py",
 		"build": "tsc --project tsconfig.build.json",
 		"test": "vitest run",
 		"prepublish": "rm -rf dist; pnpm run build"
···
 	"devDependencies": {
 		"@atcute/car": "workspace:^",
 		"@vitest/coverage-v8": "^3.2.4",
+		"valibot": "^1.1.0",
 		"vitest": "^3.2.4"
 	},
 	"dependencies": {
 		"@atcute/cbor": "workspace:^",
 		"@atcute/cid": "workspace:^",
 		"@atcute/uint8array": "workspace:^",
-		"@atcute/varint": "workspace:^",
-		"@badrap/valita": "^0.4.6"
+		"@atcute/varint": "workspace:^"
 	}
 }

pnpm-lock.yaml

···
       '@atcute/varint':
         specifier: workspace:^
         version: link:../varint
-      '@badrap/valita':
-        specifier: ^0.4.6
-        version: 0.4.6
     devDependencies:
       '@atcute/car':
         specifier: workspace:^
···
       '@vitest/coverage-v8':
         specifier: ^3.2.4
         version: 3.2.4(@vitest/browser@3.2.4)(vitest@3.2.4)
+      valibot:
+        specifier: ^1.1.0
+        version: 1.1.0(typescript@5.9.2)
       vitest:
         specifier: ^3.2.4
         version: 3.2.4(@types/node@24.3.0)(@vitest/browser@3.2.4)(tsx@4.20.6)(yaml@2.8.0)