feat(car): initial commit

README.md

···
 			<td><code>tid</code>: create and parse TID identifiers</td>
 		</tr>
 		<tr>
+			<td><code>car</code>: read AT Protocol's CAR (content-addressable archive) repositories</td>
+		</tr>
+		<tr>
 			<td><code>cid</code>: create and parse the blessed CIDv1 format</td>
 		</tr>
 		<tr>

packages/utilities/car/README.md

···
+# @atcute/car
+
+read AT Protocol's CAR (content-addressable archive) repositories
+
+```ts
+for (const { collection, rkey, record } of iterateAtpCar(buf)) {
+	// ...
+}
+```

packages/utilities/car/lib/atproto-repo.ts

···
+import * as CBOR from '@atcute/cbor';
+import * as CID from '@atcute/cid';
+
+import { fromUint8Array } from './reader.js';
+
+const decoder = new TextDecoder();
+
+export interface RepoEntry {
+	collection: string;
+	rkey: string;
+	record: unknown;
+}
+
+export function* iterateAtpCar(buf: Uint8Array): Generator<RepoEntry> {
+	const { roots, iterate } = fromUint8Array(new Uint8Array(buf));
+	assert(roots.length === 1, `expected only 1 root in the car archive; got=${roots.length}`);
+
+	const blockmap: BlockMap = new Map();
+	for (const entry of iterate()) {
+		blockmap.set(CID.format(entry.cid), entry.bytes);
+	}
+
+	const commit = readObject(blockmap, roots[0]) as Commit;
+	for (const { key, cid } of walkEntries(blockmap, commit.data)) {
+		const [collection, rkey] = key.split('/');
+		const record = readObject(blockmap, cid);
+
+		yield { collection, rkey, record };
+	}
+}
+
+function readObject(map: BlockMap, link: CBOR.CIDLink): unknown {
+	const cid = link.$link;
+
+	const bytes = map.get(cid);
+	assert(bytes != null, `cid not found in blockmap; cid=${cid}`);
+
+	const data = CBOR.decode(bytes);
+
+	return data;
+}
+
+function* walkEntries(map: BlockMap, pointer: CBOR.CIDLink): Generator<NodeEntry> {
+	const data = readObject(map, pointer) as MstNode;
+	const entries = data.e;
+
+	let lastKey = '';
+
+	if (data.l !== null) {
+		yield* walkEntries(map, data.l);
+	}
+
+	for (let i = 0, il = entries.length; i < il; i++) {
+		const entry = entries[i];
+
+		const key_str = decoder.decode(CBOR.fromBytes(entry.k));
+		const key = lastKey.slice(0, entry.p) + key_str;
+
+		lastKey = key;
+
+		yield { key: key, cid: entry.v };
+
+		if (entry.t !== null) {
+			yield* walkEntries(map, entry.t);
+		}
+	}
+}
+
+function assert(condition: boolean, message: string): asserts condition {
+	if (!condition) {
+		throw new Error(message);
+	}
+}
+
+type BlockMap = Map<string, Uint8Array>;
+
+interface Commit {
+	version: 3;
+	did: string;
+	data: CBOR.CIDLink;
+	rev: string;
+	prev: CBOR.CIDLink | null;
+	sig: CBOR.Bytes;
+}
+
+interface TreeEntry {
+	/** count of bytes shared with previous TreeEntry in this Node (if any) */
+	p: number;
+	/** remainder of key for this TreeEntry, after "prefixlen" have been removed */
+	k: CBOR.Bytes;
+	/** link to a sub-tree Node at a lower level which has keys sorting after this TreeEntry's key (to the "right"), but before the next TreeEntry's key in this Node (if any) */
+	v: CBOR.CIDLink;
+	/** next subtree (to the right of leaf) */
+	t: CBOR.CIDLink | null;
+}
+
+interface MstNode {
+	/** link to sub-tree Node on a lower level and with all keys sorting before keys at this node */
+	l: CBOR.CIDLink | null;
+	/** ordered list of TreeEntry objects */
+	e: TreeEntry[];
+}
+
+interface NodeEntry {
+	key: string;
+	cid: CBOR.CIDLink;
+}

packages/utilities/car/lib/index.test.ts

···
+import { expect, it } from 'bun:test';
+
+import { iterateAtpCar } from './index.js';
+
+it('decodes atproto car files', () => {
+	const buf = Buffer.from(
+		'OqJlcm9vdHOB2CpYJQABcRIgkD8I0DL+GsJ3OKREpf9k73yHguuSEYzEiXPGueoJg8FndmVy' +
+			'c2lvbgGPAQFxEiDqG8o/D37K3hldhQTMRq9/Uvyf7X9evn9eB9ZdgpYq6qRlJHR5cGV2YXBw' +
+			'LmJza3kuYWN0b3IucHJvZmlsZWljcmVhdGVkQXR4GDIwMjQtMDItMjRUMTI6MTU6NDEuMjE5' +
+			'WmtkZXNjcmlwdGlvbm90ZXN0aW5nIGFjY291bnRrZGlzcGxheU5hbWVg4AEBcRIgkD8I0DL+' +
+			'GsJ3OKREpf9k73yHguuSEYzEiXPGueoJg8GmY2RpZHggZGlkOnBsYzpzcmNxb3UybTd1cXVv' +
+			'Z3lkNXhrNGI1eTVjcmV2bTNsNXE1ZmplbnRjMmRjc2lnWEDeWWEO5/vV6SmnbUrLRu9WhWqI' +
+			'kHKANGFOin3xqFc4fgtuYzkbFXFJDMQU06nBWxict8FQ8Kas9Mr2fDAh++vVZGRhdGHYKlgl' +
+			'AAFxEiB2ibkpj3r4cdTag9v2ipIe8fxyjUFOgCjZbtYnfhyJ2GRwcmV29md2ZXJzaW9uA6QB' +
+			'AXESIHaJuSmPevhx1NqD2/aKkh7x/HKNQU6AKNlu1id+HInYomFlgaRha1gbYXBwLmJza3ku' +
+			'YWN0b3IucHJvZmlsZS9zZWxmYXAAYXTYKlglAAFxEiBvSJJSaF/w/fee+UmoLV84FDwZRC7p' +
+			'pJX484MghY0rM2F22CpYJQABcRIg6hvKPw9+yt4ZXYUEzEavf1L8n+1/Xr5/XgfWXYKWKuph' +
+			'bPaBAQFxEiBvSJJSaF/w/fee+UmoLV84FDwZRC7ppJX484MghY0rM6JhZYGkYWtYIGFwcC5i' +
+			'c2t5LmZlZWQucG9zdC8za201eW1rNGhoazJ6YXAAYXT2YXbYKlglAAFxEiDj+gU903L3F3Ar' +
+			'WCg+aeQZYEiM3ooIxqHbVvbQPZvEbGFs9qECAXESIOP6BT3TcvcXcCtYKD5p5BlgSIzeigjG' +
+			'odtW9tA9m8RspWR0ZXh0dWJlZXAgYm9vcCBAbWFyeS5teS5pZGUkdHlwZXJhcHAuYnNreS5m' +
+			'ZWVkLnBvc3RlbGFuZ3OBYmVuZmZhY2V0c4GjZSR0eXBld2FwcC5ic2t5LnJpY2h0ZXh0LmZh' +
+			'Y2V0ZWluZGV4omdieXRlRW5kFWlieXRlU3RhcnQKaGZlYXR1cmVzgaJjZGlkeCBkaWQ6cGxj' +
+			'OmlhNzZrdm5uZGp1dGdlZGdneDJpYnJlbWUkdHlwZXgfYXBwLmJza3kucmljaHRleHQuZmFj' +
+			'ZXQjbWVudGlvbmljcmVhdGVkQXR4GDIwMjQtMDItMjRUMTI6MTY6MjAuNjM3Wg==',
+		'base64',
+	);
+
+	// @ts-expect-error: node.js buffer it no likey
+	const result = Array.from(iterateAtpCar(buf));
+
+	expect(result).toEqual([
+		{
+			collection: 'app.bsky.actor.profile',
+			rkey: 'self',
+			record: {
+				$type: 'app.bsky.actor.profile',
+				createdAt: '2024-02-24T12:15:41.219Z',
+				displayName: '',
+				description: 'testing account',
+			},
+		},
+		{
+			collection: 'app.bsky.feed.post',
+			rkey: '3km5ymk4hhk2z',
+			record: {
+				$type: 'app.bsky.feed.post',
+				createdAt: '2024-02-24T12:16:20.637Z',
+				langs: ['en'],
+				text: 'beep boop @mary.my.id',
+				facets: [
+					{
+						$type: 'app.bsky.richtext.facet',
+						index: {
+							byteEnd: 21,
+							byteStart: 10,
+						},
+						features: [
+							{
+								did: 'did:plc:ia76kvnndjutgedggx2ibrem',
+								$type: 'app.bsky.richtext.facet#mention',
+							},
+						],
+					},
+				],
+			},
+		},
+	]);
+});

packages/utilities/car/lib/index.ts

···
+export { iterateAtpCar, type RepoEntry } from './atproto-repo.js';
+export { fromUint8Array } from './reader.js';

packages/utilities/car/lib/reader.ts

···
+import { createUint8Reader } from './utilities/byte-reader.js';
+import { createCarReader } from './utilities/sync-car-reader.js';
+
+export const fromUint8Array = (buffer: Uint8Array) => {
+	const reader = createUint8Reader(buffer);
+	return createCarReader(reader);
+};

packages/utilities/car/lib/utilities/byte-reader.ts

···
+export interface SyncByteReader {
+	readonly pos: number;
+	upto(size: number): Uint8Array;
+	exactly(size: number, seek: boolean): Uint8Array;
+	seek(size: number): void;
+}
+
+export const createUint8Reader = (buf: Uint8Array): SyncByteReader => {
+	let pos = 0;
+
+	return {
+		get pos() {
+			return pos;
+		},
+
+		seek(size) {
+			pos += size;
+		},
+		upto(size) {
+			return buf.subarray(pos, pos + Math.min(size, buf.length - pos));
+		},
+		exactly(size, seek) {
+			if (size > buf.length - pos) {
+				throw new RangeError('unexpected end of data');
+			}
+
+			const slice = buf.subarray(pos, pos + size);
+			if (seek) {
+				pos += size;
+			}
+
+			return slice;
+		},
+	};
+};

packages/utilities/car/lib/utilities/sync-car-reader.ts

···
+import * as CBOR from '@atcute/cbor';
+import * as varint from '@atcute/varint';
+import * as CID from '@atcute/cid';
+
+import type { SyncByteReader } from './byte-reader.js';
+
+interface CarV1Header {
+	version: 1;
+	roots: CBOR.CIDLink[];
+}
+
+const isCarV1Header = (value: unknown): value is CarV1Header => {
+	if (value === null || typeof value !== 'object') {
+		return false;
+	}
+
+	const { version, roots } = value as CarV1Header;
+	return version === 1 && Array.isArray(roots) && roots.every((root) => root instanceof CBOR.CIDLinkWrapper);
+};
+
+const readVarint = (reader: SyncByteReader, size: number): number => {
+	const buf = reader.upto(size);
+	if (buf.length === 0) {
+		throw new RangeError(`unexpected end of data`);
+	}
+
+	const [int, read] = varint.decode(buf);
+	reader.seek(read);
+
+	return int;
+};
+
+const readHeader = (reader: SyncByteReader): CarV1Header => {
+	const length = readVarint(reader, 8);
+	if (length === 0) {
+		throw new RangeError(`invalid car header; length=0`);
+	}
+
+	const rawHeader = reader.exactly(length, true);
+	const header = CBOR.decode(rawHeader);
+	if (!isCarV1Header(header)) {
+		throw new TypeError(`expected a car v1 archive`);
+	}
+
+	return header;
+};
+
+const readMultihashDigest = (reader: SyncByteReader): CID.Digest => {
+	const first = reader.upto(8);
+
+	const [code, codeOffset] = varint.decode(first);
+	const [size, sizeOffset] = varint.decode(first.subarray(codeOffset));
+
+	const offset = codeOffset + sizeOffset;
+
+	const bytes = reader.exactly(offset + size, true);
+	const digest = bytes.subarray(offset);
+
+	return {
+		code: code,
+		size: size,
+		digest: digest,
+		bytes: bytes,
+	};
+};
+
+const readCid = (reader: SyncByteReader): CID.CID => {
+	const version = readVarint(reader, 8);
+	if (version !== 1) {
+		throw new Error(`expected a cidv1`);
+	}
+
+	const codec = readVarint(reader, 8);
+	const digest = readMultihashDigest(reader);
+
+	const cid: CID.CID = {
+		version: version,
+		code: codec,
+		digest: digest,
+		bytes: CID.encode(version, codec, digest.bytes),
+	};
+
+	return cid;
+};
+
+const readBlockHeader = (reader: SyncByteReader): { cid: CID.CID; blockSize: number } => {
+	const start = reader.pos;
+
+	let size = readVarint(reader, 8);
+	if (size === 0) {
+		throw new Error(`invalid car section; length=0`);
+	}
+
+	size += reader.pos - start;
+
+	const cid = readCid(reader);
+	const blockSize = size - Number(reader.pos - start);
+
+	return { cid, blockSize };
+};
+
+export const createCarReader = (reader: SyncByteReader) => {
+	const { roots } = readHeader(reader);
+
+	return {
+		roots,
+		*iterate(): Generator<{ cid: CID.CID; bytes: Uint8Array }> {
+			while (reader.upto(8).length > 0) {
+				const { cid, blockSize } = readBlockHeader(reader);
+				const bytes = reader.exactly(blockSize, true);
+
+				yield { cid, bytes };
+			}
+		},
+	};
+};

packages/utilities/car/package.json

···
+{
+	"type": "module",
+	"name": "@atcute/car",
+	"version": "1.0.0",
+	"description": "read AT Protocol's CAR (content-addressable archive) repositories",
+	"license": "MIT",
+	"repository": {
+		"url": "https://codeberg.org/mary-ext/atcute"
+	},
+	"files": [
+		"dist/"
+	],
+	"exports": {
+		".": "./dist/index.js"
+	},
+	"sideEffects": false,
+	"scripts": {
+		"build": "tsc --project tsconfig.build.json",
+		"test": "bun test --coverage",
+		"prepublish": "rm -rf dist; pnpm run build"
+	},
+	"devDependencies": {
+		"@types/bun": "^1.1.10"
+	},
+	"dependencies": {
+		"@atcute/cbor": "workspace:^",
+		"@atcute/cid": "workspace:^",
+		"@atcute/varint": "workspace:^"
+	}
+}

packages/utilities/car/tsconfig.build.json

···
+{
+	"extends": "./tsconfig.json",
+	"exclude": ["**/*.test.ts", "**/*.bench.ts"]
+}

packages/utilities/car/tsconfig.json

···
+{
+	"compilerOptions": {
+		"types": ["bun"],
+		"outDir": "dist/",
+		"esModuleInterop": true,
+		"skipLibCheck": true,
+		"target": "ESNext",
+		"allowJs": true,
+		"resolveJsonModule": true,
+		"moduleDetection": "force",
+		"isolatedModules": true,
+		"verbatimModuleSyntax": true,
+		"strict": true,
+		"noImplicitOverride": true,
+		"noUnusedLocals": true,
+		"noUnusedParameters": true,
+		"noFallthroughCasesInSwitch": true,
+		"module": "NodeNext",
+		"sourceMap": true,
+		"declaration": true,
+	},
+	"include": ["lib"],
+}

pnpm-lock.yaml

···
         specifier: ^1.1.10
         version: 1.1.10
 
+  packages/utilities/car:
+    dependencies:
+      '@atcute/cbor':
+        specifier: workspace:^
+        version: link:../cbor
+      '@atcute/cid':
+        specifier: workspace:^
+        version: link:../cid
+      '@atcute/varint':
+        specifier: workspace:^
+        version: link:../varint
+    devDependencies:
+      '@types/bun':
+        specifier: ^1.1.10
+        version: 1.1.10
+
   packages/utilities/cbor:
     dependencies:
       '@atcute/base32':