import type {} from '@atcute/atproto'; import type { Client } from '@atcute/client'; import type { Blob as BlobRef, Cid, Did, LegacyBlob, ResourceUri } from '@atcute/lexicons'; import { parseCanonicalResourceUri, safeParse } from '@atcute/lexicons'; import { DevNiblingProject } from '@nibling/lexicons'; import { ProjectConflictError, SdkError, SlugTakenError } from '../errors.ts'; import { publicClient, repoClient, resolveActor } from '../identity.ts'; import { getRecord, listRecords, writtenUri, type RepoRecord } from '../records.ts'; import type { Session } from '../session.ts'; import { normalizePath } from './paths.ts'; import { isSlug, looksLikeRkey, slugify, uniqueSlug, validateSlug } from './slug.ts'; import { applyOverlay, blobCid, diffAgainstBase, hashTree, type FileTree, type Overlay } from './tree.ts'; const COLLECTION = 'dev.nibling.project'; const MAX_BLOB_BYTES = 5_000_000; const MAX_PROJECT_BYTES = 100_000_000; const BLOB_CONCURRENCY = 8; export type ProjectRecord = DevNiblingProject.Main; export interface Project { uri: ResourceUri; cid: Cid; did: Did; rkey: string; record: ProjectRecord; } /** Fetches the git tree a project's overlay applies to. The tangled module provides one. */ export type LoadBase = (base: DevNiblingProject.GitBase) => Promise; export interface LoadedProject { project: Project; files: FileTree; /** The base tree, when the record has one. Pass it back to saveProject. */ base?: FileTree; } /** Record fields a caller sets. `files`, timestamps, and `forkOf` are the store's. */ export type ProjectFields = Pick; export interface CreateProjectInput extends ProjectFields { files: FileTree; base?: { ref: DevNiblingProject.GitBase; files: FileTree }; } export async function listProjects(did: Did): Promise { const client = await publicClient(did); const projects: Project[] = []; for await (const item of listRecords(client, did, COLLECTION)) { const project = toProject(item); if (project) projects.push(project); } return projects; } export interface LoadProjectOptions { loadBase?: LoadBase; /** Caps the bytes downloaded for the project's blobs. Default 100 MB. */ maxBytes?: number; } /** A project's AT-URI, or the author and the name a URL gives it: a slug or a record key. */ export type ProjectRef = string | { actor: string; name: string }; export async function loadProject(ref: ProjectRef, options: LoadProjectOptions = {}): Promise { const { project, client } = await readProjectRef(ref); const uri = project.uri; const overlay: Overlay = new Map(); const blobs = await fetchBlobs(client, project.did, project.record, options.maxBytes); for (const entry of project.record.files) { overlay.set(entry.path, isFile(entry) ? blobs.get(blobLink(entry.content))! : null); } if (!project.record.base) return { project, files: applyOverlay(new Map(), overlay) }; if (!options.loadBase) throw new SdkError(`${uri} has a git base and no loadBase was given`, 'ERR_NIB_BASE_REQUIRED'); const base = await options.loadBase(project.record.base); return { project, files: applyOverlay(base, overlay), base }; } /** * Creates a project. Its slug is `input.slug` or one made from the name, with `-2`, `-3`, and so on * appended when another of the author's projects already has it. */ export async function createProject(session: Session, input: CreateProjectInput): Promise { const { files, base, ...fields } = input; const { entries, uploads } = planEntries([], await overlayFor(files, base?.files)); const record = checkRecord({ ...fields, slug: await freeSlug(session.did, fields.slug === undefined ? slugify(fields.name) : validateSlug(fields.slug)), $type: COLLECTION, files: entries, ...(base ? { base: base.ref } : {}), createdAt: new Date().toISOString(), }); return createRecord(session, await withUploads(session, record, uploads)); } /** * Writes `files` as the project's new contents. Uploads only blobs the loaded record does not already reference, * and fails with ProjectConflictError if the record changed since `project` was read. A new `fields.slug` * that another project has fails with SlugTakenError. */ export async function saveProject( session: Session, project: Project, files: FileTree, options: { base?: FileTree; fields?: Partial } = {}, ): Promise { assertOwner(session, project.did, project.uri); if (project.record.base && !options.base) { throw new SdkError(`${project.uri} has a git base; pass the base tree loadProject returned`, 'ERR_NIB_BASE_REQUIRED'); } if (options.fields?.slug !== undefined) await assertSlugFree(session.did, project, options.fields.slug); const { entries, uploads } = planEntries(project.record.files, await overlayFor(files, options.base)); const draft = checkRecord({ ...project.record, ...options.fields, files: entries, updatedAt: new Date().toISOString(), }); return putProject(session, project, await withUploads(session, draft, uploads)); } /** * Changes a project's name and slug and nothing else. The record key, and so the URI that likes, * forks, and comments point at, stays the same. A slug another project has fails with SlugTakenError. */ export async function renameProject(session: Session, project: Project, fields: { name?: string; slug?: string }): Promise { assertOwner(session, project.did, project.uri); if (fields.slug !== undefined) await assertSlugFree(session.did, project, fields.slug); const record = checkRecord({ ...project.record, ...fields, updatedAt: new Date().toISOString() }); return putProject(session, project, record); } /** * The record key of `actor`'s project that `name` names. A name shaped like a TID is read as a record * key first; anything else, or a key with no record, is matched against the author's slugs, and when * two projects share a slug the older one wins. Answers are kept for a minute, so a page that resolves * the same name twice lists the repo once. */ export async function resolveProjectName(actor: string, name: string): Promise { const { did } = await resolveActor(actor); const key = `${did}/${name}`; const cached = names.get(key); if (cached && cached.until > Date.now()) return cached.rkey; const rkey = await findProjectName(did, name); names.set(key, { rkey, until: Date.now() + NAME_TTL_MS }); return rkey; } async function findProjectName(did: Did, name: string): Promise { if (looksLikeRkey(name)) { const res = await getRecord(repoClient(did), did, COLLECTION, name); if (res.ok) return name; } const match = (await listProjects(did)).filter((p) => p.record.slug === name).sort((a, b) => a.rkey.localeCompare(b.rkey))[0]; if (!match) throw new SdkError(`${did} has no project named ${JSON.stringify(name)}`, 'ERR_NIB_NOT_FOUND'); return match.rkey; } const NAME_TTL_MS = 60_000; const names = new Map(); /** * Reads the project `ref` names. A name shaped like a record key is read directly, so the record that * answers it is fetched once; only when there is none is the name matched against slugs. */ async function readProjectRef(ref: ProjectRef): Promise<{ project: Project; client: Client }> { if (typeof ref === 'string') return readProject(ref); const { did, pds } = await resolveActor(ref.actor); if (looksLikeRkey(ref.name)) { const client = repoClient(did, pds); const res = await getRecord(client, did, COLLECTION, ref.name); if (res.ok) { const project = toProject(res.record); if (!project) throw new SdkError(`${res.record.uri} is not a valid ${COLLECTION} record`, 'ERR_NIB_RECORD_INVALID'); return { project, client }; } if (res.error !== 'RecordNotFound') throw new SdkError(`could not read ${recordUriOf(did, ref.name)}: ${res.error}`, 'ERR_NIB_RECORD_INVALID'); } return readProject(recordUriOf(did, await resolveProjectName(did, ref.name))); } const recordUriOf = (did: Did, rkey: string): ResourceUri => `at://${did}/${COLLECTION}/${rkey}`; /** `base`, or the first of `base-2`, `base-3`, … that none of `did`'s projects has. */ async function freeSlug(did: Did, base: string): Promise { const taken = new Set((await listProjects(did)).flatMap((p) => (p.record.slug ? [p.record.slug] : []))); return uniqueSlug(base, taken); } async function assertSlugFree(did: Did, project: Project, slug: string): Promise { validateSlug(slug); if (slug === project.record.slug) return; const holder = (await listProjects(did)).find((p) => p.rkey !== project.rkey && p.record.slug === slug); if (holder) throw new SlugTakenError(slug, holder.uri); } async function putProject(session: Session, project: Project, record: ProjectRecord): Promise { const res = await session.client.post('com.atproto.repo.putRecord', { input: { repo: session.did, collection: COLLECTION, rkey: project.rkey, record, swapRecord: project.cid }, }); if (!res.ok) { if (res.data.error === 'InvalidSwap') throw new ProjectConflictError(project.uri, await currentCid(session, project)); throw new SdkError(`putRecord ${project.uri} failed: ${res.data.error}`, 'ERR_NIB_RECORD_INVALID'); } if (project.record.slug !== record.slug) names.delete(`${session.did}/${project.record.slug}`); if (record.slug) names.set(`${session.did}/${record.slug}`, { rkey: project.rkey, until: Date.now() + NAME_TTL_MS }); return { ...project, uri: writtenUri(res.data.uri, session.did, COLLECTION).uri, cid: res.data.cid, record }; } /** Copies a project into the session's repo. Blobs live per repo, so each one is fetched from the source PDS and uploaded again. */ export async function forkProject(session: Session, uri: string): Promise { const { project: source, client } = await readProject(uri); const blobs = await fetchBlobs(client, source.did, source.record); const { $type, createdAt, updatedAt, forkOf, thumbnail, slug, ...rest } = source.record; const record = checkRecord({ ...rest, slug: await freeSlug(session.did, slug && isSlug(slug) ? slug : slugify(source.record.name)), $type: COLLECTION, forkOf: { uri: source.uri, cid: source.cid }, createdAt: new Date().toISOString(), }); return createRecord(session, await withUploads(session, record, blobs)); } export async function deleteProject(session: Session, uri: string): Promise { const { repo, rkey } = await parseProjectUri(uri); assertOwner(session, repo, uri); const res = await session.client.post('com.atproto.repo.deleteRecord', { input: { repo, collection: COLLECTION, rkey }, }); if (!res.ok) throw new SdkError(`deleteRecord ${uri} failed: ${res.data.error}`, 'ERR_NIB_RECORD_INVALID'); } type Entry = ProjectRecord['files'][number]; type FileEntry = DevNiblingProject.File & { $type: 'dev.nibling.project#file' }; function isFile(entry: Entry): entry is FileEntry { return entry.$type === 'dev.nibling.project#file'; } /** Old records can carry the pre-2023 blob shape, which names its CID directly. */ function blobLink(blob: BlobRef | LegacyBlob): Cid { return 'ref' in blob ? blob.ref.$link : blob.cid; } async function readProject(uri: string): Promise<{ project: Project; client: Client }> { const { repo, rkey, pds } = await parseProjectUri(uri); const client = repoClient(repo, pds); const res = await getRecord(client, repo, COLLECTION, rkey); if (!res.ok) throw new SdkError(`could not read ${uri}: ${res.error}`, 'ERR_NIB_RECORD_INVALID'); const project = toProject(res.record); if (!project) throw new SdkError(`${uri} is not a valid ${COLLECTION} record`, 'ERR_NIB_RECORD_INVALID'); return { project, client }; } function toProject({ uri, did, rkey, cid, value }: RepoRecord): Project | undefined { const parsed = safeParse(DevNiblingProject.mainSchema, value); if (!parsed.ok || !cid) return undefined; const paths = parsed.value.files.map((entry) => entry.path); if (new Set(paths).size !== paths.length) return undefined; try { paths.forEach(normalizePath); } catch { return undefined; } return { uri, cid, did, rkey, record: parsed.value }; } /** Accepts a handle or DID authority, since links people share often carry the handle. */ async function parseProjectUri(uri: string): Promise<{ repo: Did; rkey: string; pds: string }> { const match = /^at:\/\/([^/]+)\/([^/]+)\/([^/]+)$/.exec(uri); if (!match || match[2] !== COLLECTION) throw new SdkError(`${uri} is not a ${COLLECTION} URI`, 'ERR_NIB_RECORD_INVALID'); const { did, pds } = await resolveActor(match[1]!); const { repo, rkey } = parseCanonicalResourceUri(`at://${did}/${COLLECTION}/${match[3]}`); return { repo, rkey, pds }; } function assertOwner(session: Session, did: Did, uri: string): void { if (session.did !== did) throw new SdkError(`${uri} belongs to ${did}, not ${session.did}`, 'ERR_NIB_NOT_OWNER'); } async function overlayFor(files: FileTree, base: FileTree | undefined): Promise> { const current = new Map(); for (const [path, bytes] of files) current.set(normalizePath(path), bytes); const cids = await hashTree(current); const diff = diffAgainstBase(base && (await hashTree(base)), cids); return new Map([...diff].map(([path, cid]) => [path, cid === null ? null : { cid, bytes: current.get(path)! }])); } /** * Turns an overlay into record entries. A blob that `previous` already references keeps its ref; any other * gets a provisional ref with its real CID and size, and is returned in `uploads` for withUploads. */ function planEntries(previous: readonly Entry[], overlay: Overlay<{ cid: Cid; bytes: Uint8Array }>): { entries: Entry[]; uploads: Map } { const known = new Map(); const uploads = new Map(); const executable = new Set(); for (const entry of previous) { if (!isFile(entry)) continue; if ('ref' in entry.content) known.set(entry.content.ref.$link, entry.content); if (entry.executable) executable.add(entry.path); } const entries: Entry[] = []; for (const [path, file] of overlay) { if (!file) { entries.push({ $type: 'dev.nibling.project#removed', path }); continue; } let content = known.get(file.cid); if (!content) { content = { $type: 'blob', ref: { $link: file.cid }, mimeType: 'application/octet-stream', size: file.bytes.length }; known.set(file.cid, content); uploads.set(file.cid, file.bytes); } entries.push({ $type: 'dev.nibling.project#file', path, content, ...(executable.has(path) ? { executable: true } : {}), }); } return { entries, uploads }; } /** * Checks a record against the lexicon before anything is uploaded, so the SDK never writes a record its own * reader would reject. The lexicon validator does not enforce blob sizes, so they are checked here. */ function checkRecord(record: ProjectRecord): ProjectRecord { for (const entry of record.files) { if (isFile(entry) && 'size' in entry.content && entry.content.size > MAX_BLOB_BYTES) { throw new SdkError(`${entry.path} is ${entry.content.size} bytes; a project file can be at most 5 MB`, 'ERR_NIB_LIMIT'); } } const parsed = safeParse(DevNiblingProject.mainSchema, record); if (!parsed.ok) throw new SdkError(`the project record breaks the lexicon: ${parsed.message}`, 'ERR_NIB_LIMIT'); return record; } /** Uploads `uploads` and swaps each provisional ref in `record` for the one the PDS returned. */ async function withUploads(session: Session, record: ProjectRecord, uploads: ReadonlyMap): Promise { const refs = new Map(); for (const [cid, bytes] of uploads) { const ref = await uploadBlob(session, bytes); if (ref.ref.$link !== cid) throw new SdkError(`the PDS stored blob ${cid} as ${ref.ref.$link}`, 'ERR_NIB_BLOB_INVALID'); refs.set(cid, ref); } const files = record.files.map((entry) => { const ref = isFile(entry) ? refs.get(blobLink(entry.content)) : undefined; return ref ? { ...entry, content: ref } : entry; }); return { ...record, files }; } async function uploadBlob(session: Session, bytes: Uint8Array): Promise { const res = await session.client.post('com.atproto.repo.uploadBlob', { input: bytes, headers: { 'content-type': 'application/octet-stream' }, }); if (!res.ok) throw new SdkError(`uploadBlob failed: ${res.data.error}`, 'ERR_NIB_BLOB_INVALID'); return res.data.blob; } /** * Each distinct blob the record references, checked against its CID. Downloads run 8 at a time, and each * stops past the lexicon's 5 MB or past what is left of `maxBytes` for the whole project, whatever the * record or the response claims. */ async function fetchBlobs(client: Client, did: Did, record: ProjectRecord, maxBytes = MAX_PROJECT_BYTES): Promise> { const declared = new Map(); for (const entry of record.files.filter(isFile)) declared.set(blobLink(entry.content), 'size' in entry.content ? entry.content.size : 0); let total = 0; for (const [cid, size] of declared) { if (size > MAX_BLOB_BYTES) throw new SdkError(`blob ${cid} is ${size} bytes, over the 5 MB limit`, 'ERR_NIB_LIMIT'); total += size; } if (total > maxBytes) throw new SdkError(`project blobs total ${total} bytes, over the ${maxBytes}-byte limit`, 'ERR_NIB_LIMIT'); const budget = { left: maxBytes }; const blobs = new Map(); await eachLimited([...declared.keys()], BLOB_CONCURRENCY, async (cid) => { const res = await client.get('com.atproto.sync.getBlob', { params: { did, cid }, as: 'stream' }); if (!res.ok) throw new SdkError(`getBlob ${cid} from ${did} failed: ${res.data.error}`, 'ERR_NIB_BLOB_INVALID'); const bytes = await readCapped(res.data, Number(res.headers.get('content-length') ?? 0), budget, `blob ${cid} from ${did}`); if ((await blobCid(bytes)) !== cid) throw new SdkError(`blob ${cid} from ${did} does not match its CID`, 'ERR_NIB_BLOB_INVALID'); blobs.set(cid, bytes); }); return blobs; } /** Reads `stream` whole, failing once it passes 5 MB or the shared `budget`. */ async function readCapped(stream: ReadableStream, length: number, budget: { left: number }, what: string): Promise { const limit = Math.min(MAX_BLOB_BYTES, budget.left); if (length > limit) { await stream.cancel(); throw new SdkError(`${what} is ${length} bytes, over the limit`, 'ERR_NIB_LIMIT'); } const reader = stream.getReader(); const chunks: Uint8Array[] = []; let size = 0; for (let next = await reader.read(); !next.done; next = await reader.read()) { size += next.value.byteLength; budget.left -= next.value.byteLength; if (size > MAX_BLOB_BYTES || budget.left < 0) { await reader.cancel(); throw new SdkError(`${what} is over the limit`, 'ERR_NIB_LIMIT'); } chunks.push(next.value); } const bytes = new Uint8Array(size); let offset = 0; for (const chunk of chunks) { bytes.set(chunk, offset); offset += chunk.byteLength; } return bytes; } /** Runs `fn` over `items` with at most `limit` in flight, and starts no more after one fails. */ async function eachLimited(items: readonly T[], limit: number, fn: (item: T) => Promise): Promise { let next = 0; let failed = false; const worker = async () => { while (!failed && next < items.length) { try { await fn(items[next++]!); } catch (error) { failed = true; throw error; } } }; await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker)); } async function createRecord(session: Session, record: ProjectRecord): Promise { const res = await session.client.post('com.atproto.repo.createRecord', { input: { repo: session.did, collection: COLLECTION, record }, }); if (!res.ok) throw new SdkError(`createRecord failed: ${res.data.error}`, 'ERR_NIB_RECORD_INVALID'); const { uri, rkey } = writtenUri(res.data.uri, session.did, COLLECTION); if (record.slug) names.set(`${session.did}/${record.slug}`, { rkey, until: Date.now() + NAME_TTL_MS }); return { uri, cid: res.data.cid, did: session.did, rkey, record }; } async function currentCid(session: Session, project: Project): Promise { const res = await session.client.get('com.atproto.repo.getRecord', { params: { repo: session.did, collection: COLLECTION, rkey: project.rkey }, }); return res.ok ? res.data.cid : undefined; }