Something went wrong. Try again.
Monorepo for the nibling.run platform
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455import type {} from '@atcute/atproto';import type { Client } from '@atcute/client';import type { Blob as BlobRef, Cid, Did, LegacyBlob, ResourceUri } from '@atcute/lexicons';import { parseCanonicalResourceUri, safeParse } from '@atcute/lexicons';import { DevNiblingProject } from '@nibling/lexicons';import { ProjectConflictError, SdkError, SlugTakenError } from '../errors.ts';import { publicClient, repoClient, resolveActor } from '../identity.ts';import { getRecord, listRecords, writtenUri, type RepoRecord } from '../records.ts';import type { Session } from '../session.ts';import { normalizePath } from './paths.ts';import { isSlug, looksLikeRkey, slugify, uniqueSlug, validateSlug } from './slug.ts';import { applyOverlay, blobCid, diffAgainstBase, hashTree, type FileTree, type Overlay } from './tree.ts';
const COLLECTION = 'dev.nibling.project';const MAX_BLOB_BYTES = 5_000_000;const MAX_PROJECT_BYTES = 100_000_000;const BLOB_CONCURRENCY = 8;
export type ProjectRecord = DevNiblingProject.Main;
export interface Project { uri: ResourceUri; cid: Cid; did: Did; rkey: string; record: ProjectRecord;}
/** Fetches the git tree a project's overlay applies to. The tangled module provides one. */export type LoadBase = (base: DevNiblingProject.GitBase) => Promise<FileTree>;
export interface LoadedProject { project: Project; files: FileTree; /** The base tree, when the record has one. Pass it back to saveProject. */ base?: FileTree;}
/** Record fields a caller sets. `files`, timestamps, and `forkOf` are the store's. */export type ProjectFields = Pick<ProjectRecord, 'name' | 'slug' | 'description' | 'preview' | 'tags'>;
export interface CreateProjectInput extends ProjectFields { files: FileTree; base?: { ref: DevNiblingProject.GitBase; files: FileTree };}
export async function listProjects(did: Did): Promise<Project[]> { const client = await publicClient(did); const projects: Project[] = []; for await (const item of listRecords(client, did, COLLECTION)) { const project = toProject(item); if (project) projects.push(project); } return projects;}
export interface LoadProjectOptions { loadBase?: LoadBase; /** Caps the bytes downloaded for the project's blobs. Default 100 MB. */ maxBytes?: number;}
/** A project's AT-URI, or the author and the name a URL gives it: a slug or a record key. */export type ProjectRef = string | { actor: string; name: string };
export async function loadProject(ref: ProjectRef, options: LoadProjectOptions = {}): Promise<LoadedProject> { const { project, client } = await readProjectRef(ref); const uri = project.uri; const overlay: Overlay<Uint8Array> = new Map(); const blobs = await fetchBlobs(client, project.did, project.record, options.maxBytes); for (const entry of project.record.files) { overlay.set(entry.path, isFile(entry) ? blobs.get(blobLink(entry.content))! : null); }
if (!project.record.base) return { project, files: applyOverlay(new Map(), overlay) }; if (!options.loadBase) throw new SdkError(`${uri} has a git base and no loadBase was given`, 'ERR_NIB_BASE_REQUIRED'); const base = await options.loadBase(project.record.base); return { project, files: applyOverlay(base, overlay), base };}
/** * Creates a project. Its slug is `input.slug` or one made from the name, with `-2`, `-3`, and so on * appended when another of the author's projects already has it. */export async function createProject(session: Session, input: CreateProjectInput): Promise<Project> { const { files, base, ...fields } = input; const { entries, uploads } = planEntries([], await overlayFor(files, base?.files)); const record = checkRecord({ ...fields, slug: await freeSlug(session.did, fields.slug === undefined ? slugify(fields.name) : validateSlug(fields.slug)), $type: COLLECTION, files: entries, ...(base ? { base: base.ref } : {}), createdAt: new Date().toISOString(), }); return createRecord(session, await withUploads(session, record, uploads));}
/** * Writes `files` as the project's new contents. Uploads only blobs the loaded record does not already reference, * and fails with ProjectConflictError if the record changed since `project` was read. A new `fields.slug` * that another project has fails with SlugTakenError. */export async function saveProject( session: Session, project: Project, files: FileTree, options: { base?: FileTree; fields?: Partial<ProjectFields> } = {},): Promise<Project> { assertOwner(session, project.did, project.uri); if (project.record.base && !options.base) { throw new SdkError(`${project.uri} has a git base; pass the base tree loadProject returned`, 'ERR_NIB_BASE_REQUIRED'); } if (options.fields?.slug !== undefined) await assertSlugFree(session.did, project, options.fields.slug); const { entries, uploads } = planEntries(project.record.files, await overlayFor(files, options.base)); const draft = checkRecord({ ...project.record, ...options.fields, files: entries, updatedAt: new Date().toISOString(), }); return putProject(session, project, await withUploads(session, draft, uploads));}
/** * Changes a project's name and slug and nothing else. The record key, and so the URI that likes, * forks, and comments point at, stays the same. A slug another project has fails with SlugTakenError. */export async function renameProject(session: Session, project: Project, fields: { name?: string; slug?: string }): Promise<Project> { assertOwner(session, project.did, project.uri); if (fields.slug !== undefined) await assertSlugFree(session.did, project, fields.slug); const record = checkRecord({ ...project.record, ...fields, updatedAt: new Date().toISOString() }); return putProject(session, project, record);}
/** * The record key of `actor`'s project that `name` names. A name shaped like a TID is read as a record * key first; anything else, or a key with no record, is matched against the author's slugs, and when * two projects share a slug the older one wins. Answers are kept for a minute, so a page that resolves * the same name twice lists the repo once. */export async function resolveProjectName(actor: string, name: string): Promise<string> { const { did } = await resolveActor(actor); const key = `${did}/${name}`; const cached = names.get(key); if (cached && cached.until > Date.now()) return cached.rkey; const rkey = await findProjectName(did, name); names.set(key, { rkey, until: Date.now() + NAME_TTL_MS }); return rkey;}
async function findProjectName(did: Did, name: string): Promise<string> { if (looksLikeRkey(name)) { const res = await getRecord(repoClient(did), did, COLLECTION, name); if (res.ok) return name; } const match = (await listProjects(did)).filter((p) => p.record.slug === name).sort((a, b) => a.rkey.localeCompare(b.rkey))[0]; if (!match) throw new SdkError(`${did} has no project named ${JSON.stringify(name)}`, 'ERR_NIB_NOT_FOUND'); return match.rkey;}
const NAME_TTL_MS = 60_000;const names = new Map<string, { rkey: string; until: number }>();
/** * Reads the project `ref` names. A name shaped like a record key is read directly, so the record that * answers it is fetched once; only when there is none is the name matched against slugs. */async function readProjectRef(ref: ProjectRef): Promise<{ project: Project; client: Client }> { if (typeof ref === 'string') return readProject(ref); const { did, pds } = await resolveActor(ref.actor); if (looksLikeRkey(ref.name)) { const client = repoClient(did, pds); const res = await getRecord(client, did, COLLECTION, ref.name); if (res.ok) { const project = toProject(res.record); if (!project) throw new SdkError(`${res.record.uri} is not a valid ${COLLECTION} record`, 'ERR_NIB_RECORD_INVALID'); return { project, client }; } if (res.error !== 'RecordNotFound') throw new SdkError(`could not read ${recordUriOf(did, ref.name)}: ${res.error}`, 'ERR_NIB_RECORD_INVALID'); } return readProject(recordUriOf(did, await resolveProjectName(did, ref.name)));}
const recordUriOf = (did: Did, rkey: string): ResourceUri => `at://${did}/${COLLECTION}/${rkey}`;
/** `base`, or the first of `base-2`, `base-3`, … that none of `did`'s projects has. */async function freeSlug(did: Did, base: string): Promise<string> { const taken = new Set((await listProjects(did)).flatMap((p) => (p.record.slug ? [p.record.slug] : []))); return uniqueSlug(base, taken);}
async function assertSlugFree(did: Did, project: Project, slug: string): Promise<void> { validateSlug(slug); if (slug === project.record.slug) return; const holder = (await listProjects(did)).find((p) => p.rkey !== project.rkey && p.record.slug === slug); if (holder) throw new SlugTakenError(slug, holder.uri);}
async function putProject(session: Session, project: Project, record: ProjectRecord): Promise<Project> { const res = await session.client.post('com.atproto.repo.putRecord', { input: { repo: session.did, collection: COLLECTION, rkey: project.rkey, record, swapRecord: project.cid }, }); if (!res.ok) { if (res.data.error === 'InvalidSwap') throw new ProjectConflictError(project.uri, await currentCid(session, project)); throw new SdkError(`putRecord ${project.uri} failed: ${res.data.error}`, 'ERR_NIB_RECORD_INVALID'); } if (project.record.slug !== record.slug) names.delete(`${session.did}/${project.record.slug}`); if (record.slug) names.set(`${session.did}/${record.slug}`, { rkey: project.rkey, until: Date.now() + NAME_TTL_MS }); return { ...project, uri: writtenUri(res.data.uri, session.did, COLLECTION).uri, cid: res.data.cid, record };}
/** Copies a project into the session's repo. Blobs live per repo, so each one is fetched from the source PDS and uploaded again. */export async function forkProject(session: Session, uri: string): Promise<Project> { const { project: source, client } = await readProject(uri); const blobs = await fetchBlobs(client, source.did, source.record); const { $type, createdAt, updatedAt, forkOf, thumbnail, slug, ...rest } = source.record; const record = checkRecord({ ...rest, slug: await freeSlug(session.did, slug && isSlug(slug) ? slug : slugify(source.record.name)), $type: COLLECTION, forkOf: { uri: source.uri, cid: source.cid }, createdAt: new Date().toISOString(), }); return createRecord(session, await withUploads(session, record, blobs));}
export async function deleteProject(session: Session, uri: string): Promise<void> { const { repo, rkey } = await parseProjectUri(uri); assertOwner(session, repo, uri); const res = await session.client.post('com.atproto.repo.deleteRecord', { input: { repo, collection: COLLECTION, rkey }, }); if (!res.ok) throw new SdkError(`deleteRecord ${uri} failed: ${res.data.error}`, 'ERR_NIB_RECORD_INVALID');}
type Entry = ProjectRecord['files'][number];type FileEntry = DevNiblingProject.File & { $type: 'dev.nibling.project#file' };
function isFile(entry: Entry): entry is FileEntry { return entry.$type === 'dev.nibling.project#file';}
/** Old records can carry the pre-2023 blob shape, which names its CID directly. */function blobLink(blob: BlobRef | LegacyBlob): Cid { return 'ref' in blob ? blob.ref.$link : blob.cid;}
async function readProject(uri: string): Promise<{ project: Project; client: Client }> { const { repo, rkey, pds } = await parseProjectUri(uri); const client = repoClient(repo, pds); const res = await getRecord(client, repo, COLLECTION, rkey); if (!res.ok) throw new SdkError(`could not read ${uri}: ${res.error}`, 'ERR_NIB_RECORD_INVALID'); const project = toProject(res.record); if (!project) throw new SdkError(`${uri} is not a valid ${COLLECTION} record`, 'ERR_NIB_RECORD_INVALID'); return { project, client };}
function toProject({ uri, did, rkey, cid, value }: RepoRecord): Project | undefined { const parsed = safeParse(DevNiblingProject.mainSchema, value); if (!parsed.ok || !cid) return undefined; const paths = parsed.value.files.map((entry) => entry.path); if (new Set(paths).size !== paths.length) return undefined; try { paths.forEach(normalizePath); } catch { return undefined; } return { uri, cid, did, rkey, record: parsed.value };}
/** Accepts a handle or DID authority, since links people share often carry the handle. */async function parseProjectUri(uri: string): Promise<{ repo: Did; rkey: string; pds: string }> { const match = /^at:\/\/([^/]+)\/([^/]+)\/([^/]+)$/.exec(uri); if (!match || match[2] !== COLLECTION) throw new SdkError(`${uri} is not a ${COLLECTION} URI`, 'ERR_NIB_RECORD_INVALID'); const { did, pds } = await resolveActor(match[1]!); const { repo, rkey } = parseCanonicalResourceUri(`at://${did}/${COLLECTION}/${match[3]}`); return { repo, rkey, pds };}
function assertOwner(session: Session, did: Did, uri: string): void { if (session.did !== did) throw new SdkError(`${uri} belongs to ${did}, not ${session.did}`, 'ERR_NIB_NOT_OWNER');}
async function overlayFor(files: FileTree, base: FileTree | undefined): Promise<Overlay<{ cid: Cid; bytes: Uint8Array }>> { const current = new Map<string, Uint8Array>(); for (const [path, bytes] of files) current.set(normalizePath(path), bytes); const cids = await hashTree(current); const diff = diffAgainstBase(base && (await hashTree(base)), cids); return new Map([...diff].map(([path, cid]) => [path, cid === null ? null : { cid, bytes: current.get(path)! }]));}
/** * Turns an overlay into record entries. A blob that `previous` already references keeps its ref; any other * gets a provisional ref with its real CID and size, and is returned in `uploads` for withUploads. */function planEntries(previous: readonly Entry[], overlay: Overlay<{ cid: Cid; bytes: Uint8Array }>): { entries: Entry[]; uploads: Map<Cid, Uint8Array> } { const known = new Map<Cid, BlobRef>(); const uploads = new Map<Cid, Uint8Array>(); const executable = new Set<string>(); for (const entry of previous) { if (!isFile(entry)) continue; if ('ref' in entry.content) known.set(entry.content.ref.$link, entry.content); if (entry.executable) executable.add(entry.path); } const entries: Entry[] = []; for (const [path, file] of overlay) { if (!file) { entries.push({ $type: 'dev.nibling.project#removed', path }); continue; } let content = known.get(file.cid); if (!content) { content = { $type: 'blob', ref: { $link: file.cid }, mimeType: 'application/octet-stream', size: file.bytes.length }; known.set(file.cid, content); uploads.set(file.cid, file.bytes); } entries.push({ $type: 'dev.nibling.project#file', path, content, ...(executable.has(path) ? { executable: true } : {}), }); } return { entries, uploads };}
/** * Checks a record against the lexicon before anything is uploaded, so the SDK never writes a record its own * reader would reject. The lexicon validator does not enforce blob sizes, so they are checked here. */function checkRecord(record: ProjectRecord): ProjectRecord { for (const entry of record.files) { if (isFile(entry) && 'size' in entry.content && entry.content.size > MAX_BLOB_BYTES) { throw new SdkError(`${entry.path} is ${entry.content.size} bytes; a project file can be at most 5 MB`, 'ERR_NIB_LIMIT'); } } const parsed = safeParse(DevNiblingProject.mainSchema, record); if (!parsed.ok) throw new SdkError(`the project record breaks the lexicon: ${parsed.message}`, 'ERR_NIB_LIMIT'); return record;}
/** Uploads `uploads` and swaps each provisional ref in `record` for the one the PDS returned. */async function withUploads(session: Session, record: ProjectRecord, uploads: ReadonlyMap<Cid, Uint8Array>): Promise<ProjectRecord> { const refs = new Map<Cid, BlobRef>(); for (const [cid, bytes] of uploads) { const ref = await uploadBlob(session, bytes); if (ref.ref.$link !== cid) throw new SdkError(`the PDS stored blob ${cid} as ${ref.ref.$link}`, 'ERR_NIB_BLOB_INVALID'); refs.set(cid, ref); } const files = record.files.map((entry) => { const ref = isFile(entry) ? refs.get(blobLink(entry.content)) : undefined; return ref ? { ...entry, content: ref } : entry; }); return { ...record, files };}
async function uploadBlob(session: Session, bytes: Uint8Array): Promise<BlobRef> { const res = await session.client.post('com.atproto.repo.uploadBlob', { input: bytes, headers: { 'content-type': 'application/octet-stream' }, }); if (!res.ok) throw new SdkError(`uploadBlob failed: ${res.data.error}`, 'ERR_NIB_BLOB_INVALID'); return res.data.blob;}
/** * Each distinct blob the record references, checked against its CID. Downloads run 8 at a time, and each * stops past the lexicon's 5 MB or past what is left of `maxBytes` for the whole project, whatever the * record or the response claims. */async function fetchBlobs(client: Client, did: Did, record: ProjectRecord, maxBytes = MAX_PROJECT_BYTES): Promise<Map<Cid, Uint8Array>> { const declared = new Map<Cid, number>(); for (const entry of record.files.filter(isFile)) declared.set(blobLink(entry.content), 'size' in entry.content ? entry.content.size : 0); let total = 0; for (const [cid, size] of declared) { if (size > MAX_BLOB_BYTES) throw new SdkError(`blob ${cid} is ${size} bytes, over the 5 MB limit`, 'ERR_NIB_LIMIT'); total += size; } if (total > maxBytes) throw new SdkError(`project blobs total ${total} bytes, over the ${maxBytes}-byte limit`, 'ERR_NIB_LIMIT');
const budget = { left: maxBytes }; const blobs = new Map<Cid, Uint8Array>(); await eachLimited([...declared.keys()], BLOB_CONCURRENCY, async (cid) => { const res = await client.get('com.atproto.sync.getBlob', { params: { did, cid }, as: 'stream' }); if (!res.ok) throw new SdkError(`getBlob ${cid} from ${did} failed: ${res.data.error}`, 'ERR_NIB_BLOB_INVALID'); const bytes = await readCapped(res.data, Number(res.headers.get('content-length') ?? 0), budget, `blob ${cid} from ${did}`); if ((await blobCid(bytes)) !== cid) throw new SdkError(`blob ${cid} from ${did} does not match its CID`, 'ERR_NIB_BLOB_INVALID'); blobs.set(cid, bytes); }); return blobs;}
/** Reads `stream` whole, failing once it passes 5 MB or the shared `budget`. */async function readCapped(stream: ReadableStream<Uint8Array>, length: number, budget: { left: number }, what: string): Promise<Uint8Array> { const limit = Math.min(MAX_BLOB_BYTES, budget.left); if (length > limit) { await stream.cancel(); throw new SdkError(`${what} is ${length} bytes, over the limit`, 'ERR_NIB_LIMIT'); } const reader = stream.getReader(); const chunks: Uint8Array[] = []; let size = 0; for (let next = await reader.read(); !next.done; next = await reader.read()) { size += next.value.byteLength; budget.left -= next.value.byteLength; if (size > MAX_BLOB_BYTES || budget.left < 0) { await reader.cancel(); throw new SdkError(`${what} is over the limit`, 'ERR_NIB_LIMIT'); } chunks.push(next.value); } const bytes = new Uint8Array(size); let offset = 0; for (const chunk of chunks) { bytes.set(chunk, offset); offset += chunk.byteLength; } return bytes;}
/** Runs `fn` over `items` with at most `limit` in flight, and starts no more after one fails. */async function eachLimited<T>(items: readonly T[], limit: number, fn: (item: T) => Promise<void>): Promise<void> { let next = 0; let failed = false; const worker = async () => { while (!failed && next < items.length) { try { await fn(items[next++]!); } catch (error) { failed = true; throw error; } } }; await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));}
async function createRecord(session: Session, record: ProjectRecord): Promise<Project> { const res = await session.client.post('com.atproto.repo.createRecord', { input: { repo: session.did, collection: COLLECTION, record }, }); if (!res.ok) throw new SdkError(`createRecord failed: ${res.data.error}`, 'ERR_NIB_RECORD_INVALID'); const { uri, rkey } = writtenUri(res.data.uri, session.did, COLLECTION); if (record.slug) names.set(`${session.did}/${record.slug}`, { rkey, until: Date.now() + NAME_TTL_MS }); return { uri, cid: res.data.cid, did: session.did, rkey, record };}
async function currentCid(session: Session, project: Project): Promise<Cid | undefined> { const res = await session.client.get('com.atproto.repo.getRecord', { params: { repo: session.did, collection: COLLECTION, rkey: project.rkey }, }); return res.ok ? res.data.cid : undefined;}