diff --git a/server/utils/job-handlers.ts b/server/utils/job-handlers.ts index c8f5780..145c75f 100644 --- a/server/utils/job-handlers.ts +++ b/server/utils/job-handlers.ts @@ -5,14 +5,15 @@ import { useDb } from './db' import { installationOctokit } from './github-app' import type { JobEnvelope } from './queue' import { enqueue } from './queue' +import { type CreateRefPayload, type DeleteRefPayload, syncCreateRef, syncDeleteRef } from './sync-ref' import { syncPush, type PushPayload } from './sync-push' import { generateAndPublishKey } from './tangled-pubkey' import { enrollRepo } from './tangled-repo' /** * Map of job kind → handler. Each commit fills in its slice: - * - 'github.push' → this commit (sync push events) - * - 'github.create' / 'github.delete' → commit 13 (branch/tag ref ops) + * - 'github.push' → commit 12 (sync push events) + * - 'github.create' / 'github.delete' → this commit (branch/tag ref ops) * - 'github.repository' → commit 14/15 (description, lifecycle) * - 'github.installation_repositories' → commit 10 (fan-out enrolment) * - 'tangled.backfill-installation' → commit 10 (paginate + fan-out) @@ -88,6 +89,24 @@ function backfillInstallationPayload(value: unknown): BackfillInstallationPayloa return { installationId: o.installationId, page: o.page } } +function refPayload(kind: 'create' | 'delete', value: unknown): CreateRefPayload { + const o = asObject(value) + if ( + typeof o.installationId !== 'number' + || typeof o.githubRepoId !== 'number' + || (o.refType !== 'branch' && o.refType !== 'tag') + || typeof o.ref !== 'string' + ) { + throw new TypeError(`invalid github.${kind} payload`) + } + return { + installationId: o.installationId, + githubRepoId: o.githubRepoId, + refType: o.refType, + ref: o.ref, + } +} + function installationRepositoriesPayload(value: unknown): InstallationRepositoriesPayload { const o = asObject(value) if ( @@ -116,6 +135,16 @@ export async function dispatch(envelope: JobEnvelope): Promise { return } + if (envelope.kind === 'github.create') { + await syncCreateRef(refPayload('create', envelope.payload)) + return + } + + if (envelope.kind === 'github.delete') { + await syncDeleteRef(refPayload('delete', envelope.payload) as DeleteRefPayload) + return + } + if (envelope.kind === 'atproto.publish-pubkey') { const { did, installationId } = publishPubkeyPayload(envelope.payload) const client = await useOAuthClient() diff --git a/server/utils/sync-push-host.ts b/server/utils/sync-push-host.ts new file mode 100644 index 0000000..ef8bdae --- /dev/null +++ b/server/utils/sync-push-host.ts @@ -0,0 +1,16 @@ +/** + * Map a knot hostname (as stored on `sh.tangled.repo`) to the SSH host we + * actually push to. For the appview-hosted knot, the HTTPS XRPC endpoint is + * `knot1.tangled.sh` (Cloudflare-fronted) but SSH lives on `tangled.org`. + * Self-hosted knots serve both on the same host (their `knot` value may + * include a `:port` suffix for non-default SSH; git URL parsing handles it). + * + * The official UI does this same mapping in + * `appview/pages/templates/repo/empty.html`. If tangled adds more + * appview-hosted knots in future this'll need updating \u2014 see PLAN.md + * "Deferred / follow-ups". + */ +export function sshHostForKnot(knot: string): string { + if (knot === 'knot1.tangled.sh') return 'tangled.org' + return knot +} diff --git a/server/utils/sync-push.ts b/server/utils/sync-push.ts index 9096e9e..273ee3b 100644 --- a/server/utils/sync-push.ts +++ b/server/utils/sync-push.ts @@ -7,6 +7,7 @@ import { useDb } from './db' import { classifyPushFailure, git, RemoteRejectedPushError } from './git' import { installationOctokit } from './github-app' import { loadSshCommandForInstall } from './ssh-cmd' +import { sshHostForKnot } from './sync-push-host' const ZERO_SHA = '0000000000000000000000000000000000000000' @@ -145,22 +146,6 @@ function jsonbPath(ref: string): string { return `"${ref.replaceAll('"', '\\"')}"` } -/** - * Map a knot hostname (as stored on `sh.tangled.repo`) to the SSH host we - * actually push to. For the appview-hosted knot, the HTTPS XRPC endpoint is - * `knot1.tangled.sh` (Cloudflare-fronted) but SSH lives on `tangled.org`. - * Self-hosted knots serve both on the same host (their `knot` value may - * include a `:port` suffix for non-default SSH; git URL parsing handles it). - * - * The official UI does this same mapping in - * `appview/pages/templates/repo/empty.html`. If tangled adds more - * appview-hosted knots in future this'll need updating. - */ -function sshHostForKnot(knot: string): string { - if (knot === 'knot1.tangled.sh') return 'tangled.org' - return knot -} - async function markMappingError(mappingId: number, message: string): Promise { const db = useDb() await db.update(repoMapping) diff --git a/server/utils/sync-ref.ts b/server/utils/sync-ref.ts new file mode 100644 index 0000000..2509b4f --- /dev/null +++ b/server/utils/sync-ref.ts @@ -0,0 +1,238 @@ +import { mkdtempSync, rmSync } from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { and, eq, sql } from 'drizzle-orm' +import { repoMapping } from '../db/schema' +import { useDb } from './db' +import { classifyPushFailure, git } from './git' +import { installationOctokit } from './github-app' +import { loadSshCommandForInstall } from './ssh-cmd' +import { sshHostForKnot } from './sync-push-host' + +export type RefType = 'branch' | 'tag' + +export interface CreateRefPayload { + installationId: number + githubRepoId: number + refType: RefType + /** Short ref name as GitHub delivers it (e.g. `v1.0`, `feature-x`) \u2014 NOT + * the `refs/...` qualified form. */ + ref: string +} + +export interface DeleteRefPayload extends CreateRefPayload {} + +export interface RefResult { + status: 'synced' | 'skipped' + reason?: 'no-mapping' | 'disabled' | 'not-branch-or-tag' | 'repo-gone' +} + +/** + * Mirror a branch or tag creation from GitHub to the configured knot. + * + * Triggered by GitHub's `create` webhook event. For branches, GitHub also + * sends a parallel `push` event (with `before = 0000…`), so the branch will + * usually have been created already by the time this fires \u2014 the push to + * knot is then a no-op via ref-tip dedupe. For lightweight and annotated + * tags, no `push` event is sent, so this is the only path that creates them + * on the knot. + */ +export async function syncCreateRef(payload: CreateRefPayload): Promise { + if (payload.refType !== 'branch' && payload.refType !== 'tag') { + return { status: 'skipped', reason: 'not-branch-or-tag' } + } + + const mapping = await loadActiveMapping(payload.installationId, payload.githubRepoId) + if ('skip' in mapping) return mapping.skip + + const fullRef = qualifyRef(payload.refType, payload.ref) + const tmpDir = mkdtempSync(path.join(os.tmpdir(), 'synchub-create-')) + let sshCleanup: (() => void) | undefined + + try { + await git(['init', '--bare', '-q'], { cwd: tmpDir }) + + const octokit = await installationOctokit(payload.installationId) + const { token } = (await octokit.auth({ type: 'installation' })) as { token: string } + const githubUrl = `https://x-access-token:${token}@github.com/${mapping.githubFullName}.git` + + // Fetch the ref by name. Tags carry whatever object git stores at the + // ref (commit for lightweight; tag object for annotated); fetch gives + // us all the reachable objects either way. + await git( + ['fetch', '--no-tags', '-q', githubUrl, `+${fullRef}:${fullRef}`], + { cwd: tmpDir, timeout: 120_000 }, + ) + + const { gitSshCommand, cleanup } = await loadSshCommandForInstall(payload.installationId) + sshCleanup = cleanup + const knotUrl = `ssh://git@${sshHostForKnot(mapping.knot)}/${mapping.tangledRepoDid}` + + try { + await git( + ['push', '-q', knotUrl, `+${fullRef}:${fullRef}`], + { cwd: tmpDir, env: { GIT_SSH_COMMAND: gitSshCommand }, timeout: 120_000 }, + ) + } + catch (err) { + const stderr = err instanceof Error && 'stderr' in err ? String((err as { stderr: unknown }).stderr) : '' + const classified = classifyPushFailure(stderr) + if (classified?.reason === 'repo-gone') { + await markMappingError(mapping.id, 'knot reports repo no longer exists; stopping sync') + return { status: 'skipped', reason: 'repo-gone' } + } + throw classified ?? err + } + + // For branches we get the SHA from the local ref after fetch; for tags + // we still update lastSyncedRefs so a subsequent push event with the + // same SHA short-circuits via ref-tip dedupe. + const { stdout: sha } = await git(['rev-parse', fullRef], { cwd: tmpDir }) + await updateLastSyncedRef(mapping.id, fullRef, sha.trim()) + + return { status: 'synced' } + } + finally { + sshCleanup?.() + try { + rmSync(tmpDir, { recursive: true, force: true }) + } + catch { + // best-effort + } + } +} + +/** + * Mirror a branch or tag deletion from GitHub to the configured knot. + * + * Triggered by GitHub's `delete` webhook event. For branches, GitHub also + * sends a parallel `push` event with `after = 0000\u2026`, which `syncPush` + * currently skips (`reason: 'deletion'`) \u2014 this is the path that actually + * removes the ref on the knot. Tag deletion arrives only via this event. + * + * Deletion is idempotent: if the ref doesn't exist on the knot we treat it + * as success. We wanted it gone, it's gone. + */ +export async function syncDeleteRef(payload: DeleteRefPayload): Promise { + if (payload.refType !== 'branch' && payload.refType !== 'tag') { + return { status: 'skipped', reason: 'not-branch-or-tag' } + } + + const mapping = await loadActiveMapping(payload.installationId, payload.githubRepoId) + if ('skip' in mapping) return mapping.skip + + const fullRef = qualifyRef(payload.refType, payload.ref) + const tmpDir = mkdtempSync(path.join(os.tmpdir(), 'synchub-delete-')) + let sshCleanup: (() => void) | undefined + + try { + // No fetch needed; we're only telling the remote to drop a ref. + await git(['init', '--bare', '-q'], { cwd: tmpDir }) + + const { gitSshCommand, cleanup } = await loadSshCommandForInstall(payload.installationId) + sshCleanup = cleanup + const knotUrl = `ssh://git@${sshHostForKnot(mapping.knot)}/${mapping.tangledRepoDid}` + + // The `:` (empty source) refspec means "delete on the remote". + try { + await git( + ['push', '-q', knotUrl, `:${fullRef}`], + { cwd: tmpDir, env: { GIT_SSH_COMMAND: gitSshCommand }, timeout: 60_000 }, + ) + } + catch (err) { + const stderr = err instanceof Error && 'stderr' in err ? String((err as { stderr: unknown }).stderr) : '' + // "remote ref does not exist" is success for our purposes \u2014 the ref is + // gone, which is what we wanted. + if (/remote ref does not exist|unable to delete.*does not exist/i.test(stderr)) { + await clearLastSyncedRef(mapping.id, fullRef) + return { status: 'synced' } + } + const classified = classifyPushFailure(stderr) + if (classified?.reason === 'repo-gone') { + await markMappingError(mapping.id, 'knot reports repo no longer exists; stopping sync') + return { status: 'skipped', reason: 'repo-gone' } + } + throw classified ?? err + } + + await clearLastSyncedRef(mapping.id, fullRef) + return { status: 'synced' } + } + finally { + sshCleanup?.() + try { + rmSync(tmpDir, { recursive: true, force: true }) + } + catch { + // best-effort + } + } +} + +function qualifyRef(refType: RefType, ref: string): string { + return refType === 'tag' ? `refs/tags/${ref}` : `refs/heads/${ref}` +} + +type ActiveMapping = { + id: number + githubFullName: string + tangledRepoDid: string + knot: string +} | { skip: RefResult } + +async function loadActiveMapping(installationId: number, githubRepoId: number): Promise { + const db = useDb() + const rows = await db.select().from(repoMapping).where( + and( + eq(repoMapping.installationId, installationId), + eq(repoMapping.githubRepoId, githubRepoId), + ), + ).limit(1) + + if (rows.length === 0) return { skip: { status: 'skipped', reason: 'no-mapping' } } + const row = rows[0]! + + if (row.disabledAt) return { skip: { status: 'skipped', reason: 'disabled' } } + if (!row.tangledRepoDid || !row.knot) return { skip: { status: 'skipped', reason: 'no-mapping' } } + + return { + id: row.id, + githubFullName: row.githubFullName, + tangledRepoDid: row.tangledRepoDid, + knot: row.knot, + } +} + +async function updateLastSyncedRef(mappingId: number, fullRef: string, sha: string): Promise { + const db = useDb() + await db.update(repoMapping) + .set({ + lastSyncedRefs: sql`jsonb_set(${repoMapping.lastSyncedRefs}, ${`{${jsonbPath(fullRef)}}`}::text[], ${`"${sha}"`}::jsonb, true)`, + updatedAt: new Date(), + }) + .where(eq(repoMapping.id, mappingId)) +} + +async function clearLastSyncedRef(mappingId: number, fullRef: string): Promise { + const db = useDb() + // jsonb minus text removes a top-level key. Safe no-op if absent. + await db.update(repoMapping) + .set({ + lastSyncedRefs: sql`${repoMapping.lastSyncedRefs} - ${fullRef}`, + updatedAt: new Date(), + }) + .where(eq(repoMapping.id, mappingId)) +} + +async function markMappingError(mappingId: number, message: string): Promise { + const db = useDb() + await db.update(repoMapping) + .set({ status: 'error', lastError: message, updatedAt: new Date() }) + .where(eq(repoMapping.id, mappingId)) +} + +function jsonbPath(ref: string): string { + return `"${ref.replaceAll('"', '\\"')}"` +} diff --git a/test/unit/sync-ref.spec.ts b/test/unit/sync-ref.spec.ts new file mode 100644 index 0000000..15a37d7 --- /dev/null +++ b/test/unit/sync-ref.spec.ts @@ -0,0 +1,254 @@ +import crypto from 'node:crypto' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { installation, repoMapping } from '../../server/db/schema' +import { clearDb, setDb, useDb } from '../../server/utils/db' +import { clearEncryptionKeyCache } from '../../server/utils/encryption' +import { createTestDb } from '../utils/db' + +const ORIGINAL_ENC_KEY = process.env.NUXT_ENCRYPTION_KEY + +// Mock git + ssh + octokit so we can exercise mapping-lookup + envelope +// branches without invoking real binaries. +const gitMock = vi.fn<(args: string[], opts?: unknown) => Promise<{ stdout: string, stderr: string }>>() +const sshLoadMock = vi.fn<(installationId: number) => Promise<{ gitSshCommand: string, cleanup: () => void }>>() +const octokitAuthMock = vi.fn<(input: { type: 'installation' }) => Promise<{ token: string }>>() + +vi.mock('../../server/utils/git', async () => { + const actual = await vi.importActual('../../server/utils/git') + return { + ...actual, + git: (args: string[], opts: unknown) => gitMock(args, opts), + } +}) + +vi.mock('../../server/utils/ssh-cmd', () => ({ + loadSshCommandForInstall: (id: number) => sshLoadMock(id), +})) + +vi.mock('../../server/utils/github-app', () => ({ + installationOctokit: async () => ({ + auth: octokitAuthMock, + }), +})) + +const { syncCreateRef, syncDeleteRef } = await import('../../server/utils/sync-ref') + +describe('sync-ref', () => { + beforeEach(async () => { + process.env.NUXT_ENCRYPTION_KEY = crypto.randomBytes(32).toString('base64') + clearEncryptionKeyCache() + + setDb(await createTestDb()) + const db = useDb() + await db.insert(installation).values({ + id: 1, accountLogin: 'alice', accountId: 100, accountType: 'User', + }) + + gitMock.mockReset() + sshLoadMock.mockReset() + octokitAuthMock.mockReset() + + sshLoadMock.mockResolvedValue({ gitSshCommand: 'ssh -i /tmp/key', cleanup: () => {} }) + octokitAuthMock.mockResolvedValue({ token: 'install-token' }) + gitMock.mockResolvedValue({ stdout: 'abc1234567890abc1234567890abc1234567890a', stderr: '' }) + }) + + afterEach(() => { + if (ORIGINAL_ENC_KEY === undefined) delete process.env.NUXT_ENCRYPTION_KEY + else process.env.NUXT_ENCRYPTION_KEY = ORIGINAL_ENC_KEY + clearEncryptionKeyCache() + clearDb() + }) + + async function seedMapping(over: Partial = {}) { + const db = useDb() + await db.insert(repoMapping).values({ + installationId: 1, + githubRepoId: 9001, + githubFullName: 'alice/my-project', + tangledRepoDid: 'did:plc:repo-xyz', + tangledFullName: 'did:plc:abc/my-project', + knot: 'knot1.tangled.sh', + status: 'active', + ...over, + }) + } + + describe('syncCreateRef', () => { + it('skips non-branch/tag ref types', async () => { + await seedMapping() + const result = await syncCreateRef({ + installationId: 1, + githubRepoId: 9001, + refType: 'repository' as never, + ref: 'whatever', + }) + expect(result).toEqual({ status: 'skipped', reason: 'not-branch-or-tag' }) + expect(gitMock).not.toHaveBeenCalled() + }) + + it('skips when no mapping exists', async () => { + const result = await syncCreateRef({ + installationId: 1, + githubRepoId: 9001, + refType: 'branch', + ref: 'main', + }) + expect(result).toEqual({ status: 'skipped', reason: 'no-mapping' }) + }) + + it('skips when mapping is disabled', async () => { + await seedMapping({ disabledAt: new Date() }) + const result = await syncCreateRef({ + installationId: 1, + githubRepoId: 9001, + refType: 'branch', + ref: 'main', + }) + expect(result).toEqual({ status: 'skipped', reason: 'disabled' }) + }) + + it('qualifies branch refs as refs/heads/', async () => { + await seedMapping() + await syncCreateRef({ + installationId: 1, + githubRepoId: 9001, + refType: 'branch', + ref: 'feature-x', + }) + + // git init, git fetch, git push, git rev-parse + const calls = gitMock.mock.calls.map(c => c[0]) + const fetch = calls.find(args => args[0] === 'fetch') + const push = calls.find(args => args[0] === 'push') + expect(fetch).toBeDefined() + expect(fetch).toContain('+refs/heads/feature-x:refs/heads/feature-x') + expect(push).toContain('+refs/heads/feature-x:refs/heads/feature-x') + }) + + it('qualifies tag refs as refs/tags/', async () => { + await seedMapping() + await syncCreateRef({ + installationId: 1, + githubRepoId: 9001, + refType: 'tag', + ref: 'v1.0.0', + }) + const calls = gitMock.mock.calls.map(c => c[0]) + const fetch = calls.find(args => args[0] === 'fetch') + expect(fetch).toContain('+refs/tags/v1.0.0:refs/tags/v1.0.0') + }) + + it('routes push for knot1.tangled.sh via tangled.org', async () => { + await seedMapping() + await syncCreateRef({ + installationId: 1, + githubRepoId: 9001, + refType: 'branch', + ref: 'main', + }) + const push = gitMock.mock.calls.map(c => c[0]).find(a => a[0] === 'push')! + const url = push.find(a => a.startsWith('ssh://'))! + expect(url).toContain('@tangled.org/') + }) + + it('updates lastSyncedRefs with the fetched SHA', async () => { + await seedMapping() + gitMock.mockImplementation(async args => { + if (args[0] === 'rev-parse') { + return { stdout: 'deadbeef1234567890deadbeef1234567890dead', stderr: '' } + } + return { stdout: '', stderr: '' } + }) + + await syncCreateRef({ + installationId: 1, + githubRepoId: 9001, + refType: 'branch', + ref: 'main', + }) + + const db = useDb() + const rows = await db.select().from(repoMapping) + const refs = rows[0].lastSyncedRefs as Record + expect(refs['refs/heads/main']).toBe('deadbeef1234567890deadbeef1234567890dead') + }) + }) + + describe('syncDeleteRef', () => { + it('uses the empty-source refspec to delete on the remote', async () => { + await seedMapping() + await syncDeleteRef({ + installationId: 1, + githubRepoId: 9001, + refType: 'branch', + ref: 'old-branch', + }) + const push = gitMock.mock.calls.map(c => c[0]).find(a => a[0] === 'push')! + expect(push).toContain(':refs/heads/old-branch') + expect(push.some(s => s.startsWith(':'))).toBe(true) + }) + + it('treats "remote ref does not exist" as success', async () => { + await seedMapping() + gitMock.mockImplementation(async args => { + if (args[0] === 'push') { + throw Object.assign(new Error('exit 1'), { + stderr: 'error: unable to delete \'refs/tags/v9\': remote ref does not exist\n', + }) + } + return { stdout: '', stderr: '' } + }) + + const result = await syncDeleteRef({ + installationId: 1, + githubRepoId: 9001, + refType: 'tag', + ref: 'v9', + }) + expect(result).toEqual({ status: 'synced' }) + }) + + it('removes the ref from lastSyncedRefs', async () => { + const db = useDb() + await seedMapping({ + lastSyncedRefs: { 'refs/heads/main': 'abc', 'refs/heads/old': 'def' }, + }) + + await syncDeleteRef({ + installationId: 1, + githubRepoId: 9001, + refType: 'branch', + ref: 'old', + }) + + const rows = await db.select().from(repoMapping) + const refs = rows[0].lastSyncedRefs as Record + expect(refs).toEqual({ 'refs/heads/main': 'abc' }) + }) + + it('marks mapping as error if knot reports repo gone', async () => { + await seedMapping() + gitMock.mockImplementation(async args => { + if (args[0] === 'push') { + throw Object.assign(new Error('exit 1'), { + stderr: 'fatal: repository not found\n', + }) + } + return { stdout: '', stderr: '' } + }) + + const result = await syncDeleteRef({ + installationId: 1, + githubRepoId: 9001, + refType: 'branch', + ref: 'main', + }) + expect(result).toEqual({ status: 'skipped', reason: 'repo-gone' }) + + const db = useDb() + const rows = await db.select().from(repoMapping) + expect(rows[0].status).toBe('error') + }) + }) +})