diff --git a/packages/web/server/api/_tasks/[name].post.ts b/packages/web/server/api/_tasks/[name].post.ts index fc2d6c0..a32c038 100644 --- a/packages/web/server/api/_tasks/[name].post.ts +++ b/packages/web/server/api/_tasks/[name].post.ts @@ -1,6 +1,5 @@ import { Octokit } from '@octokit/rest' -import { indexRepo } from '~~/server/utils/index-repo' -import { getMetadataForRepo } from '~~/server/utils/metadata' +import { indexRepos } from '~~/server/utils/index-repos' export default defineEventHandler(async (event) => { const config = useRuntimeConfig(event) @@ -17,7 +16,11 @@ export default defineEventHandler(async (event) => { try { if (name === 'index-repo') { - return await indexRepoHandler(config.github, body) + const octokit = new Octokit({ auth: config.github.token }) + return await indexRepos(octokit, { + filter: body.filter as string[] | undefined, + timeBudgetMs: typeof body.timeBudgetMs === 'number' ? body.timeBudgetMs : undefined, + }) } const result = await runTask(name, { @@ -36,59 +39,3 @@ export default defineEventHandler(async (event) => { }) } }) - -async function indexRepoHandler(github: { token: string }, body: Record) { - const octokit = new Octokit({ auth: github.token }) - const filter = body.filter as string[] | undefined - const timeBudgetMs = typeof body.timeBudgetMs === 'number' ? body.timeBudgetMs : undefined - - const repos = await $fetch('/api/repos') - const indexed: string[] = [] - const partial: string[] = [] - - // Also check for repos with a non-zero cursor (partially indexed from a previous run) - const allRepos = await useDrizzle().select({ - fullName: tables.repos.full_name, - indexCursor: tables.repos.indexCursor, - indexedVersion: tables.repos.indexed, - }).from(tables.repos).all() - const cursorMap = new Map(allRepos.map(r => [r.fullName, r.indexCursor])) - - for (const repo of repos as { repo: string, indexed: boolean }[]) { - const cursor = cursorMap.get(repo.repo) || 0 - const needsIndexing = !repo.indexed || cursor > 0 - if (!needsIndexing || (filter && !filter.includes(repo.repo))) - continue - - try { - const [owner, name] = repo.repo.split('/') - const meta = await getMetadataForRepo(owner!, name!) - if (!meta) - continue - - const result = await indexRepo(octokit, meta, { timeBudgetMs }) - - if (result.complete) { - indexed.push(repo.repo) - } - else { - partial.push(repo.repo) - // Stop processing more repos — this one consumed the time budget - break - } - } - catch (e) { - console.error('Error indexing', repo.repo, e) - } - } - - const parts = [] - if (indexed.length) - parts.push(`Fully indexed: ${indexed.join(', ')}`) - if (partial.length) - parts.push(`Partially indexed (will resume): ${partial.join(', ')}`) - if (!parts.length) - parts.push('No repos needed indexing') - - return { result: `${parts.join('. ')}.` } -} diff --git a/packages/web/server/tasks/index-repo.ts b/packages/web/server/tasks/index-repo.ts index 932da7c..7f8594a 100644 --- a/packages/web/server/tasks/index-repo.ts +++ b/packages/web/server/tasks/index-repo.ts @@ -1,6 +1,6 @@ import { Octokit } from '@octokit/rest' import { defineTask } from 'nitropack/runtime' -import { indexRepo } from '~~/server/utils/index-repo' +import { indexRepos } from '~~/server/utils/index-repos' export interface TaskPayload { /** @@ -23,55 +23,9 @@ export default defineTask({ const github = payload._context?.github || useRuntimeConfig().github const octokit = new Octokit({ auth: github.token }) - const repos = await $fetch('/api/repos') - const indexed: string[] = [] - const partial: string[] = [] - - // Also check for repos with a non-zero cursor (partially indexed from a previous run) - const allRepos = await useDrizzle().select({ - fullName: tables.repos.full_name, - indexCursor: tables.repos.indexCursor, - }).from(tables.repos).all() - const cursorMap = new Map(allRepos.map(r => [r.fullName, r.indexCursor])) - - for (const repo of repos) { - const cursor = cursorMap.get(repo.repo) || 0 - const needsIndexing = !repo.indexed || cursor > 0 - if (!needsIndexing || (payload.filter && !payload.filter.includes(repo.repo))) - continue - - try { - const [owner, name] = repo.repo.split('/') - const meta = await getMetadataForRepo(owner!, name!) - if (!meta) - continue - - const result = await indexRepo(octokit, meta, { timeBudgetMs: payload.timeBudgetMs }) - - if (result.complete) { - indexed.push(repo.repo) - } - else { - partial.push(repo.repo) - // Stop processing more repos — this one consumed the time budget - break - } - } - catch (e) { - console.error(e) - } - } - - const parts = [] - if (indexed.length) - parts.push(`Fully indexed: ${indexed.join(', ')}`) - if (partial.length) - parts.push(`Partially indexed (will resume): ${partial.join(', ')}`) - if (!parts.length) - parts.push('No repos needed indexing') - - return { - result: `${parts.join('. ')}.`, - } + return await indexRepos(octokit, { + filter: payload.filter, + timeBudgetMs: payload.timeBudgetMs, + }) }, }) diff --git a/packages/web/server/utils/index-repo.ts b/packages/web/server/utils/index-repo.ts index 7fb8009..b4971c6 100644 --- a/packages/web/server/utils/index-repo.ts +++ b/packages/web/server/utils/index-repo.ts @@ -1,5 +1,7 @@ import type { Octokit } from '@octokit/rest' import type { InstallationRepo } from '~~/server/utils/metadata' + +import { count } from 'drizzle-orm' import { indexIssue } from '~~/server/utils/embeddings' import { currentIndexVersion } from '~~/server/utils/metadata' @@ -13,6 +15,10 @@ export interface IndexRepoResult { skipped: number /** Whether all pages were processed (false = timed out, needs another invocation) */ complete: boolean + /** Current page the cursor is at (0 if complete) */ + currentPage: number + /** Number of issues already stored in the DB for this repo */ + totalStored: number } /** @@ -29,7 +35,7 @@ export async function indexRepo(octokit: Octokit, repo: InstallationRepo, option const { timeBudgetMs = 20_000 } = options if (repo.private) { - return { indexed: 0, skipped: 0, complete: true } + return { indexed: 0, skipped: 0, complete: true, currentPage: 0, totalStored: 0 } } const [owner, name] = repo.full_name.split('/') @@ -38,7 +44,7 @@ export async function indexRepo(octokit: Octokit, repo: InstallationRepo, option const repoRow = await drizzle.select({ repoId: tables.repos.id, indexCursor: tables.repos.indexCursor }).from(tables.repos).where(eq(tables.repos.full_name, `${owner}/${name}`.toLowerCase())).get() if (!repoRow) { console.error('Failed to find repo row for', `${owner}/${name}`) - return { indexed: 0, skipped: 0, complete: true } + return { indexed: 0, skipped: 0, complete: true, currentPage: 0, totalStored: 0 } } const startPage = repoRow.indexCursor || 1 @@ -49,6 +55,12 @@ export async function indexRepo(octokit: Octokit, repo: InstallationRepo, option let currentPage = startPage const deadline = Date.now() + timeBudgetMs + /** Count how many issues are stored in the DB for this repo */ + async function getStoredCount(): Promise { + const result = await drizzle.select({ count: count() }).from(tables.issues).where(eq(tables.issues.repoId, repoRow!.repoId)).get() + return result?.count ?? 0 + } + for await (const response of octokit.paginate.iterator(octokit.rest.issues.listForRepo, { owner: owner!, repo: name!, @@ -60,8 +72,9 @@ export async function indexRepo(octokit: Octokit, repo: InstallationRepo, option if (Date.now() >= deadline) { console.log(`Time budget exhausted for ${repo.full_name} at page ${currentPage}, saving cursor`) await drizzle.update(tables.repos).set({ indexCursor: currentPage }).where(eq(tables.repos.id, repo.id)) - console.log('indexed', indexed, 'issues (skipped', skipped, ') from', `${owner}/${name}`, '(partial, will resume)') - return { indexed, skipped, complete: false } + const totalStored = await getStoredCount() + console.log('indexed', indexed, 'issues (skipped', skipped, `, total stored: ${totalStored}) from`, `${owner}/${name}`, '(partial, will resume)') + return { indexed, skipped, complete: false, currentPage, totalStored } } // Process each page in batches of 10 concurrent requests @@ -87,8 +100,9 @@ export async function indexRepo(octokit: Octokit, repo: InstallationRepo, option // Save cursor at current page so we re-process it (some issues in this page may not have been done) console.log(`Time budget exhausted for ${repo.full_name} mid-page ${currentPage}, saving cursor`) await drizzle.update(tables.repos).set({ indexCursor: currentPage }).where(eq(tables.repos.id, repo.id)) - console.log('indexed', indexed, 'issues (skipped', skipped, ') from', `${owner}/${name}`, '(partial, will resume)') - return { indexed, skipped, complete: false } + const totalStored = await getStoredCount() + console.log('indexed', indexed, 'issues (skipped', skipped, `, total stored: ${totalStored}) from`, `${owner}/${name}`, '(partial, will resume)') + return { indexed, skipped, complete: false, currentPage, totalStored } } } @@ -104,7 +118,8 @@ export async function indexRepo(octokit: Octokit, repo: InstallationRepo, option if (Date.now() + waitMs >= deadline) { console.warn(`Rate limit pause (${Math.ceil(waitMs / 1000)}s) would exceed time budget, saving cursor at page ${currentPage}`) await drizzle.update(tables.repos).set({ indexCursor: currentPage + 1 }).where(eq(tables.repos.id, repo.id)) - return { indexed, skipped, complete: false } + const totalStored = await getStoredCount() + return { indexed, skipped, complete: false, currentPage: currentPage + 1, totalStored } } console.warn(`Rate limit low (${remaining}), pausing for ${Math.ceil(waitMs / 1000)}s`) await new Promise(resolve => setTimeout(resolve, waitMs)) @@ -119,6 +134,7 @@ export async function indexRepo(octokit: Octokit, repo: InstallationRepo, option indexCursor: 0, }).where(eq(tables.repos.id, repo.id)) - console.log('indexed', indexed, 'issues (skipped', skipped, ') from', `${owner}/${name}`, '(complete)') - return { indexed, skipped, complete: true } + const totalStored = await getStoredCount() + console.log('indexed', indexed, 'issues (skipped', skipped, `, total stored: ${totalStored}) from`, `${owner}/${name}`, '(complete)') + return { indexed, skipped, complete: true, currentPage: 0, totalStored } } diff --git a/packages/web/server/utils/index-repos.ts b/packages/web/server/utils/index-repos.ts new file mode 100644 index 0000000..cc66fb7 --- /dev/null +++ b/packages/web/server/utils/index-repos.ts @@ -0,0 +1,98 @@ +import type { Octokit } from '@octokit/rest' +import { indexRepo } from '~~/server/utils/index-repo' +import { getMetadataForRepo } from '~~/server/utils/metadata' + +export interface IndexReposOptions { + filter?: string[] + timeBudgetMs?: number +} + +interface RepoProgress { + repo: string + indexed: number + skipped: number + totalStored: number + currentPage: number + complete: boolean +} + +export interface IndexReposResult { + repos: RepoProgress[] + result: string +} + +/** + * Orchestrate indexing across all repos that need it. + * Returns per-repo progress details and a human-readable summary. + */ +export async function indexRepos(octokit: Octokit, options: IndexReposOptions = {}): Promise { + const { filter, timeBudgetMs } = options + + const repos = await $fetch('/api/repos') + + // Check for repos with a non-zero cursor (partially indexed from a previous run) + const allRepos = await useDrizzle().select({ + fullName: tables.repos.full_name, + indexCursor: tables.repos.indexCursor, + }).from(tables.repos).all() + const cursorMap = new Map(allRepos.map(r => [r.fullName, r.indexCursor])) + + const progress: RepoProgress[] = [] + + for (const repo of repos) { + const cursor = cursorMap.get(repo.repo) || 0 + const needsIndexing = !repo.indexed || cursor > 0 + if (!needsIndexing || (filter && !filter.includes(repo.repo))) + continue + + try { + const [owner, name] = repo.repo.split('/') + const meta = await getMetadataForRepo(owner!, name!) + if (!meta) + continue + + const result = await indexRepo(octokit, meta, { timeBudgetMs }) + + progress.push({ + repo: repo.repo, + indexed: result.indexed, + skipped: result.skipped, + totalStored: result.totalStored, + currentPage: result.currentPage, + complete: result.complete, + }) + + if (!result.complete) { + // Stop processing more repos — this one consumed the time budget + break + } + } + catch (e) { + console.error('Error indexing', repo.repo, e) + } + } + + return { + repos: progress, + result: formatResult(progress), + } +} + +function formatResult(progress: RepoProgress[]): string { + if (!progress.length) + return 'No repos needed indexing.' + + const parts: string[] = [] + + const complete = progress.filter(r => r.complete) + const partial = progress.filter(r => !r.complete) + + for (const r of complete) { + parts.push(`Indexed ${r.repo}: ${r.totalStored} issues stored (${r.indexed} new, ${r.skipped} unchanged)`) + } + for (const r of partial) { + parts.push(`Partially indexed ${r.repo}: ${r.totalStored} issues stored so far, page ${r.currentPage} (${r.indexed} new this run, ${r.skipped} unchanged). Will resume on next invocation.`) + } + + return parts.join(' ') +}