import type { RestEndpointMethodTypes } from '@octokit/rest' import type { Issue } from '@octokit/webhooks-types' import { inArray } from 'drizzle-orm' import { hash } from 'ohash' import { invalidateCluster } from '#server/api/clusters/[owner]/[repo].get' import { invalidateDuplicates } from '#server/api/duplicates/[owner]/[repo].get' type RestIssue = RestEndpointMethodTypes['issues']['get']['response']['data'] export type IssueSegments = 'labels' | 'title' | 'body' // Cache for repository IDs to avoid repeated lookups (only caches found repos) const repoIdCache = new Map() // Function to get repository ID with caching. // Only caches successful lookups — null results are not cached so that // a repo added after a miss will be found on the next call. async function getRepoId(owner: string, repo: string): Promise { const cacheKey = `${owner}/${repo}`.toLowerCase() const cached = repoIdCache.get(cacheKey) if (cached !== undefined) { return cached } // Query database if not in cache const result = await useDrizzle() .select({ repoId: tables.repos.id }) .from(tables.repos) .where(eq(tables.repos.full_name, cacheKey)) .get() const repoId = result?.repoId ?? null if (repoId !== null) { repoIdCache.set(cacheKey, repoId) } return repoId } export function storageKeyForIssue(owner: string, repo: string, number: number | string) { return `issue:${owner}:${repo}:${number}` } export function storagePrefixForRepo(owner: string, repo: string) { return `issue:${owner}:${repo}:` } export async function getStoredEmbeddingsForRepo(owner: string, repo: string, state: 'open' | 'closed' | 'all' = 'open') { const repoId = await getRepoId(owner, repo) if (!repoId) { return [] } const issues = await useDrizzle().select().from(tables.issues).where( state === 'all' ? eq(tables.issues.repoId, repoId) : and(eq(tables.issues.repoId, repoId), eq(tables.issues.state, state)), ) return issues.map(i => ({ ...i, metadata: JSON.parse(i.metadata) as IssueMetadata, embeddings: JSON.parse(i.embeddings) as number[], })) } export async function getStoredEmbeddingsForRepos(repoIds: number[], state: 'open' | 'closed' | 'all' = 'open') { if (!repoIds.length) { return [] } const issues = await useDrizzle().select().from(tables.issues).where( state === 'all' ? inArray(tables.issues.repoId, repoIds) : and(inArray(tables.issues.repoId, repoIds), eq(tables.issues.state, state)), ) return issues.map(i => ({ ...i, metadata: JSON.parse(i.metadata) as IssueMetadata, embeddings: JSON.parse(i.embeddings) as number[], })) } export async function removeStoredEmbeddingsForRepo(owner: string, repo: string) { const repoId = await getRepoId(owner, repo) if (!repoId) { return } const vectorize = useEvent()?.context.cloudflare?.env?.VECTORIZE_ISSUES || null if (vectorize) { const issues = await useDrizzle() .select({ number: tables.issues.number }) .from(tables.issues) .where(eq(tables.issues.repoId, repoId)) const vectorIds = issues.map(i => storageKeyForIssue(owner, repo, i.number)) // Vectorize deleteByIds supports up to 1000 IDs per call for (let i = 0; i < vectorIds.length; i += 1000) { await vectorize.deleteByIds(vectorIds.slice(i, i + 1000)) } } const result = await useDrizzle().delete(tables.issues).where(eq(tables.issues.repoId, repoId)) console.log('removed stored embeddings for repo:', owner, repo, result) } export async function removeIssue(issue: Pick, repo: { owner: { login: string }, name: string }) { const repoId = await getRepoId(repo.owner.login, repo.name) if (!repoId) { return } const vectorize = useEvent()?.context.cloudflare?.env?.VECTORIZE_ISSUES || null await Promise.all([ useDrizzle().delete(tables.issues).where( and(eq(tables.issues.repoId, repoId), eq(tables.issues.number, issue.number)), ), vectorize?.deleteByIds([ storageKeyForIssue(repo.owner.login, repo.name, issue.number), ]), ]) console.log('removed issue:', repo.owner.login, repo.name, issue.number) } const knownBots = new Set(['renovate', 'renovate[bot]']) export async function indexIssue(issue: Issue | RestIssue, repo: { owner: { login: string }, name: string }) { if (issue.user && (knownBots.has(issue.user.login) || issue.user.login.endsWith('[bot]'))) { return } if (issue.pull_request) { return } const repoId = await getRepoId(repo.owner.login, repo.name) if (!repoId) { console.error('repo not found:', repo.owner.login, repo.name) return } const vectorize = useEvent()!.context.cloudflare?.env?.VECTORIZE_ISSUES || null const issueUpdatedTime = new Date(issue.updated_at).getTime() const text = chunkIssue(issue) const issueHash = hash(text) const issueMetadata: IssueMetadata = { owner: repo.owner.login, repository: repo.name, number: issue.number, description: issue.body || '', title: issue.title, url: issue.html_url, updated_at: issue.updated_at, labels: issue.labels?.map(label => (typeof label === 'string' ? label : { name: label.name!, color: label.color || undefined })) || [], state: issue.state || 'open', } const drizzle = useDrizzle() // Use both repoId and number to narrow the query const res = await drizzle.select().from(tables.issues).where(and( eq(tables.issues.repoId, repoId), eq(tables.issues.number, issue.number), )).get() if (res && (res.mtime >= issueUpdatedTime || res.hash === issueHash)) { return await Promise.all([ vectorize?.upsert([ { id: storageKeyForIssue(repo.owner.login, repo.name, issue.number), values: JSON.parse(res.embeddings) as number[], metadata: { ...issueMetadata, labels: JSON.stringify(issueMetadata.labels), }, }, ]), drizzle.update(tables.issues).set({ title: issue.title, state: issue.state || 'open', metadata: JSON.stringify(issueMetadata), mtime: issueUpdatedTime, }).where(eq(tables.issues.id, res.id)).execute(), ]) } const embeddings = await generateEmbedding(text) if (!embeddings.length) { console.warn('failed to generate embedding for issue:', repo.owner.login, repo.name, issue.number) return } const updatedValues = { repoId, hash: issueHash, title: issue.title, state: issue.state || 'open', number: issue.number, metadata: JSON.stringify(issueMetadata), mtime: issueUpdatedTime, embeddings: JSON.stringify(embeddings), } try { await Promise.all([ vectorize?.upsert([ { id: storageKeyForIssue(repo.owner.login, repo.name, issue.number), values: embeddings, metadata: { ...issueMetadata, labels: JSON.stringify(issueMetadata.labels), }, }, ]), drizzle.insert(tables.issues) .values(updatedValues) .onConflictDoUpdate({ target: [tables.issues.repoId, tables.issues.number], set: updatedValues, }) .execute(), ]) } catch (e) { console.error(`Failed to upsert issue ${repo.owner.login}/${repo.name}#${issue.number}:`, e) throw e } // invalidate clusters and duplicates for this repo await Promise.all([ invalidateCluster(repo.owner.login, repo.name), invalidateDuplicates(repo.owner.login, repo.name), ]) console.log('indexed issue:', issueMetadata) return embeddings } // Clear the repo ID cache when repos are updated export function clearRepoIdCache(owner?: string, repo?: string) { if (owner && repo) { repoIdCache.delete(`${owner}/${repo}`.toLowerCase()) } else { repoIdCache.clear() } } export type Label = string | { name: string, color?: string } export interface IssueMetadata { owner: string repository: string description: string number: number title: string url: string updated_at: string labels: Label[] state: string } export type IssueKeys = 'owner' | 'repository' | 'number' | 'title' | 'url' | 'updated_at' | 'labels' | 'state' export interface StoredEmbeddings { mtime: number hash: string metadata: IssueMetadata embeddings: number[] } const STRIP_ENV_SECTION_RE = /### (Environment|Reproduction).*?### /s const STRIP_NO_RESPONSE_RE = /### \w+\s+_No response_/ const STRIP_SPECIAL_CHARS_RE = /[^\w\s]/g function preprocessText(text: string): string { return text .toLowerCase() .replace(STRIP_ENV_SECTION_RE, '### ') .replace(STRIP_NO_RESPONSE_RE, '') .replace(STRIP_SPECIAL_CHARS_RE, '') .trim() } export function chunkIssue(issue: Pick, exclude?: Set) { const labels = getLabels(issue).filter(l => !exclude?.has(l)) return preprocessText(`${issue.title}\n${labels.join(', ')}\n${issue.body || ''}`) } async function generateEmbedding(text: string, retries = 2): Promise { const ai = useAI() if (!ai) { console.warn('generateEmbedding: no AI binding or REST API configured, skipping') return [] } for (let attempt = 0; attempt <= retries; attempt++) { try { const res = await ai.run('@cf/baai/bge-large-en-v1.5', { text }) return (res as { data?: number[][] }).data?.[0] || [] } catch (e) { if (attempt < retries) { console.warn(`generateEmbedding attempt ${attempt + 1} failed, retrying...`, e) await new Promise(resolve => setTimeout(resolve, 500 * (attempt + 1))) } else { console.error(`generateEmbedding failed after ${retries + 1} attempts:`, e) return [] } } } return [] } function getLabels(issue: Pick) { return issue.labels?.map(label => (typeof label === 'string' ? label : label.name!).toLowerCase()).filter(Boolean) || [] }