Something went wrong. Try again.
[READ-ONLY] Mirror of https://github.com/danielroe/unsight.dev. Detect duplicate GitHub issues, areas of concern and more across related rep unsight.dev
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325import type { RestEndpointMethodTypes } from '@octokit/rest'import type { Issue } from '@octokit/webhooks-types'import { inArray } from 'drizzle-orm'import { hash } from 'ohash'import { invalidateCluster } from '#server/api/clusters/[owner]/[repo].get'import { invalidateDuplicates } from '#server/api/duplicates/[owner]/[repo].get'
type RestIssue = RestEndpointMethodTypes['issues']['get']['response']['data']
export type IssueSegments = 'labels' | 'title' | 'body'
// Cache for repository IDs to avoid repeated lookups (only caches found repos)const repoIdCache = new Map<string, number>()
// Function to get repository ID with caching.// Only caches successful lookups — null results are not cached so that// a repo added after a miss will be found on the next call.async function getRepoId(owner: string, repo: string): Promise<number | null> { const cacheKey = `${owner}/${repo}`.toLowerCase()
const cached = repoIdCache.get(cacheKey) if (cached !== undefined) { return cached }
// Query database if not in cache const result = await useDrizzle() .select({ repoId: tables.repos.id }) .from(tables.repos) .where(eq(tables.repos.full_name, cacheKey)) .get()
const repoId = result?.repoId ?? null if (repoId !== null) { repoIdCache.set(cacheKey, repoId) } return repoId}
export function storageKeyForIssue(owner: string, repo: string, number: number | string) { return `issue:${owner}:${repo}:${number}`}
export function storagePrefixForRepo(owner: string, repo: string) { return `issue:${owner}:${repo}:`}
export async function getStoredEmbeddingsForRepo(owner: string, repo: string, state: 'open' | 'closed' | 'all' = 'open') { const repoId = await getRepoId(owner, repo) if (!repoId) { return [] }
const issues = await useDrizzle().select().from(tables.issues).where( state === 'all' ? eq(tables.issues.repoId, repoId) : and(eq(tables.issues.repoId, repoId), eq(tables.issues.state, state)), )
return issues.map(i => ({ ...i, metadata: JSON.parse(i.metadata) as IssueMetadata, embeddings: JSON.parse(i.embeddings) as number[], }))}
export async function getStoredEmbeddingsForRepos(repoIds: number[], state: 'open' | 'closed' | 'all' = 'open') { if (!repoIds.length) { return [] }
const issues = await useDrizzle().select().from(tables.issues).where( state === 'all' ? inArray(tables.issues.repoId, repoIds) : and(inArray(tables.issues.repoId, repoIds), eq(tables.issues.state, state)), )
return issues.map(i => ({ ...i, metadata: JSON.parse(i.metadata) as IssueMetadata, embeddings: JSON.parse(i.embeddings) as number[], }))}
export async function removeStoredEmbeddingsForRepo(owner: string, repo: string) { const repoId = await getRepoId(owner, repo) if (!repoId) { return }
const vectorize = useEvent()?.context.cloudflare?.env?.VECTORIZE_ISSUES || null if (vectorize) { const issues = await useDrizzle() .select({ number: tables.issues.number }) .from(tables.issues) .where(eq(tables.issues.repoId, repoId)) const vectorIds = issues.map(i => storageKeyForIssue(owner, repo, i.number)) // Vectorize deleteByIds supports up to 1000 IDs per call for (let i = 0; i < vectorIds.length; i += 1000) { await vectorize.deleteByIds(vectorIds.slice(i, i + 1000)) } }
const result = await useDrizzle().delete(tables.issues).where(eq(tables.issues.repoId, repoId)) console.log('removed stored embeddings for repo:', owner, repo, result)}
export async function removeIssue(issue: Pick<Issue, 'number'>, repo: { owner: { login: string }, name: string }) { const repoId = await getRepoId(repo.owner.login, repo.name) if (!repoId) { return } const vectorize = useEvent()?.context.cloudflare?.env?.VECTORIZE_ISSUES || null await Promise.all([ useDrizzle().delete(tables.issues).where( and(eq(tables.issues.repoId, repoId), eq(tables.issues.number, issue.number)), ), vectorize?.deleteByIds([ storageKeyForIssue(repo.owner.login, repo.name, issue.number), ]), ]) console.log('removed issue:', repo.owner.login, repo.name, issue.number)}
const knownBots = new Set(['renovate', 'renovate[bot]'])export async function indexIssue(issue: Issue | RestIssue, repo: { owner: { login: string }, name: string }) { if (issue.user && (knownBots.has(issue.user.login) || issue.user.login.endsWith('[bot]'))) { return } if (issue.pull_request) { return } const repoId = await getRepoId(repo.owner.login, repo.name) if (!repoId) { console.error('repo not found:', repo.owner.login, repo.name) return }
const vectorize = useEvent()!.context.cloudflare?.env?.VECTORIZE_ISSUES || null
const issueUpdatedTime = new Date(issue.updated_at).getTime()
const text = chunkIssue(issue) const issueHash = hash(text)
const issueMetadata: IssueMetadata = { owner: repo.owner.login, repository: repo.name, number: issue.number, description: issue.body || '', title: issue.title, url: issue.html_url, updated_at: issue.updated_at, labels: issue.labels?.map(label => (typeof label === 'string' ? label : { name: label.name!, color: label.color || undefined })) || [], state: issue.state || 'open', }
const drizzle = useDrizzle() // Use both repoId and number to narrow the query const res = await drizzle.select().from(tables.issues).where(and( eq(tables.issues.repoId, repoId), eq(tables.issues.number, issue.number), )).get()
if (res && (res.mtime >= issueUpdatedTime || res.hash === issueHash)) { return await Promise.all([ vectorize?.upsert([ { id: storageKeyForIssue(repo.owner.login, repo.name, issue.number), values: JSON.parse(res.embeddings) as number[], metadata: { ...issueMetadata, labels: JSON.stringify(issueMetadata.labels), }, }, ]), drizzle.update(tables.issues).set({ title: issue.title, state: issue.state || 'open', metadata: JSON.stringify(issueMetadata), mtime: issueUpdatedTime, }).where(eq(tables.issues.id, res.id)).execute(), ]) }
const embeddings = await generateEmbedding(text) if (!embeddings.length) { console.warn('failed to generate embedding for issue:', repo.owner.login, repo.name, issue.number) return }
const updatedValues = { repoId, hash: issueHash, title: issue.title, state: issue.state || 'open', number: issue.number, metadata: JSON.stringify(issueMetadata), mtime: issueUpdatedTime, embeddings: JSON.stringify(embeddings), }
try { await Promise.all([ vectorize?.upsert([ { id: storageKeyForIssue(repo.owner.login, repo.name, issue.number), values: embeddings, metadata: { ...issueMetadata, labels: JSON.stringify(issueMetadata.labels), }, }, ]), drizzle.insert(tables.issues) .values(updatedValues) .onConflictDoUpdate({ target: [tables.issues.repoId, tables.issues.number], set: updatedValues, }) .execute(), ]) } catch (e) { console.error(`Failed to upsert issue ${repo.owner.login}/${repo.name}#${issue.number}:`, e) throw e }
// invalidate clusters and duplicates for this repo await Promise.all([ invalidateCluster(repo.owner.login, repo.name), invalidateDuplicates(repo.owner.login, repo.name), ])
console.log('indexed issue:', issueMetadata)
return embeddings}
// Clear the repo ID cache when repos are updatedexport function clearRepoIdCache(owner?: string, repo?: string) { if (owner && repo) { repoIdCache.delete(`${owner}/${repo}`.toLowerCase()) } else { repoIdCache.clear() }}
export type Label = string | { name: string, color?: string }
export interface IssueMetadata { owner: string repository: string description: string number: number title: string url: string updated_at: string labels: Label[] state: string}
export type IssueKeys = 'owner' | 'repository' | 'number' | 'title' | 'url' | 'updated_at' | 'labels' | 'state'
export interface StoredEmbeddings { mtime: number hash: string metadata: IssueMetadata embeddings: number[]}
const STRIP_ENV_SECTION_RE = /### (Environment|Reproduction).*?### /sconst STRIP_NO_RESPONSE_RE = /### \w+\s+_No response_/const STRIP_SPECIAL_CHARS_RE = /[^\w\s]/g
function preprocessText(text: string): string { return text .toLowerCase() .replace(STRIP_ENV_SECTION_RE, '### ') .replace(STRIP_NO_RESPONSE_RE, '') .replace(STRIP_SPECIAL_CHARS_RE, '') .trim()}
export function chunkIssue(issue: Pick<Issue | RestIssue, IssueSegments>, exclude?: Set<string>) { const labels = getLabels(issue).filter(l => !exclude?.has(l)) return preprocessText(`${issue.title}\n${labels.join(', ')}\n${issue.body || ''}`)}
async function generateEmbedding(text: string, retries = 2): Promise<number[]> { const ai = useAI() if (!ai) { console.warn('generateEmbedding: no AI binding or REST API configured, skipping') return [] }
for (let attempt = 0; attempt <= retries; attempt++) { try { const res = await ai.run('@cf/baai/bge-large-en-v1.5', { text }) return (res as { data?: number[][] }).data?.[0] || [] } catch (e) { if (attempt < retries) { console.warn(`generateEmbedding attempt ${attempt + 1} failed, retrying...`, e) await new Promise(resolve => setTimeout(resolve, 500 * (attempt + 1))) } else { console.error(`generateEmbedding failed after ${retries + 1} attempts:`, e) return [] } } }
return []}
function getLabels(issue: Pick<Issue | RestIssue, IssueSegments>) { return issue.labels?.map(label => (typeof label === 'string' ? label : label.name!).toLowerCase()).filter(Boolean) || []}