[READ-ONLY] a fast, modern browser for the npm registry

perf: use osv batch api for dep analysis (#665)

authored by

Daniel Roe and committed by
GitHub
0df32a3f 89c7e489

+557 -226
+33 -41
app/composables/useNpmRegistry.ts
··· 9 9 PackageVersionInfo, 10 10 } from '#shared/types' 11 11 import type { ReleaseType } from 'semver' 12 + import { mapWithConcurrency } from '#shared/utils/async' 12 13 import { maxSatisfying, prerelease, major, minor, diff, gt, compare } from 'semver' 13 14 import { isExactVersion } from '~/utils/versions' 14 15 import { extractInstallScriptsInfo } from '~/utils/install-scripts' ··· 546 547 547 548 // Fetch packuments and downloads in parallel 548 549 const [packuments, downloads] = await Promise.all([ 549 - // Fetch packuments in parallel (with concurrency limit) 550 + // Fetch packuments with concurrency limit 550 551 (async () => { 551 - const concurrency = 10 552 - const results: MinimalPackument[] = [] 553 - for (let i = 0; i < packageNames.length; i += concurrency) { 554 - const batch = packageNames.slice(i, i + concurrency) 555 - const batchResults = await Promise.all( 556 - batch.map(async name => { 557 - try { 558 - const encoded = encodePackageName(name) 559 - const { data: pkg } = await cachedFetch<MinimalPackument>( 560 - `${NPM_REGISTRY}/${encoded}`, 561 - { signal }, 562 - ) 563 - return pkg 564 - } catch { 565 - return null 566 - } 567 - }), 568 - ) 569 - for (const pkg of batchResults) { 570 - // Filter out any unpublished packages (missing dist-tags) 571 - if (pkg && pkg['dist-tags']) { 572 - results.push(pkg) 552 + const results = await mapWithConcurrency( 553 + packageNames, 554 + async name => { 555 + try { 556 + const encoded = encodePackageName(name) 557 + const { data: pkg } = await cachedFetch<MinimalPackument>( 558 + `${NPM_REGISTRY}/${encoded}`, 559 + { signal }, 560 + ) 561 + return pkg 562 + } catch { 563 + return null 573 564 } 574 - } 575 - } 576 - return results 565 + }, 566 + 10, 567 + ) 568 + // Filter out any unpublished packages (missing dist-tags) 569 + return results.filter( 570 + (pkg): pkg is MinimalPackument => pkg !== null && !!pkg['dist-tags'], 571 + ) 577 572 })(), 578 573 // Fetch downloads in bulk 579 574 fetchBulkDownloads(packageNames, { signal }), ··· 772 767 return 773 768 } 774 769 775 - const results: Record<string, OutdatedDependencyInfo> = {} 776 770 const entries = Object.entries(deps) 777 - const batchSize = 5 771 + const batchResults = await mapWithConcurrency( 772 + entries, 773 + async ([name, constraint]) => { 774 + const info = await checkDependencyOutdated(cachedFetch, name, constraint) 775 + return [name, info] as const 776 + }, 777 + 5, 778 + ) 778 779 779 - for (let i = 0; i < entries.length; i += batchSize) { 780 - const batch = entries.slice(i, i + batchSize) 781 - const batchResults = await Promise.all( 782 - batch.map(async ([name, constraint]) => { 783 - const info = await checkDependencyOutdated(cachedFetch, name, constraint) 784 - return [name, info] as const 785 - }), 786 - ) 787 - 788 - for (const [name, info] of batchResults) { 789 - if (info) { 790 - results[name] = info 791 - } 780 + const results: Record<string, OutdatedDependencyInfo> = {} 781 + for (const [name, info] of batchResults) { 782 + if (info) { 783 + results[name] = info 792 784 } 793 785 } 794 786
+106 -34
server/utils/dependency-analysis.ts
··· 1 1 import type { 2 2 OsvQueryResponse, 3 + OsvBatchResponse, 3 4 OsvVulnerability, 4 5 OsvSeverityLevel, 5 6 VulnerabilitySummary, ··· 8 9 VulnerabilityTreeResult, 9 10 DeprecatedPackageInfo, 10 11 } from '#shared/types/dependency-analysis' 12 + import { mapWithConcurrency } from '#shared/utils/async' 11 13 import { resolveDependencyTree } from './dependency-resolver' 12 14 13 - /** Result of a single OSV query */ 14 - type OsvQueryResult = { status: 'ok'; data: PackageVulnerabilityInfo | null } | { status: 'error' } 15 + /** Maximum concurrent requests for fetching vulnerability details */ 16 + const OSV_DETAIL_CONCURRENCY = 25 17 + 18 + /** Package info needed for OSV queries */ 19 + interface PackageQueryInfo { 20 + name: string 21 + version: string 22 + depth: DependencyDepth 23 + path: string[] 24 + } 15 25 16 26 /** 17 - * Query OSV for vulnerabilities in a package 27 + * Query OSV batch API to find which packages have vulnerabilities. 28 + * Returns indices of packages that have vulnerabilities (for follow-up detailed queries). 29 + * @see https://google.github.io/osv.dev/post-v1-querybatch/ 18 30 */ 19 - async function queryOsv( 20 - name: string, 21 - version: string, 22 - depth: DependencyDepth, 23 - path: string[], 24 - ): Promise<OsvQueryResult> { 31 + async function queryOsvBatch( 32 + packages: PackageQueryInfo[], 33 + ): Promise<{ vulnerableIndices: number[]; failed: boolean }> { 34 + if (packages.length === 0) return { vulnerableIndices: [], failed: false } 35 + 36 + try { 37 + const response = await $fetch<OsvBatchResponse>('https://api.osv.dev/v1/querybatch', { 38 + method: 'POST', 39 + body: { 40 + queries: packages.map(pkg => ({ 41 + package: { name: pkg.name, ecosystem: 'npm' }, 42 + version: pkg.version, 43 + })), 44 + }, 45 + }) 46 + 47 + // Find indices of packages that have vulnerabilities 48 + const vulnerableIndices: number[] = [] 49 + for (let i = 0; i < response.results.length; i++) { 50 + const result = response.results[i] 51 + if (result?.vulns && result.vulns.length > 0) { 52 + vulnerableIndices.push(i) 53 + } 54 + // Warn if pagination token present (>1000 vulns for single query or >3000 total) 55 + // This is extremely unlikely for npm packages but log for visibility 56 + if (result?.next_page_token) { 57 + // oxlint-disable-next-line no-console -- warn about paginated results 58 + console.warn( 59 + `[dep-analysis] OSV batch result has pagination token for package index ${i} ` + 60 + `(${packages[i]?.name}@${packages[i]?.version}) - some vulnerabilities may be missing`, 61 + ) 62 + } 63 + } 64 + 65 + return { vulnerableIndices, failed: false } 66 + } catch (error) { 67 + // oxlint-disable-next-line no-console -- log OSV API failures for debugging 68 + console.warn(`[dep-analysis] OSV batch query failed:`, error) 69 + return { vulnerableIndices: [], failed: true } 70 + } 71 + } 72 + 73 + /** 74 + * Query OSV for full vulnerability details for a single package. 75 + * Only called for packages known to have vulnerabilities. 76 + */ 77 + async function queryOsvDetails(pkg: PackageQueryInfo): Promise<PackageVulnerabilityInfo | null> { 25 78 try { 26 79 const response = await $fetch<OsvQueryResponse>('https://api.osv.dev/v1/query', { 27 80 method: 'POST', 28 81 body: { 29 - package: { name, ecosystem: 'npm' }, 30 - version, 82 + package: { name: pkg.name, ecosystem: 'npm' }, 83 + version: pkg.version, 31 84 }, 32 85 }) 33 86 34 87 const vulns = response.vulns || [] 35 - if (vulns.length === 0) return { status: 'ok', data: null } 88 + if (vulns.length === 0) return null 36 89 37 90 const counts = { total: vulns.length, critical: 0, high: 0, moderate: 0, low: 0 } 38 91 const vulnerabilities: VulnerabilitySummary[] = [] ··· 65 118 }) 66 119 } 67 120 68 - return { status: 'ok', data: { name, version, depth, path, vulnerabilities, counts } } 121 + return { 122 + name: pkg.name, 123 + version: pkg.version, 124 + depth: pkg.depth, 125 + path: pkg.path, 126 + vulnerabilities, 127 + counts, 128 + } 69 129 } catch (error) { 70 130 // oxlint-disable-next-line no-console -- log OSV API failures for debugging 71 - console.warn(`[dep-analysis] OSV query failed for ${name}@${version}:`, error) 72 - return { status: 'error' } 131 + console.warn(`[dep-analysis] OSV detail query failed for ${pkg.name}@${pkg.version}:`, error) 132 + return null 73 133 } 74 134 } 75 135 ··· 110 170 111 171 /** 112 172 * Analyze entire dependency tree for vulnerabilities and deprecated packages. 173 + * Uses OSV batch API for efficient vulnerability discovery, then fetches 174 + * full details only for packages with known vulnerabilities. 113 175 */ 114 176 export const analyzeDependencyTree = defineCachedFunction( 115 177 async (name: string, version: string): Promise<VulnerabilityTreeResult> => { 116 178 // Resolve all packages in the tree with depth tracking 117 179 const resolved = await resolveDependencyTree(name, version, { trackDepth: true }) 118 180 119 - // Convert to array for OSV querying 120 - const packages = [...resolved.values()] 181 + // Convert to array with query info 182 + const packages: PackageQueryInfo[] = [...resolved.values()].map(pkg => ({ 183 + name: pkg.name, 184 + version: pkg.version, 185 + depth: pkg.depth!, 186 + path: pkg.path || [], 187 + })) 121 188 122 189 // Collect deprecated packages (no API call needed - already in packument data) 123 - const deprecatedPackages: DeprecatedPackageInfo[] = packages 190 + const deprecatedPackages: DeprecatedPackageInfo[] = [...resolved.values()] 124 191 .filter(pkg => pkg.deprecated) 125 192 .map(pkg => ({ 126 193 name: pkg.name, ··· 135 202 return depthOrder[a.depth] - depthOrder[b.depth] 136 203 }) 137 204 138 - // Query OSV for all packages in parallel batches 139 - const vulnerablePackages: PackageVulnerabilityInfo[] = [] 140 - let failedQueries = 0 141 - const batchSize = 10 205 + // Step 1: Use batch API to find which packages have vulnerabilities 206 + // This is much faster than individual queries - one request for all packages 207 + const { vulnerableIndices, failed: batchFailed } = await queryOsvBatch(packages) 142 208 143 - for (let i = 0; i < packages.length; i += batchSize) { 144 - const batch = packages.slice(i, i + batchSize) 145 - const results = await Promise.all( 146 - batch.map(pkg => queryOsv(pkg.name, pkg.version, pkg.depth!, pkg.path || [])), 209 + let vulnerablePackages: PackageVulnerabilityInfo[] = [] 210 + let failedQueries = batchFailed ? packages.length : 0 211 + 212 + if (!batchFailed && vulnerableIndices.length > 0) { 213 + // Step 2: Fetch full vulnerability details only for packages with vulns 214 + // This is typically a small fraction of total packages 215 + const detailResults = await mapWithConcurrency( 216 + vulnerableIndices, 217 + i => queryOsvDetails(packages[i]!), 218 + OSV_DETAIL_CONCURRENCY, 147 219 ) 148 220 149 - for (const result of results) { 150 - if (result.status === 'error') { 221 + for (const result of detailResults) { 222 + if (result) { 223 + vulnerablePackages.push(result) 224 + } else { 151 225 failedQueries++ 152 - } else if (result.data) { 153 - vulnerablePackages.push(result.data) 154 226 } 155 227 } 156 228 } ··· 175 247 totalCounts.low += pkg.counts.low 176 248 } 177 249 178 - // Log critical failures (>50% of queries failed) 179 - if (failedQueries > 0 && failedQueries > packages.length / 2) { 250 + // Log if batch query failed entirely 251 + if (batchFailed) { 180 252 // oxlint-disable-next-line no-console -- critical error logging 181 253 console.error( 182 - `[dep-analysis] Critical: ${failedQueries}/${packages.length} OSV queries failed for ${name}@${version}`, 254 + `[dep-analysis] Critical: OSV batch query failed for ${name}@${version} (${packages.length} packages)`, 183 255 ) 184 256 } 185 257 ··· 197 269 maxAge: 60 * 60, 198 270 swr: true, 199 271 name: 'dependency-analysis', 200 - getKey: (name: string, version: string) => `v1:${name}@${version}`, 272 + getKey: (name: string, version: string) => `v2:${name}@${version}`, 201 273 }, 202 274 )
+45 -43
server/utils/dependency-resolver.ts
··· 1 1 import type { Packument, PackumentVersion, DependencyDepth } from '#shared/types' 2 + import { mapWithConcurrency } from '#shared/utils/async' 2 3 import { maxSatisfying } from 'semver' 4 + 5 + /** Concurrency limit for fetching packuments during dependency resolution */ 6 + const PACKUMENT_FETCH_CONCURRENCY = 20 3 7 4 8 /** 5 9 * Target platform for dependency resolution. ··· 142 146 seen.add(name) 143 147 } 144 148 145 - // Process current level in batches 149 + // Process current level with concurrency limit 146 150 const entries = [...currentLevel.entries()] 147 - for (let i = 0; i < entries.length; i += 20) { 148 - const batch = entries.slice(i, i + 20) 151 + await mapWithConcurrency( 152 + entries, 153 + async ([name, { range, optional, path }]) => { 154 + const packument = await fetchPackument(name) 155 + if (!packument) return 149 156 150 - await Promise.all( 151 - batch.map(async ([name, { range, optional, path }]) => { 152 - const packument = await fetchPackument(name) 153 - if (!packument) return 154 - 155 - const versions = Object.keys(packument.versions) 156 - const version = resolveVersion(range, versions) 157 - if (!version) return 157 + const versions = Object.keys(packument.versions) 158 + const version = resolveVersion(range, versions) 159 + if (!version) return 158 160 159 - const versionData = packument.versions[version] 160 - if (!versionData) return 161 + const versionData = packument.versions[version] 162 + if (!versionData) return 161 163 162 - if (!matchesPlatform(versionData)) return 164 + if (!matchesPlatform(versionData)) return 163 165 164 - const size = (versionData.dist as { unpackedSize?: number })?.unpackedSize ?? 0 165 - const key = `${name}@${version}` 166 + const size = (versionData.dist as { unpackedSize?: number })?.unpackedSize ?? 0 167 + const key = `${name}@${version}` 166 168 167 - // Build path for this package (path to parent + this package with version) 168 - const currentPath = [...path, `${name}@${version}`] 169 + // Build path for this package (path to parent + this package with version) 170 + const currentPath = [...path, `${name}@${version}`] 169 171 170 - if (!resolved.has(key)) { 171 - const pkg: ResolvedPackage = { name, version, size, optional } 172 - if (options.trackDepth) { 173 - pkg.depth = level === 0 ? 'root' : level === 1 ? 'direct' : 'transitive' 174 - pkg.path = currentPath 175 - } 176 - if (versionData.deprecated) { 177 - pkg.deprecated = versionData.deprecated 178 - } 179 - resolved.set(key, pkg) 172 + if (!resolved.has(key)) { 173 + const pkg: ResolvedPackage = { name, version, size, optional } 174 + if (options.trackDepth) { 175 + pkg.depth = level === 0 ? 'root' : level === 1 ? 'direct' : 'transitive' 176 + pkg.path = currentPath 177 + } 178 + if (versionData.deprecated) { 179 + pkg.deprecated = versionData.deprecated 180 180 } 181 + resolved.set(key, pkg) 182 + } 181 183 182 - // Collect dependencies for next level 183 - if (versionData.dependencies) { 184 - for (const [depName, depRange] of Object.entries(versionData.dependencies)) { 185 - if (!seen.has(depName) && !nextLevel.has(depName)) { 186 - nextLevel.set(depName, { range: depRange, optional: false, path: currentPath }) 187 - } 184 + // Collect dependencies for next level 185 + if (versionData.dependencies) { 186 + for (const [depName, depRange] of Object.entries(versionData.dependencies)) { 187 + if (!seen.has(depName) && !nextLevel.has(depName)) { 188 + nextLevel.set(depName, { range: depRange, optional: false, path: currentPath }) 188 189 } 189 190 } 191 + } 190 192 191 - // Collect optional dependencies 192 - if (versionData.optionalDependencies) { 193 - for (const [depName, depRange] of Object.entries(versionData.optionalDependencies)) { 194 - if (!seen.has(depName) && !nextLevel.has(depName)) { 195 - nextLevel.set(depName, { range: depRange, optional: true, path: currentPath }) 196 - } 193 + // Collect optional dependencies 194 + if (versionData.optionalDependencies) { 195 + for (const [depName, depRange] of Object.entries(versionData.optionalDependencies)) { 196 + if (!seen.has(depName) && !nextLevel.has(depName)) { 197 + nextLevel.set(depName, { range: depRange, optional: true, path: currentPath }) 197 198 } 198 199 } 199 - }), 200 - ) 201 - } 200 + } 201 + }, 202 + PACKUMENT_FETCH_CONCURRENCY, 203 + ) 202 204 203 205 currentLevel = nextLevel 204 206 level++
+24
shared/types/dependency-analysis.ts
··· 65 65 } 66 66 67 67 /** 68 + * Single result from OSV batch query (minimal info - just ID and modified) 69 + */ 70 + export interface OsvBatchVulnRef { 71 + id: string 72 + modified: string 73 + } 74 + 75 + /** 76 + * Single result in OSV batch response 77 + */ 78 + export interface OsvBatchResult { 79 + vulns?: OsvBatchVulnRef[] 80 + next_page_token?: string 81 + } 82 + 83 + /** 84 + * OSV batch query response 85 + * @see https://google.github.io/osv.dev/post-v1-querybatch/ 86 + */ 87 + export interface OsvBatchResponse { 88 + results: OsvBatchResult[] 89 + } 90 + 91 + /** 68 92 * Simplified vulnerability info for display 69 93 */ 70 94 export interface VulnerabilitySummary {
+37
shared/utils/async.ts
··· 1 + /** 2 + * Async utilities for controlled concurrency and parallel execution. 3 + */ 4 + 5 + /** 6 + * Map over an array with limited concurrency. 7 + * Similar to Promise.all but limits how many promises run simultaneously. 8 + * 9 + * @param items - Array of items to process 10 + * @param fn - Async function to apply to each item 11 + * @param concurrency - Maximum number of concurrent operations (default: 10) 12 + * @returns Array of results in the same order as input 13 + * 14 + * @example 15 + * const results = await mapWithConcurrency(urls, fetchUrl, 5) 16 + */ 17 + export async function mapWithConcurrency<T, R>( 18 + items: T[], 19 + fn: (item: T, index: number) => Promise<R>, 20 + concurrency = 10, 21 + ): Promise<R[]> { 22 + const results: R[] = Array.from({ length: items.length }) as R[] 23 + let currentIndex = 0 24 + 25 + async function worker(): Promise<void> { 26 + while (currentIndex < items.length) { 27 + const index = currentIndex++ 28 + results[index] = await fn(items[index]!, index) 29 + } 30 + } 31 + 32 + // Start workers up to concurrency limit 33 + const workers = Array.from({ length: Math.min(concurrency, items.length) }, () => worker()) 34 + await Promise.all(workers) 35 + 36 + return results 37 + }
+217 -108
test/unit/server/utils/dependency-analysis.spec.ts
··· 14 14 15 15 const { resolveDependencyTree } = await import('../../../../server/utils/dependency-resolver') 16 16 17 + /** 18 + * Helper to create mock $fetch that handles the two-step OSV API pattern: 19 + * 1. Batch query to /v1/querybatch - returns which packages have vulns (just IDs) 20 + * 2. Individual queries to /v1/query - returns full vuln details for affected packages 21 + * 22 + * @param batchResults - Array of results for batch query (vulns array per package, in order) 23 + * @param detailResults - Map of package key to full vuln details for individual queries 24 + */ 25 + function mockOsvApi( 26 + batchResults: Array<{ vulns?: Array<{ id: string; modified: string }> }>, 27 + detailResults: Map<string, { vulns?: Array<Record<string, unknown>> }> = new Map(), 28 + ) { 29 + vi.mocked($fetch).mockImplementation(async (url: string, options?: { body?: unknown }) => { 30 + if (url === 'https://api.osv.dev/v1/querybatch') { 31 + return { results: batchResults } 32 + } 33 + if (url === 'https://api.osv.dev/v1/query') { 34 + const body = options?.body as { package: { name: string }; version: string } 35 + const key = `${body.package.name}@${body.version}` 36 + return detailResults.get(key) || { vulns: [] } 37 + } 38 + throw new Error(`Unexpected fetch to ${url}`) 39 + }) 40 + } 41 + 17 42 describe('dependency-analysis', () => { 18 43 beforeEach(() => { 19 44 vi.clearAllMocks() ··· 36 61 ]) 37 62 vi.mocked(resolveDependencyTree).mockResolvedValue(mockResolved) 38 63 39 - // Mock OSV API returning no vulnerabilities 40 - vi.mocked($fetch).mockResolvedValue({ vulns: [] }) 64 + // Mock batch API returning no vulnerabilities 65 + mockOsvApi([{ vulns: [] }]) 41 66 42 67 const result = await analyzeDependencyTree('test-pkg', '1.0.0') 43 68 ··· 49 74 expect(result.totalCounts).toEqual({ total: 0, critical: 0, high: 0, moderate: 0, low: 0 }) 50 75 }) 51 76 52 - it('tracks failed queries when OSV API fails', async () => { 77 + it('tracks failed queries when OSV batch API fails', async () => { 53 78 const mockResolved = new Map([ 54 79 [ 55 80 'test-pkg@1.0.0', ··· 76 101 ]) 77 102 vi.mocked(resolveDependencyTree).mockResolvedValue(mockResolved) 78 103 79 - // First call succeeds, second fails 80 - vi.mocked($fetch) 81 - .mockResolvedValueOnce({ vulns: [] }) 82 - .mockRejectedValueOnce(new Error('OSV API error')) 104 + // Batch query fails entirely 105 + vi.mocked($fetch).mockRejectedValue(new Error('OSV API error')) 83 106 84 107 const result = await analyzeDependencyTree('test-pkg', '1.0.0') 85 108 86 - expect(result.failedQueries).toBe(1) 109 + // When batch fails, all packages are counted as failed 110 + expect(result.failedQueries).toBe(2) 87 111 expect(result.totalPackages).toBe(2) 88 112 }) 89 113 ··· 103 127 ]) 104 128 vi.mocked(resolveDependencyTree).mockResolvedValue(mockResolved) 105 129 106 - // Mock OSV API returning vulnerabilities with different severities 107 - vi.mocked($fetch).mockResolvedValue({ 108 - vulns: [ 109 - { id: 'GHSA-1', summary: 'Critical vuln', database_specific: { severity: 'CRITICAL' } }, 110 - { id: 'GHSA-2', summary: 'High vuln', database_specific: { severity: 'HIGH' } }, 111 - { id: 'GHSA-3', summary: 'Moderate vuln', database_specific: { severity: 'MODERATE' } }, 112 - { id: 'GHSA-4', summary: 'Low vuln', database_specific: { severity: 'LOW' } }, 113 - ], 114 - }) 130 + // Mock batch API returns vuln IDs, then detail query returns full info 131 + mockOsvApi( 132 + [{ vulns: [{ id: 'GHSA-1', modified: '2024-01-01' }] }], 133 + new Map([ 134 + [ 135 + 'vuln-pkg@1.0.0', 136 + { 137 + vulns: [ 138 + { 139 + id: 'GHSA-1', 140 + summary: 'Critical vuln', 141 + database_specific: { severity: 'CRITICAL' }, 142 + }, 143 + { id: 'GHSA-2', summary: 'High vuln', database_specific: { severity: 'HIGH' } }, 144 + { 145 + id: 'GHSA-3', 146 + summary: 'Moderate vuln', 147 + database_specific: { severity: 'MODERATE' }, 148 + }, 149 + { id: 'GHSA-4', summary: 'Low vuln', database_specific: { severity: 'LOW' } }, 150 + ], 151 + }, 152 + ], 153 + ]), 154 + ) 115 155 116 156 const result = await analyzeDependencyTree('vuln-pkg', '1.0.0') 117 157 ··· 119 159 expect(result.totalCounts).toEqual({ total: 4, critical: 1, high: 1, moderate: 1, low: 1 }) 120 160 121 161 const pkg = result.vulnerablePackages[0] 122 - expect(pkg.counts.critical).toBe(1) 123 - expect(pkg.counts.high).toBe(1) 124 - expect(pkg.counts.moderate).toBe(1) 125 - expect(pkg.counts.low).toBe(1) 162 + expect(pkg?.counts.critical).toBe(1) 163 + expect(pkg?.counts.high).toBe(1) 164 + expect(pkg?.counts.moderate).toBe(1) 165 + expect(pkg?.counts.low).toBe(1) 126 166 }) 127 167 128 168 it('includes dependency path in vulnerable packages', async () => { ··· 152 192 ]) 153 193 vi.mocked(resolveDependencyTree).mockResolvedValue(mockResolved) 154 194 155 - vi.mocked($fetch) 156 - .mockResolvedValueOnce({ vulns: [] }) // root has no vulns 157 - .mockResolvedValueOnce({ 158 - vulns: [ 159 - { id: 'GHSA-test', summary: 'Test vuln', database_specific: { severity: 'HIGH' } }, 195 + // Batch: root has no vulns, vuln-dep has vulns 196 + mockOsvApi( 197 + [{ vulns: [] }, { vulns: [{ id: 'GHSA-test', modified: '2024-01-01' }] }], 198 + new Map([ 199 + [ 200 + 'vuln-dep@2.0.0', 201 + { 202 + vulns: [ 203 + { id: 'GHSA-test', summary: 'Test vuln', database_specific: { severity: 'HIGH' } }, 204 + ], 205 + }, 160 206 ], 161 - }) // vuln-dep has vuln 207 + ]), 208 + ) 162 209 163 210 const result = await analyzeDependencyTree('root', '1.0.0') 164 211 165 212 expect(result.vulnerablePackages).toHaveLength(1) 166 213 const vulnPkg = result.vulnerablePackages[0] 167 - expect(vulnPkg.path).toEqual(['root@1.0.0', 'middle@1.5.0', 'vuln-dep@2.0.0']) 168 - expect(vulnPkg.depth).toBe('transitive') 214 + expect(vulnPkg?.path).toEqual(['root@1.0.0', 'middle@1.5.0', 'vuln-dep@2.0.0']) 215 + expect(vulnPkg?.depth).toBe('transitive') 169 216 }) 170 217 171 218 it('sorts vulnerable packages by depth then severity', async () => { ··· 206 253 ]) 207 254 vi.mocked(resolveDependencyTree).mockResolvedValue(mockResolved) 208 255 209 - // All have vulnerabilities 210 - vi.mocked($fetch) 211 - .mockResolvedValueOnce({ 212 - vulns: [ 213 - { id: 'GHSA-root', summary: 'Root vuln', database_specific: { severity: 'LOW' } }, 256 + // All packages have vulnerabilities 257 + mockOsvApi( 258 + [ 259 + { vulns: [{ id: 'GHSA-root', modified: '2024-01-01' }] }, 260 + { vulns: [{ id: 'GHSA-direct', modified: '2024-01-01' }] }, 261 + { vulns: [{ id: 'GHSA-trans', modified: '2024-01-01' }] }, 262 + ], 263 + new Map([ 264 + [ 265 + 'root@1.0.0', 266 + { 267 + vulns: [ 268 + { id: 'GHSA-root', summary: 'Root vuln', database_specific: { severity: 'LOW' } }, 269 + ], 270 + }, 214 271 ], 215 - }) 216 - .mockResolvedValueOnce({ 217 - vulns: [ 272 + [ 273 + 'direct-dep@1.0.0', 218 274 { 219 - id: 'GHSA-direct', 220 - summary: 'Direct vuln', 221 - database_specific: { severity: 'CRITICAL' }, 275 + vulns: [ 276 + { 277 + id: 'GHSA-direct', 278 + summary: 'Direct vuln', 279 + database_specific: { severity: 'CRITICAL' }, 280 + }, 281 + ], 222 282 }, 223 283 ], 224 - }) 225 - .mockResolvedValueOnce({ 226 - vulns: [ 227 - { id: 'GHSA-trans', summary: 'Trans vuln', database_specific: { severity: 'HIGH' } }, 284 + [ 285 + 'transitive-dep@1.0.0', 286 + { 287 + vulns: [ 288 + { 289 + id: 'GHSA-trans', 290 + summary: 'Trans vuln', 291 + database_specific: { severity: 'HIGH' }, 292 + }, 293 + ], 294 + }, 228 295 ], 229 - }) 296 + ]), 297 + ) 230 298 231 299 const result = await analyzeDependencyTree('root', '1.0.0') 232 300 233 301 expect(result.vulnerablePackages).toHaveLength(3) 234 302 // Should be sorted: root first, then direct, then transitive 235 - expect(result.vulnerablePackages[0].name).toBe('root') 236 - expect(result.vulnerablePackages[1].name).toBe('direct-dep') 237 - expect(result.vulnerablePackages[2].name).toBe('transitive-dep') 303 + expect(result.vulnerablePackages[0]?.name).toBe('root') 304 + expect(result.vulnerablePackages[1]?.name).toBe('direct-dep') 305 + expect(result.vulnerablePackages[2]?.name).toBe('transitive-dep') 238 306 }) 239 307 240 308 it('generates correct vulnerability URLs for GHSA', async () => { ··· 253 321 ]) 254 322 vi.mocked(resolveDependencyTree).mockResolvedValue(mockResolved) 255 323 256 - vi.mocked($fetch).mockResolvedValue({ 257 - vulns: [ 258 - { 259 - id: 'GHSA-xxxx-yyyy-zzzz', 260 - summary: 'Test vuln', 261 - database_specific: { severity: 'HIGH' }, 262 - }, 263 - ], 264 - }) 324 + mockOsvApi( 325 + [{ vulns: [{ id: 'GHSA-xxxx-yyyy-zzzz', modified: '2024-01-01' }] }], 326 + new Map([ 327 + [ 328 + 'pkg@1.0.0', 329 + { 330 + vulns: [ 331 + { 332 + id: 'GHSA-xxxx-yyyy-zzzz', 333 + summary: 'Test vuln', 334 + database_specific: { severity: 'HIGH' }, 335 + }, 336 + ], 337 + }, 338 + ], 339 + ]), 340 + ) 265 341 266 342 const result = await analyzeDependencyTree('pkg', '1.0.0') 267 343 268 - expect(result.vulnerablePackages[0].vulnerabilities[0].url).toBe( 344 + expect(result.vulnerablePackages[0]?.vulnerabilities[0]?.url).toBe( 269 345 'https://github.com/advisories/GHSA-xxxx-yyyy-zzzz', 270 346 ) 271 347 }) ··· 286 362 ]) 287 363 vi.mocked(resolveDependencyTree).mockResolvedValue(mockResolved) 288 364 289 - vi.mocked($fetch).mockResolvedValue({ 290 - vulns: [ 291 - { 292 - id: 'OSV-2024-001', 293 - summary: 'Test vuln', 294 - aliases: ['CVE-2024-12345'], 295 - database_specific: { severity: 'HIGH' }, 296 - }, 297 - ], 298 - }) 365 + mockOsvApi( 366 + [{ vulns: [{ id: 'OSV-2024-001', modified: '2024-01-01' }] }], 367 + new Map([ 368 + [ 369 + 'pkg@1.0.0', 370 + { 371 + vulns: [ 372 + { 373 + id: 'OSV-2024-001', 374 + summary: 'Test vuln', 375 + aliases: ['CVE-2024-12345'], 376 + database_specific: { severity: 'HIGH' }, 377 + }, 378 + ], 379 + }, 380 + ], 381 + ]), 382 + ) 299 383 300 384 const result = await analyzeDependencyTree('pkg', '1.0.0') 301 385 302 - expect(result.vulnerablePackages[0].vulnerabilities[0].url).toBe( 386 + expect(result.vulnerablePackages[0]?.vulnerabilities[0]?.url).toBe( 303 387 'https://nvd.nist.gov/vuln/detail/CVE-2024-12345', 304 388 ) 305 389 }) ··· 320 404 ]) 321 405 vi.mocked(resolveDependencyTree).mockResolvedValue(mockResolved) 322 406 323 - vi.mocked($fetch).mockResolvedValue({ 324 - vulns: [ 325 - { id: 'PYSEC-2024-001', summary: 'Test vuln', database_specific: { severity: 'HIGH' } }, 326 - ], 327 - }) 407 + mockOsvApi( 408 + [{ vulns: [{ id: 'PYSEC-2024-001', modified: '2024-01-01' }] }], 409 + new Map([ 410 + [ 411 + 'pkg@1.0.0', 412 + { 413 + vulns: [ 414 + { 415 + id: 'PYSEC-2024-001', 416 + summary: 'Test vuln', 417 + database_specific: { severity: 'HIGH' }, 418 + }, 419 + ], 420 + }, 421 + ], 422 + ]), 423 + ) 328 424 329 425 const result = await analyzeDependencyTree('pkg', '1.0.0') 330 426 331 - expect(result.vulnerablePackages[0].vulnerabilities[0].url).toBe( 427 + expect(result.vulnerablePackages[0]?.vulnerabilities[0]?.url).toBe( 332 428 'https://osv.dev/vulnerability/PYSEC-2024-001', 333 429 ) 334 430 }) ··· 349 445 ]) 350 446 vi.mocked(resolveDependencyTree).mockResolvedValue(mockResolved) 351 447 352 - vi.mocked($fetch).mockResolvedValue({ 353 - vulns: [ 354 - { 355 - id: 'GHSA-1', 356 - summary: 'Critical (9.5)', 357 - severity: [{ score: 'CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H/9.5' }], 358 - }, 359 - { id: 'GHSA-2', summary: 'High (7.5)', severity: [{ score: '7.5' }] }, 360 - { id: 'GHSA-3', summary: 'Moderate (5.0)', severity: [{ score: '5.0' }] }, 361 - { id: 'GHSA-4', summary: 'Low (2.0)', severity: [{ score: '2.0' }] }, 362 - ], 363 - }) 448 + mockOsvApi( 449 + [{ vulns: [{ id: 'GHSA-1', modified: '2024-01-01' }] }], 450 + new Map([ 451 + [ 452 + 'pkg@1.0.0', 453 + { 454 + vulns: [ 455 + { 456 + id: 'GHSA-1', 457 + summary: 'Critical (9.5)', 458 + severity: [{ score: 'CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H/9.5' }], 459 + }, 460 + { id: 'GHSA-2', summary: 'High (7.5)', severity: [{ score: '7.5' }] }, 461 + { id: 'GHSA-3', summary: 'Moderate (5.0)', severity: [{ score: '5.0' }] }, 462 + { id: 'GHSA-4', summary: 'Low (2.0)', severity: [{ score: '2.0' }] }, 463 + ], 464 + }, 465 + ], 466 + ]), 467 + ) 364 468 365 469 const result = await analyzeDependencyTree('pkg', '1.0.0') 366 470 ··· 397 501 ], 398 502 ]) 399 503 vi.mocked(resolveDependencyTree).mockResolvedValue(mockResolved) 400 - vi.mocked($fetch).mockResolvedValue({ vulns: [] }) 504 + mockOsvApi([{ vulns: [] }, { vulns: [] }]) 401 505 402 506 const result = await analyzeDependencyTree('root', '1.0.0') 403 507 ··· 426 530 ], 427 531 ]) 428 532 vi.mocked(resolveDependencyTree).mockResolvedValue(mockResolved) 429 - vi.mocked($fetch).mockResolvedValue({ vulns: [] }) 533 + mockOsvApi([{ vulns: [] }]) 430 534 431 535 const result = await analyzeDependencyTree('root', '1.0.0') 432 536 ··· 473 577 ], 474 578 ]) 475 579 vi.mocked(resolveDependencyTree).mockResolvedValue(mockResolved) 476 - vi.mocked($fetch).mockResolvedValue({ vulns: [] }) 580 + mockOsvApi([{ vulns: [] }, { vulns: [] }, { vulns: [] }]) 477 581 478 582 const result = await analyzeDependencyTree('root', '1.0.0') 479 583 480 584 expect(result.deprecatedPackages).toHaveLength(3) 481 - expect(result.deprecatedPackages[0].name).toBe('root') 482 - expect(result.deprecatedPackages[0].depth).toBe('root') 483 - expect(result.deprecatedPackages[1].name).toBe('direct-dep') 484 - expect(result.deprecatedPackages[1].depth).toBe('direct') 485 - expect(result.deprecatedPackages[2].name).toBe('transitive-dep') 486 - expect(result.deprecatedPackages[2].depth).toBe('transitive') 585 + expect(result.deprecatedPackages[0]?.name).toBe('root') 586 + expect(result.deprecatedPackages[0]?.depth).toBe('root') 587 + expect(result.deprecatedPackages[1]?.name).toBe('direct-dep') 588 + expect(result.deprecatedPackages[1]?.depth).toBe('direct') 589 + expect(result.deprecatedPackages[2]?.name).toBe('transitive-dep') 590 + expect(result.deprecatedPackages[2]?.depth).toBe('transitive') 487 591 }) 488 592 489 593 it('returns both vulnerabilities and deprecated packages together', async () => { ··· 525 629 ]) 526 630 vi.mocked(resolveDependencyTree).mockResolvedValue(mockResolved) 527 631 528 - // root and deprecated-pkg have no vulns, vuln-pkg has one 529 - vi.mocked($fetch) 530 - .mockResolvedValueOnce({ vulns: [] }) // root 531 - .mockResolvedValueOnce({ 532 - vulns: [ 632 + // Batch: root and deprecated-pkg have no vulns, vuln-pkg has one 633 + mockOsvApi( 634 + [{ vulns: [] }, { vulns: [{ id: 'GHSA-vuln', modified: '2024-01-01' }] }, { vulns: [] }], 635 + new Map([ 636 + [ 637 + 'vuln-pkg@1.0.0', 533 638 { 534 - id: 'GHSA-vuln', 535 - summary: 'A vulnerability', 536 - database_specific: { severity: 'HIGH' }, 639 + vulns: [ 640 + { 641 + id: 'GHSA-vuln', 642 + summary: 'A vulnerability', 643 + database_specific: { severity: 'HIGH' }, 644 + }, 645 + ], 537 646 }, 538 647 ], 539 - }) // vuln-pkg 540 - .mockResolvedValueOnce({ vulns: [] }) // deprecated-pkg 648 + ]), 649 + ) 541 650 542 651 const result = await analyzeDependencyTree('root', '1.0.0') 543 652 544 653 expect(result.vulnerablePackages).toHaveLength(1) 545 - expect(result.vulnerablePackages[0].name).toBe('vuln-pkg') 654 + expect(result.vulnerablePackages[0]?.name).toBe('vuln-pkg') 546 655 expect(result.deprecatedPackages).toHaveLength(1) 547 - expect(result.deprecatedPackages[0].name).toBe('deprecated-pkg') 656 + expect(result.deprecatedPackages[0]?.name).toBe('deprecated-pkg') 548 657 expect(result.totalPackages).toBe(3) 549 658 }) 550 659 })
+95
test/unit/shared/utils/async.spec.ts
··· 1 + import { describe, expect, it, vi } from 'vitest' 2 + import { mapWithConcurrency } from '../../../../shared/utils/async' 3 + 4 + describe('mapWithConcurrency', () => { 5 + it('processes all items and returns results in order', async () => { 6 + const items = [1, 2, 3, 4, 5] 7 + const results = await mapWithConcurrency(items, async x => x * 2) 8 + 9 + expect(results).toEqual([2, 4, 6, 8, 10]) 10 + }) 11 + 12 + it('respects concurrency limit', async () => { 13 + let concurrent = 0 14 + let maxConcurrent = 0 15 + 16 + const items = Array.from({ length: 10 }, (_, i) => i) 17 + 18 + await mapWithConcurrency( 19 + items, 20 + async () => { 21 + concurrent++ 22 + maxConcurrent = Math.max(maxConcurrent, concurrent) 23 + await new Promise(resolve => setTimeout(resolve, 10)) 24 + concurrent-- 25 + }, 26 + 3, 27 + ) 28 + 29 + expect(maxConcurrent).toBe(3) 30 + }) 31 + 32 + it('handles empty array', async () => { 33 + const results = await mapWithConcurrency([], async x => x) 34 + expect(results).toEqual([]) 35 + }) 36 + 37 + it('handles single item', async () => { 38 + const results = await mapWithConcurrency([42], async x => x * 2) 39 + expect(results).toEqual([84]) 40 + }) 41 + 42 + it('passes index to callback', async () => { 43 + const items = ['a', 'b', 'c'] 44 + const results = await mapWithConcurrency(items, async (item, index) => `${item}${index}`) 45 + 46 + expect(results).toEqual(['a0', 'b1', 'c2']) 47 + }) 48 + 49 + it('propagates errors', async () => { 50 + const items = [1, 2, 3] 51 + const fn = vi.fn(async (x: number) => { 52 + if (x === 2) throw new Error('test error') 53 + return x 54 + }) 55 + 56 + await expect(mapWithConcurrency(items, fn)).rejects.toThrow('test error') 57 + }) 58 + 59 + it('uses default concurrency of 10', async () => { 60 + let concurrent = 0 61 + let maxConcurrent = 0 62 + 63 + const items = Array.from({ length: 20 }, (_, i) => i) 64 + 65 + await mapWithConcurrency(items, async () => { 66 + concurrent++ 67 + maxConcurrent = Math.max(maxConcurrent, concurrent) 68 + await new Promise(resolve => setTimeout(resolve, 5)) 69 + concurrent-- 70 + }) 71 + 72 + expect(maxConcurrent).toBe(10) 73 + }) 74 + 75 + it('handles fewer items than concurrency limit', async () => { 76 + let concurrent = 0 77 + let maxConcurrent = 0 78 + 79 + const items = [1, 2, 3] 80 + 81 + await mapWithConcurrency( 82 + items, 83 + async () => { 84 + concurrent++ 85 + maxConcurrent = Math.max(maxConcurrent, concurrent) 86 + await new Promise(resolve => setTimeout(resolve, 10)) 87 + concurrent-- 88 + }, 89 + 10, 90 + ) 91 + 92 + // Should only have 3 concurrent since we only have 3 items 93 + expect(maxConcurrent).toBe(3) 94 + }) 95 + })