tjh.dev
nixpkgs mirror (for testing)
github.com/NixOS/nixpkgs
nix
1#!/usr/bin/env node
2'use strict'
3
4const fs = require('fs')
5const crypto = require('crypto')
6const process = require('process')
7const https = require('https')
8const child_process = require('child_process')
9const path = require('path')
10const lockfile = require('./yarnpkg-lockfile.js')
11const { promisify } = require('util')
12const url = require('url')
13const { urlToName } = require('./common.js')
14
15const execFile = promisify(child_process.execFile)
16
17const exec = async (...args) => {
18 const res = await execFile(...args)
19 if (res.error) throw new Error(res.stderr)
20 return res
21}
22
23const downloadFileHttps = (fileName, url, expectedHash, hashType = 'sha1') => {
24 return new Promise((resolve, reject) => {
25 const get = (url, redirects = 0) => https.get(url, (res) => {
26 if(redirects > 10) {
27 reject('Too many redirects!');
28 return;
29 }
30 if(res.statusCode === 301 || res.statusCode === 302) {
31 return get(res.headers.location, redirects + 1)
32 }
33 const file = fs.createWriteStream(fileName)
34 const hash = crypto.createHash(hashType)
35 res.pipe(file)
36 res.pipe(hash).setEncoding('hex')
37 res.on('end', () => {
38 file.close()
39 const h = hash.read()
40 if (expectedHash === undefined){
41 console.log(`Warning: lockfile url ${url} doesn't end in "#<hash>" to validate against. Downloaded file had hash ${h}.`);
42 } else if (h != expectedHash) return reject(new Error(`hash mismatch, expected ${expectedHash}, got ${h} for ${url}`))
43 resolve()
44 })
45 res.on('error', e => reject(e))
46 })
47 get(url)
48 })
49}
50
51const downloadGit = async (fileName, url, rev) => {
52 await exec('nix-prefetch-git', [
53 '--out', fileName + '.tmp',
54 '--url', url,
55 '--rev', rev,
56 '--builder'
57 ])
58
59 await exec('tar', [
60 // hopefully make it reproducible across runs and systems
61 '--owner=0', '--group=0', '--numeric-owner', '--format=gnu', '--sort=name', '--mtime=@1',
62
63 // Set u+w because tar-fs can't unpack archives with read-only dirs: https://github.com/mafintosh/tar-fs/issues/79
64 '--mode', 'u+w',
65
66 '-C', fileName + '.tmp',
67 '-cf', fileName, '.'
68 ])
69
70 await exec('rm', [ '-rf', fileName + '.tmp', ])
71}
72
73const isGitUrl = pattern => {
74 // https://github.com/yarnpkg/yarn/blob/3119382885ea373d3c13d6a846de743eca8c914b/src/resolvers/exotics/git-resolver.js#L15-L47
75 const GIT_HOSTS = ['github.com', 'gitlab.com', 'bitbucket.com', 'bitbucket.org']
76 const GIT_PATTERN_MATCHERS = [/^git:/, /^git\+.+:/, /^ssh:/, /^https?:.+\.git$/, /^https?:.+\.git#.+/]
77
78 for (const matcher of GIT_PATTERN_MATCHERS) if (matcher.test(pattern)) return true
79
80 const {hostname, path} = url.parse(pattern)
81 if (hostname && path && GIT_HOSTS.indexOf(hostname) >= 0
82 // only if dependency is pointing to a git repo,
83 // e.g. facebook/flow and not file in a git repo facebook/flow/archive/v1.0.0.tar.gz
84 && path.split('/').filter(p => !!p).length === 2
85 ) return true
86
87 return false
88}
89
90const downloadPkg = (pkg, verbose) => {
91 for (let marker of ['@file:', '@link:']) {
92 const split = pkg.key.split(marker)
93 if (split.length == 2) {
94 console.info(`ignoring lockfile entry "${split[0]}" which points at path "${split[1]}"`)
95 return
96 } else if (split.length > 2) {
97 throw new Error(`The lockfile entry key "${pkg.key}" contains "${marker}" more than once. Processing is not implemented.`)
98 }
99 }
100
101 if (pkg.resolved === undefined) {
102 throw new Error(`The lockfile entry with key "${pkg.key}" cannot be downloaded because it is missing the "resolved" attribute, which should contain the URL to download from. The lockfile might be invalid.`)
103 }
104
105 const [ url, hash ] = pkg.resolved.split('#')
106 if (verbose) console.log('downloading ' + url)
107 const fileName = urlToName(url)
108 const s = url.split('/')
109 if (url.startsWith('https://codeload.github.com/') && url.includes('/tar.gz/')) {
110 return downloadGit(fileName, `https://github.com/${s[3]}/${s[4]}.git`, s[s.length-1])
111 } else if (url.startsWith('https://github.com/') && url.endsWith('.tar.gz') &&
112 (
113 s.length <= 5 || // https://github.com/owner/repo.tgz#feedface...
114 s[5] == "archive" // https://github.com/owner/repo/archive/refs/tags/v0.220.1.tar.gz
115 )) {
116 return downloadGit(fileName, `https://github.com/${s[3]}/${s[4]}.git`, s[s.length-1].replace(/.tar.gz$/, ''))
117 } else if (isGitUrl(url)) {
118 return downloadGit(fileName, url.replace(/^git\+/, ''), hash)
119 } else if (url.startsWith('https://')) {
120 if (typeof pkg.integrity === 'string' || pkg.integrity instanceof String) {
121 const [ type, checksum ] = pkg.integrity.split('-')
122 return downloadFileHttps(fileName, url, Buffer.from(checksum, 'base64').toString('hex'), type)
123 }
124 return downloadFileHttps(fileName, url, hash)
125 } else if (url.startsWith('file:')) {
126 console.warn(`ignoring unsupported file:path url "${url}"`)
127 } else {
128 throw new Error('don\'t know how to download "' + url + '"')
129 }
130}
131
132const performParallel = tasks => {
133 const worker = async () => {
134 while (tasks.length > 0) await tasks.shift()()
135 }
136
137 const workers = []
138 for (let i = 0; i < 4; i++) {
139 workers.push(worker())
140 }
141
142 return Promise.all(workers)
143}
144
145// This could be implemented using [`Map.groupBy`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Map/groupBy),
146// but that method is only supported starting with Node 21
147const uniqueBy = (arr, callback) => {
148 const map = new Map()
149 for (const elem of arr) {
150 map.set(callback(elem), elem)
151 }
152 return [...map.values()]
153}
154
155const prefetchYarnDeps = async (lockContents, verbose) => {
156 const lockData = lockfile.parse(lockContents)
157 await performParallel(
158 uniqueBy(Object.entries(lockData.object), ([_, value]) => value.resolved)
159 .map(([key, value]) => () => downloadPkg({ key, ...value }, verbose))
160 )
161 await fs.promises.writeFile('yarn.lock', lockContents)
162 if (verbose) console.log('Done')
163}
164
165const showUsage = async () => {
166 process.stderr.write(`
167syntax: prefetch-yarn-deps [path to yarn.lock] [options]
168
169Options:
170 -h --help Show this help
171 -v --verbose Verbose output
172 --builder Only perform the download to current directory, then exit
173`)
174 process.exit(1)
175}
176
177const main = async () => {
178 const args = process.argv.slice(2)
179 let next, lockFile, verbose, isBuilder
180 while (next = args.shift()) {
181 if (next == '--builder') {
182 isBuilder = true
183 } else if (next == '--verbose' || next == '-v') {
184 verbose = true
185 } else if (next == '--help' || next == '-h') {
186 showUsage()
187 } else if (!lockFile) {
188 lockFile = next
189 } else {
190 showUsage()
191 }
192 }
193 let lockContents
194 try {
195 lockContents = await fs.promises.readFile(lockFile || 'yarn.lock', 'utf-8')
196 } catch {
197 showUsage()
198 }
199
200 if (isBuilder) {
201 await prefetchYarnDeps(lockContents, verbose)
202 } else {
203 const { stdout: tmpDir } = await exec('mktemp', [ '-d' ])
204
205 try {
206 process.chdir(tmpDir.trim())
207 await prefetchYarnDeps(lockContents, verbose)
208 const { stdout: hash } = await exec('nix-hash', [ '--type', 'sha256', '--base32', tmpDir.trim() ])
209 console.log(hash)
210 } finally {
211 await exec('rm', [ '-rf', tmpDir.trim() ])
212 }
213 }
214}
215
216main()
217 .catch(e => {
218 console.error(e)
219 process.exit(1)
220 })