tjh.dev
1#! /usr/bin/env nix-shell
2/*
3#! nix-shell -i zx -p zx
4*/
5
6cd(__dirname)
7const nixpkgs = (await $`git rev-parse --show-toplevel`).stdout.trim()
8const $nixpkgs = $({
9 cwd: nixpkgs
10})
11
12const dummy_hash = 'sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA='
13
14const lockfile_file = './info.json'
15const lockfile_initial = fs.readJsonSync(lockfile_file)
16function flush_to_file() {
17 fs.writeJsonSync(lockfile_file, lockfile, { spaces: 2 })
18}
19const flush_to_file_proxy = {
20 get(obj, prop) {
21 const value = obj[prop]
22 return typeof value == 'object' ? new Proxy(value, flush_to_file_proxy) : value
23 },
24
25 set(obj, prop, value) {
26 obj[prop] = value
27 flush_to_file()
28 return true
29 },
30}
31const lockfile = new Proxy(structuredClone(lockfile_initial), flush_to_file_proxy)
32const ungoogled_rev = argv['ungoogled-chromium-rev']
33
34for (const attr_path of Object.keys(lockfile)) {
35 const ungoogled = attr_path === 'ungoogled-chromium'
36
37 if (!argv[attr_path] && !(ungoogled && ungoogled_rev)) {
38 console.log(`[${attr_path}] Skipping ${attr_path}. Pass --${attr_path} as argument to update.`)
39 continue
40 }
41
42 const version_nixpkgs = !ungoogled ? lockfile[attr_path].version : lockfile[attr_path].deps['ungoogled-patches'].rev
43 const version_upstream = !ungoogled ? await get_latest_chromium_release('linux') :
44 ungoogled_rev ?? await get_latest_ungoogled_release()
45
46 console.log(`[${attr_path}] ${chalk.red(version_nixpkgs)} (nixpkgs)`)
47 console.log(`[${attr_path}] ${chalk.green(version_upstream)} (upstream)`)
48
49 if (ungoogled_rev || version_greater_than(version_upstream, version_nixpkgs)) {
50 console.log(`[${attr_path}] ${chalk.green(version_upstream)} from upstream is newer than our ${chalk.red(version_nixpkgs)}...`)
51
52 let ungoogled_patches = ungoogled ? await fetch_ungoogled(version_upstream) : undefined
53
54 // For ungoogled-chromium we need to remove the patch revision (e.g. 130.0.6723.116-1 -> 130.0.6723.116)
55 // by just using the chromium_version.txt content from the patches checkout (to also work with commit revs).
56 const version_chromium = ungoogled_patches?.chromium_version ?? version_upstream
57
58 const chromium_rev = await chromium_resolve_tag_to_rev(version_chromium)
59
60 lockfile[attr_path] = {
61 version: version_chromium,
62 chromedriver: !ungoogled ? await fetch_chromedriver_binaries(await get_latest_chromium_release('mac')) : undefined,
63 deps: {
64 depot_tools: {},
65 gn: {},
66 'ungoogled-patches': !ungoogled ? undefined : {
67 rev: ungoogled_patches.rev,
68 hash: ungoogled_patches.hash,
69 },
70 npmHash: dummy_hash,
71 },
72 DEPS: {},
73 }
74
75 const depot_tools = await fetch_depot_tools(chromium_rev, lockfile_initial[attr_path].deps.depot_tools)
76 lockfile[attr_path].deps.depot_tools = {
77 rev: depot_tools.rev,
78 hash: depot_tools.hash,
79 }
80
81 const gn = await fetch_gn(chromium_rev, lockfile_initial[attr_path].deps.gn)
82 lockfile[attr_path].deps.gn = {
83 rev: gn.rev,
84 hash: gn.hash,
85 }
86
87 // DEPS update loop
88 lockfile[attr_path].DEPS = await resolve_DEPS(depot_tools.out, chromium_rev)
89 for (const [path, value] of Object.entries(lockfile[attr_path].DEPS)) {
90 delete value.fetcher
91 delete value.postFetch
92
93 if (value.url === 'https://chromium.googlesource.com/chromium/src.git') {
94 value.recompress = true
95 }
96
97 const cache_hit = (() => {
98 for (const attr_path in lockfile_initial) {
99 const cache = lockfile_initial[attr_path].DEPS[path]
100 const hits_cache =
101 cache !== undefined &&
102 value.url === cache.url &&
103 value.rev === cache.rev &&
104 value.recompress === cache.recompress &&
105 cache.hash !== undefined &&
106 cache.hash !== '' &&
107 cache.hash !== dummy_hash
108
109 if (hits_cache) {
110 cache.attr_path = attr_path
111 return cache;
112 }
113 }
114 })();
115
116 if (cache_hit) {
117 console.log(`[${chalk.green(path)}] Reusing hash from previous info.json for ${cache_hit.url}@${cache_hit.rev} from ${cache_hit.attr_path}`)
118 value.hash = cache_hit.hash
119 continue
120 }
121
122 console.log(`[${chalk.red(path)}] FOD prefetching ${value.url}@${value.rev}...`)
123 value.hash = await prefetch_FOD('-A', `${attr_path}.browser.passthru.chromiumDeps."${path}"`)
124 console.log(`[${chalk.green(path)}] FOD prefetching successful`)
125 }
126
127 lockfile[attr_path].deps.npmHash = await prefetch_FOD('-A', `${attr_path}.browser.passthru.npmDeps`)
128
129 console.log(chalk.green(`[${attr_path}] Done updating ${attr_path} from ${version_nixpkgs} to ${version_upstream}!`))
130 }
131}
132
133
134async function fetch_gn(chromium_rev, gn_previous) {
135 const DEPS_file = await get_gitiles_file('https://chromium.googlesource.com/chromium/src', chromium_rev, 'DEPS')
136 const gn_rev = /^\s+'gn_version': 'git_revision:(?<rev>.+)',$/m.exec(DEPS_file).groups.rev
137 const hash = gn_rev === gn_previous.rev ? gn_previous.hash : ''
138
139 return await prefetch_gitiles('https://gn.googlesource.com/gn', gn_rev, hash)
140}
141
142
143async function fetch_chromedriver_binaries(version) {
144 // https://developer.chrome.com/docs/chromedriver/downloads/version-selection
145 const prefetch = async (url) => {
146 const expr = [`(import ./. {}).fetchzip { url = "${url}"; hash = ""; }`]
147 const derivation = await $nixpkgs`nix-instantiate --expr ${expr}`
148 return await prefetch_FOD(derivation)
149 }
150
151 // if the URL ever changes, the URLs in the chromedriver derivations need updating as well!
152 const url = (platform) => `https://storage.googleapis.com/chrome-for-testing-public/${version}/${platform}/chromedriver-${platform}.zip`
153 return {
154 version,
155 hash_darwin: await prefetch(url('mac-x64')),
156 hash_darwin_aarch64: await prefetch(url('mac-arm64')),
157 }
158}
159
160
161async function chromium_resolve_tag_to_rev(tag) {
162 const url = `https://chromium.googlesource.com/chromium/src/+/refs/tags/${tag}?format=json`
163 const response = await (await fetch(url)).text()
164 const json = JSON.parse(response.replace(`)]}'\n`, ''))
165 return json.commit
166}
167
168
169async function resolve_DEPS(depot_tools_checkout, chromium_rev) {
170 const { stdout } = await $`./depot_tools.py ${depot_tools_checkout} ${chromium_rev}`
171 const deps = JSON.parse(stdout)
172 return Object.fromEntries(Object.entries(deps).map(([k, { url, rev, hash }]) => [k, { url, rev, hash }]))
173}
174
175
176async function get_latest_chromium_release(platform) {
177 const url = `https://versionhistory.googleapis.com/v1/chrome/platforms/${platform}/channels/stable/versions/all/releases?` + new URLSearchParams({
178 order_by: 'version desc',
179 filter: 'endtime=none,fraction>=0.5'
180 })
181
182 const response = await (await fetch(url)).json()
183 return response.releases[0].version
184}
185
186
187async function get_latest_ungoogled_release() {
188 const ungoogled_tags = await (await fetch('https://api.github.com/repos/ungoogled-software/ungoogled-chromium/tags')).json()
189 const chromium_releases = await (await fetch('https://versionhistory.googleapis.com/v1/chrome/platforms/linux/channels/stable/versions/all/releases')).json()
190 const chromium_release_map = chromium_releases.releases.map((x) => x.version)
191 return ungoogled_tags.find((x) => chromium_release_map.includes(x.name.split('-')[0])).name
192}
193
194
195async function fetch_ungoogled(rev) {
196 const expr = (hash) => [`(import ./. {}).fetchFromGitHub { owner = "ungoogled-software"; repo = "ungoogled-chromium"; rev = "${rev}"; hash = "${hash}"; }`]
197 const hash = await prefetch_FOD('--expr', expr(''))
198
199 const checkout = await $nixpkgs`nix-build --expr ${expr(hash)}`
200 const checkout_path = checkout.stdout.trim()
201
202 await fs.copy(path.join(checkout_path, 'flags.gn'), './ungoogled-flags.toml')
203
204 const chromium_version = (await fs.readFile(path.join(checkout_path, 'chromium_version.txt'))).toString().trim()
205
206 console.log(`[ungoogled-chromium] ${chalk.green(rev)} patch revision resolves to chromium version ${chalk.green(chromium_version)}`)
207
208 return {
209 rev,
210 hash,
211 chromium_version,
212 }
213}
214
215
216function version_greater_than(greater, than) {
217 return greater.localeCompare(than, undefined, { numeric: true, sensitivity: 'base' }) === 1
218}
219
220
221async function get_gitiles_file(repo, rev, path) {
222 const base64 = await (await fetch(`${repo}/+/${rev}/${path}?format=TEXT`)).text()
223 return Buffer.from(base64, 'base64').toString('utf-8')
224}
225
226
227async function fetch_depot_tools(chromium_rev, depot_tools_previous) {
228 const depot_tools_rev = await get_gitiles_file('https://chromium.googlesource.com/chromium/src', chromium_rev, 'third_party/depot_tools')
229 const hash = depot_tools_rev === depot_tools_previous.rev ? depot_tools_previous.hash : ''
230 return await prefetch_gitiles('https://chromium.googlesource.com/chromium/tools/depot_tools', depot_tools_rev, hash)
231}
232
233
234async function prefetch_gitiles(url, rev, hash = '') {
235 const expr = () => [`(import ./. {}).fetchFromGitiles { url = "${url}"; rev = "${rev}"; hash = "${hash}"; }`]
236
237 if (hash === '') {
238 hash = await prefetch_FOD('--expr', expr())
239 }
240
241 const { stdout } = await $nixpkgs`nix-build --expr ${expr()}`
242
243 return {
244 url,
245 rev,
246 hash,
247 out: stdout.trim(),
248 }
249}
250
251
252async function prefetch_FOD(...args) {
253 const { stderr } = await $nixpkgs`nix-build ${args}`.nothrow()
254 const hash = /\s+got:\s+(?<hash>.+)$/m.exec(stderr)?.groups?.hash
255
256 if (hash == undefined) {
257 throw new Error(chalk.red('Expected to find hash in nix-build stderr output:') + stderr)
258 }
259
260 return hash
261}
262