pyrox.dev
1#! /usr/bin/env nix-shell
2#! nix-shell -i python3 -p python3Packages.pyyaml
3
4import shutil
5import json
6import urllib.request
7import tempfile
8from sys import exit
9import os
10import subprocess
11import re
12import json
13import argparse
14import yaml
15import json
16
17FAKE_HASH = 'sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA='
18
19NIXPKGS_ROOT = subprocess.Popen(['git',
20 'rev-parse',
21 '--show-toplevel'],
22 stdout=subprocess.PIPE,
23 text=True).communicate()[0].strip()
24
25
26def load_code(name, **kwargs):
27 with open(f"{NIXPKGS_ROOT}/pkgs/development/compilers/flutter/update/{name}.in", 'r') as f:
28 code = f.read()
29
30 for (key, value) in kwargs.items():
31 code = code.replace(f"@{key}@", value)
32
33 return code
34
35
36# Return out paths
37def nix_build(code):
38 temp = tempfile.NamedTemporaryFile(mode='w')
39 temp.write(code)
40 temp.flush()
41 os.fsync(temp.fileno())
42
43 process = subprocess.Popen(
44 [
45 "nix-build",
46 "--impure",
47 "--no-out-link",
48 "--expr",
49 f"with import {NIXPKGS_ROOT} {{}}; callPackage {temp.name} {{}}"],
50 stdout=subprocess.PIPE,
51 text=True)
52
53 process.wait()
54 temp.close()
55 return process.stdout.read().strip().splitlines()[0]
56
57
58# Return errors
59def nix_build_to_fail(code):
60 temp = tempfile.NamedTemporaryFile(mode='w')
61 temp.write(code)
62 temp.flush()
63 os.fsync(temp.fileno())
64
65 process = subprocess.Popen(
66 [
67 "nix-build",
68 "--impure",
69 "--keep-going",
70 "--no-link",
71 "--expr",
72 f"with import {NIXPKGS_ROOT} {{}}; callPackage {temp.name} {{}}"],
73 stderr=subprocess.PIPE,
74 text=True)
75
76 stderr = ""
77 while True:
78 line = process.stderr.readline()
79 if not line:
80 break
81 stderr += line
82 print(line.strip())
83
84 process.wait()
85 temp.close()
86 return stderr
87
88
89def get_engine_hashes(engine_version, flutter_version):
90 code = load_code("get-engine-hashes.nix",
91 nixpkgs_root=NIXPKGS_ROOT,
92 flutter_version=flutter_version,
93 engine_version=engine_version)
94
95 stderr = nix_build_to_fail(code)
96
97 pattern = re.compile(
98 rf"/nix/store/.*-flutter-engine-source-{engine_version}-(.+?-.+?)-(.+?-.+?).drv':\n\s+specified: .*\n\s+got:\s+(.+?)\n")
99 matches = pattern.findall(stderr)
100 result_dict = {}
101
102 for match in matches:
103 flutter_platform, architecture, got = match
104 result_dict.setdefault(flutter_platform, {})[architecture] = got
105
106 def sort_dict_recursive(d):
107 return {
108 k: sort_dict_recursive(v) if isinstance(
109 v, dict) else v for k, v in sorted(
110 d.items())}
111 result_dict = sort_dict_recursive(result_dict)
112
113 return result_dict
114
115
116def get_artifact_hashes(flutter_compact_version):
117 code = load_code("get-artifact-hashes.nix",
118 nixpkgs_root=NIXPKGS_ROOT,
119 flutter_compact_version=flutter_compact_version)
120
121 stderr = nix_build_to_fail(code)
122
123 pattern = re.compile(
124 r"/nix/store/.*-flutter-artifacts-(.+?)-(.+?).drv':\n\s+specified: .*\n\s+got:\s+(.+?)\n")
125 matches = pattern.findall(stderr)
126 result_dict = {}
127
128 for match in matches:
129 flutter_platform, architecture, got = match
130 result_dict.setdefault(flutter_platform, {})[architecture] = got
131
132 def sort_dict_recursive(d):
133 return {
134 k: sort_dict_recursive(v) if isinstance(
135 v, dict) else v for k, v in sorted(
136 d.items())}
137 result_dict = sort_dict_recursive(result_dict)
138
139 return result_dict
140
141
142def get_dart_hashes(dart_version, channel):
143 platforms = [
144 "x86_64-linux",
145 "aarch64-linux",
146 "x86_64-darwin",
147 "aarch64-darwin"]
148 result_dict = {}
149 for platform in platforms:
150 code = load_code(
151 "get-dart-hashes.nix",
152 dart_version=dart_version,
153 channel=channel,
154 platform=platform)
155 stderr = nix_build_to_fail(code)
156
157 pattern = re.compile(r"got:\s+(.+?)\n")
158 result_dict[platform] = pattern.findall(stderr)[0]
159
160 return result_dict
161
162
163def get_flutter_hash_and_src(flutter_version):
164 code = load_code(
165 "get-flutter.nix",
166 flutter_version=flutter_version,
167 hash="")
168
169 stderr = nix_build_to_fail(code)
170 pattern = re.compile(r"got:\s+(.+?)\n")
171 hash = pattern.findall(stderr)[0]
172
173 code = load_code(
174 "get-flutter.nix",
175 flutter_version=flutter_version,
176 hash=hash)
177
178 return (hash, nix_build(code))
179
180
181def get_pubspec_lock(flutter_compact_version, flutter_src):
182 code = load_code(
183 "get-pubspec-lock.nix",
184 flutter_compact_version=flutter_compact_version,
185 flutter_src=flutter_src,
186 hash="")
187
188 stderr = nix_build_to_fail(code)
189 pattern = re.compile(r"got:\s+(.+?)\n")
190 hash = pattern.findall(stderr)[0]
191
192 code = load_code(
193 "get-pubspec-lock.nix",
194 flutter_compact_version=flutter_compact_version,
195 flutter_src=flutter_src,
196 hash=hash)
197
198 pubspec_lock_file = nix_build(code)
199
200 with open(pubspec_lock_file, 'r') as f:
201 pubspec_lock_yaml = f.read()
202
203 return yaml.safe_load(pubspec_lock_yaml)
204
205def get_engine_swiftshader_rev(engine_version):
206 with urllib.request.urlopen(f"https://github.com/flutter/flutter/raw/{engine_version}/DEPS") as f:
207 deps = f.read().decode('utf-8')
208 pattern = re.compile(r"Var\('swiftshader_git'\) \+ '\/SwiftShader\.git' \+ '@' \+ \'([0-9a-fA-F]{40})\'\,")
209 rev = pattern.findall(deps)[0]
210 return rev
211
212def get_engine_swiftshader_hash(engine_swiftshader_rev):
213 code = load_code(
214 "get-engine-swiftshader.nix",
215 engine_swiftshader_rev=engine_swiftshader_rev,
216 hash="")
217
218 stderr = nix_build_to_fail(code)
219 pattern = re.compile(r"got:\s+(.+?)\n")
220 return pattern.findall(stderr)[0]
221
222def write_data(
223 nixpkgs_flutter_version_directory,
224 flutter_version,
225 channel,
226 engine_hash,
227 engine_hashes,
228 engine_swiftshader_hash,
229 engine_swiftshader_rev,
230 dart_version,
231 dart_hash,
232 flutter_hash,
233 artifact_hashes,
234 pubspec_lock):
235 with open(f"{nixpkgs_flutter_version_directory}/data.json", "w") as f:
236 f.write(json.dumps({
237 "version": flutter_version,
238 "engineVersion": engine_hash,
239 "engineSwiftShaderHash": engine_swiftshader_hash,
240 "engineSwiftShaderRev": engine_swiftshader_rev,
241 "channel": channel,
242 "engineHashes": engine_hashes,
243 "dartVersion": dart_version,
244 "dartHash": dart_hash,
245 "flutterHash": flutter_hash,
246 "artifactHashes": artifact_hashes,
247 "pubspecLock": pubspec_lock,
248 }, indent=2).strip() + "\n")
249
250
251def update_all_packages():
252 versions_directory = f"{NIXPKGS_ROOT}/pkgs/development/compilers/flutter/versions"
253 versions = [directory for directory in os.listdir(versions_directory)]
254 versions = sorted(versions, key=lambda x: (
255 int(x.split('_')[0]), int(x.split('_')[1])), reverse=True)
256
257 new_content = [
258 "flutterPackages-bin = recurseIntoAttrs (callPackage ../development/compilers/flutter { });",
259 "flutterPackages-source = recurseIntoAttrs (callPackage ../development/compilers/flutter { useNixpkgsEngine = true; });",
260 "flutterPackages = flutterPackages-bin;",
261 "flutter = flutterPackages.stable;",
262 ] + [f"flutter{version.replace('_', '')} = flutterPackages.v{version};" for version in versions]
263
264 with open(f"{NIXPKGS_ROOT}/pkgs/top-level/all-packages.nix", 'r') as file:
265 lines = file.read().splitlines(keepends=True)
266
267 start = -1
268 end = -1
269 for i, line in enumerate(lines):
270 if "flutterPackages-bin = recurseIntoAttrs (callPackage ../development/compilers/flutter { });" in line:
271 start = i
272 if start != -1 and len(line.strip()) == 0:
273 end = i
274 break
275
276 if start != -1 and end != -1:
277 del lines[start:end]
278 lines[start:start] = [f" {l}\n" for l in new_content]
279
280 with open(f"{NIXPKGS_ROOT}/pkgs/top-level/all-packages.nix", 'w') as file:
281 file.write("".join(lines))
282
283
284# Finds Flutter version, Dart version, and Engine hash.
285# If the Flutter version is given, it uses that. Otherwise finds the
286# latest stable Flutter version.
287def find_versions(flutter_version=None, channel=None):
288 engine_hash = None
289 dart_version = None
290
291 releases = json.load(urllib.request.urlopen(
292 "https://storage.googleapis.com/flutter_infra_release/releases/releases_linux.json"))
293
294 if not channel:
295 channel = 'stable'
296
297 if not flutter_version:
298 hash = releases['current_release'][channel]
299 release = next(
300 filter(
301 lambda release: release['hash'] == hash,
302 releases['releases']))
303 flutter_version = release['version']
304
305 tags = subprocess.Popen(['git',
306 'ls-remote',
307 '--tags',
308 'https://github.com/flutter/flutter.git'],
309 stdout=subprocess.PIPE,
310 text=True).communicate()[0].strip()
311
312 try:
313 flutter_hash = next(
314 filter(
315 lambda line: line.endswith(f'refs/tags/{flutter_version}'),
316 tags.splitlines())).split('refs')[0].strip()
317
318 engine_hash = urllib.request.urlopen(f'https://github.com/flutter/flutter/raw/{flutter_hash}/bin/internal/engine.version').read().decode('utf-8').strip()
319 except StopIteration:
320 exit(
321 f"Couldn't find Engine hash for Flutter version: {flutter_version}")
322
323 try:
324 dart_version = next(
325 filter(
326 lambda release: release['version'] == flutter_version,
327 releases['releases']))['dart_sdk_version']
328
329 if " " in dart_version:
330 dart_version = dart_version.split(' ')[2][:-1]
331 except StopIteration:
332 exit(
333 f"Couldn't find Dart version for Flutter version: {flutter_version}")
334
335 return (flutter_version, engine_hash, dart_version, channel)
336
337
338def main():
339 parser = argparse.ArgumentParser(description='Update Flutter in Nixpkgs')
340 parser.add_argument('--version', type=str, help='Specify Flutter version')
341 parser.add_argument('--channel', type=str, help='Specify Flutter release channel')
342 parser.add_argument('--artifact-hashes', action='store_true',
343 help='Whether to get artifact hashes')
344 args = parser.parse_args()
345
346 (flutter_version, engine_hash, dart_version, channel) = find_versions(args.version, args.channel)
347
348 flutter_compact_version = '_'.join(flutter_version.split('.')[:2])
349
350 if args.artifact_hashes:
351 print(
352 json.dumps(
353 get_artifact_hashes(flutter_compact_version),
354 indent=2).strip() +
355 "\n")
356 return
357
358 print(f"Flutter version: {flutter_version} ({flutter_compact_version}) on ({channel})")
359 print(f"Engine hash: {engine_hash}")
360 print(f"Dart version: {dart_version}")
361
362 dart_hash = get_dart_hashes(dart_version, channel)
363 (flutter_hash, flutter_src) = get_flutter_hash_and_src(flutter_version)
364
365 nixpkgs_flutter_version_directory = f"{NIXPKGS_ROOT}/pkgs/development/compilers/flutter/versions/{flutter_compact_version}"
366
367 if os.path.exists(f"{nixpkgs_flutter_version_directory}/data.json"):
368 os.remove(f"{nixpkgs_flutter_version_directory}/data.json")
369 os.makedirs(nixpkgs_flutter_version_directory, exist_ok=True)
370
371 update_all_packages()
372
373 common_data_args = {
374 "nixpkgs_flutter_version_directory": nixpkgs_flutter_version_directory,
375 "flutter_version": flutter_version,
376 "channel": channel,
377 "dart_version": dart_version,
378 "engine_hash": engine_hash,
379 "flutter_hash": flutter_hash,
380 "dart_hash": dart_hash,
381 }
382
383 write_data(
384 pubspec_lock={},
385 artifact_hashes={},
386 engine_hashes={},
387 engine_swiftshader_hash=FAKE_HASH,
388 engine_swiftshader_rev='0',
389 **common_data_args)
390
391 pubspec_lock = get_pubspec_lock(flutter_compact_version, flutter_src)
392
393 write_data(
394 pubspec_lock=pubspec_lock,
395 artifact_hashes={},
396 engine_hashes={},
397 engine_swiftshader_hash=FAKE_HASH,
398 engine_swiftshader_rev='0',
399 **common_data_args)
400
401 artifact_hashes = get_artifact_hashes(flutter_compact_version)
402
403 write_data(
404 pubspec_lock=pubspec_lock,
405 artifact_hashes=artifact_hashes,
406 engine_hashes={},
407 engine_swiftshader_hash=FAKE_HASH,
408 engine_swiftshader_rev='0',
409 **common_data_args)
410
411 engine_hashes = get_engine_hashes(engine_hash, flutter_version)
412
413 write_data(
414 pubspec_lock=pubspec_lock,
415 artifact_hashes=artifact_hashes,
416 engine_hashes=engine_hashes,
417 engine_swiftshader_hash=FAKE_HASH,
418 engine_swiftshader_rev='0',
419 **common_data_args)
420
421 engine_swiftshader_rev = get_engine_swiftshader_rev(engine_hash)
422 engine_swiftshader_hash = get_engine_swiftshader_hash(engine_swiftshader_rev)
423
424 write_data(
425 pubspec_lock=pubspec_lock,
426 artifact_hashes=artifact_hashes,
427 engine_hashes=engine_hashes,
428 engine_swiftshader_hash=engine_swiftshader_hash,
429 engine_swiftshader_rev=engine_swiftshader_rev,
430 **common_data_args)
431
432
433if __name__ == "__main__":
434 main()