pyrox.dev
1
2import json
3from pathlib import Path
4import multiprocessing
5import subprocess
6import sys
7import toml
8from urllib.parse import urlparse
9import yaml
10
11import dag
12
13# This should match the behavior of the default unpackPhase.
14# See https://github.com/NixOS/nixpkgs/blob/59fa082abdbf462515facc8800d517f5728c909d/pkgs/stdenv/generic/setup.sh#L1044
15archive_extensions = [
16 # xz extensions
17 ".tar.xz",
18 ".tar.lzma",
19 ".txz",
20
21 # *.tar or *.tar.*
22 ".tar",
23 ".tar.Z",
24 ".tar.bz2",
25 ".tar.gz",
26
27 # Other tar extensions
28 ".tgz",
29 ".tbz2",
30 ".tbz",
31
32 ".zip"
33 ]
34
35dependencies_path = Path(sys.argv[1])
36closure_yaml_path = Path(sys.argv[2])
37julia_path = Path(sys.argv[3])
38extract_artifacts_script = Path(sys.argv[4])
39extra_libs = json.loads(sys.argv[5])
40out_path = Path(sys.argv[6])
41
42with open(dependencies_path, "r") as f:
43 dependencies = yaml.safe_load(f)
44 dependency_uuids = dependencies.keys()
45
46with open(closure_yaml_path, "r") as f:
47 # Build up a map of UUID -> closure information
48 closure_yaml_list = yaml.safe_load(f) or []
49 closure_yaml = {}
50 for item in closure_yaml_list:
51 closure_yaml[item["uuid"]] = item
52
53 # Build up a dependency graph of UUIDs
54 closure_dependencies_dag = dag.DAG()
55 for uuid, contents in closure_yaml.items():
56 if contents.get("depends_on"):
57 closure_dependencies_dag.add_node(uuid, dependencies=contents["depends_on"].values())
58
59def get_archive_derivation(uuid, artifact_name, url, sha256):
60 depends_on = set()
61 if closure_dependencies_dag.has_node(uuid):
62 depends_on = set(closure_dependencies_dag.get_dependencies(uuid)).intersection(dependency_uuids)
63
64 other_libs = extra_libs.get(uuid, [])
65
66 fixup = f"""fixupPhase = let
67 libs = lib.concatMap (lib.mapAttrsToList (k: v: v.path))
68 [{" ".join(["uuid-" + x for x in depends_on])}];
69 in ''
70 find $out -type f -executable -exec \
71 patchelf --set-rpath \$ORIGIN:\$ORIGIN/../lib:${{lib.makeLibraryPath (["$out" glibc] ++ libs ++ (with pkgs; [{" ".join(other_libs)}]))}} {{}} \;
72 find $out -type f -executable -exec \
73 patchelf --set-interpreter ${{glibc}}/lib/ld-linux-x86-64.so.2 {{}} \;
74 ''"""
75
76 return f"""stdenv.mkDerivation {{
77 name = "{artifact_name}";
78 src = fetchurl {{
79 url = "{url}";
80 sha256 = "{sha256}";
81 }};
82 preUnpack = ''
83 mkdir unpacked
84 cd unpacked
85 '';
86 sourceRoot = ".";
87 dontConfigure = true;
88 dontBuild = true;
89 installPhase = "cp -r . $out";
90 {fixup};
91 }}"""
92
93def get_plain_derivation(url, sha256):
94 return f"""fetchurl {{
95 url = "{url}";
96 sha256 = "{sha256}";
97 }}"""
98
99with open(out_path, "w") as f:
100 f.write("{ lib, fetchurl, glibc, pkgs, stdenv }:\n\n")
101 f.write("rec {\n")
102
103 def process_item(item):
104 uuid, src = item
105 lines = []
106 artifacts = toml.loads(subprocess.check_output([julia_path, extract_artifacts_script, uuid, src]).decode())
107 if not artifacts: return f' uuid-{uuid} = {{}};\n'
108
109 lines.append(f' uuid-{uuid} = {{')
110
111 for artifact_name, details in artifacts.items():
112 if len(details["download"]) == 0: continue
113 download = details["download"][0]
114 url = download["url"]
115 sha256 = download["sha256"]
116
117 git_tree_sha1 = details["git-tree-sha1"]
118
119 parsed_url = urlparse(url)
120 if any(parsed_url.path.endswith(x) for x in archive_extensions):
121 derivation = get_archive_derivation(uuid, artifact_name, url, sha256)
122 else:
123 derivation = get_plain_derivation(url, sha256)
124
125 lines.append(f""" "{artifact_name}" = {{
126 sha1 = "{git_tree_sha1}";
127 path = {derivation};
128 }};\n""")
129
130 lines.append(' };\n')
131
132 return "\n".join(lines)
133
134 with multiprocessing.Pool(10) as pool:
135 for s in pool.map(process_item, dependencies.items()):
136 f.write(s)
137
138 f.write(f"""
139}}\n""")