tjh.dev
1{
2 callPackage,
3 fetchgit,
4 fontconfig,
5 git,
6 lib,
7 makeWrapper,
8 python3,
9 runCommand,
10 system,
11 writeText,
12 writeTextFile,
13
14 # Artifacts dependencies
15 fetchurl,
16 glibc,
17 pkgs,
18 stdenv,
19
20 julia,
21
22 # Special registry which is equal to JuliaRegistries/General, but every Versions.toml
23 # entry is augmented with a Nix sha256 hash
24 augmentedRegistry ? callPackage ./registry.nix { },
25
26 # Other overridable arguments
27 extraLibs ? [ ],
28 juliaCpuTarget ? null,
29 makeTransitiveDependenciesImportable ? false, # Used to support symbol indexing
30 makeWrapperArgs ? "",
31 packageOverrides ? { },
32 precompile ? true,
33 setDefaultDepot ? true,
34}:
35
36packageNames:
37
38let
39 util = callPackage ./util.nix { };
40
41 # Some Julia packages require access to Python. Provide a Nixpkgs version so it
42 # doesn't try to install its own.
43 pythonToUse =
44 let
45 extraPythonPackages = (
46 (callPackage ./extra-python-packages.nix { inherit python3; }).getExtraPythonPackages packageNames
47 );
48 in
49 (
50 if extraPythonPackages == [ ] then
51 python3
52 else
53 util.addPackagesToPython python3 (map (pkg: lib.getAttr pkg python3.pkgs) extraPythonPackages)
54 );
55
56 # Start by wrapping Julia so it has access to Python and any other extra libs.
57 # Also, prevent various packages (CondaPkg.jl, PythonCall.jl) from trying to do network calls.
58 juliaWrapped =
59 runCommand "julia-${julia.version}-wrapped"
60 {
61 nativeBuildInputs = [ makeWrapper ];
62 inherit makeWrapperArgs;
63 }
64 ''
65 mkdir -p $out/bin
66 makeWrapper ${julia}/bin/julia $out/bin/julia \
67 --suffix LD_LIBRARY_PATH : "${lib.makeLibraryPath extraLibs}" \
68 --set FONTCONFIG_FILE ${fontconfig.out}/etc/fonts/fonts.conf \
69 --set PYTHONHOME "${pythonToUse}" \
70 --prefix PYTHONPATH : "${pythonToUse}/${pythonToUse.sitePackages}" \
71 --set PYTHON ${pythonToUse}/bin/python $makeWrapperArgs \
72 --set JULIA_CONDAPKG_OFFLINE yes \
73 --set JULIA_CONDAPKG_BACKEND Null \
74 --set JULIA_PYTHONCALL_EXE "@PyCall"
75 '';
76
77 # If our closure ends up with certain packages, add others.
78 packageImplications = {
79 # Because we want to put PythonCall in PyCall mode so it doesn't try to download
80 # Python packages
81 PythonCall = [ "PyCall" ];
82 };
83
84 # Invoke Julia resolution logic to determine the full dependency closure
85 packageOverridesRepoified = lib.mapAttrs util.repoifySimple packageOverrides;
86 closureYaml = callPackage ./package-closure.nix {
87 inherit
88 augmentedRegistry
89 julia
90 packageNames
91 packageImplications
92 ;
93 packageOverrides = packageOverridesRepoified;
94 };
95
96 # Generate a Nix file consisting of a map from dependency UUID --> package info with fetchgit call:
97 # {
98 # "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3" = {
99 # src = fetchgit {...};
100 # name = "...";
101 # version = "...";
102 # treehash = "...";
103 # };
104 # ...
105 # }
106 dependencies =
107 runCommand "julia-sources.nix"
108 {
109 buildInputs = [
110 (python3.withPackages (
111 ps: with ps; [
112 toml
113 pyyaml
114 ]
115 ))
116 git
117 ];
118 }
119 ''
120 python ${./python}/sources_nix.py \
121 "${augmentedRegistry}" \
122 '${lib.generators.toJSON { } packageOverridesRepoified}' \
123 "${closureYaml}" \
124 "$out"
125 '';
126
127 # Import the Nix file from the previous step (IFD) and turn each dependency repo into
128 # a dummy Git repository, as Julia expects. Format the results as a YAML map from
129 # dependency UUID -> Nix store location:
130 # {
131 # "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3":"/nix/store/...-NaNMath.jl-0877504",
132 # ...
133 # }
134 # This is also the point where we apply the packageOverrides.
135 dependencyUuidToInfo = import dependencies { inherit fetchgit; };
136 fillInOverrideSrc =
137 uuid: info:
138 if lib.hasAttr info.name packageOverrides then
139 (info // { src = lib.getAttr info.name packageOverrides; })
140 else
141 info;
142 dependencyUuidToRepo = lib.mapAttrs util.repoifyInfo (
143 lib.mapAttrs fillInOverrideSrc dependencyUuidToInfo
144 );
145 dependencyUuidToRepoYaml = writeTextFile {
146 name = "dependency-uuid-to-repo.yml";
147 text = lib.generators.toYAML { } dependencyUuidToRepo;
148 };
149
150 # Given the augmented registry, closure info yaml, and dependency path yaml, construct a complete
151 # Julia registry containing all the necessary packages
152 dependencyUuidToInfoYaml = writeTextFile {
153 name = "dependency-uuid-to-info.yml";
154 text = lib.generators.toYAML { } dependencyUuidToInfo;
155 };
156 fillInOverrideSrc' =
157 uuid: info:
158 if lib.hasAttr info.name packageOverridesRepoified then
159 (info // { src = lib.getAttr info.name packageOverridesRepoified; })
160 else
161 info;
162 overridesOnly = lib.mapAttrs fillInOverrideSrc' (
163 lib.filterAttrs (uuid: info: info.src == null) dependencyUuidToInfo
164 );
165 minimalRegistry =
166 runCommand "minimal-julia-registry"
167 {
168 buildInputs = [
169 (python3.withPackages (
170 ps: with ps; [
171 toml
172 pyyaml
173 ]
174 ))
175 git
176 ];
177 }
178 ''
179 python ${./python}/minimal_registry.py \
180 "${augmentedRegistry}" \
181 "${closureYaml}" \
182 '${lib.generators.toJSON { } overridesOnly}' \
183 "${dependencyUuidToRepoYaml}" \
184 "$out"
185 '';
186
187 # Next, deal with artifacts. Scan each artifacts file individually and generate a Nix file that
188 # produces the desired Overrides.toml.
189 artifactsNix =
190 runCommand "julia-artifacts.nix"
191 {
192 buildInputs = [
193 (python3.withPackages (
194 ps: with ps; [
195 toml
196 pyyaml
197 ]
198 ))
199 ];
200 }
201 ''
202 python ${./python}/extract_artifacts.py \
203 "${dependencyUuidToRepoYaml}" \
204 "${closureYaml}" \
205 "${juliaWrapped}/bin/julia" \
206 "${
207 if lib.versionAtLeast julia.version "1.7" then ./extract_artifacts.jl else ./extract_artifacts_16.jl
208 }" \
209 '${lib.generators.toJSON { } (import ./extra-libs.nix)}' \
210 '${lib.generators.toJSON { } (stdenv.hostPlatform.isDarwin)}' \
211 "$out"
212 '';
213
214 # Import the artifacts Nix to build Overrides.toml (IFD)
215 artifacts = import artifactsNix (
216 {
217 inherit
218 lib
219 fetchurl
220 pkgs
221 stdenv
222 ;
223 }
224 // lib.optionalAttrs (!stdenv.targetPlatform.isDarwin) {
225 inherit glibc;
226 }
227 );
228 overridesJson = writeTextFile {
229 name = "Overrides.json";
230 text = lib.generators.toJSON { } artifacts;
231 };
232 overridesToml =
233 runCommand "Overrides.toml" { buildInputs = [ (python3.withPackages (ps: with ps; [ toml ])) ]; }
234 ''
235 python ${./python}/format_overrides.py \
236 "${overridesJson}" \
237 "$out"
238 '';
239
240 # Build a Julia project and depot. The project contains Project.toml/Manifest.toml, while the
241 # depot contains package build products (including the precompiled libraries, if precompile=true)
242 projectAndDepot = callPackage ./depot.nix {
243 inherit
244 closureYaml
245 extraLibs
246 juliaCpuTarget
247 overridesToml
248 packageImplications
249 precompile
250 ;
251 julia = juliaWrapped;
252 registry = minimalRegistry;
253 packageNames =
254 if makeTransitiveDependenciesImportable then
255 lib.mapAttrsToList (uuid: info: info.name) dependencyUuidToInfo
256 else
257 packageNames;
258 };
259
260in
261
262runCommand "julia-${julia.version}-env"
263 {
264 nativeBuildInputs = [ makeWrapper ];
265
266 passthru = {
267 inherit julia;
268 inherit juliaWrapped;
269 inherit (julia) pname version meta;
270
271 # Expose the steps we used along the way in case the user wants to use them, for example to build
272 # expressions and build them separately to avoid IFD.
273 inherit dependencies;
274 inherit closureYaml;
275 inherit dependencyUuidToInfoYaml;
276 inherit dependencyUuidToRepoYaml;
277 inherit minimalRegistry;
278 inherit artifactsNix;
279 inherit overridesJson;
280 inherit overridesToml;
281 inherit projectAndDepot;
282 };
283 }
284 (
285 ''
286 mkdir -p $out/bin
287 makeWrapper ${juliaWrapped}/bin/julia $out/bin/julia \
288 --suffix JULIA_DEPOT_PATH : "${projectAndDepot}/depot" \
289 --set-default JULIA_PROJECT "${projectAndDepot}/project" \
290 --set-default JULIA_LOAD_PATH '@:${projectAndDepot}/project/Project.toml:@v#.#:@stdlib'
291 ''
292 + lib.optionalString setDefaultDepot ''
293 sed -i '2 i\JULIA_DEPOT_PATH=''${JULIA_DEPOT_PATH-"$HOME/.julia"}' $out/bin/julia
294 ''
295 )