tjh.dev
1{
2 callPackage,
3 fetchgit,
4 fontconfig,
5 git,
6 lib,
7 makeWrapper,
8 python3,
9 runCommand,
10 writeTextFile,
11
12 # Artifacts dependencies
13 fetchurl,
14 glibc,
15 pkgs,
16 stdenv,
17
18 julia,
19
20 # Special registry which is equal to JuliaRegistries/General, but every Versions.toml
21 # entry is augmented with a Nix sha256 hash
22 augmentedRegistry ? callPackage ./registry.nix { },
23
24 # Other overridable arguments
25 extraLibs ? [ ],
26 juliaCpuTarget ? null,
27 makeTransitiveDependenciesImportable ? false, # Used to support symbol indexing
28 makeWrapperArgs ? "",
29 packageOverrides ? { },
30 precompile ? true,
31 setDefaultDepot ? true,
32}:
33
34packageNames:
35
36let
37 util = callPackage ./util.nix { };
38
39 # Some Julia packages require access to Python. Provide a Nixpkgs version so it
40 # doesn't try to install its own.
41 pythonToUse =
42 let
43 extraPythonPackages = (
44 (callPackage ./extra-python-packages.nix { inherit python3; }).getExtraPythonPackages packageNames
45 );
46 in
47 (
48 if extraPythonPackages == [ ] then
49 python3
50 else
51 util.addPackagesToPython python3 (map (pkg: lib.getAttr pkg python3.pkgs) extraPythonPackages)
52 );
53
54 # Start by wrapping Julia so it has access to Python and any other extra libs.
55 # Also, prevent various packages (CondaPkg.jl, PythonCall.jl) from trying to do network calls.
56 juliaWrapped =
57 runCommand "julia-${julia.version}-wrapped"
58 {
59 nativeBuildInputs = [ makeWrapper ];
60 inherit makeWrapperArgs;
61 }
62 ''
63 mkdir -p $out/bin
64 makeWrapper ${julia}/bin/julia $out/bin/julia \
65 --suffix LD_LIBRARY_PATH : "${lib.makeLibraryPath extraLibs}" \
66 --set FONTCONFIG_FILE ${fontconfig.out}/etc/fonts/fonts.conf \
67 --set PYTHONHOME "${pythonToUse}" \
68 --prefix PYTHONPATH : "${pythonToUse}/${pythonToUse.sitePackages}" \
69 --set PYTHON ${pythonToUse}/bin/python $makeWrapperArgs \
70 --set JULIA_CONDAPKG_OFFLINE yes \
71 --set JULIA_CONDAPKG_BACKEND Null \
72 --set JULIA_PYTHONCALL_EXE "@PyCall"
73 '';
74
75 # If our closure ends up with certain packages, add others.
76 packageImplications = {
77 # Because we want to put PythonCall in PyCall mode so it doesn't try to download
78 # Python packages
79 PythonCall = [ "PyCall" ];
80 };
81
82 # Invoke Julia resolution logic to determine the full dependency closure
83 packageOverridesRepoified = lib.mapAttrs util.repoifySimple packageOverrides;
84 closureYaml = callPackage ./package-closure.nix {
85 inherit
86 augmentedRegistry
87 julia
88 packageNames
89 packageImplications
90 ;
91 packageOverrides = packageOverridesRepoified;
92 };
93
94 # Generate a Nix file consisting of a map from dependency UUID --> package info with fetchgit call:
95 # {
96 # "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3" = {
97 # src = fetchgit {...};
98 # name = "...";
99 # version = "...";
100 # treehash = "...";
101 # };
102 # ...
103 # }
104 dependencies =
105 runCommand "julia-sources.nix"
106 {
107 buildInputs = [
108 (python3.withPackages (
109 ps: with ps; [
110 toml
111 pyyaml
112 ]
113 ))
114 git
115 ];
116 }
117 ''
118 python ${./python}/sources_nix.py \
119 "${augmentedRegistry}" \
120 '${lib.generators.toJSON { } packageOverridesRepoified}' \
121 "${closureYaml}" \
122 "$out"
123 '';
124
125 # Import the Nix file from the previous step (IFD) and turn each dependency repo into
126 # a dummy Git repository, as Julia expects. Format the results as a YAML map from
127 # dependency UUID -> Nix store location:
128 # {
129 # "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3":"/nix/store/...-NaNMath.jl-0877504",
130 # ...
131 # }
132 # This is also the point where we apply the packageOverrides.
133 dependencyUuidToInfo = import dependencies { inherit fetchgit; };
134 fillInOverrideSrc =
135 uuid: info:
136 if lib.hasAttr info.name packageOverrides then
137 (info // { src = lib.getAttr info.name packageOverrides; })
138 else
139 info;
140 dependencyUuidToRepo = lib.mapAttrs util.repoifyInfo (
141 lib.mapAttrs fillInOverrideSrc dependencyUuidToInfo
142 );
143 dependencyUuidToRepoYaml = writeTextFile {
144 name = "dependency-uuid-to-repo.yml";
145 text = lib.generators.toYAML { } dependencyUuidToRepo;
146 };
147
148 # Given the augmented registry, closure info yaml, and dependency path yaml, construct a complete
149 # Julia registry containing all the necessary packages
150 dependencyUuidToInfoYaml = writeTextFile {
151 name = "dependency-uuid-to-info.yml";
152 text = lib.generators.toYAML { } dependencyUuidToInfo;
153 };
154 fillInOverrideSrc' =
155 uuid: info:
156 if lib.hasAttr info.name packageOverridesRepoified then
157 (info // { src = lib.getAttr info.name packageOverridesRepoified; })
158 else
159 info;
160 overridesOnly = lib.mapAttrs fillInOverrideSrc' (
161 lib.filterAttrs (uuid: info: info.src == null) dependencyUuidToInfo
162 );
163 minimalRegistry =
164 runCommand "minimal-julia-registry"
165 {
166 buildInputs = [
167 (python3.withPackages (
168 ps: with ps; [
169 toml
170 pyyaml
171 ]
172 ))
173 git
174 ];
175 }
176 ''
177 python ${./python}/minimal_registry.py \
178 "${augmentedRegistry}" \
179 "${closureYaml}" \
180 '${lib.generators.toJSON { } overridesOnly}' \
181 "${dependencyUuidToRepoYaml}" \
182 "$out"
183 '';
184
185 # Next, deal with artifacts. Scan each artifacts file individually and generate a Nix file that
186 # produces the desired Overrides.toml.
187 artifactsNix =
188 runCommand "julia-artifacts.nix"
189 {
190 buildInputs = [
191 (python3.withPackages (
192 ps: with ps; [
193 toml
194 pyyaml
195 ]
196 ))
197 ];
198 }
199 ''
200 python ${./python}/extract_artifacts.py \
201 "${dependencyUuidToRepoYaml}" \
202 "${closureYaml}" \
203 "${juliaWrapped}/bin/julia" \
204 "${
205 if lib.versionAtLeast julia.version "1.7" then ./extract_artifacts.jl else ./extract_artifacts_16.jl
206 }" \
207 '${lib.generators.toJSON { } (import ./extra-libs.nix)}' \
208 '${lib.generators.toJSON { } (stdenv.hostPlatform.isDarwin)}' \
209 "$out"
210 '';
211
212 # Import the artifacts Nix to build Overrides.toml (IFD)
213 artifacts = import artifactsNix (
214 {
215 inherit
216 lib
217 fetchurl
218 pkgs
219 stdenv
220 ;
221 }
222 // lib.optionalAttrs (!stdenv.targetPlatform.isDarwin) {
223 inherit glibc;
224 }
225 );
226 overridesJson = writeTextFile {
227 name = "Overrides.json";
228 text = lib.generators.toJSON { } artifacts;
229 };
230 overridesToml =
231 runCommand "Overrides.toml" { buildInputs = [ (python3.withPackages (ps: with ps; [ toml ])) ]; }
232 ''
233 python ${./python}/format_overrides.py \
234 "${overridesJson}" \
235 "$out"
236 '';
237
238 # Build a Julia project and depot. The project contains Project.toml/Manifest.toml, while the
239 # depot contains package build products (including the precompiled libraries, if precompile=true)
240 projectAndDepot = callPackage ./depot.nix {
241 inherit
242 closureYaml
243 extraLibs
244 juliaCpuTarget
245 overridesToml
246 packageImplications
247 precompile
248 ;
249 julia = juliaWrapped;
250 registry = minimalRegistry;
251 packageNames =
252 if makeTransitiveDependenciesImportable then
253 lib.mapAttrsToList (uuid: info: info.name) dependencyUuidToInfo
254 else
255 packageNames;
256 };
257
258in
259
260runCommand "julia-${julia.version}-env"
261 {
262 nativeBuildInputs = [ makeWrapper ];
263
264 passthru = {
265 inherit julia;
266 inherit juliaWrapped;
267 inherit (julia) pname version meta;
268
269 # Expose the steps we used along the way in case the user wants to use them, for example to build
270 # expressions and build them separately to avoid IFD.
271 inherit dependencies;
272 inherit closureYaml;
273 inherit dependencyUuidToInfoYaml;
274 inherit dependencyUuidToRepoYaml;
275 inherit minimalRegistry;
276 inherit artifactsNix;
277 inherit overridesJson;
278 inherit overridesToml;
279 inherit projectAndDepot;
280 };
281 }
282 (
283 ''
284 mkdir -p $out/bin
285 makeWrapper ${juliaWrapped}/bin/julia $out/bin/julia \
286 --suffix JULIA_DEPOT_PATH : "${projectAndDepot}/depot" \
287 --set-default JULIA_PROJECT "${projectAndDepot}/project" \
288 --set-default JULIA_LOAD_PATH '@:${projectAndDepot}/project/Project.toml:@v#.#:@stdlib'
289 ''
290 + lib.optionalString setDefaultDepot ''
291 sed -i '2 i\JULIA_DEPOT_PATH=''${JULIA_DEPOT_PATH-"$HOME/.julia"}' $out/bin/julia
292 ''
293 )