tjh.dev
nixpkgs mirror (for testing)
github.com/NixOS/nixpkgs
nix
1# NOTE: buildRedist should never take manifests or fixups as callPackage-provided arguments,
2# since we want to provide the flexibility to call it directly with a different fixup or manifest.
3{
4 _cuda,
5 autoAddCudaCompatRunpath,
6 autoAddDriverRunpath,
7 autoPatchelfHook,
8 backendStdenv,
9 cudaMajorMinorVersion,
10 cudaMajorVersion,
11 cudaNamePrefix,
12 fetchurl,
13 lib,
14 manifests,
15 markForCudatoolkitRootHook,
16 removeStubsFromRunpathHook,
17 srcOnly,
18 stdenv,
19 stdenvNoCC,
20}:
21let
22 inherit (backendStdenv) hostRedistSystem;
23 inherit (_cuda.lib) getNixSystems _mkCudaVariant mkRedistUrl;
24 inherit (lib.attrsets)
25 foldlAttrs
26 getDev
27 hasAttr
28 isAttrs
29 attrNames
30 optionalAttrs
31 ;
32 inherit (lib.customisation) extendMkDerivation;
33 inherit (lib.lists)
34 naturalSort
35 concatMap
36 unique
37 ;
38 inherit (lib.trivial) mapNullable pipe;
39 inherit (_cuda.lib) _mkMetaBadPlatforms _mkMetaBroken _redistSystemIsSupported;
40 inherit (lib)
41 licenses
42 sourceTypes
43 teams
44 ;
45 inherit (lib.asserts) assertMsg;
46 inherit (lib.lists)
47 elem
48 findFirst
49 findFirstIndex
50 foldl'
51 intersectLists
52 map
53 subtractLists
54 tail
55 ;
56 inherit (lib.strings)
57 concatMapStringsSep
58 optionalString
59 toUpper
60 stringLength
61 substring
62 ;
63 inherit (lib.trivial) flip;
64
65 mkOutputNameVar =
66 output:
67 assert assertMsg (output != "") "mkOutputNameVar: output name variable must not be empty";
68 "output" + toUpper (substring 0 1 output) + substring 1 (stringLength output - 1) output;
69
70 getSupportedReleases =
71 let
72 desiredCudaVariant = _mkCudaVariant cudaMajorVersion;
73 in
74 release:
75 # Always show preference to the "source", then "linux-all" redistSystem if they are available, as they are
76 # the most general.
77 if release ? source then
78 {
79 inherit (release) source;
80 }
81 else if release ? linux-all then
82 {
83 inherit (release) linux-all;
84 }
85 else
86 let
87 hasCudaVariants = release ? cuda_variant;
88 in
89 foldlAttrs (
90 acc: name: value:
91 acc
92 # If the value is an attribute, and when hasCudaVariants is true it has the relevant CUDA variant,
93 # then add it to the set.
94 // optionalAttrs (isAttrs value && (hasCudaVariants -> hasAttr desiredCudaVariant value)) {
95 ${name} = value.${desiredCudaVariant} or value;
96 }
97 ) { } release;
98
99 getPreferredRelease =
100 supportedReleases:
101 supportedReleases.source or supportedReleases.linux-all or supportedReleases.${hostRedistSystem}
102 or null;
103in
104extendMkDerivation {
105 constructDrv = backendStdenv.mkDerivation;
106 # These attributes are moved to passthru to avoid changing derivation hashes.
107 excludeDrvArgNames = [
108 # Core
109 "redistName"
110 "release"
111
112 # Misc
113 "brokenAssertions"
114 "platformAssertions"
115 "expectedOutputs"
116 "outputToPatterns"
117 "outputNameVarFallbacks"
118 ];
119 extendDrvArgs =
120 finalAttrs:
121 {
122 # Core
123 redistName,
124 pname,
125 release ? manifests.${finalAttrs.passthru.redistName}.${finalAttrs.pname} or null,
126
127 # Outputs
128 outputs ? [ "out" ],
129 propagatedBuildOutputs ? [ ],
130
131 # Inputs
132 nativeBuildInputs ? [ ],
133 propagatedBuildInputs ? [ ],
134 buildInputs ? [ ],
135
136 # Checking
137 doInstallCheck ? true,
138 allowFHSReferences ? false,
139
140 # Fixups
141 appendRunpaths ? [ ],
142 includeRemoveStubsFromRunpathHook ? elem "stubs" finalAttrs.outputs,
143 postFixup ? "",
144
145 # Extra
146 passthru ? { },
147 meta ? { },
148
149 # Misc
150 brokenAssertions ? [ ],
151 platformAssertions ? [ ],
152
153 # Order is important here so we use a list.
154 expectedOutputs ? [
155 "out"
156 "doc"
157 "samples"
158 "python"
159 "bin"
160 "dev"
161 "include"
162 "lib"
163 "static"
164 "stubs"
165 ],
166
167 # Traversed in the order of the outputs speficied in outputs;
168 # entries are skipped if they don't exist in outputs.
169 # NOTE: The nil LSP gets angry if we do not parenthesize the default attrset.
170 outputToPatterns ? {
171 bin = [ "bin" ];
172 dev = [
173 "**/*.pc"
174 "**/*.cmake"
175 ];
176 include = [ "include" ];
177 lib = [
178 "lib"
179 "lib64"
180 ];
181 static = [ "**/*.a" ];
182 samples = [ "samples" ];
183 python = [ "**/*.whl" ];
184 stubs = [
185 "stubs"
186 "lib/stubs"
187 ];
188 },
189
190 # Defines a list of fallbacks for each potential output.
191 # The last fallback is the out output.
192 # Taken and modified from:
193 # https://github.com/NixOS/nixpkgs/blob/fe5e11faed6241aacf7220436088789287507494/pkgs/build-support/setup-hooks/multiple-outputs.sh#L45-L62
194 outputNameVarFallbacks ? {
195 outputBin = [ "bin" ];
196 outputDev = [ "dev" ];
197 outputDoc = [ "doc" ];
198 outputInclude = [
199 "include"
200 "dev"
201 ];
202 outputLib = [ "lib" ];
203 outputOut = [ "out" ];
204 outputPython = [ "python" ];
205 outputSamples = [ "samples" ];
206 outputStatic = [ "static" ];
207 outputStubs = [
208 "stubs"
209 "lib"
210 ];
211 },
212 ...
213 }:
214 {
215 __structuredAttrs = true;
216 strictDeps = true;
217
218 # NOTE: `release` may be null if a redistributable isn't available.
219 version = finalAttrs.passthru.release.version or "0-unsupported";
220
221 # Name should be prefixed by cudaNamePrefix to create more descriptive path names.
222 name = "${cudaNamePrefix}-${finalAttrs.pname}-${finalAttrs.version}";
223
224 # We should only have the output `out` when `src` is null.
225 # lists.intersectLists iterates over the second list, checking if the elements are in the first list.
226 # As such, the order of the output is dictated by the order of the second list.
227 outputs =
228 if finalAttrs.src == null then
229 [ "out" ]
230 else
231 intersectLists outputs finalAttrs.passthru.expectedOutputs;
232
233 # NOTE: Because the `dev` output is special in Nixpkgs -- make-derivation.nix uses it as the default if
234 # it is present -- we must ensure that it brings in the expected dependencies. For us, this means that `dev`
235 # should include `bin`, `include`, and `lib` -- `static` is notably absent because it is quite large.
236 # We do not include `stubs`, as a number of packages contain stubs for libraries they already ship with!
237 # Only a few, like cuda_cudart, actually provide stubs for libraries we're missing.
238 # As such, these packages should override propagatedBuildOutputs to add `stubs`.
239 propagatedBuildOutputs =
240 intersectLists [
241 "bin"
242 "include"
243 "lib"
244 ] finalAttrs.outputs
245 ++ propagatedBuildOutputs;
246
247 # src :: null | Derivation
248 src = mapNullable (
249 { relative_path, sha256, ... }:
250 srcOnly {
251 __structuredAttrs = true;
252 strictDeps = true;
253 stdenv = stdenvNoCC;
254 inherit (finalAttrs) pname version;
255 src = fetchurl {
256 url = mkRedistUrl finalAttrs.passthru.redistName relative_path;
257 inherit sha256;
258 };
259 }
260 ) (getPreferredRelease finalAttrs.passthru.supportedReleases);
261
262 # Required for the hook.
263 inherit cudaMajorMinorVersion cudaMajorVersion;
264
265 # We do need some other phases, like configurePhase, so the multiple-output setup hook works.
266 dontBuild = true;
267
268 nativeBuildInputs = [
269 ./buildRedistHook.bash
270 autoPatchelfHook
271 # This hook will make sure libcuda can be found
272 # in typically /lib/opengl-driver by adding that
273 # directory to the rpath of all ELF binaries.
274 # Check e.g. with `patchelf --print-rpath path/to/my/binary
275 # TODO(@connorbaker): Given we'll have stubs available, we can switch from autoPatchelfIgnoreMissingDeps to
276 # allowing autoPatchelf to find and link against the stub files and rely on removeStubsFromRunpathHook to
277 # automatically find and replace those references with ones to the driver link lib directory.
278 autoAddDriverRunpath
279 markForCudatoolkitRootHook
280 ]
281 # autoAddCudaCompatRunpath depends on cuda_compat and would cause
282 # infinite recursion if applied to `cuda_compat` itself (beside the fact
283 # that it doesn't make sense in the first place)
284 ++ lib.optionals (finalAttrs.pname != "cuda_compat" && autoAddCudaCompatRunpath.enableHook) [
285 # autoAddCudaCompatRunpath must appear AFTER autoAddDriverRunpath.
286 # See its documentation in ./setup-hooks/extension.nix.
287 autoAddCudaCompatRunpath
288 ]
289 ++ nativeBuildInputs;
290
291 buildInputs = [
292 # autoPatchelfHook will search for a libstdc++ and we're giving it
293 # one that is compatible with the rest of nixpkgs, even when
294 # nvcc forces us to use an older gcc
295 # NB: We don't actually know if this is the right thing to do
296 # NOTE: Not all packages actually need this, but it's easier to just add it than create overrides for nearly all
297 # of them.
298 (lib.getLib stdenv.cc.cc)
299 ]
300 ++ buildInputs;
301
302 # Picked up by autoPatchelf
303 # Needed e.g. for libnvrtc to locate (dlopen) libnvrtc-builtins
304 appendRunpaths = [ "$ORIGIN" ] ++ appendRunpaths;
305
306 # NOTE: We don't need to check for dev or doc, because those outputs are handled by
307 # the multiple-outputs setup hook.
308 # NOTE: moveToOutput operates on all outputs:
309 # https://github.com/NixOS/nixpkgs/blob/2920b6fc16a9ed5d51429e94238b28306ceda79e/pkgs/build-support/setup-hooks/multiple-outputs.sh#L105-L107
310 # NOTE: installPhase is not moved into the builder hook because we do a lot of Nix templating.
311 installPhase =
312 let
313 mkMoveToOutputCommand =
314 output:
315 let
316 template = pattern: ''
317 moveToOutput "${pattern}" "${"$" + output}"
318 '';
319 patterns = finalAttrs.passthru.outputToPatterns.${output} or [ ];
320 in
321 concatMapStringsSep "\n" template patterns;
322 in
323 # Pre-install hook
324 ''
325 runHook preInstall
326 ''
327 # Create the primary output, out, and move the other outputs into it.
328 + ''
329 mkdir -p "$out"
330 nixLog "moving tree to output out"
331 mv * "$out"
332 ''
333 # Move the outputs into their respective outputs.
334 + ''
335 ${concatMapStringsSep "\n" mkMoveToOutputCommand (tail finalAttrs.outputs)}
336 ''
337 # Post-install hook
338 + ''
339 runHook postInstall
340 '';
341
342 inherit doInstallCheck;
343 inherit allowFHSReferences;
344 inherit includeRemoveStubsFromRunpathHook;
345
346 postFixup =
347 postFixup
348 + optionalString finalAttrs.includeRemoveStubsFromRunpathHook ''
349 nixLog "installing stub removal runpath hook"
350 mkdir -p "''${!outputStubs:?}/nix-support"
351 printWords >>"''${!outputStubs:?}/nix-support/propagated-build-inputs" \
352 "${getDev removeStubsFromRunpathHook}"
353 '';
354
355 passthru = passthru // {
356 inherit redistName release;
357
358 supportedReleases =
359 passthru.supportedReleases
360 # NOTE: `release` may be null, so we must use `lib.defaultTo`
361 or (getSupportedReleases (lib.defaultTo { } finalAttrs.passthru.release));
362
363 supportedNixSystems =
364 passthru.supportedNixSystems or (pipe finalAttrs.passthru.supportedReleases [
365 attrNames
366 (concatMap getNixSystems)
367 naturalSort
368 unique
369 ]);
370
371 supportedRedistSystems =
372 passthru.supportedRedistSystems or (naturalSort (attrNames finalAttrs.passthru.supportedReleases));
373
374 # NOTE: Downstream may expand this to include other outputs, but they must remember to set the appropriate
375 # outputNameVarFallbacks!
376 inherit expectedOutputs;
377
378 # Traversed in the order of the outputs speficied in outputs;
379 # entries are skipped if they don't exist in outputs.
380 inherit outputToPatterns;
381
382 # Defines a list of fallbacks for each potential output.
383 # The last fallback is the out output.
384 # Taken and modified from:
385 # https://github.com/NixOS/nixpkgs/blob/fe5e11faed6241aacf7220436088789287507494/pkgs/build-support/setup-hooks/multiple-outputs.sh#L45-L62
386 inherit outputNameVarFallbacks;
387
388 # brokenAssertions :: [Attrs]
389 # Used by mkMetaBroken to set `meta.broken`.
390 # Example: Broken on a specific version of CUDA or when a dependency has a specific version.
391 # NOTE: Do not use this when a broken assertion means evaluation will fail! For example, if
392 # a package is missing and is required for the build -- that should go in platformAssertions,
393 # because attempts to access attributes on the package will cause evaluation errors.
394 brokenAssertions = [
395 {
396 message = "lib output precedes static output";
397 assertion =
398 let
399 libIndex = findFirstIndex (x: x == "lib") null finalAttrs.outputs;
400 staticIndex = findFirstIndex (x: x == "static") null finalAttrs.outputs;
401 in
402 libIndex == null || staticIndex == null || libIndex < staticIndex;
403 }
404 {
405 # NOTE: We cannot (easily) check that all expected outputs have a corresponding outputNameVar attribute in
406 # finalAttrs because of the presence of attributes which use the "output" prefix but are not outputNameVars
407 # (e.g., outputChecks and outputName).
408 message = "outputNameVarFallbacks is a super set of expectedOutputs";
409 assertion =
410 subtractLists (map mkOutputNameVar finalAttrs.passthru.expectedOutputs) (
411 attrNames finalAttrs.passthru.outputNameVarFallbacks
412 ) == [ ];
413 }
414 {
415 message = "outputToPatterns is a super set of expectedOutputs";
416 assertion =
417 subtractLists finalAttrs.passthru.expectedOutputs (attrNames finalAttrs.passthru.outputToPatterns)
418 == [ ];
419 }
420 {
421 message = "propagatedBuildOutputs is a subset of outputs";
422 assertion = subtractLists finalAttrs.outputs finalAttrs.propagatedBuildOutputs == [ ];
423 }
424 ]
425 ++ brokenAssertions;
426
427 # platformAssertions :: [Attrs]
428 # Used by mkMetaBadPlatforms to set `meta.badPlatforms`.
429 # Example: Broken on a specific system when some condition is met, like targeting Jetson or
430 # a required package missing.
431 # NOTE: Use this when a failed assertion means evaluation can fail!
432 platformAssertions =
433 let
434 isSupportedRedistSystem = _redistSystemIsSupported hostRedistSystem finalAttrs.passthru.supportedRedistSystems;
435 in
436 [
437 {
438 message = "src is null if and only if hostRedistSystem is unsupported";
439 assertion = (finalAttrs.src == null) == !isSupportedRedistSystem;
440 }
441 {
442 message = "hostRedistSystem (${hostRedistSystem}) is supported (${builtins.toJSON finalAttrs.passthru.supportedRedistSystems})";
443 assertion = isSupportedRedistSystem;
444 }
445 ]
446 ++ platformAssertions;
447 };
448
449 meta = meta // {
450 longDescription = meta.longDescription or "" + ''
451 By downloading and using this package you accept the terms and conditions of the associated license(s).
452 '';
453 sourceProvenance = meta.sourceProvenance or [ sourceTypes.binaryNativeCode ];
454 platforms = finalAttrs.passthru.supportedNixSystems;
455 broken = _mkMetaBroken finalAttrs;
456 badPlatforms = _mkMetaBadPlatforms finalAttrs;
457 downloadPage =
458 meta.downloadPage
459 or "https://developer.download.nvidia.com/compute/${finalAttrs.passthru.redistName}/redist/${finalAttrs.pname}";
460 # NOTE:
461 # Every redistributable should set its own license; since that's a lot of manual work, we default to
462 # nvidiaCudaRedist if the redistributable is from the CUDA redistributables and nvidiaCuda otherwise.
463 # Despite calling them "redistributable" and the download URL containing "redist", a number of these
464 # packages are not licensed such that redistribution is allowed.
465 license =
466 if meta ? license then
467 lib.toList meta.license
468 else if finalAttrs.passthru.redistName == "cuda" then
469 [ licenses.nvidiaCudaRedist ]
470 else
471 [ licenses.nvidiaCuda ];
472 teams = meta.teams or [ ] ++ [ teams.cuda ];
473 };
474 }
475 # Setup the outputNameVar variables to gracefully handle missing outputs.
476 # NOTE: We cannot use expectedOutputs from finalAttrs.passthru because we will infinitely recurse: presence of
477 # attributes in finalAttrs cannot depend on finalAttrs.
478 // foldl' (
479 acc: output:
480 let
481 outputNameVar = mkOutputNameVar output;
482 in
483 acc
484 // {
485 ${outputNameVar} =
486 findFirst (flip elem finalAttrs.outputs) "out"
487 finalAttrs.passthru.outputNameVarFallbacks.${outputNameVar};
488 }
489 ) { } expectedOutputs;
490
491 # Don't inherit any of stdenv.mkDerivation's arguments.
492 inheritFunctionArgs = false;
493}