tjh.dev
nixpkgs mirror (for testing)
github.com/NixOS/nixpkgs
nix
1# Generic builder.
2
3{
4 lib,
5 config,
6 python,
7 # Allow passing in a custom stdenv to buildPython*.override
8 stdenv,
9 wrapPython,
10 unzip,
11 ensureNewerSourcesForZipFilesHook,
12 # Whether the derivation provides a Python module or not.
13 toPythonModule,
14 namePrefix,
15 nix-update-script,
16 setuptools,
17 pypaBuildHook,
18 pypaInstallHook,
19 pythonCatchConflictsHook,
20 pythonImportsCheckHook,
21 pythonNamespacesHook,
22 pythonOutputDistHook,
23 pythonRelaxDepsHook,
24 pythonRemoveBinBytecodeHook,
25 pythonRemoveTestsDirHook,
26 pythonRuntimeDepsCheckHook,
27 setuptoolsBuildHook,
28 wheelUnpackHook,
29 eggUnpackHook,
30 eggBuildHook,
31 eggInstallHook,
32}:
33
34let
35 inherit (builtins) unsafeGetAttrPos;
36 inherit (lib)
37 elem
38 extendDerivation
39 fixedWidthString
40 flip
41 getName
42 hasSuffix
43 head
44 isBool
45 max
46 optional
47 optionalAttrs
48 optionals
49 optionalString
50 removePrefix
51 splitString
52 stringLength
53 ;
54
55 getOptionalAttrs =
56 names: attrs: lib.getAttrs (lib.intersectLists names (lib.attrNames attrs)) attrs;
57
58 leftPadName =
59 name: against:
60 let
61 len = max (stringLength name) (stringLength against);
62 in
63 fixedWidthString len " " name;
64
65 isPythonModule =
66 drv:
67 # all pythonModules have the pythonModule attribute
68 (drv ? "pythonModule")
69 # Some pythonModules are turned in to a pythonApplication by setting the field to false
70 && (!isBool drv.pythonModule);
71
72 isMismatchedPython = drv: drv.pythonModule != python;
73
74 withDistOutput' = flip elem [
75 "pyproject"
76 "setuptools"
77 "wheel"
78 ];
79
80 isBootstrapInstallPackage' = flip elem [
81 "flit-core"
82 "installer"
83 ];
84
85 isBootstrapPackage' = flip elem (
86 [
87 "build"
88 "packaging"
89 "pyproject-hooks"
90 "wheel"
91 ]
92 ++ optionals (python.pythonOlder "3.11") [
93 "tomli"
94 ]
95 );
96
97 isSetuptoolsDependency' = flip elem [
98 "setuptools"
99 "wheel"
100 ];
101
102in
103
104lib.extendMkDerivation {
105 constructDrv = stdenv.mkDerivation;
106
107 excludeDrvArgNames = [
108 "disabled"
109 "checkPhase"
110 "checkInputs"
111 "nativeCheckInputs"
112 "doCheck"
113 "doInstallCheck"
114 "pyproject"
115 "format"
116 "stdenv"
117 "dependencies"
118 "optional-dependencies"
119 "build-system"
120 ];
121
122 extendDrvArgs =
123 finalAttrs:
124 {
125 # Build-time dependencies for the package
126 nativeBuildInputs ? [ ],
127
128 # Run-time dependencies for the package
129 buildInputs ? [ ],
130
131 # Dependencies needed for running the checkPhase.
132 # These are added to buildInputs when doCheck = true.
133 checkInputs ? [ ],
134 nativeCheckInputs ? [ ],
135
136 # propagate build dependencies so in case we have A -> B -> C,
137 # C can import package A propagated by B
138 propagatedBuildInputs ? [ ],
139
140 # Python module dependencies.
141 # These are named after PEP-621.
142 dependencies ? [ ],
143 optional-dependencies ? { },
144
145 # Python PEP-517 build systems.
146 build-system ? [ ],
147
148 # DEPRECATED: use propagatedBuildInputs
149 pythonPath ? [ ],
150
151 # Enabled to detect some (native)BuildInputs mistakes
152 strictDeps ? true,
153
154 outputs ? [ "out" ],
155
156 # used to disable derivation, useful for specific python versions
157 disabled ? false,
158
159 # Raise an error if two packages are installed with the same name
160 # TODO: For cross we probably need a different PYTHONPATH, or not
161 # add the runtime deps until after buildPhase.
162 catchConflicts ? (python.stdenv.hostPlatform == python.stdenv.buildPlatform),
163
164 # Additional arguments to pass to the makeWrapper function, which wraps
165 # generated binaries.
166 makeWrapperArgs ? [ ],
167
168 # Skip wrapping of python programs altogether
169 dontWrapPythonPrograms ? false,
170
171 # Don't use Pip to install a wheel
172 # Note this is actually a variable for the pipInstallPhase in pip's setupHook.
173 # It's included here to prevent an infinite recursion.
174 dontUsePipInstall ? false,
175
176 # Skip setting the PYTHONNOUSERSITE environment variable in wrapped programs
177 permitUserSite ? false,
178
179 # Remove bytecode from bin folder.
180 # When a Python script has the extension `.py`, bytecode is generated
181 # Typically, executables in bin have no extension, so no bytecode is generated.
182 # However, some packages do provide executables with extensions, and thus bytecode is generated.
183 removeBinBytecode ? true,
184
185 # pyproject = true <-> format = "pyproject"
186 # pyproject = false <-> format = "other"
187 # https://github.com/NixOS/nixpkgs/issues/253154
188 pyproject ? null,
189
190 # Several package formats are supported.
191 # "setuptools" : Install a common setuptools/distutils based package. This builds a wheel.
192 # "wheel" : Install from a pre-compiled wheel.
193 # "pyproject": Install a package using a ``pyproject.toml`` file (PEP517). This builds a wheel.
194 # "egg": Install a package from an egg.
195 # "other" : Provide your own buildPhase and installPhase.
196 format ? null,
197
198 meta ? { },
199
200 doCheck ? true,
201
202 ...
203 }@attrs:
204
205 let
206 getFinalPassthru =
207 let
208 pos = unsafeGetAttrPos "passthru" finalAttrs;
209 in
210 attrName:
211 finalAttrs.passthru.${attrName} or (throw (
212 ''
213 ${finalAttrs.name}: passthru.${attrName} missing after overrideAttrs overriding.
214 ''
215 + optionalString (pos != null) ''
216 Last overridden at ${pos.file}:${toString pos.line}
217 ''
218 ));
219
220 format' =
221 assert (getFinalPassthru "pyproject" != null) -> (format == null);
222 if getFinalPassthru "pyproject" != null then
223 if getFinalPassthru "pyproject" then "pyproject" else "other"
224 else if format != null then
225 format
226 else
227 throw "${name} does not configure a `format`. To build with setuptools as before, set `pyproject = true` and `build-system = [ setuptools ]`.";
228
229 withDistOutput = withDistOutput' format';
230
231 validatePythonMatches =
232 let
233 throwMismatch =
234 attrName: drv:
235 let
236 myName = "'${finalAttrs.name}'";
237 theirName = "'${drv.name}'";
238 optionalLocation =
239 let
240 pos = unsafeGetAttrPos (if attrs ? "pname" then "pname" else "name") attrs;
241 in
242 optionalString (pos != null) " at ${pos.file}:${toString pos.line}:${toString pos.column}";
243 in
244 throw ''
245 Python version mismatch in ${myName}:
246
247 The Python derivation ${myName} depends on a Python derivation
248 named ${theirName}, but the two derivations use different versions
249 of Python:
250
251 ${leftPadName myName theirName} uses ${python}
252 ${leftPadName theirName myName} uses ${toString drv.pythonModule}
253
254 Possible solutions:
255
256 * If ${theirName} is a Python library, change the reference to ${theirName}
257 in the ${attrName} of ${myName} to use a ${theirName} built from the same
258 version of Python
259
260 * If ${theirName} is used as a tool during the build, move the reference to
261 ${theirName} in ${myName} from ${attrName} to nativeBuildInputs
262
263 * If ${theirName} provides executables that are called at run time, pass its
264 bin path to makeWrapperArgs:
265
266 makeWrapperArgs = [ "--prefix PATH : ''${lib.makeBinPath [ ${getName drv} ] }" ];
267
268 ${optionalLocation}
269 '';
270
271 checkDrv =
272 attrName: drv:
273 if (isPythonModule drv) && (isMismatchedPython drv) then throwMismatch attrName drv else drv;
274
275 in
276 attrName: inputs: map (checkDrv attrName) inputs;
277
278 isBootstrapInstallPackage = isBootstrapInstallPackage' (finalAttrs.pname or null);
279
280 isBootstrapPackage = isBootstrapInstallPackage || isBootstrapPackage' (finalAttrs.pname or null);
281
282 isSetuptoolsDependency = isSetuptoolsDependency' (finalAttrs.pname or null);
283
284 name = namePrefix + attrs.name or "${finalAttrs.pname}-${finalAttrs.version}";
285
286 runtimeDepsCheckHook =
287 if isBootstrapPackage then
288 pythonRuntimeDepsCheckHook.override {
289 inherit (python.pythonOnBuildForHost.pkgs.bootstrap) packaging;
290 }
291 else
292 pythonRuntimeDepsCheckHook;
293
294 in
295 {
296 inherit name;
297
298 inherit catchConflicts;
299
300 nativeBuildInputs = [
301 python
302 wrapPython
303 ensureNewerSourcesForZipFilesHook # move to wheel installer (pip) or builder (setuptools, flit, ...)?
304 pythonRemoveTestsDirHook
305 ]
306 ++ optionals (finalAttrs.catchConflicts && !isBootstrapPackage && !isSetuptoolsDependency) [
307 #
308 # 1. When building a package that is also part of the bootstrap chain, we
309 # must ignore conflicts after installation, because there will be one with
310 # the package in the bootstrap.
311 #
312 # 2. When a package is a dependency of setuptools, we must ignore conflicts
313 # because the hook that checks for conflicts uses setuptools.
314 #
315 pythonCatchConflictsHook
316 ]
317 ++
318 optionals (finalAttrs.pythonRelaxDeps or [ ] != [ ] || finalAttrs.pythonRemoveDeps or [ ] != [ ])
319 [
320 pythonRelaxDepsHook
321 ]
322 ++ optionals removeBinBytecode [
323 pythonRemoveBinBytecodeHook
324 ]
325 ++ optionals (hasSuffix "zip" (finalAttrs.src.name or "")) [
326 unzip
327 ]
328 ++ optionals (format' == "setuptools") [
329 setuptoolsBuildHook
330 ]
331 ++ optionals (format' == "pyproject") [
332 (
333 if isBootstrapPackage then
334 pypaBuildHook.override {
335 inherit (python.pythonOnBuildForHost.pkgs.bootstrap) build;
336 wheel = null;
337 }
338 else
339 pypaBuildHook
340 )
341 runtimeDepsCheckHook
342 ]
343 ++ optionals (format' == "wheel") [
344 wheelUnpackHook
345 runtimeDepsCheckHook
346 ]
347 ++ optionals (format' == "egg") [
348 eggUnpackHook
349 eggBuildHook
350 eggInstallHook
351 ]
352 ++ optionals (format' != "other") [
353 (
354 if isBootstrapInstallPackage then
355 pypaInstallHook.override {
356 inherit (python.pythonOnBuildForHost.pkgs.bootstrap) installer;
357 }
358 else
359 pypaInstallHook
360 )
361 ]
362 ++ optionals (stdenv.buildPlatform == stdenv.hostPlatform) [
363 # This is a test, however, it should be ran independent of the checkPhase and checkInputs
364 pythonImportsCheckHook
365 ]
366 ++ optionals (python.pythonAtLeast "3.3") [
367 # Optionally enforce PEP420 for python3
368 pythonNamespacesHook
369 ]
370 ++ optionals withDistOutput [
371 pythonOutputDistHook
372 ]
373 ++ nativeBuildInputs
374 ++ getFinalPassthru "build-system";
375
376 buildInputs = validatePythonMatches "buildInputs" (buildInputs ++ pythonPath);
377
378 propagatedBuildInputs = validatePythonMatches "propagatedBuildInputs" (
379 propagatedBuildInputs
380 ++ getFinalPassthru "dependencies"
381 ++ [
382 # we propagate python even for packages transformed with 'toPythonApplication'
383 # this pollutes the PATH but avoids rebuilds
384 # see https://github.com/NixOS/nixpkgs/issues/170887 for more context
385 python
386 ]
387 );
388
389 inherit strictDeps;
390
391 LANG = "${if python.stdenv.hostPlatform.isDarwin then "en_US" else "C"}.UTF-8";
392
393 # Python packages don't have a checkPhase, only an installCheckPhase
394 doCheck = false;
395 doInstallCheck = attrs.doCheck or true;
396 nativeInstallCheckInputs = nativeCheckInputs ++ attrs.nativeInstallCheckInputs or [ ];
397 installCheckInputs = checkInputs ++ attrs.installCheckInputs or [ ];
398
399 inherit dontWrapPythonPrograms;
400
401 postFixup =
402 optionalString (!finalAttrs.dontWrapPythonPrograms) ''
403 wrapPythonPrograms
404 ''
405 + attrs.postFixup or "";
406
407 # Python packages built through cross-compilation are always for the host platform.
408 disallowedReferences = optionals (python.stdenv.hostPlatform != python.stdenv.buildPlatform) [
409 python.pythonOnBuildForHost
410 ];
411
412 outputs = outputs ++ optional withDistOutput "dist";
413
414 passthru = {
415 inherit
416 disabled
417 pyproject
418 build-system
419 dependencies
420 optional-dependencies
421 ;
422 updateScript = nix-update-script { };
423 ${if attrs ? stdenv then "__stdenvPythonCompat" else null} = attrs.stdenv;
424 }
425 // attrs.passthru or { };
426
427 meta = {
428 # default to python's platforms
429 platforms = python.meta.platforms;
430 isBuildPythonPackage = python.meta.platforms;
431 }
432 // meta;
433 }
434 // optionalAttrs (attrs ? checkPhase) {
435 # If given use the specified checkPhase, otherwise use the setup hook.
436 # Longer-term we should get rid of `checkPhase` and use `installCheckPhase`.
437 installCheckPhase = attrs.checkPhase;
438 }
439 //
440 lib.mapAttrs
441 (
442 name: value:
443 lib.throwIf (
444 attrs.${name} == [ ]
445 ) "${lib.getName finalAttrs}: ${name} must be unspecified, null or a non-empty list." attrs.${name}
446 )
447 (
448 getOptionalAttrs [
449 "enabledTestMarks"
450 "enabledTestPaths"
451 "enabledTests"
452 ] attrs
453 );
454
455 # This derivation transformation function must be independent to `attrs`
456 # for fixed-point arguments support in the future.
457 transformDrv =
458 let
459 # Workaround to make the `lib.extendDerivation`-based disabled functionality
460 # respect `<pkg>.overrideAttrs`
461 # It doesn't cover `<pkg>.<output>.overrideAttrs`.
462 disablePythonPackage =
463 drv:
464 extendDerivation (
465 drv.disabled
466 -> throw "${removePrefix namePrefix drv.name} not supported for interpreter ${python.executable}"
467 ) { } drv
468 // {
469 overrideAttrs = fdrv: disablePythonPackage (drv.overrideAttrs fdrv);
470 };
471 in
472 drv: disablePythonPackage (toPythonModule drv);
473}