tjh.dev
1# Generic builder.
2
3{
4 lib,
5 config,
6 python,
7 wrapPython,
8 unzip,
9 ensureNewerSourcesForZipFilesHook,
10 # Whether the derivation provides a Python module or not.
11 toPythonModule,
12 namePrefix,
13 nix-update-script,
14 setuptools,
15 pypaBuildHook,
16 pypaInstallHook,
17 pythonCatchConflictsHook,
18 pythonImportsCheckHook,
19 pythonNamespacesHook,
20 pythonOutputDistHook,
21 pythonRelaxDepsHook,
22 pythonRemoveBinBytecodeHook,
23 pythonRemoveTestsDirHook,
24 pythonRuntimeDepsCheckHook,
25 setuptoolsBuildHook,
26 wheelUnpackHook,
27 eggUnpackHook,
28 eggBuildHook,
29 eggInstallHook,
30}:
31
32let
33 inherit (builtins) unsafeGetAttrPos;
34 inherit (lib)
35 elem
36 extendDerivation
37 fixedWidthString
38 flip
39 getName
40 hasSuffix
41 head
42 isBool
43 max
44 optional
45 optionalAttrs
46 optionals
47 optionalString
48 removePrefix
49 splitString
50 stringLength
51 ;
52
53 getOptionalAttrs =
54 names: attrs: lib.getAttrs (lib.intersectLists names (lib.attrNames attrs)) attrs;
55
56 leftPadName =
57 name: against:
58 let
59 len = max (stringLength name) (stringLength against);
60 in
61 fixedWidthString len " " name;
62
63 isPythonModule =
64 drv:
65 # all pythonModules have the pythonModule attribute
66 (drv ? "pythonModule")
67 # Some pythonModules are turned in to a pythonApplication by setting the field to false
68 && (!isBool drv.pythonModule);
69
70 isMismatchedPython = drv: drv.pythonModule != python;
71
72 withDistOutput' = flip elem [
73 "pyproject"
74 "setuptools"
75 "wheel"
76 ];
77
78 isBootstrapInstallPackage' = flip elem [
79 "flit-core"
80 "installer"
81 ];
82
83 isBootstrapPackage' = flip elem (
84 [
85 "build"
86 "packaging"
87 "pyproject-hooks"
88 "wheel"
89 ]
90 ++ optionals (python.pythonOlder "3.11") [
91 "tomli"
92 ]
93 );
94
95 isSetuptoolsDependency' = flip elem [
96 "setuptools"
97 "wheel"
98 ];
99
100 cleanAttrs = flip removeAttrs [
101 "disabled"
102 "checkPhase"
103 "checkInputs"
104 "nativeCheckInputs"
105 "doCheck"
106 "doInstallCheck"
107 "pyproject"
108 "format"
109 "stdenv"
110 "dependencies"
111 "optional-dependencies"
112 "build-system"
113 ];
114
115in
116
117{
118 # Build-time dependencies for the package
119 nativeBuildInputs ? [ ],
120
121 # Run-time dependencies for the package
122 buildInputs ? [ ],
123
124 # Dependencies needed for running the checkPhase.
125 # These are added to buildInputs when doCheck = true.
126 checkInputs ? [ ],
127 nativeCheckInputs ? [ ],
128
129 # propagate build dependencies so in case we have A -> B -> C,
130 # C can import package A propagated by B
131 propagatedBuildInputs ? [ ],
132
133 # Python module dependencies.
134 # These are named after PEP-621.
135 dependencies ? [ ],
136 optional-dependencies ? { },
137
138 # Python PEP-517 build systems.
139 build-system ? [ ],
140
141 # DEPRECATED: use propagatedBuildInputs
142 pythonPath ? [ ],
143
144 # Enabled to detect some (native)BuildInputs mistakes
145 strictDeps ? true,
146
147 outputs ? [ "out" ],
148
149 # used to disable derivation, useful for specific python versions
150 disabled ? false,
151
152 # Raise an error if two packages are installed with the same name
153 # TODO: For cross we probably need a different PYTHONPATH, or not
154 # add the runtime deps until after buildPhase.
155 catchConflicts ? (python.stdenv.hostPlatform == python.stdenv.buildPlatform),
156
157 # Additional arguments to pass to the makeWrapper function, which wraps
158 # generated binaries.
159 makeWrapperArgs ? [ ],
160
161 # Skip wrapping of python programs altogether
162 dontWrapPythonPrograms ? false,
163
164 # Don't use Pip to install a wheel
165 # Note this is actually a variable for the pipInstallPhase in pip's setupHook.
166 # It's included here to prevent an infinite recursion.
167 dontUsePipInstall ? false,
168
169 # Skip setting the PYTHONNOUSERSITE environment variable in wrapped programs
170 permitUserSite ? false,
171
172 # Remove bytecode from bin folder.
173 # When a Python script has the extension `.py`, bytecode is generated
174 # Typically, executables in bin have no extension, so no bytecode is generated.
175 # However, some packages do provide executables with extensions, and thus bytecode is generated.
176 removeBinBytecode ? true,
177
178 # pyproject = true <-> format = "pyproject"
179 # pyproject = false <-> format = "other"
180 # https://github.com/NixOS/nixpkgs/issues/253154
181 pyproject ? null,
182
183 # Several package formats are supported.
184 # "setuptools" : Install a common setuptools/distutils based package. This builds a wheel.
185 # "wheel" : Install from a pre-compiled wheel.
186 # "pyproject": Install a package using a ``pyproject.toml`` file (PEP517). This builds a wheel.
187 # "egg": Install a package from an egg.
188 # "other" : Provide your own buildPhase and installPhase.
189 format ? null,
190
191 meta ? { },
192
193 doCheck ? true,
194
195 # Allow passing in a custom stdenv to buildPython*
196 stdenv ? python.stdenv,
197
198 ...
199}@attrs:
200
201let
202 # Keep extra attributes from `attrs`, e.g., `patchPhase', etc.
203 self = stdenv.mkDerivation (
204 finalAttrs:
205 let
206 getFinalPassthru =
207 let
208 pos = unsafeGetAttrPos "passthru" finalAttrs;
209 in
210 attrName:
211 finalAttrs.passthru.${attrName} or (throw (
212 ''
213 ${finalAttrs.name}: passthru.${attrName} missing after overrideAttrs overriding.
214 ''
215 + optionalString (pos != null) ''
216 Last overridden at ${pos.file}:${toString pos.line}
217 ''
218 ));
219
220 format' =
221 assert (getFinalPassthru "pyproject" != null) -> (format == null);
222 if getFinalPassthru "pyproject" != null then
223 if getFinalPassthru "pyproject" then "pyproject" else "other"
224 else if format != null then
225 format
226 else
227 throw "${name} does not configure a `format`. To build with setuptools as before, set `pyproject = true` and `build-system = [ setuptools ]`.`";
228
229 withDistOutput = withDistOutput' format';
230
231 validatePythonMatches =
232 let
233 throwMismatch =
234 attrName: drv:
235 let
236 myName = "'${finalAttrs.name}'";
237 theirName = "'${drv.name}'";
238 optionalLocation =
239 let
240 pos = unsafeGetAttrPos (if attrs ? "pname" then "pname" else "name") attrs;
241 in
242 optionalString (pos != null) " at ${pos.file}:${toString pos.line}:${toString pos.column}";
243 in
244 throw ''
245 Python version mismatch in ${myName}:
246
247 The Python derivation ${myName} depends on a Python derivation
248 named ${theirName}, but the two derivations use different versions
249 of Python:
250
251 ${leftPadName myName theirName} uses ${python}
252 ${leftPadName theirName myName} uses ${toString drv.pythonModule}
253
254 Possible solutions:
255
256 * If ${theirName} is a Python library, change the reference to ${theirName}
257 in the ${attrName} of ${myName} to use a ${theirName} built from the same
258 version of Python
259
260 * If ${theirName} is used as a tool during the build, move the reference to
261 ${theirName} in ${myName} from ${attrName} to nativeBuildInputs
262
263 * If ${theirName} provides executables that are called at run time, pass its
264 bin path to makeWrapperArgs:
265
266 makeWrapperArgs = [ "--prefix PATH : ''${lib.makeBinPath [ ${getName drv} ] }" ];
267
268 ${optionalLocation}
269 '';
270
271 checkDrv =
272 attrName: drv:
273 if (isPythonModule drv) && (isMismatchedPython drv) then throwMismatch attrName drv else drv;
274
275 in
276 attrName: inputs: map (checkDrv attrName) inputs;
277
278 isBootstrapInstallPackage = isBootstrapInstallPackage' (finalAttrs.pname or null);
279
280 isBootstrapPackage = isBootstrapInstallPackage || isBootstrapPackage' (finalAttrs.pname or null);
281
282 isSetuptoolsDependency = isSetuptoolsDependency' (finalAttrs.pname or null);
283
284 name = namePrefix + attrs.name or "${finalAttrs.pname}-${finalAttrs.version}";
285
286 in
287 (cleanAttrs attrs)
288 // {
289 inherit name;
290
291 inherit catchConflicts;
292
293 nativeBuildInputs = [
294 python
295 wrapPython
296 ensureNewerSourcesForZipFilesHook # move to wheel installer (pip) or builder (setuptools, flit, ...)?
297 pythonRemoveTestsDirHook
298 ]
299 ++ optionals (finalAttrs.catchConflicts && !isBootstrapPackage && !isSetuptoolsDependency) [
300 #
301 # 1. When building a package that is also part of the bootstrap chain, we
302 # must ignore conflicts after installation, because there will be one with
303 # the package in the bootstrap.
304 #
305 # 2. When a package is a dependency of setuptools, we must ignore conflicts
306 # because the hook that checks for conflicts uses setuptools.
307 #
308 pythonCatchConflictsHook
309 ]
310 ++
311 optionals (finalAttrs.pythonRelaxDeps or [ ] != [ ] || finalAttrs.pythonRemoveDeps or [ ] != [ ])
312 [
313 pythonRelaxDepsHook
314 ]
315 ++ optionals removeBinBytecode [
316 pythonRemoveBinBytecodeHook
317 ]
318 ++ optionals (hasSuffix "zip" (finalAttrs.src.name or "")) [
319 unzip
320 ]
321 ++ optionals (format' == "setuptools") [
322 setuptoolsBuildHook
323 ]
324 ++ optionals (format' == "pyproject") [
325 (
326 if isBootstrapPackage then
327 pypaBuildHook.override {
328 inherit (python.pythonOnBuildForHost.pkgs.bootstrap) build;
329 wheel = null;
330 }
331 else
332 pypaBuildHook
333 )
334 (
335 if isBootstrapPackage then
336 pythonRuntimeDepsCheckHook.override {
337 inherit (python.pythonOnBuildForHost.pkgs.bootstrap) packaging;
338 }
339 else
340 pythonRuntimeDepsCheckHook
341 )
342 ]
343 ++ optionals (format' == "wheel") [
344 wheelUnpackHook
345 ]
346 ++ optionals (format' == "egg") [
347 eggUnpackHook
348 eggBuildHook
349 eggInstallHook
350 ]
351 ++ optionals (format' != "other") [
352 (
353 if isBootstrapInstallPackage then
354 pypaInstallHook.override {
355 inherit (python.pythonOnBuildForHost.pkgs.bootstrap) installer;
356 }
357 else
358 pypaInstallHook
359 )
360 ]
361 ++ optionals (stdenv.buildPlatform == stdenv.hostPlatform) [
362 # This is a test, however, it should be ran independent of the checkPhase and checkInputs
363 pythonImportsCheckHook
364 ]
365 ++ optionals (python.pythonAtLeast "3.3") [
366 # Optionally enforce PEP420 for python3
367 pythonNamespacesHook
368 ]
369 ++ optionals withDistOutput [
370 pythonOutputDistHook
371 ]
372 ++ nativeBuildInputs
373 ++ getFinalPassthru "build-system";
374
375 buildInputs = validatePythonMatches "buildInputs" (buildInputs ++ pythonPath);
376
377 propagatedBuildInputs = validatePythonMatches "propagatedBuildInputs" (
378 propagatedBuildInputs
379 ++ getFinalPassthru "dependencies"
380 ++ [
381 # we propagate python even for packages transformed with 'toPythonApplication'
382 # this pollutes the PATH but avoids rebuilds
383 # see https://github.com/NixOS/nixpkgs/issues/170887 for more context
384 python
385 ]
386 );
387
388 inherit strictDeps;
389
390 LANG = "${if python.stdenv.hostPlatform.isDarwin then "en_US" else "C"}.UTF-8";
391
392 # Python packages don't have a checkPhase, only an installCheckPhase
393 doCheck = false;
394 doInstallCheck = attrs.doCheck or true;
395 nativeInstallCheckInputs = nativeCheckInputs ++ attrs.nativeInstallCheckInputs or [ ];
396 installCheckInputs = checkInputs ++ attrs.installCheckInputs or [ ];
397
398 inherit dontWrapPythonPrograms;
399
400 postFixup =
401 optionalString (!finalAttrs.dontWrapPythonPrograms) ''
402 wrapPythonPrograms
403 ''
404 + attrs.postFixup or "";
405
406 # Python packages built through cross-compilation are always for the host platform.
407 disallowedReferences = optionals (python.stdenv.hostPlatform != python.stdenv.buildPlatform) [
408 python.pythonOnBuildForHost
409 ];
410
411 outputs = outputs ++ optional withDistOutput "dist";
412
413 passthru = {
414 inherit
415 disabled
416 pyproject
417 build-system
418 dependencies
419 optional-dependencies
420 ;
421 }
422 // {
423 updateScript = nix-update-script { };
424 }
425 // attrs.passthru or { };
426
427 meta = {
428 # default to python's platforms
429 platforms = python.meta.platforms;
430 isBuildPythonPackage = python.meta.platforms;
431 }
432 // meta;
433 }
434 // optionalAttrs (attrs ? checkPhase) {
435 # If given use the specified checkPhase, otherwise use the setup hook.
436 # Longer-term we should get rid of `checkPhase` and use `installCheckPhase`.
437 installCheckPhase = attrs.checkPhase;
438 }
439 //
440 lib.mapAttrs
441 (
442 name: value:
443 lib.throwIf (
444 attrs.${name} == [ ]
445 ) "${lib.getName finalAttrs}: ${name} must be unspecified, null or a non-empty list." attrs.${name}
446 )
447 (
448 getOptionalAttrs [
449 "enabledTestMarks"
450 "enabledTestPaths"
451 "enabledTests"
452 ] attrs
453 )
454 );
455
456 # This derivation transformation function must be independent to `attrs`
457 # for fixed-point arguments support in the future.
458 transformDrv =
459 let
460 # Workaround to make the `lib.extendDerivation`-based disabled functionality
461 # respect `<pkg>.overrideAttrs`
462 # It doesn't cover `<pkg>.<output>.overrideAttrs`.
463 disablePythonPackage =
464 drv:
465 extendDerivation (
466 drv.disabled
467 -> throw "${removePrefix namePrefix drv.name} not supported for interpreter ${python.executable}"
468 ) { } drv
469 // {
470 overrideAttrs = fdrv: disablePythonPackage (drv.overrideAttrs fdrv);
471 };
472 in
473 drv: disablePythonPackage (toPythonModule drv);
474
475in
476transformDrv self