pyrox.dev
1{ lib, stdenv, fetchurl, fetchpatch
2, bzip2
3, expat
4, libffi
5, gdbm
6, xz
7, mime-types ? null, mimetypesSupport ? true
8, ncurses
9, openssl
10, readline
11, sqlite
12, tcl ? null, tk ? null, tix ? null, libX11 ? null, xorgproto ? null, x11Support ? false
13, bluez ? null, bluezSupport ? false
14, zlib
15, tzdata ? null
16, self
17, configd
18, autoreconfHook
19, autoconf-archive
20, python-setup-hook
21, nukeReferences
22# For the Python package set
23, packageOverrides ? (self: super: {})
24, pkgsBuildBuild
25, pkgsBuildHost
26, pkgsBuildTarget
27, pkgsHostHost
28, pkgsTargetTarget
29, sourceVersion
30, sha256
31, passthruFun
32, bash
33, stripConfig ? false
34, stripIdlelib ? false
35, stripTests ? false
36, stripTkinter ? false
37, rebuildBytecode ? true
38, stripBytecode ? true
39, includeSiteCustomize ? true
40, static ? stdenv.hostPlatform.isStatic
41, enableOptimizations ? false
42# enableNoSemanticInterposition is a subset of the enableOptimizations flag that doesn't harm reproducibility.
43# clang starts supporting `-fno-sematic-interposition` with version 10
44, enableNoSemanticInterposition ? (!stdenv.cc.isClang || (stdenv.cc.isClang && lib.versionAtLeast stdenv.cc.version "10"))
45# enableLTO is a subset of the enableOptimizations flag that doesn't harm reproducibility.
46# enabling LTO on 32bit arch causes downstream packages to fail when linking
47# enabling LTO on *-darwin causes python3 to fail when linking.
48, enableLTO ? stdenv.is64bit && stdenv.isLinux
49, reproducibleBuild ? false
50, pythonAttr ? "python${sourceVersion.major}${sourceVersion.minor}"
51}:
52
53# Note: this package is used for bootstrapping fetchurl, and thus
54# cannot use fetchpatch! All mutable patches (generated by GitHub or
55# cgit) that are needed here should be included directly in Nixpkgs as
56# files.
57
58assert x11Support -> tcl != null
59 && tk != null
60 && xorgproto != null
61 && libX11 != null;
62
63assert bluezSupport -> bluez != null;
64
65assert mimetypesSupport -> mime-types != null;
66
67assert lib.assertMsg (enableOptimizations -> (!stdenv.cc.isClang))
68 "Optimizations with clang are not supported. configure: error: llvm-profdata is required for a --enable-optimizations build but could not be found.";
69
70assert lib.assertMsg (reproducibleBuild -> stripBytecode)
71 "Deterministic builds require stripping bytecode.";
72
73assert lib.assertMsg (reproducibleBuild -> (!enableOptimizations))
74 "Deterministic builds are not achieved when optimizations are enabled.";
75
76assert lib.assertMsg (reproducibleBuild -> (!rebuildBytecode))
77 "Deterministic builds are not achieved when (default unoptimized) bytecode is created.";
78
79with lib;
80
81let
82 buildPackages = pkgsBuildHost;
83 inherit (passthru) pythonForBuild;
84
85 tzdataSupport = tzdata != null && passthru.pythonAtLeast "3.9";
86
87 passthru = passthruFun rec {
88 inherit self sourceVersion packageOverrides;
89 implementation = "cpython";
90 libPrefix = "python${pythonVersion}";
91 executable = libPrefix;
92 pythonVersion = with sourceVersion; "${major}.${minor}";
93 sitePackages = "lib/${libPrefix}/site-packages";
94 inherit hasDistutilsCxxPatch;
95 pythonOnBuildForBuild = pkgsBuildBuild.${pythonAttr};
96 pythonOnBuildForHost = pkgsBuildHost.${pythonAttr};
97 pythonOnBuildForTarget = pkgsBuildTarget.${pythonAttr};
98 pythonOnHostForHost = pkgsHostHost.${pythonAttr};
99 pythonOnTargetForTarget = pkgsTargetTarget.${pythonAttr} or {};
100 };
101
102 version = with sourceVersion; "${major}.${minor}.${patch}${suffix}";
103
104 strictDeps = true;
105
106 nativeBuildInputs = optionals (!stdenv.isDarwin) [
107 autoreconfHook
108 ] ++ optionals (!stdenv.isDarwin && passthru.pythonAtLeast "3.10") [
109 autoconf-archive # needed for AX_CHECK_COMPILE_FLAG
110 ] ++ [
111 nukeReferences
112 ] ++ optionals (stdenv.hostPlatform != stdenv.buildPlatform) [
113 buildPackages.stdenv.cc
114 pythonForBuild
115 ] ++ optionals (stdenv.cc.isClang && enableLTO) [
116 stdenv.cc.cc.libllvm.out
117 ];
118
119 buildInputs = filter (p: p != null) ([
120 zlib bzip2 expat xz libffi gdbm sqlite readline ncurses openssl ]
121 ++ optionals x11Support [ tcl tk libX11 xorgproto ]
122 ++ optionals (bluezSupport && stdenv.isLinux) [ bluez ]
123 ++ optionals stdenv.isDarwin [ configd ])
124 ++ optionals tzdataSupport [ tzdata ]; # `zoneinfo` module
125
126 hasDistutilsCxxPatch = !(stdenv.cc.isGNU or false);
127
128 pythonForBuildInterpreter = if stdenv.hostPlatform == stdenv.buildPlatform then
129 "$out/bin/python"
130 else pythonForBuild.interpreter;
131
132 # The CPython interpreter contains a _sysconfigdata_<platform specific suffix>
133 # module that is imported by the sysconfig and distutils.sysconfig modules.
134 # The sysconfigdata module is generated at build time and contains settings
135 # required for building Python extension modules, such as include paths and
136 # other compiler flags. By default, the sysconfigdata module is loaded from
137 # the currently running interpreter (ie. the build platform interpreter), but
138 # when cross-compiling we want to load it from the host platform interpreter.
139 # This can be done using the _PYTHON_SYSCONFIGDATA_NAME environment variable.
140 # The _PYTHON_HOST_PLATFORM variable also needs to be set to get the correct
141 # platform suffix on extension modules. The correct values for these variables
142 # are not documented, and must be derived from the configure script (see links
143 # below).
144 sysconfigdataHook = with stdenv.hostPlatform; with passthru; let
145 # https://github.com/python/cpython/blob/e488e300f5c01289c10906c2e53a8e43d6de32d8/configure.ac#L428
146 # The configure script uses "arm" as the CPU name for all 32-bit ARM
147 # variants when cross-compiling, but native builds include the version
148 # suffix, so we do the same.
149 pythonHostPlatform = "${parsed.kernel.name}-${parsed.cpu.name}";
150
151 # https://github.com/python/cpython/blob/e488e300f5c01289c10906c2e53a8e43d6de32d8/configure.ac#L724
152 multiarchCpu =
153 if isAarch32 then
154 if parsed.cpu.significantByte.name == "littleEndian" then "arm" else "armeb"
155 else if isx86_32 then "i386"
156 else parsed.cpu.name;
157 pythonAbiName =
158 # python's build doesn't differentiate between musl and glibc in its
159 # abi detection, our wrapper should match.
160 if stdenv.hostPlatform.isMusl then
161 replaceStrings [ "musl" ] [ "gnu" ] parsed.abi.name
162 else parsed.abi.name;
163 multiarch =
164 if isDarwin then "darwin"
165 else "${multiarchCpu}-${parsed.kernel.name}-${pythonAbiName}";
166
167 abiFlags = optionalString (isPy36 || isPy37) "m";
168
169 # https://github.com/python/cpython/blob/e488e300f5c01289c10906c2e53a8e43d6de32d8/configure.ac#L78
170 pythonSysconfigdataName = "_sysconfigdata_${abiFlags}_${parsed.kernel.name}_${multiarch}";
171 in ''
172 sysconfigdataHook() {
173 if [ "$1" = '${placeholder "out"}' ]; then
174 export _PYTHON_HOST_PLATFORM='${pythonHostPlatform}'
175 export _PYTHON_SYSCONFIGDATA_NAME='${pythonSysconfigdataName}'
176 fi
177 }
178
179 addEnvHooks "$hostOffset" sysconfigdataHook
180 '';
181
182in with passthru; stdenv.mkDerivation {
183 pname = "python3";
184 inherit version;
185
186 inherit buildInputs nativeBuildInputs;
187
188 src = fetchurl {
189 url = with sourceVersion; "https://www.python.org/ftp/python/${major}.${minor}.${patch}/Python-${version}.tar.xz";
190 inherit sha256;
191 };
192
193 prePatch = optionalString stdenv.isDarwin ''
194 substituteInPlace configure --replace '`/usr/bin/arch`' '"i386"'
195 substituteInPlace configure --replace '-Wl,-stack_size,1000000' ' '
196 '' + optionalString (stdenv.isDarwin && x11Support) ''
197 substituteInPlace setup.py --replace /Library/Frameworks /no-such-path
198 '';
199
200 patches = [
201 # Disable the use of ldconfig in ctypes.util.find_library (since
202 # ldconfig doesn't work on NixOS), and don't use
203 # ctypes.util.find_library during the loading of the uuid module
204 # (since it will do a futile invocation of gcc (!) to find
205 # libuuid, slowing down program startup a lot).
206 (./. + "/${sourceVersion.major}.${sourceVersion.minor}/no-ldconfig.patch")
207 # Make sure that the virtualenv activation scripts are
208 # owner-writable, so venvs can be recreated without permission
209 # errors.
210 ./virtualenv-permissions.patch
211 ] ++ optionals mimetypesSupport [
212 # Make the mimetypes module refer to the right file
213 ./mimetypes.patch
214 ] ++ optionals (isPy35 || isPy36) [
215 # Determinism: Write null timestamps when compiling python files.
216 ./3.5/force_bytecode_determinism.patch
217 ] ++ optionals isPy35 [
218 # Backports support for LD_LIBRARY_PATH from 3.6
219 ./3.5/ld_library_path.patch
220 ] ++ optionals (isPy35 || isPy36 || isPy37) [
221 # Backport a fix for discovering `rpmbuild` command when doing `python setup.py bdist_rpm` to 3.5, 3.6, 3.7.
222 # See: https://bugs.python.org/issue11122
223 ./3.7/fix-hardcoded-path-checking-for-rpmbuild.patch
224 # The workaround is for unittests on Win64, which we don't support.
225 # It does break aarch64-darwin, which we do support. See:
226 # * https://bugs.python.org/issue35523
227 # * https://github.com/python/cpython/commit/e6b247c8e524
228 ./3.7/no-win64-workaround.patch
229 ] ++ optionals (pythonAtLeast "3.7") [
230 # Fix darwin build https://bugs.python.org/issue34027
231 ./3.7/darwin-libutil.patch
232 ] ++ optionals (pythonOlder "3.8") [
233 # Backport from CPython 3.8 of a good list of tests to run for PGO.
234 (
235 if isPy36 || isPy37 then
236 ./3.6/profile-task.patch
237 else
238 ./3.5/profile-task.patch
239 )
240 ] ++ optionals (pythonAtLeast "3.9" && stdenv.isDarwin) [
241 # Stop checking for TCL/TK in global macOS locations
242 ./3.9/darwin-tcl-tk.patch
243 ] ++ optionals (isPy3k && hasDistutilsCxxPatch) [
244 # Fix for http://bugs.python.org/issue1222585
245 # Upstream distutils is calling C compiler to compile C++ code, which
246 # only works for GCC and Apple Clang. This makes distutils to call C++
247 # compiler when needed.
248 (
249 if isPy35 then
250 ./3.5/python-3.x-distutils-C++.patch
251 else if pythonAtLeast "3.7" then
252 ./3.7/python-3.x-distutils-C++.patch
253 else
254 fetchpatch {
255 url = "https://bugs.python.org/file48016/python-3.x-distutils-C++.patch";
256 sha256 = "1h18lnpx539h5lfxyk379dxwr8m2raigcjixkf133l4xy3f4bzi2";
257 }
258 )
259 ] ++ [
260 # LDSHARED now uses $CC instead of gcc. Fixes cross-compilation of extension modules.
261 ./3.8/0001-On-all-posix-systems-not-just-Darwin-set-LDSHARED-if.patch
262 # Use sysconfigdata to find headers. Fixes cross-compilation of extension modules.
263 (
264 if isPy36 then
265 ./3.6/fix-finding-headers-when-cross-compiling.patch
266 else
267 ./3.7/fix-finding-headers-when-cross-compiling.patch
268 )
269 ] ++ optionals (isPy36) [
270 # Backport a fix for ctypes.util.find_library.
271 ./3.6/find_library.patch
272 ];
273
274 postPatch = ''
275 substituteInPlace Lib/subprocess.py \
276 --replace "'/bin/sh'" "'${bash}/bin/sh'"
277 '' + optionalString mimetypesSupport ''
278 substituteInPlace Lib/mimetypes.py \
279 --replace "@mime-types@" "${mime-types}"
280 '' + optionalString (x11Support && (tix != null)) ''
281 substituteInPlace "Lib/tkinter/tix.py" --replace "os.environ.get('TIX_LIBRARY')" "os.environ.get('TIX_LIBRARY') or '${tix}/lib'"
282 '';
283
284 CPPFLAGS = concatStringsSep " " (map (p: "-I${getDev p}/include") buildInputs);
285 LDFLAGS = concatStringsSep " " (map (p: "-L${getLib p}/lib") buildInputs);
286 LIBS = "${optionalString (!stdenv.isDarwin) "-lcrypt"} ${optionalString (ncurses != null) "-lncurses"}";
287 NIX_LDFLAGS = lib.optionalString stdenv.cc.isGNU ({
288 "glibc" = "-lgcc_s";
289 "musl" = "-lgcc_eh";
290 }."${stdenv.hostPlatform.libc}" or "");
291 # Determinism: We fix the hashes of str, bytes and datetime objects.
292 PYTHONHASHSEED=0;
293
294 configureFlags = [
295 "--without-ensurepip"
296 "--with-system-expat"
297 "--with-system-ffi"
298 ] ++ optionals (!static) [
299 "--enable-shared"
300 ] ++ optionals enableOptimizations [
301 "--enable-optimizations"
302 ] ++ optionals enableLTO [
303 "--with-lto"
304 ] ++ optionals (pythonOlder "3.7") [
305 # This is unconditionally true starting in CPython 3.7.
306 "--with-threads"
307 ] ++ optionals (sqlite != null && isPy3k) [
308 "--enable-loadable-sqlite-extensions"
309 ] ++ optionals (openssl != null) [
310 "--with-openssl=${openssl.dev}"
311 ] ++ optionals (stdenv.hostPlatform != stdenv.buildPlatform) [
312 "ac_cv_buggy_getaddrinfo=no"
313 # Assume little-endian IEEE 754 floating point when cross compiling
314 "ac_cv_little_endian_double=yes"
315 "ac_cv_big_endian_double=no"
316 "ac_cv_mixed_endian_double=no"
317 "ac_cv_x87_double_rounding=yes"
318 "ac_cv_tanh_preserves_zero_sign=yes"
319 # Generally assume that things are present and work
320 "ac_cv_posix_semaphores_enabled=yes"
321 "ac_cv_broken_sem_getvalue=no"
322 "ac_cv_wchar_t_signed=yes"
323 "ac_cv_rshift_extends_sign=yes"
324 "ac_cv_broken_nice=no"
325 "ac_cv_broken_poll=no"
326 "ac_cv_working_tzset=yes"
327 "ac_cv_have_long_long_format=yes"
328 "ac_cv_have_size_t_format=yes"
329 "ac_cv_computed_gotos=yes"
330 "ac_cv_file__dev_ptmx=yes"
331 "ac_cv_file__dev_ptc=yes"
332 ] ++ optionals stdenv.hostPlatform.isLinux [
333 # Never even try to use lchmod on linux,
334 # don't rely on detecting glibc-isms.
335 "ac_cv_func_lchmod=no"
336 ] ++ optionals tzdataSupport [
337 "--with-tzpath=${tzdata}/share/zoneinfo"
338 ] ++ optional static "LDFLAGS=-static";
339
340 preConfigure = ''
341 for i in /usr /sw /opt /pkg; do # improve purity
342 substituteInPlace ./setup.py --replace $i /no-such-path
343 done
344 '' + optionalString stdenv.isDarwin ''
345 export NIX_CFLAGS_COMPILE="$NIX_CFLAGS_COMPILE -msse2"
346 export MACOSX_DEPLOYMENT_TARGET=10.6
347 # Override the auto-detection in setup.py, which assumes a universal build
348 export PYTHON_DECIMAL_WITH_MACHINE=${if stdenv.isAarch64 then "uint128" else "x64"}
349 '' + optionalString (isPy3k && pythonOlder "3.7") ''
350 # Determinism: The interpreter is patched to write null timestamps when compiling Python files
351 # so Python doesn't try to update the bytecode when seeing frozen timestamps in Nix's store.
352 export DETERMINISTIC_BUILD=1;
353 '' + optionalString stdenv.hostPlatform.isMusl ''
354 export NIX_CFLAGS_COMPILE+=" -DTHREAD_STACK_SIZE=0x100000"
355 '' +
356
357 # enableNoSemanticInterposition essentially sets that CFLAG -fno-semantic-interposition
358 # which changes how symbols are looked up. This essentially means we can't override
359 # libpython symbols via LD_PRELOAD anymore. This is common enough as every build
360 # that uses --enable-optimizations has the same "issue".
361 #
362 # The Fedora wiki has a good article about their journey towards enabling this flag:
363 # https://fedoraproject.org/wiki/Changes/PythonNoSemanticInterpositionSpeedup
364 optionalString enableNoSemanticInterposition ''
365 export CFLAGS_NODIST="-fno-semantic-interposition"
366 '';
367
368 setupHook = python-setup-hook sitePackages;
369
370 postInstall = let
371 # References *not* to nuke from (sys)config files
372 keep-references = concatMapStringsSep " " (val: "-e ${val}") ([
373 (placeholder "out")
374 ] ++ optionals tzdataSupport [
375 tzdata
376 ]);
377 in ''
378 # needed for some packages, especially packages that backport functionality
379 # to 2.x from 3.x
380 for item in $out/lib/${libPrefix}/test/*; do
381 if [[ "$item" != */test_support.py*
382 && "$item" != */test/support
383 && "$item" != */test/libregrtest
384 && "$item" != */test/regrtest.py* ]]; then
385 rm -rf "$item"
386 else
387 echo $item
388 fi
389 done
390 touch $out/lib/${libPrefix}/test/__init__.py
391
392 ln -s "$out/include/${executable}m" "$out/include/${executable}"
393
394 # Determinism: Windows installers were not deterministic.
395 # We're also not interested in building Windows installers.
396 find "$out" -name 'wininst*.exe' | xargs -r rm -f
397
398 # Use Python3 as default python
399 ln -s "$out/bin/idle3" "$out/bin/idle"
400 ln -s "$out/bin/pydoc3" "$out/bin/pydoc"
401 ln -s "$out/bin/python3" "$out/bin/python"
402 ln -s "$out/bin/python3-config" "$out/bin/python-config"
403 ln -s "$out/lib/pkgconfig/python3.pc" "$out/lib/pkgconfig/python.pc"
404
405 # Get rid of retained dependencies on -dev packages, and remove
406 # some $TMPDIR references to improve binary reproducibility.
407 # Note that the .pyc file of _sysconfigdata.py should be regenerated!
408 for i in $out/lib/${libPrefix}/_sysconfigdata*.py $out/lib/${libPrefix}/config-${sourceVersion.major}${sourceVersion.minor}*/Makefile; do
409 sed -i $i -e "s|$TMPDIR|/no-such-path|g"
410 done
411
412 # Further get rid of references. https://github.com/NixOS/nixpkgs/issues/51668
413 find $out/lib/python*/config-* -type f -print -exec nuke-refs ${keep-references} '{}' +
414 find $out/lib -name '_sysconfigdata*.py*' -print -exec nuke-refs ${keep-references} '{}' +
415
416 # Make the sysconfigdata module accessible on PYTHONPATH
417 # This allows build Python to import host Python's sysconfigdata
418 mkdir -p "$out/${sitePackages}"
419 ln -s "$out/lib/${libPrefix}/"_sysconfigdata*.py "$out/${sitePackages}/"
420
421 # debug info can't be separated from a static library and would otherwise be
422 # left in place by a separateDebugInfo build. force its removal here to save
423 # space in output.
424 $STRIP -S $out/lib/${libPrefix}/config-*/libpython*.a || true
425 '' + optionalString stripConfig ''
426 rm -R $out/bin/python*-config $out/lib/python*/config-*
427 '' + optionalString stripIdlelib ''
428 # Strip IDLE (and turtledemo, which uses it)
429 rm -R $out/bin/idle* $out/lib/python*/{idlelib,turtledemo}
430 '' + optionalString stripTkinter ''
431 rm -R $out/lib/python*/tkinter
432 '' + optionalString stripTests ''
433 # Strip tests
434 rm -R $out/lib/python*/test $out/lib/python*/**/test{,s}
435 '' + optionalString includeSiteCustomize ''
436 # Include a sitecustomize.py file
437 cp ${../sitecustomize.py} $out/${sitePackages}/sitecustomize.py
438
439 '' + optionalString stripBytecode ''
440 # Determinism: deterministic bytecode
441 # First we delete all old bytecode.
442 find $out -type d -name __pycache__ -print0 | xargs -0 -I {} rm -rf "{}"
443 '' + optionalString rebuildBytecode ''
444 # Python 3.7 implements PEP 552, introducing support for deterministic bytecode.
445 # compileall uses the therein introduced checked-hash method by default when
446 # `SOURCE_DATE_EPOCH` is set.
447 # We exclude lib2to3 because that's Python 2 code which fails
448 # We build 3 levels of optimized bytecode. Note the default level, without optimizations,
449 # is not reproducible yet. https://bugs.python.org/issue29708
450 # Not creating bytecode will result in a large performance loss however, so we do build it.
451 find $out -name "*.py" | ${pythonForBuildInterpreter} -m compileall -q -f -x "lib2to3" -i -
452 find $out -name "*.py" | ${pythonForBuildInterpreter} -O -m compileall -q -f -x "lib2to3" -i -
453 find $out -name "*.py" | ${pythonForBuildInterpreter} -OO -m compileall -q -f -x "lib2to3" -i -
454 '' + ''
455 # *strip* shebang from libpython gdb script - it should be dual-syntax and
456 # interpretable by whatever python the gdb in question is using, which may
457 # not even match the major version of this python. doing this after the
458 # bytecode compilations for the same reason - we don't want bytecode generated.
459 mkdir -p $out/share/gdb
460 sed '/^#!/d' Tools/gdb/libpython.py > $out/share/gdb/libpython.py
461 '';
462
463 preFixup = lib.optionalString (stdenv.hostPlatform != stdenv.buildPlatform) ''
464 # Ensure patch-shebangs uses shebangs of host interpreter.
465 export PATH=${lib.makeBinPath [ "$out" bash ]}:$PATH
466 '';
467
468 # Add CPython specific setup-hook that configures distutils.sysconfig to
469 # always load sysconfigdata from host Python.
470 postFixup = lib.optionalString (!stdenv.hostPlatform.isDarwin) ''
471 cat << "EOF" >> "$out/nix-support/setup-hook"
472 ${sysconfigdataHook}
473 EOF
474 '';
475
476 # Enforce that we don't have references to the OpenSSL -dev package, which we
477 # explicitly specify in our configure flags above.
478 disallowedReferences =
479 lib.optionals (openssl != null && !static) [ openssl.dev ]
480 ++ lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [
481 # Ensure we don't have references to build-time packages.
482 # These typically end up in shebangs.
483 pythonForBuild buildPackages.bash
484 ];
485
486 separateDebugInfo = true;
487
488 inherit passthru;
489
490 enableParallelBuilding = true;
491
492 meta = {
493 homepage = "http://python.org";
494 description = "A high-level dynamically-typed programming language";
495 longDescription = ''
496 Python is a remarkably powerful dynamic programming language that
497 is used in a wide variety of application domains. Some of its key
498 distinguishing features include: clear, readable syntax; strong
499 introspection capabilities; intuitive object orientation; natural
500 expression of procedural code; full modularity, supporting
501 hierarchical packages; exception-based error handling; and very
502 high level dynamic data types.
503 '';
504 license = licenses.psfl;
505 platforms = with platforms; linux ++ darwin;
506 maintainers = with maintainers; [ fridh ];
507 };
508}