tjh.dev
nixpkgs mirror (for testing)
github.com/NixOS/nixpkgs
nix
1{ lib, stdenv, fetchurl, fetchpatch
2, bzip2
3, expat
4, libffi
5, gdbm
6, xz
7, mime-types ? null, mimetypesSupport ? true
8, ncurses
9, openssl
10, readline
11, sqlite
12, tcl ? null, tk ? null, tix ? null, libX11 ? null, xorgproto ? null, x11Support ? false
13, bluez ? null, bluezSupport ? false
14, zlib
15, tzdata ? null
16, self
17, configd
18, autoreconfHook
19, autoconf-archive
20, python-setup-hook
21, nukeReferences
22# For the Python package set
23, packageOverrides ? (self: super: {})
24, pkgsBuildBuild
25, pkgsBuildHost
26, pkgsBuildTarget
27, pkgsHostHost
28, pkgsTargetTarget
29, sourceVersion
30, sha256
31, passthruFun
32, bash
33, stripConfig ? false
34, stripIdlelib ? false
35, stripTests ? false
36, stripTkinter ? false
37, rebuildBytecode ? true
38, stripBytecode ? reproducibleBuild
39, includeSiteCustomize ? true
40, static ? stdenv.hostPlatform.isStatic
41, enableOptimizations ? false
42, reproducibleBuild ? true
43, pythonAttr ? "python${sourceVersion.major}${sourceVersion.minor}"
44}:
45
46# Note: this package is used for bootstrapping fetchurl, and thus
47# cannot use fetchpatch! All mutable patches (generated by GitHub or
48# cgit) that are needed here should be included directly in Nixpkgs as
49# files.
50
51assert x11Support -> tcl != null
52 && tk != null
53 && xorgproto != null
54 && libX11 != null;
55
56assert bluezSupport -> bluez != null;
57
58assert mimetypesSupport -> mime-types != null;
59
60assert lib.assertMsg (enableOptimizations -> (!stdenv.cc.isClang))
61 "Optimizations with clang are not supported. configure: error: llvm-profdata is required for a --enable-optimizations build but could not be found.";
62
63assert lib.assertMsg (reproducibleBuild -> stripBytecode)
64 "Deterministic builds require stripping bytecode.";
65
66assert lib.assertMsg (reproducibleBuild -> (!enableOptimizations))
67 "Deterministic builds are not achieved when optimizations are enabled.";
68
69with lib;
70
71let
72 buildPackages = pkgsBuildHost;
73 inherit (passthru) pythonForBuild;
74
75 tzdataSupport = tzdata != null && passthru.pythonAtLeast "3.9";
76
77 passthru = passthruFun rec {
78 inherit self sourceVersion packageOverrides;
79 implementation = "cpython";
80 libPrefix = "python${pythonVersion}";
81 executable = libPrefix;
82 pythonVersion = with sourceVersion; "${major}.${minor}";
83 sitePackages = "lib/${libPrefix}/site-packages";
84 inherit hasDistutilsCxxPatch;
85 pythonOnBuildForBuild = pkgsBuildBuild.${pythonAttr};
86 pythonOnBuildForHost = pkgsBuildHost.${pythonAttr};
87 pythonOnBuildForTarget = pkgsBuildTarget.${pythonAttr};
88 pythonOnHostForHost = pkgsHostHost.${pythonAttr};
89 pythonOnTargetForTarget = pkgsTargetTarget.${pythonAttr} or {};
90 };
91
92 version = with sourceVersion; "${major}.${minor}.${patch}${suffix}";
93
94 nativeBuildInputs = optionals (!stdenv.isDarwin) [
95 autoreconfHook
96 ] ++ optionals (!stdenv.isDarwin && passthru.pythonAtLeast "3.10") [
97 autoconf-archive # needed for AX_CHECK_COMPILE_FLAG
98 ] ++ [
99 nukeReferences
100 ] ++ optionals (stdenv.hostPlatform != stdenv.buildPlatform) [
101 buildPackages.stdenv.cc
102 pythonForBuild
103 ];
104
105 buildInputs = filter (p: p != null) ([
106 zlib bzip2 expat xz libffi gdbm sqlite readline ncurses openssl ]
107 ++ optionals x11Support [ tcl tk libX11 xorgproto ]
108 ++ optionals (bluezSupport && stdenv.isLinux) [ bluez ]
109 ++ optionals stdenv.isDarwin [ configd ])
110 ++ optionals tzdataSupport [ tzdata ]; # `zoneinfo` module
111
112 hasDistutilsCxxPatch = !(stdenv.cc.isGNU or false);
113
114 pythonForBuildInterpreter = if stdenv.hostPlatform == stdenv.buildPlatform then
115 "$out/bin/python"
116 else pythonForBuild.interpreter;
117
118 # The CPython interpreter contains a _sysconfigdata_<platform specific suffix>
119 # module that is imported by the sysconfig and distutils.sysconfig modules.
120 # The sysconfigdata module is generated at build time and contains settings
121 # required for building Python extension modules, such as include paths and
122 # other compiler flags. By default, the sysconfigdata module is loaded from
123 # the currently running interpreter (ie. the build platform interpreter), but
124 # when cross-compiling we want to load it from the host platform interpreter.
125 # This can be done using the _PYTHON_SYSCONFIGDATA_NAME environment variable.
126 # The _PYTHON_HOST_PLATFORM variable also needs to be set to get the correct
127 # platform suffix on extension modules. The correct values for these variables
128 # are not documented, and must be derived from the configure script (see links
129 # below).
130 sysconfigdataHook = with stdenv.hostPlatform; with passthru; let
131 # https://github.com/python/cpython/blob/e488e300f5c01289c10906c2e53a8e43d6de32d8/configure.ac#L428
132 # The configure script uses "arm" as the CPU name for all 32-bit ARM
133 # variants when cross-compiling, but native builds include the version
134 # suffix, so we do the same.
135 pythonHostPlatform = "${parsed.kernel.name}-${parsed.cpu.name}";
136
137 # https://github.com/python/cpython/blob/e488e300f5c01289c10906c2e53a8e43d6de32d8/configure.ac#L724
138 multiarchCpu =
139 if isAarch32 then
140 if parsed.cpu.significantByte.name == "littleEndian" then "arm" else "armeb"
141 else if isx86_32 then "i386"
142 else parsed.cpu.name;
143 pythonAbiName =
144 # python's build doesn't differentiate between musl and glibc in its
145 # abi detection, our wrapper should match.
146 if stdenv.hostPlatform.isMusl then
147 replaceStrings [ "musl" ] [ "gnu" ] parsed.abi.name
148 else parsed.abi.name;
149 multiarch =
150 if isDarwin then "darwin"
151 else "${multiarchCpu}-${parsed.kernel.name}-${pythonAbiName}";
152
153 abiFlags = optionalString (isPy36 || isPy37) "m";
154
155 # https://github.com/python/cpython/blob/e488e300f5c01289c10906c2e53a8e43d6de32d8/configure.ac#L78
156 pythonSysconfigdataName = "_sysconfigdata_${abiFlags}_${parsed.kernel.name}_${multiarch}";
157 in ''
158 sysconfigdataHook() {
159 if [ "$1" = '${placeholder "out"}' ]; then
160 export _PYTHON_HOST_PLATFORM='${pythonHostPlatform}'
161 export _PYTHON_SYSCONFIGDATA_NAME='${pythonSysconfigdataName}'
162 fi
163 }
164
165 addEnvHooks "$hostOffset" sysconfigdataHook
166 '';
167
168in with passthru; stdenv.mkDerivation {
169 pname = "python3";
170 inherit version;
171
172 inherit buildInputs nativeBuildInputs;
173
174 src = fetchurl {
175 url = with sourceVersion; "https://www.python.org/ftp/python/${major}.${minor}.${patch}/Python-${version}.tar.xz";
176 inherit sha256;
177 };
178
179 prePatch = optionalString stdenv.isDarwin ''
180 substituteInPlace configure --replace '`/usr/bin/arch`' '"i386"'
181 substituteInPlace configure --replace '-Wl,-stack_size,1000000' ' '
182 '' + optionalString (stdenv.isDarwin && x11Support) ''
183 substituteInPlace setup.py --replace /Library/Frameworks /no-such-path
184 '';
185
186 patches = [
187 # Disable the use of ldconfig in ctypes.util.find_library (since
188 # ldconfig doesn't work on NixOS), and don't use
189 # ctypes.util.find_library during the loading of the uuid module
190 # (since it will do a futile invocation of gcc (!) to find
191 # libuuid, slowing down program startup a lot).
192 (./. + "/${sourceVersion.major}.${sourceVersion.minor}/no-ldconfig.patch")
193 ] ++ optionals mimetypesSupport [
194 # Make the mimetypes module refer to the right file
195 ./mimetypes.patch
196 ] ++ optionals (isPy35 || isPy36) [
197 # Determinism: Write null timestamps when compiling python files.
198 ./3.5/force_bytecode_determinism.patch
199 ] ++ optionals isPy35 [
200 # Backports support for LD_LIBRARY_PATH from 3.6
201 ./3.5/ld_library_path.patch
202 ] ++ optionals (isPy35 || isPy36 || isPy37) [
203 # Backport a fix for discovering `rpmbuild` command when doing `python setup.py bdist_rpm` to 3.5, 3.6, 3.7.
204 # See: https://bugs.python.org/issue11122
205 ./3.7/fix-hardcoded-path-checking-for-rpmbuild.patch
206 # The workaround is for unittests on Win64, which we don't support.
207 # It does break aarch64-darwin, which we do support. See:
208 # * https://bugs.python.org/issue35523
209 # * https://github.com/python/cpython/commit/e6b247c8e524
210 ./3.7/no-win64-workaround.patch
211 ] ++ optionals (isPy37 || isPy38 || isPy39) [
212 # Fix darwin build https://bugs.python.org/issue34027
213 ./3.7/darwin-libutil.patch
214 ] ++ optionals (pythonOlder "3.8") [
215 # Backport from CPython 3.8 of a good list of tests to run for PGO.
216 (
217 if isPy36 || isPy37 then
218 ./3.6/profile-task.patch
219 else
220 ./3.5/profile-task.patch
221 )
222 ] ++ optionals (isPy3k && hasDistutilsCxxPatch) [
223 # Fix for http://bugs.python.org/issue1222585
224 # Upstream distutils is calling C compiler to compile C++ code, which
225 # only works for GCC and Apple Clang. This makes distutils to call C++
226 # compiler when needed.
227 (
228 if isPy35 then
229 ./3.5/python-3.x-distutils-C++.patch
230 else if isPy37 || isPy38 || isPy39 then
231 ./3.7/python-3.x-distutils-C++.patch
232 else
233 fetchpatch {
234 url = "https://bugs.python.org/file48016/python-3.x-distutils-C++.patch";
235 sha256 = "1h18lnpx539h5lfxyk379dxwr8m2raigcjixkf133l4xy3f4bzi2";
236 }
237 )
238 ] ++ [
239 # LDSHARED now uses $CC instead of gcc. Fixes cross-compilation of extension modules.
240 ./3.8/0001-On-all-posix-systems-not-just-Darwin-set-LDSHARED-if.patch
241 # Use sysconfigdata to find headers. Fixes cross-compilation of extension modules.
242 (
243 if isPy36 then
244 ./3.6/fix-finding-headers-when-cross-compiling.patch
245 else
246 ./3.7/fix-finding-headers-when-cross-compiling.patch
247 )
248 ] ++ optionals (isPy36) [
249 # Backport a fix for ctypes.util.find_library.
250 ./3.6/find_library.patch
251 ];
252
253 postPatch = ''
254 substituteInPlace Lib/subprocess.py \
255 --replace "'/bin/sh'" "'${bash}/bin/sh'"
256 '' + optionalString mimetypesSupport ''
257 substituteInPlace Lib/mimetypes.py \
258 --replace "@mime-types@" "${mime-types}"
259 '' + optionalString (x11Support && (tix != null)) ''
260 substituteInPlace "Lib/tkinter/tix.py" --replace "os.environ.get('TIX_LIBRARY')" "os.environ.get('TIX_LIBRARY') or '${tix}/lib'"
261 '';
262
263 CPPFLAGS = concatStringsSep " " (map (p: "-I${getDev p}/include") buildInputs);
264 LDFLAGS = concatStringsSep " " (map (p: "-L${getLib p}/lib") buildInputs);
265 LIBS = "${optionalString (!stdenv.isDarwin) "-lcrypt"} ${optionalString (ncurses != null) "-lncurses"}";
266 NIX_LDFLAGS = optionalString (stdenv.isLinux && !stdenv.hostPlatform.isMusl) "-lgcc_s" + optionalString stdenv.hostPlatform.isMusl "-lgcc_eh";
267 # Determinism: We fix the hashes of str, bytes and datetime objects.
268 PYTHONHASHSEED=0;
269
270 configureFlags = [
271 "--enable-shared"
272 "--without-ensurepip"
273 "--with-system-expat"
274 "--with-system-ffi"
275 ] ++ optionals enableOptimizations [
276 "--enable-optimizations"
277 ] ++ optionals (pythonOlder "3.7") [
278 # This is unconditionally true starting in CPython 3.7.
279 "--with-threads"
280 ] ++ optionals (sqlite != null && isPy3k) [
281 "--enable-loadable-sqlite-extensions"
282 ] ++ optionals (openssl != null) [
283 "--with-openssl=${openssl.dev}"
284 ] ++ optionals (stdenv.hostPlatform != stdenv.buildPlatform) [
285 "ac_cv_buggy_getaddrinfo=no"
286 # Assume little-endian IEEE 754 floating point when cross compiling
287 "ac_cv_little_endian_double=yes"
288 "ac_cv_big_endian_double=no"
289 "ac_cv_mixed_endian_double=no"
290 "ac_cv_x87_double_rounding=yes"
291 "ac_cv_tanh_preserves_zero_sign=yes"
292 # Generally assume that things are present and work
293 "ac_cv_posix_semaphores_enabled=yes"
294 "ac_cv_broken_sem_getvalue=no"
295 "ac_cv_wchar_t_signed=yes"
296 "ac_cv_rshift_extends_sign=yes"
297 "ac_cv_broken_nice=no"
298 "ac_cv_broken_poll=no"
299 "ac_cv_working_tzset=yes"
300 "ac_cv_have_long_long_format=yes"
301 "ac_cv_have_size_t_format=yes"
302 "ac_cv_computed_gotos=yes"
303 "ac_cv_file__dev_ptmx=yes"
304 "ac_cv_file__dev_ptc=yes"
305 ] ++ optionals stdenv.hostPlatform.isLinux [
306 # Never even try to use lchmod on linux,
307 # don't rely on detecting glibc-isms.
308 "ac_cv_func_lchmod=no"
309 ] ++ optionals tzdataSupport [
310 "--with-tzpath=${tzdata}/share/zoneinfo"
311 ] ++ optional static "LDFLAGS=-static";
312
313 preConfigure = ''
314 for i in /usr /sw /opt /pkg; do # improve purity
315 substituteInPlace ./setup.py --replace $i /no-such-path
316 done
317 '' + optionalString stdenv.isDarwin ''
318 export NIX_CFLAGS_COMPILE="$NIX_CFLAGS_COMPILE -msse2"
319 export MACOSX_DEPLOYMENT_TARGET=10.6
320 '' + optionalString (isPy3k && pythonOlder "3.7") ''
321 # Determinism: The interpreter is patched to write null timestamps when compiling Python files
322 # so Python doesn't try to update the bytecode when seeing frozen timestamps in Nix's store.
323 export DETERMINISTIC_BUILD=1;
324 '' + optionalString stdenv.hostPlatform.isMusl ''
325 export NIX_CFLAGS_COMPILE+=" -DTHREAD_STACK_SIZE=0x100000"
326 '';
327
328 setupHook = python-setup-hook sitePackages;
329
330 postInstall = let
331 # References *not* to nuke from (sys)config files
332 keep-references = concatMapStringsSep " " (val: "-e ${val}") ([
333 (placeholder "out")
334 ] ++ optionals tzdataSupport [
335 tzdata
336 ]);
337 in ''
338 # needed for some packages, especially packages that backport functionality
339 # to 2.x from 3.x
340 for item in $out/lib/${libPrefix}/test/*; do
341 if [[ "$item" != */test_support.py*
342 && "$item" != */test/support
343 && "$item" != */test/libregrtest
344 && "$item" != */test/regrtest.py* ]]; then
345 rm -rf "$item"
346 else
347 echo $item
348 fi
349 done
350 touch $out/lib/${libPrefix}/test/__init__.py
351
352 ln -s "$out/include/${executable}m" "$out/include/${executable}"
353
354 # Determinism: Windows installers were not deterministic.
355 # We're also not interested in building Windows installers.
356 find "$out" -name 'wininst*.exe' | xargs -r rm -f
357
358 # Use Python3 as default python
359 ln -s "$out/bin/idle3" "$out/bin/idle"
360 ln -s "$out/bin/pydoc3" "$out/bin/pydoc"
361 ln -s "$out/bin/python3" "$out/bin/python"
362 ln -s "$out/bin/python3-config" "$out/bin/python-config"
363 ln -s "$out/lib/pkgconfig/python3.pc" "$out/lib/pkgconfig/python.pc"
364
365 # Get rid of retained dependencies on -dev packages, and remove
366 # some $TMPDIR references to improve binary reproducibility.
367 # Note that the .pyc file of _sysconfigdata.py should be regenerated!
368 for i in $out/lib/${libPrefix}/_sysconfigdata*.py $out/lib/${libPrefix}/config-${sourceVersion.major}${sourceVersion.minor}*/Makefile; do
369 sed -i $i -e "s|$TMPDIR|/no-such-path|g"
370 done
371
372 # Further get rid of references. https://github.com/NixOS/nixpkgs/issues/51668
373 find $out/lib/python*/config-* -type f -print -exec nuke-refs ${keep-references} '{}' +
374 find $out/lib -name '_sysconfigdata*.py*' -print -exec nuke-refs ${keep-references} '{}' +
375
376 # Make the sysconfigdata module accessible on PYTHONPATH
377 # This allows build Python to import host Python's sysconfigdata
378 mkdir -p "$out/${sitePackages}"
379 ln -s "$out/lib/${libPrefix}/"_sysconfigdata*.py "$out/${sitePackages}/"
380 '' + optionalString stripConfig ''
381 rm -R $out/bin/python*-config $out/lib/python*/config-*
382 '' + optionalString stripIdlelib ''
383 # Strip IDLE (and turtledemo, which uses it)
384 rm -R $out/bin/idle* $out/lib/python*/{idlelib,turtledemo}
385 '' + optionalString stripTkinter ''
386 rm -R $out/lib/python*/tkinter
387 '' + optionalString stripTests ''
388 # Strip tests
389 rm -R $out/lib/python*/test $out/lib/python*/**/test{,s}
390 '' + optionalString includeSiteCustomize ''
391 # Include a sitecustomize.py file
392 cp ${../sitecustomize.py} $out/${sitePackages}/sitecustomize.py
393
394 '' + optionalString stripBytecode ''
395 # Determinism: deterministic bytecode
396 # First we delete all old bytecode.
397 find $out -type d -name __pycache__ -print0 | xargs -0 -I {} rm -rf "{}"
398 '' + optionalString rebuildBytecode ''
399 # Then, we build for the two optimization levels.
400 # We do not build unoptimized bytecode, because its not entirely deterministic yet.
401 # Python 3.7 implements PEP 552, introducing support for deterministic bytecode.
402 # compileall uses this checked-hash method by default when `SOURCE_DATE_EPOCH` is set.
403 # We exclude lib2to3 because that's Python 2 code which fails
404 find $out -name "*.py" | ${pythonForBuildInterpreter} -O -m compileall -q -f -x "lib2to3" -i -
405 find $out -name "*.py" | ${pythonForBuildInterpreter} -OO -m compileall -q -f -x "lib2to3" -i -
406 '';
407
408 preFixup = lib.optionalString (stdenv.hostPlatform != stdenv.buildPlatform) ''
409 # Ensure patch-shebangs uses shebangs of host interpreter.
410 export PATH=${lib.makeBinPath [ "$out" bash ]}:$PATH
411 '';
412
413 # Add CPython specific setup-hook that configures distutils.sysconfig to
414 # always load sysconfigdata from host Python.
415 postFixup = lib.optionalString (!stdenv.hostPlatform.isDarwin) ''
416 cat << "EOF" >> "$out/nix-support/setup-hook"
417 ${sysconfigdataHook}
418 EOF
419 '';
420
421 # Enforce that we don't have references to the OpenSSL -dev package, which we
422 # explicitly specify in our configure flags above.
423 disallowedReferences =
424 lib.optionals (openssl != null && !static) [ openssl.dev ]
425 ++ lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [
426 # Ensure we don't have references to build-time packages.
427 # These typically end up in shebangs.
428 pythonForBuild buildPackages.bash
429 ];
430
431 inherit passthru;
432
433 enableParallelBuilding = true;
434
435 meta = {
436 homepage = "http://python.org";
437 description = "A high-level dynamically-typed programming language";
438 longDescription = ''
439 Python is a remarkably powerful dynamic programming language that
440 is used in a wide variety of application domains. Some of its key
441 distinguishing features include: clear, readable syntax; strong
442 introspection capabilities; intuitive object orientation; natural
443 expression of procedural code; full modularity, supporting
444 hierarchical packages; exception-based error handling; and very
445 high level dynamic data types.
446 '';
447 license = licenses.psfl;
448 platforms = with platforms; linux ++ darwin;
449 maintainers = with maintainers; [ fridh ];
450 };
451}