pyrox.dev
1{ lib, stdenv, fetchurl, fetchpatch
2, bzip2
3, expat
4, libffi
5, gdbm
6, db
7, ncurses
8, openssl
9, readline
10, sqlite
11, tcl ? null, tk ? null, tix ? null, xlibsWrapper ? null, libX11 ? null, x11Support ? false
12, zlib
13, self
14, configd, coreutils
15, autoreconfHook
16, python-setup-hook
17# Some proprietary libs assume UCS2 unicode, especially on darwin :(
18, ucsEncoding ? 4
19# For the Python package set
20, packageOverrides ? (self: super: {})
21, pkgsBuildBuild
22, pkgsBuildHost
23, pkgsBuildTarget
24, pkgsHostHost
25, pkgsTargetTarget
26, sourceVersion
27, sha256
28, passthruFun
29, static ? stdenv.hostPlatform.isStatic
30, stripBytecode ? reproducibleBuild
31, rebuildBytecode ? true
32, reproducibleBuild ? false
33, enableOptimizations ? false
34, pythonAttr ? "python${sourceVersion.major}${sourceVersion.minor}"
35}:
36
37assert x11Support -> tcl != null
38 && tk != null
39 && xlibsWrapper != null
40 && libX11 != null;
41
42assert lib.assertMsg (enableOptimizations -> (!stdenv.cc.isClang))
43 "Optimizations with clang are not supported. configure: error: llvm-profdata is required for a --enable-optimizations build but could not be found.";
44
45assert lib.assertMsg (reproducibleBuild -> stripBytecode)
46 "Deterministic builds require stripping bytecode.";
47
48assert lib.assertMsg (reproducibleBuild -> (!enableOptimizations))
49 "Deterministic builds are not achieved when optimizations are enabled.";
50
51assert lib.assertMsg (reproducibleBuild -> (!rebuildBytecode))
52 "Deterministic builds are not achieved when (default unoptimized) bytecode is created.";
53
54with lib;
55
56let
57 buildPackages = pkgsBuildHost;
58 inherit (passthru) pythonForBuild;
59
60 pythonForBuildInterpreter = if stdenv.hostPlatform == stdenv.buildPlatform then
61 "$out/bin/python"
62 else pythonForBuild.interpreter;
63
64 passthru = passthruFun rec {
65 inherit self sourceVersion packageOverrides;
66 implementation = "cpython";
67 libPrefix = "python${pythonVersion}";
68 executable = libPrefix;
69 pythonVersion = with sourceVersion; "${major}.${minor}";
70 sitePackages = "lib/${libPrefix}/site-packages";
71 inherit hasDistutilsCxxPatch;
72 pythonOnBuildForBuild = pkgsBuildBuild.${pythonAttr};
73 pythonOnBuildForHost = pkgsBuildHost.${pythonAttr};
74 pythonOnBuildForTarget = pkgsBuildTarget.${pythonAttr};
75 pythonOnHostForHost = pkgsHostHost.${pythonAttr};
76 pythonOnTargetForTarget = pkgsTargetTarget.${pythonAttr} or {};
77 } // {
78 inherit ucsEncoding;
79 };
80
81 version = with sourceVersion; "${major}.${minor}.${patch}${suffix}";
82
83 src = fetchurl {
84 url = with sourceVersion; "https://www.python.org/ftp/python/${major}.${minor}.${patch}/Python-${version}.tar.xz";
85 inherit sha256;
86 };
87
88 hasDistutilsCxxPatch = !(stdenv.cc.isGNU or false);
89 patches =
90 [ # Look in C_INCLUDE_PATH and LIBRARY_PATH for stuff.
91 ./search-path.patch
92
93 # Python recompiles a Python if the mtime stored *in* the
94 # pyc/pyo file differs from the mtime of the source file. This
95 # doesn't work in Nix because Nix changes the mtime of files in
96 # the Nix store to 1. So treat that as a special case.
97 ./nix-store-mtime.patch
98
99 # patch python to put zero timestamp into pyc
100 # if DETERMINISTIC_BUILD env var is set
101 ./deterministic-build.patch
102
103 # Fix python bug #27177 (https://bugs.python.org/issue27177)
104 # The issue is that `match.group` only recognizes python integers
105 # instead of everything that has `__index__`.
106 # This bug was fixed upstream, but not backported to 2.7
107 (fetchpatch {
108 name = "re_match_index.patch";
109 url = "https://bugs.python.org/file43084/re_match_index.patch";
110 sha256 = "0l9rw6r5r90iybdkp3hhl2pf0h0s1izc68h5d3ywrm92pq32wz57";
111 })
112
113 # Fix race-condition during pyc creation. Has a slight backwards
114 # incompatible effect: pyc symlinks will now be overridden
115 # (https://bugs.python.org/issue17222). Included in python >= 3.4,
116 # backported in debian since 2013.
117 # https://bugs.python.org/issue13146
118 ./atomic_pyc.patch
119
120 # Backport from CPython 3.8 of a good list of tests to run for PGO.
121 ./profile-task.patch
122
123 # Patch is likely to go away in the next release (if there is any)
124 ./CVE-2019-20907.patch
125
126 ./CVE-2021-3177.patch
127
128 ./CVE-2021-23336.patch
129
130 # The workaround is for unittests on Win64, which we don't support.
131 # It does break aarch64-darwin, which we do support. See:
132 # * https://bugs.python.org/issue35523
133 # * https://github.com/python/cpython/commit/e6b247c8e524
134 ../3.7/no-win64-workaround.patch
135
136 ] ++ optionals (x11Support && stdenv.isDarwin) [
137 ./use-correct-tcl-tk-on-darwin.patch
138 ] ++ optionals stdenv.isLinux [
139
140 # Disable the use of ldconfig in ctypes.util.find_library (since
141 # ldconfig doesn't work on NixOS), and don't use
142 # ctypes.util.find_library during the loading of the uuid module
143 # (since it will do a futile invocation of gcc (!) to find
144 # libuuid, slowing down program startup a lot).
145 ./no-ldconfig.patch
146
147 # Fix ctypes.util.find_library with gcc10.
148 ./find_library-gcc10.patch
149
150 ] ++ optionals stdenv.hostPlatform.isCygwin [
151 ./2.5.2-ctypes-util-find_library.patch
152 ./2.5.2-tkinter-x11.patch
153 ./2.6.2-ssl-threads.patch
154 ./2.6.5-export-PySignal_SetWakeupFd.patch
155 ./2.6.5-FD_SETSIZE.patch
156 ./2.6.5-ncurses-abi6.patch
157 ./2.7.3-dbm.patch
158 ./2.7.3-dylib.patch
159 ./2.7.3-getpath-exe-extension.patch
160 ./2.7.3-no-libm.patch
161 ] ++ optionals hasDistutilsCxxPatch [
162
163 # Patch from http://bugs.python.org/issue1222585 adapted to work with
164 # `patch -p1' and with a last hunk removed
165 # Upstream distutils is calling C compiler to compile C++ code, which
166 # only works for GCC and Apple Clang. This makes distutils to call C++
167 # compiler when needed.
168 ./python-2.7-distutils-C++.patch
169 ] ++ optional (stdenv.hostPlatform != stdenv.buildPlatform) [
170 ./cross-compile.patch
171 ];
172
173 preConfigure = ''
174 # Purity.
175 for i in /usr /sw /opt /pkg; do
176 substituteInPlace ./setup.py --replace $i /no-such-path
177 done
178 '' + optionalString (stdenv ? cc && stdenv.cc.libc != null) ''
179 for i in Lib/plat-*/regen; do
180 substituteInPlace $i --replace /usr/include/ ${stdenv.cc.libc}/include/
181 done
182 '' + optionalString stdenv.isDarwin ''
183 substituteInPlace configure --replace '`/usr/bin/arch`' '"i386"'
184 substituteInPlace Lib/multiprocessing/__init__.py \
185 --replace 'os.popen(comm)' 'os.popen("${coreutils}/bin/nproc")'
186 '';
187
188 configureFlags = optionals enableOptimizations [
189 "--enable-optimizations"
190 ] ++ optionals (!static) [
191 "--enable-shared"
192 ] ++ [
193 "--with-threads"
194 "--enable-unicode=ucs${toString ucsEncoding}"
195 ] ++ optionals (stdenv.hostPlatform.isCygwin || stdenv.hostPlatform.isAarch64) [
196 "--with-system-ffi"
197 ] ++ optionals stdenv.hostPlatform.isCygwin [
198 "--with-system-expat"
199 "ac_cv_func_bind_textdomain_codeset=yes"
200 ] ++ optionals stdenv.isDarwin [
201 "--disable-toolbox-glue"
202 ] ++ optionals (stdenv.hostPlatform != stdenv.buildPlatform) [
203 "PYTHON_FOR_BUILD=${getBin buildPackages.python}/bin/python"
204 "ac_cv_buggy_getaddrinfo=no"
205 # Assume little-endian IEEE 754 floating point when cross compiling
206 "ac_cv_little_endian_double=yes"
207 "ac_cv_big_endian_double=no"
208 "ac_cv_mixed_endian_double=no"
209 "ac_cv_x87_double_rounding=yes"
210 "ac_cv_tanh_preserves_zero_sign=yes"
211 # Generally assume that things are present and work
212 "ac_cv_posix_semaphores_enabled=yes"
213 "ac_cv_broken_sem_getvalue=no"
214 "ac_cv_wchar_t_signed=yes"
215 "ac_cv_rshift_extends_sign=yes"
216 "ac_cv_broken_nice=no"
217 "ac_cv_broken_poll=no"
218 "ac_cv_working_tzset=yes"
219 "ac_cv_have_long_long_format=yes"
220 "ac_cv_have_size_t_format=yes"
221 "ac_cv_computed_gotos=yes"
222 "ac_cv_file__dev_ptmx=yes"
223 "ac_cv_file__dev_ptc=yes"
224 ]
225 # Never even try to use lchmod on linux,
226 # don't rely on detecting glibc-isms.
227 ++ optional stdenv.hostPlatform.isLinux "ac_cv_func_lchmod=no"
228 ++ optional static "LDFLAGS=-static";
229
230 strictDeps = true;
231 buildInputs =
232 optional (stdenv ? cc && stdenv.cc.libc != null) stdenv.cc.libc ++
233 [ bzip2 openssl zlib ]
234 ++ optional (stdenv.hostPlatform.isCygwin || stdenv.hostPlatform.isAarch64) libffi
235 ++ optional stdenv.hostPlatform.isCygwin expat
236 ++ [ db gdbm ncurses sqlite readline ]
237 ++ optionals x11Support [ tcl tk xlibsWrapper libX11 ]
238 ++ optional (stdenv.isDarwin && configd != null) configd;
239 nativeBuildInputs =
240 [ autoreconfHook ]
241 ++ optionals (stdenv.hostPlatform != stdenv.buildPlatform)
242 [ buildPackages.stdenv.cc buildPackages.python ];
243
244 mkPaths = paths: {
245 C_INCLUDE_PATH = makeSearchPathOutput "dev" "include" paths;
246 LIBRARY_PATH = makeLibraryPath paths;
247 };
248
249 # Python 2.7 needs this
250 crossCompileEnv = lib.optionalAttrs (stdenv.hostPlatform != stdenv.buildPlatform)
251 { _PYTHON_HOST_PLATFORM = stdenv.hostPlatform.config; };
252
253 # Build the basic Python interpreter without modules that have
254 # external dependencies.
255
256in with passthru; stdenv.mkDerivation ({
257 pname = "python";
258 inherit version;
259
260 inherit src patches buildInputs nativeBuildInputs preConfigure configureFlags;
261
262 LDFLAGS = lib.optionalString (!stdenv.isDarwin) "-lgcc_s";
263 inherit (mkPaths buildInputs) C_INCLUDE_PATH LIBRARY_PATH;
264
265 NIX_CFLAGS_COMPILE = optionalString (stdenv.targetPlatform.system == "x86_64-darwin") "-msse2"
266 + optionalString stdenv.hostPlatform.isMusl " -DTHREAD_STACK_SIZE=0x100000";
267 DETERMINISTIC_BUILD = 1;
268
269 setupHook = python-setup-hook sitePackages;
270
271 postPatch = optionalString (x11Support && (tix != null)) ''
272 substituteInPlace "Lib/lib-tk/Tix.py" --replace "os.environ.get('TIX_LIBRARY')" "os.environ.get('TIX_LIBRARY') or '${tix}/lib'"
273 '';
274
275 postInstall =
276 ''
277 # needed for some packages, especially packages that backport
278 # functionality to 2.x from 3.x
279 for item in $out/lib/${libPrefix}/test/*; do
280 if [[ "$item" != */test_support.py*
281 && "$item" != */test/support
282 && "$item" != */test/regrtest.py* ]]; then
283 rm -rf "$item"
284 else
285 echo $item
286 fi
287 done
288 touch $out/lib/${libPrefix}/test/__init__.py
289 ln -s $out/lib/${libPrefix}/pdb.py $out/bin/pdb
290 ln -s $out/lib/${libPrefix}/pdb.py $out/bin/pdb${sourceVersion.major}.${sourceVersion.minor}
291 ln -s $out/share/man/man1/{python2.7.1.gz,python.1.gz}
292
293 rm "$out"/lib/python*/plat-*/regen # refers to glibc.dev
294
295 # Determinism: Windows installers were not deterministic.
296 # We're also not interested in building Windows installers.
297 find "$out" -name 'wininst*.exe' | xargs -r rm -f
298 '' + optionalString stripBytecode ''
299 # Determinism: deterministic bytecode
300 # First we delete all old bytecode.
301 find $out -name "*.pyc" -delete
302 '' + optionalString rebuildBytecode ''
303 # We build 3 levels of optimized bytecode. Note the default level, without optimizations,
304 # is not reproducible yet. https://bugs.python.org/issue29708
305 # Not creating bytecode will result in a large performance loss however, so we do build it.
306 find $out -name "*.py" | ${pythonForBuildInterpreter} -m compileall -q -f -x "lib2to3" -i -
307 find $out -name "*.py" | ${pythonForBuildInterpreter} -O -m compileall -q -f -x "lib2to3" -i -
308 find $out -name "*.py" | ${pythonForBuildInterpreter} -OO -m compileall -q -f -x "lib2to3" -i -
309 '' + optionalString stdenv.hostPlatform.isCygwin ''
310 cp libpython2.7.dll.a $out/lib
311 '';
312
313 inherit passthru;
314
315 postFixup = ''
316 # Include a sitecustomize.py file. Note it causes an error when it's in postInstall with 2.7.
317 cp ${../../sitecustomize.py} $out/${sitePackages}/sitecustomize.py
318 '';
319
320 enableParallelBuilding = true;
321
322 doCheck = false; # expensive, and fails
323
324 meta = {
325 homepage = "http://python.org";
326 description = "A high-level dynamically-typed programming language";
327 longDescription = ''
328 Python is a remarkably powerful dynamic programming language that
329 is used in a wide variety of application domains. Some of its key
330 distinguishing features include: clear, readable syntax; strong
331 introspection capabilities; intuitive object orientation; natural
332 expression of procedural code; full modularity, supporting
333 hierarchical packages; exception-based error handling; and very
334 high level dynamic data types.
335 '';
336 license = lib.licenses.psfl;
337 platforms = lib.platforms.all;
338 maintainers = with lib.maintainers; [ fridh ];
339 # Higher priority than Python 3.x so that `/bin/python` points to `/bin/python2`
340 # in case both 2 and 3 are installed.
341 priority = -100;
342 };
343 } // crossCompileEnv)