pyrox.dev
1{ lib, stdenv, fetchurl, fetchpatch
2, bzip2
3, expat
4, libffi
5, gdbm
6, db
7, ncurses
8, openssl
9, readline
10, sqlite
11, tcl ? null, tk ? null, tix ? null, libX11 ? null, x11Support ? false
12, zlib
13, self
14, configd, coreutils
15, autoreconfHook
16, python-setup-hook
17# Some proprietary libs assume UCS2 unicode, especially on darwin :(
18, ucsEncoding ? 4
19# For the Python package set
20, packageOverrides ? (self: super: {})
21, pkgsBuildBuild
22, pkgsBuildHost
23, pkgsBuildTarget
24, pkgsHostHost
25, pkgsTargetTarget
26, sourceVersion
27, sha256
28, passthruFun
29, static ? stdenv.hostPlatform.isStatic
30, stripBytecode ? reproducibleBuild
31, rebuildBytecode ? true
32, reproducibleBuild ? false
33, enableOptimizations ? false
34, pythonAttr ? "python${sourceVersion.major}${sourceVersion.minor}"
35}:
36
37assert x11Support -> tcl != null
38 && tk != null
39 && libX11 != null;
40
41assert lib.assertMsg (enableOptimizations -> (!stdenv.cc.isClang))
42 "Optimizations with clang are not supported. configure: error: llvm-profdata is required for a --enable-optimizations build but could not be found.";
43
44assert lib.assertMsg (reproducibleBuild -> stripBytecode)
45 "Deterministic builds require stripping bytecode.";
46
47assert lib.assertMsg (reproducibleBuild -> (!enableOptimizations))
48 "Deterministic builds are not achieved when optimizations are enabled.";
49
50assert lib.assertMsg (reproducibleBuild -> (!rebuildBytecode))
51 "Deterministic builds are not achieved when (default unoptimized) bytecode is created.";
52
53with lib;
54
55let
56 buildPackages = pkgsBuildHost;
57 inherit (passthru) pythonForBuild;
58
59 pythonForBuildInterpreter = if stdenv.hostPlatform == stdenv.buildPlatform then
60 "$out/bin/python"
61 else pythonForBuild.interpreter;
62
63 passthru = passthruFun rec {
64 inherit self sourceVersion packageOverrides;
65 implementation = "cpython";
66 libPrefix = "python${pythonVersion}";
67 executable = libPrefix;
68 pythonVersion = with sourceVersion; "${major}.${minor}";
69 sitePackages = "lib/${libPrefix}/site-packages";
70 inherit hasDistutilsCxxPatch pythonAttr;
71 pythonOnBuildForBuild = pkgsBuildBuild.${pythonAttr};
72 pythonOnBuildForHost = pkgsBuildHost.${pythonAttr};
73 pythonOnBuildForTarget = pkgsBuildTarget.${pythonAttr};
74 pythonOnHostForHost = pkgsHostHost.${pythonAttr};
75 pythonOnTargetForTarget = pkgsTargetTarget.${pythonAttr} or {};
76 } // {
77 inherit ucsEncoding;
78 };
79
80 version = with sourceVersion; "${major}.${minor}.${patch}${suffix}";
81
82 src = fetchurl {
83 url = with sourceVersion; "https://www.python.org/ftp/python/${major}.${minor}.${patch}/Python-${version}.tar.xz";
84 inherit sha256;
85 };
86
87 hasDistutilsCxxPatch = !(stdenv.cc.isGNU or false);
88 patches =
89 [ # Look in C_INCLUDE_PATH and LIBRARY_PATH for stuff.
90 ./search-path.patch
91
92 # Python recompiles a Python if the mtime stored *in* the
93 # pyc/pyo file differs from the mtime of the source file. This
94 # doesn't work in Nix because Nix changes the mtime of files in
95 # the Nix store to 1. So treat that as a special case.
96 ./nix-store-mtime.patch
97
98 # patch python to put zero timestamp into pyc
99 # if DETERMINISTIC_BUILD env var is set
100 ./deterministic-build.patch
101
102 # Fix python bug #27177 (https://bugs.python.org/issue27177)
103 # The issue is that `match.group` only recognizes python integers
104 # instead of everything that has `__index__`.
105 # This bug was fixed upstream, but not backported to 2.7
106 (fetchpatch {
107 name = "re_match_index.patch";
108 url = "https://bugs.python.org/file43084/re_match_index.patch";
109 sha256 = "0l9rw6r5r90iybdkp3hhl2pf0h0s1izc68h5d3ywrm92pq32wz57";
110 })
111
112 # Fix race-condition during pyc creation. Has a slight backwards
113 # incompatible effect: pyc symlinks will now be overridden
114 # (https://bugs.python.org/issue17222). Included in python >= 3.4,
115 # backported in debian since 2013.
116 # https://bugs.python.org/issue13146
117 ./atomic_pyc.patch
118
119 # Backport from CPython 3.8 of a good list of tests to run for PGO.
120 ./profile-task.patch
121
122 # Patch is likely to go away in the next release (if there is any)
123 ./CVE-2019-20907.patch
124
125 ./CVE-2021-3177.patch
126
127 ./CVE-2021-23336.patch
128
129 # The workaround is for unittests on Win64, which we don't support.
130 # It does break aarch64-darwin, which we do support. See:
131 # * https://bugs.python.org/issue35523
132 # * https://github.com/python/cpython/commit/e6b247c8e524
133 ../3.7/no-win64-workaround.patch
134
135 ] ++ optionals (x11Support && stdenv.isDarwin) [
136 ./use-correct-tcl-tk-on-darwin.patch
137 ] ++ optionals stdenv.isLinux [
138
139 # Disable the use of ldconfig in ctypes.util.find_library (since
140 # ldconfig doesn't work on NixOS), and don't use
141 # ctypes.util.find_library during the loading of the uuid module
142 # (since it will do a futile invocation of gcc (!) to find
143 # libuuid, slowing down program startup a lot).
144 ./no-ldconfig.patch
145
146 # Fix ctypes.util.find_library with gcc10.
147 ./find_library-gcc10.patch
148
149 ] ++ optionals stdenv.hostPlatform.isCygwin [
150 ./2.5.2-ctypes-util-find_library.patch
151 ./2.5.2-tkinter-x11.patch
152 ./2.6.2-ssl-threads.patch
153 ./2.6.5-export-PySignal_SetWakeupFd.patch
154 ./2.6.5-FD_SETSIZE.patch
155 ./2.6.5-ncurses-abi6.patch
156 ./2.7.3-dbm.patch
157 ./2.7.3-dylib.patch
158 ./2.7.3-getpath-exe-extension.patch
159 ./2.7.3-no-libm.patch
160 ] ++ optionals hasDistutilsCxxPatch [
161
162 # Patch from http://bugs.python.org/issue1222585 adapted to work with
163 # `patch -p1' and with a last hunk removed
164 # Upstream distutils is calling C compiler to compile C++ code, which
165 # only works for GCC and Apple Clang. This makes distutils to call C++
166 # compiler when needed.
167 ./python-2.7-distutils-C++.patch
168 ] ++ optionals (stdenv.hostPlatform != stdenv.buildPlatform) [
169 ./cross-compile.patch
170 ];
171
172 preConfigure = ''
173 # Purity.
174 for i in /usr /sw /opt /pkg; do
175 substituteInPlace ./setup.py --replace $i /no-such-path
176 done
177 '' + optionalString (stdenv ? cc && stdenv.cc.libc != null) ''
178 for i in Lib/plat-*/regen; do
179 substituteInPlace $i --replace /usr/include/ ${stdenv.cc.libc}/include/
180 done
181 '' + optionalString stdenv.isDarwin ''
182 substituteInPlace configure --replace '`/usr/bin/arch`' '"i386"'
183 substituteInPlace Lib/multiprocessing/__init__.py \
184 --replace 'os.popen(comm)' 'os.popen("${coreutils}/bin/nproc")'
185 '';
186
187 configureFlags = optionals enableOptimizations [
188 "--enable-optimizations"
189 ] ++ optionals (!static) [
190 "--enable-shared"
191 ] ++ [
192 "--with-threads"
193 "--enable-unicode=ucs${toString ucsEncoding}"
194 ] ++ optionals (stdenv.hostPlatform.isCygwin || stdenv.hostPlatform.isAarch64) [
195 "--with-system-ffi"
196 ] ++ optionals stdenv.hostPlatform.isCygwin [
197 "--with-system-expat"
198 "ac_cv_func_bind_textdomain_codeset=yes"
199 ] ++ optionals stdenv.isDarwin [
200 "--disable-toolbox-glue"
201 ] ++ optionals (stdenv.hostPlatform != stdenv.buildPlatform) [
202 "PYTHON_FOR_BUILD=${getBin buildPackages.python}/bin/python"
203 "ac_cv_buggy_getaddrinfo=no"
204 # Assume little-endian IEEE 754 floating point when cross compiling
205 "ac_cv_little_endian_double=yes"
206 "ac_cv_big_endian_double=no"
207 "ac_cv_mixed_endian_double=no"
208 "ac_cv_x87_double_rounding=yes"
209 "ac_cv_tanh_preserves_zero_sign=yes"
210 # Generally assume that things are present and work
211 "ac_cv_posix_semaphores_enabled=yes"
212 "ac_cv_broken_sem_getvalue=no"
213 "ac_cv_wchar_t_signed=yes"
214 "ac_cv_rshift_extends_sign=yes"
215 "ac_cv_broken_nice=no"
216 "ac_cv_broken_poll=no"
217 "ac_cv_working_tzset=yes"
218 "ac_cv_have_long_long_format=yes"
219 "ac_cv_have_size_t_format=yes"
220 "ac_cv_computed_gotos=yes"
221 "ac_cv_file__dev_ptmx=yes"
222 "ac_cv_file__dev_ptc=yes"
223 ]
224 # Never even try to use lchmod on linux,
225 # don't rely on detecting glibc-isms.
226 ++ optional stdenv.hostPlatform.isLinux "ac_cv_func_lchmod=no"
227 ++ optional static "LDFLAGS=-static";
228
229 strictDeps = true;
230 buildInputs =
231 optional (stdenv ? cc && stdenv.cc.libc != null) stdenv.cc.libc ++
232 [ bzip2 openssl zlib ]
233 ++ optional (stdenv.hostPlatform.isCygwin || stdenv.hostPlatform.isAarch64) libffi
234 ++ optional stdenv.hostPlatform.isCygwin expat
235 ++ [ db gdbm ncurses sqlite readline ]
236 ++ optionals x11Support [ tcl tk libX11 ]
237 ++ optional (stdenv.isDarwin && configd != null) configd;
238 nativeBuildInputs =
239 [ autoreconfHook ]
240 ++ optionals (stdenv.hostPlatform != stdenv.buildPlatform)
241 [ buildPackages.stdenv.cc buildPackages.python ];
242
243 mkPaths = paths: {
244 C_INCLUDE_PATH = makeSearchPathOutput "dev" "include" paths;
245 LIBRARY_PATH = makeLibraryPath paths;
246 };
247
248 # Python 2.7 needs this
249 crossCompileEnv = lib.optionalAttrs (stdenv.hostPlatform != stdenv.buildPlatform)
250 { _PYTHON_HOST_PLATFORM = stdenv.hostPlatform.config; };
251
252 # Build the basic Python interpreter without modules that have
253 # external dependencies.
254
255in with passthru; stdenv.mkDerivation ({
256 pname = "python";
257 inherit version;
258
259 inherit src patches buildInputs nativeBuildInputs preConfigure configureFlags;
260
261 LDFLAGS = lib.optionalString (!stdenv.isDarwin) "-lgcc_s";
262 inherit (mkPaths buildInputs) C_INCLUDE_PATH LIBRARY_PATH;
263
264 NIX_CFLAGS_COMPILE = optionalString (stdenv.targetPlatform.system == "x86_64-darwin") "-msse2"
265 + optionalString stdenv.hostPlatform.isMusl " -DTHREAD_STACK_SIZE=0x100000";
266 DETERMINISTIC_BUILD = 1;
267
268 setupHook = python-setup-hook sitePackages;
269
270 postPatch = optionalString (x11Support && (tix != null)) ''
271 substituteInPlace "Lib/lib-tk/Tix.py" --replace "os.environ.get('TIX_LIBRARY')" "os.environ.get('TIX_LIBRARY') or '${tix}/lib'"
272 '';
273
274 postInstall =
275 ''
276 # needed for some packages, especially packages that backport
277 # functionality to 2.x from 3.x
278 for item in $out/lib/${libPrefix}/test/*; do
279 if [[ "$item" != */test_support.py*
280 && "$item" != */test/support
281 && "$item" != */test/regrtest.py* ]]; then
282 rm -rf "$item"
283 else
284 echo $item
285 fi
286 done
287 touch $out/lib/${libPrefix}/test/__init__.py
288 ln -s $out/lib/${libPrefix}/pdb.py $out/bin/pdb
289 ln -s $out/lib/${libPrefix}/pdb.py $out/bin/pdb${sourceVersion.major}.${sourceVersion.minor}
290 ln -s $out/share/man/man1/{python2.7.1.gz,python.1.gz}
291
292 rm "$out"/lib/python*/plat-*/regen # refers to glibc.dev
293
294 # Determinism: Windows installers were not deterministic.
295 # We're also not interested in building Windows installers.
296 find "$out" -name 'wininst*.exe' | xargs -r rm -f
297 '' + optionalString stripBytecode ''
298 # Determinism: deterministic bytecode
299 # First we delete all old bytecode.
300 find $out -name "*.pyc" -delete
301 '' + optionalString rebuildBytecode ''
302 # We build 3 levels of optimized bytecode. Note the default level, without optimizations,
303 # is not reproducible yet. https://bugs.python.org/issue29708
304 # Not creating bytecode will result in a large performance loss however, so we do build it.
305 find $out -name "*.py" | ${pythonForBuildInterpreter} -m compileall -q -f -x "lib2to3" -i -
306 find $out -name "*.py" | ${pythonForBuildInterpreter} -O -m compileall -q -f -x "lib2to3" -i -
307 find $out -name "*.py" | ${pythonForBuildInterpreter} -OO -m compileall -q -f -x "lib2to3" -i -
308 '' + optionalString stdenv.hostPlatform.isCygwin ''
309 cp libpython2.7.dll.a $out/lib
310 '';
311
312 inherit passthru;
313
314 postFixup = ''
315 # Include a sitecustomize.py file. Note it causes an error when it's in postInstall with 2.7.
316 cp ${../../sitecustomize.py} $out/${sitePackages}/sitecustomize.py
317 '';
318
319 enableParallelBuilding = true;
320
321 doCheck = false; # expensive, and fails
322
323 meta = {
324 homepage = "http://python.org";
325 description = "A high-level dynamically-typed programming language";
326 longDescription = ''
327 Python is a remarkably powerful dynamic programming language that
328 is used in a wide variety of application domains. Some of its key
329 distinguishing features include: clear, readable syntax; strong
330 introspection capabilities; intuitive object orientation; natural
331 expression of procedural code; full modularity, supporting
332 hierarchical packages; exception-based error handling; and very
333 high level dynamic data types.
334 '';
335 license = lib.licenses.psfl;
336 platforms = lib.platforms.all;
337 maintainers = with lib.maintainers; [ fridh ];
338 # Higher priority than Python 3.x so that `/bin/python` points to `/bin/python2`
339 # in case both 2 and 3 are installed.
340 priority = -100;
341 };
342 } // crossCompileEnv)