pyrox.dev
1{ lib, stdenv, fetchFromGitHub, fetchpatch
2, bzip2
3, expat
4, libffi
5, gdbm
6, db
7, ncurses
8, openssl
9, readline
10, sqlite
11, tcl ? null, tk ? null, tix ? null, libX11 ? null, x11Support ? false
12, zlib
13, self
14, configd, coreutils
15, autoreconfHook
16, python-setup-hook
17# Some proprietary libs assume UCS2 unicode, especially on darwin :(
18, ucsEncoding ? 4
19# For the Python package set
20, packageOverrides ? (self: super: {})
21, pkgsBuildBuild
22, pkgsBuildHost
23, pkgsBuildTarget
24, pkgsHostHost
25, pkgsTargetTarget
26, sourceVersion
27, hash
28, passthruFun
29, static ? stdenv.hostPlatform.isStatic
30, stripBytecode ? reproducibleBuild
31, rebuildBytecode ? true
32, reproducibleBuild ? false
33, enableOptimizations ? false
34, strip2to3 ? false
35, stripConfig ? false
36, stripIdlelib ? false
37, stripTests ? false
38, pythonAttr ? "python${sourceVersion.major}${sourceVersion.minor}"
39}:
40
41assert x11Support -> tcl != null
42 && tk != null
43 && libX11 != null;
44
45assert lib.assertMsg (enableOptimizations -> (!stdenv.cc.isClang))
46 "Optimizations with clang are not supported. configure: error: llvm-profdata is required for a --enable-optimizations build but could not be found.";
47
48assert lib.assertMsg (reproducibleBuild -> stripBytecode)
49 "Deterministic builds require stripping bytecode.";
50
51assert lib.assertMsg (reproducibleBuild -> (!enableOptimizations))
52 "Deterministic builds are not achieved when optimizations are enabled.";
53
54assert lib.assertMsg (reproducibleBuild -> (!rebuildBytecode))
55 "Deterministic builds are not achieved when (default unoptimized) bytecode is created.";
56
57let
58 buildPackages = pkgsBuildHost;
59 inherit (passthru) pythonOnBuildForHost;
60
61 pythonOnBuildForHostInterpreter = if stdenv.hostPlatform == stdenv.buildPlatform then
62 "$out/bin/python"
63 else pythonOnBuildForHost.interpreter;
64
65 passthru = passthruFun rec {
66 inherit self sourceVersion packageOverrides;
67 implementation = "cpython";
68 libPrefix = "python${pythonVersion}";
69 executable = libPrefix;
70 pythonVersion = with sourceVersion; "${major}.${minor}";
71 sitePackages = "lib/${libPrefix}/site-packages";
72 inherit hasDistutilsCxxPatch pythonAttr;
73 pythonOnBuildForBuild = pkgsBuildBuild.${pythonAttr};
74 pythonOnBuildForHost = pkgsBuildHost.${pythonAttr};
75 pythonOnBuildForTarget = pkgsBuildTarget.${pythonAttr};
76 pythonOnHostForHost = pkgsHostHost.${pythonAttr};
77 pythonOnTargetForTarget = pkgsTargetTarget.${pythonAttr} or {};
78 } // {
79 inherit ucsEncoding;
80 };
81
82 version = with sourceVersion; "${major}.${minor}.${patch}${suffix}";
83
84 # ActiveState is a fork of cpython that includes fixes for security
85 # issues after its EOL
86 src = fetchFromGitHub {
87 owner = "ActiveState";
88 repo = "cpython";
89 rev = "v${version}";
90 inherit hash;
91 };
92
93 hasDistutilsCxxPatch = !(stdenv.cc.isGNU or false);
94 patches =
95 [ # Look in C_INCLUDE_PATH and LIBRARY_PATH for stuff.
96 ./search-path.patch
97
98 # Python recompiles a Python if the mtime stored *in* the
99 # pyc/pyo file differs from the mtime of the source file. This
100 # doesn't work in Nix because Nix changes the mtime of files in
101 # the Nix store to 1. So treat that as a special case.
102 ./nix-store-mtime.patch
103
104 # patch python to put zero timestamp into pyc
105 # if DETERMINISTIC_BUILD env var is set
106 ./deterministic-build.patch
107
108 # Fix python bug #27177 (https://bugs.python.org/issue27177)
109 # The issue is that `match.group` only recognizes python integers
110 # instead of everything that has `__index__`.
111 # This bug was fixed upstream, but not backported to 2.7
112 (fetchpatch {
113 name = "re_match_index.patch";
114 url = "https://bugs.python.org/file43084/re_match_index.patch";
115 sha256 = "0l9rw6r5r90iybdkp3hhl2pf0h0s1izc68h5d3ywrm92pq32wz57";
116 })
117
118 # Fix race-condition during pyc creation. Has a slight backwards
119 # incompatible effect: pyc symlinks will now be overridden
120 # (https://bugs.python.org/issue17222). Included in python >= 3.4,
121 # backported in debian since 2013.
122 # https://bugs.python.org/issue13146
123 ./atomic_pyc.patch
124
125 # Backport from CPython 3.8 of a good list of tests to run for PGO.
126 ./profile-task.patch
127
128 # The workaround is for unittests on Win64, which we don't support.
129 # It does break aarch64-darwin, which we do support. See:
130 # * https://bugs.python.org/issue35523
131 # * https://github.com/python/cpython/commit/e6b247c8e524
132 ../3.7/no-win64-workaround.patch
133
134 # fix openssl detection by reverting irrelevant change for us, to enable hashlib which is required by pip
135 (fetchpatch {
136 url = "https://github.com/ActiveState/cpython/pull/35/commits/20ea5b46aaf1e7bdf9d6905ba8bece2cc73b05b0.patch";
137 revert = true;
138 hash = "sha256-Lp5fGlcfJJ6p6vKmcLckJiAA2AZz4prjFE0aMEJxotw=";
139 })
140 ] ++ lib.optionals (x11Support && stdenv.isDarwin) [
141 ./use-correct-tcl-tk-on-darwin.patch
142
143 ] ++ lib.optionals stdenv.isDarwin [
144 # Fix darwin build https://bugs.python.org/issue34027
145 ../3.7/darwin-libutil.patch
146
147 ] ++ lib.optionals stdenv.isLinux [
148
149 # Disable the use of ldconfig in ctypes.util.find_library (since
150 # ldconfig doesn't work on NixOS), and don't use
151 # ctypes.util.find_library during the loading of the uuid module
152 # (since it will do a futile invocation of gcc (!) to find
153 # libuuid, slowing down program startup a lot).
154 ./no-ldconfig.patch
155
156 # Fix ctypes.util.find_library with gcc10.
157 ./find_library-gcc10.patch
158
159 ] ++ lib.optionals stdenv.hostPlatform.isCygwin [
160 ./2.5.2-ctypes-util-find_library.patch
161 ./2.5.2-tkinter-x11.patch
162 ./2.6.2-ssl-threads.patch
163 ./2.6.5-export-PySignal_SetWakeupFd.patch
164 ./2.6.5-FD_SETSIZE.patch
165 ./2.6.5-ncurses-abi6.patch
166 ./2.7.3-dbm.patch
167 ./2.7.3-dylib.patch
168 ./2.7.3-getpath-exe-extension.patch
169 ./2.7.3-no-libm.patch
170 ] ++ lib.optionals hasDistutilsCxxPatch [
171
172 # Patch from http://bugs.python.org/issue1222585 adapted to work with
173 # `patch -p1' and with a last hunk removed
174 # Upstream distutils is calling C compiler to compile C++ code, which
175 # only works for GCC and Apple Clang. This makes distutils to call C++
176 # compiler when needed.
177 ./python-2.7-distutils-C++.patch
178 ] ++ lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [
179 ./cross-compile.patch
180 ];
181
182 preConfigure = ''
183 # Purity.
184 for i in /usr /sw /opt /pkg; do
185 substituteInPlace ./setup.py --replace $i /no-such-path
186 done
187 '' + lib.optionalString (stdenv ? cc && stdenv.cc.libc != null) ''
188 for i in Lib/plat-*/regen; do
189 substituteInPlace $i --replace /usr/include/ ${stdenv.cc.libc}/include/
190 done
191 '' + lib.optionalString stdenv.isDarwin ''
192 substituteInPlace configure --replace '`/usr/bin/arch`' '"i386"'
193 substituteInPlace Lib/multiprocessing/__init__.py \
194 --replace 'os.popen(comm)' 'os.popen("${coreutils}/bin/nproc")'
195 '';
196
197 configureFlags = lib.optionals enableOptimizations [
198 "--enable-optimizations"
199 ] ++ lib.optionals (!static) [
200 "--enable-shared"
201 ] ++ [
202 "--with-threads"
203 "--with-system-ffi"
204 "--with-system-expat"
205 "--enable-unicode=ucs${toString ucsEncoding}"
206 ] ++ lib.optionals stdenv.hostPlatform.isCygwin [
207 "ac_cv_func_bind_textdomain_codeset=yes"
208 ] ++ lib.optionals stdenv.isDarwin [
209 "--disable-toolbox-glue"
210 ] ++ lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [
211 "PYTHON_FOR_BUILD=${lib.getBin buildPackages.python}/bin/python"
212 "ac_cv_buggy_getaddrinfo=no"
213 # Assume little-endian IEEE 754 floating point when cross compiling
214 "ac_cv_little_endian_double=yes"
215 "ac_cv_big_endian_double=no"
216 "ac_cv_mixed_endian_double=no"
217 "ac_cv_x87_double_rounding=yes"
218 "ac_cv_tanh_preserves_zero_sign=yes"
219 # Generally assume that things are present and work
220 "ac_cv_posix_semaphores_enabled=yes"
221 "ac_cv_broken_sem_getvalue=no"
222 "ac_cv_wchar_t_signed=yes"
223 "ac_cv_rshift_extends_sign=yes"
224 "ac_cv_broken_nice=no"
225 "ac_cv_broken_poll=no"
226 "ac_cv_working_tzset=yes"
227 "ac_cv_have_long_long_format=yes"
228 "ac_cv_have_size_t_format=yes"
229 "ac_cv_computed_gotos=yes"
230 "ac_cv_file__dev_ptmx=yes"
231 "ac_cv_file__dev_ptc=yes"
232 ]
233 # Never even try to use lchmod on linux,
234 # don't rely on detecting glibc-isms.
235 ++ lib.optional stdenv.hostPlatform.isLinux "ac_cv_func_lchmod=no"
236 ++ lib.optional static "LDFLAGS=-static";
237
238 strictDeps = true;
239 buildInputs =
240 lib.optional (stdenv ? cc && stdenv.cc.libc != null) stdenv.cc.libc ++
241 [ bzip2 openssl zlib libffi expat db gdbm ncurses sqlite readline ]
242 ++ lib.optionals x11Support [ tcl tk libX11 ]
243 ++ lib.optional (stdenv.isDarwin && configd != null) configd;
244 nativeBuildInputs =
245 [ autoreconfHook ]
246 ++ lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform)
247 [ buildPackages.stdenv.cc buildPackages.python ];
248
249 mkPaths = paths: {
250 C_INCLUDE_PATH = lib.makeSearchPathOutput "dev" "include" paths;
251 LIBRARY_PATH = lib.makeLibraryPath paths;
252 };
253
254 # Python 2.7 needs this
255 crossCompileEnv = lib.optionalAttrs (stdenv.hostPlatform != stdenv.buildPlatform)
256 { _PYTHON_HOST_PLATFORM = stdenv.hostPlatform.config; };
257
258 # Build the basic Python interpreter without modules that have
259 # external dependencies.
260
261in with passthru; stdenv.mkDerivation ({
262 pname = "python";
263 inherit version;
264
265 inherit src patches buildInputs nativeBuildInputs preConfigure configureFlags;
266
267 LDFLAGS = lib.optionalString (!stdenv.isDarwin) "-lgcc_s";
268 inherit (mkPaths buildInputs) C_INCLUDE_PATH LIBRARY_PATH;
269
270 env.NIX_CFLAGS_COMPILE = lib.optionalString (stdenv.targetPlatform.system == "x86_64-darwin") "-msse2"
271 + lib.optionalString stdenv.hostPlatform.isMusl " -DTHREAD_STACK_SIZE=0x100000";
272 DETERMINISTIC_BUILD = 1;
273
274 setupHook = python-setup-hook sitePackages;
275
276 postPatch = lib.optionalString (x11Support && (tix != null)) ''
277 substituteInPlace "Lib/lib-tk/Tix.py" --replace "os.environ.get('TIX_LIBRARY')" "os.environ.get('TIX_LIBRARY') or '${tix}/lib'"
278 '';
279
280 postInstall =
281 ''
282 # needed for some packages, especially packages that backport
283 # functionality to 2.x from 3.x
284 for item in $out/lib/${libPrefix}/test/*; do
285 if [[ "$item" != */test_support.py*
286 && "$item" != */test/support
287 && "$item" != */test/regrtest.py* ]]; then
288 rm -rf "$item"
289 else
290 echo $item
291 fi
292 done
293 touch $out/lib/${libPrefix}/test/__init__.py
294 ln -s $out/lib/${libPrefix}/pdb.py $out/bin/pdb
295 ln -s $out/lib/${libPrefix}/pdb.py $out/bin/pdb${sourceVersion.major}.${sourceVersion.minor}
296 ln -s $out/share/man/man1/{python2.7.1.gz,python.1.gz}
297
298 rm "$out"/lib/python*/plat-*/regen # refers to glibc.dev
299
300 # Determinism: Windows installers were not deterministic.
301 # We're also not interested in building Windows installers.
302 find "$out" -name 'wininst*.exe' | xargs -r rm -f
303 '' + lib.optionalString stripBytecode ''
304 # Determinism: deterministic bytecode
305 # First we delete all old bytecode.
306 find $out -name "*.pyc" -delete
307 '' + lib.optionalString rebuildBytecode ''
308 # We build 3 levels of optimized bytecode. Note the default level, without optimizations,
309 # is not reproducible yet. https://bugs.python.org/issue29708
310 # Not creating bytecode will result in a large performance loss however, so we do build it.
311 find $out -name "*.py" | ${pythonOnBuildForHostInterpreter} -m compileall -q -f -x "lib2to3" -i -
312 find $out -name "*.py" | ${pythonOnBuildForHostInterpreter} -O -m compileall -q -f -x "lib2to3" -i -
313 find $out -name "*.py" | ${pythonOnBuildForHostInterpreter} -OO -m compileall -q -f -x "lib2to3" -i -
314 '' + lib.optionalString stdenv.hostPlatform.isCygwin ''
315 cp libpython2.7.dll.a $out/lib
316 '';
317
318 inherit passthru;
319
320 postFixup = ''
321 # Include a sitecustomize.py file. Note it causes an error when it's in postInstall with 2.7.
322 cp ${../../sitecustomize.py} $out/${sitePackages}/sitecustomize.py
323 '' + lib.optionalString strip2to3 ''
324 rm -R $out/bin/2to3 $out/lib/python*/lib2to3
325 '' + lib.optionalString stripConfig ''
326 rm -R $out/bin/python*-config $out/lib/python*/config*
327 '' + lib.optionalString stripIdlelib ''
328 # Strip IDLE
329 rm -R $out/bin/idle* $out/lib/python*/idlelib
330 '' + lib.optionalString stripTests ''
331 # Strip tests
332 rm -R $out/lib/python*/test $out/lib/python*/**/test{,s}
333 '';
334
335 enableParallelBuilding = true;
336
337 doCheck = false; # expensive, and fails
338
339 meta = {
340 homepage = "http://python.org";
341 description = "A high-level dynamically-typed programming language";
342 longDescription = ''
343 Python is a remarkably powerful dynamic programming language that
344 is used in a wide variety of application domains. Some of its key
345 distinguishing features include: clear, readable syntax; strong
346 introspection capabilities; intuitive object orientation; natural
347 expression of procedural code; full modularity, supporting
348 hierarchical packages; exception-based error handling; and very
349 high level dynamic data types.
350 '';
351 license = lib.licenses.psfl;
352 platforms = lib.platforms.all;
353 knownVulnerabilities = [
354 "Python 2.7 has reached its end of life after 2020-01-01. See https://www.python.org/doc/sunset-python-2/."
355 # Quote: That means that we will not improve it anymore after that day,
356 # even if someone finds a security problem in it. You should upgrade to
357 # Python 3 as soon as you can. [..] So, in 2008, we announced that we
358 # would sunset Python 2 in 2015, and asked people to upgrade before
359 # then. Some did, but many did not. So, in 2014, we extended that
360 # sunset till 2020.
361 ];
362 };
363 } // crossCompileEnv)