tjh.dev
nixpkgs mirror (for testing)
github.com/NixOS/nixpkgs
nix
1{
2 lib,
3 stdenv,
4 replaceVars,
5 fetchurl,
6 autoconf,
7 zlibSupport ? true,
8 zlib,
9 bzip2,
10 pkg-config,
11 lndir,
12 libffi,
13 sqlite,
14 openssl,
15 ncurses,
16 python,
17 expat,
18 tcl,
19 tk,
20 tclPackages,
21 libX11,
22 gdbm,
23 db,
24 xz,
25 python-setup-hook,
26 optimizationLevel ? "jit",
27 boehmgc,
28 # For the Python package set
29 hash,
30 self,
31 packageOverrides ? (self: super: { }),
32 pkgsBuildBuild,
33 pkgsBuildHost,
34 pkgsBuildTarget,
35 pkgsHostHost,
36 pkgsTargetTarget,
37 sourceVersion,
38 pythonVersion,
39 passthruFun,
40 pythonAttr ? "pypy${lib.substring 0 1 pythonVersion}${lib.substring 2 3 pythonVersion}",
41}:
42
43assert zlibSupport -> zlib != null;
44
45let
46 isPy3k = (lib.versions.major pythonVersion) == "3";
47 isPy38OrNewer = lib.versionAtLeast pythonVersion "3.8";
48 isPy39OrNewer = lib.versionAtLeast pythonVersion "3.9";
49 passthru = passthruFun rec {
50 inherit
51 self
52 sourceVersion
53 pythonVersion
54 packageOverrides
55 ;
56 implementation = "pypy";
57 libPrefix = "pypy${pythonVersion}";
58 executable = "pypy${
59 if isPy39OrNewer then lib.versions.majorMinor pythonVersion else lib.optionalString isPy3k "3"
60 }";
61 sitePackages = "${lib.optionalString isPy38OrNewer "lib/${libPrefix}/"}site-packages";
62 hasDistutilsCxxPatch = false;
63 inherit pythonAttr;
64
65 pythonOnBuildForBuild = pkgsBuildBuild.${pythonAttr};
66 pythonOnBuildForHost = pkgsBuildHost.${pythonAttr};
67 pythonOnBuildForTarget = pkgsBuildTarget.${pythonAttr};
68 pythonOnHostForHost = pkgsHostHost.${pythonAttr};
69 pythonOnTargetForTarget = pkgsTargetTarget.${pythonAttr} or { };
70
71 pythonABITags = [
72 "none"
73 "pypy${lib.concatStrings (lib.take 2 (lib.splitString "." pythonVersion))}_pp${sourceVersion.major}${sourceVersion.minor}"
74 ];
75 };
76 pname = passthru.executable;
77 version = with sourceVersion; "${major}.${minor}.${patch}";
78 pythonForPypy = python.withPackages (ppkgs: [ ]);
79
80in
81with passthru;
82stdenv.mkDerivation rec {
83 inherit pname version;
84
85 src = fetchurl {
86 url = "https://downloads.python.org/pypy/pypy${pythonVersion}-v${version}-src.tar.bz2";
87 inherit hash;
88 };
89
90 nativeBuildInputs = [
91 pkg-config
92 lndir
93 ];
94 buildInputs = [
95 bzip2
96 openssl
97 pythonForPypy
98 libffi
99 ncurses
100 expat
101 sqlite
102 tk
103 tcl
104 libX11
105 gdbm
106 db
107 ]
108 ++ lib.optionals isPy3k [
109 xz
110 ]
111 ++ lib.optionals (stdenv ? cc && stdenv.cc.libc != null) [
112 stdenv.cc.libc
113 ]
114 ++ lib.optionals zlibSupport [
115 zlib
116 ]
117 ++
118 lib.optionals
119 (lib.elem optimizationLevel [
120 "0"
121 "1"
122 "2"
123 "3"
124 ])
125 [
126 boehmgc
127 ];
128
129 # Remove bootstrap python from closure
130 dontPatchShebangs = true;
131 disallowedReferences = [ python ];
132
133 env =
134 lib.optionalAttrs stdenv.cc.isClang {
135 # fix compiler error in curses cffi module, where char* != const char*
136 NIX_CFLAGS_COMPILE = "-Wno-error=incompatible-function-pointer-types";
137 }
138 // {
139 C_INCLUDE_PATH = lib.makeSearchPathOutput "dev" "include" buildInputs;
140 LIBRARY_PATH = lib.makeLibraryPath buildInputs;
141 LD_LIBRARY_PATH = lib.makeLibraryPath (
142 builtins.filter (x: x.outPath != stdenv.cc.libc.outPath or "") buildInputs
143 );
144 };
145
146 patches = [
147 ./dont_fetch_vendored_deps.patch
148
149 (replaceVars ./tk_tcl_paths.patch {
150 inherit tk tcl;
151 tk_dev = tk.dev;
152 tcl_dev = tcl;
153 tk_libprefix = tk.libPrefix;
154 tcl_libprefix = tcl.libPrefix;
155 })
156
157 # Python ctypes.util uses three different strategies to find a library (on Linux):
158 # 1. /sbin/ldconfig
159 # 2. cc -Wl,-t -l"$libname"; objdump -p
160 # 3. ld -t (where it attaches the values in $LD_LIBRARY_PATH as -L arguments)
161 # The first is disabled in Nix (and wouldn't work in the build sandbox or on NixOS anyway), and
162 # the third was only introduced in Python 3.6 (see bugs.python.org/issue9998), so is not
163 # available when building PyPy (which is built using Python/PyPy 2.7).
164 # The second requires SONAME to be set for the dynamic library for the second part not to fail.
165 # As libsqlite3 stopped shipping with SONAME after the switch to autosetup (>= 3.50 in Nixpkgs;
166 # see https://www.sqlite.org/src/forumpost/5a3b44f510df8ded). This makes the Python CFFI module
167 # unable to find the SQLite library.
168 # To circumvent these issues, we hardcode the path during build.
169 # For more information, see https://github.com/NixOS/nixpkgs/issues/419942.
170 (replaceVars (if isPy3k then ./sqlite_paths.patch else ./sqlite_paths_2_7.patch) {
171 inherit (sqlite) out dev;
172 libsqlite = "${sqlite.out}/lib/libsqlite3${stdenv.hostPlatform.extensions.sharedLibrary}";
173 })
174
175 # PyPy sets an explicit minimum SDK version for darwin that is much older
176 # than what we default to on nixpkgs.
177 # Simply removing the explicit flag makes it use our default instead.
178 ./darwin_version_min.patch
179 ];
180
181 postPatch = ''
182 substituteInPlace lib_pypy/pypy_tools/build_cffi_imports.py \
183 --replace "multiprocessing.cpu_count()" "$NIX_BUILD_CORES"
184
185 substituteInPlace "lib-python/${if isPy3k then "3/tkinter/tix.py" else "2.7/lib-tk/Tix.py"}" \
186 --replace "os.environ.get('TIX_LIBRARY')" "os.environ.get('TIX_LIBRARY') or '${tclPackages.tix}/lib'"
187 '';
188
189 buildPhase = ''
190 runHook preBuild
191
192 ${pythonForPypy.interpreter} rpython/bin/rpython \
193 --make-jobs="$NIX_BUILD_CORES" \
194 -O${optimizationLevel} \
195 --batch \
196 pypy/goal/targetpypystandalone.py \
197 ${lib.optionalString ((toString optimizationLevel) == "1") "--withoutmod-cpyext"}
198
199 runHook postBuild
200 '';
201
202 installPhase = ''
203 runHook preInstall
204
205 mkdir -p $out/{bin,lib/${libPrefix}}
206
207 cp -R {include,lib_pypy,lib-python} $out
208 install -Dm755 lib${executable}-c${stdenv.hostPlatform.extensions.sharedLibrary} $out/lib/
209 install -Dm755 ${executable}-c $out/bin/${executable}
210 ${lib.optionalString isPy39OrNewer "ln -s $out/bin/${executable} $out/bin/pypy3"}
211
212 # other packages expect to find stuff according to libPrefix
213 ln -s $out/include $out/include/${libPrefix}
214 lndir $out/lib-python/${if isPy3k then "3" else pythonVersion} $out/lib/${libPrefix}
215 lndir $out/lib_pypy $out/lib/${libPrefix}
216
217 # Include a sitecustomize.py file
218 cp ${../sitecustomize.py} $out/${
219 if isPy38OrNewer then sitePackages else "lib/${libPrefix}/${sitePackages}"
220 }/sitecustomize.py
221
222 runHook postInstall
223 '';
224
225 preFixup =
226 lib.optionalString (stdenv.hostPlatform.isDarwin) ''
227 install_name_tool -change @rpath/lib${executable}-c.dylib $out/lib/lib${executable}-c.dylib $out/bin/${executable}
228 ''
229 # Create platform specific _sysconfigdata__*.py (eg: _sysconfigdata__linux_x86_64-linux-gnu.py)
230 # Can be tested by building: pypy3Packages.bcrypt
231 # Based on the upstream build code found here:
232 # https://github.com/pypy/pypy/blob/release-pypy3.11-v7.3.20/pypy/tool/release/package.py#L176-L189
233 # Upstream is not shipping config.guess, just take one from autoconf
234 + lib.optionalString isPy3k ''
235 $out/bin/pypy3 -m sysconfig --generate-posix-vars HOST_GNU_TYPE "$(${autoconf}/share/autoconf/build-aux/config.guess)"
236 buildir="$(cat pybuilddir.txt)"
237 quadruplet=$(ls $buildir | sed -E 's/_sysconfigdata__(.*).py/\1/')
238 cp "$buildir/_sysconfigdata__$quadruplet.py" $out/lib_pypy/
239 ln -rs "$out/lib_pypy/_sysconfigdata__$quadruplet.py" $out/lib/pypy*/
240 ''
241 # _testcapi is compiled dynamically, into the store.
242 # This would fail if we don't do it here.
243 + lib.optionalString (stdenv.buildPlatform.canExecute stdenv.hostPlatform) ''
244 pushd /
245 $out/bin/${executable} -c "from test import support"
246 popd
247 '';
248
249 setupHook = python-setup-hook sitePackages;
250
251 # TODO: Investigate why so many tests are failing.
252 checkPhase =
253 let
254 disabledTests = [
255 # disable shutils because it assumes gid 0 exists
256 "test_shutil"
257 # disable socket because it has two actual network tests that fail
258 "test_socket"
259 ]
260 ++ lib.optionals (!isPy3k) [
261 # disable test_urllib2net, test_urllib2_localnet, and test_urllibnet because they require networking (example.com)
262 "test_urllib2net"
263 "test_urllibnet"
264 "test_urllib2_localnet"
265 # test_subclass fails with "internal error"
266 # test_load_default_certs_env fails for unknown reason
267 "test_ssl"
268 ]
269 ++ lib.optionals isPy3k [
270 # disable asyncio due to https://github.com/NixOS/nix/issues/1238
271 "test_asyncio"
272 # disable os due to https://github.com/NixOS/nixpkgs/issues/10496
273 "test_os"
274 # disable pathlib due to https://bitbucket.org/pypy/pypy/pull-requests/594
275 "test_pathlib"
276 # disable tarfile because it assumes gid 0 exists
277 "test_tarfile"
278 # disable __all__ because of spurious imp/importlib warning and
279 # warning-to-error test policy
280 "test___all__"
281 # fail for multiple reasons, TODO: investigate
282 "test__opcode"
283 "test_ast"
284 "test_audit"
285 "test_builtin"
286 "test_c_locale_coercion"
287 "test_call"
288 "test_class"
289 "test_cmd_line"
290 "test_cmd_line_script"
291 "test_code"
292 "test_code_module"
293 "test_codeop"
294 "test_compile"
295 "test_coroutines"
296 "test_cprofile"
297 "test_ctypes"
298 "test_embed"
299 "test_exceptions"
300 "test_extcall"
301 "test_frame"
302 "test_generators"
303 "test_grammar"
304 "test_idle"
305 "test_iter"
306 "test_itertools"
307 "test_list"
308 "test_marshal"
309 "test_memoryio"
310 "test_memoryview"
311 "test_metaclass"
312 "test_mmap"
313 "test_multibytecodec"
314 "test_opcache"
315 "test_pdb"
316 "test_peepholer"
317 "test_positional_only_arg"
318 "test_print"
319 "test_property"
320 "test_pyclbr"
321 "test_range"
322 "test_re"
323 "test_readline"
324 "test_regrtest"
325 "test_repl"
326 "test_rlcompleter"
327 "test_signal"
328 "test_sort"
329 "test_source_encoding"
330 "test_ssl"
331 "test_string_literals"
332 "test_structseq"
333 "test_subprocess"
334 "test_super"
335 "test_support"
336 "test_syntax"
337 "test_sys"
338 "test_sys_settrace"
339 "test_tcl"
340 "test_termios"
341 "test_threading"
342 "test_trace"
343 "test_tty"
344 "test_unpack_ex"
345 "test_utf8_mode"
346 "test_weakref"
347 "test_capi"
348 "test_concurrent_futures"
349 "test_dataclasses"
350 "test_doctest"
351 "test_future_stmt"
352 "test_importlib"
353 "test_inspect"
354 "test_pydoc"
355 "test_warnings"
356 ]
357 ++ lib.optionals isPy310 [
358 "test_contextlib_async"
359 "test_future"
360 "test_lzma"
361 "test_module"
362 "test_typing"
363 ];
364 in
365 ''
366 export TERMINFO="${ncurses.out}/share/terminfo/";
367 export TERM="xterm";
368 export HOME="$TMPDIR";
369
370 ${pythonForPypy.interpreter} ./pypy/test_all.py --pypy=./${executable}-c -k 'not (${lib.concatStringsSep " or " disabledTests})' lib-python
371 '';
372
373 # verify cffi modules
374 doInstallCheck = true;
375 installCheckPhase =
376 let
377 modules = [
378 "curses"
379 "sqlite3"
380 ]
381 ++ lib.optionals (!isPy3k) [
382 "Tkinter"
383 ]
384 ++ lib.optionals isPy3k [
385 "tkinter"
386 "lzma"
387 ];
388 imports = lib.concatMapStringsSep "; " (x: "import ${x}") modules;
389 in
390 ''
391 echo "Testing whether we can import modules"
392 $out/bin/${executable} -c '${imports}'
393 '';
394
395 inherit passthru;
396 enableParallelBuilding = true; # almost no parallelization without STM
397
398 meta = {
399 homepage = "https://www.pypy.org/";
400 changelog = "https://doc.pypy.org/en/stable/release-v${version}.html";
401 description = "Fast, compliant alternative implementation of the Python language (${pythonVersion})";
402 mainProgram = "pypy";
403 license = lib.licenses.mit;
404 platforms = [
405 "aarch64-linux"
406 "x86_64-linux"
407 "aarch64-darwin"
408 "x86_64-darwin"
409 ];
410 broken = optimizationLevel == "0"; # generates invalid code
411 maintainers = with lib.maintainers; [
412 andersk
413 fliegendewurst
414 ];
415 };
416}