pyrox.dev
1{
2 lib,
3 stdenv,
4 replaceVars,
5 fetchurl,
6 autoconf,
7 zlibSupport ? true,
8 zlib,
9 bzip2,
10 pkg-config,
11 lndir,
12 libffi,
13 sqlite,
14 openssl,
15 ncurses,
16 python,
17 expat,
18 tcl,
19 tk,
20 tclPackages,
21 libX11,
22 gdbm,
23 db,
24 xz,
25 python-setup-hook,
26 optimizationLevel ? "jit",
27 boehmgc,
28 # For the Python package set
29 hash,
30 self,
31 packageOverrides ? (self: super: { }),
32 pkgsBuildBuild,
33 pkgsBuildHost,
34 pkgsBuildTarget,
35 pkgsHostHost,
36 pkgsTargetTarget,
37 sourceVersion,
38 pythonVersion,
39 passthruFun,
40 pythonAttr ? "pypy${lib.substring 0 1 pythonVersion}${lib.substring 2 3 pythonVersion}",
41}:
42
43assert zlibSupport -> zlib != null;
44
45let
46 isPy3k = (lib.versions.major pythonVersion) == "3";
47 isPy38OrNewer = lib.versionAtLeast pythonVersion "3.8";
48 isPy39OrNewer = lib.versionAtLeast pythonVersion "3.9";
49 passthru = passthruFun rec {
50 inherit
51 self
52 sourceVersion
53 pythonVersion
54 packageOverrides
55 ;
56 implementation = "pypy";
57 libPrefix = "pypy${pythonVersion}";
58 executable = "pypy${
59 if isPy39OrNewer then lib.versions.majorMinor pythonVersion else lib.optionalString isPy3k "3"
60 }";
61 sitePackages = "${lib.optionalString isPy38OrNewer "lib/${libPrefix}/"}site-packages";
62 hasDistutilsCxxPatch = false;
63 inherit pythonAttr;
64
65 pythonOnBuildForBuild = pkgsBuildBuild.${pythonAttr};
66 pythonOnBuildForHost = pkgsBuildHost.${pythonAttr};
67 pythonOnBuildForTarget = pkgsBuildTarget.${pythonAttr};
68 pythonOnHostForHost = pkgsHostHost.${pythonAttr};
69 pythonOnTargetForTarget = pkgsTargetTarget.${pythonAttr} or { };
70 };
71 pname = passthru.executable;
72 version = with sourceVersion; "${major}.${minor}.${patch}";
73 pythonForPypy = python.withPackages (ppkgs: [ ]);
74
75in
76with passthru;
77stdenv.mkDerivation rec {
78 inherit pname version;
79
80 src = fetchurl {
81 url = "https://downloads.python.org/pypy/pypy${pythonVersion}-v${version}-src.tar.bz2";
82 inherit hash;
83 };
84
85 nativeBuildInputs = [
86 pkg-config
87 lndir
88 ];
89 buildInputs = [
90 bzip2
91 openssl
92 pythonForPypy
93 libffi
94 ncurses
95 expat
96 sqlite
97 tk
98 tcl
99 libX11
100 gdbm
101 db
102 ]
103 ++ lib.optionals isPy3k [
104 xz
105 ]
106 ++ lib.optionals (stdenv ? cc && stdenv.cc.libc != null) [
107 stdenv.cc.libc
108 ]
109 ++ lib.optionals zlibSupport [
110 zlib
111 ]
112 ++
113 lib.optionals
114 (lib.any (l: l == optimizationLevel) [
115 "0"
116 "1"
117 "2"
118 "3"
119 ])
120 [
121 boehmgc
122 ];
123
124 # Remove bootstrap python from closure
125 dontPatchShebangs = true;
126 disallowedReferences = [ python ];
127
128 env =
129 lib.optionalAttrs stdenv.cc.isClang {
130 # fix compiler error in curses cffi module, where char* != const char*
131 NIX_CFLAGS_COMPILE = "-Wno-error=incompatible-function-pointer-types";
132 }
133 // {
134 C_INCLUDE_PATH = lib.makeSearchPathOutput "dev" "include" buildInputs;
135 LIBRARY_PATH = lib.makeLibraryPath buildInputs;
136 LD_LIBRARY_PATH = lib.makeLibraryPath (
137 builtins.filter (x: x.outPath != stdenv.cc.libc.outPath or "") buildInputs
138 );
139 };
140
141 patches = [
142 ./dont_fetch_vendored_deps.patch
143
144 (replaceVars ./tk_tcl_paths.patch {
145 inherit tk tcl;
146 tk_dev = tk.dev;
147 tcl_dev = tcl;
148 tk_libprefix = tk.libPrefix;
149 tcl_libprefix = tcl.libPrefix;
150 })
151
152 # Python ctypes.util uses three different strategies to find a library (on Linux):
153 # 1. /sbin/ldconfig
154 # 2. cc -Wl,-t -l"$libname"; objdump -p
155 # 3. ld -t (where it attaches the values in $LD_LIBRARY_PATH as -L arguments)
156 # The first is disabled in Nix (and wouldn't work in the build sandbox or on NixOS anyway), and
157 # the third was only introduced in Python 3.6 (see bugs.python.org/issue9998), so is not
158 # available when building PyPy (which is built using Python/PyPy 2.7).
159 # The second requires SONAME to be set for the dynamic library for the second part not to fail.
160 # As libsqlite3 stopped shipping with SONAME after the switch to autosetup (>= 3.50 in Nixpkgs;
161 # see https://www.sqlite.org/src/forumpost/5a3b44f510df8ded). This makes the Python CFFI module
162 # unable to find the SQLite library.
163 # To circumvent these issues, we hardcode the path during build.
164 # For more information, see https://github.com/NixOS/nixpkgs/issues/419942.
165 (replaceVars (if isPy3k then ./sqlite_paths.patch else ./sqlite_paths_2_7.patch) {
166 inherit (sqlite) out dev;
167 libsqlite = "${sqlite.out}/lib/libsqlite3${stdenv.hostPlatform.extensions.sharedLibrary}";
168 })
169 ];
170
171 postPatch = ''
172 substituteInPlace lib_pypy/pypy_tools/build_cffi_imports.py \
173 --replace "multiprocessing.cpu_count()" "$NIX_BUILD_CORES"
174
175 substituteInPlace "lib-python/${if isPy3k then "3/tkinter/tix.py" else "2.7/lib-tk/Tix.py"}" \
176 --replace "os.environ.get('TIX_LIBRARY')" "os.environ.get('TIX_LIBRARY') or '${tclPackages.tix}/lib'"
177 '';
178
179 buildPhase = ''
180 runHook preBuild
181
182 ${pythonForPypy.interpreter} rpython/bin/rpython \
183 --make-jobs="$NIX_BUILD_CORES" \
184 -O${optimizationLevel} \
185 --batch \
186 pypy/goal/targetpypystandalone.py \
187 ${lib.optionalString ((toString optimizationLevel) == "1") "--withoutmod-cpyext"}
188
189 runHook postBuild
190 '';
191
192 installPhase = ''
193 runHook preInstall
194
195 mkdir -p $out/{bin,lib/${libPrefix}}
196
197 cp -R {include,lib_pypy,lib-python} $out
198 install -Dm755 lib${executable}-c${stdenv.hostPlatform.extensions.sharedLibrary} $out/lib/
199 install -Dm755 ${executable}-c $out/bin/${executable}
200 ${lib.optionalString isPy39OrNewer "ln -s $out/bin/${executable} $out/bin/pypy3"}
201
202 # other packages expect to find stuff according to libPrefix
203 ln -s $out/include $out/include/${libPrefix}
204 lndir $out/lib-python/${if isPy3k then "3" else pythonVersion} $out/lib/${libPrefix}
205 lndir $out/lib_pypy $out/lib/${libPrefix}
206
207 # Include a sitecustomize.py file
208 cp ${../sitecustomize.py} $out/${
209 if isPy38OrNewer then sitePackages else "lib/${libPrefix}/${sitePackages}"
210 }/sitecustomize.py
211
212 runHook postInstall
213 '';
214
215 preFixup =
216 lib.optionalString (stdenv.hostPlatform.isDarwin) ''
217 install_name_tool -change @rpath/lib${executable}-c.dylib $out/lib/lib${executable}-c.dylib $out/bin/${executable}
218 ''
219 # Create platform specific _sysconfigdata__*.py (eg: _sysconfigdata__linux_x86_64-linux-gnu.py)
220 # Can be tested by building: pypy3Packages.bcrypt
221 # Based on the upstream build code found here:
222 # https://github.com/pypy/pypy/blob/release-pypy3.11-v7.3.20/pypy/tool/release/package.py#L176-L189
223 # Upstream is not shipping config.guess, just take one from autoconf
224 + lib.optionalString isPy3k ''
225 $out/bin/pypy3 -m sysconfig --generate-posix-vars HOST_GNU_TYPE "$(${autoconf}/share/autoconf/build-aux/config.guess)"
226 buildir="$(cat pybuilddir.txt)"
227 quadruplet=$(ls $buildir | sed -E 's/_sysconfigdata__(.*).py/\1/')
228 cp "$buildir/_sysconfigdata__$quadruplet.py" $out/lib_pypy/
229 ln -rs "$out/lib_pypy/_sysconfigdata__$quadruplet.py" $out/lib/pypy*/
230 ''
231 # _testcapi is compiled dynamically, into the store.
232 # This would fail if we don't do it here.
233 + lib.optionalString (stdenv.buildPlatform.canExecute stdenv.hostPlatform) ''
234 pushd /
235 $out/bin/${executable} -c "from test import support"
236 popd
237 '';
238
239 setupHook = python-setup-hook sitePackages;
240
241 # TODO: Investigate why so many tests are failing.
242 checkPhase =
243 let
244 disabledTests = [
245 # disable shutils because it assumes gid 0 exists
246 "test_shutil"
247 # disable socket because it has two actual network tests that fail
248 "test_socket"
249 ]
250 ++ lib.optionals (!isPy3k) [
251 # disable test_urllib2net, test_urllib2_localnet, and test_urllibnet because they require networking (example.com)
252 "test_urllib2net"
253 "test_urllibnet"
254 "test_urllib2_localnet"
255 # test_subclass fails with "internal error"
256 # test_load_default_certs_env fails for unknown reason
257 "test_ssl"
258 ]
259 ++ lib.optionals isPy3k [
260 # disable asyncio due to https://github.com/NixOS/nix/issues/1238
261 "test_asyncio"
262 # disable os due to https://github.com/NixOS/nixpkgs/issues/10496
263 "test_os"
264 # disable pathlib due to https://bitbucket.org/pypy/pypy/pull-requests/594
265 "test_pathlib"
266 # disable tarfile because it assumes gid 0 exists
267 "test_tarfile"
268 # disable __all__ because of spurious imp/importlib warning and
269 # warning-to-error test policy
270 "test___all__"
271 # fail for multiple reasons, TODO: investigate
272 "test__opcode"
273 "test_ast"
274 "test_audit"
275 "test_builtin"
276 "test_c_locale_coercion"
277 "test_call"
278 "test_class"
279 "test_cmd_line"
280 "test_cmd_line_script"
281 "test_code"
282 "test_code_module"
283 "test_codeop"
284 "test_compile"
285 "test_coroutines"
286 "test_cprofile"
287 "test_ctypes"
288 "test_embed"
289 "test_exceptions"
290 "test_extcall"
291 "test_frame"
292 "test_generators"
293 "test_grammar"
294 "test_idle"
295 "test_iter"
296 "test_itertools"
297 "test_list"
298 "test_marshal"
299 "test_memoryio"
300 "test_memoryview"
301 "test_metaclass"
302 "test_mmap"
303 "test_multibytecodec"
304 "test_opcache"
305 "test_pdb"
306 "test_peepholer"
307 "test_positional_only_arg"
308 "test_print"
309 "test_property"
310 "test_pyclbr"
311 "test_range"
312 "test_re"
313 "test_readline"
314 "test_regrtest"
315 "test_repl"
316 "test_rlcompleter"
317 "test_signal"
318 "test_sort"
319 "test_source_encoding"
320 "test_ssl"
321 "test_string_literals"
322 "test_structseq"
323 "test_subprocess"
324 "test_super"
325 "test_support"
326 "test_syntax"
327 "test_sys"
328 "test_sys_settrace"
329 "test_tcl"
330 "test_termios"
331 "test_threading"
332 "test_trace"
333 "test_tty"
334 "test_unpack_ex"
335 "test_utf8_mode"
336 "test_weakref"
337 "test_capi"
338 "test_concurrent_futures"
339 "test_dataclasses"
340 "test_doctest"
341 "test_future_stmt"
342 "test_importlib"
343 "test_inspect"
344 "test_pydoc"
345 "test_warnings"
346 ]
347 ++ lib.optionals isPy310 [
348 "test_contextlib_async"
349 "test_future"
350 "test_lzma"
351 "test_module"
352 "test_typing"
353 ];
354 in
355 ''
356 export TERMINFO="${ncurses.out}/share/terminfo/";
357 export TERM="xterm";
358 export HOME="$TMPDIR";
359
360 ${pythonForPypy.interpreter} ./pypy/test_all.py --pypy=./${executable}-c -k 'not (${lib.concatStringsSep " or " disabledTests})' lib-python
361 '';
362
363 # verify cffi modules
364 doInstallCheck = true;
365 installCheckPhase =
366 let
367 modules = [
368 "curses"
369 "sqlite3"
370 ]
371 ++ lib.optionals (!isPy3k) [
372 "Tkinter"
373 ]
374 ++ lib.optionals isPy3k [
375 "tkinter"
376 "lzma"
377 ];
378 imports = lib.concatMapStringsSep "; " (x: "import ${x}") modules;
379 in
380 ''
381 echo "Testing whether we can import modules"
382 $out/bin/${executable} -c '${imports}'
383 '';
384
385 inherit passthru;
386 enableParallelBuilding = true; # almost no parallelization without STM
387
388 meta = with lib; {
389 homepage = "https://www.pypy.org/";
390 changelog = "https://doc.pypy.org/en/stable/release-v${version}.html";
391 description = "Fast, compliant alternative implementation of the Python language (${pythonVersion})";
392 mainProgram = "pypy";
393 license = licenses.mit;
394 platforms = [
395 "aarch64-linux"
396 "x86_64-linux"
397 "aarch64-darwin"
398 "x86_64-darwin"
399 ];
400 broken = optimizationLevel == "0"; # generates invalid code
401 maintainers = with maintainers; [
402 andersk
403 fliegendewurst
404 ];
405 };
406}