tjh.dev
1{
2 lib,
3 stdenv,
4 replaceVars,
5 fetchurl,
6 zlibSupport ? true,
7 zlib,
8 bzip2,
9 pkg-config,
10 libffi,
11 sqlite,
12 openssl,
13 ncurses,
14 python,
15 expat,
16 tcl,
17 tk,
18 tclPackages,
19 libX11,
20 gdbm,
21 db,
22 xz,
23 python-setup-hook,
24 optimizationLevel ? "jit",
25 boehmgc,
26 # For the Python package set
27 hash,
28 self,
29 packageOverrides ? (self: super: { }),
30 pkgsBuildBuild,
31 pkgsBuildHost,
32 pkgsBuildTarget,
33 pkgsHostHost,
34 pkgsTargetTarget,
35 sourceVersion,
36 pythonVersion,
37 passthruFun,
38 pythonAttr ? "pypy${lib.substring 0 1 pythonVersion}${lib.substring 2 3 pythonVersion}",
39}:
40
41assert zlibSupport -> zlib != null;
42
43let
44 isPy3k = (lib.versions.major pythonVersion) == "3";
45 isPy38OrNewer = lib.versionAtLeast pythonVersion "3.8";
46 isPy39OrNewer = lib.versionAtLeast pythonVersion "3.9";
47 passthru = passthruFun rec {
48 inherit
49 self
50 sourceVersion
51 pythonVersion
52 packageOverrides
53 ;
54 implementation = "pypy";
55 libPrefix = "pypy${pythonVersion}";
56 executable = "pypy${
57 if isPy39OrNewer then lib.versions.majorMinor pythonVersion else lib.optionalString isPy3k "3"
58 }";
59 sitePackages = "${lib.optionalString isPy38OrNewer "lib/${libPrefix}/"}site-packages";
60 hasDistutilsCxxPatch = false;
61 inherit pythonAttr;
62
63 pythonOnBuildForBuild = pkgsBuildBuild.${pythonAttr};
64 pythonOnBuildForHost = pkgsBuildHost.${pythonAttr};
65 pythonOnBuildForTarget = pkgsBuildTarget.${pythonAttr};
66 pythonOnHostForHost = pkgsHostHost.${pythonAttr};
67 pythonOnTargetForTarget = pkgsTargetTarget.${pythonAttr} or { };
68 };
69 pname = passthru.executable;
70 version = with sourceVersion; "${major}.${minor}.${patch}";
71 pythonForPypy = python.withPackages (ppkgs: [ ]);
72
73in
74with passthru;
75stdenv.mkDerivation rec {
76 inherit pname version;
77
78 src = fetchurl {
79 url = "https://downloads.python.org/pypy/pypy${pythonVersion}-v${version}-src.tar.bz2";
80 inherit hash;
81 };
82
83 nativeBuildInputs = [ pkg-config ];
84 buildInputs =
85 [
86 bzip2
87 openssl
88 pythonForPypy
89 libffi
90 ncurses
91 expat
92 sqlite
93 tk
94 tcl
95 libX11
96 gdbm
97 db
98 ]
99 ++ lib.optionals isPy3k [
100 xz
101 ]
102 ++ lib.optionals (stdenv ? cc && stdenv.cc.libc != null) [
103 stdenv.cc.libc
104 ]
105 ++ lib.optionals zlibSupport [
106 zlib
107 ]
108 ++
109 lib.optionals
110 (lib.any (l: l == optimizationLevel) [
111 "0"
112 "1"
113 "2"
114 "3"
115 ])
116 [
117 boehmgc
118 ];
119
120 # Remove bootstrap python from closure
121 dontPatchShebangs = true;
122 disallowedReferences = [ python ];
123
124 # fix compiler error in curses cffi module, where char* != const char*
125 NIX_CFLAGS_COMPILE =
126 if stdenv.cc.isClang then "-Wno-error=incompatible-function-pointer-types" else null;
127 C_INCLUDE_PATH = lib.makeSearchPathOutput "dev" "include" buildInputs;
128 LIBRARY_PATH = lib.makeLibraryPath buildInputs;
129 LD_LIBRARY_PATH = lib.makeLibraryPath (
130 builtins.filter (x: x.outPath != stdenv.cc.libc.outPath or "") buildInputs
131 );
132
133 patches = [
134 ./dont_fetch_vendored_deps.patch
135
136 (replaceVars ./tk_tcl_paths.patch {
137 inherit tk tcl;
138 tk_dev = tk.dev;
139 tcl_dev = tcl;
140 tk_libprefix = tk.libPrefix;
141 tcl_libprefix = tcl.libPrefix;
142 })
143
144 (replaceVars ./sqlite_paths.patch {
145 inherit (sqlite) out dev;
146 })
147 ];
148
149 postPatch = ''
150 substituteInPlace lib_pypy/pypy_tools/build_cffi_imports.py \
151 --replace "multiprocessing.cpu_count()" "$NIX_BUILD_CORES"
152
153 substituteInPlace "lib-python/${if isPy3k then "3/tkinter/tix.py" else "2.7/lib-tk/Tix.py"}" \
154 --replace "os.environ.get('TIX_LIBRARY')" "os.environ.get('TIX_LIBRARY') or '${tclPackages.tix}/lib'"
155 '';
156
157 buildPhase = ''
158 runHook preBuild
159
160 ${pythonForPypy.interpreter} rpython/bin/rpython \
161 --make-jobs="$NIX_BUILD_CORES" \
162 -O${optimizationLevel} \
163 --batch pypy/goal/targetpypystandalone.py
164
165 runHook postBuild
166 '';
167
168 installPhase = ''
169 runHook preInstall
170
171 mkdir -p $out/{bin,include,lib,${executable}-c}
172
173 cp -R {include,lib_pypy,lib-python,${executable}-c} $out/${executable}-c
174 cp lib${executable}-c${stdenv.hostPlatform.extensions.sharedLibrary} $out/lib/
175 ln -s $out/${executable}-c/${executable}-c $out/bin/${executable}
176 ${lib.optionalString isPy39OrNewer "ln -s $out/bin/${executable} $out/bin/pypy3"}
177
178 # other packages expect to find stuff according to libPrefix
179 ln -s $out/${executable}-c/include $out/include/${libPrefix}
180 ln -s $out/${executable}-c/lib-python/${if isPy3k then "3" else pythonVersion} $out/lib/${libPrefix}
181
182 # Include a sitecustomize.py file
183 cp ${../sitecustomize.py} $out/${
184 if isPy38OrNewer then sitePackages else "lib/${libPrefix}/${sitePackages}"
185 }/sitecustomize.py
186
187 runHook postInstall
188 '';
189
190 preFixup =
191 lib.optionalString (stdenv.hostPlatform.isDarwin) ''
192 install_name_tool -change @rpath/lib${executable}-c.dylib $out/lib/lib${executable}-c.dylib $out/bin/${executable}
193 ''
194 + lib.optionalString (stdenv.hostPlatform.isDarwin && stdenv.hostPlatform.isAarch64) ''
195 mkdir -p $out/${executable}-c/pypy/bin
196 mv $out/bin/${executable} $out/${executable}-c/pypy/bin/${executable}
197 ln -s $out/${executable}-c/pypy/bin/${executable} $out/bin/${executable}
198 ''
199 # _testcapi is compiled dynamically, into the store.
200 # This would fail if we don't do it here.
201 + lib.optionalString (stdenv.buildPlatform.canExecute stdenv.hostPlatform) ''
202 pushd /
203 $out/bin/${executable} -c "from test import support"
204 popd
205 '';
206
207 setupHook = python-setup-hook sitePackages;
208
209 # TODO: Investigate why so many tests are failing.
210 checkPhase =
211 let
212 disabledTests =
213 [
214 # disable shutils because it assumes gid 0 exists
215 "test_shutil"
216 # disable socket because it has two actual network tests that fail
217 "test_socket"
218 ]
219 ++ lib.optionals (!isPy3k) [
220 # disable test_urllib2net, test_urllib2_localnet, and test_urllibnet because they require networking (example.com)
221 "test_urllib2net"
222 "test_urllibnet"
223 "test_urllib2_localnet"
224 # test_subclass fails with "internal error"
225 # test_load_default_certs_env fails for unknown reason
226 "test_ssl"
227 ]
228 ++ lib.optionals isPy3k [
229 # disable asyncio due to https://github.com/NixOS/nix/issues/1238
230 "test_asyncio"
231 # disable os due to https://github.com/NixOS/nixpkgs/issues/10496
232 "test_os"
233 # disable pathlib due to https://bitbucket.org/pypy/pypy/pull-requests/594
234 "test_pathlib"
235 # disable tarfile because it assumes gid 0 exists
236 "test_tarfile"
237 # disable __all__ because of spurious imp/importlib warning and
238 # warning-to-error test policy
239 "test___all__"
240 # fail for multiple reasons, TODO: investigate
241 "test__opcode"
242 "test_ast"
243 "test_audit"
244 "test_builtin"
245 "test_c_locale_coercion"
246 "test_call"
247 "test_class"
248 "test_cmd_line"
249 "test_cmd_line_script"
250 "test_code"
251 "test_code_module"
252 "test_codeop"
253 "test_compile"
254 "test_coroutines"
255 "test_cprofile"
256 "test_ctypes"
257 "test_embed"
258 "test_exceptions"
259 "test_extcall"
260 "test_frame"
261 "test_generators"
262 "test_grammar"
263 "test_idle"
264 "test_iter"
265 "test_itertools"
266 "test_list"
267 "test_marshal"
268 "test_memoryio"
269 "test_memoryview"
270 "test_metaclass"
271 "test_mmap"
272 "test_multibytecodec"
273 "test_opcache"
274 "test_pdb"
275 "test_peepholer"
276 "test_positional_only_arg"
277 "test_print"
278 "test_property"
279 "test_pyclbr"
280 "test_range"
281 "test_re"
282 "test_readline"
283 "test_regrtest"
284 "test_repl"
285 "test_rlcompleter"
286 "test_signal"
287 "test_sort"
288 "test_source_encoding"
289 "test_ssl"
290 "test_string_literals"
291 "test_structseq"
292 "test_subprocess"
293 "test_super"
294 "test_support"
295 "test_syntax"
296 "test_sys"
297 "test_sys_settrace"
298 "test_tcl"
299 "test_termios"
300 "test_threading"
301 "test_trace"
302 "test_tty"
303 "test_unpack_ex"
304 "test_utf8_mode"
305 "test_weakref"
306 "test_capi"
307 "test_concurrent_futures"
308 "test_dataclasses"
309 "test_doctest"
310 "test_future_stmt"
311 "test_importlib"
312 "test_inspect"
313 "test_pydoc"
314 "test_warnings"
315 ]
316 ++ lib.optionals isPy310 [
317 "test_contextlib_async"
318 "test_future"
319 "test_lzma"
320 "test_module"
321 "test_typing"
322 ];
323 in
324 ''
325 export TERMINFO="${ncurses.out}/share/terminfo/";
326 export TERM="xterm";
327 export HOME="$TMPDIR";
328
329 ${pythonForPypy.interpreter} ./pypy/test_all.py --pypy=./${executable}-c -k 'not (${lib.concatStringsSep " or " disabledTests})' lib-python
330 '';
331
332 # verify cffi modules
333 doInstallCheck = true;
334 installCheckPhase =
335 let
336 modules =
337 [
338 "curses"
339 "sqlite3"
340 ]
341 ++ lib.optionals (!isPy3k) [
342 "Tkinter"
343 ]
344 ++ lib.optionals isPy3k [
345 "tkinter"
346 "lzma"
347 ];
348 imports = lib.concatMapStringsSep "; " (x: "import ${x}") modules;
349 in
350 ''
351 echo "Testing whether we can import modules"
352 $out/bin/${executable} -c '${imports}'
353 '';
354
355 inherit passthru;
356 enableParallelBuilding = true; # almost no parallelization without STM
357
358 meta = with lib; {
359 homepage = "https://www.pypy.org/";
360 changelog = "https://doc.pypy.org/en/stable/release-v${version}.html";
361 description = "Fast, compliant alternative implementation of the Python language (${pythonVersion})";
362 mainProgram = "pypy";
363 license = licenses.mit;
364 platforms = [
365 "aarch64-linux"
366 "x86_64-linux"
367 "aarch64-darwin"
368 "x86_64-darwin"
369 ];
370 broken = optimizationLevel == "0"; # generates invalid code
371 maintainers = with maintainers; [
372 andersk
373 fliegendewurst
374 ];
375 };
376}