tjh.dev
1{
2 lib,
3 stdenv,
4 replaceVars,
5 fetchurl,
6 zlibSupport ? true,
7 zlib,
8 bzip2,
9 pkg-config,
10 libffi,
11 sqlite,
12 openssl,
13 ncurses,
14 python,
15 expat,
16 tcl,
17 tk,
18 tclPackages,
19 libX11,
20 gdbm,
21 db,
22 xz,
23 python-setup-hook,
24 optimizationLevel ? "jit",
25 boehmgc,
26 # For the Python package set
27 hash,
28 self,
29 packageOverrides ? (self: super: { }),
30 pkgsBuildBuild,
31 pkgsBuildHost,
32 pkgsBuildTarget,
33 pkgsHostHost,
34 pkgsTargetTarget,
35 sourceVersion,
36 pythonVersion,
37 passthruFun,
38 pythonAttr ? "pypy${lib.substring 0 1 pythonVersion}${lib.substring 2 3 pythonVersion}",
39}:
40
41assert zlibSupport -> zlib != null;
42
43let
44 isPy3k = (lib.versions.major pythonVersion) == "3";
45 isPy38OrNewer = lib.versionAtLeast pythonVersion "3.8";
46 isPy39OrNewer = lib.versionAtLeast pythonVersion "3.9";
47 passthru = passthruFun rec {
48 inherit
49 self
50 sourceVersion
51 pythonVersion
52 packageOverrides
53 ;
54 implementation = "pypy";
55 libPrefix = "pypy${pythonVersion}";
56 executable = "pypy${
57 if isPy39OrNewer then lib.versions.majorMinor pythonVersion else lib.optionalString isPy3k "3"
58 }";
59 sitePackages = "${lib.optionalString isPy38OrNewer "lib/${libPrefix}/"}site-packages";
60 hasDistutilsCxxPatch = false;
61 inherit pythonAttr;
62
63 pythonOnBuildForBuild = pkgsBuildBuild.${pythonAttr};
64 pythonOnBuildForHost = pkgsBuildHost.${pythonAttr};
65 pythonOnBuildForTarget = pkgsBuildTarget.${pythonAttr};
66 pythonOnHostForHost = pkgsHostHost.${pythonAttr};
67 pythonOnTargetForTarget = pkgsTargetTarget.${pythonAttr} or { };
68 };
69 pname = passthru.executable;
70 version = with sourceVersion; "${major}.${minor}.${patch}";
71 pythonForPypy = python.withPackages (ppkgs: [ ]);
72
73in
74with passthru;
75stdenv.mkDerivation rec {
76 inherit pname version;
77
78 src = fetchurl {
79 url = "https://downloads.python.org/pypy/pypy${pythonVersion}-v${version}-src.tar.bz2";
80 inherit hash;
81 };
82
83 nativeBuildInputs = [ pkg-config ];
84 buildInputs = [
85 bzip2
86 openssl
87 pythonForPypy
88 libffi
89 ncurses
90 expat
91 sqlite
92 tk
93 tcl
94 libX11
95 gdbm
96 db
97 ]
98 ++ lib.optionals isPy3k [
99 xz
100 ]
101 ++ lib.optionals (stdenv ? cc && stdenv.cc.libc != null) [
102 stdenv.cc.libc
103 ]
104 ++ lib.optionals zlibSupport [
105 zlib
106 ]
107 ++
108 lib.optionals
109 (lib.any (l: l == optimizationLevel) [
110 "0"
111 "1"
112 "2"
113 "3"
114 ])
115 [
116 boehmgc
117 ];
118
119 # Remove bootstrap python from closure
120 dontPatchShebangs = true;
121 disallowedReferences = [ python ];
122
123 # fix compiler error in curses cffi module, where char* != const char*
124 NIX_CFLAGS_COMPILE =
125 if stdenv.cc.isClang then "-Wno-error=incompatible-function-pointer-types" else null;
126 C_INCLUDE_PATH = lib.makeSearchPathOutput "dev" "include" buildInputs;
127 LIBRARY_PATH = lib.makeLibraryPath buildInputs;
128 LD_LIBRARY_PATH = lib.makeLibraryPath (
129 builtins.filter (x: x.outPath != stdenv.cc.libc.outPath or "") buildInputs
130 );
131
132 patches = [
133 ./dont_fetch_vendored_deps.patch
134
135 (replaceVars ./tk_tcl_paths.patch {
136 inherit tk tcl;
137 tk_dev = tk.dev;
138 tcl_dev = tcl;
139 tk_libprefix = tk.libPrefix;
140 tcl_libprefix = tcl.libPrefix;
141 })
142
143 (replaceVars ./sqlite_paths.patch {
144 inherit (sqlite) out dev;
145 })
146 ];
147
148 postPatch = ''
149 substituteInPlace lib_pypy/pypy_tools/build_cffi_imports.py \
150 --replace "multiprocessing.cpu_count()" "$NIX_BUILD_CORES"
151
152 substituteInPlace "lib-python/${if isPy3k then "3/tkinter/tix.py" else "2.7/lib-tk/Tix.py"}" \
153 --replace "os.environ.get('TIX_LIBRARY')" "os.environ.get('TIX_LIBRARY') or '${tclPackages.tix}/lib'"
154 '';
155
156 buildPhase = ''
157 runHook preBuild
158
159 ${pythonForPypy.interpreter} rpython/bin/rpython \
160 --make-jobs="$NIX_BUILD_CORES" \
161 -O${optimizationLevel} \
162 --batch pypy/goal/targetpypystandalone.py
163
164 runHook postBuild
165 '';
166
167 installPhase = ''
168 runHook preInstall
169
170 mkdir -p $out/{bin,include,lib,${executable}-c}
171
172 cp -R {include,lib_pypy,lib-python,${executable}-c} $out/${executable}-c
173 cp lib${executable}-c${stdenv.hostPlatform.extensions.sharedLibrary} $out/lib/
174 ln -s $out/${executable}-c/${executable}-c $out/bin/${executable}
175 ${lib.optionalString isPy39OrNewer "ln -s $out/bin/${executable} $out/bin/pypy3"}
176
177 # other packages expect to find stuff according to libPrefix
178 ln -s $out/${executable}-c/include $out/include/${libPrefix}
179 ln -s $out/${executable}-c/lib-python/${if isPy3k then "3" else pythonVersion} $out/lib/${libPrefix}
180
181 # Include a sitecustomize.py file
182 cp ${../sitecustomize.py} $out/${
183 if isPy38OrNewer then sitePackages else "lib/${libPrefix}/${sitePackages}"
184 }/sitecustomize.py
185
186 runHook postInstall
187 '';
188
189 preFixup =
190 lib.optionalString (stdenv.hostPlatform.isDarwin) ''
191 install_name_tool -change @rpath/lib${executable}-c.dylib $out/lib/lib${executable}-c.dylib $out/bin/${executable}
192 ''
193 + lib.optionalString (stdenv.hostPlatform.isDarwin && stdenv.hostPlatform.isAarch64) ''
194 mkdir -p $out/${executable}-c/pypy/bin
195 mv $out/bin/${executable} $out/${executable}-c/pypy/bin/${executable}
196 ln -s $out/${executable}-c/pypy/bin/${executable} $out/bin/${executable}
197 ''
198 # _testcapi is compiled dynamically, into the store.
199 # This would fail if we don't do it here.
200 + lib.optionalString (stdenv.buildPlatform.canExecute stdenv.hostPlatform) ''
201 pushd /
202 $out/bin/${executable} -c "from test import support"
203 popd
204 '';
205
206 setupHook = python-setup-hook sitePackages;
207
208 # TODO: Investigate why so many tests are failing.
209 checkPhase =
210 let
211 disabledTests = [
212 # disable shutils because it assumes gid 0 exists
213 "test_shutil"
214 # disable socket because it has two actual network tests that fail
215 "test_socket"
216 ]
217 ++ lib.optionals (!isPy3k) [
218 # disable test_urllib2net, test_urllib2_localnet, and test_urllibnet because they require networking (example.com)
219 "test_urllib2net"
220 "test_urllibnet"
221 "test_urllib2_localnet"
222 # test_subclass fails with "internal error"
223 # test_load_default_certs_env fails for unknown reason
224 "test_ssl"
225 ]
226 ++ lib.optionals isPy3k [
227 # disable asyncio due to https://github.com/NixOS/nix/issues/1238
228 "test_asyncio"
229 # disable os due to https://github.com/NixOS/nixpkgs/issues/10496
230 "test_os"
231 # disable pathlib due to https://bitbucket.org/pypy/pypy/pull-requests/594
232 "test_pathlib"
233 # disable tarfile because it assumes gid 0 exists
234 "test_tarfile"
235 # disable __all__ because of spurious imp/importlib warning and
236 # warning-to-error test policy
237 "test___all__"
238 # fail for multiple reasons, TODO: investigate
239 "test__opcode"
240 "test_ast"
241 "test_audit"
242 "test_builtin"
243 "test_c_locale_coercion"
244 "test_call"
245 "test_class"
246 "test_cmd_line"
247 "test_cmd_line_script"
248 "test_code"
249 "test_code_module"
250 "test_codeop"
251 "test_compile"
252 "test_coroutines"
253 "test_cprofile"
254 "test_ctypes"
255 "test_embed"
256 "test_exceptions"
257 "test_extcall"
258 "test_frame"
259 "test_generators"
260 "test_grammar"
261 "test_idle"
262 "test_iter"
263 "test_itertools"
264 "test_list"
265 "test_marshal"
266 "test_memoryio"
267 "test_memoryview"
268 "test_metaclass"
269 "test_mmap"
270 "test_multibytecodec"
271 "test_opcache"
272 "test_pdb"
273 "test_peepholer"
274 "test_positional_only_arg"
275 "test_print"
276 "test_property"
277 "test_pyclbr"
278 "test_range"
279 "test_re"
280 "test_readline"
281 "test_regrtest"
282 "test_repl"
283 "test_rlcompleter"
284 "test_signal"
285 "test_sort"
286 "test_source_encoding"
287 "test_ssl"
288 "test_string_literals"
289 "test_structseq"
290 "test_subprocess"
291 "test_super"
292 "test_support"
293 "test_syntax"
294 "test_sys"
295 "test_sys_settrace"
296 "test_tcl"
297 "test_termios"
298 "test_threading"
299 "test_trace"
300 "test_tty"
301 "test_unpack_ex"
302 "test_utf8_mode"
303 "test_weakref"
304 "test_capi"
305 "test_concurrent_futures"
306 "test_dataclasses"
307 "test_doctest"
308 "test_future_stmt"
309 "test_importlib"
310 "test_inspect"
311 "test_pydoc"
312 "test_warnings"
313 ]
314 ++ lib.optionals isPy310 [
315 "test_contextlib_async"
316 "test_future"
317 "test_lzma"
318 "test_module"
319 "test_typing"
320 ];
321 in
322 ''
323 export TERMINFO="${ncurses.out}/share/terminfo/";
324 export TERM="xterm";
325 export HOME="$TMPDIR";
326
327 ${pythonForPypy.interpreter} ./pypy/test_all.py --pypy=./${executable}-c -k 'not (${lib.concatStringsSep " or " disabledTests})' lib-python
328 '';
329
330 # verify cffi modules
331 doInstallCheck = true;
332 installCheckPhase =
333 let
334 modules = [
335 "curses"
336 "sqlite3"
337 ]
338 ++ lib.optionals (!isPy3k) [
339 "Tkinter"
340 ]
341 ++ lib.optionals isPy3k [
342 "tkinter"
343 "lzma"
344 ];
345 imports = lib.concatMapStringsSep "; " (x: "import ${x}") modules;
346 in
347 ''
348 echo "Testing whether we can import modules"
349 $out/bin/${executable} -c '${imports}'
350 '';
351
352 inherit passthru;
353 enableParallelBuilding = true; # almost no parallelization without STM
354
355 meta = with lib; {
356 homepage = "https://www.pypy.org/";
357 changelog = "https://doc.pypy.org/en/stable/release-v${version}.html";
358 description = "Fast, compliant alternative implementation of the Python language (${pythonVersion})";
359 mainProgram = "pypy";
360 license = licenses.mit;
361 platforms = [
362 "aarch64-linux"
363 "x86_64-linux"
364 "aarch64-darwin"
365 "x86_64-darwin"
366 ];
367 broken = optimizationLevel == "0"; # generates invalid code
368 maintainers = with maintainers; [
369 andersk
370 fliegendewurst
371 ];
372 };
373}