pyrox.dev
1{ lib, stdenv
2, cmake
3, coreutils
4, glibc
5, gccForLibs
6, which
7, perl
8, libedit
9, ninja
10, pkg-config
11, sqlite
12, libxml2
13, clang_13
14, python3
15, ncurses
16, libuuid
17, libxcrypt
18, icu
19, libgcc
20, libblocksruntime
21, curl
22, rsync
23, git
24, libgit2
25, fetchFromGitHub
26, makeWrapper
27, gnumake
28, file
29}:
30
31let
32 # The Swift toolchain script builds projects with separate repos. By convention, some of them share
33 # the same version with the main Swift compiler project per release. We fetch these with
34 # `fetchSwiftRelease`. The rest have their own versions locked to each Swift release, as defined in the
35 # Swift compiler repo:
36 # utils/update_checkout/update_checkout-config.json.
37 #
38 # ... among projects listed in that file, we provide our own:
39 # - CMake
40 # - ninja
41 # - icu
42 #
43 # ... we'd like to include the following in the future:
44 # - stress-tester
45 # - integration-tests
46
47 versions = {
48 swift = "5.6.2";
49 yams = "4.0.2";
50 argumentParser = "1.0.3";
51 format = "release/5.6";
52 crypto = "1.1.5";
53 nio = "2.31.2";
54 nio-ssl = "2.15.0";
55 };
56
57 fetchAppleRepo = { repo, rev, sha256 }:
58 fetchFromGitHub {
59 owner = "apple";
60 inherit repo rev sha256;
61 name = "${repo}-${rev}-src";
62 };
63
64 fetchSwiftRelease = { repo, sha256, fetchSubmodules ? false }:
65 fetchFromGitHub {
66 owner = "apple";
67 inherit repo sha256 fetchSubmodules;
68 rev = "swift-${versions.swift}-RELEASE";
69 name = "${repo}-${versions.swift}-src";
70 };
71
72 sources = {
73 # Projects that share `versions.swift` for each release.
74
75 swift = fetchSwiftRelease {
76 repo = "swift";
77 sha256 = "sha256-wiRXAXWEksJuy+YQQ+B7tzr2iLkSVkgV6o+wIz7yKJA=";
78 };
79 cmark = fetchSwiftRelease {
80 repo = "swift-cmark";
81 sha256 = "sha256-f0BoTs4HYdx/aJ9HIGCWMalhl8PvClWD6R4QK3qSgAw=";
82 };
83 llbuild = fetchSwiftRelease {
84 repo = "swift-llbuild";
85 sha256 = "sha256-SQ6V0zVshIYMjayx+ZpYuLijgQ89tqRnPlXBPf2FYqM=";
86 };
87 driver = fetchSwiftRelease {
88 repo = "swift-driver";
89 sha256 = "sha256-D5/C4Rbv5KIsKpy6YbuMxGIGaQkn80PD4Cp0l6bPKzY=";
90 };
91 toolsSupportCore = fetchSwiftRelease {
92 repo = "swift-tools-support-core";
93 sha256 = "sha256-FbtQCq1sSlzrskCrgzD4iYuo5eGaXrAUUxoNX/BiOfg=";
94 };
95 swiftpm = fetchSwiftRelease {
96 repo = "swift-package-manager";
97 sha256 = "sha256-esO4Swz3UYngbVgxoV+fkhSC0AU3IaxVjWkgK/s3x68=";
98 };
99 syntax = fetchSwiftRelease {
100 repo = "swift-syntax";
101 sha256 = "sha256-C9FPCtq49BvKXtTWWeReYWNrU70pHzT2DhAv3NiTbPU=";
102 };
103 corelibsXctest = fetchSwiftRelease {
104 repo = "swift-corelibs-xctest";
105 sha256 = "sha256-0hizfnKJaUUA+jXuXzXWk72FmlSyc+UGEf7BTLdJrx4=";
106 };
107 corelibsFoundation = fetchSwiftRelease {
108 repo = "swift-corelibs-foundation";
109 sha256 = "sha256-8sCL8Ia6yb6bRsJZ52gUJH0jN3lwClM573G8jgUdEhw=";
110 };
111 corelibsLibdispatch = fetchSwiftRelease {
112 repo = "swift-corelibs-libdispatch";
113 sha256 = "sha256-1tIskUMnfblnvZaFDQPUMBfWTmBYG98s7rEww7PwZO8=";
114 fetchSubmodules = true;
115 };
116 indexstoreDb = fetchSwiftRelease {
117 repo = "indexstore-db";
118 sha256 = "sha256-/PO4eMiASZN3pjFjBQ1r8vYwGRn6xm3SWaB2HDZlkPs=";
119 };
120 sourcekitLsp = fetchSwiftRelease {
121 repo = "sourcekit-lsp";
122 sha256 = "sha256-ttgUC4ZHD3P/xLHllEbACtHVrJ6HXqeVWccXcoPMkts=";
123 };
124 llvmProject = fetchSwiftRelease {
125 repo = "llvm-project";
126 sha256 = "sha256-YVs3lKV2RlaovpYkdGO+vzypolrmXmbKBBP4+osNMYw=";
127 };
128 docc = fetchSwiftRelease {
129 repo = "swift-docc";
130 sha256 = "sha256-rWiaNamZoHTO1bKpubxuT7m1IBOl7amT5M71mNauilY=";
131 };
132 docc-render-artifact = fetchSwiftRelease {
133 repo = "swift-docc-render-artifact";
134 sha256 = "sha256-AX+rtDLhq8drk7N6/hoH3fQioudmmTCnEhR45bME8uU=";
135 };
136 docc-symbolkit = fetchSwiftRelease {
137 repo = "swift-docc-symbolkit";
138 sha256 = "sha256-Xy1TQ5ucDW+MnkeOvVznsATBmwcQ3p1x+ofQ22ofk+o=";
139 };
140 lmdb = fetchSwiftRelease {
141 repo = "swift-lmdb";
142 sha256 = "sha256-i2GkWRWq1W5j8rF4PiHwWgT4Dur5FCY2o44HvUU3vtQ=";
143 };
144 markdown = fetchSwiftRelease {
145 repo = "swift-markdown";
146 sha256 = "sha256-XtFSBiNHhmULjS4OqSpMgUetLu3peRg7l6HpjwVsTj8=";
147 };
148
149 cmark-gfm = fetchAppleRepo {
150 repo = "swift-cmark";
151 rev = "swift-${versions.swift}-RELEASE-gfm";
152 sha256 = "sha256-g28iKmMR2W0r1urf8Fk1HBxAp5OlonNYSVN3Ril66tQ=";
153 };
154
155 # Projects that have their own versions during each release
156
157 argumentParser = fetchAppleRepo {
158 repo = "swift-argument-parser";
159 rev = "${versions.argumentParser}";
160 sha256 = "sha256-vNqkuAwSZNCWvwe6E5BqbXQdIbmIia0dENmmSQ9P8Mo=";
161 };
162 format = fetchAppleRepo {
163 repo = "swift-format";
164 rev = "${versions.format}";
165 sha256 = "sha256-1f5sIrv9IbPB7Vnahq1VwH8gT41dcjWldRwvVEaMdto=";
166 };
167 crypto = fetchAppleRepo {
168 repo = "swift-crypto";
169 rev = "${versions.crypto}";
170 sha256 = "sha256-jwxXQuOF+CnpLMwTZ2z52Fgx2b97yWzXiPTx0Ye8KCQ=";
171 };
172 nio = fetchAppleRepo {
173 repo = "swift-nio";
174 rev = versions.nio;
175 sha256 = "sha256-FscOA/S7on31QCR/MZFjg4ZB3FGJ+rdptZ6MRZJXexE=";
176 };
177 nio-ssl = fetchAppleRepo {
178 repo = "swift-nio-ssl";
179 rev = versions.nio-ssl;
180 sha256 = "sha256-5QGkmkCOXhG3uOdf0bd3Fo1MFekB8/WcveBXGhtVZKo=";
181 };
182 yams = fetchFromGitHub {
183 owner = "jpsim";
184 repo = "Yams";
185 rev = versions.yams;
186 sha256 = "sha256-cTkCAwxxLc35laOon1ZXXV8eAxX02oDolJyPauhZado=";
187 name = "Yams-${versions.yams}-src";
188 };
189 };
190
191 devInputs = [
192 curl
193 glibc
194 icu
195 libblocksruntime
196 libedit
197 libgcc
198 libuuid
199 libxcrypt
200 libxml2
201 ncurses
202 sqlite
203 ];
204
205 python = (python3.withPackages (ps: [ps.six]));
206
207 cmakeFlags = [
208 "-DGLIBC_INCLUDE_PATH=${stdenv.cc.libc.dev}/include"
209 "-DC_INCLUDE_DIRS=${lib.makeSearchPathOutput "dev" "include" devInputs}:${libxml2.dev}/include/libxml2"
210 "-DGCC_INSTALL_PREFIX=${gccForLibs}"
211 ];
212
213in
214stdenv.mkDerivation {
215 pname = "swift";
216 version = versions.swift;
217
218 nativeBuildInputs = [
219 cmake
220 git
221 makeWrapper
222 ninja
223 perl
224 pkg-config
225 python
226 rsync
227 which
228 ];
229 buildInputs = devInputs ++ [
230 clang_13
231 ];
232
233 # TODO: Revisit what needs to be propagated and how.
234 propagatedBuildInputs = [
235 libgcc
236 libgit2
237 python
238 ];
239 propagatedUserEnvPkgs = [ git pkg-config ];
240
241 hardeningDisable = [ "format" ]; # for LLDB
242
243 unpackPhase = ''
244 mkdir src
245 cd src
246 export SWIFT_SOURCE_ROOT=$PWD
247
248 cp -r ${sources.swift} swift
249 cp -r ${sources.cmark} cmark
250 cp -r ${sources.llbuild} llbuild
251 cp -r ${sources.argumentParser} swift-argument-parser
252 cp -r ${sources.driver} swift-driver
253 cp -r ${sources.toolsSupportCore} swift-tools-support-core
254 cp -r ${sources.swiftpm} swiftpm
255 cp -r ${sources.syntax} swift-syntax
256 cp -r ${sources.corelibsXctest} swift-corelibs-xctest
257 cp -r ${sources.corelibsFoundation} swift-corelibs-foundation
258 cp -r ${sources.corelibsLibdispatch} swift-corelibs-libdispatch
259 cp -r ${sources.yams} yams
260 cp -r ${sources.indexstoreDb} indexstore-db
261 cp -r ${sources.sourcekitLsp} sourcekit-lsp
262 cp -r ${sources.format} swift-format
263 cp -r ${sources.crypto} swift-crypto
264 cp -r ${sources.llvmProject} llvm-project
265 cp -r ${sources.cmark-gfm} swift-cmark-gfm
266 cp -r ${sources.docc} swift-docc
267 cp -r ${sources.docc-render-artifact} swift-docc-render-artifact
268 cp -r ${sources.docc-symbolkit} swift-docc-symbolkit
269 cp -r ${sources.lmdb} swift-lmdb
270 cp -r ${sources.markdown} swift-markdown
271 cp -r ${sources.nio} swift-nio
272 cp -r ${sources.nio-ssl} swift-nio-ssl
273
274 chmod -R u+w .
275 '';
276
277 patchPhase = ''
278 # Just patch all the things for now, we can focus this later.
279 patchShebangs $SWIFT_SOURCE_ROOT
280
281 # TODO: eliminate use of env.
282 find -type f -print0 | xargs -0 sed -i \
283 -e 's|/usr/bin/env|${coreutils}/bin/env|g' \
284 -e 's|/usr/bin/make|${gnumake}/bin/make|g' \
285 -e 's|/bin/mkdir|${coreutils}/bin/mkdir|g' \
286 -e 's|/bin/cp|${coreutils}/bin/cp|g' \
287 -e 's|/usr/bin/file|${file}/bin/file|g'
288
289 # Build configuration patches.
290 patch -p1 -d swift -i ${./patches/0001-build-presets-linux-don-t-require-using-Ninja.patch}
291 patch -p1 -d swift -i ${./patches/0002-build-presets-linux-allow-custom-install-prefix.patch}
292 patch -p1 -d swift -i ${./patches/0003-build-presets-linux-don-t-build-extra-libs.patch}
293 patch -p1 -d swift -i ${./patches/0004-build-presets-linux-plumb-extra-cmake-options.patch}
294 patch -p1 -d swift -i ${./patches/0007-build-presets-linux-os-stdlib.patch}
295 substituteInPlace swift/cmake/modules/SwiftConfigureSDK.cmake \
296 --replace '/usr/include' "${stdenv.cc.libc.dev}/include"
297 sed -i swift/utils/build-presets.ini \
298 -e 's/^test-installable-package$/# \0/' \
299 -e 's/^test$/# \0/' \
300 -e 's/^validation-test$/# \0/' \
301 -e 's/^long-test$/# \0/' \
302 -e 's/^stress-test$/# \0/' \
303 -e 's/^test-optimized$/# \0/' \
304 -e 's/^swift-install-components=autolink.*$/\0;editor-integration/'
305
306 # LLVM toolchain patches.
307 patch -p1 -d llvm-project/clang -i ${./patches/0005-clang-toolchain-dir.patch}
308 patch -p1 -d llvm-project/clang -i ${./patches/0006-clang-purity.patch}
309 substituteInPlace llvm-project/clang/lib/Driver/ToolChains/Linux.cpp \
310 --replace 'SysRoot + "/lib' '"${glibc}/lib" "' \
311 --replace 'SysRoot + "/usr/lib' '"${glibc}/lib" "' \
312 --replace 'LibDir = "lib";' 'LibDir = "${glibc}/lib";' \
313 --replace 'LibDir = "lib64";' 'LibDir = "${glibc}/lib";' \
314 --replace 'LibDir = X32 ? "libx32" : "lib64";' 'LibDir = "${glibc}/lib";'
315
316 # Substitute ncurses for curses in llbuild.
317 sed -i 's/curses/ncurses/' llbuild/*/*/CMakeLists.txt
318 sed -i 's/curses/ncurses/' llbuild/*/*/*/CMakeLists.txt
319
320 # uuid.h is not part of glibc, but of libuuid.
321 sed -i 's|''${GLIBC_INCLUDE_PATH}/uuid/uuid.h|${libuuid.dev}/include/uuid/uuid.h|' swift/stdlib/public/Platform/glibc.modulemap.gyb
322
323 # Support library build script patches.
324 PREFIX=''${out/#\/}
325 substituteInPlace swift/utils/swift_build_support/swift_build_support/products/benchmarks.py \
326 --replace \
327 "'--toolchain', toolchain_path," \
328 "'--toolchain', '/build/install/$PREFIX',"
329 substituteInPlace swift/benchmark/scripts/build_script_helper.py \
330 --replace \
331 "swiftbuild_path = os.path.join(args.toolchain, \"usr\", \"bin\", \"swift-build\")" \
332 "swiftbuild_path = os.path.join(args.toolchain, \"bin\", \"swift-build\")"
333 substituteInPlace swift-corelibs-xctest/build_script.py \
334 --replace usr "$PREFIX"
335
336 # Can be removed in later swift-docc versions, see
337 # https://github.com/apple/swift-docc/commit/bff70b847008f91ac729cfd299a85481eef3f581
338 substituteInPlace swift-docc/build-script-helper.py \
339 --replace \
340 "subprocess.check_output(cmd, env=env).strip(), 'docc')" \
341 "subprocess.check_output(cmd, env=env).strip().decode(), 'docc')"
342
343 # Can be removed in later Swift versions, see
344 # https://github.com/apple/swift/pull/58755
345 substituteInPlace swift/utils/process-stats-dir.py \
346 --replace \
347 "type=argparse.FileType('wb', 0)," \
348 "type=argparse.FileType('w', 0),"
349
350 # Apply Python 3 fix, see
351 # https://github.com/apple/swift/commit/ec6bc595092974628b27b114a472e84162261bbd
352 substituteInPlace swift/utils/swift_build_support/swift_build_support/productpipeline_list_builder.py \
353 --replace \
354 "filter(lambda x: x is not None, pipeline)" \
355 "[p for p in pipeline if p is not None]"
356 '';
357
358 configurePhase = ''
359 cd ..
360
361 mkdir build install
362 export SWIFT_BUILD_ROOT=$PWD/build
363 export SWIFT_INSTALL_DIR=$PWD/install
364
365 export INSTALLABLE_PACKAGE=$PWD/swift.tar.gz
366 export NIX_ENFORCE_PURITY=
367
368 cd $SWIFT_BUILD_ROOT
369 '';
370
371 buildPhase = ''
372 # Explicitly include C++ headers to prevent errors where stdlib.h is not found from cstdlib.
373 export NIX_CFLAGS_COMPILE="$(< ${clang_13}/nix-support/libcxx-cxxflags) $NIX_CFLAGS_COMPILE"
374
375 # During the Swift build, a full local LLVM build is performed and the resulting clang is
376 # invoked. This compiler is not using the Nix wrappers, so it needs some help to find things.
377 export NIX_LDFLAGS_BEFORE="-rpath ${gccForLibs.lib}/lib -L${gccForLibs.lib}/lib $NIX_LDFLAGS_BEFORE"
378
379 # However, we want to use the wrapped compiler whenever possible.
380 export CC="${clang_13}/bin/clang"
381
382 $SWIFT_SOURCE_ROOT/swift/utils/build-script \
383 --preset=buildbot_linux \
384 installable_package=$INSTALLABLE_PACKAGE \
385 install_prefix=$out \
386 install_destdir=$SWIFT_INSTALL_DIR \
387 extra_cmake_options="${lib.concatStringsSep "," cmakeFlags}"
388 '';
389
390 doCheck = true;
391
392 checkInputs = [ file ];
393
394 checkPhase = ''
395 # Remove compiler build system tests which fail due to our modified default build profile and
396 # nixpkgs-provided version of CMake.
397 rm $SWIFT_SOURCE_ROOT/swift/validation-test/BuildSystem/infer_implies_install_all.test
398 rm $SWIFT_SOURCE_ROOT/swift/validation-test/BuildSystem/infer_dumps_deps_if_verbose_build.test
399
400 # This test apparently requires Python 2 (strings are assumed to be bytes-like), but the build
401 # process overall now otherwise requires Python 3 (which is what we have updated to). A fix PR
402 # has been submitted upstream.
403 rm $SWIFT_SOURCE_ROOT/swift/validation-test/SIL/verify_all_overlays.py
404
405 # TODO: consider fixing and re-adding. This test fails due to a non-standard "install_prefix".
406 rm $SWIFT_SOURCE_ROOT/swift/validation-test/Python/build_swift.swift
407
408 # We cannot handle the SDK location being in "Weird Location" due to Nix isolation.
409 rm $SWIFT_SOURCE_ROOT/swift/test/DebugInfo/compiler-flags.swift
410
411 # TODO: Fix issue with ld.gold invoked from script finding crtbeginS.o and crtendS.o.
412 rm $SWIFT_SOURCE_ROOT/swift/test/IRGen/ELF-remove-autolink-section.swift
413
414 # The following two tests fail because we use don't use the bundled libicu:
415 # [SOURCE_DIR/utils/build-script] ERROR: can't find source directory for libicu (tried /build/src/icu)
416 rm $SWIFT_SOURCE_ROOT/swift/validation-test/BuildSystem/default_build_still_performs_epilogue_opts_after_split.test
417 rm $SWIFT_SOURCE_ROOT/swift/validation-test/BuildSystem/test_early_swift_driver_and_infer.swift
418
419 # TODO: This test fails for some unknown reason
420 rm $SWIFT_SOURCE_ROOT/swift/test/Serialization/restrict-swiftmodule-to-revision.swift
421
422 # This test was flaky in ofborg, see #186476
423 rm $SWIFT_SOURCE_ROOT/swift/test/AutoDiff/compiler_crashers_fixed/sr14290-missing-debug-scopes-in-pullback-trampoline.swift
424
425 # TODO: consider using stress-tester and integration-test.
426
427 # Match the wrapped version of Swift to be installed.
428 export LIBRARY_PATH=${lib.makeLibraryPath [icu libgcc libuuid]}:$l
429
430 checkTarget=check-swift-all-${stdenv.hostPlatform.parsed.kernel.name}-${stdenv.hostPlatform.parsed.cpu.name}
431 ninjaFlags='-C buildbot_linux/swift-${stdenv.hostPlatform.parsed.kernel.name}-${stdenv.hostPlatform.parsed.cpu.name}'
432 ninjaCheckPhase
433 '';
434
435 installPhase = ''
436 mkdir -p $out
437
438 # Extract the generated tarball into the store.
439 tar xf $INSTALLABLE_PACKAGE -C $out --strip-components=3 ''${out/#\/}
440 find $out -type d -empty -delete
441
442 # Fix installation weirdness, also present in Apple’s official tarballs.
443 mv $out/local/include/indexstore $out/include
444 rmdir $out/local/include $out/local
445 rm -r $out/bin/sdk-module-lists $out/bin/swift-api-checker.py
446
447 wrapProgram $out/bin/swift \
448 --set CC $out/bin/clang \
449 --suffix C_INCLUDE_PATH : $out/lib/swift/clang/include \
450 --suffix CPLUS_INCLUDE_PATH : $out/lib/swift/clang/include \
451 --suffix LIBRARY_PATH : ${lib.makeLibraryPath [icu libgcc libuuid]} \
452 --suffix PATH : ${lib.makeBinPath [ stdenv.cc.bintools ]}
453
454 wrapProgram $out/bin/swiftc \
455 --set CC $out/bin/clang \
456 --suffix C_INCLUDE_PATH : $out/lib/swift/clang/include \
457 --suffix CPLUS_INCLUDE_PATH : $out/lib/swift/clang/include \
458 --suffix LIBRARY_PATH : ${lib.makeLibraryPath [icu libgcc libuuid]} \
459 --suffix PATH : ${lib.makeBinPath [ stdenv.cc.bintools ]}
460 '';
461
462 # Hack to avoid build and install directories in RPATHs.
463 preFixup = "rm -rf $SWIFT_BUILD_ROOT $SWIFT_INSTALL_DIR";
464
465 meta = with lib; {
466 description = "The Swift Programming Language";
467 homepage = "https://github.com/apple/swift";
468 maintainers = with maintainers; [ dtzWill trepetti dduan trundle ];
469 license = licenses.asl20;
470 # Swift doesn't support 32-bit Linux, unknown on other platforms.
471 platforms = platforms.linux;
472 badPlatforms = platforms.i686;
473 timeout = 86400; # 24 hours.
474 };
475}