···1+{ lib, stdenv, llvm_meta
2+, pkgsBuildBuild
3+, monorepoSrc
4+, runCommand
5+, fetchpatch
6+, cmake
7+, darwin
8+, ninja
9+, python3
10+, python3Packages
11+, libffi
12+# TODO: Gold plugin on LLVM16 has a severe memory corruption bug: https://github.com/llvm/llvm-project/issues/61350.
13+, enableGoldPlugin ? false
14+, libbfd
15+, libpfm
16+, libxml2
17+, ncurses
18+, version
19+, release_version
20+, zlib
21+, which
22+, sysctl
23+, buildLlvmTools
24+, debugVersion ? false
25+, doCheck ? (!stdenv.isx86_32 /* TODO: why */) && (!stdenv.hostPlatform.isMusl)
26+ && (stdenv.hostPlatform == stdenv.buildPlatform)
27+, enableManpages ? false
28+, enableSharedLibraries ? !stdenv.hostPlatform.isStatic
29+, enablePFM ? stdenv.isLinux /* PFM only supports Linux */
30+ # broken for Ampere eMAG 8180 (c2.large.arm on Packet) #56245
31+ # broken for the armv7l builder
32+ && !stdenv.hostPlatform.isAarch
33+, enablePolly ? true
34+} @args:
35+36+let
37+ inherit (lib) optional optionals optionalString;
38+39+ # Used when creating a version-suffixed symlink of libLLVM.dylib
40+ shortVersion = with lib;
41+ concatStringsSep "." (take 1 (splitString "." release_version));
42+43+ # Ordinarily we would just the `doCheck` and `checkDeps` functionality
44+ # `mkDerivation` gives us to manage our test dependencies (instead of breaking
45+ # out `doCheck` as a package level attribute).
46+ #
47+ # Unfortunately `lit` does not forward `$PYTHONPATH` to children processes, in
48+ # particular the children it uses to do feature detection.
49+ #
50+ # This means that python deps we add to `checkDeps` (which the python
51+ # interpreter is made aware of via `$PYTHONPATH` – populated by the python
52+ # setup hook) are not picked up by `lit` which causes it to skip tests.
53+ #
54+ # Adding `python3.withPackages (ps: [ ... ])` to `checkDeps` also doesn't work
55+ # because this package is shadowed in `$PATH` by the regular `python3`
56+ # package.
57+ #
58+ # So, we "manually" assemble one python derivation for the package to depend
59+ # on, taking into account whether checks are enabled or not:
60+ python = if doCheck then
61+ # Note that we _explicitly_ ask for a python interpreter for our host
62+ # platform here; the splicing that would ordinarily take care of this for
63+ # us does not seem to work once we use `withPackages`.
64+ let
65+ checkDeps = ps: with ps; [ psutil ];
66+ in pkgsBuildBuild.targetPackages.python3.withPackages checkDeps
67+ else python3;
68+69+in
70+ assert (lib.assertMsg (!enableGoldPlugin) "Gold plugin cannot be enabled on LLVM16 due to a upstream issue: https://github.com/llvm/llvm-project/issues/61350");
71+ stdenv.mkDerivation (rec {
72+ pname = "llvm";
73+ inherit version;
74+75+ src = runCommand "${pname}-src-${version}" {} (''
76+ mkdir -p "$out"
77+ cp -r ${monorepoSrc}/cmake "$out"
78+ cp -r ${monorepoSrc}/${pname} "$out"
79+ cp -r ${monorepoSrc}/third-party "$out"
80+ '' + lib.optionalString enablePolly ''
81+ chmod u+w "$out/${pname}/tools"
82+ cp -r ${monorepoSrc}/polly "$out/${pname}/tools"
83+ '');
84+85+ sourceRoot = "${src.name}/${pname}";
86+87+ outputs = [ "out" "lib" "dev" "python" ];
88+89+ nativeBuildInputs = [ cmake ninja python ]
90+ ++ optionals enableManpages [
91+ # Note: we intentionally use `python3Packages` instead of `python3.pkgs`;
92+ # splicing does *not* work with the latter. (TODO: fix)
93+ python3Packages.sphinx python3Packages.recommonmark
94+ ];
95+96+ buildInputs = [ libxml2 libffi ]
97+ ++ optional enablePFM libpfm; # exegesis
98+99+ propagatedBuildInputs = [ ncurses zlib ];
100+101+ nativeCheckInputs = [
102+ which
103+ ] ++ lib.optional stdenv.isDarwin sysctl;
104+105+ patches = [
106+ ./gnu-install-dirs.patch
107+108+ # Running the tests involves invoking binaries (like `opt`) that depend on
109+ # the LLVM dylibs and reference them by absolute install path (i.e. their
110+ # nix store path).
111+ #
112+ # Because we have not yet run the install phase (we're running these tests
113+ # as part of `checkPhase` instead of `installCheckPhase`) these absolute
114+ # paths do not exist yet; to work around this we point the loader (`ld` on
115+ # unix, `dyld` on macOS) at the `lib` directory which will later become this
116+ # package's `lib` output.
117+ #
118+ # Previously we would just set `LD_LIBRARY_PATH` to include the build `lib`
119+ # dir but:
120+ # - this doesn't generalize well to other platforms; `lit` doesn't forward
121+ # `DYLD_LIBRARY_PATH` (macOS):
122+ # + https://github.com/llvm/llvm-project/blob/0d89963df354ee309c15f67dc47c8ab3cb5d0fb2/llvm/utils/lit/lit/TestingConfig.py#L26
123+ # - even if `lit` forwarded this env var, we actually cannot set
124+ # `DYLD_LIBRARY_PATH` in the child processes `lit` launches because
125+ # `DYLD_LIBRARY_PATH` (and `DYLD_FALLBACK_LIBRARY_PATH`) is cleared for
126+ # "protected processes" (i.e. the python interpreter that runs `lit`):
127+ # https://stackoverflow.com/a/35570229
128+ # - other LLVM subprojects deal with this issue by having their `lit`
129+ # configuration set these env vars for us; it makes sense to do the same
130+ # for LLVM:
131+ # + https://github.com/llvm/llvm-project/blob/4c106cfdf7cf7eec861ad3983a3dd9a9e8f3a8ae/clang-tools-extra/test/Unit/lit.cfg.py#L22-L31
132+ #
133+ # !!! TODO: look into upstreaming this patch
134+ ./llvm-lit-cfg-add-libs-to-dylib-path.patch
135+136+ # `lit` has a mode where it executes run lines as a shell script which is
137+ # constructs; this is problematic for macOS because it means that there's
138+ # another process in between `lit` and the binaries being tested. As noted
139+ # above, this means that `DYLD_LIBRARY_PATH` is cleared which means that our
140+ # tests fail with dyld errors.
141+ #
142+ # To get around this we patch `lit` to reintroduce `DYLD_LIBRARY_PATH`, when
143+ # present in the test configuration.
144+ #
145+ # It's not clear to me why this isn't an issue for LLVM developers running
146+ # on macOS (nothing about this _seems_ nix specific)..
147+ ./lit-shell-script-runner-set-dyld-library-path.patch
148+ ] ++ lib.optionals enablePolly [
149+ ./gnu-install-dirs-polly.patch
150+151+ # Just like the `llvm-lit-cfg` patch, but for `polly`.
152+ ./polly-lit-cfg-add-libs-to-dylib-path.patch
153+ ];
154+155+ postPatch = optionalString stdenv.isDarwin ''
156+ substituteInPlace cmake/modules/AddLLVM.cmake \
157+ --replace 'set(_install_name_dir INSTALL_NAME_DIR "@rpath")' "set(_install_name_dir)" \
158+ --replace 'set(_install_rpath "@loader_path/../''${CMAKE_INSTALL_LIBDIR}''${LLVM_LIBDIR_SUFFIX}" ''${extra_libdir})' ""
159+160+ # As of LLVM 15, marked as XFAIL on arm64 macOS but lit doesn't seem to pick
161+ # this up: https://github.com/llvm/llvm-project/blob/c344d97a125b18f8fed0a64aace73c49a870e079/llvm/test/MC/ELF/cfi-version.ll#L7
162+ rm test/MC/ELF/cfi-version.ll
163+164+ # This test tries to call `sw_vers` by absolute path (`/usr/bin/sw_vers`)
165+ # and thus fails under the sandbox:
166+ substituteInPlace unittests/Support/Host.cpp \
167+ --replace '/usr/bin/sw_vers' "${(builtins.toString darwin.DarwinTools) + "/bin/sw_vers" }"
168+ '' + optionalString (stdenv.isDarwin && stdenv.hostPlatform.isx86) ''
169+ # This test tries to call the intrinsics `@llvm.roundeven.f32` and
170+ # `@llvm.roundeven.f64` which seem to (incorrectly?) lower to `roundevenf`
171+ # and `roundeven` on x86_64 macOS.
172+ #
173+ # However these functions are glibc specific so the test fails:
174+ # - https://www.gnu.org/software/gnulib/manual/html_node/roundevenf.html
175+ # - https://www.gnu.org/software/gnulib/manual/html_node/roundeven.html
176+ #
177+ # TODO(@rrbutani): this seems to run fine on `aarch64-darwin`, why does it
178+ # pass there?
179+ substituteInPlace test/ExecutionEngine/Interpreter/intrinsics.ll \
180+ --replace "%roundeven32 = call float @llvm.roundeven.f32(float 0.000000e+00)" "" \
181+ --replace "%roundeven64 = call double @llvm.roundeven.f64(double 0.000000e+00)" ""
182+183+ # This test fails on darwin x86_64 because `sw_vers` reports a different
184+ # macOS version than what LLVM finds by reading
185+ # `/System/Library/CoreServices/SystemVersion.plist` (which is passed into
186+ # the sandbox on macOS).
187+ #
188+ # The `sw_vers` provided by nixpkgs reports the macOS version associated
189+ # with the `CoreFoundation` framework with which it was built. Because
190+ # nixpkgs pins the SDK for `aarch64-darwin` and `x86_64-darwin` what
191+ # `sw_vers` reports is not guaranteed to match the macOS version of the host
192+ # that's building this derivation.
193+ #
194+ # Astute readers will note that we only _patch_ this test on aarch64-darwin
195+ # (to use the nixpkgs provided `sw_vers`) instead of disabling it outright.
196+ # So why does this test pass on aarch64?
197+ #
198+ # Well, it seems that `sw_vers` on aarch64 actually links against the _host_
199+ # CoreFoundation framework instead of the nixpkgs provided one.
200+ #
201+ # Not entirely sure what the right fix is here. I'm assuming aarch64
202+ # `sw_vers` doesn't intentionally link against the host `CoreFoundation`
203+ # (still digging into how this ends up happening, will follow up) but that
204+ # aside I think the more pertinent question is: should we be patching LLVM's
205+ # macOS version detection logic to use `sw_vers` instead of reading host
206+ # paths? This *is* a way in which details about builder machines can creep
207+ # into the artifacts that are produced, affecting reproducibility, but it's
208+ # not clear to me when/where/for what this even gets used in LLVM.
209+ #
210+ # TODO(@rrbutani): fix/follow-up
211+ substituteInPlace unittests/Support/Host.cpp \
212+ --replace "getMacOSHostVersion" "DISABLED_getMacOSHostVersion"
213+214+ # This test fails with a `dysmutil` crash; have not yet dug into what's
215+ # going on here (TODO(@rrbutani)).
216+ rm test/tools/dsymutil/ARM/obfuscated.test
217+ '' + ''
218+ # FileSystem permissions tests fail with various special bits
219+ substituteInPlace unittests/Support/CMakeLists.txt \
220+ --replace "Path.cpp" ""
221+ rm unittests/Support/Path.cpp
222+ substituteInPlace unittests/IR/CMakeLists.txt \
223+ --replace "PassBuilderCallbacksTest.cpp" ""
224+ rm unittests/IR/PassBuilderCallbacksTest.cpp
225+ rm test/tools/llvm-objcopy/ELF/mirror-permissions-unix.test
226+ '' + optionalString stdenv.hostPlatform.isMusl ''
227+ patch -p1 -i ${../../TLI-musl.patch}
228+ substituteInPlace unittests/Support/CMakeLists.txt \
229+ --replace "add_subdirectory(DynamicLibrary)" ""
230+ rm unittests/Support/DynamicLibrary/DynamicLibraryTest.cpp
231+ # valgrind unhappy with musl or glibc, but fails w/musl only
232+ rm test/CodeGen/AArch64/wineh4.mir
233+ '' + optionalString stdenv.hostPlatform.isAarch32 ''
234+ # skip failing X86 test cases on 32-bit ARM
235+ rm test/DebugInfo/X86/convert-debugloc.ll
236+ rm test/DebugInfo/X86/convert-inlined.ll
237+ rm test/DebugInfo/X86/convert-linked.ll
238+ rm test/tools/dsymutil/X86/op-convert.test
239+ rm test/tools/gold/X86/split-dwarf.ll
240+ rm test/tools/llvm-dwarfdump/X86/prettyprint_types.s
241+ rm test/tools/llvm-dwarfdump/X86/simplified-template-names.s
242+243+ # !!! Note: these tests are removed in LLVM 16.
244+ #
245+ # See here for context: https://github.com/NixOS/nixpkgs/pull/194634#discussion_r999790443
246+ rm test/CodeGen/RISCV/rv32zbp.ll
247+ rm test/CodeGen/RISCV/rv64zbp.ll
248+ '' + optionalString (stdenv.hostPlatform.system == "armv6l-linux") ''
249+ # Seems to require certain floating point hardware (NEON?)
250+ rm test/ExecutionEngine/frem.ll
251+ '' + ''
252+ patchShebangs test/BugPoint/compile-custom.ll.py
253+ '';
254+255+ preConfigure = ''
256+ # Workaround for configure flags that need to have spaces
257+ cmakeFlagsArray+=(
258+ -DLLVM_LIT_ARGS="-svj''${NIX_BUILD_CORES} --no-progress-bar"
259+ )
260+ '';
261+262+ # Defensive check: some paths (that we make symlinks to) depend on the release
263+ # version, for example:
264+ # - https://github.com/llvm/llvm-project/blob/406bde9a15136254f2b10d9ef3a42033b3cb1b16/clang/lib/Headers/CMakeLists.txt#L185
265+ #
266+ # So we want to sure that the version in the source matches the release
267+ # version we were given.
268+ #
269+ # We do this check here, in the LLVM build, because it happens early.
270+ postConfigure = let
271+ v = lib.versions;
272+ major = v.major release_version;
273+ minor = v.minor release_version;
274+ patch = v.patch release_version;
275+ in ''
276+ # $1: part, $2: expected
277+ check_version() {
278+ part="''${1^^}"
279+ part="$(cat include/llvm/Config/llvm-config.h | grep "#define LLVM_VERSION_''${part} " | cut -d' ' -f3)"
280+281+ if [[ "$part" != "$2" ]]; then
282+ echo >&2 \
283+ "mismatch in the $1 version! we have version ${release_version}" \
284+ "and expected the $1 version to be '$2'; the source has '$part' instead"
285+ exit 3
286+ fi
287+ }
288+289+ check_version major ${major}
290+ check_version minor ${minor}
291+ check_version patch ${patch}
292+ '';
293+294+ # E.g. mesa.drivers use the build-id as a cache key (see #93946):
295+ LDFLAGS = optionalString (enableSharedLibraries && !stdenv.isDarwin) "-Wl,--build-id=sha1";
296+297+ cmakeFlags = with stdenv; let
298+ # These flags influence llvm-config's BuildVariables.inc in addition to the
299+ # general build. We need to make sure these are also passed via
300+ # CROSS_TOOLCHAIN_FLAGS_NATIVE when cross-compiling or llvm-config-native
301+ # will return different results from the cross llvm-config.
302+ #
303+ # Some flags don't need to be repassed because LLVM already does so (like
304+ # CMAKE_BUILD_TYPE), others are irrelevant to the result.
305+ flagsForLlvmConfig = [
306+ "-DLLVM_INSTALL_PACKAGE_DIR=${placeholder "dev"}/lib/cmake/llvm"
307+ "-DLLVM_ENABLE_RTTI=ON"
308+ ] ++ optionals enableSharedLibraries [
309+ "-DLLVM_LINK_LLVM_DYLIB=ON"
310+ ];
311+ in flagsForLlvmConfig ++ [
312+ "-DCMAKE_BUILD_TYPE=${if debugVersion then "Debug" else "Release"}"
313+ "-DLLVM_INSTALL_UTILS=ON" # Needed by rustc
314+ "-DLLVM_BUILD_TESTS=${if doCheck then "ON" else "OFF"}"
315+ "-DLLVM_ENABLE_FFI=ON"
316+ "-DLLVM_HOST_TRIPLE=${stdenv.hostPlatform.config}"
317+ "-DLLVM_DEFAULT_TARGET_TRIPLE=${stdenv.hostPlatform.config}"
318+ "-DLLVM_ENABLE_DUMP=ON"
319+ ] ++ optionals stdenv.hostPlatform.isStatic [
320+ # Disables building of shared libs, -fPIC is still injected by cc-wrapper
321+ "-DLLVM_ENABLE_PIC=OFF"
322+ "-DLLVM_BUILD_STATIC=ON"
323+ "-DLLVM_LINK_LLVM_DYLIB=off"
324+ # libxml2 needs to be disabled because the LLVM build system ignores its .la
325+ # file and doesn't link zlib as well.
326+ # https://github.com/ClangBuiltLinux/tc-build/issues/150#issuecomment-845418812
327+ "-DLLVM_ENABLE_LIBXML2=OFF"
328+ ] ++ optionals enableManpages [
329+ "-DLLVM_BUILD_DOCS=ON"
330+ "-DLLVM_ENABLE_SPHINX=ON"
331+ "-DSPHINX_OUTPUT_MAN=ON"
332+ "-DSPHINX_OUTPUT_HTML=OFF"
333+ "-DSPHINX_WARNINGS_AS_ERRORS=OFF"
334+ ] ++ optionals (false) [
335+ "-DLLVM_BINUTILS_INCDIR=${libbfd.dev}/include"
336+ ] ++ optionals isDarwin [
337+ "-DLLVM_ENABLE_LIBCXX=ON"
338+ "-DCAN_TARGET_i386=false"
339+ ] ++ optionals (stdenv.hostPlatform != stdenv.buildPlatform) [
340+ "-DCMAKE_CROSSCOMPILING=True"
341+ "-DLLVM_TABLEGEN=${buildLlvmTools.llvm}/bin/llvm-tblgen"
342+ (
343+ let
344+ nativeCC = pkgsBuildBuild.targetPackages.stdenv.cc;
345+ nativeBintools = nativeCC.bintools.bintools;
346+ nativeToolchainFlags = [
347+ "-DCMAKE_C_COMPILER=${nativeCC}/bin/${nativeCC.targetPrefix}cc"
348+ "-DCMAKE_CXX_COMPILER=${nativeCC}/bin/${nativeCC.targetPrefix}c++"
349+ "-DCMAKE_AR=${nativeBintools}/bin/${nativeBintools.targetPrefix}ar"
350+ "-DCMAKE_STRIP=${nativeBintools}/bin/${nativeBintools.targetPrefix}strip"
351+ "-DCMAKE_RANLIB=${nativeBintools}/bin/${nativeBintools.targetPrefix}ranlib"
352+ ];
353+ # We need to repass the custom GNUInstallDirs values, otherwise CMake
354+ # will choose them for us, leading to wrong results in llvm-config-native
355+ nativeInstallFlags = [
356+ "-DCMAKE_INSTALL_PREFIX=${placeholder "out"}"
357+ "-DCMAKE_INSTALL_BINDIR=${placeholder "out"}/bin"
358+ "-DCMAKE_INSTALL_INCLUDEDIR=${placeholder "dev"}/include"
359+ "-DCMAKE_INSTALL_LIBDIR=${placeholder "lib"}/lib"
360+ "-DCMAKE_INSTALL_LIBEXECDIR=${placeholder "lib"}/libexec"
361+ ];
362+ in "-DCROSS_TOOLCHAIN_FLAGS_NATIVE:list="
363+ + lib.concatStringsSep ";" (lib.concatLists [
364+ flagsForLlvmConfig
365+ nativeToolchainFlags
366+ nativeInstallFlags
367+ ])
368+ )
369+ ];
370+371+ postInstall = ''
372+ mkdir -p $python/share
373+ mv $out/share/opt-viewer $python/share/opt-viewer
374+ moveToOutput "bin/llvm-config*" "$dev"
375+ substituteInPlace "$dev/lib/cmake/llvm/LLVMExports-${if debugVersion then "debug" else "release"}.cmake" \
376+ --replace "\''${_IMPORT_PREFIX}/lib/lib" "$lib/lib/lib" \
377+ --replace "$out/bin/llvm-config" "$dev/bin/llvm-config"
378+ substituteInPlace "$dev/lib/cmake/llvm/LLVMConfig.cmake" \
379+ --replace 'set(LLVM_BINARY_DIR "''${LLVM_INSTALL_PREFIX}")' 'set(LLVM_BINARY_DIR "'"$lib"'")'
380+ ''
381+ + optionalString (stdenv.isDarwin && enableSharedLibraries) ''
382+ ln -s $lib/lib/libLLVM.dylib $lib/lib/libLLVM-${shortVersion}.dylib
383+ ln -s $lib/lib/libLLVM.dylib $lib/lib/libLLVM-${release_version}.dylib
384+ ''
385+ + optionalString (stdenv.buildPlatform != stdenv.hostPlatform) ''
386+ cp NATIVE/bin/llvm-config $dev/bin/llvm-config-native
387+ '';
388+389+ inherit doCheck;
390+391+ checkTarget = "check-all";
392+393+ # For the update script:
394+ passthru.monorepoSrc = monorepoSrc;
395+396+ requiredSystemFeatures = [ "big-parallel" ];
397+ meta = llvm_meta // {
398+ homepage = "https://llvm.org/";
399+ description = "A collection of modular and reusable compiler and toolchain technologies";
400+ longDescription = ''
401+ The LLVM Project is a collection of modular and reusable compiler and
402+ toolchain technologies. Despite its name, LLVM has little to do with
403+ traditional virtual machines. The name "LLVM" itself is not an acronym; it
404+ is the full name of the project.
405+ LLVM began as a research project at the University of Illinois, with the
406+ goal of providing a modern, SSA-based compilation strategy capable of
407+ supporting both static and dynamic compilation of arbitrary programming
408+ languages. Since then, LLVM has grown to be an umbrella project consisting
409+ of a number of subprojects, many of which are being used in production by
410+ a wide variety of commercial and open source projects as well as being
411+ widely used in academic research. Code in the LLVM project is licensed
412+ under the "Apache 2.0 License with LLVM exceptions".
413+ '';
414+ };
415+} // lib.optionalAttrs enableManpages {
416+ pname = "llvm-manpages";
417+418+ propagatedBuildInputs = [];
419+420+ ninjaFlags = [ "docs-llvm-man" ];
421+ installTargets = [ "install-docs-llvm-man" ];
422+423+ postPatch = null;
424+ postInstall = null;
425+426+ outputs = [ "out" ];
427+428+ doCheck = false;
429+430+ meta = llvm_meta // {
431+ description = "man pages for LLVM ${version}";
432+ };
433+})
···1+diff --git a/utils/lit/lit/TestRunner.py b/utils/lit/lit/TestRunner.py
2+index 0242e0b75af3..d732011306f7 100644
3+--- a/utils/lit/lit/TestRunner.py
4++++ b/utils/lit/lit/TestRunner.py
5+@@ -1029,6 +1029,12 @@ def executeScript(test, litConfig, tmpBase, commands, cwd):
6+ f.write('@echo off\n')
7+ f.write('\n@if %ERRORLEVEL% NEQ 0 EXIT\n'.join(commands))
8+ else:
9++ # This env var is *purged* when invoking subprocesses so we have to
10++ # manually set it from within the bash script in order for the commands
11++ # in run lines to see this var:
12++ if "DYLD_LIBRARY_PATH" in test.config.environment:
13++ f.write(f'export DYLD_LIBRARY_PATH="{test.config.environment["DYLD_LIBRARY_PATH"]}"\n')
14++
15+ for i, ln in enumerate(commands):
16+ match = re.match(kPdbgRegex, ln)
17+ if match: