Merge pull request #298388 from GZGavinZhao/rocm-gfx-compat

rocmPackages: extend ISA compatibility

pkgs/development/rocm-modules/6/clr/default.nix

···
       url = "https://github.com/ROCm/clr/commit/77c581a3ebd47b5e2908973b70adea66891159ee.patch";
       hash = "sha256-auBedbd7rghlKav7A9V6l64J7VmtE9GizIdi5gWj+fs=";
     })
+    (fetchpatch {
+      name = "extend-hip-isa-compatibility-check.patch";
+      url = "https://salsa.debian.org/rocm-team/rocm-hipamd/-/raw/d6d20142c37e1dff820950b16ff8f0523241d935/debian/patches/0026-extend-hip-isa-compatibility-check.patch";
+      hash = "sha256-eG0ALZZQLRzD7zJueJFhi2emontmYy6xx8Rsm346nQI=";
+    })
+    (fetchpatch {
+      name = "improve-rocclr-isa-compatibility-check.patch";
+      url = "https://salsa.debian.org/rocm-team/rocm-hipamd/-/raw/d6d20142c37e1dff820950b16ff8f0523241d935/debian/patches/0025-improve-rocclr-isa-compatibility-check.patch";
+      hash = "sha256-8eowuRiOAdd9ucKv4Eg9FPU7c6367H3eP3fRAGfXc6Y=";
+    })
   ];
 
   postPatch = ''
···
 
     substituteInPlace hipamd/src/hip_embed_pch.sh \
       --replace "\''$LLVM_DIR/bin/clang" "${clang}/bin/clang"
+
+    # https://lists.debian.org/debian-ai/2024/02/msg00178.html
+    substituteInPlace rocclr/utils/flags.hpp \
+      --replace-fail "HIP_USE_RUNTIME_UNBUNDLER, false" "HIP_USE_RUNTIME_UNBUNDLER, true"
   '';
 
   postInstall = ''

pkgs/development/rocm-modules/6/default.nix

···
   };
 
   rocblas = callPackage ./rocblas {
-    inherit rocblas rocmUpdateScript rocm-cmake clr tensile;
+    inherit rocmUpdateScript rocm-cmake clr tensile;
     inherit (llvm) openmp;
     stdenv = llvm.rocmClangStdenv;
   };

pkgs/development/rocm-modules/6/miopen/default.nix

···
       url = "https://github.com/ROCm/MIOpen/commit/3413d2daaeb44b7d6eadcc03033a5954a118491e.patch";
       hash = "sha256-ST4snUcTmmSI1Ogx815KEX9GdMnmubsavDzXCGJkiKs=";
     })
+    (fetchpatch {
+      name = "Extend-MIOpen-ISA-compatibility.patch";
+      url = "https://github.com/GZGavinZhao/MIOpen/commit/416088b534618bd669a765afce59cfc7197064c1.patch";
+      hash = "sha256-OwONCA68y8s2GqtQj+OtotXwUXQ5jM8tpeM92iaD4MU=";
+    })
   ];
 
   outputs = [

pkgs/development/rocm-modules/6/rccl/default.nix

···
 
     # Really strange behavior, `#!/usr/bin/env perl` should work...
     substituteInPlace CMakeLists.txt \
-      --replace "\''$ \''${hipify-perl_executable}" "${perl}/bin/perl ${hipify}/bin/hipify-perl"
+      --replace "\''$ \''${hipify-perl_executable}" "${perl}/bin/perl ${hipify}/bin/hipify-perl" \
+      --replace-warn "-parallel-jobs=12" "-parallel-jobs=1" \
+      --replace-warn "-parallel-jobs=16" "-parallel-jobs=1"
   '';
 
   postInstall = lib.optionalString buildTests ''

pkgs/development/rocm-modules/6/rocblas/default.nix

···
-{ rocblas
-, lib
+{ lib
 , stdenv
 , fetchFromGitHub
+, fetchpatch
 , rocmUpdateScript
 , runCommand
 , cmake
···
 , buildBenchmarks ? false
 , tensileLogic ? "asm_full"
 , tensileCOVersion ? "default"
-, tensileSepArch ? true
-, tensileLazyLib ? true
+# https://github.com/ROCm/Tensile/issues/1757
+# Allows gfx101* users to use rocBLAS normally.
+# Turn the below two values to `true` after the fix has been cherry-picked
+# into a release. Just backporting that single fix is not enough because it
+# depends on some previous commits.
+, tensileSepArch ? false
+, tensileLazyLib ? false
 , tensileLibFormat ? "msgpack"
-, gpuTargets ? [ "all" ]
+# `gfx940`, `gfx941` are not present in this list because they are early
+# engineering samples, and all final MI300 hardware are `gfx942`:
+# https://github.com/NixOS/nixpkgs/pull/298388#issuecomment-2032791130
+#
+# `gfx1012` is not present in this list because the ISA compatibility patches
+# would force all `gfx101*` GPUs to run as `gfx1010`, so `gfx101*` GPUs will
+# always try to use `gfx1010` code objects, hence building for `gfx1012` is
+# useless: https://github.com/NixOS/nixpkgs/pull/298388#issuecomment-2076327152
+, gpuTargets ? [ "gfx900;gfx906:xnack-;gfx908:xnack-;gfx90a:xnack+;gfx90a:xnack-;gfx942;gfx1010;gfx1030;gfx1100;gfx1101;gfx1102" ]
 }:
 
-let
-  # NOTE: Update the default GPU targets on every update
-  gfx80 = (rocblas.override {
-    gpuTargets = [
-      "gfx803"
-    ];
-  }).overrideAttrs { pname = "rocblas-tensile-gfx80"; };
-
-  gfx90 = (rocblas.override {
-    gpuTargets = [
-      "gfx900"
-      "gfx906:xnack-"
-      "gfx908:xnack-"
-      "gfx90a:xnack+"
-      "gfx90a:xnack-"
-    ];
-  }).overrideAttrs { pname = "rocblas-tensile-gfx90"; };
-
-  gfx94 = (rocblas.override {
-    gpuTargets = [
-      "gfx940"
-      "gfx941"
-      "gfx942"
-    ];
-  }).overrideAttrs { pname = "rocblas-tensile-gfx94"; };
-
-  gfx10 = (rocblas.override {
-    gpuTargets = [
-      "gfx1010"
-      "gfx1012"
-      "gfx1030"
-    ];
-  }).overrideAttrs { pname = "rocblas-tensile-gfx10"; };
-
-  gfx11 = (rocblas.override {
-    gpuTargets = [
-      "gfx1100"
-      "gfx1101"
-      "gfx1102"
-    ];
-  }).overrideAttrs { pname = "rocblas-tensile-gfx11"; };
-
-  # Unfortunately, we have to do two full builds, otherwise we get overlapping _fallback.dat files
-  fallbacks = rocblas.overrideAttrs { pname = "rocblas-tensile-fallbacks"; };
-in stdenv.mkDerivation (finalAttrs: {
+stdenv.mkDerivation (finalAttrs: {
   pname = "rocblas";
   version = "6.0.2";
 
···
     cmake
     rocm-cmake
     clr
+  ] ++ lib.optionals buildTensile [
+    tensile
   ];
 
   buildInputs = [
···
   ];
 
   cmakeFlags = [
-    "-DCMAKE_C_COMPILER=hipcc"
-    "-DCMAKE_CXX_COMPILER=hipcc"
-    "-Dpython=python3"
-    "-DAMDGPU_TARGETS=${lib.concatStringsSep ";" gpuTargets}"
-    "-DBUILD_WITH_TENSILE=${if buildTensile then "ON" else "OFF"}"
-    # Manually define CMAKE_INSTALL_<DIR>
-    # See: https://github.com/NixOS/nixpkgs/pull/197838
-    "-DCMAKE_INSTALL_BINDIR=bin"
-    "-DCMAKE_INSTALL_LIBDIR=lib"
-    "-DCMAKE_INSTALL_INCLUDEDIR=include"
+    (lib.cmakeFeature "CMAKE_C_COMPILER" "hipcc")
+    (lib.cmakeFeature "CMAKE_CXX_COMPILER" "hipcc")
+    (lib.cmakeFeature "python" "python3")
+    (lib.cmakeFeature "AMDGPU_TARGETS" (lib.concatStringsSep ";" gpuTargets))
+    (lib.cmakeBool "BUILD_WITH_TENSILE" buildTensile)
+    (lib.cmakeBool "ROCM_SYMLINK_LIBS" false)
+    (lib.cmakeFeature "ROCBLAS_TENSILE_LIBRARY_DIR" "lib/rocblas")
+    (lib.cmakeBool "BUILD_CLIENTS_TESTS" buildTests)
+    (lib.cmakeBool "BUILD_CLIENTS_BENCHMARKS" buildBenchmarks)
+    # rocblas header files are not installed unless we set this
+    (lib.cmakeFeature "CMAKE_INSTALL_INCLUDEDIR" "include")
   ] ++ lib.optionals buildTensile [
-    "-DVIRTUALENV_HOME_DIR=/build/source/tensile"
-    "-DTensile_TEST_LOCAL_PATH=/build/source/tensile"
-    "-DTensile_ROOT=/build/source/tensile/${python3.sitePackages}/Tensile"
-    "-DTensile_LOGIC=${tensileLogic}"
-    "-DTensile_CODE_OBJECT_VERSION=${tensileCOVersion}"
-    "-DTensile_SEPARATE_ARCHITECTURES=${if tensileSepArch then "ON" else "OFF"}"
-    "-DTensile_LAZY_LIBRARY_LOADING=${if tensileLazyLib then "ON" else "OFF"}"
-    "-DTensile_LIBRARY_FORMAT=${tensileLibFormat}"
-  ] ++ lib.optionals buildTests [
-    "-DBUILD_CLIENTS_TESTS=ON"
-  ] ++ lib.optionals buildBenchmarks [
-    "-DBUILD_CLIENTS_BENCHMARKS=ON"
+    (lib.cmakeBool "BUILD_WITH_PIP" false)
+    (lib.cmakeFeature "Tensile_LOGIC" tensileLogic)
+    (lib.cmakeFeature "Tensile_CODE_OBJECT_VERSION" tensileCOVersion)
+    (lib.cmakeBool "Tensile_SEPARATE_ARCHITECTURES" tensileSepArch)
+    (lib.cmakeBool "Tensile_LAZY_LIBRARY_LOADING" tensileLazyLib)
+    (lib.cmakeFeature "Tensile_LIBRARY_FORMAT" tensileLibFormat)
+    (lib.cmakeBool "Tensile_PRINT_DEBUG" true)
   ] ++ lib.optionals (buildTests || buildBenchmarks) [
-    "-DCMAKE_CXX_FLAGS=-I${amd-blis}/include/blis"
+    (lib.cmakeFeature "CMAKE_CXX_FLAGS" "-I${amd-blis}/include/blis")
   ];
 
-  postPatch = lib.optionalString (finalAttrs.pname != "rocblas") ''
-    # Return early and install tensile files manually
-    substituteInPlace library/src/CMakeLists.txt \
-      --replace "set_target_properties( TensileHost PROPERTIES OUTPUT_NAME" "return()''\nset_target_properties( TensileHost PROPERTIES OUTPUT_NAME"
-  '' + lib.optionalString (buildTensile && finalAttrs.pname == "rocblas") ''
-    # Link the prebuilt Tensile files
-    mkdir -p build/Tensile/library
+  patches = [
+    (fetchpatch {
+      name = "Extend-rocBLAS-HIP-ISA-compatibility.patch";
+      url = "https://github.com/GZGavinZhao/rocBLAS/commit/89b75ff9cc731f71f370fad90517395e117b03bb.patch";
+      hash = "sha256-W/ohOOyNCcYYLOiQlPzsrTlNtCBdJpKVxO8s+4G7sjo=";
+    })
+  ];
 
-    for path in ${gfx80} ${gfx90} ${gfx94} ${gfx10} ${gfx11} ${fallbacks}; do
-      ln -s $path/lib/rocblas/library/* build/Tensile/library
-    done
-
-    unlink build/Tensile/library/TensileManifest.txt
-  '' + lib.optionalString buildTensile ''
-    # Tensile REALLY wants to write to the nix directory if we include it normally
-    cp -a ${tensile} tensile
-    chmod +w -R tensile
-
-    # Rewrap Tensile
-    substituteInPlace tensile/bin/{.t*,.T*,*} \
-      --replace "${tensile}" "/build/source/tensile"
-
-    substituteInPlace CMakeLists.txt \
-      --replace "include(virtualenv)" "" \
-      --replace "virtualenv_install(\''${Tensile_TEST_LOCAL_PATH})" ""
-  '';
-
-  postInstall = lib.optionalString (finalAttrs.pname == "rocblas") ''
-    ln -sf ${fallbacks}/lib/rocblas/library/TensileManifest.txt $out/lib/rocblas/library
-  '' + lib.optionalString (finalAttrs.pname != "rocblas") ''
-    mkdir -p $out/lib/rocblas/library
-    rm -rf $out/share
-  '' + lib.optionalString (finalAttrs.pname != "rocblas" && finalAttrs.pname != "rocblas-tensile-fallbacks") ''
-    rm Tensile/library/{TensileManifest.txt,*_fallback.dat}
-    mv Tensile/library/* $out/lib/rocblas/library
-  '' + lib.optionalString (finalAttrs.pname == "rocblas-tensile-fallbacks") ''
-    mv Tensile/library/{TensileManifest.txt,*_fallback.dat} $out/lib/rocblas/library
-  '' + lib.optionalString buildTests ''
-    mkdir -p $test/bin
-    cp -a $out/bin/* $test/bin
-    rm $test/bin/*-bench || true
-  '' + lib.optionalString buildBenchmarks ''
-    mkdir -p $benchmark/bin
-    cp -a $out/bin/* $benchmark/bin
-    rm $benchmark/bin/*-test || true
-  '' + lib.optionalString (buildTests || buildBenchmarks ) ''
-    rm -rf $out/bin
+  # Pass $NIX_BUILD_CORES to Tensile
+  postPatch = ''
+    substituteInPlace cmake/build-options.cmake \
+      --replace-fail 'Tensile_CPU_THREADS ""' 'Tensile_CPU_THREADS "$ENV{NIX_BUILD_CORES}"'
   '';
 
   passthru.updateScript = rocmUpdateScript {

pkgs/development/rocm-modules/6/rocm-runtime/default.nix

···
 { lib
 , stdenv
 , fetchFromGitHub
+, fetchpatch
 , rocmUpdateScript
 , pkg-config
 , cmake
···
     numactl
     valgrind
     libxml2
+  ];
+
+  patches = [
+    (fetchpatch {
+      name = "extend-isa-compatibility-check.patch";
+      url = "https://salsa.debian.org/rocm-team/rocr-runtime/-/raw/076026d43bbee7f816b81fea72f984213a9ff961/debian/patches/0004-extend-isa-compatibility-check.patch";
+      hash = "sha256-cC030zVGS4kNXwaztv5cwfXfVwOldpLGV9iYgEfPEnY=";
+      stripLen = 1;
+    })
   ];
 
   postPatch = ''

pkgs/development/rocm-modules/6/rocprim/default.nix

···
 { lib
+, fetchpatch
 , stdenv
 , fetchFromGitHub
 , rocmUpdateScript
···
     rev = "rocm-${finalAttrs.version}";
     hash = "sha256-nWvq26qRPZ6Au1rc5cR74TKArcdUFg7O9djFi8SvMeM=";
   };
+
+  patches = [
+    (fetchpatch {
+      name = "arch-conversion-marco.patch";
+      url = "https://salsa.debian.org/rocm-team/rocprim/-/raw/70c8aaee3cf545d92685f4ed9bf8f41e3d4d570c/debian/patches/arch-conversion-macro.patch";
+      hash = "sha256-oXdmbCArOB5bKE8ozDFrSh4opbO+c4VI6PNhljeUSms=";
+    })
+  ];
 
   nativeBuildInputs = [
     cmake

pkgs/development/rocm-modules/6/tensile/default.nix

···
 { lib
 , stdenv
 , fetchFromGitHub
+, fetchpatch
 , rocmUpdateScript
 , buildPythonPackage
 , pytestCheckHook
···
     joblib
   ];
 
+  patches = [
+    (fetchpatch {
+      name = "Extend-Tensile-HIP-ISA-compatibility.patch";
+      url = "https://github.com/GZGavinZhao/Tensile/commit/855cb15839849addb0816a6dde45772034a3e41f.patch";
+      hash = "sha256-d+fVf/vz+sxGqJ96vuxe0jRMgbC5K6j5FQ5SJ1e3Sl8=";
+    })
+    (fetchpatch {
+      name = "Don-t-copy-file-twice-in-copyStaticFiles.patch";
+      url = "https://github.com/GZGavinZhao/Tensile/commit/9e14d5a00a096bddac605910a0e4dfb4c35bb0d5.patch";
+      hash = "sha256-gOzjJyD1K056OFQ+hK5nbUeBhxLTIgQLoT+0K12SypI=";
+    })
+  ];
+
   doCheck = false; # Too many errors, not sure how to set this up properly
 
   nativeCheckInputs = [
···
     rocminfo
   ];
 
-  preCheck = ''
-    export ROCM_PATH=${rocminfo}
-  '';
+  env = {
+    ROCM_PATH = rocminfo;
+  };
 
   pythonImportsCheck = [ "Tensile" ];