tjh.dev
nixpkgs mirror (for testing)
github.com/NixOS/nixpkgs
nix
1{
2 lib,
3 stdenv,
4 callPackage,
5 fetchFromGitHub,
6 fetchpatch,
7 rocmUpdateScript,
8 runCommand,
9 pkg-config,
10 cmake,
11 rocm-cmake,
12 rocblas,
13 rocmlir,
14 rocrand,
15 rocm-runtime,
16 hipblas-common,
17 hipblas,
18 hipblaslt,
19 clr,
20 composable_kernel,
21 frugally-deep,
22 rocm-docs-core,
23 half,
24 boost,
25 sqlite,
26 bzip2,
27 lbzip2,
28 nlohmann_json,
29 texliveSmall,
30 doxygen,
31 sphinx,
32 zlib,
33 gtest,
34 rocm-comgr,
35 roctracer,
36 python3Packages,
37 gpuTargets ? clr.localGpuTargets or clr.gpuTargets,
38 buildDocs ? false, # Needs internet because of rocm-docs-core
39 buildTests ? false,
40 withComposableKernel ? true,
41}:
42
43let
44 # FIXME: cmake files need patched to include this properly
45 cFlags = "-Wno-documentation-pedantic --offload-compress -I${hipblas-common}/include -I${hipblas}/include -I${roctracer}/include -I${nlohmann_json}/include -I${sqlite.dev}/include -I${rocrand}/include";
46 version = "7.1.1";
47
48 # Targets outside this list will get
49 # error: use of undeclared identifier 'CK_BUFFER_RESOURCE_3RD_DWORD'
50 supportedTargets = lib.intersectLists [
51 "gfx900"
52 "gfx906"
53 "gfx908"
54 "gfx90a"
55 "gfx942"
56 "gfx950"
57 "gfx1030"
58 "gfx1031"
59 "gfx1100"
60 "gfx1101"
61 "gfx1102"
62 "gfx1150"
63 "gfx1151"
64 "gfx1200"
65 "gfx1201"
66 ] gpuTargets;
67
68 src = fetchFromGitHub {
69 owner = "ROCm";
70 repo = "MIOpen";
71 rev = "rocm-${version}";
72 hash = "sha256-g0AEpuUiwKCu/doiRh9mW34W04m4ynHoarSyl6tR/aE=";
73 fetchLFS = true;
74 fetchSubmodules = true;
75 # WORKAROUND: .lfsconfig is incorrectly set to exclude everything upstream
76 leaveDotGit = true;
77 # FIXME: if someone can reduce the level of awful here that would be really nice
78 postFetch = ''
79 export HOME=$(mktemp -d)
80 cd $out
81 git remote add origin $url
82 git fetch origin +refs/tags/rocm-${version}:refs/tags/rocm-${version}
83 git clean -fdx
84 git switch -c rocm-${version} refs/tags/rocm-${version}
85 git config lfs.fetchexclude "none"
86 rm .lfsconfig
87 git lfs install
88 git lfs track "*.kdb.bz2"
89 git lfs fetch --include="src/kernels/**"
90 git lfs pull --include="src/kernels/**"
91 git lfs checkout
92
93 rm -rf .git
94 '';
95 };
96
97 latex = lib.optionalAttrs buildDocs (
98 texliveSmall.withPackages (
99 ps: with ps; [
100 latexmk
101 tex-gyre
102 fncychap
103 wrapfig
104 capt-of
105 framed
106 needspace
107 tabulary
108 varwidth
109 titlesec
110 ]
111 )
112 );
113
114 gfx900 = runCommand "miopen-gfx900.kdb" { preferLocalBuild = true; } ''
115 ${lbzip2}/bin/lbzip2 -ckd ${src}/src/kernels/gfx900.kdb.bz2 > $out
116 '';
117
118 gfx906 = runCommand "miopen-gfx906.kdb" { preferLocalBuild = true; } ''
119 ${lbzip2}/bin/lbzip2 -ckd ${src}/src/kernels/gfx906.kdb.bz2 > $out
120 '';
121
122 gfx908 = runCommand "miopen-gfx908.kdb" { preferLocalBuild = true; } ''
123 ${lbzip2}/bin/lbzip2 -ckd ${src}/src/kernels/gfx908.kdb.bz2 > $out
124 '';
125
126 gfx90a = runCommand "miopen-gfx90a.kdb" { preferLocalBuild = true; } ''
127 ${lbzip2}/bin/lbzip2 -ckd ${src}/src/kernels/gfx90a.kdb.bz2 > $out
128 '';
129
130 gfx1030 = runCommand "miopen-gfx1030.kdb" { preferLocalBuild = true; } ''
131 ${lbzip2}/bin/lbzip2 -ckd ${src}/src/kernels/gfx1030.kdb.bz2 > $out
132 '';
133in
134stdenv.mkDerivation (finalAttrs: {
135 inherit version src;
136 pname = "miopen";
137
138 env.CFLAGS = cFlags;
139 env.CXXFLAGS = cFlags;
140
141 # Find zstd and add to target. Mainly for torch.
142 patches = [
143 ./skip-preexisting-dbs.patch
144 (fetchpatch {
145 url = "https://github.com/ROCm/MIOpen/commit/e608b4325646afeabb5e52846997b926d2019d19.patch";
146 hash = "sha256-oxa3qlIC2bzbwGxrQOZXoY/S7CpLsMrnWRB7Og0tk0M=";
147 })
148 (fetchpatch {
149 url = "https://github.com/ROCm/MIOpen/commit/3413d2daaeb44b7d6eadcc03033a5954a118491e.patch";
150 hash = "sha256-ST4snUcTmmSI1Ogx815KEX9GdMnmubsavDzXCGJkiKs=";
151 })
152 # FIXME: We need to rebase or drop this arch compat patch
153 # https://github.com/ROCm/MIOpen/issues/3540 suggests that
154 # arch compat patching doesn't work correctly for gfx1031
155 # (fetchpatch {
156 # name = "Extend-MIOpen-ISA-compatibility.patch";
157 # url = "https://github.com/GZGavinZhao/MIOpen/commit/416088b534618bd669a765afce59cfc7197064c1.patch";
158 # hash = "sha256-OwONCA68y8s2GqtQj+OtotXwUXQ5jM8tpeM92iaD4MU=";
159 # })
160 ];
161
162 outputs = [
163 "out"
164 ]
165 ++ lib.optionals buildDocs [
166 "doc"
167 ]
168 ++ lib.optionals buildTests [
169 "test"
170 ];
171 enableParallelBuilding = true;
172 env.ROCM_PATH = clr;
173 env.LD_LIBRARY_PATH = lib.makeLibraryPath [ rocm-runtime ];
174
175 nativeBuildInputs = [
176 pkg-config
177 cmake
178 rocm-cmake
179 clr
180 ];
181
182 buildInputs = [
183 hipblas
184 hipblas-common
185 rocblas
186 rocmlir
187 half
188 boost
189 sqlite
190 bzip2
191 nlohmann_json
192 frugally-deep
193 roctracer
194 rocrand
195 hipblaslt
196 ]
197 ++ lib.optionals withComposableKernel [
198 composable_kernel
199 ]
200 ++ lib.optionals buildDocs [
201 latex
202 doxygen
203 sphinx
204 rocm-docs-core
205 python3Packages.sphinx-rtd-theme
206 python3Packages.breathe
207 python3Packages.myst-parser
208 ]
209 ++ lib.optionals buildTests [
210 gtest
211 zlib
212 ];
213
214 cmakeFlags = [
215 "-DAMDGPU_TARGETS=${lib.concatStringsSep ";" supportedTargets}"
216 "-DGPU_TARGETS=${lib.concatStringsSep ";" supportedTargets}"
217 "-DGPU_ARCHS=${lib.concatStringsSep ";" supportedTargets}"
218 "-DMIOPEN_USE_SQLITE_PERFDB=ON"
219 "-DCMAKE_VERBOSE_MAKEFILE=ON"
220 "-DCMAKE_MODULE_PATH=${clr}/hip/cmake"
221 "-DCMAKE_BUILD_TYPE=Release"
222
223 # needs to stream to stdout so bzcat rather than bunzip2
224 "-DUNZIPPER=${bzip2}/bin/bzcat"
225
226 "-DCMAKE_C_COMPILER=amdclang"
227 "-DCMAKE_CXX_COMPILER=amdclang++"
228 "-DROCM_PATH=${clr}"
229 "-DHIP_ROOT_DIR=${clr}"
230 (lib.cmakeBool "MIOPEN_USE_ROCBLAS" true)
231 (lib.cmakeBool "MIOPEN_USE_HIPBLASLT" true)
232 (lib.cmakeBool "MIOPEN_USE_COMPOSABLEKERNEL" withComposableKernel)
233 (lib.cmakeBool "MIOPEN_USE_HIPRTC" true)
234 (lib.cmakeBool "MIOPEN_USE_COMGR" true)
235 "-DCMAKE_HIP_COMPILER_ROCM_ROOT=${clr}"
236 # Manually define CMAKE_INSTALL_<DIR>
237 # See: https://github.com/NixOS/nixpkgs/pull/197838
238 "-DCMAKE_INSTALL_BINDIR=bin"
239 "-DCMAKE_INSTALL_LIBDIR=lib"
240 "-DCMAKE_INSTALL_INCLUDEDIR=include"
241 "-DMIOPEN_BACKEND=HIP"
242 ]
243 ++ lib.optionals buildTests [
244 "-DBUILD_TESTS=ON"
245 "-DMIOPEN_TEST_ALL=ON"
246 ];
247
248 postPatch = ''
249 substituteInPlace cmake/ClangTidy.cmake \
250 --replace-fail 'macro(enable_clang_tidy)' 'macro(enable_clang_tidy)
251 endmacro()
252 macro(enable_clang_tidy_unused)' \
253 --replace-fail 'function(clang_tidy_check TARGET)' 'function(clang_tidy_check TARGET)
254 return()'
255
256 patchShebangs test src/composable_kernel fin utils install_deps.cmake
257
258 ln -sf ${gfx900} src/kernels/gfx900.kdb
259 ln -sf ${gfx906} src/kernels/gfx906.kdb
260 ln -sf ${gfx908} src/kernels/gfx908.kdb
261 ln -sf ${gfx90a} src/kernels/gfx90a.kdb
262 ln -sf ${gfx1030} src/kernels/gfx1030.kdb
263 mkdir -p build/share/miopen/db/
264 ln -sf ${gfx900} build/share/miopen/db/gfx900.kdb
265 ln -sf ${gfx906} build/share/miopen/db/gfx906.kdb
266 ln -sf ${gfx908} build/share/miopen/db/gfx908.kdb
267 ln -sf ${gfx90a} build/share/miopen/db/gfx90a.kdb
268 ln -sf ${gfx1030} build/share/miopen/db/gfx1030.kdb
269 '';
270
271 # Unfortunately, it seems like we have to call make on these manually
272 postBuild =
273 lib.optionalString buildDocs ''
274 python -m sphinx -T -E -b html -d _build/doctrees -D language=en ../docs _build/html
275 ''
276 + lib.optionalString buildTests ''
277 make -j$NIX_BUILD_CORES check
278 '';
279
280 postInstall = ''
281 rm $out/libexec/miopen/install_precompiled_kernels.sh
282 ln -sf ${gfx900} $out/share/miopen/db/gfx900.kdb
283 ln -sf ${gfx906} $out/share/miopen/db/gfx906.kdb
284 ln -sf ${gfx908} $out/share/miopen/db/gfx908.kdb
285 ln -sf ${gfx90a} $out/share/miopen/db/gfx90a.kdb
286 ln -sf ${gfx1030} $out/share/miopen/db/gfx1030.kdb
287 ''
288 + lib.optionalString buildDocs ''
289 mv ../doc/html $out/share/doc/miopen-hip
290 ''
291 + lib.optionalString buildTests ''
292 mkdir -p $test/bin
293 mv bin/test_* $test/bin
294 patchelf --set-rpath $out/lib:${
295 lib.makeLibraryPath (
296 finalAttrs.buildInputs
297 ++ [
298 clr
299 rocm-comgr
300 ]
301 )
302 } $test/bin/*
303 '';
304
305 requiredSystemFeatures = [ "big-parallel" ];
306
307 passthru.tests = {
308 # Ensure all .tn.model files can be loaded by whatever version of frugally-deep we have
309 # This is otherwise hard to verify as MIOpen will only use these models on specific,
310 # expensive Instinct GPUs
311 # If MIOpen stops embedding .tn.model files the test will also fail, and can be deleted,
312 # likely along with the frugally-deep dependency
313 can-load-models = callPackage ./test-frugally-deep-model-loading.nix {
314 inherit (finalAttrs) src version;
315 inherit frugally-deep nlohmann_json;
316 };
317 };
318 passthru.updateScript = rocmUpdateScript {
319 name = finalAttrs.pname;
320 inherit (finalAttrs.src) owner;
321 inherit (finalAttrs.src) repo;
322 };
323
324 meta = {
325 description = "Machine intelligence library for ROCm";
326 homepage = "https://github.com/ROCm/MIOpen";
327 license = with lib.licenses; [ mit ];
328 teams = [ lib.teams.rocm ];
329 platforms = lib.platforms.linux;
330 };
331})