tjh.dev
nixpkgs mirror (for testing)
github.com/NixOS/nixpkgs
nix
1{
2 stdenv,
3 cacert,
4 lib,
5 writeCBin,
6}:
7
8args@{
9 name ? "${args.pname}-${args.version}",
10 bazel,
11 bazelFlags ? [ ],
12 bazelBuildFlags ? [ ],
13 bazelTestFlags ? [ ],
14 bazelRunFlags ? [ ],
15 runTargetFlags ? [ ],
16 bazelFetchFlags ? [ ],
17 bazelTargets ? [ ],
18 bazelTestTargets ? [ ],
19 bazelRunTarget ? null,
20 buildAttrs,
21 fetchAttrs,
22
23 # Newer versions of Bazel are moving away from built-in rules_cc and instead
24 # allow fetching it as an external dependency in a WORKSPACE file[1]. If
25 # removed in the fixed-output fetch phase, building will fail to download it.
26 # This can be seen e.g. in #73097
27 #
28 # This option allows configuring the removal of rules_cc in cases where a
29 # project depends on it via an external dependency.
30 #
31 # [1]: https://github.com/bazelbuild/rules_cc
32 removeRulesCC ? true,
33 removeLocalConfigCc ? true,
34 removeLocalConfigSh ? true,
35 removeLocal ? true,
36
37 # Use build --nobuild instead of fetch. This allows fetching the dependencies
38 # required for the build as configured, rather than fetching all the dependencies
39 # which may not work in some situations (e.g. Java code which ends up relying on
40 # Debian-specific /usr/share/java paths, but doesn't in the configured build).
41 fetchConfigured ? true,
42
43 # Don’t add Bazel --copt and --linkopt from NIX_CFLAGS_COMPILE /
44 # NIX_LDFLAGS. This is necessary when using a custom toolchain which
45 # Bazel wants all headers / libraries to come from, like when using
46 # CROSSTOOL. Weirdly, we can still get the flags through the wrapped
47 # compiler.
48 dontAddBazelOpts ? false,
49 ...
50}:
51
52let
53 fArgs =
54 removeAttrs args [
55 "buildAttrs"
56 "fetchAttrs"
57 "removeRulesCC"
58 ]
59 // {
60 inherit
61 name
62 bazelFlags
63 bazelBuildFlags
64 bazelTestFlags
65 bazelRunFlags
66 runTargetFlags
67 bazelFetchFlags
68 bazelTargets
69 bazelTestTargets
70 bazelRunTarget
71 dontAddBazelOpts
72 ;
73 };
74 fBuildAttrs = fArgs // buildAttrs;
75 fFetchAttrs =
76 fArgs
77 // removeAttrs fetchAttrs [
78 "hash"
79 "sha256"
80 ];
81 bazelCmd =
82 {
83 cmd,
84 additionalFlags,
85 targets,
86 targetRunFlags ? [ ],
87 }:
88 lib.optionalString (targets != [ ]) ''
89 # See footnote called [USER and BAZEL_USE_CPP_ONLY_TOOLCHAIN variables]
90 BAZEL_USE_CPP_ONLY_TOOLCHAIN=1 \
91 USER=homeless-shelter \
92 bazel \
93 --batch \
94 --output_base="$bazelOut" \
95 --output_user_root="$bazelUserRoot" \
96 ${cmd} \
97 --curses=no \
98 "''${copts[@]}" \
99 "''${host_copts[@]}" \
100 "''${linkopts[@]}" \
101 "''${host_linkopts[@]}" \
102 $bazelFlags \
103 ${lib.strings.concatStringsSep " " additionalFlags} \
104 ${lib.strings.concatStringsSep " " targets} \
105 ${
106 lib.optionalString (targetRunFlags != [ ]) " -- " + lib.strings.concatStringsSep " " targetRunFlags
107 }
108 '';
109 # we need this to chmod dangling symlinks on darwin, gnu coreutils refuses to do so:
110 # chmod: cannot operate on dangling symlink '$symlink'
111 chmodder = writeCBin "chmodder" ''
112 #include <stdio.h>
113 #include <stdlib.h>
114 #include <sys/types.h>
115 #include <sys/stat.h>
116 #include <errno.h>
117 #include <string.h>
118
119 int main(int argc, char** argv) {
120 mode_t mode = S_IRWXU | S_IRWXG | S_IRWXO;
121 if (argc != 2) {
122 fprintf(stderr, "usage: chmodder file");
123 exit(EXIT_FAILURE);
124 }
125 if (lchmod(argv[1], mode) != 0) {
126 fprintf(stderr, "failed to lchmod '%s': %s", argv[0], strerror(errno));
127 exit(EXIT_FAILURE);
128 }
129 }
130 '';
131in
132stdenv.mkDerivation (
133 fBuildAttrs
134 // {
135
136 deps = stdenv.mkDerivation (
137 fFetchAttrs
138 // {
139 name = "${name}-deps.tar.gz";
140
141 impureEnvVars = lib.fetchers.proxyImpureEnvVars ++ fFetchAttrs.impureEnvVars or [ ];
142
143 nativeBuildInputs = fFetchAttrs.nativeBuildInputs or [ ] ++ [ bazel ];
144
145 preHook = fFetchAttrs.preHook or "" + ''
146 export bazelOut="$(echo ''${NIX_BUILD_TOP}/output | sed -e 's,//,/,g')"
147 export bazelUserRoot="$(echo ''${NIX_BUILD_TOP}/tmp | sed -e 's,//,/,g')"
148 export HOME="$NIX_BUILD_TOP"
149 export USER="nix"
150 # This is needed for git_repository with https remotes
151 export GIT_SSL_CAINFO="${cacert}/etc/ssl/certs/ca-bundle.crt"
152 # This is needed for Bazel fetchers that are themselves programs (e.g.
153 # rules_go using the go toolchain)
154 export SSL_CERT_FILE="${cacert}/etc/ssl/certs/ca-bundle.crt"
155 '';
156
157 buildPhase =
158 fFetchAttrs.buildPhase or ''
159 runHook preBuild
160
161 ${bazelCmd {
162 cmd = if fetchConfigured then "build --nobuild" else "fetch";
163 additionalFlags = [
164 # We disable multithreading for the fetching phase since it can lead to timeouts with many dependencies/threads:
165 # https://github.com/bazelbuild/bazel/issues/6502
166 "--loading_phase_threads=1"
167 "$bazelFetchFlags"
168 ]
169 ++ (
170 if fetchConfigured then
171 [
172 "--jobs"
173 "$NIX_BUILD_CORES"
174 ]
175 else
176 [ ]
177 );
178 targets = fFetchAttrs.bazelTargets ++ fFetchAttrs.bazelTestTargets;
179 }}
180
181 runHook postBuild
182 '';
183
184 installPhase =
185 fFetchAttrs.installPhase or (
186 ''
187 runHook preInstall
188
189 # Remove all built in external workspaces, Bazel will recreate them when building
190 rm -rf $bazelOut/external/{bazel_tools,\@bazel_tools.marker}
191 ${lib.optionalString removeRulesCC "rm -rf $bazelOut/external/{rules_cc,\\@rules_cc.marker}"}
192
193 rm -rf $bazelOut/external/{embedded_jdk,\@embedded_jdk.marker}
194 ${lib.optionalString removeLocalConfigCc "rm -rf $bazelOut/external/{local_config_cc,\\@local_config_cc.marker}"}
195 ${lib.optionalString removeLocal "rm -rf $bazelOut/external/{local_*,\\@local_*.marker}"}
196
197 # For bazel version >= 6 with bzlmod.
198 ${lib.optionalString removeLocalConfigCc "rm -rf $bazelOut/external/*[~+]{local_config_cc,local_config_cc.marker}"}
199 ${lib.optionalString removeLocalConfigSh "rm -rf $bazelOut/external/*[~+]{local_config_sh,local_config_sh.marker}"}
200 ${lib.optionalString removeLocal "rm -rf $bazelOut/external/*[~+]{local_jdk,local_jdk.marker}"}
201
202 # Clear markers
203 find $bazelOut/external -name '@*\.marker' -exec sh -c 'echo > {}' \;
204
205 # Remove all vcs files
206 rm -rf $(find $bazelOut/external -type d -name .git)
207 rm -rf $(find $bazelOut/external -type d -name .svn)
208 rm -rf $(find $bazelOut/external -type d -name .hg)
209
210 # Removing top-level symlinks along with their markers.
211 # This is needed because they sometimes point to temporary paths (?).
212 # For example, in Tensorflow-gpu build:
213 # platforms -> NIX_BUILD_TOP/tmp/install/35282f5123611afa742331368e9ae529/_embedded_binaries/platforms
214 find $bazelOut/external -maxdepth 1 -type l | while read symlink; do
215 name="$(basename "$symlink")"
216 rm "$symlink"
217 test -f "$bazelOut/external/@$name.marker" && rm "$bazelOut/external/@$name.marker" || true
218 done
219
220 # Patching symlinks to remove build directory reference
221 find $bazelOut/external -type l | while read symlink; do
222 new_target="$(readlink "$symlink" | sed "s,$NIX_BUILD_TOP,NIX_BUILD_TOP,")"
223 rm "$symlink"
224 ln -sf "$new_target" "$symlink"
225 ''
226 + lib.optionalString stdenv.hostPlatform.isDarwin ''
227 # on linux symlink permissions cannot be modified, so we modify those on darwin to match the linux ones
228 ${chmodder}/bin/chmodder "$symlink"
229 ''
230 + ''
231 done
232
233 echo '${bazel.name}' > $bazelOut/external/.nix-bazel-version
234
235 (cd $bazelOut/ && tar czf $out --sort=name --mtime='@1' --owner=0 --group=0 --numeric-owner external/)
236
237 runHook postInstall
238 ''
239 );
240
241 dontFixup = true;
242 allowedRequisites = [ ];
243
244 inherit (lib.fetchers.normalizeHash { hashTypes = [ "sha256" ]; } fetchAttrs)
245 outputHash
246 outputHashAlgo
247 ;
248 }
249 );
250
251 nativeBuildInputs = fBuildAttrs.nativeBuildInputs or [ ] ++ [
252 (bazel.override { enableNixHacks = true; })
253 ];
254
255 preHook = fBuildAttrs.preHook or "" + ''
256 export bazelOut="$NIX_BUILD_TOP/output"
257 export bazelUserRoot="$NIX_BUILD_TOP/tmp"
258 export HOME="$NIX_BUILD_TOP"
259 '';
260
261 preConfigure = ''
262 mkdir -p "$bazelOut"
263
264 (cd $bazelOut && tar xfz $deps)
265
266 test "${bazel.name}" = "$(<$bazelOut/external/.nix-bazel-version)" || {
267 echo "fixed output derivation was built for a different bazel version" >&2
268 echo " got: $(<$bazelOut/external/.nix-bazel-version)" >&2
269 echo "expected: ${bazel.name}" >&2
270 exit 1
271 }
272
273 chmod -R +w $bazelOut
274 find $bazelOut -type l | while read symlink; do
275 if [[ $(readlink "$symlink") == *NIX_BUILD_TOP* ]]; then
276 ln -sf $(readlink "$symlink" | sed "s,NIX_BUILD_TOP,$NIX_BUILD_TOP,") "$symlink"
277 fi
278 done
279 ''
280 + fBuildAttrs.preConfigure or "";
281
282 buildPhase =
283 fBuildAttrs.buildPhase or ''
284 runHook preBuild
285
286 # Bazel sandboxes the execution of the tools it invokes, so even though we are
287 # calling the correct nix wrappers, the values of the environment variables
288 # the wrappers are expecting will not be set. So instead of relying on the
289 # wrappers picking them up, pass them in explicitly via `--copt`, `--linkopt`
290 # and related flags.
291
292 copts=()
293 host_copts=()
294 linkopts=()
295 host_linkopts=()
296 if [ -z "''${dontAddBazelOpts:-}" ]; then
297 for flag in $NIX_CFLAGS_COMPILE; do
298 copts+=( "--copt=$flag" )
299 host_copts+=( "--host_copt=$flag" )
300 done
301 for flag in $NIX_CXXSTDLIB_COMPILE; do
302 copts+=( "--copt=$flag" )
303 host_copts+=( "--host_copt=$flag" )
304 done
305 for flag in $NIX_LDFLAGS; do
306 linkopts+=( "--linkopt=-Wl,$flag" )
307 host_linkopts+=( "--host_linkopt=-Wl,$flag" )
308 done
309 fi
310
311 ${bazelCmd {
312 cmd = "test";
313 additionalFlags = [
314 "--test_output=errors"
315 ]
316 ++ fBuildAttrs.bazelTestFlags
317 ++ [
318 "--jobs"
319 "$NIX_BUILD_CORES"
320 ];
321 targets = fBuildAttrs.bazelTestTargets;
322 }}
323 ${bazelCmd {
324 cmd = "build";
325 additionalFlags = fBuildAttrs.bazelBuildFlags ++ [
326 "--jobs"
327 "$NIX_BUILD_CORES"
328 ];
329 targets = fBuildAttrs.bazelTargets;
330 }}
331 ${bazelCmd {
332 cmd = "run";
333 additionalFlags = fBuildAttrs.bazelRunFlags ++ [
334 "--jobs"
335 "$NIX_BUILD_CORES"
336 ];
337 # Bazel run only accepts a single target, but `bazelCmd` expects `targets` to be a list.
338 targets = lib.optionals (fBuildAttrs.bazelRunTarget != null) [ fBuildAttrs.bazelRunTarget ];
339 targetRunFlags = fBuildAttrs.runTargetFlags;
340 }}
341 runHook postBuild
342 '';
343 }
344)
345
346# [USER and BAZEL_USE_CPP_ONLY_TOOLCHAIN variables]:
347# Bazel computes the default value of output_user_root before parsing the
348# flag. The computation of the default value involves getting the $USER
349# from the environment. Code here :
350# https://github.com/bazelbuild/bazel/blob/9323c57607d37f9c949b60e293b573584906da46/src/main/cpp/startup_options.cc#L123-L124
351#
352# On macOS Bazel will use the system installed Xcode or CLT toolchain instead of the one in the PATH unless we pass BAZEL_USE_CPP_ONLY_TOOLCHAIN.