tjh.dev
1# Evaluates all the accessible paths in nixpkgs.
2# *This only builds on Linux* since it requires the Linux sandbox isolation to
3# be able to write in various places while evaluating inside the sandbox.
4#
5# This file is used by nixpkgs CI (see .github/workflows/eval.yml) as well as
6# being used directly as an entry point in Lix's CI (in `flake.nix` in the Lix
7# repo).
8#
9# If you know you are doing a breaking API change, please ping the nixpkgs CI
10# maintainers and the Lix maintainers (`nix eval -f . lib.teams.lix`).
11{
12 callPackage,
13 lib,
14 runCommand,
15 writeShellScript,
16 symlinkJoin,
17 busybox,
18 jq,
19 nix,
20}:
21
22let
23 nixpkgs =
24 with lib.fileset;
25 toSource {
26 root = ../..;
27 fileset = unions (
28 map (lib.path.append ../..) [
29 "default.nix"
30 "doc"
31 "lib"
32 "maintainers"
33 "nixos"
34 "pkgs"
35 ".version"
36 "ci/supportedSystems.json"
37 ]
38 );
39 };
40
41 supportedSystems = builtins.fromJSON (builtins.readFile ../supportedSystems.json);
42
43 attrpathsSuperset =
44 {
45 evalSystem,
46 }:
47 runCommand "attrpaths-superset.json"
48 {
49 src = nixpkgs;
50 # Don't depend on -dev outputs to reduce closure size for CI.
51 nativeBuildInputs = map lib.getBin [
52 busybox
53 nix
54 ];
55 }
56 ''
57 export NIX_STATE_DIR=$(mktemp -d)
58 mkdir $out
59 export GC_INITIAL_HEAP_SIZE=4g
60 command time -f "Attribute eval done [%MKB max resident, %Es elapsed] %C" \
61 nix-instantiate --eval --strict --json --show-trace \
62 "$src/pkgs/top-level/release-attrpaths-superset.nix" \
63 -A paths \
64 -I "$src" \
65 --option restrict-eval true \
66 --option allow-import-from-derivation false \
67 --option eval-system "${evalSystem}" > $out/paths.json
68 '';
69
70 singleSystem =
71 {
72 # The system to evaluate.
73 # Note that this is intentionally not called `system`,
74 # because `--argstr system` would only be passed to the ci/default.nix file!
75 evalSystem ? builtins.currentSystem,
76 # The path to the `paths.json` file from `attrpathsSuperset`
77 attrpathFile ? "${attrpathsSuperset { inherit evalSystem; }}/paths.json",
78 # The number of attributes per chunk, see ./README.md for more info.
79 chunkSize ? 5000,
80 checkMeta ? true,
81
82 # Don't try to eval packages marked as broken.
83 includeBroken ? false,
84 # Whether to just evaluate a single chunk for quick testing
85 quickTest ? false,
86 }:
87 let
88 singleChunk = writeShellScript "single-chunk" ''
89 set -euo pipefail
90 chunkSize=$1
91 myChunk=$2
92 system=$3
93 outputDir=$4
94
95 export NIX_SHOW_STATS=1
96 export NIX_SHOW_STATS_PATH="$outputDir/stats/$myChunk"
97 echo "Chunk $myChunk on $system start"
98 set +e
99 command time -o "$outputDir/timestats/$myChunk" \
100 -f "Chunk $myChunk on $system done [%MKB max resident, %Es elapsed] %C" \
101 nix-env -f "${nixpkgs}/pkgs/top-level/release-outpaths-parallel.nix" \
102 --eval-system "$system" \
103 --option restrict-eval true \
104 --option allow-import-from-derivation false \
105 --query --available \
106 --out-path --json \
107 --show-trace \
108 --arg chunkSize "$chunkSize" \
109 --arg myChunk "$myChunk" \
110 --arg attrpathFile "${attrpathFile}" \
111 --arg systems "[ \"$system\" ]" \
112 --arg checkMeta ${lib.boolToString checkMeta} \
113 --arg includeBroken ${lib.boolToString includeBroken} \
114 -I ${nixpkgs} \
115 -I ${attrpathFile} \
116 > "$outputDir/result/$myChunk" \
117 2> "$outputDir/stderr/$myChunk"
118 exitCode=$?
119 set -e
120 cat "$outputDir/stderr/$myChunk"
121 cat "$outputDir/timestats/$myChunk"
122 if (( exitCode != 0 )); then
123 echo "Evaluation failed with exit code $exitCode"
124 # This immediately halts all xargs processes
125 kill $PPID
126 elif [[ -s "$outputDir/stderr/$myChunk" ]]; then
127 echo "Nixpkgs on $system evaluated with warnings, aborting"
128 kill $PPID
129 fi
130 '';
131 in
132 runCommand "nixpkgs-eval-${evalSystem}"
133 {
134 # Don't depend on -dev outputs to reduce closure size for CI.
135 nativeBuildInputs = map lib.getBin [
136 busybox
137 jq
138 nix
139 ];
140 env = {
141 inherit evalSystem chunkSize;
142 };
143 }
144 ''
145 export NIX_STATE_DIR=$(mktemp -d)
146 nix-store --init
147
148 echo "System: $evalSystem"
149 cores=$NIX_BUILD_CORES
150 echo "Cores: $cores"
151 attrCount=$(jq length "${attrpathFile}")
152 echo "Attribute count: $attrCount"
153 echo "Chunk size: $chunkSize"
154 # Same as `attrCount / chunkSize` but rounded up
155 chunkCount=$(( (attrCount - 1) / chunkSize + 1 ))
156 echo "Chunk count: $chunkCount"
157
158 mkdir -p $out/${evalSystem}
159
160 # Record and print stats on free memory and swap in the background
161 (
162 while true; do
163 availMemory=$(free -m | grep Mem | awk '{print $7}')
164 freeSwap=$(free -m | grep Swap | awk '{print $4}')
165 echo "Available memory: $(( availMemory )) MiB, free swap: $(( freeSwap )) MiB"
166
167 if [[ ! -f "$out/${evalSystem}/min-avail-memory" ]] || (( availMemory < $(<$out/${evalSystem}/min-avail-memory) )); then
168 echo "$availMemory" > $out/${evalSystem}/min-avail-memory
169 fi
170 if [[ ! -f $out/${evalSystem}/min-free-swap ]] || (( freeSwap < $(<$out/${evalSystem}/min-free-swap) )); then
171 echo "$freeSwap" > $out/${evalSystem}/min-free-swap
172 fi
173 sleep 4
174 done
175 ) &
176
177 seq_end=$(( chunkCount - 1 ))
178
179 ${lib.optionalString quickTest ''
180 seq_end=0
181 ''}
182
183 chunkOutputDir=$(mktemp -d)
184 mkdir "$chunkOutputDir"/{result,stats,timestats,stderr}
185
186 seq -w 0 "$seq_end" |
187 command time -f "%e" -o "$out/${evalSystem}/total-time" \
188 xargs -I{} -P"$cores" \
189 ${singleChunk} "$chunkSize" {} "$evalSystem" "$chunkOutputDir"
190
191 cp -r "$chunkOutputDir"/stats $out/${evalSystem}/stats-by-chunk
192
193 if (( chunkSize * chunkCount != attrCount )); then
194 # A final incomplete chunk would mess up the stats, don't include it
195 rm "$chunkOutputDir"/stats/"$seq_end"
196 fi
197
198 cat "$chunkOutputDir"/result/* | jq -s 'add | map_values(.outputs)' > $out/${evalSystem}/paths.json
199 '';
200
201 diff = callPackage ./diff.nix { };
202
203 combine =
204 {
205 diffDir,
206 }:
207 runCommand "combined-eval"
208 {
209 # Don't depend on -dev outputs to reduce closure size for CI.
210 nativeBuildInputs = map lib.getBin [
211 jq
212 ];
213 }
214 ''
215 mkdir -p $out
216
217 # Combine output paths from all systems
218 cat ${diffDir}/*/diff.json | jq -s '
219 reduce .[] as $item ({}; {
220 added: (.added + $item.added),
221 changed: (.changed + $item.changed),
222 removed: (.removed + $item.removed),
223 rebuilds: (.rebuilds + $item.rebuilds)
224 })
225 ' > $out/combined-diff.json
226
227 mkdir -p $out/before/stats
228 for d in ${diffDir}/before/*; do
229 cp -r "$d"/stats-by-chunk $out/before/stats/$(basename "$d")
230 done
231
232 mkdir -p $out/after/stats
233 for d in ${diffDir}/after/*; do
234 cp -r "$d"/stats-by-chunk $out/after/stats/$(basename "$d")
235 done
236 '';
237
238 compare = callPackage ./compare { };
239
240 full =
241 {
242 # Whether to evaluate on a specific set of systems, by default all are evaluated
243 evalSystems ? if quickTest then [ "x86_64-linux" ] else supportedSystems,
244 # The number of attributes per chunk, see ./README.md for more info.
245 chunkSize,
246 quickTest ? false,
247 }:
248 let
249 diffs = symlinkJoin {
250 name = "diffs";
251 paths = map (
252 evalSystem:
253 let
254 eval = singleSystem {
255 inherit quickTest evalSystem chunkSize;
256 };
257 in
258 diff {
259 inherit evalSystem;
260 # Local "full" evaluation doesn't do a real diff.
261 beforeDir = eval;
262 afterDir = eval;
263 }
264 ) evalSystems;
265 };
266 in
267 combine {
268 diffDir = diffs;
269 };
270
271in
272{
273 inherit
274 attrpathsSuperset
275 singleSystem
276 diff
277 combine
278 compare
279 # The above three are used by separate VMs in a GitHub workflow,
280 # while the below is intended for testing on a single local machine
281 full
282 ;
283}