tjh.dev
1# Evaluates all the accessible paths in nixpkgs.
2# *This only builds on Linux* since it requires the Linux sandbox isolation to
3# be able to write in various places while evaluating inside the sandbox.
4#
5# This file is used by nixpkgs CI (see .github/workflows/eval.yml) as well as
6# being used directly as an entry point in Lix's CI (in `flake.nix` in the Lix
7# repo).
8#
9# If you know you are doing a breaking API change, please ping the nixpkgs CI
10# maintainers and the Lix maintainers (`nix eval -f . lib.teams.lix`).
11{
12 callPackage,
13 lib,
14 runCommand,
15 writeShellScript,
16 symlinkJoin,
17 time,
18 procps,
19 nix,
20 jq,
21}:
22
23let
24 nixpkgs =
25 with lib.fileset;
26 toSource {
27 root = ../..;
28 fileset = unions (
29 map (lib.path.append ../..) [
30 "default.nix"
31 "doc"
32 "lib"
33 "maintainers"
34 "nixos"
35 "pkgs"
36 ".version"
37 "ci/supportedSystems.json"
38 ]
39 );
40 };
41
42 supportedSystems = builtins.fromJSON (builtins.readFile ../supportedSystems.json);
43
44 attrpathsSuperset =
45 {
46 evalSystem,
47 }:
48 runCommand "attrpaths-superset.json"
49 {
50 src = nixpkgs;
51 nativeBuildInputs = [
52 nix
53 time
54 ];
55 }
56 ''
57 export NIX_STATE_DIR=$(mktemp -d)
58 mkdir $out
59 export GC_INITIAL_HEAP_SIZE=4g
60 command time -f "Attribute eval done [%MKB max resident, %Es elapsed] %C" \
61 nix-instantiate --eval --strict --json --show-trace \
62 "$src/pkgs/top-level/release-attrpaths-superset.nix" \
63 -A paths \
64 -I "$src" \
65 --option restrict-eval true \
66 --option allow-import-from-derivation false \
67 --option eval-system "${evalSystem}" > $out/paths.json
68 '';
69
70 singleSystem =
71 {
72 # The system to evaluate.
73 # Note that this is intentionally not called `system`,
74 # because `--argstr system` would only be passed to the ci/default.nix file!
75 evalSystem,
76 # The path to the `paths.json` file from `attrpathsSuperset`
77 attrpathFile ? "${attrpathsSuperset { inherit evalSystem; }}/paths.json",
78 # The number of attributes per chunk, see ./README.md for more info.
79 chunkSize,
80 checkMeta ? true,
81
82 # Don't try to eval packages marked as broken.
83 includeBroken ? false,
84 # Whether to just evaluate a single chunk for quick testing
85 quickTest ? false,
86 }:
87 let
88 singleChunk = writeShellScript "single-chunk" ''
89 set -euo pipefail
90 chunkSize=$1
91 myChunk=$2
92 system=$3
93 outputDir=$4
94
95 export NIX_SHOW_STATS=1
96 export NIX_SHOW_STATS_PATH="$outputDir/stats/$myChunk"
97 echo "Chunk $myChunk on $system start"
98 set +e
99 command time -o "$outputDir/timestats/$myChunk" \
100 -f "Chunk $myChunk on $system done [%MKB max resident, %Es elapsed] %C" \
101 nix-env -f "${nixpkgs}/pkgs/top-level/release-outpaths-parallel.nix" \
102 --eval-system "$system" \
103 --option restrict-eval true \
104 --option allow-import-from-derivation false \
105 --query --available \
106 --out-path --json \
107 --show-trace \
108 --arg chunkSize "$chunkSize" \
109 --arg myChunk "$myChunk" \
110 --arg attrpathFile "${attrpathFile}" \
111 --arg systems "[ \"$system\" ]" \
112 --arg checkMeta ${lib.boolToString checkMeta} \
113 --arg includeBroken ${lib.boolToString includeBroken} \
114 -I ${nixpkgs} \
115 -I ${attrpathFile} \
116 > "$outputDir/result/$myChunk" \
117 2> "$outputDir/stderr/$myChunk"
118 exitCode=$?
119 set -e
120 cat "$outputDir/stderr/$myChunk"
121 cat "$outputDir/timestats/$myChunk"
122 if (( exitCode != 0 )); then
123 echo "Evaluation failed with exit code $exitCode"
124 # This immediately halts all xargs processes
125 kill $PPID
126 elif [[ -s "$outputDir/stderr/$myChunk" ]]; then
127 echo "Nixpkgs on $system evaluated with warnings, aborting"
128 kill $PPID
129 fi
130 '';
131 in
132 runCommand "nixpkgs-eval-${evalSystem}"
133 {
134 nativeBuildInputs = [
135 nix
136 time
137 procps
138 jq
139 ];
140 env = {
141 inherit evalSystem chunkSize;
142 };
143 }
144 ''
145 export NIX_STATE_DIR=$(mktemp -d)
146 nix-store --init
147
148 echo "System: $evalSystem"
149 cores=$NIX_BUILD_CORES
150 echo "Cores: $cores"
151 attrCount=$(jq length "${attrpathFile}")
152 echo "Attribute count: $attrCount"
153 echo "Chunk size: $chunkSize"
154 # Same as `attrCount / chunkSize` but rounded up
155 chunkCount=$(( (attrCount - 1) / chunkSize + 1 ))
156 echo "Chunk count: $chunkCount"
157
158 mkdir -p $out/${evalSystem}
159
160 # Record and print stats on free memory and swap in the background
161 (
162 while true; do
163 availMemory=$(free -b | grep Mem | awk '{print $7}')
164 freeSwap=$(free -b | grep Swap | awk '{print $4}')
165 echo "Available memory: $(( availMemory / 1024 / 1024 )) MiB, free swap: $(( freeSwap / 1024 / 1024 )) MiB"
166
167 if [[ ! -f "$out/${evalSystem}/min-avail-memory" ]] || (( availMemory < $(<$out/${evalSystem}/min-avail-memory) )); then
168 echo "$availMemory" > $out/${evalSystem}/min-avail-memory
169 fi
170 if [[ ! -f $out/${evalSystem}/min-free-swap ]] || (( availMemory < $(<$out/${evalSystem}/min-free-swap) )); then
171 echo "$freeSwap" > $out/${evalSystem}/min-free-swap
172 fi
173 sleep 4
174 done
175 ) &
176
177 seq_end=$(( chunkCount - 1 ))
178
179 ${lib.optionalString quickTest ''
180 seq_end=0
181 ''}
182
183 chunkOutputDir=$(mktemp -d)
184 mkdir "$chunkOutputDir"/{result,stats,timestats,stderr}
185
186 seq -w 0 "$seq_end" |
187 command time -f "%e" -o "$out/${evalSystem}/total-time" \
188 xargs -I{} -P"$cores" \
189 ${singleChunk} "$chunkSize" {} "$evalSystem" "$chunkOutputDir"
190
191 cp -r "$chunkOutputDir"/stats $out/${evalSystem}/stats-by-chunk
192
193 if (( chunkSize * chunkCount != attrCount )); then
194 # A final incomplete chunk would mess up the stats, don't include it
195 rm "$chunkOutputDir"/stats/"$seq_end"
196 fi
197
198 cat "$chunkOutputDir"/result/* | jq -s 'add | map_values(.outputs)' > $out/${evalSystem}/paths.json
199 '';
200
201 diff = callPackage ./diff.nix { };
202
203 combine =
204 {
205 diffDir,
206 }:
207 runCommand "combined-eval"
208 {
209 nativeBuildInputs = [
210 jq
211 ];
212 }
213 ''
214 mkdir -p $out
215
216 # Combine output paths from all systems
217 cat ${diffDir}/*/diff.json | jq -s '
218 reduce .[] as $item ({}; {
219 added: (.added + $item.added),
220 changed: (.changed + $item.changed),
221 removed: (.removed + $item.removed)
222 })
223 ' > $out/combined-diff.json
224
225 mkdir -p $out/before/stats
226 for d in ${diffDir}/before/*; do
227 cp -r "$d"/stats-by-chunk $out/before/stats/$(basename "$d")
228 done
229
230 mkdir -p $out/after/stats
231 for d in ${diffDir}/after/*; do
232 cp -r "$d"/stats-by-chunk $out/after/stats/$(basename "$d")
233 done
234 '';
235
236 compare = callPackage ./compare { };
237
238 full =
239 {
240 # Whether to evaluate on a specific set of systems, by default all are evaluated
241 evalSystems ? if quickTest then [ "x86_64-linux" ] else supportedSystems,
242 # The number of attributes per chunk, see ./README.md for more info.
243 chunkSize,
244 quickTest ? false,
245 }:
246 let
247 diffs = symlinkJoin {
248 name = "diffs";
249 paths = map (
250 evalSystem:
251 let
252 eval = singleSystem {
253 inherit quickTest evalSystem chunkSize;
254 };
255 in
256 diff {
257 inherit evalSystem;
258 # Local "full" evaluation doesn't do a real diff.
259 beforeDir = eval;
260 afterDir = eval;
261 }
262 ) evalSystems;
263 };
264 in
265 combine {
266 diffDir = diffs;
267 };
268
269in
270{
271 inherit
272 attrpathsSuperset
273 singleSystem
274 diff
275 combine
276 compare
277 # The above three are used by separate VMs in a GitHub workflow,
278 # while the below is intended for testing on a single local machine
279 full
280 ;
281}