tjh.dev
nixpkgs mirror (for testing)
github.com/NixOS/nixpkgs
nix
1name: Eval
2
3on:
4 workflow_call:
5 inputs:
6 artifact-prefix:
7 required: true
8 type: string
9 mergedSha:
10 required: true
11 type: string
12 headSha:
13 required: false # only required when testVersions is true
14 type: string
15 targetSha:
16 required: true
17 type: string
18 systems:
19 required: true
20 type: string
21 testVersions:
22 required: false
23 default: false
24 type: boolean
25 secrets:
26 # Should only be provided in the merge queue, not in pull requests,
27 # where we're evaluating untrusted code.
28 CACHIX_AUTH_TOKEN_GHA:
29 required: false
30
31permissions: {}
32
33defaults:
34 run:
35 shell: bash
36
37jobs:
38 versions:
39 if: inputs.testVersions
40 runs-on: ubuntu-slim
41 outputs:
42 versions: ${{ steps.versions.outputs.versions }}
43 ciPinBumpCommit: ${{ steps.find-pinned-commit.outputs.ciPinBumpCommit }}
44 ciPinBumpCommitShort: ${{ steps.find-pinned-commit.outputs.ciPinBumpCommitShort }}
45 steps:
46 - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
47 with:
48 persist-credentials: false
49 path: trusted
50 sparse-checkout: |
51 ci/supportedVersions.nix
52
53 - name: Check out the PR at the test merge commit
54 uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
55 with:
56 persist-credentials: false
57 ref: ${{ inputs.mergedSha }}
58 path: untrusted
59 sparse-checkout: |
60 ci/pinned.json
61
62 - name: Find commit that touched ci/pinned.json
63 id: find-pinned-commit
64 uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
65 env:
66 TARGET_SHA: ${{ inputs.targetSha }}
67 HEAD_SHA: ${{ inputs.headSha }}
68 with:
69 script: |
70 const targetSha = process.env.TARGET_SHA
71 const headSha = process.env.HEAD_SHA
72
73 if (!targetSha || !headSha) {
74 core.setFailed('Error: Both targetSha and headSha inputs are required when testVersions is true.')
75 return
76 }
77
78 // Compare the two commits to get the list of commits in between
79 const comparison = await github.rest.repos.compareCommitsWithBasehead({
80 ...context.repo,
81 basehead: `${targetSha}...${headSha}`,
82 })
83
84 if(comparison.data.commits.length > 50) {
85 core.setFailed('Error: Too many commits in comparison, cannot reliably find pinned.json change.')
86 return
87 }
88
89 const logRateLimit = async (label) => {
90 const { data } = await github.rest.rateLimit.get()
91 const { remaining, limit, used } = data.rate
92 core.info(`[Rate Limit ${label}] ${remaining}/${limit} remaining (${used} used)`)
93 }
94
95 await logRateLimit('before commit filtering')
96
97 // Filter commits that modified ci/pinned.json
98 const commitsModifyingPinned = (
99 await Promise.all(
100 comparison.data.commits.map(async (commit) => {
101 const commitDetails = await github.rest.repos.getCommit({
102 ...context.repo,
103 ref: commit.sha,
104 })
105 const modifiesPinned = commitDetails.data.files?.some(
106 (file) => file.filename === "ci/pinned.json"
107 )
108 return modifiesPinned ? commit.sha : null
109 })
110 )
111 ).filter((sha) => sha !== null)
112
113 await logRateLimit('after commit filtering')
114
115 if (commitsModifyingPinned.length === 0) {
116 // This should not happen as testVersions should only be true
117 // when ci/pinned.json was modified in the PR.
118 core.setFailed("Error: ci/pinned.json was not modified in this PR")
119 return
120 } else if (commitsModifyingPinned.length > 1) {
121 core.setFailed([
122 "Error: Multiple commits touch ci/pinned.json in this PR:",
123 ...commitsModifyingPinned,
124 "Please ensure only a single commit modifies ci/pinned.json for accurate version matrix evaluation."
125 ].join("\n"))
126 return
127 }
128
129 const ciPinBumpCommit = commitsModifyingPinned[0]
130 core.setOutput("ciPinBumpCommit", ciPinBumpCommit)
131 core.setOutput("ciPinBumpCommitShort", ciPinBumpCommit.substring(0, 7))
132 core.info(`Found pinned.json commit: ${ciPinBumpCommit}`)
133
134 - name: Install Nix
135 uses: cachix/install-nix-action@1ca7d21a94afc7c957383a2d217460d980de4934 # v31.10.1
136
137 - name: Load supported versions
138 id: versions
139 run: |
140 echo "versions=$(trusted/ci/supportedVersions.nix --arg pinnedJson untrusted/ci/pinned.json)" >> "$GITHUB_OUTPUT"
141
142 eval:
143 runs-on: ubuntu-24.04-arm
144 needs: versions
145 if: ${{ !cancelled() && !failure() }}
146 strategy:
147 fail-fast: false
148 matrix:
149 system: ${{ fromJSON(inputs.systems) }}
150 version:
151 - "" # Default Eval triggering rebuild labels and such.
152 - ${{ fromJSON(needs.versions.outputs.versions || '[]') }} # Only for ci/pinned.json updates.
153 # Failures for versioned Evals will be collected in a separate job below
154 # to not interrupt main Eval's compare step.
155 continue-on-error: ${{ matrix.version != '' }}
156 name: ${{ matrix.system }}${{ matrix.version && format(' @ {0} ({1})', matrix.version, needs.versions.outputs.ciPinBumpCommitShort) || '' }}
157 timeout-minutes: 15
158 steps:
159 # This is not supposed to be used and just acts as a fallback.
160 # Without swap, when Eval runs OOM, it will fail badly with a
161 # job that is sometimes not interruptible anymore.
162 # If Eval starts swapping, decrease chunkSize to keep it fast.
163 - name: Enable swap
164 run: |
165 sudo fallocate -l 10G /swap
166 sudo chmod 600 /swap
167 sudo mkswap /swap
168 sudo swapon /swap
169
170 - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
171 with:
172 persist-credentials: false
173 sparse-checkout: .github/actions
174 - name: Check out the PR at merged and target commits
175 uses: ./.github/actions/checkout
176 with:
177 # For versioned evals, use the target as the untrusted base and apply the pin-bump commit
178 merged-as-untrusted-at: ${{ matrix.version && inputs.targetSha || inputs.mergedSha }}
179 untrusted-pin-bump: ${{ matrix.version && needs.versions.outputs.ciPinBumpCommit }}
180 target-as-trusted-at: ${{ inputs.targetSha }}
181
182 - name: Install Nix
183 uses: cachix/install-nix-action@1ca7d21a94afc7c957383a2d217460d980de4934 # v31.10.1
184
185 - uses: cachix/cachix-action@1eb2ef646ac0255473d23a5907ad7b04ce94065c # v17
186 continue-on-error: true
187 with:
188 # The nixpkgs-gha cache should not be trusted or used outside of Nixpkgs and its forks' CI.
189 name: ${{ vars.CACHIX_NAME || 'nixpkgs-gha' }}
190 extraPullNames: nixpkgs-gha
191 authToken: ${{ secrets.CACHIX_AUTH_TOKEN_GHA }}
192 pushFilter: '(-source|-single-chunk)$'
193
194 - name: Evaluate the ${{ matrix.system }} output paths at the merge commit
195 env:
196 MATRIX_SYSTEM: ${{ matrix.system }}
197 MATRIX_VERSION: ${{ matrix.version || 'nixVersions.latest' }}
198 run: |
199 nix-build nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A eval.singleSystem \
200 --argstr evalSystem "$MATRIX_SYSTEM" \
201 --arg chunkSize 8000 \
202 --argstr nixPath "$MATRIX_VERSION" \
203 --out-link merged
204 # If it uses too much memory, slightly decrease chunkSize.
205 # Note: Keep the same further down in sync!
206
207 - name: Evaluate the ${{ matrix.system }} output paths at the target commit
208 env:
209 MATRIX_SYSTEM: ${{ matrix.system }}
210 run: |
211 TARGET_DRV=$(nix-instantiate nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A eval.singleSystem \
212 --argstr evalSystem "$MATRIX_SYSTEM" \
213 --arg chunkSize 8000 \
214 --argstr nixPath "nixVersions.latest")
215
216 # Try to fetch this from Cachix a few times, for up to 30 seconds. This avoids running Eval
217 # twice in the Merge Queue, when a later item finishes Eval at the merge commit earlier.
218 for _i in {1..6}; do
219 # Using --max-jobs 0 will cause nix-build to fail if this can't be substituted from cachix.
220 if nix-build "$TARGET_DRV" --max-jobs 0; then
221 break
222 fi
223 sleep 5
224 done
225
226 # Either fetches from Cachix or runs Eval itself. The fallback is required
227 # for pull requests into wip-branches without merge queue.
228 nix-build "$TARGET_DRV" --out-link target
229
230 - name: Compare outpaths against the target branch
231 env:
232 MATRIX_SYSTEM: ${{ matrix.system }}
233 run: |
234 nix-build nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A eval.diff \
235 --arg beforeDir ./target \
236 --arg afterDir ./merged \
237 --argstr evalSystem "$MATRIX_SYSTEM" \
238 --out-link diff
239
240 - name: Upload outpaths diff and stats
241 uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
242 with:
243 name: ${{ inputs.artifact-prefix }}${{ matrix.version && format('{0}-', matrix.version) || '' }}diff-${{ matrix.system }}
244 path: diff/*
245
246 compare:
247 runs-on: ubuntu-24.04-arm
248 needs: [eval]
249 if: ${{ !cancelled() && !failure() }}
250 permissions:
251 pull-requests: write # submitting 'wrong branch' reviews
252 statuses: write # creating 'Eval Summary' commit statuses
253 timeout-minutes: 5
254 steps:
255 - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
256 with:
257 persist-credentials: false
258 sparse-checkout: .github/actions
259 - name: Check out the PR at the target commit
260 uses: ./.github/actions/checkout
261 with:
262 merged-as-untrusted-at: ${{ inputs.mergedSha }}
263 target-as-trusted-at: ${{ inputs.targetSha }}
264
265 - name: Download output paths and eval stats for all systems
266 uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
267 with:
268 pattern: ${{ inputs.artifact-prefix }}diff-*
269 path: diff
270 merge-multiple: true
271
272 - name: Install Nix
273 uses: cachix/install-nix-action@1ca7d21a94afc7c957383a2d217460d980de4934 # v31.10.1
274
275 - name: Combine all output paths and eval stats
276 run: |
277 nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A eval.combine \
278 --arg diffDir ./diff \
279 --out-link combined
280
281 - name: Upload the maintainer list
282 uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
283 with:
284 name: ${{ inputs.artifact-prefix }}maintainers
285 path: combined/maintainers.json
286
287 - name: Compare against the target branch
288 env:
289 TARGET_SHA: ${{ inputs.mergedSha }}
290 run: |
291 git -C nixpkgs/trusted diff --name-only "$TARGET_SHA" \
292 | jq --raw-input --slurp 'split("\n")[:-1]' > touched-files.json
293
294 # Use the target branch to get accurate maintainer info
295 nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A eval.compare \
296 --arg combinedDir ./combined \
297 --arg touchedFilesJson ./touched-files.json \
298 --out-link comparison
299
300 cat comparison/step-summary.md >> "$GITHUB_STEP_SUMMARY"
301
302 - name: Upload the comparison results
303 uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
304 with:
305 name: ${{ inputs.artifact-prefix }}comparison
306 path: comparison/*
307
308 - name: Add eval summary to commit statuses
309 if: ${{ github.event_name == 'pull_request_target' }}
310 uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
311 with:
312 script: |
313 const { readFile } = require('node:fs/promises')
314 const changed = JSON.parse(await readFile('comparison/changed-paths.json', 'utf-8'))
315 const description =
316 'Package: ' + [
317 `added ${changed.attrdiff.added.length}`,
318 `removed ${changed.attrdiff.removed.length}`,
319 `changed ${changed.attrdiff.changed.length}`
320 ].join(', ') +
321 ' — Rebuild: ' + [
322 `linux ${changed.rebuildCountByKernel.linux}`,
323 `darwin ${changed.rebuildCountByKernel.darwin}`
324 ].join(', ')
325
326 const { serverUrl, repo, runId, payload } = context
327 const target_url =
328 `${serverUrl}/${repo.owner}/${repo.repo}/actions/runs/${runId}?pr=${payload.pull_request.number}`
329
330 await github.rest.repos.createCommitStatus({
331 ...repo,
332 sha: payload.pull_request.head.sha,
333 context: 'Eval Summary',
334 state: 'success',
335 description,
336 target_url
337 })
338 - name: Request changes if PR is against an inappropriate branch
339 if: ${{ github.event_name == 'pull_request_target' }}
340 uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
341 with:
342 script: |
343 require('./nixpkgs/trusted/ci/github-script/check-target-branch.js')({
344 github,
345 context,
346 core,
347 dry: context.eventName == 'pull_request',
348 })
349
350 # Creates a matrix of Eval performance for various versions and systems.
351 report:
352 runs-on: ubuntu-slim
353 needs: [versions, eval]
354 steps:
355 - name: Download output paths and eval stats for all versions
356 uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
357 with:
358 pattern: "*-diff-*"
359 path: versions
360
361 - name: Add version comparison table to job summary
362 uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
363 env:
364 ARTIFACT_PREFIX: ${{ inputs.artifact-prefix }}
365 SYSTEMS: ${{ inputs.systems }}
366 VERSIONS: ${{ needs.versions.outputs.versions }}
367 CI_PIN_BUMP_COMMIT: ${{ needs.versions.outputs.ciPinBumpCommit }}
368 with:
369 script: |
370 const { readFileSync } = require('node:fs')
371 const path = require('node:path')
372
373 const prefix = process.env.ARTIFACT_PREFIX
374 const systems = JSON.parse(process.env.SYSTEMS)
375 const versions = JSON.parse(process.env.VERSIONS)
376 const ciPinBumpCommit = process.env.CI_PIN_BUMP_COMMIT
377
378 core.summary.addHeading('Lix/Nix version comparison')
379 core.summary.addRaw(`\n*Evaluated at commit: \`${ciPinBumpCommit}\` (commit that modified ci/pinned.json)*\n`, true)
380 core.summary.addTable(
381 [].concat(
382 [
383 [{ data: 'Version', header: true }].concat(
384 systems.map((system) => ({ data: system, header: true })),
385 ),
386 ],
387 versions.map((version) =>
388 [{ data: version }].concat(
389 systems.map((system) => {
390 try {
391 const artifact = path.join('versions', `${prefix}${version}-diff-${system}`)
392 const time = Math.round(
393 parseFloat(
394 readFileSync(
395 path.join(artifact, 'after', system, 'total-time'),
396 'utf-8',
397 ),
398 ),
399 )
400 const diff = JSON.parse(
401 readFileSync(path.join(artifact, system, 'diff.json'), 'utf-8'),
402 )
403 const attrs = []
404 .concat(diff.added, diff.removed, diff.changed, diff.rebuilds)
405 // There are some special attributes, which are ignored for rebuilds.
406 // These only have a single path component, because they lack the `.<system>` suffix.
407 .filter((attr) => attr.split('.').length > 1)
408 if (attrs.length > 0) {
409 core.setFailed(
410 `${version} on ${system} has changed outpaths!\n` +
411 `Note: This indicates that commit ${ciPinBumpCommit} ` +
412 `(which modified ci/pinned.json) also contains other ` +
413 `changes affecting package outputs. ` +
414 `Please ensure ci/pinned.json is updated in a standalone commit.`
415 )
416 return { data: ':x:' }
417 }
418 return { data: time }
419 } catch {
420 core.warning(`${version} on ${system} did not produce artifact.`)
421 return { data: ':warning:' }
422 }
423 }),
424 ),
425 ),
426 ),
427 )
428 core.summary.addRaw(
429 '\n*Evaluation time in seconds without downloading dependencies.*',
430 true,
431 )
432 core.summary.addRaw('\n*:warning: Job did not report a result.*', true)
433 core.summary.addRaw(
434 '\n*:x: Job produced different outpaths than the target branch.*',
435 true,
436 )
437 core.summary.write()
438
439 misc:
440 if: ${{ github.event_name != 'push' }}
441 runs-on: ubuntu-24.04-arm
442 timeout-minutes: 10
443 steps:
444 - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
445 with:
446 persist-credentials: false
447 sparse-checkout: .github/actions
448 - name: Checkout the merge commit
449 uses: ./.github/actions/checkout
450 with:
451 merged-as-untrusted-at: ${{ inputs.mergedSha }}
452
453 - name: Install Nix
454 uses: cachix/install-nix-action@1ca7d21a94afc7c957383a2d217460d980de4934 # v31.10.1
455
456 - name: Ensure flake outputs on all systems still evaluate
457 run: nix flake check --all-systems --no-build './nixpkgs/untrusted?shallow=1'
458
459 - name: Query nixpkgs with aliases enabled to check for basic syntax errors
460 run: |
461 time nix-env -I ./nixpkgs/untrusted -f ./nixpkgs/untrusted -qa '*' --option restrict-eval true --option allow-import-from-derivation false >/dev/null
462
463 - name: Ensure NixOS modules meta is valid
464 run: |
465 time nix-instantiate -I ./nixpkgs/untrusted --strict --eval --json ./nixpkgs/untrusted/nixos --arg configuration '{}' --attr config.meta --option restrict-eval true --option allow-import-from-derivation false