tjh.dev
1name: Eval
2
3on:
4 pull_request:
5 paths:
6 - .github/workflows/eval.yml
7 pull_request_target:
8 types: [opened, ready_for_review, synchronize, reopened]
9 push:
10 # Keep this synced with ci/request-reviews/dev-branches.txt
11 branches:
12 - master
13 - staging
14 - release-*
15 - staging-*
16 - haskell-updates
17 - python-updates
18
19permissions: {}
20
21jobs:
22 get-merge-commit:
23 uses: ./.github/workflows/get-merge-commit.yml
24
25 outpaths:
26 name: Outpaths
27 runs-on: ubuntu-24.04-arm
28 needs: [ get-merge-commit ]
29 strategy:
30 fail-fast: false
31 matrix:
32 system: ${{ fromJSON(needs.get-merge-commit.outputs.systems) }}
33 steps:
34 - name: Enable swap
35 run: |
36 sudo fallocate -l 10G /swap
37 sudo chmod 600 /swap
38 sudo mkswap /swap
39 sudo swapon /swap
40
41 - name: Check out the PR at the test merge commit
42 uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
43 with:
44 ref: ${{ needs.get-merge-commit.outputs.mergedSha }}
45 path: nixpkgs
46
47 - name: Install Nix
48 uses: cachix/install-nix-action@526118121621777ccd86f79b04685a9319637641 # v31
49 with:
50 extra_nix_config: sandbox = true
51
52 - name: Evaluate the ${{ matrix.system }} output paths for all derivation attributes
53 env:
54 MATRIX_SYSTEM: ${{ matrix.system }}
55 run: |
56 nix-build nixpkgs/ci -A eval.singleSystem \
57 --argstr evalSystem "$MATRIX_SYSTEM" \
58 --arg chunkSize 10000
59 # If it uses too much memory, slightly decrease chunkSize
60
61 - name: Upload the output paths and eval stats
62 uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
63 with:
64 name: intermediate-${{ matrix.system }}
65 path: result/*
66
67 process:
68 name: Process
69 runs-on: ubuntu-24.04-arm
70 needs: [ outpaths, get-merge-commit ]
71 outputs:
72 targetRunId: ${{ steps.targetRunId.outputs.targetRunId }}
73 steps:
74 - name: Download output paths and eval stats for all systems
75 uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
76 with:
77 pattern: intermediate-*
78 path: intermediate
79
80 - name: Check out the PR at the test merge commit
81 uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
82 with:
83 ref: ${{ needs.get-merge-commit.outputs.mergedSha }}
84 fetch-depth: 2
85 path: nixpkgs
86
87 - name: Install Nix
88 uses: cachix/install-nix-action@526118121621777ccd86f79b04685a9319637641 # v31
89 with:
90 extra_nix_config: sandbox = true
91
92 - name: Combine all output paths and eval stats
93 run: |
94 nix-build nixpkgs/ci -A eval.combine \
95 --arg resultsDir ./intermediate \
96 -o prResult
97
98 - name: Upload the combined results
99 uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
100 with:
101 name: result
102 path: prResult/*
103
104 - name: Get target run id
105 if: needs.get-merge-commit.outputs.targetSha
106 id: targetRunId
107 run: |
108 # Get the latest eval.yml workflow run for the PR's target commit
109 if ! run=$(gh api --method GET /repos/"$REPOSITORY"/actions/workflows/eval.yml/runs \
110 -f head_sha="$TARGET_SHA" -f event=push \
111 --jq '.workflow_runs | sort_by(.run_started_at) | .[-1]') \
112 || [[ -z "$run" ]]; then
113 echo "Could not find an eval.yml workflow run for $TARGET_SHA, cannot make comparison"
114 exit 1
115 fi
116 echo "Comparing against $(jq .html_url <<< "$run")"
117 runId=$(jq .id <<< "$run")
118 conclusion=$(jq -r .conclusion <<< "$run")
119
120 while [[ "$conclusion" == null || "$conclusion" == "" ]]; do
121 echo "Workflow not done, waiting 10 seconds before checking again"
122 sleep 10
123 conclusion=$(gh api /repos/"$REPOSITORY"/actions/runs/"$runId" --jq '.conclusion')
124 done
125
126 if [[ "$conclusion" != "success" ]]; then
127 echo "Workflow was not successful (conclusion: $conclusion), cannot make comparison"
128 exit 1
129 fi
130
131 echo "targetRunId=$runId" >> "$GITHUB_OUTPUT"
132 env:
133 REPOSITORY: ${{ github.repository }}
134 TARGET_SHA: ${{ needs.get-merge-commit.outputs.targetSha }}
135 GH_TOKEN: ${{ github.token }}
136
137 - uses: actions/download-artifact@v4
138 if: steps.targetRunId.outputs.targetRunId
139 with:
140 name: result
141 path: targetResult
142 github-token: ${{ github.token }}
143 run-id: ${{ steps.targetRunId.outputs.targetRunId }}
144
145 - name: Compare against the target branch
146 if: steps.targetRunId.outputs.targetRunId
147 run: |
148 git -C nixpkgs worktree add ../target ${{ needs.get-merge-commit.outputs.targetSha }}
149 git -C nixpkgs diff --name-only ${{ needs.get-merge-commit.outputs.targetSha }} \
150 | jq --raw-input --slurp 'split("\n")[:-1]' > touched-files.json
151
152 # Use the target branch to get accurate maintainer info
153 nix-build target/ci -A eval.compare \
154 --arg beforeResultDir ./targetResult \
155 --arg afterResultDir "$(realpath prResult)" \
156 --arg touchedFilesJson ./touched-files.json \
157 --argstr githubAuthorId "$AUTHOR_ID" \
158 -o comparison
159
160 cat comparison/step-summary.md >> "$GITHUB_STEP_SUMMARY"
161 env:
162 AUTHOR_ID: ${{ github.event.pull_request.user.id }}
163
164 - name: Upload the combined results
165 if: steps.targetRunId.outputs.targetRunId
166 uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
167 with:
168 name: comparison
169 path: comparison/*
170
171 # Separate job to have a very tightly scoped PR write token
172 tag:
173 name: Tag
174 runs-on: ubuntu-24.04-arm
175 needs: [ get-merge-commit, process ]
176 if: needs.process.outputs.targetRunId
177 permissions:
178 pull-requests: write
179 statuses: write
180 steps:
181 # See ./codeowners-v2.yml, reuse the same App because we need the same permissions
182 # Can't use the token received from permissions above, because it can't get enough permissions
183 - uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v2.0.6
184 if: vars.OWNER_APP_ID
185 id: app-token
186 with:
187 app-id: ${{ vars.OWNER_APP_ID }}
188 private-key: ${{ secrets.OWNER_APP_PRIVATE_KEY }}
189 permission-administration: read
190 permission-members: read
191 permission-pull-requests: write
192
193 - name: Download process result
194 uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
195 with:
196 name: comparison
197 path: comparison
198
199 - name: Install Nix
200 uses: cachix/install-nix-action@526118121621777ccd86f79b04685a9319637641 # v31
201
202 # Important: This workflow job runs with extra permissions,
203 # so we need to make sure to not run untrusted code from PRs
204 - name: Check out Nixpkgs at the base commit
205 uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
206 with:
207 ref: ${{ needs.get-merge-commit.outputs.targetSha }}
208 path: base
209 sparse-checkout: ci
210
211 - name: Build the requestReviews derivation
212 run: nix-build base/ci -A requestReviews
213
214 - name: Labelling pull request
215 if: ${{ github.event_name == 'pull_request_target' && github.repository_owner == 'NixOS' }}
216 run: |
217 # Get all currently set labels that we manage
218 gh api \
219 /repos/"$REPOSITORY"/issues/"$NUMBER"/labels \
220 --jq '.[].name | select(startswith("10.rebuild") or . == "11.by: package-maintainer")' \
221 | sort > before
222
223 # And the labels that should be there
224 jq -r '.labels[]' comparison/changed-paths.json \
225 | sort > after
226
227 # Remove the ones not needed anymore
228 while read -r toRemove; do
229 echo "Removing label $toRemove"
230 gh api \
231 --method DELETE \
232 /repos/"$REPOSITORY"/issues/"$NUMBER"/labels/"$toRemove"
233 done < <(comm -23 before after)
234
235 # And add the ones that aren't set already
236 while read -r toAdd; do
237 echo "Adding label $toAdd"
238 gh api \
239 --method POST \
240 /repos/"$REPOSITORY"/issues/"$NUMBER"/labels \
241 -f "labels[]=$toAdd"
242 done < <(comm -13 before after)
243
244 env:
245 GH_TOKEN: ${{ github.token }}
246 REPOSITORY: ${{ github.repository }}
247 NUMBER: ${{ github.event.number }}
248
249 - name: Add eval summary to commit statuses
250 if: ${{ github.event_name == 'pull_request_target' && github.repository_owner == 'NixOS' }}
251 run: |
252 description=$(jq -r '
253 "Package: added " + (.attrdiff.added | length | tostring) +
254 ", removed " + (.attrdiff.removed | length | tostring) +
255 ", changed " + (.attrdiff.changed | length | tostring) +
256 ", Rebuild: linux " + (.rebuildCountByKernel.linux | tostring) +
257 ", darwin " + (.rebuildCountByKernel.darwin | tostring)
258 ' <comparison/changed-paths.json)
259 target_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID?pr=$NUMBER"
260 gh api --method POST \
261 -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" \
262 "/repos/$GITHUB_REPOSITORY/statuses/$PR_HEAD_SHA" \
263 -f "context=Eval / Summary" -f "state=success" -f "description=$description" -f "target_url=$target_url"
264 env:
265 GH_TOKEN: ${{ github.token }}
266 PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
267 NUMBER: ${{ github.event.number }}
268
269 - name: Requesting maintainer reviews
270 if: ${{ steps.app-token.outputs.token && github.repository_owner == 'NixOS' }}
271 run: |
272 # maintainers.json contains GitHub IDs. Look up handles to request reviews from.
273 # There appears to be no API to request reviews based on GitHub IDs
274 jq -r 'keys[]' comparison/maintainers.json \
275 | while read -r id; do gh api /user/"$id" --jq .login; done \
276 | GH_TOKEN=${{ steps.app-token.outputs.token }} result/bin/request-reviewers.sh "$REPOSITORY" "$NUMBER" "$AUTHOR"
277
278 env:
279 GH_TOKEN: ${{ github.token }}
280 REPOSITORY: ${{ github.repository }}
281 NUMBER: ${{ github.event.number }}
282 AUTHOR: ${{ github.event.pull_request.user.login }}
283 # Don't request reviewers on draft PRs
284 DRY_MODE: ${{ github.event.pull_request.draft && '1' || '' }}