tjh.dev
Clone of https://github.com/NixOS/nixpkgs.git (to stress-test knotserver)
1name: Eval
2
3on:
4 pull_request:
5 paths:
6 - .github/workflows/eval.yml
7 pull_request_target:
8 types: [opened, ready_for_review, synchronize, reopened]
9 push:
10 # Keep this synced with ci/request-reviews/dev-branches.txt
11 branches:
12 - master
13 - staging
14 - release-*
15 - staging-*
16 - haskell-updates
17 - python-updates
18
19permissions: {}
20
21jobs:
22 get-merge-commit:
23 uses: ./.github/workflows/get-merge-commit.yml
24
25 outpaths:
26 name: Outpaths
27 runs-on: ubuntu-24.04-arm
28 needs: [ get-merge-commit ]
29 strategy:
30 fail-fast: false
31 matrix:
32 system: ${{ fromJSON(needs.get-merge-commit.outputs.systems) }}
33 steps:
34 - name: Enable swap
35 run: |
36 sudo fallocate -l 10G /swap
37 sudo chmod 600 /swap
38 sudo mkswap /swap
39 sudo swapon /swap
40
41 - name: Check out the PR at the test merge commit
42 uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
43 with:
44 ref: ${{ needs.get-merge-commit.outputs.mergedSha }}
45 path: nixpkgs
46
47 - name: Install Nix
48 uses: cachix/install-nix-action@526118121621777ccd86f79b04685a9319637641 # v31
49 with:
50 extra_nix_config: sandbox = true
51
52 - name: Evaluate the ${{ matrix.system }} output paths for all derivation attributes
53 env:
54 MATRIX_SYSTEM: ${{ matrix.system }}
55 run: |
56 nix-build nixpkgs/ci -A eval.singleSystem \
57 --argstr evalSystem "$MATRIX_SYSTEM" \
58 --arg chunkSize 10000
59 # If it uses too much memory, slightly decrease chunkSize
60
61 - name: Upload the output paths and eval stats
62 uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
63 with:
64 name: intermediate-${{ matrix.system }}
65 path: result/*
66
67 process:
68 name: Process
69 runs-on: ubuntu-24.04-arm
70 needs: [ outpaths, get-merge-commit ]
71 outputs:
72 targetRunId: ${{ steps.targetRunId.outputs.targetRunId }}
73 steps:
74 - name: Download output paths and eval stats for all systems
75 uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
76 with:
77 pattern: intermediate-*
78 path: intermediate
79
80 - name: Check out the PR at the test merge commit
81 uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
82 with:
83 ref: ${{ needs.get-merge-commit.outputs.mergedSha }}
84 fetch-depth: 2
85 path: nixpkgs
86
87 - name: Install Nix
88 uses: cachix/install-nix-action@526118121621777ccd86f79b04685a9319637641 # v31
89 with:
90 extra_nix_config: sandbox = true
91
92 - name: Combine all output paths and eval stats
93 run: |
94 nix-build nixpkgs/ci -A eval.combine \
95 --arg resultsDir ./intermediate \
96 -o prResult
97
98 - name: Upload the combined results
99 uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
100 with:
101 name: result
102 path: prResult/*
103
104 - name: Get target run id
105 if: needs.get-merge-commit.outputs.targetSha
106 id: targetRunId
107 run: |
108 # Get the latest eval.yml workflow run for the PR's target commit
109 if ! run=$(gh api --method GET /repos/"$REPOSITORY"/actions/workflows/eval.yml/runs \
110 -f head_sha="$TARGET_SHA" -f event=push \
111 --jq '.workflow_runs | sort_by(.run_started_at) | .[-1]') \
112 || [[ -z "$run" ]]; then
113 echo "Could not find an eval.yml workflow run for $TARGET_SHA, cannot make comparison"
114 exit 1
115 fi
116 echo "Comparing against $(jq .html_url <<< "$run")"
117 runId=$(jq .id <<< "$run")
118 conclusion=$(jq -r .conclusion <<< "$run")
119
120 while [[ "$conclusion" == null || "$conclusion" == "" ]]; do
121 echo "Workflow not done, waiting 10 seconds before checking again"
122 sleep 10
123 conclusion=$(gh api /repos/"$REPOSITORY"/actions/runs/"$runId" --jq '.conclusion')
124 done
125
126 if [[ "$conclusion" != "success" ]]; then
127 echo "Workflow was not successful (conclusion: $conclusion), cannot make comparison"
128 exit 1
129 fi
130
131 echo "targetRunId=$runId" >> "$GITHUB_OUTPUT"
132 env:
133 REPOSITORY: ${{ github.repository }}
134 TARGET_SHA: ${{ needs.get-merge-commit.outputs.targetSha }}
135 GH_TOKEN: ${{ github.token }}
136
137 - uses: actions/download-artifact@v4
138 if: steps.targetRunId.outputs.targetRunId
139 with:
140 name: result
141 path: targetResult
142 github-token: ${{ github.token }}
143 run-id: ${{ steps.targetRunId.outputs.targetRunId }}
144
145 - name: Compare against the target branch
146 if: steps.targetRunId.outputs.targetRunId
147 run: |
148 git -C nixpkgs worktree add ../target ${{ needs.get-merge-commit.outputs.targetSha }}
149 git -C nixpkgs diff --name-only ${{ needs.get-merge-commit.outputs.targetSha }} \
150 | jq --raw-input --slurp 'split("\n")[:-1]' > touched-files.json
151
152 # Use the target branch to get accurate maintainer info
153 nix-build target/ci -A eval.compare \
154 --arg beforeResultDir ./targetResult \
155 --arg afterResultDir "$(realpath prResult)" \
156 --arg touchedFilesJson ./touched-files.json \
157 -o comparison
158
159 cat comparison/step-summary.md >> "$GITHUB_STEP_SUMMARY"
160
161 - name: Upload the combined results
162 if: steps.targetRunId.outputs.targetRunId
163 uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
164 with:
165 name: comparison
166 path: comparison/*
167
168 # Separate job to have a very tightly scoped PR write token
169 tag:
170 name: Tag
171 runs-on: ubuntu-24.04-arm
172 needs: [ get-merge-commit, process ]
173 if: needs.process.outputs.targetRunId
174 permissions:
175 pull-requests: write
176 statuses: write
177 steps:
178 # See ./codeowners-v2.yml, reuse the same App because we need the same permissions
179 # Can't use the token received from permissions above, because it can't get enough permissions
180 - uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v2.0.6
181 if: vars.OWNER_APP_ID
182 id: app-token
183 with:
184 app-id: ${{ vars.OWNER_APP_ID }}
185 private-key: ${{ secrets.OWNER_APP_PRIVATE_KEY }}
186 permission-administration: read
187 permission-members: read
188 permission-pull-requests: write
189
190 - name: Download process result
191 uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
192 with:
193 name: comparison
194 path: comparison
195
196 - name: Install Nix
197 uses: cachix/install-nix-action@526118121621777ccd86f79b04685a9319637641 # v31
198
199 # Important: This workflow job runs with extra permissions,
200 # so we need to make sure to not run untrusted code from PRs
201 - name: Check out Nixpkgs at the base commit
202 uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
203 with:
204 ref: ${{ needs.get-merge-commit.outputs.targetSha }}
205 path: base
206 sparse-checkout: ci
207
208 - name: Build the requestReviews derivation
209 run: nix-build base/ci -A requestReviews
210
211 - name: Labelling pull request
212 if: ${{ github.event_name == 'pull_request_target' && github.repository_owner == 'NixOS' }}
213 run: |
214 # Get all currently set rebuild labels
215 gh api \
216 /repos/"$REPOSITORY"/issues/"$NUMBER"/labels \
217 --jq '.[].name | select(startswith("10.rebuild"))' \
218 | sort > before
219
220 # And the labels that should be there
221 jq -r '.labels[]' comparison/changed-paths.json \
222 | sort > after
223
224 # Remove the ones not needed anymore
225 while read -r toRemove; do
226 echo "Removing label $toRemove"
227 gh api \
228 --method DELETE \
229 /repos/"$REPOSITORY"/issues/"$NUMBER"/labels/"$toRemove"
230 done < <(comm -23 before after)
231
232 # And add the ones that aren't set already
233 while read -r toAdd; do
234 echo "Adding label $toAdd"
235 gh api \
236 --method POST \
237 /repos/"$REPOSITORY"/issues/"$NUMBER"/labels \
238 -f "labels[]=$toAdd"
239 done < <(comm -13 before after)
240
241 env:
242 GH_TOKEN: ${{ github.token }}
243 REPOSITORY: ${{ github.repository }}
244 NUMBER: ${{ github.event.number }}
245
246 - name: Add eval summary to commit statuses
247 if: ${{ github.event_name == 'pull_request_target' && github.repository_owner == 'NixOS' }}
248 run: |
249 description=$(jq -r '
250 "Package: added " + (.attrdiff.added | length | tostring) +
251 ", removed " + (.attrdiff.removed | length | tostring) +
252 ", changed " + (.attrdiff.changed | length | tostring) +
253 ", Rebuild: linux " + (.rebuildCountByKernel.linux | tostring) +
254 ", darwin " + (.rebuildCountByKernel.darwin | tostring)
255 ' <comparison/changed-paths.json)
256 target_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID?pr=$NUMBER"
257 gh api --method POST \
258 -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" \
259 "/repos/$GITHUB_REPOSITORY/statuses/$PR_HEAD_SHA" \
260 -f "context=Eval / Summary" -f "state=success" -f "description=$description" -f "target_url=$target_url"
261 env:
262 GH_TOKEN: ${{ github.token }}
263 PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
264 NUMBER: ${{ github.event.number }}
265
266 - name: Requesting maintainer reviews
267 if: ${{ steps.app-token.outputs.token && github.repository_owner == 'NixOS' }}
268 run: |
269 # maintainers.json contains GitHub IDs. Look up handles to request reviews from.
270 # There appears to be no API to request reviews based on GitHub IDs
271 jq -r 'keys[]' comparison/maintainers.json \
272 | while read -r id; do gh api /user/"$id" --jq .login; done \
273 | GH_TOKEN=${{ steps.app-token.outputs.token }} result/bin/request-reviewers.sh "$REPOSITORY" "$NUMBER" "$AUTHOR"
274
275 env:
276 GH_TOKEN: ${{ github.token }}
277 REPOSITORY: ${{ github.repository }}
278 NUMBER: ${{ github.event.number }}
279 AUTHOR: ${{ github.event.pull_request.user.login }}
280 # Don't request reviewers on draft PRs
281 DRY_MODE: ${{ github.event.pull_request.draft && '1' || '' }}