tjh.dev
1name: Eval
2
3on:
4 pull_request_target:
5 push:
6 # Keep this synced with ci/request-reviews/dev-branches.txt
7 branches:
8 - master
9 - staging
10 - release-*
11 - staging-*
12 - haskell-updates
13 - python-updates
14
15permissions:
16 contents: read
17
18jobs:
19 get-merge-commit:
20 uses: ./.github/workflows/get-merge-commit.yml
21
22 attrs:
23 name: Attributes
24 runs-on: ubuntu-latest
25 needs: get-merge-commit
26 # Skip this and dependent steps if the PR can't be merged
27 if: needs.get-merge-commit.outputs.mergedSha
28 outputs:
29 baseSha: ${{ steps.baseSha.outputs.baseSha }}
30 systems: ${{ steps.systems.outputs.systems }}
31 steps:
32 - name: Check out the PR at the test merge commit
33 uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
34 with:
35 ref: ${{ needs.get-merge-commit.outputs.mergedSha }}
36 fetch-depth: 2
37 path: nixpkgs
38
39 - name: Determine base commit
40 if: github.event_name == 'pull_request_target'
41 id: baseSha
42 run: |
43 baseSha=$(git -C nixpkgs rev-parse HEAD^1)
44 echo "baseSha=$baseSha" >> "$GITHUB_OUTPUT"
45
46 - name: Install Nix
47 uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # v30
48
49 - name: Evaluate the list of all attributes and get the systems matrix
50 id: systems
51 run: |
52 nix-build nixpkgs/ci -A eval.attrpathsSuperset
53 echo "systems=$(<result/systems.json)" >> "$GITHUB_OUTPUT"
54
55 - name: Upload the list of all attributes
56 uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
57 with:
58 name: paths
59 path: result/*
60
61 eval-aliases:
62 name: Eval nixpkgs with aliases enabled
63 runs-on: ubuntu-latest
64 needs: [ attrs, get-merge-commit ]
65 steps:
66 - name: Check out the PR at the test merge commit
67 uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
68 with:
69 ref: ${{ needs.get-merge-commit.outputs.mergedSha }}
70 path: nixpkgs
71
72 - name: Install Nix
73 uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # v30
74
75 - name: Query nixpkgs with aliases enabled to check for basic syntax errors
76 run: |
77 time nix-env -I ./nixpkgs -f ./nixpkgs -qa '*' --option restrict-eval true --option allow-import-from-derivation false >/dev/null
78
79 outpaths:
80 name: Outpaths
81 runs-on: ubuntu-latest
82 needs: [ attrs, get-merge-commit ]
83 strategy:
84 matrix:
85 system: ${{ fromJSON(needs.attrs.outputs.systems) }}
86 steps:
87 - name: Download the list of all attributes
88 uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
89 with:
90 name: paths
91 path: paths
92
93 - name: Check out the PR at the test merge commit
94 uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
95 with:
96 ref: ${{ needs.get-merge-commit.outputs.mergedSha }}
97 path: nixpkgs
98
99 - name: Install Nix
100 uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # v30
101
102 - name: Evaluate the ${{ matrix.system }} output paths for all derivation attributes
103 env:
104 MATRIX_SYSTEM: ${{ matrix.system }}
105 run: |
106 nix-build nixpkgs/ci -A eval.singleSystem \
107 --argstr evalSystem "$MATRIX_SYSTEM" \
108 --arg attrpathFile ./paths/paths.json \
109 --arg chunkSize 10000
110 # If it uses too much memory, slightly decrease chunkSize
111
112 - name: Upload the output paths and eval stats
113 uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
114 with:
115 name: intermediate-${{ matrix.system }}
116 path: result/*
117
118 process:
119 name: Process
120 runs-on: ubuntu-latest
121 needs: [ outpaths, attrs, get-merge-commit ]
122 outputs:
123 baseRunId: ${{ steps.baseRunId.outputs.baseRunId }}
124 steps:
125 - name: Download output paths and eval stats for all systems
126 uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
127 with:
128 pattern: intermediate-*
129 path: intermediate
130
131 - name: Check out the PR at the test merge commit
132 uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
133 with:
134 ref: ${{ needs.get-merge-commit.outputs.mergedSha }}
135 path: nixpkgs
136
137 - name: Install Nix
138 uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # v30
139
140 - name: Combine all output paths and eval stats
141 run: |
142 nix-build nixpkgs/ci -A eval.combine \
143 --arg resultsDir ./intermediate \
144 -o prResult
145
146 - name: Upload the combined results
147 uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
148 with:
149 name: result
150 path: prResult/*
151
152 - name: Get base run id
153 if: needs.attrs.outputs.baseSha
154 id: baseRunId
155 run: |
156 # Get the latest eval.yml workflow run for the PR's base commit
157 if ! run=$(gh api --method GET /repos/"$REPOSITORY"/actions/workflows/eval.yml/runs \
158 -f head_sha="$BASE_SHA" -f event=push \
159 --jq '.workflow_runs | sort_by(.run_started_at) | .[-1]') \
160 || [[ -z "$run" ]]; then
161 echo "Could not find an eval.yml workflow run for $BASE_SHA, cannot make comparison"
162 exit 0
163 fi
164 echo "Comparing against $(jq .html_url <<< "$run")"
165 runId=$(jq .id <<< "$run")
166 conclusion=$(jq -r .conclusion <<< "$run")
167
168 while [[ "$conclusion" == null || "$conclusion" == "" ]]; do
169 echo "Workflow not done, waiting 10 seconds before checking again"
170 sleep 10
171 conclusion=$(gh api /repos/"$REPOSITORY"/actions/runs/"$runId" --jq '.conclusion')
172 done
173
174 if [[ "$conclusion" != "success" ]]; then
175 echo "Workflow was not successful (conclusion: $conclusion), cannot make comparison"
176 exit 0
177 fi
178
179 echo "baseRunId=$runId" >> "$GITHUB_OUTPUT"
180 env:
181 REPOSITORY: ${{ github.repository }}
182 BASE_SHA: ${{ needs.attrs.outputs.baseSha }}
183 GH_TOKEN: ${{ github.token }}
184
185 - uses: actions/download-artifact@v4
186 if: steps.baseRunId.outputs.baseRunId
187 with:
188 name: result
189 path: baseResult
190 github-token: ${{ github.token }}
191 run-id: ${{ steps.baseRunId.outputs.baseRunId }}
192
193 - name: Compare against the base branch
194 if: steps.baseRunId.outputs.baseRunId
195 run: |
196 nix-build nixpkgs/ci -A eval.compare \
197 --arg beforeResultDir ./baseResult \
198 --arg afterResultDir ./prResult \
199 -o comparison
200 cat comparison/step-summary.md >> "$GITHUB_STEP_SUMMARY"
201 # TODO: Request reviews from maintainers for packages whose files are modified in the PR
202
203 - name: Upload the combined results
204 if: steps.baseRunId.outputs.baseRunId
205 uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
206 with:
207 name: comparison
208 path: comparison/*
209
210 # Separate job to have a very tightly scoped PR write token
211 tag:
212 name: Tag
213 runs-on: ubuntu-latest
214 needs: process
215 if: needs.process.outputs.baseRunId
216 permissions:
217 pull-requests: write
218 statuses: write
219 steps:
220 - name: Download process result
221 uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
222 with:
223 name: comparison
224 path: comparison
225
226 - name: Tagging pull request
227 run: |
228 # Get all currently set rebuild labels
229 gh api \
230 /repos/"$REPOSITORY"/issues/"$NUMBER"/labels \
231 --jq '.[].name | select(startswith("10.rebuild"))' \
232 | sort > before
233
234 # And the labels that should be there
235 jq -r '.labels[]' comparison/changed-paths.json \
236 | sort > after
237
238 # Remove the ones not needed anymore
239 while read -r toRemove; do
240 echo "Removing label $toRemove"
241 gh api \
242 --method DELETE \
243 /repos/"$REPOSITORY"/issues/"$NUMBER"/labels/"$toRemove"
244 done < <(comm -23 before after)
245
246 # And add the ones that aren't set already
247 while read -r toAdd; do
248 echo "Adding label $toAdd"
249 gh api \
250 --method POST \
251 /repos/"$REPOSITORY"/issues/"$NUMBER"/labels \
252 -f "labels[]=$toAdd"
253 done < <(comm -13 before after)
254 env:
255 GH_TOKEN: ${{ github.token }}
256 REPOSITORY: ${{ github.repository }}
257 NUMBER: ${{ github.event.number }}
258
259 - name: Add eval summary to commit statuses
260 if: ${{ github.event_name == 'pull_request_target' }}
261 run: |
262 description=$(jq -r '
263 "Package: added " + (.attrdiff.added | length | tostring) +
264 ", removed " + (.attrdiff.removed | length | tostring) +
265 ", changed " + (.attrdiff.changed | length | tostring) +
266 ", Rebuild: linux " + (.rebuildCountByKernel.linux | tostring) +
267 ", darwin " + (.rebuildCountByKernel.darwin | tostring)
268 ' <comparison/changed-paths.json)
269 target_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID?pr=$NUMBER"
270 gh api --method POST \
271 -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" \
272 "/repos/$GITHUB_REPOSITORY/statuses/$PR_HEAD_SHA" \
273 -f "context=Eval / Summary" -f "state=success" -f "description=$description" -f "target_url=$target_url"
274 env:
275 GH_TOKEN: ${{ github.token }}
276 PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
277 NUMBER: ${{ github.event.number }}