tjh.dev
1name: Eval
2
3on:
4 workflow_call:
5 inputs:
6 mergedSha:
7 required: true
8 type: string
9 targetSha:
10 type: string
11 systems:
12 required: true
13 type: string
14 secrets:
15 OWNER_APP_PRIVATE_KEY:
16 required: false
17
18permissions: {}
19
20defaults:
21 run:
22 shell: bash
23
24jobs:
25 eval:
26 runs-on: ubuntu-24.04-arm
27 strategy:
28 fail-fast: false
29 matrix:
30 system: ${{ fromJSON(inputs.systems) }}
31 name: ${{ matrix.system }}
32 outputs:
33 targetRunId: ${{ steps.targetRunId.outputs.targetRunId }}
34 steps:
35 # We'd rather fail than run slow eval on swap.
36 # Failing allows us to adjust the chunkSize to remain fast.
37 - name: Disable swap
38 run: |
39 sudo swapoff -a
40
41 - name: Check out the PR at the test merge commit
42 uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
43 with:
44 ref: ${{ inputs.mergedSha }}
45 path: untrusted
46
47 - name: Install Nix
48 uses: cachix/install-nix-action@f0fe604f8a612776892427721526b4c7cfb23aba # v31
49 with:
50 extra_nix_config: sandbox = true
51
52 - name: Evaluate the ${{ matrix.system }} output paths for all derivation attributes
53 env:
54 MATRIX_SYSTEM: ${{ matrix.system }}
55 run: |
56 nix-build untrusted/ci -A eval.singleSystem \
57 --argstr evalSystem "$MATRIX_SYSTEM" \
58 --arg chunkSize 8000 \
59 --out-link merged
60 # If it uses too much memory, slightly decrease chunkSize
61
62 - name: Upload the output paths and eval stats
63 uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
64 with:
65 name: merged-${{ matrix.system }}
66 path: merged/*
67
68 - name: Log current API rate limits
69 env:
70 GH_TOKEN: ${{ github.token }}
71 run: gh api /rate_limit | jq
72
73 - name: Get target run id
74 if: inputs.targetSha
75 id: targetRunId
76 uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
77 env:
78 MATRIX_SYSTEM: ${{ matrix.system }}
79 TARGET_SHA: ${{ inputs.targetSha }}
80 with:
81 script: |
82 const system = process.env.MATRIX_SYSTEM
83 const targetSha = process.env.TARGET_SHA
84
85 let run_id
86 try {
87 run_id = (await github.rest.actions.listWorkflowRuns({
88 ...context.repo,
89 workflow_id: 'push.yml',
90 event: 'push',
91 head_sha: targetSha
92 })).data.workflow_runs[0].id
93 } catch {
94 throw new Error(`Could not find a push.yml workflow run for ${targetSha}.`)
95 }
96
97 // Waiting 120 * 5 sec = 10 min. max.
98 // Eval takes max 5-6 minutes, normally.
99 for (let i = 0; i < 120; i++) {
100 const result = await github.rest.actions.listWorkflowRunArtifacts({
101 ...context.repo,
102 run_id,
103 name: `merged-${system}`
104 })
105 if (result.data.total_count > 0) {
106 core.setOutput('targetRunId', run_id)
107 return
108 }
109 await new Promise(resolve => setTimeout(resolve, 5000))
110 }
111 // No artifact found at this stage. This usually means that Eval failed on the target branch.
112 // This should only happen when Eval is broken on the target branch and this PR fixes it.
113 // Continue without targetRunId to skip the remaining steps, but pass the job.
114
115 - name: Log current API rate limits
116 env:
117 GH_TOKEN: ${{ github.token }}
118 run: gh api /rate_limit | jq
119
120 - uses: actions/download-artifact@v4
121 if: steps.targetRunId.outputs.targetRunId
122 with:
123 run-id: ${{ steps.targetRunId.outputs.targetRunId }}
124 name: merged-${{ matrix.system }}
125 path: target
126 github-token: ${{ github.token }}
127 merge-multiple: true
128
129 - name: Compare outpaths against the target branch
130 if: steps.targetRunId.outputs.targetRunId
131 env:
132 MATRIX_SYSTEM: ${{ matrix.system }}
133 run: |
134 nix-build untrusted/ci -A eval.diff \
135 --arg beforeDir ./target \
136 --arg afterDir "$(readlink ./merged)" \
137 --argstr evalSystem "$MATRIX_SYSTEM" \
138 --out-link diff
139
140 - name: Upload outpaths diff and stats
141 if: steps.targetRunId.outputs.targetRunId
142 uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
143 with:
144 name: diff-${{ matrix.system }}
145 path: diff/*
146
147 compare:
148 runs-on: ubuntu-24.04-arm
149 needs: [eval]
150 if: needs.eval.outputs.targetRunId
151 permissions:
152 statuses: write
153 steps:
154 - name: Download output paths and eval stats for all systems
155 uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
156 with:
157 pattern: diff-*
158 path: diff
159 merge-multiple: true
160
161 - name: Check out the PR at the target commit
162 uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
163 with:
164 ref: ${{ inputs.targetSha }}
165 path: trusted
166
167 - name: Install Nix
168 uses: cachix/install-nix-action@f0fe604f8a612776892427721526b4c7cfb23aba # v31
169 with:
170 extra_nix_config: sandbox = true
171
172 - name: Combine all output paths and eval stats
173 run: |
174 nix-build trusted/ci -A eval.combine \
175 --arg diffDir ./diff \
176 --out-link combined
177
178 - name: Compare against the target branch
179 env:
180 AUTHOR_ID: ${{ github.event.pull_request.user.id }}
181 run: |
182 git -C trusted fetch --depth 1 origin ${{ inputs.mergedSha }}
183 git -C trusted diff --name-only ${{ inputs.mergedSha }} \
184 | jq --raw-input --slurp 'split("\n")[:-1]' > touched-files.json
185
186 # Use the target branch to get accurate maintainer info
187 nix-build trusted/ci -A eval.compare \
188 --arg combinedDir "$(realpath ./combined)" \
189 --arg touchedFilesJson ./touched-files.json \
190 --argstr githubAuthorId "$AUTHOR_ID" \
191 --out-link comparison
192
193 cat comparison/step-summary.md >> "$GITHUB_STEP_SUMMARY"
194
195 - name: Upload the comparison results
196 uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
197 with:
198 name: comparison
199 path: comparison/*
200
201 - name: Add eval summary to commit statuses
202 if: ${{ github.event_name == 'pull_request_target' }}
203 uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
204 with:
205 script: |
206 const { readFile } = require('node:fs/promises')
207 const changed = JSON.parse(await readFile('comparison/changed-paths.json', 'utf-8'))
208 const description =
209 'Package: ' + [
210 `added ${changed.attrdiff.added.length}`,
211 `removed ${changed.attrdiff.removed.length}`,
212 `changed ${changed.attrdiff.changed.length}`
213 ].join(', ') +
214 ' — Rebuild: ' + [
215 `linux ${changed.rebuildCountByKernel.linux}`,
216 `darwin ${changed.rebuildCountByKernel.darwin}`
217 ].join(', ')
218
219 const { serverUrl, repo, runId, payload } = context
220 const target_url =
221 `${serverUrl}/${repo.owner}/${repo.repo}/actions/runs/${runId}?pr=${payload.pull_request.number}`
222
223 await github.rest.repos.createCommitStatus({
224 ...repo,
225 sha: payload.pull_request.head.sha,
226 context: 'Eval Summary',
227 state: 'success',
228 description,
229 target_url
230 })
231
232 misc:
233 if: ${{ github.event_name != 'push' }}
234 runs-on: ubuntu-24.04-arm
235 steps:
236 - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
237 with:
238 sparse-checkout: .github/actions
239 - name: Check if the PR can be merged and checkout the merge commit
240 uses: ./.github/actions/get-merge-commit
241 with:
242 merged-as-untrusted: true
243
244 - name: Install Nix
245 uses: cachix/install-nix-action@f0fe604f8a612776892427721526b4c7cfb23aba # v31
246 with:
247 extra_nix_config: sandbox = true
248
249 - name: Ensure flake outputs on all systems still evaluate
250 run: nix flake check --all-systems --no-build ./untrusted
251
252 - name: Query nixpkgs with aliases enabled to check for basic syntax errors
253 run: |
254 time nix-env -I ./untrusted -f ./untrusted -qa '*' --option restrict-eval true --option allow-import-from-derivation false >/dev/null