tjh.dev
1name: Eval
2
3on:
4 workflow_call:
5 inputs:
6 mergedSha:
7 required: true
8 type: string
9 targetSha:
10 type: string
11 systems:
12 required: true
13 type: string
14 secrets:
15 OWNER_APP_PRIVATE_KEY:
16 required: false
17
18permissions: {}
19
20defaults:
21 run:
22 shell: bash
23
24jobs:
25 eval:
26 runs-on: ubuntu-24.04-arm
27 strategy:
28 fail-fast: false
29 matrix:
30 system: ${{ fromJSON(inputs.systems) }}
31 name: ${{ matrix.system }}
32 outputs:
33 targetRunId: ${{ steps.targetRunId.outputs.targetRunId }}
34 steps:
35 - name: Enable swap
36 run: |
37 sudo fallocate -l 10G /swap
38 sudo chmod 600 /swap
39 sudo mkswap /swap
40 sudo swapon /swap
41
42 - name: Check out the PR at the test merge commit
43 uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
44 with:
45 ref: ${{ inputs.mergedSha }}
46 path: untrusted
47
48 - name: Install Nix
49 uses: cachix/install-nix-action@f0fe604f8a612776892427721526b4c7cfb23aba # v31
50 with:
51 extra_nix_config: sandbox = true
52
53 - name: Evaluate the ${{ matrix.system }} output paths for all derivation attributes
54 env:
55 MATRIX_SYSTEM: ${{ matrix.system }}
56 run: |
57 nix-build untrusted/ci -A eval.singleSystem \
58 --argstr evalSystem "$MATRIX_SYSTEM" \
59 --arg chunkSize 10000 \
60 --out-link merged
61 # If it uses too much memory, slightly decrease chunkSize
62
63 - name: Upload the output paths and eval stats
64 uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
65 with:
66 name: merged-${{ matrix.system }}
67 path: merged/*
68
69 - name: Log current API rate limits
70 env:
71 GH_TOKEN: ${{ github.token }}
72 run: gh api /rate_limit | jq
73
74 - name: Get target run id
75 if: inputs.targetSha
76 id: targetRunId
77 uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
78 env:
79 MATRIX_SYSTEM: ${{ matrix.system }}
80 TARGET_SHA: ${{ inputs.targetSha }}
81 with:
82 script: |
83 const system = process.env.MATRIX_SYSTEM
84 const targetSha = process.env.TARGET_SHA
85
86 let run_id
87 try {
88 run_id = (await github.rest.actions.listWorkflowRuns({
89 ...context.repo,
90 workflow_id: 'push.yml',
91 event: 'push',
92 head_sha: targetSha
93 })).data.workflow_runs[0].id
94 } catch {
95 throw new Error(`Could not find a push.yml workflow run for ${targetSha}.`)
96 }
97
98 // Waiting 120 * 5 sec = 10 min. max.
99 // Eval takes max 5-6 minutes, normally.
100 for (let i = 0; i < 120; i++) {
101 const result = await github.rest.actions.listWorkflowRunArtifacts({
102 ...context.repo,
103 run_id,
104 name: `merged-${system}`
105 })
106 if (result.data.total_count > 0) {
107 core.setOutput('targetRunId', run_id)
108 return
109 }
110 await new Promise(resolve => setTimeout(resolve, 5000))
111 }
112 // No artifact found at this stage. This usually means that Eval failed on the target branch.
113 // This should only happen when Eval is broken on the target branch and this PR fixes it.
114 // Continue without targetRunId to skip the remaining steps, but pass the job.
115
116 - name: Log current API rate limits
117 env:
118 GH_TOKEN: ${{ github.token }}
119 run: gh api /rate_limit | jq
120
121 - uses: actions/download-artifact@v4
122 if: steps.targetRunId.outputs.targetRunId
123 with:
124 run-id: ${{ steps.targetRunId.outputs.targetRunId }}
125 name: merged-${{ matrix.system }}
126 path: target
127 github-token: ${{ github.token }}
128 merge-multiple: true
129
130 - name: Compare outpaths against the target branch
131 if: steps.targetRunId.outputs.targetRunId
132 env:
133 MATRIX_SYSTEM: ${{ matrix.system }}
134 run: |
135 nix-build untrusted/ci -A eval.diff \
136 --arg beforeDir ./target \
137 --arg afterDir "$(readlink ./merged)" \
138 --argstr evalSystem "$MATRIX_SYSTEM" \
139 --out-link diff
140
141 - name: Upload outpaths diff and stats
142 if: steps.targetRunId.outputs.targetRunId
143 uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
144 with:
145 name: diff-${{ matrix.system }}
146 path: diff/*
147
148 compare:
149 runs-on: ubuntu-24.04-arm
150 needs: [eval]
151 if: needs.eval.outputs.targetRunId
152 permissions:
153 statuses: write
154 steps:
155 - name: Download output paths and eval stats for all systems
156 uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
157 with:
158 pattern: diff-*
159 path: diff
160 merge-multiple: true
161
162 - name: Check out the PR at the target commit
163 uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
164 with:
165 ref: ${{ inputs.targetSha }}
166 path: trusted
167
168 - name: Install Nix
169 uses: cachix/install-nix-action@f0fe604f8a612776892427721526b4c7cfb23aba # v31
170 with:
171 extra_nix_config: sandbox = true
172
173 - name: Combine all output paths and eval stats
174 run: |
175 nix-build trusted/ci -A eval.combine \
176 --arg diffDir ./diff \
177 --out-link combined
178
179 - name: Compare against the target branch
180 env:
181 AUTHOR_ID: ${{ github.event.pull_request.user.id }}
182 run: |
183 git -C trusted fetch --depth 1 origin ${{ inputs.mergedSha }}
184 git -C trusted diff --name-only ${{ inputs.mergedSha }} \
185 | jq --raw-input --slurp 'split("\n")[:-1]' > touched-files.json
186
187 # Use the target branch to get accurate maintainer info
188 nix-build trusted/ci -A eval.compare \
189 --arg combinedDir "$(realpath ./combined)" \
190 --arg touchedFilesJson ./touched-files.json \
191 --argstr githubAuthorId "$AUTHOR_ID" \
192 --out-link comparison
193
194 cat comparison/step-summary.md >> "$GITHUB_STEP_SUMMARY"
195
196 - name: Upload the comparison results
197 uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
198 with:
199 name: comparison
200 path: comparison/*
201
202 - name: Add eval summary to commit statuses
203 if: ${{ github.event_name == 'pull_request_target' }}
204 uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
205 with:
206 script: |
207 const { readFile } = require('node:fs/promises')
208 const changed = JSON.parse(await readFile('comparison/changed-paths.json', 'utf-8'))
209 const description =
210 'Package: ' + [
211 `added ${changed.attrdiff.added.length}`,
212 `removed ${changed.attrdiff.removed.length}`,
213 `changed ${changed.attrdiff.changed.length}`
214 ].join(', ') +
215 ' — Rebuild: ' + [
216 `linux ${changed.rebuildCountByKernel.linux}`,
217 `darwin ${changed.rebuildCountByKernel.darwin}`
218 ].join(', ')
219
220 const { serverUrl, repo, runId, payload } = context
221 const target_url =
222 `${serverUrl}/${repo.owner}/${repo.repo}/actions/runs/${runId}?pr=${payload.pull_request.number}`
223
224 await github.rest.repos.createCommitStatus({
225 ...repo,
226 sha: payload.pull_request.head.sha,
227 context: 'Eval Summary',
228 state: 'success',
229 description,
230 target_url
231 })
232
233 misc:
234 if: ${{ github.event_name != 'push' }}
235 runs-on: ubuntu-24.04-arm
236 steps:
237 - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
238 with:
239 sparse-checkout: .github/actions
240 - name: Check if the PR can be merged and checkout the merge commit
241 uses: ./.github/actions/get-merge-commit
242 with:
243 merged-as-untrusted: true
244
245 - name: Install Nix
246 uses: cachix/install-nix-action@f0fe604f8a612776892427721526b4c7cfb23aba # v31
247 with:
248 extra_nix_config: sandbox = true
249
250 - name: Ensure flake outputs on all systems still evaluate
251 run: nix flake check --all-systems --no-build ./untrusted
252
253 - name: Query nixpkgs with aliases enabled to check for basic syntax errors
254 run: |
255 time nix-env -I ./untrusted -f ./untrusted -qa '*' --option restrict-eval true --option allow-import-from-derivation false >/dev/null