name: Eval

on:
  workflow_call:
    inputs:
      mergedSha:
        required: true
        type: string
      targetSha:
        type: string
      systems:
        required: true
        type: string
      testVersions:
        required: false
        default: false
        type: boolean
    secrets:
      CACHIX_AUTH_TOKEN:
        required: true

permissions: {}

defaults:
  run:
    shell: bash

jobs:
  versions:
    if: inputs.testVersions
    runs-on: ubuntu-24.04-arm
    outputs:
      versions: ${{ steps.versions.outputs.versions }}
    steps:
      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
        with:
          path: trusted
          sparse-checkout: |
            ci/supportedVersions.nix

      - name: Check out the PR at the test merge commit
        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
        with:
          ref: ${{ inputs.mergedSha }}
          path: untrusted
          sparse-checkout: |
            ci/pinned.json

      - name: Install Nix
        uses: cachix/install-nix-action@9280e7aca88deada44c930f1e2c78e21c3ae3edd # v31

      - name: Load supported versions
        id: versions
        run: |
          echo "versions=$(trusted/ci/supportedVersions.nix --arg pinnedJson untrusted/ci/pinned.json)" >> "$GITHUB_OUTPUT"

  eval:
    runs-on: ubuntu-24.04-arm
    needs: versions
    if: ${{ !cancelled() && !failure() }}
    strategy:
      fail-fast: false
      matrix:
        system: ${{ fromJSON(inputs.systems) }}
        version:
          - "" # Default Eval triggering rebuild labels and such.
          - ${{ fromJSON(needs.versions.outputs.versions || '[]') }} # Only for ci/pinned.json updates.
    # Failures for versioned Evals will be collected in a separate job below
    # to not interrupt main Eval's compare step.
    continue-on-error: ${{ matrix.version != '' }}
    name: ${{ matrix.system }}${{ matrix.version && format(' @ {0}', matrix.version) || '' }}
    timeout-minutes: 15
    steps:
      # This is not supposed to be used and just acts as a fallback.
      # Without swap, when Eval runs OOM, it will fail badly with a
      # job that is sometimes not interruptible anymore.
      # If Eval starts swapping, decrease chunkSize to keep it fast.
      - name: Enable swap
        run: |
          sudo fallocate -l 10G /swap
          sudo chmod 600 /swap
          sudo mkswap /swap
          sudo swapon /swap

      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
        with:
          sparse-checkout: .github/actions
      - name: Check out the PR at merged and target commits
        uses: ./.github/actions/checkout
        with:
          merged-as-untrusted-at: ${{ inputs.mergedSha }}
          target-as-trusted-at: ${{ inputs.targetSha }}

      - name: Install Nix
        uses: cachix/install-nix-action@9280e7aca88deada44c930f1e2c78e21c3ae3edd # v31

      - uses: cachix/cachix-action@0fc020193b5a1fa3ac4575aa3a7d3aa6a35435ad # v16
        with:
          # The nixpkgs-ci cache should not be trusted or used outside of Nixpkgs and its forks' CI.
          name: ${{ vars.CACHIX_NAME || 'nixpkgs-ci' }}
          extraPullNames: nixpkgs-ci
          authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
          pushFilter: '(-source|-single-chunk)$'

      - name: Evaluate the ${{ matrix.system }} output paths at the merge commit
        env:
          MATRIX_SYSTEM: ${{ matrix.system }}
          MATRIX_VERSION: ${{ matrix.version || 'nixVersions.latest' }}
        run: |
          nix-build nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned  -A eval.singleSystem \
            --argstr evalSystem "$MATRIX_SYSTEM" \
            --arg chunkSize 8000 \
            --argstr nixPath "$MATRIX_VERSION" \
            --out-link merged
          # If it uses too much memory, slightly decrease chunkSize.
          # Note: Keep the same further down in sync!

      # Running the attrpath generation step separately from the outpath step afterwards.
      # The idea is that, *if* Eval on the target branch has not finished, yet, we will
      # generate the attrpaths in the meantime - and the separate command command afterwards
      # will check cachix again for whether Eval has finished. If no Eval result from the
      # target branch can be found the second time, we proceed to run it in here. Attrpaths
      # generation takes roughly 30 seconds, so for every normal use-case this should be more
      # than enough of a head start for Eval on the target branch to finish.
      # This edge-case, that Eval on the target branch is delayed is unlikely to happen anyway:
      # For a commit to become the target commit of a PR, it must *already* be on the branch.
      # Normally, CI should always start running on that push event *before* it starts running
      # on the PR.
      - name: Evaluate the ${{ matrix.system }} attribute paths at the target commit
        if: inputs.targetSha
        env:
          MATRIX_SYSTEM: ${{ matrix.system }}
        run: |
          nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A eval.attrpathsSuperset \
            --argstr evalSystem "$MATRIX_SYSTEM" \
            --argstr nixPath "nixVersions.latest"

      - name: Evaluate the ${{ matrix.system }} output paths at the target commit
        if: inputs.targetSha
        env:
          MATRIX_SYSTEM: ${{ matrix.system }}
        # This should be very quick, because it pulls the eval results from Cachix.
        run: |
          nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A eval.singleSystem \
            --argstr evalSystem "$MATRIX_SYSTEM" \
            --arg chunkSize 8000 \
            --argstr nixPath "nixVersions.latest" \
            --out-link target

      - name: Compare outpaths against the target branch
        if: inputs.targetSha
        env:
          MATRIX_SYSTEM: ${{ matrix.system }}
        run: |
          nix-build nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A eval.diff \
            --arg beforeDir ./target \
            --arg afterDir ./merged \
            --argstr evalSystem "$MATRIX_SYSTEM" \
            --out-link diff

      - name: Upload outpaths diff and stats
        if: inputs.targetSha
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        with:
          name: ${{ matrix.version && format('{0}-', matrix.version) || '' }}diff-${{ matrix.system }}
          path: diff/*

  compare:
    runs-on: ubuntu-24.04-arm
    needs: [eval]
    if: inputs.targetSha && !cancelled() && !failure()
    permissions:
      statuses: write
    timeout-minutes: 5
    steps:
      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
        with:
          sparse-checkout: .github/actions
      - name: Check out the PR at the target commit
        uses: ./.github/actions/checkout
        with:
          merged-as-untrusted-at: ${{ inputs.mergedSha }}
          target-as-trusted-at: ${{ inputs.targetSha }}

      - name: Download output paths and eval stats for all systems
        uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
        with:
          pattern: diff-*
          path: diff
          merge-multiple: true

      - name: Install Nix
        uses: cachix/install-nix-action@9280e7aca88deada44c930f1e2c78e21c3ae3edd # v31

      - name: Combine all output paths and eval stats
        run: |
          nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A eval.combine \
            --arg diffDir ./diff \
            --out-link combined

      - name: Compare against the target branch
        env:
          AUTHOR_ID: ${{ github.event.pull_request.user.id }}
        run: |
          git -C nixpkgs/trusted diff --name-only ${{ inputs.mergedSha }} \
            | jq --raw-input --slurp 'split("\n")[:-1]' > touched-files.json

          # Use the target branch to get accurate maintainer info
          nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A eval.compare \
            --arg combinedDir ./combined \
            --arg touchedFilesJson ./touched-files.json \
            --argstr githubAuthorId "$AUTHOR_ID" \
            --out-link comparison

          cat comparison/step-summary.md >> "$GITHUB_STEP_SUMMARY"

      - name: Upload the comparison results
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        with:
          name: comparison
          path: comparison/*

      - name: Add eval summary to commit statuses
        if: ${{ github.event_name == 'pull_request_target' }}
        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
        with:
          script: |
            const { readFile } = require('node:fs/promises')
            const changed = JSON.parse(await readFile('comparison/changed-paths.json', 'utf-8'))
            const description =
              'Package: ' + [
                `added ${changed.attrdiff.added.length}`,
                `removed ${changed.attrdiff.removed.length}`,
                `changed ${changed.attrdiff.changed.length}`
              ].join(', ') +
              ' — Rebuild: ' + [
                `linux ${changed.rebuildCountByKernel.linux}`,
                `darwin ${changed.rebuildCountByKernel.darwin}`
              ].join(', ')

            const { serverUrl, repo, runId, payload } = context
            const target_url =
              `${serverUrl}/${repo.owner}/${repo.repo}/actions/runs/${runId}?pr=${payload.pull_request.number}`

            await github.rest.repos.createCommitStatus({
              ...repo,
              sha: payload.pull_request.head.sha,
              context: 'Eval Summary',
              state: 'success',
              description,
              target_url
            })

  # Creates a matrix of Eval performance for various versions and systems.
  report:
    runs-on: ubuntu-24.04-arm
    needs: [versions, eval]
    steps:
      - name: Download output paths and eval stats for all versions
        uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
        with:
          pattern: "*-diff-*"
          path: versions

      - name: Add version comparison table to job summary
        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
        env:
          SYSTEMS: ${{ inputs.systems }}
          VERSIONS: ${{ needs.versions.outputs.versions }}
        with:
          script: |
            const { readFileSync } = require('node:fs')
            const path = require('node:path')

            const systems = JSON.parse(process.env.SYSTEMS)
            const versions = JSON.parse(process.env.VERSIONS)

            core.summary.addHeading('Lix/Nix version comparison')
            core.summary.addTable(
              [].concat(
                [
                  [{ data: 'Version', header: true }].concat(
                    systems.map((system) => ({ data: system, header: true })),
                  ),
                ],
                versions.map((version) =>
                  [{ data: version }].concat(
                    systems.map((system) => {
                      try {
                        const artifact = path.join('versions', `${version}-diff-${system}`)
                        const time = Math.round(
                          parseFloat(
                            readFileSync(
                              path.join(artifact, 'after', system, 'total-time'),
                              'utf-8',
                            ),
                          ),
                        )
                        const diff = JSON.parse(
                          readFileSync(path.join(artifact, system, 'diff.json'), 'utf-8'),
                        )
                        const attrs = [].concat(
                          diff.added,
                          diff.removed,
                          diff.changed,
                          diff.rebuilds
                        ).filter(attr =>
                          // Exceptions related to dev shells, which changed at some time between 2.18 and 2.24.
                          !attr.startsWith('tests.devShellTools.nixos.') &&
                          !attr.startsWith('tests.devShellTools.unstructuredDerivationInputEnv.')
                        )
                        if (attrs.length > 0) {
                          core.setFailed(
                            `${version} on ${system} has changed outpaths!\nNote: Please make sure to update ci/pinned.json separately from changes to other packages.`,
                          )
                          return { data: ':x:' }
                        }
                        return { data: time }
                      } catch {
                        core.warning(`${version} on ${system} did not produce artifact.`)
                        return { data: ':warning:' }
                      }
                    }),
                  ),
                ),
              ),
            )
            core.summary.addRaw(
              '\n*Evaluation time in seconds without downloading dependencies.*',
              true,
            )
            core.summary.addRaw('\n*:warning: Job did not report a result.*', true)
            core.summary.addRaw(
              '\n*:x: Job produced different outpaths than the target branch.*',
              true,
            )
            core.summary.write()

  misc:
    if: ${{ github.event_name != 'push' }}
    runs-on: ubuntu-24.04-arm
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
        with:
          sparse-checkout: .github/actions
      - name: Checkout the merge commit
        uses: ./.github/actions/checkout
        with:
          merged-as-untrusted-at: ${{ inputs.mergedSha }}

      - name: Install Nix
        uses: cachix/install-nix-action@9280e7aca88deada44c930f1e2c78e21c3ae3edd # v31

      - name: Run misc eval tasks in parallel
        run: |
          # Ensure flake outputs on all systems still evaluate
          nix flake check --all-systems --no-build './nixpkgs/untrusted?shallow=1' &
          # Query nixpkgs with aliases enabled to check for basic syntax errors
          nix-env -I ./nixpkgs/untrusted -f ./nixpkgs/untrusted -qa '*' --option restrict-eval true --option allow-import-from-derivation false >/dev/null &
          wait