Merge pull request #297901 from tweag/check-by-name-separate-repo

Switch the `pkgs/by-name` check to a separate repository

.github/workflows/check-by-name.yml

···
 # Checks pkgs/by-name (see pkgs/by-name/README.md)
-# using the nixpkgs-check-by-name tool (see pkgs/test/nixpkgs-check-by-name)
+# using the nixpkgs-check-by-name tool (see https://github.com/NixOS/nixpkgs-check-by-name)
 #
-# When you make changes to this workflow, also update pkgs/test/nixpkgs-check-by-name/scripts/run-local.sh adequately
+# When you make changes to this workflow, also update pkgs/test/check-by-name/run-local.sh adequately
 name: Check pkgs/by-name
 
-# The tool is pinned to a pre-built version on Hydra,
-# see pkgs/test/nixpkgs-check-by-name/scripts/README.md
 on:
   # Using pull_request_target instead of pull_request avoids having to approve first time contributors
   pull_request_target:
···
 
 jobs:
   check:
-    # This is x86_64-linux, for which the tool is always prebuilt on the nixos-* channels,
-    # as specified in nixos/release-combined.nix
+    # This needs to be x86_64-linux, because we depend on the tooling being pre-built in the GitHub releases
     runs-on: ubuntu-latest
     # This should take 1 minute at most, but let's be generous.
     # The default of 6 hours is definitely too long
···
           echo "base=$base" >> "$GITHUB_ENV"
       - uses: cachix/install-nix-action@8887e596b4ee1134dae06b98d573bd674693f47c # v26
       - name: Fetching the pinned tool
-        # Update the pinned version using pkgs/test/nixpkgs-check-by-name/scripts/update-pinned-tool.sh
+        # Update the pinned version using pkgs/test/check-by-name/update-pinned-tool.sh
         run: |
-          # Get the direct /nix/store path from the pin to avoid having to evaluate Nixpkgs
-          toolPath=$(jq -r '."ci-path"' pkgs/test/nixpkgs-check-by-name/scripts/pinned-tool.json)
-          # This asks the substituter for the path, which should be there because Hydra will have pre-built and pushed it
+          # The pinned version of the tooling to use
+          toolVersion=$(<pkgs/test/check-by-name/pinned-version.txt)
+          # Fetch the x86_64-linux-specific release artifact containing the Gzipped NAR of the pre-built tool
+          toolPath=$(curl -sSfL https://github.com/NixOS/nixpkgs-check-by-name/releases/download/"$toolVersion"/x86_64-linux.nar.gz \
+            | gzip -cd | nix-store --import | tail -1)
+          # Adds a result symlink as a GC root
           nix-store --realise "$toolPath" --add-root result
       - name: Running nixpkgs-check-by-name
         run: |

maintainers/scripts/README.md

···
 
 ### `check-by-name.sh`
 
-An alias for `pkgs/test/nixpkgs-check-by-name/scripts/run-local.sh`, see [documentation](../../pkgs/test/nixpkgs-check-by-name/scripts/README.md).
+An alias for `pkgs/test/check-by-name/run-local.sh`, see [documentation](../../pkgs/test/check-by-name/README.md).
 
 ### `get-maintainer.sh`
 

maintainers/scripts/check-by-name.sh

···
-../../pkgs/test/nixpkgs-check-by-name/scripts/run-local.sh
+../../pkgs/test/check-by-name/run-local.sh

nixos/release-combined.nix

···
         (onFullSupported "nixpkgs.jdk")
         (onSystems ["x86_64-linux"] "nixpkgs.mesa_i686") # i686 sanity check + useful
         ["nixpkgs.tarball"]
-
-        # Ensure that nixpkgs-check-by-name is available in nixos-unstable,
-        # so that a pre-built version can be used in CI for PR's
-        # See ../pkgs/test/nixpkgs-check-by-name/README.md
-        (onSystems ["x86_64-linux"] "nixpkgs.tests.nixpkgs-check-by-name")
       ];
     };
 }

pkgs/by-name/README.md

···
 
 ## Validation
 
-CI performs [certain checks](../test/nixpkgs-check-by-name/README.md#validity-checks) on the `pkgs/by-name` structure.
-This is done using the [`nixpkgs-check-by-name` tool](../test/nixpkgs-check-by-name).
+CI performs [certain checks](https://github.com/NixOS/nixpkgs-check-by-name?tab=readme-ov-file#validity-checks) on the `pkgs/by-name` structure.
+This is done using the [`nixpkgs-check-by-name` tool](https://github.com/NixOS/nixpkgs-check-by-name).
 
 You can locally emulate the CI check using
 
 ```
-$ ./pkgs/test/nixpkgs-check-by-name/scripts/run-local.sh master
+$ ./maintainers/scripts/check-by-name.sh master
 ```
 
 See [here](../../.github/workflows/check-by-name.yml) for more info.

pkgs/test/check-by-name/pinned-version.txt

···
+0.1.0

pkgs/test/check-by-name/run-local.sh

···
+#!/usr/bin/env nix-shell
+#!nix-shell -i bash -p jq
+
+set -o pipefail -o errexit -o nounset
+
+trace() { echo >&2 "$@"; }
+
+tmp=$(mktemp -d)
+cleanup() {
+    # Don't exit early if anything fails to cleanup
+    set +o errexit
+
+    trace -n "Cleaning up.. "
+
+    [[ -e "$tmp/base" ]] && git worktree remove --force "$tmp/base"
+    [[ -e "$tmp/merged" ]] && git worktree remove --force "$tmp/merged"
+
+    rm -rf "$tmp"
+
+    trace "Done"
+}
+trap cleanup exit
+
+
+repo=https://github.com/NixOS/nixpkgs.git
+
+if (( $# != 0 )); then
+    baseBranch=$1
+    shift
+else
+    trace "Usage: $0 BASE_BRANCH [REPOSITORY]"
+    trace "BASE_BRANCH: The base branch to use, e.g. master or release-23.11"
+    trace "REPOSITORY: The repository to fetch the base branch from, defaults to $repo"
+    exit 1
+fi
+
+if (( $# != 0 )); then
+    repo=$1
+    shift
+fi
+
+if [[ -n "$(git status --porcelain)" ]]; then
+    trace -e "\e[33mWarning: Dirty tree, uncommitted changes won't be taken into account\e[0m"
+fi
+headSha=$(git rev-parse HEAD)
+trace -e "Using HEAD commit \e[34m$headSha\e[0m"
+
+trace -n "Creating Git worktree for the HEAD commit in $tmp/merged.. "
+git worktree add --detach -q "$tmp/merged" HEAD
+trace "Done"
+
+trace -n "Fetching base branch $baseBranch to compare against.. "
+git fetch -q "$repo" refs/heads/"$baseBranch"
+baseSha=$(git rev-parse FETCH_HEAD)
+trace -e "\e[34m$baseSha\e[0m"
+
+trace -n "Creating Git worktree for the base branch in $tmp/base.. "
+git worktree add -q "$tmp/base" "$baseSha"
+trace "Done"
+
+trace -n "Merging base branch into the HEAD commit in $tmp/merged.. "
+git -C "$tmp/merged" merge -q --no-edit "$baseSha"
+trace -e "\e[34m$(git -C "$tmp/merged" rev-parse HEAD)\e[0m"
+
+trace -n "Reading pinned nixpkgs-check-by-name version from pinned-version.txt.. "
+toolVersion=$(<"$tmp/merged/pkgs/test/check-by-name/pinned-version.txt")
+trace -e "\e[34m$toolVersion\e[0m"
+
+trace -n "Building tool.. "
+nix-build https://github.com/NixOS/nixpkgs-check-by-name/tarball/"$toolVersion" -o "$tmp/tool" -A build
+
+trace "Running nixpkgs-check-by-name.."
+"$tmp/tool/bin/nixpkgs-check-by-name" --base "$tmp/base" "$tmp/merged"

pkgs/test/check-by-name/update-pinned-tool.sh

···
+#!/usr/bin/env nix-shell
+#!nix-shell -i bash -p jq curl
+
+set -o pipefail -o errexit -o nounset
+
+trace() { echo >&2 "$@"; }
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+repository=NixOS/nixpkgs-check-by-name
+pin_file=$SCRIPT_DIR/pinned-version.txt
+
+trace -n "Fetching latest release of $repository.. "
+latestRelease=$(curl -sSfL \
+  -H "Accept: application/vnd.github+json" \
+  -H "X-GitHub-Api-Version: 2022-11-28" \
+  https://api.github.com/repos/"$repository"/releases/latest)
+latestVersion=$(jq .tag_name -r <<< "$latestRelease")
+trace "$latestVersion"
+
+trace "Updating $pin_file"
+echo "$latestVersion" > "$pin_file"

pkgs/test/nixpkgs-check-by-name/scripts/README.md pkgs/test/check-by-name/README.md

···
-# CI-related Scripts
+# `pkgs/by-name` check CI scripts
 
 This directory contains scripts and files used and related to the CI running the `pkgs/by-name` checks in Nixpkgs.
-See also the [CI GitHub Action](../../../../.github/workflows/check-by-name.yml).
+See also the [CI GitHub Action](../../../.github/workflows/check-by-name.yml).
 
 ## `./run-local.sh BASE_BRANCH [REPOSITORY]`
 
···
 
 ## `./update-pinned-tool.sh`
 
-Updates the pinned CI tool in [`./pinned-tool.json`](./pinned-tool.json) to the
-[latest version from the `nixos-unstable` channel](https://hydra.nixos.org/job/nixos/trunk-combined/nixpkgs.tests.nixpkgs-check-by-name.x86_64-linux).
+Updates the pinned [nixpkgs-check-by-name tool](https://github.com/NixOS/nixpkgs-check-by-name) in [`./pinned-version.txt`](./pinned-version.txt) to the latest [release](https://github.com/NixOS/nixpkgs-check-by-name/releases).
+Each release contains a pre-built x86_64-linux version of the tool which is used by CI.
 
-This script needs to be called manually when the CI tooling needs to be updated.
-
-The `pinned-tool.json` file gets populated with both:
-- The `/nix/store` path for `x86_64-linux`, such that CI doesn't have to evaluate Nixpkgs and can directly fetch it from the cache instead.
-- The Nixpkgs revision, such that the `./run-local.sh` script can be used to run the checks locally on any system.
-
-To ensure that the tool is always pre-built for `x86_64-linux` in the `nixos-unstable` channel,
-it's included in the `tested` jobset description in [`nixos/release-combined.nix`](../../../nixos/release-combined.nix).
+This script currently needs to be called manually when the CI tooling needs to be updated.
 
 Why not just build the tooling right from the PRs Nixpkgs version?
 - Because it allows CI to check all PRs, even if they would break the CI tooling.

pkgs/test/nixpkgs-check-by-name/scripts/run-local.sh

···
-#!/usr/bin/env nix-shell
-#!nix-shell -i bash -p jq
-
-set -o pipefail -o errexit -o nounset
-
-trace() { echo >&2 "$@"; }
-
-tmp=$(mktemp -d)
-cleanup() {
-    # Don't exit early if anything fails to cleanup
-    set +o errexit
-
-    trace -n "Cleaning up.. "
-
-    [[ -e "$tmp/base" ]] && git worktree remove --force "$tmp/base"
-    [[ -e "$tmp/merged" ]] && git worktree remove --force "$tmp/merged"
-    [[ -e "$tmp/tool-nixpkgs" ]] && git worktree remove --force "$tmp/tool-nixpkgs"
-
-    rm -rf "$tmp"
-
-    trace "Done"
-}
-trap cleanup exit
-
-
-repo=https://github.com/NixOS/nixpkgs.git
-
-if (( $# != 0 )); then
-    baseBranch=$1
-    shift
-else
-    trace "Usage: $0 BASE_BRANCH [REPOSITORY]"
-    trace "BASE_BRANCH: The base branch to use, e.g. master or release-23.11"
-    trace "REPOSITORY: The repository to fetch the base branch from, defaults to $repo"
-    exit 1
-fi
-
-if (( $# != 0 )); then
-    repo=$1
-    shift
-fi
-
-if [[ -n "$(git status --porcelain)" ]]; then
-    trace -e "\e[33mWarning: Dirty tree, uncommitted changes won't be taken into account\e[0m"
-fi
-headSha=$(git rev-parse HEAD)
-trace -e "Using HEAD commit \e[34m$headSha\e[0m"
-
-trace -n "Creating Git worktree for the HEAD commit in $tmp/merged.. "
-git worktree add --detach -q "$tmp/merged" HEAD
-trace "Done"
-
-trace -n "Fetching base branch $baseBranch to compare against.. "
-git fetch -q "$repo" refs/heads/"$baseBranch"
-baseSha=$(git rev-parse FETCH_HEAD)
-trace -e "\e[34m$baseSha\e[0m"
-
-trace -n "Creating Git worktree for the base branch in $tmp/base.. "
-git worktree add -q "$tmp/base" "$baseSha"
-trace "Done"
-
-trace -n "Merging base branch into the HEAD commit in $tmp/merged.. "
-git -C "$tmp/merged" merge -q --no-edit "$baseSha"
-trace -e "\e[34m$(git -C "$tmp/merged" rev-parse HEAD)\e[0m"
-
-trace -n "Reading pinned nixpkgs-check-by-name revision from pinned-tool.json.. "
-toolSha=$(jq -r .rev "$tmp/merged/pkgs/test/nixpkgs-check-by-name/scripts/pinned-tool.json")
-trace -e "\e[34m$toolSha\e[0m"
-
-trace -n "Creating Git worktree for the nixpkgs-check-by-name revision in $tmp/tool-nixpkgs.. "
-git worktree add -q "$tmp/tool-nixpkgs" "$toolSha"
-trace "Done"
-
-trace "Building/fetching nixpkgs-check-by-name.."
-nix-build -o "$tmp/tool" "$tmp/tool-nixpkgs" \
-    -A tests.nixpkgs-check-by-name \
-    --arg config '{}' \
-    --arg overlays '[]' \
-    -j 0
-
-trace "Running nixpkgs-check-by-name.."
-"$tmp/tool/bin/nixpkgs-check-by-name" --base "$tmp/base" "$tmp/merged"

pkgs/test/nixpkgs-check-by-name/scripts/update-pinned-tool.sh

···
-#!/usr/bin/env nix-shell
-#!nix-shell -i bash -p jq
-
-set -o pipefail -o errexit -o nounset
-
-trace() { echo >&2 "$@"; }
-
-SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
-
-# Determined by `runs-on: ubuntu-latest` in .github/workflows/check-by-name.yml
-CI_SYSTEM=x86_64-linux
-
-channel=nixos-unstable
-pin_file=$SCRIPT_DIR/pinned-tool.json
-
-trace -n "Fetching latest version of channel $channel.. "
-# This is probably the easiest way to get Nix to output the path to a downloaded channel!
-nixpkgs=$(nix-instantiate --find-file nixpkgs -I nixpkgs=channel:"$channel")
-trace "$nixpkgs"
-
-# This file only exists in channels
-rev=$(<"$nixpkgs/.git-revision")
-trace -e "Git revision of channel $channel is \e[34m$rev\e[0m"
-
-trace -n "Fetching the prebuilt version of nixpkgs-check-by-name for $CI_SYSTEM.. "
-# This is the architecture used by CI, we want to prefetch the exact path to avoid having to evaluate Nixpkgs
-ci_path=$(nix-build --no-out-link "$nixpkgs" \
-    -A tests.nixpkgs-check-by-name \
-    --arg config '{}' \
-    --argstr system "$CI_SYSTEM" \
-    --arg overlays '[]' \
-    -j 0 \
-    | tee /dev/stderr)
-
-trace "Updating $pin_file"
-jq -n \
-    --arg rev "$rev" \
-    --arg ci-path "$ci_path" \
-    '$ARGS.named' \
-    > "$pin_file"