···158158 (onFullSupported "nixpkgs.emacs")159159 (onFullSupported "nixpkgs.jdk")160160 ["nixpkgs.tarball"]161161+162162+ # Ensure that nixpkgs-check-by-name is available in all release channels and nixos-unstable,163163+ # so that a pre-built version can be used in CI for PR's on the corresponding development branches.164164+ # See ../pkgs/test/nixpkgs-check-by-name/README.md165165+ (onSystems ["x86_64-linux"] "nixpkgs.tests.nixpkgs-check-by-name")161166 ];162167 };163168}
+13-5
pkgs/applications/audio/patchance/default.nix
···11-{ lib, fetchurl, buildPythonApplication, libjack2, pyqt5, qttools, which }:11+{ lib, fetchurl, buildPythonApplication, libjack2, pyqt5, qt5, which, bash }:2233buildPythonApplication rec {44 pname = "patchance";55- version = "1.0.0";55+ version = "1.1.0";6677 src = fetchurl {88 url = "https://github.com/Houston4444/Patchance/releases/download/v${version}/Patchance-${version}-source.tar.gz";99- sha256 = "sha256-8Zn6xcDv4hBFXnaXK9xslYEB8uHEfIP+1NKvcPAyHj0=";99+ sha256 = "sha256-wlkEKkPH2C/y7TQicIVycWbtLUdX2hICcUWi7nFN51w=";1010 };11111212 format = "other";13131414 nativeBuildInputs = [1515 pyqt5 # pyuic5 and pyrcc5 to build resources.1616- qttools # lrelease to build translations.1616+ qt5.qttools # lrelease to build translations.1717 which # which to find lrelease.1818+ qt5.wrapQtAppsHook1819 ];1919- buildInputs = [ libjack2 ];2020+ buildInputs = [ libjack2 bash ];2021 propagatedBuildInputs = [ pyqt5 ];21222223 dontWrapQtApps = true; # The program is a python script.···2827 "--prefix" "LD_LIBRARY_PATH" ":" (lib.makeLibraryPath [ libjack2 ])2928 ];30293030+ preFixup = ''3131+ makeWrapperArgs+=("''${qtWrapperArgs[@]}")3232+ '';3333+3134 postFixup = ''3235 wrapPythonProgramsIn "$out/share/patchance/src" "$out $pythonPath"3636+ for file in $out/bin/*; do3737+ wrapQtApp "$file"3838+ done3339 '';34403541 meta = with lib; {
+6-2
pkgs/applications/audio/raysession/default.nix
···11-{ lib, fetchurl, buildPythonApplication, libjack2, pydbus, pyliblo, pyqt5, qttools, which, bash }:11+{ lib, fetchurl, buildPythonApplication, libjack2, pydbus, pyliblo, pyqt5, which, bash, qt5 }:2233buildPythonApplication rec {44 pname = "raysession";···20202121 nativeBuildInputs = [2222 pyqt5 # pyuic5 and pyrcc5 to build resources.2323- qttools # lrelease to build translations.2323+ qt5.qttools # lrelease to build translations.2424 which # which to find lrelease.2525+ qt5.wrapQtAppsHook2526 ];2627 buildInputs = [ libjack2 bash ];2728 propagatedBuildInputs = [ pydbus pyliblo pyqt5 ];···37363837 postFixup = ''3938 wrapPythonProgramsIn "$out/share/raysession/src" "$out $pythonPath"3939+ for file in $out/bin/*; do4040+ wrapQtApp "$file"4141+ done4042 '';41434244 meta = with lib; {
···13131414buildPythonPackage rec {1515 pname = "dbus-fast";1616- version = "1.94.0";1616+ version = "1.94.1";1717 format = "pyproject";18181919 disabled = pythonOlder "3.7";···2222 owner = "Bluetooth-Devices";2323 repo = pname;2424 rev = "refs/tags/v${version}";2525- hash = "sha256-0xfoo131l0c1hA7aZpYQJ/kBp8AtAF6aYctBEk++Fqg=";2525+ hash = "sha256-Ttz6AX/NH6/NNLgU2cMSb5e1jV/cq0LGW3ENARRP7H4=";2626 };27272828 # The project can build both an optimized cython version and an unoptimized
···11+# Nixpkgs pkgs/by-name checker22+33+This directory implements a program to check the [validity](#validity-checks) of the `pkgs/by-name` Nixpkgs directory once introduced.44+This is part of the implementation of [RFC 140](https://github.com/NixOS/rfcs/pull/140).55+66+## API77+88+This API may be changed over time if the CI making use of it is adjusted to deal with the change appropriately, see [Hydra builds](#hydra-builds).99+1010+- Command line: `nixpkgs-check-by-name <NIXPKGS>`1111+- Arguments:1212+ - `<NIXPKGS>`: The path to the Nixpkgs to check1313+- Exit code:1414+ - `0`: If the [validation](#validity-checks) is successful1515+ - `1`: If the [validation](#validity-checks) is not successful1616+ - `2`: If an unexpected I/O error occurs1717+- Standard error:1818+ - Informative messages1919+ - Error messages if validation is not successful2020+2121+## Validity checks2222+2323+These checks are performed by this tool:2424+2525+### File structure checks2626+- `pkgs/by-name` must only contain subdirectories of the form `${shard}/${name}`, called _package directories_.2727+- The `name`'s of package directories must be unique when lowercased2828+- `name` is a string only consisting of the ASCII characters `a-z`, `A-Z`, `0-9`, `-` or `_`.2929+- `shard` is the lowercased first two letters of `name`, expressed in Nix: `shard = toLower (substring 0 2 name)`.3030+- Each package directory must contain a `package.nix` file and may contain arbitrary other files.3131+3232+### Nix parser checks3333+- Each package directory must not refer to files outside itself using symlinks or Nix path expressions.3434+3535+### Nix evaluation checks3636+- `pkgs.${name}` is defined as `callPackage pkgs/by-name/${shard}/${name}/package.nix args` for some `args`.3737+- `pkgs.lib.isDerivation pkgs.${name}` is `true`.3838+3939+## Development4040+4141+Enter the development environment in this directory either automatically with `direnv` or with4242+```4343+nix-shell4444+```4545+4646+Then use `cargo`:4747+```4848+cargo build4949+cargo test5050+cargo fmt5151+cargo clippy5252+```5353+5454+## Tests5555+5656+Tests are declared in [`./tests`](./tests) as subdirectories imitating Nixpkgs with these files:5757+- `default.nix`:5858+ Always contains5959+ ```nix6060+ import ../mock-nixpkgs.nix { root = ./.; }6161+ ```6262+ which makes6363+ ```6464+ nix-instantiate <subdir> --eval -A <attr> --arg overlays <overlays>6565+ ```6666+ work very similarly to the real Nixpkgs, just enough for the program to be able to test it.6767+- `pkgs/by-name`:6868+ The `pkgs/by-name` directory to check.6969+7070+- `all-packages.nix` (optional):7171+ Contains an overlay of the form7272+ ```nix7373+ self: super: {7474+ # ...7575+ }7676+ ```7777+ allowing the simulation of package overrides to the real [`pkgs/top-level/all-packages.nix`](../../top-level/all-packages.nix`).7878+ The default is an empty overlay.7979+8080+- `expected` (optional):8181+ A file containing the expected standard output.8282+ The default is expecting an empty standard output.8383+8484+## Hydra builds8585+8686+This program will always be available pre-built for `x86_64-linux` on the `nixos-unstable` channel and `nixos-XX.YY` channels.8787+This is ensured by including it in the `tested` jobset description in [`nixos/release-combined.nix`](../../../nixos/release-combined.nix).8888+8989+This allows CI for PRs to development branches `master` and `release-XX.YY` to fetch the pre-built program from the corresponding channel and use that to check the PR. This has the following benefits:9090+- It allows CI to check all PRs, even if they would break the CI tooling.9191+- It makes the CI check very fast, since no Nix builds need to be done, even for mass rebuilds.9292+- It improves security, since we don't have to build potentially untrusted code from PRs.9393+ The tool only needs a very minimal Nix evaluation at runtime, which can work with [readonly-mode](https://nixos.org/manual/nix/stable/command-ref/opt-common.html#opt-readonly-mode) and [restrict-eval](https://nixos.org/manual/nix/stable/command-ref/conf-file.html#conf-restrict-eval).9494+- It allows anybody to make updates to the tooling and for those updates to be automatically used by CI without needing a separate release mechanism.9595+9696+The tradeoff is that there's a delay between updates to the tool and those updates being used by CI.9797+This needs to be considered when updating the [API](#api).
···11+# Takes a path to nixpkgs and a path to the json-encoded list of attributes to check.22+# Returns an attribute set containing information on each requested attribute.33+# If the attribute is missing from Nixpkgs it's also missing from the result.44+#55+# The returned information is an attribute set with:66+# - call_package_path: The <path> from `<attr> = callPackage <path> { ... }`,77+# or null if it's not defined as with callPackage, or if the <path> is not a path88+# - is_derivation: The result of `lib.isDerivation <attr>`99+{1010+ attrsPath,1111+ nixpkgsPath,1212+}:1313+let1414+ attrs = builtins.fromJSON (builtins.readFile attrsPath);1515+1616+ # This overlay mocks callPackage to persist the path of the first argument1717+ callPackageOverlay = self: super: {1818+ callPackage = fn: args:1919+ let2020+ result = super.callPackage fn args;2121+ in2222+ if builtins.isAttrs result then2323+ # If this was the last overlay to be applied, we could just only return the `_callPackagePath`,2424+ # but that's not the case because stdenv has another overlays on top of user-provided ones.2525+ # So to not break the stdenv build we need to return the mostly proper result here2626+ result // {2727+ _callPackagePath = fn;2828+ }2929+ else3030+ # It's very rare that callPackage doesn't return an attribute set, but it can occur.3131+ {3232+ _callPackagePath = fn;3333+ };3434+ };3535+3636+ pkgs = import nixpkgsPath {3737+ # Don't let the users home directory influence this result3838+ config = { };3939+ overlays = [ callPackageOverlay ];4040+ };4141+4242+ attrInfo = attr: {4343+ # These names are used by the deserializer on the Rust side4444+ call_package_path =4545+ if pkgs.${attr} ? _callPackagePath && builtins.isPath pkgs.${attr}._callPackagePath then4646+ toString pkgs.${attr}._callPackagePath4747+ else4848+ null;4949+ is_derivation = pkgs.lib.isDerivation pkgs.${attr};5050+ };5151+5252+ attrInfos = builtins.listToAttrs (map (name: {5353+ inherit name;5454+ value = attrInfo name;5555+ }) attrs);5656+5757+in5858+# Filter out attributes not in Nixpkgs5959+builtins.intersectAttrs pkgs attrInfos
+124
pkgs/test/nixpkgs-check-by-name/src/eval.rs
···11+use crate::structure;22+use crate::utils::ErrorWriter;33+use std::path::Path;44+55+use anyhow::Context;66+use serde::Deserialize;77+use std::collections::HashMap;88+use std::io;99+use std::path::PathBuf;1010+use std::process;1111+use tempfile::NamedTempFile;1212+1313+/// Attribute set of this structure is returned by eval.nix1414+#[derive(Deserialize)]1515+struct AttributeInfo {1616+ call_package_path: Option<PathBuf>,1717+ is_derivation: bool,1818+}1919+2020+const EXPR: &str = include_str!("eval.nix");2121+2222+/// Check that the Nixpkgs attribute values corresponding to the packages in pkgs/by-name are2323+/// of the form `callPackage <package_file> { ... }`.2424+/// See the `eval.nix` file for how this is achieved on the Nix side2525+pub fn check_values<W: io::Write>(2626+ error_writer: &mut ErrorWriter<W>,2727+ nixpkgs: &structure::Nixpkgs,2828+ eval_accessible_paths: Vec<&Path>,2929+) -> anyhow::Result<()> {3030+ // Write the list of packages we need to check into a temporary JSON file.3131+ // This can then get read by the Nix evaluation.3232+ let attrs_file = NamedTempFile::new().context("Failed to create a temporary file")?;3333+ serde_json::to_writer(&attrs_file, &nixpkgs.package_names).context(format!(3434+ "Failed to serialise the package names to the temporary path {}",3535+ attrs_file.path().display()3636+ ))?;3737+3838+ // With restrict-eval, only paths in NIX_PATH can be accessed, so we explicitly specify the3939+ // ones needed needed4040+4141+ let mut command = process::Command::new("nix-instantiate");4242+ command4343+ // Inherit stderr so that error messages always get shown4444+ .stderr(process::Stdio::inherit())4545+ // Clear NIX_PATH to be sure it doesn't influence the result4646+ .env_remove("NIX_PATH")4747+ .args([4848+ "--eval",4949+ "--json",5050+ "--strict",5151+ "--readonly-mode",5252+ "--restrict-eval",5353+ "--show-trace",5454+ "--expr",5555+ EXPR,5656+ ])5757+ // Pass the path to the attrs_file as an argument and add it to the NIX_PATH so it can be5858+ // accessed in restrict-eval mode5959+ .args(["--arg", "attrsPath"])6060+ .arg(attrs_file.path())6161+ .arg("-I")6262+ .arg(attrs_file.path())6363+ // Same for the nixpkgs to test6464+ .args(["--arg", "nixpkgsPath"])6565+ .arg(&nixpkgs.path)6666+ .arg("-I")6767+ .arg(&nixpkgs.path);6868+6969+ // Also add extra paths that need to be accessible7070+ for path in eval_accessible_paths {7171+ command.arg("-I");7272+ command.arg(path);7373+ }7474+7575+ let result = command7676+ .output()7777+ .context(format!("Failed to run command {command:?}"))?;7878+7979+ if !result.status.success() {8080+ anyhow::bail!("Failed to run command {command:?}");8181+ }8282+ // Parse the resulting JSON value8383+ let actual_files: HashMap<String, AttributeInfo> = serde_json::from_slice(&result.stdout)8484+ .context(format!(8585+ "Failed to deserialise {}",8686+ String::from_utf8_lossy(&result.stdout)8787+ ))?;8888+8989+ for package_name in &nixpkgs.package_names {9090+ let relative_package_file = structure::Nixpkgs::relative_file_for_package(package_name);9191+ let absolute_package_file = nixpkgs.path.join(&relative_package_file);9292+9393+ if let Some(attribute_info) = actual_files.get(package_name) {9494+ let is_expected_file =9595+ if let Some(call_package_path) = &attribute_info.call_package_path {9696+ absolute_package_file == *call_package_path9797+ } else {9898+ false9999+ };100100+101101+ if !is_expected_file {102102+ error_writer.write(&format!(103103+ "pkgs.{package_name}: This attribute is not defined as `pkgs.callPackage {} {{ ... }}`.",104104+ relative_package_file.display()105105+ ))?;106106+ continue;107107+ }108108+109109+ if !attribute_info.is_derivation {110110+ error_writer.write(&format!(111111+ "pkgs.{package_name}: This attribute defined by {} is not a derivation",112112+ relative_package_file.display()113113+ ))?;114114+ }115115+ } else {116116+ error_writer.write(&format!(117117+ "pkgs.{package_name}: This attribute is not defined but it should be defined automatically as {}",118118+ relative_package_file.display()119119+ ))?;120120+ continue;121121+ }122122+ }123123+ Ok(())124124+}
+133
pkgs/test/nixpkgs-check-by-name/src/main.rs
···11+mod eval;22+mod references;33+mod structure;44+mod utils;55+66+use anyhow::Context;77+use clap::Parser;88+use colored::Colorize;99+use std::io;1010+use std::path::{Path, PathBuf};1111+use std::process::ExitCode;1212+use structure::Nixpkgs;1313+use utils::ErrorWriter;1414+1515+/// Program to check the validity of pkgs/by-name1616+#[derive(Parser, Debug)]1717+#[command(about)]1818+struct Args {1919+ /// Path to nixpkgs2020+ nixpkgs: PathBuf,2121+}2222+2323+fn main() -> ExitCode {2424+ let args = Args::parse();2525+ match check_nixpkgs(&args.nixpkgs, vec![], &mut io::stderr()) {2626+ Ok(true) => {2727+ eprintln!("{}", "Validated successfully".green());2828+ ExitCode::SUCCESS2929+ }3030+ Ok(false) => {3131+ eprintln!("{}", "Validation failed, see above errors".yellow());3232+ ExitCode::from(1)3333+ }3434+ Err(e) => {3535+ eprintln!("{} {:#}", "I/O error: ".yellow(), e);3636+ ExitCode::from(2)3737+ }3838+ }3939+}4040+4141+/// Checks whether the pkgs/by-name structure in Nixpkgs is valid.4242+///4343+/// # Arguments4444+/// - `nixpkgs_path`: The path to the Nixpkgs to check4545+/// - `eval_accessible_paths`:4646+/// Extra paths that need to be accessible to evaluate Nixpkgs using `restrict-eval`.4747+/// This is used to allow the tests to access the mock-nixpkgs.nix file4848+/// - `error_writer`: An `io::Write` value to write validation errors to, if any.4949+///5050+/// # Return value5151+/// - `Err(e)` if an I/O-related error `e` occurred.5252+/// - `Ok(false)` if the structure is invalid, all the structural errors have been written to `error_writer`.5353+/// - `Ok(true)` if the structure is valid, nothing will have been written to `error_writer`.5454+pub fn check_nixpkgs<W: io::Write>(5555+ nixpkgs_path: &Path,5656+ eval_accessible_paths: Vec<&Path>,5757+ error_writer: &mut W,5858+) -> anyhow::Result<bool> {5959+ let nixpkgs_path = nixpkgs_path.canonicalize().context(format!(6060+ "Nixpkgs path {} could not be resolved",6161+ nixpkgs_path.display()6262+ ))?;6363+6464+ // Wraps the error_writer to print everything in red, and tracks whether anything was printed6565+ // at all. Later used to figure out if the structure was valid or not.6666+ let mut error_writer = ErrorWriter::new(error_writer);6767+6868+ if !nixpkgs_path.join(structure::BASE_SUBPATH).exists() {6969+ eprintln!(7070+ "Given Nixpkgs path does not contain a {} subdirectory, no check necessary.",7171+ structure::BASE_SUBPATH7272+ );7373+ } else {7474+ let nixpkgs = Nixpkgs::new(&nixpkgs_path, &mut error_writer)?;7575+7676+ if error_writer.empty {7777+ // Only if we could successfully parse the structure, we do the semantic checks7878+ eval::check_values(&mut error_writer, &nixpkgs, eval_accessible_paths)?;7979+ references::check_references(&mut error_writer, &nixpkgs)?;8080+ }8181+ }8282+ Ok(error_writer.empty)8383+}8484+8585+#[cfg(test)]8686+mod tests {8787+ use crate::check_nixpkgs;8888+ use anyhow::Context;8989+ use std::env;9090+ use std::fs;9191+ use std::path::PathBuf;9292+9393+ #[test]9494+ fn test_cases() -> anyhow::Result<()> {9595+ let extra_nix_path = PathBuf::from("tests/mock-nixpkgs.nix");9696+9797+ // We don't want coloring to mess up the tests9898+ env::set_var("NO_COLOR", "1");9999+100100+ for entry in PathBuf::from("tests").read_dir()? {101101+ let entry = entry?;102102+ let path = entry.path();103103+ let name = entry.file_name().to_string_lossy().into_owned();104104+105105+ if !entry.path().is_dir() {106106+ continue;107107+ }108108+109109+ // This test explicitly makes sure we don't add files that would cause problems on110110+ // Darwin, so we cannot test it on Darwin itself111111+ #[cfg(not(target_os = "linux"))]112112+ if name == "case-sensitive-duplicate-package" {113113+ continue;114114+ }115115+116116+ let mut writer = vec![];117117+ check_nixpkgs(&path, vec![&extra_nix_path], &mut writer)118118+ .context(format!("Failed test case {name}"))?;119119+120120+ let actual_errors = String::from_utf8_lossy(&writer);121121+ let expected_errors =122122+ fs::read_to_string(path.join("expected")).unwrap_or(String::new());123123+124124+ if actual_errors != expected_errors {125125+ panic!(126126+ "Failed test case {name}, expected these errors:\n\n{}\n\nbut got these:\n\n{}",127127+ expected_errors, actual_errors128128+ );129129+ }130130+ }131131+ Ok(())132132+ }133133+}
+184
pkgs/test/nixpkgs-check-by-name/src/references.rs
···11+use crate::structure::Nixpkgs;22+use crate::utils;33+use crate::utils::{ErrorWriter, LineIndex};44+55+use anyhow::Context;66+use rnix::{Root, SyntaxKind::NODE_PATH};77+use std::ffi::OsStr;88+use std::fs::read_to_string;99+use std::io;1010+use std::path::{Path, PathBuf};1111+1212+/// Small helper so we don't need to pass in the same arguments to all functions1313+struct PackageContext<'a, W: io::Write> {1414+ error_writer: &'a mut ErrorWriter<W>,1515+ /// The package directory relative to Nixpkgs, such as `pkgs/by-name/fo/foo`1616+ relative_package_dir: &'a PathBuf,1717+ /// The absolute package directory1818+ absolute_package_dir: &'a PathBuf,1919+}2020+2121+/// Check that every package directory in pkgs/by-name doesn't link to outside that directory.2222+/// Both symlinks and Nix path expressions are checked.2323+pub fn check_references<W: io::Write>(2424+ error_writer: &mut ErrorWriter<W>,2525+ nixpkgs: &Nixpkgs,2626+) -> anyhow::Result<()> {2727+ // Check the directories for each package separately2828+ for package_name in &nixpkgs.package_names {2929+ let relative_package_dir = Nixpkgs::relative_dir_for_package(package_name);3030+ let mut context = PackageContext {3131+ error_writer,3232+ relative_package_dir: &relative_package_dir,3333+ absolute_package_dir: &nixpkgs.path.join(&relative_package_dir),3434+ };3535+3636+ // The empty argument here is the subpath under the package directory to check3737+ // An empty one means the package directory itself3838+ check_path(&mut context, Path::new("")).context(format!(3939+ "While checking the references in package directory {}",4040+ relative_package_dir.display()4141+ ))?;4242+ }4343+ Ok(())4444+}4545+4646+/// Checks for a specific path to not have references outside4747+fn check_path<W: io::Write>(context: &mut PackageContext<W>, subpath: &Path) -> anyhow::Result<()> {4848+ let path = context.absolute_package_dir.join(subpath);4949+5050+ if path.is_symlink() {5151+ // Check whether the symlink resolves to outside the package directory5252+ match path.canonicalize() {5353+ Ok(target) => {5454+ // No need to handle the case of it being inside the directory, since we scan through the5555+ // entire directory recursively anyways5656+ if let Err(_prefix_error) = target.strip_prefix(context.absolute_package_dir) {5757+ context.error_writer.write(&format!(5858+ "{}: Path {} is a symlink pointing to a path outside the directory of that package.",5959+ context.relative_package_dir.display(),6060+ subpath.display(),6161+ ))?;6262+ }6363+ }6464+ Err(e) => {6565+ context.error_writer.write(&format!(6666+ "{}: Path {} is a symlink which cannot be resolved: {e}.",6767+ context.relative_package_dir.display(),6868+ subpath.display(),6969+ ))?;7070+ }7171+ }7272+ } else if path.is_dir() {7373+ // Recursively check each entry7474+ for entry in utils::read_dir_sorted(&path)? {7575+ let entry_subpath = subpath.join(entry.file_name());7676+ check_path(context, &entry_subpath)7777+ .context(format!("Error while recursing into {}", subpath.display()))?7878+ }7979+ } else if path.is_file() {8080+ // Only check Nix files8181+ if let Some(ext) = path.extension() {8282+ if ext == OsStr::new("nix") {8383+ check_nix_file(context, subpath).context(format!(8484+ "Error while checking Nix file {}",8585+ subpath.display()8686+ ))?8787+ }8888+ }8989+ } else {9090+ // This should never happen, git doesn't support other file types9191+ anyhow::bail!("Unsupported file type for path {}", subpath.display());9292+ }9393+ Ok(())9494+}9595+9696+/// Check whether a nix file contains path expression references pointing outside the package9797+/// directory9898+fn check_nix_file<W: io::Write>(9999+ context: &mut PackageContext<W>,100100+ subpath: &Path,101101+) -> anyhow::Result<()> {102102+ let path = context.absolute_package_dir.join(subpath);103103+ let parent_dir = path.parent().context(format!(104104+ "Could not get parent of path {}",105105+ subpath.display()106106+ ))?;107107+108108+ let contents =109109+ read_to_string(&path).context(format!("Could not read file {}", subpath.display()))?;110110+111111+ let root = Root::parse(&contents);112112+ if let Some(error) = root.errors().first() {113113+ context.error_writer.write(&format!(114114+ "{}: File {} could not be parsed by rnix: {}",115115+ context.relative_package_dir.display(),116116+ subpath.display(),117117+ error,118118+ ))?;119119+ return Ok(());120120+ }121121+122122+ let line_index = LineIndex::new(&contents);123123+124124+ for node in root.syntax().descendants() {125125+ // We're only interested in Path expressions126126+ if node.kind() != NODE_PATH {127127+ continue;128128+ }129129+130130+ let text = node.text().to_string();131131+ let line = line_index.line(node.text_range().start().into());132132+133133+ // Filters out ./foo/${bar}/baz134134+ // TODO: We can just check ./foo135135+ if node.children().count() != 0 {136136+ context.error_writer.write(&format!(137137+ "{}: File {} at line {line} contains the path expression \"{}\", which is not yet supported and may point outside the directory of that package.",138138+ context.relative_package_dir.display(),139139+ subpath.display(),140140+ text141141+ ))?;142142+ continue;143143+ }144144+145145+ // Filters out search paths like <nixpkgs>146146+ if text.starts_with('<') {147147+ context.error_writer.write(&format!(148148+ "{}: File {} at line {line} contains the nix search path expression \"{}\" which may point outside the directory of that package.",149149+ context.relative_package_dir.display(),150150+ subpath.display(),151151+ text152152+ ))?;153153+ continue;154154+ }155155+156156+ // Resolves the reference of the Nix path157157+ // turning `../baz` inside `/foo/bar/default.nix` to `/foo/baz`158158+ match parent_dir.join(Path::new(&text)).canonicalize() {159159+ Ok(target) => {160160+ // Then checking if it's still in the package directory161161+ // No need to handle the case of it being inside the directory, since we scan through the162162+ // entire directory recursively anyways163163+ if let Err(_prefix_error) = target.strip_prefix(context.absolute_package_dir) {164164+ context.error_writer.write(&format!(165165+ "{}: File {} at line {line} contains the path expression \"{}\" which may point outside the directory of that package.",166166+ context.relative_package_dir.display(),167167+ subpath.display(),168168+ text,169169+ ))?;170170+ }171171+ }172172+ Err(e) => {173173+ context.error_writer.write(&format!(174174+ "{}: File {} at line {line} contains the path expression \"{}\" which cannot be resolved: {e}.",175175+ context.relative_package_dir.display(),176176+ subpath.display(),177177+ text,178178+ ))?;179179+ }180180+ };181181+ }182182+183183+ Ok(())184184+}
+152
pkgs/test/nixpkgs-check-by-name/src/structure.rs
···11+use crate::utils;22+use crate::utils::ErrorWriter;33+use lazy_static::lazy_static;44+use regex::Regex;55+use std::collections::HashMap;66+use std::io;77+use std::path::{Path, PathBuf};88+99+pub const BASE_SUBPATH: &str = "pkgs/by-name";1010+pub const PACKAGE_NIX_FILENAME: &str = "package.nix";1111+1212+lazy_static! {1313+ static ref SHARD_NAME_REGEX: Regex = Regex::new(r"^[a-z0-9_-]{1,2}$").unwrap();1414+ static ref PACKAGE_NAME_REGEX: Regex = Regex::new(r"^[a-zA-Z0-9_-]+$").unwrap();1515+}1616+1717+/// Contains information about the structure of the pkgs/by-name directory of a Nixpkgs1818+pub struct Nixpkgs {1919+ /// The path to nixpkgs2020+ pub path: PathBuf,2121+ /// The names of all packages declared in pkgs/by-name2222+ pub package_names: Vec<String>,2323+}2424+2525+impl Nixpkgs {2626+ // Some utility functions for the basic structure2727+2828+ pub fn shard_for_package(package_name: &str) -> String {2929+ package_name.to_lowercase().chars().take(2).collect()3030+ }3131+3232+ pub fn relative_dir_for_shard(shard_name: &str) -> PathBuf {3333+ PathBuf::from(format!("{BASE_SUBPATH}/{shard_name}"))3434+ }3535+3636+ pub fn relative_dir_for_package(package_name: &str) -> PathBuf {3737+ Nixpkgs::relative_dir_for_shard(&Nixpkgs::shard_for_package(package_name))3838+ .join(package_name)3939+ }4040+4141+ pub fn relative_file_for_package(package_name: &str) -> PathBuf {4242+ Nixpkgs::relative_dir_for_package(package_name).join(PACKAGE_NIX_FILENAME)4343+ }4444+}4545+4646+impl Nixpkgs {4747+ /// Read the structure of a Nixpkgs directory, displaying errors on the writer.4848+ /// May return early with I/O errors.4949+ pub fn new<W: io::Write>(5050+ path: &Path,5151+ error_writer: &mut ErrorWriter<W>,5252+ ) -> anyhow::Result<Nixpkgs> {5353+ let base_dir = path.join(BASE_SUBPATH);5454+5555+ let mut package_names = Vec::new();5656+5757+ for shard_entry in utils::read_dir_sorted(&base_dir)? {5858+ let shard_path = shard_entry.path();5959+ let shard_name = shard_entry.file_name().to_string_lossy().into_owned();6060+ let relative_shard_path = Nixpkgs::relative_dir_for_shard(&shard_name);6161+6262+ if shard_name == "README.md" {6363+ // README.md is allowed to be a file and not checked6464+ continue;6565+ }6666+6767+ if !shard_path.is_dir() {6868+ error_writer.write(&format!(6969+ "{}: This is a file, but it should be a directory.",7070+ relative_shard_path.display(),7171+ ))?;7272+ // we can't check for any other errors if it's a file, since there's no subdirectories to check7373+ continue;7474+ }7575+7676+ let shard_name_valid = SHARD_NAME_REGEX.is_match(&shard_name);7777+ if !shard_name_valid {7878+ error_writer.write(&format!(7979+ "{}: Invalid directory name \"{shard_name}\", must be at most 2 ASCII characters consisting of a-z, 0-9, \"-\" or \"_\".",8080+ relative_shard_path.display()8181+ ))?;8282+ }8383+8484+ let mut unique_package_names = HashMap::new();8585+8686+ for package_entry in utils::read_dir_sorted(&shard_path)? {8787+ let package_path = package_entry.path();8888+ let package_name = package_entry.file_name().to_string_lossy().into_owned();8989+ let relative_package_dir =9090+ PathBuf::from(format!("{BASE_SUBPATH}/{shard_name}/{package_name}"));9191+9292+ if !package_path.is_dir() {9393+ error_writer.write(&format!(9494+ "{}: This path is a file, but it should be a directory.",9595+ relative_package_dir.display(),9696+ ))?;9797+ continue;9898+ }9999+100100+ if let Some(duplicate_package_name) =101101+ unique_package_names.insert(package_name.to_lowercase(), package_name.clone())102102+ {103103+ error_writer.write(&format!(104104+ "{}: Duplicate case-sensitive package directories \"{duplicate_package_name}\" and \"{package_name}\".",105105+ relative_shard_path.display(),106106+ ))?;107107+ }108108+109109+ let package_name_valid = PACKAGE_NAME_REGEX.is_match(&package_name);110110+ if !package_name_valid {111111+ error_writer.write(&format!(112112+ "{}: Invalid package directory name \"{package_name}\", must be ASCII characters consisting of a-z, A-Z, 0-9, \"-\" or \"_\".",113113+ relative_package_dir.display(),114114+ ))?;115115+ }116116+117117+ let correct_relative_package_dir = Nixpkgs::relative_dir_for_package(&package_name);118118+ if relative_package_dir != correct_relative_package_dir {119119+ // Only show this error if we have a valid shard and package name120120+ // Because if one of those is wrong, you should fix that first121121+ if shard_name_valid && package_name_valid {122122+ error_writer.write(&format!(123123+ "{}: Incorrect directory location, should be {} instead.",124124+ relative_package_dir.display(),125125+ correct_relative_package_dir.display(),126126+ ))?;127127+ }128128+ }129129+130130+ let package_nix_path = package_path.join(PACKAGE_NIX_FILENAME);131131+ if !package_nix_path.exists() {132132+ error_writer.write(&format!(133133+ "{}: Missing required \"{PACKAGE_NIX_FILENAME}\" file.",134134+ relative_package_dir.display(),135135+ ))?;136136+ } else if package_nix_path.is_dir() {137137+ error_writer.write(&format!(138138+ "{}: \"{PACKAGE_NIX_FILENAME}\" must be a file.",139139+ relative_package_dir.display(),140140+ ))?;141141+ }142142+143143+ package_names.push(package_name.clone());144144+ }145145+ }146146+147147+ Ok(Nixpkgs {148148+ path: path.to_owned(),149149+ package_names,150150+ })151151+ }152152+}
+72
pkgs/test/nixpkgs-check-by-name/src/utils.rs
···11+use anyhow::Context;22+use colored::Colorize;33+use std::fs;44+use std::io;55+use std::path::Path;66+77+/// Deterministic file listing so that tests are reproducible88+pub fn read_dir_sorted(base_dir: &Path) -> anyhow::Result<Vec<fs::DirEntry>> {99+ let listing = base_dir1010+ .read_dir()1111+ .context(format!("Could not list directory {}", base_dir.display()))?;1212+ let mut shard_entries = listing1313+ .collect::<io::Result<Vec<_>>>()1414+ .context(format!("Could not list directory {}", base_dir.display()))?;1515+ shard_entries.sort_by_key(|entry| entry.file_name());1616+ Ok(shard_entries)1717+}1818+1919+/// A simple utility for calculating the line for a string offset.2020+/// This doesn't do any Unicode handling, though that probably doesn't matter2121+/// because newlines can't split up Unicode characters. Also this is only used2222+/// for error reporting2323+pub struct LineIndex {2424+ /// Stores the indices of newlines2525+ newlines: Vec<usize>,2626+}2727+2828+impl LineIndex {2929+ pub fn new(s: &str) -> LineIndex {3030+ let mut newlines = vec![];3131+ let mut index = 0;3232+ // Iterates over all newline-split parts of the string, adding the index of the newline to3333+ // the vec3434+ for split in s.split_inclusive('\n') {3535+ index += split.len();3636+ newlines.push(index);3737+ }3838+ LineIndex { newlines }3939+ }4040+4141+ /// Returns the line number for a string index4242+ pub fn line(&self, index: usize) -> usize {4343+ match self.newlines.binary_search(&index) {4444+ // +1 because lines are 1-indexed4545+ Ok(x) => x + 1,4646+ Err(x) => x + 1,4747+ }4848+ }4949+}5050+5151+/// A small wrapper around a generic io::Write specifically for errors:5252+/// - Print everything in red to signal it's an error5353+/// - Keep track of whether anything was printed at all, so that5454+/// it can be queried whether any errors were encountered at all5555+pub struct ErrorWriter<W> {5656+ pub writer: W,5757+ pub empty: bool,5858+}5959+6060+impl<W: io::Write> ErrorWriter<W> {6161+ pub fn new(writer: W) -> ErrorWriter<W> {6262+ ErrorWriter {6363+ writer,6464+ empty: true,6565+ }6666+ }6767+6868+ pub fn write(&mut self, string: &str) -> io::Result<()> {6969+ self.empty = false;7070+ writeln!(self.writer, "{}", string.red())7171+ }7272+}
···11+/*22+This file returns a mocked version of Nixpkgs' default.nix for testing purposes.33+It does not depend on Nixpkgs itself for the sake of simplicity.44+55+It takes one attribute as an argument:66+- `root`: The root of Nixpkgs to read other files from, including:77+ - `./pkgs/by-name`: The `pkgs/by-name` directory to test88+ - `./all-packages.nix`: A file containing an overlay to mirror the real `pkgs/top-level/all-packages.nix`.99+ This allows adding overrides on top of the auto-called packages in `pkgs/by-name`.1010+1111+It returns a Nixpkgs-like function that can be auto-called and evaluates to an attribute set.1212+*/1313+{1414+ root,1515+}:1616+# The arguments for the Nixpkgs function1717+{1818+ # Passed by the checker to modify `callPackage`1919+ overlays ? [],2020+ # Passed by the checker to make sure a real Nixpkgs isn't influenced by impurities2121+ config ? {},2222+}:2323+let2424+2525+ # Simplified versions of lib functions2626+ lib = {2727+ fix = f: let x = f x; in x;2828+2929+ extends = overlay: f: final:3030+ let3131+ prev = f final;3232+ in3333+ prev // overlay final prev;3434+3535+ callPackageWith = autoArgs: fn: args:3636+ let3737+ f = if builtins.isFunction fn then fn else import fn;3838+ fargs = builtins.functionArgs f;3939+ allArgs = builtins.intersectAttrs fargs autoArgs // args;4040+ in4141+ f allArgs;4242+4343+ isDerivation = value: value.type or null == "derivation";4444+ };4545+4646+ # The base fixed-point function to populate the resulting attribute set4747+ pkgsFun = self: {4848+ inherit lib;4949+ callPackage = lib.callPackageWith self;5050+ someDrv = { type = "derivation"; };5151+ };5252+5353+ baseDirectory = root + "/pkgs/by-name";5454+5555+ # Generates { <name> = <file>; } entries mapping package names to their `package.nix` files in `pkgs/by-name`.5656+ # Could be more efficient, but this is only for testing.5757+ autoCalledPackageFiles =5858+ let5959+ entries = builtins.readDir baseDirectory;6060+6161+ namesForShard = shard:6262+ if entries.${shard} != "directory" then6363+ # Only README.md is allowed to be a file, but it's not this code's job to check for that6464+ { }6565+ else6666+ builtins.mapAttrs6767+ (name: _: baseDirectory + "/${shard}/${name}/package.nix")6868+ (builtins.readDir (baseDirectory + "/${shard}"));6969+7070+ in7171+ builtins.foldl'7272+ (acc: el: acc // el)7373+ { }7474+ (map namesForShard (builtins.attrNames entries));7575+7676+ # Turns autoCalledPackageFiles into an overlay that `callPackage`'s all of them7777+ autoCalledPackages = self: super:7878+ builtins.mapAttrs (name: file:7979+ self.callPackage file { }8080+ ) autoCalledPackageFiles;8181+8282+ # A list optionally containing the `all-packages.nix` file from the test case as an overlay8383+ optionalAllPackagesOverlay =8484+ if builtins.pathExists (root + "/all-packages.nix") then8585+ [ (import (root + "/all-packages.nix")) ]8686+ else8787+ [ ];8888+8989+ # All the overlays in the right order, including the user-supplied ones9090+ allOverlays =9191+ [9292+ autoCalledPackages9393+ ]9494+ ++ optionalAllPackagesOverlay9595+ ++ overlays;9696+9797+ # Apply all the overlays in order to the base fixed-point function pkgsFun9898+ f = builtins.foldl' (f: overlay: lib.extends overlay f) pkgsFun allOverlays;9999+in100100+# Evaluate the fixed-point101101+lib.fix f
···11+pkgs/by-name/aa/aa: File package.nix at line 2 contains the path expression "/foo" which cannot be resolved: No such file or directory (os error 2).
···11+pkgs/by-name/aa/aa: File package.nix at line 2 contains the nix search path expression "<nixpkgs>" which may point outside the directory of that package.
···11+pkgs/by-name/aa/aa: File package.nix at line 2 contains the path expression "./${"test"}", which is not yet supported and may point outside the directory of that package.