Clone of https://github.com/NixOS/nixpkgs.git (to stress-test knotserver)

prefetch-npm-deps: download dev deps for git deps with install scripts

Git dependencies with install scripts are built isolated from the main
package, so their development dependencies are required.

To take advantage of this, #206477 is needed.

authored by winter.bsky.social and committed by

Lily Foster 7e247e6f 3e68a278

+73 -10
+4 -1
pkgs/build-support/node/build-npm-package/default.nix
··· 12 12 # The output hash of the dependencies for this project. 13 13 # Can be calculated in advance with prefetch-npm-deps. 14 14 , npmDepsHash ? "" 15 + # Whether to force the usage of Git dependencies that have install scripts, but not a lockfile. 16 + # Use with care. 17 + , forceGitDeps ? false 15 18 # Whether to make the cache writable prior to installing dependencies. 16 19 # Don't set this unless npm tries to write to the cache directory, as it can slow down the build. 17 20 , makeCacheWritable ? false ··· 32 35 33 36 let 34 37 npmDeps = fetchNpmDeps { 35 - inherit src srcs sourceRoot prePatch patches postPatch; 38 + inherit forceGitDeps src srcs sourceRoot prePatch patches postPatch; 36 39 name = "${name}-npm-deps"; 37 40 hash = npmDepsHash; 38 41 };
+8 -3
pkgs/build-support/node/fetch-npm-deps/default.nix
··· 36 36 ''; 37 37 }; 38 38 39 - makeTest = { name, src, hash }: testers.invalidateFetcherByDrvHash fetchNpmDeps { 40 - inherit name hash; 39 + makeTest = { name, src, hash, forceGitDeps ? false }: testers.invalidateFetcherByDrvHash fetchNpmDeps { 40 + inherit name hash forceGitDeps; 41 41 42 42 src = makeTestSrc { inherit name src; }; 43 43 }; ··· 108 108 }; 109 109 110 110 hash = "sha256-+KA8/orSBJ4EhuSyQO8IKSxsN/FAsYU3lOzq+awuxNQ="; 111 + 112 + forceGitDeps = true; 111 113 }; 112 114 }; 113 115 ··· 121 123 fetchNpmDeps = 122 124 { name ? "npm-deps" 123 125 , hash ? "" 126 + , forceGitDeps ? false 124 127 , ... 125 128 } @ args: 126 129 let ··· 131 134 outputHash = ""; 132 135 outputHashAlgo = "sha256"; 133 136 }; 137 + 138 + forceGitDeps_ = lib.optionalAttrs forceGitDeps { FORCE_GIT_DEPS = true; }; 134 139 in 135 140 stdenvNoCC.mkDerivation (args // { 136 141 inherit name; ··· 161 166 dontInstall = true; 162 167 163 168 outputHashMode = "recursive"; 164 - } // hash_); 169 + } // hash_ // forceGitDeps_); 165 170 }
+1 -1
pkgs/build-support/node/fetch-npm-deps/src/main.rs
··· 97 97 (out_tempdir.path(), true) 98 98 }; 99 99 100 - let packages = parse::lockfile(&lock_content)?; 100 + let packages = parse::lockfile(&lock_content, env::var("FORCE_GIT_DEPS").is_ok())?; 101 101 102 102 let cache = Cache::new(out.join("_cacache")); 103 103
+60 -5
pkgs/build-support/node/fetch-npm-deps/src/parse/mod.rs
··· 1 1 use anyhow::{anyhow, bail, Context}; 2 2 use lock::UrlOrString; 3 3 use rayon::prelude::*; 4 + use serde_json::{Map, Value}; 4 5 use std::{ 5 6 fs, io, 6 7 process::{Command, Stdio}, ··· 10 11 11 12 mod lock; 12 13 13 - pub fn lockfile(lockfile: &str) -> anyhow::Result<Vec<Package>> { 14 - let packages = lock::packages(lockfile).context("failed to extract packages from lockfile")?; 15 - 16 - packages 14 + pub fn lockfile(content: &str, force_git_deps: bool) -> anyhow::Result<Vec<Package>> { 15 + let mut packages = lock::packages(content) 16 + .context("failed to extract packages from lockfile")? 17 17 .into_par_iter() 18 18 .map(|p| { 19 19 let n = p.name.clone().unwrap(); 20 20 21 21 Package::from_lock(p).with_context(|| format!("failed to parse data for {n}")) 22 22 }) 23 - .collect() 23 + .collect::<anyhow::Result<Vec<_>>>()?; 24 + 25 + let mut new = Vec::new(); 26 + 27 + for pkg in packages 28 + .iter() 29 + .filter(|p| matches!(p.specifics, Specifics::Git { .. })) 30 + { 31 + let dir = match &pkg.specifics { 32 + Specifics::Git { workdir } => workdir, 33 + Specifics::Registry { .. } => unimplemented!(), 34 + }; 35 + 36 + let path = dir.path().join("package"); 37 + 38 + let lockfile_contents = fs::read_to_string(path.join("package-lock.json")); 39 + 40 + let package_json_path = path.join("package.json"); 41 + let mut package_json: Map<String, Value> = 42 + serde_json::from_str(&fs::read_to_string(package_json_path)?)?; 43 + 44 + if let Some(scripts) = package_json 45 + .get_mut("scripts") 46 + .and_then(Value::as_object_mut) 47 + { 48 + // https://github.com/npm/pacote/blob/272edc1bac06991fc5f95d06342334bbacfbaa4b/lib/git.js#L166-L172 49 + for typ in [ 50 + "postinstall", 51 + "build", 52 + "preinstall", 53 + "install", 54 + "prepack", 55 + "prepare", 56 + ] { 57 + if scripts.contains_key(typ) && lockfile_contents.is_err() && !force_git_deps { 58 + bail!("Git dependency {} contains install scripts, but has no lockfile, which is something that will probably break. Open an issue if you can't feasibly patch this dependency out, and we'll come up with a workaround.\nIf you'd like to attempt to try to use this dependency anyways, set `forceGitDeps = true`.", pkg.name); 59 + } 60 + } 61 + } 62 + 63 + if let Ok(lockfile_contents) = lockfile_contents { 64 + new.append(&mut lockfile(&lockfile_contents, force_git_deps)?); 65 + } 66 + } 67 + 68 + packages.append(&mut new); 69 + 70 + packages.par_sort_by(|x, y| { 71 + x.url 72 + .partial_cmp(&y.url) 73 + .expect("resolved should be comparable") 74 + }); 75 + 76 + packages.dedup_by(|x, y| x.url == y.url); 77 + 78 + Ok(packages) 24 79 } 25 80 26 81 #[derive(Debug)]