tjh.dev
nixpkgs mirror (for testing)
github.com/NixOS/nixpkgs
nix
1{
2 pkgs ? import <nixpkgs> { },
3 nodejs ? pkgs.nodejs,
4 yarn ? pkgs.yarn,
5 allowAliases ? pkgs.config.allowAliases,
6}@inputs:
7
8let
9 inherit (pkgs)
10 stdenv
11 lib
12 callPackage
13 git
14 rsync
15 runCommandLocal
16 ;
17
18 compose =
19 f: g: x:
20 f (g x);
21 id = x: x;
22 composeAll = builtins.foldl' compose id;
23
24 # https://docs.npmjs.com/files/package.json#license
25 # TODO: support expression syntax (OR, AND, etc)
26 getLicenseFromSpdxId =
27 licstr: if licstr == "UNLICENSED" then lib.licenses.unfree else lib.getLicenseFromSpdxId licstr;
28in
29rec {
30 # Export yarn again to make it easier to find out which yarn was used.
31 inherit yarn;
32
33 # Re-export pkgs
34 inherit pkgs;
35
36 unlessNull = item: alt: if item == null then alt else item;
37
38 reformatPackageName =
39 pname:
40 let
41 # regex adapted from `validate-npm-package-name`
42 # will produce 3 parts e.g.
43 # "@someorg/somepackage" -> [ "@someorg/" "someorg" "somepackage" ]
44 # "somepackage" -> [ null null "somepackage" ]
45 parts = builtins.tail (builtins.match "^(@([^/]+)/)?([^/]+)$" pname);
46 # if there is no organisation we need to filter out null values.
47 non-null = builtins.filter (x: x != null) parts;
48 in
49 builtins.concatStringsSep "-" non-null;
50
51 inherit getLicenseFromSpdxId;
52
53 # Generates the yarn.nix from the yarn.lock file
54 mkYarnNix =
55 {
56 yarnLock,
57 flags ? [ ],
58 }:
59 pkgs.runCommand "yarn.nix" { }
60 "${yarn2nix}/bin/yarn2nix --lockfile ${yarnLock} --no-patch --builtin-fetchgit ${lib.escapeShellArgs flags} > $out";
61
62 # Loads the generated offline cache. This will be used by yarn as
63 # the package source.
64 importOfflineCache =
65 yarnNix:
66 let
67 pkg = callPackage yarnNix { };
68 in
69 pkg.offline_cache;
70
71 defaultYarnFlags = [
72 "--offline"
73 "--frozen-lockfile"
74 "--ignore-engines"
75 ];
76
77 mkYarnModules =
78 {
79 name ? "${pname}-${version}", # safe name and version, e.g. testcompany-one-modules-1.0.0
80 pname, # original name, e.g @testcompany/one
81 version,
82 packageJSON,
83 yarnLock,
84 yarnNix ? mkYarnNix { inherit yarnLock; },
85 offlineCache ? importOfflineCache yarnNix,
86 yarnFlags ? [ ],
87 ignoreScripts ? true,
88 nodejs ? inputs.nodejs,
89 yarn ? inputs.yarn.override { inherit nodejs; },
90 pkgConfig ? { },
91 preBuild ? "",
92 postBuild ? "",
93 workspaceDependencies ? [ ], # List of yarn packages
94 packageResolutions ? { },
95 }:
96 let
97 extraNativeBuildInputs = lib.concatMap (key: pkgConfig.${key}.nativeBuildInputs or [ ]) (
98 builtins.attrNames pkgConfig
99 );
100 extraBuildInputs = lib.concatMap (key: pkgConfig.${key}.buildInputs or [ ]) (
101 builtins.attrNames pkgConfig
102 );
103
104 postInstall = builtins.map (
105 key:
106 if (pkgConfig.${key} ? postInstall) then
107 ''
108 for f in $(find -L -path '*/node_modules/${key}' -type d); do
109 (cd "$f" && (${pkgConfig.${key}.postInstall}))
110 done
111 ''
112 else
113 ""
114 ) (builtins.attrNames pkgConfig);
115
116 # build-time JSON generation to avoid IFD
117 # see https://wiki.nixos.org/wiki/Import_From_Derivation
118 workspaceJSON =
119 pkgs.runCommand "${name}-workspace-package.json"
120 {
121 nativeBuildInputs = [ pkgs.jq ];
122 inherit packageJSON;
123 passAsFile = [ "baseJSON" ];
124 baseJSON = builtins.toJSON {
125 private = true;
126 workspaces = [ "deps/**" ];
127 resolutions = packageResolutions;
128 };
129 }
130 ''
131 jq --slurpfile packageJSON "$packageJSON" '.resolutions = $packageJSON[0].resolutions + .resolutions' <"$baseJSONPath" >$out
132 '';
133
134 workspaceDependencyLinks = lib.concatMapStringsSep "\n" (dep: ''
135 mkdir -p "deps/${dep.pname}"
136 ln -sf ${dep.packageJSON} "deps/${dep.pname}/package.json"
137 '') workspaceDependencies;
138
139 in
140 stdenv.mkDerivation {
141 inherit preBuild postBuild name;
142 dontUnpack = true;
143 dontInstall = true;
144 nativeBuildInputs = [
145 yarn
146 nodejs
147 git
148 ]
149 ++ extraNativeBuildInputs;
150 buildInputs = extraBuildInputs;
151
152 configurePhase =
153 lib.optionalString (offlineCache ? outputHash) ''
154 if ! cmp -s ${yarnLock} ${offlineCache}/yarn.lock; then
155 echo "yarn.lock changed, you need to update the fetchYarnDeps hash"
156 exit 1
157 fi
158 ''
159 + ''
160 # Yarn writes cache directories etc to $HOME.
161 export HOME=$PWD/yarn_home
162 '';
163
164 buildPhase = ''
165 runHook preBuild
166
167 mkdir -p "deps/${pname}"
168 cp ${packageJSON} "deps/${pname}/package.json"
169 cp ${workspaceJSON} ./package.json
170 cp ${yarnLock} ./yarn.lock
171 chmod +w ./yarn.lock
172
173 yarn config --offline set yarn-offline-mirror ${offlineCache}
174
175 # Do not look up in the registry, but in the offline cache.
176 ${fixup_yarn_lock}/bin/fixup_yarn_lock yarn.lock
177
178 ${workspaceDependencyLinks}
179
180 yarn install ${
181 lib.escapeShellArgs (defaultYarnFlags ++ lib.optional ignoreScripts "--ignore-scripts" ++ yarnFlags)
182 }
183
184 ${lib.concatStringsSep "\n" postInstall}
185
186 mkdir $out
187 mv node_modules $out/
188 mv deps $out/
189 patchShebangs $out
190
191 runHook postBuild
192 '';
193
194 dontCheckForBrokenSymlinks = true;
195 };
196
197 # This can be used as a shellHook in mkYarnPackage. It brings the built node_modules into
198 # the shell-hook environment.
199 linkNodeModulesHook = ''
200 if [[ -d node_modules || -L node_modules ]]; then
201 echo "./node_modules is present. Replacing."
202 rm -rf node_modules
203 fi
204
205 ln -s "$node_modules" node_modules
206 '';
207
208 mkYarnWorkspace =
209 {
210 src,
211 packageJSON ? src + "/package.json",
212 yarnLock ? src + "/yarn.lock",
213 nodejs ? inputs.nodejs,
214 yarn ? inputs.yarn.override { inherit nodejs; },
215 packageOverrides ? { },
216 ...
217 }@attrs:
218 let
219 package = lib.importJSON packageJSON;
220
221 packageGlobs =
222 if lib.isList package.workspaces then package.workspaces else package.workspaces.packages;
223
224 packageResolutions = package.resolutions or { };
225
226 globElemToRegex = lib.replaceStrings [ "*" ] [ ".*" ];
227
228 # PathGlob -> [PathGlobElem]
229 splitGlob = lib.splitString "/";
230
231 # Path -> [PathGlobElem] -> [Path]
232 # Note: Only directories are included, everything else is filtered out
233 expandGlobList =
234 base: globElems:
235 let
236 elemRegex = globElemToRegex (lib.head globElems);
237 rest = lib.tail globElems;
238 children = lib.attrNames (
239 lib.filterAttrs (name: type: type == "directory") (builtins.readDir base)
240 );
241 matchingChildren = lib.filter (child: builtins.match elemRegex child != null) children;
242 in
243 if globElems == [ ] then
244 [ base ]
245 else
246 lib.concatMap (child: expandGlobList (base + ("/" + child)) rest) matchingChildren;
247
248 # Path -> PathGlob -> [Path]
249 expandGlob = base: glob: expandGlobList base (splitGlob glob);
250
251 packagePaths = lib.concatMap (expandGlob src) packageGlobs;
252
253 packages = lib.listToAttrs (
254 map (
255 src:
256 let
257 packageJSON = src + "/package.json";
258
259 package = lib.importJSON packageJSON;
260
261 allDependencies = lib.foldl (a: b: a // b) { } (
262 map (field: lib.attrByPath [ field ] { } package) [
263 "dependencies"
264 "devDependencies"
265 ]
266 );
267
268 # { [name: String] : { pname : String, packageJSON : String, ... } } -> { [pname: String] : version } -> [{ pname : String, packageJSON : String, ... }]
269 getWorkspaceDependencies =
270 packages: allDependencies:
271 let
272 packageList = lib.attrValues packages;
273 in
274 composeAll [
275 (lib.filter (x: x != null))
276 (lib.mapAttrsToList (
277 pname: _version: lib.findFirst (package: package.pname == pname) null packageList
278 ))
279 ] allDependencies;
280
281 workspaceDependencies = getWorkspaceDependencies packages allDependencies;
282
283 name = reformatPackageName package.name;
284 in
285 {
286 inherit name;
287 value = mkYarnPackage (
288 builtins.removeAttrs attrs [ "packageOverrides" ]
289 // {
290 inherit
291 src
292 packageJSON
293 yarnLock
294 nodejs
295 yarn
296 packageResolutions
297 workspaceDependencies
298 ;
299 }
300 // lib.attrByPath [ name ] { } packageOverrides
301 );
302 }
303 ) packagePaths
304 );
305 in
306 packages;
307
308 mkYarnPackage =
309 {
310 name ? null,
311 src,
312 packageJSON ? src + "/package.json",
313 yarnLock ? src + "/yarn.lock",
314 yarnNix ? mkYarnNix { inherit yarnLock; },
315 offlineCache ? importOfflineCache yarnNix,
316 nodejs ? inputs.nodejs,
317 yarn ? inputs.yarn.override { inherit nodejs; },
318 yarnFlags ? [ ],
319 yarnPreBuild ? "",
320 yarnPostBuild ? "",
321 pkgConfig ? { },
322 extraBuildInputs ? [ ],
323 publishBinsFor ? null,
324 workspaceDependencies ? [ ], # List of yarnPackages
325 packageResolutions ? { },
326 ...
327 }@attrs:
328 let
329 package = lib.importJSON packageJSON;
330 pname = attrs.pname or package.name;
331 safeName = reformatPackageName package.name;
332 version = attrs.version or package.version;
333 baseName = unlessNull name "${safeName}-${version}";
334
335 workspaceDependenciesTransitive = lib.unique (
336 (lib.flatten (builtins.map (dep: dep.workspaceDependencies) workspaceDependencies))
337 ++ workspaceDependencies
338 );
339
340 deps = mkYarnModules {
341 pname = package.name;
342 name = "${safeName}-modules-${version}";
343 preBuild = yarnPreBuild;
344 postBuild = yarnPostBuild;
345 workspaceDependencies = workspaceDependenciesTransitive;
346 inherit
347 packageJSON
348 version
349 yarnLock
350 offlineCache
351 nodejs
352 yarn
353 yarnFlags
354 pkgConfig
355 packageResolutions
356 ;
357 };
358
359 publishBinsFor_ = unlessNull publishBinsFor [ package.name ];
360
361 linkDirFunction = ''
362 linkDirToDirLinks() {
363 target=$1
364 if [ ! -f "$target" ]; then
365 mkdir -p "$target"
366 elif [ -L "$target" ]; then
367 local new=$(mktemp -d)
368 trueSource=$(realpath "$target")
369 if [ "$(ls $trueSource | wc -l)" -gt 0 ]; then
370 ln -s $trueSource/* $new/
371 fi
372 rm -r "$target"
373 mv "$new" "$target"
374 fi
375 }
376 '';
377
378 workspaceDependencyCopy = lib.concatMapStringsSep "\n" (dep: ''
379 # ensure any existing scope directory is not a symlink
380 linkDirToDirLinks "$(dirname node_modules/${dep.package.name})"
381 mkdir -p "deps/${dep.package.name}"
382 tar -xf "${dep}/tarballs/${dep.name}.tgz" --directory "deps/${dep.package.name}" --strip-components=1
383 if [ ! -e "deps/${dep.package.name}/node_modules" ]; then
384 ln -s "${deps}/deps/${dep.package.name}/node_modules" "deps/${dep.package.name}/node_modules"
385 fi
386 '') workspaceDependenciesTransitive;
387
388 in
389 stdenv.mkDerivation (
390 builtins.removeAttrs attrs [
391 "yarnNix"
392 "pkgConfig"
393 "workspaceDependencies"
394 "packageResolutions"
395 ]
396 // {
397 inherit pname version src;
398
399 name = baseName;
400
401 buildInputs = [
402 yarn
403 nodejs
404 rsync
405 ]
406 ++ extraBuildInputs;
407
408 node_modules = deps + "/node_modules";
409
410 configurePhase =
411 attrs.configurePhase or ''
412 runHook preConfigure
413
414 for localDir in npm-packages-offline-cache node_modules; do
415 if [[ -d $localDir || -L $localDir ]]; then
416 echo "$localDir dir present. Removing."
417 rm -rf $localDir
418 fi
419 done
420
421 # move convent of . to ./deps/${package.name}
422 mv $PWD $NIX_BUILD_TOP/temp
423 mkdir -p "$PWD/deps/${package.name}"
424 rm -fd "$PWD/deps/${package.name}"
425 mv $NIX_BUILD_TOP/temp "$PWD/deps/${package.name}"
426 cd $PWD
427
428 ln -s ${deps}/deps/${package.name}/node_modules "deps/${package.name}/node_modules"
429
430 cp -r $node_modules node_modules
431 chmod -R +w node_modules
432
433 ${linkDirFunction}
434
435 linkDirToDirLinks "$(dirname node_modules/${package.name})"
436 ln -s "deps/${package.name}" "node_modules/${package.name}"
437
438 ${workspaceDependencyCopy}
439
440 # Help yarn commands run in other phases find the package
441 echo "--cwd deps/${package.name}" > .yarnrc
442 runHook postConfigure
443 '';
444
445 # Replace this phase on frontend packages where only the generated
446 # files are an interesting output.
447 installPhase =
448 attrs.installPhase or ''
449 runHook preInstall
450
451 mkdir -p $out/{bin,libexec/${package.name}}
452 mv node_modules $out/libexec/${package.name}/node_modules
453 mv deps $out/libexec/${package.name}/deps
454
455 node ${./internal/fixup_bin.js} $out/bin $out/libexec/${package.name}/node_modules ${lib.concatStringsSep " " publishBinsFor_}
456
457 runHook postInstall
458 '';
459
460 dontCheckForBrokenSymlinks = true;
461
462 doDist = attrs.doDist or true;
463
464 distPhase =
465 attrs.distPhase or ''
466 # pack command ignores cwd option
467 rm -f .yarnrc
468 cd $out/libexec/${package.name}/deps/${package.name}
469 mkdir -p $out/tarballs/
470 yarn pack --offline --ignore-scripts --filename $out/tarballs/${baseName}.tgz
471 '';
472
473 passthru = {
474 inherit package packageJSON deps;
475 workspaceDependencies = workspaceDependenciesTransitive;
476 }
477 // (attrs.passthru or { });
478
479 meta = {
480 inherit (nodejs.meta) platforms;
481 }
482 // lib.optionalAttrs (package ? description) { inherit (package) description; }
483 // lib.optionalAttrs (package ? homepage) { inherit (package) homepage; }
484 // lib.optionalAttrs (package ? license) { license = getLicenseFromSpdxId package.license; }
485 // (attrs.meta or { });
486 }
487 );
488
489 yarn2nix = mkYarnPackage {
490 src = ./yarn2nix;
491
492 # yarn2nix is the only package that requires the yarnNix option.
493 # All the other projects can auto-generate that file.
494 yarnNix = ./yarn.nix;
495
496 # Using the filter above and importing package.json from the filtered
497 # source results in an error in restricted mode. To circumvent this,
498 # we import package.json from the unfiltered source
499 packageJSON = ./yarn2nix/package.json;
500
501 yarnFlags = defaultYarnFlags ++ [
502 "--ignore-scripts"
503 "--production=true"
504 ];
505
506 nativeBuildInputs = [ pkgs.makeWrapper ];
507
508 buildPhase = ''
509 source ${./nix/expectShFunctions.sh}
510
511 expectFilePresent ./node_modules/.yarn-integrity
512
513 # check dependencies are installed
514 expectFilePresent ./node_modules/@yarnpkg/lockfile/package.json
515
516 # check devDependencies are not installed
517 expectFileOrDirAbsent ./node_modules/.bin/eslint
518 expectFileOrDirAbsent ./node_modules/eslint/package.json
519 '';
520
521 postInstall = ''
522 wrapProgram $out/bin/yarn2nix --prefix PATH : "${pkgs.nix-prefetch-git}/bin"
523 '';
524 };
525
526 fixup_yarn_lock =
527 runCommandLocal "fixup_yarn_lock"
528 {
529 buildInputs = [ nodejs ];
530 }
531 ''
532 mkdir -p $out/lib
533 mkdir -p $out/bin
534
535 cp ${./yarn2nix/lib/urlToName.js} $out/lib/urlToName.js
536 cp ${./internal/fixup_yarn_lock.js} $out/bin/fixup_yarn_lock
537
538 patchShebangs $out
539 '';
540}
541// lib.optionalAttrs allowAliases {
542 # Aliases
543 spdxLicense = getLicenseFromSpdxId; # added 2021-12-01
544}