tjh.dev
nixpkgs mirror (for testing)
github.com/NixOS/nixpkgs
nix
1{
2 pkgs ? import <nixpkgs> { },
3 nodejs ? pkgs.nodejs,
4 yarn ? pkgs.yarn,
5 allowAliases ? pkgs.config.allowAliases,
6}@inputs:
7
8let
9 inherit (pkgs)
10 stdenv
11 lib
12 callPackage
13 git
14 rsync
15 runCommandLocal
16 ;
17
18 compose =
19 f: g: x:
20 f (g x);
21 id = x: x;
22 composeAll = builtins.foldl' compose id;
23
24 # https://docs.npmjs.com/files/package.json#license
25 # TODO: support expression syntax (OR, AND, etc)
26 getLicenseFromSpdxId =
27 licstr: if licstr == "UNLICENSED" then lib.licenses.unfree else lib.getLicenseFromSpdxId licstr;
28in
29rec {
30 # Export yarn again to make it easier to find out which yarn was used.
31 inherit yarn;
32
33 # Re-export pkgs
34 inherit pkgs;
35
36 unlessNull = item: alt: if item == null then alt else item;
37
38 reformatPackageName =
39 pname:
40 let
41 # regex adapted from `validate-npm-package-name`
42 # will produce 3 parts e.g.
43 # "@someorg/somepackage" -> [ "@someorg/" "someorg" "somepackage" ]
44 # "somepackage" -> [ null null "somepackage" ]
45 parts = builtins.tail (builtins.match "^(@([^/]+)/)?([^/]+)$" pname);
46 # if there is no organisation we need to filter out null values.
47 non-null = builtins.filter (x: x != null) parts;
48 in
49 builtins.concatStringsSep "-" non-null;
50
51 inherit getLicenseFromSpdxId;
52
53 # Generates the yarn.nix from the yarn.lock file
54 mkYarnNix =
55 {
56 yarnLock,
57 flags ? [ ],
58 }:
59 pkgs.runCommand "yarn.nix" { }
60 "${yarn2nix}/bin/yarn2nix --lockfile ${yarnLock} --no-patch --builtin-fetchgit ${lib.escapeShellArgs flags} > $out";
61
62 # Loads the generated offline cache. This will be used by yarn as
63 # the package source.
64 importOfflineCache =
65 yarnNix:
66 let
67 pkg = callPackage yarnNix { };
68 in
69 pkg.offline_cache;
70
71 defaultYarnFlags = [
72 "--offline"
73 "--frozen-lockfile"
74 "--ignore-engines"
75 ];
76
77 mkYarnModules =
78 {
79 name ? "${pname}-${version}", # safe name and version, e.g. testcompany-one-modules-1.0.0
80 pname, # original name, e.g @testcompany/one
81 version,
82 packageJSON,
83 yarnLock,
84 yarnNix ? mkYarnNix { inherit yarnLock; },
85 offlineCache ? importOfflineCache yarnNix,
86 yarnFlags ? [ ],
87 ignoreScripts ? true,
88 nodejs ? inputs.nodejs,
89 yarn ? inputs.yarn.override { inherit nodejs; },
90 pkgConfig ? { },
91 preBuild ? "",
92 postBuild ? "",
93 workspaceDependencies ? [ ], # List of yarn packages
94 packageResolutions ? { },
95 }:
96 let
97 extraNativeBuildInputs = lib.concatMap (key: pkgConfig.${key}.nativeBuildInputs or [ ]) (
98 builtins.attrNames pkgConfig
99 );
100 extraBuildInputs = lib.concatMap (key: pkgConfig.${key}.buildInputs or [ ]) (
101 builtins.attrNames pkgConfig
102 );
103
104 postInstall = map (
105 key:
106 if (pkgConfig.${key} ? postInstall) then
107 ''
108 for f in $(find -L -path '*/node_modules/${key}' -type d); do
109 (cd "$f" && (${pkgConfig.${key}.postInstall}))
110 done
111 ''
112 else
113 ""
114 ) (builtins.attrNames pkgConfig);
115
116 # build-time JSON generation to avoid IFD
117 # see https://wiki.nixos.org/wiki/Import_From_Derivation
118 workspaceJSON =
119 pkgs.runCommand "${name}-workspace-package.json"
120 {
121 nativeBuildInputs = [ pkgs.jq ];
122 inherit packageJSON;
123 passAsFile = [ "baseJSON" ];
124 baseJSON = builtins.toJSON {
125 private = true;
126 workspaces = [ "deps/**" ];
127 resolutions = packageResolutions;
128 };
129 }
130 ''
131 jq --slurpfile packageJSON "$packageJSON" '.resolutions = $packageJSON[0].resolutions + .resolutions' <"$baseJSONPath" >$out
132 '';
133
134 workspaceDependencyLinks = lib.concatMapStringsSep "\n" (dep: ''
135 mkdir -p "deps/${dep.pname}"
136 ln -sf ${dep.packageJSON} "deps/${dep.pname}/package.json"
137 '') workspaceDependencies;
138
139 in
140 stdenv.mkDerivation {
141 inherit preBuild postBuild name;
142 dontUnpack = true;
143 dontInstall = true;
144 nativeBuildInputs = [
145 yarn
146 nodejs
147 git
148 ]
149 ++ extraNativeBuildInputs;
150 buildInputs = extraBuildInputs;
151
152 configurePhase =
153 lib.optionalString (offlineCache ? outputHash) ''
154 if ! cmp -s ${yarnLock} ${offlineCache}/yarn.lock; then
155 echo "yarn.lock changed, you need to update the fetchYarnDeps hash"
156 exit 1
157 fi
158 ''
159 + ''
160 # Yarn writes cache directories etc to $HOME.
161 export HOME=$PWD/yarn_home
162 '';
163
164 buildPhase = ''
165 runHook preBuild
166
167 mkdir -p "deps/${pname}"
168 cp ${packageJSON} "deps/${pname}/package.json"
169 cp ${workspaceJSON} ./package.json
170 cp ${yarnLock} ./yarn.lock
171 chmod +w ./yarn.lock
172
173 yarn config --offline set yarn-offline-mirror ${offlineCache}
174
175 # Do not look up in the registry, but in the offline cache.
176 ${fixup_yarn_lock}/bin/fixup_yarn_lock yarn.lock
177
178 ${workspaceDependencyLinks}
179
180 yarn install ${
181 lib.escapeShellArgs (defaultYarnFlags ++ lib.optional ignoreScripts "--ignore-scripts" ++ yarnFlags)
182 }
183
184 ${lib.concatStringsSep "\n" postInstall}
185
186 mkdir $out
187 mv node_modules $out/
188 mv deps $out/
189 patchShebangs $out
190
191 runHook postBuild
192 '';
193
194 dontCheckForBrokenSymlinks = true;
195 };
196
197 # This can be used as a shellHook in mkYarnPackage. It brings the built node_modules into
198 # the shell-hook environment.
199 linkNodeModulesHook = ''
200 if [[ -d node_modules || -L node_modules ]]; then
201 echo "./node_modules is present. Replacing."
202 rm -rf node_modules
203 fi
204
205 ln -s "$node_modules" node_modules
206 '';
207
208 mkYarnWorkspace =
209 {
210 src,
211 packageJSON ? src + "/package.json",
212 yarnLock ? src + "/yarn.lock",
213 nodejs ? inputs.nodejs,
214 yarn ? inputs.yarn.override { inherit nodejs; },
215 packageOverrides ? { },
216 ...
217 }@attrs:
218 let
219 package = lib.importJSON packageJSON;
220
221 packageGlobs =
222 if lib.isList package.workspaces then package.workspaces else package.workspaces.packages;
223
224 packageResolutions = package.resolutions or { };
225
226 globElemToRegex = lib.replaceStrings [ "*" ] [ ".*" ];
227
228 # PathGlob -> [PathGlobElem]
229 splitGlob = lib.splitString "/";
230
231 # Path -> [PathGlobElem] -> [Path]
232 # Note: Only directories are included, everything else is filtered out
233 expandGlobList =
234 base: globElems:
235 let
236 elemRegex = globElemToRegex (lib.head globElems);
237 rest = lib.tail globElems;
238 children = lib.attrNames (
239 lib.filterAttrs (name: type: type == "directory") (builtins.readDir base)
240 );
241 matchingChildren = lib.filter (child: builtins.match elemRegex child != null) children;
242 in
243 if globElems == [ ] then
244 [ base ]
245 else
246 lib.concatMap (child: expandGlobList (base + ("/" + child)) rest) matchingChildren;
247
248 # Path -> PathGlob -> [Path]
249 expandGlob = base: glob: expandGlobList base (splitGlob glob);
250
251 packagePaths = lib.concatMap (expandGlob src) packageGlobs;
252
253 packages = lib.listToAttrs (
254 map (
255 src:
256 let
257 packageJSON = src + "/package.json";
258
259 package = lib.importJSON packageJSON;
260
261 allDependencies = lib.foldl (a: b: a // b) { } (
262 map (field: lib.attrByPath [ field ] { } package) [
263 "dependencies"
264 "devDependencies"
265 ]
266 );
267
268 # { [name: String] : { pname : String, packageJSON : String, ... } } -> { [pname: String] : version } -> [{ pname : String, packageJSON : String, ... }]
269 getWorkspaceDependencies =
270 packages: allDependencies:
271 let
272 packageList = lib.attrValues packages;
273 in
274 composeAll [
275 (lib.filter (x: x != null))
276 (lib.mapAttrsToList (
277 pname: _version: lib.findFirst (package: package.pname == pname) null packageList
278 ))
279 ] allDependencies;
280
281 workspaceDependencies = getWorkspaceDependencies packages allDependencies;
282
283 name = reformatPackageName package.name;
284 in
285 {
286 inherit name;
287 value = mkYarnPackage (
288 removeAttrs attrs [ "packageOverrides" ]
289 // {
290 inherit
291 src
292 packageJSON
293 yarnLock
294 nodejs
295 yarn
296 packageResolutions
297 workspaceDependencies
298 ;
299 }
300 // lib.attrByPath [ name ] { } packageOverrides
301 );
302 }
303 ) packagePaths
304 );
305 in
306 packages;
307
308 mkYarnPackage =
309 {
310 name ? null,
311 src,
312 packageJSON ? src + "/package.json",
313 yarnLock ? src + "/yarn.lock",
314 yarnNix ? mkYarnNix { inherit yarnLock; },
315 offlineCache ? importOfflineCache yarnNix,
316 nodejs ? inputs.nodejs,
317 yarn ? inputs.yarn.override { inherit nodejs; },
318 yarnFlags ? [ ],
319 yarnPreBuild ? "",
320 yarnPostBuild ? "",
321 pkgConfig ? { },
322 extraBuildInputs ? [ ],
323 publishBinsFor ? null,
324 workspaceDependencies ? [ ], # List of yarnPackages
325 packageResolutions ? { },
326 ...
327 }@attrs:
328 let
329 package = lib.importJSON packageJSON;
330 pname = attrs.pname or package.name;
331 safeName = reformatPackageName package.name;
332 version = attrs.version or package.version;
333 baseName = unlessNull name "${safeName}-${version}";
334
335 workspaceDependenciesTransitive = lib.unique (
336 (lib.flatten (map (dep: dep.workspaceDependencies) workspaceDependencies)) ++ workspaceDependencies
337 );
338
339 deps = mkYarnModules {
340 pname = package.name;
341 name = "${safeName}-modules-${version}";
342 preBuild = yarnPreBuild;
343 postBuild = yarnPostBuild;
344 workspaceDependencies = workspaceDependenciesTransitive;
345 inherit
346 packageJSON
347 version
348 yarnLock
349 offlineCache
350 nodejs
351 yarn
352 yarnFlags
353 pkgConfig
354 packageResolutions
355 ;
356 };
357
358 publishBinsFor_ = unlessNull publishBinsFor [ package.name ];
359
360 linkDirFunction = ''
361 linkDirToDirLinks() {
362 target=$1
363 if [ ! -f "$target" ]; then
364 mkdir -p "$target"
365 elif [ -L "$target" ]; then
366 local new=$(mktemp -d)
367 trueSource=$(realpath "$target")
368 if [ "$(ls $trueSource | wc -l)" -gt 0 ]; then
369 ln -s $trueSource/* $new/
370 fi
371 rm -r "$target"
372 mv "$new" "$target"
373 fi
374 }
375 '';
376
377 workspaceDependencyCopy = lib.concatMapStringsSep "\n" (dep: ''
378 # ensure any existing scope directory is not a symlink
379 linkDirToDirLinks "$(dirname node_modules/${dep.package.name})"
380 mkdir -p "deps/${dep.package.name}"
381 tar -xf "${dep}/tarballs/${dep.name}.tgz" --directory "deps/${dep.package.name}" --strip-components=1
382 if [ ! -e "deps/${dep.package.name}/node_modules" ]; then
383 ln -s "${deps}/deps/${dep.package.name}/node_modules" "deps/${dep.package.name}/node_modules"
384 fi
385 '') workspaceDependenciesTransitive;
386
387 in
388 stdenv.mkDerivation (
389 removeAttrs attrs [
390 "yarnNix"
391 "pkgConfig"
392 "workspaceDependencies"
393 "packageResolutions"
394 ]
395 // {
396 inherit pname version src;
397
398 name = baseName;
399
400 buildInputs = [
401 yarn
402 nodejs
403 rsync
404 ]
405 ++ extraBuildInputs;
406
407 node_modules = deps + "/node_modules";
408
409 configurePhase =
410 attrs.configurePhase or ''
411 runHook preConfigure
412
413 for localDir in npm-packages-offline-cache node_modules; do
414 if [[ -d $localDir || -L $localDir ]]; then
415 echo "$localDir dir present. Removing."
416 rm -rf $localDir
417 fi
418 done
419
420 # move convent of . to ./deps/${package.name}
421 mv $PWD $NIX_BUILD_TOP/temp
422 mkdir -p "$PWD/deps/${package.name}"
423 rm -fd "$PWD/deps/${package.name}"
424 mv $NIX_BUILD_TOP/temp "$PWD/deps/${package.name}"
425 cd $PWD
426
427 ln -s ${deps}/deps/${package.name}/node_modules "deps/${package.name}/node_modules"
428
429 cp -r $node_modules node_modules
430 chmod -R +w node_modules
431
432 ${linkDirFunction}
433
434 linkDirToDirLinks "$(dirname node_modules/${package.name})"
435 ln -s "deps/${package.name}" "node_modules/${package.name}"
436
437 ${workspaceDependencyCopy}
438
439 # Help yarn commands run in other phases find the package
440 echo "--cwd deps/${package.name}" > .yarnrc
441 runHook postConfigure
442 '';
443
444 # Replace this phase on frontend packages where only the generated
445 # files are an interesting output.
446 installPhase =
447 attrs.installPhase or ''
448 runHook preInstall
449
450 mkdir -p $out/{bin,libexec/${package.name}}
451 mv node_modules $out/libexec/${package.name}/node_modules
452 mv deps $out/libexec/${package.name}/deps
453
454 node ${./internal/fixup_bin.js} $out/bin $out/libexec/${package.name}/node_modules ${lib.concatStringsSep " " publishBinsFor_}
455
456 runHook postInstall
457 '';
458
459 dontCheckForBrokenSymlinks = true;
460
461 doDist = attrs.doDist or true;
462
463 distPhase =
464 attrs.distPhase or ''
465 # pack command ignores cwd option
466 rm -f .yarnrc
467 cd $out/libexec/${package.name}/deps/${package.name}
468 mkdir -p $out/tarballs/
469 yarn pack --offline --ignore-scripts --filename $out/tarballs/${baseName}.tgz
470 '';
471
472 passthru = {
473 inherit package packageJSON deps;
474 workspaceDependencies = workspaceDependenciesTransitive;
475 }
476 // (attrs.passthru or { });
477
478 meta = {
479 inherit (nodejs.meta) platforms;
480 }
481 // lib.optionalAttrs (package ? description) { inherit (package) description; }
482 // lib.optionalAttrs (package ? homepage) { inherit (package) homepage; }
483 // lib.optionalAttrs (package ? license) { license = getLicenseFromSpdxId package.license; }
484 // (attrs.meta or { });
485 }
486 );
487
488 yarn2nix = mkYarnPackage {
489 src = ./yarn2nix;
490
491 # yarn2nix is the only package that requires the yarnNix option.
492 # All the other projects can auto-generate that file.
493 yarnNix = ./yarn.nix;
494
495 # Using the filter above and importing package.json from the filtered
496 # source results in an error in restricted mode. To circumvent this,
497 # we import package.json from the unfiltered source
498 packageJSON = ./yarn2nix/package.json;
499
500 yarnFlags = defaultYarnFlags ++ [
501 "--ignore-scripts"
502 "--production=true"
503 ];
504
505 nativeBuildInputs = [ pkgs.makeWrapper ];
506
507 buildPhase = ''
508 source ${./nix/expectShFunctions.sh}
509
510 expectFilePresent ./node_modules/.yarn-integrity
511
512 # check dependencies are installed
513 expectFilePresent ./node_modules/@yarnpkg/lockfile/package.json
514
515 # check devDependencies are not installed
516 expectFileOrDirAbsent ./node_modules/.bin/eslint
517 expectFileOrDirAbsent ./node_modules/eslint/package.json
518 '';
519
520 postInstall = ''
521 wrapProgram $out/bin/yarn2nix --prefix PATH : "${pkgs.nix-prefetch-git}/bin"
522 '';
523 };
524
525 fixup_yarn_lock =
526 runCommandLocal "fixup_yarn_lock"
527 {
528 buildInputs = [ nodejs ];
529 }
530 ''
531 mkdir -p $out/lib
532 mkdir -p $out/bin
533
534 cp ${./yarn2nix/lib/urlToName.js} $out/lib/urlToName.js
535 cp ${./internal/fixup_yarn_lock.js} $out/bin/fixup_yarn_lock
536
537 patchShebangs $out
538 '';
539}
540// lib.optionalAttrs allowAliases {
541 # Aliases
542 spdxLicense = getLicenseFromSpdxId; # added 2021-12-01
543}