tjh.dev
1# Checks derivation meta and attrs for problems (like brokenness,
2# licenses, etc).
3
4{
5 lib,
6 config,
7 hostPlatform,
8}:
9
10let
11 inherit (lib)
12 all
13 attrNames
14 concatMapStrings
15 concatMapStringsSep
16 concatStrings
17 findFirst
18 isDerivation
19 length
20 concatMap
21 mutuallyExclusive
22 optional
23 optionalAttrs
24 optionalString
25 optionals
26 isAttrs
27 isString
28 mapAttrs
29 filterAttrs
30 ;
31
32 inherit (lib.lists)
33 any
34 toList
35 isList
36 elem
37 ;
38
39 inherit (lib.meta)
40 availableOn
41 ;
42
43 inherit (lib.generators)
44 toPretty
45 ;
46
47 # If we're in hydra, we can dispense with the more verbose error
48 # messages and make problems easier to spot.
49 inHydra = config.inHydra or false;
50 # Allow the user to opt-into additional warnings, e.g.
51 # import <nixpkgs> { config = { showDerivationWarnings = [ "maintainerless" ]; }; }
52 showWarnings = config.showDerivationWarnings;
53
54 getNameWithVersion =
55 attrs: attrs.name or ("${attrs.pname or "«name-missing»"}-${attrs.version or "«version-missing»"}");
56
57 allowUnfree = config.allowUnfree || builtins.getEnv "NIXPKGS_ALLOW_UNFREE" == "1";
58
59 allowNonSource =
60 let
61 envVar = builtins.getEnv "NIXPKGS_ALLOW_NONSOURCE";
62 in
63 if envVar != "" then envVar != "0" else config.allowNonSource or true;
64
65 allowlist = config.allowlistedLicenses or config.whitelistedLicenses or [ ];
66 blocklist = config.blocklistedLicenses or config.blacklistedLicenses or [ ];
67
68 areLicenseListsValid =
69 if mutuallyExclusive allowlist blocklist then
70 true
71 else
72 throw "allowlistedLicenses and blocklistedLicenses are not mutually exclusive.";
73
74 hasLicense = attrs: attrs ? meta.license;
75
76 hasListedLicense =
77 assert areLicenseListsValid;
78 list: attrs:
79 length list > 0
80 && hasLicense attrs
81 && (
82 if isList attrs.meta.license then
83 any (l: elem l list) attrs.meta.license
84 else
85 elem attrs.meta.license list
86 );
87
88 hasAllowlistedLicense = attrs: hasListedLicense allowlist attrs;
89
90 hasBlocklistedLicense = attrs: hasListedLicense blocklist attrs;
91
92 allowBroken = config.allowBroken || builtins.getEnv "NIXPKGS_ALLOW_BROKEN" == "1";
93
94 allowUnsupportedSystem =
95 config.allowUnsupportedSystem || builtins.getEnv "NIXPKGS_ALLOW_UNSUPPORTED_SYSTEM" == "1";
96
97 isUnfree =
98 licenses:
99 if isAttrs licenses then
100 !licenses.free or true
101 # TODO: Returning false in the case of a string is a bug that should be fixed.
102 # In a previous implementation of this function the function body
103 # was `licenses: lib.lists.any (l: !l.free or true) licenses;`
104 # which always evaluates to `!true` for strings.
105 else if isString licenses then
106 false
107 else
108 any (l: !l.free or true) licenses;
109
110 hasUnfreeLicense = attrs: hasLicense attrs && isUnfree attrs.meta.license;
111
112 hasNoMaintainers =
113 # To get usable output, we want to avoid flagging "internal" derivations.
114 # Because we do not have a way to reliably decide between internal or
115 # external derivation, some heuristics are required to decide.
116 #
117 # If `outputHash` is defined, the derivation is a FOD, such as the output of a fetcher.
118 # If `description` is not defined, the derivation is probably not a package.
119 # Simply checking whether `meta` is defined is insufficient,
120 # as some fetchers and trivial builders do define meta.
121 attrs:
122 (!attrs ? outputHash)
123 && (attrs ? meta.description)
124 && (attrs.meta.maintainers or [ ] == [ ])
125 && (attrs.meta.teams or [ ] == [ ]);
126
127 isMarkedBroken = attrs: attrs.meta.broken or false;
128
129 hasUnsupportedPlatform = pkg: !(availableOn hostPlatform pkg);
130
131 isMarkedInsecure = attrs: (attrs.meta.knownVulnerabilities or [ ]) != [ ];
132
133 # Allow granular checks to allow only some unfree packages
134 # Example:
135 # {pkgs, ...}:
136 # {
137 # allowUnfree = false;
138 # allowUnfreePredicate = (x: pkgs.lib.hasPrefix "vscode" x.name);
139 # }
140 allowUnfreePredicate = config.allowUnfreePredicate or (x: false);
141
142 # Check whether unfree packages are allowed and if not, whether the
143 # package has an unfree license and is not explicitly allowed by the
144 # `allowUnfreePredicate` function.
145 hasDeniedUnfreeLicense =
146 attrs: hasUnfreeLicense attrs && !allowUnfree && !allowUnfreePredicate attrs;
147
148 allowInsecureDefaultPredicate =
149 x: builtins.elem (getNameWithVersion x) (config.permittedInsecurePackages or [ ]);
150 allowInsecurePredicate = x: (config.allowInsecurePredicate or allowInsecureDefaultPredicate) x;
151
152 hasAllowedInsecure =
153 attrs:
154 !(isMarkedInsecure attrs)
155 || allowInsecurePredicate attrs
156 || builtins.getEnv "NIXPKGS_ALLOW_INSECURE" == "1";
157
158 isNonSource = sourceTypes: any (t: !t.isSource) sourceTypes;
159
160 hasNonSourceProvenance =
161 attrs: (attrs ? meta.sourceProvenance) && isNonSource attrs.meta.sourceProvenance;
162
163 # Allow granular checks to allow only some non-source-built packages
164 # Example:
165 # { pkgs, ... }:
166 # {
167 # allowNonSource = false;
168 # allowNonSourcePredicate = with pkgs.lib.lists; pkg: !(any (p: !p.isSource && p != lib.sourceTypes.binaryFirmware) pkg.meta.sourceProvenance);
169 # }
170 allowNonSourcePredicate = config.allowNonSourcePredicate or (x: false);
171
172 # Check whether non-source packages are allowed and if not, whether the
173 # package has non-source provenance and is not explicitly allowed by the
174 # `allowNonSourcePredicate` function.
175 hasDeniedNonSourceProvenance =
176 attrs: hasNonSourceProvenance attrs && !allowNonSource && !allowNonSourcePredicate attrs;
177
178 showLicenseOrSourceType = value: toString (map (v: v.shortName or "unknown") (toList value));
179 showLicense = showLicenseOrSourceType;
180 showSourceType = showLicenseOrSourceType;
181
182 pos_str = meta: meta.position or "«unknown-file»";
183
184 remediation = {
185 unfree = remediate_allowlist "Unfree" (remediate_predicate "allowUnfreePredicate");
186 non-source = remediate_allowlist "NonSource" (remediate_predicate "allowNonSourcePredicate");
187 broken = remediate_allowlist "Broken" (x: "");
188 unsupported = remediate_allowlist "UnsupportedSystem" (x: "");
189 blocklisted = x: "";
190 insecure = remediate_insecure;
191 broken-outputs = remediateOutputsToInstall;
192 unknown-meta = x: "";
193 maintainerless = x: "";
194 };
195 remediation_env_var =
196 allow_attr:
197 {
198 Unfree = "NIXPKGS_ALLOW_UNFREE";
199 Broken = "NIXPKGS_ALLOW_BROKEN";
200 UnsupportedSystem = "NIXPKGS_ALLOW_UNSUPPORTED_SYSTEM";
201 NonSource = "NIXPKGS_ALLOW_NONSOURCE";
202 }
203 .${allow_attr};
204 remediation_phrase =
205 allow_attr:
206 {
207 Unfree = "unfree packages";
208 Broken = "broken packages";
209 UnsupportedSystem = "packages that are unsupported for this system";
210 NonSource = "packages not built from source";
211 }
212 .${allow_attr};
213 remediate_predicate = predicateConfigAttr: attrs: ''
214
215 Alternatively you can configure a predicate to allow specific packages:
216 { nixpkgs.config.${predicateConfigAttr} = pkg: builtins.elem (lib.getName pkg) [
217 "${lib.getName attrs}"
218 ];
219 }
220 '';
221
222 # flakeNote will be printed in the remediation messages below.
223 flakeNote = "
224 Note: When using `nix shell`, `nix build`, `nix develop`, etc with a flake,
225 then pass `--impure` in order to allow use of environment variables.
226 ";
227
228 remediate_allowlist = allow_attr: rebuild_amendment: attrs: ''
229 a) To temporarily allow ${remediation_phrase allow_attr}, you can use an environment variable
230 for a single invocation of the nix tools.
231
232 $ export ${remediation_env_var allow_attr}=1
233 ${flakeNote}
234 b) For `nixos-rebuild` you can set
235 { nixpkgs.config.allow${allow_attr} = true; }
236 in configuration.nix to override this.
237 ${rebuild_amendment attrs}
238 c) For `nix-env`, `nix-build`, `nix-shell` or any other Nix command you can add
239 { allow${allow_attr} = true; }
240 to ~/.config/nixpkgs/config.nix.
241 '';
242
243 remediate_insecure =
244 attrs:
245 ''
246
247 Known issues:
248 ''
249 + (concatStrings (map (issue: " - ${issue}\n") attrs.meta.knownVulnerabilities))
250 + ''
251
252 You can install it anyway by allowing this package, using the
253 following methods:
254
255 a) To temporarily allow all insecure packages, you can use an environment
256 variable for a single invocation of the nix tools:
257
258 $ export NIXPKGS_ALLOW_INSECURE=1
259 ${flakeNote}
260 b) for `nixos-rebuild` you can add ‘${getNameWithVersion attrs}’ to
261 `nixpkgs.config.permittedInsecurePackages` in the configuration.nix,
262 like so:
263
264 {
265 nixpkgs.config.permittedInsecurePackages = [
266 "${getNameWithVersion attrs}"
267 ];
268 }
269
270 c) For `nix-env`, `nix-build`, `nix-shell` or any other Nix command you can add
271 ‘${getNameWithVersion attrs}’ to `permittedInsecurePackages` in
272 ~/.config/nixpkgs/config.nix, like so:
273
274 {
275 permittedInsecurePackages = [
276 "${getNameWithVersion attrs}"
277 ];
278 }
279
280 '';
281
282 remediateOutputsToInstall =
283 attrs:
284 let
285 expectedOutputs = attrs.meta.outputsToInstall or [ ];
286 actualOutputs = attrs.outputs or [ "out" ];
287 missingOutputs = builtins.filter (output: !builtins.elem output actualOutputs) expectedOutputs;
288 in
289 ''
290 The package ${getNameWithVersion attrs} has set meta.outputsToInstall to: ${builtins.concatStringsSep ", " expectedOutputs}
291
292 however ${getNameWithVersion attrs} only has the outputs: ${builtins.concatStringsSep ", " actualOutputs}
293
294 and is missing the following ouputs:
295
296 ${concatStrings (builtins.map (output: " - ${output}\n") missingOutputs)}
297 '';
298
299 handleEvalIssue =
300 { meta, attrs }:
301 {
302 reason,
303 errormsg ? "",
304 }:
305 let
306 msg =
307 if inHydra then
308 "Failed to evaluate ${getNameWithVersion attrs}: «${reason}»: ${errormsg}"
309 else
310 ''
311 Package ‘${getNameWithVersion attrs}’ in ${pos_str meta} ${errormsg}, refusing to evaluate.
312
313 ''
314 + (builtins.getAttr reason remediation) attrs;
315
316 handler = if config ? handleEvalIssue then config.handleEvalIssue reason else throw;
317 in
318 handler msg;
319
320 handleEvalWarning =
321 { meta, attrs }:
322 {
323 reason,
324 errormsg ? "",
325 }:
326 let
327 remediationMsg = (builtins.getAttr reason remediation) attrs;
328 msg =
329 if inHydra then
330 "Warning while evaluating ${getNameWithVersion attrs}: «${reason}»: ${errormsg}"
331 else
332 "Package ${getNameWithVersion attrs} in ${pos_str meta} ${errormsg}, continuing anyway."
333 + (optionalString (remediationMsg != "") "\n${remediationMsg}");
334 isEnabled = findFirst (x: x == reason) null showWarnings;
335 in
336 if isEnabled != null then builtins.trace msg true else true;
337
338 metaTypes =
339 let
340 types = import ./meta-types.nix { inherit lib; };
341 inherit (types)
342 str
343 union
344 int
345 attrs
346 attrsOf
347 any
348 listOf
349 bool
350 ;
351 platforms = listOf (union [
352 str
353 (attrsOf any)
354 ]); # see lib.meta.platformMatch
355 in
356 {
357 # These keys are documented
358 description = str;
359 mainProgram = str;
360 longDescription = str;
361 branch = str;
362 homepage = union [
363 (listOf str)
364 str
365 ];
366 downloadPage = str;
367 changelog = union [
368 (listOf str)
369 str
370 ];
371 license =
372 let
373 # TODO disallow `str` licenses, use a module
374 licenseType = union [
375 (attrsOf any)
376 str
377 ];
378 in
379 union [
380 (listOf licenseType)
381 licenseType
382 ];
383 sourceProvenance = listOf attrs;
384 maintainers = listOf (attrsOf any); # TODO use the maintainer type from lib/tests/maintainer-module.nix
385 teams = listOf (attrsOf any); # TODO similar to maintainers, use a teams type
386 priority = int;
387 pkgConfigModules = listOf str;
388 inherit platforms;
389 hydraPlatforms = listOf str;
390 broken = bool;
391 unfree = bool;
392 unsupported = bool;
393 insecure = bool;
394 tests = {
395 name = "test";
396 verify =
397 x:
398 x == { }
399 ||
400 # Accept {} for tests that are unsupported
401 (isDerivation x && x ? meta.timeout);
402 };
403 timeout = int;
404 knownVulnerabilities = listOf str;
405 badPlatforms = platforms;
406
407 # Needed for Hydra to expose channel tarballs:
408 # https://github.com/NixOS/hydra/blob/53335323ae79ca1a42643f58e520b376898ce641/doc/manual/src/jobs.md#meta-fields
409 isHydraChannel = bool;
410
411 # Weirder stuff that doesn't appear in the documentation?
412 maxSilent = int;
413 name = str;
414 version = str;
415 tag = str;
416 executables = listOf str;
417 outputsToInstall = listOf str;
418 position = str;
419 available = any;
420 isBuildPythonPackage = platforms;
421 schedulingPriority = int;
422 isFcitxEngine = bool;
423 isIbusEngine = bool;
424 isGutenprint = bool;
425
426 # Used for the original location of the maintainer and team attributes to assist with pings.
427 maintainersPosition = any;
428 teamsPosition = any;
429 };
430
431 checkMetaAttr =
432 let
433 # Map attrs directly to the verify function for performance
434 metaTypes' = mapAttrs (_: t: t.verify) metaTypes;
435 in
436 k: v:
437 if metaTypes ? ${k} then
438 if metaTypes'.${k} v then
439 [ ]
440 else
441 [
442 "key 'meta.${k}' has invalid value; expected ${metaTypes.${k}.name}, got\n ${
443 toPretty { indent = " "; } v
444 }"
445 ]
446 else
447 [
448 "key 'meta.${k}' is unrecognized; expected one of: \n [${
449 concatMapStringsSep ", " (x: "'${x}'") (attrNames metaTypes)
450 }]"
451 ];
452 checkMeta =
453 meta:
454 optionals config.checkMeta (concatMap (attr: checkMetaAttr attr meta.${attr}) (attrNames meta));
455
456 checkOutputsToInstall =
457 attrs:
458 let
459 expectedOutputs = attrs.meta.outputsToInstall or [ ];
460 actualOutputs = attrs.outputs or [ "out" ];
461 missingOutputs = builtins.filter (output: !builtins.elem output actualOutputs) expectedOutputs;
462 in
463 if config.checkMeta then builtins.length missingOutputs > 0 else false;
464
465 # Check if a derivation is valid, that is whether it passes checks for
466 # e.g brokenness or license.
467 #
468 # Return { valid: "yes", "warn" or "no" } and additionally
469 # { reason: String; errormsg: String } if it is not valid, where
470 # reason is one of "unfree", "blocklisted", "broken", "insecure", ...
471 # !!! reason strings are hardcoded into OfBorg, make sure to keep them in sync
472 # Along with a boolean flag for each reason
473 checkValidity =
474 let
475 validYes = {
476 valid = "yes";
477 handled = true;
478 };
479 in
480 attrs:
481 # Check meta attribute types first, to make sure it is always called even when there are other issues
482 # Note that this is not a full type check and functions below still need to by careful about their inputs!
483 let
484 res = checkMeta (attrs.meta or { });
485 in
486 if res != [ ] then
487 {
488 valid = "no";
489 reason = "unknown-meta";
490 errormsg = "has an invalid meta attrset:${concatMapStrings (x: "\n - " + x) res}\n";
491 }
492
493 # --- Put checks that cannot be ignored here ---
494 else if checkOutputsToInstall attrs then
495 {
496 valid = "no";
497 reason = "broken-outputs";
498 errormsg = "has invalid meta.outputsToInstall";
499 }
500
501 # --- Put checks that can be ignored here ---
502 else if hasDeniedUnfreeLicense attrs && !(hasAllowlistedLicense attrs) then
503 {
504 valid = "no";
505 reason = "unfree";
506 errormsg = "has an unfree license (‘${showLicense attrs.meta.license}’)";
507 }
508 else if hasBlocklistedLicense attrs then
509 {
510 valid = "no";
511 reason = "blocklisted";
512 errormsg = "has a blocklisted license (‘${showLicense attrs.meta.license}’)";
513 }
514 else if hasDeniedNonSourceProvenance attrs then
515 {
516 valid = "no";
517 reason = "non-source";
518 errormsg = "contains elements not built from source (‘${showSourceType attrs.meta.sourceProvenance}’)";
519 }
520 else if !allowBroken && attrs.meta.broken or false then
521 {
522 valid = "no";
523 reason = "broken";
524 errormsg = "is marked as broken";
525 }
526 else if !allowUnsupportedSystem && hasUnsupportedPlatform attrs then
527 let
528 toPretty' = toPretty {
529 allowPrettyValues = true;
530 indent = " ";
531 };
532 in
533 {
534 valid = "no";
535 reason = "unsupported";
536 errormsg = ''
537 is not available on the requested hostPlatform:
538 hostPlatform.config = "${hostPlatform.config}"
539 package.meta.platforms = ${toPretty' (attrs.meta.platforms or [ ])}
540 package.meta.badPlatforms = ${toPretty' (attrs.meta.badPlatforms or [ ])}
541 '';
542 }
543 else if !(hasAllowedInsecure attrs) then
544 {
545 valid = "no";
546 reason = "insecure";
547 errormsg = "is marked as insecure";
548 }
549
550 # --- warnings ---
551 # Please also update the type in /pkgs/top-level/config.nix alongside this.
552 else if hasNoMaintainers attrs then
553 {
554 valid = "warn";
555 reason = "maintainerless";
556 errormsg = "has no maintainers or teams";
557 }
558 # -----
559 else
560 validYes;
561
562 # The meta attribute is passed in the resulting attribute set,
563 # but it's not part of the actual derivation, i.e., it's not
564 # passed to the builder and is not a dependency. But since we
565 # include it in the result, it *is* available to nix-env for queries.
566 # Example:
567 # meta = checkMeta.commonMeta { inherit validity attrs pos references; };
568 # validity = checkMeta.assertValidity { inherit meta attrs; };
569 commonMeta =
570 {
571 validity,
572 attrs,
573 pos ? null,
574 references ? [ ],
575 }:
576 let
577 outputs = attrs.outputs or [ "out" ];
578 hasOutput = out: builtins.elem out outputs;
579 in
580 {
581 # `name` derivation attribute includes cross-compilation cruft,
582 # is under assert, and is sanitized.
583 # Let's have a clean always accessible version here.
584 name = attrs.name or "${attrs.pname}-${attrs.version}";
585
586 # If the packager hasn't specified `outputsToInstall`, choose a default,
587 # which is the name of `p.bin or p.out or p` along with `p.man` when
588 # present.
589 #
590 # If the packager has specified it, it will be overridden below in
591 # `// meta`.
592 #
593 # Note: This default probably shouldn't be globally configurable.
594 # Services and users should specify outputs explicitly,
595 # unless they are comfortable with this default.
596 outputsToInstall = [
597 (
598 if hasOutput "bin" then
599 "bin"
600 else if hasOutput "out" then
601 "out"
602 else
603 findFirst hasOutput null outputs
604 )
605 ]
606 ++ optional (hasOutput "man") "man";
607 }
608 // (filterAttrs (_: v: v != null) {
609 # CI scripts look at these to determine pings. Note that we should filter nulls out of this,
610 # or nix-env complains: https://github.com/NixOS/nix/blob/2.18.8/src/nix-env/nix-env.cc#L963
611 maintainersPosition = builtins.unsafeGetAttrPos "maintainers" (attrs.meta or { });
612 teamsPosition = builtins.unsafeGetAttrPos "teams" (attrs.meta or { });
613 })
614 // attrs.meta or { }
615 # Fill `meta.position` to identify the source location of the package.
616 // optionalAttrs (pos != null) {
617 position = pos.file + ":" + toString pos.line;
618 }
619 // {
620 # Maintainers should be inclusive of teams.
621 # Note that there may be external consumers of this API (repology, for instance) -
622 # if you add a new maintainer or team attribute please ensure that this expectation is still met.
623 maintainers =
624 attrs.meta.maintainers or [ ] ++ concatMap (team: team.members or [ ]) attrs.meta.teams or [ ];
625 }
626 // {
627 # Expose the result of the checks for everyone to see.
628 unfree = hasUnfreeLicense attrs;
629 broken = isMarkedBroken attrs;
630 unsupported = hasUnsupportedPlatform attrs;
631 insecure = isMarkedInsecure attrs;
632
633 available =
634 validity.valid != "no"
635 && (
636 if config.checkMetaRecursively or false then all (d: d.meta.available or true) references else true
637 );
638 };
639
640 assertValidity =
641 { meta, attrs }:
642 let
643 validity = checkValidity attrs;
644 inherit (validity) valid;
645 in
646 if validity ? handled then
647 validity
648 else
649 validity
650 // {
651 # Throw an error if trying to evaluate a non-valid derivation
652 # or, alternatively, just output a warning message.
653 handled = (
654 if valid == "yes" then
655 true
656 else if valid == "no" then
657 (handleEvalIssue { inherit meta attrs; } { inherit (validity) reason errormsg; })
658 else if valid == "warn" then
659 (handleEvalWarning { inherit meta attrs; } { inherit (validity) reason errormsg; })
660 else
661 throw "Unknown validitiy: '${valid}'"
662 );
663 };
664
665in
666{
667 inherit assertValidity commonMeta;
668}