tjh.dev
1{
2 callPackage,
3 lib,
4 jq,
5 runCommand,
6 writeText,
7 python3,
8}:
9{
10 combinedDir,
11 touchedFilesJson,
12 githubAuthorId,
13 byName ? false,
14}:
15let
16 /*
17 Derivation that computes which packages are affected (added, changed or removed) between two revisions of nixpkgs.
18 Note: "platforms" are "x86_64-linux", "aarch64-darwin", ...
19
20 ---
21 Inputs:
22 - beforeDir, afterDir: The evaluation result from before and after the change.
23 They can be obtained by running `nix-build -A ci.eval.full` on both revisions.
24
25 ---
26 Outputs:
27 - changed-paths.json: Various information about the changes:
28 {
29 attrdiff: {
30 added: ["package1"],
31 changed: ["package2", "package3"],
32 removed: ["package4"],
33 },
34 labels: {
35 "10.rebuild-darwin: 1-10": true,
36 "10.rebuild-linux: 1-10": true
37 },
38 rebuildsByKernel: {
39 darwin: ["package1", "package2"],
40 linux: ["package1", "package2", "package3"]
41 },
42 rebuildCountByKernel: {
43 darwin: 2,
44 linux: 3,
45 },
46 rebuildsByPlatform: {
47 aarch64-darwin: ["package1", "package2"],
48 aarch64-linux: ["package1", "package2"],
49 x86_64-linux: ["package1", "package2", "package3"],
50 x86_64-darwin: ["package1"],
51 },
52 }
53 - step-summary.md: A markdown render of the changes
54
55 ---
56 Implementation details:
57
58 Helper functions can be found in ./utils.nix.
59 Two main "types" are important:
60
61 - `packagePlatformPath`: A string of the form "<PACKAGE_PATH>.<PLATFORM>"
62 Example: "python312Packages.numpy.x86_64-linux"
63
64 - `packagePlatformAttr`: An attrs representation of a packagePlatformPath:
65 Example: { name = "python312Packages.numpy"; platform = "x86_64-linux"; }
66 */
67 inherit (import ./utils.nix { inherit lib; })
68 groupByKernel
69 convertToPackagePlatformAttrs
70 groupByPlatform
71 extractPackageNames
72 getLabels
73 ;
74
75 # Attrs
76 # - keys: "added", "changed", "removed" and "rebuilds"
77 # - values: lists of `packagePlatformPath`s
78 diffAttrs = builtins.fromJSON (builtins.readFile "${combinedDir}/combined-diff.json");
79
80 rebuildsPackagePlatformAttrs = convertToPackagePlatformAttrs diffAttrs.rebuilds;
81
82 changed-paths =
83 let
84 rebuildsByPlatform = groupByPlatform rebuildsPackagePlatformAttrs;
85 rebuildsByKernel = groupByKernel rebuildsPackagePlatformAttrs;
86 rebuildCountByKernel = lib.mapAttrs (
87 kernel: kernelRebuilds: lib.length kernelRebuilds
88 ) rebuildsByKernel;
89 in
90 writeText "changed-paths.json" (
91 builtins.toJSON {
92 attrdiff = lib.mapAttrs (_: extractPackageNames) { inherit (diffAttrs) added changed removed; };
93 inherit
94 rebuildsByPlatform
95 rebuildsByKernel
96 rebuildCountByKernel
97 ;
98 labels =
99 getLabels rebuildCountByKernel
100 # Sets "10.rebuild-*-stdenv" label to whether the "stdenv" attribute was changed.
101 // lib.mapAttrs' (
102 kernel: rebuilds: lib.nameValuePair "10.rebuild-${kernel}-stdenv" (lib.elem "stdenv" rebuilds)
103 ) rebuildsByKernel
104 # Set the "11.by: package-maintainer" label to whether all packages directly
105 # changed are maintained by the PR's author.
106 # (https://github.com/NixOS/ofborg/blob/df400f44502d4a4a80fa283d33f2e55a4e43ee90/ofborg/src/tagger.rs#L83-L88)
107 // {
108 "11.by: package-maintainer" =
109 maintainers ? ${githubAuthorId}
110 && lib.all (lib.flip lib.elem maintainers.${githubAuthorId}) (
111 lib.flatten (lib.attrValues maintainers)
112 );
113 };
114 }
115 );
116
117 maintainers = callPackage ./maintainers.nix { } {
118 changedattrs = lib.attrNames (lib.groupBy (a: a.name) rebuildsPackagePlatformAttrs);
119 changedpathsjson = touchedFilesJson;
120 inherit byName;
121 };
122in
123runCommand "compare"
124 {
125 # Don't depend on -dev outputs to reduce closure size for CI.
126 nativeBuildInputs = map lib.getBin [
127 jq
128 (python3.withPackages (
129 ps: with ps; [
130 numpy
131 pandas
132 scipy
133 ]
134 ))
135
136 ];
137 maintainers = builtins.toJSON maintainers;
138 passAsFile = [ "maintainers" ];
139 env = {
140 BEFORE_DIR = "${combinedDir}/before";
141 AFTER_DIR = "${combinedDir}/after";
142 };
143 }
144 ''
145 mkdir $out
146
147 cp ${changed-paths} $out/changed-paths.json
148
149
150 if jq -e '(.attrdiff.added | length == 0) and (.attrdiff.removed | length == 0)' "${changed-paths}" > /dev/null; then
151 # Chunks have changed between revisions
152 # We cannot generate a performance comparison
153 {
154 echo
155 echo "# Performance comparison"
156 echo
157 echo "This compares the performance of this branch against its pull request base branch (e.g., 'master')"
158 echo
159 echo "For further help please refer to: [ci/README.md](https://github.com/NixOS/nixpkgs/blob/master/ci/README.md)"
160 echo
161 } >> $out/step-summary.md
162
163 python3 ${./cmp-stats.py} >> $out/step-summary.md
164
165 else
166 # Package chunks are the same in both revisions
167 # We can use the to generate a performance comparison
168 {
169 echo
170 echo "# Performance Comparison"
171 echo
172 echo "Performance stats were skipped because the package sets differ between the two revisions."
173 echo
174 echo "For further help please refer to: [ci/README.md](https://github.com/NixOS/nixpkgs/blob/master/ci/README.md)"
175 } >> $out/step-summary.md
176 fi
177
178 {
179 echo
180 echo "# Packages"
181 echo
182 jq -r -f ${./generate-step-summary.jq} < ${changed-paths}
183 } >> $out/step-summary.md
184
185 cp "$maintainersPath" "$out/maintainers.json"
186 ''