pyrox.dev
maintainers/scripts/rebuild-amount.sh: report parallelism, add example, cleanup (#137695)
* maintainers/scripts/rebuild-amount.sh: report parallelism setting
This slightly helps discovering parallelism support.
* maintainers/scripts/rebuild-amount.sh: add basic example
It's not clear if script should be used against HEAD or HEAD^.
Let's have a copyable example.
* maintainers/scripts/rebuild-amount.sh: convert tabs to four spaces
One notable change is here-document conversion.
Before:
<TAB>cat <<-FOO
<TAB>...
<TAB>FOO
After:
<sp><sp><sp><sp>cat <<FOO
<sp><sp><sp><sp>...
FOO
Note seemingly misaligned 'FOO': '-FOO' understands leading tabs,
but not spaces.
authored by Sergei Trofimovich and committed by GitHub 39e8ec2d 1922e879
+77
-70
maintainers/scripts/rebuild-amount.sh
+77
-70
maintainers/scripts/rebuild-amount.sh
···
4
4
# --print: avoid dependency on environment
5
5
optPrint=
6
6
if [ "$1" == "--print" ]; then
7
-
optPrint=true
8
-
shift
7
+
optPrint=true
8
+
shift
9
9
fi
10
10
11
11
if [ "$#" != 1 ] && [ "$#" != 2 ]; then
12
-
cat <<-EOF
13
-
Usage: $0 [--print] commit-spec [commit-spec]
14
-
You need to be in a git-controlled nixpkgs tree.
15
-
The current state of the tree will be used if the second commit is missing.
16
-
EOF
17
-
exit 1
12
+
cat <<EOF
13
+
Usage: $0 [--print] from-commit-spec [to-commit-spec]
14
+
You need to be in a git-controlled nixpkgs tree.
15
+
The current state of the tree will be used if the second commit is missing.
16
+
17
+
Examples:
18
+
effect of latest commit:
19
+
$ $0 HEAD^
20
+
$ $0 --print HEAD^
21
+
effect of the whole patch series for 'staging' branch:
22
+
$ $0 origin/staging staging
23
+
EOF
24
+
exit 1
18
25
fi
19
26
20
27
# A slightly hacky way to get the config.
21
28
parallel="$(echo 'config.rebuild-amount.parallel or false' | nix-repl . 2>/dev/null \
22
-
| grep -v '^\(nix-repl.*\)\?$' | tail -n 1 || true)"
29
+
| grep -v '^\(nix-repl.*\)\?$' | tail -n 1 || true)"
23
30
24
-
echo "Estimating rebuild amount by counting changed Hydra jobs."
31
+
echo "Estimating rebuild amount by counting changed Hydra jobs (parallel=${parallel:-unset})."
25
32
26
33
toRemove=()
27
34
28
35
cleanup() {
29
-
rm -rf "${toRemove[@]}"
36
+
rm -rf "${toRemove[@]}"
30
37
}
31
38
trap cleanup EXIT SIGINT SIGQUIT ERR
32
39
33
40
MKTEMP='mktemp --tmpdir nix-rebuild-amount-XXXXXXXX'
34
41
35
42
nixexpr() {
36
-
cat <<-EONIX
37
-
let
38
-
lib = import $1/lib;
39
-
hydraJobs = import $1/pkgs/top-level/release.nix
40
-
# Compromise: accuracy vs. resources needed for evaluation.
41
-
{ supportedSystems = cfg.systems or [ "x86_64-linux" "x86_64-darwin" ]; };
42
-
cfg = (import $1 {}).config.rebuild-amount or {};
43
+
cat <<EONIX
44
+
let
45
+
lib = import $1/lib;
46
+
hydraJobs = import $1/pkgs/top-level/release.nix
47
+
# Compromise: accuracy vs. resources needed for evaluation.
48
+
{ supportedSystems = cfg.systems or [ "x86_64-linux" "x86_64-darwin" ]; };
49
+
cfg = (import $1 {}).config.rebuild-amount or {};
43
50
44
-
recurseIntoAttrs = attrs: attrs // { recurseForDerivations = true; };
51
+
recurseIntoAttrs = attrs: attrs // { recurseForDerivations = true; };
45
52
46
-
# hydraJobs leaves recurseForDerivations as empty attrmaps;
47
-
# that would break nix-env and we also need to recurse everywhere.
48
-
tweak = lib.mapAttrs
49
-
(name: val:
50
-
if name == "recurseForDerivations" then true
51
-
else if lib.isAttrs val && val.type or null != "derivation"
52
-
then recurseIntoAttrs (tweak val)
53
-
else val
54
-
);
53
+
# hydraJobs leaves recurseForDerivations as empty attrmaps;
54
+
# that would break nix-env and we also need to recurse everywhere.
55
+
tweak = lib.mapAttrs
56
+
(name: val:
57
+
if name == "recurseForDerivations" then true
58
+
else if lib.isAttrs val && val.type or null != "derivation"
59
+
then recurseIntoAttrs (tweak val)
60
+
else val
61
+
);
55
62
56
-
# Some of these contain explicit references to platform(s) we want to avoid;
57
-
# some even (transitively) depend on ~/.nixpkgs/config.nix (!)
58
-
blacklist = [
59
-
"tarball" "metrics" "manual"
60
-
"darwin-tested" "unstable" "stdenvBootstrapTools"
61
-
"moduleSystem" "lib-tests" # these just confuse the output
62
-
];
63
+
# Some of these contain explicit references to platform(s) we want to avoid;
64
+
# some even (transitively) depend on ~/.nixpkgs/config.nix (!)
65
+
blacklist = [
66
+
"tarball" "metrics" "manual"
67
+
"darwin-tested" "unstable" "stdenvBootstrapTools"
68
+
"moduleSystem" "lib-tests" # these just confuse the output
69
+
];
63
70
64
-
in
65
-
tweak (builtins.removeAttrs hydraJobs blacklist)
66
-
EONIX
71
+
in
72
+
tweak (builtins.removeAttrs hydraJobs blacklist)
73
+
EONIX
67
74
}
68
75
69
76
# Output packages in tree $2 that weren't in $1.
70
77
# Changing the output hash or name is taken as a change.
71
78
# Extra nix-env parameters can be in $3
72
79
newPkgs() {
73
-
# We use files instead of pipes, as running multiple nix-env processes
74
-
# could eat too much memory for a standard 4GiB machine.
75
-
local -a list
76
-
for i in 1 2; do
77
-
local l="$($MKTEMP)"
78
-
list[$i]="$l"
79
-
toRemove+=("$l")
80
+
# We use files instead of pipes, as running multiple nix-env processes
81
+
# could eat too much memory for a standard 4GiB machine.
82
+
local -a list
83
+
for i in 1 2; do
84
+
local l="$($MKTEMP)"
85
+
list[$i]="$l"
86
+
toRemove+=("$l")
80
87
81
-
local expr="$($MKTEMP)"
82
-
toRemove+=("$expr")
83
-
nixexpr "${!i}" > "$expr"
88
+
local expr="$($MKTEMP)"
89
+
toRemove+=("$expr")
90
+
nixexpr "${!i}" > "$expr"
84
91
85
-
nix-env -f "$expr" -qaP --no-name --out-path --show-trace $3 \
86
-
| sort > "${list[$i]}" &
92
+
nix-env -f "$expr" -qaP --no-name --out-path --show-trace $3 \
93
+
| sort > "${list[$i]}" &
87
94
88
-
if [ "$parallel" != "true" ]; then
89
-
wait
90
-
fi
91
-
done
95
+
if [ "$parallel" != "true" ]; then
96
+
wait
97
+
fi
98
+
done
92
99
93
-
wait
94
-
comm -13 "${list[@]}"
100
+
wait
101
+
comm -13 "${list[@]}"
95
102
}
96
103
97
104
# Prepare nixpkgs trees.
98
105
declare -a tree
99
106
for i in 1 2; do
100
-
if [ -n "${!i}" ]; then # use the given commit
101
-
dir="$($MKTEMP -d)"
102
-
tree[$i]="$dir"
103
-
toRemove+=("$dir")
107
+
if [ -n "${!i}" ]; then # use the given commit
108
+
dir="$($MKTEMP -d)"
109
+
tree[$i]="$dir"
110
+
toRemove+=("$dir")
104
111
105
-
git clone --shared --no-checkout --quiet . "${tree[$i]}"
106
-
(cd "${tree[$i]}" && git checkout --quiet "${!i}")
107
-
else #use the current tree
108
-
tree[$i]="$(pwd)"
109
-
fi
112
+
git clone --shared --no-checkout --quiet . "${tree[$i]}"
113
+
(cd "${tree[$i]}" && git checkout --quiet "${!i}")
114
+
else #use the current tree
115
+
tree[$i]="$(pwd)"
116
+
fi
110
117
done
111
118
112
119
newlist="$($MKTEMP)"
113
120
toRemove+=("$newlist")
114
121
# Notes:
115
-
# - the evaluation is done on x86_64-linux, like on Hydra.
116
-
# - using $newlist file so that newPkgs() isn't in a sub-shell (because of toRemove)
122
+
# - the evaluation is done on x86_64-linux, like on Hydra.
123
+
# - using $newlist file so that newPkgs() isn't in a sub-shell (because of toRemove)
117
124
newPkgs "${tree[1]}" "${tree[2]}" '--argstr system "x86_64-linux"' > "$newlist"
118
125
119
126
# Hacky: keep only the last word of each attribute path and sort.
120
127
sed -n 's/\([^. ]*\.\)*\([^. ]*\) .*$/\2/p' < "$newlist" \
121
-
| sort | uniq -c
128
+
| sort | uniq -c
122
129
123
130
if [ -n "$optPrint" ]; then
124
-
echo
125
-
cat "$newlist"
131
+
echo
132
+
cat "$newlist"
126
133
fi