pyrox.dev
Merge pull request #148121 from milahu/fetch-kde-qt-sh-get-sha256-from-server
+148
-12
maintainers/scripts/fetch-kde-qt.sh
+148
-12
maintainers/scripts/fetch-kde-qt.sh
···
2
2
#! nix-shell -i bash -p coreutils findutils gnused nix wget
3
3
4
4
set -efuo pipefail
5
+
export LC_COLLATE=C # fix sort order
5
6
6
-
SRCS=
7
-
if [ -d "$1" ]; then
8
-
SRCS="$(pwd)/$1/srcs.nix"
9
-
. "$1/fetch.sh"
7
+
# parse files and folders from https://download.kde.org/ and https://download.qt.io/
8
+
# you can override this function in fetch.sh
9
+
function PARSE_INDEX() {
10
+
cat "$1" | grep -o -E -e '\s+href="[^"]+\.tar\.xz"' -e '\s+href="[-_a-zA-Z0-9]+/"' | cut -d'"' -f2 | sort | uniq
11
+
}
12
+
13
+
if [ $# != 1 ]; then
14
+
echo "example use:" >&2
15
+
echo "cd nixpkgs/" >&2
16
+
echo "./maintainers/scripts/fetch-kde-qt.sh pkgs/development/libraries/qt-5/5.12" >&2
17
+
exit 1
18
+
fi
19
+
20
+
if ! echo "$1" | grep -q '^pkgs/'; then
21
+
echo "error: path argument must start with pkgs/" >&2
22
+
exit 1
23
+
fi
24
+
25
+
# need absolute path for the pushd-popd block
26
+
if [ -f "$1" ]; then
27
+
echo "ok: using fetchfile $1"
28
+
fetchfilerel="$1"
29
+
fetchfile="$(readlink -f "$fetchfilerel")" # resolve absolute path
30
+
basedir="$(dirname "$fetchfile")"
31
+
basedirrel="$(dirname "$fetchfilerel")"
32
+
elif [ -d "$1" ]; then
33
+
echo "ok: using basedir $1"
34
+
basedirrel="$1"
35
+
basedir="$(readlink -f "$basedirrel")" # resolve absolute path
36
+
if ! [ -d "$basedir" ]; then
37
+
basedir="$(dirname "$basedir")"
38
+
fi
39
+
fetchfile="$basedir/fetch.sh"
10
40
else
11
-
SRCS="$(pwd)/$(dirname $1)/srcs.nix"
12
-
. "$1"
41
+
echo 'error: $1 must be file or dir' >&2
42
+
exit 1
13
43
fi
14
44
15
-
tmp=$(mktemp -d)
45
+
pkgname=$(basename "$basedir")
46
+
SRCS="$basedir/srcs.nix"
47
+
srcsrel="$basedirrel/srcs.nix"
48
+
49
+
source "$fetchfile"
50
+
51
+
if [ -n "$WGET_ARGS" ]; then # old format
52
+
BASE_URL="${WGET_ARGS[0]}" # convert to new format
53
+
# validate
54
+
if ! echo "$BASE_URL" | grep -q -E '^(http|https|ftp)://'; then
55
+
printf 'error: from WGET_ARGS, converted invalid BASE_URL: %q\n' "$BASE_URL" >&2
56
+
exit 1
57
+
fi
58
+
printf 'ok: from WGET_ARGS, converted BASE_URL: %q\n' "$BASE_URL"
59
+
elif [ -n "$BASE_URL" ]; then # new format
60
+
:
61
+
else
62
+
echo "error: fetch.sh must set either WGET_ARGS or BASE_URL" >&2
63
+
exit 1
64
+
fi
65
+
66
+
tmptpl=tmp.fetch-kde-qt.$pkgname.XXXXXXXXXX
67
+
68
+
tmp=$(mktemp -d $tmptpl)
16
69
pushd $tmp >/dev/null
17
-
wget -nH -r -c --no-parent "${WGET_ARGS[@]}" >/dev/null
70
+
echo "tempdir is $tmp"
18
71
19
-
csv=$(mktemp)
20
-
find . -type f | while read src; do
72
+
wgetargs='--quiet --show-progress'
73
+
#wgetargs='' # debug
74
+
75
+
dirlist="$BASE_URL"
76
+
filelist=""
77
+
base_url_len=${#BASE_URL}
78
+
79
+
clean_urls() {
80
+
# // -> /
81
+
sed -E 's,//+,/,g' | sed -E 's,^(http|https|ftp):/,&/,'
82
+
}
83
+
84
+
while [ -n "$dirlist" ]
85
+
do
86
+
for dirurl in $dirlist
87
+
do
88
+
echo "fetching index.html from $dirurl"
89
+
relpath=$(echo "./${dirurl:$base_url_len}" | clean_urls)
90
+
mkdir -p "$relpath"
91
+
indexfile=$(echo "$relpath/index.html" | clean_urls)
92
+
wget $wgetargs -O "$indexfile" "$dirurl"
93
+
echo "parsing $indexfile"
94
+
filedirlist="$(PARSE_INDEX "$indexfile")"
95
+
filelist_next="$(echo "$filedirlist" | grep '\.tar\.xz$' | while read file; do echo "$dirurl/$file"; done)"
96
+
filelist_next="$(echo "$filelist_next" | clean_urls)"
97
+
[ -n "$filelist" ] && filelist+=$'\n'
98
+
filelist+="$filelist_next"
99
+
dirlist="$(echo "$filedirlist" | grep -v '\.tar\.xz$' | while read dir; do echo "$dirurl/$dir"; done || true)"
100
+
dirlist="$(echo "$dirlist" | clean_urls)"
101
+
done
102
+
done
103
+
104
+
filecount=$(echo "$filelist" | wc -l)
105
+
106
+
if [ -z "$filelist" ]
107
+
then
108
+
echo "error: no files parsed from $tmp/index.html"
109
+
exit 1
110
+
fi
111
+
112
+
echo "parsed $filecount tar.xz files:"; echo "$filelist"
113
+
114
+
# most time is spent here
115
+
echo "fetching $filecount sha256 files ..."
116
+
urllist="$(echo "$filelist" | while read file; do echo "$file.sha256"; done)"
117
+
# wget -r: keep directory structure
118
+
echo "$urllist" | xargs wget $wgetargs -nH -r -c --no-parent && {
119
+
actual=$(find . -type f -name '*.sha256' | wc -l)
120
+
echo "fetching $filecount sha256 files done: got $actual files"
121
+
} || {
122
+
# workaround: in rare cases, the server does not provide the sha256 files
123
+
# for example when the release is just a few hours old
124
+
# and the servers are not yet fully synced
125
+
actual=$(find . -type f -name '*.sha256' | wc -l)
126
+
echo "fetching $filecount sha256 files failed: got only $actual files"
127
+
128
+
# TODO fetch only missing tar.xz files
129
+
echo "fetching $filecount tar.xz files ..."
130
+
urllist="$(echo "$filelist" | while read file; do echo "$BASE_URL/$file"; done)"
131
+
echo "$urllist" | xargs wget $wgetargs -nH -r -c --no-parent
132
+
133
+
echo "generating sha256 files ..."
134
+
find . -type f -name '*.tar.xz' | while read src; do
135
+
name=$(basename "$src")
136
+
sha256=$(sha256sum "$src" | cut -d' ' -f1)
137
+
echo "$sha256 $name" >"$src.sha256"
138
+
done
139
+
}
140
+
141
+
csv=$(mktemp $tmptpl.csv)
142
+
echo "writing temporary file $csv ..."
143
+
find . -type f -name '*.sha256' | while read sha256file; do
144
+
src="${sha256file%.*}" # remove extension
145
+
sha256=$(cat $sha256file | cut -d' ' -f1) # base16
146
+
sha256=$(nix-hash --type sha256 --to-base32 $sha256)
21
147
# Sanitize file name
22
148
filename=$(basename "$src" | tr '@' '_')
23
149
nameVersion="${filename%.tar.*}"
24
150
name=$(echo "$nameVersion" | sed -e 's,-[[:digit:]].*,,' | sed -e 's,-opensource-src$,,' | sed -e 's,-everywhere-src$,,')
25
151
version=$(echo "$nameVersion" | sed -e 's,^\([[:alpha:]][[:alnum:]]*-\)\+,,')
26
-
echo "$name,$version,$src,$filename" >>$csv
152
+
echo "$name,$version,$src,$filename,$sha256" >>$csv
27
153
done
28
154
155
+
files_before=$(grep -c 'src = ' "$SRCS")
156
+
157
+
echo "writing output file $SRCS ..."
29
158
cat >"$SRCS" <<EOF
30
159
# DO NOT EDIT! This file is generated automatically.
31
160
# Command: $0 $@
···
39
168
latestVersion=$(echo "$versions" | sort -rV | head -n 1)
40
169
src=$(gawk -F , "/^$name,$latestVersion,/ { print \$3 }" $csv)
41
170
filename=$(gawk -F , "/^$name,$latestVersion,/ { print \$4 }" $csv)
171
+
sha256=$(gawk -F , "/^$name,$latestVersion,/ { print \$5 }" $csv)
42
172
url="${src:2}"
43
-
sha256=$(nix-hash --type sha256 --base32 --flat "$src")
44
173
cat >>"$SRCS" <<EOF
45
174
$name = {
46
175
version = "$latestVersion";
···
54
183
done
55
184
56
185
echo "}" >>"$SRCS"
186
+
187
+
files_after=$(grep -c 'src = ' "$SRCS")
188
+
echo "files before: $files_before"
189
+
echo "files after: $files_after"
190
+
191
+
echo "compare:"
192
+
echo "git diff $srcsrel"
57
193
58
194
popd >/dev/null
59
195
rm -fr $tmp >/dev/null