pyrox.dev
1#! /usr/bin/env bash
2
3set -e -o pipefail
4
5url=
6rev=
7expHash=
8hashType=$NIX_HASH_ALGO
9deepClone=$NIX_PREFETCH_GIT_DEEP_CLONE
10leaveDotGit=$NIX_PREFETCH_GIT_LEAVE_DOT_GIT
11fetchSubmodules=
12builder=
13branchName=$NIX_PREFETCH_GIT_BRANCH_NAME
14
15# ENV params
16out=${out:-}
17http_proxy=${http_proxy:-}
18
19# populated by clone_user_rev()
20fullRev=
21humanReadableRev=
22commitDate=
23commitDateStrict8601=
24
25if test -n "$deepClone"; then
26 deepClone=true
27else
28 deepClone=false
29fi
30
31if test "$leaveDotGit" != 1; then
32 leaveDotGit=
33else
34 leaveDotGit=true
35fi
36
37usage(){
38 echo >&2 "syntax: nix-prefetch-git [options] [URL [REVISION [EXPECTED-HASH]]]
39
40Options:
41 --out path Path where the output would be stored.
42 --url url Any url understand by 'git clone'.
43 --rev ref Any sha1 or references (such as refs/heads/master)
44 --hash h Expected hash.
45 --deepClone Clone the entire repository.
46 --no-deepClone Make a shallow clone of just the required ref.
47 --leave-dotGit Keep the .git directories.
48 --fetch-submodules Fetch submodules.
49 --builder Clone as fetchgit does, but url, rev, and out option are mandatory.
50 --quiet Only print the final json summary.
51"
52 exit 1
53}
54
55argi=0
56argfun=""
57for arg; do
58 if test -z "$argfun"; then
59 case $arg in
60 --out) argfun=set_out;;
61 --url) argfun=set_url;;
62 --rev) argfun=set_rev;;
63 --hash) argfun=set_hashType;;
64 --branch-name) argfun=set_branchName;;
65 --deepClone) deepClone=true;;
66 --quiet) QUIET=true;;
67 --no-deepClone) deepClone=false;;
68 --leave-dotGit) leaveDotGit=true;;
69 --fetch-submodules) fetchSubmodules=true;;
70 --builder) builder=true;;
71 --help) usage; exit;;
72 *)
73 : $((++argi))
74 case $argi in
75 1) url=$arg;;
76 2) rev=$arg;;
77 3) expHash=$arg;;
78 *) exit 1;;
79 esac
80 ;;
81 esac
82 else
83 case $argfun in
84 set_*)
85 var=${argfun#set_}
86 eval $var=$arg
87 ;;
88 esac
89 argfun=""
90 fi
91done
92
93if test -z "$url"; then
94 usage
95fi
96
97
98init_remote(){
99 local url=$1
100 git init
101 git remote add origin "$url"
102 ( [ -n "$http_proxy" ] && git config http.proxy "$http_proxy" ) || true
103}
104
105# Return the reference of an hash if it exists on the remote repository.
106ref_from_hash(){
107 local hash=$1
108 git ls-remote origin | sed -n "\,$hash\t, { s,\(.*\)\t\(.*\),\2,; p; q}"
109}
110
111# Return the hash of a reference if it exists on the remote repository.
112hash_from_ref(){
113 local ref=$1
114 git ls-remote origin | sed -n "\,\t$ref, { s,\(.*\)\t\(.*\),\1,; p; q}"
115}
116
117# Returns a name based on the url and reference
118#
119# This function needs to be in sync with nix's fetchgit implementation
120# of urlToName() to re-use the same nix store paths.
121url_to_name(){
122 local url=$1
123 local ref=$2
124 local base
125 base=$(basename "$url" .git | cut -d: -f2)
126
127 if [[ $ref =~ ^[a-z0-9]+$ ]]; then
128 echo "$base-${ref:0:7}"
129 else
130 echo "$base"
131 fi
132}
133
134# Fetch everything and checkout the right sha1
135checkout_hash(){
136 local hash="$1"
137 local ref="$2"
138
139 if test -z "$hash"; then
140 hash=$(hash_from_ref "$ref")
141 fi
142
143 git fetch -t ${builder:+--progress} origin || return 1
144 git checkout -b "$branchName" "$hash" || return 1
145}
146
147# Fetch only a branch/tag and checkout it.
148checkout_ref(){
149 local hash="$1"
150 local ref="$2"
151
152 if "$deepClone"; then
153 # The caller explicitly asked for a deep clone. Deep clones
154 # allow "git describe" and similar tools to work. See
155 # http://thread.gmane.org/gmane.linux.distributions.nixos/3569
156 # for a discussion.
157 return 1
158 fi
159
160 if test -z "$ref"; then
161 ref=$(ref_from_hash "$hash")
162 fi
163
164 if test -n "$ref"; then
165 # --depth option is ignored on http repository.
166 git fetch ${builder:+--progress} --depth 1 origin +"$ref" || return 1
167 git checkout -b "$branchName" FETCH_HEAD || return 1
168 else
169 return 1
170 fi
171}
172
173# Update submodules
174init_submodules(){
175 # Add urls into .git/config file
176 git submodule init
177
178 # list submodule directories and their hashes
179 git submodule status |
180 while read -r l; do
181 local hash
182 local dir
183 local name
184 local url
185
186 # checkout each submodule
187 hash=$(echo "$l" | awk '{print substr($1,2)}')
188 dir=$(echo "$l" | awk '{print $2}')
189 name=$(
190 git config -f .gitmodules --get-regexp submodule\..*\.path |
191 sed -n "s,^\(.*\)\.path $dir\$,\\1,p")
192 url=$(git config --get "${name}.url")
193
194 clone "$dir" "$url" "$hash" ""
195 done
196}
197
198clone(){
199 local top=$PWD
200 local dir="$1"
201 local url="$2"
202 local hash="$3"
203 local ref="$4"
204
205 cd "$dir"
206
207 # Initialize the repository.
208 init_remote "$url"
209
210 # Download data from the repository.
211 checkout_ref "$hash" "$ref" ||
212 checkout_hash "$hash" "$ref" || (
213 echo 1>&2 "Unable to checkout $hash$ref from $url."
214 exit 1
215 )
216
217 # Checkout linked sources.
218 if test -n "$fetchSubmodules"; then
219 init_submodules
220 fi
221
222 if [ -z "$builder" ] && [ -f .topdeps ]; then
223 if tg help &>/dev/null; then
224 echo "populating TopGit branches..."
225 tg remote --populate origin
226 else
227 echo "WARNING: would populate TopGit branches but TopGit is not available" >&2
228 echo "WARNING: install TopGit to fix the problem" >&2
229 fi
230 fi
231
232 cd "$top"
233}
234
235# Remove all remote branches, remove tags not reachable from HEAD, do a full
236# repack and then garbage collect unreferenced objects.
237make_deterministic_repo(){
238 local repo="$1"
239
240 # run in sub-shell to not touch current working directory
241 (
242 cd "$repo"
243 # Remove files that contain timestamps or otherwise have non-deterministic
244 # properties.
245 rm -rf .git/logs/ .git/hooks/ .git/index .git/FETCH_HEAD .git/ORIG_HEAD \
246 .git/refs/remotes/origin/HEAD .git/config
247
248 # Remove all remote branches.
249 git branch -r | while read -r branch; do
250 git branch -rD "$branch" >&2
251 done
252
253 # Remove tags not reachable from HEAD. If we're exactly on a tag, don't
254 # delete it.
255 maybe_tag=$(git tag --points-at HEAD)
256 git tag --contains HEAD | while read -r tag; do
257 if [ "$tag" != "$maybe_tag" ]; then
258 git tag -d "$tag" >&2
259 fi
260 done
261
262 # Do a full repack. Must run single-threaded, or else we lose determinism.
263 git config pack.threads 1
264 git repack -A -d -f
265 rm -f .git/config
266
267 # Garbage collect unreferenced objects.
268 git gc --prune=all
269 )
270}
271
272
273_clone_user_rev() {
274 local dir="$1"
275 local url="$2"
276 local rev="${3:-HEAD}"
277
278 # Perform the checkout.
279 case "$rev" in
280 HEAD|refs/*)
281 clone "$dir" "$url" "" "$rev" 1>&2;;
282 *)
283 if test -z "$(echo "$rev" | tr -d 0123456789abcdef)"; then
284 clone "$dir" "$url" "$rev" "" 1>&2
285 else
286 # if revision is not hexadecimal it might be a tag
287 clone "$dir" "$url" "" "refs/tags/$rev" 1>&2
288 fi;;
289 esac
290
291 pushd "$dir" >/dev/null
292 fullRev=$( (git rev-parse "$rev" 2>/dev/null || git rev-parse "refs/heads/$branchName") | tail -n1)
293 humanReadableRev=$(git describe "$fullRev" 2> /dev/null || git describe --tags "$fullRev" 2> /dev/null || echo -- none --)
294 commitDate=$(git show -1 --no-patch --pretty=%ci "$fullRev")
295 commitDateStrict8601=$(git show -1 --no-patch --pretty=%cI "$fullRev")
296 popd >/dev/null
297
298 # Allow doing additional processing before .git removal
299 eval "$NIX_PREFETCH_GIT_CHECKOUT_HOOK"
300 if test -z "$leaveDotGit"; then
301 echo "removing \`.git'..." >&2
302 find "$dir" -name .git -print0 | xargs -0 rm -rf
303 else
304 find "$dir" -name .git | while read -r gitdir; do
305 make_deterministic_repo "$(readlink -f "$gitdir/..")"
306 done
307 fi
308}
309
310clone_user_rev() {
311 if ! test -n "$QUIET"; then
312 _clone_user_rev "$@"
313 else
314 errfile="$(mktemp "${TMPDIR:-/tmp}/git-checkout-err-XXXXXXXX")"
315 # shellcheck disable=SC2064
316 trap "rm -rf \"$errfile\"" EXIT
317 _clone_user_rev "$@" 2> "$errfile" || (
318 status="$?"
319 cat "$errfile" >&2
320 exit "$status"
321 )
322 fi
323}
324
325json_escape() {
326 local s="$1"
327 s="${s//\\/\\\\}" # \
328 s="${s//\"/\\\"}" # "
329 s="${s//^H/\\\b}" # \b (backspace)
330 s="${s//^L/\\\f}" # \f (form feed)
331 s="${s//
332/\\\n}" # \n (newline)
333 s="${s//^M/\\\r}" # \r (carriage return)
334 s="${s// /\\t}" # \t (tab)
335 echo "$s"
336}
337
338print_results() {
339 hash="$1"
340 if ! test -n "$QUIET"; then
341 echo "" >&2
342 echo "git revision is $fullRev" >&2
343 if test -n "$finalPath"; then
344 echo "path is $finalPath" >&2
345 fi
346 echo "git human-readable version is $humanReadableRev" >&2
347 echo "Commit date is $commitDate" >&2
348 if test -n "$hash"; then
349 echo "hash is $hash" >&2
350 fi
351 fi
352 if test -n "$hash"; then
353 cat <<EOF
354{
355 "url": "$(json_escape "$url")",
356 "rev": "$(json_escape "$fullRev")",
357 "date": "$(json_escape "$commitDateStrict8601")",
358 "$(json_escape "$hashType")": "$(json_escape "$hash")",
359 "fetchSubmodules": $([[ -n "fetchSubmodules" ]] && echo true || echo false)
360}
361EOF
362 fi
363}
364
365if test -z "$branchName"; then
366 branchName=fetchgit
367fi
368
369if test -n "$builder"; then
370 test -n "$out" -a -n "$url" -a -n "$rev" || usage
371 mkdir -p "$out"
372 clone_user_rev "$out" "$url" "$rev"
373else
374 if test -z "$hashType"; then
375 hashType=sha256
376 fi
377
378 # If the hash was given, a file with that hash may already be in the
379 # store.
380 if test -n "$expHash"; then
381 finalPath=$(nix-store --print-fixed-path --recursive "$hashType" "$expHash" "$(url_to_name "$url" "$rev")")
382 if ! nix-store --check-validity "$finalPath" 2> /dev/null; then
383 finalPath=
384 fi
385 hash=$expHash
386 fi
387
388 # If we don't know the hash or a path with that hash doesn't exist,
389 # download the file and add it to the store.
390 if test -z "$finalPath"; then
391
392 tmpPath="$(mktemp -d "${TMPDIR:-/tmp}/git-checkout-tmp-XXXXXXXX")"
393 # shellcheck disable=SC2064
394 trap "rm -rf \"$tmpPath\"" EXIT
395
396 tmpFile="$tmpPath/$(url_to_name "$url" "$rev")"
397 mkdir -p "$tmpFile"
398
399 # Perform the checkout.
400 clone_user_rev "$tmpFile" "$url" "$rev"
401
402 # Compute the hash.
403 hash=$(nix-hash --type $hashType --base32 "$tmpFile")
404
405 # Add the downloaded file to the Nix store.
406 finalPath=$(nix-store --add-fixed --recursive "$hashType" "$tmpFile")
407
408 if test -n "$expHash" -a "$expHash" != "$hash"; then
409 echo "hash mismatch for URL \`$url'. Got \`$hash'; expected \`$expHash'." >&2
410 exit 1
411 fi
412 fi
413
414 print_results "$hash"
415
416 if test -n "$PRINT_PATH"; then
417 echo "$finalPath"
418 fi
419fi