pyrox.dev
1#! /usr/bin/env bash
2
3set -e -o pipefail
4
5url=
6rev=
7expHash=
8hashType=$NIX_HASH_ALGO
9deepClone=$NIX_PREFETCH_GIT_DEEP_CLONE
10leaveDotGit=$NIX_PREFETCH_GIT_LEAVE_DOT_GIT
11fetchSubmodules=
12builder=
13branchName=$NIX_PREFETCH_GIT_BRANCH_NAME
14
15# ENV params
16out=${out:-}
17http_proxy=${http_proxy:-}
18
19# populated by clone_user_rev()
20fullRev=
21humanReadableRev=
22commitDate=
23commitDateStrict8601=
24
25if test -n "$deepClone"; then
26 deepClone=true
27else
28 deepClone=false
29fi
30
31if test "$leaveDotGit" != 1; then
32 leaveDotGit=
33else
34 leaveDotGit=true
35fi
36
37usage(){
38 echo >&2 "syntax: nix-prefetch-git [options] [URL [REVISION [EXPECTED-HASH]]]
39
40Options:
41 --out path Path where the output would be stored.
42 --url url Any url understand by 'git clone'.
43 --rev ref Any sha1 or references (such as refs/heads/master)
44 --hash h Expected hash.
45 --deepClone Clone the entire repository.
46 --no-deepClone Make a shallow clone of just the required ref.
47 --leave-dotGit Keep the .git directories.
48 --fetch-submodules Fetch submodules.
49 --builder Clone as fetchgit does, but url, rev, and out option are mandatory.
50"
51 exit 1
52}
53
54argi=0
55argfun=""
56for arg; do
57 if test -z "$argfun"; then
58 case $arg in
59 --out) argfun=set_out;;
60 --url) argfun=set_url;;
61 --rev) argfun=set_rev;;
62 --hash) argfun=set_hashType;;
63 --branch-name) argfun=set_branchName;;
64 --deepClone) deepClone=true;;
65 --quiet) QUIET=true;;
66 --no-deepClone) deepClone=false;;
67 --leave-dotGit) leaveDotGit=true;;
68 --fetch-submodules) fetchSubmodules=true;;
69 --builder) builder=true;;
70 --help) usage; exit;;
71 *)
72 : $((++argi))
73 case $argi in
74 1) url=$arg;;
75 2) rev=$arg;;
76 3) expHash=$arg;;
77 *) exit 1;;
78 esac
79 ;;
80 esac
81 else
82 case $argfun in
83 set_*)
84 var=${argfun#set_}
85 eval $var=$arg
86 ;;
87 esac
88 argfun=""
89 fi
90done
91
92if test -z "$url"; then
93 usage
94fi
95
96
97init_remote(){
98 local url=$1
99 git init
100 git remote add origin "$url"
101 ( [ -n "$http_proxy" ] && git config http.proxy "$http_proxy" ) || true
102}
103
104# Return the reference of an hash if it exists on the remote repository.
105ref_from_hash(){
106 local hash=$1
107 git ls-remote origin | sed -n "\,$hash\t, { s,\(.*\)\t\(.*\),\2,; p; q}"
108}
109
110# Return the hash of a reference if it exists on the remote repository.
111hash_from_ref(){
112 local ref=$1
113 git ls-remote origin | sed -n "\,\t$ref, { s,\(.*\)\t\(.*\),\1,; p; q}"
114}
115
116# Returns a name based on the url and reference
117#
118# This function needs to be in sync with nix's fetchgit implementation
119# of urlToName() to re-use the same nix store paths.
120url_to_name(){
121 local url=$1
122 local ref=$2
123 # basename removes the / and .git suffixes
124 local base
125 base=$(basename "$url" .git)
126
127 if [[ $ref =~ ^[a-z0-9]+$ ]]; then
128 echo "$base-${ref:0:7}"
129 else
130 echo "$base"
131 fi
132}
133
134# Fetch everything and checkout the right sha1
135checkout_hash(){
136 local hash="$1"
137 local ref="$2"
138
139 if test -z "$hash"; then
140 hash=$(hash_from_ref "$ref")
141 fi
142
143 git fetch -t ${builder:+--progress} origin || return 1
144 git checkout -b "$branchName" "$hash" || return 1
145}
146
147# Fetch only a branch/tag and checkout it.
148checkout_ref(){
149 local hash="$1"
150 local ref="$2"
151
152 if "$deepClone"; then
153 # The caller explicitly asked for a deep clone. Deep clones
154 # allow "git describe" and similar tools to work. See
155 # http://thread.gmane.org/gmane.linux.distributions.nixos/3569
156 # for a discussion.
157 return 1
158 fi
159
160 if test -z "$ref"; then
161 ref=$(ref_from_hash "$hash")
162 fi
163
164 if test -n "$ref"; then
165 # --depth option is ignored on http repository.
166 git fetch ${builder:+--progress} --depth 1 origin +"$ref" || return 1
167 git checkout -b "$branchName" FETCH_HEAD || return 1
168 else
169 return 1
170 fi
171}
172
173# Update submodules
174init_submodules(){
175 # Add urls into .git/config file
176 git submodule init
177
178 # list submodule directories and their hashes
179 git submodule status |
180 while read -r l; do
181 local hash
182 local dir
183 local name
184 local url
185
186 # checkout each submodule
187 hash=$(echo "$l" | awk '{print substr($1,2)}')
188 dir=$(echo "$l" | awk '{print $2}')
189 name=$(
190 git config -f .gitmodules --get-regexp submodule\..*\.path |
191 sed -n "s,^\(.*\)\.path $dir\$,\\1,p")
192 url=$(git config --get "${name}.url")
193
194 clone "$dir" "$url" "$hash" ""
195 done
196}
197
198clone(){
199 local top=$PWD
200 local dir="$1"
201 local url="$2"
202 local hash="$3"
203 local ref="$4"
204
205 cd "$dir"
206
207 # Initialize the repository.
208 init_remote "$url"
209
210 # Download data from the repository.
211 checkout_ref "$hash" "$ref" ||
212 checkout_hash "$hash" "$ref" || (
213 echo 1>&2 "Unable to checkout $hash$ref from $url."
214 exit 1
215 )
216
217 # Checkout linked sources.
218 if test -n "$fetchSubmodules"; then
219 init_submodules
220 fi
221
222 if [ -z "$builder" ] && [ -f .topdeps ]; then
223 if tg help &>/dev/null; then
224 echo "populating TopGit branches..."
225 tg remote --populate origin
226 else
227 echo "WARNING: would populate TopGit branches but TopGit is not available" >&2
228 echo "WARNING: install TopGit to fix the problem" >&2
229 fi
230 fi
231
232 cd "$top"
233}
234
235# Remove all remote branches, remove tags not reachable from HEAD, do a full
236# repack and then garbage collect unreferenced objects.
237make_deterministic_repo(){
238 local repo="$1"
239
240 # run in sub-shell to not touch current working directory
241 (
242 cd "$repo"
243 # Remove files that contain timestamps or otherwise have non-deterministic
244 # properties.
245 rm -rf .git/logs/ .git/hooks/ .git/index .git/FETCH_HEAD .git/ORIG_HEAD \
246 .git/refs/remotes/origin/HEAD .git/config
247
248 # Remove all remote branches.
249 git branch -r | while read -r branch; do
250 git branch -rD "$branch" >&2
251 done
252
253 # Remove tags not reachable from HEAD. If we're exactly on a tag, don't
254 # delete it.
255 maybe_tag=$(git tag --points-at HEAD)
256 git tag --contains HEAD | while read -r tag; do
257 if [ "$tag" != "$maybe_tag" ]; then
258 git tag -d "$tag" >&2
259 fi
260 done
261
262 # Do a full repack. Must run single-threaded, or else we lose determinism.
263 git config pack.threads 1
264 git repack -A -d -f
265 rm -f .git/config
266
267 # Garbage collect unreferenced objects.
268 git gc --prune=all
269 )
270}
271
272
273_clone_user_rev() {
274 local dir="$1"
275 local url="$2"
276 local rev="${3:-HEAD}"
277
278 # Perform the checkout.
279 case "$rev" in
280 HEAD|refs/*)
281 clone "$dir" "$url" "" "$rev" 1>&2;;
282 *)
283 if test -z "$(echo "$rev" | tr -d 0123456789abcdef)"; then
284 clone "$dir" "$url" "$rev" "" 1>&2
285 else
286 echo 1>&2 "Bad commit hash or bad reference."
287 exit 1
288 fi;;
289 esac
290
291 pushd "$dir" >/dev/null
292 fullRev=$( (git rev-parse "$rev" 2>/dev/null || git rev-parse "refs/heads/$branchName") | tail -n1)
293 humanReadableRev=$(git describe "$fullRev" 2> /dev/null || git describe --tags "$fullRev" 2> /dev/null || echo -- none --)
294 commitDate=$(git show --no-patch --pretty=%ci "$fullRev")
295 commitDateStrict8601=$(git show --no-patch --pretty=%cI "$fullRev")
296 popd >/dev/null
297
298 # Allow doing additional processing before .git removal
299 eval "$NIX_PREFETCH_GIT_CHECKOUT_HOOK"
300 if test -z "$leaveDotGit"; then
301 echo "removing \`.git'..." >&2
302 find "$dir" -name .git -print0 | xargs -0 rm -rf
303 else
304 find "$dir" -name .git | while read -r gitdir; do
305 make_deterministic_repo "$(readlink -f "$gitdir/..")"
306 done
307 fi
308}
309
310clone_user_rev() {
311 if ! test -n "$QUIET"; then
312 _clone_user_rev "$@"
313 else
314 errfile="$(mktemp "${TMPDIR:-/tmp}/git-checkout-err-XXXXXXXX")"
315 # shellcheck disable=SC2064
316 trap "rm -rf \"$errfile\"" EXIT
317 _clone_user_rev "$@" 2> "$errfile" || (
318 status="$?"
319 cat "$errfile" >&2
320 exit "$status"
321 )
322 fi
323}
324
325
326print_results() {
327 hash="$1"
328 if ! test -n "$QUIET"; then
329 echo "" >&2
330 echo "git revision is $fullRev" >&2
331 if test -n "$finalPath"; then
332 echo "path is $finalPath" >&2
333 fi
334 echo "git human-readable version is $humanReadableRev" >&2
335 echo "Commit date is $commitDate" >&2
336 if test -n "$hash"; then
337 echo "hash is $hash" >&2
338 fi
339 fi
340 if test -n "$hash"; then
341 echo "{"
342 echo " \"url\": \"$url\","
343 echo " \"rev\": \"$fullRev\","
344 echo " \"date\": \"$commitDateStrict8601\","
345 echo -n " \"$hashType\": \"$hash\""
346 if test -n "$fetchSubmodules"; then
347 echo ","
348 echo -n " \"fetchSubmodules\": true"
349 fi
350 echo ""
351 echo "}"
352 fi
353}
354
355if test -z "$branchName"; then
356 branchName=fetchgit
357fi
358
359if test -n "$builder"; then
360 test -n "$out" -a -n "$url" -a -n "$rev" || usage
361 mkdir -p "$out"
362 clone_user_rev "$out" "$url" "$rev"
363else
364 if test -z "$hashType"; then
365 hashType=sha256
366 fi
367
368 # If the hash was given, a file with that hash may already be in the
369 # store.
370 if test -n "$expHash"; then
371 finalPath=$(nix-store --print-fixed-path --recursive "$hashType" "$expHash" "$(url_to_name "$url" "$rev")")
372 if ! nix-store --check-validity "$finalPath" 2> /dev/null; then
373 finalPath=
374 fi
375 hash=$expHash
376 fi
377
378 # If we don't know the hash or a path with that hash doesn't exist,
379 # download the file and add it to the store.
380 if test -z "$finalPath"; then
381
382 tmpPath="$(mktemp -d "${TMPDIR:-/tmp}/git-checkout-tmp-XXXXXXXX")"
383 # shellcheck disable=SC2064
384 trap "rm -rf \"$tmpPath\"" EXIT
385
386 tmpFile="$tmpPath/$(url_to_name "$url" "$rev")"
387 mkdir -p "$tmpFile"
388
389 # Perform the checkout.
390 clone_user_rev "$tmpFile" "$url" "$rev"
391
392 # Compute the hash.
393 hash=$(nix-hash --type $hashType --base32 "$tmpFile")
394
395 # Add the downloaded file to the Nix store.
396 finalPath=$(nix-store --add-fixed --recursive "$hashType" "$tmpFile")
397
398 if test -n "$expHash" -a "$expHash" != "$hash"; then
399 print_metadata
400 echo "hash mismatch for URL \`$url'" >&2
401 exit 1
402 fi
403 fi
404
405 print_results "$hash"
406
407 if test -n "$PRINT_PATH"; then
408 echo "$finalPath"
409 fi
410fi