tjh.dev
1#! /usr/bin/env bash
2
3set -e -o pipefail
4
5url=
6rev=
7expHash=
8hashType=$NIX_HASH_ALGO
9deepClone=$NIX_PREFETCH_GIT_DEEP_CLONE
10leaveDotGit=$NIX_PREFETCH_GIT_LEAVE_DOT_GIT
11fetchSubmodules=
12builder=
13branchName=$NIX_PREFETCH_GIT_BRANCH_NAME
14
15# populated by clone_user_rev()
16fullRev=
17humanReadableRev=
18commitDate=
19
20if test -n "$deepClone"; then
21 deepClone=true
22else
23 deepClone=false
24fi
25
26if test "$leaveDotGit" != 1; then
27 leaveDotGit=
28else
29 leaveDotGit=true
30fi
31
32usage(){
33 echo >&2 "syntax: nix-prefetch-git [options] [URL [REVISION [EXPECTED-HASH]]]
34
35Options:
36 --out path Path where the output would be stored.
37 --url url Any url understand by 'git clone'.
38 --rev ref Any sha1 or references (such as refs/heads/master)
39 --hash h Expected hash.
40 --deepClone Clone the entire repository.
41 --no-deepClone Make a shallow clone of just the required ref.
42 --leave-dotGit Keep the .git directories.
43 --fetch-submodules Fetch submodules.
44 --builder Clone as fetchgit does, but url, rev, and out option are mandatory.
45"
46 exit 1
47}
48
49argi=0
50argfun=""
51for arg; do
52 if test -z "$argfun"; then
53 case $arg in
54 --out) argfun=set_out;;
55 --url) argfun=set_url;;
56 --rev) argfun=set_rev;;
57 --hash) argfun=set_hashType;;
58 --branch-name) argfun=set_branchName;;
59 --deepClone) deepClone=true;;
60 --quiet) QUIET=true;;
61 --no-deepClone) deepClone=false;;
62 --leave-dotGit) leaveDotGit=true;;
63 --fetch-submodules) fetchSubmodules=true;;
64 --builder) builder=true;;
65 --help) usage; exit;;
66 *)
67 argi=$(($argi + 1))
68 case $argi in
69 1) url=$arg;;
70 2) rev=$arg;;
71 3) expHash=$arg;;
72 *) exit 1;;
73 esac
74 ;;
75 esac
76 else
77 case $argfun in
78 set_*)
79 var=$(echo $argfun | sed 's,^set_,,')
80 eval $var=$arg
81 ;;
82 esac
83 argfun=""
84 fi
85done
86
87if test -z "$url"; then
88 usage
89fi
90
91
92init_remote(){
93 local url=$1
94 git init
95 git remote add origin $url
96 ( [ -n "$http_proxy" ] && git config http.proxy $http_proxy ) || true
97}
98
99# Return the reference of an hash if it exists on the remote repository.
100ref_from_hash(){
101 local hash=$1
102 git ls-remote origin | sed -n "\,$hash\t, { s,\(.*\)\t\(.*\),\2,; p; q}"
103}
104
105# Return the hash of a reference if it exists on the remote repository.
106hash_from_ref(){
107 local ref=$1
108 git ls-remote origin | sed -n "\,\t$ref, { s,\(.*\)\t\(.*\),\1,; p; q}"
109}
110
111# Returns a name based on the url and reference
112#
113# This function needs to be in sync with nix's fetchgit implementation
114# of urlToName() to re-use the same nix store paths.
115url_to_name(){
116 local url=$1
117 local ref=$2
118 # basename removes the / and .git suffixes
119 local base=$(basename "$url" .git)
120
121 if [[ $ref =~ [a-z0-9]+ ]]; then
122 echo "$base-${ref:0:7}"
123 else
124 echo $base
125 fi
126}
127
128# Fetch everything and checkout the right sha1
129checkout_hash(){
130 local hash="$1"
131 local ref="$2"
132
133 if test -z "$hash"; then
134 hash=$(hash_from_ref $ref)
135 fi
136
137 git fetch -t ${builder:+--progress} origin || return 1
138 git checkout -b $branchName $hash || return 1
139}
140
141# Fetch only a branch/tag and checkout it.
142checkout_ref(){
143 local hash="$1"
144 local ref="$2"
145
146 if "$deepClone"; then
147 # The caller explicitly asked for a deep clone. Deep clones
148 # allow "git describe" and similar tools to work. See
149 # http://thread.gmane.org/gmane.linux.distributions.nixos/3569
150 # for a discussion.
151 return 1
152 fi
153
154 if test -z "$ref"; then
155 ref=$(ref_from_hash $hash)
156 fi
157
158 if test -n "$ref"; then
159 # --depth option is ignored on http repository.
160 git fetch ${builder:+--progress} --depth 1 origin +"$ref" || return 1
161 git checkout -b $branchName FETCH_HEAD || return 1
162 else
163 return 1
164 fi
165}
166
167# Update submodules
168init_submodules(){
169 # Add urls into .git/config file
170 git submodule init
171
172 # list submodule directories and their hashes
173 git submodule status |
174 while read l; do
175 # checkout each submodule
176 local hash=$(echo $l | awk '{print substr($1,2)}')
177 local dir=$(echo $l | awk '{print $2}')
178 local name=$(
179 git config -f .gitmodules --get-regexp submodule\..*\.path |
180 sed -n "s,^\(.*\)\.path $dir\$,\\1,p")
181 local url=$(git config --get ${name}.url)
182
183 clone "$dir" "$url" "$hash" ""
184 done
185}
186
187clone(){
188 local top=$(pwd)
189 local dir="$1"
190 local url="$2"
191 local hash="$3"
192 local ref="$4"
193
194 cd $dir
195
196 # Initialize the repository.
197 init_remote "$url"
198
199 # Download data from the repository.
200 checkout_ref "$hash" "$ref" ||
201 checkout_hash "$hash" "$ref" || (
202 echo 1>&2 "Unable to checkout $hash$ref from $url."
203 exit 1
204 )
205
206 # Checkout linked sources.
207 if test -n "$fetchSubmodules"; then
208 init_submodules
209 fi
210
211 if [ -z "$builder" -a -f .topdeps ]; then
212 if tg help 2>&1 > /dev/null
213 then
214 echo "populating TopGit branches..."
215 tg remote --populate origin
216 else
217 echo "WARNING: would populate TopGit branches but TopGit is not available" >&2
218 echo "WARNING: install TopGit to fix the problem" >&2
219 fi
220 fi
221
222 cd $top
223}
224
225# Remove all remote branches, remove tags not reachable from HEAD, do a full
226# repack and then garbage collect unreferenced objects.
227make_deterministic_repo(){
228 local repo="$1"
229
230 # run in sub-shell to not touch current working directory
231 (
232 cd "$repo"
233 # Remove files that contain timestamps or otherwise have non-deterministic
234 # properties.
235 rm -rf .git/logs/ .git/hooks/ .git/index .git/FETCH_HEAD .git/ORIG_HEAD \
236 .git/refs/remotes/origin/HEAD .git/config
237
238 # Remove all remote branches.
239 git branch -r | while read branch; do
240 git branch -rD "$branch" >&2
241 done
242
243 # Remove tags not reachable from HEAD. If we're exactly on a tag, don't
244 # delete it.
245 maybe_tag=$(git tag --points-at HEAD)
246 git tag --contains HEAD | while read tag; do
247 if [ "$tag" != "$maybe_tag" ]; then
248 git tag -d "$tag" >&2
249 fi
250 done
251
252 # Do a full repack. Must run single-threaded, or else we lose determinism.
253 git config pack.threads 1
254 git repack -A -d -f
255 rm -f .git/config
256
257 # Garbage collect unreferenced objects.
258 git gc --prune=all
259 )
260}
261
262
263_clone_user_rev() {
264 local dir="$1"
265 local url="$2"
266 local rev="${3:-HEAD}"
267
268 # Perform the checkout.
269 case "$rev" in
270 HEAD|refs/*)
271 clone "$dir" "$url" "" "$rev" 1>&2;;
272 *)
273 if test -z "$(echo $rev | tr -d 0123456789abcdef)"; then
274 clone "$dir" "$url" "$rev" "" 1>&2
275 else
276 echo 1>&2 "Bad commit hash or bad reference."
277 exit 1
278 fi;;
279 esac
280
281 fullRev="$(cd $dir && (git rev-parse $rev 2> /dev/null || git rev-parse refs/heads/$branchName) | tail -n1)"
282 humanReadableRev="$(cd $dir && (git describe $fullRev 2> /dev/null || git describe --tags $fullRev 2> /dev/null || echo -- none --))"
283 commitDate="$(cd $dir && git show --no-patch --pretty=%ci $fullRev)"
284
285 # Allow doing additional processing before .git removal
286 eval "$NIX_PREFETCH_GIT_CHECKOUT_HOOK"
287 if test -z "$leaveDotGit"; then
288 echo "removing \`.git'..." >&2
289 find "$dir" -name .git\* -print0 | xargs -0 rm -rf
290 else
291 find "$dir" -name .git | while read gitdir; do
292 make_deterministic_repo "$(readlink -f "$gitdir/..")"
293 done
294 fi
295}
296
297clone_user_rev() {
298 if ! test -n "$QUIET"; then
299 _clone_user_rev "$@"
300 else
301 errfile="$(mktemp "${TMPDIR:-/tmp}/git-checkout-err-XXXXXXXX")"
302 trap "rm -rf \"$errfile\"" EXIT
303 _clone_user_rev "$@" 2> "$errfile" || (
304 status="$?"
305 cat "$errfile" >&2
306 exit "$status"
307 )
308 fi
309}
310
311
312print_results() {
313 hash="$1"
314 if ! test -n "$QUIET"; then
315 echo "" >&2
316 echo "git revision is $fullRev" >&2
317 if test -n "$finalPath"; then
318 echo "path is $finalPath" >&2
319 fi
320 echo "git human-readable version is $humanReadableRev" >&2
321 echo "Commit date is $commitDate" >&2
322 if test -n "$hash"; then
323 echo "hash is $hash" >&2
324 fi
325 fi
326 if test -n "$hash"; then
327 echo "{"
328 echo " url = \"$url\";"
329 echo " rev = \"$fullRev\";"
330 echo " $hashType = \"$hash\";"
331 echo "}"
332 fi
333}
334
335if test -z "$branchName"; then
336 branchName=fetchgit
337fi
338
339if test -n "$builder"; then
340 test -n "$out" -a -n "$url" -a -n "$rev" || usage
341 mkdir $out
342 clone_user_rev "$out" "$url" "$rev"
343else
344 if test -z "$hashType"; then
345 hashType=sha256
346 fi
347
348 # If the hash was given, a file with that hash may already be in the
349 # store.
350 if test -n "$expHash"; then
351 finalPath=$(nix-store --print-fixed-path --recursive "$hashType" "$expHash" "$(url_to_name "$url" "$rev")")
352 if ! nix-store --check-validity "$finalPath" 2> /dev/null; then
353 finalPath=
354 fi
355 hash=$expHash
356 fi
357
358 # If we don't know the hash or a path with that hash doesn't exist,
359 # download the file and add it to the store.
360 if test -z "$finalPath"; then
361
362 tmpPath="$(mktemp -d "${TMPDIR:-/tmp}/git-checkout-tmp-XXXXXXXX")"
363 trap "rm -rf \"$tmpPath\"" EXIT
364
365 tmpFile="$tmpPath/$(url_to_name "$url" "$rev")"
366 mkdir "$tmpFile"
367
368 # Perform the checkout.
369 clone_user_rev "$tmpFile" "$url" "$rev"
370
371 # Compute the hash.
372 hash=$(nix-hash --type $hashType --base32 $tmpFile)
373
374 # Add the downloaded file to the Nix store.
375 finalPath=$(nix-store --add-fixed --recursive "$hashType" "$tmpFile")
376
377 if test -n "$expHash" -a "$expHash" != "$hash"; then
378 print_metadata
379 echo "hash mismatch for URL \`$url'" >&2
380 exit 1
381 fi
382 fi
383
384 print_results "$hash"
385
386 if test -n "$PRINT_PATH"; then
387 echo $finalPath
388 fi
389fi