tjh.dev
1#! /usr/bin/env bash
2
3set -e -o pipefail
4
5url=
6rev=
7expHash=
8hashType=$NIX_HASH_ALGO
9deepClone=$NIX_PREFETCH_GIT_DEEP_CLONE
10leaveDotGit=$NIX_PREFETCH_GIT_LEAVE_DOT_GIT
11fetchSubmodules=
12builder=
13branchName=$NIX_PREFETCH_GIT_BRANCH_NAME
14
15# ENV params
16out=${out:-}
17http_proxy=${http_proxy:-}
18
19# populated by clone_user_rev()
20fullRev=
21humanReadableRev=
22commitDate=
23commitDateStrict8601=
24
25if test -n "$deepClone"; then
26 deepClone=true
27else
28 deepClone=false
29fi
30
31if test "$leaveDotGit" != 1; then
32 leaveDotGit=
33else
34 leaveDotGit=true
35fi
36
37usage(){
38 echo >&2 "syntax: nix-prefetch-git [options] [URL [REVISION [EXPECTED-HASH]]]
39
40Options:
41 --out path Path where the output would be stored.
42 --url url Any url understood by 'git clone'.
43 --rev ref Any sha1 or references (such as refs/heads/master)
44 --hash h Expected hash.
45 --branch-name Branch name to check out into
46 --deepClone Clone the entire repository.
47 --no-deepClone Make a shallow clone of just the required ref.
48 --leave-dotGit Keep the .git directories.
49 --fetch-submodules Fetch submodules.
50 --builder Clone as fetchgit does, but url, rev, and out option are mandatory.
51 --quiet Only print the final json summary.
52"
53 exit 1
54}
55
56argi=0
57argfun=""
58for arg; do
59 if test -z "$argfun"; then
60 case $arg in
61 --out) argfun=set_out;;
62 --url) argfun=set_url;;
63 --rev) argfun=set_rev;;
64 --hash) argfun=set_hashType;;
65 --branch-name) argfun=set_branchName;;
66 --deepClone) deepClone=true;;
67 --quiet) QUIET=true;;
68 --no-deepClone) deepClone=false;;
69 --leave-dotGit) leaveDotGit=true;;
70 --fetch-submodules) fetchSubmodules=true;;
71 --builder) builder=true;;
72 -h|--help) usage; exit;;
73 *)
74 : $((++argi))
75 case $argi in
76 1) url=$arg;;
77 2) rev=$arg;;
78 3) expHash=$arg;;
79 *) exit 1;;
80 esac
81 ;;
82 esac
83 else
84 case $argfun in
85 set_*)
86 var=${argfun#set_}
87 eval $var=$arg
88 ;;
89 esac
90 argfun=""
91 fi
92done
93
94if test -z "$url"; then
95 usage
96fi
97
98
99init_remote(){
100 local url=$1
101 git init
102 git remote add origin "$url"
103 ( [ -n "$http_proxy" ] && git config http.proxy "$http_proxy" ) || true
104}
105
106# Return the reference of an hash if it exists on the remote repository.
107ref_from_hash(){
108 local hash=$1
109 git ls-remote origin | sed -n "\,$hash\t, { s,\(.*\)\t\(.*\),\2,; p; q}"
110}
111
112# Return the hash of a reference if it exists on the remote repository.
113hash_from_ref(){
114 local ref=$1
115 git ls-remote origin | sed -n "\,\t$ref, { s,\(.*\)\t\(.*\),\1,; p; q}"
116}
117
118# Returns a name based on the url and reference
119#
120# This function needs to be in sync with nix's fetchgit implementation
121# of urlToName() to re-use the same nix store paths.
122url_to_name(){
123 local url=$1
124 local ref=$2
125 local base
126 base=$(basename "$url" .git | cut -d: -f2)
127
128 if [[ $ref =~ ^[a-z0-9]+$ ]]; then
129 echo "$base-${ref:0:7}"
130 else
131 echo "$base"
132 fi
133}
134
135# Fetch everything and checkout the right sha1
136checkout_hash(){
137 local hash="$1"
138 local ref="$2"
139
140 if test -z "$hash"; then
141 hash=$(hash_from_ref "$ref")
142 fi
143
144 git fetch -t ${builder:+--progress} origin || return 1
145 git checkout -b "$branchName" "$hash" || return 1
146}
147
148# Fetch only a branch/tag and checkout it.
149checkout_ref(){
150 local hash="$1"
151 local ref="$2"
152
153 if "$deepClone"; then
154 # The caller explicitly asked for a deep clone. Deep clones
155 # allow "git describe" and similar tools to work. See
156 # https://marc.info/?l=nix-dev&m=139641582514772
157 # for a discussion.
158 return 1
159 fi
160
161 if test -z "$ref"; then
162 ref=$(ref_from_hash "$hash")
163 fi
164
165 if test -n "$ref"; then
166 # --depth option is ignored on http repository.
167 git fetch ${builder:+--progress} --depth 1 origin +"$ref" || return 1
168 git checkout -b "$branchName" FETCH_HEAD || return 1
169 else
170 return 1
171 fi
172}
173
174# Update submodules
175init_submodules(){
176 # Add urls into .git/config file
177 git submodule init
178
179 # list submodule directories and their hashes
180 git submodule status |
181 while read -r l; do
182 local hash
183 local dir
184 local name
185 local url
186
187 # checkout each submodule
188 hash=$(echo "$l" | awk '{print $1}' | tr -d '-')
189 dir=$(echo "$l" | sed -n 's/^.[0-9a-f]\+ \(.*[^)]*\)\( (.*)\)\?$/\1/p')
190 name=$(
191 git config -f .gitmodules --get-regexp submodule\..*\.path |
192 sed -n "s,^\(.*\)\.path $dir\$,\\1,p")
193 url=$(git config --get "${name}.url")
194
195 clone "$dir" "$url" "$hash" ""
196 done
197}
198
199clone(){
200 local top=$PWD
201 local dir="$1"
202 local url="$2"
203 local hash="$3"
204 local ref="$4"
205
206 cd "$dir"
207
208 # Initialize the repository.
209 init_remote "$url"
210
211 # Download data from the repository.
212 checkout_ref "$hash" "$ref" ||
213 checkout_hash "$hash" "$ref" || (
214 echo 1>&2 "Unable to checkout $hash$ref from $url."
215 exit 1
216 )
217
218 # Checkout linked sources.
219 if test -n "$fetchSubmodules"; then
220 init_submodules
221 fi
222
223 if [ -z "$builder" ] && [ -f .topdeps ]; then
224 if tg help &>/dev/null; then
225 echo "populating TopGit branches..."
226 tg remote --populate origin
227 else
228 echo "WARNING: would populate TopGit branches but TopGit is not available" >&2
229 echo "WARNING: install TopGit to fix the problem" >&2
230 fi
231 fi
232
233 cd "$top"
234}
235
236# Remove all remote branches, remove tags not reachable from HEAD, do a full
237# repack and then garbage collect unreferenced objects.
238make_deterministic_repo(){
239 local repo="$1"
240
241 # run in sub-shell to not touch current working directory
242 (
243 cd "$repo"
244 # Remove files that contain timestamps or otherwise have non-deterministic
245 # properties.
246 rm -rf .git/logs/ .git/hooks/ .git/index .git/FETCH_HEAD .git/ORIG_HEAD \
247 .git/refs/remotes/origin/HEAD .git/config
248
249 # Remove all remote branches.
250 git branch -r | while read -r branch; do
251 git branch -rD "$branch" >&2
252 done
253
254 # Remove tags not reachable from HEAD. If we're exactly on a tag, don't
255 # delete it.
256 maybe_tag=$(git tag --points-at HEAD)
257 git tag --contains HEAD | while read -r tag; do
258 if [ "$tag" != "$maybe_tag" ]; then
259 git tag -d "$tag" >&2
260 fi
261 done
262
263 # Do a full repack. Must run single-threaded, or else we lose determinism.
264 git config pack.threads 1
265 git repack -A -d -f
266 rm -f .git/config
267
268 # Garbage collect unreferenced objects.
269 # Note: --keep-largest-pack prevents non-deterministic ordering of packs
270 # listed in .git/objects/info/packs by only using a single pack
271 git gc --prune=all --keep-largest-pack
272 )
273}
274
275
276clone_user_rev() {
277 local dir="$1"
278 local url="$2"
279 local rev="${3:-HEAD}"
280
281 # Perform the checkout.
282 case "$rev" in
283 HEAD|refs/*)
284 clone "$dir" "$url" "" "$rev" 1>&2;;
285 *)
286 if test -z "$(echo "$rev" | tr -d 0123456789abcdef)"; then
287 clone "$dir" "$url" "$rev" "" 1>&2
288 else
289 # if revision is not hexadecimal it might be a tag
290 clone "$dir" "$url" "" "refs/tags/$rev" 1>&2
291 fi;;
292 esac
293
294 pushd "$dir" >/dev/null
295 fullRev=$( (git rev-parse "$rev" 2>/dev/null || git rev-parse "refs/heads/$branchName") | tail -n1)
296 humanReadableRev=$(git describe "$fullRev" 2> /dev/null || git describe --tags "$fullRev" 2> /dev/null || echo -- none --)
297 commitDate=$(git show -1 --no-patch --pretty=%ci "$fullRev")
298 commitDateStrict8601=$(git show -1 --no-patch --pretty=%cI "$fullRev")
299 popd >/dev/null
300
301 # Allow doing additional processing before .git removal
302 eval "$NIX_PREFETCH_GIT_CHECKOUT_HOOK"
303 if test -z "$leaveDotGit"; then
304 echo "removing \`.git'..." >&2
305 find "$dir" -name .git -print0 | xargs -0 rm -rf
306 else
307 find "$dir" -name .git | while read -r gitdir; do
308 make_deterministic_repo "$(readlink -f "$gitdir/..")"
309 done
310 fi
311}
312
313exit_handlers=()
314
315run_exit_handlers() {
316 exit_status=$?
317 for handler in "${exit_handlers[@]}"; do
318 eval "$handler $exit_status"
319 done
320}
321
322trap run_exit_handlers EXIT
323
324quiet_exit_handler() {
325 exec 2>&3 3>&-
326 if [ $1 -ne 0 ]; then
327 cat "$errfile" >&2
328 fi
329 rm -f "$errfile"
330}
331
332quiet_mode() {
333 errfile="$(mktemp "${TMPDIR:-/tmp}/git-checkout-err-XXXXXXXX")"
334 exit_handlers+=(quiet_exit_handler)
335 exec 3>&2 2>"$errfile"
336}
337
338json_escape() {
339 local s="$1"
340 s="${s//\\/\\\\}" # \
341 s="${s//\"/\\\"}" # "
342 s="${s//^H/\\\b}" # \b (backspace)
343 s="${s//^L/\\\f}" # \f (form feed)
344 s="${s//
345/\\\n}" # \n (newline)
346 s="${s//^M/\\\r}" # \r (carriage return)
347 s="${s// /\\t}" # \t (tab)
348 echo "$s"
349}
350
351print_results() {
352 hash="$1"
353 if ! test -n "$QUIET"; then
354 echo "" >&2
355 echo "git revision is $fullRev" >&2
356 if test -n "$finalPath"; then
357 echo "path is $finalPath" >&2
358 fi
359 echo "git human-readable version is $humanReadableRev" >&2
360 echo "Commit date is $commitDate" >&2
361 if test -n "$hash"; then
362 echo "hash is $hash" >&2
363 fi
364 fi
365 if test -n "$hash"; then
366 cat <<EOF
367{
368 "url": "$(json_escape "$url")",
369 "rev": "$(json_escape "$fullRev")",
370 "date": "$(json_escape "$commitDateStrict8601")",
371 "$(json_escape "$hashType")": "$(json_escape "$hash")",
372 "fetchSubmodules": $([[ -n "$fetchSubmodules" ]] && echo true || echo false)
373}
374EOF
375 fi
376}
377
378remove_tmpPath() {
379 rm -rf "$tmpPath"
380}
381
382if test -n "$QUIET"; then
383 quiet_mode
384fi
385
386if test -z "$branchName"; then
387 branchName=fetchgit
388fi
389
390if test -n "$builder"; then
391 test -n "$out" -a -n "$url" -a -n "$rev" || usage
392 mkdir -p "$out"
393 clone_user_rev "$out" "$url" "$rev"
394else
395 if test -z "$hashType"; then
396 hashType=sha256
397 fi
398
399 # If the hash was given, a file with that hash may already be in the
400 # store.
401 if test -n "$expHash"; then
402 finalPath=$(nix-store --print-fixed-path --recursive "$hashType" "$expHash" "$(url_to_name "$url" "$rev")")
403 if ! nix-store --check-validity "$finalPath" 2> /dev/null; then
404 finalPath=
405 fi
406 hash=$expHash
407 fi
408
409 # If we don't know the hash or a path with that hash doesn't exist,
410 # download the file and add it to the store.
411 if test -z "$finalPath"; then
412
413 tmpPath="$(mktemp -d "${TMPDIR:-/tmp}/git-checkout-tmp-XXXXXXXX")"
414 exit_handlers+=(remove_tmpPath)
415
416 tmpFile="$tmpPath/$(url_to_name "$url" "$rev")"
417 mkdir -p "$tmpFile"
418
419 # Perform the checkout.
420 clone_user_rev "$tmpFile" "$url" "$rev"
421
422 # Compute the hash.
423 hash=$(nix-hash --type $hashType --base32 "$tmpFile")
424
425 # Add the downloaded file to the Nix store.
426 finalPath=$(nix-store --add-fixed --recursive "$hashType" "$tmpFile")
427
428 if test -n "$expHash" -a "$expHash" != "$hash"; then
429 echo "hash mismatch for URL \`$url'. Got \`$hash'; expected \`$expHash'." >&2
430 exit 1
431 fi
432 fi
433
434 print_results "$hash"
435
436 if test -n "$PRINT_PATH"; then
437 echo "$finalPath"
438 fi
439fi