tjh.dev
1#! /usr/bin/env bash
2
3set -e -o pipefail
4
5url=
6rev=
7expHash=
8hashType=$NIX_HASH_ALGO
9deepClone=$NIX_PREFETCH_GIT_DEEP_CLONE
10leaveDotGit=$NIX_PREFETCH_GIT_LEAVE_DOT_GIT
11fetchSubmodules=
12fetchLFS=
13builder=
14branchName=$NIX_PREFETCH_GIT_BRANCH_NAME
15
16# ENV params
17out=${out:-}
18http_proxy=${http_proxy:-}
19
20# NOTE: use of NIX_GIT_SSL_CAINFO is for backwards compatibility; NIX_SSL_CERT_FILE is preferred
21# as of PR#303307
22GIT_SSL_CAINFO=${NIX_GIT_SSL_CAINFO:-$NIX_SSL_CERT_FILE}
23
24# populated by clone_user_rev()
25fullRev=
26humanReadableRev=
27commitDate=
28commitDateStrict8601=
29
30if test -n "$deepClone"; then
31 deepClone=true
32else
33 deepClone=
34fi
35
36if test "$leaveDotGit" != 1; then
37 leaveDotGit=
38else
39 leaveDotGit=true
40fi
41
42usage(){
43 echo >&2 "syntax: nix-prefetch-git [options] [URL [REVISION [EXPECTED-HASH]]]
44
45Options:
46 --out path Path where the output would be stored.
47 --url url Any url understood by 'git clone'.
48 --rev ref Any sha1 or references (such as refs/heads/master)
49 --hash h Expected hash.
50 --branch-name Branch name to check out into
51 --sparse-checkout Only fetch and checkout part of the repository.
52 --non-cone-mode Use non-cone mode for sparse checkouts.
53 --deepClone Clone the entire repository.
54 --no-deepClone Make a shallow clone of just the required ref.
55 --leave-dotGit Keep the .git directories.
56 --fetch-lfs Fetch git Large File Storage (LFS) files.
57 --fetch-submodules Fetch submodules.
58 --builder Clone as fetchgit does, but url, rev, and out option are mandatory.
59 --quiet Only print the final json summary.
60"
61 exit 1
62}
63
64# some git commands print to stdout, which would contaminate our JSON output
65clean_git(){
66 git "$@" >&2
67}
68
69argi=0
70argfun=""
71for arg; do
72 if test -z "$argfun"; then
73 case $arg in
74 --out) argfun=set_out;;
75 --url) argfun=set_url;;
76 --rev) argfun=set_rev;;
77 --hash) argfun=set_hashType;;
78 --branch-name) argfun=set_branchName;;
79 --deepClone) deepClone=true;;
80 --sparse-checkout) argfun=set_sparseCheckout;;
81 --non-cone-mode) nonConeMode=true;;
82 --quiet) QUIET=true;;
83 --no-deepClone) deepClone=;;
84 --leave-dotGit) leaveDotGit=true;;
85 --fetch-lfs) fetchLFS=true;;
86 --fetch-submodules) fetchSubmodules=true;;
87 --builder) builder=true;;
88 -h|--help) usage; exit;;
89 *)
90 : $((++argi))
91 case $argi in
92 1) url=$arg;;
93 2) rev=$arg;;
94 3) expHash=$arg;;
95 *) exit 1;;
96 esac
97 ;;
98 esac
99 else
100 case $argfun in
101 set_*)
102 var=${argfun#set_}
103 eval "$var=$(printf %q "$arg")"
104 ;;
105 esac
106 argfun=""
107 fi
108done
109
110if test -z "$url"; then
111 usage
112fi
113
114
115init_remote(){
116 local url=$1
117 clean_git init --initial-branch=master
118 clean_git remote add origin "$url"
119 if [ -n "$sparseCheckout" ]; then
120 git config remote.origin.partialclonefilter "blob:none"
121 echo "$sparseCheckout" | git sparse-checkout set --stdin ${nonConeMode:+--no-cone}
122 fi
123 ( [ -n "$http_proxy" ] && clean_git config --global http.proxy "$http_proxy" ) || true
124}
125
126# Return the reference of an hash if it exists on the remote repository.
127ref_from_hash(){
128 local hash=$1
129 git ls-remote origin | sed -n "\,$hash\t, { s,\(.*\)\t\(.*\),\2,; p; q}"
130}
131
132# Return the hash of a reference if it exists on the remote repository.
133hash_from_ref(){
134 local ref=$1
135 git ls-remote origin | sed -n "\,\t$ref, { s,\(.*\)\t\(.*\),\1,; p; q}"
136}
137
138# Returns a name based on the url and reference
139#
140# This function needs to be in sync with nix's fetchgit implementation
141# of urlToName() to re-use the same nix store paths.
142url_to_name(){
143 local url=$1
144 local ref=$2
145 local base
146 base=$(basename "$url" .git | cut -d: -f2)
147
148 if [[ $ref =~ ^[a-z0-9]+$ ]]; then
149 echo "$base-${ref:0:7}"
150 else
151 echo "$base"
152 fi
153}
154
155# Fetch and checkout the right sha1
156checkout_hash(){
157 local hash="$1"
158 local ref="$2"
159
160 if test -z "$hash"; then
161 hash=$(hash_from_ref "$ref")
162 fi
163
164 [[ -z "$deepClone" ]] && \
165 clean_git fetch ${builder:+--progress} --depth=1 origin "$hash" || \
166 clean_git fetch -t ${builder:+--progress} origin || return 1
167
168 local object_type=$(git cat-file -t "$hash")
169 if [[ "$object_type" == "commit" || "$object_type" == "tag" ]]; then
170 clean_git checkout -b "$branchName" "$hash" || return 1
171 elif [[ "$object_type" == "tree" ]]; then
172 clean_git config user.email "nix-prefetch-git@localhost"
173 clean_git config user.name "nix-prefetch-git"
174 local commit_id=$(git commit-tree "$hash" -m "Commit created from tree hash $hash")
175 clean_git checkout -b "$branchName" "$commit_id" || return 1
176 else
177 echo "Unrecognized git object type: $object_type"
178 return 1
179 fi
180}
181
182# Fetch only a branch/tag and checkout it.
183checkout_ref(){
184 local hash="$1"
185 local ref="$2"
186
187 if [[ -n "$deepClone" ]]; then
188 # The caller explicitly asked for a deep clone. Deep clones
189 # allow "git describe" and similar tools to work. See
190 # https://marc.info/?l=nix-dev&m=139641582514772
191 # for a discussion.
192 return 1
193 fi
194
195 if test -z "$ref"; then
196 ref=$(ref_from_hash "$hash")
197 fi
198
199 if test -n "$ref"; then
200 # --depth option is ignored on http repository.
201 clean_git fetch ${builder:+--progress} --depth 1 origin +"$ref" || return 1
202 clean_git checkout -b "$branchName" FETCH_HEAD || return 1
203 else
204 return 1
205 fi
206}
207
208# Update submodules
209init_submodules(){
210 # shallow with leaveDotGit will change hashes
211 [[ -z "$deepClone" ]] && [[ -z "$leaveDotGit" ]] && \
212 clean_git submodule update --init --recursive -j ${NIX_BUILD_CORES:-1} --progress --depth 1 || \
213 clean_git submodule update --init --recursive -j ${NIX_BUILD_CORES:-1} --progress
214}
215
216clone(){
217 local top=$PWD
218 local dir="$1"
219 local url="$2"
220 local hash="$3"
221 local ref="$4"
222
223 cd "$dir"
224
225 # Initialize the repository.
226 init_remote "$url"
227
228 # Download data from the repository.
229 checkout_ref "$hash" "$ref" ||
230 checkout_hash "$hash" "$ref" || (
231 echo 1>&2 "Unable to checkout $hash$ref from $url."
232 exit 1
233 )
234
235 # Checkout linked sources.
236 if test -n "$fetchSubmodules"; then
237 init_submodules
238 fi
239
240 if [ -z "$builder" ] && [ -f .topdeps ]; then
241 if tg help &>/dev/null; then
242 echo "populating TopGit branches..."
243 tg remote --populate origin
244 else
245 echo "WARNING: would populate TopGit branches but TopGit is not available" >&2
246 echo "WARNING: install TopGit to fix the problem" >&2
247 fi
248 fi
249
250 cd "$top"
251}
252
253# Remove all remote branches, remove tags not reachable from HEAD, do a full
254# repack and then garbage collect unreferenced objects.
255make_deterministic_repo(){
256 local repo="$1"
257
258 # run in sub-shell to not touch current working directory
259 (
260 cd "$repo"
261 # Remove files that contain timestamps or otherwise have non-deterministic
262 # properties.
263 if [ -f .git ]; then
264 local dotgit_content=$(<.git)
265 local dotgit_dir="${dotgit_content#gitdir: }"
266 else
267 local dotgit_dir=".git"
268 fi
269 pushd "$dotgit_dir" >/dev/null
270 rm -rf logs/ hooks/ index FETCH_HEAD ORIG_HEAD refs/remotes/origin/HEAD config
271 popd >/dev/null
272 # Remove all remote branches.
273 git branch -r | while read -r branch; do
274 clean_git branch -rD "$branch"
275 done
276
277 # Remove tags not reachable from HEAD. If we're exactly on a tag, don't
278 # delete it.
279 maybe_tag=$(git tag --points-at HEAD)
280 git tag --contains HEAD | while read -r tag; do
281 if [ "$tag" != "$maybe_tag" ]; then
282 clean_git tag -d "$tag"
283 fi
284 done
285
286 # Do a full repack. Must run single-threaded, or else we lose determinism.
287 clean_git config pack.threads 1
288 clean_git repack -A -d -f
289 rm -f "$dotgit_dir/config"
290
291 # Garbage collect unreferenced objects.
292 # Note: --keep-largest-pack prevents non-deterministic ordering of packs
293 # listed in .git/objects/info/packs by only using a single pack
294 clean_git gc --prune=all --keep-largest-pack
295 )
296}
297
298
299clone_user_rev() {
300 local dir="$1"
301 local url="$2"
302 local rev="${3:-HEAD}"
303
304 if [ -n "$fetchLFS" ]; then
305 clean_git lfs install
306 fi
307
308 # Perform the checkout.
309 case "$rev" in
310 HEAD|refs/*)
311 clone "$dir" "$url" "" "$rev" 1>&2;;
312 *)
313 if test -z "$(echo "$rev" | tr -d 0123456789abcdef)"; then
314 clone "$dir" "$url" "$rev" "" 1>&2
315 else
316 # if revision is not hexadecimal it might be a tag
317 clone "$dir" "$url" "" "refs/tags/$rev" 1>&2
318 fi;;
319 esac
320
321 pushd "$dir" >/dev/null
322 fullRev=$( (git rev-parse "$rev" 2>/dev/null || git rev-parse "refs/heads/$branchName") | tail -n1)
323 humanReadableRev=$(git describe "$fullRev" 2> /dev/null || git describe --tags "$fullRev" 2> /dev/null || echo -- none --)
324 commitDate=$(git show -1 --no-patch --pretty=%ci "$fullRev")
325 commitDateStrict8601=$(git show -1 --no-patch --pretty=%cI "$fullRev")
326 popd >/dev/null
327
328 # Allow doing additional processing before .git removal
329 eval "$NIX_PREFETCH_GIT_CHECKOUT_HOOK"
330 if test -z "$leaveDotGit"; then
331 echo "removing \`.git'..." >&2
332 find "$dir" -name .git -print0 | xargs -0 rm -rf
333 else
334 find "$dir" -name .git | while read -r gitdir; do
335 make_deterministic_repo "$(readlink -f "$(dirname "$gitdir")")"
336 done
337 fi
338}
339
340exit_handlers=()
341
342run_exit_handlers() {
343 exit_status=$?
344 for handler in "${exit_handlers[@]}"; do
345 eval "$handler $exit_status"
346 done
347}
348
349trap run_exit_handlers EXIT
350
351quiet_exit_handler() {
352 exec 2>&3 3>&-
353 if [ $1 -ne 0 ]; then
354 cat "$errfile" >&2
355 fi
356 rm -f "$errfile"
357}
358
359quiet_mode() {
360 errfile="$(mktemp "${TMPDIR:-/tmp}/git-checkout-err-XXXXXXXX")"
361 exit_handlers+=(quiet_exit_handler)
362 exec 3>&2 2>"$errfile"
363}
364
365json_escape() {
366 local s="$1"
367 s="${s//\\/\\\\}" # \
368 s="${s//\"/\\\"}" # "
369 s="${s//^H/\\\b}" # \b (backspace)
370 s="${s//^L/\\\f}" # \f (form feed)
371 s="${s//
372/\\\n}" # \n (newline)
373 s="${s//^M/\\\r}" # \r (carriage return)
374 s="${s// /\\t}" # \t (tab)
375 echo "$s"
376}
377
378print_results() {
379 hash="$1"
380 if ! test -n "$QUIET"; then
381 echo "" >&2
382 echo "git revision is $fullRev" >&2
383 if test -n "$finalPath"; then
384 echo "path is $finalPath" >&2
385 fi
386 echo "git human-readable version is $humanReadableRev" >&2
387 echo "Commit date is $commitDate" >&2
388 if test -n "$hash"; then
389 echo "hash is $hash" >&2
390 fi
391 fi
392 if test -n "$hash"; then
393 cat <<EOF
394{
395 "url": "$(json_escape "$url")",
396 "rev": "$(json_escape "$fullRev")",
397 "date": "$(json_escape "$commitDateStrict8601")",
398 "path": "$(json_escape "$finalPath")",
399 "$(json_escape "$hashType")": "$(json_escape "$hash")",
400 "hash": "$(nix-hash --to-sri --type $hashType $hash)",
401 "fetchLFS": $([[ -n "$fetchLFS" ]] && echo true || echo false),
402 "fetchSubmodules": $([[ -n "$fetchSubmodules" ]] && echo true || echo false),
403 "deepClone": $([[ -n "$deepClone" ]] && echo true || echo false),
404 "leaveDotGit": $([[ -n "$leaveDotGit" ]] && echo true || echo false)
405}
406EOF
407 fi
408}
409
410remove_tmpPath() {
411 rm -rf "$tmpPath"
412}
413
414remove_tmpHomePath() {
415 chmod -R u+w "$tmpHomePath"
416 rm -rf "$tmpHomePath"
417}
418
419if test -n "$QUIET"; then
420 quiet_mode
421fi
422
423if test -z "$branchName"; then
424 branchName=fetchgit
425fi
426
427tmpHomePath="$(mktemp -d "${TMPDIR:-/tmp}/nix-prefetch-git-tmp-home-XXXXXXXXXX")"
428exit_handlers+=(remove_tmpHomePath)
429ln -s "${NETRC:-$HOME/.netrc}" "$tmpHomePath/.netrc"
430HOME="$tmpHomePath"
431unset XDG_CONFIG_HOME
432export GIT_CONFIG_NOSYSTEM=1
433
434if test -n "$builder"; then
435 test -n "$out" -a -n "$url" -a -n "$rev" || usage
436 mkdir -p "$out"
437 clone_user_rev "$out" "$url" "$rev"
438else
439 if test -z "$hashType"; then
440 hashType=sha256
441 fi
442
443 # If the hash was given, a file with that hash may already be in the
444 # store.
445 if test -n "$expHash"; then
446 finalPath=$(nix-store --print-fixed-path --recursive "$hashType" "$expHash" "$(url_to_name "$url" "$rev")")
447 if ! nix-store --check-validity "$finalPath" 2> /dev/null; then
448 finalPath=
449 fi
450 hash=$expHash
451 fi
452
453 # If we don't know the hash or a path with that hash doesn't exist,
454 # download the file and add it to the store.
455 if test -z "$finalPath"; then
456 # nix>=2.20 rejects adding symlinked paths to the store, so use realpath
457 # to resolve to a physical path. https://github.com/NixOS/nix/issues/11941
458 tmpPath="$(realpath "$(mktemp -d --tmpdir git-checkout-tmp-XXXXXXXX)")"
459 exit_handlers+=(remove_tmpPath)
460
461 tmpFile="$tmpPath/$(url_to_name "$url" "$rev")"
462 mkdir -p "$tmpFile"
463
464 # Perform the checkout.
465 clone_user_rev "$tmpFile" "$url" "$rev"
466
467 # Compute the hash.
468 hash=$(nix-hash --type $hashType --base32 "$tmpFile")
469
470 # Add the downloaded file to the Nix store.
471 finalPath=$(nix-store --add-fixed --recursive "$hashType" "$tmpFile")
472
473 if test -n "$expHash" -a "$expHash" != "$hash"; then
474 echo "hash mismatch for URL \`$url'. Got \`$hash'; expected \`$expHash'." >&2
475 exit 1
476 fi
477 fi
478
479 print_results "$hash"
480
481 if test -n "$PRINT_PATH"; then
482 echo "$finalPath"
483 fi
484fi