tjh.dev
nixpkgs mirror (for testing)
github.com/NixOS/nixpkgs
nix
1#! /usr/bin/env bash
2
3set -e -o pipefail
4
5url=
6rev=
7expHash=
8hashType=$NIX_HASH_ALGO
9deepClone=$NIX_PREFETCH_GIT_DEEP_CLONE
10leaveDotGit=$NIX_PREFETCH_GIT_LEAVE_DOT_GIT
11fetchSubmodules=
12fetchLFS=
13builder=
14fetchTags=
15branchName=$NIX_PREFETCH_GIT_BRANCH_NAME
16
17# ENV params
18out=${out:-}
19http_proxy=${http_proxy:-}
20
21# NOTE: use of NIX_GIT_SSL_CAINFO is for backwards compatibility; NIX_SSL_CERT_FILE is preferred
22# as of PR#303307
23GIT_SSL_CAINFO=${NIX_GIT_SSL_CAINFO:-$NIX_SSL_CERT_FILE}
24
25# populated by clone_user_rev()
26fullRev=
27humanReadableRev=
28commitDate=
29commitDateStrict8601=
30
31if test -n "$deepClone"; then
32 deepClone=true
33else
34 deepClone=
35fi
36
37if test "$leaveDotGit" != 1; then
38 leaveDotGit=
39else
40 leaveDotGit=true
41fi
42
43usage(){
44 echo >&2 "syntax: nix-prefetch-git [options] [URL [REVISION [EXPECTED-HASH]]]
45
46Options:
47 --out path Path where the output would be stored.
48 --url url Any url understood by 'git clone'.
49 --rev ref Any sha1 or references (such as refs/heads/master)
50 --hash h Expected hash.
51 --name n Symbolic store path name to use for the result (default: based on URL)
52 --branch-name Branch name to check out into
53 --sparse-checkout Only fetch and checkout part of the repository.
54 --non-cone-mode Use non-cone mode for sparse checkouts.
55 --deepClone Clone the entire repository.
56 --no-deepClone Make a shallow clone of just the required ref.
57 --leave-dotGit Keep the .git directories.
58 --fetch-lfs Fetch git Large File Storage (LFS) files.
59 --fetch-submodules Fetch submodules.
60 --fetch-tags Fetch all tags (useful for git describe).
61 --builder Clone as fetchgit does, but url, rev, and out option are mandatory.
62 --no-add-path Do not actually add the contents of the git repo to the store.
63 --root-dir dir Directory in the repository that will be copied to the output instead of the full repository.
64 --quiet Only print the final json summary.
65"
66 exit 1
67}
68
69# some git commands print to stdout, which would contaminate our JSON output
70clean_git(){
71 git "$@" >&2
72}
73
74argi=0
75argfun=""
76for arg; do
77 if test -z "$argfun"; then
78 case $arg in
79 --out) argfun=set_out;;
80 --url) argfun=set_url;;
81 --rev) argfun=set_rev;;
82 --hash) argfun=set_hashType;;
83 --name) argfun=set_symbolicName;;
84 --branch-name) argfun=set_branchName;;
85 --deepClone) deepClone=true;;
86 --sparse-checkout) argfun=set_sparseCheckout;;
87 --non-cone-mode) nonConeMode=true;;
88 --quiet) QUIET=true;;
89 --no-deepClone) deepClone=;;
90 --leave-dotGit) leaveDotGit=true;;
91 --fetch-lfs) fetchLFS=true;;
92 --fetch-submodules) fetchSubmodules=true;;
93 --fetch-tags) fetchTags=true;;
94 --builder) builder=true;;
95 --no-add-path) noAddPath=true;;
96 --root-dir) argfun=set_rootDir;;
97 -h|--help) usage; exit;;
98 *)
99 : $((++argi))
100 case $argi in
101 1) url=$arg;;
102 2) rev=$arg;;
103 3) expHash=$arg;;
104 *) exit 1;;
105 esac
106 ;;
107 esac
108 else
109 case $argfun in
110 set_*)
111 var=${argfun#set_}
112 eval "$var=$(printf %q "$arg")"
113 ;;
114 esac
115 argfun=""
116 fi
117done
118
119if test -z "$url"; then
120 usage
121fi
122
123init_remote(){
124 local url=$1
125 clean_git init --initial-branch=master
126 clean_git remote add origin "$url"
127 if [ -n "$sparseCheckout" ]; then
128 git config remote.origin.partialclonefilter "blob:none"
129 echo "$sparseCheckout" | git sparse-checkout set --stdin ${nonConeMode:+--no-cone}
130 fi
131 ( [ -n "$http_proxy" ] && clean_git config --global http.proxy "$http_proxy" ) || true
132 local proxy_pairs i
133 read -a proxy_pairs <<< "${FETCHGIT_HTTP_PROXIES:-}"
134 for ((i = 1; i < ${#proxy_pairs[@]}; i += 2)); do
135 clean_git config --global "http.${proxy_pairs[$i - 1]}.proxy" "${proxy_pairs[$i]}"
136 done
137}
138
139# Return the reference of an hash if it exists on the remote repository.
140ref_from_hash(){
141 local hash=$1
142 git ls-remote origin | sed -n "\,$hash\t, { s,\(.*\)\t\(.*\),\2,; p; q}"
143}
144
145# Return the hash of a reference if it exists on the remote repository.
146hash_from_ref(){
147 local ref=$1
148 git ls-remote origin | sed -n "\,\t$ref, { s,\(.*\)\t\(.*\),\1,; p; q}"
149}
150
151# Returns a name based on the url and reference
152#
153# This function needs to be in sync with nix's fetchgit implementation
154# of urlToName() to re-use the same nix store paths.
155url_to_name(){
156 local url=$1
157 local ref=$2
158 local base
159 base=$(basename "$url" .git | cut -d: -f2)
160
161 if test -n "$rootDir"; then
162 # Sanitize by removing leading dots and replacing all invalid character sequences with dashes.
163 # See sanitizeDerivationName in ../../../lib/strings.nix for reference.
164 echo "$base-$(sed -E 's/^\.+//;s/[^[:alnum:]+._?=-]+/-/g' <<< $rootDir)"
165 else
166 if [[ $ref =~ ^[a-z0-9]+$ ]]; then
167 echo "$base-${ref:0:7}"
168 else
169 echo "$base"
170 fi
171 fi
172}
173
174# Fetch and checkout the right sha1
175checkout_hash(){
176 local hash="$1"
177 local ref="$2"
178
179 if test -z "$hash"; then
180 hash=$(hash_from_ref "$ref")
181 fi
182
183 local -a fetchTagsArgs=()
184 # Avoid fetching all tags at clone time when not leaving .git,
185 # but keep/restore the behaviour before
186 # commit 7e085677f996 ("nix-prefetch-git: dont't fetch tags when deep clone unless leaving .git")
187 # to preserve the fragile hashes of existing .git sources.
188 if [[ -z "$leaveDotGit" ]]; then
189 fetchTagsArgs=(--no-tags)
190 elif [[ -n "$deepClone" ]]; then
191 fetchTagsArgs=(--tags)
192 fi
193 local -a fetchTargetArgs
194 if [[ -n "$deepClone" ]]; then
195 fetchTargetArgs=(origin)
196 else
197 fetchTargetArgs=(--depth=1 origin "$hash")
198 fi
199 if ! clean_git fetch "${fetchTagsArgs[@]}" ${builder:+--progress} "${fetchTargetArgs[@]}"; then
200 echo "ERROR: \`git fetch' failed." >&2
201 # Git remotes using the "dumb" protocol does not support shallow fetch;
202 # fall back to deep fetch if shallow fetch failed.
203 # TODO(@ShamrockLee): Determine whether the transfer protocol is smart reliably.
204 if [[ -z "$deepClone" ]] || [[ -z "$fetchTags" ]]; then
205 echo "This might be due to the dumb transfer protocol not supporting shallow fetch or no-tag cloning. Trying with \`--tags' and without \`--depth=1'..." >&2
206 clean_git fetch --tags ${builder:+--progress} origin || return 1
207 else
208 return 1
209 fi
210 fi
211
212 local object_type=$(git cat-file -t "$hash")
213 if [[ "$object_type" == "commit" || "$object_type" == "tag" ]]; then
214 clean_git checkout -b "$branchName" "$hash" || return 1
215 elif [[ "$object_type" == "tree" ]]; then
216 clean_git config user.email "nix-prefetch-git@localhost"
217 clean_git config user.name "nix-prefetch-git"
218 local commit_id=$(git commit-tree "$hash" -m "Commit created from tree hash $hash")
219 clean_git checkout -b "$branchName" "$commit_id" || return 1
220 else
221 echo "Unrecognized git object type: $object_type"
222 return 1
223 fi
224}
225
226# Fetch only a branch/tag and checkout it.
227checkout_ref(){
228 local hash="$1"
229 local ref="$2"
230
231 if [[ -n "$deepClone" ]]; then
232 # The caller explicitly asked for a deep clone. Deep clones
233 # allow "git describe" and similar tools to work. See
234 # https://marc.info/?l=nix-dev&m=139641582514772
235 # for a discussion.
236 return 1
237 fi
238
239 if test -z "$ref"; then
240 ref=$(ref_from_hash "$hash")
241 fi
242
243 if test -n "$ref"; then
244 # --depth option is ignored on http repository.
245 clean_git fetch ${builder:+--progress} --depth 1 origin +"$ref" || return 1
246 clean_git checkout -b "$branchName" FETCH_HEAD || return 1
247 else
248 return 1
249 fi
250}
251
252# Update submodules
253init_submodules(){
254 # shallow with leaveDotGit will change hashes
255 [[ -z "$deepClone" ]] && [[ -z "$leaveDotGit" ]] && \
256 clean_git submodule update --init --recursive --checkout -j ${NIX_BUILD_CORES:-1} --progress --depth 1 || \
257 clean_git submodule update --init --recursive --checkout -j ${NIX_BUILD_CORES:-1} --progress
258}
259
260clone(){
261 local top=$PWD
262 local dir="$1"
263 local url="$2"
264 local hash="$3"
265 local ref="$4"
266
267 cd "$dir"
268
269 # Initialize the repository.
270 init_remote "$url"
271
272 # Download data from the repository.
273 checkout_ref "$hash" "$ref" ||
274 checkout_hash "$hash" "$ref" || (
275 echo 1>&2 "Unable to checkout $hash$ref from $url."
276 exit 1
277 )
278
279 # Fetch all tags if requested
280 # The fetched tags are potentially non-reproducible, as tags are mutable parts of the Git tree.
281 if [[ -n "$fetchTags" ]]; then
282 echo "fetching all tags..." >&2
283 clean_git fetch origin 'refs/tags/*:refs/tags/*' || echo "warning: failed to fetch some tags" >&2
284 fi
285
286 # Name tag "$ref" to make `git describe` work reproducibly in `NIX_PREFETCH_GIT_CHECKOUT_HOOK`.
287 # Name only when not leaving `.git` for compatibility purposes.
288 if [[ "$ref" =~ ^refs/tags/ ]] && [[ -z "$leaveDotGit" ]]; then
289 echo "refer to FETCH_HEAD as its original name $ref"
290 clean_git update-ref "$ref" FETCH_HEAD
291 fi
292
293 # Checkout linked sources.
294 if test -n "$fetchSubmodules"; then
295 init_submodules
296 fi
297
298 if [ -z "$builder" ] && [ -f .topdeps ]; then
299 if tg help &>/dev/null; then
300 echo "populating TopGit branches..."
301 tg remote --populate origin
302 else
303 echo "WARNING: would populate TopGit branches but TopGit is not available" >&2
304 echo "WARNING: install TopGit to fix the problem" >&2
305 fi
306 fi
307
308 cd "$top"
309}
310
311# Remove all remote branches, remove tags not reachable from HEAD, do a full
312# repack and then garbage collect unreferenced objects.
313make_deterministic_repo(){
314 local repo="$1"
315
316 # run in sub-shell to not touch current working directory
317 (
318 cd "$repo"
319 # Remove files that contain timestamps or otherwise have non-deterministic
320 # properties.
321 if [ -f .git ]; then
322 local dotgit_content=$(<.git)
323 local dotgit_dir="${dotgit_content#gitdir: }"
324 else
325 local dotgit_dir=".git"
326 fi
327 pushd "$dotgit_dir" >/dev/null
328 rm -rf logs/ hooks/ index FETCH_HEAD ORIG_HEAD refs/remotes/origin/HEAD config
329 popd >/dev/null
330 # Remove all remote branches.
331 git branch -r | while read -r branch; do
332 clean_git branch -rD "$branch"
333 done
334
335 # Remove tags not reachable from HEAD. If we're exactly on a tag, don't
336 # delete it.
337 maybe_tag=$(git tag --points-at HEAD)
338 git tag --contains HEAD | while read -r tag; do
339 if [ "$tag" != "$maybe_tag" ]; then
340 clean_git tag -d "$tag"
341 fi
342 done
343
344 # Do a full repack. Must run single-threaded, or else we lose determinism.
345 clean_git config pack.threads 1
346 clean_git repack -A -d -f
347 rm -f "$dotgit_dir/config"
348
349 # Garbage collect unreferenced objects.
350 # Note: --keep-largest-pack prevents non-deterministic ordering of packs
351 # listed in .git/objects/info/packs by only using a single pack
352 clean_git gc --prune=all --keep-largest-pack
353 )
354}
355
356
357clone_user_rev() {
358 local dir="$1"
359 local url="$2"
360 local rev="${3:-HEAD}"
361
362 if [ -n "$fetchLFS" ]; then
363 clean_git lfs install
364 fi
365
366 # Perform the checkout.
367 case "$rev" in
368 HEAD|refs/*)
369 clone "$dir" "$url" "" "$rev" 1>&2;;
370 *)
371 if test -z "$(echo "$rev" | tr -d 0123456789abcdef)"; then
372 clone "$dir" "$url" "$rev" "" 1>&2
373 else
374 # if revision is not hexadecimal it might be a tag
375 clone "$dir" "$url" "" "refs/tags/$rev" 1>&2
376 fi;;
377 esac
378
379 pushd "$dir" >/dev/null
380 fullRev=$( (git rev-parse "$rev" 2>/dev/null || git rev-parse "refs/heads/$branchName") | tail -n1)
381 humanReadableRev=$(git describe "$fullRev" 2> /dev/null || git describe --tags "$fullRev" 2> /dev/null || echo -- none --)
382 commitDate=$(git show -1 --no-patch --pretty=%ci "$fullRev")
383 commitDateStrict8601=$(git show -1 --no-patch --pretty=%cI "$fullRev")
384 popd >/dev/null
385
386 # Allow doing additional processing before .git removal
387 eval "$NIX_PREFETCH_GIT_CHECKOUT_HOOK"
388 if test -z "$leaveDotGit"; then
389 echo "removing \`.git'..." >&2
390 find "$dir" -name .git -print0 | xargs -0 rm -rf
391 else
392 find "$dir" -name .git | while read -r gitdir; do
393 make_deterministic_repo "$(readlink -f "$(dirname "$gitdir")")"
394 done
395 fi
396}
397
398clone_user_rev_to_tmpfile(){
399 local url="$1"
400 local rev="${2:-HEAD}"
401
402 # nix>=2.20 rejects adding symlinked paths to the store, so use realpath
403 # to resolve to a physical path. https://github.com/NixOS/nix/issues/11941
404 tmpPath="$(realpath "$(mktemp -d --tmpdir git-checkout-tmp-XXXXXXXX)")"
405 exit_handlers+=(remove_tmpPath)
406
407 tmpOut="$tmpPath/out/$storePathName"
408 tmpClone="$tmpPath/clone"
409 mkdir -p "$tmpPath/out" "$tmpClone"
410
411 # Perform the checkout.
412 clone_user_rev "$tmpClone" "$url" "$rev"
413}
414
415exit_handlers=()
416
417run_exit_handlers() {
418 exit_status=$?
419 for handler in "${exit_handlers[@]}"; do
420 eval "$handler $exit_status"
421 done
422}
423
424trap run_exit_handlers EXIT
425
426quiet_exit_handler() {
427 exec 2>&3 3>&-
428 if [ $1 -ne 0 ]; then
429 cat "$errfile" >&2
430 fi
431 rm -f "$errfile"
432}
433
434quiet_mode() {
435 errfile="$(mktemp "${TMPDIR:-/tmp}/git-checkout-err-XXXXXXXX")"
436 exit_handlers+=(quiet_exit_handler)
437 exec 3>&2 2>"$errfile"
438}
439
440json_escape() {
441 local s="$1"
442 s="${s//\\/\\\\}" # \
443 s="${s//\"/\\\"}" # "
444 s="${s//^H/\\\b}" # \b (backspace)
445 s="${s//^L/\\\f}" # \f (form feed)
446 s="${s//
447/\\\n}" # \n (newline)
448 s="${s//^M/\\\r}" # \r (carriage return)
449 s="${s// /\\t}" # \t (tab)
450 echo "$s"
451}
452
453print_results() {
454 hash="$1"
455 if ! test -n "$QUIET"; then
456 echo "" >&2
457 echo "git revision is $fullRev" >&2
458 if test -n "$finalPath"; then
459 echo "path is $finalPath" >&2
460 fi
461 echo "git human-readable version is $humanReadableRev" >&2
462 echo "Commit date is $commitDate" >&2
463 if test -n "$hash"; then
464 echo "hash is $hash" >&2
465 fi
466 fi
467 if test -n "$hash"; then
468 cat <<EOF
469{
470 "url": "$(json_escape "$url")",
471 "rev": "$(json_escape "$fullRev")",
472 "date": "$(json_escape "$commitDateStrict8601")",
473 "path": "$(json_escape "$finalPath")",
474 "$(json_escape "$hashType")": "$(json_escape "$hash")",
475 "hash": "$(nix-hash --to-sri --type $hashType $hash)",
476 "fetchLFS": $([[ -n "$fetchLFS" ]] && echo true || echo false),
477 "fetchSubmodules": $([[ -n "$fetchSubmodules" ]] && echo true || echo false),
478 "deepClone": $([[ -n "$deepClone" ]] && echo true || echo false),
479 "fetchTags": $([[ -n "$fetchTags" ]] && echo true || echo false),
480 "leaveDotGit": $([[ -n "$leaveDotGit" ]] && echo true || echo false),
481 "rootDir": "$(json_escape "$rootDir")"
482}
483EOF
484 fi
485}
486
487remove_tmpPath() {
488 rm -rf "$tmpPath"
489}
490
491remove_tmpHomePath() {
492 chmod -R u+w "$tmpHomePath"
493 rm -rf "$tmpHomePath"
494}
495
496if test -n "$QUIET"; then
497 quiet_mode
498fi
499
500if test -z "$branchName"; then
501 branchName=fetchgit
502fi
503
504if [ -v symbolicName ]; then
505 storePathName="$symbolicName"
506else
507 storePathName="$(url_to_name "$url" "$rev")"
508fi
509
510tmpHomePath="$(mktemp -d "${TMPDIR:-/tmp}/nix-prefetch-git-tmp-home-XXXXXXXXXX")"
511exit_handlers+=(remove_tmpHomePath)
512ln -s "${NETRC:-$HOME/.netrc}" "$tmpHomePath/.netrc"
513HOME="$tmpHomePath"
514unset XDG_CONFIG_HOME
515export GIT_CONFIG_NOSYSTEM=1
516
517if test -n "$builder"; then
518 test -n "$out" -a -n "$url" -a -n "$rev" || usage
519 if test -n "$rootDir"; then
520 clone_user_rev_to_tmpfile "$url" "$rev"
521 mv "$tmpClone/$rootDir" "$out"
522 else
523 mkdir -p "$out"
524 clone_user_rev "$out" "$url" "$rev"
525 fi
526else
527 if test -z "$hashType"; then
528 hashType=sha256
529 fi
530
531 # If the hash was given, a file with that hash may already be in the
532 # store.
533 if test -n "$expHash"; then
534 finalPath=$(nix-store --print-fixed-path --recursive "$hashType" "$expHash" "$storePathName")
535 if ! nix-store --check-validity "$finalPath" 2> /dev/null; then
536 finalPath=
537 fi
538 hash=$expHash
539 fi
540
541 # If we don't know the hash or a path with that hash doesn't exist,
542 # download the file and add it to the store.
543 if test -z "$finalPath"; then
544 clone_user_rev_to_tmpfile "$url" "$rev"
545
546 if test -z "$rootDir"; then
547 mv "$tmpClone" "$tmpOut"
548 else
549 mv "$tmpClone/$rootDir" "$tmpOut"
550 fi
551
552 # Compute the hash.
553 hash=$(nix-hash --type $hashType --base32 "$tmpOut")
554
555 # Add the downloaded file to the Nix store.
556
557 if test -z "$noAddPath"; then
558 finalPath=$(nix-store --add-fixed --recursive "$hashType" "$tmpOut") \
559 || { printf "maybe try again with \`nix-prefetch-git --no-add-path <...>' ?\n" >&2; exit 1; }
560 else
561 printf "the path for \`%s' has NOT been added to the store\n" "$url" >&2
562 finalPath=$(nix-store --print-fixed-path --recursive "$hashType" "$hash" "$storePathName")
563 fi
564
565 if test -n "$expHash" -a "$expHash" != "$hash"; then
566 echo "hash mismatch for URL \`$url'. Got \`$hash'; expected \`$expHash'." >&2
567 exit 1
568 fi
569 fi
570
571 print_results "$hash"
572
573 if test -n "$PRINT_PATH"; then
574 echo "$finalPath"
575 fi
576fi