tjh.dev
Clone of https://github.com/NixOS/nixpkgs.git (to stress-test knotserver)
1#! /usr/bin/env bash
2
3set -e -o pipefail
4
5url=
6rev=
7expHash=
8hashType=$NIX_HASH_ALGO
9deepClone=$NIX_PREFETCH_GIT_DEEP_CLONE
10leaveDotGit=$NIX_PREFETCH_GIT_LEAVE_DOT_GIT
11fetchSubmodules=
12fetchLFS=
13builder=
14fetchTags=
15branchName=$NIX_PREFETCH_GIT_BRANCH_NAME
16
17# ENV params
18out=${out:-}
19http_proxy=${http_proxy:-}
20
21# NOTE: use of NIX_GIT_SSL_CAINFO is for backwards compatibility; NIX_SSL_CERT_FILE is preferred
22# as of PR#303307
23GIT_SSL_CAINFO=${NIX_GIT_SSL_CAINFO:-$NIX_SSL_CERT_FILE}
24
25# populated by clone_user_rev()
26fullRev=
27humanReadableRev=
28commitDate=
29commitDateStrict8601=
30
31if test -n "$deepClone"; then
32 deepClone=true
33else
34 deepClone=
35fi
36
37if test "$leaveDotGit" != 1; then
38 leaveDotGit=
39else
40 leaveDotGit=true
41fi
42
43usage(){
44 echo >&2 "syntax: nix-prefetch-git [options] [URL [REVISION [EXPECTED-HASH]]]
45
46Options:
47 --out path Path where the output would be stored.
48 --url url Any url understood by 'git clone'.
49 --rev ref Any sha1 or references (such as refs/heads/master)
50 --hash h Expected hash.
51 --name n Symbolic store path name to use for the result (default: based on URL)
52 --branch-name Branch name to check out into
53 --sparse-checkout Only fetch and checkout part of the repository.
54 --non-cone-mode Use non-cone mode for sparse checkouts.
55 --deepClone Clone the entire repository.
56 --no-deepClone Make a shallow clone of just the required ref.
57 --leave-dotGit Keep the .git directories.
58 --fetch-lfs Fetch git Large File Storage (LFS) files.
59 --fetch-submodules Fetch submodules.
60 --fetch-tags Fetch all tags (useful for git describe).
61 --builder Clone as fetchgit does, but url, rev, and out option are mandatory.
62 --quiet Only print the final json summary.
63"
64 exit 1
65}
66
67# some git commands print to stdout, which would contaminate our JSON output
68clean_git(){
69 git "$@" >&2
70}
71
72argi=0
73argfun=""
74for arg; do
75 if test -z "$argfun"; then
76 case $arg in
77 --out) argfun=set_out;;
78 --url) argfun=set_url;;
79 --rev) argfun=set_rev;;
80 --hash) argfun=set_hashType;;
81 --name) argfun=set_symbolicName;;
82 --branch-name) argfun=set_branchName;;
83 --deepClone) deepClone=true;;
84 --sparse-checkout) argfun=set_sparseCheckout;;
85 --non-cone-mode) nonConeMode=true;;
86 --quiet) QUIET=true;;
87 --no-deepClone) deepClone=;;
88 --leave-dotGit) leaveDotGit=true;;
89 --fetch-lfs) fetchLFS=true;;
90 --fetch-submodules) fetchSubmodules=true;;
91 --fetch-tags) fetchTags=true;;
92 --builder) builder=true;;
93 -h|--help) usage; exit;;
94 *)
95 : $((++argi))
96 case $argi in
97 1) url=$arg;;
98 2) rev=$arg;;
99 3) expHash=$arg;;
100 *) exit 1;;
101 esac
102 ;;
103 esac
104 else
105 case $argfun in
106 set_*)
107 var=${argfun#set_}
108 eval "$var=$(printf %q "$arg")"
109 ;;
110 esac
111 argfun=""
112 fi
113done
114
115if test -z "$url"; then
116 usage
117fi
118
119
120init_remote(){
121 local url=$1
122 clean_git init --initial-branch=master
123 clean_git remote add origin "$url"
124 if [ -n "$sparseCheckout" ]; then
125 git config remote.origin.partialclonefilter "blob:none"
126 echo "$sparseCheckout" | git sparse-checkout set --stdin ${nonConeMode:+--no-cone}
127 fi
128 ( [ -n "$http_proxy" ] && clean_git config --global http.proxy "$http_proxy" ) || true
129 local proxy_pairs i
130 read -a proxy_pairs <<< "${FETCHGIT_HTTP_PROXIES:-}"
131 for ((i = 1; i < ${#proxy_pairs[@]}; i += 2)); do
132 clean_git config --global "http.${proxy_pairs[$i - 1]}.proxy" "${proxy_pairs[$i]}"
133 done
134}
135
136# Return the reference of an hash if it exists on the remote repository.
137ref_from_hash(){
138 local hash=$1
139 git ls-remote origin | sed -n "\,$hash\t, { s,\(.*\)\t\(.*\),\2,; p; q}"
140}
141
142# Return the hash of a reference if it exists on the remote repository.
143hash_from_ref(){
144 local ref=$1
145 git ls-remote origin | sed -n "\,\t$ref, { s,\(.*\)\t\(.*\),\1,; p; q}"
146}
147
148# Returns a name based on the url and reference
149#
150# This function needs to be in sync with nix's fetchgit implementation
151# of urlToName() to re-use the same nix store paths.
152url_to_name(){
153 local url=$1
154 local ref=$2
155 local base
156 base=$(basename "$url" .git | cut -d: -f2)
157
158 if [[ $ref =~ ^[a-z0-9]+$ ]]; then
159 echo "$base-${ref:0:7}"
160 else
161 echo "$base"
162 fi
163}
164
165# Fetch and checkout the right sha1
166checkout_hash(){
167 local hash="$1"
168 local ref="$2"
169
170 if test -z "$hash"; then
171 hash=$(hash_from_ref "$ref")
172 fi
173
174 [[ -z "$deepClone" ]] && \
175 clean_git fetch ${builder:+--progress} --depth=1 origin "$hash" || \
176 clean_git fetch -t ${builder:+--progress} origin || return 1
177
178 local object_type=$(git cat-file -t "$hash")
179 if [[ "$object_type" == "commit" || "$object_type" == "tag" ]]; then
180 clean_git checkout -b "$branchName" "$hash" || return 1
181 elif [[ "$object_type" == "tree" ]]; then
182 clean_git config user.email "nix-prefetch-git@localhost"
183 clean_git config user.name "nix-prefetch-git"
184 local commit_id=$(git commit-tree "$hash" -m "Commit created from tree hash $hash")
185 clean_git checkout -b "$branchName" "$commit_id" || return 1
186 else
187 echo "Unrecognized git object type: $object_type"
188 return 1
189 fi
190}
191
192# Fetch only a branch/tag and checkout it.
193checkout_ref(){
194 local hash="$1"
195 local ref="$2"
196
197 if [[ -n "$deepClone" ]]; then
198 # The caller explicitly asked for a deep clone. Deep clones
199 # allow "git describe" and similar tools to work. See
200 # https://marc.info/?l=nix-dev&m=139641582514772
201 # for a discussion.
202 return 1
203 fi
204
205 if test -z "$ref"; then
206 ref=$(ref_from_hash "$hash")
207 fi
208
209 if test -n "$ref"; then
210 # --depth option is ignored on http repository.
211 clean_git fetch ${builder:+--progress} --depth 1 origin +"$ref" || return 1
212 clean_git checkout -b "$branchName" FETCH_HEAD || return 1
213 else
214 return 1
215 fi
216}
217
218# Update submodules
219init_submodules(){
220 # shallow with leaveDotGit will change hashes
221 [[ -z "$deepClone" ]] && [[ -z "$leaveDotGit" ]] && \
222 clean_git submodule update --init --recursive --checkout -j ${NIX_BUILD_CORES:-1} --progress --depth 1 || \
223 clean_git submodule update --init --recursive --checkout -j ${NIX_BUILD_CORES:-1} --progress
224}
225
226clone(){
227 local top=$PWD
228 local dir="$1"
229 local url="$2"
230 local hash="$3"
231 local ref="$4"
232
233 cd "$dir"
234
235 # Initialize the repository.
236 init_remote "$url"
237
238 # Download data from the repository.
239 checkout_ref "$hash" "$ref" ||
240 checkout_hash "$hash" "$ref" || (
241 echo 1>&2 "Unable to checkout $hash$ref from $url."
242 exit 1
243 )
244
245 # Fetch all tags if requested
246 if test -n "$fetchTags"; then
247 echo "fetching all tags..." >&2
248 clean_git fetch origin 'refs/tags/*:refs/tags/*' || echo "warning: failed to fetch some tags" >&2
249 fi
250
251 # Checkout linked sources.
252 if test -n "$fetchSubmodules"; then
253 init_submodules
254 fi
255
256 if [ -z "$builder" ] && [ -f .topdeps ]; then
257 if tg help &>/dev/null; then
258 echo "populating TopGit branches..."
259 tg remote --populate origin
260 else
261 echo "WARNING: would populate TopGit branches but TopGit is not available" >&2
262 echo "WARNING: install TopGit to fix the problem" >&2
263 fi
264 fi
265
266 cd "$top"
267}
268
269# Remove all remote branches, remove tags not reachable from HEAD, do a full
270# repack and then garbage collect unreferenced objects.
271make_deterministic_repo(){
272 local repo="$1"
273
274 # run in sub-shell to not touch current working directory
275 (
276 cd "$repo"
277 # Remove files that contain timestamps or otherwise have non-deterministic
278 # properties.
279 if [ -f .git ]; then
280 local dotgit_content=$(<.git)
281 local dotgit_dir="${dotgit_content#gitdir: }"
282 else
283 local dotgit_dir=".git"
284 fi
285 pushd "$dotgit_dir" >/dev/null
286 rm -rf logs/ hooks/ index FETCH_HEAD ORIG_HEAD refs/remotes/origin/HEAD config
287 popd >/dev/null
288 # Remove all remote branches.
289 git branch -r | while read -r branch; do
290 clean_git branch -rD "$branch"
291 done
292
293 # Remove tags not reachable from HEAD. If we're exactly on a tag, don't
294 # delete it.
295 maybe_tag=$(git tag --points-at HEAD)
296 git tag --contains HEAD | while read -r tag; do
297 if [ "$tag" != "$maybe_tag" ]; then
298 clean_git tag -d "$tag"
299 fi
300 done
301
302 # Do a full repack. Must run single-threaded, or else we lose determinism.
303 clean_git config pack.threads 1
304 clean_git repack -A -d -f
305 rm -f "$dotgit_dir/config"
306
307 # Garbage collect unreferenced objects.
308 # Note: --keep-largest-pack prevents non-deterministic ordering of packs
309 # listed in .git/objects/info/packs by only using a single pack
310 clean_git gc --prune=all --keep-largest-pack
311 )
312}
313
314
315clone_user_rev() {
316 local dir="$1"
317 local url="$2"
318 local rev="${3:-HEAD}"
319
320 if [ -n "$fetchLFS" ]; then
321 clean_git lfs install
322 fi
323
324 # Perform the checkout.
325 case "$rev" in
326 HEAD|refs/*)
327 clone "$dir" "$url" "" "$rev" 1>&2;;
328 *)
329 if test -z "$(echo "$rev" | tr -d 0123456789abcdef)"; then
330 clone "$dir" "$url" "$rev" "" 1>&2
331 else
332 # if revision is not hexadecimal it might be a tag
333 clone "$dir" "$url" "" "refs/tags/$rev" 1>&2
334 fi;;
335 esac
336
337 pushd "$dir" >/dev/null
338 fullRev=$( (git rev-parse "$rev" 2>/dev/null || git rev-parse "refs/heads/$branchName") | tail -n1)
339 humanReadableRev=$(git describe "$fullRev" 2> /dev/null || git describe --tags "$fullRev" 2> /dev/null || echo -- none --)
340 commitDate=$(git show -1 --no-patch --pretty=%ci "$fullRev")
341 commitDateStrict8601=$(git show -1 --no-patch --pretty=%cI "$fullRev")
342 popd >/dev/null
343
344 # Allow doing additional processing before .git removal
345 eval "$NIX_PREFETCH_GIT_CHECKOUT_HOOK"
346 if test -z "$leaveDotGit"; then
347 echo "removing \`.git'..." >&2
348 find "$dir" -name .git -print0 | xargs -0 rm -rf
349 else
350 find "$dir" -name .git | while read -r gitdir; do
351 make_deterministic_repo "$(readlink -f "$(dirname "$gitdir")")"
352 done
353 fi
354}
355
356exit_handlers=()
357
358run_exit_handlers() {
359 exit_status=$?
360 for handler in "${exit_handlers[@]}"; do
361 eval "$handler $exit_status"
362 done
363}
364
365trap run_exit_handlers EXIT
366
367quiet_exit_handler() {
368 exec 2>&3 3>&-
369 if [ $1 -ne 0 ]; then
370 cat "$errfile" >&2
371 fi
372 rm -f "$errfile"
373}
374
375quiet_mode() {
376 errfile="$(mktemp "${TMPDIR:-/tmp}/git-checkout-err-XXXXXXXX")"
377 exit_handlers+=(quiet_exit_handler)
378 exec 3>&2 2>"$errfile"
379}
380
381json_escape() {
382 local s="$1"
383 s="${s//\\/\\\\}" # \
384 s="${s//\"/\\\"}" # "
385 s="${s//^H/\\\b}" # \b (backspace)
386 s="${s//^L/\\\f}" # \f (form feed)
387 s="${s//
388/\\\n}" # \n (newline)
389 s="${s//^M/\\\r}" # \r (carriage return)
390 s="${s// /\\t}" # \t (tab)
391 echo "$s"
392}
393
394print_results() {
395 hash="$1"
396 if ! test -n "$QUIET"; then
397 echo "" >&2
398 echo "git revision is $fullRev" >&2
399 if test -n "$finalPath"; then
400 echo "path is $finalPath" >&2
401 fi
402 echo "git human-readable version is $humanReadableRev" >&2
403 echo "Commit date is $commitDate" >&2
404 if test -n "$hash"; then
405 echo "hash is $hash" >&2
406 fi
407 fi
408 if test -n "$hash"; then
409 cat <<EOF
410{
411 "url": "$(json_escape "$url")",
412 "rev": "$(json_escape "$fullRev")",
413 "date": "$(json_escape "$commitDateStrict8601")",
414 "path": "$(json_escape "$finalPath")",
415 "$(json_escape "$hashType")": "$(json_escape "$hash")",
416 "hash": "$(nix-hash --to-sri --type $hashType $hash)",
417 "fetchLFS": $([[ -n "$fetchLFS" ]] && echo true || echo false),
418 "fetchSubmodules": $([[ -n "$fetchSubmodules" ]] && echo true || echo false),
419 "deepClone": $([[ -n "$deepClone" ]] && echo true || echo false),
420 "fetchTags": $([[ -n "$fetchTags" ]] && echo true || echo false),
421 "leaveDotGit": $([[ -n "$leaveDotGit" ]] && echo true || echo false)
422}
423EOF
424 fi
425}
426
427remove_tmpPath() {
428 rm -rf "$tmpPath"
429}
430
431remove_tmpHomePath() {
432 chmod -R u+w "$tmpHomePath"
433 rm -rf "$tmpHomePath"
434}
435
436if test -n "$QUIET"; then
437 quiet_mode
438fi
439
440if test -z "$branchName"; then
441 branchName=fetchgit
442fi
443
444if [ -v symbolicName ]; then
445 storePathName="$symbolicName"
446else
447 storePathName="$(url_to_name "$url" "$rev")"
448fi
449
450tmpHomePath="$(mktemp -d "${TMPDIR:-/tmp}/nix-prefetch-git-tmp-home-XXXXXXXXXX")"
451exit_handlers+=(remove_tmpHomePath)
452ln -s "${NETRC:-$HOME/.netrc}" "$tmpHomePath/.netrc"
453HOME="$tmpHomePath"
454unset XDG_CONFIG_HOME
455export GIT_CONFIG_NOSYSTEM=1
456
457if test -n "$builder"; then
458 test -n "$out" -a -n "$url" -a -n "$rev" || usage
459 mkdir -p "$out"
460 clone_user_rev "$out" "$url" "$rev"
461else
462 if test -z "$hashType"; then
463 hashType=sha256
464 fi
465
466 # If the hash was given, a file with that hash may already be in the
467 # store.
468 if test -n "$expHash"; then
469 finalPath=$(nix-store --print-fixed-path --recursive "$hashType" "$expHash" "$storePathName")
470 if ! nix-store --check-validity "$finalPath" 2> /dev/null; then
471 finalPath=
472 fi
473 hash=$expHash
474 fi
475
476 # If we don't know the hash or a path with that hash doesn't exist,
477 # download the file and add it to the store.
478 if test -z "$finalPath"; then
479 # nix>=2.20 rejects adding symlinked paths to the store, so use realpath
480 # to resolve to a physical path. https://github.com/NixOS/nix/issues/11941
481 tmpPath="$(realpath "$(mktemp -d --tmpdir git-checkout-tmp-XXXXXXXX)")"
482 exit_handlers+=(remove_tmpPath)
483
484 tmpFile="$tmpPath/$storePathName"
485 mkdir -p "$tmpFile"
486
487 # Perform the checkout.
488 clone_user_rev "$tmpFile" "$url" "$rev"
489
490 # Compute the hash.
491 hash=$(nix-hash --type $hashType --base32 "$tmpFile")
492
493 # Add the downloaded file to the Nix store.
494 finalPath=$(nix-store --add-fixed --recursive "$hashType" "$tmpFile")
495
496 if test -n "$expHash" -a "$expHash" != "$hash"; then
497 echo "hash mismatch for URL \`$url'. Got \`$hash'; expected \`$expHash'." >&2
498 exit 1
499 fi
500 fi
501
502 print_results "$hash"
503
504 if test -n "$PRINT_PATH"; then
505 echo "$finalPath"
506 fi
507fi