freshlybakedca.ke
Git fork
1# Library of functions shared by all CI scripts
2
3if test true = "$GITHUB_ACTIONS"
4then
5 begin_group () {
6 need_to_end_group=t
7 echo "::group::$1" >&2
8 set -x
9 }
10
11 end_group () {
12 test -n "$need_to_end_group" || return 0
13 set +x
14 need_to_end_group=
15 echo '::endgroup::' >&2
16 }
17elif test true = "$GITLAB_CI"
18then
19 begin_group () {
20 need_to_end_group=t
21 printf '\e[0Ksection_start:%s:%s[collapsed=true]\r\e[0K%s\n' \
22 "$(date +%s)" "$(echo "$1" | tr ' ' _)" "$1"
23 trap "end_group '$1'" EXIT
24 set -x
25 }
26
27 end_group () {
28 test -n "$need_to_end_group" || return 0
29 set +x
30 need_to_end_group=
31 printf '\e[0Ksection_end:%s:%s\r\e[0K\n' \
32 "$(date +%s)" "$(echo "$1" | tr ' ' _)"
33 trap - EXIT
34 }
35else
36 begin_group () { :; }
37 end_group () { :; }
38
39 set -x
40fi
41
42group () {
43 group="$1"
44 shift
45 begin_group "$group"
46
47 # work around `dash` not supporting `set -o pipefail`
48 (
49 "$@" 2>&1
50 echo $? >exit.status
51 ) |
52 sed 's/^\(\([^ ]*\):\([0-9]*\):\([0-9]*:\) \)\(error\|warning\): /::\5 file=\2,line=\3::\1/'
53 res=$(cat exit.status)
54 rm exit.status
55
56 end_group "$group"
57 return $res
58}
59
60begin_group "CI setup via $(basename $0)"
61
62# Set 'exit on error' for all CI scripts to let the caller know that
63# something went wrong.
64#
65# We already enabled tracing executed commands earlier. This helps by showing
66# how # environment variables are set and dependencies are installed.
67set -e
68
69skip_branch_tip_with_tag () {
70 # Sometimes, a branch is pushed at the same time the tag that points
71 # at the same commit as the tip of the branch is pushed, and building
72 # both at the same time is a waste.
73 #
74 # When the build is triggered by a push to a tag, $CI_BRANCH will
75 # have that tagname, e.g. v2.14.0. Let's see if $CI_BRANCH is
76 # exactly at a tag, and if so, if it is different from $CI_BRANCH.
77 # That way, we can tell if we are building the tip of a branch that
78 # is tagged and we can skip the build because we won't be skipping a
79 # build of a tag.
80
81 if TAG=$(git describe --exact-match "$CI_BRANCH" 2>/dev/null) &&
82 test "$TAG" != "$CI_BRANCH"
83 then
84 echo "$(tput setaf 2)Tip of $CI_BRANCH is exactly at $TAG$(tput sgr0)"
85 exit 0
86 fi
87}
88
89# Check whether we can use the path passed via the first argument as Git
90# repository.
91is_usable_git_repository () {
92 # We require Git in our PATH, otherwise we cannot access repositories
93 # at all.
94 if ! command -v git >/dev/null
95 then
96 return 1
97 fi
98
99 # And the target directory needs to be a proper Git repository.
100 if ! git -C "$1" rev-parse 2>/dev/null
101 then
102 return 1
103 fi
104}
105
106# Save some info about the current commit's tree, so we can skip the build
107# job if we encounter the same tree again and can provide a useful info
108# message.
109save_good_tree () {
110 if ! is_usable_git_repository .
111 then
112 return
113 fi
114
115 echo "$(git rev-parse $CI_COMMIT^{tree}) $CI_COMMIT $CI_JOB_NUMBER $CI_JOB_ID" >>"$good_trees_file"
116 # limit the file size
117 tail -1000 "$good_trees_file" >"$good_trees_file".tmp
118 mv "$good_trees_file".tmp "$good_trees_file"
119}
120
121# Skip the build job if the same tree has already been built and tested
122# successfully before (e.g. because the branch got rebased, changing only
123# the commit messages).
124skip_good_tree () {
125 if test true = "$GITHUB_ACTIONS"
126 then
127 return
128 fi
129
130 if ! is_usable_git_repository .
131 then
132 return
133 fi
134
135 if ! good_tree_info="$(grep "^$(git rev-parse $CI_COMMIT^{tree}) " "$good_trees_file")"
136 then
137 # Haven't seen this tree yet, or no cached good trees file yet.
138 # Continue the build job.
139 return
140 fi
141
142 echo "$good_tree_info" | {
143 read tree prev_good_commit prev_good_job_number prev_good_job_id
144
145 if test "$CI_JOB_ID" = "$prev_good_job_id"
146 then
147 cat <<-EOF
148 $(tput setaf 2)Skipping build job for commit $CI_COMMIT.$(tput sgr0)
149 This commit has already been built and tested successfully by this build job.
150 To force a re-build delete the branch's cache and then hit 'Restart job'.
151 EOF
152 else
153 cat <<-EOF
154 $(tput setaf 2)Skipping build job for commit $CI_COMMIT.$(tput sgr0)
155 This commit's tree has already been built and tested successfully in build job $prev_good_job_number for commit $prev_good_commit.
156 The log of that build job is available at $SYSTEM_TASKDEFINITIONSURI$SYSTEM_TEAMPROJECT/_build/results?buildId=$prev_good_job_id
157 To force a re-build delete the branch's cache and then hit 'Restart job'.
158 EOF
159 fi
160 }
161
162 exit 0
163}
164
165check_unignored_build_artifacts () {
166 if ! is_usable_git_repository .
167 then
168 return
169 fi
170
171 ! git ls-files --other --exclude-standard --error-unmatch \
172 -- ':/*' 2>/dev/null ||
173 {
174 echo "$(tput setaf 1)error: found unignored build artifacts$(tput sgr0)"
175 false
176 }
177}
178
179handle_failed_tests () {
180 return 1
181}
182
183create_failed_test_artifacts () {
184 mkdir -p "${TEST_OUTPUT_DIRECTORY:-t}"/failed-test-artifacts
185
186 for test_exit in "${TEST_OUTPUT_DIRECTORY:-t}"/test-results/*.exit
187 do
188 test 0 != "$(cat "$test_exit")" || continue
189
190 test_name="${test_exit%.exit}"
191 test_name="${test_name##*/}"
192 printf "\\e[33m\\e[1m=== Failed test: ${test_name} ===\\e[m\\n"
193 echo "The full logs are in the 'print test failures' step below."
194 echo "See also the 'failed-tests-*' artifacts attached to this run."
195 cat "${TEST_OUTPUT_DIRECTORY:-t}/test-results/$test_name.markup"
196
197 trash_dir="${TEST_OUTPUT_DIRECTORY:-t}/trash directory.$test_name"
198 cp "${TEST_OUTPUT_DIRECTORY:-t}/test-results/$test_name.out" "${TEST_OUTPUT_DIRECTORY:-t}"/failed-test-artifacts/
199 tar czf "${TEST_OUTPUT_DIRECTORY:-t}/failed-test-artifacts/$test_name.trash.tar.gz" "$trash_dir"
200 done
201}
202
203# GitHub Action doesn't set TERM, which is required by tput
204export TERM=${TERM:-dumb}
205
206# Clear MAKEFLAGS that may come from the outside world.
207export MAKEFLAGS=
208
209if test true = "$GITHUB_ACTIONS"
210then
211 CI_TYPE=github-actions
212 CI_BRANCH="$GITHUB_REF"
213 CI_COMMIT="$GITHUB_SHA"
214 CI_OS_NAME="$(echo "$RUNNER_OS" | tr A-Z a-z)"
215 test macos != "$CI_OS_NAME" || CI_OS_NAME=osx
216 CI_REPO_SLUG="$GITHUB_REPOSITORY"
217 CI_JOB_ID="$GITHUB_RUN_ID"
218 CC="${CC_PACKAGE:-${CC:-gcc}}"
219 DONT_SKIP_TAGS=t
220 handle_failed_tests () {
221 echo "FAILED_TEST_ARTIFACTS=${TEST_OUTPUT_DIRECTORY:-t}/failed-test-artifacts" >>$GITHUB_ENV
222 create_failed_test_artifacts
223 return 1
224 }
225
226 cache_dir="$HOME/none"
227
228 GIT_TEST_OPTS="--github-workflow-markup"
229 JOBS=10
230
231 distro=$(echo "$CI_JOB_IMAGE" | tr : -)
232elif test true = "$GITLAB_CI"
233then
234 CI_TYPE=gitlab-ci
235 CI_BRANCH="$CI_COMMIT_REF_NAME"
236 CI_COMMIT="$CI_COMMIT_SHA"
237
238 case "$OS,$CI_JOB_IMAGE" in
239 Windows_NT,*)
240 CI_OS_NAME=windows
241 JOBS=$NUMBER_OF_PROCESSORS
242 ;;
243 *,macos-*)
244 # GitLab CI has Python installed via multiple package managers,
245 # most notably via asdf and Homebrew. Ensure that our builds
246 # pick up the Homebrew one by prepending it to our PATH as the
247 # asdf one breaks tests.
248 export PATH="$(brew --prefix)/bin:$PATH"
249
250 CI_OS_NAME=osx
251 JOBS=$(nproc)
252 ;;
253 *,alpine:*|*,fedora:*|*,ubuntu:*|*,i386/ubuntu:*)
254 CI_OS_NAME=linux
255 JOBS=$(nproc)
256 ;;
257 *)
258 echo "Could not identify OS image" >&2
259 env >&2
260 exit 1
261 ;;
262 esac
263 CI_REPO_SLUG="$CI_PROJECT_PATH"
264 CI_JOB_ID="$CI_JOB_ID"
265 CC="${CC_PACKAGE:-${CC:-gcc}}"
266 DONT_SKIP_TAGS=t
267
268 handle_failed_tests () {
269 create_failed_test_artifacts
270 return 1
271 }
272
273 cache_dir="$HOME/none"
274
275 distro=$(echo "$CI_JOB_IMAGE" | tr : -)
276else
277 echo "Could not identify CI type" >&2
278 env >&2
279 exit 1
280fi
281
282MAKEFLAGS="$MAKEFLAGS --jobs=$JOBS"
283GIT_PROVE_OPTS="--timer --jobs $JOBS"
284
285GIT_TEST_OPTS="$GIT_TEST_OPTS --verbose-log -x"
286case "$CI_OS_NAME" in
287windows|windows_nt)
288 GIT_TEST_OPTS="$GIT_TEST_OPTS --no-chain-lint --no-bin-wrappers"
289 ;;
290esac
291
292export GIT_TEST_OPTS
293export GIT_PROVE_OPTS
294
295good_trees_file="$cache_dir/good-trees"
296
297mkdir -p "$cache_dir"
298
299test -n "${DONT_SKIP_TAGS-}" ||
300skip_branch_tip_with_tag
301skip_good_tree
302
303if test -z "$jobname"
304then
305 jobname="$CI_OS_NAME-$CC"
306fi
307
308export DEVELOPER=1
309export DEFAULT_TEST_TARGET=prove
310export GIT_TEST_CLONE_2GB=true
311export SKIP_DASHED_BUILT_INS=YesPlease
312
313case "$distro" in
314ubuntu-*)
315 # Python 2 is end of life, and Ubuntu 23.04 and newer don't actually
316 # have it anymore. We thus only test with Python 2 on older LTS
317 # releases.
318 if test "$distro" = "ubuntu-20.04"
319 then
320 PYTHON_PACKAGE=python2
321 else
322 PYTHON_PACKAGE=python3
323 fi
324 MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=/usr/bin/$PYTHON_PACKAGE"
325
326 export GIT_TEST_HTTPD=true
327
328 # The Linux build installs the defined dependency versions below.
329 # The OS X build installs much more recent versions, whichever
330 # were recorded in the Homebrew database upon creating the OS X
331 # image.
332 # Keep that in mind when you encounter a broken OS X build!
333 export LINUX_GIT_LFS_VERSION="1.5.2"
334 ;;
335macos-*)
336 MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=$(which python3)"
337 if [ "$jobname" != osx-gcc ]
338 then
339 MAKEFLAGS="$MAKEFLAGS APPLE_COMMON_CRYPTO_SHA1=Yes"
340 fi
341 ;;
342esac
343
344CUSTOM_PATH="${CUSTOM_PATH:-$HOME/path}"
345export PATH="$CUSTOM_PATH:$PATH"
346
347case "$jobname" in
348linux32)
349 CC=gcc
350 ;;
351linux-meson)
352 MESONFLAGS="$MESONFLAGS -Dcredential_helpers=libsecret,netrc"
353 ;;
354linux-musl-meson)
355 MESONFLAGS="$MESONFLAGS -Dtest_utf8_locale=C.UTF-8"
356 ;;
357linux-leaks|linux-reftable-leaks)
358 export SANITIZE=leak
359 ;;
360linux-asan-ubsan)
361 export SANITIZE=address,undefined
362 export NO_SVN_TESTS=LetsSaveSomeTime
363 MAKEFLAGS="$MAKEFLAGS NO_PYTHON=YepBecauseP4FlakesTooOften"
364 ;;
365osx-meson)
366 MESONFLAGS="$MESONFLAGS -Dcredential_helpers=osxkeychain"
367 ;;
368esac
369
370MAKEFLAGS="$MAKEFLAGS CC=${CC:-cc}"
371
372end_group "CI setup via $(basename $0)"
373set -x