#!/bin/bash
set -euo pipefail
cd "$(dirname "$0")/../.."
unset PARCEL_TEST_SKIP_BUILD PARCEL_TEST_XUNIT_OUTPUT
work=$(mktemp -d "${TMPDIR:-/tmp}/test-runner.XXXXXX")
trap 'rm -rf "$work"' EXIT
mkdir -p "$work/project/Scripts" "$work/project/Tests/Lanes" "$work/bin"
cp Scripts/test.sh Scripts/test-unit.sh Scripts/test-lane.sh Scripts/test-native-host.sh Scripts/test-native-runner.sh Scripts/NativeTestHost.plist "$work/project/Scripts/"
cp Tests/Lanes/*.txt "$work/project/Tests/Lanes/"
selector=${PARCEL_TEST_SELECTION_BINARY:-$(swift build --show-bin-path)/ParcelTestSelection}
cp "$selector" "$work/bin/"
cat > "$work/project/Scripts/swift-build.sh" <<'MOCK'
#!/bin/sh
set -eu
exec swift build "$@"
MOCK
chmod +x "$work/project/Scripts/swift-build.sh"
export TEST_BIN="$work/bin" TEST_EVENTS="$work/events" TEST_ENVIRONMENT="$work/environment"
export TEST_DISCOVERY="$work/discovered" TEST_REPORT="$work/report.xml"
python3 -I - "$work/project/Tests/Lanes/unit.txt" "$TEST_DISCOVERY" "$TEST_REPORT" <<'PY'
import sys
from pathlib import Path
from xml.etree.ElementTree import Element, SubElement, ElementTree
suites = [s for s in Path(sys.argv[1]).read_text().splitlines() if s and not s.startswith('#')]
other = [s for lane in ['native-host', 'bench'] for s in Path(sys.argv[1]).with_name(lane + '.txt').read_text().splitlines() if s and not s.startswith('#')]
Path(sys.argv[2]).write_text('\n'.join(s if s.endswith(')') else s + '/example()' for s in suites + other) + '\n')
root = Element('testsuites')
for suite in suites:
if suite.endswith(')'):
classname, name = suite.split('.', 1)
else:
classname, name = suite.replace('/', '.'), 'example()'
SubElement(root, 'testcase', classname=classname, name=name)
ElementTree(root).write(sys.argv[3])
PY
cat > "$work/bin/swift" <<'MOCK'
#!/bin/bash
set -euo pipefail
printf '%s\n' "$*" >> "$TEST_EVENTS"
if [[ $* == 'build --show-bin-path' ]]; then printf '%s\n' "$TEST_BIN"; exit; fi
if [[ $* == 'build --build-tests' ]]; then exit "${TEST_BUILD_STATUS:-0}"; fi
if [[ $* == 'test list --skip-build' ]]; then
printf '%s\n' "$(< "$TEST_DISCOVERY")"
exit "${TEST_LIST_STATUS:-0}"
fi
printf '%s|%s|%s|%s|%s\n' "${PARCEL_NATIVE_HOST:-unset}" "${PARCEL_E2E:-unset}" "${PARCEL_BENCH:-unset}" "${PARCEL_BENCH_SMOKE:-unset}" "${PARCEL_LOCALIZATION_SCREENSHOTS:-unset}" > "$TEST_ENVIRONMENT"
previous=
for arg in "$@"; do
if [[ $previous == --xunit-output && ${TEST_NO_REPORT:-0} != 1 ]]; then cp "$TEST_REPORT" "$arg"; fi
previous=$arg
done
exit "${TEST_SWIFT_STATUS:-0}"
MOCK
chmod +x "$work/bin/swift"
export PATH="$work/bin:$PATH"
count=0
check() { "$@" || { printf 'Test runner check failed: %s\n' "$*" >&2; exit 1; }; count=$((count + 1)); }
fails_with() {
local expected=$1 status=0
shift
"$@" > "$work/failure.log" 2>&1 || status=$?
[[ $status == "$expected" ]]
}
runner="$work/project/Scripts/test.sh"
check env PARCEL_NATIVE_HOST=1 PARCEL_E2E=1 PARCEL_BENCH=1 PARCEL_BENCH_SMOKE=1 PARCEL_LOCALIZATION_SCREENSHOTS=/private/captures bash "$runner"
printf 'unset|unset|unset|unset|unset\n' > "$work/expected"
check cmp "$work/expected" "$TEST_ENVIRONMENT"
check test -f "$work/project/build/junit.xml"
check grep -q '^build --build-tests$' "$TEST_EVENTS"
check grep -q '^test list --skip-build$' "$TEST_EVENTS"
check grep -q '^test --skip-build --disable-xctest --parallel --filter \^' "$TEST_EVENTS"
: > "$TEST_EVENTS"
check env PARCEL_TEST_SKIP_BUILD=1 PARCEL_TEST_XUNIT_OUTPUT="$work/custom.xml" bash "$runner" --no-parallel
check test -f "$work/custom.xml"
check test "$(grep -c '^build --build-tests$' "$TEST_EVENTS" || true)" == 0
check grep -q '^test --skip-build --disable-xctest --no-parallel --filter ' "$TEST_EVENTS"
check bash "$runner" --skip-build --xunit-output "$work/argument.xml"
check test -f "$work/argument.xml"
check fails_with 2 bash "$runner" --package-path /other
check fails_with 2 bash "$runner" --filter
check fails_with 2 bash "$runner" --skip unsafe
check fails_with 1 bash "$runner" --filter 'NoSuchTest'
check fails_with 1 bash "$runner" --filter 'AppKitTestSuite|ParcelPerformanceTests'
check bash "$runner" --skip-build
check fails_with 17 env TEST_BUILD_STATUS=17 bash "$runner"
check test ! -e "$work/project/build/junit.xml"
check fails_with 19 env TEST_LIST_STATUS=19 bash "$runner" --skip-build
check fails_with 7 env TEST_SWIFT_STATUS=7 bash "$runner" --skip-build --xunit-output "$work/failed.xml"
check test -f "$work/failed.xml"
check fails_with 1 env TEST_NO_REPORT=1 bash "$runner" --skip-build
check test ! -e "$work/project/build/junit.xml"
check bash "$runner" --skip-build
check fails_with 19 env TEST_LIST_STATUS=19 bash "$runner" --skip-build
check test ! -e "$work/project/build/junit.xml"
cp "$TEST_DISCOVERY" "$work/saved-discovery"
for identifier in 'ParcelE2ETests.NewScreenFreeTests/escape()' 'ParcelPerformanceTests.NewBenchmarkTests/escape()'; do
printf '%s\n' "$identifier" >> "$TEST_DISCOVERY"
check fails_with 1 bash "$runner" --skip-build
cp "$work/saved-discovery" "$TEST_DISCOVERY"
done
cp "$work/project/Tests/Lanes/unit.txt" "$work/saved-unit"
printf 'ParcelPerformanceTests.PerformanceTestSuite\n' >> "$work/project/Tests/Lanes/unit.txt"
check fails_with 1 bash "$runner" --skip-build
cp "$work/saved-unit" "$work/project/Tests/Lanes/unit.txt"
"$TEST_BIN/ParcelTestSelection" select "$work/project/Tests/Lanes/bench.txt" "$TEST_DISCOVERY" '' "$TEST_REPORT" > "$work/bench-filter"
check grep -Fq 'ParcelPerformanceTests\.PerformanceTestSuite' "$work/bench-filter"
: > "$TEST_EVENTS"
check env PARCEL_TEST_SKIP_BUILD=1 bash "$work/project/Scripts/test-lane.sh" inventory
check grep -q '^build --build-tests$' "$TEST_EVENTS"
check test "$(grep -c '^test --skip-build ' "$TEST_EVENTS" || true)" == 0
check fails_with 2 bash "$work/project/Scripts/test-lane.sh" inventory --filter 'Geometry'
printf 'NewTests.Unclassified/test()\n' >> "$TEST_DISCOVERY"
check fails_with 1 bash "$work/project/Scripts/test-lane.sh" inventory
cp "$work/saved-discovery" "$TEST_DISCOVERY"
check grep -Fq 'ParcelE2ETests.FixtureReplyTests' "$work/project/Tests/Lanes/unit.txt"
: > "$TEST_DISCOVERY"
check fails_with 1 bash "$runner" --skip-build
cp "$work/saved-discovery" "$TEST_DISCOVERY"
python3 -I - "$TEST_DISCOVERY" <<'PY'
import sys
from pathlib import Path
p = Path(sys.argv[1]); p.write_text('\n'.join(p.read_text().splitlines()[1:]) + '\n')
PY
check fails_with 1 bash "$runner" --skip-build
cp "$work/saved-discovery" "$TEST_DISCOVERY"
cp "$TEST_REPORT" "$work/saved-report"
printf '\n' > "$TEST_REPORT"
check fails_with 1 bash "$runner" --skip-build
python3 -I - "$work/saved-report" "$TEST_REPORT" <<'PY'
import sys
from xml.etree.ElementTree import parse, SubElement
report = parse(sys.argv[1])
report.getroot().remove(report.getroot()[0])
report.write(sys.argv[2])
PY
check fails_with 1 bash "$runner" --skip-build
python3 -I - "$work/saved-report" "$TEST_REPORT" <<'PY'
import sys
from xml.etree.ElementTree import parse, SubElement
report = parse(sys.argv[1])
SubElement(report.getroot()[0], 'skipped')
report.write(sys.argv[2])
PY
check fails_with 1 bash "$runner" --skip-build
cp "$work/saved-report" "$TEST_REPORT"
printf 'UnexpectedTests.NewSuite/escape()\n' >> "$TEST_DISCOVERY"
check fails_with 1 bash "$runner" --skip-build
cp "$work/saved-discovery" "$TEST_DISCOVERY"
python3 -I - "$work/saved-report" "$TEST_REPORT" <<'PY'
import sys
from xml.etree.ElementTree import parse, SubElement
report = parse(sys.argv[1])
SubElement(report.getroot(), 'testcase', classname='UnexpectedTests.NewSuite', name='escape()')
report.write(sys.argv[2])
PY
check fails_with 1 bash "$runner" --skip-build
printf '\n' > "$TEST_REPORT"
check fails_with 1 bash "$runner" --skip-build
for mode in missing-global skipped-global unexpected-global; do
python3 -I - "$work/saved-report" "$TEST_REPORT" "$mode" <<'PY'
import sys
from xml.etree.ElementTree import parse, SubElement
report = parse(sys.argv[1])
case = next(c for c in report.getroot() if '.' not in c.attrib['classname'])
if sys.argv[3] == 'missing-global':
report.getroot().remove(case)
elif sys.argv[3] == 'skipped-global':
SubElement(case, 'skipped')
else:
SubElement(report.getroot(), 'testcase', classname=case.attrib['classname'], name='unexpectedGlobal()')
report.write(sys.argv[2])
PY
check fails_with 1 bash "$runner" --skip-build
done
cp "$work/saved-report" "$TEST_REPORT"
printf 'UnexpectedTests.globalEscape()\n' >> "$TEST_DISCOVERY"
check fails_with 1 bash "$runner" --skip-build
cp "$work/saved-discovery" "$TEST_DISCOVERY"
python3 -I - "$work/saved-report" "$TEST_REPORT" "$work/global-filter" <<'PY'
import re, sys
from pathlib import Path
from xml.etree.ElementTree import parse, Element, ElementTree
report = parse(sys.argv[1])
case = next(c for c in report.getroot() if '.' not in c.attrib['classname'])
root = Element('testsuites'); root.append(case)
ElementTree(root).write(sys.argv[2])
Path(sys.argv[3]).write_text('^' + re.escape(case.attrib['classname'] + '.' + case.attrib['name']) + '$')
PY
check bash "$runner" --skip-build --filter "$(< "$work/global-filter")"
python3 -I - "$work/project/Tests/Lanes/bench.txt" "$TEST_REPORT" <<'PY'
import sys
from pathlib import Path
from xml.etree.ElementTree import Element, SubElement, ElementTree
root = Element('testsuites')
for suite in Path(sys.argv[1]).read_text().splitlines():
if suite and not suite.startswith('#'):
SubElement(root, 'testcase', classname=suite.replace('/', '.'), name='example()')
ElementTree(root).write(sys.argv[2])
PY
: > "$TEST_EVENTS"
check env PARCEL_NATIVE_HOST=1 PARCEL_E2E=1 PARCEL_BENCH=1 PARCEL_LOCALIZATION_SCREENSHOTS=/private/captures bash "$work/project/Scripts/test-lane.sh" bench --skip-build
printf 'unset|unset|unset|1|unset\n' > "$work/expected"
check cmp "$work/expected" "$TEST_ENVIRONMENT"
check test -f "$work/project/build/junit-bench.xml"
check grep -q '^test --skip-build --disable-xctest --no-parallel --filter ' "$TEST_EVENTS"
check fails_with 2 bash "$work/project/Scripts/test-lane.sh" bench --parallel
check fails_with 1 bash "$work/project/Scripts/test-lane.sh" bench --skip-build --filter 'SettingsTests'
cp "$TEST_REPORT" "$work/bench-report"
for mode in missing-suite skipped-suite unexpected-suite empty; do
python3 -I - "$work/bench-report" "$TEST_REPORT" "$mode" <<'PY'
import sys
from xml.etree.ElementTree import parse, SubElement
report = parse(sys.argv[1]); root = report.getroot()
if sys.argv[3] == 'missing-suite':
root.remove(root[0])
elif sys.argv[3] == 'skipped-suite':
SubElement(root[0], 'skipped')
elif sys.argv[3] == 'unexpected-suite':
SubElement(root, 'testcase', classname='UnexpectedTests.Suite', name='escape()')
else:
root.clear()
report.write(sys.argv[2])
PY
check fails_with 1 bash "$work/project/Scripts/test-lane.sh" bench --skip-build
done
python3 -I - "$work/saved-discovery" "$TEST_DISCOVERY" "$work/bench-report" "$TEST_REPORT" <<'PY'
import sys
from pathlib import Path
from xml.etree.ElementTree import parse, SubElement
parent = 'ParcelPerformanceTests.EngineCorePerformanceTests'
text = Path(sys.argv[1]).read_text()
old = parent + '/example()'
assert text.count(old) == 1
Path(sys.argv[2]).write_text(text.replace(old, parent + '/Counters/counter()\n' + parent + '/Workloads/workload()'))
report = parse(sys.argv[3]); root = report.getroot()
case = next(c for c in root if c.attrib['classname'] == parent)
root.remove(case)
SubElement(root, 'testcase', classname=parent + '.Counters', name='counter()')
SubElement(root, 'testcase', classname=parent + '.Workloads', name='workload()')
report.write(sys.argv[4])
PY
check bash "$work/project/Scripts/test-lane.sh" bench --skip-build
cp "$TEST_REPORT" "$work/nested-bench-report"
for child in Counters Workloads; do
python3 -I - "$work/nested-bench-report" "$TEST_REPORT" "$child" <<'PY'
import sys
from xml.etree.ElementTree import parse, SubElement
report = parse(sys.argv[1])
case = next(c for c in report.getroot() if c.attrib['classname'].endswith('.' + sys.argv[3]))
SubElement(case, 'skipped')
report.write(sys.argv[2])
PY
check fails_with 1 bash "$work/project/Scripts/test-lane.sh" bench --skip-build
check grep -q 'Required suite did not execute' "$work/failure.log"
done
cp "$work/saved-discovery" "$TEST_DISCOVERY"
cp "$work/bench-report" "$TEST_REPORT"
mkdir -p "$work/toolchain/usr/bin" "$work/toolchain/usr/libexec/swift/pm" "$work/sdk/Developer/Library/Frameworks" "$TEST_BIN/ParcelPackageTests.xctest/Contents/MacOS"
: > "$TEST_BIN/ParcelPackageTests.xctest/Contents/MacOS/ParcelPackageTests"
export TEST_TOOLCHAIN="$work/toolchain" TEST_SDK="$work/sdk"
export TEST_NATIVE_ARGUMENTS="$work/native-arguments" TEST_HOST_INFO="$work/host.plist"
cat > "$work/bin/xcrun" <<'MOCK'
#!/bin/bash
set -euo pipefail
case "$*" in
'--find swift') printf '%s\n' "$TEST_TOOLCHAIN/usr/bin/swift" ;;
'--show-sdk-platform-path') printf '%s\n' "$TEST_SDK" ;;
*) exit 2 ;;
esac
MOCK
cat > "$work/toolchain/usr/libexec/swift/pm/swiftpm-testing-helper" <<'MOCK'
#!/bin/bash
set -euo pipefail
cp "$(dirname "$0")/../Info.plist" "$TEST_HOST_INFO"
printf '%s\n' "$@" > "$TEST_NATIVE_ARGUMENTS"
exec "$TEST_BIN/swift" test "$@"
MOCK
chmod +x "$work/bin/xcrun" "$work/toolchain/usr/libexec/swift/pm/swiftpm-testing-helper"
python3 -I - "$work/project/Tests/Lanes/native-host.txt" "$TEST_REPORT" <<'PY'
import sys
from pathlib import Path
from xml.etree.ElementTree import Element, SubElement, ElementTree
root = Element('testsuites')
for suite in Path(sys.argv[1]).read_text().splitlines():
if suite and not suite.startswith('#'):
SubElement(root, 'testcase', classname=suite.replace('/', '.'), name='example()')
ElementTree(root).write(sys.argv[2])
PY
check env PARCEL_NATIVE_HOST=1 PARCEL_LOCALIZATION_LANGUAGE=de bash "$work/project/Scripts/test-lane.sh" native-host --skip-build
check grep -q '^--no-parallel$' "$TEST_NATIVE_ARGUMENTS"
check grep -q '^-AppleLanguages$' "$TEST_NATIVE_ARGUMENTS"
check grep -q '^(de)$' "$TEST_NATIVE_ARGUMENTS"
check test "$(/usr/libexec/PlistBuddy -c 'Print :CFBundleAllowMixedLocalizations' "$TEST_HOST_INFO")" == true
check test "$(find "$work/project/.build" -name 'native-test-host.*' -print | wc -l | tr -d ' ')" == 0
check env PARCEL_NATIVE_HOST=1 PARCEL_PSEUDO_LOCALIZATION=1 PARCEL_LOCALIZATION_LANGUAGE=de bash "$work/project/Scripts/test-lane.sh" native-host --skip-build
check grep -q '^(en-XA)$' "$TEST_NATIVE_ARGUMENTS"
check fails_with 2 env PARCEL_NATIVE_HOST=1 PARCEL_LOCALIZATION_LANGUAGE=invalid bash "$work/project/Scripts/test-lane.sh" native-host --skip-build
for mode in all-skipped permission-skipped; do
python3 -I - "$work/project/Tests/Lanes/native-host.txt" "$TEST_REPORT" "$mode" <<'PY'
import sys
from pathlib import Path
from xml.etree.ElementTree import Element, SubElement, ElementTree
suites = [s for s in Path(sys.argv[1]).read_text().splitlines() if s and not s.startswith('#')]
root = Element('testsuites')
for suite in suites:
case = SubElement(root, 'testcase', classname=suite.replace('/', '.'), name='example()')
if sys.argv[3] == 'all-skipped' or suite.endswith('/InputPermissionTests'):
SubElement(case, 'skipped', message='Accessibility is not granted')
ElementTree(root).write(sys.argv[2])
PY
if [[ $mode == all-skipped ]]; then
check fails_with 1 "$TEST_BIN/ParcelTestSelection" check-native "$work/project/Tests/Lanes/native-host.txt" "$TEST_DISCOVERY" '' "$TEST_REPORT"
else
check "$TEST_BIN/ParcelTestSelection" check-native "$work/project/Tests/Lanes/native-host.txt" "$TEST_DISCOVERY" '' "$TEST_REPORT"
fi
done
check fails_with 2 env PARCEL_NATIVE_HOST=0 bash "$work/project/Scripts/test-native-host.sh"
cat > "$work/project/Scripts/screen-lock.sh" <<'MOCK'
#!/bin/bash
set -euo pipefail
printf '%s\n' "${PARCEL_SCREEN_OWNER:-}" > "$TEST_LOCK_OWNER"
exit 29
MOCK
chmod +x "$work/project/Scripts/screen-lock.sh"
check fails_with 29 env PARCEL_NATIVE_HOST=1 TEST_LOCK_OWNER="$work/owner" bash "$work/project/Scripts/test-native-host.sh"
printf 'native-host\n' > "$work/expected"
check cmp "$work/expected" "$work/owner"
for target in test test-serial test-scripts; do
make --no-print-directory --dry-run "$target" > "$work/actual"
case "$target" in
test) printf '%s\n' 'mise run test' > "$work/expected" ;;
test-serial) printf '%s\n' 'mise run test:serial' > "$work/expected" ;;
test-scripts) printf '%s\n' 'mise run test:scripts' > "$work/expected" ;;
esac
check cmp "$work/expected" "$work/actual"
done
printf 'Test runner checks passed: %s\n' "$count"