#!/bin/bash set -euo pipefail cd "$(dirname "$0")/../.." unset PARCEL_TEST_SKIP_BUILD PARCEL_TEST_XUNIT_OUTPUT work=$(mktemp -d "${TMPDIR:-/tmp}/test-runner.XXXXXX") trap 'rm -rf "$work"' EXIT mkdir -p "$work/project/Scripts" "$work/project/Tests/Lanes" "$work/bin" cp Scripts/test.sh Scripts/test-unit.sh Scripts/test-lane.sh Scripts/test-native-host.sh Scripts/test-native-runner.sh Scripts/NativeTestHost.plist "$work/project/Scripts/" cp Tests/Lanes/*.txt "$work/project/Tests/Lanes/" selector=${PARCEL_TEST_SELECTION_BINARY:-$(swift build --show-bin-path)/ParcelTestSelection} cp "$selector" "$work/bin/" cat > "$work/project/Scripts/swift-build.sh" <<'MOCK' #!/bin/sh set -eu exec swift build "$@" MOCK chmod +x "$work/project/Scripts/swift-build.sh" export TEST_BIN="$work/bin" TEST_EVENTS="$work/events" TEST_ENVIRONMENT="$work/environment" export TEST_DISCOVERY="$work/discovered" TEST_REPORT="$work/report.xml" python3 -I - "$work/project/Tests/Lanes/unit.txt" "$TEST_DISCOVERY" "$TEST_REPORT" <<'PY' import sys from pathlib import Path from xml.etree.ElementTree import Element, SubElement, ElementTree suites = [s for s in Path(sys.argv[1]).read_text().splitlines() if s and not s.startswith('#')] other = [s for lane in ['native-host', 'bench'] for s in Path(sys.argv[1]).with_name(lane + '.txt').read_text().splitlines() if s and not s.startswith('#')] Path(sys.argv[2]).write_text('\n'.join(s if s.endswith(')') else s + '/example()' for s in suites + other) + '\n') root = Element('testsuites') for suite in suites: if suite.endswith(')'): classname, name = suite.split('.', 1) else: classname, name = suite.replace('/', '.'), 'example()' SubElement(root, 'testcase', classname=classname, name=name) ElementTree(root).write(sys.argv[3]) PY cat > "$work/bin/swift" <<'MOCK' #!/bin/bash set -euo pipefail printf '%s\n' "$*" >> "$TEST_EVENTS" if [[ $* == 'build --show-bin-path' ]]; then printf '%s\n' "$TEST_BIN"; exit; fi if [[ $* == 'build --build-tests' ]]; then exit "${TEST_BUILD_STATUS:-0}"; fi if [[ $* == 'test list --skip-build' ]]; then printf '%s\n' "$(< "$TEST_DISCOVERY")" exit "${TEST_LIST_STATUS:-0}" fi printf '%s|%s|%s|%s|%s\n' "${PARCEL_NATIVE_HOST:-unset}" "${PARCEL_E2E:-unset}" "${PARCEL_BENCH:-unset}" "${PARCEL_BENCH_SMOKE:-unset}" "${PARCEL_LOCALIZATION_SCREENSHOTS:-unset}" > "$TEST_ENVIRONMENT" previous= for arg in "$@"; do if [[ $previous == --xunit-output && ${TEST_NO_REPORT:-0} != 1 ]]; then cp "$TEST_REPORT" "$arg"; fi previous=$arg done exit "${TEST_SWIFT_STATUS:-0}" MOCK chmod +x "$work/bin/swift" export PATH="$work/bin:$PATH" count=0 check() { "$@" || { printf 'Test runner check failed: %s\n' "$*" >&2; exit 1; }; count=$((count + 1)); } fails_with() { local expected=$1 status=0 shift "$@" > "$work/failure.log" 2>&1 || status=$? [[ $status == "$expected" ]] } runner="$work/project/Scripts/test.sh" check env PARCEL_NATIVE_HOST=1 PARCEL_E2E=1 PARCEL_BENCH=1 PARCEL_BENCH_SMOKE=1 PARCEL_LOCALIZATION_SCREENSHOTS=/private/captures bash "$runner" printf 'unset|unset|unset|unset|unset\n' > "$work/expected" check cmp "$work/expected" "$TEST_ENVIRONMENT" check test -f "$work/project/build/junit.xml" check grep -q '^build --build-tests$' "$TEST_EVENTS" check grep -q '^test list --skip-build$' "$TEST_EVENTS" check grep -q '^test --skip-build --disable-xctest --parallel --filter \^' "$TEST_EVENTS" : > "$TEST_EVENTS" check env PARCEL_TEST_SKIP_BUILD=1 PARCEL_TEST_XUNIT_OUTPUT="$work/custom.xml" bash "$runner" --no-parallel check test -f "$work/custom.xml" check test "$(grep -c '^build --build-tests$' "$TEST_EVENTS" || true)" == 0 check grep -q '^test --skip-build --disable-xctest --no-parallel --filter ' "$TEST_EVENTS" check bash "$runner" --skip-build --xunit-output "$work/argument.xml" check test -f "$work/argument.xml" check fails_with 2 bash "$runner" --package-path /other check fails_with 2 bash "$runner" --filter check fails_with 2 bash "$runner" --skip unsafe check fails_with 1 bash "$runner" --filter 'NoSuchTest' check fails_with 1 bash "$runner" --filter 'AppKitTestSuite|ParcelPerformanceTests' check bash "$runner" --skip-build check fails_with 17 env TEST_BUILD_STATUS=17 bash "$runner" check test ! -e "$work/project/build/junit.xml" check fails_with 19 env TEST_LIST_STATUS=19 bash "$runner" --skip-build check fails_with 7 env TEST_SWIFT_STATUS=7 bash "$runner" --skip-build --xunit-output "$work/failed.xml" check test -f "$work/failed.xml" check fails_with 1 env TEST_NO_REPORT=1 bash "$runner" --skip-build check test ! -e "$work/project/build/junit.xml" check bash "$runner" --skip-build check fails_with 19 env TEST_LIST_STATUS=19 bash "$runner" --skip-build check test ! -e "$work/project/build/junit.xml" cp "$TEST_DISCOVERY" "$work/saved-discovery" for identifier in 'ParcelE2ETests.NewScreenFreeTests/escape()' 'ParcelPerformanceTests.NewBenchmarkTests/escape()'; do printf '%s\n' "$identifier" >> "$TEST_DISCOVERY" check fails_with 1 bash "$runner" --skip-build cp "$work/saved-discovery" "$TEST_DISCOVERY" done cp "$work/project/Tests/Lanes/unit.txt" "$work/saved-unit" printf 'ParcelPerformanceTests.PerformanceTestSuite\n' >> "$work/project/Tests/Lanes/unit.txt" check fails_with 1 bash "$runner" --skip-build cp "$work/saved-unit" "$work/project/Tests/Lanes/unit.txt" "$TEST_BIN/ParcelTestSelection" select "$work/project/Tests/Lanes/bench.txt" "$TEST_DISCOVERY" '' "$TEST_REPORT" > "$work/bench-filter" check grep -Fq 'ParcelPerformanceTests\.PerformanceTestSuite' "$work/bench-filter" : > "$TEST_EVENTS" check env PARCEL_TEST_SKIP_BUILD=1 bash "$work/project/Scripts/test-lane.sh" inventory check grep -q '^build --build-tests$' "$TEST_EVENTS" check test "$(grep -c '^test --skip-build ' "$TEST_EVENTS" || true)" == 0 check fails_with 2 bash "$work/project/Scripts/test-lane.sh" inventory --filter 'Geometry' printf 'NewTests.Unclassified/test()\n' >> "$TEST_DISCOVERY" check fails_with 1 bash "$work/project/Scripts/test-lane.sh" inventory cp "$work/saved-discovery" "$TEST_DISCOVERY" check grep -Fq 'ParcelE2ETests.FixtureReplyTests' "$work/project/Tests/Lanes/unit.txt" : > "$TEST_DISCOVERY" check fails_with 1 bash "$runner" --skip-build cp "$work/saved-discovery" "$TEST_DISCOVERY" python3 -I - "$TEST_DISCOVERY" <<'PY' import sys from pathlib import Path p = Path(sys.argv[1]); p.write_text('\n'.join(p.read_text().splitlines()[1:]) + '\n') PY check fails_with 1 bash "$runner" --skip-build cp "$work/saved-discovery" "$TEST_DISCOVERY" cp "$TEST_REPORT" "$work/saved-report" printf '\n' > "$TEST_REPORT" check fails_with 1 bash "$runner" --skip-build python3 -I - "$work/saved-report" "$TEST_REPORT" <<'PY' import sys from xml.etree.ElementTree import parse, SubElement report = parse(sys.argv[1]) report.getroot().remove(report.getroot()[0]) report.write(sys.argv[2]) PY check fails_with 1 bash "$runner" --skip-build python3 -I - "$work/saved-report" "$TEST_REPORT" <<'PY' import sys from xml.etree.ElementTree import parse, SubElement report = parse(sys.argv[1]) SubElement(report.getroot()[0], 'skipped') report.write(sys.argv[2]) PY check fails_with 1 bash "$runner" --skip-build cp "$work/saved-report" "$TEST_REPORT" printf 'UnexpectedTests.NewSuite/escape()\n' >> "$TEST_DISCOVERY" check fails_with 1 bash "$runner" --skip-build cp "$work/saved-discovery" "$TEST_DISCOVERY" python3 -I - "$work/saved-report" "$TEST_REPORT" <<'PY' import sys from xml.etree.ElementTree import parse, SubElement report = parse(sys.argv[1]) SubElement(report.getroot(), 'testcase', classname='UnexpectedTests.NewSuite', name='escape()') report.write(sys.argv[2]) PY check fails_with 1 bash "$runner" --skip-build printf '\n' > "$TEST_REPORT" check fails_with 1 bash "$runner" --skip-build for mode in missing-global skipped-global unexpected-global; do python3 -I - "$work/saved-report" "$TEST_REPORT" "$mode" <<'PY' import sys from xml.etree.ElementTree import parse, SubElement report = parse(sys.argv[1]) case = next(c for c in report.getroot() if '.' not in c.attrib['classname']) if sys.argv[3] == 'missing-global': report.getroot().remove(case) elif sys.argv[3] == 'skipped-global': SubElement(case, 'skipped') else: SubElement(report.getroot(), 'testcase', classname=case.attrib['classname'], name='unexpectedGlobal()') report.write(sys.argv[2]) PY check fails_with 1 bash "$runner" --skip-build done cp "$work/saved-report" "$TEST_REPORT" printf 'UnexpectedTests.globalEscape()\n' >> "$TEST_DISCOVERY" check fails_with 1 bash "$runner" --skip-build cp "$work/saved-discovery" "$TEST_DISCOVERY" python3 -I - "$work/saved-report" "$TEST_REPORT" "$work/global-filter" <<'PY' import re, sys from pathlib import Path from xml.etree.ElementTree import parse, Element, ElementTree report = parse(sys.argv[1]) case = next(c for c in report.getroot() if '.' not in c.attrib['classname']) root = Element('testsuites'); root.append(case) ElementTree(root).write(sys.argv[2]) Path(sys.argv[3]).write_text('^' + re.escape(case.attrib['classname'] + '.' + case.attrib['name']) + '$') PY check bash "$runner" --skip-build --filter "$(< "$work/global-filter")" python3 -I - "$work/project/Tests/Lanes/bench.txt" "$TEST_REPORT" <<'PY' import sys from pathlib import Path from xml.etree.ElementTree import Element, SubElement, ElementTree root = Element('testsuites') for suite in Path(sys.argv[1]).read_text().splitlines(): if suite and not suite.startswith('#'): SubElement(root, 'testcase', classname=suite.replace('/', '.'), name='example()') ElementTree(root).write(sys.argv[2]) PY : > "$TEST_EVENTS" check env PARCEL_NATIVE_HOST=1 PARCEL_E2E=1 PARCEL_BENCH=1 PARCEL_LOCALIZATION_SCREENSHOTS=/private/captures bash "$work/project/Scripts/test-lane.sh" bench --skip-build printf 'unset|unset|unset|1|unset\n' > "$work/expected" check cmp "$work/expected" "$TEST_ENVIRONMENT" check test -f "$work/project/build/junit-bench.xml" check grep -q '^test --skip-build --disable-xctest --no-parallel --filter ' "$TEST_EVENTS" check fails_with 2 bash "$work/project/Scripts/test-lane.sh" bench --parallel check fails_with 1 bash "$work/project/Scripts/test-lane.sh" bench --skip-build --filter 'SettingsTests' cp "$TEST_REPORT" "$work/bench-report" for mode in missing-suite skipped-suite unexpected-suite empty; do python3 -I - "$work/bench-report" "$TEST_REPORT" "$mode" <<'PY' import sys from xml.etree.ElementTree import parse, SubElement report = parse(sys.argv[1]); root = report.getroot() if sys.argv[3] == 'missing-suite': root.remove(root[0]) elif sys.argv[3] == 'skipped-suite': SubElement(root[0], 'skipped') elif sys.argv[3] == 'unexpected-suite': SubElement(root, 'testcase', classname='UnexpectedTests.Suite', name='escape()') else: root.clear() report.write(sys.argv[2]) PY check fails_with 1 bash "$work/project/Scripts/test-lane.sh" bench --skip-build done python3 -I - "$work/saved-discovery" "$TEST_DISCOVERY" "$work/bench-report" "$TEST_REPORT" <<'PY' import sys from pathlib import Path from xml.etree.ElementTree import parse, SubElement parent = 'ParcelPerformanceTests.EngineCorePerformanceTests' text = Path(sys.argv[1]).read_text() old = parent + '/example()' assert text.count(old) == 1 Path(sys.argv[2]).write_text(text.replace(old, parent + '/Counters/counter()\n' + parent + '/Workloads/workload()')) report = parse(sys.argv[3]); root = report.getroot() case = next(c for c in root if c.attrib['classname'] == parent) root.remove(case) SubElement(root, 'testcase', classname=parent + '.Counters', name='counter()') SubElement(root, 'testcase', classname=parent + '.Workloads', name='workload()') report.write(sys.argv[4]) PY check bash "$work/project/Scripts/test-lane.sh" bench --skip-build cp "$TEST_REPORT" "$work/nested-bench-report" for child in Counters Workloads; do python3 -I - "$work/nested-bench-report" "$TEST_REPORT" "$child" <<'PY' import sys from xml.etree.ElementTree import parse, SubElement report = parse(sys.argv[1]) case = next(c for c in report.getroot() if c.attrib['classname'].endswith('.' + sys.argv[3])) SubElement(case, 'skipped') report.write(sys.argv[2]) PY check fails_with 1 bash "$work/project/Scripts/test-lane.sh" bench --skip-build check grep -q 'Required suite did not execute' "$work/failure.log" done cp "$work/saved-discovery" "$TEST_DISCOVERY" cp "$work/bench-report" "$TEST_REPORT" mkdir -p "$work/toolchain/usr/bin" "$work/toolchain/usr/libexec/swift/pm" "$work/sdk/Developer/Library/Frameworks" "$TEST_BIN/ParcelPackageTests.xctest/Contents/MacOS" : > "$TEST_BIN/ParcelPackageTests.xctest/Contents/MacOS/ParcelPackageTests" export TEST_TOOLCHAIN="$work/toolchain" TEST_SDK="$work/sdk" export TEST_NATIVE_ARGUMENTS="$work/native-arguments" TEST_HOST_INFO="$work/host.plist" cat > "$work/bin/xcrun" <<'MOCK' #!/bin/bash set -euo pipefail case "$*" in '--find swift') printf '%s\n' "$TEST_TOOLCHAIN/usr/bin/swift" ;; '--show-sdk-platform-path') printf '%s\n' "$TEST_SDK" ;; *) exit 2 ;; esac MOCK cat > "$work/toolchain/usr/libexec/swift/pm/swiftpm-testing-helper" <<'MOCK' #!/bin/bash set -euo pipefail cp "$(dirname "$0")/../Info.plist" "$TEST_HOST_INFO" printf '%s\n' "$@" > "$TEST_NATIVE_ARGUMENTS" exec "$TEST_BIN/swift" test "$@" MOCK chmod +x "$work/bin/xcrun" "$work/toolchain/usr/libexec/swift/pm/swiftpm-testing-helper" python3 -I - "$work/project/Tests/Lanes/native-host.txt" "$TEST_REPORT" <<'PY' import sys from pathlib import Path from xml.etree.ElementTree import Element, SubElement, ElementTree root = Element('testsuites') for suite in Path(sys.argv[1]).read_text().splitlines(): if suite and not suite.startswith('#'): SubElement(root, 'testcase', classname=suite.replace('/', '.'), name='example()') ElementTree(root).write(sys.argv[2]) PY check env PARCEL_NATIVE_HOST=1 PARCEL_LOCALIZATION_LANGUAGE=de bash "$work/project/Scripts/test-lane.sh" native-host --skip-build check grep -q '^--no-parallel$' "$TEST_NATIVE_ARGUMENTS" check grep -q '^-AppleLanguages$' "$TEST_NATIVE_ARGUMENTS" check grep -q '^(de)$' "$TEST_NATIVE_ARGUMENTS" check test "$(/usr/libexec/PlistBuddy -c 'Print :CFBundleAllowMixedLocalizations' "$TEST_HOST_INFO")" == true check test "$(find "$work/project/.build" -name 'native-test-host.*' -print | wc -l | tr -d ' ')" == 0 check env PARCEL_NATIVE_HOST=1 PARCEL_PSEUDO_LOCALIZATION=1 PARCEL_LOCALIZATION_LANGUAGE=de bash "$work/project/Scripts/test-lane.sh" native-host --skip-build check grep -q '^(en-XA)$' "$TEST_NATIVE_ARGUMENTS" check fails_with 2 env PARCEL_NATIVE_HOST=1 PARCEL_LOCALIZATION_LANGUAGE=invalid bash "$work/project/Scripts/test-lane.sh" native-host --skip-build for mode in all-skipped permission-skipped; do python3 -I - "$work/project/Tests/Lanes/native-host.txt" "$TEST_REPORT" "$mode" <<'PY' import sys from pathlib import Path from xml.etree.ElementTree import Element, SubElement, ElementTree suites = [s for s in Path(sys.argv[1]).read_text().splitlines() if s and not s.startswith('#')] root = Element('testsuites') for suite in suites: case = SubElement(root, 'testcase', classname=suite.replace('/', '.'), name='example()') if sys.argv[3] == 'all-skipped' or suite.endswith('/InputPermissionTests'): SubElement(case, 'skipped', message='Accessibility is not granted') ElementTree(root).write(sys.argv[2]) PY if [[ $mode == all-skipped ]]; then check fails_with 1 "$TEST_BIN/ParcelTestSelection" check-native "$work/project/Tests/Lanes/native-host.txt" "$TEST_DISCOVERY" '' "$TEST_REPORT" else check "$TEST_BIN/ParcelTestSelection" check-native "$work/project/Tests/Lanes/native-host.txt" "$TEST_DISCOVERY" '' "$TEST_REPORT" fi done check fails_with 2 env PARCEL_NATIVE_HOST=0 bash "$work/project/Scripts/test-native-host.sh" cat > "$work/project/Scripts/screen-lock.sh" <<'MOCK' #!/bin/bash set -euo pipefail printf '%s\n' "${PARCEL_SCREEN_OWNER:-}" > "$TEST_LOCK_OWNER" exit 29 MOCK chmod +x "$work/project/Scripts/screen-lock.sh" check fails_with 29 env PARCEL_NATIVE_HOST=1 TEST_LOCK_OWNER="$work/owner" bash "$work/project/Scripts/test-native-host.sh" printf 'native-host\n' > "$work/expected" check cmp "$work/expected" "$work/owner" for target in test test-serial test-scripts; do make --no-print-directory --dry-run "$target" > "$work/actual" case "$target" in test) printf '%s\n' 'mise run test' > "$work/expected" ;; test-serial) printf '%s\n' 'mise run test:serial' > "$work/expected" ;; test-scripts) printf '%s\n' 'mise run test:scripts' > "$work/expected" ;; esac check cmp "$work/expected" "$work/actual" done printf 'Test runner checks passed: %s\n' "$count"