import Foundation
import Testing
@testable import ParcelDev
struct ScriptLaneToolsTests {
private struct Fixture {
let work: URL
let root: URL
let output: URL
let calls: URL
var environment: [String: String]
init() throws {
work = FileManager.default.temporaryDirectory.appendingPathComponent("parcel-script-lane-\(UUID())")
root = work.appendingPathComponent("repository with spaces")
output = work.appendingPathComponent("reports with spaces")
calls = work.appendingPathComponent("swift-calls")
for directory in [root.appendingPathComponent("Scripts/tests"), work.appendingPathComponent("bin")] {
try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true)
}
environment = ProcessInfo.processInfo.environment
environment["PATH"] = work.appendingPathComponent("bin").path + ":/usr/bin:/bin"
environment["MOCK_CALLS"] = calls.path
environment["MOCK_DISCOVERY"] = "NameGuardTests.GuardTests/valid()\nParcelDevTests.ToolTests/valid()"
environment["MOCK_DISCOVERY_STATUS"] = "0"
environment["MOCK_ROOT_STATUS"] = "0"
environment["MOCK_ROOT_XML"] =
#""#
environment["PARCEL_SCRIPT_ROOT_TESTS_DONE"] = nil
environment["SHELL_ORDER"] = work.appendingPathComponent("shell-order").path
try write(
work.appendingPathComponent("bin/swift"),
#"""
#!/bin/bash
set -euo pipefail
printf '%s\n' "$*" >> "$MOCK_CALLS"
[[ $1 == test ]] || exit 90
[[ "$*" == *--skip-build* ]] || exit 91
[[ "$*" == *'^(NameGuardTests|ParcelDevTests)\.'* ]] || exit 92
for name in PARCEL_E2E PARCEL_E2E_PREFLIGHT_ONLY PARCEL_NATIVE_HOST PARCEL_BENCH PARCEL_BENCH_SMOKE PARCEL_REALITY PARCEL_REALITY_VOICEOVER PARCEL_REALITY_SCREEN_CAPTURE PARCEL_TEST_SKIP_BUILD PARCEL_TEST_XUNIT_OUTPUT PARCEL_SCRIPT_RESULTS; do
[[ -z ${!name+x} ]] || { printf 'live flag leaked: %s\n' "$name" >&2; exit 93; }
done
[[ -z ${PARCEL_SCRIPT_ROOT_TESTS_DONE+x} ]] || exit 95
if [[ ${2:-} == list ]]; then
printf '%s\n' "$MOCK_DISCOVERY"
exit "$MOCK_DISCOVERY_STATUS"
fi
report=''
while [[ $# -gt 0 ]]; do
if [[ $1 == --xunit-output ]]; then
shift
report=$1
break
fi
shift
done
[[ -n $report ]] || exit 94
if [[ $MOCK_ROOT_XML != MISSING ]]; then
printf '%s\n' "$MOCK_ROOT_XML" > "$report"
fi
printf 'mock root stdout\n'
printf 'mock root stderr\n' >&2
exit "$MOCK_ROOT_STATUS"
"""#, executable: true)
}
func write(_ url: URL, _ text: String, executable: Bool = false) throws {
try Data(text.utf8).write(to: url)
try FileManager.default.setAttributes(
[.posixPermissions: executable ? 0o755 : 0o644], ofItemAtPath: url.path)
}
func shell(_ name: String, status: Int = 0) throws {
try write(
root.appendingPathComponent("Scripts/tests/" + name),
"""
#!/bin/bash
set -euo pipefail
[[ ${PARCEL_SCRIPT_ROOT_TESTS_DONE:-} == 1 ]] || exit 80
for name in PARCEL_E2E PARCEL_E2E_PREFLIGHT_ONLY PARCEL_NATIVE_HOST PARCEL_BENCH PARCEL_BENCH_SMOKE PARCEL_REALITY PARCEL_REALITY_VOICEOVER PARCEL_REALITY_SCREEN_CAPTURE PARCEL_TEST_SKIP_BUILD PARCEL_TEST_XUNIT_OUTPUT PARCEL_SCRIPT_RESULTS; do
[[ -z ${!name+x} ]] || exit 81
done
printf '%s\\n' "$(basename "$0")" >> "$SHELL_ORDER"
printf 'fixture stdout\\n'
printf 'fixture stderr\\n' >&2
exit \(status)
""")
}
func run(_ arguments: [String]? = nil) throws -> DevProcessResult {
try DevProcess.run(
try PolicyTestSupport.binary.path,
arguments ?? ["run-script-tests", root.path, output.path], environment: environment)
}
var results: [ScriptLaneResult] {
get throws {
try JSONDecoder().decode(
[ScriptLaneResult].self,
from: Data(contentsOf: output.appendingPathComponent("script-results.json")))
}
}
var swiftCalls: [String] {
get throws {
guard FileManager.default.fileExists(atPath: calls.path) else { return [] }
return try String(contentsOf: calls, encoding: .utf8).split(separator: "\n").map(String.init)
}
}
func clean() throws { try FileManager.default.removeItem(at: work) }
}
@Test func redirectedProgressIsVisibleBeforeTheNextFixtureRuns() throws {
var fixture = try Fixture()
defer { #expect(throws: Never.self) { try fixture.clean() } }
let progress = fixture.work.appendingPathComponent("progress.log")
fixture.environment["MOCK_PROGRESS"] = progress.path
try fixture.write(
fixture.root.appendingPathComponent("Scripts/tests/progress.sh"),
#"""
#!/bin/bash
set -euo pipefail
grep -q '^PASS discovery ' "$MOCK_PROGRESS"
grep -q '^PASS root-swift ' "$MOCK_PROGRESS"
"""#)
try fixture.write(progress, "")
let output = try FileHandle(forWritingTo: progress)
defer { #expect(throws: Never.self) { try output.close() } }
let process = Process()
process.executableURL = try PolicyTestSupport.binary
process.arguments = ["run-script-tests", fixture.root.path, fixture.output.path]
process.environment = fixture.environment
process.standardOutput = output
process.standardError = FileHandle.nullDevice
try process.run()
process.waitUntilExit()
#expect(process.terminationStatus == 0)
#expect(try fixture.results.last?.name == "progress.sh")
#expect(try fixture.results.last?.status == 0)
}
@Test func rootRunsExactlyOnceWithoutBuildAndShellFixturesAreSortedIncludingSpaces() throws {
let fixture = try Fixture()
defer { #expect(throws: Never.self) { try fixture.clean() } }
try fixture.shell("z-last.sh")
try fixture.shell("a first.sh")
try fixture.shell("middle.sh")
try fixture.write(fixture.root.appendingPathComponent("Scripts/tests/not-a-fixture.txt"), "ignored")
let process = try fixture.run()
#expect(process.status == 0, Comment(rawValue: process.errorOutput + process.output))
let results = try fixture.results
#expect(results.map(\.name) == ["discovery", "root-swift", "a first.sh", "middle.sh", "z-last.sh"])
#expect(results.allSatisfy { $0.status == 0 && $0.seconds >= 0 && $0.detail.isEmpty })
let calls = try fixture.swiftCalls
#expect(calls.count == 2)
#expect(calls.first == "test list --skip-build --filter " + ScriptLaneTools.rootFilter)
#expect(
calls.last == "test --skip-build --disable-xctest --filter " + ScriptLaneTools.rootFilter
+ " --xunit-output "
+ fixture.output.appendingPathComponent("junit-scripts.xml").path)
#expect(calls.allSatisfy { !$0.split(separator: " ").contains("build") })
let order = try String(contentsOf: fixture.work.appendingPathComponent("shell-order"), encoding: .utf8)
#expect(order == "a first.sh\nmiddle.sh\nz-last.sh\n")
for name in results.map(\.name) {
#expect(FileManager.default.fileExists(atPath: fixture.output.appendingPathComponent(name + ".log").path))
}
#expect(try ScriptLaneTools.executedTests(in: fixture.output.appendingPathComponent("junit-shell.xml")) == 3)
}
@Test(arguments: ["missing-dev", "missing-guard", "zero", "malformed", "discovery-failure"])
func discoveryFailureDoesNotRunRootButStillRunsAllShellFixtures(_ scenario: String) throws {
var fixture = try Fixture()
defer { #expect(throws: Never.self) { try fixture.clean() } }
try fixture.shell("one.sh")
try fixture.shell("two.sh")
switch scenario {
case "missing-dev": fixture.environment["MOCK_DISCOVERY"] = "NameGuardTests.GuardTests/valid()"
case "missing-guard": fixture.environment["MOCK_DISCOVERY"] = "ParcelDevTests.ToolTests/valid()"
case "zero": fixture.environment["MOCK_DISCOVERY"] = ""
case "malformed": fixture.environment["MOCK_DISCOVERY"] = "NameGuardTests.GuardTests\nParcelDevTests.ToolTests"
case "discovery-failure": fixture.environment["MOCK_DISCOVERY_STATUS"] = "7"
default: Issue.record("Unknown scenario")
}
try FileManager.default.createDirectory(at: fixture.output, withIntermediateDirectories: true)
for name in ["junit-scripts.xml", "junit-scripts-swift-testing.xml"] {
try fixture.write(fixture.output.appendingPathComponent(name), "")
}
#expect(try fixture.run().status == 1)
let results = try fixture.results
#expect(results.map(\.name) == ["discovery", "root-swift", "one.sh", "two.sh"])
#expect(results[1].status == 1)
#expect(results[1].detail.contains("Discovery failed or missing targets"))
#expect(results.suffix(2).allSatisfy { $0.status == 0 })
#expect(try fixture.swiftCalls.count == 1)
#expect(
!FileManager.default.fileExists(
atPath: fixture.output.appendingPathComponent("junit-scripts-swift-testing.xml").path))
}
@Test(arguments: [
"zero", "skipped", "malformed", "wrong-root", "missing", "xml-failure", "xml-error", "root-exit", "guard-only",
"dev-only",
])
func rootFailuresAndInvalidJUnitNeverSuppressShellResults(_ scenario: String) throws {
var fixture = try Fixture()
defer { #expect(throws: Never.self) { try fixture.clean() } }
try fixture.shell("after.sh")
switch scenario {
case "zero": fixture.environment["MOCK_ROOT_XML"] = ""
case "skipped": fixture.environment["MOCK_ROOT_XML"] = ""
case "malformed": fixture.environment["MOCK_ROOT_XML"] = ""
case "wrong-root": fixture.environment["MOCK_ROOT_XML"] = ""
case "missing": fixture.environment["MOCK_ROOT_XML"] = "MISSING"
case "xml-failure":
fixture.environment["MOCK_ROOT_XML"] = ""
case "xml-error": fixture.environment["MOCK_ROOT_XML"] = ""
case "root-exit": fixture.environment["MOCK_ROOT_STATUS"] = "9"
case "guard-only":
fixture.environment["MOCK_ROOT_XML"] =
#""#
case "dev-only":
fixture.environment["MOCK_ROOT_XML"] =
#""#
default: Issue.record("Unknown scenario")
}
#expect(try fixture.run().status == 1)
let results = try fixture.results
#expect(results.count == 3)
#expect(results[0].status == 0)
#expect(results[1].status != 0)
#expect(!results[1].detail.isEmpty)
#expect(results[2].name == "after.sh" && results[2].status == 0)
#expect(try fixture.swiftCalls.count == 2)
if scenario == "root-exit" {
#expect(results[1].status == 9)
#expect(results[1].detail.contains("mock root stdout") && results[1].detail.contains("mock root stderr"))
}
}
@Test func oneShellFailureDoesNotOmitLaterOutcomesOrDiagnostics() throws {
let fixture = try Fixture()
defer { #expect(throws: Never.self) { try fixture.clean() } }
try fixture.shell("a-fail.sh", status: 17)
try fixture.shell("b-pass.sh")
try fixture.shell("c-fail.sh", status: 23)
#expect(try fixture.run().status == 1)
let results = try fixture.results
#expect(results.map(\.name) == ["discovery", "root-swift", "a-fail.sh", "b-pass.sh", "c-fail.sh"])
#expect(results.map(\.status) == [0, 0, 17, 0, 23])
#expect(results[2].detail.contains("fixture stdout") && results[2].detail.contains("fixture stderr"))
let xml = try String(contentsOf: fixture.output.appendingPathComponent("junit-shell.xml"), encoding: .utf8)
#expect(xml.contains("tests=\"3\" failures=\"2\""))
#expect(xml.contains("b-pass.sh"))
let summary = try String(
contentsOf: fixture.output.appendingPathComponent("script-summary.md"), encoding: .utf8)
#expect(
summary.contains("a-fail.sh | fail (17)") && summary.contains("b-pass.sh | pass")
&& summary.contains("c-fail.sh | fail (23)"))
}
@Test func liveAndLaneControlFlagsAreClearedForSubprocesses() throws {
var fixture = try Fixture()
defer { #expect(throws: Never.self) { try fixture.clean() } }
for name in [
"PARCEL_E2E", "PARCEL_E2E_PREFLIGHT_ONLY", "PARCEL_NATIVE_HOST", "PARCEL_BENCH", "PARCEL_BENCH_SMOKE",
"PARCEL_REALITY",
"PARCEL_REALITY_VOICEOVER", "PARCEL_REALITY_SCREEN_CAPTURE",
"PARCEL_TEST_SKIP_BUILD", "PARCEL_TEST_XUNIT_OUTPUT", "PARCEL_SCRIPT_RESULTS",
"PARCEL_SCRIPT_ROOT_TESTS_DONE",
] {
fixture.environment[name] = "1"
}
try fixture.shell("safe.sh")
#expect(try fixture.run().status == 0)
#expect(try fixture.results.allSatisfy { $0.status == 0 })
}
@Test(arguments: ["no-args", "one-arg", "extra-arg", "missing-directory", "no-fixtures", "output-file"])
func commandRejectsInvalidArgumentsAndPaths(_ scenario: String) throws {
let fixture = try Fixture()
defer { #expect(throws: Never.self) { try fixture.clean() } }
var args = ["run-script-tests", fixture.root.path, fixture.output.path]
switch scenario {
case "no-args": args = ["run-script-tests"]
case "one-arg": args = ["run-script-tests", fixture.root.path]
case "extra-arg": args.append("unexpected")
case "missing-directory":
try FileManager.default.removeItem(at: fixture.root.appendingPathComponent("Scripts/tests"))
case "no-fixtures": break
case "output-file":
try fixture.shell("fixture.sh")
try fixture.write(fixture.output, "not a directory")
default: Issue.record("Unknown scenario")
}
let process = try fixture.run(args)
#expect(process.status == 2)
#expect(!process.errorOutput.isEmpty)
#expect(try fixture.swiftCalls.isEmpty)
#expect(
!FileManager.default.fileExists(atPath: fixture.output.appendingPathComponent("script-results.json").path))
}
@Test func xmlEscapesAttributesAndTextAndRemovesInvalidXMLScalars() throws {
let original = "&<>\"'\u{0}\u{1}\u{8}\u{B}\u{C}\u{1F}\u{FFFE}\u{FFFF}\t\n\r日本語"
let escaped = ScriptLaneTools.xml(original)
#expect(escaped == "&<>"'\t\n\r日本語")
}
@Test func generatedReportsRemainValidWithEscapedNamesAndFailureDetails() throws {
let fixture = try Fixture()
defer { #expect(throws: Never.self) { try fixture.clean() } }
try FileManager.default.createDirectory(at: fixture.output, withIntermediateDirectories: true)
let results: [ScriptLaneResult] = [
.init(name: "root-swift", seconds: 0.25, status: 1, detail: "root failed"),
.init(name: "a |<&\"'.sh", seconds: 0.5, status: 7, detail: "&\"'\u{1}\u{FFFF}"),
.init(name: "ok.sh", seconds: 0.75, status: 0, detail: ""),
]
try ScriptLaneTools.write(results, to: fixture.output)
#expect(try fixture.results == results)
let xmlURL = fixture.output.appendingPathComponent("junit-shell.xml")
let data = try Data(contentsOf: xmlURL)
#expect(XMLParser(data: data).parse())
let xml = String(decoding: data, as: UTF8.self)
#expect(xml.contains("tests=\"2\" failures=\"1\""))
#expect(xml.contains("<bad>&"'"))
#expect(!xml.contains("root-swift"))
let summary = try String(
contentsOf: fixture.output.appendingPathComponent("script-summary.md"), encoding: .utf8)
#expect(summary.contains("a \\|"))
#expect(summary.contains("| root-swift | fail (1) | 0.250 |"))
}
@Test(arguments: ["suite", "suites", "skips", "counts-not-trusted"])
func junitCountsActualExecutedCasesNotDeclaredTotals(_ scenario: String) throws {
let fixture = try Fixture()
defer { #expect(throws: Never.self) { try fixture.clean() } }
let file = fixture.work.appendingPathComponent("report.xml")
let text =
switch scenario {
case "suite": ""
case "suites":
""
case "skips": ""
default: ""
}
try fixture.write(file, text)
#expect(try ScriptLaneTools.executedTests(in: file) == 2)
}
@Test(arguments: ["missing", "malformed", "wrong-root", "failure", "error"])
func invalidOrFailingJUnitThrows(_ scenario: String) throws {
let fixture = try Fixture()
defer { #expect(throws: Never.self) { try fixture.clean() } }
let file = fixture.work.appendingPathComponent("report.xml")
let text =
switch scenario {
case "malformed": ""
case "wrong-root": ""
case "failure": ""
default: ""
}
if scenario != "missing" { try fixture.write(file, text) }
#expect(throws: (any Error).self) { try ScriptLaneTools.executedTests(in: file) }
}
}