import Foundation import Testing @testable import ParcelDev struct ScriptLaneToolsTests { private struct Fixture { let work: URL let root: URL let output: URL let calls: URL var environment: [String: String] init() throws { work = FileManager.default.temporaryDirectory.appendingPathComponent("parcel-script-lane-\(UUID())") root = work.appendingPathComponent("repository with spaces") output = work.appendingPathComponent("reports with spaces") calls = work.appendingPathComponent("swift-calls") for directory in [root.appendingPathComponent("Scripts/tests"), work.appendingPathComponent("bin")] { try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) } environment = ProcessInfo.processInfo.environment environment["PATH"] = work.appendingPathComponent("bin").path + ":/usr/bin:/bin" environment["MOCK_CALLS"] = calls.path environment["MOCK_DISCOVERY"] = "NameGuardTests.GuardTests/valid()\nParcelDevTests.ToolTests/valid()" environment["MOCK_DISCOVERY_STATUS"] = "0" environment["MOCK_ROOT_STATUS"] = "0" environment["MOCK_ROOT_XML"] = #""# environment["PARCEL_SCRIPT_ROOT_TESTS_DONE"] = nil environment["SHELL_ORDER"] = work.appendingPathComponent("shell-order").path try write( work.appendingPathComponent("bin/swift"), #""" #!/bin/bash set -euo pipefail printf '%s\n' "$*" >> "$MOCK_CALLS" [[ $1 == test ]] || exit 90 [[ "$*" == *--skip-build* ]] || exit 91 [[ "$*" == *'^(NameGuardTests|ParcelDevTests)\.'* ]] || exit 92 for name in PARCEL_E2E PARCEL_E2E_PREFLIGHT_ONLY PARCEL_NATIVE_HOST PARCEL_BENCH PARCEL_BENCH_SMOKE PARCEL_REALITY PARCEL_REALITY_VOICEOVER PARCEL_REALITY_SCREEN_CAPTURE PARCEL_TEST_SKIP_BUILD PARCEL_TEST_XUNIT_OUTPUT PARCEL_SCRIPT_RESULTS; do [[ -z ${!name+x} ]] || { printf 'live flag leaked: %s\n' "$name" >&2; exit 93; } done [[ -z ${PARCEL_SCRIPT_ROOT_TESTS_DONE+x} ]] || exit 95 if [[ ${2:-} == list ]]; then printf '%s\n' "$MOCK_DISCOVERY" exit "$MOCK_DISCOVERY_STATUS" fi report='' while [[ $# -gt 0 ]]; do if [[ $1 == --xunit-output ]]; then shift report=$1 break fi shift done [[ -n $report ]] || exit 94 if [[ $MOCK_ROOT_XML != MISSING ]]; then printf '%s\n' "$MOCK_ROOT_XML" > "$report" fi printf 'mock root stdout\n' printf 'mock root stderr\n' >&2 exit "$MOCK_ROOT_STATUS" """#, executable: true) } func write(_ url: URL, _ text: String, executable: Bool = false) throws { try Data(text.utf8).write(to: url) try FileManager.default.setAttributes( [.posixPermissions: executable ? 0o755 : 0o644], ofItemAtPath: url.path) } func shell(_ name: String, status: Int = 0) throws { try write( root.appendingPathComponent("Scripts/tests/" + name), """ #!/bin/bash set -euo pipefail [[ ${PARCEL_SCRIPT_ROOT_TESTS_DONE:-} == 1 ]] || exit 80 for name in PARCEL_E2E PARCEL_E2E_PREFLIGHT_ONLY PARCEL_NATIVE_HOST PARCEL_BENCH PARCEL_BENCH_SMOKE PARCEL_REALITY PARCEL_REALITY_VOICEOVER PARCEL_REALITY_SCREEN_CAPTURE PARCEL_TEST_SKIP_BUILD PARCEL_TEST_XUNIT_OUTPUT PARCEL_SCRIPT_RESULTS; do [[ -z ${!name+x} ]] || exit 81 done printf '%s\\n' "$(basename "$0")" >> "$SHELL_ORDER" printf 'fixture stdout\\n' printf 'fixture stderr\\n' >&2 exit \(status) """) } func run(_ arguments: [String]? = nil) throws -> DevProcessResult { try DevProcess.run( try PolicyTestSupport.binary.path, arguments ?? ["run-script-tests", root.path, output.path], environment: environment) } var results: [ScriptLaneResult] { get throws { try JSONDecoder().decode( [ScriptLaneResult].self, from: Data(contentsOf: output.appendingPathComponent("script-results.json"))) } } var swiftCalls: [String] { get throws { guard FileManager.default.fileExists(atPath: calls.path) else { return [] } return try String(contentsOf: calls, encoding: .utf8).split(separator: "\n").map(String.init) } } func clean() throws { try FileManager.default.removeItem(at: work) } } @Test func redirectedProgressIsVisibleBeforeTheNextFixtureRuns() throws { var fixture = try Fixture() defer { #expect(throws: Never.self) { try fixture.clean() } } let progress = fixture.work.appendingPathComponent("progress.log") fixture.environment["MOCK_PROGRESS"] = progress.path try fixture.write( fixture.root.appendingPathComponent("Scripts/tests/progress.sh"), #""" #!/bin/bash set -euo pipefail grep -q '^PASS discovery ' "$MOCK_PROGRESS" grep -q '^PASS root-swift ' "$MOCK_PROGRESS" """#) try fixture.write(progress, "") let output = try FileHandle(forWritingTo: progress) defer { #expect(throws: Never.self) { try output.close() } } let process = Process() process.executableURL = try PolicyTestSupport.binary process.arguments = ["run-script-tests", fixture.root.path, fixture.output.path] process.environment = fixture.environment process.standardOutput = output process.standardError = FileHandle.nullDevice try process.run() process.waitUntilExit() #expect(process.terminationStatus == 0) #expect(try fixture.results.last?.name == "progress.sh") #expect(try fixture.results.last?.status == 0) } @Test func rootRunsExactlyOnceWithoutBuildAndShellFixturesAreSortedIncludingSpaces() throws { let fixture = try Fixture() defer { #expect(throws: Never.self) { try fixture.clean() } } try fixture.shell("z-last.sh") try fixture.shell("a first.sh") try fixture.shell("middle.sh") try fixture.write(fixture.root.appendingPathComponent("Scripts/tests/not-a-fixture.txt"), "ignored") let process = try fixture.run() #expect(process.status == 0, Comment(rawValue: process.errorOutput + process.output)) let results = try fixture.results #expect(results.map(\.name) == ["discovery", "root-swift", "a first.sh", "middle.sh", "z-last.sh"]) #expect(results.allSatisfy { $0.status == 0 && $0.seconds >= 0 && $0.detail.isEmpty }) let calls = try fixture.swiftCalls #expect(calls.count == 2) #expect(calls.first == "test list --skip-build --filter " + ScriptLaneTools.rootFilter) #expect( calls.last == "test --skip-build --disable-xctest --filter " + ScriptLaneTools.rootFilter + " --xunit-output " + fixture.output.appendingPathComponent("junit-scripts.xml").path) #expect(calls.allSatisfy { !$0.split(separator: " ").contains("build") }) let order = try String(contentsOf: fixture.work.appendingPathComponent("shell-order"), encoding: .utf8) #expect(order == "a first.sh\nmiddle.sh\nz-last.sh\n") for name in results.map(\.name) { #expect(FileManager.default.fileExists(atPath: fixture.output.appendingPathComponent(name + ".log").path)) } #expect(try ScriptLaneTools.executedTests(in: fixture.output.appendingPathComponent("junit-shell.xml")) == 3) } @Test(arguments: ["missing-dev", "missing-guard", "zero", "malformed", "discovery-failure"]) func discoveryFailureDoesNotRunRootButStillRunsAllShellFixtures(_ scenario: String) throws { var fixture = try Fixture() defer { #expect(throws: Never.self) { try fixture.clean() } } try fixture.shell("one.sh") try fixture.shell("two.sh") switch scenario { case "missing-dev": fixture.environment["MOCK_DISCOVERY"] = "NameGuardTests.GuardTests/valid()" case "missing-guard": fixture.environment["MOCK_DISCOVERY"] = "ParcelDevTests.ToolTests/valid()" case "zero": fixture.environment["MOCK_DISCOVERY"] = "" case "malformed": fixture.environment["MOCK_DISCOVERY"] = "NameGuardTests.GuardTests\nParcelDevTests.ToolTests" case "discovery-failure": fixture.environment["MOCK_DISCOVERY_STATUS"] = "7" default: Issue.record("Unknown scenario") } try FileManager.default.createDirectory(at: fixture.output, withIntermediateDirectories: true) for name in ["junit-scripts.xml", "junit-scripts-swift-testing.xml"] { try fixture.write(fixture.output.appendingPathComponent(name), "") } #expect(try fixture.run().status == 1) let results = try fixture.results #expect(results.map(\.name) == ["discovery", "root-swift", "one.sh", "two.sh"]) #expect(results[1].status == 1) #expect(results[1].detail.contains("Discovery failed or missing targets")) #expect(results.suffix(2).allSatisfy { $0.status == 0 }) #expect(try fixture.swiftCalls.count == 1) #expect( !FileManager.default.fileExists( atPath: fixture.output.appendingPathComponent("junit-scripts-swift-testing.xml").path)) } @Test(arguments: [ "zero", "skipped", "malformed", "wrong-root", "missing", "xml-failure", "xml-error", "root-exit", "guard-only", "dev-only", ]) func rootFailuresAndInvalidJUnitNeverSuppressShellResults(_ scenario: String) throws { var fixture = try Fixture() defer { #expect(throws: Never.self) { try fixture.clean() } } try fixture.shell("after.sh") switch scenario { case "zero": fixture.environment["MOCK_ROOT_XML"] = "" case "skipped": fixture.environment["MOCK_ROOT_XML"] = "" case "malformed": fixture.environment["MOCK_ROOT_XML"] = "" case "wrong-root": fixture.environment["MOCK_ROOT_XML"] = "" case "missing": fixture.environment["MOCK_ROOT_XML"] = "MISSING" case "xml-failure": fixture.environment["MOCK_ROOT_XML"] = "" case "xml-error": fixture.environment["MOCK_ROOT_XML"] = "" case "root-exit": fixture.environment["MOCK_ROOT_STATUS"] = "9" case "guard-only": fixture.environment["MOCK_ROOT_XML"] = #""# case "dev-only": fixture.environment["MOCK_ROOT_XML"] = #""# default: Issue.record("Unknown scenario") } #expect(try fixture.run().status == 1) let results = try fixture.results #expect(results.count == 3) #expect(results[0].status == 0) #expect(results[1].status != 0) #expect(!results[1].detail.isEmpty) #expect(results[2].name == "after.sh" && results[2].status == 0) #expect(try fixture.swiftCalls.count == 2) if scenario == "root-exit" { #expect(results[1].status == 9) #expect(results[1].detail.contains("mock root stdout") && results[1].detail.contains("mock root stderr")) } } @Test func oneShellFailureDoesNotOmitLaterOutcomesOrDiagnostics() throws { let fixture = try Fixture() defer { #expect(throws: Never.self) { try fixture.clean() } } try fixture.shell("a-fail.sh", status: 17) try fixture.shell("b-pass.sh") try fixture.shell("c-fail.sh", status: 23) #expect(try fixture.run().status == 1) let results = try fixture.results #expect(results.map(\.name) == ["discovery", "root-swift", "a-fail.sh", "b-pass.sh", "c-fail.sh"]) #expect(results.map(\.status) == [0, 0, 17, 0, 23]) #expect(results[2].detail.contains("fixture stdout") && results[2].detail.contains("fixture stderr")) let xml = try String(contentsOf: fixture.output.appendingPathComponent("junit-shell.xml"), encoding: .utf8) #expect(xml.contains("tests=\"3\" failures=\"2\"")) #expect(xml.contains("b-pass.sh")) let summary = try String( contentsOf: fixture.output.appendingPathComponent("script-summary.md"), encoding: .utf8) #expect( summary.contains("a-fail.sh | fail (17)") && summary.contains("b-pass.sh | pass") && summary.contains("c-fail.sh | fail (23)")) } @Test func liveAndLaneControlFlagsAreClearedForSubprocesses() throws { var fixture = try Fixture() defer { #expect(throws: Never.self) { try fixture.clean() } } for name in [ "PARCEL_E2E", "PARCEL_E2E_PREFLIGHT_ONLY", "PARCEL_NATIVE_HOST", "PARCEL_BENCH", "PARCEL_BENCH_SMOKE", "PARCEL_REALITY", "PARCEL_REALITY_VOICEOVER", "PARCEL_REALITY_SCREEN_CAPTURE", "PARCEL_TEST_SKIP_BUILD", "PARCEL_TEST_XUNIT_OUTPUT", "PARCEL_SCRIPT_RESULTS", "PARCEL_SCRIPT_ROOT_TESTS_DONE", ] { fixture.environment[name] = "1" } try fixture.shell("safe.sh") #expect(try fixture.run().status == 0) #expect(try fixture.results.allSatisfy { $0.status == 0 }) } @Test(arguments: ["no-args", "one-arg", "extra-arg", "missing-directory", "no-fixtures", "output-file"]) func commandRejectsInvalidArgumentsAndPaths(_ scenario: String) throws { let fixture = try Fixture() defer { #expect(throws: Never.self) { try fixture.clean() } } var args = ["run-script-tests", fixture.root.path, fixture.output.path] switch scenario { case "no-args": args = ["run-script-tests"] case "one-arg": args = ["run-script-tests", fixture.root.path] case "extra-arg": args.append("unexpected") case "missing-directory": try FileManager.default.removeItem(at: fixture.root.appendingPathComponent("Scripts/tests")) case "no-fixtures": break case "output-file": try fixture.shell("fixture.sh") try fixture.write(fixture.output, "not a directory") default: Issue.record("Unknown scenario") } let process = try fixture.run(args) #expect(process.status == 2) #expect(!process.errorOutput.isEmpty) #expect(try fixture.swiftCalls.isEmpty) #expect( !FileManager.default.fileExists(atPath: fixture.output.appendingPathComponent("script-results.json").path)) } @Test func xmlEscapesAttributesAndTextAndRemovesInvalidXMLScalars() throws { let original = "&<>\"'\u{0}\u{1}\u{8}\u{B}\u{C}\u{1F}\u{FFFE}\u{FFFF}\t\n\r日本語" let escaped = ScriptLaneTools.xml(original) #expect(escaped == "&<>"'\t\n\r日本語") } @Test func generatedReportsRemainValidWithEscapedNamesAndFailureDetails() throws { let fixture = try Fixture() defer { #expect(throws: Never.self) { try fixture.clean() } } try FileManager.default.createDirectory(at: fixture.output, withIntermediateDirectories: true) let results: [ScriptLaneResult] = [ .init(name: "root-swift", seconds: 0.25, status: 1, detail: "root failed"), .init(name: "a |<&\"'.sh", seconds: 0.5, status: 7, detail: "&\"'\u{1}\u{FFFF}"), .init(name: "ok.sh", seconds: 0.75, status: 0, detail: ""), ] try ScriptLaneTools.write(results, to: fixture.output) #expect(try fixture.results == results) let xmlURL = fixture.output.appendingPathComponent("junit-shell.xml") let data = try Data(contentsOf: xmlURL) #expect(XMLParser(data: data).parse()) let xml = String(decoding: data, as: UTF8.self) #expect(xml.contains("tests=\"2\" failures=\"1\"")) #expect(xml.contains("<bad>&"'")) #expect(!xml.contains("root-swift")) let summary = try String( contentsOf: fixture.output.appendingPathComponent("script-summary.md"), encoding: .utf8) #expect(summary.contains("a \\|")) #expect(summary.contains("| root-swift | fail (1) | 0.250 |")) } @Test(arguments: ["suite", "suites", "skips", "counts-not-trusted"]) func junitCountsActualExecutedCasesNotDeclaredTotals(_ scenario: String) throws { let fixture = try Fixture() defer { #expect(throws: Never.self) { try fixture.clean() } } let file = fixture.work.appendingPathComponent("report.xml") let text = switch scenario { case "suite": "" case "suites": "" case "skips": "" default: "" } try fixture.write(file, text) #expect(try ScriptLaneTools.executedTests(in: file) == 2) } @Test(arguments: ["missing", "malformed", "wrong-root", "failure", "error"]) func invalidOrFailingJUnitThrows(_ scenario: String) throws { let fixture = try Fixture() defer { #expect(throws: Never.self) { try fixture.clean() } } let file = fixture.work.appendingPathComponent("report.xml") let text = switch scenario { case "malformed": "" case "wrong-root": "" case "failure": "" default: "" } if scenario != "missing" { try fixture.write(file, text) } #expect(throws: (any Error).self) { try ScriptLaneTools.executedTests(in: file) } } }