Something went wrong. Try again.
An auto tiler for macOS
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304import Foundation
enum MeasurementAB { struct Host: Decodable { let hostKey: String let model: String let cores: Int let osBuild: String let swiftVersion: String let buildConfiguration: String let counterSource: String let json: DevJSON enum CodingKeys: String, CodingKey { case hostKey, model, cores, osBuild, swiftVersion, buildConfiguration, counterSource } init(from decoder: any Decoder) throws { let container = try decoder.container(keyedBy: CodingKeys.self) hostKey = try container.decode(String.self, forKey: .hostKey) model = try container.decode(String.self, forKey: .model) cores = try container.decode(Int.self, forKey: .cores) osBuild = try container.decode(String.self, forKey: .osBuild) swiftVersion = try container.decode(String.self, forKey: .swiftVersion) buildConfiguration = try container.decode(String.self, forKey: .buildConfiguration) counterSource = try container.decode(String.self, forKey: .counterSource) json = try DevJSON(from: decoder) } func compatible(with other: Host) -> Bool { hostKey == other.hostKey && model == other.model && cores == other.cores && osBuild == other.osBuild && swiftVersion == other.swiftVersion && buildConfiguration == other.buildConfiguration && counterSource == other.counterSource } } struct Result: Codable { var name: String var metric: String var reportingOnly: Bool var minimumNanoseconds: Double var instructionsPerOp: Double? var counterSource: String var checksum: Int var budget: DevJSON? = nil var reason: DevJSON? = nil } struct Report: Decodable { var version: Int var machine: String var host: Host var results: [Result] } struct CaseSummary: Encodable { let name: String let metric: String let counterSource: String let reportingOnly: Bool let gating: Bool let status: String let budget: DevJSON let reason: DevJSON let instructionsRatio: Double? let minimumNanosecondsRatio: Double let minimumNanosecondsRatio95CI: [Double] let checksum: Int let failures: [String] enum CodingKeys: String, CodingKey { case name, metric, counterSource, reportingOnly, gating, status, budget, reason, instructionsRatio case minimumNanosecondsRatio, minimumNanosecondsRatio95CI, checksum, failures } func encode(to encoder: any Encoder) throws { var container = encoder.container(keyedBy: CodingKeys.self) try container.encode(name, forKey: .name) try container.encode(metric, forKey: .metric) try container.encode(counterSource, forKey: .counterSource) try container.encode(reportingOnly, forKey: .reportingOnly) try container.encode(gating, forKey: .gating) try container.encode(status, forKey: .status) try container.encode(budget, forKey: .budget) try container.encode(reason, forKey: .reason) try container.encode(instructionsRatio, forKey: .instructionsRatio) try container.encode(minimumNanosecondsRatio, forKey: .minimumNanosecondsRatio) try container.encode(minimumNanosecondsRatio95CI, forKey: .minimumNanosecondsRatio95CI) try container.encode(checksum, forKey: .checksum) try container.encode(failures, forKey: .failures) } } struct Summary: Encodable { let version = 1 let host: DevJSON let hostReadings: [String: [DevJSON]] let rounds = 5 let bootstrapResamples = 10_000 let bootstrapMethod = "paired round ratios, median, percentile 95%" let results: [CaseSummary] let regressions: [String] let passed: Bool var baseRevision: DevJSON = .null var headRevision: DevJSON = .null let measurementOverlay = "working-tree Package.swift and Tests/ParcelPerformanceTests on base production sources" var caseScope = "all-cases" }
static func read(_ path: String) throws -> Report { struct Schema: Decodable { let version: Int? } let data = try Data(contentsOf: URL(fileURLWithPath: path)) guard try JSONDecoder().decode(Schema.self, from: data).version == 5 else { throw DevError("\(path): A/B requires schema v5; rebuild both revisions with the measurement harness") } let report = try JSONDecoder().decode(Report.self, from: data) guard report.host.cores > 0, [ report.host.hostKey, report.host.model, report.host.osBuild, report.host.swiftVersion, report.host.buildConfiguration, report.host.counterSource, ].allSatisfy({ !$0.isEmpty }) else { throw DevError("\(path): invalid host metadata") } guard !report.machine.isEmpty else { throw DevError("\(path): machine identity is missing") } guard report.host.buildConfiguration == "release", ["selfcounts", "rusage", "cpu-only"].contains(report.host.counterSource) else { throw DevError("\(path): full release measurements are required, not smoke mode") } guard !report.results.isEmpty else { throw DevError("\(path): benchmark results are empty") } var names: Set<String> = [] for result in report.results { guard !result.name.isEmpty, names.insert(result.name).inserted else { throw DevError("\(path): duplicate or missing case name") } guard ["cpu", "wall"].contains(result.metric) else { throw DevError("\(path): \(result.name): metric invalid") } guard result.minimumNanoseconds.isFinite, result.minimumNanoseconds > 0 else { throw DevError("\(path): \(result.name): minimumNanoseconds must be positive and finite") } if let instructions = result.instructionsPerOp, !instructions.isFinite || instructions <= 0 { throw DevError("\(path): \(result.name): instructionsPerOp must be positive and finite") } guard ["selfcounts", "rusage", "cpu-only", "smoke"].contains(result.counterSource) else { throw DevError("\(path): \(result.name): unrecognized counter source") } if result.metric == "cpu", result.counterSource != "smoke", result.counterSource != report.host.counterSource { throw DevError("\(path): \(result.name): CPU result counter source differs from host") } guard result.checksum != 0 else { throw DevError("\(path): \(result.name): checksum must be a nonzero integer") } } return report }
static func compare(base: [String], head: [String]) throws -> Summary { guard base.count == 5, head.count == 5 else { throw DevError("exactly five paired A/B rounds are required") } let reports = try zip(base, head).flatMap { try [read($0), read($1)] } guard let reference = reports.first else { throw DevError("Missing A/B reports") } let names = Set(reference.results.map(\.name)) for report in reports { guard report.machine == reference.machine, report.host.compatible(with: reference.host) else { throw DevError("host, OS, toolchain or counter source changed between A/B runs") } guard Set(report.results.map(\.name)) == names else { throw DevError("benchmark cases differ between A/B runs; use compatible measurement harnesses") } } var rng = BootstrapRandom() var output: [CaseSummary] = [] var regressions: [String] = [] for name in names.sorted() { let entries = reports.compactMap { $0.results.first { $0.name == name } } guard entries.count == 10, let first = entries.first else { throw DevError("Missing benchmark case: \(name)") } guard entries.allSatisfy({ $0.metric == first.metric && $0.counterSource == first.counterSource && $0.reportingOnly == first.reportingOnly }) else { throw DevError("\(name): measurement metric, source or reporting policy changed") } let reporting = first.reportingOnly || first.metric != "cpu" || first.counterSource == "smoke" guard reporting || entries.allSatisfy({ $0.checksum == first.checksum }) else { throw DevError("\(name): checksum differs; a faster result must do the same work") } let cpu = stride(from: 0, to: 10, by: 2).map { entries[$0 + 1].minimumNanoseconds / entries[$0].minimumNanoseconds } guard cpu.allSatisfy({ $0.isFinite && $0 > 0 }) else { throw DevError("\(name): non-finite paired CPU ratio") } let counts = entries.compactMap(\.instructionsPerOp) guard counts.isEmpty || counts.count == 10 else { throw DevError("\(name): instruction availability changed between runs") } if !reporting, first.counterSource != "cpu-only", counts.isEmpty { throw DevError("\(name): instruction counter source reported without instruction readings") } let instructionRatio: Double? if counts.isEmpty { instructionRatio = nil } else { let ratios = stride(from: 0, to: 10, by: 2).map { counts[$0 + 1] / counts[$0] } guard ratios.allSatisfy({ $0.isFinite && $0 > 0 }) else { throw DevError("\(name): non-finite paired instruction ratio") } instructionRatio = try MeasurementTools.median(ratios) } var samples: [Double] = [] for _ in 0..<10_000 { samples.append(try MeasurementTools.median((0..<5).map { _ in cpu[Int(rng.next() * 5)] })) } samples.sort() let ci = [samples[249], samples[9749]] var failures: [String] = [] if !reporting { if let instructionRatio, instructionRatio > 1.03 { failures.append("instructions exceed +3%") } if ci[0] > 1.05 { failures.append("95% bootstrap minimum-CPU lower bound exceeds +5%") } } regressions.append(contentsOf: failures.map { "\(name): \($0)" }) let status = first.counterSource == "smoke" && first.metric == "cpu" ? "unsupported-counter" : reporting ? "report-only" : instructionRatio == nil ? "cpu-only" : "gated" output.append( CaseSummary( name: name, metric: first.metric, counterSource: first.counterSource, reportingOnly: reporting, gating: !reporting, status: status, budget: first.budget ?? .null, reason: first.reason ?? .null, instructionsRatio: instructionRatio, minimumNanosecondsRatio: try MeasurementTools.median(cpu), minimumNanosecondsRatio95CI: ci, checksum: first.checksum, failures: failures)) } return Summary( host: reference.host.json, hostReadings: [ "base": stride(from: 0, to: 10, by: 2).map { reports[$0].host.json }, "head": stride(from: 1, to: 10, by: 2).map { reports[$0].host.json }, ], results: output, regressions: regressions, passed: regressions.isEmpty) }
static func run(_ args: [String]) throws -> Int32 { let options = try MeasurementTools.options( args, allowed: ["--base", "--head", "--output", "--base-revision", "--head-revision"]) var summary = try compare(base: options["--base"] ?? [], head: options["--head"] ?? []) summary.baseRevision = try MeasurementTools.optional("--base-revision", options).map(DevJSON.string) ?? .null summary.headRevision = try MeasurementTools.optional("--head-revision", options).map(DevJSON.string) ?? .null summary.caseScope = ProcessInfo.processInfo.environment["PARCEL_BENCH_CORE_ONLY"] == "1" ? "common-core" : "all-cases" _ = try MeasurementTools.write(summary, output: MeasurementTools.required("--output", options)) for result in summary.results { let instructions = result.instructionsRatio.map { String(format: "%+.2f%%", ($0 - 1) * 100) } ?? "unavailable" print( String( format: "%@ [%@]: instructions %@; minimum %@ ratio %.4f (95%% CI %.4f–%.4f)", result.name, result.status, instructions, result.metric, result.minimumNanosecondsRatio, result.minimumNanosecondsRatio95CI[0], result.minimumNanosecondsRatio95CI[1])) } for failure in summary.regressions { FileHandle.standardError.write(Data("REGRESSION \(failure)\n".utf8)) } return summary.passed ? 0 : 1 }
// Seed-zero MT19937 and 53-bit draws keep paired bootstrap reports reproducible. struct BootstrapRandom { private var state = [UInt32](repeating: 0, count: 624) private var index = 624 init() { state[0] = 19_650_218 for i in 1..<624 { state[i] = 1_812_433_253 &* (state[i - 1] ^ (state[i - 1] >> 30)) &+ UInt32(i) } var i = 1 for _ in 0..<624 { state[i] = (state[i] ^ ((state[i - 1] ^ (state[i - 1] >> 30)) &* 1_664_525)) i += 1 if i >= 624 { state[0] = state[623]; i = 1 } } for _ in 0..<623 { state[i] = (state[i] ^ ((state[i - 1] ^ (state[i - 1] >> 30)) &* 1_566_083_941)) &- UInt32(i) i += 1 if i >= 624 { state[0] = state[623]; i = 1 } } state[0] = 0x8000_0000 } private mutating func word() -> UInt32 { if index >= 624 { for i in 0..<624 { let value = (state[i] & 0x8000_0000) | (state[(i + 1) % 624] & 0x7fff_ffff) state[i] = state[(i + 397) % 624] ^ (value >> 1) ^ (value & 1 == 0 ? 0 : 0x9908_b0df) } index = 0 } var value = state[index] index += 1 value ^= value >> 11 value ^= (value << 7) & 0x9d2c_5680 value ^= (value << 15) & 0xefc6_0000 value ^= value >> 18 return value } mutating func next() -> Double { let a = word() >> 5 let b = word() >> 6 return (Double(a) * 67_108_864 + Double(b)) / 9_007_199_254_740_992 } }}