import CoreGraphics import Foundation import ImageIO import RealityDriverSupport struct PixelMetrics: Codable, Sendable { let luminance: Double let edgeContrast: Double let red: Double let green: Double let blue: Double } struct VisualPixels: Sendable { let bytes: [UInt8] let width: Int let height: Int let frame: ScreenRect init(capture: DriverCapture) throws { guard let source = CGImageSourceCreateWithURL(URL(fileURLWithPath: capture.path) as CFURL, nil), let image = CGImageSourceCreateImageAtIndex(source, 0, nil), image.width == capture.pixelWidth, image.height == capture.pixelHeight else { throw RealityError("Cannot decode driver capture: \(capture.path)") } let width = image.width let height = image.height self.width = width self.height = height frame = ScreenRect( x: capture.frame.x, y: capture.frame.y, width: capture.frame.width, height: capture.frame.height) var bytes = [UInt8](repeating: 0, count: width * height * 4) let rendered = bytes.withUnsafeMutableBytes { buffer -> Bool in guard let context = CGContext( data: buffer.baseAddress, width: width, height: height, bitsPerComponent: 8, bytesPerRow: width * 4, space: CGColorSpaceCreateDeviceRGB(), bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue | CGBitmapInfo.byteOrder32Big.rawValue) else { return false } context.draw(image, in: CGRect(x: 0, y: 0, width: width, height: height)) return true } guard rendered else { throw RealityError("Cannot render driver capture for pixel comparison") } self.bytes = bytes } init(bytes: [UInt8], width: Int, height: Int, frame: ScreenRect) { precondition(width > 0 && height > 0 && bytes.count == width * height * 4 && frame.isValid) self.bytes = bytes self.width = width self.height = height self.frame = frame } func metrics(in region: ScreenRect) throws -> PixelMetrics { guard region.isValid, frame.contains(region, tolerance: 0) else { throw RealityError("Pixel oracle region is outside the captured display: \(region)") } let x0 = Int((region.x - frame.x) / frame.width * Double(width)) let x1 = Int((region.maxX - frame.x) / frame.width * Double(width)) let y0 = Int((region.y - frame.y) / frame.height * Double(height)) let y1 = Int((region.maxY - frame.y) / frame.height * Double(height)) guard x1 > x0 + 1, y1 > y0 + 1 else { throw RealityError("Pixel oracle region is too small") } var channels = [Double](repeating: 0, count: 3) var edge = 0.0 var pixels = 0 var edges = 0 for y in y0.. x0 { edge += abs(value - luminance(at: offset - 4)); edges += 1 } if y > y0 { edge += abs(value - luminance(at: offset - width * 4)); edges += 1 } pixels += 1 } } channels = channels.map { $0 / Double(pixels) } return PixelMetrics( luminance: 0.2126 * channels[0] + 0.7152 * channels[1] + 0.0722 * channels[2], edgeContrast: edge / Double(edges), red: channels[0], green: channels[1], blue: channels[2]) } func difference(from other: Self, in region: ScreenRect) throws -> Double { guard width == other.width, height == other.height, frame == other.frame else { throw RealityError("Driver capture dimensions changed during the visual check") } _ = try metrics(in: region) let x0 = Int((region.x - frame.x) / frame.width * Double(width)) let x1 = Int((region.maxX - frame.x) / frame.width * Double(width)) let y0 = Int((region.y - frame.y) / frame.height * Double(height)) let y1 = Int((region.maxY - frame.y) / frame.height * Double(height)) var difference = 0.0 for y in y0.. Double { (0.2126 * Double(bytes[offset]) + 0.7152 * Double(bytes[offset + 1]) + 0.0722 * Double(bytes[offset + 2])) / 255 } } struct VisualFocusRegions: Codable, Sendable { let focusedID: UInt32 let focused: ScreenRect let background: [ScreenRect] } enum NoPeekExpectation: String, Codable, Sendable { case blurAndDim = "blur+dim" case dimOnly = "dim-only (NO blur)" init(reduceTransparency: Bool, increaseContrast: Bool) { self = reduceTransparency || increaseContrast ? .dimOnly : .blurAndDim } func matches(before: PixelMetrics, after: PixelMetrics) -> Bool { switch self { case .blurAndDim: VisualOracle.dimmedAndBlurred(before: before, after: after) case .dimOnly: VisualOracle.dimmedWithoutBlur(before: before, after: after) } } } enum VisualOracle { static func focusRegions(ids: [UInt32], frames: [ScreenRect], focusedID: UInt32?) throws -> VisualFocusRegions { guard ids.count == frames.count, Set(ids).count == ids.count, let focusedID, let index = ids.firstIndex(of: focusedID), frames.allSatisfy(\.isValid) else { throw RealityError("Visual check requires an exact owned native focus and matching frame baseline") } return VisualFocusRegions( focusedID: focusedID, focused: frames[index], background: frames.enumerated().compactMap { $0.offset == index ? nil : $0.element }) } static func dimmedAndBlurred(before: PixelMetrics, after: PixelMetrics) -> Bool { guard dimmed(before: before, after: after) else { return false } return normalizedContrast(before: before, after: after) < 0.9 } static func dimmedWithoutBlur(before: PixelMetrics, after: PixelMetrics) -> Bool { guard dimmed(before: before, after: after) else { return false } // Dimming scales edges and luminance together; blur additionally erases detail. return (0.9...1.1).contains(normalizedContrast(before: before, after: after)) } private static func normalizedContrast(before: PixelMetrics, after: PixelMetrics) -> Double { (after.edgeContrast / after.luminance) / (before.edgeContrast / before.luminance) } private static func dimmed(before: PixelMetrics, after: PixelMetrics) -> Bool { before.luminance.isFinite && before.edgeContrast.isFinite && after.luminance.isFinite && after.edgeContrast.isFinite && before.luminance > 0.05 && before.edgeContrast > 0.001 && after.luminance > 0.001 && after.edgeContrast >= 0 && after.luminance < before.luminance * 0.85 } static func graphite(_ metrics: PixelMetrics) -> Bool { [metrics.red, metrics.green, metrics.blue].allSatisfy { abs($0 - 0.16) < 0.07 } && metrics.edgeContrast < 0.01 } static func overlayWindows(_ rows: [RealWindow], pid: Int32, display: ScreenRect) -> [RealWindow] { rows.filter { $0.pid == pid && $0.onscreen && $0.frame.matches(display) && $0.alpha > 0.95 } } } @MainActor func checkVisualMode(_ session: RealitySession, command: String, owned: [OwnedWindow], before: [ScreenRect]) async throws { guard let parcel = session.parcel else { throw RealityError("Visual check requires the tested Parcel process") } var noPeek: NoPeekExpectation? if command == "no-peek-toggle" { let settings = try AccessibilitySettingsSnapshot.capture() let expectation = NoPeekExpectation( reduceTransparency: settings.reduceTransparency, increaseContrast: settings.increaseContrast) noPeek = expectation struct Branch: Codable { let expectation: NoPeekExpectation let settings: AccessibilitySettingsSnapshot } try encodeJSON( Branch(expectation: expectation, settings: settings), to: session.directory.appendingPathComponent("no-peek-branch.json")) session.recordExpectation( "No Peek branch: \(expectation.rawValue); settings read only, never changed by scenario") } let display = try RealityDisplays.primaryFrame() let initial = try FrameOracle.read(allLayers: true) guard !initial.contains(where: { $0.pid == parcel.processIdentifier && $0.onscreen && $0.layer == 0 }) else { throw RealityError("Reality visual baseline contains an unexpected Parcel window; no tour should be presented") } let regions = try VisualOracle.focusRegions( ids: owned.map(\.id), frames: before, focusedID: RealApps.focusedID(initial.filter { $0.layer == 0 })) try encodeJSON(regions, to: session.directory.appendingPathComponent("visual-focus-before.json")) try encodeJSON(initial, to: session.directory.appendingPathComponent("overlays-before.json")) let captureEnabled = ProcessInfo.processInfo.environment["PARCEL_REALITY_SCREEN_CAPTURE"] == "1" var pixelsBefore: VisualPixels? if captureEnabled { pixelsBefore = try await captureRealityPixels(session, stage: "before") } try await session.command(command) try await session.expect("\(command) preserves real tiled frames \(before)") { try FrameOracle.frames($0, ids: owned.map(\.id)) == before && RealApps.focusedID($0) == regions.focusedID } let initialIDs = Set(initial.map(\.id)) let clock = ContinuousClock() let deadline = clock.now.advanced(by: .seconds(4)) var overlays: [RealWindow] = [] repeat { overlays = VisualOracle.overlayWindows( try FrameOracle.read(allLayers: true), pid: parcel.processIdentifier, display: display ) .filter { !initialIDs.contains($0.id) } if !overlays.isEmpty { break } try await Task.sleep(for: .milliseconds(40)) } while clock.now < deadline try encodeJSON( try FrameOracle.read(allLayers: true), to: session.directory.appendingPathComponent("overlays-enabled.json")) if captureEnabled { guard !overlays.isEmpty else { throw RealityError("Visual mode created no independently observed display overlay") } let enabled = try await captureRealityPixels(session, stage: "enabled") guard let pixelsBefore else { throw RealityError("Visual baseline was not captured") } try checkPrivacyPixels( session, command: command, baseline: pixelsBefore, enabled: enabled, regions: regions, stage: "enabled", noPeek: noPeek) for window in owned.reversed() where window.id != regions.focusedID { try await session.apps.focus(window) try await session.expect("Visual mode follows native focus \(window.id) without changing frames") { try RealApps.focusedID($0) == window.id && FrameOracle.frames($0, ids: owned.map(\.id)) == before } let currentRegions = try VisualOracle.focusRegions( ids: owned.map(\.id), frames: before, focusedID: RealApps.focusedID(try FrameOracle.read())) let stage = "focused-\(window.id)" try encodeJSON(currentRegions, to: session.directory.appendingPathComponent("visual-\(stage).json")) let current = try await captureRealityPixels(session, stage: stage) try checkPrivacyPixels( session, command: command, baseline: pixelsBefore, enabled: current, regions: currentRegions, stage: stage, noPeek: noPeek) } } try await session.command(command) try await session.expect("\(command) restores real tiled frames \(before)") { try FrameOracle.frames($0, ids: owned.map(\.id)) == before } let enabledIDs = Set(overlays.map(\.id)) let offDeadline = clock.now.advanced(by: .seconds(4)) while try FrameOracle.read(allLayers: true).contains(where: { enabledIDs.contains($0.id) && $0.onscreen }) { guard clock.now < offDeadline else { throw RealityError("Visual overlay remained onscreen after mode was disabled") } try await Task.sleep(for: .milliseconds(40)) } try encodeJSON( try FrameOracle.read(allLayers: true), to: session.directory.appendingPathComponent("overlays-disabled.json")) if let pixelsBefore { let restored = try await captureRealityPixels(session, stage: "disabled") for frame in before { guard try restored.difference(from: pixelsBefore, in: frame.inset(40)) < 0.03 else { throw RealityError("Visual mode did not restore baseline document pixels") } } } else { throw RealitySkip( "needs Screen Recording for the responsible reality host (opt-in): PARCEL_REALITY_SCREEN_CAPTURE=1 and an existing grant for \(session.driver.appURL.path); no consent requested; overlay metadata does not prove dimming, blur, or backdrop pixels" ) } } @MainActor private func checkPrivacyPixels( _ session: RealitySession, command: String, baseline: VisualPixels, enabled: VisualPixels, regions: VisualFocusRegions, stage: String, noPeek: NoPeekExpectation? ) throws { guard try enabled.difference(from: baseline, in: regions.focused.inset(40)) < 0.03 else { throw RealityError("Visual mode altered native focused document \(regions.focusedID) pixels") } var metrics: [PixelMetrics] = [] for frame in regions.background { let region = frame.inset(40) let before = try baseline.metrics(in: region) let after = try enabled.metrics(in: region) metrics += [before, after] try encodeJSON(metrics, to: session.directory.appendingPathComponent("pixel-metrics-\(stage).json")) let valid: Bool let expectation: String if command == "no-peek-toggle" { guard let noPeek else { throw RealityError("No Peek visual branch was not recorded") } valid = noPeek.matches(before: before, after: after) expectation = noPeek.rawValue } else { valid = VisualOracle.graphite(after) expectation = "graphite backdrop" } guard valid else { throw RealityError( "Visual pixels failed \(expectation) expectation: before=\(before), enabled=\(after)") } } } @MainActor func captureRealityPixels(_ session: RealitySession, stage: String) async throws -> VisualPixels { let directory = session.directory.appendingPathComponent("capture-\(stage)") try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: false) let response = try await session.driver.send(.capture(directory: directory.path, optedIn: true)) guard let screenshot = response.captures?.first else { throw RealityError("Driver returned no screen capture") } try encodeJSON(screenshot, to: session.directory.appendingPathComponent("capture-\(stage).json")) return try VisualPixels(capture: screenshot) }