import Foundation import Models import os /// Talks to Twitter's web GraphQL API with the cookies from the in-app sign-in. actor TwitterWebSession { static let userAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.0 Safari/605.1.15" /// The public bearer token embedded in x.com's web client for every visitor (not a secret). static let webBearer = "AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA" /// How long a catalog is used before it's rebuilt in the background. private static let catalogLifetime: TimeInterval = 6 * 3600 /// How long a saved catalog is used at all. Query IDs rotate every few weeks, and a rotated one forces a rebuild. private static let catalogMaximumAge: TimeInterval = 7 * 24 * 3600 private let log = Logger(subsystem: "gold.samhenri.Twitter", category: "api") let session: URLSession private var cookies: [String: String] private let onCookiesChanged: @Sendable ([String: String]) -> Void var catalog: CatalogSnapshot? private var catalogTask: Task? /// Debug-only logged-out mode used to preview the UI with public data. private var guestToken: String? func activateGuestSession() async throws { var request = URLRequest(url: URL(string: "https://api.x.com/1.1/guest/activate.json")!) request.httpMethod = "POST" request.setValue("Bearer \(Self.webBearer)", forHTTPHeaderField: "Authorization") let (data, _) = try await session.data(for: request) guestToken = (try JSONSerialization.jsonObject(with: data) as? [String: Any])?["guest_token"] as? String } init(cookies: [String: String], onCookiesChanged: @escaping @Sendable ([String: String]) -> Void) { let configuration = URLSessionConfiguration.ephemeral configuration.httpCookieStorage = nil configuration.httpShouldSetCookies = false configuration.timeoutIntervalForRequest = 30 session = URLSession(configuration: configuration) self.cookies = cookies self.onCookiesChanged = onCookiesChanged } // MARK: Catalog private static var catalogFile: URL { let dir = URL.applicationSupportDirectory.appending(path: "Twitter", directoryHint: .isDirectory) try? FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) return dir.appending(path: "web-client-catalog.json") } /// The catalog to make requests with. Building one downloads and scans megabytes of x.com's JavaScript, so a /// saved catalog is used for days, stale-while-revalidate: past `catalogLifetime` it's rebuilt in the background /// while requests keep going with it, and `graphQLData` forces a rebuild if Twitter has rotated an ID meanwhile. /// Only a launch with nothing usable on disk waits for a build. func currentCatalog(forceRefresh: Bool = false) async throws -> CatalogSnapshot { if !forceRefresh { if catalog == nil { catalog = Self.savedCatalog() } if let catalog { let age = Date.now.timeIntervalSince(catalog.createdAt) if age < Self.catalogMaximumAge { if age >= Self.catalogLifetime { _ = catalogBuild() } return catalog } } } return await catalogBuild().value } /// Joins the build in flight, or starts one. private func catalogBuild() -> Task { if let catalogTask { return catalogTask } let task = Task { await buildCatalog() } catalogTask = task return task } private func buildCatalog() async -> CatalogSnapshot { defer { catalogTask = nil } do { let snapshot = try await CatalogBuilder.build { url in try await self.fetchText(url) } catalog = snapshot if let data = try? JSONEncoder().encode(snapshot) { try? data.write(to: Self.catalogFile, options: .atomic) } log.notice( "Catalog built: \(snapshot.operations.count) operations, transaction id \(snapshot.transactionAnimationKey == nil ? "unavailable" : "ready", privacy: .public)" ) return snapshot } catch { log.error("Catalog build failed: \(error.localizedDescription, privacy: .public)") if let catalog { return catalog } let fallback = CatalogSnapshot.fallback catalog = fallback return fallback } } private static func savedCatalog() -> CatalogSnapshot? { guard let data = try? Data(contentsOf: catalogFile) else { return nil } return try? JSONDecoder().decode(CatalogSnapshot.self, from: data) } private func fetchText(_ url: URL) async throws -> String { var request = URLRequest(url: url) request.setValue(Self.userAgent, forHTTPHeaderField: "User-Agent") request.setValue("text/html,application/xhtml+xml,*/*", forHTTPHeaderField: "Accept") request.setValue("en-US,en;q=0.9", forHTTPHeaderField: "Accept-Language") if url.host() == "x.com" { request.setValue(cookieHeader, forHTTPHeaderField: "Cookie") } let (data, response) = try await session.data(for: request) if let http = response as? HTTPURLResponse { if url.host() == "x.com" { absorbCookies(from: http) } guard (200..<300).contains(http.statusCode) else { throw TwitterError.http(http.statusCode, url.absoluteString) } } return String(decoding: data, as: UTF8.self) } // MARK: Requests private var cookieHeader: String { cookies.map { "\($0.key)=\($0.value)" }.sorted().joined(separator: "; ") } func absorbCookies(from response: HTTPURLResponse) { guard let fields = response.allHeaderFields as? [String: String], let url = response.url else { return } let updated = HTTPCookie.cookies(withResponseHeaderFields: fields, for: url) guard !updated.isEmpty else { return } var changed = false for cookie in updated where cookie.domain.hasSuffix("x.com") { if let expires = cookie.expiresDate, expires < .now { changed = cookies.removeValue(forKey: cookie.name) != nil || changed } else if cookies[cookie.name] != cookie.value { cookies[cookie.name] = cookie.value changed = true } } if changed { onCookiesChanged(cookies) } } func apiRequest( url: URL, method: String, body: Data?, transactionPath: String, generator: TransactionIDGenerator? ) -> URLRequest { var request = URLRequest(url: url) request.httpMethod = method request.httpBody = body request.setValue("Bearer \(Self.webBearer)", forHTTPHeaderField: "Authorization") request.setValue(cookies["ct0"] ?? "", forHTTPHeaderField: "x-csrf-token") request.setValue("OAuth2Session", forHTTPHeaderField: "x-twitter-auth-type") request.setValue("yes", forHTTPHeaderField: "x-twitter-active-user") request.setValue("en", forHTTPHeaderField: "x-twitter-client-language") request.setValue("application/json", forHTTPHeaderField: "Content-Type") request.setValue("*/*", forHTTPHeaderField: "Accept") request.setValue("https://x.com", forHTTPHeaderField: "Origin") request.setValue("https://x.com/", forHTTPHeaderField: "Referer") request.setValue(Self.userAgent, forHTTPHeaderField: "User-Agent") request.setValue(cookieHeader, forHTTPHeaderField: "Cookie") if let guestToken { request.setValue(guestToken, forHTTPHeaderField: "x-guest-token") request.setValue(nil, forHTTPHeaderField: "x-twitter-auth-type") request.setValue(nil, forHTTPHeaderField: "x-csrf-token") } if let generator { request.setValue( generator.generate(method: method, path: transactionPath), forHTTPHeaderField: "x-client-transaction-id" ) } return request } func perform(_ request: URLRequest) async throws -> (Any, Int) { let (data, http) = try await send(request) if (200..<300).contains(http.statusCode) { return ((try? JSONSerialization.jsonObject(with: data)) ?? [:], http.statusCode) } return (try Self.failure(data, http), http.statusCode) } /// Sends a request and absorbs any rotated cookies, leaving the body undecoded. private func send(_ request: URLRequest) async throws -> (Data, HTTPURLResponse) { let (data, response) = try await session.data(for: request) guard let http = response as? HTTPURLResponse else { throw TwitterError.http(0, "No response") } absorbCookies(from: http) return (data, http) } /// Inspects a non-2xx response: throws for auth and rate-limit failures, otherwise returns the error JSON. private static func failure(_ data: Data, _ http: HTTPURLResponse) throws -> Any { let json = try? JSONSerialization.jsonObject(with: data) switch http.statusCode { case 401: throw TwitterError.unauthorized case 429: let reset = (http.value(forHTTPHeaderField: "x-rate-limit-reset")).flatMap(Double.init).map( Date.init(timeIntervalSince1970:)) throw TwitterError.rateLimited(resetAt: reset) default: let message = Self.errorMessages(json).first ?? String(decoding: data.prefix(300), as: UTF8.self) if http.statusCode == 403, message.localizedCaseInsensitiveContains("authenticat") || message.contains("(32)") { throw TwitterError.unauthorized } return json ?? ["__status": http.statusCode, "__message": message] } } static func errorMessages(_ json: Any?) -> [String] { ((json as? [String: Any])?["errors"] as? [[String: Any]] ?? []).compactMap { $0["message"] as? String } } /// Parses a 2xx GraphQL body, throwing when it carries errors and no data. private static func graphQLJSON(_ data: Data) throws -> Any { let json = (try? JSONSerialization.jsonObject(with: data)) ?? [:] if (json as? [String: Any])?["data"] == nil, let message = errorMessages(json).first { throw TwitterError.api(message) } return json } /// Parses a timeline body off the session actor, so large pages don't hold up other requests. @concurrent private static func decodePage(_ data: Data) async throws -> FeedPage { TimelineParser.page(from: try graphQLJSON(data)) } /// Runs a GraphQL operation and returns its parsed JSON. func graphQL(_ operation: String, variables: [String: JSONValue], usePOST: Bool? = nil) async throws -> Any { try Self.graphQLJSON(try await graphQLData(operation, variables: variables, usePOST: usePOST)) } /// Runs a GraphQL timeline operation, decoding the page off the session actor. func graphQLPage(_ operation: String, variables: [String: JSONValue], usePOST: Bool? = nil) async throws -> FeedPage { try await Self.decodePage(try await graphQLData(operation, variables: variables, usePOST: usePOST)) } /// Runs a GraphQL operation with variables, retrying once with a fresh catalog when Twitter has /// rotated query IDs (404) or added required feature flags (400). Returns the undecoded 2xx body. private func graphQLData(_ operation: String, variables: [String: JSONValue], usePOST: Bool?) async throws -> Data { var extraFeatures: [String: JSONValue] = [:] var refreshed = false var forceRefresh = false for _ in 0..<3 { let catalog = try await currentCatalog(forceRefresh: forceRefresh) forceRefresh = false guard let op = catalog.operations[operation] else { if !refreshed { refreshed = true forceRefresh = true continue } throw TwitterError.missingOperation(operation) } var features: [String: JSONValue] = [:] for name in op.featureSwitches { features[name] = catalog.featureValues[name] ?? false } features.merge(extraFeatures) { $1 } var toggles: [String: JSONValue] = [:] for name in op.fieldToggles { toggles[name] = false } let path = "/i/api/graphql/\(op.queryId)/\(operation)" let isPOST = usePOST ?? (op.type == "mutation") let request: URLRequest if isPOST { var body: [String: JSONValue] = ["variables": .object(variables), "queryId": .string(op.queryId)] if !features.isEmpty { body["features"] = .object(features) } if !toggles.isEmpty { body["fieldToggles"] = .object(toggles) } request = apiRequest( url: URL(string: "https://x.com\(path)")!, method: "POST", body: Data(JSONValue.object(body).encodedString().utf8), transactionPath: path, generator: catalog.transactionGenerator) } else { var components = URLComponents(string: "https://x.com\(path)")! var items = [URLQueryItem(name: "variables", value: JSONValue.object(variables).encodedString())] if !features.isEmpty { items.append(URLQueryItem(name: "features", value: JSONValue.object(features).encodedString())) } if !toggles.isEmpty { items.append(URLQueryItem(name: "fieldToggles", value: JSONValue.object(toggles).encodedString())) } components.queryItems = items components.percentEncodedQuery = components.percentEncodedQuery?.replacingOccurrences( of: "+", with: "%2B") request = apiRequest( url: components.url!, method: "GET", body: nil, transactionPath: path, generator: catalog.transactionGenerator) } let started = Date.now let (data, http) = try await send(request) let status = http.statusCode log.notice( "\(operation, privacy: .public) → HTTP \(status) in \(Int(Date.now.timeIntervalSince(started) * 1000))ms" ) if (200..<300).contains(status) { return data } let json = try Self.failure(data, http) let message = Self.errorMessages(json).first ?? ((json as? [String: Any])?["__message"] as? String ?? "") log.error("\(operation, privacy: .public) failed: HTTP \(status) \(message, privacy: .public)") if status == 400, let range = message.range(of: "cannot be null:") { for name in message[range.upperBound...].split(separator: ",") { extraFeatures[name.trimmingCharacters(in: .whitespaces)] = false } continue } if status == 404 || status == 400, !refreshed { refreshed = true forceRefresh = true continue } throw TwitterError.http(status, message) } throw TwitterError.api("\(operation) failed after retries") } func rest(_ path: String) async throws -> Any { let request = apiRequest( url: URL(string: "https://x.com\(path)")!, method: "GET", body: nil, transactionPath: URL(string: "https://x.com\(path)")!.path(), generator: catalog?.transactionGenerator) let (json, status) = try await perform(request) guard (200..<300).contains(status) else { throw TwitterError.http(status, (json as? [String: Any])?["__message"] as? String ?? "") } return json } /// `application/x-www-form-urlencoded`, where `+` means a space: so it, `&`, `=` and everything else outside the /// unreserved set is escaped (URLComponents leaves `+` alone, which turned "C++" into "C "). static func formEncoded(_ form: [String: String]) -> String { var allowed = CharacterSet.alphanumerics allowed.insert(charactersIn: "-._~") func encode(_ string: String) -> String { string.addingPercentEncoding(withAllowedCharacters: allowed) ?? string } return form.sorted { $0.key < $1.key }.map { "\(encode($0.key))=\(encode($0.value))" }.joined(separator: "&") } /// v1.1 REST POST with a form body (follow, mute, block use these on x.com). func restPost(_ path: String, form: [String: String]) async throws -> Any { let url = URL(string: "https://x.com\(path)")! var request = apiRequest( url: url, method: "POST", body: Data(Self.formEncoded(form).utf8), transactionPath: url.path(), generator: catalog?.transactionGenerator) request.setValue("application/x-www-form-urlencoded", forHTTPHeaderField: "Content-Type") let (json, status) = try await perform(request) guard (200..<300).contains(status) else { throw TwitterError.http( status, Self.errorMessages(json).first ?? ((json as? [String: Any])?["__message"] as? String ?? "")) } return json } }