jcs.org
Serenity Operating System
1/*
2 * Copyright (c) 2018-2022, Andreas Kling <kling@serenityos.org>
3 * Copyright (c) 2022, Dex♪ <dexes.ttp@gmail.com>
4 *
5 * SPDX-License-Identifier: BSD-2-Clause
6 */
7
8#include <AK/Base64.h>
9#include <AK/Debug.h>
10#include <AK/JsonObject.h>
11#include <LibCore/ElapsedTimer.h>
12#include <LibWeb/Cookie/Cookie.h>
13#include <LibWeb/Cookie/ParsedCookie.h>
14#include <LibWeb/Loader/ContentFilter.h>
15#include <LibWeb/Loader/LoadRequest.h>
16#include <LibWeb/Loader/ProxyMappings.h>
17#include <LibWeb/Loader/Resource.h>
18#include <LibWeb/Loader/ResourceLoader.h>
19#include <LibWeb/Platform/EventLoopPlugin.h>
20#include <LibWeb/Platform/Timer.h>
21
22#ifdef AK_OS_SERENITY
23# include <serenity.h>
24#endif
25
26namespace Web {
27
28ResourceLoaderConnectorRequest::ResourceLoaderConnectorRequest() = default;
29
30ResourceLoaderConnectorRequest::~ResourceLoaderConnectorRequest() = default;
31
32ResourceLoaderConnector::ResourceLoaderConnector() = default;
33
34ResourceLoaderConnector::~ResourceLoaderConnector() = default;
35
36static RefPtr<ResourceLoader> s_resource_loader;
37
38void ResourceLoader::initialize(RefPtr<ResourceLoaderConnector> connector)
39{
40 if (connector)
41 s_resource_loader = ResourceLoader::try_create(connector.release_nonnull()).release_value_but_fixme_should_propagate_errors();
42}
43
44ResourceLoader& ResourceLoader::the()
45{
46 if (!s_resource_loader) {
47 dbgln("Web::ResourceLoader was not initialized");
48 VERIFY_NOT_REACHED();
49 }
50 return *s_resource_loader;
51}
52
53ErrorOr<NonnullRefPtr<ResourceLoader>> ResourceLoader::try_create(NonnullRefPtr<ResourceLoaderConnector> connector)
54{
55 return adopt_nonnull_ref_or_enomem(new (nothrow) ResourceLoader(move(connector)));
56}
57
58ResourceLoader::ResourceLoader(NonnullRefPtr<ResourceLoaderConnector> connector)
59 : m_connector(move(connector))
60 , m_user_agent(default_user_agent)
61{
62}
63
64void ResourceLoader::prefetch_dns(AK::URL const& url)
65{
66 if (ContentFilter::the().is_filtered(url)) {
67 dbgln("ResourceLoader: Refusing to prefetch DNS for '{}': \033[31;1mURL was filtered\033[0m", url);
68 return;
69 }
70
71 m_connector->prefetch_dns(url);
72}
73
74void ResourceLoader::preconnect(AK::URL const& url)
75{
76 if (ContentFilter::the().is_filtered(url)) {
77 dbgln("ResourceLoader: Refusing to pre-connect to '{}': \033[31;1mURL was filtered\033[0m", url);
78 return;
79 }
80
81 m_connector->preconnect(url);
82}
83
84static HashMap<LoadRequest, NonnullRefPtr<Resource>> s_resource_cache;
85
86RefPtr<Resource> ResourceLoader::load_resource(Resource::Type type, LoadRequest& request)
87{
88 if (!request.is_valid())
89 return nullptr;
90
91 bool use_cache = request.url().scheme() != "file";
92
93 if (use_cache) {
94 auto it = s_resource_cache.find(request);
95 if (it != s_resource_cache.end()) {
96 if (it->value->type() != type) {
97 dbgln("FIXME: Not using cached resource for {} since there's a type mismatch.", request.url());
98 } else {
99 dbgln_if(CACHE_DEBUG, "Reusing cached resource for: {}", request.url());
100 return it->value;
101 }
102 }
103 }
104
105 auto resource = Resource::create({}, type, request);
106
107 if (use_cache)
108 s_resource_cache.set(request, resource);
109
110 load(
111 request,
112 [=](auto data, auto& headers, auto status_code) {
113 const_cast<Resource&>(*resource).did_load({}, data, headers, status_code);
114 },
115 [=](auto& error, auto status_code) {
116 const_cast<Resource&>(*resource).did_fail({}, error, status_code);
117 });
118
119 return resource;
120}
121
122static DeprecatedString sanitized_url_for_logging(AK::URL const& url)
123{
124 if (url.scheme() == "data"sv)
125 return DeprecatedString::formatted("[data URL, mime-type={}, size={}]", url.data_mime_type(), url.data_payload().length());
126 return url.to_deprecated_string();
127}
128
129static void emit_signpost(DeprecatedString const& message, int id)
130{
131#ifdef AK_OS_SERENITY
132 auto string_id = perf_register_string(message.characters(), message.length());
133 perf_event(PERF_EVENT_SIGNPOST, string_id, id);
134#else
135 (void)message;
136 (void)id;
137#endif
138}
139
140static void store_response_cookies(Page& page, AK::URL const& url, DeprecatedString const& cookies)
141{
142 auto set_cookie_json_value = MUST(JsonValue::from_string(cookies));
143 VERIFY(set_cookie_json_value.type() == JsonValue::Type::Array);
144
145 for (auto const& set_cookie_entry : set_cookie_json_value.as_array().values()) {
146 VERIFY(set_cookie_entry.type() == JsonValue::Type::String);
147
148 auto cookie = Cookie::parse_cookie(set_cookie_entry.as_string());
149 if (!cookie.has_value())
150 continue;
151
152 page.client().page_did_set_cookie(url, cookie.value(), Cookie::Source::Http); // FIXME: Determine cookie source correctly
153 }
154}
155
156static size_t resource_id = 0;
157
158void ResourceLoader::load(LoadRequest& request, Function<void(ReadonlyBytes, HashMap<DeprecatedString, DeprecatedString, CaseInsensitiveStringTraits> const& response_headers, Optional<u32> status_code)> success_callback, Function<void(DeprecatedString const&, Optional<u32> status_code)> error_callback, Optional<u32> timeout, Function<void()> timeout_callback)
159{
160 auto& url = request.url();
161 request.start_timer();
162
163 auto id = resource_id++;
164 auto url_for_logging = sanitized_url_for_logging(url);
165 emit_signpost(DeprecatedString::formatted("Starting load: {}", url_for_logging), id);
166 dbgln("ResourceLoader: Starting load of: \"{}\"", url_for_logging);
167
168 auto const log_success = [url_for_logging, id](auto const& request) {
169 auto load_time_ms = request.load_time().to_milliseconds();
170 emit_signpost(DeprecatedString::formatted("Finished load: {}", url_for_logging), id);
171 dbgln("ResourceLoader: Finished load of: \"{}\", Duration: {}ms", url_for_logging, load_time_ms);
172 };
173
174 auto const log_failure = [url_for_logging, id](auto const& request, auto const& error_message) {
175 auto load_time_ms = request.load_time().to_milliseconds();
176 emit_signpost(DeprecatedString::formatted("Failed load: {}", url_for_logging), id);
177 dbgln("ResourceLoader: Failed load of: \"{}\", \033[31;1mError: {}\033[0m, Duration: {}ms", url_for_logging, error_message, load_time_ms);
178 };
179
180 if (is_port_blocked(url.port_or_default())) {
181 log_failure(request, DeprecatedString::formatted("The port #{} is blocked", url.port_or_default()));
182 return;
183 }
184
185 if (ContentFilter::the().is_filtered(url)) {
186 auto filter_message = "URL was filtered"sv;
187 log_failure(request, filter_message);
188 error_callback(filter_message, {});
189 return;
190 }
191
192 if (url.scheme() == "about") {
193 dbgln_if(SPAM_DEBUG, "Loading about: URL {}", url);
194 log_success(request);
195
196 HashMap<DeprecatedString, DeprecatedString, CaseInsensitiveStringTraits> response_headers;
197 response_headers.set("Content-Type", "text/html; charset=UTF-8");
198
199 Platform::EventLoopPlugin::the().deferred_invoke([success_callback = move(success_callback), response_headers = move(response_headers)] {
200 success_callback(DeprecatedString::empty().to_byte_buffer(), response_headers, {});
201 });
202 return;
203 }
204
205 if (url.scheme() == "data") {
206 dbgln_if(SPAM_DEBUG, "ResourceLoader loading a data URL with mime-type: '{}', base64={}, payload='{}'",
207 url.data_mime_type(),
208 url.data_payload_is_base64(),
209 url.data_payload());
210
211 ByteBuffer data;
212 if (url.data_payload_is_base64()) {
213 auto data_maybe = decode_base64(url.data_payload());
214 if (data_maybe.is_error()) {
215 auto error_message = data_maybe.error().string_literal();
216 log_failure(request, error_message);
217 error_callback(error_message, {});
218 return;
219 }
220 data = data_maybe.value();
221 } else {
222 data = url.data_payload().to_byte_buffer();
223 }
224
225 log_success(request);
226 Platform::EventLoopPlugin::the().deferred_invoke([data = move(data), success_callback = move(success_callback)] {
227 success_callback(data, {}, {});
228 });
229 return;
230 }
231
232 if (url.scheme() == "file") {
233 if (request.page().has_value())
234 m_page = request.page().value();
235
236 if (!m_page.has_value())
237 return;
238
239 FileRequest file_request(url.path(), [this, success_callback = move(success_callback), error_callback = move(error_callback), log_success, log_failure, request](ErrorOr<i32> file_or_error) {
240 --m_pending_loads;
241 if (on_load_counter_change)
242 on_load_counter_change();
243
244 if (file_or_error.is_error()) {
245 log_failure(request, file_or_error.error());
246 if (error_callback)
247 error_callback(DeprecatedString::formatted("{}", file_or_error.error()), file_or_error.error().code());
248 return;
249 }
250
251 auto const fd = file_or_error.value();
252
253 auto maybe_file = Core::File::adopt_fd(fd, Core::File::OpenMode::Read);
254 if (maybe_file.is_error()) {
255 log_failure(request, maybe_file.error());
256 if (error_callback)
257 error_callback(DeprecatedString::formatted("{}", maybe_file.error()), maybe_file.error().code());
258 return;
259 }
260
261 auto file = maybe_file.release_value();
262 auto maybe_data = file->read_until_eof();
263 if (maybe_data.is_error()) {
264 log_failure(request, maybe_data.error());
265 if (error_callback)
266 error_callback(DeprecatedString::formatted("{}", maybe_data.error()), maybe_data.error().code());
267 return;
268 }
269 auto data = maybe_data.release_value();
270 log_success(request);
271 success_callback(data, {}, {});
272 });
273
274 m_page->client().request_file(move(file_request));
275
276 ++m_pending_loads;
277 if (on_load_counter_change)
278 on_load_counter_change();
279
280 return;
281 }
282
283 if (url.scheme() == "http" || url.scheme() == "https" || url.scheme() == "gemini") {
284 auto proxy = ProxyMappings::the().proxy_for_url(url);
285
286 HashMap<DeprecatedString, DeprecatedString> headers;
287 headers.set("User-Agent", m_user_agent);
288 headers.set("Accept-Encoding", "gzip, deflate, br");
289
290 for (auto& it : request.headers()) {
291 headers.set(it.key, it.value);
292 }
293
294 auto protocol_request = m_connector->start_request(request.method(), url, headers, request.body(), proxy);
295 if (!protocol_request) {
296 auto start_request_failure_msg = "Failed to initiate load"sv;
297 log_failure(request, start_request_failure_msg);
298 if (error_callback)
299 error_callback(start_request_failure_msg, {});
300 return;
301 }
302
303 if (timeout.has_value() && timeout.value() > 0) {
304 auto timer = Platform::Timer::create_single_shot(timeout.value(), nullptr);
305 timer->on_timeout = [timer, protocol_request, timeout_callback = move(timeout_callback)] {
306 protocol_request->stop();
307 if (timeout_callback)
308 timeout_callback();
309 };
310 timer->start();
311 }
312
313 m_active_requests.set(*protocol_request);
314
315 protocol_request->on_buffered_request_finish = [this, success_callback = move(success_callback), error_callback = move(error_callback), log_success, log_failure, request, &protocol_request = *protocol_request](bool success, auto, auto& response_headers, auto status_code, ReadonlyBytes payload) mutable {
316 --m_pending_loads;
317 if (on_load_counter_change)
318 on_load_counter_change();
319
320 if (request.page().has_value()) {
321 if (auto set_cookie = response_headers.get("Set-Cookie"); set_cookie.has_value())
322 store_response_cookies(request.page().value(), request.url(), *set_cookie);
323 }
324
325 if (!success || (status_code.has_value() && *status_code >= 400 && *status_code <= 599 && payload.is_empty())) {
326 StringBuilder error_builder;
327 if (status_code.has_value())
328 error_builder.appendff("Load failed: {}", *status_code);
329 else
330 error_builder.append("Load failed"sv);
331 log_failure(request, error_builder.string_view());
332 if (error_callback)
333 error_callback(error_builder.to_deprecated_string(), status_code);
334 return;
335 }
336 log_success(request);
337 success_callback(payload, response_headers, status_code);
338 Platform::EventLoopPlugin::the().deferred_invoke([this, &protocol_request] {
339 m_active_requests.remove(protocol_request);
340 });
341 };
342 protocol_request->set_should_buffer_all_input(true);
343 protocol_request->on_certificate_requested = []() -> ResourceLoaderConnectorRequest::CertificateAndKey {
344 return {};
345 };
346 ++m_pending_loads;
347 if (on_load_counter_change)
348 on_load_counter_change();
349 return;
350 }
351
352 auto not_implemented_error = DeprecatedString::formatted("Protocol not implemented: {}", url.scheme());
353 log_failure(request, not_implemented_error);
354 if (error_callback)
355 error_callback(not_implemented_error, {});
356}
357
358void ResourceLoader::load(const AK::URL& url, Function<void(ReadonlyBytes, HashMap<DeprecatedString, DeprecatedString, CaseInsensitiveStringTraits> const& response_headers, Optional<u32> status_code)> success_callback, Function<void(DeprecatedString const&, Optional<u32> status_code)> error_callback, Optional<u32> timeout, Function<void()> timeout_callback)
359{
360 LoadRequest request;
361 request.set_url(url);
362 load(request, move(success_callback), move(error_callback), timeout, move(timeout_callback));
363}
364
365bool ResourceLoader::is_port_blocked(int port)
366{
367 int ports[] { 1, 7, 9, 11, 13, 15, 17, 19, 20, 21, 22, 23, 25, 37, 42,
368 43, 53, 77, 79, 87, 95, 101, 102, 103, 104, 109, 110, 111, 113,
369 115, 117, 119, 123, 135, 139, 143, 179, 389, 465, 512, 513, 514,
370 515, 526, 530, 531, 532, 540, 556, 563, 587, 601, 636, 993, 995,
371 2049, 3659, 4045, 6000, 6379, 6665, 6666, 6667, 6668, 6669, 9000 };
372 for (auto blocked_port : ports)
373 if (port == blocked_port)
374 return true;
375 return false;
376}
377
378void ResourceLoader::clear_cache()
379{
380 dbgln_if(CACHE_DEBUG, "Clearing {} items from ResourceLoader cache", s_resource_cache.size());
381 s_resource_cache.clear();
382}
383
384void ResourceLoader::evict_from_cache(LoadRequest const& request)
385{
386 dbgln_if(CACHE_DEBUG, "Removing resource {} from cache", request.url());
387 s_resource_cache.remove(request);
388}
389
390}