jcs.org
Serenity Operating System
1/*
2 * Copyright (c) 2022, David Tuin <davidot@serenityos.org>
3 *
4 * SPDX-License-Identifier: BSD-2-Clause
5 */
6
7#include <AK/DeprecatedString.h>
8#include <AK/Format.h>
9#include <AK/HashMap.h>
10#include <AK/JsonObject.h>
11#include <AK/JsonParser.h>
12#include <AK/LexicalPath.h>
13#include <AK/QuickSort.h>
14#include <AK/Vector.h>
15#include <LibCore/ArgsParser.h>
16#include <LibCore/DeprecatedFile.h>
17#include <LibCore/File.h>
18#include <LibCore/Process.h>
19#include <LibCore/System.h>
20#include <LibMain/Main.h>
21#include <LibTest/TestRunnerUtil.h>
22#include <spawn.h>
23#include <sys/wait.h>
24#include <unistd.h>
25
26enum class TestResult {
27 Passed,
28 Failed,
29 Skipped,
30 MetadataError,
31 HarnessError,
32 TimeoutError,
33 ProcessError,
34 RunnerFailure,
35 TodoError,
36};
37
38static StringView name_for_result(TestResult result)
39{
40 switch (result) {
41 case TestResult::Passed:
42 return "PASSED"sv;
43 case TestResult::Failed:
44 return "FAILED"sv;
45 case TestResult::Skipped:
46 return "SKIPPED"sv;
47 case TestResult::MetadataError:
48 return "METADATA_ERROR"sv;
49 case TestResult::HarnessError:
50 return "HARNESS_ERROR"sv;
51 case TestResult::TimeoutError:
52 return "TIMEOUT_ERROR"sv;
53 case TestResult::ProcessError:
54 return "PROCESS_ERROR"sv;
55 case TestResult::RunnerFailure:
56 return "RUNNER_EXCEPTION"sv;
57 case TestResult::TodoError:
58 return "TODO_ERROR"sv;
59 }
60 VERIFY_NOT_REACHED();
61 return ""sv;
62}
63
64static TestResult result_from_string(StringView string_view)
65{
66 if (string_view == "passed"sv)
67 return TestResult::Passed;
68 if (string_view == "failed"sv)
69 return TestResult::Failed;
70 if (string_view == "skipped"sv)
71 return TestResult::Skipped;
72 if (string_view == "metadata_error"sv)
73 return TestResult::MetadataError;
74 if (string_view == "harness_error"sv)
75 return TestResult::HarnessError;
76 if (string_view == "timeout"sv)
77 return TestResult::TimeoutError;
78 if (string_view == "process_error"sv || string_view == "assert_fail"sv)
79 return TestResult::ProcessError;
80 if (string_view == "todo_error"sv)
81 return TestResult::TodoError;
82
83 return TestResult::RunnerFailure;
84}
85
86static StringView emoji_for_result(TestResult result)
87{
88 switch (result) {
89 case TestResult::Passed:
90 return "✅"sv;
91 case TestResult::Failed:
92 return "❌"sv;
93 case TestResult::Skipped:
94 return "⚠"sv;
95 case TestResult::MetadataError:
96 return "📄"sv;
97 case TestResult::HarnessError:
98 return "⚙"sv;
99 case TestResult::TimeoutError:
100 return "💀"sv;
101 case TestResult::ProcessError:
102 return "💥"sv;
103 case TestResult::RunnerFailure:
104 return "🐍"sv;
105 case TestResult::TodoError:
106 return "📝"sv;
107 }
108 VERIFY_NOT_REACHED();
109 return ""sv;
110}
111
112static constexpr StringView total_test_emoji = "🧪"sv;
113
114class Test262RunnerHandler {
115public:
116 static ErrorOr<OwnPtr<Test262RunnerHandler>> create(StringView command, char const* const arguments[])
117 {
118 auto write_pipe_fds = TRY(Core::System::pipe2(O_CLOEXEC));
119 auto read_pipe_fds = TRY(Core::System::pipe2(O_CLOEXEC));
120
121 posix_spawn_file_actions_t file_actions;
122 posix_spawn_file_actions_init(&file_actions);
123 posix_spawn_file_actions_adddup2(&file_actions, write_pipe_fds[0], STDIN_FILENO);
124 posix_spawn_file_actions_adddup2(&file_actions, read_pipe_fds[1], STDOUT_FILENO);
125
126 auto pid = TRY(Core::System::posix_spawnp(command, &file_actions, nullptr, const_cast<char**>(arguments), environ));
127
128 posix_spawn_file_actions_destroy(&file_actions);
129 ArmedScopeGuard runner_kill { [&pid] { kill(pid, SIGKILL); } };
130
131 TRY(Core::System::close(write_pipe_fds[0]));
132 TRY(Core::System::close(read_pipe_fds[1]));
133
134 auto infile = TRY(Core::File::adopt_fd(read_pipe_fds[0], Core::File::OpenMode::Read));
135
136 auto outfile = TRY(Core::File::adopt_fd(write_pipe_fds[1], Core::File::OpenMode::Write));
137
138 runner_kill.disarm();
139
140 return make<Test262RunnerHandler>(pid, move(infile), move(outfile));
141 }
142
143 Test262RunnerHandler(pid_t pid, NonnullOwnPtr<Core::File> in_file, NonnullOwnPtr<Core::File> out_file)
144 : m_pid(pid)
145 , m_input(move(in_file))
146 , m_output(move(out_file))
147 {
148 }
149
150 bool write_lines(Span<DeprecatedString> lines)
151 {
152 // It's possible the process dies before we can write all the tests
153 // to the stdin. So make sure that we don't crash but just stop writing.
154 struct sigaction action_handler {
155 .sa_handler = SIG_IGN, .sa_mask = {}, .sa_flags = 0,
156 };
157 struct sigaction old_action_handler;
158 if (sigaction(SIGPIPE, &action_handler, &old_action_handler) < 0) {
159 perror("sigaction");
160 return false;
161 }
162
163 for (DeprecatedString const& line : lines) {
164 if (m_output->write_until_depleted(DeprecatedString::formatted("{}\n", line).bytes()).is_error())
165 break;
166 }
167
168 // Ensure that the input stream ends here, whether we were able to write all lines or not
169 m_output->close();
170
171 // It's not really a problem if this signal failed
172 if (sigaction(SIGPIPE, &old_action_handler, nullptr) < 0)
173 perror("sigaction");
174
175 return true;
176 }
177
178 DeprecatedString read_all()
179 {
180 auto all_output_or_error = m_input->read_until_eof();
181 if (all_output_or_error.is_error()) {
182 warnln("Got error: {} while reading runner output", all_output_or_error.error());
183 return ""sv;
184 }
185 return DeprecatedString(all_output_or_error.value().bytes(), Chomp);
186 }
187
188 enum class ProcessResult {
189 Running,
190 DoneWithZeroExitCode,
191 Failed,
192 FailedFromTimeout,
193 Unknown,
194 };
195
196 ErrorOr<ProcessResult> status()
197 {
198 if (m_pid == -1)
199 return ProcessResult::Unknown;
200
201 m_output->close();
202
203 auto wait_result = TRY(Core::System::waitpid(m_pid, WNOHANG));
204 if (wait_result.pid == 0) {
205 // Attempt to kill it, since it has not finished yet somehow
206 return ProcessResult::Running;
207 }
208 m_pid = -1;
209
210 if (WIFSIGNALED(wait_result.status) && WTERMSIG(wait_result.status) == SIGALRM)
211 return ProcessResult::FailedFromTimeout;
212
213 if (WIFEXITED(wait_result.status) && WEXITSTATUS(wait_result.status) == 0)
214 return ProcessResult::DoneWithZeroExitCode;
215
216 return ProcessResult::Failed;
217 }
218
219public:
220 pid_t m_pid;
221 NonnullOwnPtr<Core::File> m_input;
222 NonnullOwnPtr<Core::File> m_output;
223};
224
225static ErrorOr<HashMap<size_t, TestResult>> run_test_files(Span<DeprecatedString> files, size_t offset, StringView command, char const* const arguments[])
226{
227 HashMap<size_t, TestResult> results {};
228 TRY(results.try_ensure_capacity(files.size()));
229 size_t test_index = 0;
230
231 auto fail_all_after = [&] {
232 for (; test_index < files.size(); ++test_index)
233 results.set(offset + test_index, TestResult::RunnerFailure);
234 };
235
236 while (test_index < files.size()) {
237 auto runner_process_or_error = Test262RunnerHandler::create(command, arguments);
238 if (runner_process_or_error.is_error()) {
239 fail_all_after();
240 return results;
241 }
242 auto& runner_process = runner_process_or_error.value();
243
244 if (!runner_process->write_lines(files.slice(test_index))) {
245 fail_all_after();
246 return results;
247 }
248
249 DeprecatedString output = runner_process->read_all();
250 auto status_or_error = runner_process->status();
251 bool failed = false;
252 if (!status_or_error.is_error()) {
253 VERIFY(status_or_error.value() != Test262RunnerHandler::ProcessResult::Running);
254 failed = status_or_error.value() != Test262RunnerHandler::ProcessResult::DoneWithZeroExitCode;
255 }
256
257 for (StringView line : output.split_view('\n')) {
258 if (!line.starts_with("RESULT "sv))
259 break;
260
261 auto test_for_line = test_index;
262 ++test_index;
263 if (test_for_line >= files.size())
264 break;
265
266 line = line.substring_view(7).trim("\n\0 "sv);
267 JsonParser parser { line };
268 TestResult result = TestResult::RunnerFailure;
269 auto result_object_or_error = parser.parse();
270 if (!result_object_or_error.is_error() && result_object_or_error.value().is_object()) {
271 auto& result_object = result_object_or_error.value().as_object();
272 if (auto result_string = result_object.get_deprecated_string("result"sv); result_string.has_value()) {
273 auto const& view = result_string.value();
274 // Timeout and assert fail already are the result of the stopping test
275 if (view == "timeout"sv || view == "assert_fail"sv) {
276 failed = false;
277 }
278
279 result = result_from_string(view);
280 }
281 }
282
283 results.set(test_for_line + offset, result);
284 }
285
286 if (failed) {
287 TestResult result = TestResult::ProcessError;
288 if (!status_or_error.is_error() && status_or_error.value() == Test262RunnerHandler::ProcessResult::FailedFromTimeout) {
289 result = TestResult::TimeoutError;
290 }
291 // assume the last test failed, if by SIGALRM signal it's a timeout
292 results.set(test_index + offset, result);
293 ++test_index;
294 }
295 }
296
297 return results;
298}
299
300void write_per_file(HashMap<size_t, TestResult> const& result_map, Vector<DeprecatedString> const& paths, StringView per_file_name, double time_taken_in_ms);
301
302ErrorOr<int> serenity_main(Main::Arguments arguments)
303{
304 size_t batch_size = 50;
305 StringView per_file_location;
306 StringView pass_through_parameters;
307 StringView runner_command = "test262-runner"sv;
308 StringView test_directory;
309 bool dont_print_progress = false;
310 bool dont_disable_core_dump = false;
311
312 Core::ArgsParser args_parser;
313 args_parser.add_positional_argument(test_directory, "Directory to search for tests", "tests");
314 args_parser.add_option(per_file_location, "Output a per-file containing all results", "per-file", 'o', "filename");
315 args_parser.add_option(batch_size, "Size of batches send to runner at once", "batch-size", 'b', "batch size");
316 args_parser.add_option(runner_command, "Command to run", "runner-command", 'r', "command");
317 args_parser.add_option(pass_through_parameters, "Parameters to pass through to the runner, will split on spaces", "pass-through", 'p', "parameters");
318 args_parser.add_option(dont_print_progress, "Hide progress information", "quiet", 'q');
319 args_parser.add_option(dont_disable_core_dump, "Enabled core dumps for runner (i.e. don't pass --disable-core-dump)", "enable-core-dumps", 0);
320 args_parser.parse(arguments);
321
322 // Normalize the path to ensure filenames are consistent
323 Vector<DeprecatedString> paths;
324
325 if (!Core::DeprecatedFile::is_directory(test_directory)) {
326 paths.append(test_directory);
327 } else {
328 Test::iterate_directory_recursively(LexicalPath::canonicalized_path(test_directory), [&](DeprecatedString const& file_path) {
329 if (file_path.contains("_FIXTURE"sv))
330 return;
331 // FIXME: Add ignored file set
332 paths.append(file_path);
333 });
334 quick_sort(paths);
335 }
336
337 outln("Found {} tests", paths.size());
338
339 auto parameters = pass_through_parameters.split_view(' ');
340 Vector<DeprecatedString> args;
341 args.ensure_capacity(parameters.size() + 2);
342 args.append(runner_command);
343 if (!dont_disable_core_dump)
344 args.append("--disable-core-dump"sv);
345
346 for (auto parameter : parameters)
347 args.append(parameter);
348
349 Vector<char const*> raw_args;
350 raw_args.ensure_capacity(args.size() + 1);
351 for (auto& arg : args)
352 raw_args.append(arg.characters());
353
354 raw_args.append(nullptr);
355
356 dbgln("test262 runner command: {}", args);
357
358 HashMap<size_t, TestResult> results;
359 Array<size_t, 9> result_counts {};
360 static_assert(result_counts.size() == static_cast<size_t>(TestResult::TodoError) + 1u);
361 size_t index = 0;
362
363 double start_time = Test::get_time_in_ms();
364
365 auto print_progress = [&] {
366 if (!dont_print_progress) {
367 warn("\033]9;{};{};\033\\", index, paths.size());
368 double percentage_done = (100. * index) / paths.size();
369 warn("{:04.2f}% {:3.1f}s ", percentage_done, (Test::get_time_in_ms() - start_time) / 1000.);
370 for (size_t i = 0; i < result_counts.size(); ++i) {
371 auto result_type = static_cast<TestResult>(i);
372 warn("{} {} ", emoji_for_result(result_type), result_counts[i]);
373 }
374 warn("\r");
375 }
376 };
377
378 while (index < paths.size()) {
379 print_progress();
380 auto this_batch_size = min(batch_size, paths.size() - index);
381 auto batch_results = TRY(run_test_files(paths.span().slice(index, this_batch_size), index, args[0], raw_args.data()));
382
383 TRY(results.try_ensure_capacity(results.size() + batch_results.size()));
384 for (auto& [key, value] : batch_results) {
385 results.set(key, value);
386 ++result_counts[static_cast<size_t>(value)];
387 }
388
389 index += this_batch_size;
390 }
391
392 double time_taken_in_ms = Test::get_time_in_ms() - start_time;
393
394 print_progress();
395 if (!dont_print_progress)
396 warn("\n\033]9;-1;\033\\");
397
398 outln("Took {} seconds", time_taken_in_ms / 1000.);
399 outln("{}: {}", total_test_emoji, paths.size());
400 for (size_t i = 0; i < result_counts.size(); ++i) {
401 auto result_type = static_cast<TestResult>(i);
402 outln("{}: {} ({:3.2f}%)", emoji_for_result(result_type), result_counts[i], 100. * static_cast<double>(result_counts[i]) / paths.size());
403 }
404
405 if (!per_file_location.is_empty())
406 write_per_file(results, paths, per_file_location, time_taken_in_ms);
407
408 return 0;
409}
410
411void write_per_file(HashMap<size_t, TestResult> const& result_map, Vector<DeprecatedString> const& paths, StringView per_file_name, double time_taken_in_ms)
412{
413
414 auto file_or_error = Core::File::open(per_file_name, Core::File::OpenMode::Write);
415 if (file_or_error.is_error()) {
416 warnln("Failed to open per file for writing at {}: {}", per_file_name, file_or_error.error().string_literal());
417 return;
418 }
419
420 auto& file = file_or_error.value();
421
422 JsonObject result_object;
423 for (auto& [test, value] : result_map)
424 result_object.set(paths[test], name_for_result(value));
425
426 JsonObject complete_results {};
427 complete_results.set("duration", time_taken_in_ms / 1000.);
428 complete_results.set("results", result_object);
429
430 if (file->write_until_depleted(complete_results.to_deprecated_string().bytes()).is_error())
431 warnln("Failed to write per-file");
432 file->close();
433}