jcs.org
Serenity Operating System
1/*
2 * Copyright (c) 2021, Andrew Kaster <akaster@serenityos.org>
3 *
4 * SPDX-License-Identifier: BSD-2-Clause
5 */
6
7#include <AK/LexicalPath.h>
8#include <LibCore/ArgsParser.h>
9#include <LibCore/ConfigFile.h>
10#include <LibCore/DeprecatedFile.h>
11#include <LibCore/System.h>
12#include <LibCoredump/Backtrace.h>
13#include <LibMain/Main.h>
14#include <LibRegex/Regex.h>
15#include <LibTest/TestRunner.h>
16#include <signal.h>
17#include <spawn.h>
18#include <stdlib.h>
19#include <sys/wait.h>
20#include <unistd.h>
21
22namespace Test {
23TestRunner* TestRunner::s_the = nullptr;
24}
25using Test::get_time_in_ms;
26using Test::print_modifiers;
27
28struct FileResult {
29 LexicalPath file_path;
30 double time_taken { 0 };
31 Test::Result result { Test::Result::Pass };
32 int stdout_err_fd { -1 };
33 pid_t child_pid { 0 };
34};
35
36DeprecatedString g_currently_running_test;
37
38class TestRunner : public ::Test::TestRunner {
39public:
40 TestRunner(DeprecatedString test_root, Regex<PosixExtended> exclude_regex, NonnullRefPtr<Core::ConfigFile> config, Regex<PosixExtended> skip_regex, bool run_skipped_tests, bool print_progress, bool print_json, bool print_all_output, bool print_times = true)
41 : ::Test::TestRunner(move(test_root), print_times, print_progress, print_json)
42 , m_exclude_regex(move(exclude_regex))
43 , m_config(move(config))
44 , m_skip_regex(move(skip_regex))
45 , m_run_skipped_tests(run_skipped_tests)
46 , m_print_all_output(print_all_output)
47 {
48 if (!run_skipped_tests) {
49 m_skip_directories = m_config->read_entry("Global", "SkipDirectories", "").split(' ');
50 m_skip_files = m_config->read_entry("Global", "SkipTests", "").split(' ');
51 }
52 }
53
54 virtual ~TestRunner() = default;
55
56protected:
57 virtual void do_run_single_test(DeprecatedString const& test_path, size_t current_text_index, size_t num_tests) override;
58 virtual Vector<DeprecatedString> get_test_paths() const override;
59 virtual Vector<DeprecatedString> const* get_failed_test_names() const override { return &m_failed_test_names; }
60
61 virtual FileResult run_test_file(DeprecatedString const& test_path);
62
63 bool should_skip_test(LexicalPath const& test_path);
64
65 Regex<PosixExtended> m_exclude_regex;
66 NonnullRefPtr<Core::ConfigFile> m_config;
67 Vector<DeprecatedString> m_skip_directories;
68 Vector<DeprecatedString> m_skip_files;
69 Vector<DeprecatedString> m_failed_test_names;
70 Regex<PosixExtended> m_skip_regex;
71 bool m_run_skipped_tests { false };
72 bool m_print_all_output { false };
73};
74
75Vector<DeprecatedString> TestRunner::get_test_paths() const
76{
77 Vector<DeprecatedString> paths;
78 Test::iterate_directory_recursively(m_test_root, [&](DeprecatedString const& file_path) {
79 if (access(file_path.characters(), R_OK | X_OK) != 0)
80 return;
81 auto result = m_exclude_regex.match(file_path, PosixFlags::Global);
82 if (!result.success) // must NOT match the regex to be a valid test file
83 paths.append(file_path);
84 });
85 quick_sort(paths);
86 return paths;
87}
88
89bool TestRunner::should_skip_test(LexicalPath const& test_path)
90{
91 if (m_run_skipped_tests)
92 return false;
93
94 for (DeprecatedString const& dir : m_skip_directories) {
95 if (test_path.dirname().contains(dir))
96 return true;
97 }
98 for (DeprecatedString const& file : m_skip_files) {
99 if (test_path.basename().contains(file))
100 return true;
101 }
102 auto result = m_skip_regex.match(test_path.basename(), PosixFlags::Global);
103 if (result.success)
104 return true;
105
106 return false;
107}
108
109void TestRunner::do_run_single_test(DeprecatedString const& test_path, size_t current_test_index, size_t num_tests)
110{
111 g_currently_running_test = test_path;
112 auto test_relative_path = LexicalPath::relative_path(test_path, m_test_root);
113 outln(" START {} ({}/{})", test_relative_path, current_test_index, num_tests);
114 fflush(stdout); // we really want to see the start text in case the test hangs
115 auto test_result = run_test_file(test_path);
116
117 switch (test_result.result) {
118 case Test::Result::Pass:
119 ++m_counts.tests_passed;
120 break;
121 case Test::Result::Skip:
122 ++m_counts.tests_skipped;
123 break;
124 case Test::Result::Fail:
125 ++m_counts.tests_failed;
126 break;
127 case Test::Result::Crashed:
128 ++m_counts.tests_failed; // FIXME: tests_crashed
129 break;
130 }
131 if (test_result.result != Test::Result::Skip)
132 ++m_counts.files_total;
133
134 m_total_elapsed_time_in_ms += test_result.time_taken;
135
136 bool crashed_or_failed = test_result.result == Test::Result::Fail || test_result.result == Test::Result::Crashed;
137 bool print_stdout_stderr = crashed_or_failed || m_print_all_output;
138 if (crashed_or_failed) {
139 m_failed_test_names.append(test_path);
140 print_modifiers({ Test::BG_RED, Test::FG_BLACK, Test::FG_BOLD });
141 out("{}", test_result.result == Test::Result::Fail ? " FAIL " : "CRASHED");
142 print_modifiers({ Test::CLEAR });
143 if (test_result.result == Test::Result::Crashed) {
144 auto pid_search_string = DeprecatedString::formatted("_{}_", test_result.child_pid);
145 Core::DirIterator iterator("/tmp/coredump"sv);
146 if (!iterator.has_error()) {
147 while (iterator.has_next()) {
148 auto path = iterator.next_full_path();
149 if (!path.contains(pid_search_string))
150 continue;
151
152 auto reader = Coredump::Reader::create(path);
153 if (!reader)
154 break;
155
156 dbgln("Last crash backtrace for {} (was pid {}):", test_path, test_result.child_pid);
157 reader->for_each_thread_info([&](auto thread_info) {
158 Coredump::Backtrace thread_backtrace(*reader, thread_info);
159 auto tid = thread_info.tid; // Note: Yoinking this out of the struct because we can't pass a reference to it (as it's a misaligned field in a packed struct)
160 dbgln("Thread {}", tid);
161 for (auto const& entry : thread_backtrace.entries())
162 dbgln("- {}", entry.to_deprecated_string(true));
163 return IterationDecision::Continue;
164 });
165 break;
166 }
167 }
168 }
169 } else {
170 print_modifiers({ Test::BG_GREEN, Test::FG_BLACK, Test::FG_BOLD });
171 out(" PASS ");
172 print_modifiers({ Test::CLEAR });
173 }
174
175 out(" {}", test_relative_path);
176
177 print_modifiers({ Test::CLEAR, Test::ITALIC, Test::FG_GRAY });
178 if (test_result.time_taken < 1000) {
179 outln(" ({}ms)", static_cast<int>(test_result.time_taken));
180 } else {
181 outln(" ({:3}s)", test_result.time_taken / 1000.0);
182 }
183 print_modifiers({ Test::CLEAR });
184
185 if (test_result.result != Test::Result::Pass) {
186 print_modifiers({ Test::FG_GRAY, Test::FG_BOLD });
187 out(" Test: ");
188 if (crashed_or_failed) {
189 print_modifiers({ Test::CLEAR, Test::FG_RED });
190 outln("{} ({})", test_result.file_path.basename(), test_result.result == Test::Result::Fail ? "failed" : "crashed");
191 } else {
192 print_modifiers({ Test::CLEAR, Test::FG_ORANGE });
193 outln("{} (skipped)", test_result.file_path.basename());
194 }
195 print_modifiers({ Test::CLEAR });
196 }
197
198 // Make sure our clear modifiers goes through before we dump file output via write(2)
199 fflush(stdout);
200
201 if (print_stdout_stderr && test_result.stdout_err_fd > 0) {
202 int ret = lseek(test_result.stdout_err_fd, 0, SEEK_SET);
203 VERIFY(ret == 0);
204 for (;;) {
205 char buf[32768];
206 ssize_t nread = read(test_result.stdout_err_fd, buf, sizeof(buf));
207 if (nread == 0)
208 break;
209 if (nread < 0) {
210 perror("read");
211 break;
212 }
213 size_t already_written = 0;
214 while (already_written < (size_t)nread) {
215 ssize_t nwritten = write(STDOUT_FILENO, buf + already_written, nread - already_written);
216 if (nwritten < 0) {
217 perror("write");
218 break;
219 }
220 already_written += nwritten;
221 }
222 }
223 }
224
225 close(test_result.stdout_err_fd);
226}
227
228FileResult TestRunner::run_test_file(DeprecatedString const& test_path)
229{
230 double start_time = get_time_in_ms();
231
232 auto path_for_test = LexicalPath(test_path);
233 if (should_skip_test(path_for_test)) {
234 return FileResult { move(path_for_test), 0.0, Test::Result::Skip, -1 };
235 }
236
237 // FIXME: actual error handling, mark test as :yaksplode: if any are bad instead of VERIFY
238 posix_spawn_file_actions_t file_actions;
239 posix_spawn_file_actions_init(&file_actions);
240 char child_out_err_path[] = "/tmp/run-tests.XXXXXX";
241 int child_out_err_file = mkstemp(child_out_err_path);
242 VERIFY(child_out_err_file >= 0);
243
244 DeprecatedString dirname = path_for_test.dirname();
245 DeprecatedString basename = path_for_test.basename();
246
247 (void)posix_spawn_file_actions_adddup2(&file_actions, child_out_err_file, STDOUT_FILENO);
248 (void)posix_spawn_file_actions_adddup2(&file_actions, child_out_err_file, STDERR_FILENO);
249 (void)posix_spawn_file_actions_addchdir(&file_actions, dirname.characters());
250
251 Vector<char const*, 4> argv;
252 argv.append(basename.characters());
253 auto extra_args = m_config->read_entry(path_for_test.basename(), "Arguments", "").split(' ');
254 for (auto& arg : extra_args)
255 argv.append(arg.characters());
256 argv.append(nullptr);
257
258 pid_t child_pid = -1;
259 // FIXME: Do we really want to copy test runner's entire env?
260 int ret = posix_spawn(&child_pid, test_path.characters(), &file_actions, nullptr, const_cast<char* const*>(argv.data()), environ);
261 VERIFY(ret == 0);
262 VERIFY(child_pid > 0);
263
264 int wstatus;
265
266 Test::Result test_result = Test::Result::Fail;
267 for (size_t num_waits = 0; num_waits < 2; ++num_waits) {
268 ret = waitpid(child_pid, &wstatus, 0); // intentionally not setting WCONTINUED
269 if (ret != child_pid)
270 break; // we'll end up with a failure
271
272 if (WIFEXITED(wstatus)) {
273 if (wstatus == 0) {
274 test_result = Test::Result::Pass;
275 }
276 break;
277 } else if (WIFSIGNALED(wstatus)) {
278 test_result = Test::Result::Crashed;
279 break;
280 } else if (WIFSTOPPED(wstatus)) {
281 outln("{} was stopped unexpectedly, sending SIGCONT", test_path);
282 kill(child_pid, SIGCONT);
283 }
284 }
285
286 // Remove the child's stdout from /tmp. This does cause the temp file to be observable
287 // while the test is executing, but if it hangs that might even be a bonus :)
288 ret = unlink(child_out_err_path);
289 VERIFY(ret == 0);
290
291 return FileResult { move(path_for_test), get_time_in_ms() - start_time, test_result, child_out_err_file, child_pid };
292}
293
294ErrorOr<int> serenity_main(Main::Arguments arguments)
295{
296
297 auto program_name = LexicalPath::basename(arguments.strings[0]);
298
299#ifdef SIGINFO
300 TRY(Core::System::signal(SIGINFO, [](int) {
301 static char buffer[4096];
302 auto& counts = ::Test::TestRunner::the()->counts();
303 int len = snprintf(buffer, sizeof(buffer), "Pass: %d, Fail: %d, Skip: %d\nCurrent test: %s\n", counts.tests_passed, counts.tests_failed, counts.tests_skipped, g_currently_running_test.characters());
304 write(STDOUT_FILENO, buffer, len);
305 }));
306#endif
307
308 bool print_progress =
309#ifdef AK_OS_SERENITY
310 true; // Use OSC 9 to print progress
311#else
312 false;
313#endif
314 bool print_json = false;
315 bool print_all_output = false;
316 bool run_benchmarks = false;
317 bool run_skipped_tests = false;
318 StringView specified_test_root;
319 DeprecatedString test_glob;
320 DeprecatedString exclude_pattern;
321 DeprecatedString config_file;
322
323 Core::ArgsParser args_parser;
324 args_parser.add_option(Core::ArgsParser::Option {
325 .argument_mode = Core::ArgsParser::OptionArgumentMode::Required,
326 .help_string = "Show progress with OSC 9 (true, false)",
327 .long_name = "show-progress",
328 .short_name = 'p',
329 .accept_value = [&](StringView str) {
330 if ("true"sv == str)
331 print_progress = true;
332 else if ("false"sv == str)
333 print_progress = false;
334 else
335 return false;
336 return true;
337 },
338 });
339 args_parser.add_option(print_json, "Show results as JSON", "json", 'j');
340 args_parser.add_option(print_all_output, "Show all test output", "verbose", 'v');
341 args_parser.add_option(run_benchmarks, "Run benchmarks as well", "benchmarks", 'b');
342 args_parser.add_option(run_skipped_tests, "Run all matching tests, even those marked as 'skip'", "all", 'a');
343 args_parser.add_option(test_glob, "Only run tests matching the given glob", "filter", 'f', "glob");
344 args_parser.add_option(exclude_pattern, "Regular expression to use to exclude paths from being considered tests", "exclude-pattern", 'e', "pattern");
345 args_parser.add_option(config_file, "Configuration file to use", "config-file", 'c', "filename");
346 args_parser.add_positional_argument(specified_test_root, "Tests root directory", "path", Core::ArgsParser::Required::No);
347 args_parser.parse(arguments);
348
349 test_glob = DeprecatedString::formatted("*{}*", test_glob);
350
351 if (getenv("DISABLE_DBG_OUTPUT")) {
352 AK::set_debug_enabled(false);
353 }
354
355 // Make UBSAN deadly for all tests we run by default.
356 TRY(Core::System::setenv("UBSAN_OPTIONS"sv, "halt_on_error=1"sv, true));
357
358 if (!run_benchmarks)
359 TRY(Core::System::setenv("TESTS_ONLY"sv, "1"sv, true));
360
361 DeprecatedString test_root;
362
363 if (!specified_test_root.is_empty()) {
364 test_root = DeprecatedString { specified_test_root };
365 } else {
366 test_root = "/usr/Tests";
367 }
368 if (!Core::DeprecatedFile::is_directory(test_root)) {
369 warnln("Test root is not a directory: {}", test_root);
370 return 1;
371 }
372
373 test_root = Core::DeprecatedFile::real_path_for(test_root);
374
375 auto void_or_error = Core::System::chdir(test_root);
376 if (void_or_error.is_error()) {
377 warnln("chdir failed: {}", void_or_error.error());
378 return void_or_error.release_error();
379 }
380
381 auto config_or_error = config_file.is_empty() ? Core::ConfigFile::open_for_app("Tests") : Core::ConfigFile::open(config_file);
382 if (config_or_error.is_error()) {
383 warnln("Failed to open configuration file ({}): {}", config_file.is_empty() ? "User config for Tests" : config_file.characters(), config_or_error.error());
384 return config_or_error.release_error();
385 }
386 auto config = config_or_error.release_value();
387
388 if (config->num_groups() == 0)
389 warnln("Empty configuration file ({}) loaded!", config_file.is_empty() ? "User config for Tests" : config_file.characters());
390
391 if (exclude_pattern.is_empty())
392 exclude_pattern = config->read_entry("Global", "NotTestsPattern", "$^"); // default is match nothing (aka match end then beginning)
393
394 Regex<PosixExtended> exclude_regex(exclude_pattern, {});
395 if (exclude_regex.parser_result.error != regex::Error::NoError) {
396 warnln("Exclude pattern \"{}\" is invalid", exclude_pattern);
397 return 1;
398 }
399
400 // we need to preconfigure this, because we can't autoinitialize Regex types
401 // in the Testrunner
402 auto skip_regex_pattern = config->read_entry("Global", "SkipRegex", "$^");
403 Regex<PosixExtended> skip_regex { skip_regex_pattern, {} };
404 if (skip_regex.parser_result.error != regex::Error::NoError) {
405 warnln("SkipRegex pattern \"{}\" is invalid", skip_regex_pattern);
406 return 1;
407 }
408
409 TestRunner test_runner(test_root, move(exclude_regex), move(config), move(skip_regex), run_skipped_tests, print_progress, print_json, print_all_output);
410 test_runner.run(test_glob);
411
412 return test_runner.counts().tests_failed;
413}