diff --git a/Userland/Utilities/headless-browser.cpp b/Userland/Utilities/headless-browser.cpp index 140ab9375a..dede3f66e4 100644 --- a/Userland/Utilities/headless-browser.cpp +++ b/Userland/Utilities/headless-browser.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -160,9 +161,25 @@ enum class TestMode { enum class TestResult { Pass, Fail, + Skipped, Timeout, }; +static StringView test_result_to_string(TestResult result) +{ + switch (result) { + case TestResult::Pass: + return "Pass"sv; + case TestResult::Fail: + return "Fail"sv; + case TestResult::Skipped: + return "Skipped"sv; + case TestResult::Timeout: + return "Timeout"sv; + } + VERIFY_NOT_REACHED(); +} + static ErrorOr run_dump_test(HeadlessWebContentView& view, StringView input_path, StringView expectation_path, TestMode mode, int timeout_in_milliseconds = DEFAULT_TIMEOUT_MS) { Core::EventLoop loop; @@ -315,6 +332,33 @@ struct Test { Optional result; }; +static Vector s_skipped_tests; + +static ErrorOr load_test_config(StringView test_root_path) +{ + auto config_path = LexicalPath::join(test_root_path, "TestConfig.ini"sv); + auto config_or_error = Core::ConfigFile::open(config_path.string()); + + if (config_or_error.is_error()) { + if (config_or_error.error().code() == ENOENT) + return {}; + dbgln("Unable to open test config {}", config_path); + return config_or_error.release_error(); + } + + auto config = config_or_error.release_value(); + + for (auto const& group : config->groups()) { + if (group == "Skipped"sv) { + for (auto& key : config->keys(group)) + s_skipped_tests.append(LexicalPath::join(test_root_path, key).string()); + } else { + warnln("Unknown group '{}' in config {}", group, config_path); + } + } + return {}; +} + static ErrorOr collect_dump_tests(Vector& tests, StringView path, StringView trail, TestMode mode) { Core::DirIterator it(TRY(String::formatted("{}/input/{}", path, trail)).to_deprecated_string(), Core::DirIterator::Flags::SkipDots); @@ -352,6 +396,8 @@ static ErrorOr run_tests(HeadlessWebContentView& view, StringView test_root { view.clear_content_filters(); + TRY(load_test_config(test_root_path)); + Vector tests; TRY(collect_dump_tests(tests, TRY(String::formatted("{}/Layout", test_root_path)), "."sv, TestMode::Layout)); TRY(collect_dump_tests(tests, TRY(String::formatted("{}/Text", test_root_path)), "."sv, TestMode::Text)); @@ -360,6 +406,7 @@ static ErrorOr run_tests(HeadlessWebContentView& view, StringView test_root size_t pass_count = 0; size_t fail_count = 0; size_t timeout_count = 0; + size_t skipped_count = 0; bool is_tty = isatty(STDOUT_FILENO); @@ -379,6 +426,12 @@ static ErrorOr run_tests(HeadlessWebContentView& view, StringView test_root else outln(""); + if (s_skipped_tests.contains_slow(test.input_path.bytes_as_string_view())) { + test.result = TestResult::Skipped; + ++skipped_count; + continue; + } + test.result = TRY(run_test(view, test.input_path, test.expectation_path, test.mode, dump_failed_ref_tests)); switch (*test.result) { case TestResult::Pass: @@ -390,6 +443,9 @@ static ErrorOr run_tests(HeadlessWebContentView& view, StringView test_root case TestResult::Timeout: ++timeout_count; break; + case TestResult::Skipped: + VERIFY_NOT_REACHED(); + break; } } @@ -397,12 +453,12 @@ static ErrorOr run_tests(HeadlessWebContentView& view, StringView test_root outln("\33[2K\rDone!"); outln("=================================================="); - outln("Pass: {}, Fail: {}, Timeout: {}", pass_count, fail_count, timeout_count); + outln("Pass: {}, Fail: {}, Skipped: {}, Timeout: {}", pass_count, fail_count, skipped_count, timeout_count); outln("=================================================="); for (auto& test : tests) { if (*test.result == TestResult::Pass) continue; - outln("{}: {}", *test.result == TestResult::Fail ? "Fail" : "Timeout", test.input_path); + outln("{}: {}", test_result_to_string(*test.result), test.input_path); } if (timeout_count == 0 && fail_count == 0)