From 0d66a80a0f15caac9a024c9273fda5238befbe3c Mon Sep 17 00:00:00 2001 From: Aliaksandr Kalenik Date: Mon, 14 Aug 2023 00:37:49 +0200 Subject: [PATCH] headless-browser: Add ref tests support The ref tests runner takes screenshots of both the input page and the expected page, then compares them. Ref testing allows us to catch painting bugs, which cannot be detected with the layout and text tests we already have. With ref tests, we'll likely want to reuse the same expectation page for multiple inputs. Therefore, there's a `manifest.json` file that describes the relationship between inputs and expected outputs. --- Tests/LibWeb/Ref/manifest.json | 3 + Tests/LibWeb/Ref/square-flex.html | 11 +++ Tests/LibWeb/Ref/square-ref.html | 7 ++ Userland/Utilities/headless-browser.cpp | 116 +++++++++++++++++++----- 4 files changed, 112 insertions(+), 25 deletions(-) create mode 100644 Tests/LibWeb/Ref/manifest.json create mode 100644 Tests/LibWeb/Ref/square-flex.html create mode 100644 Tests/LibWeb/Ref/square-ref.html diff --git a/Tests/LibWeb/Ref/manifest.json b/Tests/LibWeb/Ref/manifest.json new file mode 100644 index 0000000000..c65328cdf0 --- /dev/null +++ b/Tests/LibWeb/Ref/manifest.json @@ -0,0 +1,3 @@ +{ + "square-flex.html": "square-ref.html" +} diff --git a/Tests/LibWeb/Ref/square-flex.html b/Tests/LibWeb/Ref/square-flex.html new file mode 100644 index 0000000000..cbd6615536 --- /dev/null +++ b/Tests/LibWeb/Ref/square-flex.html @@ -0,0 +1,11 @@ +
diff --git a/Tests/LibWeb/Ref/square-ref.html b/Tests/LibWeb/Ref/square-ref.html new file mode 100644 index 0000000000..4f1f9eec26 --- /dev/null +++ b/Tests/LibWeb/Ref/square-ref.html @@ -0,0 +1,7 @@ +
diff --git a/Userland/Utilities/headless-browser.cpp b/Userland/Utilities/headless-browser.cpp index 35485af5a3..f70643e66f 100644 --- a/Userland/Utilities/headless-browser.cpp +++ b/Userland/Utilities/headless-browser.cpp @@ -9,6 +9,8 @@ #include #include #include +#include +#include #include #include #include @@ -184,9 +186,16 @@ static ErrorOr format_url(StringView url) enum class TestMode { Layout, Text, + Ref, }; -static ErrorOr run_one_test(HeadlessWebContentView& view, StringView input_path, StringView expectation_path, TestMode mode, int timeout_in_milliseconds = 15000) +enum class TestResult { + Pass, + Fail, + Timeout, +}; + +static ErrorOr run_dump_test(HeadlessWebContentView& view, StringView input_path, StringView expectation_path, TestMode mode, int timeout_in_milliseconds = 15000) { Core::EventLoop loop; bool did_timeout = false; @@ -197,7 +206,6 @@ static ErrorOr run_one_test(HeadlessWebContentView& view, StringView inp })); view.load(URL::create_with_file_scheme(TRY(FileSystem::real_path(input_path)).to_deprecated_string())); - (void)expectation_path; String result; @@ -225,25 +233,7 @@ static ErrorOr run_one_test(HeadlessWebContentView& view, StringView inp loop.exec(); if (did_timeout) - return Error::from_errno(ETIMEDOUT); - - return result; -} - -enum class TestResult { - Pass, - Fail, - Timeout, -}; - -static ErrorOr run_test(HeadlessWebContentView& view, StringView input_path, StringView expectation_path, TestMode mode) -{ - auto result = run_one_test(view, input_path, expectation_path, mode); - - if (result.is_error() && result.error().code() == ETIMEDOUT) return TestResult::Timeout; - if (result.is_error()) - return result.release_error(); auto expectation_file_or_error = Core::File::open(expectation_path, Core::File::OpenMode::Read); if (expectation_file_or_error.is_error()) { @@ -255,7 +245,7 @@ static ErrorOr run_test(HeadlessWebContentView& view, StringView inp auto expectation = TRY(String::from_utf8(StringView(TRY(expectation_file->read_until_eof()).bytes()))); - auto actual = result.release_value(); + auto actual = result; auto actual_trimmed = TRY(actual.trim("\n"sv, TrimMode::Right)); auto expectation_trimmed = TRY(expectation.trim("\n"sv, TrimMode::Right)); @@ -279,6 +269,58 @@ static ErrorOr run_test(HeadlessWebContentView& view, StringView inp return TestResult::Fail; } +static ErrorOr run_ref_test(HeadlessWebContentView& view, StringView input_path, StringView expectation_path, int timeout_in_milliseconds = 15000) +{ + Core::EventLoop loop; + bool did_timeout = false; + + auto timeout_timer = TRY(Core::Timer::create_single_shot(5000, [&] { + did_timeout = true; + loop.quit(0); + })); + + view.load(URL::create_with_file_scheme(TRY(FileSystem::real_path(input_path)).to_deprecated_string())); + auto expectation_real_path = TRY(FileSystem::real_path(expectation_path)).to_deprecated_string(); + + RefPtr actual_screenshot, expectation_screenshot; + view.on_load_finish = [&](auto const&) { + if (actual_screenshot) { + expectation_screenshot = view.take_screenshot(); + loop.quit(0); + } else { + actual_screenshot = view.take_screenshot(); + view.load(URL::create_with_file_scheme(expectation_real_path)); + } + }; + + timeout_timer->start(timeout_in_milliseconds); + loop.exec(); + + if (did_timeout) + return TestResult::Timeout; + + VERIFY(actual_screenshot); + VERIFY(expectation_screenshot); + + if (actual_screenshot->visually_equals(*expectation_screenshot)) + return TestResult::Pass; + + return TestResult::Fail; +} + +static ErrorOr run_test(HeadlessWebContentView& view, StringView input_path, StringView expectation_path, TestMode mode) +{ + switch (mode) { + case TestMode::Text: + case TestMode::Layout: + return run_dump_test(view, input_path, expectation_path, mode); + case TestMode::Ref: + return run_ref_test(view, input_path, expectation_path); + default: + VERIFY_NOT_REACHED(); + } +} + struct Test { String input_path; String expectation_path; @@ -286,14 +328,14 @@ struct Test { Optional result; }; -static ErrorOr collect_tests(Vector& tests, StringView path, StringView trail, TestMode mode) +static ErrorOr collect_dump_tests(Vector& tests, StringView path, StringView trail, TestMode mode) { Core::DirIterator it(TRY(String::formatted("{}/input/{}", path, trail)).to_deprecated_string(), Core::DirIterator::Flags::SkipDots); while (it.has_next()) { auto name = it.next_path(); auto input_path = TRY(FileSystem::real_path(TRY(String::formatted("{}/input/{}/{}", path, trail, name)))); if (FileSystem::is_directory(input_path)) { - TRY(collect_tests(tests, path, TRY(String::formatted("{}/{}", trail, name)), mode)); + TRY(collect_dump_tests(tests, path, TRY(String::formatted("{}/{}", trail, name)), mode)); continue; } if (!name.ends_with(".html"sv)) @@ -306,13 +348,37 @@ static ErrorOr collect_tests(Vector& tests, StringView path, StringV return {}; } +static ErrorOr collect_ref_tests(Vector& tests, StringView path) +{ + auto manifest_path = TRY(String::formatted("{}/manifest.json", path)); + auto manifest_file_or_error = Core::File::open(manifest_path, Core::File::OpenMode::Read); + if (manifest_file_or_error.is_error()) { + warnln("Failed opening '{}': {}", manifest_path, manifest_file_or_error.error()); + return manifest_file_or_error.release_error(); + } + + auto manifest_file = manifest_file_or_error.release_value(); + auto manifest = TRY(String::from_utf8(StringView(TRY(manifest_file->read_until_eof()).bytes()))); + auto manifest_json = TRY(JsonParser(manifest).parse()); + TRY(manifest_json.as_object().try_for_each_member([&](DeprecatedString const& key, AK::JsonValue const& value) -> ErrorOr { + TRY(String::from_deprecated_string(key)); + auto input_path = TRY(String::formatted("{}/{}", path, key)); + auto expectation_path = TRY(String::formatted("{}/{}", path, value.to_deprecated_string())); + tests.append({ input_path, expectation_path, TestMode::Ref, {} }); + return {}; + })); + + return {}; +} + static ErrorOr run_tests(HeadlessWebContentView& view, StringView test_root_path) { view.clear_content_filters(); Vector tests; - TRY(collect_tests(tests, TRY(String::formatted("{}/Layout", test_root_path)), "."sv, TestMode::Layout)); - TRY(collect_tests(tests, TRY(String::formatted("{}/Text", test_root_path)), "."sv, TestMode::Text)); + TRY(collect_dump_tests(tests, TRY(String::formatted("{}/Layout", test_root_path)), "."sv, TestMode::Layout)); + TRY(collect_dump_tests(tests, TRY(String::formatted("{}/Text", test_root_path)), "."sv, TestMode::Text)); + TRY(collect_ref_tests(tests, TRY(String::formatted("{}/Ref", test_root_path)))); size_t pass_count = 0; size_t fail_count = 0;