diff --git a/Tests/LibWeb/Ref/manifest.json b/Tests/LibWeb/Ref/manifest.json new file mode 100644 index 0000000000..c65328cdf0 --- /dev/null +++ b/Tests/LibWeb/Ref/manifest.json @@ -0,0 +1,3 @@ +{ + "square-flex.html": "square-ref.html" +} diff --git a/Tests/LibWeb/Ref/square-flex.html b/Tests/LibWeb/Ref/square-flex.html new file mode 100644 index 0000000000..cbd6615536 --- /dev/null +++ b/Tests/LibWeb/Ref/square-flex.html @@ -0,0 +1,11 @@ +
diff --git a/Tests/LibWeb/Ref/square-ref.html b/Tests/LibWeb/Ref/square-ref.html new file mode 100644 index 0000000000..4f1f9eec26 --- /dev/null +++ b/Tests/LibWeb/Ref/square-ref.html @@ -0,0 +1,7 @@ +
diff --git a/Userland/Utilities/headless-browser.cpp b/Userland/Utilities/headless-browser.cpp index 35485af5a3..f70643e66f 100644 --- a/Userland/Utilities/headless-browser.cpp +++ b/Userland/Utilities/headless-browser.cpp @@ -9,6 +9,8 @@ #include #include #include +#include +#include #include #include #include @@ -184,9 +186,16 @@ static ErrorOr format_url(StringView url) enum class TestMode { Layout, Text, + Ref, }; -static ErrorOr run_one_test(HeadlessWebContentView& view, StringView input_path, StringView expectation_path, TestMode mode, int timeout_in_milliseconds = 15000) +enum class TestResult { + Pass, + Fail, + Timeout, +}; + +static ErrorOr run_dump_test(HeadlessWebContentView& view, StringView input_path, StringView expectation_path, TestMode mode, int timeout_in_milliseconds = 15000) { Core::EventLoop loop; bool did_timeout = false; @@ -197,7 +206,6 @@ static ErrorOr run_one_test(HeadlessWebContentView& view, StringView inp })); view.load(URL::create_with_file_scheme(TRY(FileSystem::real_path(input_path)).to_deprecated_string())); - (void)expectation_path; String result; @@ -225,25 +233,7 @@ static ErrorOr run_one_test(HeadlessWebContentView& view, StringView inp loop.exec(); if (did_timeout) - return Error::from_errno(ETIMEDOUT); - - return result; -} - -enum class TestResult { - Pass, - Fail, - Timeout, -}; - -static ErrorOr run_test(HeadlessWebContentView& view, StringView input_path, StringView expectation_path, TestMode mode) -{ - auto result = run_one_test(view, input_path, expectation_path, mode); - - if (result.is_error() && result.error().code() == ETIMEDOUT) return TestResult::Timeout; - if (result.is_error()) - return result.release_error(); auto expectation_file_or_error = Core::File::open(expectation_path, Core::File::OpenMode::Read); if (expectation_file_or_error.is_error()) { @@ -255,7 +245,7 @@ static ErrorOr run_test(HeadlessWebContentView& view, StringView inp auto expectation = TRY(String::from_utf8(StringView(TRY(expectation_file->read_until_eof()).bytes()))); - auto actual = result.release_value(); + auto actual = result; auto actual_trimmed = TRY(actual.trim("\n"sv, TrimMode::Right)); auto expectation_trimmed = TRY(expectation.trim("\n"sv, TrimMode::Right)); @@ -279,6 +269,58 @@ static ErrorOr run_test(HeadlessWebContentView& view, StringView inp return TestResult::Fail; } +static ErrorOr run_ref_test(HeadlessWebContentView& view, StringView input_path, StringView expectation_path, int timeout_in_milliseconds = 15000) +{ + Core::EventLoop loop; + bool did_timeout = false; + + auto timeout_timer = TRY(Core::Timer::create_single_shot(5000, [&] { + did_timeout = true; + loop.quit(0); + })); + + view.load(URL::create_with_file_scheme(TRY(FileSystem::real_path(input_path)).to_deprecated_string())); + auto expectation_real_path = TRY(FileSystem::real_path(expectation_path)).to_deprecated_string(); + + RefPtr actual_screenshot, expectation_screenshot; + view.on_load_finish = [&](auto const&) { + if (actual_screenshot) { + expectation_screenshot = view.take_screenshot(); + loop.quit(0); + } else { + actual_screenshot = view.take_screenshot(); + view.load(URL::create_with_file_scheme(expectation_real_path)); + } + }; + + timeout_timer->start(timeout_in_milliseconds); + loop.exec(); + + if (did_timeout) + return TestResult::Timeout; + + VERIFY(actual_screenshot); + VERIFY(expectation_screenshot); + + if (actual_screenshot->visually_equals(*expectation_screenshot)) + return TestResult::Pass; + + return TestResult::Fail; +} + +static ErrorOr run_test(HeadlessWebContentView& view, StringView input_path, StringView expectation_path, TestMode mode) +{ + switch (mode) { + case TestMode::Text: + case TestMode::Layout: + return run_dump_test(view, input_path, expectation_path, mode); + case TestMode::Ref: + return run_ref_test(view, input_path, expectation_path); + default: + VERIFY_NOT_REACHED(); + } +} + struct Test { String input_path; String expectation_path; @@ -286,14 +328,14 @@ struct Test { Optional result; }; -static ErrorOr collect_tests(Vector& tests, StringView path, StringView trail, TestMode mode) +static ErrorOr collect_dump_tests(Vector& tests, StringView path, StringView trail, TestMode mode) { Core::DirIterator it(TRY(String::formatted("{}/input/{}", path, trail)).to_deprecated_string(), Core::DirIterator::Flags::SkipDots); while (it.has_next()) { auto name = it.next_path(); auto input_path = TRY(FileSystem::real_path(TRY(String::formatted("{}/input/{}/{}", path, trail, name)))); if (FileSystem::is_directory(input_path)) { - TRY(collect_tests(tests, path, TRY(String::formatted("{}/{}", trail, name)), mode)); + TRY(collect_dump_tests(tests, path, TRY(String::formatted("{}/{}", trail, name)), mode)); continue; } if (!name.ends_with(".html"sv)) @@ -306,13 +348,37 @@ static ErrorOr collect_tests(Vector& tests, StringView path, StringV return {}; } +static ErrorOr collect_ref_tests(Vector& tests, StringView path) +{ + auto manifest_path = TRY(String::formatted("{}/manifest.json", path)); + auto manifest_file_or_error = Core::File::open(manifest_path, Core::File::OpenMode::Read); + if (manifest_file_or_error.is_error()) { + warnln("Failed opening '{}': {}", manifest_path, manifest_file_or_error.error()); + return manifest_file_or_error.release_error(); + } + + auto manifest_file = manifest_file_or_error.release_value(); + auto manifest = TRY(String::from_utf8(StringView(TRY(manifest_file->read_until_eof()).bytes()))); + auto manifest_json = TRY(JsonParser(manifest).parse()); + TRY(manifest_json.as_object().try_for_each_member([&](DeprecatedString const& key, AK::JsonValue const& value) -> ErrorOr { + TRY(String::from_deprecated_string(key)); + auto input_path = TRY(String::formatted("{}/{}", path, key)); + auto expectation_path = TRY(String::formatted("{}/{}", path, value.to_deprecated_string())); + tests.append({ input_path, expectation_path, TestMode::Ref, {} }); + return {}; + })); + + return {}; +} + static ErrorOr run_tests(HeadlessWebContentView& view, StringView test_root_path) { view.clear_content_filters(); Vector tests; - TRY(collect_tests(tests, TRY(String::formatted("{}/Layout", test_root_path)), "."sv, TestMode::Layout)); - TRY(collect_tests(tests, TRY(String::formatted("{}/Text", test_root_path)), "."sv, TestMode::Text)); + TRY(collect_dump_tests(tests, TRY(String::formatted("{}/Layout", test_root_path)), "."sv, TestMode::Layout)); + TRY(collect_dump_tests(tests, TRY(String::formatted("{}/Text", test_root_path)), "."sv, TestMode::Text)); + TRY(collect_ref_tests(tests, TRY(String::formatted("{}/Ref", test_root_path)))); size_t pass_count = 0; size_t fail_count = 0;