From ea0561a0b398f249ee1bb0cc28d763adc7858076 Mon Sep 17 00:00:00 2001 From: Timothy Flynn Date: Thu, 10 Nov 2022 13:33:18 -0500 Subject: [PATCH] Browser+WebContent+WebDriver: Move Take Element Screenshot to WebContent --- .../Applications/Browser/BrowserWindow.cpp | 8 --- .../Browser/WebDriverConnection.cpp | 22 ------ .../Browser/WebDriverConnection.h | 2 - .../Applications/Browser/WebDriverEndpoints.h | 2 - .../Browser/WebDriverSessionClient.ipc | 2 - .../LibWebView/OutOfProcessWebView.cpp | 10 --- .../LibWebView/OutOfProcessWebView.h | 3 - .../WebContent/ConnectionFromClient.cpp | 59 --------------- .../WebContent/ConnectionFromClient.h | 2 - .../Services/WebContent/WebContentServer.ipc | 2 - .../Services/WebContent/WebDriverClient.ipc | 1 + .../WebContent/WebDriverConnection.cpp | 48 +++++++++++++ .../Services/WebContent/WebDriverConnection.h | 1 + Userland/Services/WebDriver/Client.cpp | 3 +- Userland/Services/WebDriver/Session.cpp | 71 ------------------- 15 files changed, 51 insertions(+), 185 deletions(-) diff --git a/Userland/Applications/Browser/BrowserWindow.cpp b/Userland/Applications/Browser/BrowserWindow.cpp index ac473ae670..dcb77be9e8 100644 --- a/Userland/Applications/Browser/BrowserWindow.cpp +++ b/Userland/Applications/Browser/BrowserWindow.cpp @@ -607,10 +607,6 @@ void BrowserWindow::create_new_tab(URL url, bool activate) return active_tab().view().take_screenshot(); }; - new_tab.webdriver_endpoints().on_scroll_element_into_view = [this](i32 element_id) { - active_tab().view().scroll_element_into_view(element_id); - }; - new_tab.webdriver_endpoints().on_serialize_source = [this]() { return active_tab().view().serialize_source(); }; @@ -619,10 +615,6 @@ void BrowserWindow::create_new_tab(URL url, bool activate) return active_tab().view().webdriver_execute_script(body, json_arguments, timeout, async); }; - new_tab.webdriver_endpoints().on_take_element_screenshot = [this](i32 element_id) { - return active_tab().view().take_element_screenshot(element_id); - }; - new_tab.load(url); dbgln_if(SPAM_DEBUG, "Added new tab {:p}, loading {}", &new_tab, url); diff --git a/Userland/Applications/Browser/WebDriverConnection.cpp b/Userland/Applications/Browser/WebDriverConnection.cpp index db09863973..d0c26bc3c1 100644 --- a/Userland/Applications/Browser/WebDriverConnection.cpp +++ b/Userland/Applications/Browser/WebDriverConnection.cpp @@ -136,26 +136,4 @@ void WebDriverConnection::update_cookie(Web::Cookie::Cookie const& cookie) } } -void WebDriverConnection::scroll_element_into_view(i32 element_id) -{ - dbgln("WebDriverConnection: scroll_element_into_view {}", element_id); - if (auto browser_window = m_browser_window.strong_ref()) { - auto& tab = browser_window->active_tab(); - if (tab.webdriver_endpoints().on_scroll_element_into_view) - tab.webdriver_endpoints().on_scroll_element_into_view(element_id); - } -} - -Messages::WebDriverSessionClient::TakeElementScreenshotResponse WebDriverConnection::take_element_screenshot(i32 element_id) -{ - dbgln_if(WEBDRIVER_DEBUG, "WebDriverConnection: take_element_screenshot {}", element_id); - if (auto browser_window = m_browser_window.strong_ref()) { - auto& tab = browser_window->active_tab(); - if (tab.webdriver_endpoints().on_take_element_screenshot) - return { tab.webdriver_endpoints().on_take_element_screenshot(element_id) }; - } - - return { {} }; -} - } diff --git a/Userland/Applications/Browser/WebDriverConnection.h b/Userland/Applications/Browser/WebDriverConnection.h index 5d2639e6fe..7dbd966a39 100644 --- a/Userland/Applications/Browser/WebDriverConnection.h +++ b/Userland/Applications/Browser/WebDriverConnection.h @@ -48,8 +48,6 @@ public: virtual Messages::WebDriverSessionClient::GetNamedCookieResponse get_named_cookie(String const& name) override; virtual void add_cookie(Web::Cookie::ParsedCookie const&) override; virtual void update_cookie(Web::Cookie::Cookie const&) override; - virtual void scroll_element_into_view(i32 element_id) override; - virtual Messages::WebDriverSessionClient::TakeElementScreenshotResponse take_element_screenshot(i32 element_id) override; private: WebDriverConnection(NonnullOwnPtr socket, NonnullRefPtr browser_window); diff --git a/Userland/Applications/Browser/WebDriverEndpoints.h b/Userland/Applications/Browser/WebDriverEndpoints.h index 45b21cf875..e59a1d2435 100644 --- a/Userland/Applications/Browser/WebDriverEndpoints.h +++ b/Userland/Applications/Browser/WebDriverEndpoints.h @@ -23,8 +23,6 @@ public: WebDriverEndpoints() = default; ~WebDriverEndpoints() = default; - Function on_scroll_element_into_view; - Function on_take_element_screenshot; Function on_serialize_source; Function const& json_arguments, Optional const& timeout, bool async)> on_execute_script; }; diff --git a/Userland/Applications/Browser/WebDriverSessionClient.ipc b/Userland/Applications/Browser/WebDriverSessionClient.ipc index eb68c91d46..759c4ba81f 100644 --- a/Userland/Applications/Browser/WebDriverSessionClient.ipc +++ b/Userland/Applications/Browser/WebDriverSessionClient.ipc @@ -24,6 +24,4 @@ endpoint WebDriverSessionClient { get_named_cookie(String name) => (Optional cookie) add_cookie(Web::Cookie::ParsedCookie cookie) =| update_cookie(Web::Cookie::Cookie cookie) =| - scroll_element_into_view(i32 element_id) => () - take_element_screenshot(i32 element_id) => (Gfx::ShareableBitmap data) } diff --git a/Userland/Libraries/LibWebView/OutOfProcessWebView.cpp b/Userland/Libraries/LibWebView/OutOfProcessWebView.cpp index 8a7f03570b..05924c6aa5 100644 --- a/Userland/Libraries/LibWebView/OutOfProcessWebView.cpp +++ b/Userland/Libraries/LibWebView/OutOfProcessWebView.cpp @@ -554,11 +554,6 @@ OrderedHashMap OutOfProcessWebView::get_session_storage_entries( return client().get_session_storage_entries(); } -void OutOfProcessWebView::scroll_element_into_view(i32 element_id) -{ - return client().scroll_element_into_view(element_id); -} - void OutOfProcessWebView::set_content_filters(Vector filters) { client().async_set_content_filters(filters); @@ -596,11 +591,6 @@ Gfx::ShareableBitmap OutOfProcessWebView::take_screenshot() const return {}; } -Gfx::ShareableBitmap OutOfProcessWebView::take_element_screenshot(i32 element_id) -{ - return client().take_element_screenshot(element_id); -} - Gfx::ShareableBitmap OutOfProcessWebView::take_document_screenshot() { return client().take_document_screenshot(); diff --git a/Userland/Libraries/LibWebView/OutOfProcessWebView.h b/Userland/Libraries/LibWebView/OutOfProcessWebView.h index e7c533c2d7..c24d37a336 100644 --- a/Userland/Libraries/LibWebView/OutOfProcessWebView.h +++ b/Userland/Libraries/LibWebView/OutOfProcessWebView.h @@ -63,8 +63,6 @@ public: OrderedHashMap get_local_storage_entries(); OrderedHashMap get_session_storage_entries(); - void scroll_element_into_view(i32 element_id); - void set_content_filters(Vector); void set_proxy_mappings(Vector proxies, HashMap mappings); void set_preferred_color_scheme(Web::CSS::PreferredColorScheme); @@ -76,7 +74,6 @@ public: void set_system_visibility_state(bool visible); Gfx::ShareableBitmap take_screenshot() const; - Gfx::ShareableBitmap take_element_screenshot(i32 element_id); Gfx::ShareableBitmap take_document_screenshot(); Messages::WebContentServer::WebdriverExecuteScriptResponse webdriver_execute_script(String const& body, Vector const& json_arguments, Optional const& timeout, bool async); diff --git a/Userland/Services/WebContent/ConnectionFromClient.cpp b/Userland/Services/WebContent/ConnectionFromClient.cpp index 07385fece1..6699516fd6 100644 --- a/Userland/Services/WebContent/ConnectionFromClient.cpp +++ b/Userland/Services/WebContent/ConnectionFromClient.cpp @@ -464,35 +464,6 @@ void ConnectionFromClient::js_console_request_messages(i32 start_index) m_console_client->send_messages(start_index); } -static Optional find_element_by_id(i32 element_id) -{ - auto* node = Web::DOM::Node::from_id(element_id); - if (!node || !node->is_element()) - return {}; - - return verify_cast(*node); -} - -// https://w3c.github.io/webdriver/#dfn-scrolls-into-view -void ConnectionFromClient::scroll_element_into_view(i32 element_id) -{ - auto element = find_element_by_id(element_id); - if (!element.has_value()) - return; - - // 1. Let options be the following ScrollIntoViewOptions: - Web::DOM::ScrollIntoViewOptions options {}; - // Logical scroll position "block" - // "end" - options.block = Web::Bindings::ScrollLogicalPosition::End; - // Logical scroll position "inline" - // "nearest" - options.inline_ = Web::Bindings::ScrollLogicalPosition::Nearest; - - // 2. Run Function.[[Call]](scrollIntoView, options) with element as the this value. - element->scroll_into_view(options); -} - // https://w3c.github.io/webdriver/#dfn-calculate-the-absolute-position static Gfx::IntPoint calculate_absolute_position_of_element(Web::Page const& page, JS::NonnullGCPtr rect) { @@ -511,36 +482,6 @@ static Gfx::IntPoint calculate_absolute_position_of_element(Web::Page const& pag return { x, y }; } -static Gfx::IntRect calculate_absolute_rect_of_element(Web::Page const& page, Web::DOM::Element const& element) -{ - auto bounding_rect = element.get_bounding_client_rect(); - auto coordinates = calculate_absolute_position_of_element(page, bounding_rect); - - return Gfx::IntRect { - coordinates.x(), - coordinates.y(), - static_cast(bounding_rect->width()), - static_cast(bounding_rect->height()) - }; -} - -Messages::WebContentServer::TakeElementScreenshotResponse ConnectionFromClient::take_element_screenshot(i32 element_id) -{ - auto element = find_element_by_id(element_id); - if (!element.has_value()) - return { {} }; - - auto viewport_rect = page().top_level_browsing_context().viewport_rect(); - - auto rect = calculate_absolute_rect_of_element(page(), *element); - rect.intersect(viewport_rect); - - auto bitmap = Gfx::Bitmap::try_create(Gfx::BitmapFormat::BGRA8888, rect.size()).release_value_but_fixme_should_propagate_errors(); - m_page_host->paint(rect, *bitmap); - - return { bitmap->to_shareable_bitmap() }; -} - Messages::WebContentServer::TakeDocumentScreenshotResponse ConnectionFromClient::take_document_screenshot() { auto* document = page().top_level_browsing_context().active_document(); diff --git a/Userland/Services/WebContent/ConnectionFromClient.h b/Userland/Services/WebContent/ConnectionFromClient.h index 42211eed65..8ae5606c34 100644 --- a/Userland/Services/WebContent/ConnectionFromClient.h +++ b/Userland/Services/WebContent/ConnectionFromClient.h @@ -83,8 +83,6 @@ private: virtual void run_javascript(String const&) override; virtual void js_console_request_messages(i32) override; - virtual void scroll_element_into_view(i32 element_id) override; - virtual Messages::WebContentServer::TakeElementScreenshotResponse take_element_screenshot(i32 element_id) override; virtual Messages::WebContentServer::TakeDocumentScreenshotResponse take_document_screenshot() override; virtual Messages::WebContentServer::GetLocalStorageEntriesResponse get_local_storage_entries() override; diff --git a/Userland/Services/WebContent/WebContentServer.ipc b/Userland/Services/WebContent/WebContentServer.ipc index 6178a26256..6e08649eb5 100644 --- a/Userland/Services/WebContent/WebContentServer.ipc +++ b/Userland/Services/WebContent/WebContentServer.ipc @@ -42,8 +42,6 @@ endpoint WebContentServer js_console_input(String js_source) =| js_console_request_messages(i32 start_index) =| - scroll_element_into_view(i32 element_id) => () - take_element_screenshot(i32 element_id) => (Gfx::ShareableBitmap data) take_document_screenshot() => (Gfx::ShareableBitmap data) webdriver_execute_script(String body, Vector json_arguments, Optional timeout, bool async) => (Web::WebDriver::ExecuteScriptResultType result_type, String json_result) diff --git a/Userland/Services/WebContent/WebDriverClient.ipc b/Userland/Services/WebContent/WebDriverClient.ipc index e9a629f5e4..7b9d3cc3d0 100644 --- a/Userland/Services/WebContent/WebDriverClient.ipc +++ b/Userland/Services/WebContent/WebDriverClient.ipc @@ -22,4 +22,5 @@ endpoint WebDriverClient { get_element_rect(String element_id) => (Web::WebDriver::Response response) is_element_enabled(String element_id) => (Web::WebDriver::Response response) take_screenshot() => (Web::WebDriver::Response response) + take_element_screenshot(String element_id) => (Web::WebDriver::Response response) } diff --git a/Userland/Services/WebContent/WebDriverConnection.cpp b/Userland/Services/WebContent/WebDriverConnection.cpp index eca78dabc7..cd2c58d8a8 100644 --- a/Userland/Services/WebContent/WebDriverConnection.cpp +++ b/Userland/Services/WebContent/WebDriverConnection.cpp @@ -139,6 +139,22 @@ static ErrorOr get_known_connected_el return static_cast(node); } +// https://w3c.github.io/webdriver/#dfn-scrolls-into-view +static void scroll_element_into_view(Web::DOM::Element& element) +{ + // 1. Let options be the following ScrollIntoViewOptions: + Web::DOM::ScrollIntoViewOptions options {}; + // Logical scroll position "block" + // "end" + options.block = Web::Bindings::ScrollLogicalPosition::End; + // Logical scroll position "inline" + // "nearest" + options.inline_ = Web::Bindings::ScrollLogicalPosition::Nearest; + + // 2. Run Function.[[Call]](scrollIntoView, options) with element as the this value. + element.scroll_into_view(options); +} + static ErrorOr get_property(JsonValue const& payload, StringView key) { if (!payload.is_object()) @@ -740,6 +756,38 @@ Messages::WebDriverClient::TakeScreenshotResponse WebDriverConnection::take_scre return make_success_response(move(encoded_string)); } +// 17.2 Take Element Screenshot, https://w3c.github.io/webdriver/#dfn-take-element-screenshot +Messages::WebDriverClient::TakeElementScreenshotResponse WebDriverConnection::take_element_screenshot(String const& element_id) +{ + // 1. If the current top-level browsing context is no longer open, return error with error code no such window. + TRY(ensure_open_top_level_browsing_context()); + + // FIXME: 2. Handle any user prompts and return its value if it is an error. + + // 3. Let element be the result of trying to get a known connected element with url variable element id. + auto* element = TRY(get_known_connected_element(element_id)); + + // 4. Scroll into view the element. + scroll_element_into_view(*element); + + // 5. When the user agent is next to run the animation frame callbacks: + // a. Let element rect be element’s rectangle. + // b. Let screenshot result be the result of trying to call draw a bounding box from the framebuffer, given element rect as an argument. + // c. Let canvas be a canvas element of screenshot result’s data. + // d. Let encoding result be the result of trying encoding a canvas as Base64 canvas. + // e. Let encoded string be encoding result’s data. + auto element_rect = calculate_absolute_rect_of_element(m_page_host.page(), *element); + + auto encoded_string = TRY(Web::WebDriver::capture_element_screenshot( + [&](auto const& rect, auto& bitmap) { m_page_host.paint(rect, bitmap); }, + m_page_host.page(), + *element, + element_rect)); + + // 6. Return success with data encoded string. + return make_success_response(move(encoded_string)); +} + // https://w3c.github.io/webdriver/#dfn-no-longer-open ErrorOr WebDriverConnection::ensure_open_top_level_browsing_context() { diff --git a/Userland/Services/WebContent/WebDriverConnection.h b/Userland/Services/WebContent/WebDriverConnection.h index 9eb89d01d2..ea1459bee0 100644 --- a/Userland/Services/WebContent/WebDriverConnection.h +++ b/Userland/Services/WebContent/WebDriverConnection.h @@ -52,6 +52,7 @@ private: virtual Messages::WebDriverClient::GetElementRectResponse get_element_rect(String const& element_id) override; virtual Messages::WebDriverClient::IsElementEnabledResponse is_element_enabled(String const& element_id) override; virtual Messages::WebDriverClient::TakeScreenshotResponse take_screenshot() override; + virtual Messages::WebDriverClient::TakeElementScreenshotResponse take_element_screenshot(String const& element_id) override; ErrorOr ensure_open_top_level_browsing_context(); void restore_the_window(); diff --git a/Userland/Services/WebDriver/Client.cpp b/Userland/Services/WebDriver/Client.cpp index 1d33b8be94..c8e26377a9 100644 --- a/Userland/Services/WebDriver/Client.cpp +++ b/Userland/Services/WebDriver/Client.cpp @@ -815,8 +815,7 @@ Web::WebDriver::Response Client::handle_take_element_screenshot(Vector/element//screenshot"); auto* session = TRY(find_session_with_id(parameters[0])); - auto result = TRY(session->take_element_screenshot(parameters[1])); - return make_json_value(result); + return session->web_content_connection().take_element_screenshot(parameters[1]); } } diff --git a/Userland/Services/WebDriver/Session.cpp b/Userland/Services/WebDriver/Session.cpp index 29d6fc0706..a69e9fffee 100644 --- a/Userland/Services/WebDriver/Session.cpp +++ b/Userland/Services/WebDriver/Session.cpp @@ -11,14 +11,11 @@ #include "Session.h" #include "BrowserConnection.h" #include "Client.h" -#include #include #include -#include #include #include #include -#include #include #include #include @@ -300,18 +297,6 @@ Web::WebDriver::Response Session::get_window_handles() const return JsonValue { handles }; } -// https://w3c.github.io/webdriver/#dfn-get-a-known-connected-element -static ErrorOr get_known_connected_element(StringView element_id) -{ - // NOTE: The whole concept of "connected elements" is not implemented yet. See get_or_create_a_web_element_reference(). - // For now the element is only represented by its ID. - auto maybe_element_id = element_id.to_int(); - if (!maybe_element_id.has_value()) - return Web::WebDriver::Error::from_code(Web::WebDriver::ErrorCode::InvalidArgument, "Element ID is not an integer"); - - return maybe_element_id.release_value(); -} - // 13.1 Get Page Source, https://w3c.github.io/webdriver/#dfn-get-page-source Web::WebDriver::Response Session::get_source() { @@ -669,60 +654,4 @@ Web::WebDriver::Response Session::delete_all_cookies() return JsonValue(); } -// https://w3c.github.io/webdriver/#dfn-encoding-a-canvas-as-base64 -static ErrorOr encode_bitmap_as_canvas_element(Gfx::Bitmap const& bitmap) -{ - // FIXME: 1. If the canvas element’s bitmap’s origin-clean flag is set to false, return error with error code unable to capture screen. - - // 2. If the canvas element’s bitmap has no pixels (i.e. either its horizontal dimension or vertical dimension is zero) then return error with error code unable to capture screen. - if (bitmap.width() == 0 || bitmap.height() == 0) - return Web::WebDriver::Error::from_code(Web::WebDriver::ErrorCode::UnableToCaptureScreen, "Captured screenshot is empty"sv); - - // 3. Let file be a serialization of the canvas element’s bitmap as a file, using "image/png" as an argument. - auto file = Gfx::PNGWriter::encode(bitmap); - - // 4. Let data url be a data: URL representing file. [RFC2397] - auto data_url = AK::URL::create_with_data("image/png"sv, encode_base64(file), true).to_string(); - - // 5. Let index be the index of "," in data url. - auto index = data_url.find(','); - VERIFY(index.has_value()); - - // 6. Let encoded string be a substring of data url using (index + 1) as the start argument. - auto encoded_string = data_url.substring(*index + 1); - - // 7. Return success with data encoded string. - return encoded_string; -} - -// 17.2 Take Element Screenshot, https://w3c.github.io/webdriver/#dfn-take-element-screenshot -Web::WebDriver::Response Session::take_element_screenshot(StringView parameter_element_id) -{ - // 1. If the current top-level browsing context is no longer open, return error with error code no such window. - TRY(check_for_open_top_level_browsing_context_or_return_error()); - - // FIXME: 2. Handle any user prompts and return its value if it is an error. - - // 3. Let element be the result of trying to get a known connected element with url variable element id. - auto element_id = TRY(get_known_connected_element(parameter_element_id)); - - // 4. Scroll into view the element. - m_browser_connection->scroll_element_into_view(element_id); - - // 5. When the user agent is next to run the animation frame callbacks: - // a. Let element rect be element’s rectangle. - // b. Let screenshot result be the result of trying to call draw a bounding box from the framebuffer, given element rect as an argument. - auto screenshot = m_browser_connection->take_element_screenshot(element_id); - if (!screenshot.is_valid()) - return Web::WebDriver::Error::from_code(Web::WebDriver::ErrorCode::UnableToCaptureScreen, "Unable to capture screenshot"sv); - - // c. Let canvas be a canvas element of screenshot result’s data. - // d. Let encoding result be the result of trying encoding a canvas as Base64 canvas. - // e. Let encoded string be encoding result’s data. - auto encoded_string = TRY(encode_bitmap_as_canvas_element(*screenshot.bitmap())); - - // 6. Return success with data encoded string. - return JsonValue { encoded_string }; -} - }