RGBCube/serenity
1
Fork 0
mirror of https://github.com/RGBCube/serenity synced 2025-05-20 03:55:07 +00:00
Code Issues Activity Actions

Browser+WebContent+WebDriver: Move Take Element Screenshot to WebContent

Browse source
This commit is contained in:
Timothy Flynn 2022-11-10 13:33:18 -05:00 • committed by Linus Groh
parent de1e882601
commit ea0561a0b3
15 changed files with 51 additions and 185 deletions
Download patch file Download diff file Expand all files Collapse all files

8
Userland/Applications/Browser/BrowserWindow.cpp
View file

@ -607,10 +607,6 @@ void BrowserWindow::create_new_tab(URL url, bool activate)
return active_tab().view().take_screenshot();
};
new_tab.webdriver_endpoints().on_scroll_element_into_view = [this](i32 element_id) {
active_tab().view().scroll_element_into_view(element_id);
};
new_tab.webdriver_endpoints().on_serialize_source = [this]() {
return active_tab().view().serialize_source();
};
@ -619,10 +615,6 @@ void BrowserWindow::create_new_tab(URL url, bool activate)
return active_tab().view().webdriver_execute_script(body, json_arguments, timeout, async);
};
new_tab.webdriver_endpoints().on_take_element_screenshot = [this](i32 element_id) {
return active_tab().view().take_element_screenshot(element_id);
};
new_tab.load(url);
dbgln_if(SPAM_DEBUG, "Added new tab {:p}, loading {}", &new_tab, url);

22
Userland/Applications/Browser/WebDriverConnection.cpp
View file

@ -136,26 +136,4 @@ void WebDriverConnection::update_cookie(Web::Cookie::Cookie const& cookie)
}
}
void WebDriverConnection::scroll_element_into_view(i32 element_id)
{
dbgln("WebDriverConnection: scroll_element_into_view {}", element_id);
if (auto browser_window = m_browser_window.strong_ref()) {
auto& tab = browser_window->active_tab();
if (tab.webdriver_endpoints().on_scroll_element_into_view)
tab.webdriver_endpoints().on_scroll_element_into_view(element_id);
}
}
Messages::WebDriverSessionClient::TakeElementScreenshotResponse WebDriverConnection::take_element_screenshot(i32 element_id)
{
dbgln_if(WEBDRIVER_DEBUG, "WebDriverConnection: take_element_screenshot {}", element_id);
if (auto browser_window = m_browser_window.strong_ref()) {
auto& tab = browser_window->active_tab();
if (tab.webdriver_endpoints().on_take_element_screenshot)
return { tab.webdriver_endpoints().on_take_element_screenshot(element_id) };
}
return { {} };
}
}

2
Userland/Applications/Browser/WebDriverConnection.h
View file

@ -48,8 +48,6 @@ public:
virtual Messages::WebDriverSessionClient::GetNamedCookieResponse get_named_cookie(String const& name) override;
virtual void add_cookie(Web::Cookie::ParsedCookie const&) override;
virtual void update_cookie(Web::Cookie::Cookie const&) override;
virtual void scroll_element_into_view(i32 element_id) override;
virtual Messages::WebDriverSessionClient::TakeElementScreenshotResponse take_element_screenshot(i32 element_id) override;
private:
WebDriverConnection(NonnullOwnPtr<Core::Stream::LocalSocket> socket, NonnullRefPtr<BrowserWindow> browser_window);

2
Userland/Applications/Browser/WebDriverEndpoints.h
View file

@ -23,8 +23,6 @@ public:
WebDriverEndpoints() = default;
~WebDriverEndpoints() = default;
Function<void(i32 element_id)> on_scroll_element_into_view;
Function<Gfx::ShareableBitmap(i32 element_id)> on_take_element_screenshot;
Function<String()> on_serialize_source;
Function<Messages::WebContentServer::WebdriverExecuteScriptResponse(String const& body, Vector<String> const& json_arguments, Optional<u64> const& timeout, bool async)> on_execute_script;
};

2
Userland/Applications/Browser/WebDriverSessionClient.ipc
View file

@ -24,6 +24,4 @@ endpoint WebDriverSessionClient {
get_named_cookie(String name) => (Optional<Web::Cookie::Cookie> cookie)
add_cookie(Web::Cookie::ParsedCookie cookie) =|
update_cookie(Web::Cookie::Cookie cookie) =|
scroll_element_into_view(i32 element_id) => ()
take_element_screenshot(i32 element_id) => (Gfx::ShareableBitmap data)
}

10
Userland/Libraries/LibWebView/OutOfProcessWebView.cpp
View file

@ -554,11 +554,6 @@ OrderedHashMap<String, String> OutOfProcessWebView::get_session_storage_entries(
return client().get_session_storage_entries();
}
void OutOfProcessWebView::scroll_element_into_view(i32 element_id)
{
return client().scroll_element_into_view(element_id);
}
void OutOfProcessWebView::set_content_filters(Vector<String> filters)
{
client().async_set_content_filters(filters);
@ -596,11 +591,6 @@ Gfx::ShareableBitmap OutOfProcessWebView::take_screenshot() const
return {};
}
Gfx::ShareableBitmap OutOfProcessWebView::take_element_screenshot(i32 element_id)
{
return client().take_element_screenshot(element_id);
}
Gfx::ShareableBitmap OutOfProcessWebView::take_document_screenshot()
{
return client().take_document_screenshot();

3
Userland/Libraries/LibWebView/OutOfProcessWebView.h
View file

@ -63,8 +63,6 @@ public:
OrderedHashMap<String, String> get_local_storage_entries();
OrderedHashMap<String, String> get_session_storage_entries();
void scroll_element_into_view(i32 element_id);
void set_content_filters(Vector<String>);
void set_proxy_mappings(Vector<String> proxies, HashMap<String, size_t> mappings);
void set_preferred_color_scheme(Web::CSS::PreferredColorScheme);
@ -76,7 +74,6 @@ public:
void set_system_visibility_state(bool visible);
Gfx::ShareableBitmap take_screenshot() const;
Gfx::ShareableBitmap take_element_screenshot(i32 element_id);
Gfx::ShareableBitmap take_document_screenshot();
Messages::WebContentServer::WebdriverExecuteScriptResponse webdriver_execute_script(String const& body, Vector<String> const& json_arguments, Optional<u64> const& timeout, bool async);

59
Userland/Services/WebContent/ConnectionFromClient.cpp
View file

@ -464,35 +464,6 @@ void ConnectionFromClient::js_console_request_messages(i32 start_index)
m_console_client->send_messages(start_index);
}
static Optional<Web::DOM::Element&> find_element_by_id(i32 element_id)
{
auto* node = Web::DOM::Node::from_id(element_id);
if (!node || !node->is_element())
return {};
return verify_cast<Web::DOM::Element>(*node);
}
// https://w3c.github.io/webdriver/#dfn-scrolls-into-view
void ConnectionFromClient::scroll_element_into_view(i32 element_id)
{
auto element = find_element_by_id(element_id);
if (!element.has_value())
return;
// 1. Let options be the following ScrollIntoViewOptions:
Web::DOM::ScrollIntoViewOptions options {};
// Logical scroll position "block"
// "end"
options.block = Web::Bindings::ScrollLogicalPosition::End;
// Logical scroll position "inline"
// "nearest"
options.inline_ = Web::Bindings::ScrollLogicalPosition::Nearest;
// 2. Run Function.[[Call]](scrollIntoView, options) with element as the this value.
element->scroll_into_view(options);
}
// https://w3c.github.io/webdriver/#dfn-calculate-the-absolute-position
static Gfx::IntPoint calculate_absolute_position_of_element(Web::Page const& page, JS::NonnullGCPtr<Web::Geometry::DOMRect> rect)
{
@ -511,36 +482,6 @@ static Gfx::IntPoint calculate_absolute_position_of_element(Web::Page const& pag
return { x, y };
}
static Gfx::IntRect calculate_absolute_rect_of_element(Web::Page const& page, Web::DOM::Element const& element)
{
auto bounding_rect = element.get_bounding_client_rect();
auto coordinates = calculate_absolute_position_of_element(page, bounding_rect);
return Gfx::IntRect {
coordinates.x(),
coordinates.y(),
static_cast<int>(bounding_rect->width()),
static_cast<int>(bounding_rect->height())
};
}
Messages::WebContentServer::TakeElementScreenshotResponse ConnectionFromClient::take_element_screenshot(i32 element_id)
{
auto element = find_element_by_id(element_id);
if (!element.has_value())
return { {} };
auto viewport_rect = page().top_level_browsing_context().viewport_rect();
auto rect = calculate_absolute_rect_of_element(page(), *element);
rect.intersect(viewport_rect);
auto bitmap = Gfx::Bitmap::try_create(Gfx::BitmapFormat::BGRA8888, rect.size()).release_value_but_fixme_should_propagate_errors();
m_page_host->paint(rect, *bitmap);
return { bitmap->to_shareable_bitmap() };
}
Messages::WebContentServer::TakeDocumentScreenshotResponse ConnectionFromClient::take_document_screenshot()
{
auto* document = page().top_level_browsing_context().active_document();

2
Userland/Services/WebContent/ConnectionFromClient.h
View file

@ -83,8 +83,6 @@ private:
virtual void run_javascript(String const&) override;
virtual void js_console_request_messages(i32) override;
virtual void scroll_element_into_view(i32 element_id) override;
virtual Messages::WebContentServer::TakeElementScreenshotResponse take_element_screenshot(i32 element_id) override;
virtual Messages::WebContentServer::TakeDocumentScreenshotResponse take_document_screenshot() override;
virtual Messages::WebContentServer::GetLocalStorageEntriesResponse get_local_storage_entries() override;

2
Userland/Services/WebContent/WebContentServer.ipc
View file

@ -42,8 +42,6 @@ endpoint WebContentServer
js_console_input(String js_source) =|
js_console_request_messages(i32 start_index) =|
scroll_element_into_view(i32 element_id) => ()
take_element_screenshot(i32 element_id) => (Gfx::ShareableBitmap data)
take_document_screenshot() => (Gfx::ShareableBitmap data)
webdriver_execute_script(String body, Vector<String> json_arguments, Optional<u64> timeout, bool async) => (Web::WebDriver::ExecuteScriptResultType result_type, String json_result)

1
Userland/Services/WebContent/WebDriverClient.ipc
View file

@ -22,4 +22,5 @@ endpoint WebDriverClient {
get_element_rect(String element_id) => (Web::WebDriver::Response response)
is_element_enabled(String element_id) => (Web::WebDriver::Response response)
take_screenshot() => (Web::WebDriver::Response response)
take_element_screenshot(String element_id) => (Web::WebDriver::Response response)
}

48
Userland/Services/WebContent/WebDriverConnection.cpp
View file

@ -139,6 +139,22 @@ static ErrorOr<Web::DOM::Element*, Web::WebDriver::Error> get_known_connected_el
return static_cast<Web::DOM::Element*>(node);
}
// https://w3c.github.io/webdriver/#dfn-scrolls-into-view
static void scroll_element_into_view(Web::DOM::Element& element)
{
// 1. Let options be the following ScrollIntoViewOptions:
Web::DOM::ScrollIntoViewOptions options {};
// Logical scroll position "block"
// "end"
options.block = Web::Bindings::ScrollLogicalPosition::End;
// Logical scroll position "inline"
// "nearest"
options.inline_ = Web::Bindings::ScrollLogicalPosition::Nearest;
// 2. Run Function.[[Call]](scrollIntoView, options) with element as the this value.
element.scroll_into_view(options);
}
static ErrorOr<String, Web::WebDriver::Error> get_property(JsonValue const& payload, StringView key)
{
if (!payload.is_object())
@ -740,6 +756,38 @@ Messages::WebDriverClient::TakeScreenshotResponse WebDriverConnection::take_scre
return make_success_response(move(encoded_string));
}
// 17.2 Take Element Screenshot, https://w3c.github.io/webdriver/#dfn-take-element-screenshot
Messages::WebDriverClient::TakeElementScreenshotResponse WebDriverConnection::take_element_screenshot(String const& element_id)
{
// 1. If the current top-level browsing context is no longer open, return error with error code no such window.
TRY(ensure_open_top_level_browsing_context());
// FIXME: 2. Handle any user prompts and return its value if it is an error.
// 3. Let element be the result of trying to get a known connected element with url variable element id.
auto* element = TRY(get_known_connected_element(element_id));
// 4. Scroll into view the element.
scroll_element_into_view(*element);
// 5. When the user agent is next to run the animation frame callbacks:
// a. Let element rect be element’s rectangle.
// b. Let screenshot result be the result of trying to call draw a bounding box from the framebuffer, given element rect as an argument.
// c. Let canvas be a canvas element of screenshot result’s data.
// d. Let encoding result be the result of trying encoding a canvas as Base64 canvas.
// e. Let encoded string be encoding result’s data.
auto element_rect = calculate_absolute_rect_of_element(m_page_host.page(), *element);
auto encoded_string = TRY(Web::WebDriver::capture_element_screenshot(
[&](auto const& rect, auto& bitmap) { m_page_host.paint(rect, bitmap); },
m_page_host.page(),
*element,
element_rect));
// 6. Return success with data encoded string.
return make_success_response(move(encoded_string));
}
// https://w3c.github.io/webdriver/#dfn-no-longer-open
ErrorOr<void, Web::WebDriver::Error> WebDriverConnection::ensure_open_top_level_browsing_context()
{

1
Userland/Services/WebContent/WebDriverConnection.h
View file

@ -52,6 +52,7 @@ private:
virtual Messages::WebDriverClient::GetElementRectResponse get_element_rect(String const& element_id) override;
virtual Messages::WebDriverClient::IsElementEnabledResponse is_element_enabled(String const& element_id) override;
virtual Messages::WebDriverClient::TakeScreenshotResponse take_screenshot() override;
virtual Messages::WebDriverClient::TakeElementScreenshotResponse take_element_screenshot(String const& element_id) override;
ErrorOr<void, Web::WebDriver::Error> ensure_open_top_level_browsing_context();
void restore_the_window();

3
Userland/Services/WebDriver/Client.cpp
View file

@ -815,8 +815,7 @@ Web::WebDriver::Response Client::handle_take_element_screenshot(Vector<StringVie
{
dbgln_if(WEBDRIVER_DEBUG, "Handling GET /session/<session_id>/element/<element_id>/screenshot");
auto* session = TRY(find_session_with_id(parameters[0]));
auto result = TRY(session->take_element_screenshot(parameters[1]));
return make_json_value(result);
return session->web_content_connection().take_element_screenshot(parameters[1]);
}
}

71
Userland/Services/WebDriver/Session.cpp
View file

@ -11,14 +11,11 @@
#include "Session.h"
#include "BrowserConnection.h"
#include "Client.h"
#include <AK/Base64.h>
#include <AK/NumericLimits.h>
#include <AK/Time.h>
#include <AK/URL.h>
#include <LibCore/LocalServer.h>
#include <LibCore/Stream.h>
#include <LibCore/System.h>
#include <LibGfx/PNGWriter.h>
#include <LibGfx/Point.h>
#include <LibGfx/Rect.h>
#include <LibGfx/Size.h>
@ -300,18 +297,6 @@ Web::WebDriver::Response Session::get_window_handles() const
return JsonValue { handles };
}
// https://w3c.github.io/webdriver/#dfn-get-a-known-connected-element
static ErrorOr<i32, Web::WebDriver::Error> get_known_connected_element(StringView element_id)
{
// NOTE: The whole concept of "connected elements" is not implemented yet. See get_or_create_a_web_element_reference().
// For now the element is only represented by its ID.
auto maybe_element_id = element_id.to_int();
if (!maybe_element_id.has_value())
return Web::WebDriver::Error::from_code(Web::WebDriver::ErrorCode::InvalidArgument, "Element ID is not an integer");
return maybe_element_id.release_value();
}
// 13.1 Get Page Source, https://w3c.github.io/webdriver/#dfn-get-page-source
Web::WebDriver::Response Session::get_source()
{
@ -669,60 +654,4 @@ Web::WebDriver::Response Session::delete_all_cookies()
return JsonValue();
}
// https://w3c.github.io/webdriver/#dfn-encoding-a-canvas-as-base64
static ErrorOr<String, Web::WebDriver::Error> encode_bitmap_as_canvas_element(Gfx::Bitmap const& bitmap)
{
// FIXME: 1. If the canvas element’s bitmap’s origin-clean flag is set to false, return error with error code unable to capture screen.
// 2. If the canvas element’s bitmap has no pixels (i.e. either its horizontal dimension or vertical dimension is zero) then return error with error code unable to capture screen.
if (bitmap.width() == 0 || bitmap.height() == 0)
return Web::WebDriver::Error::from_code(Web::WebDriver::ErrorCode::UnableToCaptureScreen, "Captured screenshot is empty"sv);
// 3. Let file be a serialization of the canvas element’s bitmap as a file, using "image/png" as an argument.
auto file = Gfx::PNGWriter::encode(bitmap);
// 4. Let data url be a data: URL representing file. [RFC2397]
auto data_url = AK::URL::create_with_data("image/png"sv, encode_base64(file), true).to_string();
// 5. Let index be the index of "," in data url.
auto index = data_url.find(',');
VERIFY(index.has_value());
// 6. Let encoded string be a substring of data url using (index + 1) as the start argument.
auto encoded_string = data_url.substring(*index + 1);
// 7. Return success with data encoded string.
return encoded_string;
}
// 17.2 Take Element Screenshot, https://w3c.github.io/webdriver/#dfn-take-element-screenshot
Web::WebDriver::Response Session::take_element_screenshot(StringView parameter_element_id)
{
// 1. If the current top-level browsing context is no longer open, return error with error code no such window.
TRY(check_for_open_top_level_browsing_context_or_return_error());
// FIXME: 2. Handle any user prompts and return its value if it is an error.
// 3. Let element be the result of trying to get a known connected element with url variable element id.
auto element_id = TRY(get_known_connected_element(parameter_element_id));
// 4. Scroll into view the element.
m_browser_connection->scroll_element_into_view(element_id);
// 5. When the user agent is next to run the animation frame callbacks:
// a. Let element rect be element’s rectangle.
// b. Let screenshot result be the result of trying to call draw a bounding box from the framebuffer, given element rect as an argument.
auto screenshot = m_browser_connection->take_element_screenshot(element_id);
if (!screenshot.is_valid())
return Web::WebDriver::Error::from_code(Web::WebDriver::ErrorCode::UnableToCaptureScreen, "Unable to capture screenshot"sv);
// c. Let canvas be a canvas element of screenshot result’s data.
// d. Let encoding result be the result of trying encoding a canvas as Base64 canvas.
// e. Let encoded string be encoding result’s data.
auto encoded_string = TRY(encode_bitmap_as_canvas_element(*screenshot.bitmap()));
// 6. Return success with data encoded string.
return JsonValue { encoded_string };
}
}