RGBCube/serenity
1
Fork 0
mirror of https://github.com/RGBCube/serenity synced 2025-07-26 08:47:34 +00:00
Code Issues Activity Actions

WebDriver+Browser: Implement GET /session/{id}/element/{id}/screenshot

Browse source
This commit is contained in:
Timothy Flynn 2022-11-04 20:28:42 -04:00 • committed by Linus Groh
parent 3c1c2994f1
commit b0eb45f7c7
9 changed files with 79 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

8
Userland/Applications/Browser/BrowserWindow.cpp
View file

@ -607,6 +607,10 @@ void BrowserWindow::create_new_tab(URL url, bool activate)
return active_tab().view().query_selector_all(start_node_id, selector);
};
new_tab.webdriver_endpoints().on_scroll_element_into_view = [this](i32 element_id) {
active_tab().view().scroll_element_into_view(element_id);
};
new_tab.webdriver_endpoints().on_is_element_selected = [this](i32 element_id) {
return active_tab().view().is_element_selected(element_id);
};
@ -651,6 +655,10 @@ void BrowserWindow::create_new_tab(URL url, bool activate)
return active_tab().view().webdriver_execute_script(body, json_arguments, timeout, async);
};
new_tab.webdriver_endpoints().on_take_element_screenshot = [this](i32 element_id) {
return active_tab().view().take_element_screenshot(element_id);
};
new_tab.load(url);
dbgln_if(SPAM_DEBUG, "Added new tab {:p}, loading {}", &new_tab, url);

22
Userland/Applications/Browser/WebDriverConnection.cpp
View file

@ -218,6 +218,16 @@ Messages::WebDriverSessionClient::QuerySelectorAllResponse WebDriverConnection::
return { {} };
}
void WebDriverConnection::scroll_element_into_view(i32 element_id)
{
dbgln("WebDriverConnection: scroll_element_into_view {}", element_id);
if (auto browser_window = m_browser_window.strong_ref()) {
auto& tab = browser_window->active_tab();
if (tab.webdriver_endpoints().on_scroll_element_into_view)
tab.webdriver_endpoints().on_scroll_element_into_view(element_id);
}
}
Messages::WebDriverSessionClient::IsElementSelectedResponse WebDriverConnection::is_element_selected(i32 element_id)
{
dbgln("WebDriverConnection: is_element_selected {}", element_id);
@ -329,4 +339,16 @@ Messages::WebDriverSessionClient::TakeScreenshotResponse WebDriverConnection::ta
return { {} };
}
Messages::WebDriverSessionClient::TakeElementScreenshotResponse WebDriverConnection::take_element_screenshot(i32 element_id)
{
dbgln_if(WEBDRIVER_DEBUG, "WebDriverConnection: take_element_screenshot {}", element_id);
if (auto browser_window = m_browser_window.strong_ref()) {
auto& tab = browser_window->active_tab();
if (tab.webdriver_endpoints().on_take_element_screenshot)
return { tab.webdriver_endpoints().on_take_element_screenshot(element_id) };
}
return { {} };
}
}

2
Userland/Applications/Browser/WebDriverConnection.h
View file

@ -58,6 +58,7 @@ public:
virtual void update_cookie(Web::Cookie::Cookie const&) override;
virtual Messages::WebDriverSessionClient::GetDocumentElementResponse get_document_element() override;
virtual Messages::WebDriverSessionClient::QuerySelectorAllResponse query_selector_all(i32 start_node_id, String const& selector) override;
virtual void scroll_element_into_view(i32 element_id) override;
virtual Messages::WebDriverSessionClient::IsElementSelectedResponse is_element_selected(i32 element_id) override;
virtual Messages::WebDriverSessionClient::GetElementAttributeResponse get_element_attribute(i32 element_id, String const& name) override;
virtual Messages::WebDriverSessionClient::GetElementPropertyResponse get_element_property(i32 element_id, String const& name) override;
@ -68,6 +69,7 @@ public:
virtual Messages::WebDriverSessionClient::GetElementRectResponse get_element_rect(i32 element_id) override;
virtual Messages::WebDriverSessionClient::IsElementEnabledResponse is_element_enabled(i32 element_id) override;
virtual Messages::WebDriverSessionClient::TakeScreenshotResponse take_screenshot() override;
virtual Messages::WebDriverSessionClient::TakeElementScreenshotResponse take_element_screenshot(i32 element_id) override;
private:
WebDriverConnection(NonnullOwnPtr<Core::Stream::LocalSocket> socket, NonnullRefPtr<BrowserWindow> browser_window);

2
Userland/Applications/Browser/WebDriverEndpoints.h
View file

@ -25,6 +25,7 @@ public:
Function<Optional<i32>()> on_get_document_element;
Function<Optional<Vector<i32>>(i32 start_node_id, String const&)> on_query_selector_all;
Function<void(i32 element_id)> on_scroll_element_into_view;
Function<bool(i32 element_id)> on_is_element_selected;
Function<Optional<String>(i32 element_id, String const&)> on_get_element_attribute;
Function<Optional<String>(i32 element_id, String const&)> on_get_element_property;
@ -34,6 +35,7 @@ public:
Function<String(i32 element_id)> on_get_element_tag_name;
Function<Gfx::IntRect(i32 element_id)> on_get_element_rect;
Function<bool(i32 element_id)> on_is_element_enabled;
Function<Gfx::ShareableBitmap(i32 element_id)> on_take_element_screenshot;
Function<String()> on_serialize_source;
Function<Messages::WebContentServer::WebdriverExecuteScriptResponse(String const& body, Vector<String> const& json_arguments, Optional<u64> const& timeout, bool async)> on_execute_script;
};

2
Userland/Applications/Browser/WebDriverSessionClient.ipc
View file

@ -31,6 +31,7 @@ endpoint WebDriverSessionClient {
update_cookie(Web::Cookie::Cookie cookie) =|
get_document_element() => (Optional<i32> document_element_id)
query_selector_all(i32 start_node_id, String selector) => (Optional<Vector<i32>> elements_ids)
scroll_element_into_view(i32 element_id) => ()
is_element_selected(i32 element_id) => (bool selected)
get_element_attribute(i32 element_id, String name) => (Optional<String> atttibute)
get_element_property(i32 element_id, String name) => (Optional<String> property)
@ -41,4 +42,5 @@ endpoint WebDriverSessionClient {
get_element_rect(i32 element_id) => (Gfx::IntRect rect)
is_element_enabled(i32 element_id) => (bool enabled)
take_screenshot() => (Gfx::ShareableBitmap data)
take_element_screenshot(i32 element_id) => (Gfx::ShareableBitmap data)
}

11
Userland/Services/WebDriver/Client.cpp
View file

@ -62,6 +62,7 @@ Vector<Client::Route> Client::s_routes = {
{ HTTP::HttpRequest::Method::DELETE, { "session", ":session_id", "cookie", ":name" }, &Client::handle_delete_cookie },
{ HTTP::HttpRequest::Method::DELETE, { "session", ":session_id", "cookie" }, &Client::handle_delete_all_cookies },
{ HTTP::HttpRequest::Method::GET, { "session", ":session_id", "screenshot" }, &Client::handle_take_screenshot },
{ HTTP::HttpRequest::Method::GET, { "session", ":session_id", "element", ":element_id", "screenshot" }, &Client::handle_take_element_screenshot },
};
Client::Client(NonnullOwnPtr<Core::Stream::BufferedTCPSocket> socket, Core::Object* parent)
@ -812,4 +813,14 @@ ErrorOr<JsonValue, WebDriverError> Client::handle_take_screenshot(Vector<StringV
return make_json_value(result);
}
// 17.2 Take Element Screenshot, https://w3c.github.io/webdriver/#dfn-take-element-screenshot
// GET /session/{session id}/element/{element id}/screenshot
ErrorOr<JsonValue, WebDriverError> Client::handle_take_element_screenshot(Vector<StringView> const& parameters, JsonValue const&)
{
dbgln_if(WEBDRIVER_DEBUG, "Handling GET /session/<session_id>/element/<element_id>/screenshot");
auto* session = TRY(find_session_with_id(parameters[0]));
auto result = TRY(session->take_element_screenshot(parameters[1]));
return make_json_value(result);
}
}

1
Userland/Services/WebDriver/Client.h
View file

@ -87,6 +87,7 @@ private:
ErrorOr<JsonValue, WebDriverError> handle_delete_cookie(Vector<StringView> const&, JsonValue const& payload);
ErrorOr<JsonValue, WebDriverError> handle_delete_all_cookies(Vector<StringView> const&, JsonValue const& payload);
ErrorOr<JsonValue, WebDriverError> handle_take_screenshot(Vector<StringView> const&, JsonValue const& payload);
ErrorOr<JsonValue, WebDriverError> handle_take_element_screenshot(Vector<StringView> const&, JsonValue const& payload);
ErrorOr<Session*, WebDriverError> find_session_with_id(StringView session_id);
JsonValue make_json_value(JsonValue const&);

30
Userland/Services/WebDriver/Session.cpp
View file

@ -1354,4 +1354,34 @@ ErrorOr<JsonValue, WebDriverError> Session::take_screenshot()
return encoded_string;
}
// 17.2 Take Element Screenshot, https://w3c.github.io/webdriver/#dfn-take-element-screenshot
ErrorOr<JsonValue, WebDriverError> Session::take_element_screenshot(StringView parameter_element_id)
{
// 1. If the current top-level browsing context is no longer open, return error with error code no such window.
TRY(check_for_open_top_level_browsing_context_or_return_error());
// FIXME: 2. Handle any user prompts and return its value if it is an error.
// 3. Let element be the result of trying to get a known connected element with url variable element id.
auto element_id = TRY(get_known_connected_element(parameter_element_id));
// 4. Scroll into view the element.
m_browser_connection->scroll_element_into_view(element_id);
// 5. When the user agent is next to run the animation frame callbacks:
// a. Let element rect be element’s rectangle.
// b. Let screenshot result be the result of trying to call draw a bounding box from the framebuffer, given element rect as an argument.
auto screenshot = m_browser_connection->take_element_screenshot(element_id);
if (!screenshot.is_valid())
return WebDriverError::from_code(ErrorCode::UnableToCaptureScreen, "Unable to capture screenshot"sv);
// c. Let canvas be a canvas element of screenshot result’s data.
// d. Let encoding result be the result of trying encoding a canvas as Base64 canvas.
// e. Let encoded string be encoding result’s data.
auto encoded_string = TRY(encode_bitmap_as_canvas_element(*screenshot.bitmap()));
// 6. Return success with data encoded string.
return encoded_string;
}
}

1
Userland/Services/WebDriver/Session.h
View file

@ -76,6 +76,7 @@ public:
ErrorOr<JsonValue, WebDriverError> delete_cookie(StringView name);
ErrorOr<JsonValue, WebDriverError> delete_all_cookies();
ErrorOr<JsonValue, WebDriverError> take_screenshot();
ErrorOr<JsonValue, WebDriverError> take_element_screenshot(StringView element_id);
private:
void delete_cookies(Optional<StringView> const& name = {});