From 2946a684efd1d3c34148d16121b118574b6a9132 Mon Sep 17 00:00:00 2001 From: Andreas Kling Date: Mon, 28 Sep 2020 11:55:26 +0200 Subject: [PATCH] ProtocolServer+LibWeb: Support more detailed HTTP requests This patch adds the ability for ProtocolServer clients to specify which HTTP method to use, and also to include an optional HTTP request body. --- Applications/Browser/DownloadWidget.cpp | 2 +- Libraries/LibHTTP/HttpRequest.cpp | 4 +++ Libraries/LibHTTP/HttpRequest.h | 5 ++++ Libraries/LibProtocol/Client.cpp | 4 +-- Libraries/LibProtocol/Client.h | 2 +- Libraries/LibWeb/Loader/FrameLoader.cpp | 27 +++++++++++++++----- Libraries/LibWeb/Loader/FrameLoader.h | 1 + Libraries/LibWeb/Loader/ResourceLoader.cpp | 19 +++++++++++--- Libraries/LibWeb/Loader/ResourceLoader.h | 1 + Libraries/LibWeb/Page/Page.cpp | 5 ++++ Libraries/LibWeb/Page/Page.h | 2 ++ Services/ProtocolServer/ClientConnection.cpp | 2 +- Services/ProtocolServer/GeminiProtocol.cpp | 2 +- Services/ProtocolServer/GeminiProtocol.h | 2 +- Services/ProtocolServer/HttpProtocol.cpp | 8 ++++-- Services/ProtocolServer/HttpProtocol.h | 2 +- Services/ProtocolServer/HttpsProtocol.cpp | 8 ++++-- Services/ProtocolServer/HttpsProtocol.h | 2 +- Services/ProtocolServer/Protocol.h | 2 +- Services/ProtocolServer/ProtocolServer.ipc | 2 +- Userland/pro.cpp | 2 +- 21 files changed, 78 insertions(+), 26 deletions(-) diff --git a/Applications/Browser/DownloadWidget.cpp b/Applications/Browser/DownloadWidget.cpp index 4323c65065..2551cd3f0b 100644 --- a/Applications/Browser/DownloadWidget.cpp +++ b/Applications/Browser/DownloadWidget.cpp @@ -56,7 +56,7 @@ DownloadWidget::DownloadWidget(const URL& url) } m_elapsed_timer.start(); - m_download = Web::ResourceLoader::the().protocol_client().start_download(url.to_string()); + m_download = Web::ResourceLoader::the().protocol_client().start_download("GET", url.to_string()); ASSERT(m_download); m_download->on_progress = [this](Optional total_size, u32 downloaded_size) { did_progress(total_size.value(), downloaded_size); diff --git a/Libraries/LibHTTP/HttpRequest.cpp b/Libraries/LibHTTP/HttpRequest.cpp index 8e1fc445f5..0d6699ca3f 100644 --- a/Libraries/LibHTTP/HttpRequest.cpp +++ b/Libraries/LibHTTP/HttpRequest.cpp @@ -79,6 +79,10 @@ ByteBuffer HttpRequest::to_raw_request() const builder.append("\r\n"); } builder.append("Connection: close\r\n\r\n"); + if (!m_body.is_empty()) { + builder.append((const char*)m_body.data(), m_body.size()); + builder.append("\r\n"); + } return builder.to_byte_buffer(); } diff --git a/Libraries/LibHTTP/HttpRequest.h b/Libraries/LibHTTP/HttpRequest.h index f7e0a0783e..488630dad1 100644 --- a/Libraries/LibHTTP/HttpRequest.h +++ b/Libraries/LibHTTP/HttpRequest.h @@ -26,6 +26,7 @@ #pragma once +#include #include #include #include @@ -60,6 +61,9 @@ public: Method method() const { return m_method; } void set_method(Method method) { m_method = method; } + const ByteBuffer& body() const { return m_body; } + void set_body(const ByteBuffer& body) { m_body = body; } + String method_name() const; ByteBuffer to_raw_request() const; @@ -74,6 +78,7 @@ private: String m_resource; Method m_method { GET }; Vector
m_headers; + ByteBuffer m_body; }; } diff --git a/Libraries/LibProtocol/Client.cpp b/Libraries/LibProtocol/Client.cpp index 3531bfaa76..0138ed9349 100644 --- a/Libraries/LibProtocol/Client.cpp +++ b/Libraries/LibProtocol/Client.cpp @@ -47,13 +47,13 @@ bool Client::is_supported_protocol(const String& protocol) return send_sync(protocol)->supported(); } -RefPtr Client::start_download(const String& url, const HashMap& request_headers) +RefPtr Client::start_download(const String& method, const String& url, const HashMap& request_headers, const ByteBuffer& request_body) { IPC::Dictionary header_dictionary; for (auto& it : request_headers) header_dictionary.add(it.key, it.value); - i32 download_id = send_sync(url, header_dictionary)->download_id(); + i32 download_id = send_sync(method, url, header_dictionary, String::copy(request_body))->download_id(); if (download_id < 0) return nullptr; auto download = Download::create_from_id({}, *this, download_id); diff --git a/Libraries/LibProtocol/Client.h b/Libraries/LibProtocol/Client.h index 6724a30779..ca9f93b3e6 100644 --- a/Libraries/LibProtocol/Client.h +++ b/Libraries/LibProtocol/Client.h @@ -44,7 +44,7 @@ public: virtual void handshake() override; bool is_supported_protocol(const String&); - RefPtr start_download(const String& url, const HashMap& request_headers = {}); + RefPtr start_download(const String& method, const String& url, const HashMap& request_headers = {}, const ByteBuffer& request_body = {}); bool stop_download(Badge, Download&); bool set_certificate(Badge, Download&, String, String); diff --git a/Libraries/LibWeb/Loader/FrameLoader.cpp b/Libraries/LibWeb/Loader/FrameLoader.cpp index ab5f148292..5c29b12d32 100644 --- a/Libraries/LibWeb/Loader/FrameLoader.cpp +++ b/Libraries/LibWeb/Loader/FrameLoader.cpp @@ -139,17 +139,15 @@ RefPtr FrameLoader::create_document_from_mime_type(const ByteBuff return nullptr; } -bool FrameLoader::load(const URL& url, Type type) +bool FrameLoader::load(const LoadRequest& request, Type type) { - dbg() << "FrameLoader::load: " << url; - - if (!url.is_valid()) { - load_error_page(url, "Invalid URL"); + if (!request.is_valid()) { + load_error_page(request.url(), "Invalid request"); return false; } - LoadRequest request; - request.set_url(url); + auto& url = request.url(); + set_resource(ResourceLoader::the().load_resource(Resource::Type::Generic, request)); if (type == Type::Navigation) @@ -180,6 +178,21 @@ bool FrameLoader::load(const URL& url, Type type) return true; } +bool FrameLoader::load(const URL& url, Type type) +{ + dbg() << "FrameLoader::load: " << url; + + if (!url.is_valid()) { + load_error_page(url, "Invalid URL"); + return false; + } + + LoadRequest request; + request.set_url(url); + + return load(request, type); +} + void FrameLoader::load_error_page(const URL& failed_url, const String& error) { auto error_page_url = "file:///res/html/error.html"; diff --git a/Libraries/LibWeb/Loader/FrameLoader.h b/Libraries/LibWeb/Loader/FrameLoader.h index 3da7854b8c..4e22ee11e1 100644 --- a/Libraries/LibWeb/Loader/FrameLoader.h +++ b/Libraries/LibWeb/Loader/FrameLoader.h @@ -45,6 +45,7 @@ public: ~FrameLoader(); bool load(const URL&, Type); + bool load(const LoadRequest&, Type); Frame& frame() { return m_frame; } const Frame& frame() const { return m_frame; } diff --git a/Libraries/LibWeb/Loader/ResourceLoader.cpp b/Libraries/LibWeb/Loader/ResourceLoader.cpp index 1efaf3d52c..1911c7c39e 100644 --- a/Libraries/LibWeb/Loader/ResourceLoader.cpp +++ b/Libraries/LibWeb/Loader/ResourceLoader.cpp @@ -96,7 +96,7 @@ RefPtr ResourceLoader::load_resource(Resource::Type type, const LoadRe s_resource_cache.set(request, resource); load( - request.url(), + request, [=](auto& data, auto& headers) { const_cast(*resource).did_load({}, data, headers); }, @@ -107,8 +107,9 @@ RefPtr ResourceLoader::load_resource(Resource::Type type, const LoadRe return resource; } -void ResourceLoader::load(const URL& url, Function& response_headers)> success_callback, Function error_callback) +void ResourceLoader::load(const LoadRequest& request, Function& response_headers)> success_callback, Function error_callback) { + auto& url = request.url(); if (is_port_blocked(url.port())) { dbg() << "ResourceLoader::load: Error: blocked port " << url.port() << " for URL: " << url; return; @@ -157,7 +158,12 @@ void ResourceLoader::load(const URL& url, Function headers; headers.set("User-Agent", m_user_agent); - auto download = protocol_client().start_download(url.to_string(), headers); + + for (auto& it : request.headers()) { + headers.set(it.key, it.value); + } + + auto download = protocol_client().start_download(request.method(), url.to_string(), headers, request.body()); if (!download) { if (error_callback) error_callback("Failed to initiate load"); @@ -192,6 +198,13 @@ void ResourceLoader::load(const URL& url, Function& response_headers)> success_callback, Function error_callback) +{ + LoadRequest request; + request.set_url(url); + load(request, move(success_callback), move(error_callback)); +} + bool ResourceLoader::is_port_blocked(int port) { int ports[] { 1, 7, 9, 11, 13, 15, 17, 19, 20, 21, 22, 23, 25, 37, 42, diff --git a/Libraries/LibWeb/Loader/ResourceLoader.h b/Libraries/LibWeb/Loader/ResourceLoader.h index 7f6c62d0f9..d6cb8d1ede 100644 --- a/Libraries/LibWeb/Loader/ResourceLoader.h +++ b/Libraries/LibWeb/Loader/ResourceLoader.h @@ -44,6 +44,7 @@ public: RefPtr load_resource(Resource::Type, const LoadRequest&); + void load(const LoadRequest&, Function& response_headers)> success_callback, Function error_callback = nullptr); void load(const URL&, Function& response_headers)> success_callback, Function error_callback = nullptr); void load_sync(const URL&, Function& response_headers)> success_callback, Function error_callback = nullptr); diff --git a/Libraries/LibWeb/Page/Page.cpp b/Libraries/LibWeb/Page/Page.cpp index 36e56d50b2..ffab18691a 100644 --- a/Libraries/LibWeb/Page/Page.cpp +++ b/Libraries/LibWeb/Page/Page.cpp @@ -57,6 +57,11 @@ void Page::load(const URL& url) main_frame().loader().load(url, FrameLoader::Type::Navigation); } +void Page::load(const LoadRequest& request) +{ + main_frame().loader().load(request, FrameLoader::Type::Navigation); +} + Gfx::Palette Page::palette() const { return m_client.palette(); diff --git a/Libraries/LibWeb/Page/Page.h b/Libraries/LibWeb/Page/Page.h index 0cbb992994..1805c80d79 100644 --- a/Libraries/LibWeb/Page/Page.h +++ b/Libraries/LibWeb/Page/Page.h @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -59,6 +60,7 @@ public: void set_focused_frame(Badge, Frame&); void load(const URL&); + void load(const LoadRequest&); bool handle_mouseup(const Gfx::IntPoint&, unsigned button, unsigned modifiers); bool handle_mousedown(const Gfx::IntPoint&, unsigned button, unsigned modifiers); diff --git a/Services/ProtocolServer/ClientConnection.cpp b/Services/ProtocolServer/ClientConnection.cpp index beaa55c4e1..86cc6344ce 100644 --- a/Services/ProtocolServer/ClientConnection.cpp +++ b/Services/ProtocolServer/ClientConnection.cpp @@ -66,7 +66,7 @@ OwnPtr ClientConnection::handle auto* protocol = Protocol::find_by_name(url.protocol()); if (!protocol) return make(-1); - auto download = protocol->start_download(*this, url, message.request_headers().entries()); + auto download = protocol->start_download(*this, message.method(), url, message.request_headers().entries(), message.request_body().to_byte_buffer()); if (!download) return make(-1); auto id = download->id(); diff --git a/Services/ProtocolServer/GeminiProtocol.cpp b/Services/ProtocolServer/GeminiProtocol.cpp index 9bc61d54aa..f1167ecd61 100644 --- a/Services/ProtocolServer/GeminiProtocol.cpp +++ b/Services/ProtocolServer/GeminiProtocol.cpp @@ -40,7 +40,7 @@ GeminiProtocol::~GeminiProtocol() { } -OwnPtr GeminiProtocol::start_download(ClientConnection& client, const URL& url, const HashMap&) +OwnPtr GeminiProtocol::start_download(ClientConnection& client, const String&, const URL& url, const HashMap&, const ByteBuffer&) { Gemini::GeminiRequest request; request.set_url(url); diff --git a/Services/ProtocolServer/GeminiProtocol.h b/Services/ProtocolServer/GeminiProtocol.h index e3f28bf57d..f9ed21cca3 100644 --- a/Services/ProtocolServer/GeminiProtocol.h +++ b/Services/ProtocolServer/GeminiProtocol.h @@ -35,7 +35,7 @@ public: GeminiProtocol(); virtual ~GeminiProtocol() override; - virtual OwnPtr start_download(ClientConnection&, const URL&, const HashMap&) override; + virtual OwnPtr start_download(ClientConnection&, const String& method, const URL&, const HashMap&, const ByteBuffer& request_body) override; }; } diff --git a/Services/ProtocolServer/HttpProtocol.cpp b/Services/ProtocolServer/HttpProtocol.cpp index 7eaa51287c..f6b618d4b0 100644 --- a/Services/ProtocolServer/HttpProtocol.cpp +++ b/Services/ProtocolServer/HttpProtocol.cpp @@ -40,12 +40,16 @@ HttpProtocol::~HttpProtocol() { } -OwnPtr HttpProtocol::start_download(ClientConnection& client, const URL& url, const HashMap& headers) +OwnPtr HttpProtocol::start_download(ClientConnection& client, const String& method, const URL& url, const HashMap& headers, const ByteBuffer& request_body) { HTTP::HttpRequest request; - request.set_method(HTTP::HttpRequest::Method::GET); + if (method.equals_ignoring_case("post")) + request.set_method(HTTP::HttpRequest::Method::POST); + else + request.set_method(HTTP::HttpRequest::Method::GET); request.set_url(url); request.set_headers(headers); + request.set_body(request_body); auto job = request.schedule(); if (!job) return nullptr; diff --git a/Services/ProtocolServer/HttpProtocol.h b/Services/ProtocolServer/HttpProtocol.h index ee0c0f5f3b..aa9601b8ce 100644 --- a/Services/ProtocolServer/HttpProtocol.h +++ b/Services/ProtocolServer/HttpProtocol.h @@ -35,7 +35,7 @@ public: HttpProtocol(); virtual ~HttpProtocol() override; - virtual OwnPtr start_download(ClientConnection&, const URL&, const HashMap& headers) override; + virtual OwnPtr start_download(ClientConnection&, const String& method, const URL&, const HashMap& headers, const ByteBuffer& request_body) override; }; } diff --git a/Services/ProtocolServer/HttpsProtocol.cpp b/Services/ProtocolServer/HttpsProtocol.cpp index ab1181cb69..3de9ca8e2b 100644 --- a/Services/ProtocolServer/HttpsProtocol.cpp +++ b/Services/ProtocolServer/HttpsProtocol.cpp @@ -40,12 +40,16 @@ HttpsProtocol::~HttpsProtocol() { } -OwnPtr HttpsProtocol::start_download(ClientConnection& client, const URL& url, const HashMap& headers) +OwnPtr HttpsProtocol::start_download(ClientConnection& client, const String& method, const URL& url, const HashMap& headers, const ByteBuffer& request_body) { HTTP::HttpRequest request; - request.set_method(HTTP::HttpRequest::Method::GET); + if (method.equals_ignoring_case("post")) + request.set_method(HTTP::HttpRequest::Method::POST); + else + request.set_method(HTTP::HttpRequest::Method::GET); request.set_url(url); request.set_headers(headers); + request.set_body(request_body); auto job = HTTP::HttpsJob::construct(request); auto download = HttpsDownload::create_with_job({}, client, (HTTP::HttpsJob&)*job); job->start(); diff --git a/Services/ProtocolServer/HttpsProtocol.h b/Services/ProtocolServer/HttpsProtocol.h index 92f72e8729..9cb0ce190b 100644 --- a/Services/ProtocolServer/HttpsProtocol.h +++ b/Services/ProtocolServer/HttpsProtocol.h @@ -35,7 +35,7 @@ public: HttpsProtocol(); virtual ~HttpsProtocol() override; - virtual OwnPtr start_download(ClientConnection&, const URL&, const HashMap& headers) override; + virtual OwnPtr start_download(ClientConnection&, const String& method, const URL&, const HashMap& headers, const ByteBuffer& request_body) override; }; } diff --git a/Services/ProtocolServer/Protocol.h b/Services/ProtocolServer/Protocol.h index 13cff2e7bb..035b56cb6e 100644 --- a/Services/ProtocolServer/Protocol.h +++ b/Services/ProtocolServer/Protocol.h @@ -37,7 +37,7 @@ public: virtual ~Protocol(); const String& name() const { return m_name; } - virtual OwnPtr start_download(ClientConnection&, const URL&, const HashMap& headers) = 0; + virtual OwnPtr start_download(ClientConnection&, const String& method, const URL&, const HashMap& headers, const ByteBuffer& request_body) = 0; static Protocol* find_by_name(const String&); diff --git a/Services/ProtocolServer/ProtocolServer.ipc b/Services/ProtocolServer/ProtocolServer.ipc index e819eb3234..4cf1204520 100644 --- a/Services/ProtocolServer/ProtocolServer.ipc +++ b/Services/ProtocolServer/ProtocolServer.ipc @@ -10,7 +10,7 @@ endpoint ProtocolServer = 9 IsSupportedProtocol(String protocol) => (bool supported) // Download API - StartDownload(URL url, IPC::Dictionary request_headers) => (i32 download_id) + StartDownload(String method, URL url, IPC::Dictionary request_headers, String request_body) => (i32 download_id) StopDownload(i32 download_id) => (bool success) SetCertificate(i32 download_id, String certificate, String key) => (bool success) } diff --git a/Userland/pro.cpp b/Userland/pro.cpp index 7ce7a39b45..d41df71d01 100644 --- a/Userland/pro.cpp +++ b/Userland/pro.cpp @@ -159,7 +159,7 @@ int main(int argc, char** argv) Core::EventLoop loop; auto protocol_client = Protocol::Client::construct(); - auto download = protocol_client->start_download(url.to_string()); + auto download = protocol_client->start_download("GET", url.to_string()); if (!download) { fprintf(stderr, "Failed to start download for '%s'\n", url_str); return 1;