Services: Rename ProtocolServer to RequestServer

The current ProtocolServer was really only used for requests, and with
the recent introduction of the WebSocket service, long-lasting
connections with another server are not part of it. To better reflect
this, this commit renames it to RequestServer.

This commit also changes the existing 'protocol' portal to 'request',
the existing 'protocol' user and group to 'request', and most mentions
of the 'download' aspect of the request to 'request' when relevant, to
make everything consistent across the system.

Note that LibProtocol still exists as-is, but the more generic Client
class and the more specific Download class have both been renamed to a
more accurate RequestClient and Request to match the new names.

This commit only change names, not behaviors.

Userland/Services/RequestServer/CMakeLists.txt

@@ -0,0 +1,20 @@
+compile_ipc(RequestServer.ipc RequestServerEndpoint.h)
+compile_ipc(RequestClient.ipc RequestClientEndpoint.h)
+
+set(SOURCES
+    ClientConnection.cpp
+    Request.cpp
+    RequestClientEndpoint.h
+    RequestServerEndpoint.h
+    GeminiRequest.cpp
+    GeminiProtocol.cpp
+    HttpRequest.cpp
+    HttpProtocol.cpp
+    HttpsRequest.cpp
+    HttpsProtocol.cpp
+    main.cpp
+    Protocol.cpp
+)
+
+serenity_bin(RequestServer)
+target_link_libraries(RequestServer LibCore LibIPC LibGemini LibHTTP)

Userland/Services/RequestServer/ClientConnection.cpp

@@ -0,0 +1,119 @@
+/*
+ * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <AK/Badge.h>
+#include <RequestServer/ClientConnection.h>
+#include <RequestServer/Protocol.h>
+#include <RequestServer/Request.h>
+#include <RequestServer/RequestClientEndpoint.h>
+
+namespace RequestServer {
+
+static HashMap<int, RefPtr<ClientConnection>> s_connections;
+
+ClientConnection::ClientConnection(NonnullRefPtr<Core::LocalSocket> socket, int client_id)
+    : IPC::ClientConnection<RequestClientEndpoint, RequestServerEndpoint>(*this, move(socket), client_id)
+{
+    s_connections.set(client_id, *this);
+}
+
+ClientConnection::~ClientConnection()
+{
+}
+
+void ClientConnection::die()
+{
+    s_connections.remove(client_id());
+    if (s_connections.is_empty())
+        Core::EventLoop::current().quit(0);
+}
+
+OwnPtr<Messages::RequestServer::IsSupportedProtocolResponse> ClientConnection::handle(const Messages::RequestServer::IsSupportedProtocol& message)
+{
+    bool supported = Protocol::find_by_name(message.protocol().to_lowercase());
+    return make<Messages::RequestServer::IsSupportedProtocolResponse>(supported);
+}
+
+OwnPtr<Messages::RequestServer::StartRequestResponse> ClientConnection::handle(const Messages::RequestServer::StartRequest& message)
+{
+    const auto& url = message.url();
+    if (!url.is_valid()) {
+        dbgln("StartRequest: Invalid URL requested: '{}'", url);
+        return make<Messages::RequestServer::StartRequestResponse>(-1, Optional<IPC::File> {});
+    }
+    auto* protocol = Protocol::find_by_name(url.protocol());
+    if (!protocol) {
+        dbgln("StartRequest: No protocol handler for URL: '{}'", url);
+        return make<Messages::RequestServer::StartRequestResponse>(-1, Optional<IPC::File> {});
+    }
+    auto request = protocol->start_request(*this, message.method(), url, message.request_headers().entries(), message.request_body());
+    if (!request) {
+        dbgln("StartRequest: Protocol handler failed to start request: '{}'", url);
+        return make<Messages::RequestServer::StartRequestResponse>(-1, Optional<IPC::File> {});
+    }
+    auto id = request->id();
+    auto fd = request->request_fd();
+    m_requests.set(id, move(request));
+    return make<Messages::RequestServer::StartRequestResponse>(id, IPC::File(fd, IPC::File::CloseAfterSending));
+}
+
+OwnPtr<Messages::RequestServer::StopRequestResponse> ClientConnection::handle(const Messages::RequestServer::StopRequest& message)
+{
+    auto* request = const_cast<Request*>(m_requests.get(message.request_id()).value_or(nullptr));
+    bool success = false;
+    if (request) {
+        request->stop();
+        m_requests.remove(message.request_id());
+        success = true;
+    }
+    return make<Messages::RequestServer::StopRequestResponse>(success);
+}
+
+void ClientConnection::did_receive_headers(Badge<Request>, Request& request)
+{
+    IPC::Dictionary response_headers;
+    for (auto& it : request.response_headers())
+        response_headers.add(it.key, it.value);
+
+    post_message(Messages::RequestClient::HeadersBecameAvailable(request.id(), move(response_headers), request.status_code()));
+}
+
+void ClientConnection::did_finish_request(Badge<Request>, Request& request, bool success)
+{
+    VERIFY(request.total_size().has_value());
+
+    post_message(Messages::RequestClient::RequestFinished(request.id(), success, request.total_size().value()));
+
+    m_requests.remove(request.id());
+}
+
+void ClientConnection::did_progress_request(Badge<Request>, Request& request)
+{
+    post_message(Messages::RequestClient::RequestProgress(request.id(), request.total_size(), request.downloaded_size()));
+}
+
+void ClientConnection::did_request_certificates(Badge<Request>, Request& request)
+{
+    post_message(Messages::RequestClient::CertificateRequested(request.id()));
+}
+
+OwnPtr<Messages::RequestServer::GreetResponse> ClientConnection::handle(const Messages::RequestServer::Greet&)
+{
+    return make<Messages::RequestServer::GreetResponse>();
+}
+
+OwnPtr<Messages::RequestServer::SetCertificateResponse> ClientConnection::handle(const Messages::RequestServer::SetCertificate& message)
+{
+    auto* request = const_cast<Request*>(m_requests.get(message.request_id()).value_or(nullptr));
+    bool success = false;
+    if (request) {
+        request->set_certificate(message.certificate(), message.key());
+        success = true;
+    }
+    return make<Messages::RequestServer::SetCertificateResponse>(success);
+}
+
+}

Userland/Services/RequestServer/ClientConnection.h

@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/HashMap.h>
+#include <LibIPC/ClientConnection.h>
+#include <RequestServer/Forward.h>
+#include <RequestServer/RequestClientEndpoint.h>
+#include <RequestServer/RequestServerEndpoint.h>
+
+namespace RequestServer {
+
+class ClientConnection final
+    : public IPC::ClientConnection<RequestClientEndpoint, RequestServerEndpoint>
+    , public RequestServerEndpoint {
+    C_OBJECT(ClientConnection);
+
+public:
+    explicit ClientConnection(NonnullRefPtr<Core::LocalSocket>, int client_id);
+    ~ClientConnection() override;
+
+    virtual void die() override;
+
+    void did_receive_headers(Badge<Request>, Request&);
+    void did_finish_request(Badge<Request>, Request&, bool success);
+    void did_progress_request(Badge<Request>, Request&);
+    void did_request_certificates(Badge<Request>, Request&);
+
+private:
+    virtual OwnPtr<Messages::RequestServer::GreetResponse> handle(const Messages::RequestServer::Greet&) override;
+    virtual OwnPtr<Messages::RequestServer::IsSupportedProtocolResponse> handle(const Messages::RequestServer::IsSupportedProtocol&) override;
+    virtual OwnPtr<Messages::RequestServer::StartRequestResponse> handle(const Messages::RequestServer::StartRequest&) override;
+    virtual OwnPtr<Messages::RequestServer::StopRequestResponse> handle(const Messages::RequestServer::StopRequest&) override;
+    virtual OwnPtr<Messages::RequestServer::SetCertificateResponse> handle(const Messages::RequestServer::SetCertificate&) override;
+
+    HashMap<i32, OwnPtr<Request>> m_requests;
+};
+
+}

Userland/Services/RequestServer/Forward.h

@@ -0,0 +1,20 @@
+/*
+ * Copyright (c) 2020, Andreas Kling <kling@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+namespace RequestServer {
+
+class ClientConnection;
+class Request;
+class GeminiProtocol;
+class HttpRequest;
+class HttpProtocol;
+class HttpsRequest;
+class HttpsProtocol;
+class Protocol;
+
+}

Userland/Services/RequestServer/GeminiProtocol.cpp

@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2020, The SerenityOS developers.
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <LibGemini/GeminiJob.h>
+#include <LibGemini/GeminiRequest.h>
+#include <RequestServer/GeminiProtocol.h>
+#include <RequestServer/GeminiRequest.h>
+#include <fcntl.h>
+
+namespace RequestServer {
+
+GeminiProtocol::GeminiProtocol()
+    : Protocol("gemini")
+{
+}
+
+GeminiProtocol::~GeminiProtocol()
+{
+}
+
+OwnPtr<Request> GeminiProtocol::start_request(ClientConnection& client, const String&, const URL& url, const HashMap<String, String>&, ReadonlyBytes)
+{
+    Gemini::GeminiRequest request;
+    request.set_url(url);
+
+    auto pipe_result = get_pipe_for_request();
+    if (pipe_result.is_error())
+        return {};
+
+    auto output_stream = make<OutputFileStream>(pipe_result.value().write_fd);
+    output_stream->make_unbuffered();
+    auto job = Gemini::GeminiJob::construct(request, *output_stream);
+    auto protocol_request = GeminiRequest::create_with_job({}, client, (Gemini::GeminiJob&)*job, move(output_stream));
+    protocol_request->set_request_fd(pipe_result.value().read_fd);
+    job->start();
+    return protocol_request;
+}
+
+}

Userland/Services/RequestServer/GeminiProtocol.h

@@ -0,0 +1,21 @@
+/*
+ * Copyright (c) 2020, The SerenityOS developers.
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <RequestServer/Protocol.h>
+
+namespace RequestServer {
+
+class GeminiProtocol final : public Protocol {
+public:
+    GeminiProtocol();
+    virtual ~GeminiProtocol() override;
+
+    virtual OwnPtr<Request> start_request(ClientConnection&, const String& method, const URL&, const HashMap<String, String>&, ReadonlyBytes body) override;
+};
+
+}

Userland/Services/RequestServer/GeminiRequest.cpp

@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2020, The SerenityOS developers.
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <LibGemini/GeminiJob.h>
+#include <LibGemini/GeminiResponse.h>
+#include <RequestServer/GeminiRequest.h>
+
+namespace RequestServer {
+
+GeminiRequest::GeminiRequest(ClientConnection& client, NonnullRefPtr<Gemini::GeminiJob> job, NonnullOwnPtr<OutputFileStream>&& output_stream)
+    : Request(client, move(output_stream))
+    , m_job(job)
+{
+    m_job->on_finish = [this](bool success) {
+        if (auto* response = m_job->response()) {
+            set_downloaded_size(this->output_stream().size());
+            if (!response->meta().is_empty()) {
+                HashMap<String, String, CaseInsensitiveStringTraits> headers;
+                headers.set("meta", response->meta());
+                // Note: We're setting content-type to meta only on status==SUCCESS
+                //       we should perhaps have a better mechanism for this, since we
+                //       are already shoehorning the concept of "headers" here
+                if (response->status() >= 20 && response->status() < 30) {
+                    headers.set("content-type", response->meta());
+                }
+                set_response_headers(headers);
+            }
+        }
+
+        // signal 100% request progress so any listeners can react
+        // appropriately
+        did_progress(downloaded_size(), downloaded_size());
+
+        did_finish(success);
+    };
+    m_job->on_progress = [this](Optional<u32> total, u32 current) {
+        did_progress(total, current);
+    };
+    m_job->on_certificate_requested = [this](auto&) {
+        did_request_certificates();
+    };
+}
+
+void GeminiRequest::set_certificate(String certificate, String key)
+{
+    m_job->set_certificate(move(certificate), move(key));
+}
+
+GeminiRequest::~GeminiRequest()
+{
+    m_job->on_finish = nullptr;
+    m_job->on_progress = nullptr;
+    m_job->shutdown();
+}
+
+NonnullOwnPtr<GeminiRequest> GeminiRequest::create_with_job(Badge<GeminiProtocol>, ClientConnection& client, NonnullRefPtr<Gemini::GeminiJob> job, NonnullOwnPtr<OutputFileStream>&& output_stream)
+{
+    return adopt_own(*new GeminiRequest(client, move(job), move(output_stream)));
+}
+
+}

Userland/Services/RequestServer/GeminiRequest.h

@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2020, The SerenityOS developers.
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/Badge.h>
+#include <LibCore/Forward.h>
+#include <LibGemini/Forward.h>
+#include <RequestServer/Request.h>
+
+namespace RequestServer {
+
+class GeminiRequest final : public Request {
+public:
+    virtual ~GeminiRequest() override;
+    static NonnullOwnPtr<GeminiRequest> create_with_job(Badge<GeminiProtocol>, ClientConnection&, NonnullRefPtr<Gemini::GeminiJob>, NonnullOwnPtr<OutputFileStream>&&);
+
+private:
+    explicit GeminiRequest(ClientConnection&, NonnullRefPtr<Gemini::GeminiJob>, NonnullOwnPtr<OutputFileStream>&&);
+
+    virtual void set_certificate(String certificate, String key) override;
+
+    NonnullRefPtr<Gemini::GeminiJob> m_job;
+};
+
+}

Userland/Services/RequestServer/HttpCommon.h

@@ -0,0 +1,83 @@
+/*
+ * Copyright (c) 2018-2021, Andreas Kling <kling@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/ByteBuffer.h>
+#include <AK/HashMap.h>
+#include <AK/NonnullOwnPtr.h>
+#include <AK/Optional.h>
+#include <AK/OwnPtr.h>
+#include <AK/String.h>
+#include <AK/Types.h>
+#include <LibHTTP/HttpRequest.h>
+#include <RequestServer/ClientConnection.h>
+#include <RequestServer/Request.h>
+
+namespace RequestServer::Detail {
+
+template<typename TSelf, typename TJob>
+void init(TSelf* self, TJob job)
+{
+    job->on_headers_received = [self](auto& headers, auto response_code) {
+        if (response_code.has_value())
+            self->set_status_code(response_code.value());
+        self->set_response_headers(headers);
+    };
+
+    job->on_finish = [self](bool success) {
+        if (auto* response = self->job().response()) {
+            self->set_status_code(response->code());
+            self->set_response_headers(response->headers());
+            self->set_downloaded_size(self->output_stream().size());
+        }
+
+        // if we didn't know the total size, pretend that the request finished successfully
+        // and set the total size to the downloaded size
+        if (!self->total_size().has_value())
+            self->did_progress(self->downloaded_size(), self->downloaded_size());
+
+        self->did_finish(success);
+    };
+    job->on_progress = [self](Optional<u32> total, u32 current) {
+        self->did_progress(total, current);
+    };
+    if constexpr (requires { job->on_certificate_requested; }) {
+        job->on_certificate_requested = [self](auto&) {
+            self->did_request_certificates();
+        };
+    }
+}
+
+template<typename TBadgedProtocol, typename TPipeResult>
+OwnPtr<Request> start_request(TBadgedProtocol&& protocol, ClientConnection& client, const String& method, const URL& url, const HashMap<String, String>& headers, ReadonlyBytes body, TPipeResult&& pipe_result)
+{
+    using TJob = TBadgedProtocol::Type::JobType;
+    using TRequest = TBadgedProtocol::Type::RequestType;
+
+    if (pipe_result.is_error()) {
+        return {};
+    }
+
+    HTTP::HttpRequest request;
+    if (method.equals_ignoring_case("post"))
+        request.set_method(HTTP::HttpRequest::Method::POST);
+    else
+        request.set_method(HTTP::HttpRequest::Method::GET);
+    request.set_url(url);
+    request.set_headers(headers);
+    request.set_body(body);
+
+    auto output_stream = make<OutputFileStream>(pipe_result.value().write_fd);
+    output_stream->make_unbuffered();
+    auto job = TJob::construct(request, *output_stream);
+    auto protocol_request = TRequest::create_with_job(forward<TBadgedProtocol>(protocol), client, (TJob&)*job, move(output_stream));
+    protocol_request->set_request_fd(pipe_result.value().read_fd);
+    job->start();
+    return protocol_request;
+}
+
+}

Userland/Services/RequestServer/HttpProtocol.cpp

@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <AK/Badge.h>
+#include <AK/ByteBuffer.h>
+#include <AK/HashMap.h>
+#include <AK/OwnPtr.h>
+#include <AK/String.h>
+#include <AK/URL.h>
+#include <LibHTTP/HttpJob.h>
+#include <RequestServer/ClientConnection.h>
+#include <RequestServer/HttpCommon.h>
+#include <RequestServer/HttpProtocol.h>
+#include <RequestServer/HttpRequest.h>
+#include <RequestServer/Request.h>
+
+namespace RequestServer {
+
+HttpProtocol::HttpProtocol()
+    : Protocol("http")
+{
+}
+
+OwnPtr<Request> HttpProtocol::start_request(ClientConnection& client, const String& method, const URL& url, const HashMap<String, String>& headers, ReadonlyBytes body)
+{
+    return Detail::start_request(Badge<HttpProtocol> {}, client, method, url, headers, body, get_pipe_for_request());
+}
+
+}

Userland/Services/RequestServer/HttpProtocol.h

@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/ByteBuffer.h>
+#include <AK/HashMap.h>
+#include <AK/OwnPtr.h>
+#include <AK/String.h>
+#include <AK/URL.h>
+#include <LibHTTP/HttpJob.h>
+#include <RequestServer/ClientConnection.h>
+#include <RequestServer/HttpRequest.h>
+#include <RequestServer/Protocol.h>
+#include <RequestServer/Request.h>
+
+namespace RequestServer {
+
+class HttpProtocol final : public Protocol {
+public:
+    using JobType = HTTP::HttpJob;
+    using RequestType = HttpRequest;
+
+    HttpProtocol();
+    ~HttpProtocol() override = default;
+
+    virtual OwnPtr<Request> start_request(ClientConnection&, const String& method, const URL&, const HashMap<String, String>& headers, ReadonlyBytes body) override;
+};
+
+}

Userland/Services/RequestServer/HttpRequest.cpp

@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <LibHTTP/HttpJob.h>
+#include <RequestServer/HttpCommon.h>
+#include <RequestServer/HttpProtocol.h>
+#include <RequestServer/HttpRequest.h>
+
+namespace RequestServer {
+
+HttpRequest::HttpRequest(ClientConnection& client, NonnullRefPtr<HTTP::HttpJob> job, NonnullOwnPtr<OutputFileStream>&& output_stream)
+    : Request(client, move(output_stream))
+    , m_job(job)
+{
+    Detail::init(this, job);
+}
+
+HttpRequest::~HttpRequest()
+{
+    m_job->on_finish = nullptr;
+    m_job->on_progress = nullptr;
+    m_job->shutdown();
+}
+
+NonnullOwnPtr<HttpRequest> HttpRequest::create_with_job(Badge<HttpProtocol>&&, ClientConnection& client, NonnullRefPtr<HTTP::HttpJob> job, NonnullOwnPtr<OutputFileStream>&& output_stream)
+{
+    return adopt_own(*new HttpRequest(client, move(job), move(output_stream)));
+}
+
+}

Userland/Services/RequestServer/HttpRequest.h

@@ -0,0 +1,30 @@
+/*
+ * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/Badge.h>
+#include <AK/NonnullOwnPtr.h>
+#include <LibCore/Forward.h>
+#include <LibHTTP/Forward.h>
+#include <RequestServer/Request.h>
+
+namespace RequestServer {
+
+class HttpRequest final : public Request {
+public:
+    virtual ~HttpRequest() override;
+    static NonnullOwnPtr<HttpRequest> create_with_job(Badge<HttpProtocol>&&, ClientConnection&, NonnullRefPtr<HTTP::HttpJob>, NonnullOwnPtr<OutputFileStream>&&);
+
+    HTTP::HttpJob& job() { return m_job; }
+
+private:
+    explicit HttpRequest(ClientConnection&, NonnullRefPtr<HTTP::HttpJob>, NonnullOwnPtr<OutputFileStream>&&);
+
+    NonnullRefPtr<HTTP::HttpJob> m_job;
+};
+
+}

Userland/Services/RequestServer/HttpsProtocol.cpp

@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2018-2020, The SerenityOS developers.
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <AK/Badge.h>
+#include <AK/ByteBuffer.h>
+#include <AK/HashMap.h>
+#include <AK/OwnPtr.h>
+#include <AK/String.h>
+#include <AK/URL.h>
+#include <LibHTTP/HttpsJob.h>
+#include <RequestServer/ClientConnection.h>
+#include <RequestServer/HttpCommon.h>
+#include <RequestServer/HttpsProtocol.h>
+#include <RequestServer/HttpsRequest.h>
+#include <RequestServer/Request.h>
+
+namespace RequestServer {
+
+HttpsProtocol::HttpsProtocol()
+    : Protocol("https")
+{
+}
+
+OwnPtr<Request> HttpsProtocol::start_request(ClientConnection& client, const String& method, const URL& url, const HashMap<String, String>& headers, ReadonlyBytes body)
+{
+    return Detail::start_request(Badge<HttpsProtocol> {}, client, method, url, headers, body, get_pipe_for_request());
+}
+
+}

Userland/Services/RequestServer/HttpsProtocol.h

@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2018-2020, The SerenityOS developers.
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/ByteBuffer.h>
+#include <AK/HashMap.h>
+#include <AK/OwnPtr.h>
+#include <AK/String.h>
+#include <AK/URL.h>
+#include <LibHTTP/HttpsJob.h>
+#include <RequestServer/ClientConnection.h>
+#include <RequestServer/HttpsRequest.h>
+#include <RequestServer/Protocol.h>
+#include <RequestServer/Request.h>
+
+namespace RequestServer {
+
+class HttpsProtocol final : public Protocol {
+public:
+    using JobType = HTTP::HttpsJob;
+    using RequestType = HttpsRequest;
+
+    HttpsProtocol();
+    ~HttpsProtocol() override = default;
+
+    virtual OwnPtr<Request> start_request(ClientConnection&, const String& method, const URL&, const HashMap<String, String>& headers, ReadonlyBytes body) override;
+};
+
+}

Userland/Services/RequestServer/HttpsRequest.cpp

@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2020, The SerenityOS developers.
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <LibHTTP/HttpsJob.h>
+#include <RequestServer/HttpCommon.h>
+#include <RequestServer/HttpsProtocol.h>
+#include <RequestServer/HttpsRequest.h>
+
+namespace RequestServer {
+
+HttpsRequest::HttpsRequest(ClientConnection& client, NonnullRefPtr<HTTP::HttpsJob> job, NonnullOwnPtr<OutputFileStream>&& output_stream)
+    : Request(client, move(output_stream))
+    , m_job(job)
+{
+    Detail::init(this, job);
+}
+
+void HttpsRequest::set_certificate(String certificate, String key)
+{
+    m_job->set_certificate(move(certificate), move(key));
+}
+
+HttpsRequest::~HttpsRequest()
+{
+    m_job->on_finish = nullptr;
+    m_job->on_progress = nullptr;
+    m_job->shutdown();
+}
+
+NonnullOwnPtr<HttpsRequest> HttpsRequest::create_with_job(Badge<HttpsProtocol>&&, ClientConnection& client, NonnullRefPtr<HTTP::HttpsJob> job, NonnullOwnPtr<OutputFileStream>&& output_stream)
+{
+    return adopt_own(*new HttpsRequest(client, move(job), move(output_stream)));
+}
+
+}

Userland/Services/RequestServer/HttpsRequest.h

@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2020, The SerenityOS developers.
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/Badge.h>
+#include <LibCore/Forward.h>
+#include <LibHTTP/HttpsJob.h>
+#include <RequestServer/Request.h>
+
+namespace RequestServer {
+
+class HttpsRequest final : public Request {
+public:
+    virtual ~HttpsRequest() override;
+    static NonnullOwnPtr<HttpsRequest> create_with_job(Badge<HttpsProtocol>&&, ClientConnection&, NonnullRefPtr<HTTP::HttpsJob>, NonnullOwnPtr<OutputFileStream>&&);
+
+    HTTP::HttpsJob& job() { return m_job; }
+
+private:
+    explicit HttpsRequest(ClientConnection&, NonnullRefPtr<HTTP::HttpsJob>, NonnullOwnPtr<OutputFileStream>&&);
+
+    virtual void set_certificate(String certificate, String key) override;
+
+    NonnullRefPtr<HTTP::HttpsJob> m_job;
+};
+
+}

Userland/Services/RequestServer/Protocol.cpp

@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <AK/HashMap.h>
+#include <RequestServer/Protocol.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <string.h>
+#include <unistd.h>
+
+namespace RequestServer {
+
+static HashMap<String, Protocol*>& all_protocols()
+{
+    static HashMap<String, Protocol*> map;
+    return map;
+}
+
+Protocol* Protocol::find_by_name(const String& name)
+{
+    return all_protocols().get(name).value_or(nullptr);
+}
+
+Protocol::Protocol(const String& name)
+{
+    all_protocols().set(name, this);
+}
+
+Protocol::~Protocol()
+{
+    VERIFY_NOT_REACHED();
+}
+
+Result<Protocol::Pipe, String> Protocol::get_pipe_for_request()
+{
+    int fd_pair[2] { 0 };
+    if (pipe(fd_pair) != 0) {
+        auto saved_errno = errno;
+        dbgln("Protocol: pipe() failed: {}", strerror(saved_errno));
+        return String { strerror(saved_errno) };
+    }
+    fcntl(fd_pair[1], F_SETFL, fcntl(fd_pair[1], F_GETFL) | O_NONBLOCK);
+    return Pipe { fd_pair[0], fd_pair[1] };
+}
+
+}

Userland/Services/RequestServer/Protocol.h

@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/RefPtr.h>
+#include <AK/Result.h>
+#include <AK/URL.h>
+#include <RequestServer/Forward.h>
+
+namespace RequestServer {
+
+class Protocol {
+public:
+    virtual ~Protocol();
+
+    const String& name() const { return m_name; }
+    virtual OwnPtr<Request> start_request(ClientConnection&, const String& method, const URL&, const HashMap<String, String>& headers, ReadonlyBytes body) = 0;
+
+    static Protocol* find_by_name(const String&);
+
+protected:
+    explicit Protocol(const String& name);
+    struct Pipe {
+        int read_fd { -1 };
+        int write_fd { -1 };
+    };
+    static Result<Pipe, String> get_pipe_for_request();
+
+private:
+    String m_name;
+};
+
+}

Userland/Services/RequestServer/Request.cpp

@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <AK/Badge.h>
+#include <RequestServer/ClientConnection.h>
+#include <RequestServer/Request.h>
+
+namespace RequestServer {
+
+// FIXME: What about rollover?
+static i32 s_next_id = 1;
+
+Request::Request(ClientConnection& client, NonnullOwnPtr<OutputFileStream>&& output_stream)
+    : m_client(client)
+    , m_id(s_next_id++)
+    , m_output_stream(move(output_stream))
+{
+}
+
+Request::~Request()
+{
+}
+
+void Request::stop()
+{
+    m_client.did_finish_request({}, *this, false);
+}
+
+void Request::set_response_headers(const HashMap<String, String, CaseInsensitiveStringTraits>& response_headers)
+{
+    m_response_headers = response_headers;
+    m_client.did_receive_headers({}, *this);
+}
+
+void Request::set_certificate(String, String)
+{
+}
+
+void Request::did_finish(bool success)
+{
+    m_client.did_finish_request({}, *this, success);
+}
+
+void Request::did_progress(Optional<u32> total_size, u32 downloaded_size)
+{
+    m_total_size = total_size;
+    m_downloaded_size = downloaded_size;
+    m_client.did_progress_request({}, *this);
+}
+
+void Request::did_request_certificates()
+{
+    m_client.did_request_certificates({}, *this);
+}
+
+}

Userland/Services/RequestServer/Request.h

@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/FileStream.h>
+#include <AK/HashMap.h>
+#include <AK/NonnullOwnPtr.h>
+#include <AK/Optional.h>
+#include <AK/RefCounted.h>
+#include <AK/URL.h>
+#include <RequestServer/Forward.h>
+
+namespace RequestServer {
+
+class Request {
+public:
+    virtual ~Request();
+
+    i32 id() const { return m_id; }
+    URL url() const { return m_url; }
+
+    Optional<u32> status_code() const { return m_status_code; }
+    Optional<u32> total_size() const { return m_total_size; }
+    size_t downloaded_size() const { return m_downloaded_size; }
+    const HashMap<String, String, CaseInsensitiveStringTraits>& response_headers() const { return m_response_headers; }
+
+    void stop();
+    virtual void set_certificate(String, String);
+
+    // FIXME: Want Badge<Protocol>, but can't make one from HttpProtocol, etc.
+    void set_request_fd(int fd) { m_request_fd = fd; }
+    int request_fd() const { return m_request_fd; }
+
+    void did_finish(bool success);
+    void did_progress(Optional<u32> total_size, u32 downloaded_size);
+    void set_status_code(u32 status_code) { m_status_code = status_code; }
+    void did_request_certificates();
+    void set_response_headers(const HashMap<String, String, CaseInsensitiveStringTraits>&);
+    void set_downloaded_size(size_t size) { m_downloaded_size = size; }
+    const OutputFileStream& output_stream() const { return *m_output_stream; }
+
+protected:
+    explicit Request(ClientConnection&, NonnullOwnPtr<OutputFileStream>&&);
+
+private:
+    ClientConnection& m_client;
+    i32 m_id { 0 };
+    int m_request_fd { -1 }; // Passed to client.
+    URL m_url;
+    Optional<u32> m_status_code;
+    Optional<u32> m_total_size {};
+    size_t m_downloaded_size { 0 };
+    NonnullOwnPtr<OutputFileStream> m_output_stream;
+    HashMap<String, String, CaseInsensitiveStringTraits> m_response_headers;
+};
+
+}

Userland/Services/RequestServer/RequestClient.ipc

@@ -0,0 +1,9 @@
+endpoint RequestClient
+{
+    RequestProgress(i32 request_id, Optional<u32> total_size, u32 downloaded_size) =|
+    RequestFinished(i32 request_id, bool success, u32 total_size) =|
+    HeadersBecameAvailable(i32 request_id, IPC::Dictionary response_headers, Optional<u32> status_code) =|
+
+    // Certificate requests
+    CertificateRequested(i32 request_id) => ()
+}

Userland/Services/RequestServer/RequestServer.ipc

@@ -0,0 +1,12 @@
+endpoint RequestServer
+{
+    // Basic protocol
+    Greet() => ()
+
+    // Test if a specific protocol is supported, e.g "http"
+    IsSupportedProtocol(String protocol) => (bool supported)
+
+    StartRequest(String method, URL url, IPC::Dictionary request_headers, ByteBuffer request_body) => (i32 request_id, Optional<IPC::File> response_fd)
+    StopRequest(i32 request_id) => (bool success)
+    SetCertificate(i32 request_id, String certificate, String key) => (bool success)
+}

Userland/Services/RequestServer/main.cpp

@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <LibCore/EventLoop.h>
+#include <LibCore/LocalServer.h>
+#include <LibIPC/ClientConnection.h>
+#include <LibTLS/Certificate.h>
+#include <RequestServer/ClientConnection.h>
+#include <RequestServer/GeminiProtocol.h>
+#include <RequestServer/HttpProtocol.h>
+#include <RequestServer/HttpsProtocol.h>
+
+int main(int, char**)
+{
+    if (pledge("stdio inet accept unix rpath cpath fattr sendfd recvfd", nullptr) < 0) {
+        perror("pledge");
+        return 1;
+    }
+
+    // Ensure the certificates are read out here.
+    [[maybe_unused]] auto& certs = DefaultRootCACertificates::the();
+
+    Core::EventLoop event_loop;
+    // FIXME: Establish a connection to LookupServer and then drop "unix"?
+    if (pledge("stdio inet accept unix sendfd recvfd", nullptr) < 0) {
+        perror("pledge");
+        return 1;
+    }
+    if (unveil("/tmp/portal/lookup", "rw") < 0) {
+        perror("unveil");
+        return 1;
+    }
+    if (unveil(nullptr, nullptr) < 0) {
+        perror("unveil");
+        return 1;
+    }
+
+    [[maybe_unused]] auto gemini = new RequestServer::GeminiProtocol;
+    [[maybe_unused]] auto http = new RequestServer::HttpProtocol;
+    [[maybe_unused]] auto https = new RequestServer::HttpsProtocol;
+
+    auto socket = Core::LocalSocket::take_over_accepted_socket_from_system_server();
+    VERIFY(socket);
+    IPC::new_client_connection<RequestServer::ClientConnection>(socket.release_nonnull(), 1);
+    return event_loop.exec();
+}