LibCore+LibHTTP: Move out the HTTP handler and add HTTPS

Libraries/LibHTTP/Forward.h

@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2020, The SerenityOS developers.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+namespace HTTP {
+
+class HttpRequest;
+class HttpResponse;
+class HttpJob;
+class HttpsJob;
+
+}

Libraries/LibHTTP/HttpJob.cpp

@@ -0,0 +1,216 @@
+/*
+ * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <LibCore/Gzip.h>
+#include <LibCore/TCPSocket.h>
+#include <LibHTTP/HttpJob.h>
+#include <LibHTTP/HttpResponse.h>
+#include <stdio.h>
+#include <unistd.h>
+
+//#define HTTPJOB_DEBUG
+
+namespace HTTP {
+
+static ByteBuffer handle_content_encoding(const ByteBuffer& buf, const String& content_encoding)
+{
+#ifdef CHTTPJOB_DEBUG
+    dbg() << "HttpJob::handle_content_encoding: buf has content_encoding = " << content_encoding;
+#endif
+
+    if (content_encoding == "gzip") {
+        if (!Core::Gzip::is_compressed(buf)) {
+            dbg() << "HttpJob::handle_content_encoding: buf is not gzip compressed!";
+        }
+
+#ifdef CHTTPJOB_DEBUG
+        dbg() << "HttpJob::handle_content_encoding: buf is gzip compressed!";
+#endif
+
+        auto uncompressed = Core::Gzip::decompress(buf);
+        if (!uncompressed.has_value()) {
+            dbg() << "HttpJob::handle_content_encoding: Gzip::decompress() failed. Returning original buffer.";
+            return buf;
+        }
+
+#ifdef CHTTPJOB_DEBUG
+        dbg() << "HttpJob::handle_content_encoding: Gzip::decompress() successful.\n"
+              << "  Input size = " << buf.size() << "\n"
+              << "  Output size = " << uncompressed.value().size();
+#endif
+
+        return uncompressed.value();
+    }
+
+    return buf;
+}
+
+HttpJob::HttpJob(const HttpRequest& request)
+    : m_request(request)
+{
+}
+
+HttpJob::~HttpJob()
+{
+}
+
+void HttpJob::on_socket_connected()
+{
+    auto raw_request = m_request.to_raw_request();
+#if 0
+    dbg() << "HttpJob: raw_request:";
+    dbg() << String::copy(raw_request).characters();
+#endif
+
+    bool success = m_socket->send(raw_request);
+    if (!success)
+        return deferred_invoke([this](auto&) { did_fail(Core::NetworkJob::Error::TransmissionFailed); });
+
+    m_socket->on_ready_to_read = [&] {
+        if (is_cancelled())
+            return;
+        if (m_state == State::InStatus) {
+            if (!m_socket->can_read_line())
+                return;
+            auto line = m_socket->read_line(PAGE_SIZE);
+            if (line.is_null()) {
+                fprintf(stderr, "HttpJob: Expected HTTP status\n");
+                return deferred_invoke([this](auto&) { did_fail(Core::NetworkJob::Error::TransmissionFailed); });
+            }
+            auto parts = String::copy(line, Chomp).split(' ');
+            if (parts.size() < 3) {
+                fprintf(stderr, "HttpJob: Expected 3-part HTTP status, got '%s'\n", line.data());
+                return deferred_invoke([this](auto&) { did_fail(Core::NetworkJob::Error::ProtocolFailed); });
+            }
+            bool ok;
+            m_code = parts[1].to_uint(ok);
+            if (!ok) {
+                fprintf(stderr, "HttpJob: Expected numeric HTTP status\n");
+                return deferred_invoke([this](auto&) { did_fail(Core::NetworkJob::Error::ProtocolFailed); });
+            }
+            m_state = State::InHeaders;
+            return;
+        }
+        if (m_state == State::InHeaders) {
+            if (!m_socket->can_read_line())
+                return;
+            auto line = m_socket->read_line(PAGE_SIZE);
+            if (line.is_null()) {
+                fprintf(stderr, "HttpJob: Expected HTTP header\n");
+                return did_fail(Core::NetworkJob::Error::ProtocolFailed);
+            }
+            auto chomped_line = String::copy(line, Chomp);
+            if (chomped_line.is_empty()) {
+                m_state = State::InBody;
+                return;
+            }
+            auto parts = chomped_line.split(':');
+            if (parts.is_empty()) {
+                fprintf(stderr, "HttpJob: Expected HTTP header with key/value\n");
+                return deferred_invoke([this](auto&) { did_fail(Core::NetworkJob::Error::ProtocolFailed); });
+            }
+            auto name = parts[0];
+            if (chomped_line.length() < name.length() + 2) {
+                fprintf(stderr, "HttpJob: Malformed HTTP header: '%s' (%zu)\n", chomped_line.characters(), chomped_line.length());
+                return deferred_invoke([this](auto&) { did_fail(Core::NetworkJob::Error::ProtocolFailed); });
+            }
+            auto value = chomped_line.substring(name.length() + 2, chomped_line.length() - name.length() - 2);
+            m_headers.set(name, value);
+#ifdef CHTTPJOB_DEBUG
+            dbg() << "HttpJob: [" << name << "] = '" << value << "'";
+#endif
+            return;
+        }
+        ASSERT(m_state == State::InBody);
+        ASSERT(m_socket->can_read());
+        auto payload = m_socket->receive(64 * KB);
+        if (!payload) {
+            if (m_socket->eof())
+                return finish_up();
+            return deferred_invoke([this](auto&) { did_fail(Core::NetworkJob::Error::ProtocolFailed); });
+        }
+        m_received_buffers.append(payload);
+        m_received_size += payload.size();
+
+        auto content_length_header = m_headers.get("Content-Length");
+        if (content_length_header.has_value()) {
+            bool ok;
+            if (m_received_size >= content_length_header.value().to_uint(ok) && ok)
+                return finish_up();
+        }
+    };
+}
+
+void HttpJob::finish_up()
+{
+    m_state = State::Finished;
+    auto flattened_buffer = ByteBuffer::create_uninitialized(m_received_size);
+    u8* flat_ptr = flattened_buffer.data();
+    for (auto& received_buffer : m_received_buffers) {
+        memcpy(flat_ptr, received_buffer.data(), received_buffer.size());
+        flat_ptr += received_buffer.size();
+    }
+    m_received_buffers.clear();
+
+    auto content_encoding = m_headers.get("Content-Encoding");
+    if (content_encoding.has_value()) {
+        flattened_buffer = handle_content_encoding(flattened_buffer, content_encoding.value());
+    }
+
+    auto response = HttpResponse::create(m_code, move(m_headers), move(flattened_buffer));
+    deferred_invoke([this, response](auto&) {
+        did_finish(move(response));
+    });
+}
+
+void HttpJob::start()
+{
+    ASSERT(!m_socket);
+    m_socket = Core::TCPSocket::construct(this);
+    m_socket->on_connected = [this] {
+#ifdef CHTTPJOB_DEBUG
+        dbg() << "HttpJob: on_connected callback";
+#endif
+        on_socket_connected();
+    };
+    bool success = m_socket->connect(m_request.url().host(), m_request.url().port());
+    if (!success) {
+        deferred_invoke([this](auto&) {
+            return did_fail(Core::NetworkJob::Error::ConnectionFailed);
+        });
+    }
+}
+
+void HttpJob::shutdown()
+{
+    if (!m_socket)
+        return;
+    m_socket->on_ready_to_read = nullptr;
+    m_socket->on_connected = nullptr;
+    remove_child(*m_socket);
+    m_socket = nullptr;
+}
+}

Libraries/LibHTTP/HttpJob.h

@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+#include <AK/HashMap.h>
+#include <LibCore/NetworkJob.h>
+#include <LibCore/TCPSocket.h>
+#include <LibHTTP/HttpRequest.h>
+#include <LibHTTP/HttpResponse.h>
+
+namespace HTTP {
+
+class HttpJob final : public Core::NetworkJob {
+    C_OBJECT(HttpJob)
+public:
+    explicit HttpJob(const HttpRequest&);
+    virtual ~HttpJob() override;
+
+    virtual void start() override;
+    virtual void shutdown() override;
+
+    HttpResponse* response() { return static_cast<HttpResponse*>(Core::NetworkJob::response()); }
+    const HttpResponse* response() const { return static_cast<const HttpResponse*>(Core::NetworkJob::response()); }
+
+private:
+    void on_socket_connected();
+    void finish_up();
+
+    enum class State {
+        InStatus,
+        InHeaders,
+        InBody,
+        Finished,
+    };
+
+    HttpRequest m_request;
+    RefPtr<Core::Socket> m_socket;
+    State m_state { State::InStatus };
+    int m_code { -1 };
+    HashMap<String, String> m_headers;
+    Vector<ByteBuffer> m_received_buffers;
+    size_t m_received_size { 0 };
+};
+
+}

Libraries/LibHTTP/HttpRequest.cpp

@@ -0,0 +1,180 @@
+/*
+ * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <AK/StringBuilder.h>
+#include <LibHTTP/HttpJob.h>
+#include <LibHTTP/HttpRequest.h>
+
+namespace HTTP {
+
+HttpRequest::HttpRequest()
+{
+}
+
+HttpRequest::~HttpRequest()
+{
+}
+
+RefPtr<Core::NetworkJob> HttpRequest::schedule()
+{
+    auto job = HttpJob::construct(*this);
+    job->start();
+    return job;
+}
+
+String HttpRequest::method_name() const
+{
+    switch (m_method) {
+    case Method::GET:
+        return "GET";
+    case Method::HEAD:
+        return "HEAD";
+    case Method::POST:
+        return "POST";
+    default:
+        ASSERT_NOT_REACHED();
+    }
+}
+
+ByteBuffer HttpRequest::to_raw_request() const
+{
+    StringBuilder builder;
+    builder.append(method_name());
+    builder.append(' ');
+    builder.append(m_url.path());
+    builder.append(" HTTP/1.1\r\nHost: ");
+    builder.append(m_url.host());
+    builder.append("\r\nConnection: close\r\n\r\n");
+    return builder.to_byte_buffer();
+}
+
+Optional<HttpRequest> HttpRequest::from_raw_request(const ByteBuffer& raw_request)
+{
+    enum class State {
+        InMethod,
+        InResource,
+        InProtocol,
+        InHeaderName,
+        InHeaderValue,
+    };
+
+    State state { State::InMethod };
+    size_t index = 0;
+
+    auto peek = [&](int offset = 0) -> u8 {
+        if (index + offset >= raw_request.size())
+            return 0;
+        return raw_request[index + offset];
+    };
+
+    auto consume = [&]() -> u8 {
+        ASSERT(index < raw_request.size());
+        return raw_request[index++];
+    };
+
+    Vector<u8, 256> buffer;
+
+    String method;
+    String resource;
+    String protocol;
+    Vector<Header> headers;
+    Header current_header;
+
+    auto commit_and_advance_to = [&](auto& output, State new_state) {
+        output = String::copy(buffer);
+        buffer.clear();
+        state = new_state;
+    };
+
+    while (index < raw_request.size()) {
+        // FIXME: Figure out what the appropriate limitations should be.
+        if (buffer.size() > 65536)
+            return {};
+        switch (state) {
+        case State::InMethod:
+            if (peek() == ' ') {
+                consume();
+                commit_and_advance_to(method, State::InResource);
+                break;
+            }
+            buffer.append(consume());
+            break;
+        case State::InResource:
+            if (peek() == ' ') {
+                consume();
+                commit_and_advance_to(resource, State::InProtocol);
+                break;
+            }
+            buffer.append(consume());
+            break;
+        case State::InProtocol:
+            if (peek(0) == '\r' && peek(1) == '\n') {
+                consume();
+                consume();
+                commit_and_advance_to(protocol, State::InHeaderName);
+                break;
+            }
+            buffer.append(consume());
+            break;
+        case State::InHeaderName:
+            if (peek(0) == ':' && peek(1) == ' ') {
+                consume();
+                consume();
+                commit_and_advance_to(current_header.name, State::InHeaderValue);
+                break;
+            }
+            buffer.append(consume());
+            break;
+        case State::InHeaderValue:
+            if (peek(0) == '\r' && peek(1) == '\n') {
+                consume();
+                consume();
+                commit_and_advance_to(current_header.value, State::InHeaderName);
+                headers.append(move(current_header));
+                break;
+            }
+            buffer.append(consume());
+            break;
+        }
+    }
+
+    HttpRequest request;
+    if (method == "GET")
+        request.m_method = Method::GET;
+    else if (method == "HEAD")
+        request.m_method = Method::HEAD;
+    else if (method == "POST")
+        request.m_method = Method::POST;
+    else
+        return {};
+
+    request.m_resource = resource;
+    request.m_headers = move(headers);
+
+    return request;
+}
+
+}

Libraries/LibHTTP/HttpRequest.h

@@ -0,0 +1,77 @@
+/*
+ * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+#include <AK/Optional.h>
+#include <AK/String.h>
+#include <AK/URL.h>
+#include <AK/Vector.h>
+#include <LibCore/Forward.h>
+
+namespace HTTP {
+
+class HttpRequest {
+public:
+    enum Method {
+        Invalid,
+        HEAD,
+        GET,
+        POST
+    };
+
+    struct Header {
+        String name;
+        String value;
+    };
+
+    HttpRequest();
+    ~HttpRequest();
+
+    const String& resource() const { return m_resource; }
+    const Vector<Header>& headers() const { return m_headers; }
+
+    const URL& url() const { return m_url; }
+    void set_url(const URL& url) { m_url = url; }
+
+    Method method() const { return m_method; }
+    void set_method(Method method) { m_method = method; }
+
+    String method_name() const;
+    ByteBuffer to_raw_request() const;
+
+    RefPtr<Core::NetworkJob> schedule();
+
+    static Optional<HttpRequest> from_raw_request(const ByteBuffer&);
+
+private:
+    URL m_url;
+    String m_resource;
+    Method m_method { GET };
+    Vector<Header> m_headers;
+};
+
+}

Libraries/LibHTTP/HttpResponse.cpp

@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <LibHTTP/HttpResponse.h>
+
+namespace HTTP {
+
+HttpResponse::HttpResponse(int code, HashMap<String, String>&& headers, ByteBuffer&& payload)
+    : Core::NetworkResponse(move(payload))
+    , m_code(code)
+    , m_headers(move(headers))
+{
+}
+
+HttpResponse::~HttpResponse()
+{
+}
+
+}

Libraries/LibHTTP/HttpResponse.h

@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+#include <AK/HashMap.h>
+#include <AK/String.h>
+#include <LibCore/NetworkResponse.h>
+
+namespace HTTP {
+
+class HttpResponse : public Core::NetworkResponse {
+public:
+    virtual ~HttpResponse() override;
+    static NonnullRefPtr<HttpResponse> create(int code, HashMap<String, String>&& headers, ByteBuffer&& payload)
+    {
+        return adopt(*new HttpResponse(code, move(headers), move(payload)));
+    }
+
+    int code() const { return m_code; }
+    const HashMap<String, String>& headers() const { return m_headers; }
+
+private:
+    HttpResponse(int code, HashMap<String, String>&&, ByteBuffer&&);
+
+    int m_code { 0 };
+    HashMap<String, String> m_headers;
+};
+
+}

Libraries/LibHTTP/HttpsJob.cpp

@@ -0,0 +1,237 @@
+/*
+ * Copyright (c) 2020, The SerenityOS developers.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <LibCore/EventLoop.h>
+#include <LibCore/Gzip.h>
+#include <LibHTTP/HttpResponse.h>
+#include <LibHTTP/HttpsJob.h>
+#include <LibTLS/TLSv12.h>
+#include <stdio.h>
+#include <unistd.h>
+
+//#define HTTPJOB_DEBUG
+
+namespace HTTP {
+
+static ByteBuffer handle_content_encoding(const ByteBuffer& buf, const String& content_encoding)
+{
+#ifdef CHTTPJOB_DEBUG
+    dbg() << "HttpsJob::handle_content_encoding: buf has content_encoding = " << content_encoding;
+#endif
+
+    if (content_encoding == "gzip") {
+        if (!Core::Gzip::is_compressed(buf)) {
+            dbg() << "HttpsJob::handle_content_encoding: buf is not gzip compressed!";
+        }
+
+#ifdef CHTTPJOB_DEBUG
+        dbg() << "HttpsJob::handle_content_encoding: buf is gzip compressed!";
+#endif
+
+        auto uncompressed = Core::Gzip::decompress(buf);
+        if (!uncompressed.has_value()) {
+            dbg() << "HttpsJob::handle_content_encoding: Gzip::decompress() failed. Returning original buffer.";
+            return buf;
+        }
+
+#ifdef CHTTPJOB_DEBUG
+        dbg() << "HttpsJob::handle_content_encoding: Gzip::decompress() successful.\n"
+              << "  Input size = " << buf.size() << "\n"
+              << "  Output size = " << uncompressed.value().size();
+#endif
+
+        return uncompressed.value();
+    }
+
+    return buf;
+}
+
+HttpsJob::HttpsJob(const HttpRequest& request)
+    : m_request(request)
+{
+}
+
+HttpsJob::~HttpsJob()
+{
+    m_socket = nullptr;
+}
+
+void HttpsJob::on_socket_connected()
+{
+
+    m_socket->on_tls_ready_to_write = [&](TLS::TLSv12& tls) {
+        if (m_sent_data)
+            return;
+        m_sent_data = true;
+        auto raw_request = m_request.to_raw_request();
+#if 0
+        dbg() << "HttpsJob: raw_request:";
+        dbg() << String::copy(raw_request).characters();
+#endif
+        bool success = tls.write(raw_request);
+        if (!success)
+            deferred_invoke([this](auto&) { did_fail(Core::NetworkJob::Error::TransmissionFailed); });
+    };
+
+    m_socket->on_tls_ready_to_read = [&](TLS::TLSv12& tls) {
+        dbg() << " ON TLS READY TO READ: " << (u16)m_state;
+        if (is_cancelled())
+            return;
+        if (m_state == State::InStatus) {
+            if (!tls.can_read_line()) {
+                dbg() << " cannot read line";
+                return;
+            }
+            auto line = tls.read_line(PAGE_SIZE);
+            if (line.is_null()) {
+                fprintf(stderr, "HttpsJob: Expected HTTP status\n");
+                return deferred_invoke([this](auto&) { did_fail(Core::NetworkJob::Error::TransmissionFailed); });
+            }
+            auto parts = String::copy(line, Chomp).split(' ');
+            if (parts.size() < 3) {
+                fprintf(stderr, "HttpsJob: Expected 3-part HTTP status, got '%s'\n", line.data());
+                return deferred_invoke([this](auto&) { did_fail(Core::NetworkJob::Error::ProtocolFailed); });
+            }
+            bool ok;
+            m_code = parts[1].to_uint(ok);
+            if (!ok) {
+                fprintf(stderr, "HttpsJob: Expected numeric HTTP status\n");
+                return deferred_invoke([this](auto&) { did_fail(Core::NetworkJob::Error::ProtocolFailed); });
+            }
+            m_state = State::InHeaders;
+            return;
+        }
+        if (m_state == State::InHeaders) {
+            if (!tls.can_read_line())
+                return;
+            auto line = tls.read_line(PAGE_SIZE);
+            if (line.is_null()) {
+                fprintf(stderr, "HttpsJob: Expected HTTP header\n");
+                return did_fail(Core::NetworkJob::Error::ProtocolFailed);
+            }
+            auto chomped_line = String::copy(line, Chomp);
+            if (chomped_line.is_empty()) {
+                m_state = State::InBody;
+                return;
+            }
+            auto parts = chomped_line.split(':');
+            if (parts.is_empty()) {
+                fprintf(stderr, "HttpsJob: Expected HTTP header with key/value\n");
+                return deferred_invoke([this](auto&) { did_fail(Core::NetworkJob::Error::ProtocolFailed); });
+            }
+            auto name = parts[0];
+            if (chomped_line.length() < name.length() + 2) {
+                fprintf(stderr, "HttpsJob: Malformed HTTP header: '%s' (%zu)\n", chomped_line.characters(), chomped_line.length());
+                return deferred_invoke([this](auto&) { did_fail(Core::NetworkJob::Error::ProtocolFailed); });
+            }
+            auto value = chomped_line.substring(name.length() + 2, chomped_line.length() - name.length() - 2);
+            m_headers.set(name, value);
+#ifdef CHTTPJOB_DEBUG
+            dbg() << "HttpsJob: [" << name << "] = '" << value << "'";
+#endif
+            return;
+        }
+        ASSERT(m_state == State::InBody);
+        ASSERT(tls.can_read());
+        auto payload = tls.read(64 * KB);
+        if (!payload) {
+            if (tls.eof())
+                return finish_up();
+            return deferred_invoke([this](auto&) { did_fail(Core::NetworkJob::Error::ProtocolFailed); });
+        }
+        dbg() << "Read payload, " << payload.size() << " bytes";
+        m_received_buffers.append(payload);
+        m_received_size += payload.size();
+
+        auto content_length_header = m_headers.get("Content-Length");
+        if (content_length_header.has_value()) {
+            dbg() << "content length is " << content_length_header.value() << ", we have " << m_received_size;
+            bool ok;
+            if (m_received_size >= content_length_header.value().to_uint(ok) && ok)
+                finish_up();
+        } else {
+            // no content-length, assume closed connection
+            finish_up();
+        }
+    };
+}
+
+void HttpsJob::finish_up()
+{
+    m_state = State::Finished;
+    auto flattened_buffer = ByteBuffer::create_uninitialized(m_received_size);
+    u8* flat_ptr = flattened_buffer.data();
+    for (auto& received_buffer : m_received_buffers) {
+        memcpy(flat_ptr, received_buffer.data(), received_buffer.size());
+        flat_ptr += received_buffer.size();
+    }
+    m_received_buffers.clear();
+
+    auto content_encoding = m_headers.get("Content-Encoding");
+    if (content_encoding.has_value()) {
+        flattened_buffer = handle_content_encoding(flattened_buffer, content_encoding.value());
+    }
+
+    auto response = HttpResponse::create(m_code, move(m_headers), move(flattened_buffer));
+    deferred_invoke([this, response](auto&) {
+        did_finish(move(response));
+    });
+}
+
+void HttpsJob::start()
+{
+    ASSERT(!m_socket);
+    m_socket = TLS::TLSv12::construct(this);
+    m_socket->on_tls_connected = [this] {
+#ifdef CHTTPJOB_DEBUG
+        dbg() << "HttpsJob: on_connected callback";
+#endif
+        on_socket_connected();
+    };
+    m_socket->on_tls_error = [&](auto) {
+        finish_up();
+    };
+    m_socket->on_tls_finished = [&] {
+        finish_up();
+    };
+    bool success = ((TLS::TLSv12&)*m_socket).connect(m_request.url().host(), m_request.url().port());
+    if (!success) {
+        deferred_invoke([this](auto&) {
+            return did_fail(Core::NetworkJob::Error::ConnectionFailed);
+        });
+    }
+}
+
+void HttpsJob::shutdown()
+{
+    if (!m_socket)
+        return;
+    m_socket->on_tls_ready_to_read = nullptr;
+    m_socket->on_tls_connected = nullptr;
+    remove_child(*m_socket);
+    m_socket = nullptr;
+}
+}

Libraries/LibHTTP/HttpsJob.h

@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) 2020, The SerenityOS developers.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+#include <AK/HashMap.h>
+#include <LibCore/NetworkJob.h>
+#include <LibHTTP/HttpRequest.h>
+#include <LibHTTP/HttpResponse.h>
+#include <LibTLS/TLSv12.h>
+
+namespace HTTP {
+
+class HttpsJob final : public Core::NetworkJob {
+    C_OBJECT(HttpsJob)
+public:
+    explicit HttpsJob(const HttpRequest&);
+    virtual ~HttpsJob() override;
+
+    virtual void start() override;
+    virtual void shutdown() override;
+
+    HttpResponse* response() { return static_cast<HttpResponse*>(Core::NetworkJob::response()); }
+    const HttpResponse* response() const { return static_cast<const HttpResponse*>(Core::NetworkJob::response()); }
+
+private:
+    RefPtr<TLS::TLSv12> construct_socket() { return TLS::TLSv12::construct(this); }
+    void on_socket_connected();
+    void finish_up();
+
+    enum class State {
+        InStatus,
+        InHeaders,
+        InBody,
+        Finished,
+    };
+
+    HttpRequest m_request;
+    RefPtr<TLS::TLSv12> m_socket;
+    State m_state { State::InStatus };
+    int m_code { -1 };
+    HashMap<String, String> m_headers;
+    Vector<ByteBuffer> m_received_buffers;
+    size_t m_received_size { 0 };
+    bool m_sent_data { false };
+};
+
+}

Libraries/LibHTTP/Makefile

@@ -0,0 +1,18 @@
+OBJS = HttpResponse.o \
+	   HttpRequest.o \
+	   HttpJob.o \
+	   HttpsJob.o
+
+LIBRARY = libhttp.a
+
+LIB_DEPS = Core
+
+POST_LIBRARY_BUILD = $(QUIET) $(MAKE) install
+
+install:
+	mkdir -p $(SERENITY_BASE_DIR)/Root/usr/include/sys/
+	mkdir -p $(SERENITY_BASE_DIR)/Root/usr/lib/
+	cp *.h $(SERENITY_BASE_DIR)/Root/usr/include/
+	cp $(LIBRARY) $(SERENITY_BASE_DIR)/Root/usr/lib/
+
+include ../../Makefile.common