LibHTTP: Unify and generalise response handling logic

Libraries/LibHTTP/Forward.h

@@ -30,5 +30,6 @@ class HttpRequest;
 class HttpResponse;
 class HttpJob;
 class HttpsJob;
+class Job;
 
 }

Libraries/LibHTTP/HttpJob.cpp

@@ -34,173 +34,6 @@
 //#define HTTPJOB_DEBUG
 
 namespace HTTP {
-
-static ByteBuffer handle_content_encoding(const ByteBuffer& buf, const String& content_encoding)
-{
-#ifdef CHTTPJOB_DEBUG
-    dbg() << "HttpJob::handle_content_encoding: buf has content_encoding = " << content_encoding;
-#endif
-
-    if (content_encoding == "gzip") {
-        if (!Core::Gzip::is_compressed(buf)) {
-            dbg() << "HttpJob::handle_content_encoding: buf is not gzip compressed!";
-        }
-
-#ifdef CHTTPJOB_DEBUG
-        dbg() << "HttpJob::handle_content_encoding: buf is gzip compressed!";
-#endif
-
-        auto uncompressed = Core::Gzip::decompress(buf);
-        if (!uncompressed.has_value()) {
-            dbg() << "HttpJob::handle_content_encoding: Gzip::decompress() failed. Returning original buffer.";
-            return buf;
-        }
-
-#ifdef CHTTPJOB_DEBUG
-        dbg() << "HttpJob::handle_content_encoding: Gzip::decompress() successful.\n"
-              << "  Input size = " << buf.size() << "\n"
-              << "  Output size = " << uncompressed.value().size();
-#endif
-
-        return uncompressed.value();
-    }
-
-    return buf;
-}
-
-HttpJob::HttpJob(const HttpRequest& request)
-    : m_request(request)
-{
-}
-
-HttpJob::~HttpJob()
-{
-}
-
-void HttpJob::on_socket_connected()
-{
-    auto raw_request = m_request.to_raw_request();
-#if 0
-    dbg() << "HttpJob: raw_request:";
-    dbg() << String::copy(raw_request).characters();
-#endif
-
-    bool success = m_socket->send(raw_request);
-    if (!success)
-        return deferred_invoke([this](auto&) { did_fail(Core::NetworkJob::Error::TransmissionFailed); });
-
-    m_socket->on_ready_to_read = [&] {
-        if (is_cancelled())
-            return;
-        if (m_state == State::InStatus) {
-            if (!m_socket->can_read_line())
-                return;
-            auto line = m_socket->read_line(PAGE_SIZE);
-            if (line.is_null()) {
-                fprintf(stderr, "HttpJob: Expected HTTP status\n");
-                return deferred_invoke([this](auto&) { did_fail(Core::NetworkJob::Error::TransmissionFailed); });
-            }
-            auto parts = String::copy(line, Chomp).split(' ');
-            if (parts.size() < 3) {
-                fprintf(stderr, "HttpJob: Expected 3-part HTTP status, got '%s'\n", line.data());
-                return deferred_invoke([this](auto&) { did_fail(Core::NetworkJob::Error::ProtocolFailed); });
-            }
-            bool ok;
-            m_code = parts[1].to_uint(ok);
-            if (!ok) {
-                fprintf(stderr, "HttpJob: Expected numeric HTTP status\n");
-                return deferred_invoke([this](auto&) { did_fail(Core::NetworkJob::Error::ProtocolFailed); });
-            }
-            m_state = State::InHeaders;
-            return;
-        }
-        if (m_state == State::InHeaders) {
-            if (!m_socket->can_read_line())
-                return;
-            auto line = m_socket->read_line(PAGE_SIZE);
-            if (line.is_null()) {
-                fprintf(stderr, "HttpJob: Expected HTTP header\n");
-                return did_fail(Core::NetworkJob::Error::ProtocolFailed);
-            }
-            auto chomped_line = String::copy(line, Chomp);
-            if (chomped_line.is_empty()) {
-                m_state = State::InBody;
-                return;
-            }
-            auto parts = chomped_line.split(':');
-            if (parts.is_empty()) {
-                fprintf(stderr, "HttpJob: Expected HTTP header with key/value\n");
-                return deferred_invoke([this](auto&) { did_fail(Core::NetworkJob::Error::ProtocolFailed); });
-            }
-            auto name = parts[0];
-            if (chomped_line.length() < name.length() + 2) {
-                fprintf(stderr, "HttpJob: Malformed HTTP header: '%s' (%zu)\n", chomped_line.characters(), chomped_line.length());
-                return deferred_invoke([this](auto&) { did_fail(Core::NetworkJob::Error::ProtocolFailed); });
-            }
-            auto value = chomped_line.substring(name.length() + 2, chomped_line.length() - name.length() - 2);
-            m_headers.set(name, value);
-#ifdef CHTTPJOB_DEBUG
-            dbg() << "HttpJob: [" << name << "] = '" << value << "'";
-#endif
-            return;
-        }
-        ASSERT(m_state == State::InBody);
-        ASSERT(m_socket->can_read());
-        auto payload = m_socket->receive(64 * KB);
-        if (!payload) {
-            if (m_socket->eof())
-                return finish_up();
-            return deferred_invoke([this](auto&) { did_fail(Core::NetworkJob::Error::ProtocolFailed); });
-        }
-        m_received_buffers.append(payload);
-        m_received_size += payload.size();
-
-        auto content_length_header = m_headers.get("Content-Length");
-        Optional<u32> content_length {};
-
-        if (content_length_header.has_value()) {
-            bool ok;
-            auto length = content_length_header.value().to_uint(ok);
-            if (ok)
-                content_length = length;
-        }
-
-        deferred_invoke([this, content_length](auto&) {
-            did_progress(content_length, m_received_size);
-        });
-
-        if (content_length.has_value()) {
-            auto length = content_length.value();
-            if (m_received_size >= length) {
-                m_received_size = length;
-                finish_up();
-            }
-        }
-    };
-}
-
-void HttpJob::finish_up()
-{
-    m_state = State::Finished;
-    auto flattened_buffer = ByteBuffer::create_uninitialized(m_received_size);
-    u8* flat_ptr = flattened_buffer.data();
-    for (auto& received_buffer : m_received_buffers) {
-        memcpy(flat_ptr, received_buffer.data(), received_buffer.size());
-        flat_ptr += received_buffer.size();
-    }
-    m_received_buffers.clear();
-
-    auto content_encoding = m_headers.get("Content-Encoding");
-    if (content_encoding.has_value()) {
-        flattened_buffer = handle_content_encoding(flattened_buffer, content_encoding.value());
-    }
-
-    auto response = HttpResponse::create(m_code, move(m_headers), move(flattened_buffer));
-    deferred_invoke([this, response](auto&) {
-        did_finish(move(response));
-    });
-}
-
 void HttpJob::start()
 {
     ASSERT(!m_socket);
@@ -228,4 +61,46 @@ void HttpJob::shutdown()
     remove_child(*m_socket);
     m_socket = nullptr;
 }
+
+void HttpJob::register_on_ready_to_read(Function<void()> callback)
+{
+    m_socket->on_ready_to_read = move(callback);
+}
+
+void HttpJob::register_on_ready_to_write(Function<void()> callback)
+{
+    // There is no need to wait, the connection is already established
+    callback();
+}
+
+bool HttpJob::can_read_line()
+{
+    return m_socket->can_read_line();
+}
+
+ByteBuffer HttpJob::read_line(size_t size)
+{
+    return m_socket->read_line(size);
+}
+
+ByteBuffer HttpJob::receive(size_t size)
+{
+    return m_socket->receive(size);
+}
+
+bool HttpJob::can_read() const
+{
+    return m_socket->can_read();
+}
+
+bool HttpJob::eof() const
+{
+    return m_socket->eof();
+}
+
+bool HttpJob::write(const ByteBuffer& data)
+{
+    return m_socket->write(data);
+}
+
 }

Libraries/LibHTTP/HttpJob.h

@@ -31,39 +31,38 @@
 #include <LibCore/TCPSocket.h>
 #include <LibHTTP/HttpRequest.h>
 #include <LibHTTP/HttpResponse.h>
+#include <LibHTTP/Job.h>
 
 namespace HTTP {
 
-class HttpJob final : public Core::NetworkJob {
+class HttpJob final : public Job {
     C_OBJECT(HttpJob)
 public:
-    explicit HttpJob(const HttpRequest&);
+    explicit HttpJob(const HttpRequest& request)
-    virtual ~HttpJob() override;
+        : Job(request)
+    {
+    }
+
+    virtual ~HttpJob() override
+    {
+    }
 
     virtual void start() override;
     virtual void shutdown() override;
 
-    HttpResponse* response() { return static_cast<HttpResponse*>(Core::NetworkJob::response()); }
+protected:
-    const HttpResponse* response() const { return static_cast<const HttpResponse*>(Core::NetworkJob::response()); }
+    virtual void register_on_ready_to_read(Function<void()>) override;
+    virtual void register_on_ready_to_write(Function<void()>) override;
+    virtual bool can_read_line() override;
+    virtual ByteBuffer read_line(size_t) override;
+    virtual bool can_read() const override;
+    virtual ByteBuffer receive(size_t) override;
+    virtual bool eof() const override;
+    virtual bool write(const ByteBuffer&) override;
+    virtual bool is_established() const override { return true; }
 
 private:
-    void on_socket_connected();
-    void finish_up();
-
-    enum class State {
-        InStatus,
-        InHeaders,
-        InBody,
-        Finished,
-    };
-
-    HttpRequest m_request;
     RefPtr<Core::Socket> m_socket;
-    State m_state { State::InStatus };
-    int m_code { -1 };
-    HashMap<String, String> m_headers;
-    Vector<ByteBuffer> m_received_buffers;
-    size_t m_received_size { 0 };
 };
 
 }

Libraries/LibHTTP/HttpsJob.cpp

@@ -36,193 +36,6 @@
 
 namespace HTTP {
 
-static ByteBuffer handle_content_encoding(const ByteBuffer& buf, const String& content_encoding)
-{
-#ifdef HTTPSJOB_DEBUG
-    dbg() << "HttpsJob::handle_content_encoding: buf has content_encoding = " << content_encoding;
-#endif
-
-    if (content_encoding == "gzip") {
-        if (!Core::Gzip::is_compressed(buf)) {
-            dbg() << "HttpsJob::handle_content_encoding: buf is not gzip compressed!";
-        }
-
-#ifdef HTTPSJOB_DEBUG
-        dbg() << "HttpsJob::handle_content_encoding: buf is gzip compressed!";
-#endif
-
-        auto uncompressed = Core::Gzip::decompress(buf);
-        if (!uncompressed.has_value()) {
-            dbg() << "HttpsJob::handle_content_encoding: Gzip::decompress() failed. Returning original buffer.";
-            return buf;
-        }
-
-#ifdef HTTPSJOB_DEBUG
-        dbg() << "HttpsJob::handle_content_encoding: Gzip::decompress() successful.\n"
-              << "  Input size = " << buf.size() << "\n"
-              << "  Output size = " << uncompressed.value().size();
-#endif
-
-        return uncompressed.value();
-    }
-
-    return buf;
-}
-
-HttpsJob::HttpsJob(const HttpRequest& request)
-    : m_request(request)
-{
-}
-
-HttpsJob::~HttpsJob()
-{
-    m_socket = nullptr;
-}
-
-void HttpsJob::on_socket_connected()
-{
-
-    m_socket->on_tls_ready_to_write = [&](TLS::TLSv12& tls) {
-        if (m_sent_data)
-            return;
-        m_sent_data = true;
-        auto raw_request = m_request.to_raw_request();
-#if 0
-        dbg() << "HttpsJob: raw_request:";
-        dbg() << String::copy(raw_request).characters();
-#endif
-        bool success = tls.write(raw_request);
-        if (!success)
-            deferred_invoke([this](auto&) { did_fail(Core::NetworkJob::Error::TransmissionFailed); });
-    };
-
-    m_socket->on_tls_ready_to_read = [&](TLS::TLSv12& tls) {
-#ifdef HTTPS_DEBUG
-        dbg() << " ON TLS READY TO READ: " << (u16)m_state;
-#endif
-        if (is_cancelled())
-            return;
-        if (m_state == State::InStatus) {
-            if (!tls.can_read_line()) {
-                dbg() << " cannot read line";
-                return;
-            }
-            auto line = tls.read_line(PAGE_SIZE);
-            if (line.is_null()) {
-                fprintf(stderr, "HttpsJob: Expected HTTP status\n");
-                return deferred_invoke([this](auto&) { did_fail(Core::NetworkJob::Error::TransmissionFailed); });
-            }
-            auto parts = String::copy(line, Chomp).split(' ');
-            if (parts.size() < 3) {
-                fprintf(stderr, "HttpsJob: Expected 3-part HTTP status, got '%s'\n", line.data());
-                return deferred_invoke([this](auto&) { did_fail(Core::NetworkJob::Error::ProtocolFailed); });
-            }
-            bool ok;
-            m_code = parts[1].to_uint(ok);
-            if (!ok) {
-                fprintf(stderr, "HttpsJob: Expected numeric HTTP status\n");
-                return deferred_invoke([this](auto&) { did_fail(Core::NetworkJob::Error::ProtocolFailed); });
-            }
-            m_state = State::InHeaders;
-            return;
-        }
-        if (m_state == State::InHeaders) {
-            if (!tls.can_read_line())
-                return;
-            auto line = tls.read_line(PAGE_SIZE);
-            if (line.is_null()) {
-                fprintf(stderr, "HttpsJob: Expected HTTP header\n");
-                return did_fail(Core::NetworkJob::Error::ProtocolFailed);
-            }
-            auto chomped_line = String::copy(line, Chomp);
-            if (chomped_line.is_empty()) {
-                m_state = State::InBody;
-                return;
-            }
-            auto parts = chomped_line.split(':');
-            if (parts.is_empty()) {
-                fprintf(stderr, "HttpsJob: Expected HTTP header with key/value\n");
-                return deferred_invoke([this](auto&) { did_fail(Core::NetworkJob::Error::ProtocolFailed); });
-            }
-            auto name = parts[0];
-            if (chomped_line.length() < name.length() + 2) {
-                fprintf(stderr, "HttpsJob: Malformed HTTP header: '%s' (%zu)\n", chomped_line.characters(), chomped_line.length());
-                return deferred_invoke([this](auto&) { did_fail(Core::NetworkJob::Error::ProtocolFailed); });
-            }
-            auto value = chomped_line.substring(name.length() + 2, chomped_line.length() - name.length() - 2);
-            m_headers.set(name, value);
-#ifdef HTTPSJOB_DEBUG
-            dbg() << "HttpsJob: [" << name << "] = '" << value << "'";
-#endif
-            return;
-        }
-        ASSERT(m_state == State::InBody);
-        ASSERT(tls.can_read());
-
-        while (tls.can_read())
-            read_body(tls);
-
-        if (!tls.is_established())
-            return finish_up();
-    };
-}
-
-void HttpsJob::read_body(TLS::TLSv12& tls)
-{
-    auto payload = tls.read(64 * KB);
-    if (!payload) {
-        if (tls.eof())
-            return finish_up();
-        return deferred_invoke([this](auto&) { did_fail(Core::NetworkJob::Error::ProtocolFailed); });
-    }
-    m_received_buffers.append(payload);
-    m_received_size += payload.size();
-
-    auto content_length_header = m_headers.get("Content-Length");
-    Optional<u32> content_length {};
-
-    if (content_length_header.has_value()) {
-        bool ok;
-        auto length = content_length_header.value().to_uint(ok);
-        if (ok)
-            content_length = length;
-    }
-
-    // This needs to be synchronous
-    // FIXME: Somehow enforce that this should not modify anything
-    did_progress(content_length, m_received_size);
-
-    if (content_length.has_value()) {
-        auto length = content_length.value();
-        if (m_received_size >= length) {
-            m_received_size = length;
-            finish_up();
-        }
-    }
-}
-
-void HttpsJob::finish_up()
-{
-    m_state = State::Finished;
-    auto flattened_buffer = ByteBuffer::create_uninitialized(m_received_size);
-    u8* flat_ptr = flattened_buffer.data();
-    for (auto& received_buffer : m_received_buffers) {
-        memcpy(flat_ptr, received_buffer.data(), received_buffer.size());
-        flat_ptr += received_buffer.size();
-    }
-    m_received_buffers.clear();
-
-    auto content_encoding = m_headers.get("Content-Encoding");
-    if (content_encoding.has_value()) {
-        flattened_buffer = handle_content_encoding(flattened_buffer, content_encoding.value());
-    }
-
-    auto response = HttpResponse::create(m_code, move(m_headers), move(flattened_buffer));
-    deferred_invoke([this, response](auto&) {
-        did_finish(move(response));
-    });
-}
-
 void HttpsJob::start()
 {
     ASSERT(!m_socket);
@@ -268,4 +81,57 @@ void HttpsJob::shutdown()
     remove_child(*m_socket);
     m_socket = nullptr;
 }
+
+void HttpsJob::read_while_data_available(Function<IterationDecision()> read)
+{
+    while (m_socket->can_read()) {
+        if (read() == IterationDecision::Break)
+            break;
+    }
+}
+
+void HttpsJob::register_on_ready_to_read(Function<void()> callback)
+{
+    m_socket->on_tls_ready_to_read = [callback = move(callback)](auto&) {
+        callback();
+    };
+}
+
+void HttpsJob::register_on_ready_to_write(Function<void()> callback)
+{
+    m_socket->on_tls_ready_to_write = [callback = move(callback)](auto&) {
+        callback();
+    };
+}
+
+bool HttpsJob::can_read_line()
+{
+    return m_socket->can_read_line();
+}
+
+ByteBuffer HttpsJob::read_line(size_t size)
+{
+    return m_socket->read_line(size);
+}
+
+ByteBuffer HttpsJob::receive(size_t size)
+{
+    return m_socket->read(size);
+}
+
+bool HttpsJob::can_read() const
+{
+    return m_socket->can_read();
+}
+
+bool HttpsJob::eof() const
+{
+    return m_socket->eof();
+}
+
+bool HttpsJob::write(const ByteBuffer& data)
+{
+    return m_socket->write(data);
+}
+
 }

Libraries/LibHTTP/HttpsJob.h

@@ -30,43 +30,41 @@
 #include <LibCore/NetworkJob.h>
 #include <LibHTTP/HttpRequest.h>
 #include <LibHTTP/HttpResponse.h>
+#include <LibHTTP/Job.h>
 #include <LibTLS/TLSv12.h>
 
 namespace HTTP {
 
-class HttpsJob final : public Core::NetworkJob {
+class HttpsJob final : public Job {
     C_OBJECT(HttpsJob)
 public:
-    explicit HttpsJob(const HttpRequest&);
+    explicit HttpsJob(const HttpRequest& request)
-    virtual ~HttpsJob() override;
+        : Job(request)
+    {
+    }
+
+    virtual ~HttpsJob() override
+    {
+    }
 
     virtual void start() override;
     virtual void shutdown() override;
 
-    HttpResponse* response() { return static_cast<HttpResponse*>(Core::NetworkJob::response()); }
+protected:
-    const HttpResponse* response() const { return static_cast<const HttpResponse*>(Core::NetworkJob::response()); }
+    virtual void register_on_ready_to_read(Function<void()>) override;
+    virtual void register_on_ready_to_write(Function<void()>) override;
+    virtual bool can_read_line() override;
+    virtual ByteBuffer read_line(size_t) override;
+    virtual bool can_read() const override;
+    virtual ByteBuffer receive(size_t) override;
+    virtual bool eof() const override;
+    virtual bool write(const ByteBuffer&) override;
+    virtual bool is_established() const override { return m_socket->is_established(); }
+    virtual bool should_fail_on_empty_payload() const override { return false; }
+    virtual void read_while_data_available(Function<IterationDecision()>) override;
 
 private:
-    RefPtr<TLS::TLSv12> construct_socket() { return TLS::TLSv12::construct(this); }
-    void on_socket_connected();
-    void finish_up();
-    void read_body(TLS::TLSv12&);
-
-    enum class State {
-        InStatus,
-        InHeaders,
-        InBody,
-        Finished,
-    };
-
-    HttpRequest m_request;
     RefPtr<TLS::TLSv12> m_socket;
-    State m_state { State::InStatus };
-    int m_code { -1 };
-    HashMap<String, String> m_headers;
-    Vector<ByteBuffer> m_received_buffers;
-    size_t m_received_size { 0 };
-    bool m_sent_data { false };
     bool m_queued_finish { false };
 };
 

Libraries/LibHTTP/Job.cpp

@@ -0,0 +1,222 @@
+/*
+ * Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <LibCore/Gzip.h>
+#include <LibCore/TCPSocket.h>
+#include <LibHTTP/HttpResponse.h>
+#include <LibHTTP/Job.h>
+#include <stdio.h>
+#include <unistd.h>
+
+//#define JOB_DEBUG
+
+namespace HTTP {
+
+static ByteBuffer handle_content_encoding(const ByteBuffer& buf, const String& content_encoding)
+{
+#ifdef JOB_DEBUG
+    dbg() << "Job::handle_content_encoding: buf has content_encoding = " << content_encoding;
+#endif
+
+    if (content_encoding == "gzip") {
+        if (!Core::Gzip::is_compressed(buf)) {
+            dbg() << "Job::handle_content_encoding: buf is not gzip compressed!";
+        }
+
+#ifdef JOB_DEBUG
+        dbg() << "Job::handle_content_encoding: buf is gzip compressed!";
+#endif
+
+        auto uncompressed = Core::Gzip::decompress(buf);
+        if (!uncompressed.has_value()) {
+            dbg() << "Job::handle_content_encoding: Gzip::decompress() failed. Returning original buffer.";
+            return buf;
+        }
+
+#ifdef JOB_DEBUG
+        dbg() << "Job::handle_content_encoding: Gzip::decompress() successful.\n"
+              << "  Input size = " << buf.size() << "\n"
+              << "  Output size = " << uncompressed.value().size();
+#endif
+
+        return uncompressed.value();
+    }
+
+    return buf;
+}
+
+Job::Job(const HttpRequest& request)
+    : m_request(request)
+{
+}
+
+Job::~Job()
+{
+}
+
+void Job::on_socket_connected()
+{
+    register_on_ready_to_write([&] {
+        if (m_sent_data)
+            return;
+        m_sent_data = true;
+        auto raw_request = m_request.to_raw_request();
+#ifdef JOB_DEBUG
+        dbg() << "Job: raw_request:";
+        dbg() << String::copy(raw_request).characters();
+#endif
+        bool success = write(raw_request);
+        if (!success)
+            deferred_invoke([this](auto&) { did_fail(Core::NetworkJob::Error::TransmissionFailed); });
+    });
+    register_on_ready_to_read([&] {
+        if (is_cancelled())
+            return;
+        if (m_state == State::InStatus) {
+            if (!can_read_line())
+                return;
+            auto line = read_line(PAGE_SIZE);
+            if (line.is_null()) {
+                fprintf(stderr, "Job: Expected HTTP status\n");
+                return deferred_invoke([this](auto&) { did_fail(Core::NetworkJob::Error::TransmissionFailed); });
+            }
+            auto parts = String::copy(line, Chomp).split(' ');
+            if (parts.size() < 3) {
+                fprintf(stderr, "Job: Expected 3-part HTTP status, got '%s'\n", line.data());
+                return deferred_invoke([this](auto&) { did_fail(Core::NetworkJob::Error::ProtocolFailed); });
+            }
+            bool ok;
+            m_code = parts[1].to_uint(ok);
+            if (!ok) {
+                fprintf(stderr, "Job: Expected numeric HTTP status\n");
+                return deferred_invoke([this](auto&) { did_fail(Core::NetworkJob::Error::ProtocolFailed); });
+            }
+            m_state = State::InHeaders;
+            return;
+        }
+        if (m_state == State::InHeaders) {
+            if (!can_read_line())
+                return;
+            auto line = read_line(PAGE_SIZE);
+            if (line.is_null()) {
+                fprintf(stderr, "Job: Expected HTTP header\n");
+                return did_fail(Core::NetworkJob::Error::ProtocolFailed);
+            }
+            auto chomped_line = String::copy(line, Chomp);
+            if (chomped_line.is_empty()) {
+                m_state = State::InBody;
+                return;
+            }
+            auto parts = chomped_line.split(':');
+            if (parts.is_empty()) {
+                fprintf(stderr, "Job: Expected HTTP header with key/value\n");
+                return deferred_invoke([this](auto&) { did_fail(Core::NetworkJob::Error::ProtocolFailed); });
+            }
+            auto name = parts[0];
+            if (chomped_line.length() < name.length() + 2) {
+                fprintf(stderr, "Job: Malformed HTTP header: '%s' (%zu)\n", chomped_line.characters(), chomped_line.length());
+                return deferred_invoke([this](auto&) { did_fail(Core::NetworkJob::Error::ProtocolFailed); });
+            }
+            auto value = chomped_line.substring(name.length() + 2, chomped_line.length() - name.length() - 2);
+            m_headers.set(name, value);
+#ifdef JOB_DEBUG
+            dbg() << "Job: [" << name << "] = '" << value << "'";
+#endif
+            return;
+        }
+        ASSERT(m_state == State::InBody);
+        ASSERT(can_read());
+
+        read_while_data_available([&] {
+            auto payload = receive(64 * KB);
+            if (!payload) {
+                if (eof()) {
+                    finish_up();
+                    return IterationDecision::Break;
+                }
+
+                if (should_fail_on_empty_payload()) {
+                    deferred_invoke([this](auto&) { did_fail(Core::NetworkJob::Error::ProtocolFailed); });
+                    return IterationDecision::Break;
+                }
+            }
+            m_received_buffers.append(payload);
+            m_received_size += payload.size();
+
+            auto content_length_header = m_headers.get("Content-Length");
+            Optional<u32> content_length {};
+
+            if (content_length_header.has_value()) {
+                bool ok;
+                auto length = content_length_header.value().to_uint(ok);
+                if (ok)
+                    content_length = length;
+            }
+
+            did_progress(content_length, m_received_size);
+
+            if (content_length.has_value()) {
+                auto length = content_length.value();
+                if (m_received_size >= length) {
+                    m_received_size = length;
+                    finish_up();
+                    return IterationDecision::Break;
+                }
+            }
+            return IterationDecision::Continue;
+        });
+
+        if (!is_established()) {
+#ifdef JOB_DEBUG
+            dbg() << "Connection appears to have closed, finishing up";
+#endif
+            finish_up();
+        }
+    });
+}
+
+void Job::finish_up()
+{
+    m_state = State::Finished;
+    auto flattened_buffer = ByteBuffer::create_uninitialized(m_received_size);
+    u8* flat_ptr = flattened_buffer.data();
+    for (auto& received_buffer : m_received_buffers) {
+        memcpy(flat_ptr, received_buffer.data(), received_buffer.size());
+        flat_ptr += received_buffer.size();
+    }
+    m_received_buffers.clear();
+
+    auto content_encoding = m_headers.get("Content-Encoding");
+    if (content_encoding.has_value()) {
+        flattened_buffer = handle_content_encoding(flattened_buffer, content_encoding.value());
+    }
+
+    auto response = HttpResponse::create(m_code, move(m_headers), move(flattened_buffer));
+    deferred_invoke([this, response](auto&) {
+        did_finish(move(response));
+    });
+}
+}

Libraries/LibHTTP/Job.h

@@ -0,0 +1,80 @@
+/*
+ * Copyright (c) 2020, The SerenityOS developers.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+#include <AK/HashMap.h>
+#include <LibCore/NetworkJob.h>
+#include <LibCore/TCPSocket.h>
+#include <LibHTTP/HttpRequest.h>
+#include <LibHTTP/HttpResponse.h>
+
+namespace HTTP {
+
+class Job : public Core::NetworkJob {
+public:
+    explicit Job(const HttpRequest&);
+    virtual ~Job() override;
+
+    virtual void start() override = 0;
+    virtual void shutdown() override = 0;
+
+    HttpResponse* response() { return static_cast<HttpResponse*>(Core::NetworkJob::response()); }
+    const HttpResponse* response() const { return static_cast<const HttpResponse*>(Core::NetworkJob::response()); }
+
+protected:
+    void finish_up();
+    void on_socket_connected();
+    virtual void register_on_ready_to_read(Function<void()>) = 0;
+    virtual void register_on_ready_to_write(Function<void()>) = 0;
+    // FIXME: I want const but Core::IODevice::can_read_line populates a cache with this
+    virtual bool can_read_line() = 0;
+    virtual ByteBuffer read_line(size_t) = 0;
+    virtual bool can_read() const = 0;
+    virtual ByteBuffer receive(size_t) = 0;
+    virtual bool eof() const = 0;
+    virtual bool write(const ByteBuffer&) = 0;
+    virtual bool is_established() const = 0;
+    virtual bool should_fail_on_empty_payload() const { return true; }
+    virtual void read_while_data_available(Function<IterationDecision()> read) { read(); };
+
+    enum class State {
+        InStatus,
+        InHeaders,
+        InBody,
+        Finished,
+    };
+
+    HttpRequest m_request;
+    State m_state { State::InStatus };
+    int m_code { -1 };
+    HashMap<String, String> m_headers;
+    Vector<ByteBuffer> m_received_buffers;
+    size_t m_received_size { 0 };
+    bool m_sent_data { 0 };
+};
+
+}

Libraries/LibHTTP/Makefile

@@ -1,7 +1,8 @@
 OBJS = HttpResponse.o \
 	   HttpRequest.o \
 	   HttpJob.o \
-	   HttpsJob.o
+	   HttpsJob.o \
+	   Job.o
 
 LIBRARY = libhttp.a