ProtocolServer: Stream the downloaded data if possible

This patchset makes ProtocolServer stream the downloads to its client
(LibProtocol), and as such changes the download API; a possible
download lifecycle could be as such:
notation = client->server:'>', server->client:'<', pipe activity:'*'
```
> StartDownload(GET, url, headers, {})
< Response(0, fd 8)
* {data, 1024b}
< HeadersBecameAvailable(0, response_headers, 200)
< DownloadProgress(0, 4K, 1024)
* {data, 1024b}
* {data, 1024b}
< DownloadProgress(0, 4K, 2048)
* {data, 1024b}
< DownloadProgress(0, 4K, 1024)
< DownloadFinished(0, true, 4K)
```

Since managing the received file descriptor is a pain, LibProtocol
implements `Download::stream_into(OutputStream)`, which can be used to
stream the download into any given output stream (be it a file, or
memory, or writing stuff with a delay, etc.).
Also, as some of the users of this API require all the downloaded data
upfront, LibProtocol also implements `set_should_buffer_all_input()`,
which causes the download instance to buffer all the data until the
download is complete, and to call the `on_buffered_download_finish`
hook.

Libraries/LibHTTP/HttpJob.cpp

@@ -98,9 +98,9 @@ bool HttpJob::eof() const
     return m_socket->eof();
 }
 
-bool HttpJob::write(const ByteBuffer& data)
+bool HttpJob::write(ReadonlyBytes bytes)
 {
-    return m_socket->write(data);
+    return m_socket->write(bytes);
 }
 
 }

Libraries/LibHTTP/HttpJob.h

@@ -38,8 +38,8 @@ namespace HTTP {
 class HttpJob final : public Job {
     C_OBJECT(HttpJob)
 public:
-    explicit HttpJob(const HttpRequest& request)
-        : Job(request)
+    explicit HttpJob(const HttpRequest& request, OutputStream& output_stream)
+        : Job(request, output_stream)
     {
     }
 
@@ -59,7 +59,7 @@ protected:
     virtual bool can_read() const override;
     virtual ByteBuffer receive(size_t) override;
     virtual bool eof() const override;
-    virtual bool write(const ByteBuffer&) override;
+    virtual bool write(ReadonlyBytes) override;
     virtual bool is_established() const override { return true; }
 
 private:

Libraries/LibHTTP/HttpRequest.cpp

@@ -71,11 +71,12 @@ ByteBuffer HttpRequest::to_raw_request() const
         builder.append(header.value);
         builder.append("\r\n");
     }
-    builder.append("Connection: close\r\n\r\n");
+    builder.append("Connection: close\r\n");
     if (!m_body.is_empty()) {
+        builder.appendff("Content-Length: {}\r\n\r\n", m_body.size());
         builder.append((const char*)m_body.data(), m_body.size());
-        builder.append("\r\n");
     }
+    builder.append("\r\n");
     return builder.to_byte_buffer();
 }
 

Libraries/LibHTTP/HttpRequest.h

@@ -62,7 +62,8 @@ public:
     void set_method(Method method) { m_method = method; }
 
     const ByteBuffer& body() const { return m_body; }
-    void set_body(const ByteBuffer& body) { m_body = body; }
+    void set_body(ReadonlyBytes body) { m_body = ByteBuffer::copy(body); }
+    void set_body(ByteBuffer&& body) { m_body = move(body); }
 
     String method_name() const;
     ByteBuffer to_raw_request() const;

Libraries/LibHTTP/HttpResponse.cpp

@@ -28,9 +28,8 @@
 
 namespace HTTP {
 
-HttpResponse::HttpResponse(int code, HashMap<String, String, CaseInsensitiveStringTraits>&& headers, ByteBuffer&& payload)
-    : Core::NetworkResponse(move(payload))
-    , m_code(code)
+HttpResponse::HttpResponse(int code, HashMap<String, String, CaseInsensitiveStringTraits>&& headers)
+    : m_code(code)
     , m_headers(move(headers))
 {
 }

Libraries/LibHTTP/HttpResponse.h

@@ -35,16 +35,16 @@ namespace HTTP {
 class HttpResponse : public Core::NetworkResponse {
 public:
     virtual ~HttpResponse() override;
-    static NonnullRefPtr<HttpResponse> create(int code, HashMap<String, String, CaseInsensitiveStringTraits>&& headers, ByteBuffer&& payload)
+    static NonnullRefPtr<HttpResponse> create(int code, HashMap<String, String, CaseInsensitiveStringTraits>&& headers)
     {
-        return adopt(*new HttpResponse(code, move(headers), move(payload)));
+        return adopt(*new HttpResponse(code, move(headers)));
     }
 
     int code() const { return m_code; }
     const HashMap<String, String, CaseInsensitiveStringTraits>& headers() const { return m_headers; }
 
 private:
-    HttpResponse(int code, HashMap<String, String, CaseInsensitiveStringTraits>&&, ByteBuffer&&);
+    HttpResponse(int code, HashMap<String, String, CaseInsensitiveStringTraits>&&);
 
     int m_code { 0 };
     HashMap<String, String, CaseInsensitiveStringTraits> m_headers;

Libraries/LibHTTP/HttpsJob.cpp

@@ -143,7 +143,7 @@ bool HttpsJob::eof() const
     return m_socket->eof();
 }
 
-bool HttpsJob::write(const ByteBuffer& data)
+bool HttpsJob::write(ReadonlyBytes data)
 {
     return m_socket->write(data);
 }

Libraries/LibHTTP/HttpsJob.h

@@ -38,8 +38,8 @@ namespace HTTP {
 class HttpsJob final : public Job {
     C_OBJECT(HttpsJob)
 public:
-    explicit HttpsJob(const HttpRequest& request, const Vector<Certificate>* override_certs = nullptr)
-        : Job(request)
+    explicit HttpsJob(const HttpRequest& request, OutputStream& output_stream, const Vector<Certificate>* override_certs = nullptr)
+        : Job(request, output_stream)
         , m_override_ca_certificates(override_certs)
     {
     }
@@ -62,7 +62,7 @@ protected:
     virtual bool can_read() const override;
     virtual ByteBuffer receive(size_t) override;
     virtual bool eof() const override;
-    virtual bool write(const ByteBuffer&) override;
+    virtual bool write(ReadonlyBytes) override;
     virtual bool is_established() const override { return m_socket->is_established(); }
     virtual bool should_fail_on_empty_payload() const override { return false; }
     virtual void read_while_data_available(Function<IterationDecision()>) override;

Libraries/LibHTTP/Job.cpp

@@ -68,8 +68,9 @@ static ByteBuffer handle_content_encoding(const ByteBuffer& buf, const String& c
     return buf;
 }
 
-Job::Job(const HttpRequest& request)
-    : m_request(request)
+Job::Job(const HttpRequest& request, OutputStream& output_stream)
+    : Core::NetworkJob(output_stream)
+    , m_request(request)
 {
 }
 
@@ -77,6 +78,35 @@ Job::~Job()
 {
 }
 
+void Job::flush_received_buffers()
+{
+    if (!m_can_stream_response || m_buffered_size == 0)
+        return;
+#ifdef JOB_DEBUG
+    dbg() << "Job: Flushing received buffers: have " << m_buffered_size << " bytes in " << m_received_buffers.size() << " buffers";
+#endif
+    for (size_t i = 0; i < m_received_buffers.size(); ++i) {
+        auto& payload = m_received_buffers[i];
+        auto written = do_write(payload);
+        m_buffered_size -= written;
+        if (written == payload.size()) {
+            // FIXME: Make this a take-first-friendly object?
+            m_received_buffers.take_first();
+            --i;
+            continue;
+        }
+        ASSERT(written < payload.size());
+        payload = payload.slice(written, payload.size() - written);
+#ifdef JOB_DEBUG
+        dbg() << "Job: Flushing received buffers done: have " << m_buffered_size << " bytes in " << m_received_buffers.size() << " buffers";
+#endif
+        return;
+    }
+#ifdef JOB_DEBUG
+    dbg() << "Job: Flushing received buffers done: have " << m_buffered_size << " bytes in " << m_received_buffers.size() << " buffers";
+#endif
+}
+
 void Job::on_socket_connected()
 {
     register_on_ready_to_write([&] {
@@ -135,6 +165,8 @@ void Job::on_socket_connected()
                 if (m_state == State::Trailers) {
                     return finish_up();
                 } else {
+                    if (on_headers_received)
+                        on_headers_received(m_headers, m_code > 0 ? m_code : Optional<u32> {});
                     m_state = State::InBody;
                 }
                 return;
@@ -163,6 +195,13 @@ void Job::on_socket_connected()
             }
             auto value = line.substring(name.length() + 2, line.length() - name.length() - 2);
             m_headers.set(name, value);
+            if (name.equals_ignoring_case("Content-Encoding")) {
+                // Assume that any content-encoding means that we can't decode it as a stream :(
+#ifdef JOB_DEBUG
+                dbg() << "Content-Encoding " << value << " detected, cannot stream output :(";
+#endif
+                m_can_stream_response = false;
+            }
 #ifdef JOB_DEBUG
             dbg() << "Job: [" << name << "] = '" << value << "'";
 #endif
@@ -252,7 +291,9 @@ void Job::on_socket_connected()
             }
 
             m_received_buffers.append(payload);
+            m_buffered_size += payload.size();
             m_received_size += payload.size();
+            flush_received_buffers();
 
             if (m_current_chunk_remaining_size.has_value()) {
                 auto size = m_current_chunk_remaining_size.value() - payload.size();
@@ -313,20 +354,37 @@ void Job::on_socket_connected()
 void Job::finish_up()
 {
     m_state = State::Finished;
-    auto flattened_buffer = ByteBuffer::create_uninitialized(m_received_size);
-    u8* flat_ptr = flattened_buffer.data();
-    for (auto& received_buffer : m_received_buffers) {
-        memcpy(flat_ptr, received_buffer.data(), received_buffer.size());
-        flat_ptr += received_buffer.size();
-    }
-    m_received_buffers.clear();
+    if (!m_can_stream_response) {
+        auto flattened_buffer = ByteBuffer::create_uninitialized(m_received_size);
+        u8* flat_ptr = flattened_buffer.data();
+        for (auto& received_buffer : m_received_buffers) {
+            memcpy(flat_ptr, received_buffer.data(), received_buffer.size());
+            flat_ptr += received_buffer.size();
+        }
+        m_received_buffers.clear();
 
-    auto content_encoding = m_headers.get("Content-Encoding");
-    if (content_encoding.has_value()) {
-        flattened_buffer = handle_content_encoding(flattened_buffer, content_encoding.value());
+        // For the time being, we cannot stream stuff with content-encoding set to _anything_.
+        auto content_encoding = m_headers.get("Content-Encoding");
+        if (content_encoding.has_value()) {
+            flattened_buffer = handle_content_encoding(flattened_buffer, content_encoding.value());
+        }
+
+        m_buffered_size = flattened_buffer.size();
+        m_received_buffers.append(move(flattened_buffer));
+        m_can_stream_response = true;
     }
 
-    auto response = HttpResponse::create(m_code, move(m_headers), move(flattened_buffer));
+    flush_received_buffers();
+    if (m_buffered_size != 0) {
+        // FIXME: What do we do? ignore it?
+        //        "Transmission failed" is not strictly correct, but let's roll with it for now.
+        deferred_invoke([this](auto&) {
+            did_fail(Error::TransmissionFailed);
+        });
+        return;
+    }
+
+    auto response = HttpResponse::create(m_code, move(m_headers));
     deferred_invoke([this, response](auto&) {
         did_finish(move(response));
     });

Libraries/LibHTTP/Job.h

@@ -26,6 +26,7 @@
 
 #pragma once
 
+#include <AK/FileStream.h>
 #include <AK/HashMap.h>
 #include <AK/Optional.h>
 #include <LibCore/NetworkJob.h>
@@ -37,7 +38,7 @@ namespace HTTP {
 
 class Job : public Core::NetworkJob {
 public:
-    explicit Job(const HttpRequest&);
+    explicit Job(const HttpRequest&, OutputStream&);
     virtual ~Job() override;
 
     virtual void start() override = 0;
@@ -49,6 +50,7 @@ public:
 protected:
     void finish_up();
     void on_socket_connected();
+    void flush_received_buffers();
     virtual void register_on_ready_to_read(Function<void()>) = 0;
     virtual void register_on_ready_to_write(Function<void()>) = 0;
     virtual bool can_read_line() const = 0;
@@ -56,7 +58,7 @@ protected:
     virtual bool can_read() const = 0;
     virtual ByteBuffer receive(size_t) = 0;
     virtual bool eof() const = 0;
-    virtual bool write(const ByteBuffer&) = 0;
+    virtual bool write(ReadonlyBytes) = 0;
     virtual bool is_established() const = 0;
     virtual bool should_fail_on_empty_payload() const { return true; }
     virtual void read_while_data_available(Function<IterationDecision()> read) { read(); };
@@ -73,11 +75,13 @@ protected:
     State m_state { State::InStatus };
     int m_code { -1 };
     HashMap<String, String, CaseInsensitiveStringTraits> m_headers;
-    Vector<ByteBuffer> m_received_buffers;
+    Vector<ByteBuffer, 2> m_received_buffers;
+    size_t m_buffered_size { 0 };
     size_t m_received_size { 0 };
     bool m_sent_data { 0 };
     Optional<ssize_t> m_current_chunk_remaining_size;
     Optional<size_t> m_current_chunk_total_size;
+    bool m_can_stream_response { true };
 };
 
 }