ProtocolServer: Stream the downloaded data if possible

This patchset makes ProtocolServer stream the downloads to its client
(LibProtocol), and as such changes the download API; a possible
download lifecycle could be as such:
notation = client->server:'>', server->client:'<', pipe activity:'*'
```
> StartDownload(GET, url, headers, {})
< Response(0, fd 8)
* {data, 1024b}
< HeadersBecameAvailable(0, response_headers, 200)
< DownloadProgress(0, 4K, 1024)
* {data, 1024b}
* {data, 1024b}
< DownloadProgress(0, 4K, 2048)
* {data, 1024b}
< DownloadProgress(0, 4K, 1024)
< DownloadFinished(0, true, 4K)
```

Since managing the received file descriptor is a pain, LibProtocol
implements `Download::stream_into(OutputStream)`, which can be used to
stream the download into any given output stream (be it a file, or
memory, or writing stuff with a delay, etc.).
Also, as some of the users of this API require all the downloaded data
upfront, LibProtocol also implements `set_should_buffer_all_input()`,
which causes the download instance to buffer all the data until the
download is complete, and to call the `on_buffered_download_finish`
hook.

Libraries/LibWeb/Loader/FrameLoader.cpp

@@ -171,6 +171,7 @@ bool FrameLoader::load(const LoadRequest& request, Type type)
         return true;
 
     if (url.protocol() == "http" || url.protocol() == "https") {
+#if 0
         URL favicon_url;
         favicon_url.set_protocol(url.protocol());
         favicon_url.set_host(url.host());
@@ -191,6 +192,7 @@ bool FrameLoader::load(const LoadRequest& request, Type type)
                 if (auto* page = frame().page())
                     page->client().page_did_change_favicon(*bitmap);
             });
+#endif
     }
 
     return true;

Libraries/LibWeb/Loader/Resource.cpp

@@ -84,10 +84,10 @@ static String mime_type_from_content_type(const String& content_type)
     return content_type;
 }
 
-void Resource::did_load(Badge<ResourceLoader>, const ByteBuffer& data, const HashMap<String, String, CaseInsensitiveStringTraits>& headers)
+void Resource::did_load(Badge<ResourceLoader>, ReadonlyBytes data, const HashMap<String, String, CaseInsensitiveStringTraits>& headers)
 {
     ASSERT(!m_loaded);
-    m_encoded_data = data;
+    m_encoded_data = ByteBuffer::copy(data);
     m_response_headers = headers;
     m_loaded = true;
 

Libraries/LibWeb/Loader/Resource.h

@@ -77,7 +77,7 @@ public:
 
     void for_each_client(Function<void(ResourceClient&)>);
 
-    void did_load(Badge<ResourceLoader>, const ByteBuffer& data, const HashMap<String, String, CaseInsensitiveStringTraits>& headers);
+    void did_load(Badge<ResourceLoader>, ReadonlyBytes data, const HashMap<String, String, CaseInsensitiveStringTraits>& headers);
     void did_fail(Badge<ResourceLoader>, const String& error);
 
 protected:

Libraries/LibWeb/Loader/ResourceLoader.cpp

@@ -53,13 +53,13 @@ ResourceLoader::ResourceLoader()
 {
 }
 
-void ResourceLoader::load_sync(const URL& url, Function<void(const ByteBuffer&, const HashMap<String, String, CaseInsensitiveStringTraits>& response_headers)> success_callback, Function<void(const String&)> error_callback)
+void ResourceLoader::load_sync(const URL& url, Function<void(ReadonlyBytes, const HashMap<String, String, CaseInsensitiveStringTraits>& response_headers)> success_callback, Function<void(const String&)> error_callback)
 {
     Core::EventLoop loop;
 
     load(
         url,
-        [&](auto& data, auto& response_headers) {
+        [&](auto data, auto& response_headers) {
             success_callback(data, response_headers);
             loop.quit(0);
         },
@@ -97,7 +97,7 @@ RefPtr<Resource> ResourceLoader::load_resource(Resource::Type type, const LoadRe
 
     load(
         request,
-        [=](auto& data, auto& headers) {
+        [=](auto data, auto& headers) {
             const_cast<Resource&>(*resource).did_load({}, data, headers);
         },
         [=](auto& error) {
@@ -107,7 +107,7 @@ RefPtr<Resource> ResourceLoader::load_resource(Resource::Type type, const LoadRe
     return resource;
 }
 
-void ResourceLoader::load(const LoadRequest& request, Function<void(const ByteBuffer&, const HashMap<String, String, CaseInsensitiveStringTraits>& response_headers)> success_callback, Function<void(const String&)> error_callback)
+void ResourceLoader::load(const LoadRequest& request, Function<void(ReadonlyBytes, const HashMap<String, String, CaseInsensitiveStringTraits>& response_headers)> success_callback, Function<void(const String&)> error_callback)
 {
     auto& url = request.url();
     if (is_port_blocked(url.port())) {
@@ -170,7 +170,12 @@ void ResourceLoader::load(const LoadRequest& request, Function<void(const ByteBu
                 error_callback("Failed to initiate load");
             return;
         }
-        download->on_finish = [this, success_callback = move(success_callback), error_callback = move(error_callback)](bool success, ReadonlyBytes payload, auto, auto& response_headers, auto status_code) {
+        download->on_buffered_download_finish = [this, success_callback = move(success_callback), error_callback = move(error_callback), download](bool success, auto, auto& response_headers, auto status_code, ReadonlyBytes payload) {
+            if (status_code.has_value() && status_code.value() >= 400 && status_code.value() <= 499) {
+                if (error_callback)
+                    error_callback(String::format("HTTP error (%u)", status_code.value()));
+                return;
+            }
             --m_pending_loads;
             if (on_load_counter_change)
                 on_load_counter_change();
@@ -179,13 +184,9 @@ void ResourceLoader::load(const LoadRequest& request, Function<void(const ByteBu
                     error_callback("HTTP load failed");
                 return;
             }
-            if (status_code.has_value() && status_code.value() >= 400 && status_code.value() <= 499) {
-                if (error_callback)
-                    error_callback(String::format("HTTP error (%u)", status_code.value()));
-                return;
-            }
-            success_callback(ByteBuffer::copy(payload.data(), payload.size()), response_headers);
+            success_callback(payload, response_headers);
         };
+        download->set_should_buffer_all_input(true);
         download->on_certificate_requested = []() -> Protocol::Download::CertificateAndKey {
             return {};
         };
@@ -199,7 +200,7 @@ void ResourceLoader::load(const LoadRequest& request, Function<void(const ByteBu
         error_callback(String::format("Protocol not implemented: %s", url.protocol().characters()));
 }
 
-void ResourceLoader::load(const URL& url, Function<void(const ByteBuffer&, const HashMap<String, String, CaseInsensitiveStringTraits>& response_headers)> success_callback, Function<void(const String&)> error_callback)
+void ResourceLoader::load(const URL& url, Function<void(ReadonlyBytes, const HashMap<String, String, CaseInsensitiveStringTraits>& response_headers)> success_callback, Function<void(const String&)> error_callback)
 {
     LoadRequest request;
     request.set_url(url);

Libraries/LibWeb/Loader/ResourceLoader.h

@@ -44,9 +44,9 @@ public:
 
     RefPtr<Resource> load_resource(Resource::Type, const LoadRequest&);
 
-    void load(const LoadRequest&, Function<void(const ByteBuffer&, const HashMap<String, String, CaseInsensitiveStringTraits>& response_headers)> success_callback, Function<void(const String&)> error_callback = nullptr);
-    void load(const URL&, Function<void(const ByteBuffer&, const HashMap<String, String, CaseInsensitiveStringTraits>& response_headers)> success_callback, Function<void(const String&)> error_callback = nullptr);
-    void load_sync(const URL&, Function<void(const ByteBuffer&, const HashMap<String, String, CaseInsensitiveStringTraits>& response_headers)> success_callback, Function<void(const String&)> error_callback = nullptr);
+    void load(const LoadRequest&, Function<void(ReadonlyBytes, const HashMap<String, String, CaseInsensitiveStringTraits>& response_headers)> success_callback, Function<void(const String&)> error_callback = nullptr);
+    void load(const URL&, Function<void(ReadonlyBytes, const HashMap<String, String, CaseInsensitiveStringTraits>& response_headers)> success_callback, Function<void(const String&)> error_callback = nullptr);
+    void load_sync(const URL&, Function<void(ReadonlyBytes, const HashMap<String, String, CaseInsensitiveStringTraits>& response_headers)> success_callback, Function<void(const String&)> error_callback = nullptr);
 
     Function<void()> on_load_counter_change;