1
Fork 0
mirror of https://github.com/RGBCube/serenity synced 2025-07-25 11:37:44 +00:00

ProtocolServer: Stream the downloaded data if possible

This patchset makes ProtocolServer stream the downloads to its client
(LibProtocol), and as such changes the download API; a possible
download lifecycle could be as such:
notation = client->server:'>', server->client:'<', pipe activity:'*'
```
> StartDownload(GET, url, headers, {})
< Response(0, fd 8)
* {data, 1024b}
< HeadersBecameAvailable(0, response_headers, 200)
< DownloadProgress(0, 4K, 1024)
* {data, 1024b}
* {data, 1024b}
< DownloadProgress(0, 4K, 2048)
* {data, 1024b}
< DownloadProgress(0, 4K, 1024)
< DownloadFinished(0, true, 4K)
```

Since managing the received file descriptor is a pain, LibProtocol
implements `Download::stream_into(OutputStream)`, which can be used to
stream the download into any given output stream (be it a file, or
memory, or writing stuff with a delay, etc.).
Also, as some of the users of this API require all the downloaded data
upfront, LibProtocol also implements `set_should_buffer_all_input()`,
which causes the download instance to buffer all the data until the
download is complete, and to call the `on_buffered_download_finish`
hook.
This commit is contained in:
AnotherTest 2020-12-26 17:14:12 +03:30 committed by Andreas Kling
parent 36d642ee75
commit 4a2da10e38
55 changed files with 528 additions and 235 deletions

View file

@ -98,9 +98,9 @@ bool HttpJob::eof() const
return m_socket->eof();
}
bool HttpJob::write(const ByteBuffer& data)
bool HttpJob::write(ReadonlyBytes bytes)
{
return m_socket->write(data);
return m_socket->write(bytes);
}
}

View file

@ -38,8 +38,8 @@ namespace HTTP {
class HttpJob final : public Job {
C_OBJECT(HttpJob)
public:
explicit HttpJob(const HttpRequest& request)
: Job(request)
explicit HttpJob(const HttpRequest& request, OutputStream& output_stream)
: Job(request, output_stream)
{
}
@ -59,7 +59,7 @@ protected:
virtual bool can_read() const override;
virtual ByteBuffer receive(size_t) override;
virtual bool eof() const override;
virtual bool write(const ByteBuffer&) override;
virtual bool write(ReadonlyBytes) override;
virtual bool is_established() const override { return true; }
private:

View file

@ -71,11 +71,12 @@ ByteBuffer HttpRequest::to_raw_request() const
builder.append(header.value);
builder.append("\r\n");
}
builder.append("Connection: close\r\n\r\n");
builder.append("Connection: close\r\n");
if (!m_body.is_empty()) {
builder.appendff("Content-Length: {}\r\n\r\n", m_body.size());
builder.append((const char*)m_body.data(), m_body.size());
builder.append("\r\n");
}
builder.append("\r\n");
return builder.to_byte_buffer();
}

View file

@ -62,7 +62,8 @@ public:
void set_method(Method method) { m_method = method; }
const ByteBuffer& body() const { return m_body; }
void set_body(const ByteBuffer& body) { m_body = body; }
void set_body(ReadonlyBytes body) { m_body = ByteBuffer::copy(body); }
void set_body(ByteBuffer&& body) { m_body = move(body); }
String method_name() const;
ByteBuffer to_raw_request() const;

View file

@ -28,9 +28,8 @@
namespace HTTP {
HttpResponse::HttpResponse(int code, HashMap<String, String, CaseInsensitiveStringTraits>&& headers, ByteBuffer&& payload)
: Core::NetworkResponse(move(payload))
, m_code(code)
HttpResponse::HttpResponse(int code, HashMap<String, String, CaseInsensitiveStringTraits>&& headers)
: m_code(code)
, m_headers(move(headers))
{
}

View file

@ -35,16 +35,16 @@ namespace HTTP {
class HttpResponse : public Core::NetworkResponse {
public:
virtual ~HttpResponse() override;
static NonnullRefPtr<HttpResponse> create(int code, HashMap<String, String, CaseInsensitiveStringTraits>&& headers, ByteBuffer&& payload)
static NonnullRefPtr<HttpResponse> create(int code, HashMap<String, String, CaseInsensitiveStringTraits>&& headers)
{
return adopt(*new HttpResponse(code, move(headers), move(payload)));
return adopt(*new HttpResponse(code, move(headers)));
}
int code() const { return m_code; }
const HashMap<String, String, CaseInsensitiveStringTraits>& headers() const { return m_headers; }
private:
HttpResponse(int code, HashMap<String, String, CaseInsensitiveStringTraits>&&, ByteBuffer&&);
HttpResponse(int code, HashMap<String, String, CaseInsensitiveStringTraits>&&);
int m_code { 0 };
HashMap<String, String, CaseInsensitiveStringTraits> m_headers;

View file

@ -143,7 +143,7 @@ bool HttpsJob::eof() const
return m_socket->eof();
}
bool HttpsJob::write(const ByteBuffer& data)
bool HttpsJob::write(ReadonlyBytes data)
{
return m_socket->write(data);
}

View file

@ -38,8 +38,8 @@ namespace HTTP {
class HttpsJob final : public Job {
C_OBJECT(HttpsJob)
public:
explicit HttpsJob(const HttpRequest& request, const Vector<Certificate>* override_certs = nullptr)
: Job(request)
explicit HttpsJob(const HttpRequest& request, OutputStream& output_stream, const Vector<Certificate>* override_certs = nullptr)
: Job(request, output_stream)
, m_override_ca_certificates(override_certs)
{
}
@ -62,7 +62,7 @@ protected:
virtual bool can_read() const override;
virtual ByteBuffer receive(size_t) override;
virtual bool eof() const override;
virtual bool write(const ByteBuffer&) override;
virtual bool write(ReadonlyBytes) override;
virtual bool is_established() const override { return m_socket->is_established(); }
virtual bool should_fail_on_empty_payload() const override { return false; }
virtual void read_while_data_available(Function<IterationDecision()>) override;

View file

@ -68,8 +68,9 @@ static ByteBuffer handle_content_encoding(const ByteBuffer& buf, const String& c
return buf;
}
Job::Job(const HttpRequest& request)
: m_request(request)
Job::Job(const HttpRequest& request, OutputStream& output_stream)
: Core::NetworkJob(output_stream)
, m_request(request)
{
}
@ -77,6 +78,35 @@ Job::~Job()
{
}
void Job::flush_received_buffers()
{
if (!m_can_stream_response || m_buffered_size == 0)
return;
#ifdef JOB_DEBUG
dbg() << "Job: Flushing received buffers: have " << m_buffered_size << " bytes in " << m_received_buffers.size() << " buffers";
#endif
for (size_t i = 0; i < m_received_buffers.size(); ++i) {
auto& payload = m_received_buffers[i];
auto written = do_write(payload);
m_buffered_size -= written;
if (written == payload.size()) {
// FIXME: Make this a take-first-friendly object?
m_received_buffers.take_first();
--i;
continue;
}
ASSERT(written < payload.size());
payload = payload.slice(written, payload.size() - written);
#ifdef JOB_DEBUG
dbg() << "Job: Flushing received buffers done: have " << m_buffered_size << " bytes in " << m_received_buffers.size() << " buffers";
#endif
return;
}
#ifdef JOB_DEBUG
dbg() << "Job: Flushing received buffers done: have " << m_buffered_size << " bytes in " << m_received_buffers.size() << " buffers";
#endif
}
void Job::on_socket_connected()
{
register_on_ready_to_write([&] {
@ -135,6 +165,8 @@ void Job::on_socket_connected()
if (m_state == State::Trailers) {
return finish_up();
} else {
if (on_headers_received)
on_headers_received(m_headers, m_code > 0 ? m_code : Optional<u32> {});
m_state = State::InBody;
}
return;
@ -163,6 +195,13 @@ void Job::on_socket_connected()
}
auto value = line.substring(name.length() + 2, line.length() - name.length() - 2);
m_headers.set(name, value);
if (name.equals_ignoring_case("Content-Encoding")) {
// Assume that any content-encoding means that we can't decode it as a stream :(
#ifdef JOB_DEBUG
dbg() << "Content-Encoding " << value << " detected, cannot stream output :(";
#endif
m_can_stream_response = false;
}
#ifdef JOB_DEBUG
dbg() << "Job: [" << name << "] = '" << value << "'";
#endif
@ -252,7 +291,9 @@ void Job::on_socket_connected()
}
m_received_buffers.append(payload);
m_buffered_size += payload.size();
m_received_size += payload.size();
flush_received_buffers();
if (m_current_chunk_remaining_size.has_value()) {
auto size = m_current_chunk_remaining_size.value() - payload.size();
@ -313,20 +354,37 @@ void Job::on_socket_connected()
void Job::finish_up()
{
m_state = State::Finished;
auto flattened_buffer = ByteBuffer::create_uninitialized(m_received_size);
u8* flat_ptr = flattened_buffer.data();
for (auto& received_buffer : m_received_buffers) {
memcpy(flat_ptr, received_buffer.data(), received_buffer.size());
flat_ptr += received_buffer.size();
}
m_received_buffers.clear();
if (!m_can_stream_response) {
auto flattened_buffer = ByteBuffer::create_uninitialized(m_received_size);
u8* flat_ptr = flattened_buffer.data();
for (auto& received_buffer : m_received_buffers) {
memcpy(flat_ptr, received_buffer.data(), received_buffer.size());
flat_ptr += received_buffer.size();
}
m_received_buffers.clear();
auto content_encoding = m_headers.get("Content-Encoding");
if (content_encoding.has_value()) {
flattened_buffer = handle_content_encoding(flattened_buffer, content_encoding.value());
// For the time being, we cannot stream stuff with content-encoding set to _anything_.
auto content_encoding = m_headers.get("Content-Encoding");
if (content_encoding.has_value()) {
flattened_buffer = handle_content_encoding(flattened_buffer, content_encoding.value());
}
m_buffered_size = flattened_buffer.size();
m_received_buffers.append(move(flattened_buffer));
m_can_stream_response = true;
}
auto response = HttpResponse::create(m_code, move(m_headers), move(flattened_buffer));
flush_received_buffers();
if (m_buffered_size != 0) {
// FIXME: What do we do? ignore it?
// "Transmission failed" is not strictly correct, but let's roll with it for now.
deferred_invoke([this](auto&) {
did_fail(Error::TransmissionFailed);
});
return;
}
auto response = HttpResponse::create(m_code, move(m_headers));
deferred_invoke([this, response](auto&) {
did_finish(move(response));
});

View file

@ -26,6 +26,7 @@
#pragma once
#include <AK/FileStream.h>
#include <AK/HashMap.h>
#include <AK/Optional.h>
#include <LibCore/NetworkJob.h>
@ -37,7 +38,7 @@ namespace HTTP {
class Job : public Core::NetworkJob {
public:
explicit Job(const HttpRequest&);
explicit Job(const HttpRequest&, OutputStream&);
virtual ~Job() override;
virtual void start() override = 0;
@ -49,6 +50,7 @@ public:
protected:
void finish_up();
void on_socket_connected();
void flush_received_buffers();
virtual void register_on_ready_to_read(Function<void()>) = 0;
virtual void register_on_ready_to_write(Function<void()>) = 0;
virtual bool can_read_line() const = 0;
@ -56,7 +58,7 @@ protected:
virtual bool can_read() const = 0;
virtual ByteBuffer receive(size_t) = 0;
virtual bool eof() const = 0;
virtual bool write(const ByteBuffer&) = 0;
virtual bool write(ReadonlyBytes) = 0;
virtual bool is_established() const = 0;
virtual bool should_fail_on_empty_payload() const { return true; }
virtual void read_while_data_available(Function<IterationDecision()> read) { read(); };
@ -73,11 +75,13 @@ protected:
State m_state { State::InStatus };
int m_code { -1 };
HashMap<String, String, CaseInsensitiveStringTraits> m_headers;
Vector<ByteBuffer> m_received_buffers;
Vector<ByteBuffer, 2> m_received_buffers;
size_t m_buffered_size { 0 };
size_t m_received_size { 0 };
bool m_sent_data { 0 };
Optional<ssize_t> m_current_chunk_remaining_size;
Optional<size_t> m_current_chunk_total_size;
bool m_can_stream_response { true };
};
}