mirror of
https://github.com/RGBCube/serenity
synced 2025-07-25 11:37:44 +00:00
ProtocolServer: Stream the downloaded data if possible
This patchset makes ProtocolServer stream the downloads to its client (LibProtocol), and as such changes the download API; a possible download lifecycle could be as such: notation = client->server:'>', server->client:'<', pipe activity:'*' ``` > StartDownload(GET, url, headers, {}) < Response(0, fd 8) * {data, 1024b} < HeadersBecameAvailable(0, response_headers, 200) < DownloadProgress(0, 4K, 1024) * {data, 1024b} * {data, 1024b} < DownloadProgress(0, 4K, 2048) * {data, 1024b} < DownloadProgress(0, 4K, 1024) < DownloadFinished(0, true, 4K) ``` Since managing the received file descriptor is a pain, LibProtocol implements `Download::stream_into(OutputStream)`, which can be used to stream the download into any given output stream (be it a file, or memory, or writing stuff with a delay, etc.). Also, as some of the users of this API require all the downloaded data upfront, LibProtocol also implements `set_should_buffer_all_input()`, which causes the download instance to buffer all the data until the download is complete, and to call the `on_buffered_download_finish` hook.
This commit is contained in:
parent
36d642ee75
commit
4a2da10e38
55 changed files with 528 additions and 235 deletions
|
@ -98,9 +98,9 @@ bool HttpJob::eof() const
|
|||
return m_socket->eof();
|
||||
}
|
||||
|
||||
bool HttpJob::write(const ByteBuffer& data)
|
||||
bool HttpJob::write(ReadonlyBytes bytes)
|
||||
{
|
||||
return m_socket->write(data);
|
||||
return m_socket->write(bytes);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -38,8 +38,8 @@ namespace HTTP {
|
|||
class HttpJob final : public Job {
|
||||
C_OBJECT(HttpJob)
|
||||
public:
|
||||
explicit HttpJob(const HttpRequest& request)
|
||||
: Job(request)
|
||||
explicit HttpJob(const HttpRequest& request, OutputStream& output_stream)
|
||||
: Job(request, output_stream)
|
||||
{
|
||||
}
|
||||
|
||||
|
@ -59,7 +59,7 @@ protected:
|
|||
virtual bool can_read() const override;
|
||||
virtual ByteBuffer receive(size_t) override;
|
||||
virtual bool eof() const override;
|
||||
virtual bool write(const ByteBuffer&) override;
|
||||
virtual bool write(ReadonlyBytes) override;
|
||||
virtual bool is_established() const override { return true; }
|
||||
|
||||
private:
|
||||
|
|
|
@ -71,11 +71,12 @@ ByteBuffer HttpRequest::to_raw_request() const
|
|||
builder.append(header.value);
|
||||
builder.append("\r\n");
|
||||
}
|
||||
builder.append("Connection: close\r\n\r\n");
|
||||
builder.append("Connection: close\r\n");
|
||||
if (!m_body.is_empty()) {
|
||||
builder.appendff("Content-Length: {}\r\n\r\n", m_body.size());
|
||||
builder.append((const char*)m_body.data(), m_body.size());
|
||||
builder.append("\r\n");
|
||||
}
|
||||
builder.append("\r\n");
|
||||
return builder.to_byte_buffer();
|
||||
}
|
||||
|
||||
|
|
|
@ -62,7 +62,8 @@ public:
|
|||
void set_method(Method method) { m_method = method; }
|
||||
|
||||
const ByteBuffer& body() const { return m_body; }
|
||||
void set_body(const ByteBuffer& body) { m_body = body; }
|
||||
void set_body(ReadonlyBytes body) { m_body = ByteBuffer::copy(body); }
|
||||
void set_body(ByteBuffer&& body) { m_body = move(body); }
|
||||
|
||||
String method_name() const;
|
||||
ByteBuffer to_raw_request() const;
|
||||
|
|
|
@ -28,9 +28,8 @@
|
|||
|
||||
namespace HTTP {
|
||||
|
||||
HttpResponse::HttpResponse(int code, HashMap<String, String, CaseInsensitiveStringTraits>&& headers, ByteBuffer&& payload)
|
||||
: Core::NetworkResponse(move(payload))
|
||||
, m_code(code)
|
||||
HttpResponse::HttpResponse(int code, HashMap<String, String, CaseInsensitiveStringTraits>&& headers)
|
||||
: m_code(code)
|
||||
, m_headers(move(headers))
|
||||
{
|
||||
}
|
||||
|
|
|
@ -35,16 +35,16 @@ namespace HTTP {
|
|||
class HttpResponse : public Core::NetworkResponse {
|
||||
public:
|
||||
virtual ~HttpResponse() override;
|
||||
static NonnullRefPtr<HttpResponse> create(int code, HashMap<String, String, CaseInsensitiveStringTraits>&& headers, ByteBuffer&& payload)
|
||||
static NonnullRefPtr<HttpResponse> create(int code, HashMap<String, String, CaseInsensitiveStringTraits>&& headers)
|
||||
{
|
||||
return adopt(*new HttpResponse(code, move(headers), move(payload)));
|
||||
return adopt(*new HttpResponse(code, move(headers)));
|
||||
}
|
||||
|
||||
int code() const { return m_code; }
|
||||
const HashMap<String, String, CaseInsensitiveStringTraits>& headers() const { return m_headers; }
|
||||
|
||||
private:
|
||||
HttpResponse(int code, HashMap<String, String, CaseInsensitiveStringTraits>&&, ByteBuffer&&);
|
||||
HttpResponse(int code, HashMap<String, String, CaseInsensitiveStringTraits>&&);
|
||||
|
||||
int m_code { 0 };
|
||||
HashMap<String, String, CaseInsensitiveStringTraits> m_headers;
|
||||
|
|
|
@ -143,7 +143,7 @@ bool HttpsJob::eof() const
|
|||
return m_socket->eof();
|
||||
}
|
||||
|
||||
bool HttpsJob::write(const ByteBuffer& data)
|
||||
bool HttpsJob::write(ReadonlyBytes data)
|
||||
{
|
||||
return m_socket->write(data);
|
||||
}
|
||||
|
|
|
@ -38,8 +38,8 @@ namespace HTTP {
|
|||
class HttpsJob final : public Job {
|
||||
C_OBJECT(HttpsJob)
|
||||
public:
|
||||
explicit HttpsJob(const HttpRequest& request, const Vector<Certificate>* override_certs = nullptr)
|
||||
: Job(request)
|
||||
explicit HttpsJob(const HttpRequest& request, OutputStream& output_stream, const Vector<Certificate>* override_certs = nullptr)
|
||||
: Job(request, output_stream)
|
||||
, m_override_ca_certificates(override_certs)
|
||||
{
|
||||
}
|
||||
|
@ -62,7 +62,7 @@ protected:
|
|||
virtual bool can_read() const override;
|
||||
virtual ByteBuffer receive(size_t) override;
|
||||
virtual bool eof() const override;
|
||||
virtual bool write(const ByteBuffer&) override;
|
||||
virtual bool write(ReadonlyBytes) override;
|
||||
virtual bool is_established() const override { return m_socket->is_established(); }
|
||||
virtual bool should_fail_on_empty_payload() const override { return false; }
|
||||
virtual void read_while_data_available(Function<IterationDecision()>) override;
|
||||
|
|
|
@ -68,8 +68,9 @@ static ByteBuffer handle_content_encoding(const ByteBuffer& buf, const String& c
|
|||
return buf;
|
||||
}
|
||||
|
||||
Job::Job(const HttpRequest& request)
|
||||
: m_request(request)
|
||||
Job::Job(const HttpRequest& request, OutputStream& output_stream)
|
||||
: Core::NetworkJob(output_stream)
|
||||
, m_request(request)
|
||||
{
|
||||
}
|
||||
|
||||
|
@ -77,6 +78,35 @@ Job::~Job()
|
|||
{
|
||||
}
|
||||
|
||||
void Job::flush_received_buffers()
|
||||
{
|
||||
if (!m_can_stream_response || m_buffered_size == 0)
|
||||
return;
|
||||
#ifdef JOB_DEBUG
|
||||
dbg() << "Job: Flushing received buffers: have " << m_buffered_size << " bytes in " << m_received_buffers.size() << " buffers";
|
||||
#endif
|
||||
for (size_t i = 0; i < m_received_buffers.size(); ++i) {
|
||||
auto& payload = m_received_buffers[i];
|
||||
auto written = do_write(payload);
|
||||
m_buffered_size -= written;
|
||||
if (written == payload.size()) {
|
||||
// FIXME: Make this a take-first-friendly object?
|
||||
m_received_buffers.take_first();
|
||||
--i;
|
||||
continue;
|
||||
}
|
||||
ASSERT(written < payload.size());
|
||||
payload = payload.slice(written, payload.size() - written);
|
||||
#ifdef JOB_DEBUG
|
||||
dbg() << "Job: Flushing received buffers done: have " << m_buffered_size << " bytes in " << m_received_buffers.size() << " buffers";
|
||||
#endif
|
||||
return;
|
||||
}
|
||||
#ifdef JOB_DEBUG
|
||||
dbg() << "Job: Flushing received buffers done: have " << m_buffered_size << " bytes in " << m_received_buffers.size() << " buffers";
|
||||
#endif
|
||||
}
|
||||
|
||||
void Job::on_socket_connected()
|
||||
{
|
||||
register_on_ready_to_write([&] {
|
||||
|
@ -135,6 +165,8 @@ void Job::on_socket_connected()
|
|||
if (m_state == State::Trailers) {
|
||||
return finish_up();
|
||||
} else {
|
||||
if (on_headers_received)
|
||||
on_headers_received(m_headers, m_code > 0 ? m_code : Optional<u32> {});
|
||||
m_state = State::InBody;
|
||||
}
|
||||
return;
|
||||
|
@ -163,6 +195,13 @@ void Job::on_socket_connected()
|
|||
}
|
||||
auto value = line.substring(name.length() + 2, line.length() - name.length() - 2);
|
||||
m_headers.set(name, value);
|
||||
if (name.equals_ignoring_case("Content-Encoding")) {
|
||||
// Assume that any content-encoding means that we can't decode it as a stream :(
|
||||
#ifdef JOB_DEBUG
|
||||
dbg() << "Content-Encoding " << value << " detected, cannot stream output :(";
|
||||
#endif
|
||||
m_can_stream_response = false;
|
||||
}
|
||||
#ifdef JOB_DEBUG
|
||||
dbg() << "Job: [" << name << "] = '" << value << "'";
|
||||
#endif
|
||||
|
@ -252,7 +291,9 @@ void Job::on_socket_connected()
|
|||
}
|
||||
|
||||
m_received_buffers.append(payload);
|
||||
m_buffered_size += payload.size();
|
||||
m_received_size += payload.size();
|
||||
flush_received_buffers();
|
||||
|
||||
if (m_current_chunk_remaining_size.has_value()) {
|
||||
auto size = m_current_chunk_remaining_size.value() - payload.size();
|
||||
|
@ -313,20 +354,37 @@ void Job::on_socket_connected()
|
|||
void Job::finish_up()
|
||||
{
|
||||
m_state = State::Finished;
|
||||
auto flattened_buffer = ByteBuffer::create_uninitialized(m_received_size);
|
||||
u8* flat_ptr = flattened_buffer.data();
|
||||
for (auto& received_buffer : m_received_buffers) {
|
||||
memcpy(flat_ptr, received_buffer.data(), received_buffer.size());
|
||||
flat_ptr += received_buffer.size();
|
||||
}
|
||||
m_received_buffers.clear();
|
||||
if (!m_can_stream_response) {
|
||||
auto flattened_buffer = ByteBuffer::create_uninitialized(m_received_size);
|
||||
u8* flat_ptr = flattened_buffer.data();
|
||||
for (auto& received_buffer : m_received_buffers) {
|
||||
memcpy(flat_ptr, received_buffer.data(), received_buffer.size());
|
||||
flat_ptr += received_buffer.size();
|
||||
}
|
||||
m_received_buffers.clear();
|
||||
|
||||
auto content_encoding = m_headers.get("Content-Encoding");
|
||||
if (content_encoding.has_value()) {
|
||||
flattened_buffer = handle_content_encoding(flattened_buffer, content_encoding.value());
|
||||
// For the time being, we cannot stream stuff with content-encoding set to _anything_.
|
||||
auto content_encoding = m_headers.get("Content-Encoding");
|
||||
if (content_encoding.has_value()) {
|
||||
flattened_buffer = handle_content_encoding(flattened_buffer, content_encoding.value());
|
||||
}
|
||||
|
||||
m_buffered_size = flattened_buffer.size();
|
||||
m_received_buffers.append(move(flattened_buffer));
|
||||
m_can_stream_response = true;
|
||||
}
|
||||
|
||||
auto response = HttpResponse::create(m_code, move(m_headers), move(flattened_buffer));
|
||||
flush_received_buffers();
|
||||
if (m_buffered_size != 0) {
|
||||
// FIXME: What do we do? ignore it?
|
||||
// "Transmission failed" is not strictly correct, but let's roll with it for now.
|
||||
deferred_invoke([this](auto&) {
|
||||
did_fail(Error::TransmissionFailed);
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
auto response = HttpResponse::create(m_code, move(m_headers));
|
||||
deferred_invoke([this, response](auto&) {
|
||||
did_finish(move(response));
|
||||
});
|
||||
|
|
|
@ -26,6 +26,7 @@
|
|||
|
||||
#pragma once
|
||||
|
||||
#include <AK/FileStream.h>
|
||||
#include <AK/HashMap.h>
|
||||
#include <AK/Optional.h>
|
||||
#include <LibCore/NetworkJob.h>
|
||||
|
@ -37,7 +38,7 @@ namespace HTTP {
|
|||
|
||||
class Job : public Core::NetworkJob {
|
||||
public:
|
||||
explicit Job(const HttpRequest&);
|
||||
explicit Job(const HttpRequest&, OutputStream&);
|
||||
virtual ~Job() override;
|
||||
|
||||
virtual void start() override = 0;
|
||||
|
@ -49,6 +50,7 @@ public:
|
|||
protected:
|
||||
void finish_up();
|
||||
void on_socket_connected();
|
||||
void flush_received_buffers();
|
||||
virtual void register_on_ready_to_read(Function<void()>) = 0;
|
||||
virtual void register_on_ready_to_write(Function<void()>) = 0;
|
||||
virtual bool can_read_line() const = 0;
|
||||
|
@ -56,7 +58,7 @@ protected:
|
|||
virtual bool can_read() const = 0;
|
||||
virtual ByteBuffer receive(size_t) = 0;
|
||||
virtual bool eof() const = 0;
|
||||
virtual bool write(const ByteBuffer&) = 0;
|
||||
virtual bool write(ReadonlyBytes) = 0;
|
||||
virtual bool is_established() const = 0;
|
||||
virtual bool should_fail_on_empty_payload() const { return true; }
|
||||
virtual void read_while_data_available(Function<IterationDecision()> read) { read(); };
|
||||
|
@ -73,11 +75,13 @@ protected:
|
|||
State m_state { State::InStatus };
|
||||
int m_code { -1 };
|
||||
HashMap<String, String, CaseInsensitiveStringTraits> m_headers;
|
||||
Vector<ByteBuffer> m_received_buffers;
|
||||
Vector<ByteBuffer, 2> m_received_buffers;
|
||||
size_t m_buffered_size { 0 };
|
||||
size_t m_received_size { 0 };
|
||||
bool m_sent_data { 0 };
|
||||
Optional<ssize_t> m_current_chunk_remaining_size;
|
||||
Optional<size_t> m_current_chunk_total_size;
|
||||
bool m_can_stream_response { true };
|
||||
};
|
||||
|
||||
}
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue