1
Fork 0
mirror of https://github.com/RGBCube/serenity synced 2025-07-25 20:07:35 +00:00

LibCompress: Port DeflateCompressor to Core::Stream

This commit is contained in:
Tim Schumacher 2022-12-27 13:49:42 +01:00 committed by Andreas Kling
parent a212bc3052
commit 8cd2cf2b77
7 changed files with 107 additions and 97 deletions

View file

@ -118,7 +118,7 @@ TEST_CASE(deflate_round_trip_store)
auto original = ByteBuffer::create_uninitialized(1024).release_value(); auto original = ByteBuffer::create_uninitialized(1024).release_value();
fill_with_random(original.data(), 1024); fill_with_random(original.data(), 1024);
auto compressed = Compress::DeflateCompressor::compress_all(original, Compress::DeflateCompressor::CompressionLevel::STORE); auto compressed = Compress::DeflateCompressor::compress_all(original, Compress::DeflateCompressor::CompressionLevel::STORE);
EXPECT(compressed.has_value()); EXPECT(!compressed.is_error());
auto uncompressed = Compress::DeflateDecompressor::decompress_all(compressed.value()); auto uncompressed = Compress::DeflateDecompressor::decompress_all(compressed.value());
EXPECT(!uncompressed.is_error()); EXPECT(!uncompressed.is_error());
EXPECT(uncompressed.value() == original); EXPECT(uncompressed.value() == original);
@ -130,7 +130,7 @@ TEST_CASE(deflate_round_trip_compress)
fill_with_random(original.data(), 1024); // we pre-filled the second half with 0s to make sure we test back references as well fill_with_random(original.data(), 1024); // we pre-filled the second half with 0s to make sure we test back references as well
// Since the different levels just change how much time is spent looking for better matches, just use fast here to reduce test time // Since the different levels just change how much time is spent looking for better matches, just use fast here to reduce test time
auto compressed = Compress::DeflateCompressor::compress_all(original, Compress::DeflateCompressor::CompressionLevel::FAST); auto compressed = Compress::DeflateCompressor::compress_all(original, Compress::DeflateCompressor::CompressionLevel::FAST);
EXPECT(compressed.has_value()); EXPECT(!compressed.is_error());
auto uncompressed = Compress::DeflateDecompressor::decompress_all(compressed.value()); auto uncompressed = Compress::DeflateDecompressor::decompress_all(compressed.value());
EXPECT(!uncompressed.is_error()); EXPECT(!uncompressed.is_error());
EXPECT(uncompressed.value() == original); EXPECT(uncompressed.value() == original);
@ -143,7 +143,7 @@ TEST_CASE(deflate_round_trip_compress_large)
fill_with_random(original.data(), size); fill_with_random(original.data(), size);
// Since the different levels just change how much time is spent looking for better matches, just use fast here to reduce test time // Since the different levels just change how much time is spent looking for better matches, just use fast here to reduce test time
auto compressed = Compress::DeflateCompressor::compress_all(original, Compress::DeflateCompressor::CompressionLevel::FAST); auto compressed = Compress::DeflateCompressor::compress_all(original, Compress::DeflateCompressor::CompressionLevel::FAST);
EXPECT(compressed.has_value()); EXPECT(!compressed.is_error());
auto uncompressed = Compress::DeflateDecompressor::decompress_all(compressed.value()); auto uncompressed = Compress::DeflateDecompressor::decompress_all(compressed.value());
EXPECT(!uncompressed.is_error()); EXPECT(!uncompressed.is_error());
EXPECT(uncompressed.value() == original); EXPECT(uncompressed.value() == original);
@ -154,5 +154,5 @@ TEST_CASE(deflate_compress_literals)
// This byte array is known to not produce any back references with our lz77 implementation even at the highest compression settings // This byte array is known to not produce any back references with our lz77 implementation even at the highest compression settings
Array<u8, 0x13> test { 0, 0, 0, 0, 0x72, 0, 0, 0xee, 0, 0, 0, 0x26, 0, 0, 0, 0x28, 0, 0, 0x72 }; Array<u8, 0x13> test { 0, 0, 0, 0, 0x72, 0, 0, 0xee, 0, 0, 0, 0x26, 0, 0, 0, 0x28, 0, 0, 0x72 };
auto compressed = Compress::DeflateCompressor::compress_all(test, Compress::DeflateCompressor::CompressionLevel::GOOD); auto compressed = Compress::DeflateCompressor::compress_all(test, Compress::DeflateCompressor::CompressionLevel::GOOD);
EXPECT(compressed.has_value()); EXPECT(!compressed.is_error());
} }

View file

@ -116,9 +116,10 @@ ErrorOr<u32> CanonicalCode::read_symbol(Core::Stream::LittleEndianInputBitStream
} }
} }
void CanonicalCode::write_symbol(OutputBitStream& stream, u32 symbol) const ErrorOr<void> CanonicalCode::write_symbol(Core::Stream::LittleEndianOutputBitStream& stream, u32 symbol) const
{ {
stream.write_bits(m_bit_codes[symbol], m_bit_code_lengths[symbol]); TRY(stream.write_bits(m_bit_codes[symbol], m_bit_code_lengths[symbol]));
return {};
} }
DeflateDecompressor::CompressedBlock::CompressedBlock(DeflateDecompressor& decompressor, CanonicalCode literal_codes, Optional<CanonicalCode> distance_codes) DeflateDecompressor::CompressedBlock::CompressedBlock(DeflateDecompressor& decompressor, CanonicalCode literal_codes, Optional<CanonicalCode> distance_codes)
@ -437,10 +438,10 @@ ErrorOr<void> DeflateDecompressor::decode_codes(CanonicalCode& literal_code, Opt
return {}; return {};
} }
DeflateCompressor::DeflateCompressor(OutputStream& stream, CompressionLevel compression_level) DeflateCompressor::DeflateCompressor(Core::Stream::Handle<Core::Stream::Stream> stream, CompressionLevel compression_level)
: m_compression_level(compression_level) : m_compression_level(compression_level)
, m_compression_constants(compression_constants[static_cast<int>(m_compression_level)]) , m_compression_constants(compression_constants[static_cast<int>(m_compression_level)])
, m_output_stream(stream) , m_output_stream(Core::Stream::LittleEndianOutputBitStream::construct(move(stream)).release_value_but_fixme_should_propagate_errors())
{ {
m_symbol_frequencies.fill(0); m_symbol_frequencies.fill(0);
m_distance_frequencies.fill(0); m_distance_frequencies.fill(0);
@ -451,7 +452,12 @@ DeflateCompressor::~DeflateCompressor()
VERIFY(m_finished); VERIFY(m_finished);
} }
size_t DeflateCompressor::write(ReadonlyBytes bytes) ErrorOr<Bytes> DeflateCompressor::read(Bytes)
{
return Error::from_errno(EBADF);
}
ErrorOr<size_t> DeflateCompressor::write(ReadonlyBytes bytes)
{ {
VERIFY(!m_finished); VERIFY(!m_finished);
@ -462,21 +468,25 @@ size_t DeflateCompressor::write(ReadonlyBytes bytes)
m_pending_block_size += n_written; m_pending_block_size += n_written;
if (m_pending_block_size == block_size) if (m_pending_block_size == block_size)
flush(); TRY(flush());
return n_written + write(bytes.slice(n_written)); return n_written + TRY(write(bytes.slice(n_written)));
} }
bool DeflateCompressor::write_or_error(ReadonlyBytes bytes) bool DeflateCompressor::is_eof() const
{ {
if (write(bytes) < bytes.size()) {
set_fatal_error();
return false;
}
return true; return true;
} }
bool DeflateCompressor::is_open() const
{
return m_output_stream->is_open();
}
void DeflateCompressor::close()
{
}
// Knuth's multiplicative hash on 4 bytes // Knuth's multiplicative hash on 4 bytes
u16 DeflateCompressor::hash_sequence(u8 const* bytes) u16 DeflateCompressor::hash_sequence(u8 const* bytes)
{ {
@ -730,7 +740,7 @@ size_t DeflateCompressor::huffman_block_length(Array<u8, max_huffman_literals> c
size_t DeflateCompressor::uncompressed_block_length() size_t DeflateCompressor::uncompressed_block_length()
{ {
auto padding = 8 - ((m_output_stream.bit_offset() + 3) % 8); auto padding = 8 - ((m_output_stream->bit_offset() + 3) % 8);
// 3 bit block header + align to byte + 2 * 16 bit length fields + block contents // 3 bit block header + align to byte + 2 * 16 bit length fields + block contents
return 3 + padding + (2 * 16) + m_pending_block_size * 8; return 3 + padding + (2 * 16) + m_pending_block_size * 8;
} }
@ -765,25 +775,26 @@ size_t DeflateCompressor::dynamic_block_length(Array<u8, max_huffman_literals> c
return length + huffman_block_length(literal_bit_lengths, distance_bit_lengths); return length + huffman_block_length(literal_bit_lengths, distance_bit_lengths);
} }
void DeflateCompressor::write_huffman(CanonicalCode const& literal_code, Optional<CanonicalCode> const& distance_code) ErrorOr<void> DeflateCompressor::write_huffman(CanonicalCode const& literal_code, Optional<CanonicalCode> const& distance_code)
{ {
auto has_distances = distance_code.has_value(); auto has_distances = distance_code.has_value();
for (size_t i = 0; i < m_pending_symbol_size; i++) { for (size_t i = 0; i < m_pending_symbol_size; i++) {
if (m_symbol_buffer[i].distance == 0) { if (m_symbol_buffer[i].distance == 0) {
literal_code.write_symbol(m_output_stream, m_symbol_buffer[i].literal); TRY(literal_code.write_symbol(*m_output_stream, m_symbol_buffer[i].literal));
continue; continue;
} }
VERIFY(has_distances); VERIFY(has_distances);
auto symbol = length_to_symbol[m_symbol_buffer[i].length]; auto symbol = length_to_symbol[m_symbol_buffer[i].length];
literal_code.write_symbol(m_output_stream, symbol); TRY(literal_code.write_symbol(*m_output_stream, symbol));
// Emit extra bits if needed // Emit extra bits if needed
m_output_stream.write_bits(m_symbol_buffer[i].length - packed_length_symbols[symbol - 257].base_length, packed_length_symbols[symbol - 257].extra_bits); TRY(m_output_stream->write_bits<u16>(m_symbol_buffer[i].length - packed_length_symbols[symbol - 257].base_length, packed_length_symbols[symbol - 257].extra_bits));
auto base_distance = distance_to_base(m_symbol_buffer[i].distance); auto base_distance = distance_to_base(m_symbol_buffer[i].distance);
distance_code.value().write_symbol(m_output_stream, base_distance); TRY(distance_code.value().write_symbol(*m_output_stream, base_distance));
// Emit extra bits if needed // Emit extra bits if needed
m_output_stream.write_bits(m_symbol_buffer[i].distance - packed_distances[base_distance].base_distance, packed_distances[base_distance].extra_bits); TRY(m_output_stream->write_bits<u16>(m_symbol_buffer[i].distance - packed_distances[base_distance].base_distance, packed_distances[base_distance].extra_bits));
} }
return {};
} }
size_t DeflateCompressor::encode_huffman_lengths(Array<u8, max_huffman_literals + max_huffman_distances> const& lengths, size_t lengths_count, Array<code_length_symbol, max_huffman_literals + max_huffman_distances>& encoded_lengths) size_t DeflateCompressor::encode_huffman_lengths(Array<u8, max_huffman_literals + max_huffman_distances> const& lengths, size_t lengths_count, Array<code_length_symbol, max_huffman_literals + max_huffman_distances>& encoded_lengths)
@ -854,65 +865,62 @@ size_t DeflateCompressor::encode_block_lengths(Array<u8, max_huffman_literals> c
return encode_huffman_lengths(all_lengths, lengths_count, encoded_lengths); return encode_huffman_lengths(all_lengths, lengths_count, encoded_lengths);
} }
void DeflateCompressor::write_dynamic_huffman(CanonicalCode const& literal_code, size_t literal_code_count, Optional<CanonicalCode> const& distance_code, size_t distance_code_count, Array<u8, 19> const& code_lengths_bit_lengths, size_t code_length_count, Array<code_length_symbol, max_huffman_literals + max_huffman_distances> const& encoded_lengths, size_t encoded_lengths_count) ErrorOr<void> DeflateCompressor::write_dynamic_huffman(CanonicalCode const& literal_code, size_t literal_code_count, Optional<CanonicalCode> const& distance_code, size_t distance_code_count, Array<u8, 19> const& code_lengths_bit_lengths, size_t code_length_count, Array<code_length_symbol, max_huffman_literals + max_huffman_distances> const& encoded_lengths, size_t encoded_lengths_count)
{ {
m_output_stream.write_bits(literal_code_count - 257, 5); TRY(m_output_stream->write_bits(literal_code_count - 257, 5));
m_output_stream.write_bits(distance_code_count - 1, 5); TRY(m_output_stream->write_bits(distance_code_count - 1, 5));
m_output_stream.write_bits(code_length_count - 4, 4); TRY(m_output_stream->write_bits(code_length_count - 4, 4));
for (size_t i = 0; i < code_length_count; i++) { for (size_t i = 0; i < code_length_count; i++) {
m_output_stream.write_bits(code_lengths_bit_lengths[code_lengths_code_lengths_order[i]], 3); TRY(m_output_stream->write_bits(code_lengths_bit_lengths[code_lengths_code_lengths_order[i]], 3));
} }
auto code_lengths_code = CanonicalCode::from_bytes(code_lengths_bit_lengths); auto code_lengths_code = CanonicalCode::from_bytes(code_lengths_bit_lengths);
VERIFY(code_lengths_code.has_value()); VERIFY(code_lengths_code.has_value());
for (size_t i = 0; i < encoded_lengths_count; i++) { for (size_t i = 0; i < encoded_lengths_count; i++) {
auto encoded_length = encoded_lengths[i]; auto encoded_length = encoded_lengths[i];
code_lengths_code->write_symbol(m_output_stream, encoded_length.symbol); TRY(code_lengths_code->write_symbol(*m_output_stream, encoded_length.symbol));
if (encoded_length.symbol == deflate_special_code_length_copy) { if (encoded_length.symbol == deflate_special_code_length_copy) {
m_output_stream.write_bits(encoded_length.count - 3, 2); TRY(m_output_stream->write_bits<u8>(encoded_length.count - 3, 2));
} else if (encoded_length.symbol == deflate_special_code_length_zeros) { } else if (encoded_length.symbol == deflate_special_code_length_zeros) {
m_output_stream.write_bits(encoded_length.count - 3, 3); TRY(m_output_stream->write_bits<u8>(encoded_length.count - 3, 3));
} else if (encoded_length.symbol == deflate_special_code_length_long_zeros) { } else if (encoded_length.symbol == deflate_special_code_length_long_zeros) {
m_output_stream.write_bits(encoded_length.count - 11, 7); TRY(m_output_stream->write_bits<u8>(encoded_length.count - 11, 7));
} }
} }
write_huffman(literal_code, distance_code); TRY(write_huffman(literal_code, distance_code));
return {};
} }
void DeflateCompressor::flush() ErrorOr<void> DeflateCompressor::flush()
{ {
if (m_output_stream.handle_any_error()) { TRY(m_output_stream->write_bits(m_finished, 1));
set_fatal_error();
return;
}
m_output_stream.write_bit(m_finished);
// if this is just an empty block to signify the end of the deflate stream use the smallest block possible (10 bits total) // if this is just an empty block to signify the end of the deflate stream use the smallest block possible (10 bits total)
if (m_pending_block_size == 0) { if (m_pending_block_size == 0) {
VERIFY(m_finished); // we shouldn't be writing empty blocks unless this is the final one VERIFY(m_finished); // we shouldn't be writing empty blocks unless this is the final one
m_output_stream.write_bits(0b01, 2); // fixed huffman codes TRY(m_output_stream->write_bits(0b01u, 2)); // fixed huffman codes
m_output_stream.write_bits(0b0000000, 7); // end of block symbol TRY(m_output_stream->write_bits(0b0000000u, 7)); // end of block symbol
m_output_stream.align_to_byte_boundary(); TRY(m_output_stream->align_to_byte_boundary());
return; return {};
} }
auto write_uncompressed = [&]() { auto write_uncompressed = [&]() -> ErrorOr<void> {
m_output_stream.write_bits(0b00, 2); // no compression TRY(m_output_stream->write_bits(0b00u, 2)); // no compression
m_output_stream.align_to_byte_boundary(); TRY(m_output_stream->align_to_byte_boundary());
LittleEndian<u16> len = m_pending_block_size; LittleEndian<u16> len = m_pending_block_size;
m_output_stream << len; TRY(m_output_stream->write_entire_buffer(len.bytes()));
LittleEndian<u16> nlen = ~m_pending_block_size; LittleEndian<u16> nlen = ~m_pending_block_size;
m_output_stream << nlen; TRY(m_output_stream->write_entire_buffer(nlen.bytes()));
m_output_stream.write_or_error(pending_block().slice(0, m_pending_block_size)); TRY(m_output_stream->write_entire_buffer(pending_block().slice(0, m_pending_block_size)));
return {};
}; };
if (m_compression_level == CompressionLevel::STORE) { // disabled compression fast path if (m_compression_level == CompressionLevel::STORE) { // disabled compression fast path
write_uncompressed(); TRY(write_uncompressed());
m_pending_block_size = 0; m_pending_block_size = 0;
return; return {};
} }
// The following implementation of lz77 compression and huffman encoding is based on the reference implementation by Hans Wennborg https://www.hanshq.net/zip.html // The following implementation of lz77 compression and huffman encoding is based on the reference implementation by Hans Wennborg https://www.hanshq.net/zip.html
@ -956,19 +964,21 @@ void DeflateCompressor::flush()
// If the compression somehow didn't reduce the size enough, just write out the block uncompressed as it allows for much faster decompression // If the compression somehow didn't reduce the size enough, just write out the block uncompressed as it allows for much faster decompression
if (uncompressed_size <= min(fixed_huffman_size, dynamic_huffman_size)) { if (uncompressed_size <= min(fixed_huffman_size, dynamic_huffman_size)) {
write_uncompressed(); TRY(write_uncompressed());
} else if (fixed_huffman_size <= dynamic_huffman_size) { // If the fixed and dynamic huffman codes come out the same size, prefer the fixed version, as it takes less time to decode } else if (fixed_huffman_size <= dynamic_huffman_size) {
m_output_stream.write_bits(0b01, 2); // fixed huffman codes // If the fixed and dynamic huffman codes come out the same size, prefer the fixed version, as it takes less time to decode fixed huffman codes.
write_huffman(CanonicalCode::fixed_literal_codes(), CanonicalCode::fixed_distance_codes()); TRY(m_output_stream->write_bits(0b01u, 2));
TRY(write_huffman(CanonicalCode::fixed_literal_codes(), CanonicalCode::fixed_distance_codes()));
} else { } else {
m_output_stream.write_bits(0b10, 2); // dynamic huffman codes // dynamic huffman codes
TRY(m_output_stream->write_bits(0b10u, 2));
auto literal_code = CanonicalCode::from_bytes(dynamic_literal_bit_lengths); auto literal_code = CanonicalCode::from_bytes(dynamic_literal_bit_lengths);
VERIFY(literal_code.has_value()); VERIFY(literal_code.has_value());
auto distance_code = CanonicalCode::from_bytes(dynamic_distance_bit_lengths); auto distance_code = CanonicalCode::from_bytes(dynamic_distance_bit_lengths);
write_dynamic_huffman(literal_code.value(), literal_code_count, distance_code, distance_code_count, code_lengths_bit_lengths, code_lengths_count, encoded_lengths, encoded_lengths_count); TRY(write_dynamic_huffman(literal_code.value(), literal_code_count, distance_code, distance_code_count, code_lengths_bit_lengths, code_lengths_count, encoded_lengths, encoded_lengths_count));
} }
if (m_finished) if (m_finished)
m_output_stream.align_to_byte_boundary(); TRY(m_output_stream->align_to_byte_boundary());
// reset all block specific members // reset all block specific members
m_pending_block_size = 0; m_pending_block_size = 0;
@ -977,28 +987,30 @@ void DeflateCompressor::flush()
m_distance_frequencies.fill(0); m_distance_frequencies.fill(0);
// On the final block this copy will potentially produce an invalid search window, but since its the final block we dont care // On the final block this copy will potentially produce an invalid search window, but since its the final block we dont care
pending_block().copy_trimmed_to({ m_rolling_window, block_size }); pending_block().copy_trimmed_to({ m_rolling_window, block_size });
return {};
} }
void DeflateCompressor::final_flush() ErrorOr<void> DeflateCompressor::final_flush()
{ {
VERIFY(!m_finished); VERIFY(!m_finished);
m_finished = true; m_finished = true;
flush(); TRY(flush());
return {};
} }
Optional<ByteBuffer> DeflateCompressor::compress_all(ReadonlyBytes bytes, CompressionLevel compression_level) ErrorOr<ByteBuffer> DeflateCompressor::compress_all(ReadonlyBytes bytes, CompressionLevel compression_level)
{ {
DuplexMemoryStream output_stream; auto output_stream = TRY(try_make<Core::Stream::AllocatingMemoryStream>());
DeflateCompressor deflate_stream { output_stream, compression_level }; DeflateCompressor deflate_stream { Core::Stream::Handle<Core::Stream::Stream>(*output_stream), compression_level };
deflate_stream.write_or_error(bytes); TRY(deflate_stream.write_entire_buffer(bytes));
TRY(deflate_stream.final_flush());
deflate_stream.final_flush(); auto buffer = TRY(ByteBuffer::create_uninitialized(output_stream->used_buffer_size()));
TRY(output_stream->read_entire_buffer(buffer));
if (deflate_stream.handle_any_error()) return buffer;
return {};
return output_stream.copy_into_contiguous_buffer();
} }
} }

View file

@ -22,7 +22,7 @@ class CanonicalCode {
public: public:
CanonicalCode() = default; CanonicalCode() = default;
ErrorOr<u32> read_symbol(Core::Stream::LittleEndianInputBitStream&) const; ErrorOr<u32> read_symbol(Core::Stream::LittleEndianInputBitStream&) const;
void write_symbol(OutputBitStream&, u32) const; ErrorOr<void> write_symbol(Core::Stream::LittleEndianOutputBitStream&, u32) const;
static CanonicalCode const& fixed_literal_codes(); static CanonicalCode const& fixed_literal_codes();
static CanonicalCode const& fixed_distance_codes(); static CanonicalCode const& fixed_distance_codes();
@ -104,7 +104,7 @@ private:
CircularDuplexStream<32 * KiB> m_output_stream; CircularDuplexStream<32 * KiB> m_output_stream;
}; };
class DeflateCompressor final : public OutputStream { class DeflateCompressor final : public Core::Stream::Stream {
public: public:
static constexpr size_t block_size = 32 * KiB - 1; // TODO: this can theoretically be increased to 64 KiB - 2 static constexpr size_t block_size = 32 * KiB - 1; // TODO: this can theoretically be increased to 64 KiB - 2
static constexpr size_t window_size = block_size * 2; static constexpr size_t window_size = block_size * 2;
@ -139,14 +139,17 @@ public:
BEST // WARNING: this one can take an unreasonable amount of time! BEST // WARNING: this one can take an unreasonable amount of time!
}; };
DeflateCompressor(OutputStream&, CompressionLevel = CompressionLevel::GOOD); DeflateCompressor(Core::Stream::Handle<Core::Stream::Stream>, CompressionLevel = CompressionLevel::GOOD);
~DeflateCompressor(); ~DeflateCompressor();
size_t write(ReadonlyBytes) override; virtual ErrorOr<Bytes> read(Bytes) override;
bool write_or_error(ReadonlyBytes) override; virtual ErrorOr<size_t> write(ReadonlyBytes) override;
void final_flush(); virtual bool is_eof() const override;
virtual bool is_open() const override;
virtual void close() override;
ErrorOr<void> final_flush();
static Optional<ByteBuffer> compress_all(ReadonlyBytes bytes, CompressionLevel = CompressionLevel::GOOD); static ErrorOr<ByteBuffer> compress_all(ReadonlyBytes bytes, CompressionLevel = CompressionLevel::GOOD);
private: private:
Bytes pending_block() { return { m_rolling_window + block_size, block_size }; } Bytes pending_block() { return { m_rolling_window + block_size, block_size }; }
@ -166,20 +169,20 @@ private:
template<size_t Size> template<size_t Size>
static void generate_huffman_lengths(Array<u8, Size>& lengths, Array<u16, Size> const& frequencies, size_t max_bit_length, u16 frequency_cap = UINT16_MAX); static void generate_huffman_lengths(Array<u8, Size>& lengths, Array<u16, Size> const& frequencies, size_t max_bit_length, u16 frequency_cap = UINT16_MAX);
size_t huffman_block_length(Array<u8, max_huffman_literals> const& literal_bit_lengths, Array<u8, max_huffman_distances> const& distance_bit_lengths); size_t huffman_block_length(Array<u8, max_huffman_literals> const& literal_bit_lengths, Array<u8, max_huffman_distances> const& distance_bit_lengths);
void write_huffman(CanonicalCode const& literal_code, Optional<CanonicalCode> const& distance_code); ErrorOr<void> write_huffman(CanonicalCode const& literal_code, Optional<CanonicalCode> const& distance_code);
static size_t encode_huffman_lengths(Array<u8, max_huffman_literals + max_huffman_distances> const& lengths, size_t lengths_count, Array<code_length_symbol, max_huffman_literals + max_huffman_distances>& encoded_lengths); static size_t encode_huffman_lengths(Array<u8, max_huffman_literals + max_huffman_distances> const& lengths, size_t lengths_count, Array<code_length_symbol, max_huffman_literals + max_huffman_distances>& encoded_lengths);
size_t encode_block_lengths(Array<u8, max_huffman_literals> const& literal_bit_lengths, Array<u8, max_huffman_distances> const& distance_bit_lengths, Array<code_length_symbol, max_huffman_literals + max_huffman_distances>& encoded_lengths, size_t& literal_code_count, size_t& distance_code_count); size_t encode_block_lengths(Array<u8, max_huffman_literals> const& literal_bit_lengths, Array<u8, max_huffman_distances> const& distance_bit_lengths, Array<code_length_symbol, max_huffman_literals + max_huffman_distances>& encoded_lengths, size_t& literal_code_count, size_t& distance_code_count);
void write_dynamic_huffman(CanonicalCode const& literal_code, size_t literal_code_count, Optional<CanonicalCode> const& distance_code, size_t distance_code_count, Array<u8, 19> const& code_lengths_bit_lengths, size_t code_length_count, Array<code_length_symbol, max_huffman_literals + max_huffman_distances> const& encoded_lengths, size_t encoded_lengths_count); ErrorOr<void> write_dynamic_huffman(CanonicalCode const& literal_code, size_t literal_code_count, Optional<CanonicalCode> const& distance_code, size_t distance_code_count, Array<u8, 19> const& code_lengths_bit_lengths, size_t code_length_count, Array<code_length_symbol, max_huffman_literals + max_huffman_distances> const& encoded_lengths, size_t encoded_lengths_count);
size_t uncompressed_block_length(); size_t uncompressed_block_length();
size_t fixed_block_length(); size_t fixed_block_length();
size_t dynamic_block_length(Array<u8, max_huffman_literals> const& literal_bit_lengths, Array<u8, max_huffman_distances> const& distance_bit_lengths, Array<u8, 19> const& code_lengths_bit_lengths, Array<u16, 19> const& code_lengths_frequencies, size_t code_lengths_count); size_t dynamic_block_length(Array<u8, max_huffman_literals> const& literal_bit_lengths, Array<u8, max_huffman_distances> const& distance_bit_lengths, Array<u8, 19> const& code_lengths_bit_lengths, Array<u16, 19> const& code_lengths_frequencies, size_t code_lengths_count);
void flush(); ErrorOr<void> flush();
bool m_finished { false }; bool m_finished { false };
CompressionLevel m_compression_level; CompressionLevel m_compression_level;
CompressionConstants m_compression_constants; CompressionConstants m_compression_constants;
OutputBitStream m_output_stream; NonnullOwnPtr<Core::Stream::LittleEndianOutputBitStream> m_output_stream;
u8 m_rolling_window[window_size]; u8 m_rolling_window[window_size];
size_t m_pending_block_size { 0 }; size_t m_pending_block_size { 0 };

View file

@ -194,11 +194,9 @@ ErrorOr<size_t> GzipCompressor::write(ReadonlyBytes bytes)
header.extra_flags = 3; // DEFLATE sets 2 for maximum compression and 4 for minimum compression header.extra_flags = 3; // DEFLATE sets 2 for maximum compression and 4 for minimum compression
header.operating_system = 3; // unix header.operating_system = 3; // unix
TRY(m_output_stream->write_entire_buffer({ &header, sizeof(header) })); TRY(m_output_stream->write_entire_buffer({ &header, sizeof(header) }));
Core::Stream::WrapInAKOutputStream wrapped_stream { *m_output_stream }; DeflateCompressor compressed_stream { Core::Stream::Handle(*m_output_stream) };
DeflateCompressor compressed_stream { wrapped_stream }; TRY(compressed_stream.write_entire_buffer(bytes));
if (!compressed_stream.write_or_error(bytes)) TRY(compressed_stream.final_flush());
return Error::from_string_literal("Underlying DeflateCompressor indicated an error");
compressed_stream.final_flush();
Crypto::Checksum::CRC32 crc32; Crypto::Checksum::CRC32 crc32;
crc32.update(bytes); crc32.update(bytes);
LittleEndian<u32> digest = crc32.digest(); LittleEndian<u32> digest = crc32.digest();

View file

@ -81,10 +81,9 @@ ErrorOr<NonnullOwnPtr<ZlibCompressor>> ZlibCompressor::construct(Core::Stream::H
} }
ZlibCompressor::ZlibCompressor(Core::Stream::Handle<Core::Stream::Stream> stream, ZlibCompressionLevel compression_level) ZlibCompressor::ZlibCompressor(Core::Stream::Handle<Core::Stream::Stream> stream, ZlibCompressionLevel compression_level)
: m_ak_output_stream(make<Core::Stream::WrapInAKOutputStream>(*stream)) : m_output_stream(move(stream))
, m_output_stream(move(stream))
// FIXME: Find a way to compress with Deflate's "Best" compression level. // FIXME: Find a way to compress with Deflate's "Best" compression level.
, m_compressor(make<DeflateCompressor>(*m_ak_output_stream, static_cast<DeflateCompressor::CompressionLevel>(compression_level))) , m_compressor(make<DeflateCompressor>(Core::Stream::Handle(*m_output_stream), static_cast<DeflateCompressor::CompressionLevel>(compression_level)))
{ {
} }
@ -126,7 +125,7 @@ ErrorOr<size_t> ZlibCompressor::write(ReadonlyBytes bytes)
{ {
VERIFY(!m_finished); VERIFY(!m_finished);
size_t n_written = m_compressor->write(bytes); size_t n_written = TRY(m_compressor->write(bytes));
m_adler32_checksum.update(bytes.trim(n_written)); m_adler32_checksum.update(bytes.trim(n_written));
return n_written; return n_written;
} }
@ -150,7 +149,7 @@ ErrorOr<void> ZlibCompressor::finish()
VERIFY(!m_finished); VERIFY(!m_finished);
if (is<DeflateCompressor>(m_compressor.ptr())) if (is<DeflateCompressor>(m_compressor.ptr()))
static_cast<DeflateCompressor*>(m_compressor.ptr())->final_flush(); TRY(static_cast<DeflateCompressor*>(m_compressor.ptr())->final_flush());
NetworkOrdered<u32> adler_sum = m_adler32_checksum.digest(); NetworkOrdered<u32> adler_sum = m_adler32_checksum.digest();
TRY(m_output_stream->write(adler_sum.bytes())); TRY(m_output_stream->write(adler_sum.bytes()));

View file

@ -80,10 +80,8 @@ private:
ErrorOr<void> write_header(ZlibCompressionMethod, ZlibCompressionLevel); ErrorOr<void> write_header(ZlibCompressionMethod, ZlibCompressionLevel);
bool m_finished { false }; bool m_finished { false };
// FIXME: Remove this once DeflateCompressor is ported to Core::Stream.
NonnullOwnPtr<Core::Stream::WrapInAKOutputStream> m_ak_output_stream;
Core::Stream::Handle<Core::Stream::Stream> m_output_stream; Core::Stream::Handle<Core::Stream::Stream> m_output_stream;
NonnullOwnPtr<OutputStream> m_compressor; NonnullOwnPtr<Core::Stream::Stream> m_compressor;
Crypto::Checksum::Adler32 m_adler32_checksum; Crypto::Checksum::Adler32 m_adler32_checksum;
}; };

View file

@ -59,7 +59,7 @@ ErrorOr<int> serenity_main(Main::Arguments arguments)
member.name = TRY(String::from_deprecated_string(canonicalized_path)); member.name = TRY(String::from_deprecated_string(canonicalized_path));
auto deflate_buffer = Compress::DeflateCompressor::compress_all(file_buffer); auto deflate_buffer = Compress::DeflateCompressor::compress_all(file_buffer);
if (deflate_buffer.has_value() && deflate_buffer.value().size() < file_buffer.size()) { if (!deflate_buffer.is_error() && deflate_buffer.value().size() < file_buffer.size()) {
member.compressed_data = deflate_buffer.value().bytes(); member.compressed_data = deflate_buffer.value().bytes();
member.compression_method = Archive::ZipCompressionMethod::Deflate; member.compression_method = Archive::ZipCompressionMethod::Deflate;
auto compression_ratio = (double)deflate_buffer.value().size() / file_buffer.size(); auto compression_ratio = (double)deflate_buffer.value().size() / file_buffer.size();