mirror of
https://github.com/RGBCube/serenity
synced 2025-07-25 20:07:35 +00:00
LibCompress: Port DeflateCompressor
to Core::Stream
This commit is contained in:
parent
a212bc3052
commit
8cd2cf2b77
7 changed files with 107 additions and 97 deletions
|
@ -118,7 +118,7 @@ TEST_CASE(deflate_round_trip_store)
|
||||||
auto original = ByteBuffer::create_uninitialized(1024).release_value();
|
auto original = ByteBuffer::create_uninitialized(1024).release_value();
|
||||||
fill_with_random(original.data(), 1024);
|
fill_with_random(original.data(), 1024);
|
||||||
auto compressed = Compress::DeflateCompressor::compress_all(original, Compress::DeflateCompressor::CompressionLevel::STORE);
|
auto compressed = Compress::DeflateCompressor::compress_all(original, Compress::DeflateCompressor::CompressionLevel::STORE);
|
||||||
EXPECT(compressed.has_value());
|
EXPECT(!compressed.is_error());
|
||||||
auto uncompressed = Compress::DeflateDecompressor::decompress_all(compressed.value());
|
auto uncompressed = Compress::DeflateDecompressor::decompress_all(compressed.value());
|
||||||
EXPECT(!uncompressed.is_error());
|
EXPECT(!uncompressed.is_error());
|
||||||
EXPECT(uncompressed.value() == original);
|
EXPECT(uncompressed.value() == original);
|
||||||
|
@ -130,7 +130,7 @@ TEST_CASE(deflate_round_trip_compress)
|
||||||
fill_with_random(original.data(), 1024); // we pre-filled the second half with 0s to make sure we test back references as well
|
fill_with_random(original.data(), 1024); // we pre-filled the second half with 0s to make sure we test back references as well
|
||||||
// Since the different levels just change how much time is spent looking for better matches, just use fast here to reduce test time
|
// Since the different levels just change how much time is spent looking for better matches, just use fast here to reduce test time
|
||||||
auto compressed = Compress::DeflateCompressor::compress_all(original, Compress::DeflateCompressor::CompressionLevel::FAST);
|
auto compressed = Compress::DeflateCompressor::compress_all(original, Compress::DeflateCompressor::CompressionLevel::FAST);
|
||||||
EXPECT(compressed.has_value());
|
EXPECT(!compressed.is_error());
|
||||||
auto uncompressed = Compress::DeflateDecompressor::decompress_all(compressed.value());
|
auto uncompressed = Compress::DeflateDecompressor::decompress_all(compressed.value());
|
||||||
EXPECT(!uncompressed.is_error());
|
EXPECT(!uncompressed.is_error());
|
||||||
EXPECT(uncompressed.value() == original);
|
EXPECT(uncompressed.value() == original);
|
||||||
|
@ -143,7 +143,7 @@ TEST_CASE(deflate_round_trip_compress_large)
|
||||||
fill_with_random(original.data(), size);
|
fill_with_random(original.data(), size);
|
||||||
// Since the different levels just change how much time is spent looking for better matches, just use fast here to reduce test time
|
// Since the different levels just change how much time is spent looking for better matches, just use fast here to reduce test time
|
||||||
auto compressed = Compress::DeflateCompressor::compress_all(original, Compress::DeflateCompressor::CompressionLevel::FAST);
|
auto compressed = Compress::DeflateCompressor::compress_all(original, Compress::DeflateCompressor::CompressionLevel::FAST);
|
||||||
EXPECT(compressed.has_value());
|
EXPECT(!compressed.is_error());
|
||||||
auto uncompressed = Compress::DeflateDecompressor::decompress_all(compressed.value());
|
auto uncompressed = Compress::DeflateDecompressor::decompress_all(compressed.value());
|
||||||
EXPECT(!uncompressed.is_error());
|
EXPECT(!uncompressed.is_error());
|
||||||
EXPECT(uncompressed.value() == original);
|
EXPECT(uncompressed.value() == original);
|
||||||
|
@ -154,5 +154,5 @@ TEST_CASE(deflate_compress_literals)
|
||||||
// This byte array is known to not produce any back references with our lz77 implementation even at the highest compression settings
|
// This byte array is known to not produce any back references with our lz77 implementation even at the highest compression settings
|
||||||
Array<u8, 0x13> test { 0, 0, 0, 0, 0x72, 0, 0, 0xee, 0, 0, 0, 0x26, 0, 0, 0, 0x28, 0, 0, 0x72 };
|
Array<u8, 0x13> test { 0, 0, 0, 0, 0x72, 0, 0, 0xee, 0, 0, 0, 0x26, 0, 0, 0, 0x28, 0, 0, 0x72 };
|
||||||
auto compressed = Compress::DeflateCompressor::compress_all(test, Compress::DeflateCompressor::CompressionLevel::GOOD);
|
auto compressed = Compress::DeflateCompressor::compress_all(test, Compress::DeflateCompressor::CompressionLevel::GOOD);
|
||||||
EXPECT(compressed.has_value());
|
EXPECT(!compressed.is_error());
|
||||||
}
|
}
|
||||||
|
|
|
@ -116,9 +116,10 @@ ErrorOr<u32> CanonicalCode::read_symbol(Core::Stream::LittleEndianInputBitStream
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void CanonicalCode::write_symbol(OutputBitStream& stream, u32 symbol) const
|
ErrorOr<void> CanonicalCode::write_symbol(Core::Stream::LittleEndianOutputBitStream& stream, u32 symbol) const
|
||||||
{
|
{
|
||||||
stream.write_bits(m_bit_codes[symbol], m_bit_code_lengths[symbol]);
|
TRY(stream.write_bits(m_bit_codes[symbol], m_bit_code_lengths[symbol]));
|
||||||
|
return {};
|
||||||
}
|
}
|
||||||
|
|
||||||
DeflateDecompressor::CompressedBlock::CompressedBlock(DeflateDecompressor& decompressor, CanonicalCode literal_codes, Optional<CanonicalCode> distance_codes)
|
DeflateDecompressor::CompressedBlock::CompressedBlock(DeflateDecompressor& decompressor, CanonicalCode literal_codes, Optional<CanonicalCode> distance_codes)
|
||||||
|
@ -437,10 +438,10 @@ ErrorOr<void> DeflateDecompressor::decode_codes(CanonicalCode& literal_code, Opt
|
||||||
return {};
|
return {};
|
||||||
}
|
}
|
||||||
|
|
||||||
DeflateCompressor::DeflateCompressor(OutputStream& stream, CompressionLevel compression_level)
|
DeflateCompressor::DeflateCompressor(Core::Stream::Handle<Core::Stream::Stream> stream, CompressionLevel compression_level)
|
||||||
: m_compression_level(compression_level)
|
: m_compression_level(compression_level)
|
||||||
, m_compression_constants(compression_constants[static_cast<int>(m_compression_level)])
|
, m_compression_constants(compression_constants[static_cast<int>(m_compression_level)])
|
||||||
, m_output_stream(stream)
|
, m_output_stream(Core::Stream::LittleEndianOutputBitStream::construct(move(stream)).release_value_but_fixme_should_propagate_errors())
|
||||||
{
|
{
|
||||||
m_symbol_frequencies.fill(0);
|
m_symbol_frequencies.fill(0);
|
||||||
m_distance_frequencies.fill(0);
|
m_distance_frequencies.fill(0);
|
||||||
|
@ -451,7 +452,12 @@ DeflateCompressor::~DeflateCompressor()
|
||||||
VERIFY(m_finished);
|
VERIFY(m_finished);
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t DeflateCompressor::write(ReadonlyBytes bytes)
|
ErrorOr<Bytes> DeflateCompressor::read(Bytes)
|
||||||
|
{
|
||||||
|
return Error::from_errno(EBADF);
|
||||||
|
}
|
||||||
|
|
||||||
|
ErrorOr<size_t> DeflateCompressor::write(ReadonlyBytes bytes)
|
||||||
{
|
{
|
||||||
VERIFY(!m_finished);
|
VERIFY(!m_finished);
|
||||||
|
|
||||||
|
@ -462,21 +468,25 @@ size_t DeflateCompressor::write(ReadonlyBytes bytes)
|
||||||
m_pending_block_size += n_written;
|
m_pending_block_size += n_written;
|
||||||
|
|
||||||
if (m_pending_block_size == block_size)
|
if (m_pending_block_size == block_size)
|
||||||
flush();
|
TRY(flush());
|
||||||
|
|
||||||
return n_written + write(bytes.slice(n_written));
|
return n_written + TRY(write(bytes.slice(n_written)));
|
||||||
}
|
}
|
||||||
|
|
||||||
bool DeflateCompressor::write_or_error(ReadonlyBytes bytes)
|
bool DeflateCompressor::is_eof() const
|
||||||
{
|
{
|
||||||
if (write(bytes) < bytes.size()) {
|
|
||||||
set_fatal_error();
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool DeflateCompressor::is_open() const
|
||||||
|
{
|
||||||
|
return m_output_stream->is_open();
|
||||||
|
}
|
||||||
|
|
||||||
|
void DeflateCompressor::close()
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
// Knuth's multiplicative hash on 4 bytes
|
// Knuth's multiplicative hash on 4 bytes
|
||||||
u16 DeflateCompressor::hash_sequence(u8 const* bytes)
|
u16 DeflateCompressor::hash_sequence(u8 const* bytes)
|
||||||
{
|
{
|
||||||
|
@ -730,7 +740,7 @@ size_t DeflateCompressor::huffman_block_length(Array<u8, max_huffman_literals> c
|
||||||
|
|
||||||
size_t DeflateCompressor::uncompressed_block_length()
|
size_t DeflateCompressor::uncompressed_block_length()
|
||||||
{
|
{
|
||||||
auto padding = 8 - ((m_output_stream.bit_offset() + 3) % 8);
|
auto padding = 8 - ((m_output_stream->bit_offset() + 3) % 8);
|
||||||
// 3 bit block header + align to byte + 2 * 16 bit length fields + block contents
|
// 3 bit block header + align to byte + 2 * 16 bit length fields + block contents
|
||||||
return 3 + padding + (2 * 16) + m_pending_block_size * 8;
|
return 3 + padding + (2 * 16) + m_pending_block_size * 8;
|
||||||
}
|
}
|
||||||
|
@ -765,25 +775,26 @@ size_t DeflateCompressor::dynamic_block_length(Array<u8, max_huffman_literals> c
|
||||||
return length + huffman_block_length(literal_bit_lengths, distance_bit_lengths);
|
return length + huffman_block_length(literal_bit_lengths, distance_bit_lengths);
|
||||||
}
|
}
|
||||||
|
|
||||||
void DeflateCompressor::write_huffman(CanonicalCode const& literal_code, Optional<CanonicalCode> const& distance_code)
|
ErrorOr<void> DeflateCompressor::write_huffman(CanonicalCode const& literal_code, Optional<CanonicalCode> const& distance_code)
|
||||||
{
|
{
|
||||||
auto has_distances = distance_code.has_value();
|
auto has_distances = distance_code.has_value();
|
||||||
for (size_t i = 0; i < m_pending_symbol_size; i++) {
|
for (size_t i = 0; i < m_pending_symbol_size; i++) {
|
||||||
if (m_symbol_buffer[i].distance == 0) {
|
if (m_symbol_buffer[i].distance == 0) {
|
||||||
literal_code.write_symbol(m_output_stream, m_symbol_buffer[i].literal);
|
TRY(literal_code.write_symbol(*m_output_stream, m_symbol_buffer[i].literal));
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
VERIFY(has_distances);
|
VERIFY(has_distances);
|
||||||
auto symbol = length_to_symbol[m_symbol_buffer[i].length];
|
auto symbol = length_to_symbol[m_symbol_buffer[i].length];
|
||||||
literal_code.write_symbol(m_output_stream, symbol);
|
TRY(literal_code.write_symbol(*m_output_stream, symbol));
|
||||||
// Emit extra bits if needed
|
// Emit extra bits if needed
|
||||||
m_output_stream.write_bits(m_symbol_buffer[i].length - packed_length_symbols[symbol - 257].base_length, packed_length_symbols[symbol - 257].extra_bits);
|
TRY(m_output_stream->write_bits<u16>(m_symbol_buffer[i].length - packed_length_symbols[symbol - 257].base_length, packed_length_symbols[symbol - 257].extra_bits));
|
||||||
|
|
||||||
auto base_distance = distance_to_base(m_symbol_buffer[i].distance);
|
auto base_distance = distance_to_base(m_symbol_buffer[i].distance);
|
||||||
distance_code.value().write_symbol(m_output_stream, base_distance);
|
TRY(distance_code.value().write_symbol(*m_output_stream, base_distance));
|
||||||
// Emit extra bits if needed
|
// Emit extra bits if needed
|
||||||
m_output_stream.write_bits(m_symbol_buffer[i].distance - packed_distances[base_distance].base_distance, packed_distances[base_distance].extra_bits);
|
TRY(m_output_stream->write_bits<u16>(m_symbol_buffer[i].distance - packed_distances[base_distance].base_distance, packed_distances[base_distance].extra_bits));
|
||||||
}
|
}
|
||||||
|
return {};
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t DeflateCompressor::encode_huffman_lengths(Array<u8, max_huffman_literals + max_huffman_distances> const& lengths, size_t lengths_count, Array<code_length_symbol, max_huffman_literals + max_huffman_distances>& encoded_lengths)
|
size_t DeflateCompressor::encode_huffman_lengths(Array<u8, max_huffman_literals + max_huffman_distances> const& lengths, size_t lengths_count, Array<code_length_symbol, max_huffman_literals + max_huffman_distances>& encoded_lengths)
|
||||||
|
@ -854,65 +865,62 @@ size_t DeflateCompressor::encode_block_lengths(Array<u8, max_huffman_literals> c
|
||||||
return encode_huffman_lengths(all_lengths, lengths_count, encoded_lengths);
|
return encode_huffman_lengths(all_lengths, lengths_count, encoded_lengths);
|
||||||
}
|
}
|
||||||
|
|
||||||
void DeflateCompressor::write_dynamic_huffman(CanonicalCode const& literal_code, size_t literal_code_count, Optional<CanonicalCode> const& distance_code, size_t distance_code_count, Array<u8, 19> const& code_lengths_bit_lengths, size_t code_length_count, Array<code_length_symbol, max_huffman_literals + max_huffman_distances> const& encoded_lengths, size_t encoded_lengths_count)
|
ErrorOr<void> DeflateCompressor::write_dynamic_huffman(CanonicalCode const& literal_code, size_t literal_code_count, Optional<CanonicalCode> const& distance_code, size_t distance_code_count, Array<u8, 19> const& code_lengths_bit_lengths, size_t code_length_count, Array<code_length_symbol, max_huffman_literals + max_huffman_distances> const& encoded_lengths, size_t encoded_lengths_count)
|
||||||
{
|
{
|
||||||
m_output_stream.write_bits(literal_code_count - 257, 5);
|
TRY(m_output_stream->write_bits(literal_code_count - 257, 5));
|
||||||
m_output_stream.write_bits(distance_code_count - 1, 5);
|
TRY(m_output_stream->write_bits(distance_code_count - 1, 5));
|
||||||
m_output_stream.write_bits(code_length_count - 4, 4);
|
TRY(m_output_stream->write_bits(code_length_count - 4, 4));
|
||||||
|
|
||||||
for (size_t i = 0; i < code_length_count; i++) {
|
for (size_t i = 0; i < code_length_count; i++) {
|
||||||
m_output_stream.write_bits(code_lengths_bit_lengths[code_lengths_code_lengths_order[i]], 3);
|
TRY(m_output_stream->write_bits(code_lengths_bit_lengths[code_lengths_code_lengths_order[i]], 3));
|
||||||
}
|
}
|
||||||
|
|
||||||
auto code_lengths_code = CanonicalCode::from_bytes(code_lengths_bit_lengths);
|
auto code_lengths_code = CanonicalCode::from_bytes(code_lengths_bit_lengths);
|
||||||
VERIFY(code_lengths_code.has_value());
|
VERIFY(code_lengths_code.has_value());
|
||||||
for (size_t i = 0; i < encoded_lengths_count; i++) {
|
for (size_t i = 0; i < encoded_lengths_count; i++) {
|
||||||
auto encoded_length = encoded_lengths[i];
|
auto encoded_length = encoded_lengths[i];
|
||||||
code_lengths_code->write_symbol(m_output_stream, encoded_length.symbol);
|
TRY(code_lengths_code->write_symbol(*m_output_stream, encoded_length.symbol));
|
||||||
if (encoded_length.symbol == deflate_special_code_length_copy) {
|
if (encoded_length.symbol == deflate_special_code_length_copy) {
|
||||||
m_output_stream.write_bits(encoded_length.count - 3, 2);
|
TRY(m_output_stream->write_bits<u8>(encoded_length.count - 3, 2));
|
||||||
} else if (encoded_length.symbol == deflate_special_code_length_zeros) {
|
} else if (encoded_length.symbol == deflate_special_code_length_zeros) {
|
||||||
m_output_stream.write_bits(encoded_length.count - 3, 3);
|
TRY(m_output_stream->write_bits<u8>(encoded_length.count - 3, 3));
|
||||||
} else if (encoded_length.symbol == deflate_special_code_length_long_zeros) {
|
} else if (encoded_length.symbol == deflate_special_code_length_long_zeros) {
|
||||||
m_output_stream.write_bits(encoded_length.count - 11, 7);
|
TRY(m_output_stream->write_bits<u8>(encoded_length.count - 11, 7));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
write_huffman(literal_code, distance_code);
|
TRY(write_huffman(literal_code, distance_code));
|
||||||
|
return {};
|
||||||
}
|
}
|
||||||
|
|
||||||
void DeflateCompressor::flush()
|
ErrorOr<void> DeflateCompressor::flush()
|
||||||
{
|
{
|
||||||
if (m_output_stream.handle_any_error()) {
|
TRY(m_output_stream->write_bits(m_finished, 1));
|
||||||
set_fatal_error();
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
m_output_stream.write_bit(m_finished);
|
|
||||||
|
|
||||||
// if this is just an empty block to signify the end of the deflate stream use the smallest block possible (10 bits total)
|
// if this is just an empty block to signify the end of the deflate stream use the smallest block possible (10 bits total)
|
||||||
if (m_pending_block_size == 0) {
|
if (m_pending_block_size == 0) {
|
||||||
VERIFY(m_finished); // we shouldn't be writing empty blocks unless this is the final one
|
VERIFY(m_finished); // we shouldn't be writing empty blocks unless this is the final one
|
||||||
m_output_stream.write_bits(0b01, 2); // fixed huffman codes
|
TRY(m_output_stream->write_bits(0b01u, 2)); // fixed huffman codes
|
||||||
m_output_stream.write_bits(0b0000000, 7); // end of block symbol
|
TRY(m_output_stream->write_bits(0b0000000u, 7)); // end of block symbol
|
||||||
m_output_stream.align_to_byte_boundary();
|
TRY(m_output_stream->align_to_byte_boundary());
|
||||||
return;
|
return {};
|
||||||
}
|
}
|
||||||
|
|
||||||
auto write_uncompressed = [&]() {
|
auto write_uncompressed = [&]() -> ErrorOr<void> {
|
||||||
m_output_stream.write_bits(0b00, 2); // no compression
|
TRY(m_output_stream->write_bits(0b00u, 2)); // no compression
|
||||||
m_output_stream.align_to_byte_boundary();
|
TRY(m_output_stream->align_to_byte_boundary());
|
||||||
LittleEndian<u16> len = m_pending_block_size;
|
LittleEndian<u16> len = m_pending_block_size;
|
||||||
m_output_stream << len;
|
TRY(m_output_stream->write_entire_buffer(len.bytes()));
|
||||||
LittleEndian<u16> nlen = ~m_pending_block_size;
|
LittleEndian<u16> nlen = ~m_pending_block_size;
|
||||||
m_output_stream << nlen;
|
TRY(m_output_stream->write_entire_buffer(nlen.bytes()));
|
||||||
m_output_stream.write_or_error(pending_block().slice(0, m_pending_block_size));
|
TRY(m_output_stream->write_entire_buffer(pending_block().slice(0, m_pending_block_size)));
|
||||||
|
return {};
|
||||||
};
|
};
|
||||||
|
|
||||||
if (m_compression_level == CompressionLevel::STORE) { // disabled compression fast path
|
if (m_compression_level == CompressionLevel::STORE) { // disabled compression fast path
|
||||||
write_uncompressed();
|
TRY(write_uncompressed());
|
||||||
m_pending_block_size = 0;
|
m_pending_block_size = 0;
|
||||||
return;
|
return {};
|
||||||
}
|
}
|
||||||
|
|
||||||
// The following implementation of lz77 compression and huffman encoding is based on the reference implementation by Hans Wennborg https://www.hanshq.net/zip.html
|
// The following implementation of lz77 compression and huffman encoding is based on the reference implementation by Hans Wennborg https://www.hanshq.net/zip.html
|
||||||
|
@ -956,19 +964,21 @@ void DeflateCompressor::flush()
|
||||||
|
|
||||||
// If the compression somehow didn't reduce the size enough, just write out the block uncompressed as it allows for much faster decompression
|
// If the compression somehow didn't reduce the size enough, just write out the block uncompressed as it allows for much faster decompression
|
||||||
if (uncompressed_size <= min(fixed_huffman_size, dynamic_huffman_size)) {
|
if (uncompressed_size <= min(fixed_huffman_size, dynamic_huffman_size)) {
|
||||||
write_uncompressed();
|
TRY(write_uncompressed());
|
||||||
} else if (fixed_huffman_size <= dynamic_huffman_size) { // If the fixed and dynamic huffman codes come out the same size, prefer the fixed version, as it takes less time to decode
|
} else if (fixed_huffman_size <= dynamic_huffman_size) {
|
||||||
m_output_stream.write_bits(0b01, 2); // fixed huffman codes
|
// If the fixed and dynamic huffman codes come out the same size, prefer the fixed version, as it takes less time to decode fixed huffman codes.
|
||||||
write_huffman(CanonicalCode::fixed_literal_codes(), CanonicalCode::fixed_distance_codes());
|
TRY(m_output_stream->write_bits(0b01u, 2));
|
||||||
|
TRY(write_huffman(CanonicalCode::fixed_literal_codes(), CanonicalCode::fixed_distance_codes()));
|
||||||
} else {
|
} else {
|
||||||
m_output_stream.write_bits(0b10, 2); // dynamic huffman codes
|
// dynamic huffman codes
|
||||||
|
TRY(m_output_stream->write_bits(0b10u, 2));
|
||||||
auto literal_code = CanonicalCode::from_bytes(dynamic_literal_bit_lengths);
|
auto literal_code = CanonicalCode::from_bytes(dynamic_literal_bit_lengths);
|
||||||
VERIFY(literal_code.has_value());
|
VERIFY(literal_code.has_value());
|
||||||
auto distance_code = CanonicalCode::from_bytes(dynamic_distance_bit_lengths);
|
auto distance_code = CanonicalCode::from_bytes(dynamic_distance_bit_lengths);
|
||||||
write_dynamic_huffman(literal_code.value(), literal_code_count, distance_code, distance_code_count, code_lengths_bit_lengths, code_lengths_count, encoded_lengths, encoded_lengths_count);
|
TRY(write_dynamic_huffman(literal_code.value(), literal_code_count, distance_code, distance_code_count, code_lengths_bit_lengths, code_lengths_count, encoded_lengths, encoded_lengths_count));
|
||||||
}
|
}
|
||||||
if (m_finished)
|
if (m_finished)
|
||||||
m_output_stream.align_to_byte_boundary();
|
TRY(m_output_stream->align_to_byte_boundary());
|
||||||
|
|
||||||
// reset all block specific members
|
// reset all block specific members
|
||||||
m_pending_block_size = 0;
|
m_pending_block_size = 0;
|
||||||
|
@ -977,28 +987,30 @@ void DeflateCompressor::flush()
|
||||||
m_distance_frequencies.fill(0);
|
m_distance_frequencies.fill(0);
|
||||||
// On the final block this copy will potentially produce an invalid search window, but since its the final block we dont care
|
// On the final block this copy will potentially produce an invalid search window, but since its the final block we dont care
|
||||||
pending_block().copy_trimmed_to({ m_rolling_window, block_size });
|
pending_block().copy_trimmed_to({ m_rolling_window, block_size });
|
||||||
|
|
||||||
|
return {};
|
||||||
}
|
}
|
||||||
|
|
||||||
void DeflateCompressor::final_flush()
|
ErrorOr<void> DeflateCompressor::final_flush()
|
||||||
{
|
{
|
||||||
VERIFY(!m_finished);
|
VERIFY(!m_finished);
|
||||||
m_finished = true;
|
m_finished = true;
|
||||||
flush();
|
TRY(flush());
|
||||||
|
return {};
|
||||||
}
|
}
|
||||||
|
|
||||||
Optional<ByteBuffer> DeflateCompressor::compress_all(ReadonlyBytes bytes, CompressionLevel compression_level)
|
ErrorOr<ByteBuffer> DeflateCompressor::compress_all(ReadonlyBytes bytes, CompressionLevel compression_level)
|
||||||
{
|
{
|
||||||
DuplexMemoryStream output_stream;
|
auto output_stream = TRY(try_make<Core::Stream::AllocatingMemoryStream>());
|
||||||
DeflateCompressor deflate_stream { output_stream, compression_level };
|
DeflateCompressor deflate_stream { Core::Stream::Handle<Core::Stream::Stream>(*output_stream), compression_level };
|
||||||
|
|
||||||
deflate_stream.write_or_error(bytes);
|
TRY(deflate_stream.write_entire_buffer(bytes));
|
||||||
|
TRY(deflate_stream.final_flush());
|
||||||
|
|
||||||
deflate_stream.final_flush();
|
auto buffer = TRY(ByteBuffer::create_uninitialized(output_stream->used_buffer_size()));
|
||||||
|
TRY(output_stream->read_entire_buffer(buffer));
|
||||||
|
|
||||||
if (deflate_stream.handle_any_error())
|
return buffer;
|
||||||
return {};
|
|
||||||
|
|
||||||
return output_stream.copy_into_contiguous_buffer();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -22,7 +22,7 @@ class CanonicalCode {
|
||||||
public:
|
public:
|
||||||
CanonicalCode() = default;
|
CanonicalCode() = default;
|
||||||
ErrorOr<u32> read_symbol(Core::Stream::LittleEndianInputBitStream&) const;
|
ErrorOr<u32> read_symbol(Core::Stream::LittleEndianInputBitStream&) const;
|
||||||
void write_symbol(OutputBitStream&, u32) const;
|
ErrorOr<void> write_symbol(Core::Stream::LittleEndianOutputBitStream&, u32) const;
|
||||||
|
|
||||||
static CanonicalCode const& fixed_literal_codes();
|
static CanonicalCode const& fixed_literal_codes();
|
||||||
static CanonicalCode const& fixed_distance_codes();
|
static CanonicalCode const& fixed_distance_codes();
|
||||||
|
@ -104,7 +104,7 @@ private:
|
||||||
CircularDuplexStream<32 * KiB> m_output_stream;
|
CircularDuplexStream<32 * KiB> m_output_stream;
|
||||||
};
|
};
|
||||||
|
|
||||||
class DeflateCompressor final : public OutputStream {
|
class DeflateCompressor final : public Core::Stream::Stream {
|
||||||
public:
|
public:
|
||||||
static constexpr size_t block_size = 32 * KiB - 1; // TODO: this can theoretically be increased to 64 KiB - 2
|
static constexpr size_t block_size = 32 * KiB - 1; // TODO: this can theoretically be increased to 64 KiB - 2
|
||||||
static constexpr size_t window_size = block_size * 2;
|
static constexpr size_t window_size = block_size * 2;
|
||||||
|
@ -139,14 +139,17 @@ public:
|
||||||
BEST // WARNING: this one can take an unreasonable amount of time!
|
BEST // WARNING: this one can take an unreasonable amount of time!
|
||||||
};
|
};
|
||||||
|
|
||||||
DeflateCompressor(OutputStream&, CompressionLevel = CompressionLevel::GOOD);
|
DeflateCompressor(Core::Stream::Handle<Core::Stream::Stream>, CompressionLevel = CompressionLevel::GOOD);
|
||||||
~DeflateCompressor();
|
~DeflateCompressor();
|
||||||
|
|
||||||
size_t write(ReadonlyBytes) override;
|
virtual ErrorOr<Bytes> read(Bytes) override;
|
||||||
bool write_or_error(ReadonlyBytes) override;
|
virtual ErrorOr<size_t> write(ReadonlyBytes) override;
|
||||||
void final_flush();
|
virtual bool is_eof() const override;
|
||||||
|
virtual bool is_open() const override;
|
||||||
|
virtual void close() override;
|
||||||
|
ErrorOr<void> final_flush();
|
||||||
|
|
||||||
static Optional<ByteBuffer> compress_all(ReadonlyBytes bytes, CompressionLevel = CompressionLevel::GOOD);
|
static ErrorOr<ByteBuffer> compress_all(ReadonlyBytes bytes, CompressionLevel = CompressionLevel::GOOD);
|
||||||
|
|
||||||
private:
|
private:
|
||||||
Bytes pending_block() { return { m_rolling_window + block_size, block_size }; }
|
Bytes pending_block() { return { m_rolling_window + block_size, block_size }; }
|
||||||
|
@ -166,20 +169,20 @@ private:
|
||||||
template<size_t Size>
|
template<size_t Size>
|
||||||
static void generate_huffman_lengths(Array<u8, Size>& lengths, Array<u16, Size> const& frequencies, size_t max_bit_length, u16 frequency_cap = UINT16_MAX);
|
static void generate_huffman_lengths(Array<u8, Size>& lengths, Array<u16, Size> const& frequencies, size_t max_bit_length, u16 frequency_cap = UINT16_MAX);
|
||||||
size_t huffman_block_length(Array<u8, max_huffman_literals> const& literal_bit_lengths, Array<u8, max_huffman_distances> const& distance_bit_lengths);
|
size_t huffman_block_length(Array<u8, max_huffman_literals> const& literal_bit_lengths, Array<u8, max_huffman_distances> const& distance_bit_lengths);
|
||||||
void write_huffman(CanonicalCode const& literal_code, Optional<CanonicalCode> const& distance_code);
|
ErrorOr<void> write_huffman(CanonicalCode const& literal_code, Optional<CanonicalCode> const& distance_code);
|
||||||
static size_t encode_huffman_lengths(Array<u8, max_huffman_literals + max_huffman_distances> const& lengths, size_t lengths_count, Array<code_length_symbol, max_huffman_literals + max_huffman_distances>& encoded_lengths);
|
static size_t encode_huffman_lengths(Array<u8, max_huffman_literals + max_huffman_distances> const& lengths, size_t lengths_count, Array<code_length_symbol, max_huffman_literals + max_huffman_distances>& encoded_lengths);
|
||||||
size_t encode_block_lengths(Array<u8, max_huffman_literals> const& literal_bit_lengths, Array<u8, max_huffman_distances> const& distance_bit_lengths, Array<code_length_symbol, max_huffman_literals + max_huffman_distances>& encoded_lengths, size_t& literal_code_count, size_t& distance_code_count);
|
size_t encode_block_lengths(Array<u8, max_huffman_literals> const& literal_bit_lengths, Array<u8, max_huffman_distances> const& distance_bit_lengths, Array<code_length_symbol, max_huffman_literals + max_huffman_distances>& encoded_lengths, size_t& literal_code_count, size_t& distance_code_count);
|
||||||
void write_dynamic_huffman(CanonicalCode const& literal_code, size_t literal_code_count, Optional<CanonicalCode> const& distance_code, size_t distance_code_count, Array<u8, 19> const& code_lengths_bit_lengths, size_t code_length_count, Array<code_length_symbol, max_huffman_literals + max_huffman_distances> const& encoded_lengths, size_t encoded_lengths_count);
|
ErrorOr<void> write_dynamic_huffman(CanonicalCode const& literal_code, size_t literal_code_count, Optional<CanonicalCode> const& distance_code, size_t distance_code_count, Array<u8, 19> const& code_lengths_bit_lengths, size_t code_length_count, Array<code_length_symbol, max_huffman_literals + max_huffman_distances> const& encoded_lengths, size_t encoded_lengths_count);
|
||||||
|
|
||||||
size_t uncompressed_block_length();
|
size_t uncompressed_block_length();
|
||||||
size_t fixed_block_length();
|
size_t fixed_block_length();
|
||||||
size_t dynamic_block_length(Array<u8, max_huffman_literals> const& literal_bit_lengths, Array<u8, max_huffman_distances> const& distance_bit_lengths, Array<u8, 19> const& code_lengths_bit_lengths, Array<u16, 19> const& code_lengths_frequencies, size_t code_lengths_count);
|
size_t dynamic_block_length(Array<u8, max_huffman_literals> const& literal_bit_lengths, Array<u8, max_huffman_distances> const& distance_bit_lengths, Array<u8, 19> const& code_lengths_bit_lengths, Array<u16, 19> const& code_lengths_frequencies, size_t code_lengths_count);
|
||||||
void flush();
|
ErrorOr<void> flush();
|
||||||
|
|
||||||
bool m_finished { false };
|
bool m_finished { false };
|
||||||
CompressionLevel m_compression_level;
|
CompressionLevel m_compression_level;
|
||||||
CompressionConstants m_compression_constants;
|
CompressionConstants m_compression_constants;
|
||||||
OutputBitStream m_output_stream;
|
NonnullOwnPtr<Core::Stream::LittleEndianOutputBitStream> m_output_stream;
|
||||||
|
|
||||||
u8 m_rolling_window[window_size];
|
u8 m_rolling_window[window_size];
|
||||||
size_t m_pending_block_size { 0 };
|
size_t m_pending_block_size { 0 };
|
||||||
|
|
|
@ -194,11 +194,9 @@ ErrorOr<size_t> GzipCompressor::write(ReadonlyBytes bytes)
|
||||||
header.extra_flags = 3; // DEFLATE sets 2 for maximum compression and 4 for minimum compression
|
header.extra_flags = 3; // DEFLATE sets 2 for maximum compression and 4 for minimum compression
|
||||||
header.operating_system = 3; // unix
|
header.operating_system = 3; // unix
|
||||||
TRY(m_output_stream->write_entire_buffer({ &header, sizeof(header) }));
|
TRY(m_output_stream->write_entire_buffer({ &header, sizeof(header) }));
|
||||||
Core::Stream::WrapInAKOutputStream wrapped_stream { *m_output_stream };
|
DeflateCompressor compressed_stream { Core::Stream::Handle(*m_output_stream) };
|
||||||
DeflateCompressor compressed_stream { wrapped_stream };
|
TRY(compressed_stream.write_entire_buffer(bytes));
|
||||||
if (!compressed_stream.write_or_error(bytes))
|
TRY(compressed_stream.final_flush());
|
||||||
return Error::from_string_literal("Underlying DeflateCompressor indicated an error");
|
|
||||||
compressed_stream.final_flush();
|
|
||||||
Crypto::Checksum::CRC32 crc32;
|
Crypto::Checksum::CRC32 crc32;
|
||||||
crc32.update(bytes);
|
crc32.update(bytes);
|
||||||
LittleEndian<u32> digest = crc32.digest();
|
LittleEndian<u32> digest = crc32.digest();
|
||||||
|
|
|
@ -81,10 +81,9 @@ ErrorOr<NonnullOwnPtr<ZlibCompressor>> ZlibCompressor::construct(Core::Stream::H
|
||||||
}
|
}
|
||||||
|
|
||||||
ZlibCompressor::ZlibCompressor(Core::Stream::Handle<Core::Stream::Stream> stream, ZlibCompressionLevel compression_level)
|
ZlibCompressor::ZlibCompressor(Core::Stream::Handle<Core::Stream::Stream> stream, ZlibCompressionLevel compression_level)
|
||||||
: m_ak_output_stream(make<Core::Stream::WrapInAKOutputStream>(*stream))
|
: m_output_stream(move(stream))
|
||||||
, m_output_stream(move(stream))
|
|
||||||
// FIXME: Find a way to compress with Deflate's "Best" compression level.
|
// FIXME: Find a way to compress with Deflate's "Best" compression level.
|
||||||
, m_compressor(make<DeflateCompressor>(*m_ak_output_stream, static_cast<DeflateCompressor::CompressionLevel>(compression_level)))
|
, m_compressor(make<DeflateCompressor>(Core::Stream::Handle(*m_output_stream), static_cast<DeflateCompressor::CompressionLevel>(compression_level)))
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -126,7 +125,7 @@ ErrorOr<size_t> ZlibCompressor::write(ReadonlyBytes bytes)
|
||||||
{
|
{
|
||||||
VERIFY(!m_finished);
|
VERIFY(!m_finished);
|
||||||
|
|
||||||
size_t n_written = m_compressor->write(bytes);
|
size_t n_written = TRY(m_compressor->write(bytes));
|
||||||
m_adler32_checksum.update(bytes.trim(n_written));
|
m_adler32_checksum.update(bytes.trim(n_written));
|
||||||
return n_written;
|
return n_written;
|
||||||
}
|
}
|
||||||
|
@ -150,7 +149,7 @@ ErrorOr<void> ZlibCompressor::finish()
|
||||||
VERIFY(!m_finished);
|
VERIFY(!m_finished);
|
||||||
|
|
||||||
if (is<DeflateCompressor>(m_compressor.ptr()))
|
if (is<DeflateCompressor>(m_compressor.ptr()))
|
||||||
static_cast<DeflateCompressor*>(m_compressor.ptr())->final_flush();
|
TRY(static_cast<DeflateCompressor*>(m_compressor.ptr())->final_flush());
|
||||||
|
|
||||||
NetworkOrdered<u32> adler_sum = m_adler32_checksum.digest();
|
NetworkOrdered<u32> adler_sum = m_adler32_checksum.digest();
|
||||||
TRY(m_output_stream->write(adler_sum.bytes()));
|
TRY(m_output_stream->write(adler_sum.bytes()));
|
||||||
|
|
|
@ -80,10 +80,8 @@ private:
|
||||||
ErrorOr<void> write_header(ZlibCompressionMethod, ZlibCompressionLevel);
|
ErrorOr<void> write_header(ZlibCompressionMethod, ZlibCompressionLevel);
|
||||||
|
|
||||||
bool m_finished { false };
|
bool m_finished { false };
|
||||||
// FIXME: Remove this once DeflateCompressor is ported to Core::Stream.
|
|
||||||
NonnullOwnPtr<Core::Stream::WrapInAKOutputStream> m_ak_output_stream;
|
|
||||||
Core::Stream::Handle<Core::Stream::Stream> m_output_stream;
|
Core::Stream::Handle<Core::Stream::Stream> m_output_stream;
|
||||||
NonnullOwnPtr<OutputStream> m_compressor;
|
NonnullOwnPtr<Core::Stream::Stream> m_compressor;
|
||||||
Crypto::Checksum::Adler32 m_adler32_checksum;
|
Crypto::Checksum::Adler32 m_adler32_checksum;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
@ -59,7 +59,7 @@ ErrorOr<int> serenity_main(Main::Arguments arguments)
|
||||||
member.name = TRY(String::from_deprecated_string(canonicalized_path));
|
member.name = TRY(String::from_deprecated_string(canonicalized_path));
|
||||||
|
|
||||||
auto deflate_buffer = Compress::DeflateCompressor::compress_all(file_buffer);
|
auto deflate_buffer = Compress::DeflateCompressor::compress_all(file_buffer);
|
||||||
if (deflate_buffer.has_value() && deflate_buffer.value().size() < file_buffer.size()) {
|
if (!deflate_buffer.is_error() && deflate_buffer.value().size() < file_buffer.size()) {
|
||||||
member.compressed_data = deflate_buffer.value().bytes();
|
member.compressed_data = deflate_buffer.value().bytes();
|
||||||
member.compression_method = Archive::ZipCompressionMethod::Deflate;
|
member.compression_method = Archive::ZipCompressionMethod::Deflate;
|
||||||
auto compression_ratio = (double)deflate_buffer.value().size() / file_buffer.size();
|
auto compression_ratio = (double)deflate_buffer.value().size() / file_buffer.size();
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue