mirror of
https://github.com/RGBCube/serenity
synced 2025-07-26 22:37:36 +00:00
LibCompress: Port DeflateCompressor to Core::Stream
This commit is contained in:
Tim Schumacher
Andreas Kling
parent
a212bc3052
commit
8cd2cf2b77
7 changed files with 107 additions and 97 deletions
|
|
@ -116,9 +116,10 @@ ErrorOr<u32> CanonicalCode::read_symbol(Core::Stream::LittleEndianInputBitStream
|
|||
}
|
||||
}
|
||||
|
||||
void CanonicalCode::write_symbol(OutputBitStream& stream, u32 symbol) const
|
||||
ErrorOr<void> CanonicalCode::write_symbol(Core::Stream::LittleEndianOutputBitStream& stream, u32 symbol) const
|
||||
{
|
||||
stream.write_bits(m_bit_codes[symbol], m_bit_code_lengths[symbol]);
|
||||
TRY(stream.write_bits(m_bit_codes[symbol], m_bit_code_lengths[symbol]));
|
||||
return {};
|
||||
}
|
||||
|
||||
DeflateDecompressor::CompressedBlock::CompressedBlock(DeflateDecompressor& decompressor, CanonicalCode literal_codes, Optional<CanonicalCode> distance_codes)
|
||||
|
|
@ -437,10 +438,10 @@ ErrorOr<void> DeflateDecompressor::decode_codes(CanonicalCode& literal_code, Opt
|
|||
return {};
|
||||
}
|
||||
|
||||
DeflateCompressor::DeflateCompressor(OutputStream& stream, CompressionLevel compression_level)
|
||||
DeflateCompressor::DeflateCompressor(Core::Stream::Handle<Core::Stream::Stream> stream, CompressionLevel compression_level)
|
||||
: m_compression_level(compression_level)
|
||||
, m_compression_constants(compression_constants[static_cast<int>(m_compression_level)])
|
||||
, m_output_stream(stream)
|
||||
, m_output_stream(Core::Stream::LittleEndianOutputBitStream::construct(move(stream)).release_value_but_fixme_should_propagate_errors())
|
||||
{
|
||||
m_symbol_frequencies.fill(0);
|
||||
m_distance_frequencies.fill(0);
|
||||
|
|
@ -451,7 +452,12 @@ DeflateCompressor::~DeflateCompressor()
|
|||
VERIFY(m_finished);
|
||||
}
|
||||
|
||||
size_t DeflateCompressor::write(ReadonlyBytes bytes)
|
||||
ErrorOr<Bytes> DeflateCompressor::read(Bytes)
|
||||
{
|
||||
return Error::from_errno(EBADF);
|
||||
}
|
||||
|
||||
ErrorOr<size_t> DeflateCompressor::write(ReadonlyBytes bytes)
|
||||
{
|
||||
VERIFY(!m_finished);
|
||||
|
||||
|
|
@ -462,21 +468,25 @@ size_t DeflateCompressor::write(ReadonlyBytes bytes)
|
|||
m_pending_block_size += n_written;
|
||||
|
||||
if (m_pending_block_size == block_size)
|
||||
flush();
|
||||
TRY(flush());
|
||||
|
||||
return n_written + write(bytes.slice(n_written));
|
||||
return n_written + TRY(write(bytes.slice(n_written)));
|
||||
}
|
||||
|
||||
bool DeflateCompressor::write_or_error(ReadonlyBytes bytes)
|
||||
bool DeflateCompressor::is_eof() const
|
||||
{
|
||||
if (write(bytes) < bytes.size()) {
|
||||
set_fatal_error();
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool DeflateCompressor::is_open() const
|
||||
{
|
||||
return m_output_stream->is_open();
|
||||
}
|
||||
|
||||
void DeflateCompressor::close()
|
||||
{
|
||||
}
|
||||
|
||||
// Knuth's multiplicative hash on 4 bytes
|
||||
u16 DeflateCompressor::hash_sequence(u8 const* bytes)
|
||||
{
|
||||
|
|
@ -730,7 +740,7 @@ size_t DeflateCompressor::huffman_block_length(Array<u8, max_huffman_literals> c
|
|||
|
||||
size_t DeflateCompressor::uncompressed_block_length()
|
||||
{
|
||||
auto padding = 8 - ((m_output_stream.bit_offset() + 3) % 8);
|
||||
auto padding = 8 - ((m_output_stream->bit_offset() + 3) % 8);
|
||||
// 3 bit block header + align to byte + 2 * 16 bit length fields + block contents
|
||||
return 3 + padding + (2 * 16) + m_pending_block_size * 8;
|
||||
}
|
||||
|
|
@ -765,25 +775,26 @@ size_t DeflateCompressor::dynamic_block_length(Array<u8, max_huffman_literals> c
|
|||
return length + huffman_block_length(literal_bit_lengths, distance_bit_lengths);
|
||||
}
|
||||
|
||||
void DeflateCompressor::write_huffman(CanonicalCode const& literal_code, Optional<CanonicalCode> const& distance_code)
|
||||
ErrorOr<void> DeflateCompressor::write_huffman(CanonicalCode const& literal_code, Optional<CanonicalCode> const& distance_code)
|
||||
{
|
||||
auto has_distances = distance_code.has_value();
|
||||
for (size_t i = 0; i < m_pending_symbol_size; i++) {
|
||||
if (m_symbol_buffer[i].distance == 0) {
|
||||
literal_code.write_symbol(m_output_stream, m_symbol_buffer[i].literal);
|
||||
TRY(literal_code.write_symbol(*m_output_stream, m_symbol_buffer[i].literal));
|
||||
continue;
|
||||
}
|
||||
VERIFY(has_distances);
|
||||
auto symbol = length_to_symbol[m_symbol_buffer[i].length];
|
||||
literal_code.write_symbol(m_output_stream, symbol);
|
||||
TRY(literal_code.write_symbol(*m_output_stream, symbol));
|
||||
// Emit extra bits if needed
|
||||
m_output_stream.write_bits(m_symbol_buffer[i].length - packed_length_symbols[symbol - 257].base_length, packed_length_symbols[symbol - 257].extra_bits);
|
||||
TRY(m_output_stream->write_bits<u16>(m_symbol_buffer[i].length - packed_length_symbols[symbol - 257].base_length, packed_length_symbols[symbol - 257].extra_bits));
|
||||
|
||||
auto base_distance = distance_to_base(m_symbol_buffer[i].distance);
|
||||
distance_code.value().write_symbol(m_output_stream, base_distance);
|
||||
TRY(distance_code.value().write_symbol(*m_output_stream, base_distance));
|
||||
// Emit extra bits if needed
|
||||
m_output_stream.write_bits(m_symbol_buffer[i].distance - packed_distances[base_distance].base_distance, packed_distances[base_distance].extra_bits);
|
||||
TRY(m_output_stream->write_bits<u16>(m_symbol_buffer[i].distance - packed_distances[base_distance].base_distance, packed_distances[base_distance].extra_bits));
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
size_t DeflateCompressor::encode_huffman_lengths(Array<u8, max_huffman_literals + max_huffman_distances> const& lengths, size_t lengths_count, Array<code_length_symbol, max_huffman_literals + max_huffman_distances>& encoded_lengths)
|
||||
|
|
@ -854,65 +865,62 @@ size_t DeflateCompressor::encode_block_lengths(Array<u8, max_huffman_literals> c
|
|||
return encode_huffman_lengths(all_lengths, lengths_count, encoded_lengths);
|
||||
}
|
||||
|
||||
void DeflateCompressor::write_dynamic_huffman(CanonicalCode const& literal_code, size_t literal_code_count, Optional<CanonicalCode> const& distance_code, size_t distance_code_count, Array<u8, 19> const& code_lengths_bit_lengths, size_t code_length_count, Array<code_length_symbol, max_huffman_literals + max_huffman_distances> const& encoded_lengths, size_t encoded_lengths_count)
|
||||
ErrorOr<void> DeflateCompressor::write_dynamic_huffman(CanonicalCode const& literal_code, size_t literal_code_count, Optional<CanonicalCode> const& distance_code, size_t distance_code_count, Array<u8, 19> const& code_lengths_bit_lengths, size_t code_length_count, Array<code_length_symbol, max_huffman_literals + max_huffman_distances> const& encoded_lengths, size_t encoded_lengths_count)
|
||||
{
|
||||
m_output_stream.write_bits(literal_code_count - 257, 5);
|
||||
m_output_stream.write_bits(distance_code_count - 1, 5);
|
||||
m_output_stream.write_bits(code_length_count - 4, 4);
|
||||
TRY(m_output_stream->write_bits(literal_code_count - 257, 5));
|
||||
TRY(m_output_stream->write_bits(distance_code_count - 1, 5));
|
||||
TRY(m_output_stream->write_bits(code_length_count - 4, 4));
|
||||
|
||||
for (size_t i = 0; i < code_length_count; i++) {
|
||||
m_output_stream.write_bits(code_lengths_bit_lengths[code_lengths_code_lengths_order[i]], 3);
|
||||
TRY(m_output_stream->write_bits(code_lengths_bit_lengths[code_lengths_code_lengths_order[i]], 3));
|
||||
}
|
||||
|
||||
auto code_lengths_code = CanonicalCode::from_bytes(code_lengths_bit_lengths);
|
||||
VERIFY(code_lengths_code.has_value());
|
||||
for (size_t i = 0; i < encoded_lengths_count; i++) {
|
||||
auto encoded_length = encoded_lengths[i];
|
||||
code_lengths_code->write_symbol(m_output_stream, encoded_length.symbol);
|
||||
TRY(code_lengths_code->write_symbol(*m_output_stream, encoded_length.symbol));
|
||||
if (encoded_length.symbol == deflate_special_code_length_copy) {
|
||||
m_output_stream.write_bits(encoded_length.count - 3, 2);
|
||||
TRY(m_output_stream->write_bits<u8>(encoded_length.count - 3, 2));
|
||||
} else if (encoded_length.symbol == deflate_special_code_length_zeros) {
|
||||
m_output_stream.write_bits(encoded_length.count - 3, 3);
|
||||
TRY(m_output_stream->write_bits<u8>(encoded_length.count - 3, 3));
|
||||
} else if (encoded_length.symbol == deflate_special_code_length_long_zeros) {
|
||||
m_output_stream.write_bits(encoded_length.count - 11, 7);
|
||||
TRY(m_output_stream->write_bits<u8>(encoded_length.count - 11, 7));
|
||||
}
|
||||
}
|
||||
|
||||
write_huffman(literal_code, distance_code);
|
||||
TRY(write_huffman(literal_code, distance_code));
|
||||
return {};
|
||||
}
|
||||
|
||||
void DeflateCompressor::flush()
|
||||
ErrorOr<void> DeflateCompressor::flush()
|
||||
{
|
||||
if (m_output_stream.handle_any_error()) {
|
||||
set_fatal_error();
|
||||
return;
|
||||
}
|
||||
|
||||
m_output_stream.write_bit(m_finished);
|
||||
TRY(m_output_stream->write_bits(m_finished, 1));
|
||||
|
||||
// if this is just an empty block to signify the end of the deflate stream use the smallest block possible (10 bits total)
|
||||
if (m_pending_block_size == 0) {
|
||||
VERIFY(m_finished); // we shouldn't be writing empty blocks unless this is the final one
|
||||
m_output_stream.write_bits(0b01, 2); // fixed huffman codes
|
||||
m_output_stream.write_bits(0b0000000, 7); // end of block symbol
|
||||
m_output_stream.align_to_byte_boundary();
|
||||
return;
|
||||
VERIFY(m_finished); // we shouldn't be writing empty blocks unless this is the final one
|
||||
TRY(m_output_stream->write_bits(0b01u, 2)); // fixed huffman codes
|
||||
TRY(m_output_stream->write_bits(0b0000000u, 7)); // end of block symbol
|
||||
TRY(m_output_stream->align_to_byte_boundary());
|
||||
return {};
|
||||
}
|
||||
|
||||
auto write_uncompressed = [&]() {
|
||||
m_output_stream.write_bits(0b00, 2); // no compression
|
||||
m_output_stream.align_to_byte_boundary();
|
||||
auto write_uncompressed = [&]() -> ErrorOr<void> {
|
||||
TRY(m_output_stream->write_bits(0b00u, 2)); // no compression
|
||||
TRY(m_output_stream->align_to_byte_boundary());
|
||||
LittleEndian<u16> len = m_pending_block_size;
|
||||
m_output_stream << len;
|
||||
TRY(m_output_stream->write_entire_buffer(len.bytes()));
|
||||
LittleEndian<u16> nlen = ~m_pending_block_size;
|
||||
m_output_stream << nlen;
|
||||
m_output_stream.write_or_error(pending_block().slice(0, m_pending_block_size));
|
||||
TRY(m_output_stream->write_entire_buffer(nlen.bytes()));
|
||||
TRY(m_output_stream->write_entire_buffer(pending_block().slice(0, m_pending_block_size)));
|
||||
return {};
|
||||
};
|
||||
|
||||
if (m_compression_level == CompressionLevel::STORE) { // disabled compression fast path
|
||||
write_uncompressed();
|
||||
TRY(write_uncompressed());
|
||||
m_pending_block_size = 0;
|
||||
return;
|
||||
return {};
|
||||
}
|
||||
|
||||
// The following implementation of lz77 compression and huffman encoding is based on the reference implementation by Hans Wennborg https://www.hanshq.net/zip.html
|
||||
|
|
@ -956,19 +964,21 @@ void DeflateCompressor::flush()
|
|||
|
||||
// If the compression somehow didn't reduce the size enough, just write out the block uncompressed as it allows for much faster decompression
|
||||
if (uncompressed_size <= min(fixed_huffman_size, dynamic_huffman_size)) {
|
||||
write_uncompressed();
|
||||
} else if (fixed_huffman_size <= dynamic_huffman_size) { // If the fixed and dynamic huffman codes come out the same size, prefer the fixed version, as it takes less time to decode
|
||||
m_output_stream.write_bits(0b01, 2); // fixed huffman codes
|
||||
write_huffman(CanonicalCode::fixed_literal_codes(), CanonicalCode::fixed_distance_codes());
|
||||
TRY(write_uncompressed());
|
||||
} else if (fixed_huffman_size <= dynamic_huffman_size) {
|
||||
// If the fixed and dynamic huffman codes come out the same size, prefer the fixed version, as it takes less time to decode fixed huffman codes.
|
||||
TRY(m_output_stream->write_bits(0b01u, 2));
|
||||
TRY(write_huffman(CanonicalCode::fixed_literal_codes(), CanonicalCode::fixed_distance_codes()));
|
||||
} else {
|
||||
m_output_stream.write_bits(0b10, 2); // dynamic huffman codes
|
||||
// dynamic huffman codes
|
||||
TRY(m_output_stream->write_bits(0b10u, 2));
|
||||
auto literal_code = CanonicalCode::from_bytes(dynamic_literal_bit_lengths);
|
||||
VERIFY(literal_code.has_value());
|
||||
auto distance_code = CanonicalCode::from_bytes(dynamic_distance_bit_lengths);
|
||||
write_dynamic_huffman(literal_code.value(), literal_code_count, distance_code, distance_code_count, code_lengths_bit_lengths, code_lengths_count, encoded_lengths, encoded_lengths_count);
|
||||
TRY(write_dynamic_huffman(literal_code.value(), literal_code_count, distance_code, distance_code_count, code_lengths_bit_lengths, code_lengths_count, encoded_lengths, encoded_lengths_count));
|
||||
}
|
||||
if (m_finished)
|
||||
m_output_stream.align_to_byte_boundary();
|
||||
TRY(m_output_stream->align_to_byte_boundary());
|
||||
|
||||
// reset all block specific members
|
||||
m_pending_block_size = 0;
|
||||
|
|
@ -977,28 +987,30 @@ void DeflateCompressor::flush()
|
|||
m_distance_frequencies.fill(0);
|
||||
// On the final block this copy will potentially produce an invalid search window, but since its the final block we dont care
|
||||
pending_block().copy_trimmed_to({ m_rolling_window, block_size });
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
void DeflateCompressor::final_flush()
|
||||
ErrorOr<void> DeflateCompressor::final_flush()
|
||||
{
|
||||
VERIFY(!m_finished);
|
||||
m_finished = true;
|
||||
flush();
|
||||
TRY(flush());
|
||||
return {};
|
||||
}
|
||||
|
||||
Optional<ByteBuffer> DeflateCompressor::compress_all(ReadonlyBytes bytes, CompressionLevel compression_level)
|
||||
ErrorOr<ByteBuffer> DeflateCompressor::compress_all(ReadonlyBytes bytes, CompressionLevel compression_level)
|
||||
{
|
||||
DuplexMemoryStream output_stream;
|
||||
DeflateCompressor deflate_stream { output_stream, compression_level };
|
||||
auto output_stream = TRY(try_make<Core::Stream::AllocatingMemoryStream>());
|
||||
DeflateCompressor deflate_stream { Core::Stream::Handle<Core::Stream::Stream>(*output_stream), compression_level };
|
||||
|
||||
deflate_stream.write_or_error(bytes);
|
||||
TRY(deflate_stream.write_entire_buffer(bytes));
|
||||
TRY(deflate_stream.final_flush());
|
||||
|
||||
deflate_stream.final_flush();
|
||||
auto buffer = TRY(ByteBuffer::create_uninitialized(output_stream->used_buffer_size()));
|
||||
TRY(output_stream->read_entire_buffer(buffer));
|
||||
|
||||
if (deflate_stream.handle_any_error())
|
||||
return {};
|
||||
|
||||
return output_stream.copy_into_contiguous_buffer();
|
||||
return buffer;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue