/* * Copyright (c) 2021, Hunter Salyer * Copyright (c) 2022, Gregory Bertilson * * SPDX-License-Identifier: BSD-2-Clause */ #pragma once #include #include #include #include #include #include #include #include #include "BooleanDecoder.h" #include "ContextStorage.h" #include "Enums.h" #include "LookupTables.h" #include "MotionVector.h" #include "SyntaxElementCounter.h" #include "Utilities.h" namespace Video::VP9 { enum class FrameShowMode { CreateAndShowNewFrame, ShowExistingFrame, DoNotShowFrame, }; struct Quantizers { u16 y_ac_quantizer { 0 }; u16 uv_ac_quantizer { 0 }; u16 y_dc_quantizer { 0 }; u16 uv_dc_quantizer { 0 }; }; struct FrameContext { public: static ErrorOr create(ReadonlyBytes data, Vector2D& contexts) { return FrameContext( data, TRY(try_make(data)), TRY(try_make()), contexts); } FrameContext(FrameContext const&) = delete; FrameContext(FrameContext&&) = default; ReadonlyBytes stream_data; NonnullOwnPtr stream; BigEndianInputBitStream bit_stream; DecoderErrorOr create_range_decoder(size_t size) { if (size > stream->remaining()) return DecoderError::corrupted("Range decoder size invalid"sv); auto compressed_header_data = ReadonlyBytes(stream_data.data() + stream->offset(), size); // 9.2.1: The Boolean decoding process specified in section 9.2.2 is invoked to read a marker syntax element from the // bitstream. It is a requirement of bitstream conformance that the value read is equal to 0. auto decoder = DECODER_TRY(DecoderErrorCategory::Corrupted, BooleanDecoder::initialize(compressed_header_data)); if (decoder.read_bool(128)) return DecoderError::corrupted("Range decoder marker was non-zero"sv); DECODER_TRY(DecoderErrorCategory::Corrupted, bit_stream.discard(size)); return decoder; } NonnullOwnPtr counter; u8 profile { 0 }; FrameType type { FrameType::KeyFrame }; bool is_inter_predicted() const { return type == FrameType::InterFrame; } bool error_resilient_mode { false }; bool parallel_decoding_mode { false }; bool should_replace_probability_context { false }; bool shows_a_frame() const { return m_frame_show_mode != FrameShowMode::DoNotShowFrame; } bool shows_a_new_frame() const { return m_frame_show_mode == FrameShowMode::CreateAndShowNewFrame; } bool shows_existing_frame() const { return m_frame_show_mode == FrameShowMode::ShowExistingFrame; } void set_frame_hidden() { m_frame_show_mode = FrameShowMode::DoNotShowFrame; } void set_existing_frame_to_show(u8 index) { m_frame_show_mode = FrameShowMode::ShowExistingFrame; m_existing_frame_index = index; } u8 existing_frame_index() const { return m_existing_frame_index; } bool use_previous_frame_motion_vectors { false }; ColorConfig color_config {}; u8 reference_frames_to_update_flags { 0 }; bool should_update_reference_frame_at_index(u8 index) const { return (reference_frames_to_update_flags & (1 << index)) != 0; } u8 probability_context_index { 0 }; Gfx::Size size() const { return m_size; } ErrorOr set_size(Gfx::Size size) { m_size = size; // From spec, compute_image_size( ) m_rows = pixels_to_blocks(size.height() + 7u); m_columns = pixels_to_blocks(size.width() + 7u); return m_block_contexts.try_resize(m_rows, m_columns); } u32 rows() const { return m_rows; } u32 columns() const { return m_columns; } u32 superblock_rows() const { return blocks_ceiled_to_superblocks(rows()); } u32 superblock_columns() const { return blocks_ceiled_to_superblocks(columns()); } // Calculates the output size for each plane in the frame. Gfx::Size decoded_size(bool uv) const { if (uv) { return { y_size_to_uv_size(color_config.subsampling_y, blocks_to_pixels(columns())), y_size_to_uv_size(color_config.subsampling_y, blocks_to_pixels(rows())), }; } return { blocks_to_pixels(columns()), blocks_to_pixels(rows()), }; } Vector2D const& block_contexts() const { return m_block_contexts; } Gfx::Size render_size { 0, 0 }; Gfx::Size log2_of_tile_counts { 0, 0 }; // This group of fields is only needed for inter-predicted frames. Array reference_frame_indices; Array reference_frame_sign_biases; bool high_precision_motion_vectors_allowed { false }; InterpolationFilter interpolation_filter { InterpolationFilter::Switchable }; u8 loop_filter_level { 0 }; u8 loop_filter_sharpness { 0 }; bool loop_filter_delta_enabled { false }; Array loop_filter_reference_deltas; Array loop_filter_mode_deltas; // Set based on quantization_params( ) in the spec. bool lossless { false }; Array segment_quantizers; bool segmentation_enabled { false }; // Note: We can use Optional> for these tree probabilities, but unfortunately it seems to have measurable performance overhead. bool use_full_segment_id_tree { false }; Array full_segment_id_tree_probabilities; bool use_predicted_segment_id_tree { false }; Array predicted_segment_id_tree_probabilities; bool should_use_absolute_segment_base_quantizer { false }; SegmentationFeatures segmentation_features; SegmentFeatureStatus get_segment_feature(u8 segment_id, SegmentFeature feature) const { return segmentation_features[segment_id][to_underlying(feature)]; } u16 header_size_in_bytes { 0 }; TransformMode transform_mode; // This group also is only needed for inter-predicted frames. ReferenceMode reference_mode; ReferenceFrameType fixed_reference_type; ReferenceFramePair variable_reference_types; private: friend struct TileContext; FrameContext(ReadonlyBytes data, NonnullOwnPtr stream, NonnullOwnPtr counter, Vector2D& contexts) : stream_data(data) , stream(move(stream)) , bit_stream(MaybeOwned(*this->stream)) , counter(move(counter)) , m_block_contexts(contexts) { } FrameShowMode m_frame_show_mode { FrameShowMode::CreateAndShowNewFrame }; u8 m_existing_frame_index { 0 }; Gfx::Size m_size { 0, 0 }; u32 m_rows { 0 }; u32 m_columns { 0 }; // FIXME: From spec: NOTE – We are using a 2D array to store the SubModes for clarity. It is possible to reduce memory // consumption by only storing one intra mode for each 8x8 horizontal and vertical position, i.e. to use two 1D // arrays instead. // I think should also apply to other fields that are only accessed relative to the current block. Worth looking // into how much of this context needs to be stored for the whole frame vs a row or column from the current tile. Vector2D& m_block_contexts; }; static ErrorOr create_non_zero_tokens(u32 size_in_sub_blocks, bool subsampling) { return NonZeroTokens { TRY(FixedArray::create(size_in_sub_blocks)), TRY(FixedArray::create(size_in_sub_blocks >>= subsampling)), TRY(FixedArray::create(size_in_sub_blocks)), }; } template static Span safe_slice(Span span, u32 start, u32 size) { return span.slice(start, min(size, span.size() - start)); } static NonZeroTokensView create_non_zero_tokens_view(NonZeroTokensView non_zero_tokens, u32 start_in_sub_blocks, u32 size_in_sub_blocks, bool subsampling) { NonZeroTokensView result; // Y plane result[0] = safe_slice(non_zero_tokens[0], start_in_sub_blocks, size_in_sub_blocks); // UV planes start_in_sub_blocks >>= subsampling; size_in_sub_blocks >>= subsampling; result[1] = safe_slice(non_zero_tokens[1], start_in_sub_blocks, size_in_sub_blocks); result[2] = safe_slice(non_zero_tokens[2], start_in_sub_blocks, size_in_sub_blocks); return result; } static NonZeroTokensView create_non_zero_tokens_view(NonZeroTokens& non_zero_tokens, u32 start_in_sub_blocks, u32 size_in_sub_blocks, bool subsampling) { return create_non_zero_tokens_view({ non_zero_tokens[0].span(), non_zero_tokens[1].span(), non_zero_tokens[2].span() }, start_in_sub_blocks, size_in_sub_blocks, subsampling); } struct TileContext { public: static DecoderErrorOr try_create(FrameContext& frame_context, u32 tile_size, u32 rows_start, u32 rows_end, u32 columns_start, u32 columns_end, PartitionContextView above_partition_context, NonZeroTokensView above_non_zero_tokens, SegmentationPredictionContextView above_segmentation_ids) { auto width = columns_end - columns_start; auto height = rows_end - rows_start; auto context_view = frame_context.m_block_contexts.view(rows_start, columns_start, height, width); return TileContext { frame_context, TRY(frame_context.create_range_decoder(tile_size)), DECODER_TRY_ALLOC(try_make()), rows_start, rows_end, columns_start, columns_end, context_view, above_partition_context, above_non_zero_tokens, above_segmentation_ids, DECODER_TRY_ALLOC(PartitionContext::create(superblocks_to_blocks(blocks_ceiled_to_superblocks(height)))), DECODER_TRY_ALLOC(create_non_zero_tokens(blocks_to_sub_blocks(height), frame_context.color_config.subsampling_y)), DECODER_TRY_ALLOC(SegmentationPredictionContext::create(height)), }; } Vector2D const& frame_block_contexts() const { return frame_context.block_contexts(); } FrameContext const& frame_context; BooleanDecoder decoder; NonnullOwnPtr counter; u32 rows_start { 0 }; u32 rows_end { 0 }; u32 columns_start { 0 }; u32 columns_end { 0 }; u32 rows() const { return rows_end - rows_start; } u32 columns() const { return columns_end - columns_start; } Vector2DView block_contexts_view; PartitionContextView above_partition_context; NonZeroTokensView above_non_zero_tokens; SegmentationPredictionContextView above_segmentation_ids; PartitionContext left_partition_context; NonZeroTokens left_non_zero_tokens; SegmentationPredictionContext left_segmentation_ids; }; struct BlockContext { static BlockContext create(TileContext& tile_context, u32 row, u32 column, BlockSubsize size) { auto contexts_view = tile_context.block_contexts_view.view( row - tile_context.rows_start, column - tile_context.columns_start, min(num_8x8_blocks_high_lookup[size], tile_context.frame_context.rows() - row), min(num_8x8_blocks_wide_lookup[size], tile_context.frame_context.columns() - column)); auto size_in_blocks = block_size_to_blocks(size); auto size_in_sub_blocks = block_size_to_sub_blocks(get_subsampled_block_size(size, false, false)); return BlockContext { .frame_context = tile_context.frame_context, .tile_context = tile_context, .decoder = tile_context.decoder, .counter = *tile_context.counter, .row = row, .column = column, .size = size, .contexts_view = contexts_view, .above_non_zero_tokens = create_non_zero_tokens_view(tile_context.above_non_zero_tokens, blocks_to_sub_blocks(column - tile_context.columns_start), size_in_sub_blocks.width(), tile_context.frame_context.color_config.subsampling_x), .above_segmentation_ids = safe_slice(tile_context.above_segmentation_ids, column - tile_context.columns_start, size_in_blocks.width()), .left_non_zero_tokens = create_non_zero_tokens_view(tile_context.left_non_zero_tokens, blocks_to_sub_blocks(row - tile_context.rows_start), size_in_sub_blocks.height(), tile_context.frame_context.color_config.subsampling_y), .left_segmentation_ids = safe_slice(tile_context.left_segmentation_ids.span(), row - tile_context.rows_start, size_in_blocks.height()), }; } Vector2D const& frame_block_contexts() const { return frame_context.block_contexts(); } FrameContext const& frame_context; TileContext const& tile_context; BooleanDecoder& decoder; SyntaxElementCounter& counter; u32 row { 0 }; u32 column { 0 }; BlockSubsize size; Gfx::Size get_size_in_sub_blocks() const { return block_size_to_sub_blocks(size); } Vector2DView contexts_view; u8 segment_id { 0 }; bool should_skip_residuals { false }; TransformSize transform_size { Transform_4x4 }; ReferenceFramePair reference_frame_types {}; bool is_inter_predicted() const { return reference_frame_types.primary != ReferenceFrameType::None; } bool is_compound() const { return reference_frame_types.secondary != ReferenceFrameType::None; } Array sub_block_prediction_modes {}; PredictionMode y_prediction_mode() const { return sub_block_prediction_modes.last(); } PredictionMode& y_prediction_mode() { return sub_block_prediction_modes.last(); } PredictionMode uv_prediction_mode { 0 }; InterpolationFilter interpolation_filter { EightTap }; Array sub_block_motion_vectors {}; Array residual_tokens {}; // Indexed by ReferenceFrame enum. Array mode_context {}; NonZeroTokensView above_non_zero_tokens; SegmentationPredictionContextView above_segmentation_ids; NonZeroTokensView left_non_zero_tokens; SegmentationPredictionContextView left_segmentation_ids; SegmentFeatureStatus get_segment_feature(SegmentFeature feature) const { return frame_context.get_segment_feature(segment_id, feature); } }; struct BlockMotionVectorCandidateSet { MotionVector near_vector; MotionVector nearest_vector; MotionVector best_vector; }; struct MotionVectorCandidate { ReferenceFrameType type; MotionVector vector; }; }