RGBCube/serenity
1
Fork 0
mirror of https://github.com/RGBCube/serenity synced 2025-07-25 19:27:44 +00:00
Code Issues Activity Actions

LibVideo/VP9: Increase the size of summed boolean counts in merge_probs

Browse source 
This fixes an issue where probabilities that sum to greater than 255
would wrap and cause the maximum probability adaption to take effect.
This commit is contained in:
Zaggy1024 2023-02-01 18:57:23 -06:00 • committed by Jelle Raaijmakers
parent 7c1fe32af3
commit 4224f253af
2 changed files with 8 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

9
Userland/Libraries/LibVideo/VP9/Decoder.cpp
View file

@ -216,16 +216,19 @@ DecoderErrorOr<NonnullOwnPtr<VideoFrame>> Decoder::get_decoded_frame()
return m_video_frame_queue.dequeue();
}
u8 Decoder::merge_prob(u8 pre_prob, u8 count_0, u8 count_1, u8 count_sat, u8 max_update_factor)
u8 Decoder::merge_prob(u8 pre_prob, u32 count_0, u32 count_1, u8 count_sat, u8 max_update_factor)
{
auto total_decode_count = count_0 + count_1;
auto prob = (total_decode_count == 0) ? 128 : clip_3(1, 255, (count_0 * 256 + (total_decode_count >> 1)) / total_decode_count);
u8 prob = 128;
if (total_decode_count != 0) {
prob = static_cast<u8>(clip_3(1u, 255u, (count_0 * 256 + (total_decode_count >> 1)) / total_decode_count));
}
auto count = min(total_decode_count, count_sat);
auto factor = (max_update_factor * count) / count_sat;
return round_2(pre_prob * (256 - factor) + (prob * factor), 8);
}
u8 Decoder::merge_probs(int const* tree, int index, u8* probs, u8* counts, u8 count_sat, u8 max_update_factor)
u32 Decoder::merge_probs(int const* tree, int index, u8* probs, u8* counts, u8 count_sat, u8 max_update_factor)
{
auto s = tree[index];
auto left_count = (s <= 0) ? counts[-s] : merge_probs(tree, s, probs, counts, count_sat, max_update_factor);

4
Userland/Libraries/LibVideo/VP9/Decoder.h
View file

@ -48,8 +48,8 @@ private:
Vector<u16>& get_output_buffer(u8 plane);
/* (8.4) Probability Adaptation Process */
u8 merge_prob(u8 pre_prob, u8 count_0, u8 count_1, u8 count_sat, u8 max_update_factor);
u8 merge_probs(int const* tree, int index, u8* probs, u8* counts, u8 count_sat, u8 max_update_factor);
u8 merge_prob(u8 pre_prob, u32 count_0, u32 count_1, u8 count_sat, u8 max_update_factor);
u32 merge_probs(int const* tree, int index, u8* probs, u8* counts, u8 count_sat, u8 max_update_factor);
DecoderErrorOr<void> adapt_coef_probs(bool is_inter_predicted_frame);
DecoderErrorOr<void> adapt_non_coef_probs(FrameContext const&);
void adapt_probs(int const* tree, u8* probs, u8* counts);