RGBCube/serenity
1
Fork 0
mirror of https://github.com/RGBCube/serenity synced 2025-07-27 04:27:45 +00:00
Code Issues Activity Actions

LibVideo: Implement accurate seeking to inter frames in PlaybackManager

Browse source 
This implements the PlaybackManager portion of seeking, so that it can
seek to any frame in the video by dropping all preceding frames until
it reaches the seek point.

MatroskaDemuxer currently will only seek to the start of the video.
That means that every seek has to drop all the frames until it comes
across the target timestamp.
This commit is contained in:
Zaggy1024 2022-11-14 00:54:21 -06:00 • committed by Andreas Kling
parent 9a9fe08449
commit 5c2cede2c9
3 changed files with 65 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

14
Userland/Libraries/LibVideo/Containers/Matroska/MatroskaDemuxer.cpp
View file

@ -53,15 +53,13 @@ DecoderErrorOr<MatroskaDemuxer::TrackStatus*> MatroskaDemuxer::get_track_status(
return &m_track_statuses.get(track).release_value();
}
DecoderErrorOr<void> MatroskaDemuxer::seek_to_most_recent_keyframe(Track track, Time timestamp)
DecoderErrorOr<void> MatroskaDemuxer::seek_to_most_recent_keyframe(Track track, Time)
{
if (timestamp.is_zero()) {
// Removing the track status will cause us to start from the beginning.
m_track_statuses.remove(track);
return {};
}
return DecoderError::not_implemented();
// Removing the track status will cause us to start from the beginning.
// FIXME: We just go back to the beginning always, so that the PlaybackManager seeking
// technology can be tested.
m_track_statuses.remove(track);
return {};
}
DecoderErrorOr<NonnullOwnPtr<Sample>> MatroskaDemuxer::get_next_sample_for_track(Track track)

59
Userland/Libraries/LibVideo/PlaybackManager.cpp
View file

@ -58,7 +58,7 @@ void PlaybackManager::set_playback_status(PlaybackStatus status)
restart_playback();
m_last_present_in_real_time = Time::now_monotonic();
m_present_timer->start(0);
} else {
} else if (!is_seeking()) {
m_last_present_in_media_time = current_playback_time();
m_last_present_in_real_time = Time::zero();
m_present_timer->stop();
@ -70,16 +70,27 @@ void PlaybackManager::set_playback_status(PlaybackStatus status)
void PlaybackManager::resume_playback()
{
if (is_seeking()) {
set_playback_status(PlaybackStatus::SeekingPlaying);
return;
}
set_playback_status(PlaybackStatus::Playing);
}
void PlaybackManager::pause_playback()
{
if (is_seeking()) {
set_playback_status(PlaybackStatus::SeekingPaused);
return;
}
set_playback_status(PlaybackStatus::Paused);
}
Time PlaybackManager::current_playback_time()
{
if (is_seeking())
return m_seek_to_media_time;
VERIFY(!m_last_present_in_media_time.is_negative());
if (m_status == PlaybackStatus::Playing)
return m_last_present_in_media_time + (Time::now_monotonic() - m_last_present_in_real_time);
return m_last_present_in_media_time;
@ -108,13 +119,28 @@ void PlaybackManager::on_decoder_error(DecoderError error)
}
}
void PlaybackManager::end_seek()
{
dbgln_if(PLAYBACK_MANAGER_DEBUG, "We've finished seeking, reset seek target and play");
VERIFY(!m_seek_to_media_time.is_negative());
m_last_present_in_media_time = m_seek_to_media_time;
m_seek_to_media_time = Time::min();
if (m_status == PlaybackStatus::SeekingPlaying) {
set_playback_status(PlaybackStatus::Playing);
return;
}
VERIFY(m_status == PlaybackStatus::SeekingPaused);
set_playback_status(PlaybackStatus::Paused);
}
void PlaybackManager::update_presented_frame()
{
Optional<FrameQueueItem> future_frame_item;
bool should_present_frame = false;
// Skip frames until we find a frame past the current playback time, and keep the one that precedes it to display.
while (m_status == PlaybackStatus::Playing && !m_frame_queue->is_empty()) {
while ((m_status == PlaybackStatus::Playing || is_seeking()) && !m_frame_queue->is_empty()) {
future_frame_item.emplace(m_frame_queue->dequeue());
m_decode_timer->start(0);
@ -124,7 +150,7 @@ void PlaybackManager::update_presented_frame()
break;
}
if (m_next_frame.has_value()) {
if (m_next_frame.has_value() && !is_seeking()) {
dbgln_if(PLAYBACK_MANAGER_DEBUG, "At {}ms: Dropped {} in favor of {}", current_playback_time().to_milliseconds(), m_next_frame->debug_string(), future_frame_item->debug_string());
m_skipped_frames++;
}
@ -162,7 +188,10 @@ void PlaybackManager::update_presented_frame()
// If we have a frame, send it for presentation.
if (should_present_frame) {
m_last_present_in_media_time = current_playback_time();
if (is_seeking())
end_seek();
else
m_last_present_in_media_time = current_playback_time();
m_last_present_in_real_time = Time::now_monotonic();
m_main_loop.post_event(m_event_handler, make<VideoFramePresentEvent>(m_next_frame.value().bitmap()));
dbgln_if(PLAYBACK_MANAGER_DEBUG, "Sent frame for presentation");
@ -194,16 +223,24 @@ void PlaybackManager::update_presented_frame()
void PlaybackManager::seek_to_timestamp(Time timestamp)
{
dbgln_if(PLAYBACK_MANAGER_DEBUG, "Seeking to {}ms", timestamp.to_milliseconds());
m_last_present_in_media_time = Time::zero();
m_last_present_in_real_time = Time::zero();
m_frame_queue->clear();
m_next_frame.clear();
m_skipped_frames = 0;
// FIXME: When the demuxer is getting samples off the main thread in the future, this needs to
// mutex so that seeking can't happen while that thread is getting a sample.
auto result = m_demuxer->seek_to_most_recent_keyframe(m_selected_video_track, timestamp);
if (result.is_error())
on_decoder_error(result.release_error());
if (is_playing())
set_playback_status(PlaybackStatus::SeekingPlaying);
else
set_playback_status(PlaybackStatus::SeekingPaused);
m_frame_queue->clear();
m_next_frame.clear();
m_skipped_frames = 0;
m_seek_to_media_time = timestamp;
m_last_present_in_media_time = Time::min();
m_last_present_in_real_time = Time::zero();
m_present_timer->stop();
m_decode_timer->start(0);
}
void PlaybackManager::restart_playback()
@ -232,7 +269,7 @@ bool PlaybackManager::decode_and_queue_one_sample()
if (_temporary_result.is_error()) { \
dbgln_if(PLAYBACK_MANAGER_DEBUG, "Enqueued decoder error: {}", _temporary_result.error().string_literal()); \
m_frame_queue->enqueue(FrameQueueItem::error_marker(_temporary_result.release_error())); \
m_present_timer->start(0); \
m_present_timer->start(0); \
return false; \
} \
_temporary_result.release_value(); \
@ -298,7 +335,7 @@ void PlaybackManager::on_decode_timer()
}
// Continually decode until buffering is complete
if (is_buffering())
if (is_buffering() || is_seeking())
m_decode_timer->start(0);
}

16
Userland/Libraries/LibVideo/PlaybackManager.h
View file

@ -28,7 +28,8 @@ enum class PlaybackStatus {
Playing,
Paused,
Buffering,
Seeking,
SeekingPlaying,
SeekingPaused,
Stopped,
Corrupted,
};
@ -102,7 +103,8 @@ public:
void pause_playback();
void restart_playback();
void seek_to_timestamp(Time);
bool is_playing() const { return m_status == PlaybackStatus::Playing || m_status == PlaybackStatus::Buffering; }
bool is_playing() const { return m_status == PlaybackStatus::Playing || m_status == PlaybackStatus::SeekingPlaying || m_status == PlaybackStatus::Buffering; }
bool is_seeking() const { return m_status == PlaybackStatus::SeekingPlaying || m_status == PlaybackStatus::SeekingPaused; }
bool is_buffering() const { return m_status == PlaybackStatus::Buffering; }
bool is_stopped() const { return m_status == PlaybackStatus::Stopped || m_status == PlaybackStatus::Corrupted; }
@ -118,7 +120,7 @@ public:
private:
void set_playback_status(PlaybackStatus status);
bool prepare_next_frame();
void end_seek();
void update_presented_frame();
// May run off the main thread
@ -133,6 +135,8 @@ private:
Time m_last_present_in_media_time = Time::zero();
Time m_last_present_in_real_time = Time::zero();
Time m_seek_to_media_time = Time::min();
NonnullOwnPtr<Demuxer> m_demuxer;
Track m_selected_video_track;
NonnullOwnPtr<VideoDecoder> m_decoder;
@ -213,8 +217,10 @@ inline StringView playback_status_to_string(PlaybackStatus status)
return "Paused"sv;
case PlaybackStatus::Buffering:
return "Buffering"sv;
case PlaybackStatus::Seeking:
return "Seeking"sv;
case PlaybackStatus::SeekingPlaying:
return "SeekingPlaying"sv;
case PlaybackStatus::SeekingPaused:
return "SeekingPaused"sv;
case PlaybackStatus::Stopped:
return "Stopped"sv;
case PlaybackStatus::Corrupted: