VideoPlayer: Start fleshing out the user interface

This adds player widget with working play/pause controls, a seek bar
which currently only displays the current playback position, and a
button to cycle between the scaling modes.

The player uses the new PlaybackManager class to handle demuxing,
decoding, and frame presentation timing.

Currently, the volume control is non-functional.

Userland/Applications/VideoPlayer/CMakeLists.txt

@@ -6,6 +6,8 @@ serenity_component(
 
 set(SOURCES
     main.cpp
+    VideoFrameWidget.cpp
+    VideoPlayerWidget.cpp
 )
 
 serenity_bin(VideoPlayer)

Userland/Applications/VideoPlayer/VideoFrameWidget.cpp

@@ -0,0 +1,91 @@
+/*
+ * Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <LibGUI/Painter.h>
+#include <LibGfx/Bitmap.h>
+
+#include "VideoFrameWidget.h"
+
+namespace VideoPlayer {
+
+VideoFrameWidget::VideoFrameWidget()
+{
+    set_auto_resize(true);
+}
+
+void VideoFrameWidget::set_bitmap(Gfx::Bitmap const* bitmap)
+{
+    if (m_bitmap == bitmap)
+        return;
+
+    m_bitmap = bitmap;
+    if (m_bitmap && m_auto_resize)
+        set_fixed_size(m_bitmap->size());
+
+    update();
+}
+
+void VideoFrameWidget::set_auto_resize(bool value)
+{
+    m_auto_resize = value;
+
+    if (m_bitmap)
+        set_fixed_size(m_bitmap->size());
+}
+
+void VideoFrameWidget::mousedown_event(GUI::MouseEvent&)
+{
+    if (on_click)
+        on_click();
+}
+
+void VideoFrameWidget::paint_event(GUI::PaintEvent& event)
+{
+    Frame::paint_event(event);
+
+    GUI::Painter painter(*this);
+    painter.add_clip_rect(event.rect());
+
+    painter.fill_rect(frame_inner_rect(), Gfx::Color::Black);
+
+    if (!m_bitmap)
+        return;
+
+    if (m_sizing_mode == VideoSizingMode::Stretch) {
+        painter.draw_scaled_bitmap(frame_inner_rect(), *m_bitmap, m_bitmap->rect(), 1.0f, Gfx::Painter::ScalingMode::BilinearBlend);
+        return;
+    }
+
+    auto center = frame_inner_rect().center();
+
+    if (m_sizing_mode == VideoSizingMode::FullSize) {
+        painter.blit(center.translated(-m_bitmap->width() / 2, -m_bitmap->height() / 2), *m_bitmap, m_bitmap->rect());
+        return;
+    }
+
+    VERIFY(m_sizing_mode < VideoSizingMode::Sentinel);
+
+    auto aspect_ratio = m_bitmap->width() / static_cast<float>(m_bitmap->height());
+    auto display_aspect_ratio = frame_inner_rect().width() / static_cast<float>(frame_inner_rect().height());
+
+    Gfx::IntSize display_size;
+    if ((display_aspect_ratio > aspect_ratio) == (m_sizing_mode == VideoSizingMode::Fit)) {
+        display_size = {
+            (frame_inner_rect().height() * m_bitmap->width()) / m_bitmap->height(),
+            frame_inner_rect().height(),
+        };
+    } else {
+        display_size = {
+            frame_inner_rect().width(),
+            (frame_inner_rect().width() * m_bitmap->height()) / m_bitmap->width(),
+        };
+    }
+
+    auto display_rect = Gfx::IntRect(center.translated(-display_size.width() / 2, -display_size.height() / 2), display_size);
+    painter.draw_scaled_bitmap(display_rect, *m_bitmap, m_bitmap->rect(), 1.0f, Gfx::Painter::ScalingMode::BilinearBlend);
+}
+
+}

Userland/Applications/VideoPlayer/VideoFrameWidget.h

@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/StringView.h>
+#include <LibGUI/Event.h>
+#include <LibGUI/Frame.h>
+
+namespace VideoPlayer {
+
+enum class VideoSizingMode : u8 {
+    Fit,
+    Fill,
+    Stretch,
+    FullSize,
+    Sentinel
+};
+
+class VideoFrameWidget : public GUI::Frame {
+    C_OBJECT(VideoFrameWidget)
+public:
+    virtual ~VideoFrameWidget() override = default;
+
+    void set_bitmap(Gfx::Bitmap const*);
+    Gfx::Bitmap* bitmap() { return m_bitmap.ptr(); }
+    Gfx::Bitmap const* bitmap() const { return m_bitmap.ptr(); }
+
+    void set_sizing_mode(VideoSizingMode value) { m_sizing_mode = value; }
+    VideoSizingMode sizing_mode() const { return m_sizing_mode; }
+
+    void set_auto_resize(bool value);
+    bool auto_resize() const { return m_auto_resize; }
+
+    Function<void()> on_click;
+
+protected:
+    explicit VideoFrameWidget();
+
+    virtual void mousedown_event(GUI::MouseEvent&) override;
+    virtual void paint_event(GUI::PaintEvent&) override;
+
+private:
+    RefPtr<Gfx::Bitmap> m_bitmap;
+    VideoSizingMode m_sizing_mode { VideoSizingMode::Fit };
+    bool m_auto_resize { false };
+};
+
+constexpr StringView video_sizing_mode_name(VideoSizingMode mode)
+{
+    switch (mode) {
+    case VideoSizingMode::Fit:
+        return "Fit"sv;
+        break;
+    case VideoSizingMode::Fill:
+        return "Fill"sv;
+        break;
+    case VideoSizingMode::Stretch:
+        return "Stretch"sv;
+        break;
+    case VideoSizingMode::FullSize:
+        return "Full size"sv;
+        break;
+    default:
+        VERIFY_NOT_REACHED();
+    }
+}
+
+}

Userland/Applications/VideoPlayer/VideoPlayerWidget.cpp

@@ -0,0 +1,187 @@
+/*
+ * Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <LibGUI/Action.h>
+#include <LibGUI/BoxLayout.h>
+#include <LibGUI/ImageWidget.h>
+#include <LibGUI/Label.h>
+#include <LibGUI/MessageBox.h>
+#include <LibGUI/SeparatorWidget.h>
+#include <LibGUI/Slider.h>
+#include <LibGUI/Toolbar.h>
+#include <LibGUI/ToolbarContainer.h>
+#include <LibGUI/Window.h>
+
+#include "VideoPlayerWidget.h"
+
+namespace VideoPlayer {
+
+VideoPlayerWidget::VideoPlayerWidget(GUI::Window& window)
+    : m_window(window)
+{
+    set_fill_with_background_color(true);
+
+    set_layout<GUI::VerticalBoxLayout>();
+
+    m_video_display = add<VideoFrameWidget>();
+    m_video_display->set_auto_resize(false);
+    m_video_display->on_click = [&]() { toggle_pause(); };
+
+    auto& player_controls_widget = add<GUI::Widget>();
+    player_controls_widget.set_layout<GUI::VerticalBoxLayout>();
+    player_controls_widget.set_max_height(50);
+
+    m_seek_slider = player_controls_widget.add<GUI::HorizontalSlider>();
+    m_seek_slider->set_fixed_height(20);
+    m_seek_slider->set_enabled(false);
+
+    auto& toolbar_container = player_controls_widget.add<GUI::ToolbarContainer>();
+    m_toolbar = toolbar_container.add<GUI::Toolbar>();
+
+    m_play_icon = Gfx::Bitmap::try_load_from_file("/res/icons/16x16/play.png"sv).release_value_but_fixme_should_propagate_errors();
+    m_pause_icon = Gfx::Bitmap::try_load_from_file("/res/icons/16x16/pause.png"sv).release_value_but_fixme_should_propagate_errors();
+
+    m_play_pause_action = GUI::Action::create("Play", { Key_Space }, m_play_icon, [&](auto&) {
+        toggle_pause();
+    });
+
+    m_cycle_sizing_modes_action = GUI::Action::create("Sizing", [&](auto&) {
+        cycle_sizing_modes();
+    });
+
+    m_toolbar->add_action(*m_play_pause_action);
+    m_toolbar->add<GUI::VerticalSeparator>();
+    m_timestamp_label = m_toolbar->add<GUI::Label>();
+    m_timestamp_label->set_fixed_width(50);
+
+    m_toolbar->add<GUI::Widget>(); // Filler widget
+
+    m_toolbar->add_action(*m_cycle_sizing_modes_action);
+
+    m_toolbar->add<GUI::VerticalSeparator>();
+    m_volume_slider = m_toolbar->add<GUI::HorizontalSlider>();
+    m_volume_slider->set_min(0);
+    m_volume_slider->set_max(100);
+    m_volume_slider->set_fixed_width(100);
+}
+
+void VideoPlayerWidget::open_file(StringView filename)
+{
+    auto load_file_result = Video::PlaybackManager::from_file(this, filename);
+
+    if (load_file_result.is_error()) {
+        on_decoding_error(load_file_result.release_error());
+        return;
+    }
+
+    m_playback_manager = load_file_result.release_value();
+    resume_playback();
+}
+
+void VideoPlayerWidget::update_play_pause_icon()
+{
+    if (!m_playback_manager) {
+        m_play_pause_action->set_enabled(false);
+        m_play_pause_action->set_icon(m_play_icon);
+        return;
+    }
+
+    m_play_pause_action->set_enabled(true);
+
+    if (m_playback_manager->is_playing() || m_playback_manager->is_buffering())
+        m_play_pause_action->set_icon(m_pause_icon);
+    else
+        m_play_pause_action->set_icon(m_play_icon);
+}
+
+void VideoPlayerWidget::resume_playback()
+{
+    if (!m_playback_manager)
+        return;
+    m_playback_manager->resume_playback();
+    update_play_pause_icon();
+}
+
+void VideoPlayerWidget::pause_playback()
+{
+    if (!m_playback_manager)
+        return;
+    m_playback_manager->pause_playback();
+    update_play_pause_icon();
+}
+
+void VideoPlayerWidget::toggle_pause()
+{
+    if (!m_playback_manager)
+        return;
+    if (m_playback_manager->is_playing() || m_playback_manager->is_buffering())
+        pause_playback();
+    else
+        resume_playback();
+}
+
+void VideoPlayerWidget::on_decoding_error(Video::DecoderError error)
+{
+    StringView text_format;
+
+    switch (error.category()) {
+    case Video::DecoderErrorCategory::IO:
+        text_format = "Error while reading video:\n{}"sv;
+        break;
+    case Video::DecoderErrorCategory::Memory:
+        text_format = "Ran out of memory:\n{}"sv;
+        break;
+    case Video::DecoderErrorCategory::Corrupted:
+        text_format = "Video was corrupted:\n{}"sv;
+        break;
+    case Video::DecoderErrorCategory::Invalid:
+        text_format = "Invalid call:\n{}"sv;
+        break;
+    case Video::DecoderErrorCategory::NotImplemented:
+        text_format = "Video feature is not yet implemented:\n{}"sv;
+        break;
+    default:
+        text_format = "Unexpected error:\n{}"sv;
+        break;
+    }
+
+    GUI::MessageBox::show(&m_window, String::formatted(text_format, error.string_literal()), "Video Player encountered an error"sv);
+}
+
+void VideoPlayerWidget::event(Core::Event& event)
+{
+    if (event.type() == Video::EventType::DecoderErrorOccurred) {
+        auto& error_event = static_cast<Video::DecoderErrorEvent&>(event);
+        on_decoding_error(error_event.error());
+        error_event.accept();
+    } else if (event.type() == Video::EventType::VideoFramePresent) {
+        auto& frame_event = static_cast<Video::VideoFramePresentEvent&>(event);
+
+        m_video_display->set_bitmap(frame_event.frame());
+        m_video_display->repaint();
+
+        m_seek_slider->set_max(m_playback_manager->duration().to_milliseconds());
+        m_seek_slider->set_value(m_playback_manager->current_playback_time().to_milliseconds());
+        m_seek_slider->set_enabled(true);
+
+        frame_event.accept();
+    } else if (event.type() == Video::EventType::PlaybackStatusChange) {
+        update_play_pause_icon();
+        event.accept();
+    }
+
+    Widget::event(event);
+}
+
+void VideoPlayerWidget::cycle_sizing_modes()
+{
+    auto sizing_mode = m_video_display->sizing_mode();
+    sizing_mode = static_cast<VideoSizingMode>((to_underlying(sizing_mode) + 1) % to_underlying(VideoSizingMode::Sentinel));
+    m_video_display->set_sizing_mode(sizing_mode);
+    m_video_display->update();
+}
+
+}

Userland/Applications/VideoPlayer/VideoPlayerWidget.h

@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/FixedArray.h>
+#include <AK/NonnullRefPtr.h>
+#include <LibGUI/Forward.h>
+#include <LibGUI/Widget.h>
+#include <LibGfx/Forward.h>
+#include <LibVideo/DecoderError.h>
+#include <LibVideo/PlaybackManager.h>
+
+#include "VideoFrameWidget.h"
+
+namespace VideoPlayer {
+
+class VideoPlayerWidget final : public GUI::Widget {
+    C_OBJECT(VideoPlayerWidget)
+
+public:
+    void open_file(StringView filename);
+    void resume_playback();
+    void pause_playback();
+    void toggle_pause();
+
+private:
+    VideoPlayerWidget(GUI::Window&);
+
+    void update_play_pause_icon();
+    void on_decoding_error(Video::DecoderError);
+    void display_next_frame();
+
+    void cycle_sizing_modes();
+
+    void event(Core::Event&) override;
+
+    GUI::Window& m_window;
+
+    RefPtr<VideoFrameWidget> m_video_display;
+    RefPtr<GUI::HorizontalSlider> m_seek_slider;
+
+    RefPtr<GUI::Toolbar> m_toolbar;
+
+    RefPtr<Gfx::Bitmap> m_play_icon;
+    RefPtr<Gfx::Bitmap> m_pause_icon;
+
+    RefPtr<GUI::Action> m_play_pause_action;
+    RefPtr<GUI::Label> m_timestamp_label;
+    RefPtr<GUI::Action> m_cycle_sizing_modes_action;
+    RefPtr<GUI::HorizontalSlider> m_volume_slider;
+
+    RefPtr<Video::PlaybackManager> m_playback_manager;
+};
+
+}

Userland/Applications/VideoPlayer/main.cpp

@@ -7,103 +7,39 @@
 #include "LibVideo/Color/CodingIndependentCodePoints.h"
 #include "LibVideo/MatroskaDemuxer.h"
 #include <LibCore/ArgsParser.h>
-#include <LibCore/ElapsedTimer.h>
 #include <LibGUI/Application.h>
-#include <LibGUI/BoxLayout.h>
+#include <LibGUI/FilePicker.h>
-#include <LibGUI/ImageWidget.h>
+#include <LibGUI/Menu.h>
 #include <LibGUI/Window.h>
-#include <LibGfx/Bitmap.h>
 #include <LibMain/Main.h>
-#include <LibVideo/Color/ColorConverter.h>
+#include <LibVideo/PlaybackManager.h>
-#include <LibVideo/MatroskaReader.h>
+
-#include <LibVideo/VP9/Decoder.h>
+#include "VideoPlayerWidget.h"
 
 ErrorOr<int> serenity_main(Main::Arguments arguments)
 {
-    bool benchmark = false;
+    StringView filename = ""sv;
-    StringView filename = "/home/anon/Videos/test-webm.webm"sv;
-
     Core::ArgsParser args_parser;
-    args_parser.add_option(benchmark, "Benchmark the video decoder.", "benchmark", 'b');
     args_parser.add_positional_argument(filename, "The video file to display.", "filename", Core::ArgsParser::Required::No);
     args_parser.parse(arguments);
 
     auto app = TRY(GUI::Application::try_create(arguments));
     auto window = TRY(GUI::Window::try_create());
+    window->set_title("Video Player");
+    window->resize(640, 480);
+    window->set_resizable(true);
 
-    auto demuxer_result = Video::MatroskaDemuxer::from_file(filename);
+    auto main_widget = TRY(window->try_set_main_widget<VideoPlayer::VideoPlayerWidget>(window));
-    if (demuxer_result.is_error()) {
-        outln("Error parsing Matroska: {}", demuxer_result.release_error().string_literal());
-        return 1;
-    }
-    auto demuxer = demuxer_result.release_value();
-    auto tracks = demuxer->get_tracks_for_type(Video::TrackType::Video);
-    if (tracks.is_empty()) {
-        outln("No video tracks present.");
-        return 1;
-    }
-    auto track = tracks[0];
 
-    auto main_widget = TRY(window->try_set_main_widget<GUI::Widget>());
+    if (!filename.is_empty())
-    main_widget->set_fill_with_background_color(true);
+        main_widget->open_file(filename);
-    main_widget->set_layout<GUI::VerticalBoxLayout>();
-    auto image_widget = TRY(main_widget->try_add<GUI::ImageWidget>());
 
-    OwnPtr<Video::VideoDecoder> decoder = make<Video::VP9::Decoder>();
+    auto file_menu = TRY(window->try_add_menu("&File"));
-    auto frame_number = 0u;
+    TRY(file_menu->try_add_action(GUI::CommonActions::make_open_action([&](auto&) {
-
+        Optional<String> path = GUI::FilePicker::get_open_filepath(window, "Open video file...");
-    auto display_next_frame = [&]() {
+        if (path.has_value())
-        auto sample_result = demuxer->get_next_video_sample_for_track(track);
+            main_widget->open_file(path.value());
-
+    })));
-        if (sample_result.is_error()) {
-            outln("Error demuxing next sample {}: {}", frame_number, sample_result.release_error().string_literal());
-            return;
-        }
-
-        auto sample = sample_result.release_value();
-        auto result = decoder->receive_sample(sample->data());
-
-        if (result.is_error()) {
-            outln("Error decoding frame {}: {}", frame_number, result.error().string_literal());
-            return;
-        }
-
-        auto frame_result = decoder->get_decoded_frame();
-        if (frame_result.is_error()) {
-            outln("Error retrieving frame {}: {}", frame_number, frame_result.error().string_literal());
-            return;
-        }
-        auto frame = frame_result.release_value();
-
-        auto& cicp = frame->cicp();
-        cicp.adopt_specified_values(sample->container_cicp());
-        cicp.default_code_points_if_unspecified({ Video::ColorPrimaries::BT709, Video::TransferCharacteristics::BT709, Video::MatrixCoefficients::BT709, Video::ColorRange::Studio });
-
-        auto convert_result = frame->to_bitmap();
-        if (convert_result.is_error()) {
-            outln("Error creating bitmap for frame {}: {}", frame_number, convert_result.error().string_literal());
-            return;
-        }
-
-        image_widget->set_bitmap(convert_result.release_value());
-        image_widget->set_fixed_size(frame->size());
-        image_widget->update();
-
-        frame_number++;
-    };
-
-    image_widget->on_click = [&]() { display_next_frame(); };
-
-    if (benchmark) {
-        auto timer = Core::ElapsedTimer::start_new();
-        for (auto i = 0; i < 100; i++)
-            display_next_frame();
-        auto elapsed_time = timer.elapsed_time();
-        outln("Decoding 100 frames took {} ms", elapsed_time.to_milliseconds());
-        return 0;
-    }
-
-    display_next_frame();
 
     window->show();
     return app->exec();