LibVideo: Abstract media container format demuxing

This creates an abstract Demuxer class to allow multiple container
container formats to be easily used by video playback systems.

Userland/Libraries/LibVideo/CMakeLists.txt

@@ -2,6 +2,7 @@ set(SOURCES
     Color/ColorConverter.cpp
     Color/ColorPrimaries.cpp
     Color/TransferCharacteristics.cpp
+    MatroskaDemuxer.cpp
     MatroskaReader.cpp
     VideoFrame.cpp
     VP9/BitStream.cpp

Userland/Libraries/LibVideo/DecoderError.h

@@ -22,6 +22,7 @@ using DecoderErrorOr = ErrorOr<T, DecoderError>;
 enum class DecoderErrorCategory : u32 {
     Unknown,
     IO,
+    EndOfStream,
     Memory,
     // The input is corrupted.
     Corrupted,

Userland/Libraries/LibVideo/Demuxer.h

@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/NonnullOwnPtr.h>
+#include <LibCore/Object.h>
+
+#include "DecoderError.h"
+#include "Sample.h"
+#include "Track.h"
+
+namespace Video {
+
+class Demuxer {
+public:
+    virtual ~Demuxer() = default;
+
+    virtual Vector<Track> get_tracks_for_type(TrackType type) = 0;
+
+    DecoderErrorOr<NonnullOwnPtr<VideoSample>> get_next_video_sample_for_track(Track track)
+    {
+        VERIFY(track.type() == TrackType::Video);
+        auto sample = TRY(get_next_sample_for_track(track));
+        VERIFY(sample->is_video_sample());
+        return sample.release_nonnull<VideoSample>();
+    }
+
+    virtual DecoderErrorOr<void> seek_to_most_recent_keyframe(Track track, size_t timestamp) = 0;
+
+    virtual Time duration() = 0;
+
+protected:
+    virtual DecoderErrorOr<NonnullOwnPtr<Sample>> get_next_sample_for_track(Track track) = 0;
+};
+
+}

Userland/Libraries/LibVideo/MatroskaDemuxer.cpp

@@ -0,0 +1,119 @@
+/*
+ * Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include "MatroskaDemuxer.h"
+
+namespace Video {
+
+DecoderErrorOr<NonnullOwnPtr<MatroskaDemuxer>> MatroskaDemuxer::from_file(StringView filename)
+{
+    // FIXME: MatroskaReader should return errors.
+    auto nullable_document = MatroskaReader::parse_matroska_from_file(filename);
+    if (!nullable_document)
+        return DecoderError::format(DecoderErrorCategory::IO, "Failed to open matroska from file '{}'", filename);
+    auto document = nullable_document.release_nonnull();
+    return make<MatroskaDemuxer>(document);
+}
+
+DecoderErrorOr<NonnullOwnPtr<MatroskaDemuxer>> MatroskaDemuxer::from_data(Span<u8 const> data)
+{
+    // FIXME: MatroskaReader should return errors.
+    auto nullable_document = MatroskaReader::parse_matroska_from_data(data.data(), data.size());
+    if (!nullable_document)
+        return DecoderError::format(DecoderErrorCategory::IO, "Failed to open matroska from data");
+    auto document = nullable_document.release_nonnull();
+    return make<MatroskaDemuxer>(document);
+}
+
+Vector<Track> MatroskaDemuxer::get_tracks_for_type(TrackType type)
+{
+    Video::TrackEntry::TrackType matroska_track_type;
+
+    switch (type) {
+    case TrackType::Video:
+        matroska_track_type = Video::TrackEntry::TrackType::Video;
+        break;
+    case TrackType::Audio:
+        matroska_track_type = Video::TrackEntry::TrackType::Audio;
+        break;
+    case TrackType::Subtitles:
+        matroska_track_type = Video::TrackEntry::TrackType::Subtitle;
+        break;
+    }
+
+    Vector<Track> tracks;
+
+    for (auto const& track_table_entry : m_document->tracks()) {
+        auto const& track_entry = track_table_entry.value;
+        if (matroska_track_type == track_entry->track_type())
+            tracks.append(Track(type, track_entry->track_number()));
+    }
+
+    // FIXME: Sort the vector, presumably the hashtable will not have a consistent order.
+    return tracks;
+}
+
+ErrorOr<MatroskaDemuxer::TrackStatus*> MatroskaDemuxer::get_track_status(Track track)
+{
+    if (!m_track_statuses.contains(track))
+        TRY(m_track_statuses.try_set(track, TrackStatus()));
+
+    return &m_track_statuses.get(track).release_value();
+}
+
+DecoderErrorOr<void> MatroskaDemuxer::seek_to_most_recent_keyframe(Track track, size_t timestamp)
+{
+    if (timestamp == 0) {
+        auto track_status = DECODER_TRY_ALLOC(get_track_status(track));
+        track_status->m_cluster_index = 0;
+        track_status->m_block_index = 0;
+        track_status->m_frame_index = 0;
+        return {};
+    }
+
+    return DecoderError::not_implemented();
+}
+
+DecoderErrorOr<NonnullOwnPtr<Sample>> MatroskaDemuxer::get_next_sample_for_track(Track track)
+{
+    auto track_status = DECODER_TRY_ALLOC(get_track_status(track));
+
+    for (; track_status->m_cluster_index < m_document->clusters().size(); track_status->m_cluster_index++) {
+        auto const& cluster = m_document->clusters()[track_status->m_cluster_index];
+        for (; track_status->m_block_index < cluster.blocks().size(); track_status->m_block_index++) {
+            auto const& block = cluster.blocks()[track_status->m_block_index];
+            if (block.track_number() != track.identifier())
+                continue;
+            if (track_status->m_frame_index < block.frame_count()) {
+                switch (track.type()) {
+                case TrackType::Video: {
+                    // FIXME: This makes a copy of the sample, which shouldn't be necessary.
+                    //        Matroska should make a RefPtr<ByteBuffer>, probably.
+                    auto cicp = m_document->track_for_track_number(track.identifier())->video_track()->color_format.to_cicp();
+                    Time timestamp = Time::from_nanoseconds((cluster.timestamp() + block.timestamp()) * m_document->segment_information()->timestamp_scale());
+                    return make<VideoSample>(block.frame(track_status->m_frame_index++), cicp, timestamp);
+                }
+                default:
+                    return DecoderError::not_implemented();
+                }
+            }
+            track_status->m_frame_index = 0;
+        }
+        track_status->m_block_index = 0;
+    }
+    return DecoderError::with_description(DecoderErrorCategory::EndOfStream, "End of stream reached."sv);
+}
+
+Time MatroskaDemuxer::duration()
+{
+    if (!m_document->segment_information().has_value())
+        return Time::zero();
+    if (!m_document->segment_information()->duration().has_value())
+        return Time::zero();
+    return Time::from_nanoseconds(m_document->segment_information()->duration().value() * m_document->segment_information()->timestamp_scale());
+}
+
+}

Userland/Libraries/LibVideo/MatroskaDemuxer.h

@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/HashMap.h>
+
+#include "Demuxer.h"
+#include "MatroskaReader.h"
+
+namespace Video {
+
+class MatroskaDemuxer final : public Demuxer {
+public:
+    // FIXME: We should instead accept some abstract data streaming type so that the demuxer
+    //        can work with non-contiguous data.
+    static DecoderErrorOr<NonnullOwnPtr<MatroskaDemuxer>> from_file(StringView filename);
+    static DecoderErrorOr<NonnullOwnPtr<MatroskaDemuxer>> from_data(Span<u8 const> data);
+
+    MatroskaDemuxer(NonnullOwnPtr<MatroskaDocument>& document)
+        : m_document(move(document))
+    {
+    }
+
+    Vector<Track> get_tracks_for_type(TrackType type) override;
+
+    DecoderErrorOr<void> seek_to_most_recent_keyframe(Track track, size_t timestamp) override;
+
+    Time duration() override;
+
+protected:
+    DecoderErrorOr<NonnullOwnPtr<Sample>> get_next_sample_for_track(Track track) override;
+
+private:
+    struct TrackStatus {
+        size_t m_cluster_index { 0 };
+        size_t m_block_index { 0 };
+        size_t m_frame_index { 0 };
+    };
+
+    ErrorOr<TrackStatus*> get_track_status(Track track);
+
+    NonnullOwnPtr<MatroskaDocument> m_document;
+
+    HashMap<Track, TrackStatus> m_track_statuses;
+};
+
+}

Userland/Libraries/LibVideo/Sample.h

@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/ByteBuffer.h>
+#include <AK/Time.h>
+#include <LibVideo/Color/CodingIndependentCodePoints.h>
+
+namespace Video {
+
+class Sample {
+public:
+    virtual ~Sample() = default;
+
+    virtual bool is_video_sample() const { return false; }
+};
+
+class VideoSample : public Sample {
+public:
+    VideoSample(ByteBuffer const& data, CodingIndependentCodePoints container_cicp, Time timestamp)
+        : m_data(data)
+        , m_container_cicp(container_cicp)
+        , m_timestamp(timestamp)
+    {
+    }
+
+    bool is_video_sample() const override { return true; }
+    ByteBuffer const& data() const { return m_data; }
+    CodingIndependentCodePoints container_cicp() const { return m_container_cicp; }
+    Time timestamp() const { return m_timestamp; }
+
+private:
+    ByteBuffer m_data;
+    CodingIndependentCodePoints m_container_cicp;
+    Time m_timestamp;
+};
+
+// FIXME: Add samples for audio, subtitles, etc.
+
+}

Userland/Libraries/LibVideo/Track.h

@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/HashFunctions.h>
+#include <AK/Traits.h>
+#include <AK/Types.h>
+
+namespace Video {
+
+enum class TrackType : u32 {
+    Video,
+    Audio,
+    Subtitles,
+};
+
+struct Track {
+public:
+    Track(TrackType type, size_t identifier)
+        : m_type(type)
+        , m_identifier(identifier)
+    {
+    }
+
+    TrackType type() { return m_type; }
+    size_t identifier() const { return m_identifier; }
+
+    bool operator==(Track const& other) const
+    {
+        return m_type == other.m_type && m_identifier == other.m_identifier;
+    }
+    unsigned hash() const
+    {
+        return pair_int_hash(to_underlying(m_type), m_identifier);
+    }
+
+private:
+    TrackType m_type;
+    size_t m_identifier;
+};
+
+}
+
+template<>
+struct AK::Traits<Video::Track> : public GenericTraits<Video::Track> {
+    static unsigned hash(Video::Track const& t) { return t.hash(); }
+};