From 3d9106b1b5e76149da621d898b922705397c9a4f Mon Sep 17 00:00:00 2001 From: Timothy Flynn Date: Mon, 10 Apr 2023 10:07:23 -0400 Subject: [PATCH] LibWeb: Begin tracking HTMLMediaElement playback positions There are several playback positions to be tracked, depending on the state of the media element. --- .../LibWeb/HTML/HTMLMediaElement.cpp | 172 +++++++++++++++++- .../Libraries/LibWeb/HTML/HTMLMediaElement.h | 20 ++ .../LibWeb/HTML/HTMLMediaElement.idl | 1 + Userland/Libraries/LibWeb/HTML/VideoTrack.cpp | 9 + Userland/Libraries/LibWeb/HTML/VideoTrack.h | 2 + 5 files changed, 195 insertions(+), 9 deletions(-) diff --git a/Userland/Libraries/LibWeb/HTML/HTMLMediaElement.cpp b/Userland/Libraries/LibWeb/HTML/HTMLMediaElement.cpp index 632345a113..5112dfb941 100644 --- a/Userland/Libraries/LibWeb/HTML/HTMLMediaElement.cpp +++ b/Userland/Libraries/LibWeb/HTML/HTMLMediaElement.cpp @@ -124,6 +124,55 @@ WebIDL::ExceptionOr HTMLMediaElement::load() return {}; } +// https://html.spec.whatwg.org/multipage/media.html#dom-media-currenttime +double HTMLMediaElement::current_time() const +{ + // The currentTime attribute must, on getting, return the media element's default playback start position, unless that is zero, + // in which case it must return the element's official playback position. The returned value must be expressed in seconds. + if (m_default_playback_start_position != 0) + return m_default_playback_start_position; + return m_official_playback_position; +} + +// https://html.spec.whatwg.org/multipage/media.html#dom-media-currenttime +void HTMLMediaElement::set_current_time(double current_time) +{ + // On setting, if the media element's readyState is HAVE_NOTHING, then it must set the media element's default playback start + // position to the new value; otherwise, it must set the official playback position to the new value and then seek to the new + // value. The new value must be interpreted as being in seconds. + if (m_ready_state == ReadyState::HaveNothing) { + m_default_playback_start_position = current_time; + } else { + m_official_playback_position = current_time; + + // FIXME: Seek to the provided position. + } +} + +// https://html.spec.whatwg.org/multipage/media.html#time-marches-on#playing-the-media-resource:current-playback-position-13 +void HTMLMediaElement::set_current_playback_position(double playback_position) +{ + // When the current playback position of a media element changes (e.g. due to playback or seeking), the user agent must + // run the time marches on steps. To support use cases that depend on the timing accuracy of cue event firing, such as + // synchronizing captions with shot changes in a video, user agents should fire cue events as close as possible to their + // position on the media timeline, and ideally within 20 milliseconds. If the current playback position changes while the + // steps are running, then the user agent must wait for the steps to complete, and then must immediately rerun the steps. + // These steps are thus run as often as possible or needed. + // FIXME: Detect "the current playback position changes while the steps are running". + m_current_playback_position = playback_position; + + // FIXME: Regarding the official playback position, the spec states: + // + // Any time the user agent provides a stable state, the official playback position must be set to the current playback position. + // https://html.spec.whatwg.org/multipage/media.html#time-marches-on#playing-the-media-resource:official-playback-position-2 + // + // We do not currently have a means to track a "stable state", so for now, keep the official playback position + // in sync with the current playback position. + m_official_playback_position = m_current_playback_position; + + time_marches_on(); +} + // https://html.spec.whatwg.org/multipage/media.html#dom-media-duration double HTMLMediaElement::duration() const { @@ -244,10 +293,21 @@ WebIDL::ExceptionOr HTMLMediaElement::load_element() } // FIXME: 7. If seeking is true, set it to false. - // FIXME: 8. Set the current playback position to 0. - // Set the official playback position to 0. - // If this changed the official playback position, then queue a media element task given the media element to fire an - // event named timeupdate at the media element. + + // 8. Set the current playback position to 0. + m_current_playback_position = 0; + + if (m_official_playback_position != 0) { + // Set the official playback position to 0. + m_official_playback_position = 0; + + // If this changed the official playback position, then queue a media element task given the media element to fire an + // event named timeupdate at the media element. + queue_a_media_element_task([this] { + dispatch_time_update_event().release_value_but_fixme_should_propagate_errors(); + }); + } + // FIXME: 9. Set the timeline offset to Not-a-Number (NaN). // 10. Update the duration attribute to Not-a-Number (NaN). @@ -671,7 +731,10 @@ WebIDL::ExceptionOr HTMLMediaElement::process_media_data(Function // FIXME: 1. Establish the media timeline for the purposes of the current playback position and the earliest possible position, based on the media data. // FIXME: 2. Update the timeline offset to the date and time that corresponds to the zero time in the media timeline established in the previous step, // if any. If no explicit time and date is given by the media resource, the timeline offset must be set to Not-a-Number (NaN). - // FIXME: 3. Set the current playback position and the official playback position to the earliest possible position. + + // 3. Set the current playback position and the official playback position to the earliest possible position. + m_current_playback_position = 0; + m_official_playback_position = 0; // 4. Update the duration attribute with the time of the last frame of the resource, if known, on the media timeline established above. If it is // not known (e.g. a stream that is in principle infinite), update the duration attribute to the value positive Infinity. @@ -694,9 +757,18 @@ WebIDL::ExceptionOr HTMLMediaElement::process_media_data(Function // 6. Set the readyState attribute to HAVE_METADATA. set_ready_state(ReadyState::HaveMetadata); - // FIXME: 7. Let jumped be false. - // FIXME: 8. If the media element's default playback start position is greater than zero, then seek to that time, and let jumped be true. - // FIXME: 9. Let the media element's default playback start position be zero. + // 7. Let jumped be false. + [[maybe_unused]] auto jumped = false; + + // 8. If the media element's default playback start position is greater than zero, then seek to that time, and let jumped be true. + if (m_default_playback_start_position > 0) { + // FIXME: Seek to the default playback position. + jumped = true; + } + + // 9. Let the media element's default playback start position be zero. + m_default_playback_start_position = 0; + // FIXME: 10. Let the initial playback position be zero. // FIXME: 11. If either the media resource or the URL of the current media resource indicate a particular start time, then set the initial playback // position to that time and, if jumped is still false, seek to that time. @@ -939,7 +1011,8 @@ WebIDL::ExceptionOr HTMLMediaElement::pause_element() reject_pending_play_promises(promises, "Media playback was paused"_fly_string.release_value_but_fixme_should_propagate_errors()); }); - // FIXME: 4. Set the official playback position to the current playback position. + // 4. Set the official playback position to the current playback position. + m_official_playback_position = m_current_playback_position; } return {}; @@ -985,6 +1058,87 @@ WebIDL::ExceptionOr HTMLMediaElement::dispatch_time_update_event() return {}; } +// https://html.spec.whatwg.org/multipage/media.html#time-marches-on +void HTMLMediaElement::time_marches_on(TimeMarchesOnReason reason) +{ + // FIXME: 1. Let current cues be a list of cues, initialized to contain all the cues of all the hidden or showing text tracks + // of the media element (not the disabled ones) whose start times are less than or equal to the current playback + // position and whose end times are greater than the current playback position. + // FIXME: 2. Let other cues be a list of cues, initialized to contain all the cues of hidden and showing text tracks of the + // media element that are not present in current cues. + // FIXME: 3. Let last time be the current playback position at the time this algorithm was last run for this media element, + // if this is not the first time it has run. + // FIXME: 4. If the current playback position has, since the last time this algorithm was run, only changed through its usual + // monotonic increase during normal playback, then let missed cues be the list of cues in other cues whose start times + // are greater than or equal to last time and whose end times are less than or equal to the current playback position. + // Otherwise, let missed cues be an empty list. + // FIXME: 5. Remove all the cues in missed cues that are also in the media element's list of newly introduced cues, and then + // empty the element's list of newly introduced cues. + + // 6. If the time was reached through the usual monotonic increase of the current playback position during normal + // playback, and if the user agent has not fired a timeupdate event at the element in the past 15 to 250ms and is + // not still running event handlers for such an event, then the user agent must queue a media element task given + // the media element to fire an event named timeupdate at the element. (In the other cases, such as explicit seeks, + // relevant events get fired as part of the overall process of changing the current playback position.) + if (reason == TimeMarchesOnReason::NormalPlayback && !m_running_time_update_event_handler) { + auto dispatch_event = true; + + if (m_last_time_update_event_time.has_value()) { + auto time_since_last_event = Time::now_monotonic() - *m_last_time_update_event_time; + dispatch_event = time_since_last_event.to_milliseconds() > 250; + } + + if (dispatch_event) { + queue_a_media_element_task([this]() { + dispatch_time_update_event().release_value_but_fixme_should_propagate_errors(); + }); + } + } + + // FIXME: 7. If all of the cues in current cues have their text track cue active flag set, none of the cues in other cues have + // their text track cue active flag set, and missed cues is empty, then return. + // FIXME: 8. If the time was reached through the usual monotonic increase of the current playback position during normal playback, + // and there are cues in other cues that have their text track cue pause-on-exit flag set and that either have their + // text track cue active flag set or are also in missed cues, then immediately pause the media element. + // FIXME: 9. Let events be a list of tasks, initially empty. Each task in this list will be associated with a text track, a + // text track cue, and a time, which are used to sort the list before the tasks are queued. + // + // Let affected tracks be a list of text tracks, initially empty. + // + // When the steps below say to prepare an event named event for a text track cue target with a time time, the user + // agent must run these steps: + // 1. Let track be the text track with which the text track cue target is associated. + // 2. Create a task to fire an event named event at target. + // 3. Add the newly created task to events, associated with the time time, the text track track, and the text + // track cue target. + // 4. Add track to affected tracks. + // FIXME: 10. For each text track cue in missed cues, prepare an event named enter for the TextTrackCue object with the text + // track cue start time. + // FIXME: 11. For each text track cue in other cues that either has its text track cue active flag set or is in missed cues, + // prepare an event named exit for the TextTrackCue object with the later of the text track cue end time and the + /// text track cue start time. + // FIXME: 12. For each text track cue in current cues that does not have its text track cue active flag set, prepare an event + // named enter for the TextTrackCue object with the text track cue start time. + // FIXME: 13. Sort the tasks in events in ascending time order (tasks with earlier times first). + // + // Further sort tasks in events that have the same time by the relative text track cue order of the text track cues + // associated with these tasks. + // + // Finally, sort tasks in events that have the same time and same text track cue order by placing tasks that fire + // enter events before those that fire exit events. + // FIXME: 14. Queue a media element task given the media element for each task in events, in list order. + // FIXME: 15. Sort affected tracks in the same order as the text tracks appear in the media element's list of text tracks, and + // remove duplicates. + // FIXME: 16. For each text track in affected tracks, in the list order, queue a media element task given the media element to + // fire an event named cuechange at the TextTrack object, and, if the text track has a corresponding track element, + // to then fire an event named cuechange at the track element as well. + // FIXME: 17. Set the text track cue active flag of all the cues in the current cues, and unset the text track cue active flag + // of all the cues in the other cues. + // FIXME: 18. Run the rules for updating the text track rendering of each of the text tracks in affected tracks that are showing, + // providing the text track's text track language as the fallback language if it is not the empty string. For example, + // for text tracks based on WebVTT, the rules for updating the display of WebVTT text tracks. +} + // https://html.spec.whatwg.org/multipage/media.html#take-pending-play-promises JS::MarkedVector> HTMLMediaElement::take_pending_play_promises() { diff --git a/Userland/Libraries/LibWeb/HTML/HTMLMediaElement.h b/Userland/Libraries/LibWeb/HTML/HTMLMediaElement.h index d5e958ef40..1219b3e104 100644 --- a/Userland/Libraries/LibWeb/HTML/HTMLMediaElement.h +++ b/Userland/Libraries/LibWeb/HTML/HTMLMediaElement.h @@ -47,6 +47,11 @@ public: ReadyState ready_state() const { return m_ready_state; } WebIDL::ExceptionOr load(); + + double current_time() const; + void set_current_time(double); + void set_current_playback_position(double); + double duration() const; bool paused() const { return m_paused; } WebIDL::ExceptionOr> play(); @@ -90,6 +95,12 @@ private: WebIDL::ExceptionOr dispatch_time_update_event(); + enum class TimeMarchesOnReason { + NormalPlayback, + Other, + }; + void time_marches_on(TimeMarchesOnReason = TimeMarchesOnReason::NormalPlayback); + JS::MarkedVector> take_pending_play_promises(); void resolve_pending_play_promises(ReadonlySpan> promises); void reject_pending_play_promises(ReadonlySpan> promises, JS::NonnullGCPtr error); @@ -114,6 +125,15 @@ private: ReadyState m_ready_state { ReadyState::HaveNothing }; bool m_first_data_load_event_since_load_start { false }; + // https://html.spec.whatwg.org/multipage/media.html#current-playback-position + double m_current_playback_position { 0 }; + + // https://html.spec.whatwg.org/multipage/media.html#official-playback-position + double m_official_playback_position { 0 }; + + // https://html.spec.whatwg.org/multipage/media.html#default-playback-start-position + double m_default_playback_start_position { 0 }; + // https://html.spec.whatwg.org/multipage/media.html#dom-media-duration double m_duration { NAN }; diff --git a/Userland/Libraries/LibWeb/HTML/HTMLMediaElement.idl b/Userland/Libraries/LibWeb/HTML/HTMLMediaElement.idl index 46e9bd5cd7..d655e10052 100644 --- a/Userland/Libraries/LibWeb/HTML/HTMLMediaElement.idl +++ b/Userland/Libraries/LibWeb/HTML/HTMLMediaElement.idl @@ -30,6 +30,7 @@ interface HTMLMediaElement : HTMLElement { readonly attribute unsigned short readyState; // playback state + attribute double currentTime; readonly attribute unrestricted double duration; readonly attribute boolean paused; [Reflect, CEReactions] attribute boolean autoplay; diff --git a/Userland/Libraries/LibWeb/HTML/VideoTrack.cpp b/Userland/Libraries/LibWeb/HTML/VideoTrack.cpp index c3e23dbbfc..78d6767c73 100644 --- a/Userland/Libraries/LibWeb/HTML/VideoTrack.cpp +++ b/Userland/Libraries/LibWeb/HTML/VideoTrack.cpp @@ -5,6 +5,7 @@ */ #include +#include #include #include #include @@ -31,6 +32,9 @@ VideoTrack::VideoTrack(JS::Realm& realm, JS::NonnullGCPtr medi m_playback_manager->on_video_frame = [this](auto frame) { if (is(*m_media_element)) verify_cast(*m_media_element).set_current_frame({}, move(frame)); + + auto playback_position_ms = static_cast(position().to_milliseconds()); + m_media_element->set_current_playback_position(playback_position_ms / 1000.0); }; m_playback_manager->on_decoder_error = [](auto) { @@ -77,6 +81,11 @@ void VideoTrack::pause_video(Badge) m_playback_manager->pause_playback(); } +Time VideoTrack::position() const +{ + return m_playback_manager->current_playback_time(); +} + Time VideoTrack::duration() const { return m_playback_manager->selected_video_track().video_data().duration; diff --git a/Userland/Libraries/LibWeb/HTML/VideoTrack.h b/Userland/Libraries/LibWeb/HTML/VideoTrack.h index 77e37cecf6..e0bf17e12b 100644 --- a/Userland/Libraries/LibWeb/HTML/VideoTrack.h +++ b/Userland/Libraries/LibWeb/HTML/VideoTrack.h @@ -25,7 +25,9 @@ public: void play_video(Badge); void pause_video(Badge); + Time position() const; Time duration() const; + u64 pixel_width() const; u64 pixel_height() const;