From 9df2d6ee0f7cb3ed62307948eb3513603e14db25 Mon Sep 17 00:00:00 2001 From: Timothy Flynn Date: Wed, 21 Jun 2023 17:08:50 -0400 Subject: [PATCH] LibWeb: Implement scrubbing of the media element timeline and volume This implements the ability to drag the timeline and volume buttons on UA-rendered media controls. The two behave a bit differently: Volume is updated as the user drags the volume button. This isn't a very expensive operation, so updating in real-time and hearing the volume change feels nice. The current time, on the other hand, is not committed until the user releases the mouse button. Performing a seek every time we get a mouse- move event is pretty laggy, especially for video. However, we still want to render updates on the timeline itself (so the position of the button and the timestamp update as you drag). To do so, we internally pause the media and override the timestamp provided to the layout node. In the future, we may be able to seek video periodically to provide some visual feedback. For example, we can seek after every N seconds of scrubbing, or when the user pauses scrubbing for a while. --- .../LibWeb/HTML/HTMLMediaElement.cpp | 22 ++++ .../Libraries/LibWeb/HTML/HTMLMediaElement.h | 13 ++ .../LibWeb/Painting/MediaPaintable.cpp | 120 +++++++++++++++--- .../LibWeb/Painting/MediaPaintable.h | 11 ++ 4 files changed, 146 insertions(+), 20 deletions(-) diff --git a/Userland/Libraries/LibWeb/HTML/HTMLMediaElement.cpp b/Userland/Libraries/LibWeb/HTML/HTMLMediaElement.cpp index 59f4da8db2..e6a85865a3 100644 --- a/Userland/Libraries/LibWeb/HTML/HTMLMediaElement.cpp +++ b/Userland/Libraries/LibWeb/HTML/HTMLMediaElement.cpp @@ -1836,4 +1836,26 @@ void HTMLMediaElement::reject_pending_play_promises(ReadonlySpan, Optional display_time) +{ + if (display_time.has_value() && !m_display_time.has_value()) { + if (potentially_playing()) { + m_tracking_mouse_position_while_playing = true; + on_paused(); + } + } else if (!display_time.has_value() && m_display_time.has_value()) { + if (m_tracking_mouse_position_while_playing) { + m_tracking_mouse_position_while_playing = false; + on_playing(); + } + } + + m_display_time = move(display_time); +} + +double HTMLMediaElement::layout_display_time(Badge) const +{ + return m_display_time.value_or(current_time()); +} + } diff --git a/Userland/Libraries/LibWeb/HTML/HTMLMediaElement.h b/Userland/Libraries/LibWeb/HTML/HTMLMediaElement.h index e038a855e3..044d254f94 100644 --- a/Userland/Libraries/LibWeb/HTML/HTMLMediaElement.h +++ b/Userland/Libraries/LibWeb/HTML/HTMLMediaElement.h @@ -96,9 +96,19 @@ public: JS::NonnullGCPtr audio_tracks() const { return *m_audio_tracks; } JS::NonnullGCPtr video_tracks() const { return *m_video_tracks; } + enum class MouseTrackingComponent { + Timeline, + Volume, + }; + void set_layout_mouse_tracking_component(Badge, Optional mouse_tracking_component) { m_mouse_tracking_component = move(mouse_tracking_component); } + Optional const& layout_mouse_tracking_component(Badge) const { return m_mouse_tracking_component; } + void set_layout_mouse_position(Badge, Optional mouse_position) { m_mouse_position = move(mouse_position); } Optional const& layout_mouse_position(Badge) const { return m_mouse_position; } + void set_layout_display_time(Badge, Optional display_time); + double layout_display_time(Badge) const; + struct CachedLayoutBoxes { Optional control_box_rect; Optional playback_button_rect; @@ -259,7 +269,10 @@ private: bool m_seek_in_progress = false; // Cached state for layout. + Optional m_mouse_tracking_component; + bool m_tracking_mouse_position_while_playing { false }; Optional m_mouse_position; + Optional m_display_time; mutable CachedLayoutBoxes m_layout_boxes; }; diff --git a/Userland/Libraries/LibWeb/Painting/MediaPaintable.cpp b/Userland/Libraries/LibWeb/Painting/MediaPaintable.cpp index 1831edb349..437ae54c07 100644 --- a/Userland/Libraries/LibWeb/Painting/MediaPaintable.cpp +++ b/Userland/Libraries/LibWeb/Painting/MediaPaintable.cpp @@ -9,10 +9,11 @@ #include #include #include +#include #include -#include #include #include +#include #include namespace Web::Painting { @@ -101,9 +102,9 @@ MediaPaintable::Components MediaPaintable::compute_control_bar_components(PaintC remaining_rect.take_from_right(components.speaker_button_size + component_padding); } - auto current_time = human_readable_digital_time(round(media_element.current_time())); + auto display_time = human_readable_digital_time(round(media_element.layout_display_time({}))); auto duration = human_readable_digital_time(isnan(media_element.duration()) ? 0 : round(media_element.duration())); - components.timestamp = String::formatted("{} / {}", current_time, duration).release_value_but_fixme_should_propagate_errors(); + components.timestamp = String::formatted("{} / {}", display_time, duration).release_value_but_fixme_should_propagate_errors(); auto const& scaled_font = layout_node().scaled_font(context); components.timestamp_font = scaled_font.with_size(10); @@ -138,7 +139,7 @@ void MediaPaintable::paint_control_bar_playback_button(PaintContext& context, HT auto playback_button_offset_y = (components.playback_button_rect.height() - playback_button_size) / 2; auto playback_button_location = components.playback_button_rect.top_left().translated(playback_button_offset_x, playback_button_offset_y); - auto playback_button_is_hovered = mouse_position.has_value() && components.playback_button_rect.contains(*mouse_position); + auto playback_button_is_hovered = rect_is_hovered(media_element, components.playback_button_rect, mouse_position); auto playback_button_color = control_button_color(playback_button_is_hovered); if (media_element.paused()) { @@ -172,7 +173,7 @@ void MediaPaintable::paint_control_bar_timeline(PaintContext& context, HTML::HTM auto timelime_scrub_rect = components.timeline_rect; timelime_scrub_rect.shrink(components.timeline_button_size, timelime_scrub_rect.height() - components.timeline_button_size / 2); - auto playback_percentage = isnan(media_element.duration()) ? 0.0 : media_element.current_time() / media_element.duration(); + auto playback_percentage = isnan(media_element.duration()) ? 0.0 : media_element.layout_display_time({}) / media_element.duration(); auto playback_position = static_cast(static_cast(timelime_scrub_rect.width())) * playback_percentage; auto timeline_button_offset_x = static_cast(round(playback_position)); @@ -190,7 +191,7 @@ void MediaPaintable::paint_control_bar_timeline(PaintContext& context, HTML::HTM timeline_button_rect.shrink(timelime_scrub_rect.width() - components.timeline_button_size, timelime_scrub_rect.height() - components.timeline_button_size); timeline_button_rect.set_x(timelime_scrub_rect.x() + timeline_button_offset_x - components.timeline_button_size / 2); - auto timeline_is_hovered = mouse_position.has_value() && components.timeline_rect.contains(*mouse_position); + auto timeline_is_hovered = rect_is_hovered(media_element, components.timeline_rect, mouse_position, HTML::HTMLMediaElement::MouseTrackingComponent::Timeline); auto timeline_color = control_button_color(timeline_is_hovered); painter.fill_ellipse(timeline_button_rect.to_type(), timeline_color); } @@ -220,7 +221,7 @@ void MediaPaintable::paint_control_bar_speaker(PaintContext& context, HTML::HTML return position.to_type().to_type(); }; - auto speaker_button_is_hovered = mouse_position.has_value() && components.speaker_button_rect.contains(*mouse_position); + auto speaker_button_is_hovered = rect_is_hovered(media_element, components.speaker_button_rect, mouse_position); auto speaker_button_color = control_button_color(speaker_button_is_hovered); Gfx::AntiAliasingPainter painter { context.painter() }; @@ -274,12 +275,12 @@ void MediaPaintable::paint_control_bar_volume(PaintContext& context, HTML::HTMLM volume_button_rect.shrink(volume_scrub_rect.width() - components.volume_button_size, volume_scrub_rect.height() - components.volume_button_size); volume_button_rect.set_x(volume_scrub_rect.x() + volume_button_offset_x - components.volume_button_size / 2); - auto volume_is_hovered = mouse_position.has_value() && components.volume_rect.contains(*mouse_position); + auto volume_is_hovered = rect_is_hovered(media_element, components.volume_rect, mouse_position, HTML::HTMLMediaElement::MouseTrackingComponent::Volume); auto volume_color = control_button_color(volume_is_hovered); painter.fill_ellipse(volume_button_rect.to_type(), volume_color); } -MediaPaintable::DispatchEventOfSameName MediaPaintable::handle_mouseup(Badge, CSSPixelPoint position, unsigned button, unsigned) +MediaPaintable::DispatchEventOfSameName MediaPaintable::handle_mousedown(Badge, CSSPixelPoint position, unsigned button, unsigned) { if (button != GUI::MouseButton::Primary) return DispatchEventOfSameName::Yes; @@ -287,6 +288,39 @@ MediaPaintable::DispatchEventOfSameName MediaPaintable::handle_mouseup(Badge(layout_box().dom_node()); auto const& cached_layout_boxes = media_element.cached_layout_boxes({}); + if (cached_layout_boxes.timeline_rect.has_value() && cached_layout_boxes.timeline_rect->contains(position)) + media_element.set_layout_mouse_tracking_component({}, HTML::HTMLMediaElement::MouseTrackingComponent::Timeline); + else if (cached_layout_boxes.volume_rect.has_value() && cached_layout_boxes.volume_rect->contains(position)) + media_element.set_layout_mouse_tracking_component({}, HTML::HTMLMediaElement::MouseTrackingComponent::Volume); + + if (media_element.layout_mouse_tracking_component({}).has_value()) + const_cast(browsing_context()).event_handler().set_mouse_event_tracking_layout_node(&layout_node()); + + return DispatchEventOfSameName::Yes; +} + +MediaPaintable::DispatchEventOfSameName MediaPaintable::handle_mouseup(Badge, CSSPixelPoint position, unsigned button, unsigned) +{ + auto& media_element = *verify_cast(layout_box().dom_node()); + auto const& cached_layout_boxes = media_element.cached_layout_boxes({}); + + auto was_tracking_mouse = media_element.layout_mouse_tracking_component({}).has_value(); + auto was_tracking_timeline = media_element.layout_mouse_tracking_component({}) == HTML::HTMLMediaElement::MouseTrackingComponent::Timeline; + media_element.set_layout_mouse_tracking_component({}, {}); + + if (was_tracking_mouse) { + if (was_tracking_timeline) { + set_current_time(media_element, *cached_layout_boxes.timeline_rect, position, Temporary::No); + media_element.set_layout_display_time({}, {}); + } + + const_cast(browsing_context()).event_handler().set_mouse_event_tracking_layout_node(nullptr); + return DispatchEventOfSameName::Yes; + } + + if (button != GUI::MouseButton::Primary) + return DispatchEventOfSameName::Yes; + // FIXME: This runs from outside the context of any user script, so we do not have a running execution // context. This pushes one to allow the promise creation hook to run. auto& environment_settings = document().relevant_settings_object(); @@ -309,12 +343,7 @@ MediaPaintable::DispatchEventOfSameName MediaPaintable::handle_mouseup(Badgecontains(position)) { - auto x_offset = position.x() - cached_layout_boxes.timeline_rect->x(); - auto x_percentage = static_cast(x_offset) / static_cast(cached_layout_boxes.timeline_rect->width()); - - auto position = static_cast(x_percentage) * media_element.duration(); - media_element.set_current_time(position); - + set_current_time(media_element, *cached_layout_boxes.timeline_rect, position, Temporary::No); return DispatchEventOfSameName::Yes; } @@ -324,11 +353,7 @@ MediaPaintable::DispatchEventOfSameName MediaPaintable::handle_mouseup(Badgecontains(position)) { - auto x_offset = position.x() - cached_layout_boxes.volume_rect->x(); - auto volume = static_cast(x_offset) / static_cast(cached_layout_boxes.volume_rect->width()); - - media_element.set_volume(volume).release_value_but_fixme_should_propagate_errors(); - + set_volume(media_element, *cached_layout_boxes.volume_rect, position); return DispatchEventOfSameName::Yes; } @@ -342,6 +367,21 @@ MediaPaintable::DispatchEventOfSameName MediaPaintable::handle_mouseup(Badge, CSSPixelPoint position, unsigned, unsigned) { auto& media_element = *verify_cast(layout_box().dom_node()); + auto const& cached_layout_boxes = media_element.cached_layout_boxes({}); + + if (auto const& mouse_tracking_component = media_element.layout_mouse_tracking_component({}); mouse_tracking_component.has_value()) { + switch (*mouse_tracking_component) { + case HTML::HTMLMediaElement::MouseTrackingComponent::Timeline: + if (cached_layout_boxes.timeline_rect.has_value()) + set_current_time(media_element, *cached_layout_boxes.timeline_rect, position, Temporary::Yes); + break; + + case HTML::HTMLMediaElement::MouseTrackingComponent::Volume: + if (cached_layout_boxes.volume_rect.has_value()) + set_volume(media_element, *cached_layout_boxes.volume_rect, position); + break; + } + } if (absolute_rect().contains(position)) { media_element.set_layout_mouse_position({}, position); @@ -352,4 +392,44 @@ MediaPaintable::DispatchEventOfSameName MediaPaintable::handle_mousemove(Badge(x_offset) / static_cast(timeline_rect.width()); + auto position = static_cast(x_percentage) * media_element.duration(); + + switch (temporarily) { + case Temporary::Yes: + media_element.set_layout_display_time({}, position); + break; + case Temporary::No: + media_element.set_current_time(position); + break; + } +} + +void MediaPaintable::set_volume(HTML::HTMLMediaElement& media_element, CSSPixelRect volume_rect, CSSPixelPoint mouse_position) +{ + auto x_offset = mouse_position.x() - volume_rect.x(); + x_offset = max(x_offset, 0); + x_offset = min(x_offset, volume_rect.width()); + + auto volume = static_cast(x_offset) / static_cast(volume_rect.width()); + media_element.set_volume(volume).release_value_but_fixme_should_propagate_errors(); +} + +bool MediaPaintable::rect_is_hovered(HTML::HTMLMediaElement const& media_element, Optional const& rect, Optional const& mouse_position, Optional const& allowed_mouse_tracking_component) +{ + if (auto const& mouse_tracking_component = media_element.layout_mouse_tracking_component({}); mouse_tracking_component.has_value()) + return mouse_tracking_component == allowed_mouse_tracking_component; + + if (!rect.has_value() || !mouse_position.has_value()) + return false; + + return rect->contains(*mouse_position); +} + } diff --git a/Userland/Libraries/LibWeb/Painting/MediaPaintable.h b/Userland/Libraries/LibWeb/Painting/MediaPaintable.h index e6c0cf4ee3..9337f6ffa0 100644 --- a/Userland/Libraries/LibWeb/Painting/MediaPaintable.h +++ b/Userland/Libraries/LibWeb/Painting/MediaPaintable.h @@ -7,6 +7,7 @@ #pragma once #include +#include #include #include @@ -43,6 +44,7 @@ private: }; virtual bool wants_mouse_events() const override { return true; } + virtual DispatchEventOfSameName handle_mousedown(Badge, CSSPixelPoint, unsigned button, unsigned modifiers) override; virtual DispatchEventOfSameName handle_mouseup(Badge, CSSPixelPoint, unsigned button, unsigned modifiers) override; virtual DispatchEventOfSameName handle_mousemove(Badge, CSSPixelPoint, unsigned buttons, unsigned modifiers) override; @@ -52,6 +54,15 @@ private: static void paint_control_bar_timestamp(PaintContext&, Components const&); static void paint_control_bar_speaker(PaintContext&, HTML::HTMLMediaElement const&, Components const& components, Optional const& mouse_position); static void paint_control_bar_volume(PaintContext&, HTML::HTMLMediaElement const&, Components const&, Optional const& mouse_position); + + enum class Temporary { + Yes, + No, + }; + static void set_current_time(HTML::HTMLMediaElement& media_element, CSSPixelRect timeline_rect, CSSPixelPoint mouse_position, Temporary); + static void set_volume(HTML::HTMLMediaElement& media_element, CSSPixelRect volume_rect, CSSPixelPoint mouse_position); + + static bool rect_is_hovered(HTML::HTMLMediaElement const& media_element, Optional const& rect, Optional const& mouse_position, Optional const& allowed_mouse_tracking_component = {}); }; }