WindowServer: Simplify determining transparent/opaque occlusions

By moving the logic to determine what window areas (shadow, frame,
content) into WindowFrame::opaque/transparent_render_rects we can
simplify the occlusion calculation and properly handle more
arbitrary opaque/transparent areas.

This also solves the problem where we would render the entire
window frame as transparency only because the frame had a window
shadow.

Userland/Services/WindowServer/Compositor.cpp

@@ -893,37 +893,29 @@ void Compositor::recompute_occlusions()
         Gfx::DisjointRectSet visible_rects(screen_rect);
         bool have_transparent = false;
         WindowManager::the().for_each_visible_window_from_front_to_back([&](Window& w) {
-            auto window_frame_rect = w.frame().render_rect().intersected(screen_rect);
             w.transparency_wallpaper_rects().clear();
             auto& visible_opaque = w.opaque_rects();
             auto& transparency_rects = w.transparency_rects();
-            if (w.is_minimized() || window_frame_rect.is_empty()) {
+            if (w.is_minimized()) {
                 visible_opaque.clear();
                 transparency_rects.clear();
                 return IterationDecision::Continue;
             }
 
-            Gfx::DisjointRectSet opaque_covering;
+            auto transparent_render_rects = w.frame().transparent_render_rects().intersected(screen_rect);
-            if (w.is_opaque()) {
+            auto opaque_render_rects = w.frame().opaque_render_rects().intersected(screen_rect);
-                transparency_rects.clear();
+            if (transparent_render_rects.is_empty() && opaque_render_rects.is_empty()) {
-                if (w.frame().is_opaque()) {
-                    visible_opaque = visible_rects.intersected(window_frame_rect);
-                } else {
-                    auto window_rect = w.rect().intersected(screen_rect);
-                    visible_opaque = visible_rects.intersected(window_rect);
-                    transparency_rects.add_many(window_frame_rect.shatter(window_rect));
-                }
-            } else {
                 visible_opaque.clear();
-                if (w.frame().is_opaque()) {
+                transparency_rects.clear();
-                    auto window_rect = w.rect().intersected(screen_rect);
+                return IterationDecision::Continue;
-                    visible_opaque.add_many(window_frame_rect.shatter(window_rect));
-                    transparency_rects = visible_rects.intersected(window_rect);
-                } else {
-                    transparency_rects = visible_rects.intersected(window_frame_rect);
-                }
             }
 
+            visible_opaque = visible_rects.intersected(opaque_render_rects);
+            transparency_rects = move(transparent_render_rects);
+
+            auto render_rect = w.frame().render_rect();
+
+            Gfx::DisjointRectSet opaque_covering;
             bool found_this_window = false;
             WindowManager::the().for_each_visible_window_from_back_to_front([&](Window& w2) {
                 if (!found_this_window) {
@@ -934,18 +926,22 @@ void Compositor::recompute_occlusions()
 
                 if (w2.is_minimized())
                     return IterationDecision::Continue;
-                auto window_frame_rect2 = w2.frame().render_rect().intersected(screen_rect);
+
-                auto covering_rect = window_frame_rect2.intersected(window_frame_rect);
+                if (!render_rect.intersects(w2.frame().render_rect()))
-                if (covering_rect.is_empty())
                     return IterationDecision::Continue;
 
-                auto add_opaque = [&](const Gfx::IntRect& covering) -> bool {
+                auto opaque_rects = w2.frame().opaque_render_rects().intersected(render_rect);
+                auto transparent_rects = w2.frame().transparent_render_rects().intersected(render_rect);
+                if (opaque_rects.is_empty() && transparent_rects.is_empty())
+                    return IterationDecision::Continue;
+
+                for (auto& covering : opaque_rects.rects()) {
                     opaque_covering.add(covering);
-                    if (opaque_covering.contains(window_frame_rect)) {
+                    if (opaque_covering.contains(render_rect)) {
                         // This window (including frame) is entirely covered by another opaque window
                         visible_opaque.clear();
                         transparency_rects.clear();
-                        return false;
+                        return IterationDecision::Break;
                     }
                     if (!visible_opaque.is_empty()) {
                         auto uncovered_opaque = visible_opaque.shatter(covering);
@@ -956,9 +952,8 @@ void Compositor::recompute_occlusions()
                         auto uncovered_transparency = transparency_rects.shatter(covering);
                         transparency_rects = move(uncovered_transparency);
                     }
-                    return true;
+                }
-                };
+                for (auto& covering : transparent_rects.rects()) {
-                auto add_transparent = [&](const Gfx::IntRect& covering) {
                     visible_rects.for_each_intersected(covering, [&](const Gfx::IntRect& intersected) {
                         transparency_rects.add(intersected);
                         if (!visible_opaque.is_empty()) {
@@ -967,29 +962,6 @@ void Compositor::recompute_occlusions()
                         }
                         return IterationDecision::Continue;
                     });
-                };
-                if (w2.is_opaque()) {
-                    if (w2.frame().is_opaque()) {
-                        if (!add_opaque(covering_rect))
-                            return IterationDecision::Break;
-                    } else {
-                        auto covering_window_rect = covering_rect.intersected(w2.rect());
-                        if (!add_opaque(covering_window_rect))
-                            return IterationDecision::Break;
-                        for (auto& covering_frame_rect : covering_rect.shatter(covering_window_rect))
-                            add_transparent(covering_frame_rect);
-                    }
-                } else {
-                    if (w2.frame().is_opaque()) {
-                        auto covering_window_rect = covering_rect.intersected(w2.rect());
-                        for (auto& covering_frame_rect : covering_rect.shatter(covering_window_rect)) {
-                            if (!add_opaque(covering_frame_rect))
-                                return IterationDecision::Break;
-                        }
-                        add_transparent(covering_window_rect);
-                    } else {
-                        add_transparent(covering_rect);
-                    }
                 }
 
                 return IterationDecision::Continue;
@@ -1001,20 +973,7 @@ void Compositor::recompute_occlusions()
             VERIFY(!visible_opaque.intersects(transparency_rects));
 
             // Determine visible area for the window below
-            if (w.is_opaque()) {
+            visible_rects = visible_rects.shatter(visible_opaque);
-                if (w.frame().is_opaque()) {
-                    auto visible_rects_below_window = visible_rects.shatter(window_frame_rect);
-                    visible_rects = move(visible_rects_below_window);
-                } else {
-                    auto visible_rects_below_window = visible_rects.shatter(w.rect().intersected(screen_rect));
-                    visible_rects = move(visible_rects_below_window);
-                }
-            } else if (w.frame().is_opaque()) {
-                for (auto& rect : window_frame_rect.shatter(w.rect())) {
-                    auto visible_rects_below_window = visible_rects.shatter(rect);
-                    visible_rects = move(visible_rects_below_window);
-                }
-            }
             return IterationDecision::Continue;
         });
 

Userland/Services/WindowServer/WindowFrame.cpp

@@ -196,10 +196,8 @@ Gfx::Bitmap* WindowFrame::window_shadow() const
     }
 }
 
-bool WindowFrame::frame_has_alpha() const
+bool WindowFrame::has_shadow() const
 {
-    if (m_has_alpha_channel)
-        return true;
     if (auto* shadow_bitmap = window_shadow(); shadow_bitmap && shadow_bitmap->format() == Gfx::BitmapFormat::BGRA8888)
         return true;
     return false;
@@ -514,6 +512,39 @@ Gfx::IntRect WindowFrame::render_rect() const
     return inflated_for_shadow(rect());
 }
 
+Gfx::DisjointRectSet WindowFrame::opaque_render_rects() const
+{
+    if (has_alpha_channel()) {
+        if (m_window.is_opaque())
+            return m_window.rect();
+        return {};
+    }
+    if (m_window.is_opaque())
+        return rect();
+    Gfx::DisjointRectSet opaque_rects;
+    opaque_rects.add_many(rect().shatter(m_window.rect()));
+    return opaque_rects;
+}
+
+Gfx::DisjointRectSet WindowFrame::transparent_render_rects() const
+{
+    if (has_alpha_channel()) {
+        if (m_window.is_opaque()) {
+            Gfx::DisjointRectSet transparent_rects;
+            transparent_rects.add_many(render_rect().shatter(m_window.rect()));
+            return transparent_rects;
+        }
+        return render_rect();
+    }
+
+    Gfx::DisjointRectSet transparent_rects;
+    if (has_shadow())
+        transparent_rects.add_many(render_rect().shatter(rect()));
+    if (!m_window.is_opaque())
+        transparent_rects.add(m_window.rect());
+    return transparent_rects;
+}
+
 void WindowFrame::invalidate_titlebar()
 {
     m_dirty = true;

Userland/Services/WindowServer/WindowFrame.h

@@ -29,6 +29,8 @@ public:
 
     Gfx::IntRect rect() const;
     Gfx::IntRect render_rect() const;
+    Gfx::DisjointRectSet opaque_render_rects() const;
+    Gfx::DisjointRectSet transparent_render_rects() const;
     void paint(Gfx::Painter&, const Gfx::IntRect&);
     void render(Gfx::Painter&);
     void render_to_cache();
@@ -52,8 +54,9 @@ public:
 
     void start_flash_animation();
 
-    bool has_alpha_channel() const { return m_has_alpha_channel || frame_has_alpha(); }
+    bool has_alpha_channel() const { return m_has_alpha_channel; }
     void set_has_alpha_channel(bool value) { m_has_alpha_channel = value; }
+    bool has_shadow() const;
 
     void set_opacity(float);
     float opacity() const { return m_opacity; }
@@ -86,7 +89,6 @@ private:
     void paint_tool_window_frame(Gfx::Painter&);
     void paint_menubar(Gfx::Painter&);
     Gfx::Bitmap* window_shadow() const;
-    bool frame_has_alpha() const;
     Gfx::IntRect inflated_for_shadow(const Gfx::IntRect&) const;
     Gfx::Bitmap* inflate_for_shadow(Gfx::IntRect&, Gfx::IntPoint&) const;