PixelPaint: Introduce new mask features

This patch adds a new Editing-Mask type to layers. This kind of mask
is used to restrict changes on the content bitmap only to areas where
the mask is drawn. The intensity of a pixel change is controlled by the
alpha-value of the mask.

Furthermore a function to invert and clear masks has been introduced.
When a new mask is created for a layer the edit mode of the layer is
also changed to Mask so that the user can immediately start to draw the
mask.

Userland/Applications/PixelPaint/Layer.cpp

@@ -44,6 +44,7 @@ ErrorOr<NonnullRefPtr<Layer>> Layer::create_snapshot(Image& image, Layer const&
     if (layer.is_masked()) {
         snapshot->m_mask_bitmap = TRY(layer.mask_bitmap()->clone());
         snapshot->m_edit_mode = layer.m_edit_mode;
+        snapshot->m_mask_type = layer.m_mask_type;
     }
 
     /*
@@ -273,7 +274,7 @@ ErrorOr<void> Layer::scale(Gfx::IntRect const& new_rect, Gfx::Painter::ScalingMo
 
 void Layer::update_cached_bitmap()
 {
-    if (!is_masked()) {
+    if (mask_type() == MaskType::None || mask_type() == MaskType::EditingMask) {
         if (m_content_bitmap.ptr() == m_cached_display_bitmap.ptr())
             return;
         m_cached_display_bitmap = m_content_bitmap;
@@ -296,10 +297,23 @@ void Layer::update_cached_bitmap()
     }
 }
 
-ErrorOr<void> Layer::create_mask()
+ErrorOr<void> Layer::create_mask(MaskType type)
 {
-    m_mask_bitmap = TRY(Gfx::Bitmap::create(Gfx::BitmapFormat::BGRx8888, size()));
-    m_mask_bitmap->fill(Gfx::Color::White);
+    m_mask_type = type;
+
+    switch (type) {
+    case MaskType::BasicMask:
+        m_mask_bitmap = TRY(Gfx::Bitmap::create(Gfx::BitmapFormat::BGRx8888, size()));
+        m_mask_bitmap->fill(Gfx::Color::White);
+        break;
+    case MaskType::EditingMask:
+        m_mask_bitmap = TRY(Gfx::Bitmap::create(Gfx::BitmapFormat::BGRA8888, size()));
+        break;
+    case MaskType::None:
+        VERIFY_NOT_REACHED();
+    }
+
+    set_edit_mode(EditMode::Mask);
     update_cached_bitmap();
     return {};
 }
@@ -307,6 +321,7 @@ ErrorOr<void> Layer::create_mask()
 void Layer::delete_mask()
 {
     m_mask_bitmap = nullptr;
+    m_mask_type = MaskType::None;
     set_edit_mode(EditMode::Content);
     update_cached_bitmap();
 }
@@ -319,6 +334,38 @@ void Layer::apply_mask()
     delete_mask();
 }
 
+void Layer::invert_mask()
+{
+    VERIFY(mask_type() != MaskType::None);
+
+    for (int y = 0; y < size().height(); ++y) {
+        for (int x = 0; x < size().width(); ++x) {
+            auto inverted_mask_color = m_mask_bitmap->get_pixel(x, y).inverted();
+            if (mask_type() == MaskType::EditingMask)
+                inverted_mask_color.set_alpha(255 - inverted_mask_color.alpha());
+            m_mask_bitmap->set_pixel(x, y, inverted_mask_color);
+        }
+    }
+
+    update_cached_bitmap();
+}
+
+void Layer::clear_mask()
+{
+    switch (mask_type()) {
+    case MaskType::None:
+        VERIFY_NOT_REACHED();
+    case MaskType::BasicMask:
+        m_mask_bitmap->fill(Gfx::Color::White);
+        break;
+    case MaskType::EditingMask:
+        m_mask_bitmap->fill(Gfx::Color::Transparent);
+        break;
+    }
+
+    update_cached_bitmap();
+}
+
 Gfx::Bitmap& Layer::currently_edited_bitmap()
 {
     switch (edit_mode()) {
@@ -410,4 +457,11 @@ ErrorOr<NonnullRefPtr<Layer>> Layer::duplicate(DeprecatedString name)
     return duplicated_layer;
 }
 
+Layer::MaskType Layer::mask_type()
+{
+    if (m_mask_bitmap.is_null())
+        return MaskType::None;
+    return m_mask_type;
+}
+
 }

Userland/Applications/PixelPaint/Layer.h

@@ -45,9 +45,17 @@ public:
     Gfx::Bitmap const* mask_bitmap() const { return m_mask_bitmap; }
     Gfx::Bitmap* mask_bitmap() { return m_mask_bitmap; }
 
-    ErrorOr<void> create_mask();
+    enum class MaskType {
+        None,
+        BasicMask,
+        EditingMask,
+    };
+
+    ErrorOr<void> create_mask(MaskType);
     void delete_mask();
     void apply_mask();
+    void invert_mask();
+    void clear_mask();
 
     Gfx::Bitmap& get_scratch_edited_bitmap();
 
@@ -91,6 +99,7 @@ public:
     void erase_selection(Selection const&);
 
     bool is_masked() const { return !m_mask_bitmap.is_null(); }
+    MaskType mask_type();
 
     enum class EditMode {
         Content,
@@ -104,6 +113,19 @@ public:
 
     ErrorOr<NonnullRefPtr<Layer>> duplicate(DeprecatedString name);
 
+    ALWAYS_INLINE Color modify_pixel_with_editing_mask(int x, int y, Color const& target_color, Color const& current_color)
+    {
+        if (mask_type() != MaskType::EditingMask)
+            return target_color;
+
+        auto mask = mask_bitmap()->get_pixel(x, y).alpha();
+        if (!mask)
+            return current_color;
+
+        float mask_intensity = mask / 255.0f;
+        return current_color.mixed_with(target_color, mask_intensity);
+    }
+
 private:
     Layer(Image&, NonnullRefPtr<Gfx::Bitmap>, DeprecatedString name);
 
@@ -122,6 +144,7 @@ private:
     int m_opacity_percent { 100 };
 
     EditMode m_edit_mode { EditMode::Content };
+    MaskType m_mask_type { MaskType::None };
 
     void update_cached_bitmap();
 };

Userland/Applications/PixelPaint/MainWidget.cpp

@@ -812,11 +812,19 @@ ErrorOr<void> MainWidget::initialize_menubar(GUI::Window& window)
     m_add_mask_action = GUI::Action::create(
         "Add M&ask", { Mod_Ctrl | Mod_Shift, Key_M }, g_icon_bag.add_mask, create_layer_mask_callback("Add Mask", [&](Layer* active_layer) {
             VERIFY(!active_layer->is_masked());
-            if (auto maybe_error = active_layer->create_mask(); maybe_error.is_error())
+            if (auto maybe_error = active_layer->create_mask(Layer::MaskType::BasicMask); maybe_error.is_error())
                 GUI::MessageBox::show_error(&window, MUST(String::formatted("Failed to create layer mask: {}", maybe_error.release_error())));
         }));
     TRY(m_layer_menu->try_add_action(*m_add_mask_action));
 
+    m_add_editing_mask_action = GUI::Action::create(
+        "Add E&diting Mask", { Mod_Ctrl | Mod_Alt, Key_E }, g_icon_bag.add_mask, create_layer_mask_callback("Add Editing Mask", [&](Layer* active_layer) {
+            VERIFY(!active_layer->is_masked());
+            if (auto maybe_error = active_layer->create_mask(Layer::MaskType::EditingMask); maybe_error.is_error())
+                GUI::MessageBox::show_error(&window, MUST(String::formatted("Failed to create layer mask: {}", maybe_error.release_error())));
+        }));
+    TRY(m_layer_menu->try_add_action(*m_add_editing_mask_action));
+
     m_delete_mask_action = GUI::Action::create(
         "Delete Mask", create_layer_mask_callback("Delete Mask", [&](Layer* active_layer) {
             VERIFY(active_layer->is_masked());
@@ -831,6 +839,20 @@ ErrorOr<void> MainWidget::initialize_menubar(GUI::Window& window)
         }));
     TRY(m_layer_menu->try_add_action(*m_apply_mask_action));
 
+    m_invert_mask_action = GUI::Action::create(
+        "Invert Mask", create_layer_mask_callback("Invert Mask", [&](Layer* active_layer) {
+            VERIFY(active_layer->is_masked());
+            active_layer->invert_mask();
+        }));
+    TRY(m_layer_menu->try_add_action(*m_invert_mask_action));
+
+    m_clear_mask_action = GUI::Action::create(
+        "Clear Mask", create_layer_mask_callback("Clear Mask", [&](Layer* active_layer) {
+            VERIFY(active_layer->is_masked());
+            active_layer->clear_mask();
+        }));
+    TRY(m_layer_menu->try_add_action(*m_clear_mask_action));
+
     TRY(m_layer_menu->try_add_separator());
 
     TRY(m_layer_menu->try_add_action(GUI::Action::create(
@@ -1205,8 +1227,11 @@ void MainWidget::set_mask_actions_for_layer(Layer* layer)
 
     auto masked = layer->is_masked();
     m_add_mask_action->set_visible(!masked);
+    m_add_editing_mask_action->set_visible(!masked);
+    m_invert_mask_action->set_visible(masked);
+    m_clear_mask_action->set_visible(masked);
     m_delete_mask_action->set_visible(masked);
-    m_apply_mask_action->set_visible(masked);
+    m_apply_mask_action->set_visible(layer->mask_type() == Layer::MaskType::BasicMask);
 }
 
 void MainWidget::open_image(FileSystemAccessClient::File file)

Userland/Applications/PixelPaint/MainWidget.h

@@ -114,6 +114,9 @@ private:
     RefPtr<GUI::Action> m_add_mask_action;
     RefPtr<GUI::Action> m_delete_mask_action;
     RefPtr<GUI::Action> m_apply_mask_action;
+    RefPtr<GUI::Action> m_add_editing_mask_action;
+    RefPtr<GUI::Action> m_invert_mask_action;
+    RefPtr<GUI::Action> m_clear_mask_action;
 
     Gfx::IntPoint m_last_image_editor_mouse_position;
 };

Userland/Applications/PixelPaint/Tools/Tool.cpp

@@ -83,4 +83,35 @@ Gfx::IntPoint Tool::constrain_line_angle(Gfx::IntPoint start_pos, Gfx::IntPoint
         start_pos.y() + (int)(AK::sin(constrained_angle) * line_length) };
 }
 
+template<>
+void Tool::set_pixel_with_possible_mask<Gfx::StorageFormat::BGRA8888>(int x, int y, Gfx::Color color, Gfx::Bitmap& bitmap)
+{
+    if (!m_editor || !m_editor->active_layer())
+        return;
+
+    switch (m_editor->active_layer()->edit_mode()) {
+    case Layer::EditMode::Content:
+        bitmap.set_pixel<Gfx::StorageFormat::BGRA8888>(x, y, m_editor->active_layer()->modify_pixel_with_editing_mask(x, y, color, bitmap.get_pixel(x, y)));
+        break;
+    case Layer::EditMode::Mask:
+        bitmap.set_pixel<Gfx::StorageFormat::BGRA8888>(x, y, color);
+        break;
+    }
+}
+
+void Tool::set_pixel_with_possible_mask(int x, int y, Gfx::Color color, Gfx::Bitmap& bitmap)
+{
+    if (!m_editor || !m_editor->active_layer())
+        return;
+
+    switch (m_editor->active_layer()->edit_mode()) {
+    case Layer::EditMode::Content:
+        bitmap.set_pixel(x, y, m_editor->active_layer()->modify_pixel_with_editing_mask(x, y, color, bitmap.get_pixel(x, y)));
+        break;
+    case Layer::EditMode::Mask:
+        bitmap.set_pixel(x, y, color);
+        break;
+    }
+}
+
 }

Userland/Applications/PixelPaint/Tools/Tool.h

@@ -101,6 +101,10 @@ protected:
 
     GUI::AbstractSlider* m_primary_slider { nullptr };
     GUI::AbstractSlider* m_secondary_slider { nullptr };
+
+    template<Gfx::StorageFormat>
+    void set_pixel_with_possible_mask(int x, int y, Gfx::Color color, Gfx::Bitmap& bitmap);
+    void set_pixel_with_possible_mask(int x, int y, Gfx::Color color, Gfx::Bitmap& bitmap);
 };
 
 }