PixelPaint: Change BucketTool flood fill algorithm

This resolves some cases where the existing filling algorithm could get
stuck in a recursion loop and evenutally overflow the stack.

Userland/Applications/PixelPaint/Tools/BucketTool.cpp

@@ -10,6 +10,7 @@
 #include "BucketTool.h"
 #include "../ImageEditor.h"
 #include "../Layer.h"
+#include "../Mask.h"
 #include <AK/HashTable.h>
 #include <AK/Queue.h>
 #include <LibGUI/BoxLayout.h>
@@ -35,76 +36,12 @@ static float color_distance_squared(Gfx::Color const& lhs, Gfx::Color const& rhs
     return (a * a + b * b + c * c + d * d) / (4.0f * 255.0f * 255.0f);
 }
 
-static bool can_paint(int x, int y, Gfx::Bitmap& bitmap, Gfx::Color const& target_color, float threshold_normalized_squared)
+static bool can_paint(Gfx::IntPoint point, Gfx::Bitmap& bitmap, Gfx::Color const& target_color, float threshold_normalized_squared)
 {
-    auto pixel_color = bitmap.get_pixel<Gfx::StorageFormat::BGRA8888>(x, y);
+    auto pixel_color = bitmap.get_pixel<Gfx::StorageFormat::BGRA8888>(point.x(), point.y());
     return color_distance_squared(pixel_color, target_color) <= threshold_normalized_squared;
 }
 
-static void fill_core(int, int, Gfx::Bitmap&, Gfx::Color const&, Gfx::Color const&, float);
-
-static void fill_start(int x, int y, Gfx::Bitmap& bitmap, Gfx::Color const& target_color, Gfx::Color const& fill_color, float threshold_normalized_squared)
-{
-    // Move as far up and to the left as we can first and then start filling
-    while (true) {
-        int previous_x = x;
-        int previous_y = y;
-        while (y != 0 && can_paint(x, y - 1, bitmap, target_color, threshold_normalized_squared))
-            y--;
-        while (x != 0 && can_paint(x - 1, y, bitmap, target_color, threshold_normalized_squared))
-            x--;
-        if (x == previous_x && y == previous_y)
-            break;
-    }
-    fill_core(x, y, bitmap, target_color, fill_color, threshold_normalized_squared);
-}
-
-static void fill_core(int x, int y, Gfx::Bitmap& bitmap, Gfx::Color const& target_color, Gfx::Color const& fill_color, float threshold_normalized_squared)
-{
-    int prev_row_length = 0;
-    do {
-        int row_length = 0;
-        int start_x = x;
-        // To handle the case where the previous row overhangs the current row without recursion, move x to the right until we can paint.
-        // If we can't fill any more without extending beyond the previous row, return.
-        // The else branch handles the opposite case, and moves x to the left and checks above it for a spot to start filling from.
-        if (prev_row_length != 0 && !can_paint(x, y, bitmap, target_color, threshold_normalized_squared)) {
-            do {
-                if (--prev_row_length == 0)
-                    return;
-            } while (!can_paint(++x, y, bitmap, target_color, threshold_normalized_squared));
-            start_x = x;
-        } else {
-            for (; x != 0 && can_paint(x - 1, y, bitmap, target_color, threshold_normalized_squared); row_length++, prev_row_length++) {
-                bitmap.set_pixel<Gfx::StorageFormat::BGRA8888>(--x, y, fill_color);
-                if (y != 0 && can_paint(x, y - 1, bitmap, target_color, threshold_normalized_squared))
-                    fill_start(x, y - 1, bitmap, target_color, fill_color, threshold_normalized_squared);
-            }
-        }
-
-        // Fill this row to the right as much as we can
-        for (; start_x < bitmap.width() && can_paint(start_x, y, bitmap, target_color, threshold_normalized_squared); row_length++, start_x++) {
-            bitmap.set_pixel<Gfx::StorageFormat::BGRA8888>(start_x, y, fill_color);
-        }
-
-        // If the previous row was longer than this row, look beyond the end of this row for pixels to fill until reaching the end of the previous row.
-        // The else branch handles the opposite case, where the previous row was shorter. Look beyond the end of the previous row up into it for pixels to fill.
-        if (row_length < prev_row_length) {
-            for (int prev_row_end = x + prev_row_length; ++start_x < prev_row_end;) {
-                if (can_paint(start_x, y, bitmap, target_color, threshold_normalized_squared))
-                    fill_core(start_x, y, bitmap, target_color, fill_color, threshold_normalized_squared);
-            }
-        } else if (row_length > prev_row_length && y != 0) {
-            for (int upward_x = x + prev_row_length; ++upward_x < start_x;) {
-                if (can_paint(upward_x, y - 1, bitmap, target_color, threshold_normalized_squared))
-                    fill_start(upward_x, y - 1, bitmap, target_color, fill_color, threshold_normalized_squared);
-            }
-        }
-
-        prev_row_length = row_length;
-    } while (prev_row_length != 0 && ++y < bitmap.height());
-}
-
 static void flood_fill(Gfx::Bitmap& bitmap, Gfx::IntPoint const& start_position, Color target_color, Color fill_color, int threshold)
 {
     VERIFY(bitmap.bpp() == 32);
@@ -117,8 +54,35 @@ static void flood_fill(Gfx::Bitmap& bitmap, Gfx::IntPoint const& start_position,
 
     float threshold_normalized_squared = (threshold / 100.0f) * (threshold / 100.0f);
 
-    if (can_paint(start_position.x(), start_position.y(), bitmap, target_color, threshold_normalized_squared))
+    // Create Mask which will track already-colored pixels
-        fill_start(start_position.x(), start_position.y(), bitmap, target_color, fill_color, threshold_normalized_squared);
+    Mask flood_mask = Mask::empty(bitmap.rect());
+
+    Queue<Gfx::IntPoint> points_to_visit = Queue<Gfx::IntPoint>();
+
+    points_to_visit.enqueue({ start_position.x(), start_position.y() });
+    bitmap.set_pixel<Gfx::StorageFormat::BGRA8888>(start_position.x(), start_position.y(), fill_color);
+    flood_mask.set(start_position.x(), start_position.y(), 1);
+
+    // This implements a non-recursive flood fill. This is a breadth-first search of paintable neighbors
+    // As we find neighbors that are paintable we update their pixel, add them to the queue, and mark them in the mask
+    while (!points_to_visit.is_empty()) {
+        auto current_point = points_to_visit.dequeue();
+        auto candidate_points = Array {
+            current_point.moved_left(1),
+            current_point.moved_right(1),
+            current_point.moved_up(1),
+            current_point.moved_down(1)
+        };
+        for (auto candidate_point : candidate_points) {
+            if (!bitmap.rect().contains(candidate_point))
+                continue;
+            if (flood_mask.get(candidate_point.x(), candidate_point.y()) == 0 && can_paint(candidate_point, bitmap, target_color, threshold_normalized_squared)) {
+                points_to_visit.enqueue(candidate_point);
+                bitmap.set_pixel<Gfx::StorageFormat::BGRA8888>(candidate_point.x(), candidate_point.y(), fill_color);
+            }
+            flood_mask.set(candidate_point.x(), candidate_point.y(), 0xFF);
+        }
+    }
 }
 
 void BucketTool::on_mousedown(Layer* layer, MouseEvent& event)