LibLine: Use grapheme clusters for cursor management

This makes using the line editor much nicer when multi-code-point
graphemes are present in the input (e.g. flag emojis, or some cjk
glyphs), and avoids messing up the buffer when deleting text, or
cursoring around.

Userland/Libraries/LibLine/CMakeLists.txt

@@ -7,4 +7,4 @@ set(SOURCES
 )
 
 serenity_lib(LibLine line)
-target_link_libraries(LibLine PRIVATE LibCore)
+target_link_libraries(LibLine PRIVATE LibCore LibUnicode)

Userland/Libraries/LibLine/Editor.cpp

@@ -21,6 +21,7 @@
 #include <LibCore/Event.h>
 #include <LibCore/EventLoop.h>
 #include <LibCore/Notifier.h>
+#include <LibUnicode/Segmentation.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <signal.h>
@@ -1865,66 +1866,53 @@ static MaskedSelectionDecision resolve_masked_selection(Optional<Style::Mask>& m
 
 StringMetrics Editor::actual_rendered_string_metrics(StringView string, RedBlackTree<u32, Optional<Style::Mask>> const& masks, Optional<size_t> maximum_line_width)
 {
-    StringMetrics metrics;
-    StringMetrics::LineMetrics current_line;
-    VTState state { Free };
-    Utf8View view { string };
-    size_t last_return {};
-    auto it = view.begin();
-    Optional<Style::Mask> mask;
-    size_t i = 0;
-    auto mask_it = masks.begin();
+    Vector<u32> utf32_buffer;
+    utf32_buffer.ensure_capacity(string.length());
+    for (auto c : Utf8View { string })
+        utf32_buffer.append(c);
 
-    for (; it != view.end(); ++it) {
-        if (!mask_it.is_end() && mask_it.key() <= i)
-            mask = *mask_it;
-        auto c = *it;
-        auto it_copy = it;
-        ++it_copy;
-
-        if (resolve_masked_selection(mask, i, mask_it, view, state, metrics, current_line) == MaskedSelectionDecision::Skip)
-            continue;
-
-        auto next_c = it_copy == view.end() ? 0 : *it_copy;
-        state = actual_rendered_string_length_step(metrics, view.iterator_offset(it), current_line, c, next_c, state, mask, maximum_line_width, last_return);
-        if (!mask_it.is_end() && mask_it.key() <= i) {
-            auto mask_it_peek = mask_it;
-            ++mask_it_peek;
-            if (!mask_it_peek.is_end() && mask_it_peek.key() > i)
-                mask_it = mask_it_peek;
-        }
-        ++i;
-    }
-
-    metrics.line_metrics.append(current_line);
-
-    for (auto& line : metrics.line_metrics)
-        metrics.max_line_length = max(line.total_length(), metrics.max_line_length);
-
-    return metrics;
+    return actual_rendered_string_metrics(Utf32View { utf32_buffer.data(), utf32_buffer.size() }, masks, maximum_line_width);
 }
 
-StringMetrics Editor::actual_rendered_string_metrics(Utf32View const& view, RedBlackTree<u32, Optional<Style::Mask>> const& masks)
+StringMetrics Editor::actual_rendered_string_metrics(Utf32View const& view, RedBlackTree<u32, Optional<Style::Mask>> const& masks, Optional<size_t> maximum_line_width)
 {
     StringMetrics metrics;
     StringMetrics::LineMetrics current_line;
     VTState state { Free };
     Optional<Style::Mask> mask;
+    size_t last_return { 0 };
 
     auto mask_it = masks.begin();
 
-    for (size_t i = 0; i < view.length(); ++i) {
+    Vector<size_t> grapheme_breaks;
+    Unicode::for_each_grapheme_segmentation_boundary(view, [&](size_t offset) -> IterationDecision {
+        if (offset >= view.length())
+            return IterationDecision::Break;
+
+        grapheme_breaks.append(offset);
+        return IterationDecision::Continue;
+    });
+
+    // In case Unicode data isn't available, default to using code points as grapheme boundaries.
+    if (grapheme_breaks.is_empty()) {
+        for (size_t i = 0; i < view.length(); ++i)
+            grapheme_breaks.append(i);
+    }
+
+    for (size_t break_index = 0; break_index < grapheme_breaks.size(); ++break_index) {
+        auto i = grapheme_breaks[break_index];
         auto c = view[i];
         if (!mask_it.is_end() && mask_it.key() <= i)
             mask = *mask_it;
 
         if (resolve_masked_selection(mask, i, mask_it, view, state, metrics, current_line) == MaskedSelectionDecision::Skip) {
             --i;
+            binary_search(grapheme_breaks, i, &break_index);
             continue;
         }
 
-        auto next_c = i + 1 < view.length() ? view.code_points()[i + 1] : 0;
-        state = actual_rendered_string_length_step(metrics, i, current_line, c, next_c, state, mask);
+        auto next_c = break_index + 1 < grapheme_breaks.size() ? view.code_points()[grapheme_breaks[break_index + 1]] : 0;
+        state = actual_rendered_string_length_step(metrics, i, current_line, c, next_c, state, mask, maximum_line_width, last_return);
         if (!mask_it.is_end() && mask_it.key() <= i) {
             auto mask_it_peek = mask_it;
             ++mask_it_peek;
@@ -1938,6 +1926,8 @@ StringMetrics Editor::actual_rendered_string_metrics(Utf32View const& view, RedB
     for (auto& line : metrics.line_metrics)
         metrics.max_line_length = max(line.total_length(), metrics.max_line_length);
 
+    metrics.grapheme_breaks = move(grapheme_breaks);
+
     return metrics;
 }
 

Userland/Libraries/LibLine/Editor.h

@@ -161,7 +161,7 @@ public:
     void register_key_input_callback(Key key, Function<bool(Editor&)> callback) { register_key_input_callback(Vector<Key> { key }, move(callback)); }
 
     static StringMetrics actual_rendered_string_metrics(StringView, RedBlackTree<u32, Optional<Style::Mask>> const& masks = {}, Optional<size_t> maximum_line_width = {});
-    static StringMetrics actual_rendered_string_metrics(Utf32View const&, RedBlackTree<u32, Optional<Style::Mask>> const& masks = {});
+    static StringMetrics actual_rendered_string_metrics(Utf32View const&, RedBlackTree<u32, Optional<Style::Mask>> const& masks = {}, Optional<size_t> maximum_line_width = {});
 
     Function<Vector<CompletionSuggestion>(Editor const&)> on_tab_complete;
     Function<void(Utf32View, Editor&)> on_paste;

Userland/Libraries/LibLine/InternalFunctions.cpp

@@ -94,8 +94,11 @@ void Editor::cursor_left_word()
 
 void Editor::cursor_left_character()
 {
-    if (m_cursor > 0)
-        --m_cursor;
+    if (m_cursor > 0) {
+        size_t closest_cursor_left_offset;
+        binary_search(m_cached_buffer_metrics.grapheme_breaks, m_cursor - 1, &closest_cursor_left_offset);
+        m_cursor = m_cached_buffer_metrics.grapheme_breaks[closest_cursor_left_offset];
+    }
     m_inline_search_cursor = m_cursor;
 }
 
@@ -120,7 +123,11 @@ void Editor::cursor_right_word()
 void Editor::cursor_right_character()
 {
     if (m_cursor < m_buffer.size()) {
-        ++m_cursor;
+        size_t closest_cursor_left_offset;
+        binary_search(m_cached_buffer_metrics.grapheme_breaks, m_cursor, &closest_cursor_left_offset);
+        m_cursor = closest_cursor_left_offset + 1 >= m_cached_buffer_metrics.grapheme_breaks.size()
+            ? m_buffer.size()
+            : m_cached_buffer_metrics.grapheme_breaks[closest_cursor_left_offset + 1];
     }
     m_inline_search_cursor = m_cursor;
     m_search_offset = 0;
@@ -136,8 +143,13 @@ void Editor::erase_character_backwards()
         fflush(stderr);
         return;
     }
-    remove_at_index(m_cursor - 1);
-    --m_cursor;
+
+    size_t closest_cursor_left_offset;
+    binary_search(m_cached_buffer_metrics.grapheme_breaks, m_cursor - 1, &closest_cursor_left_offset);
+    auto start_of_previous_grapheme = m_cached_buffer_metrics.grapheme_breaks[closest_cursor_left_offset];
+    for (; m_cursor > start_of_previous_grapheme; --m_cursor)
+        remove_at_index(m_cursor - 1);
+
     m_inline_search_cursor = m_cursor;
     // We will have to redraw :(
     m_refresh_needed = true;
@@ -150,7 +162,14 @@ void Editor::erase_character_forwards()
         fflush(stderr);
         return;
     }
-    remove_at_index(m_cursor);
+
+    size_t closest_cursor_left_offset;
+    binary_search(m_cached_buffer_metrics.grapheme_breaks, m_cursor, &closest_cursor_left_offset);
+    auto end_of_next_grapheme = closest_cursor_left_offset + 1 >= m_cached_buffer_metrics.grapheme_breaks.size()
+        ? m_buffer.size()
+        : m_cached_buffer_metrics.grapheme_breaks[closest_cursor_left_offset + 1];
+    for (; m_cursor < end_of_next_grapheme;)
+        remove_at_index(m_cursor);
     m_refresh_needed = true;
 }
 

Userland/Libraries/LibLine/StringMetrics.h

@@ -27,6 +27,7 @@ struct StringMetrics {
     };
 
     Vector<LineMetrics> line_metrics;
+    Vector<size_t> grapheme_breaks {};
     size_t total_length { 0 };
     size_t max_line_length { 0 };