RGBCube/serenity
1
Fork 0
mirror of https://github.com/RGBCube/serenity synced 2025-07-25 19:47:44 +00:00
Code Issues Activity Actions

LibLine: Correctly handle multibyte codepoints in suggestions

Browse source
This commit is contained in:
AnotherTest 2020-05-21 18:06:43 +04:30 • committed by Andreas Kling
parent 2b3e9c28b2
commit 65adf2aea2
2 changed files with 66 additions and 38 deletions
Download patch file Download diff file Expand all files Collapse all files

65
Libraries/LibLine/Editor.cpp
View file

@ -78,6 +78,12 @@ void Editor::clear_line()
m_inline_search_cursor = m_cursor; m_inline_search_cursor = m_cursor;
} }
void Editor::insert(const Utf32View& string)
{
for (size_t i = 0; i < string.length(); ++i)
insert(string.codepoints()[i]);
}
void Editor::insert(const String& string) void Editor::insert(const String& string)
{ {
for (auto ch : Utf8View { string }) for (auto ch : Utf8View { string })
@ -494,17 +500,18 @@ String Editor::get_line(const String& prompt)
m_suggestions = on_tab_complete(*this); m_suggestions = on_tab_complete(*this);
size_t common_suggestion_prefix { 0 }; size_t common_suggestion_prefix { 0 };
if (m_suggestions.size() == 1) { if (m_suggestions.size() == 1) {
m_largest_common_suggestion_prefix_length = m_suggestions[0].text.length(); m_largest_common_suggestion_prefix_length = m_suggestions[0].text_view.length();
} else if (m_suggestions.size()) { } else if (m_suggestions.size()) {
char last_valid_suggestion_char; u32 last_valid_suggestion_codepoint;
for (;; ++common_suggestion_prefix) { for (;; ++common_suggestion_prefix) {
if (m_suggestions[0].text.length() <= common_suggestion_prefix) if (m_suggestions[0].text_view.length() <= common_suggestion_prefix)
goto no_more_commons; goto no_more_commons;
last_valid_suggestion_char = m_suggestions[0].text[common_suggestion_prefix]; last_valid_suggestion_codepoint = m_suggestions[0].text_view.codepoints()[common_suggestion_prefix];
for (const auto& suggestion : m_suggestions) { for (auto& suggestion : m_suggestions) {
if (suggestion.text.length() <= common_suggestion_prefix || suggestion.text[common_suggestion_prefix] != last_valid_suggestion_char) { if (suggestion.text_view.length() <= common_suggestion_prefix || suggestion.text_view.codepoints()[common_suggestion_prefix] != last_valid_suggestion_codepoint) {
goto no_more_commons; goto no_more_commons;
} }
} }
@ -546,7 +553,7 @@ String Editor::get_line(const String& prompt)
case 2: case 2:
actual_offset = m_cursor - m_largest_common_suggestion_prefix_length + m_next_suggestion_invariant_offset; actual_offset = m_cursor - m_largest_common_suggestion_prefix_length + m_next_suggestion_invariant_offset;
if (can_complete) if (can_complete)
shown_length = m_largest_common_suggestion_prefix_length + m_last_shown_suggestion.trailing_trivia.length(); shown_length = m_largest_common_suggestion_prefix_length + m_last_shown_suggestion.trivia_view.length();
break; break;
default: default:
if (m_last_shown_suggestion_display_length == 0) if (m_last_shown_suggestion_display_length == 0)
@ -558,28 +565,31 @@ String Editor::get_line(const String& prompt)
for (size_t i = m_next_suggestion_invariant_offset; i < shown_length; ++i) for (size_t i = m_next_suggestion_invariant_offset; i < shown_length; ++i)
remove_at_index(actual_offset); remove_at_index(actual_offset);
m_cursor = actual_offset; m_cursor = actual_offset;
m_inline_search_cursor = m_cursor; m_inline_search_cursor = m_cursor;
m_refresh_needed = true; m_refresh_needed = true;
} }
m_last_shown_suggestion = m_suggestions[m_next_suggestion_index]; m_last_shown_suggestion = m_suggestions[m_next_suggestion_index];
m_last_shown_suggestion.token_start_index = token_start - m_next_suggestion_invariant_offset - m_next_suggestion_static_offset;
#ifdef SUGGESTIONS_DEBUG if (m_last_shown_suggestion_display_length)
dbg() << "Last shown suggestion token start index: " << m_last_shown_suggestion.token_start_index << " Token Start " << token_start << " invariant offset " << m_next_suggestion_invariant_offset; m_last_shown_suggestion.token_start_index = token_start - m_next_suggestion_static_offset - m_last_shown_suggestion_display_length;
#endif else
m_last_shown_suggestion_display_length = m_last_shown_suggestion.text.length(); m_last_shown_suggestion.token_start_index = token_start - m_next_suggestion_static_offset - m_next_suggestion_invariant_offset;
m_last_shown_suggestion_display_length = m_last_shown_suggestion.text_view.length();
m_last_shown_suggestion_was_complete = true; m_last_shown_suggestion_was_complete = true;
if (m_times_tab_pressed == 1) { if (m_times_tab_pressed == 1) {
// This is the first time, so only auto-complete *if possible* // This is the first time, so only auto-complete *if possible*
if (can_complete) { if (can_complete) {
insert(m_last_shown_suggestion.text.substring_view(m_next_suggestion_invariant_offset, m_largest_common_suggestion_prefix_length - m_next_suggestion_invariant_offset)); insert(m_last_shown_suggestion.text_view.substring_view(m_next_suggestion_invariant_offset, m_largest_common_suggestion_prefix_length - m_next_suggestion_invariant_offset));
m_last_shown_suggestion_display_length = m_largest_common_suggestion_prefix_length; m_last_shown_suggestion_display_length = m_largest_common_suggestion_prefix_length;
// do not increment the suggestion index, as the first tab should only be a *peek* // do not increment the suggestion index, as the first tab should only be a *peek*
if (m_suggestions.size() == 1) { if (m_suggestions.size() == 1) {
// if there's one suggestion, commit and forget // if there's one suggestion, commit and forget
m_times_tab_pressed = 0; m_times_tab_pressed = 0;
// add in the trivia of the last selected suggestion // add in the trivia of the last selected suggestion
insert(m_last_shown_suggestion.trailing_trivia); insert(m_last_shown_suggestion.trivia_view);
m_last_shown_suggestion_display_length = 0; m_last_shown_suggestion_display_length = 0;
readjust_anchored_styles(m_last_shown_suggestion.token_start_index, ModificationKind::ForcedOverlapRemoval); readjust_anchored_styles(m_last_shown_suggestion.token_start_index, ModificationKind::ForcedOverlapRemoval);
stylize({ m_last_shown_suggestion.token_start_index, m_cursor, Span::Mode::CodepointOriented }, m_last_shown_suggestion.style); stylize({ m_last_shown_suggestion.token_start_index, m_cursor, Span::Mode::CodepointOriented }, m_last_shown_suggestion.style);
@ -590,10 +600,10 @@ String Editor::get_line(const String& prompt)
++m_times_tab_pressed; ++m_times_tab_pressed;
m_last_shown_suggestion_was_complete = false; m_last_shown_suggestion_was_complete = false;
} else { } else {
insert(m_last_shown_suggestion.text.substring_view(m_next_suggestion_invariant_offset, m_last_shown_suggestion.text.length() - m_next_suggestion_invariant_offset)); insert(m_last_shown_suggestion.text_view.substring_view(m_next_suggestion_invariant_offset, m_last_shown_suggestion.text_view.length() - m_next_suggestion_invariant_offset));
// add in the trivia of the last selected suggestion // add in the trivia of the last selected suggestion
insert(m_last_shown_suggestion.trailing_trivia); insert(m_last_shown_suggestion.trivia_view);
m_last_shown_suggestion_display_length += m_last_shown_suggestion.trailing_trivia.length(); m_last_shown_suggestion_display_length += m_last_shown_suggestion.trivia_view.length();
if (m_tab_direction == TabDirection::Forward) if (m_tab_direction == TabDirection::Forward)
increment_suggestion_index(); increment_suggestion_index();
else else
@ -605,12 +615,14 @@ String Editor::get_line(const String& prompt)
if (m_times_tab_pressed > 1 && !m_suggestions.is_empty()) { if (m_times_tab_pressed > 1 && !m_suggestions.is_empty()) {
size_t longest_suggestion_length = 0; size_t longest_suggestion_length = 0;
size_t longest_suggestion_byte_length = 0;
size_t start_index = 0; size_t start_index = 0;
for (auto& suggestion : m_suggestions) { for (auto& suggestion : m_suggestions) {
if (start_index++ < m_last_displayed_suggestion_index) if (start_index++ < m_last_displayed_suggestion_index)
continue; continue;
longest_suggestion_length = max(longest_suggestion_length, suggestion.text.length()); longest_suggestion_length = max(longest_suggestion_length, suggestion.text_view.length());
longest_suggestion_byte_length = max(longest_suggestion_byte_length, suggestion.text_string.length());
} }
size_t num_printed = 0; size_t num_printed = 0;
@ -638,10 +650,10 @@ String Editor::get_line(const String& prompt)
++index; ++index;
continue; continue;
} }
size_t next_column = num_printed + suggestion.text.length() + longest_suggestion_length + 2; size_t next_column = num_printed + suggestion.text_view.length() + longest_suggestion_length + 2;
if (next_column > m_num_columns) { if (next_column > m_num_columns) {
auto lines = (suggestion.text.length() + m_num_columns - 1) / m_num_columns; auto lines = (suggestion.text_view.length() + m_num_columns - 1) / m_num_columns;
lines_used += lines; lines_used += lines;
putchar('\n'); putchar('\n');
num_printed = 0; num_printed = 0;
@ -660,9 +672,10 @@ String Editor::get_line(const String& prompt)
if (spans_entire_line) { if (spans_entire_line) {
num_printed += m_num_columns; num_printed += m_num_columns;
fprintf(stderr, "%s", suggestion.text.characters()); fprintf(stderr, "%s", suggestion.text_string.characters());
} else { } else {
num_printed += fprintf(stderr, "%-*s", static_cast<int>(longest_suggestion_length) + 2, suggestion.text.characters()); fprintf(stderr, "%-*s", static_cast<int>(longest_suggestion_byte_length) + 2, suggestion.text_string.characters());
num_printed += longest_suggestion_length + 2;
} }
if (m_last_shown_suggestion_was_complete && index == current_suggestion_index) { if (m_last_shown_suggestion_was_complete && index == current_suggestion_index) {
@ -703,9 +716,6 @@ String Editor::get_line(const String& prompt)
if (m_times_tab_pressed) { if (m_times_tab_pressed) {
// Apply the style of the last suggestion // Apply the style of the last suggestion
#ifdef SUGGESTIONS_DEBUG
dbg() << "Last shown suggestion token start index: " << m_last_shown_suggestion.token_start_index << " invariant offset " << m_next_suggestion_invariant_offset << " static offset " << m_next_suggestion_static_offset;
#endif
readjust_anchored_styles(m_last_shown_suggestion.token_start_index, ModificationKind::ForcedOverlapRemoval); readjust_anchored_styles(m_last_shown_suggestion.token_start_index, ModificationKind::ForcedOverlapRemoval);
stylize({ m_last_shown_suggestion.token_start_index, m_cursor, Span::Mode::CodepointOriented }, m_last_shown_suggestion.style); stylize({ m_last_shown_suggestion.token_start_index, m_cursor, Span::Mode::CodepointOriented }, m_last_shown_suggestion.style);
// we probably have some suggestions drawn // we probably have some suggestions drawn
@ -858,11 +868,12 @@ String Editor::get_line(const String& prompt)
// manually cleanup the search line // manually cleanup the search line
reposition_cursor(); reposition_cursor();
vt_clear_lines(0, (search_string.length() + actual_rendered_string_length(search_prompt) + m_num_columns - 1) / m_num_columns); auto search_string_codepoint_length = Utf8View { search_string }.length_in_codepoints();
vt_clear_lines(0, (search_string_codepoint_length + actual_rendered_string_length(search_prompt) + m_num_columns - 1) / m_num_columns);
reposition_cursor(); reposition_cursor();
if (!m_reset_buffer_on_search_end || search_string.length() == 0) { if (!m_reset_buffer_on_search_end || search_string_codepoint_length == 0) {
// if the entry was empty, or we purposely quit without a newline, // if the entry was empty, or we purposely quit without a newline,
// do not return anything // do not return anything
// instead, just end the search // instead, just end the search

39
Libraries/LibLine/Editor.h
View file

@ -34,6 +34,8 @@
#include <AK/NonnullOwnPtr.h> #include <AK/NonnullOwnPtr.h>
#include <AK/QuickSort.h> #include <AK/QuickSort.h>
#include <AK/String.h> #include <AK/String.h>
#include <AK/Utf32View.h>
#include <AK/Utf8View.h>
#include <AK/Vector.h> #include <AK/Vector.h>
#include <LibCore/DirIterator.h> #include <LibCore/DirIterator.h>
#include <LibLine/Span.h> #include <LibLine/Span.h>
@ -45,25 +47,34 @@ namespace Line {
class Editor; class Editor;
// FIXME: These objects are pretty heavy since they store two copies of text
// somehow get rid of one
struct CompletionSuggestion { struct CompletionSuggestion {
friend class Editor;
// intentionally not explicit (allows suggesting bare strings) // intentionally not explicit (allows suggesting bare strings)
CompletionSuggestion(const String& completion) CompletionSuggestion(const String& completion)
: text(completion) : CompletionSuggestion(completion, "", {})
, trailing_trivia("")
, style()
{ {
} }
CompletionSuggestion(const StringView& completion, const StringView& trailing_trivia) CompletionSuggestion(const StringView& completion, const StringView& trailing_trivia)
: text(completion) : CompletionSuggestion(completion, trailing_trivia, {})
, trailing_trivia(trailing_trivia)
, style()
{ {
} }
CompletionSuggestion(const StringView& completion, const StringView& trailing_trivia, Style style) CompletionSuggestion(const StringView& completion, const StringView& trailing_trivia, Style style)
: text(completion) : style(style)
, trailing_trivia(trailing_trivia) , text_string(completion)
, style(style)
{ {
Utf8View text_u8 { completion };
Utf8View trivia_u8 { trailing_trivia };
for (auto cp : text_u8)
text.append(cp);
for (auto cp : trivia_u8)
this->trailing_trivia.append(cp);
text_view = Utf32View { text.data(), text.size() };
trivia_view = Utf32View { this->trailing_trivia.data(), this->trailing_trivia.size() };
} }
bool operator==(const CompletionSuggestion& suggestion) const bool operator==(const CompletionSuggestion& suggestion) const
@ -71,10 +82,15 @@ struct CompletionSuggestion {
return suggestion.text == text; return suggestion.text == text;
} }
String text; Vector<u32> text;
String trailing_trivia; Vector<u32> trailing_trivia;
Style style; Style style;
size_t token_start_index { 0 }; size_t token_start_index { 0 };
private:
Utf32View text_view;
Utf32View trivia_view;
String text_string;
}; };
struct Configuration { struct Configuration {
@ -165,6 +181,7 @@ public:
void clear_line(); void clear_line();
void insert(const String&); void insert(const String&);
void insert(const Utf32View&);
void insert(const u32); void insert(const u32);
void stylize(const Span&, const Style&); void stylize(const Span&, const Style&);
void strip_styles(bool strip_anchored = false); void strip_styles(bool strip_anchored = false);