RGBCube/serenity
1
Fork 0
mirror of https://github.com/RGBCube/serenity synced 2025-05-31 12:48:10 +00:00
Code Issues Activity Actions

LibUnicode: Fix typos causing text segmentation on mid-word punctuation

Browse source 
For example the words "can't" and "32.3" should not have boundaries
detected on the "'" and "." code points, respectively.

The String test cases fixed here are because "b'ar" is now considered
one word.
This commit is contained in:
Timothy Flynn 2023-02-14 13:56:19 -05:00 • committed by Linus Groh
parent 6e7a6e2d02
commit 5cbf054651
4 changed files with 45 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

4
Userland/Libraries/LibUnicode/Segmentation.cpp
View file

@ -215,7 +215,7 @@ static void for_each_word_segmentation_boundary_impl([[maybe_unused]] ViewType c
auto it_copy = it;
++it_copy;
if (it_copy != view.end())
next_next_code_point = *it;
next_next_code_point = *it_copy;
}
bool next_next_code_point_is_hebrew_letter = next_next_code_point.has_value() && has_any_wbp(*next_next_code_point, WBP::Hebrew_Letter);
bool next_next_code_point_is_ah_letter = next_next_code_point_is_hebrew_letter || (next_next_code_point.has_value() && has_any_wbp(*next_next_code_point, WBP::ALetter));
@ -256,7 +256,7 @@ static void for_each_word_segmentation_boundary_impl([[maybe_unused]] ViewType c
if (code_point_is_numeric && next_code_point_is_ah_letter)
continue;
auto previous_code_point_is_numeric = previous_code_point.has_value() && has_any_wbp(code_point, WBP::Numeric);
auto previous_code_point_is_numeric = previous_code_point.has_value() && has_any_wbp(*previous_code_point, WBP::Numeric);
// WB11
if (previous_code_point_is_numeric && next_code_point_is_numeric && (code_point_is_mid_num_let_q || has_any_wbp(code_point, WBP::MidNum)))