From 2e45e52993a47b7aa6d61aed7298fb78ea746695 Mon Sep 17 00:00:00 2001 From: Timothy Flynn Date: Tue, 20 Jul 2021 22:25:48 -0400 Subject: [PATCH] AK: Add UTF-16 helper methods required for use within LibRegex To be used as a RegexStringView variant, Utf16View must provide a couple more helper methods. It must also not default its assignment operators, because that implicitly deletes move/copy constructors. --- AK/Utf16View.cpp | 80 +++++++++++++++++++++++++++++++++++++++++++----- AK/Utf16View.h | 10 ++++-- 2 files changed, 81 insertions(+), 9 deletions(-) diff --git a/AK/Utf16View.cpp b/AK/Utf16View.cpp index 83d95d4543..a784822167 100644 --- a/AK/Utf16View.cpp +++ b/AK/Utf16View.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include namespace AK { @@ -18,16 +19,12 @@ static constexpr u16 low_surrogate_max = 0xdfff; static constexpr u32 replacement_code_point = 0xfffd; static constexpr u32 first_supplementary_plane_code_point = 0x10000; -Vector utf8_to_utf16(StringView const& utf8_view) -{ - return utf8_to_utf16(Utf8View { utf8_view }); -} - -Vector utf8_to_utf16(Utf8View const& utf8_view) +template +static Vector to_utf16_impl(UtfViewType const& view) requires(IsSame || IsSame) { Vector utf16_data; - for (auto code_point : utf8_view) { + for (auto code_point : view) { if (code_point < first_supplementary_plane_code_point) { utf16_data.append(static_cast(code_point)); } else { @@ -40,6 +37,21 @@ Vector utf8_to_utf16(Utf8View const& utf8_view) return utf16_data; } +Vector utf8_to_utf16(StringView const& utf8_view) +{ + return to_utf16_impl(Utf8View { utf8_view }); +} + +Vector utf8_to_utf16(Utf8View const& utf8_view) +{ + return to_utf16_impl(utf8_view); +} + +Vector utf32_to_utf16(Utf32View const& utf32_view) +{ + return to_utf16_impl(utf32_view); +} + bool Utf16View::is_high_surrogate(u16 code_unit) { return (code_unit >= high_surrogate_min) && (code_unit <= high_surrogate_max); @@ -98,6 +110,36 @@ u16 Utf16View::code_unit_at(size_t index) const return m_code_units[index]; } +size_t Utf16View::code_point_offset_of(size_t code_unit_offset) const +{ + size_t code_point_offset = 0; + + for (auto it = begin(); it != end(); ++it) { + if (code_unit_offset == 0) + return code_point_offset; + + code_unit_offset -= it.length_in_code_units(); + ++code_point_offset; + } + + return code_point_offset; +} + +size_t Utf16View::code_unit_offset_of(size_t code_point_offset) const +{ + size_t code_unit_offset = 0; + + for (auto it = begin(); it != end(); ++it) { + if (code_point_offset == 0) + return code_unit_offset; + + code_unit_offset += it.length_in_code_units(); + --code_point_offset; + } + + return code_unit_offset; +} + Utf16View Utf16View::substring_view(size_t code_unit_offset, size_t code_unit_length) const { VERIFY(!Checked::addition_would_overflow(code_unit_offset, code_unit_length)); @@ -106,6 +148,30 @@ Utf16View Utf16View::substring_view(size_t code_unit_offset, size_t code_unit_le return Utf16View { m_code_units.slice(code_unit_offset, code_unit_length) }; } +Utf16View Utf16View::unicode_substring_view(size_t code_point_offset, size_t code_point_length) const +{ + if (code_point_length == 0) + return {}; + + auto code_unit_offset_of = [&](Utf16CodePointIterator const& it) { return it.m_ptr - begin_ptr(); }; + size_t code_point_index = 0; + size_t code_unit_offset = 0; + + for (auto it = begin(); it != end(); ++it) { + if (code_point_index == code_point_offset) + code_unit_offset = code_unit_offset_of(it); + + if (code_point_index == (code_point_offset + code_point_length - 1)) { + size_t code_unit_length = code_unit_offset_of(++it) - code_unit_offset; + return substring_view(code_unit_offset, code_unit_length); + } + + ++code_point_index; + } + + VERIFY_NOT_REACHED(); +} + bool Utf16View::validate(size_t& valid_code_units) const { valid_code_units = 0; diff --git a/AK/Utf16View.h b/AK/Utf16View.h index 09f269df0f..fc1c4ca87d 100644 --- a/AK/Utf16View.h +++ b/AK/Utf16View.h @@ -17,6 +17,7 @@ namespace AK { Vector utf8_to_utf16(StringView const&); Vector utf8_to_utf16(Utf8View const&); +Vector utf32_to_utf16(Utf32View const&); class Utf16View; @@ -67,8 +68,6 @@ public: { } - Utf16View& operator=(Utf16View const&) = default; - Utf16View& operator=(Utf16View&&) = default; bool operator==(Utf16View const& other) const; enum class AllowInvalidCodeUnits { @@ -78,6 +77,7 @@ public: String to_utf8(AllowInvalidCodeUnits = AllowInvalidCodeUnits::No) const; + bool is_null() const { return m_code_units.is_null(); } bool is_empty() const { return m_code_units.is_empty(); } size_t length_in_code_units() const { return m_code_units.size(); } size_t length_in_code_points() const; @@ -88,9 +88,15 @@ public: u16 const* data() const { return m_code_units.data(); } u16 code_unit_at(size_t index) const; + size_t code_point_offset_of(size_t code_unit_offset) const; + size_t code_unit_offset_of(size_t code_point_offset) const; + Utf16View substring_view(size_t code_unit_offset, size_t code_unit_length) const; Utf16View substring_view(size_t code_unit_offset) const { return substring_view(code_unit_offset, length_in_code_units() - code_unit_offset); } + Utf16View unicode_substring_view(size_t code_point_offset, size_t code_point_length) const; + Utf16View unicode_substring_view(size_t code_point_offset) const { return unicode_substring_view(code_point_offset, length_in_code_points() - code_point_offset); } + bool validate(size_t& valid_code_units) const; bool validate() const {