mirror of
https://github.com/SerenityOS/serenity
synced 2024-10-06 16:09:30 +00:00
AK: Add UTF-16 helper methods required for use within LibRegex
To be used as a RegexStringView variant, Utf16View must provide a couple more helper methods. It must also not default its assignment operators, because that implicitly deletes move/copy constructors.
This commit is contained in:
parent
1d76be97f5
commit
2e45e52993
|
@ -7,6 +7,7 @@
|
||||||
#include <AK/StringBuilder.h>
|
#include <AK/StringBuilder.h>
|
||||||
#include <AK/StringView.h>
|
#include <AK/StringView.h>
|
||||||
#include <AK/Utf16View.h>
|
#include <AK/Utf16View.h>
|
||||||
|
#include <AK/Utf32View.h>
|
||||||
#include <AK/Utf8View.h>
|
#include <AK/Utf8View.h>
|
||||||
|
|
||||||
namespace AK {
|
namespace AK {
|
||||||
|
@ -18,16 +19,12 @@ static constexpr u16 low_surrogate_max = 0xdfff;
|
||||||
static constexpr u32 replacement_code_point = 0xfffd;
|
static constexpr u32 replacement_code_point = 0xfffd;
|
||||||
static constexpr u32 first_supplementary_plane_code_point = 0x10000;
|
static constexpr u32 first_supplementary_plane_code_point = 0x10000;
|
||||||
|
|
||||||
Vector<u16> utf8_to_utf16(StringView const& utf8_view)
|
template<typename UtfViewType>
|
||||||
{
|
static Vector<u16> to_utf16_impl(UtfViewType const& view) requires(IsSame<UtfViewType, Utf8View> || IsSame<UtfViewType, Utf32View>)
|
||||||
return utf8_to_utf16(Utf8View { utf8_view });
|
|
||||||
}
|
|
||||||
|
|
||||||
Vector<u16> utf8_to_utf16(Utf8View const& utf8_view)
|
|
||||||
{
|
{
|
||||||
Vector<u16> utf16_data;
|
Vector<u16> utf16_data;
|
||||||
|
|
||||||
for (auto code_point : utf8_view) {
|
for (auto code_point : view) {
|
||||||
if (code_point < first_supplementary_plane_code_point) {
|
if (code_point < first_supplementary_plane_code_point) {
|
||||||
utf16_data.append(static_cast<u16>(code_point));
|
utf16_data.append(static_cast<u16>(code_point));
|
||||||
} else {
|
} else {
|
||||||
|
@ -40,6 +37,21 @@ Vector<u16> utf8_to_utf16(Utf8View const& utf8_view)
|
||||||
return utf16_data;
|
return utf16_data;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Vector<u16> utf8_to_utf16(StringView const& utf8_view)
|
||||||
|
{
|
||||||
|
return to_utf16_impl(Utf8View { utf8_view });
|
||||||
|
}
|
||||||
|
|
||||||
|
Vector<u16> utf8_to_utf16(Utf8View const& utf8_view)
|
||||||
|
{
|
||||||
|
return to_utf16_impl(utf8_view);
|
||||||
|
}
|
||||||
|
|
||||||
|
Vector<u16> utf32_to_utf16(Utf32View const& utf32_view)
|
||||||
|
{
|
||||||
|
return to_utf16_impl(utf32_view);
|
||||||
|
}
|
||||||
|
|
||||||
bool Utf16View::is_high_surrogate(u16 code_unit)
|
bool Utf16View::is_high_surrogate(u16 code_unit)
|
||||||
{
|
{
|
||||||
return (code_unit >= high_surrogate_min) && (code_unit <= high_surrogate_max);
|
return (code_unit >= high_surrogate_min) && (code_unit <= high_surrogate_max);
|
||||||
|
@ -98,6 +110,36 @@ u16 Utf16View::code_unit_at(size_t index) const
|
||||||
return m_code_units[index];
|
return m_code_units[index];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
size_t Utf16View::code_point_offset_of(size_t code_unit_offset) const
|
||||||
|
{
|
||||||
|
size_t code_point_offset = 0;
|
||||||
|
|
||||||
|
for (auto it = begin(); it != end(); ++it) {
|
||||||
|
if (code_unit_offset == 0)
|
||||||
|
return code_point_offset;
|
||||||
|
|
||||||
|
code_unit_offset -= it.length_in_code_units();
|
||||||
|
++code_point_offset;
|
||||||
|
}
|
||||||
|
|
||||||
|
return code_point_offset;
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t Utf16View::code_unit_offset_of(size_t code_point_offset) const
|
||||||
|
{
|
||||||
|
size_t code_unit_offset = 0;
|
||||||
|
|
||||||
|
for (auto it = begin(); it != end(); ++it) {
|
||||||
|
if (code_point_offset == 0)
|
||||||
|
return code_unit_offset;
|
||||||
|
|
||||||
|
code_unit_offset += it.length_in_code_units();
|
||||||
|
--code_point_offset;
|
||||||
|
}
|
||||||
|
|
||||||
|
return code_unit_offset;
|
||||||
|
}
|
||||||
|
|
||||||
Utf16View Utf16View::substring_view(size_t code_unit_offset, size_t code_unit_length) const
|
Utf16View Utf16View::substring_view(size_t code_unit_offset, size_t code_unit_length) const
|
||||||
{
|
{
|
||||||
VERIFY(!Checked<size_t>::addition_would_overflow(code_unit_offset, code_unit_length));
|
VERIFY(!Checked<size_t>::addition_would_overflow(code_unit_offset, code_unit_length));
|
||||||
|
@ -106,6 +148,30 @@ Utf16View Utf16View::substring_view(size_t code_unit_offset, size_t code_unit_le
|
||||||
return Utf16View { m_code_units.slice(code_unit_offset, code_unit_length) };
|
return Utf16View { m_code_units.slice(code_unit_offset, code_unit_length) };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Utf16View Utf16View::unicode_substring_view(size_t code_point_offset, size_t code_point_length) const
|
||||||
|
{
|
||||||
|
if (code_point_length == 0)
|
||||||
|
return {};
|
||||||
|
|
||||||
|
auto code_unit_offset_of = [&](Utf16CodePointIterator const& it) { return it.m_ptr - begin_ptr(); };
|
||||||
|
size_t code_point_index = 0;
|
||||||
|
size_t code_unit_offset = 0;
|
||||||
|
|
||||||
|
for (auto it = begin(); it != end(); ++it) {
|
||||||
|
if (code_point_index == code_point_offset)
|
||||||
|
code_unit_offset = code_unit_offset_of(it);
|
||||||
|
|
||||||
|
if (code_point_index == (code_point_offset + code_point_length - 1)) {
|
||||||
|
size_t code_unit_length = code_unit_offset_of(++it) - code_unit_offset;
|
||||||
|
return substring_view(code_unit_offset, code_unit_length);
|
||||||
|
}
|
||||||
|
|
||||||
|
++code_point_index;
|
||||||
|
}
|
||||||
|
|
||||||
|
VERIFY_NOT_REACHED();
|
||||||
|
}
|
||||||
|
|
||||||
bool Utf16View::validate(size_t& valid_code_units) const
|
bool Utf16View::validate(size_t& valid_code_units) const
|
||||||
{
|
{
|
||||||
valid_code_units = 0;
|
valid_code_units = 0;
|
||||||
|
|
|
@ -17,6 +17,7 @@ namespace AK {
|
||||||
|
|
||||||
Vector<u16> utf8_to_utf16(StringView const&);
|
Vector<u16> utf8_to_utf16(StringView const&);
|
||||||
Vector<u16> utf8_to_utf16(Utf8View const&);
|
Vector<u16> utf8_to_utf16(Utf8View const&);
|
||||||
|
Vector<u16> utf32_to_utf16(Utf32View const&);
|
||||||
|
|
||||||
class Utf16View;
|
class Utf16View;
|
||||||
|
|
||||||
|
@ -67,8 +68,6 @@ public:
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
Utf16View& operator=(Utf16View const&) = default;
|
|
||||||
Utf16View& operator=(Utf16View&&) = default;
|
|
||||||
bool operator==(Utf16View const& other) const;
|
bool operator==(Utf16View const& other) const;
|
||||||
|
|
||||||
enum class AllowInvalidCodeUnits {
|
enum class AllowInvalidCodeUnits {
|
||||||
|
@ -78,6 +77,7 @@ public:
|
||||||
|
|
||||||
String to_utf8(AllowInvalidCodeUnits = AllowInvalidCodeUnits::No) const;
|
String to_utf8(AllowInvalidCodeUnits = AllowInvalidCodeUnits::No) const;
|
||||||
|
|
||||||
|
bool is_null() const { return m_code_units.is_null(); }
|
||||||
bool is_empty() const { return m_code_units.is_empty(); }
|
bool is_empty() const { return m_code_units.is_empty(); }
|
||||||
size_t length_in_code_units() const { return m_code_units.size(); }
|
size_t length_in_code_units() const { return m_code_units.size(); }
|
||||||
size_t length_in_code_points() const;
|
size_t length_in_code_points() const;
|
||||||
|
@ -88,9 +88,15 @@ public:
|
||||||
u16 const* data() const { return m_code_units.data(); }
|
u16 const* data() const { return m_code_units.data(); }
|
||||||
u16 code_unit_at(size_t index) const;
|
u16 code_unit_at(size_t index) const;
|
||||||
|
|
||||||
|
size_t code_point_offset_of(size_t code_unit_offset) const;
|
||||||
|
size_t code_unit_offset_of(size_t code_point_offset) const;
|
||||||
|
|
||||||
Utf16View substring_view(size_t code_unit_offset, size_t code_unit_length) const;
|
Utf16View substring_view(size_t code_unit_offset, size_t code_unit_length) const;
|
||||||
Utf16View substring_view(size_t code_unit_offset) const { return substring_view(code_unit_offset, length_in_code_units() - code_unit_offset); }
|
Utf16View substring_view(size_t code_unit_offset) const { return substring_view(code_unit_offset, length_in_code_units() - code_unit_offset); }
|
||||||
|
|
||||||
|
Utf16View unicode_substring_view(size_t code_point_offset, size_t code_point_length) const;
|
||||||
|
Utf16View unicode_substring_view(size_t code_point_offset) const { return unicode_substring_view(code_point_offset, length_in_code_points() - code_point_offset); }
|
||||||
|
|
||||||
bool validate(size_t& valid_code_units) const;
|
bool validate(size_t& valid_code_units) const;
|
||||||
bool validate() const
|
bool validate() const
|
||||||
{
|
{
|
||||||
|
|
Loading…
Reference in a new issue