mirror of
https://github.com/dart-lang/sdk
synced 2024-11-02 15:01:30 +00:00
8b0f4a8c61
TEST=build Change-Id: I61ebb04f414d36813a99b424fb1bcdb440f68ad5 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/293801 Commit-Queue: Ryan Macnak <rmacnak@google.com> Reviewed-by: Alexander Aprelev <aam@google.com>
280 lines
8.4 KiB
C++
280 lines
8.4 KiB
C++
// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file
|
|
// for details. All rights reserved. Use of this source code is governed by a
|
|
// BSD-style license that can be found in the LICENSE file.
|
|
|
|
#include "platform/unicode.h"
|
|
|
|
#include "platform/allocation.h"
|
|
#include "platform/globals.h"
|
|
#include "platform/syslog.h"
|
|
|
|
namespace dart {
|
|
|
|
// clang-format off
|
|
const int8_t Utf8::kTrailBytes[256] = {
|
|
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
|
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
|
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
|
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
|
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
|
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
|
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
|
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
|
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
|
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
|
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
|
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
|
|
4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 0, 0
|
|
};
|
|
// clang-format on
|
|
|
|
const uint32_t Utf8::kMagicBits[7] = {0, // Padding.
|
|
0x00000000, 0x00003080, 0x000E2080,
|
|
0x03C82080, 0xFA082080, 0x82082080};
|
|
|
|
// Minimum values of code points used to check shortest form.
|
|
const uint32_t Utf8::kOverlongMinimum[7] = {0, // Padding.
|
|
0x0, 0x80, 0x800,
|
|
0x10000, 0xFFFFFFFF, 0xFFFFFFFF};
|
|
|
|
// Returns the most restricted coding form in which the sequence of utf8
|
|
// characters in 'utf8_array' can be represented in, and the number of
|
|
// code units needed in that form.
|
|
intptr_t Utf8::CodeUnitCount(const uint8_t* utf8_array,
|
|
intptr_t array_len,
|
|
Type* type) {
|
|
intptr_t len = 0;
|
|
Type char_type = kLatin1;
|
|
for (intptr_t i = 0; i < array_len; i++) {
|
|
uint8_t code_unit = utf8_array[i];
|
|
if (!IsTrailByte(code_unit)) {
|
|
++len;
|
|
if (!IsLatin1SequenceStart(code_unit)) { // > U+00FF
|
|
if (IsSupplementarySequenceStart(code_unit)) { // >= U+10000
|
|
char_type = kSupplementary;
|
|
++len;
|
|
} else if (char_type == kLatin1) {
|
|
char_type = kBMP;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
*type = char_type;
|
|
return len;
|
|
}
|
|
|
|
// Returns true if str is a valid NUL-terminated UTF-8 string.
|
|
bool Utf8::IsValid(const uint8_t* utf8_array, intptr_t array_len) {
|
|
intptr_t i = 0;
|
|
while (i < array_len) {
|
|
uint32_t ch = utf8_array[i] & 0xFF;
|
|
intptr_t j = 1;
|
|
if (ch >= 0x80) {
|
|
int8_t num_trail_bytes = kTrailBytes[ch];
|
|
bool is_malformed = false;
|
|
for (; j < num_trail_bytes; ++j) {
|
|
if ((i + j) < array_len) {
|
|
uint8_t code_unit = utf8_array[i + j];
|
|
is_malformed |= !IsTrailByte(code_unit);
|
|
ch = (ch << 6) + code_unit;
|
|
} else {
|
|
return false;
|
|
}
|
|
}
|
|
ch -= kMagicBits[num_trail_bytes];
|
|
if (!((is_malformed == false) && (j == num_trail_bytes) &&
|
|
!Utf::IsOutOfRange(ch) && !IsNonShortestForm(ch, j))) {
|
|
return false;
|
|
}
|
|
}
|
|
i += j;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
intptr_t Utf8::Length(int32_t ch) {
|
|
if (ch <= kMaxOneByteChar) {
|
|
return 1;
|
|
} else if (ch <= kMaxTwoByteChar) {
|
|
return 2;
|
|
} else if (ch <= kMaxThreeByteChar) {
|
|
return 3;
|
|
}
|
|
ASSERT(ch <= kMaxFourByteChar);
|
|
return 4;
|
|
}
|
|
|
|
intptr_t Utf8::Encode(int32_t ch, char* dst) {
|
|
constexpr int kMask = ~(1 << 6);
|
|
if (ch <= kMaxOneByteChar) {
|
|
dst[0] = ch;
|
|
return 1;
|
|
}
|
|
if (ch <= kMaxTwoByteChar) {
|
|
dst[0] = 0xC0 | (ch >> 6);
|
|
dst[1] = 0x80 | (ch & kMask);
|
|
return 2;
|
|
}
|
|
if (ch <= kMaxThreeByteChar) {
|
|
dst[0] = 0xE0 | (ch >> 12);
|
|
dst[1] = 0x80 | ((ch >> 6) & kMask);
|
|
dst[2] = 0x80 | (ch & kMask);
|
|
return 3;
|
|
}
|
|
ASSERT(ch <= kMaxFourByteChar);
|
|
dst[0] = 0xF0 | (ch >> 18);
|
|
dst[1] = 0x80 | ((ch >> 12) & kMask);
|
|
dst[2] = 0x80 | ((ch >> 6) & kMask);
|
|
dst[3] = 0x80 | (ch & kMask);
|
|
return 4;
|
|
}
|
|
|
|
intptr_t Utf8::Decode(const uint8_t* utf8_array,
|
|
intptr_t array_len,
|
|
int32_t* dst) {
|
|
uint32_t ch = utf8_array[0] & 0xFF;
|
|
intptr_t i = 1;
|
|
if (ch >= 0x80) {
|
|
intptr_t num_trail_bytes = kTrailBytes[ch];
|
|
bool is_malformed = false;
|
|
for (; i < num_trail_bytes; ++i) {
|
|
if (i < array_len) {
|
|
uint8_t code_unit = utf8_array[i];
|
|
is_malformed |= !IsTrailByte(code_unit);
|
|
ch = (ch << 6) + code_unit;
|
|
} else {
|
|
*dst = -1;
|
|
return 0;
|
|
}
|
|
}
|
|
ch -= kMagicBits[num_trail_bytes];
|
|
if (!((is_malformed == false) && (i == num_trail_bytes) &&
|
|
!Utf::IsOutOfRange(ch) && !IsNonShortestForm(ch, i))) {
|
|
*dst = -1;
|
|
return 0;
|
|
}
|
|
}
|
|
*dst = ch;
|
|
return i;
|
|
}
|
|
intptr_t Utf8::ReportInvalidByte(const uint8_t* utf8_array,
|
|
intptr_t array_len,
|
|
intptr_t len) {
|
|
intptr_t i = 0;
|
|
intptr_t j = 0;
|
|
intptr_t num_bytes;
|
|
for (; (i < array_len) && (j < len); i += num_bytes, ++j) {
|
|
int32_t ch;
|
|
bool is_supplementary = IsSupplementarySequenceStart(utf8_array[i]);
|
|
num_bytes = Utf8::Decode(&utf8_array[i], (array_len - i), &ch);
|
|
if (ch == -1) {
|
|
break; // Invalid input.
|
|
}
|
|
if (is_supplementary) {
|
|
j = j + 1;
|
|
}
|
|
}
|
|
#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
|
|
// Remain silent while libFuzzer is active, since
|
|
// the output only slows down the in-process fuzzing.
|
|
#else
|
|
Syslog::PrintErr("Invalid UTF8 sequence encountered, ");
|
|
for (intptr_t idx = 0; idx < 10 && (i + idx) < array_len; idx++) {
|
|
Syslog::PrintErr("(Error Code: %X + idx: %" Pd " )", utf8_array[idx + i],
|
|
(idx + i));
|
|
}
|
|
Syslog::PrintErr("\n");
|
|
#endif
|
|
return i;
|
|
}
|
|
|
|
bool Utf8::DecodeToLatin1(const uint8_t* utf8_array,
|
|
intptr_t array_len,
|
|
uint8_t* dst,
|
|
intptr_t len) {
|
|
intptr_t i = 0;
|
|
intptr_t j = 0;
|
|
intptr_t num_bytes;
|
|
for (; (i < array_len) && (j < len); i += num_bytes, ++j) {
|
|
int32_t ch;
|
|
ASSERT(IsLatin1SequenceStart(utf8_array[i]));
|
|
num_bytes = Utf8::Decode(&utf8_array[i], (array_len - i), &ch);
|
|
if (ch == -1) {
|
|
return false; // Invalid input.
|
|
}
|
|
ASSERT(Utf::IsLatin1(ch));
|
|
dst[j] = ch;
|
|
}
|
|
if ((i < array_len) && (j == len)) {
|
|
return false; // Output overflow.
|
|
}
|
|
return true; // Success.
|
|
}
|
|
|
|
bool Utf8::DecodeToUTF16(const uint8_t* utf8_array,
|
|
intptr_t array_len,
|
|
uint16_t* dst,
|
|
intptr_t len) {
|
|
intptr_t i = 0;
|
|
intptr_t j = 0;
|
|
intptr_t num_bytes;
|
|
for (; (i < array_len) && (j < len); i += num_bytes, ++j) {
|
|
int32_t ch;
|
|
bool is_supplementary = IsSupplementarySequenceStart(utf8_array[i]);
|
|
num_bytes = Utf8::Decode(&utf8_array[i], (array_len - i), &ch);
|
|
if (ch == -1) {
|
|
return false; // Invalid input.
|
|
}
|
|
if (is_supplementary) {
|
|
if (j == (len - 1)) return false; // Output overflow.
|
|
Utf16::Encode(ch, &dst[j]);
|
|
j = j + 1;
|
|
} else {
|
|
dst[j] = ch;
|
|
}
|
|
}
|
|
if ((i < array_len) && (j == len)) {
|
|
return false; // Output overflow.
|
|
}
|
|
return true; // Success.
|
|
}
|
|
|
|
bool Utf8::DecodeToUTF32(const uint8_t* utf8_array,
|
|
intptr_t array_len,
|
|
int32_t* dst,
|
|
intptr_t len) {
|
|
intptr_t i = 0;
|
|
intptr_t j = 0;
|
|
intptr_t num_bytes;
|
|
for (; (i < array_len) && (j < len); i += num_bytes, ++j) {
|
|
int32_t ch;
|
|
num_bytes = Utf8::Decode(&utf8_array[i], (array_len - i), &ch);
|
|
if (ch == -1) {
|
|
return false; // Invalid input.
|
|
}
|
|
dst[j] = ch;
|
|
}
|
|
if ((i < array_len) && (j == len)) {
|
|
return false; // Output overflow.
|
|
}
|
|
return true; // Success.
|
|
}
|
|
|
|
bool Utf8::DecodeCStringToUTF32(const char* str, int32_t* dst, intptr_t len) {
|
|
ASSERT(str != nullptr);
|
|
intptr_t array_len = strlen(str);
|
|
const uint8_t* utf8_array = reinterpret_cast<const uint8_t*>(str);
|
|
return Utf8::DecodeToUTF32(utf8_array, array_len, dst, len);
|
|
}
|
|
|
|
void Utf16::Encode(int32_t codepoint, uint16_t* dst) {
|
|
ASSERT(codepoint > Utf16::kMaxCodeUnit);
|
|
ASSERT(dst != nullptr);
|
|
dst[0] = (Utf16::kLeadSurrogateOffset + (codepoint >> 10));
|
|
dst[1] = (0xDC00 + (codepoint & 0x3FF));
|
|
}
|
|
|
|
} // namespace dart
|