mirror of
https://github.com/dart-lang/sdk
synced 2024-09-15 21:20:36 +00:00
Remove deprecated dart:utf library.
BUG= http://dartbug.com/12843 R=fschneider@google.com, sgjesse@google.com Review URL: https://codereview.chromium.org//48133002 git-svn-id: https://dart.googlecode.com/svn/branches/bleeding_edge/dart@29537 260f80e4-7a28-3924-810f-c04153c831b5
This commit is contained in:
lrn@google.com
parent
7f9dfb1890
commit
41ff075e31
|
|
@ -436,7 +436,6 @@ BENCHMARK(StandaloneSnapshotSize) {
|
|||
"import 'dart:isolate';\n"
|
||||
"import 'dart:mirrors';\n"
|
||||
"import 'dart:typed_data';\n"
|
||||
"import 'dart:utf';\n"
|
||||
"import 'dart:builtin';\n"
|
||||
"import 'dart:io';\n"
|
||||
"\n";
|
||||
|
|
|
|||
|
|
@ -66,10 +66,6 @@ static bootstrap_lib_props bootstrap_libraries[] = {
|
|||
typed_data,
|
||||
Bootstrap::typed_data_source_paths_,
|
||||
Bootstrap::typed_data_patch_paths_),
|
||||
INIT_LIBRARY(ObjectStore::kUtf,
|
||||
utf,
|
||||
Bootstrap::utf_source_paths_,
|
||||
NULL),
|
||||
|
||||
{ ObjectStore::kNone, NULL, NULL, NULL, NULL }
|
||||
};
|
||||
|
|
|
|||
|
|
@ -7951,11 +7951,6 @@ RawLibrary* Library::TypedDataLibrary() {
|
|||
}
|
||||
|
||||
|
||||
RawLibrary* Library::UtfLibrary() {
|
||||
return Isolate::Current()->object_store()->utf_library();
|
||||
}
|
||||
|
||||
|
||||
const char* Library::ToCString() const {
|
||||
const char* kFormat = "Library:'%s'";
|
||||
const String& name = String::Handle(url());
|
||||
|
|
|
|||
|
|
@ -2544,7 +2544,6 @@ class Library : public Object {
|
|||
static RawLibrary* MirrorsLibrary();
|
||||
static RawLibrary* NativeWrappersLibrary();
|
||||
static RawLibrary* TypedDataLibrary();
|
||||
static RawLibrary* UtfLibrary();
|
||||
|
||||
// Eagerly compile all classes and functions in the library.
|
||||
static RawError* CompileAll();
|
||||
|
|
|
|||
|
|
@ -59,7 +59,6 @@ ObjectStore::ObjectStore()
|
|||
native_wrappers_library_(Library::null()),
|
||||
root_library_(Library::null()),
|
||||
typed_data_library_(Library::null()),
|
||||
utf_library_(Library::null()),
|
||||
libraries_(GrowableObjectArray::null()),
|
||||
pending_classes_(GrowableObjectArray::null()),
|
||||
pending_functions_(GrowableObjectArray::null()),
|
||||
|
|
|
|||
|
|
@ -30,7 +30,6 @@ class ObjectStore {
|
|||
kMath,
|
||||
kMirrors,
|
||||
kTypedData,
|
||||
kUtf,
|
||||
};
|
||||
|
||||
~ObjectStore();
|
||||
|
|
@ -279,7 +278,6 @@ class ObjectStore {
|
|||
RawLibrary* math_library() const { return math_library_; }
|
||||
RawLibrary* mirrors_library() const { return mirrors_library_; }
|
||||
RawLibrary* typed_data_library() const { return typed_data_library_; }
|
||||
RawLibrary* utf_library() const { return utf_library_; }
|
||||
void set_bootstrap_library(BootstrapLibraryId index, const Library& value) {
|
||||
switch (index) {
|
||||
case kAsync:
|
||||
|
|
@ -309,9 +307,6 @@ class ObjectStore {
|
|||
case kTypedData:
|
||||
typed_data_library_ = value.raw();
|
||||
break;
|
||||
case kUtf:
|
||||
utf_library_ = value.raw();
|
||||
break;
|
||||
default:
|
||||
UNREACHABLE();
|
||||
}
|
||||
|
|
@ -480,7 +475,6 @@ class ObjectStore {
|
|||
RawLibrary* native_wrappers_library_;
|
||||
RawLibrary* root_library_;
|
||||
RawLibrary* typed_data_library_;
|
||||
RawLibrary* utf_library_;
|
||||
RawGrowableObjectArray* libraries_;
|
||||
RawGrowableObjectArray* pending_classes_;
|
||||
RawGrowableObjectArray* pending_functions_;
|
||||
|
|
|
|||
|
|
@ -269,7 +269,6 @@ class ObjectPointerVisitor;
|
|||
V(DartMath, "dart:math") \
|
||||
V(DartMirrors, "dart:mirrors") \
|
||||
V(DartTypedData, "dart:typed_data") \
|
||||
V(DartUtf, "dart:utf") \
|
||||
V(_Random, "_Random") \
|
||||
V(_state, "_state") \
|
||||
V(_A, "_A") \
|
||||
|
|
|
|||
|
|
@ -25,7 +25,6 @@
|
|||
'isolate_patch_cc_file': '<(gen_source_dir)/isolate_patch_gen.cc',
|
||||
'typed_data_cc_file': '<(gen_source_dir)/typed_data_gen.cc',
|
||||
'typed_data_patch_cc_file': '<(gen_source_dir)/typed_data_patch_gen.cc',
|
||||
'utf_cc_file': '<(gen_source_dir)/utf_gen.cc',
|
||||
'snapshot_test_dat_file': '<(gen_source_dir)/snapshot_test.dat',
|
||||
'snapshot_test_in_dat_file': 'snapshot_test_in.dat',
|
||||
'snapshot_test_dart_file': 'snapshot_test.dart',
|
||||
|
|
@ -111,7 +110,6 @@
|
|||
'generate_mirrors_patch_cc_file#host',
|
||||
'generate_typed_data_cc_file#host',
|
||||
'generate_typed_data_patch_cc_file#host',
|
||||
'generate_utf_cc_file#host',
|
||||
],
|
||||
'includes': [
|
||||
'../lib/async_sources.gypi',
|
||||
|
|
@ -143,7 +141,6 @@
|
|||
'<(mirrors_patch_cc_file)',
|
||||
'<(typed_data_cc_file)',
|
||||
'<(typed_data_patch_cc_file)',
|
||||
'<(utf_cc_file)',
|
||||
],
|
||||
'include_dirs': [
|
||||
'..',
|
||||
|
|
@ -888,39 +885,6 @@
|
|||
},
|
||||
]
|
||||
},
|
||||
{
|
||||
'target_name': 'generate_utf_cc_file',
|
||||
'type': 'none',
|
||||
'toolsets':['host'],
|
||||
'includes': [
|
||||
# Load the shared utf sources.
|
||||
'../../sdk/lib/utf/utf_sources.gypi',
|
||||
],
|
||||
'actions': [
|
||||
{
|
||||
'action_name': 'generate_utf_cc',
|
||||
'inputs': [
|
||||
'../tools/gen_library_src_paths.py',
|
||||
'<(libgen_in_cc_file)',
|
||||
'<@(_sources)',
|
||||
],
|
||||
'outputs': [
|
||||
'<(utf_cc_file)',
|
||||
],
|
||||
'action': [
|
||||
'python',
|
||||
'tools/gen_library_src_paths.py',
|
||||
'--output', '<(utf_cc_file)',
|
||||
'--input_cc', '<(libgen_in_cc_file)',
|
||||
'--include', 'vm/bootstrap.h',
|
||||
'--var_name', 'dart::Bootstrap::utf_source_paths_',
|
||||
'--library_name', 'dart:utf',
|
||||
'<@(_sources)',
|
||||
],
|
||||
'message': 'Generating ''<(utf_cc_file)'' file.'
|
||||
},
|
||||
]
|
||||
},
|
||||
{
|
||||
'target_name': 'generate_snapshot_test_dat_file',
|
||||
'type': 'none',
|
||||
|
|
|
|||
|
|
@ -112,10 +112,6 @@ const Map<String, LibraryInfo> LIBRARIES = const {
|
|||
maturity: Maturity.WEB_STABLE,
|
||||
dart2jsPath: "svg/dart2js/svg_dart2js.dart"),
|
||||
|
||||
"utf": const LibraryInfo(
|
||||
"utf/utf.dart",
|
||||
maturity: Maturity.DEPRECATED),
|
||||
|
||||
"web_audio": const LibraryInfo(
|
||||
"web_audio/dartium/web_audio_dartium.dart",
|
||||
category: "Client",
|
||||
|
|
|
|||
|
|
@ -1,289 +0,0 @@
|
|||
// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file
|
||||
// for details. All rights reserved. Use of this source code is governed by a
|
||||
// BSD-style license that can be found in the LICENSE file.
|
||||
|
||||
/**
|
||||
* Support for encoding and decoding Unicode characters in UTF-8, UTF-16, and
|
||||
* UTF-32.
|
||||
*/
|
||||
@deprecated
|
||||
library dart.utf;
|
||||
import "dart:_collection-dev" show deprecated, EfficientLength;
|
||||
import "dart:async";
|
||||
import "dart:collection";
|
||||
part "utf_stream.dart";
|
||||
part "utf8.dart";
|
||||
part "utf16.dart";
|
||||
part "utf32.dart";
|
||||
|
||||
// TODO(jmesserly): would be nice to have this on String (dartbug.com/6501).
|
||||
/**
|
||||
* *DEPRECATED*: Use `dart:convert` or `package:utf/utf.dart` instead.
|
||||
*
|
||||
* Provide a list of Unicode codepoints for a given string.
|
||||
*/
|
||||
@deprecated
|
||||
List<int> stringToCodepoints(String str) {
|
||||
// Note: str.codeUnits gives us 16-bit code units on all Dart implementations.
|
||||
// So we need to convert.
|
||||
return _utf16CodeUnitsToCodepoints(str.codeUnits);
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a string from the provided Unicode codepoints.
|
||||
*
|
||||
* *Deprecated* Use [String.fromCharCodes] instead.
|
||||
*/
|
||||
@deprecated
|
||||
String codepointsToString(List<int> codepoints) {
|
||||
return new String.fromCharCodes(codepoints);
|
||||
}
|
||||
|
||||
/**
|
||||
* Invalid codepoints or encodings may be substituted with the value U+fffd.
|
||||
*/
|
||||
@deprecated
|
||||
const int UNICODE_REPLACEMENT_CHARACTER_CODEPOINT = 0xfffd;
|
||||
@deprecated
|
||||
const int UNICODE_BOM = 0xfeff;
|
||||
@deprecated
|
||||
const int UNICODE_UTF_BOM_LO = 0xff;
|
||||
@deprecated
|
||||
const int UNICODE_UTF_BOM_HI = 0xfe;
|
||||
|
||||
@deprecated
|
||||
const int UNICODE_BYTE_ZERO_MASK = 0xff;
|
||||
@deprecated
|
||||
const int UNICODE_BYTE_ONE_MASK = 0xff00;
|
||||
@deprecated
|
||||
const int UNICODE_VALID_RANGE_MAX = 0x10ffff;
|
||||
@deprecated
|
||||
const int UNICODE_PLANE_ONE_MAX = 0xffff;
|
||||
@deprecated
|
||||
const int UNICODE_UTF16_RESERVED_LO = 0xd800;
|
||||
@deprecated
|
||||
const int UNICODE_UTF16_RESERVED_HI = 0xdfff;
|
||||
@deprecated
|
||||
const int UNICODE_UTF16_OFFSET = 0x10000;
|
||||
@deprecated
|
||||
const int UNICODE_UTF16_SURROGATE_UNIT_0_BASE = 0xd800;
|
||||
@deprecated
|
||||
const int UNICODE_UTF16_SURROGATE_UNIT_1_BASE = 0xdc00;
|
||||
@deprecated
|
||||
const int UNICODE_UTF16_HI_MASK = 0xffc00;
|
||||
@deprecated
|
||||
const int UNICODE_UTF16_LO_MASK = 0x3ff;
|
||||
|
||||
/**
|
||||
* Encode code points as UTF16 code units.
|
||||
*/
|
||||
List<int> _codepointsToUtf16CodeUnits(
|
||||
List<int> codepoints,
|
||||
[int offset = 0,
|
||||
int length,
|
||||
int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) {
|
||||
|
||||
_ListRange listRange = new _ListRange(codepoints, offset, length);
|
||||
int encodedLength = 0;
|
||||
for (int value in listRange) {
|
||||
if ((value >= 0 && value < UNICODE_UTF16_RESERVED_LO) ||
|
||||
(value > UNICODE_UTF16_RESERVED_HI && value <= UNICODE_PLANE_ONE_MAX)) {
|
||||
encodedLength++;
|
||||
} else if (value > UNICODE_PLANE_ONE_MAX &&
|
||||
value <= UNICODE_VALID_RANGE_MAX) {
|
||||
encodedLength += 2;
|
||||
} else {
|
||||
encodedLength++;
|
||||
}
|
||||
}
|
||||
|
||||
List<int> codeUnitsBuffer = new List<int>(encodedLength);
|
||||
int j = 0;
|
||||
for (int value in listRange) {
|
||||
if ((value >= 0 && value < UNICODE_UTF16_RESERVED_LO) ||
|
||||
(value > UNICODE_UTF16_RESERVED_HI && value <= UNICODE_PLANE_ONE_MAX)) {
|
||||
codeUnitsBuffer[j++] = value;
|
||||
} else if (value > UNICODE_PLANE_ONE_MAX &&
|
||||
value <= UNICODE_VALID_RANGE_MAX) {
|
||||
int base = value - UNICODE_UTF16_OFFSET;
|
||||
codeUnitsBuffer[j++] = UNICODE_UTF16_SURROGATE_UNIT_0_BASE +
|
||||
((base & UNICODE_UTF16_HI_MASK) >> 10);
|
||||
codeUnitsBuffer[j++] = UNICODE_UTF16_SURROGATE_UNIT_1_BASE +
|
||||
(base & UNICODE_UTF16_LO_MASK);
|
||||
} else if (replacementCodepoint != null) {
|
||||
codeUnitsBuffer[j++] = replacementCodepoint;
|
||||
} else {
|
||||
throw new ArgumentError("Invalid encoding");
|
||||
}
|
||||
}
|
||||
return codeUnitsBuffer;
|
||||
}
|
||||
|
||||
/**
|
||||
* Decodes the utf16 codeunits to codepoints.
|
||||
*/
|
||||
List<int> _utf16CodeUnitsToCodepoints(
|
||||
List<int> utf16CodeUnits, [int offset = 0, int length,
|
||||
int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) {
|
||||
_ListRangeIterator source =
|
||||
(new _ListRange(utf16CodeUnits, offset, length)).iterator;
|
||||
Utf16CodeUnitDecoder decoder = new Utf16CodeUnitDecoder
|
||||
.fromListRangeIterator(source, replacementCodepoint);
|
||||
List<int> codepoints = new List<int>(source.remaining);
|
||||
int i = 0;
|
||||
while (decoder.moveNext()) {
|
||||
codepoints[i++] = decoder.current;
|
||||
}
|
||||
if (i == codepoints.length) {
|
||||
return codepoints;
|
||||
} else {
|
||||
List<int> codepointTrunc = new List<int>(i);
|
||||
codepointTrunc.setRange(0, i, codepoints);
|
||||
return codepointTrunc;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
class Utf16CodeUnitDecoder implements Iterator<int> {
|
||||
final _ListRangeIterator utf16CodeUnitIterator;
|
||||
final int replacementCodepoint;
|
||||
int _current = null;
|
||||
|
||||
Utf16CodeUnitDecoder(List<int> utf16CodeUnits, [int offset = 0, int length,
|
||||
int this.replacementCodepoint =
|
||||
UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) :
|
||||
utf16CodeUnitIterator =
|
||||
(new _ListRange(utf16CodeUnits, offset, length)).iterator;
|
||||
|
||||
Utf16CodeUnitDecoder.fromListRangeIterator(
|
||||
_ListRangeIterator this.utf16CodeUnitIterator,
|
||||
int this.replacementCodepoint);
|
||||
|
||||
Iterator<int> get iterator => this;
|
||||
|
||||
int get current => _current;
|
||||
|
||||
bool moveNext() {
|
||||
_current = null;
|
||||
if (!utf16CodeUnitIterator.moveNext()) return false;
|
||||
|
||||
int value = utf16CodeUnitIterator.current;
|
||||
if (value < 0) {
|
||||
if (replacementCodepoint != null) {
|
||||
_current = replacementCodepoint;
|
||||
} else {
|
||||
throw new ArgumentError(
|
||||
"Invalid UTF16 at ${utf16CodeUnitIterator.position}");
|
||||
}
|
||||
} else if (value < UNICODE_UTF16_RESERVED_LO ||
|
||||
(value > UNICODE_UTF16_RESERVED_HI && value <= UNICODE_PLANE_ONE_MAX)) {
|
||||
// transfer directly
|
||||
_current = value;
|
||||
} else if (value < UNICODE_UTF16_SURROGATE_UNIT_1_BASE &&
|
||||
utf16CodeUnitIterator.moveNext()) {
|
||||
// merge surrogate pair
|
||||
int nextValue = utf16CodeUnitIterator.current;
|
||||
if (nextValue >= UNICODE_UTF16_SURROGATE_UNIT_1_BASE &&
|
||||
nextValue <= UNICODE_UTF16_RESERVED_HI) {
|
||||
value = (value - UNICODE_UTF16_SURROGATE_UNIT_0_BASE) << 10;
|
||||
value += UNICODE_UTF16_OFFSET +
|
||||
(nextValue - UNICODE_UTF16_SURROGATE_UNIT_1_BASE);
|
||||
_current = value;
|
||||
} else {
|
||||
if (nextValue >= UNICODE_UTF16_SURROGATE_UNIT_0_BASE &&
|
||||
nextValue < UNICODE_UTF16_SURROGATE_UNIT_1_BASE) {
|
||||
utf16CodeUnitIterator.backup();
|
||||
}
|
||||
if (replacementCodepoint != null) {
|
||||
_current = replacementCodepoint;
|
||||
} else {
|
||||
throw new ArgumentError(
|
||||
"Invalid UTF16 at ${utf16CodeUnitIterator.position}");
|
||||
}
|
||||
}
|
||||
} else if (replacementCodepoint != null) {
|
||||
_current = replacementCodepoint;
|
||||
} else {
|
||||
throw new ArgumentError(
|
||||
"Invalid UTF16 at ${utf16CodeUnitIterator.position}");
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* _ListRange in an internal type used to create a lightweight Interable on a
|
||||
* range within a source list. DO NOT MODIFY the underlying list while
|
||||
* iterating over it. The results of doing so are undefined.
|
||||
*/
|
||||
// TODO(floitsch): Consider removing the extend and switch to implements since
|
||||
// that's cheaper to allocate.
|
||||
class _ListRange extends IterableBase implements EfficientLength {
|
||||
final List _source;
|
||||
final int _offset;
|
||||
final int _length;
|
||||
|
||||
_ListRange(source, [offset = 0, length]) :
|
||||
this._source = source,
|
||||
this._offset = offset,
|
||||
this._length = (length == null ? source.length - offset : length) {
|
||||
if (_offset < 0 || _offset > _source.length) {
|
||||
throw new RangeError.value(_offset);
|
||||
}
|
||||
if (_length != null && (_length < 0)) {
|
||||
throw new RangeError.value(_length);
|
||||
}
|
||||
if (_length + _offset > _source.length) {
|
||||
throw new RangeError.value(_length + _offset);
|
||||
}
|
||||
}
|
||||
|
||||
_ListRangeIterator get iterator =>
|
||||
new _ListRangeIteratorImpl(_source, _offset, _offset + _length);
|
||||
|
||||
int get length => _length;
|
||||
}
|
||||
|
||||
/**
|
||||
* The _ListRangeIterator provides more capabilities than a standard iterator,
|
||||
* including the ability to get the current position, count remaining items,
|
||||
* and move forward/backward within the iterator.
|
||||
*/
|
||||
abstract class _ListRangeIterator implements Iterator<int> {
|
||||
bool moveNext();
|
||||
int get current;
|
||||
int get position;
|
||||
void backup([by]);
|
||||
int get remaining;
|
||||
void skip([count]);
|
||||
}
|
||||
|
||||
class _ListRangeIteratorImpl implements _ListRangeIterator {
|
||||
final List<int> _source;
|
||||
int _offset;
|
||||
final int _end;
|
||||
|
||||
_ListRangeIteratorImpl(this._source, int offset, this._end)
|
||||
: _offset = offset - 1;
|
||||
|
||||
int get current => _source[_offset];
|
||||
|
||||
bool moveNext() => ++_offset < _end;
|
||||
|
||||
int get position => _offset;
|
||||
|
||||
void backup([int by = 1]) {
|
||||
_offset -= by;
|
||||
}
|
||||
|
||||
int get remaining => _end - _offset - 1;
|
||||
|
||||
void skip([int count = 1]) {
|
||||
_offset += count;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1,329 +0,0 @@
|
|||
// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file
|
||||
// for details. All rights reserved. Use of this source code is governed by a
|
||||
// BSD-style license that can be found in the LICENSE file.
|
||||
|
||||
part of dart.utf;
|
||||
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
IterableUtf16Decoder decodeUtf16AsIterable(List<int> bytes, [int offset = 0,
|
||||
int length, int replacementCodepoint =
|
||||
UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) {
|
||||
return new IterableUtf16Decoder._(
|
||||
() => new Utf16BytesToCodeUnitsDecoder(bytes, offset, length,
|
||||
replacementCodepoint), replacementCodepoint);
|
||||
}
|
||||
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
IterableUtf16Decoder decodeUtf16beAsIterable(List<int> bytes, [int offset = 0,
|
||||
int length, bool stripBom = true, int replacementCodepoint =
|
||||
UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) {
|
||||
return new IterableUtf16Decoder._(
|
||||
() => new Utf16beBytesToCodeUnitsDecoder(bytes, offset, length, stripBom,
|
||||
replacementCodepoint), replacementCodepoint);
|
||||
}
|
||||
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
IterableUtf16Decoder decodeUtf16leAsIterable(List<int> bytes, [int offset = 0,
|
||||
int length, bool stripBom = true, int replacementCodepoint =
|
||||
UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) {
|
||||
return new IterableUtf16Decoder._(
|
||||
() => new Utf16leBytesToCodeUnitsDecoder(bytes, offset, length, stripBom,
|
||||
replacementCodepoint), replacementCodepoint);
|
||||
}
|
||||
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
String decodeUtf16(List<int> bytes, [int offset = 0, int length,
|
||||
int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) {
|
||||
Utf16BytesToCodeUnitsDecoder decoder = new Utf16BytesToCodeUnitsDecoder(bytes,
|
||||
offset, length, replacementCodepoint);
|
||||
List<int> codeunits = decoder.decodeRest();
|
||||
return new String.fromCharCodes(
|
||||
_utf16CodeUnitsToCodepoints(codeunits, 0, null, replacementCodepoint));
|
||||
}
|
||||
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
String decodeUtf16be(List<int> bytes, [int offset = 0, int length,
|
||||
bool stripBom = true,
|
||||
int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) {
|
||||
List<int> codeunits = (new Utf16beBytesToCodeUnitsDecoder(bytes, offset,
|
||||
length, stripBom, replacementCodepoint)).decodeRest();
|
||||
return new String.fromCharCodes(
|
||||
_utf16CodeUnitsToCodepoints(codeunits, 0, null, replacementCodepoint));
|
||||
}
|
||||
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
String decodeUtf16le(List<int> bytes, [int offset = 0, int length,
|
||||
bool stripBom = true,
|
||||
int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) {
|
||||
List<int> codeunits = (new Utf16leBytesToCodeUnitsDecoder(bytes, offset,
|
||||
length, stripBom, replacementCodepoint)).decodeRest();
|
||||
return new String.fromCharCodes(
|
||||
_utf16CodeUnitsToCodepoints(codeunits, 0, null, replacementCodepoint));
|
||||
}
|
||||
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
List<int> encodeUtf16(String str) =>
|
||||
encodeUtf16be(str, true);
|
||||
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
List<int> encodeUtf16be(String str, [bool writeBOM = false]) {
|
||||
List<int> utf16CodeUnits = _stringToUtf16CodeUnits(str);
|
||||
List<int> encoding =
|
||||
new List<int>(2 * utf16CodeUnits.length + (writeBOM ? 2 : 0));
|
||||
int i = 0;
|
||||
if (writeBOM) {
|
||||
encoding[i++] = UNICODE_UTF_BOM_HI;
|
||||
encoding[i++] = UNICODE_UTF_BOM_LO;
|
||||
}
|
||||
for (int unit in utf16CodeUnits) {
|
||||
encoding[i++] = (unit & UNICODE_BYTE_ONE_MASK) >> 8;
|
||||
encoding[i++] = unit & UNICODE_BYTE_ZERO_MASK;
|
||||
}
|
||||
return encoding;
|
||||
}
|
||||
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
List<int> encodeUtf16le(String str, [bool writeBOM = false]) {
|
||||
List<int> utf16CodeUnits = _stringToUtf16CodeUnits(str);
|
||||
List<int> encoding =
|
||||
new List<int>(2 * utf16CodeUnits.length + (writeBOM ? 2 : 0));
|
||||
int i = 0;
|
||||
if (writeBOM) {
|
||||
encoding[i++] = UNICODE_UTF_BOM_LO;
|
||||
encoding[i++] = UNICODE_UTF_BOM_HI;
|
||||
}
|
||||
for (int unit in utf16CodeUnits) {
|
||||
encoding[i++] = unit & UNICODE_BYTE_ZERO_MASK;
|
||||
encoding[i++] = (unit & UNICODE_BYTE_ONE_MASK) >> 8;
|
||||
}
|
||||
return encoding;
|
||||
}
|
||||
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
bool hasUtf16Bom(List<int> utf32EncodedBytes, [int offset = 0, int length]) {
|
||||
return hasUtf16beBom(utf32EncodedBytes, offset, length) ||
|
||||
hasUtf16leBom(utf32EncodedBytes, offset, length);
|
||||
}
|
||||
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
bool hasUtf16beBom(List<int> utf16EncodedBytes, [int offset = 0, int length]) {
|
||||
int end = length != null ? offset + length : utf16EncodedBytes.length;
|
||||
return (offset + 2) <= end &&
|
||||
utf16EncodedBytes[offset] == UNICODE_UTF_BOM_HI &&
|
||||
utf16EncodedBytes[offset + 1] == UNICODE_UTF_BOM_LO;
|
||||
}
|
||||
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
bool hasUtf16leBom(List<int> utf16EncodedBytes, [int offset = 0, int length]) {
|
||||
int end = length != null ? offset + length : utf16EncodedBytes.length;
|
||||
return (offset + 2) <= end &&
|
||||
utf16EncodedBytes[offset] == UNICODE_UTF_BOM_LO &&
|
||||
utf16EncodedBytes[offset + 1] == UNICODE_UTF_BOM_HI;
|
||||
}
|
||||
|
||||
List<int> _stringToUtf16CodeUnits(String str) {
|
||||
return _codepointsToUtf16CodeUnits(str.codeUnits);
|
||||
}
|
||||
|
||||
typedef _ListRangeIterator _CodeUnitsProvider();
|
||||
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
class IterableUtf16Decoder extends IterableBase<int> {
|
||||
final _CodeUnitsProvider codeunitsProvider;
|
||||
final int replacementCodepoint;
|
||||
|
||||
IterableUtf16Decoder._(this.codeunitsProvider, this.replacementCodepoint);
|
||||
|
||||
Utf16CodeUnitDecoder get iterator =>
|
||||
new Utf16CodeUnitDecoder.fromListRangeIterator(codeunitsProvider(),
|
||||
replacementCodepoint);
|
||||
}
|
||||
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
abstract class Utf16BytesToCodeUnitsDecoder implements _ListRangeIterator {
|
||||
final _ListRangeIterator utf16EncodedBytesIterator;
|
||||
final int replacementCodepoint;
|
||||
int _current = null;
|
||||
|
||||
Utf16BytesToCodeUnitsDecoder._fromListRangeIterator(
|
||||
this.utf16EncodedBytesIterator, this.replacementCodepoint);
|
||||
|
||||
factory Utf16BytesToCodeUnitsDecoder(List<int> utf16EncodedBytes, [
|
||||
int offset = 0, int length,
|
||||
int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) {
|
||||
if (length == null) {
|
||||
length = utf16EncodedBytes.length - offset;
|
||||
}
|
||||
if (hasUtf16beBom(utf16EncodedBytes, offset, length)) {
|
||||
return new Utf16beBytesToCodeUnitsDecoder(utf16EncodedBytes, offset + 2,
|
||||
length - 2, false, replacementCodepoint);
|
||||
} else if (hasUtf16leBom(utf16EncodedBytes, offset, length)) {
|
||||
return new Utf16leBytesToCodeUnitsDecoder(utf16EncodedBytes, offset + 2,
|
||||
length - 2, false, replacementCodepoint);
|
||||
} else {
|
||||
return new Utf16beBytesToCodeUnitsDecoder(utf16EncodedBytes, offset,
|
||||
length, false, replacementCodepoint);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Provides a fast way to decode the rest of the source bytes in a single
|
||||
* call. This method trades memory for improved speed in that it potentially
|
||||
* over-allocates the List containing results.
|
||||
*/
|
||||
List<int> decodeRest() {
|
||||
List<int> codeunits = new List<int>(remaining);
|
||||
int i = 0;
|
||||
while (moveNext()) {
|
||||
codeunits[i++] = current;
|
||||
}
|
||||
if (i == codeunits.length) {
|
||||
return codeunits;
|
||||
} else {
|
||||
List<int> truncCodeunits = new List<int>(i);
|
||||
truncCodeunits.setRange(0, i, codeunits);
|
||||
return truncCodeunits;
|
||||
}
|
||||
}
|
||||
|
||||
int get current => _current;
|
||||
|
||||
bool moveNext() {
|
||||
_current = null;
|
||||
if (utf16EncodedBytesIterator.remaining < 2) {
|
||||
utf16EncodedBytesIterator.moveNext();
|
||||
if (replacementCodepoint != null) {
|
||||
_current = replacementCodepoint;
|
||||
return true;
|
||||
} else {
|
||||
throw new ArgumentError(
|
||||
"Invalid UTF16 at ${utf16EncodedBytesIterator.position}");
|
||||
}
|
||||
} else {
|
||||
_current = decode();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
int get position => utf16EncodedBytesIterator.position ~/ 2;
|
||||
|
||||
void backup([int by = 1]) {
|
||||
utf16EncodedBytesIterator.backup(2 * by);
|
||||
}
|
||||
|
||||
int get remaining => (utf16EncodedBytesIterator.remaining + 1) ~/ 2;
|
||||
|
||||
void skip([int count = 1]) {
|
||||
utf16EncodedBytesIterator.skip(2 * count);
|
||||
}
|
||||
|
||||
int decode();
|
||||
}
|
||||
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
class Utf16beBytesToCodeUnitsDecoder extends Utf16BytesToCodeUnitsDecoder {
|
||||
Utf16beBytesToCodeUnitsDecoder(List<int> utf16EncodedBytes, [
|
||||
int offset = 0, int length, bool stripBom = true,
|
||||
int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) :
|
||||
super._fromListRangeIterator(
|
||||
(new _ListRange(utf16EncodedBytes, offset, length)).iterator,
|
||||
replacementCodepoint) {
|
||||
if (stripBom && hasUtf16beBom(utf16EncodedBytes, offset, length)) {
|
||||
skip();
|
||||
}
|
||||
}
|
||||
|
||||
int decode() {
|
||||
utf16EncodedBytesIterator.moveNext();
|
||||
int hi = utf16EncodedBytesIterator.current;
|
||||
utf16EncodedBytesIterator.moveNext();
|
||||
int lo = utf16EncodedBytesIterator.current;
|
||||
return (hi << 8) + lo;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
class Utf16leBytesToCodeUnitsDecoder extends Utf16BytesToCodeUnitsDecoder {
|
||||
Utf16leBytesToCodeUnitsDecoder(List<int> utf16EncodedBytes, [
|
||||
int offset = 0, int length, bool stripBom = true,
|
||||
int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) :
|
||||
super._fromListRangeIterator(
|
||||
(new _ListRange(utf16EncodedBytes, offset, length)).iterator,
|
||||
replacementCodepoint) {
|
||||
if (stripBom && hasUtf16leBom(utf16EncodedBytes, offset, length)) {
|
||||
skip();
|
||||
}
|
||||
}
|
||||
|
||||
int decode() {
|
||||
utf16EncodedBytesIterator.moveNext();
|
||||
int lo = utf16EncodedBytesIterator.current;
|
||||
utf16EncodedBytesIterator.moveNext();
|
||||
int hi = utf16EncodedBytesIterator.current;
|
||||
return (hi << 8) + lo;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,335 +0,0 @@
|
|||
// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file
|
||||
// for details. All rights reserved. Use of this source code is governed by a
|
||||
// BSD-style license that can be found in the LICENSE file.
|
||||
|
||||
part of dart.utf;
|
||||
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
IterableUtf32Decoder decodeUtf32AsIterable(List<int> bytes, [
|
||||
int offset = 0, int length,
|
||||
int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) {
|
||||
return new IterableUtf32Decoder._(
|
||||
() => new Utf32BytesDecoder(bytes, offset, length, replacementCodepoint));
|
||||
}
|
||||
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
IterableUtf32Decoder decodeUtf32beAsIterable(List<int> bytes, [
|
||||
int offset = 0, int length, bool stripBom = true,
|
||||
int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) {
|
||||
return new IterableUtf32Decoder._(
|
||||
() => new Utf32beBytesDecoder(bytes, offset, length, stripBom,
|
||||
replacementCodepoint));
|
||||
}
|
||||
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
IterableUtf32Decoder decodeUtf32leAsIterable(List<int> bytes, [
|
||||
int offset = 0, int length, bool stripBom = true,
|
||||
int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) {
|
||||
return new IterableUtf32Decoder._(
|
||||
() => new Utf32leBytesDecoder(bytes, offset, length, stripBom,
|
||||
replacementCodepoint));
|
||||
}
|
||||
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
String decodeUtf32(List<int> bytes, [int offset = 0, int length,
|
||||
int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) {
|
||||
return new String.fromCharCodes((new Utf32BytesDecoder(bytes, offset, length,
|
||||
replacementCodepoint)).decodeRest());
|
||||
}
|
||||
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
String decodeUtf32be(
|
||||
List<int> bytes, [int offset = 0, int length, bool stripBom = true,
|
||||
int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) =>
|
||||
new String.fromCharCodes((new Utf32beBytesDecoder(bytes, offset, length,
|
||||
stripBom, replacementCodepoint)).decodeRest());
|
||||
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
String decodeUtf32le(
|
||||
List<int> bytes, [int offset = 0, int length, bool stripBom = true,
|
||||
int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) =>
|
||||
new String.fromCharCodes((new Utf32leBytesDecoder(bytes, offset, length,
|
||||
stripBom, replacementCodepoint)).decodeRest());
|
||||
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
List<int> encodeUtf32(String str) =>
|
||||
encodeUtf32be(str, true);
|
||||
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
List<int> encodeUtf32be(String str, [bool writeBOM = false]) {
|
||||
List<int> utf32CodeUnits = stringToCodepoints(str);
|
||||
List<int> encoding = new List<int>(4 * utf32CodeUnits.length +
|
||||
(writeBOM ? 4 : 0));
|
||||
int i = 0;
|
||||
if (writeBOM) {
|
||||
encoding[i++] = 0;
|
||||
encoding[i++] = 0;
|
||||
encoding[i++] = UNICODE_UTF_BOM_HI;
|
||||
encoding[i++] = UNICODE_UTF_BOM_LO;
|
||||
}
|
||||
for (int unit in utf32CodeUnits) {
|
||||
encoding[i++] = (unit >> 24) & UNICODE_BYTE_ZERO_MASK;
|
||||
encoding[i++] = (unit >> 16) & UNICODE_BYTE_ZERO_MASK;
|
||||
encoding[i++] = (unit >> 8) & UNICODE_BYTE_ZERO_MASK;
|
||||
encoding[i++] = unit & UNICODE_BYTE_ZERO_MASK;
|
||||
}
|
||||
return encoding;
|
||||
}
|
||||
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
List<int> encodeUtf32le(String str, [bool writeBOM = false]) {
|
||||
List<int> utf32CodeUnits = stringToCodepoints(str);
|
||||
List<int> encoding = new List<int>(4 * utf32CodeUnits.length +
|
||||
(writeBOM ? 4 : 0));
|
||||
int i = 0;
|
||||
if (writeBOM) {
|
||||
encoding[i++] = UNICODE_UTF_BOM_LO;
|
||||
encoding[i++] = UNICODE_UTF_BOM_HI;
|
||||
encoding[i++] = 0;
|
||||
encoding[i++] = 0;
|
||||
}
|
||||
for (int unit in utf32CodeUnits) {
|
||||
encoding[i++] = unit & UNICODE_BYTE_ZERO_MASK;
|
||||
encoding[i++] = (unit >> 8) & UNICODE_BYTE_ZERO_MASK;
|
||||
encoding[i++] = (unit >> 16) & UNICODE_BYTE_ZERO_MASK;
|
||||
encoding[i++] = (unit >> 24) & UNICODE_BYTE_ZERO_MASK;
|
||||
}
|
||||
return encoding;
|
||||
}
|
||||
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
bool hasUtf32Bom(
|
||||
List<int> utf32EncodedBytes, [int offset = 0, int length]) {
|
||||
return hasUtf32beBom(utf32EncodedBytes, offset, length) ||
|
||||
hasUtf32leBom(utf32EncodedBytes, offset, length);
|
||||
}
|
||||
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
bool hasUtf32beBom(List<int> utf32EncodedBytes, [int offset = 0, int length]) {
|
||||
int end = length != null ? offset + length : utf32EncodedBytes.length;
|
||||
return (offset + 4) <= end &&
|
||||
utf32EncodedBytes[offset] == 0 && utf32EncodedBytes[offset + 1] == 0 &&
|
||||
utf32EncodedBytes[offset + 2] == UNICODE_UTF_BOM_HI &&
|
||||
utf32EncodedBytes[offset + 3] == UNICODE_UTF_BOM_LO;
|
||||
}
|
||||
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
bool hasUtf32leBom(List<int> utf32EncodedBytes, [int offset = 0, int length]) {
|
||||
int end = length != null ? offset + length : utf32EncodedBytes.length;
|
||||
return (offset + 4) <= end &&
|
||||
utf32EncodedBytes[offset] == UNICODE_UTF_BOM_LO &&
|
||||
utf32EncodedBytes[offset + 1] == UNICODE_UTF_BOM_HI &&
|
||||
utf32EncodedBytes[offset + 2] == 0 && utf32EncodedBytes[offset + 3] == 0;
|
||||
}
|
||||
|
||||
typedef Utf32BytesDecoder Utf32BytesDecoderProvider();
|
||||
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
class IterableUtf32Decoder extends IterableBase<int> {
|
||||
final Utf32BytesDecoderProvider codeunitsProvider;
|
||||
|
||||
IterableUtf32Decoder._(this.codeunitsProvider);
|
||||
|
||||
Utf32BytesDecoder get iterator => codeunitsProvider();
|
||||
}
|
||||
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
abstract class Utf32BytesDecoder implements _ListRangeIterator {
|
||||
final _ListRangeIterator utf32EncodedBytesIterator;
|
||||
final int replacementCodepoint;
|
||||
int _current = null;
|
||||
|
||||
Utf32BytesDecoder._fromListRangeIterator(
|
||||
this.utf32EncodedBytesIterator, this.replacementCodepoint);
|
||||
|
||||
factory Utf32BytesDecoder(List<int> utf32EncodedBytes, [
|
||||
int offset = 0, int length,
|
||||
int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) {
|
||||
if (length == null) {
|
||||
length = utf32EncodedBytes.length - offset;
|
||||
}
|
||||
if (hasUtf32beBom(utf32EncodedBytes, offset, length)) {
|
||||
return new Utf32beBytesDecoder(utf32EncodedBytes, offset + 4, length - 4,
|
||||
false, replacementCodepoint);
|
||||
} else if (hasUtf32leBom(utf32EncodedBytes, offset, length)) {
|
||||
return new Utf32leBytesDecoder(utf32EncodedBytes, offset + 4, length - 4,
|
||||
false, replacementCodepoint);
|
||||
} else {
|
||||
return new Utf32beBytesDecoder(utf32EncodedBytes, offset, length, false,
|
||||
replacementCodepoint);
|
||||
}
|
||||
}
|
||||
|
||||
List<int> decodeRest() {
|
||||
List<int> codeunits = new List<int>(remaining);
|
||||
int i = 0;
|
||||
while (moveNext()) {
|
||||
codeunits[i++] = current;
|
||||
}
|
||||
return codeunits;
|
||||
}
|
||||
|
||||
int get current => _current;
|
||||
|
||||
bool moveNext() {
|
||||
_current = null;
|
||||
if (utf32EncodedBytesIterator.remaining < 4) {
|
||||
utf32EncodedBytesIterator.skip(utf32EncodedBytesIterator.remaining);
|
||||
if (replacementCodepoint != null) {
|
||||
_current = replacementCodepoint;
|
||||
return true;
|
||||
} else {
|
||||
throw new ArgumentError(
|
||||
"Invalid UTF32 at ${utf32EncodedBytesIterator.position}");
|
||||
}
|
||||
} else {
|
||||
int codepoint = decode();
|
||||
if (_validCodepoint(codepoint)) {
|
||||
_current = codepoint;
|
||||
return true;
|
||||
} else if (replacementCodepoint != null) {
|
||||
_current = replacementCodepoint;
|
||||
return true;
|
||||
} else {
|
||||
throw new ArgumentError(
|
||||
"Invalid UTF32 at ${utf32EncodedBytesIterator.position}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int get position => utf32EncodedBytesIterator.position ~/ 4;
|
||||
|
||||
void backup([int by = 1]) {
|
||||
utf32EncodedBytesIterator.backup(4 * by);
|
||||
}
|
||||
|
||||
int get remaining => (utf32EncodedBytesIterator.remaining + 3) ~/ 4;
|
||||
|
||||
void skip([int count = 1]) {
|
||||
utf32EncodedBytesIterator.skip(4 * count);
|
||||
}
|
||||
|
||||
int decode();
|
||||
}
|
||||
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
class Utf32beBytesDecoder extends Utf32BytesDecoder {
|
||||
Utf32beBytesDecoder(List<int> utf32EncodedBytes, [int offset = 0,
|
||||
int length, bool stripBom = true,
|
||||
int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) :
|
||||
super._fromListRangeIterator(
|
||||
(new _ListRange(utf32EncodedBytes, offset, length)).iterator,
|
||||
replacementCodepoint) {
|
||||
if (stripBom && hasUtf32beBom(utf32EncodedBytes, offset, length)) {
|
||||
skip();
|
||||
}
|
||||
}
|
||||
|
||||
int decode() {
|
||||
utf32EncodedBytesIterator.moveNext();
|
||||
int value = utf32EncodedBytesIterator.current;
|
||||
utf32EncodedBytesIterator.moveNext();
|
||||
value = (value << 8) + utf32EncodedBytesIterator.current;
|
||||
utf32EncodedBytesIterator.moveNext();
|
||||
value = (value << 8) + utf32EncodedBytesIterator.current;
|
||||
utf32EncodedBytesIterator.moveNext();
|
||||
value = (value << 8) + utf32EncodedBytesIterator.current;
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
class Utf32leBytesDecoder extends Utf32BytesDecoder {
|
||||
Utf32leBytesDecoder(List<int> utf32EncodedBytes, [int offset = 0,
|
||||
int length, bool stripBom = true,
|
||||
int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) :
|
||||
super._fromListRangeIterator(
|
||||
(new _ListRange(utf32EncodedBytes, offset, length)).iterator,
|
||||
replacementCodepoint) {
|
||||
if (stripBom && hasUtf32leBom(utf32EncodedBytes, offset, length)) {
|
||||
skip();
|
||||
}
|
||||
}
|
||||
|
||||
int decode() {
|
||||
utf32EncodedBytesIterator.moveNext();
|
||||
int value = utf32EncodedBytesIterator.current;
|
||||
utf32EncodedBytesIterator.moveNext();
|
||||
value += (utf32EncodedBytesIterator.current << 8);
|
||||
utf32EncodedBytesIterator.moveNext();
|
||||
value += (utf32EncodedBytesIterator.current << 16);
|
||||
utf32EncodedBytesIterator.moveNext();
|
||||
value += (utf32EncodedBytesIterator.current << 24);
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
bool _validCodepoint(int codepoint) {
|
||||
return (codepoint >= 0 && codepoint < UNICODE_UTF16_RESERVED_LO) ||
|
||||
(codepoint > UNICODE_UTF16_RESERVED_HI &&
|
||||
codepoint < UNICODE_VALID_RANGE_MAX);
|
||||
}
|
||||
|
|
@ -1,345 +0,0 @@
|
|||
// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file
|
||||
// for details. All rights reserved. Use of this source code is governed by a
|
||||
// BSD-style license that can be found in the LICENSE file.
|
||||
|
||||
part of dart.utf;
|
||||
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
const int _UTF8_ONE_BYTE_MAX = 0x7f;
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
const int _UTF8_TWO_BYTE_MAX = 0x7ff;
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
const int _UTF8_THREE_BYTE_MAX = 0xffff;
|
||||
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
const int _UTF8_LO_SIX_BIT_MASK = 0x3f;
|
||||
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
const int _UTF8_FIRST_BYTE_OF_TWO_BASE = 0xc0;
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
const int _UTF8_FIRST_BYTE_OF_THREE_BASE = 0xe0;
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
const int _UTF8_FIRST_BYTE_OF_FOUR_BASE = 0xf0;
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
const int _UTF8_FIRST_BYTE_OF_FIVE_BASE = 0xf8;
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
const int _UTF8_FIRST_BYTE_OF_SIX_BASE = 0xfc;
|
||||
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
const int _UTF8_FIRST_BYTE_OF_TWO_MASK = 0x1f;
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
const int _UTF8_FIRST_BYTE_OF_THREE_MASK = 0xf;
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
const int _UTF8_FIRST_BYTE_OF_FOUR_MASK = 0x7;
|
||||
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
const int _UTF8_FIRST_BYTE_BOUND_EXCL = 0xfe;
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
const int _UTF8_SUBSEQUENT_BYTE_BASE = 0x80;
|
||||
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
IterableUtf8Decoder decodeUtf8AsIterable(List<int> bytes, [int offset = 0,
|
||||
int length,
|
||||
int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) {
|
||||
return new IterableUtf8Decoder(bytes, offset, length, replacementCodepoint);
|
||||
}
|
||||
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
String decodeUtf8(List<int> bytes, [int offset = 0, int length,
|
||||
int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) {
|
||||
return new String.fromCharCodes(
|
||||
(new Utf8Decoder(bytes, offset, length, replacementCodepoint))
|
||||
.decodeRest());
|
||||
}
|
||||
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
List<int> encodeUtf8(String str) =>
|
||||
codepointsToUtf8(stringToCodepoints(str));
|
||||
|
||||
int _addToEncoding(int offset, int bytes, int value, List<int> buffer) {
|
||||
while (bytes > 0) {
|
||||
buffer[offset + bytes] = _UTF8_SUBSEQUENT_BYTE_BASE |
|
||||
(value & _UTF8_LO_SIX_BIT_MASK);
|
||||
value = value >> 6;
|
||||
bytes--;
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
List<int> codepointsToUtf8(
|
||||
List<int> codepoints, [int offset = 0, int length]) {
|
||||
_ListRange source = new _ListRange(codepoints, offset, length);
|
||||
|
||||
int encodedLength = 0;
|
||||
for (int value in source) {
|
||||
if (value < 0 || value > UNICODE_VALID_RANGE_MAX) {
|
||||
encodedLength += 3;
|
||||
} else if (value <= _UTF8_ONE_BYTE_MAX) {
|
||||
encodedLength++;
|
||||
} else if (value <= _UTF8_TWO_BYTE_MAX) {
|
||||
encodedLength += 2;
|
||||
} else if (value <= _UTF8_THREE_BYTE_MAX) {
|
||||
encodedLength += 3;
|
||||
} else if (value <= UNICODE_VALID_RANGE_MAX) {
|
||||
encodedLength += 4;
|
||||
}
|
||||
}
|
||||
|
||||
List<int> encoded = new List<int>(encodedLength);
|
||||
int insertAt = 0;
|
||||
for (int value in source) {
|
||||
if (value < 0 || value > UNICODE_VALID_RANGE_MAX) {
|
||||
encoded.setRange(insertAt, insertAt + 3, [0xef, 0xbf, 0xbd]);
|
||||
insertAt += 3;
|
||||
} else if (value <= _UTF8_ONE_BYTE_MAX) {
|
||||
encoded[insertAt] = value;
|
||||
insertAt++;
|
||||
} else if (value <= _UTF8_TWO_BYTE_MAX) {
|
||||
encoded[insertAt] = _UTF8_FIRST_BYTE_OF_TWO_BASE | (
|
||||
_UTF8_FIRST_BYTE_OF_TWO_MASK &
|
||||
_addToEncoding(insertAt, 1, value, encoded));
|
||||
insertAt += 2;
|
||||
} else if (value <= _UTF8_THREE_BYTE_MAX) {
|
||||
encoded[insertAt] = _UTF8_FIRST_BYTE_OF_THREE_BASE | (
|
||||
_UTF8_FIRST_BYTE_OF_THREE_MASK &
|
||||
_addToEncoding(insertAt, 2, value, encoded));
|
||||
insertAt += 3;
|
||||
} else if (value <= UNICODE_VALID_RANGE_MAX) {
|
||||
encoded[insertAt] = _UTF8_FIRST_BYTE_OF_FOUR_BASE | (
|
||||
_UTF8_FIRST_BYTE_OF_FOUR_MASK &
|
||||
_addToEncoding(insertAt, 3, value, encoded));
|
||||
insertAt += 4;
|
||||
}
|
||||
}
|
||||
return encoded;
|
||||
}
|
||||
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
List<int> utf8ToCodepoints(
|
||||
List<int> utf8EncodedBytes, [int offset = 0, int length,
|
||||
int replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) {
|
||||
return new Utf8Decoder(utf8EncodedBytes, offset, length,
|
||||
replacementCodepoint).decodeRest();
|
||||
}
|
||||
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
class IterableUtf8Decoder extends IterableBase<int> {
|
||||
final List<int> bytes;
|
||||
final int offset;
|
||||
final int length;
|
||||
final int replacementCodepoint;
|
||||
|
||||
IterableUtf8Decoder(this.bytes, [this.offset = 0, this.length = null,
|
||||
this.replacementCodepoint = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]);
|
||||
|
||||
Utf8Decoder get iterator =>
|
||||
new Utf8Decoder(bytes, offset, length, replacementCodepoint);
|
||||
}
|
||||
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
class Utf8Decoder implements Iterator<int> {
|
||||
final _ListRangeIterator utf8EncodedBytesIterator;
|
||||
final int replacementCodepoint;
|
||||
int _current = null;
|
||||
|
||||
Utf8Decoder(List<int> utf8EncodedBytes, [int offset = 0, int length,
|
||||
this.replacementCodepoint =
|
||||
UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) :
|
||||
utf8EncodedBytesIterator =
|
||||
(new _ListRange(utf8EncodedBytes, offset, length)).iterator;
|
||||
|
||||
|
||||
Utf8Decoder._fromListRangeIterator(_ListRange source, [
|
||||
this.replacementCodepoint =
|
||||
UNICODE_REPLACEMENT_CHARACTER_CODEPOINT]) :
|
||||
utf8EncodedBytesIterator = source.iterator;
|
||||
|
||||
/** Decode the remaininder of the characters in this decoder
|
||||
* into a [List<int>].
|
||||
*/
|
||||
List<int> decodeRest() {
|
||||
List<int> codepoints = new List<int>(utf8EncodedBytesIterator.remaining);
|
||||
int i = 0;
|
||||
while (moveNext()) {
|
||||
codepoints[i++] = current;
|
||||
}
|
||||
if (i == codepoints.length) {
|
||||
return codepoints;
|
||||
} else {
|
||||
List<int> truncCodepoints = new List<int>(i);
|
||||
truncCodepoints.setRange(0, i, codepoints);
|
||||
return truncCodepoints;
|
||||
}
|
||||
}
|
||||
|
||||
int get current => _current;
|
||||
|
||||
bool moveNext() {
|
||||
_current = null;
|
||||
|
||||
if (!utf8EncodedBytesIterator.moveNext()) return false;
|
||||
|
||||
int value = utf8EncodedBytesIterator.current;
|
||||
int additionalBytes = 0;
|
||||
|
||||
if (value < 0) {
|
||||
if (replacementCodepoint != null) {
|
||||
_current = replacementCodepoint;
|
||||
return true;
|
||||
} else {
|
||||
throw new ArgumentError(
|
||||
"Invalid UTF8 at ${utf8EncodedBytesIterator.position}");
|
||||
}
|
||||
} else if (value <= _UTF8_ONE_BYTE_MAX) {
|
||||
_current = value;
|
||||
return true;
|
||||
} else if (value < _UTF8_FIRST_BYTE_OF_TWO_BASE) {
|
||||
if (replacementCodepoint != null) {
|
||||
_current = replacementCodepoint;
|
||||
return true;
|
||||
} else {
|
||||
throw new ArgumentError(
|
||||
"Invalid UTF8 at ${utf8EncodedBytesIterator.position}");
|
||||
}
|
||||
} else if (value < _UTF8_FIRST_BYTE_OF_THREE_BASE) {
|
||||
value -= _UTF8_FIRST_BYTE_OF_TWO_BASE;
|
||||
additionalBytes = 1;
|
||||
} else if (value < _UTF8_FIRST_BYTE_OF_FOUR_BASE) {
|
||||
value -= _UTF8_FIRST_BYTE_OF_THREE_BASE;
|
||||
additionalBytes = 2;
|
||||
} else if (value < _UTF8_FIRST_BYTE_OF_FIVE_BASE) {
|
||||
value -= _UTF8_FIRST_BYTE_OF_FOUR_BASE;
|
||||
additionalBytes = 3;
|
||||
} else if (value < _UTF8_FIRST_BYTE_OF_SIX_BASE) {
|
||||
value -= _UTF8_FIRST_BYTE_OF_FIVE_BASE;
|
||||
additionalBytes = 4;
|
||||
} else if (value < _UTF8_FIRST_BYTE_BOUND_EXCL) {
|
||||
value -= _UTF8_FIRST_BYTE_OF_SIX_BASE;
|
||||
additionalBytes = 5;
|
||||
} else if (replacementCodepoint != null) {
|
||||
_current = replacementCodepoint;
|
||||
return true;
|
||||
} else {
|
||||
throw new ArgumentError(
|
||||
"Invalid UTF8 at ${utf8EncodedBytesIterator.position}");
|
||||
}
|
||||
int j = 0;
|
||||
while (j < additionalBytes && utf8EncodedBytesIterator.moveNext()) {
|
||||
int nextValue = utf8EncodedBytesIterator.current;
|
||||
if (nextValue > _UTF8_ONE_BYTE_MAX &&
|
||||
nextValue < _UTF8_FIRST_BYTE_OF_TWO_BASE) {
|
||||
value = ((value << 6) | (nextValue & _UTF8_LO_SIX_BIT_MASK));
|
||||
} else {
|
||||
// if sequence-starting code unit, reposition cursor to start here
|
||||
if (nextValue >= _UTF8_FIRST_BYTE_OF_TWO_BASE) {
|
||||
utf8EncodedBytesIterator.backup();
|
||||
}
|
||||
break;
|
||||
}
|
||||
j++;
|
||||
}
|
||||
bool validSequence = (j == additionalBytes && (
|
||||
value < UNICODE_UTF16_RESERVED_LO ||
|
||||
value > UNICODE_UTF16_RESERVED_HI));
|
||||
bool nonOverlong =
|
||||
(additionalBytes == 1 && value > _UTF8_ONE_BYTE_MAX) ||
|
||||
(additionalBytes == 2 && value > _UTF8_TWO_BYTE_MAX) ||
|
||||
(additionalBytes == 3 && value > _UTF8_THREE_BYTE_MAX);
|
||||
bool inRange = value <= UNICODE_VALID_RANGE_MAX;
|
||||
if (validSequence && nonOverlong && inRange) {
|
||||
_current = value;
|
||||
return true;
|
||||
} else if (replacementCodepoint != null) {
|
||||
_current = replacementCodepoint;
|
||||
return true;
|
||||
} else {
|
||||
throw new ArgumentError(
|
||||
"Invalid UTF8 at ${utf8EncodedBytesIterator.position - j}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
# Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file
|
||||
# for details. All rights reserved. Use of this source code is governed by a
|
||||
# BSD-style license that can be found in the LICENSE file.
|
||||
|
||||
# This file contains all sources for the dart:utf library.
|
||||
{
|
||||
'sources': [
|
||||
'utf.dart',
|
||||
# The above file needs to be first as it lists the parts below.
|
||||
'utf_stream.dart',
|
||||
'utf8.dart',
|
||||
'utf16.dart',
|
||||
'utf32.dart',
|
||||
],
|
||||
}
|
||||
|
|
@ -1,240 +0,0 @@
|
|||
// Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file
|
||||
// for details. All rights reserved. Use of this source code is governed by a
|
||||
// BSD-style license that can be found in the LICENSE file.
|
||||
|
||||
part of dart.utf;
|
||||
|
||||
abstract class _StringDecoder
|
||||
implements StreamTransformer<List<int>, String>, EventSink<List<int>> {
|
||||
List<int> _carry;
|
||||
List<int> _buffer;
|
||||
int _replacementChar;
|
||||
|
||||
EventSink<String> _outSink;
|
||||
|
||||
_StringDecoder(int this._replacementChar);
|
||||
|
||||
Stream<String> bind(Stream<List<int>> stream) {
|
||||
return new Stream.eventTransformed(
|
||||
stream,
|
||||
(EventSink<String> sink) {
|
||||
if (_outSink != null) {
|
||||
throw new StateError("String decoder already used");
|
||||
}
|
||||
_outSink = sink;
|
||||
return this;
|
||||
});
|
||||
}
|
||||
|
||||
void add(List<int> bytes) {
|
||||
try {
|
||||
_buffer = <int>[];
|
||||
List<int> carry = _carry;
|
||||
_carry = null;
|
||||
int pos = 0;
|
||||
int available = bytes.length;
|
||||
// If we have carry-over data, start from negative index, indicating carry
|
||||
// index.
|
||||
int goodChars = 0;
|
||||
if (carry != null) pos = -carry.length;
|
||||
while (pos < available) {
|
||||
int currentPos = pos;
|
||||
int getNext() {
|
||||
if (pos < 0) {
|
||||
return carry[pos++ + carry.length];
|
||||
} else if (pos < available) {
|
||||
return bytes[pos++];
|
||||
}
|
||||
return null;
|
||||
}
|
||||
int consumed = _processBytes(getNext);
|
||||
if (consumed > 0) {
|
||||
goodChars = _buffer.length;
|
||||
} else if (consumed == 0) {
|
||||
_buffer.length = goodChars;
|
||||
if (currentPos < 0) {
|
||||
_carry = [];
|
||||
_carry.addAll(carry);
|
||||
_carry.addAll(bytes);
|
||||
} else {
|
||||
_carry = bytes.sublist(currentPos);
|
||||
}
|
||||
break;
|
||||
} else {
|
||||
// Invalid byte at position pos - 1
|
||||
_buffer.length = goodChars;
|
||||
_addChar(-1);
|
||||
goodChars = _buffer.length;
|
||||
}
|
||||
}
|
||||
if (_buffer.length > 0) {
|
||||
// Limit to 'goodChars', if lower than actual charCodes in the buffer.
|
||||
_outSink.add(new String.fromCharCodes(_buffer));
|
||||
}
|
||||
_buffer = null;
|
||||
} catch (e, stackTrace) {
|
||||
_outSink.addError(e, stackTrace);
|
||||
}
|
||||
}
|
||||
|
||||
void addError(Object error, [StackTrace stackTrace]) {
|
||||
_outSink.addError(error, stackTrace);
|
||||
}
|
||||
|
||||
void close() {
|
||||
if (_carry != null) {
|
||||
if (_replacementChar != null) {
|
||||
_outSink.add(new String.fromCharCodes(
|
||||
new List.filled(_carry.length, _replacementChar)));
|
||||
} else {
|
||||
throw new ArgumentError('Invalid codepoint');
|
||||
}
|
||||
}
|
||||
_outSink.close();
|
||||
}
|
||||
|
||||
int _processBytes(int getNext());
|
||||
|
||||
void _addChar(int char) {
|
||||
void error() {
|
||||
if (_replacementChar != null) {
|
||||
char = _replacementChar;
|
||||
} else {
|
||||
throw new ArgumentError('Invalid codepoint');
|
||||
}
|
||||
}
|
||||
if (char < 0) error();
|
||||
if (char >= 0xD800 && char <= 0xDFFF) error();
|
||||
if (char > 0x10FFFF) error();
|
||||
_buffer.add(char);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
class Utf8DecoderTransformer extends _StringDecoder {
|
||||
Utf8DecoderTransformer(
|
||||
[int replacementChar = UNICODE_REPLACEMENT_CHARACTER_CODEPOINT])
|
||||
: super(replacementChar);
|
||||
|
||||
int _processBytes(int getNext()) {
|
||||
int value = getNext();
|
||||
if ((value & 0xFF) != value) return -1; // Not a byte.
|
||||
if ((value & 0x80) == 0x80) {
|
||||
int additionalBytes;
|
||||
int min;
|
||||
if ((value & 0xe0) == 0xc0) { // 110xxxxx
|
||||
value = value & 0x1F;
|
||||
additionalBytes = 1;
|
||||
min = 0x80;
|
||||
} else if ((value & 0xf0) == 0xe0) { // 1110xxxx
|
||||
value = value & 0x0F;
|
||||
additionalBytes = 2;
|
||||
min = 0x800;
|
||||
} else if ((value & 0xf8) == 0xf0) { // 11110xxx
|
||||
value = value & 0x07;
|
||||
additionalBytes = 3;
|
||||
min = 0x10000;
|
||||
} else if ((value & 0xfc) == 0xf8) { // 111110xx
|
||||
value = value & 0x03;
|
||||
additionalBytes = 4;
|
||||
min = 0x200000;
|
||||
} else if ((value & 0xfe) == 0xfc) { // 1111110x
|
||||
value = value & 0x01;
|
||||
additionalBytes = 5;
|
||||
min = 0x4000000;
|
||||
} else {
|
||||
return -1;
|
||||
}
|
||||
for (int i = 0; i < additionalBytes; i++) {
|
||||
int next = getNext();
|
||||
if (next == null) return 0; // Not enough chars, reset.
|
||||
if ((next & 0xc0) != 0x80 || (next & 0xff) != next) return -1;
|
||||
value = value << 6 | (next & 0x3f);
|
||||
if (additionalBytes >= 3 && i == 0 && value << 12 > 0x10FFFF) {
|
||||
_addChar(-1);
|
||||
}
|
||||
}
|
||||
// Invalid charCode if less then minimum expected.
|
||||
if (value < min) value = -1;
|
||||
_addChar(value);
|
||||
return 1 + additionalBytes;
|
||||
}
|
||||
_addChar(value);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
abstract class _StringEncoder
|
||||
implements StreamTransformer<String, List<int>>, EventSink<String> {
|
||||
|
||||
EventSink<List<int>> _outSink;
|
||||
|
||||
Stream<List<int>> bind(Stream<String> stream) {
|
||||
return new Stream.eventTransformed(
|
||||
stream,
|
||||
(EventSink<List<int>> sink) {
|
||||
if (_outSink != null) {
|
||||
throw new StateError("String encoder already used");
|
||||
}
|
||||
_outSink = sink;
|
||||
return this;
|
||||
});
|
||||
}
|
||||
|
||||
void add(String data) {
|
||||
_outSink.add(_processString(data));
|
||||
}
|
||||
|
||||
void addError(Object error, [StackTrace stackTrace]) {
|
||||
_outSink.addError(error, stackTrace);
|
||||
}
|
||||
|
||||
void close() { _outSink.close(); }
|
||||
|
||||
List<int> _processString(String string);
|
||||
}
|
||||
|
||||
/**
|
||||
* *DEPRECATED*: Use `package:utf/utf.dart` or, when applicable, `dart:convert`
|
||||
* instead.
|
||||
*/
|
||||
@deprecated
|
||||
class Utf8EncoderTransformer extends _StringEncoder {
|
||||
List<int> _processString(String string) {
|
||||
var bytes = [];
|
||||
int pos = 0;
|
||||
List<int> codepoints = _utf16CodeUnitsToCodepoints(string.codeUnits);
|
||||
int length = codepoints.length;
|
||||
for (int i = 0; i < length; i++) {
|
||||
int additionalBytes;
|
||||
int charCode = codepoints[i];
|
||||
if (charCode <= 0x007F) {
|
||||
additionalBytes = 0;
|
||||
bytes.add(charCode);
|
||||
} else if (charCode <= 0x07FF) {
|
||||
// 110xxxxx (xxxxx is top 5 bits).
|
||||
bytes.add(((charCode >> 6) & 0x1F) | 0xC0);
|
||||
additionalBytes = 1;
|
||||
} else if (charCode <= 0xFFFF) {
|
||||
// 1110xxxx (xxxx is top 4 bits)
|
||||
bytes.add(((charCode >> 12) & 0x0F)| 0xE0);
|
||||
additionalBytes = 2;
|
||||
} else {
|
||||
// 11110xxx (xxx is top 3 bits)
|
||||
bytes.add(((charCode >> 18) & 0x07) | 0xF0);
|
||||
additionalBytes = 3;
|
||||
}
|
||||
for (int i = additionalBytes; i > 0; i--) {
|
||||
// 10xxxxxx (xxxxxx is next 6 bits from the top).
|
||||
bytes.add(((charCode >> (6 * (i - 1))) & 0x3F) | 0x80);
|
||||
}
|
||||
pos += additionalBytes + 1;
|
||||
}
|
||||
return bytes;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,47 +0,0 @@
|
|||
// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file
|
||||
// for details. All rights reserved. Use of this source code is governed by a
|
||||
// BSD-style license that can be found in the LICENSE file.
|
||||
|
||||
library utf8_test;
|
||||
import "package:expect/expect.dart";
|
||||
import 'dart:utf';
|
||||
|
||||
String decode(List<int> bytes) => decodeUtf8(bytes);
|
||||
|
||||
main() {
|
||||
// Google favorite: "Îñţérñåţîöñåļîžåţîờñ".
|
||||
String string = decode([0xc3, 0x8e, 0xc3, 0xb1, 0xc5, 0xa3, 0xc3, 0xa9, 0x72,
|
||||
0xc3, 0xb1, 0xc3, 0xa5, 0xc5, 0xa3, 0xc3, 0xae, 0xc3,
|
||||
0xb6, 0xc3, 0xb1, 0xc3, 0xa5, 0xc4, 0xbc, 0xc3, 0xae,
|
||||
0xc5, 0xbe, 0xc3, 0xa5, 0xc5, 0xa3, 0xc3, 0xae, 0xe1,
|
||||
0xbb, 0x9d, 0xc3, 0xb1]);
|
||||
Expect.stringEquals("Îñţérñåţîöñåļîžåţîờñ", string);
|
||||
|
||||
// Blueberry porridge in Danish: "blåbærgrød".
|
||||
string = decode([0x62, 0x6c, 0xc3, 0xa5, 0x62, 0xc3, 0xa6, 0x72, 0x67, 0x72,
|
||||
0xc3, 0xb8, 0x64]);
|
||||
Expect.stringEquals("blåbærgrød", string);
|
||||
|
||||
// "சிவா அணாமாைல", that is "Siva Annamalai" in Tamil.
|
||||
string = decode([0xe0, 0xae, 0x9a, 0xe0, 0xae, 0xbf, 0xe0, 0xae, 0xb5, 0xe0,
|
||||
0xae, 0xbe, 0x20, 0xe0, 0xae, 0x85, 0xe0, 0xae, 0xa3, 0xe0,
|
||||
0xae, 0xbe, 0xe0, 0xae, 0xae, 0xe0, 0xae, 0xbe, 0xe0, 0xaf,
|
||||
0x88, 0xe0, 0xae, 0xb2]);
|
||||
Expect.stringEquals("சிவா அணாமாைல", string);
|
||||
|
||||
// "िसवा अणामालै", that is "Siva Annamalai" in Devanagari.
|
||||
string = decode([0xe0, 0xa4, 0xbf, 0xe0, 0xa4, 0xb8, 0xe0, 0xa4, 0xb5, 0xe0,
|
||||
0xa4, 0xbe, 0x20, 0xe0, 0xa4, 0x85, 0xe0, 0xa4, 0xa3, 0xe0,
|
||||
0xa4, 0xbe, 0xe0, 0xa4, 0xae, 0xe0, 0xa4, 0xbe, 0xe0, 0xa4,
|
||||
0xb2, 0xe0, 0xa5, 0x88]);
|
||||
Expect.stringEquals("िसवा अणामालै", string);
|
||||
|
||||
// DESERET CAPITAL LETTER BEE, unicode 0x10412(0xD801+0xDC12)
|
||||
// UTF-8: F0 90 90 92
|
||||
string = decode([0xf0, 0x90, 0x90, 0x92]);
|
||||
Expect.equals(string.length, 2);
|
||||
Expect.equals("𐐒".length, 2);
|
||||
Expect.stringEquals("𐐒", string);
|
||||
|
||||
// TODO(ahe): Add tests of bad input.
|
||||
}
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file
|
||||
// for details. All rights reserved. Use of this source code is governed by a
|
||||
// BSD-style license that can be found in the LICENSE file.
|
||||
|
||||
library utf_test;
|
||||
import "package:expect/expect.dart";
|
||||
import 'dart:utf';
|
||||
|
||||
main() {
|
||||
String str = new String.fromCharCodes([0x1d537]);
|
||||
// String.codeUnits gives 16-bit code units, but stringToCodepoints gives
|
||||
// back the original code points.
|
||||
Expect.listEquals([0xd835, 0xdd37], str.codeUnits);
|
||||
Expect.listEquals([0x1d537], stringToCodepoints(str));
|
||||
}
|
||||
|
|
@ -6,7 +6,7 @@ library record_and_replay;
|
|||
|
||||
import 'dart:io';
|
||||
import 'dart:convert';
|
||||
import 'dart:utf';
|
||||
import 'dart:convert';
|
||||
|
||||
import 'test_runner.dart';
|
||||
import 'utils.dart' show Path;
|
||||
|
|
@ -123,8 +123,8 @@ class TestCaseOutputArchive {
|
|||
command,
|
||||
command_output['exit_code'],
|
||||
command_output['did_timeout'],
|
||||
encodeUtf8(command_output['stdout']),
|
||||
encodeUtf8(command_output['stderr']),
|
||||
UTF8.encode(command_output['stdout']),
|
||||
UTF8.encode(command_output['stderr']),
|
||||
duration,
|
||||
false);
|
||||
return commandOutput;
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ library utils;
|
|||
import 'dart:async';
|
||||
import 'dart:io';
|
||||
import 'dart:math' show min;
|
||||
import 'dart:utf' as utf;
|
||||
import 'dart:convert';
|
||||
|
||||
part 'legacy_path.dart';
|
||||
|
||||
|
|
@ -117,14 +117,14 @@ int findBytes(List<int> data, List<int> pattern, [int startPos=0]) {
|
|||
}
|
||||
|
||||
List<int> encodeUtf8(String string) {
|
||||
return utf.encodeUtf8(string);
|
||||
return UTF8.encode(string);
|
||||
}
|
||||
|
||||
// TODO(kustermann,ricow): As soon we have a debug log we should log
|
||||
// invalid utf8-encoded input to the log.
|
||||
// Currently invalid bytes will be replaced by a replacement character.
|
||||
String decodeUtf8(List<int> bytes) {
|
||||
return utf.decodeUtf8(bytes);
|
||||
return UTF8.decode(bytes, allowMalformed: true);
|
||||
}
|
||||
|
||||
class Locations {
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@
|
|||
|
||||
library unicode_tests;
|
||||
import 'dunit.dart';
|
||||
import '../../../sdk/lib/utf/utf.dart';
|
||||
|
||||
void main() {
|
||||
TestSuite suite = new TestSuite();
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@
|
|||
library BenchmarkTests;
|
||||
import 'dart:io';
|
||||
import 'dart:math' as Math;
|
||||
import '../../../lib/utf/utf.dart' as SE;
|
||||
import '../../../lib/convert/convert.dart' as SE;
|
||||
part 'benchmark_runner.dart';
|
||||
|
||||
void main() {
|
||||
|
|
@ -102,25 +102,20 @@ void main() {
|
|||
new TimedTestConfig(100, 1 * 1000, blocksize: 1000);
|
||||
|
||||
BenchmarkRunner.runTimed("SE_EN1","string_encoding/decodeUtf8-English1",
|
||||
testConfig_1sec, () =>
|
||||
(new SE.Utf8Decoder(testEnglishUtf8)).decodeRest());
|
||||
testConfig_1sec, () => SE.UTF8.decode(testEnglishUtf8));
|
||||
|
||||
BenchmarkRunner.runTimed("SE_DA1","string_encoding/decodeUtf8-Danish1",
|
||||
testConfig_1sec, () => (new SE.Utf8Decoder(testDanishUtf8)).decodeRest());
|
||||
testConfig_1sec, () => SE.UTF8.decode(testDanishUtf8));
|
||||
|
||||
BenchmarkRunner.runTimed("SE_HE1","string_encoding/decodeUtf8-Hebrew1",
|
||||
testConfig_1sec, () => (new SE.Utf8Decoder(testHebrewUtf8)).decodeRest());
|
||||
testConfig_1sec, () => SE.UTF8.decode(testHebrewUtf8));
|
||||
|
||||
BenchmarkRunner.runTimed("SE_RU1","string_encoding/decodeUtf8-Russian1",
|
||||
testConfig_1sec, () =>
|
||||
(new SE.Utf8Decoder(testRussianUtf8)).decodeRest());
|
||||
testConfig_1sec, () => SE.UTF8.decode(testRussianUtf8));
|
||||
|
||||
BenchmarkRunner.runTimed("SE_EL1","string_encoding/decodeUtf8-Greek",
|
||||
testConfig_1sec, () =>
|
||||
(new SE.Utf8Decoder(testGreekUtf8)).decodeRest());
|
||||
testConfig_1sec, () => SE.UTF8.decode(testGreekUtf8));
|
||||
|
||||
BenchmarkRunner.runTimed("SE_JA1","string_encoding/decodeUtf8-Katakana",
|
||||
testConfig_1sec, () =>
|
||||
(new SE.Utf8Decoder(testKatakanaUtf8)).decodeRest());
|
||||
testConfig_1sec, () => SE.UTF8.decode(testKatakanaUtf8));
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
|
||||
library utf8_tests;
|
||||
import 'dunit.dart';
|
||||
import '../../../lib/utf/utf.dart';
|
||||
import '../../../lib/convert/convert.dart';
|
||||
|
||||
void main() {
|
||||
TestSuite suite = new TestSuite();
|
||||
|
|
@ -149,6 +149,10 @@ class Utf8Tests extends TestClass {
|
|||
"Katakana to utf8");
|
||||
}
|
||||
|
||||
List encodeUtf8(String text) => UTF8.encode(text);
|
||||
String decodeUtf8(List bytes) => UTF8.decode(bytes);
|
||||
List utf8ToCodePoints(List bytes) => UTF8.decode(bytes).runes.toList();
|
||||
|
||||
void testUtf8bytesToCodepoints() {
|
||||
Expect.listEquals([954, 972, 963, 956, 949],
|
||||
utf8ToCodepoints([0xce, 0xba, 0xcf, 0x8c, 0xcf,
|
||||
|
|
|
|||
Loading…
Reference in a new issue