mirror of
https://github.com/Microsoft/vscode
synced 2024-08-28 21:39:43 +00:00
This reverts commit 1dbfecd37f
.
This commit is contained in:
parent
1dbfecd37f
commit
495112ded5
|
@ -38,7 +38,7 @@
|
|||
"gulp-bom": "^1.0.0",
|
||||
"gulp-sourcemaps": "^1.11.0",
|
||||
"gulp-uglify": "^3.0.0",
|
||||
"iconv-lite": "0.6.0",
|
||||
"iconv-lite": "0.4.23",
|
||||
"mime": "^1.3.4",
|
||||
"minimatch": "3.0.4",
|
||||
"minimist": "^1.2.3",
|
||||
|
|
|
@ -1415,10 +1415,10 @@ http-signature@~1.2.0:
|
|||
jsprim "^1.2.2"
|
||||
sshpk "^1.7.0"
|
||||
|
||||
iconv-lite@0.6.0:
|
||||
version "0.6.0"
|
||||
resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.0.tgz#66a93b80df0bd05d2a43a7426296b7f91073f125"
|
||||
integrity sha512-43ZpGYZ9QtuutX5l6WC1DSO8ane9N+Ct5qPLF2OV7vM9abM69gnAbVkh66ibaZd3aOGkoP1ZmringlKhLBkw2Q==
|
||||
iconv-lite@0.4.23:
|
||||
version "0.4.23"
|
||||
resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.23.tgz#297871f63be507adcfbfca715d0cd0eed84e9a63"
|
||||
integrity sha512-neyTUVFtahjf0mB3dZT77u+8O0QB89jFdnBkd5P1JgYPbPaia3gXXOVL2fq8VyU2gMMD7SaN7QukTB/pmXYvDA==
|
||||
dependencies:
|
||||
safer-buffer ">= 2.1.2 < 3"
|
||||
|
||||
|
|
|
@ -1878,7 +1878,7 @@
|
|||
"dependencies": {
|
||||
"byline": "^5.0.0",
|
||||
"file-type": "^7.2.0",
|
||||
"iconv-lite": "0.6.0",
|
||||
"iconv-lite": "^0.4.24",
|
||||
"jschardet": "2.1.1",
|
||||
"vscode-extension-telemetry": "0.1.1",
|
||||
"vscode-nls": "^4.0.0",
|
||||
|
|
|
@ -425,10 +425,10 @@ https-proxy-agent@^2.2.1:
|
|||
agent-base "^4.3.0"
|
||||
debug "^3.1.0"
|
||||
|
||||
iconv-lite@0.6.0:
|
||||
version "0.6.0"
|
||||
resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.0.tgz#66a93b80df0bd05d2a43a7426296b7f91073f125"
|
||||
integrity sha512-43ZpGYZ9QtuutX5l6WC1DSO8ane9N+Ct5qPLF2OV7vM9abM69gnAbVkh66ibaZd3aOGkoP1ZmringlKhLBkw2Q==
|
||||
iconv-lite@^0.4.24:
|
||||
version "0.4.24"
|
||||
resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.24.tgz#2022b4b25fbddc21d2f524974a474aafe733908b"
|
||||
integrity sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==
|
||||
dependencies:
|
||||
safer-buffer ">= 2.1.2 < 3"
|
||||
|
||||
|
|
|
@ -41,7 +41,7 @@
|
|||
"graceful-fs": "4.2.3",
|
||||
"http-proxy-agent": "^2.1.0",
|
||||
"https-proxy-agent": "^2.2.3",
|
||||
"iconv-lite": "0.6.0",
|
||||
"iconv-lite": "0.5.0",
|
||||
"jschardet": "2.1.1",
|
||||
"keytar": "^5.5.0",
|
||||
"minimist": "^1.2.5",
|
||||
|
|
|
@ -8,7 +8,7 @@
|
|||
"graceful-fs": "4.2.3",
|
||||
"http-proxy-agent": "^2.1.0",
|
||||
"https-proxy-agent": "^2.2.3",
|
||||
"iconv-lite": "0.6.0",
|
||||
"iconv-lite": "0.5.0",
|
||||
"jschardet": "2.1.1",
|
||||
"minimist": "^1.2.5",
|
||||
"native-watchdog": "1.3.0",
|
||||
|
|
|
@ -176,10 +176,10 @@ https-proxy-agent@^2.2.3:
|
|||
agent-base "^4.3.0"
|
||||
debug "^3.1.0"
|
||||
|
||||
iconv-lite@0.6.0:
|
||||
version "0.6.0"
|
||||
resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.0.tgz#66a93b80df0bd05d2a43a7426296b7f91073f125"
|
||||
integrity sha512-43ZpGYZ9QtuutX5l6WC1DSO8ane9N+Ct5qPLF2OV7vM9abM69gnAbVkh66ibaZd3aOGkoP1ZmringlKhLBkw2Q==
|
||||
iconv-lite@0.5.0:
|
||||
version "0.5.0"
|
||||
resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.5.0.tgz#59cdde0a2a297cc2aeb0c6445a195ee89f127550"
|
||||
integrity sha512-NnEhI9hIEKHOzJ4f697DMz9IQEXr/MMJ5w64vN2/4Ai+wRnvV7SBrL0KLoRlwaKVghOc7LQ5YkPLuX146b6Ydw==
|
||||
dependencies:
|
||||
safer-buffer ">= 2.1.2 < 3"
|
||||
|
||||
|
|
|
@ -4,9 +4,8 @@
|
|||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
import * as iconv from 'iconv-lite';
|
||||
import { Readable, ReadableStream, newWriteableStream } from 'vs/base/common/stream';
|
||||
import { isUndefinedOrNull, isUndefined, isNumber } from 'vs/base/common/types';
|
||||
import { VSBuffer, VSBufferReadable, VSBufferReadableStream } from 'vs/base/common/buffer';
|
||||
import { Readable, Writable } from 'stream';
|
||||
import { VSBuffer } from 'vs/base/common/buffer';
|
||||
|
||||
export const UTF8 = 'utf8';
|
||||
export const UTF8_with_bom = 'utf8bom';
|
||||
|
@ -36,135 +35,121 @@ export interface IDecodeStreamOptions {
|
|||
}
|
||||
|
||||
export interface IDecodeStreamResult {
|
||||
stream: ReadableStream<string>;
|
||||
stream: NodeJS.ReadableStream;
|
||||
detected: IDetectedEncodingResult;
|
||||
}
|
||||
|
||||
export function toDecodeStream(source: VSBufferReadableStream, options: IDecodeStreamOptions): Promise<IDecodeStreamResult> {
|
||||
export function toDecodeStream(readable: Readable, options: IDecodeStreamOptions): Promise<IDecodeStreamResult> {
|
||||
if (!options.minBytesRequiredForDetection) {
|
||||
options.minBytesRequiredForDetection = options.guessEncoding ? AUTO_ENCODING_GUESS_MIN_BYTES : NO_ENCODING_GUESS_MIN_BYTES;
|
||||
}
|
||||
|
||||
return new Promise<IDecodeStreamResult>((resolve, reject) => {
|
||||
const target = newWriteableStream<string>(strings => strings.join(''));
|
||||
const writer = new class extends Writable {
|
||||
private decodeStream: NodeJS.ReadWriteStream | undefined;
|
||||
private decodeStreamPromise: Promise<void> | undefined;
|
||||
|
||||
const bufferedChunks: VSBuffer[] = [];
|
||||
let bytesBuffered = 0;
|
||||
let decoder: iconv.DecoderStream | null = null;
|
||||
private bufferedChunks: Buffer[] = [];
|
||||
private bytesBuffered = 0;
|
||||
|
||||
_write(chunk: Buffer, encoding: string, callback: (error: Error | null | undefined) => void): void {
|
||||
if (!Buffer.isBuffer(chunk)) {
|
||||
return callback(new Error('toDecodeStream(): data must be a buffer'));
|
||||
}
|
||||
|
||||
// if the decode stream is ready, we just write directly
|
||||
if (this.decodeStream) {
|
||||
this.decodeStream.write(chunk, callback);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
// otherwise we need to buffer the data until the stream is ready
|
||||
this.bufferedChunks.push(chunk);
|
||||
this.bytesBuffered += chunk.byteLength;
|
||||
|
||||
// waiting for the decoder to be ready
|
||||
if (this.decodeStreamPromise) {
|
||||
this.decodeStreamPromise.then(() => callback(null), error => callback(error));
|
||||
}
|
||||
|
||||
// buffered enough data for encoding detection, create stream and forward data
|
||||
else if (typeof options.minBytesRequiredForDetection === 'number' && this.bytesBuffered >= options.minBytesRequiredForDetection) {
|
||||
this._startDecodeStream(callback);
|
||||
}
|
||||
|
||||
// only buffering until enough data for encoding detection is there
|
||||
else {
|
||||
callback(null);
|
||||
}
|
||||
}
|
||||
|
||||
_startDecodeStream(callback: (error: Error | null | undefined) => void): void {
|
||||
|
||||
// detect encoding from buffer
|
||||
this.decodeStreamPromise = Promise.resolve(detectEncodingFromBuffer({
|
||||
buffer: Buffer.concat(this.bufferedChunks),
|
||||
bytesRead: this.bytesBuffered
|
||||
}, options.guessEncoding)).then(detected => {
|
||||
|
||||
const startDecodeStream = () => {
|
||||
return Promise.resolve()
|
||||
.then(() =>
|
||||
// detect encoding from buffer
|
||||
detectEncodingFromBuffer({
|
||||
buffer: Buffer.from(VSBuffer.concat(bufferedChunks).buffer),
|
||||
bytesRead: bytesBuffered
|
||||
}, options.guessEncoding)
|
||||
)
|
||||
.then(detected => {
|
||||
// ensure to respect overwrite of encoding
|
||||
detected.encoding = options.overwriteEncoding(detected.encoding);
|
||||
|
||||
// decode and write buffered content
|
||||
decoder = iconv.getDecoder(toNodeEncoding(detected.encoding));
|
||||
const nodeBuffer = Buffer.from(VSBuffer.concat(bufferedChunks).buffer);
|
||||
target.write(decoder.write(nodeBuffer));
|
||||
bufferedChunks.length = 0;
|
||||
// decode and write buffer
|
||||
this.decodeStream = decodeStream(detected.encoding);
|
||||
this.decodeStream.write(Buffer.concat(this.bufferedChunks), callback);
|
||||
this.bufferedChunks.length = 0;
|
||||
|
||||
// signal to the outside our detected encoding
|
||||
// and final decoder stream
|
||||
resolve({
|
||||
stream: target,
|
||||
detected,
|
||||
});
|
||||
})
|
||||
.catch(reject);
|
||||
};
|
||||
resolve({ detected, stream: this.decodeStream });
|
||||
}, error => {
|
||||
this.emit('error', error);
|
||||
|
||||
source.on('error', target.error);
|
||||
source.on('data', (chunk) => {
|
||||
// if the decoder is ready, we just write directly
|
||||
if (!isUndefinedOrNull(decoder)) {
|
||||
target.write(decoder.write(Buffer.from(chunk.buffer)));
|
||||
return;
|
||||
}
|
||||
|
||||
// otherwise we need to buffer the data until the stream is ready
|
||||
bufferedChunks.push(chunk);
|
||||
bytesBuffered += chunk.byteLength;
|
||||
|
||||
// buffered enough data for encoding detection, create stream and forward data
|
||||
if (isNumber(options.minBytesRequiredForDetection) && bytesBuffered >= options.minBytesRequiredForDetection) {
|
||||
startDecodeStream();
|
||||
}
|
||||
});
|
||||
source.on('end', () => {
|
||||
// normal finish
|
||||
if (!isUndefinedOrNull(decoder)) {
|
||||
target.end(decoder.end());
|
||||
}
|
||||
|
||||
// we were still waiting for data to do the encoding
|
||||
// detection. thus, wrap up starting the stream even
|
||||
// without all the data to get things going
|
||||
else {
|
||||
startDecodeStream().then(() => {
|
||||
target.end(decoder?.end());
|
||||
callback(error);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
_final(callback: () => void) {
|
||||
|
||||
// normal finish
|
||||
if (this.decodeStream) {
|
||||
this.decodeStream.end(callback);
|
||||
}
|
||||
|
||||
// we were still waiting for data to do the encoding
|
||||
// detection. thus, wrap up starting the stream even
|
||||
// without all the data to get things going
|
||||
else {
|
||||
this._startDecodeStream(() => {
|
||||
if (this.decodeStream) {
|
||||
this.decodeStream.end(callback);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// errors
|
||||
readable.on('error', reject);
|
||||
|
||||
// pipe through
|
||||
readable.pipe(writer);
|
||||
});
|
||||
}
|
||||
|
||||
export function toEncodeReadable(readable: Readable<string>, encoding: string, options?: { addBOM?: boolean }): VSBufferReadable {
|
||||
const encoder = iconv.getEncoder(toNodeEncoding(encoding), options);
|
||||
let bytesRead = 0;
|
||||
let done = false;
|
||||
|
||||
return {
|
||||
read() {
|
||||
if (done) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const chunk = readable.read();
|
||||
if (isUndefinedOrNull(chunk)) {
|
||||
done = true;
|
||||
|
||||
// If we are instructed to add a BOM but we detect that no
|
||||
// bytes have been read, we must ensure to return the BOM
|
||||
// ourselves so that we comply with the contract.
|
||||
if (bytesRead === 0 && options?.addBOM) {
|
||||
switch (encoding) {
|
||||
case UTF8:
|
||||
case UTF8_with_bom:
|
||||
return VSBuffer.wrap(Buffer.from(UTF8_BOM));
|
||||
case UTF16be:
|
||||
return VSBuffer.wrap(Buffer.from(UTF16be_BOM));
|
||||
case UTF16le:
|
||||
return VSBuffer.wrap(Buffer.from(UTF16le_BOM));
|
||||
}
|
||||
}
|
||||
|
||||
const leftovers = encoder.end();
|
||||
if (!isUndefined(leftovers) && leftovers.length > 0) {
|
||||
return VSBuffer.wrap(leftovers);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
bytesRead += chunk.length;
|
||||
|
||||
return VSBuffer.wrap(encoder.write(chunk));
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
export function encodingExists(encoding: string): boolean {
|
||||
return iconv.encodingExists(toNodeEncoding(encoding));
|
||||
}
|
||||
|
||||
function decodeStream(encoding: string | null): NodeJS.ReadWriteStream {
|
||||
return iconv.decodeStream(toNodeEncoding(encoding));
|
||||
}
|
||||
|
||||
export function encodeStream(encoding: string, options?: { addBOM?: boolean }): NodeJS.ReadWriteStream {
|
||||
return iconv.encodeStream(toNodeEncoding(encoding), options);
|
||||
}
|
||||
|
||||
export function toNodeEncoding(enc: string | null): string {
|
||||
if (enc === UTF8_with_bom || enc === null) {
|
||||
return UTF8; // iconv does not distinguish UTF 8 with or without BOM, so we need to help it
|
||||
|
|
|
@ -3,8 +3,10 @@
|
|||
* Licensed under the MIT License. See License.txt in the project root for license information.
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
import { VSBufferReadableStream } from 'vs/base/common/buffer';
|
||||
import { VSBufferReadableStream, VSBufferReadable, VSBuffer } from 'vs/base/common/buffer';
|
||||
import { Readable } from 'stream';
|
||||
import { isUndefinedOrNull } from 'vs/base/common/types';
|
||||
import { UTF8, UTF8_with_bom, UTF8_BOM, UTF16be, UTF16le_BOM, UTF16be_BOM, UTF16le, UTF_ENCODING } from 'vs/base/node/encoding';
|
||||
|
||||
export function streamToNodeReadable(stream: VSBufferReadableStream): Readable {
|
||||
return new class extends Readable {
|
||||
|
@ -49,3 +51,62 @@ export function streamToNodeReadable(stream: VSBufferReadableStream): Readable {
|
|||
}
|
||||
};
|
||||
}
|
||||
|
||||
export function nodeReadableToString(stream: NodeJS.ReadableStream): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
let result = '';
|
||||
|
||||
stream.on('data', chunk => result += chunk);
|
||||
stream.on('error', reject);
|
||||
stream.on('end', () => resolve(result));
|
||||
});
|
||||
}
|
||||
|
||||
export function nodeStreamToVSBufferReadable(stream: NodeJS.ReadWriteStream, addBOM?: { encoding: UTF_ENCODING }): VSBufferReadable {
|
||||
let bytesRead = 0;
|
||||
let done = false;
|
||||
|
||||
return {
|
||||
read(): VSBuffer | null {
|
||||
if (done) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const res = stream.read();
|
||||
if (isUndefinedOrNull(res)) {
|
||||
done = true;
|
||||
|
||||
// If we are instructed to add a BOM but we detect that no
|
||||
// bytes have been read, we must ensure to return the BOM
|
||||
// ourselves so that we comply with the contract.
|
||||
if (bytesRead === 0 && addBOM) {
|
||||
switch (addBOM.encoding) {
|
||||
case UTF8:
|
||||
case UTF8_with_bom:
|
||||
return VSBuffer.wrap(Buffer.from(UTF8_BOM));
|
||||
case UTF16be:
|
||||
return VSBuffer.wrap(Buffer.from(UTF16be_BOM));
|
||||
case UTF16le:
|
||||
return VSBuffer.wrap(Buffer.from(UTF16le_BOM));
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
// Handle String
|
||||
if (typeof res === 'string') {
|
||||
bytesRead += res.length;
|
||||
|
||||
return VSBuffer.fromString(res);
|
||||
}
|
||||
|
||||
// Handle Buffer
|
||||
else {
|
||||
bytesRead += res.byteLength;
|
||||
|
||||
return VSBuffer.wrap(res);
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
|
|
@ -7,10 +7,9 @@ import * as assert from 'assert';
|
|||
import * as fs from 'fs';
|
||||
import * as encoding from 'vs/base/node/encoding';
|
||||
import * as terminalEncoding from 'vs/base/node/terminalEncoding';
|
||||
import * as streams from 'vs/base/common/stream';
|
||||
import { Readable } from 'stream';
|
||||
import * as iconv from 'iconv-lite';
|
||||
import { getPathFromAmdModule } from 'vs/base/common/amd';
|
||||
import { newWriteableBufferStream, VSBuffer, VSBufferReadableStream, streamToBufferReadableStream } from 'vs/base/common/buffer';
|
||||
|
||||
export async function detectEncodingByBOM(file: string): Promise<typeof encoding.UTF16be | typeof encoding.UTF16le | typeof encoding.UTF8_with_bom | null> {
|
||||
try {
|
||||
|
@ -232,32 +231,30 @@ suite('Encoding', () => {
|
|||
});
|
||||
}
|
||||
|
||||
function newTestReadableStream(buffers: Buffer[]): VSBufferReadableStream {
|
||||
const stream = newWriteableBufferStream();
|
||||
buffers
|
||||
.map(VSBuffer.wrap)
|
||||
.forEach(buffer => {
|
||||
setTimeout(() => {
|
||||
stream.write(buffer);
|
||||
});
|
||||
async function readAllAsString(stream: NodeJS.ReadableStream) {
|
||||
return new Promise<string>((resolve, reject) => {
|
||||
let all = '';
|
||||
stream.on('data', chunk => {
|
||||
all += chunk;
|
||||
assert.equal(typeof chunk, 'string');
|
||||
});
|
||||
setTimeout(() => {
|
||||
stream.end();
|
||||
stream.on('end', () => {
|
||||
resolve(all);
|
||||
});
|
||||
stream.on('error', reject);
|
||||
});
|
||||
return stream;
|
||||
}
|
||||
|
||||
async function readAllAsString(stream: streams.ReadableStream<string>) {
|
||||
return streams.consumeStream(stream, strings => strings.join(''));
|
||||
}
|
||||
|
||||
test('toDecodeStream - some stream', async function () {
|
||||
|
||||
let source = newTestReadableStream([
|
||||
Buffer.from([65, 66, 67]),
|
||||
Buffer.from([65, 66, 67]),
|
||||
Buffer.from([65, 66, 67]),
|
||||
]);
|
||||
let source = new Readable({
|
||||
read(size) {
|
||||
this.push(Buffer.from([65, 66, 67]));
|
||||
this.push(Buffer.from([65, 66, 67]));
|
||||
this.push(Buffer.from([65, 66, 67]));
|
||||
this.push(null);
|
||||
}
|
||||
});
|
||||
|
||||
let { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 4, guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 });
|
||||
|
||||
|
@ -270,11 +267,14 @@ suite('Encoding', () => {
|
|||
|
||||
test('toDecodeStream - some stream, expect too much data', async function () {
|
||||
|
||||
let source = newTestReadableStream([
|
||||
Buffer.from([65, 66, 67]),
|
||||
Buffer.from([65, 66, 67]),
|
||||
Buffer.from([65, 66, 67]),
|
||||
]);
|
||||
let source = new Readable({
|
||||
read(size) {
|
||||
this.push(Buffer.from([65, 66, 67]));
|
||||
this.push(Buffer.from([65, 66, 67]));
|
||||
this.push(Buffer.from([65, 66, 67]));
|
||||
this.push(null);
|
||||
}
|
||||
});
|
||||
|
||||
let { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 64, guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 });
|
||||
|
||||
|
@ -287,8 +287,11 @@ suite('Encoding', () => {
|
|||
|
||||
test('toDecodeStream - some stream, no data', async function () {
|
||||
|
||||
let source = newWriteableBufferStream();
|
||||
source.end();
|
||||
let source = new Readable({
|
||||
read(size) {
|
||||
this.push(null); // empty
|
||||
}
|
||||
});
|
||||
|
||||
let { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 512, guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 });
|
||||
|
||||
|
@ -303,7 +306,7 @@ suite('Encoding', () => {
|
|||
test('toDecodeStream - encoding, utf16be', async function () {
|
||||
|
||||
let path = getPathFromAmdModule(require, './fixtures/some_utf16be.css');
|
||||
let source = streamToBufferReadableStream(fs.createReadStream(path));
|
||||
let source = fs.createReadStream(path);
|
||||
|
||||
let { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 64, guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 });
|
||||
|
||||
|
@ -319,91 +322,11 @@ suite('Encoding', () => {
|
|||
test('toDecodeStream - empty file', async function () {
|
||||
|
||||
let path = getPathFromAmdModule(require, './fixtures/empty.txt');
|
||||
let source = streamToBufferReadableStream(fs.createReadStream(path));
|
||||
let source = fs.createReadStream(path);
|
||||
let { detected, stream } = await encoding.toDecodeStream(source, { guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 });
|
||||
|
||||
let expected = await readAndDecodeFromDisk(path, detected.encoding);
|
||||
let actual = await readAllAsString(stream);
|
||||
assert.equal(actual, expected);
|
||||
});
|
||||
|
||||
test('toDecodeStream - decodes buffer entirely', async function () {
|
||||
let emojis = Buffer.from('🖥️💻💾');
|
||||
let incompleteEmojis = emojis.slice(0, emojis.length - 1);
|
||||
|
||||
let buffers = [];
|
||||
for (let i = 0; i < incompleteEmojis.length; i++) {
|
||||
buffers.push(incompleteEmojis.slice(i, i + 1));
|
||||
}
|
||||
|
||||
const source = newTestReadableStream(buffers);
|
||||
let { stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 4, guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 });
|
||||
|
||||
let expected = incompleteEmojis.toString(encoding.UTF8);
|
||||
let actual = await readAllAsString(stream);
|
||||
|
||||
assert.equal(actual, expected);
|
||||
});
|
||||
|
||||
test('toEncodeReadable - encoding, utf16be', async function () {
|
||||
|
||||
let path = getPathFromAmdModule(require, './fixtures/some_utf16be.css');
|
||||
let source = await readAndDecodeFromDisk(path, encoding.UTF16be);
|
||||
|
||||
let expected = VSBuffer.wrap(
|
||||
iconv.encode(source, encoding.toNodeEncoding(encoding.UTF16be))
|
||||
).toString();
|
||||
let actual = streams.consumeReadable(
|
||||
encoding.toEncodeReadable(streams.toReadable(source), encoding.UTF16be),
|
||||
VSBuffer.concat
|
||||
).toString();
|
||||
|
||||
assert.equal(actual, expected);
|
||||
});
|
||||
|
||||
test('toEncodeReadable - empty readable to utf8', async function () {
|
||||
|
||||
const source: streams.Readable<string> = {
|
||||
read() {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
let actual = streams.consumeReadable(
|
||||
encoding.toEncodeReadable(source, encoding.UTF8),
|
||||
VSBuffer.concat
|
||||
).toString();
|
||||
|
||||
assert.equal(actual, '');
|
||||
});
|
||||
|
||||
[{
|
||||
utfEncoding: encoding.UTF8,
|
||||
relatedBom: encoding.UTF8_BOM
|
||||
}, {
|
||||
utfEncoding: encoding.UTF8_with_bom,
|
||||
relatedBom: encoding.UTF8_BOM
|
||||
}, {
|
||||
utfEncoding: encoding.UTF16be,
|
||||
relatedBom: encoding.UTF16be_BOM,
|
||||
}, {
|
||||
utfEncoding: encoding.UTF16le,
|
||||
relatedBom: encoding.UTF16le_BOM
|
||||
}].forEach(({ utfEncoding, relatedBom }) => {
|
||||
test(`toEncodeReadable - empty readable to ${utfEncoding} with BOM`, async function () {
|
||||
|
||||
const source: streams.Readable<string> = {
|
||||
read() {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
let encodedReadable = encoding.toEncodeReadable(source, utfEncoding, { addBOM: true });
|
||||
|
||||
const expected = VSBuffer.wrap(Buffer.from(relatedBom)).toString();
|
||||
const actual = streams.consumeReadable(encodedReadable, VSBuffer.concat).toString();
|
||||
|
||||
assert.equal(actual, expected);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
@ -17,15 +17,16 @@ import { isMacintosh } from 'vs/base/common/platform';
|
|||
import { IProductService } from 'vs/platform/product/common/productService';
|
||||
import { ITextResourceConfigurationService } from 'vs/editor/common/services/textResourceConfigurationService';
|
||||
import { IWorkspaceContextService } from 'vs/platform/workspace/common/workspace';
|
||||
import { UTF8, UTF8_with_bom, UTF16be, UTF16le, encodingExists, UTF8_BOM, toDecodeStream, toEncodeReadable, IDecodeStreamResult, detectEncodingByBOMFromBuffer } from 'vs/base/node/encoding';
|
||||
import { UTF8, UTF8_with_bom, UTF16be, UTF16le, encodingExists, encodeStream, UTF8_BOM, toDecodeStream, IDecodeStreamResult, detectEncodingByBOMFromBuffer, isUTFEncoding } from 'vs/base/node/encoding';
|
||||
import { WORKSPACE_EXTENSION } from 'vs/platform/workspaces/common/workspaces';
|
||||
import { joinPath, extname, isEqualOrParent } from 'vs/base/common/resources';
|
||||
import { Disposable } from 'vs/base/common/lifecycle';
|
||||
import { IEnvironmentService } from 'vs/platform/environment/common/environment';
|
||||
import { bufferToStream, VSBufferReadable } from 'vs/base/common/buffer';
|
||||
import { VSBufferReadable, bufferToStream } from 'vs/base/common/buffer';
|
||||
import { Readable } from 'stream';
|
||||
import { createTextBufferFactoryFromStream } from 'vs/editor/common/model/textModel';
|
||||
import { ITextSnapshot } from 'vs/editor/common/model';
|
||||
import { consumeStream } from 'vs/base/common/stream';
|
||||
import { nodeReadableToString, streamToNodeReadable, nodeStreamToVSBufferReadable } from 'vs/base/node/stream';
|
||||
import { IUntitledTextEditorService } from 'vs/workbench/services/untitled/common/untitledTextEditorService';
|
||||
import { ILifecycleService } from 'vs/platform/lifecycle/common/lifecycle';
|
||||
import { IInstantiationService } from 'vs/platform/instantiation/common/instantiation';
|
||||
|
@ -88,7 +89,7 @@ export class NativeTextFileService extends AbstractTextFileService {
|
|||
return {
|
||||
...bufferStream,
|
||||
encoding: decoder.detected.encoding || UTF8,
|
||||
value: await consumeStream(decoder.stream, strings => strings.join(''))
|
||||
value: await nodeReadableToString(decoder.stream)
|
||||
};
|
||||
}
|
||||
|
||||
|
@ -120,7 +121,7 @@ export class NativeTextFileService extends AbstractTextFileService {
|
|||
}
|
||||
|
||||
// read through encoding library
|
||||
const decoder = await toDecodeStream(bufferStream.value, {
|
||||
const decoder = await toDecodeStream(streamToNodeReadable(bufferStream.value), {
|
||||
guessEncoding: options?.autoGuessEncoding || this.textResourceConfigurationService.getValue(resource, 'files.autoGuessEncoding'),
|
||||
overwriteEncoding: detectedEncoding => this.encoding.getReadEncoding(resource, options, detectedEncoding)
|
||||
});
|
||||
|
@ -231,8 +232,37 @@ export class NativeTextFileService extends AbstractTextFileService {
|
|||
}
|
||||
|
||||
private getEncodedReadable(value: string | ITextSnapshot, encoding: string, addBOM: boolean): VSBufferReadable {
|
||||
const snapshot = typeof value === 'string' ? stringToSnapshot(value) : value;
|
||||
return toEncodeReadable(snapshot, encoding, { addBOM });
|
||||
const readable = this.snapshotToNodeReadable(typeof value === 'string' ? stringToSnapshot(value) : value);
|
||||
const encoder = encodeStream(encoding, { addBOM });
|
||||
|
||||
const encodedReadable = readable.pipe(encoder);
|
||||
|
||||
return nodeStreamToVSBufferReadable(encodedReadable, addBOM && isUTFEncoding(encoding) ? { encoding } : undefined);
|
||||
}
|
||||
|
||||
private snapshotToNodeReadable(snapshot: ITextSnapshot): Readable {
|
||||
return new Readable({
|
||||
read: function () {
|
||||
try {
|
||||
let chunk: string | null = null;
|
||||
let canPush = true;
|
||||
|
||||
// Push all chunks as long as we can push and as long as
|
||||
// the underlying snapshot returns strings to us
|
||||
while (canPush && typeof (chunk = snapshot.read()) === 'string') {
|
||||
canPush = this.push(chunk);
|
||||
}
|
||||
|
||||
// Signal EOS by pushing NULL
|
||||
if (typeof chunk !== 'string') {
|
||||
this.push(null);
|
||||
}
|
||||
} catch (error) {
|
||||
this.emit('error', error);
|
||||
}
|
||||
},
|
||||
encoding: UTF8 // very important, so that strings are passed around and not buffers!
|
||||
});
|
||||
}
|
||||
|
||||
private async writeElevated(resource: URI, value: string | ITextSnapshot, options?: IWriteTextFileOptions): Promise<IFileStatWithMetadata> {
|
||||
|
|
|
@ -4579,10 +4579,10 @@ husky@^0.13.1:
|
|||
is-ci "^1.0.9"
|
||||
normalize-path "^1.0.0"
|
||||
|
||||
iconv-lite@0.6.0:
|
||||
version "0.6.0"
|
||||
resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.0.tgz#66a93b80df0bd05d2a43a7426296b7f91073f125"
|
||||
integrity sha512-43ZpGYZ9QtuutX5l6WC1DSO8ane9N+Ct5qPLF2OV7vM9abM69gnAbVkh66ibaZd3aOGkoP1ZmringlKhLBkw2Q==
|
||||
iconv-lite@0.5.0:
|
||||
version "0.5.0"
|
||||
resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.5.0.tgz#59cdde0a2a297cc2aeb0c6445a195ee89f127550"
|
||||
integrity sha512-NnEhI9hIEKHOzJ4f697DMz9IQEXr/MMJ5w64vN2/4Ai+wRnvV7SBrL0KLoRlwaKVghOc7LQ5YkPLuX146b6Ydw==
|
||||
dependencies:
|
||||
safer-buffer ">= 2.1.2 < 3"
|
||||
|
||||
|
|
Loading…
Reference in a new issue