diff --git a/build/package.json b/build/package.json index 3daba3dd482..79862e3ce5d 100644 --- a/build/package.json +++ b/build/package.json @@ -38,7 +38,7 @@ "gulp-bom": "^1.0.0", "gulp-sourcemaps": "^1.11.0", "gulp-uglify": "^3.0.0", - "iconv-lite": "0.4.23", + "iconv-lite": "0.6.0", "mime": "^1.3.4", "minimatch": "3.0.4", "minimist": "^1.2.3", diff --git a/build/yarn.lock b/build/yarn.lock index 3f7d72a7438..9bf1e1c7095 100644 --- a/build/yarn.lock +++ b/build/yarn.lock @@ -1415,10 +1415,10 @@ http-signature@~1.2.0: jsprim "^1.2.2" sshpk "^1.7.0" -iconv-lite@0.4.23: - version "0.4.23" - resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.23.tgz#297871f63be507adcfbfca715d0cd0eed84e9a63" - integrity sha512-neyTUVFtahjf0mB3dZT77u+8O0QB89jFdnBkd5P1JgYPbPaia3gXXOVL2fq8VyU2gMMD7SaN7QukTB/pmXYvDA== +iconv-lite@0.6.0: + version "0.6.0" + resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.0.tgz#66a93b80df0bd05d2a43a7426296b7f91073f125" + integrity sha512-43ZpGYZ9QtuutX5l6WC1DSO8ane9N+Ct5qPLF2OV7vM9abM69gnAbVkh66ibaZd3aOGkoP1ZmringlKhLBkw2Q== dependencies: safer-buffer ">= 2.1.2 < 3" diff --git a/extensions/git/package.json b/extensions/git/package.json index d6472c6e4e4..318c6e733c7 100644 --- a/extensions/git/package.json +++ b/extensions/git/package.json @@ -1878,7 +1878,7 @@ "dependencies": { "byline": "^5.0.0", "file-type": "^7.2.0", - "iconv-lite": "^0.4.24", + "iconv-lite": "0.6.0", "jschardet": "2.1.1", "vscode-extension-telemetry": "0.1.1", "vscode-nls": "^4.0.0", diff --git a/extensions/git/yarn.lock b/extensions/git/yarn.lock index f72600de7bc..f03bee0ad56 100644 --- a/extensions/git/yarn.lock +++ b/extensions/git/yarn.lock @@ -425,10 +425,10 @@ https-proxy-agent@^2.2.1: agent-base "^4.3.0" debug "^3.1.0" -iconv-lite@^0.4.24: - version "0.4.24" - resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.24.tgz#2022b4b25fbddc21d2f524974a474aafe733908b" - integrity sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA== +iconv-lite@0.6.0: + version "0.6.0" + resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.0.tgz#66a93b80df0bd05d2a43a7426296b7f91073f125" + integrity sha512-43ZpGYZ9QtuutX5l6WC1DSO8ane9N+Ct5qPLF2OV7vM9abM69gnAbVkh66ibaZd3aOGkoP1ZmringlKhLBkw2Q== dependencies: safer-buffer ">= 2.1.2 < 3" diff --git a/package.json b/package.json index 458b3bffa52..20f054a2a3f 100644 --- a/package.json +++ b/package.json @@ -41,7 +41,7 @@ "graceful-fs": "4.2.3", "http-proxy-agent": "^2.1.0", "https-proxy-agent": "^2.2.3", - "iconv-lite": "0.5.0", + "iconv-lite": "0.6.0", "jschardet": "2.1.1", "keytar": "^5.5.0", "minimist": "^1.2.5", diff --git a/remote/package.json b/remote/package.json index cba584d935d..a1e1cd5d802 100644 --- a/remote/package.json +++ b/remote/package.json @@ -8,7 +8,7 @@ "graceful-fs": "4.2.3", "http-proxy-agent": "^2.1.0", "https-proxy-agent": "^2.2.3", - "iconv-lite": "0.5.0", + "iconv-lite": "0.6.0", "jschardet": "2.1.1", "minimist": "^1.2.5", "native-watchdog": "1.3.0", diff --git a/remote/yarn.lock b/remote/yarn.lock index 7ba3b88854f..4effe6f9fa0 100644 --- a/remote/yarn.lock +++ b/remote/yarn.lock @@ -176,10 +176,10 @@ https-proxy-agent@^2.2.3: agent-base "^4.3.0" debug "^3.1.0" -iconv-lite@0.5.0: - version "0.5.0" - resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.5.0.tgz#59cdde0a2a297cc2aeb0c6445a195ee89f127550" - integrity sha512-NnEhI9hIEKHOzJ4f697DMz9IQEXr/MMJ5w64vN2/4Ai+wRnvV7SBrL0KLoRlwaKVghOc7LQ5YkPLuX146b6Ydw== +iconv-lite@0.6.0: + version "0.6.0" + resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.0.tgz#66a93b80df0bd05d2a43a7426296b7f91073f125" + integrity sha512-43ZpGYZ9QtuutX5l6WC1DSO8ane9N+Ct5qPLF2OV7vM9abM69gnAbVkh66ibaZd3aOGkoP1ZmringlKhLBkw2Q== dependencies: safer-buffer ">= 2.1.2 < 3" diff --git a/src/vs/base/node/encoding.ts b/src/vs/base/node/encoding.ts index ec3392a780e..a7034737930 100644 --- a/src/vs/base/node/encoding.ts +++ b/src/vs/base/node/encoding.ts @@ -4,8 +4,9 @@ *--------------------------------------------------------------------------------------------*/ import * as iconv from 'iconv-lite'; -import { Readable, Writable } from 'stream'; -import { VSBuffer } from 'vs/base/common/buffer'; +import { Readable, ReadableStream, newWriteableStream } from 'vs/base/common/stream'; +import { isUndefinedOrNull, isUndefined, isNumber } from 'vs/base/common/types'; +import { VSBuffer, VSBufferReadable, VSBufferReadableStream } from 'vs/base/common/buffer'; export const UTF8 = 'utf8'; export const UTF8_with_bom = 'utf8bom'; @@ -35,121 +36,135 @@ export interface IDecodeStreamOptions { } export interface IDecodeStreamResult { - stream: NodeJS.ReadableStream; + stream: ReadableStream; detected: IDetectedEncodingResult; } -export function toDecodeStream(readable: Readable, options: IDecodeStreamOptions): Promise { +export function toDecodeStream(source: VSBufferReadableStream, options: IDecodeStreamOptions): Promise { if (!options.minBytesRequiredForDetection) { options.minBytesRequiredForDetection = options.guessEncoding ? AUTO_ENCODING_GUESS_MIN_BYTES : NO_ENCODING_GUESS_MIN_BYTES; } return new Promise((resolve, reject) => { - const writer = new class extends Writable { - private decodeStream: NodeJS.ReadWriteStream | undefined; - private decodeStreamPromise: Promise | undefined; + const target = newWriteableStream(strings => strings.join('')); - private bufferedChunks: Buffer[] = []; - private bytesBuffered = 0; - - _write(chunk: Buffer, encoding: string, callback: (error: Error | null | undefined) => void): void { - if (!Buffer.isBuffer(chunk)) { - return callback(new Error('toDecodeStream(): data must be a buffer')); - } - - // if the decode stream is ready, we just write directly - if (this.decodeStream) { - this.decodeStream.write(chunk, callback); - - return; - } - - // otherwise we need to buffer the data until the stream is ready - this.bufferedChunks.push(chunk); - this.bytesBuffered += chunk.byteLength; - - // waiting for the decoder to be ready - if (this.decodeStreamPromise) { - this.decodeStreamPromise.then(() => callback(null), error => callback(error)); - } - - // buffered enough data for encoding detection, create stream and forward data - else if (typeof options.minBytesRequiredForDetection === 'number' && this.bytesBuffered >= options.minBytesRequiredForDetection) { - this._startDecodeStream(callback); - } - - // only buffering until enough data for encoding detection is there - else { - callback(null); - } - } - - _startDecodeStream(callback: (error: Error | null | undefined) => void): void { - - // detect encoding from buffer - this.decodeStreamPromise = Promise.resolve(detectEncodingFromBuffer({ - buffer: Buffer.concat(this.bufferedChunks), - bytesRead: this.bytesBuffered - }, options.guessEncoding)).then(detected => { + const bufferedChunks: VSBuffer[] = []; + let bytesBuffered = 0; + let decoder: iconv.DecoderStream | null = null; + const startDecodeStream = () => { + return Promise.resolve() + .then(() => + // detect encoding from buffer + detectEncodingFromBuffer({ + buffer: Buffer.from(VSBuffer.concat(bufferedChunks).buffer), + bytesRead: bytesBuffered + }, options.guessEncoding) + ) + .then(detected => { // ensure to respect overwrite of encoding detected.encoding = options.overwriteEncoding(detected.encoding); - // decode and write buffer - this.decodeStream = decodeStream(detected.encoding); - this.decodeStream.write(Buffer.concat(this.bufferedChunks), callback); - this.bufferedChunks.length = 0; + // decode and write buffered content + decoder = iconv.getDecoder(toNodeEncoding(detected.encoding)); + const nodeBuffer = Buffer.from(VSBuffer.concat(bufferedChunks).buffer); + target.write(decoder.write(nodeBuffer)); + bufferedChunks.length = 0; // signal to the outside our detected encoding // and final decoder stream - resolve({ detected, stream: this.decodeStream }); - }, error => { - this.emit('error', error); - - callback(error); - }); - } - - _final(callback: () => void) { - - // normal finish - if (this.decodeStream) { - this.decodeStream.end(callback); - } - - // we were still waiting for data to do the encoding - // detection. thus, wrap up starting the stream even - // without all the data to get things going - else { - this._startDecodeStream(() => { - if (this.decodeStream) { - this.decodeStream.end(callback); - } + resolve({ + stream: target, + detected, }); - } - } + }) + .catch(reject); }; - // errors - readable.on('error', reject); + source.on('error', target.error); + source.on('data', (chunk) => { + // if the decoder is ready, we just write directly + if (!isUndefinedOrNull(decoder)) { + target.write(decoder.write(Buffer.from(chunk.buffer))); + return; + } - // pipe through - readable.pipe(writer); + // otherwise we need to buffer the data until the stream is ready + bufferedChunks.push(chunk); + bytesBuffered += chunk.byteLength; + + // buffered enough data for encoding detection, create stream and forward data + if (isNumber(options.minBytesRequiredForDetection) && bytesBuffered >= options.minBytesRequiredForDetection) { + startDecodeStream(); + } + }); + source.on('end', () => { + // normal finish + if (!isUndefinedOrNull(decoder)) { + target.end(decoder.end()); + } + + // we were still waiting for data to do the encoding + // detection. thus, wrap up starting the stream even + // without all the data to get things going + else { + startDecodeStream().then(() => { + target.end(decoder?.end()); + }); + } + }); }); } +export function toEncodeReadable(readable: Readable, encoding: string, options?: { addBOM?: boolean }): VSBufferReadable { + const encoder = iconv.getEncoder(toNodeEncoding(encoding), options); + let bytesRead = 0; + let done = false; + + return { + read() { + if (done) { + return null; + } + + const chunk = readable.read(); + if (isUndefinedOrNull(chunk)) { + done = true; + + // If we are instructed to add a BOM but we detect that no + // bytes have been read, we must ensure to return the BOM + // ourselves so that we comply with the contract. + if (bytesRead === 0 && options?.addBOM) { + switch (encoding) { + case UTF8: + case UTF8_with_bom: + return VSBuffer.wrap(Buffer.from(UTF8_BOM)); + case UTF16be: + return VSBuffer.wrap(Buffer.from(UTF16be_BOM)); + case UTF16le: + return VSBuffer.wrap(Buffer.from(UTF16le_BOM)); + } + } + + const leftovers = encoder.end(); + if (!isUndefined(leftovers) && leftovers.length > 0) { + return VSBuffer.wrap(leftovers); + } + + return null; + } + + bytesRead += chunk.length; + + return VSBuffer.wrap(encoder.write(chunk)); + } + }; +} + export function encodingExists(encoding: string): boolean { return iconv.encodingExists(toNodeEncoding(encoding)); } -function decodeStream(encoding: string | null): NodeJS.ReadWriteStream { - return iconv.decodeStream(toNodeEncoding(encoding)); -} - -export function encodeStream(encoding: string, options?: { addBOM?: boolean }): NodeJS.ReadWriteStream { - return iconv.encodeStream(toNodeEncoding(encoding), options); -} - export function toNodeEncoding(enc: string | null): string { if (enc === UTF8_with_bom || enc === null) { return UTF8; // iconv does not distinguish UTF 8 with or without BOM, so we need to help it diff --git a/src/vs/base/node/stream.ts b/src/vs/base/node/stream.ts index 12ba5e5d929..b280d2fb3ea 100644 --- a/src/vs/base/node/stream.ts +++ b/src/vs/base/node/stream.ts @@ -3,10 +3,8 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { VSBufferReadableStream, VSBufferReadable, VSBuffer } from 'vs/base/common/buffer'; +import { VSBufferReadableStream } from 'vs/base/common/buffer'; import { Readable } from 'stream'; -import { isUndefinedOrNull } from 'vs/base/common/types'; -import { UTF8, UTF8_with_bom, UTF8_BOM, UTF16be, UTF16le_BOM, UTF16be_BOM, UTF16le, UTF_ENCODING } from 'vs/base/node/encoding'; export function streamToNodeReadable(stream: VSBufferReadableStream): Readable { return new class extends Readable { @@ -51,62 +49,3 @@ export function streamToNodeReadable(stream: VSBufferReadableStream): Readable { } }; } - -export function nodeReadableToString(stream: NodeJS.ReadableStream): Promise { - return new Promise((resolve, reject) => { - let result = ''; - - stream.on('data', chunk => result += chunk); - stream.on('error', reject); - stream.on('end', () => resolve(result)); - }); -} - -export function nodeStreamToVSBufferReadable(stream: NodeJS.ReadWriteStream, addBOM?: { encoding: UTF_ENCODING }): VSBufferReadable { - let bytesRead = 0; - let done = false; - - return { - read(): VSBuffer | null { - if (done) { - return null; - } - - const res = stream.read(); - if (isUndefinedOrNull(res)) { - done = true; - - // If we are instructed to add a BOM but we detect that no - // bytes have been read, we must ensure to return the BOM - // ourselves so that we comply with the contract. - if (bytesRead === 0 && addBOM) { - switch (addBOM.encoding) { - case UTF8: - case UTF8_with_bom: - return VSBuffer.wrap(Buffer.from(UTF8_BOM)); - case UTF16be: - return VSBuffer.wrap(Buffer.from(UTF16be_BOM)); - case UTF16le: - return VSBuffer.wrap(Buffer.from(UTF16le_BOM)); - } - } - - return null; - } - - // Handle String - if (typeof res === 'string') { - bytesRead += res.length; - - return VSBuffer.fromString(res); - } - - // Handle Buffer - else { - bytesRead += res.byteLength; - - return VSBuffer.wrap(res); - } - } - }; -} diff --git a/src/vs/base/test/node/encoding/encoding.test.ts b/src/vs/base/test/node/encoding/encoding.test.ts index 024f7084d72..66eb9f3c013 100644 --- a/src/vs/base/test/node/encoding/encoding.test.ts +++ b/src/vs/base/test/node/encoding/encoding.test.ts @@ -7,9 +7,10 @@ import * as assert from 'assert'; import * as fs from 'fs'; import * as encoding from 'vs/base/node/encoding'; import * as terminalEncoding from 'vs/base/node/terminalEncoding'; -import { Readable } from 'stream'; +import * as streams from 'vs/base/common/stream'; import * as iconv from 'iconv-lite'; import { getPathFromAmdModule } from 'vs/base/common/amd'; +import { newWriteableBufferStream, VSBuffer, VSBufferReadableStream, streamToBufferReadableStream } from 'vs/base/common/buffer'; export async function detectEncodingByBOM(file: string): Promise { try { @@ -231,30 +232,32 @@ suite('Encoding', () => { }); } - async function readAllAsString(stream: NodeJS.ReadableStream) { - return new Promise((resolve, reject) => { - let all = ''; - stream.on('data', chunk => { - all += chunk; - assert.equal(typeof chunk, 'string'); + function newTestReadableStream(buffers: Buffer[]): VSBufferReadableStream { + const stream = newWriteableBufferStream(); + buffers + .map(VSBuffer.wrap) + .forEach(buffer => { + setTimeout(() => { + stream.write(buffer); + }); }); - stream.on('end', () => { - resolve(all); - }); - stream.on('error', reject); + setTimeout(() => { + stream.end(); }); + return stream; + } + + async function readAllAsString(stream: streams.ReadableStream) { + return streams.consumeStream(stream, strings => strings.join('')); } test('toDecodeStream - some stream', async function () { - let source = new Readable({ - read(size) { - this.push(Buffer.from([65, 66, 67])); - this.push(Buffer.from([65, 66, 67])); - this.push(Buffer.from([65, 66, 67])); - this.push(null); - } - }); + let source = newTestReadableStream([ + Buffer.from([65, 66, 67]), + Buffer.from([65, 66, 67]), + Buffer.from([65, 66, 67]), + ]); let { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 4, guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 }); @@ -267,14 +270,11 @@ suite('Encoding', () => { test('toDecodeStream - some stream, expect too much data', async function () { - let source = new Readable({ - read(size) { - this.push(Buffer.from([65, 66, 67])); - this.push(Buffer.from([65, 66, 67])); - this.push(Buffer.from([65, 66, 67])); - this.push(null); - } - }); + let source = newTestReadableStream([ + Buffer.from([65, 66, 67]), + Buffer.from([65, 66, 67]), + Buffer.from([65, 66, 67]), + ]); let { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 64, guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 }); @@ -287,11 +287,8 @@ suite('Encoding', () => { test('toDecodeStream - some stream, no data', async function () { - let source = new Readable({ - read(size) { - this.push(null); // empty - } - }); + let source = newWriteableBufferStream(); + source.end(); let { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 512, guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 }); @@ -306,7 +303,7 @@ suite('Encoding', () => { test('toDecodeStream - encoding, utf16be', async function () { let path = getPathFromAmdModule(require, './fixtures/some_utf16be.css'); - let source = fs.createReadStream(path); + let source = streamToBufferReadableStream(fs.createReadStream(path)); let { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 64, guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 }); @@ -322,11 +319,91 @@ suite('Encoding', () => { test('toDecodeStream - empty file', async function () { let path = getPathFromAmdModule(require, './fixtures/empty.txt'); - let source = fs.createReadStream(path); + let source = streamToBufferReadableStream(fs.createReadStream(path)); let { detected, stream } = await encoding.toDecodeStream(source, { guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 }); let expected = await readAndDecodeFromDisk(path, detected.encoding); let actual = await readAllAsString(stream); assert.equal(actual, expected); }); + + test('toDecodeStream - decodes buffer entirely', async function () { + let emojis = Buffer.from('🖥️💻💾'); + let incompleteEmojis = emojis.slice(0, emojis.length - 1); + + let buffers = []; + for (let i = 0; i < incompleteEmojis.length; i++) { + buffers.push(incompleteEmojis.slice(i, i + 1)); + } + + const source = newTestReadableStream(buffers); + let { stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 4, guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 }); + + let expected = incompleteEmojis.toString(encoding.UTF8); + let actual = await readAllAsString(stream); + + assert.equal(actual, expected); + }); + + test('toEncodeReadable - encoding, utf16be', async function () { + + let path = getPathFromAmdModule(require, './fixtures/some_utf16be.css'); + let source = await readAndDecodeFromDisk(path, encoding.UTF16be); + + let expected = VSBuffer.wrap( + iconv.encode(source, encoding.toNodeEncoding(encoding.UTF16be)) + ).toString(); + let actual = streams.consumeReadable( + encoding.toEncodeReadable(streams.toReadable(source), encoding.UTF16be), + VSBuffer.concat + ).toString(); + + assert.equal(actual, expected); + }); + + test('toEncodeReadable - empty readable to utf8', async function () { + + const source: streams.Readable = { + read() { + return null; + } + }; + + let actual = streams.consumeReadable( + encoding.toEncodeReadable(source, encoding.UTF8), + VSBuffer.concat + ).toString(); + + assert.equal(actual, ''); + }); + + [{ + utfEncoding: encoding.UTF8, + relatedBom: encoding.UTF8_BOM + }, { + utfEncoding: encoding.UTF8_with_bom, + relatedBom: encoding.UTF8_BOM + }, { + utfEncoding: encoding.UTF16be, + relatedBom: encoding.UTF16be_BOM, + }, { + utfEncoding: encoding.UTF16le, + relatedBom: encoding.UTF16le_BOM + }].forEach(({ utfEncoding, relatedBom }) => { + test(`toEncodeReadable - empty readable to ${utfEncoding} with BOM`, async function () { + + const source: streams.Readable = { + read() { + return null; + } + }; + + let encodedReadable = encoding.toEncodeReadable(source, utfEncoding, { addBOM: true }); + + const expected = VSBuffer.wrap(Buffer.from(relatedBom)).toString(); + const actual = streams.consumeReadable(encodedReadable, VSBuffer.concat).toString(); + + assert.equal(actual, expected); + }); + }); }); diff --git a/src/vs/workbench/services/textfile/electron-browser/nativeTextFileService.ts b/src/vs/workbench/services/textfile/electron-browser/nativeTextFileService.ts index 557622abf89..beecdcd2fbd 100644 --- a/src/vs/workbench/services/textfile/electron-browser/nativeTextFileService.ts +++ b/src/vs/workbench/services/textfile/electron-browser/nativeTextFileService.ts @@ -17,16 +17,15 @@ import { isMacintosh } from 'vs/base/common/platform'; import { IProductService } from 'vs/platform/product/common/productService'; import { ITextResourceConfigurationService } from 'vs/editor/common/services/textResourceConfigurationService'; import { IWorkspaceContextService } from 'vs/platform/workspace/common/workspace'; -import { UTF8, UTF8_with_bom, UTF16be, UTF16le, encodingExists, encodeStream, UTF8_BOM, toDecodeStream, IDecodeStreamResult, detectEncodingByBOMFromBuffer, isUTFEncoding } from 'vs/base/node/encoding'; +import { UTF8, UTF8_with_bom, UTF16be, UTF16le, encodingExists, UTF8_BOM, toDecodeStream, toEncodeReadable, IDecodeStreamResult, detectEncodingByBOMFromBuffer } from 'vs/base/node/encoding'; import { WORKSPACE_EXTENSION } from 'vs/platform/workspaces/common/workspaces'; import { joinPath, extname, isEqualOrParent } from 'vs/base/common/resources'; import { Disposable } from 'vs/base/common/lifecycle'; import { IEnvironmentService } from 'vs/platform/environment/common/environment'; -import { VSBufferReadable, bufferToStream } from 'vs/base/common/buffer'; -import { Readable } from 'stream'; +import { bufferToStream, VSBufferReadable } from 'vs/base/common/buffer'; import { createTextBufferFactoryFromStream } from 'vs/editor/common/model/textModel'; import { ITextSnapshot } from 'vs/editor/common/model'; -import { nodeReadableToString, streamToNodeReadable, nodeStreamToVSBufferReadable } from 'vs/base/node/stream'; +import { consumeStream } from 'vs/base/common/stream'; import { IUntitledTextEditorService } from 'vs/workbench/services/untitled/common/untitledTextEditorService'; import { ILifecycleService } from 'vs/platform/lifecycle/common/lifecycle'; import { IInstantiationService } from 'vs/platform/instantiation/common/instantiation'; @@ -89,7 +88,7 @@ export class NativeTextFileService extends AbstractTextFileService { return { ...bufferStream, encoding: decoder.detected.encoding || UTF8, - value: await nodeReadableToString(decoder.stream) + value: await consumeStream(decoder.stream, strings => strings.join('')) }; } @@ -121,7 +120,7 @@ export class NativeTextFileService extends AbstractTextFileService { } // read through encoding library - const decoder = await toDecodeStream(streamToNodeReadable(bufferStream.value), { + const decoder = await toDecodeStream(bufferStream.value, { guessEncoding: options?.autoGuessEncoding || this.textResourceConfigurationService.getValue(resource, 'files.autoGuessEncoding'), overwriteEncoding: detectedEncoding => this.encoding.getReadEncoding(resource, options, detectedEncoding) }); @@ -232,37 +231,8 @@ export class NativeTextFileService extends AbstractTextFileService { } private getEncodedReadable(value: string | ITextSnapshot, encoding: string, addBOM: boolean): VSBufferReadable { - const readable = this.snapshotToNodeReadable(typeof value === 'string' ? stringToSnapshot(value) : value); - const encoder = encodeStream(encoding, { addBOM }); - - const encodedReadable = readable.pipe(encoder); - - return nodeStreamToVSBufferReadable(encodedReadable, addBOM && isUTFEncoding(encoding) ? { encoding } : undefined); - } - - private snapshotToNodeReadable(snapshot: ITextSnapshot): Readable { - return new Readable({ - read: function () { - try { - let chunk: string | null = null; - let canPush = true; - - // Push all chunks as long as we can push and as long as - // the underlying snapshot returns strings to us - while (canPush && typeof (chunk = snapshot.read()) === 'string') { - canPush = this.push(chunk); - } - - // Signal EOS by pushing NULL - if (typeof chunk !== 'string') { - this.push(null); - } - } catch (error) { - this.emit('error', error); - } - }, - encoding: UTF8 // very important, so that strings are passed around and not buffers! - }); + const snapshot = typeof value === 'string' ? stringToSnapshot(value) : value; + return toEncodeReadable(snapshot, encoding, { addBOM }); } private async writeElevated(resource: URI, value: string | ITextSnapshot, options?: IWriteTextFileOptions): Promise { diff --git a/yarn.lock b/yarn.lock index b14decdc251..02d22b42d28 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4579,10 +4579,10 @@ husky@^0.13.1: is-ci "^1.0.9" normalize-path "^1.0.0" -iconv-lite@0.5.0: - version "0.5.0" - resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.5.0.tgz#59cdde0a2a297cc2aeb0c6445a195ee89f127550" - integrity sha512-NnEhI9hIEKHOzJ4f697DMz9IQEXr/MMJ5w64vN2/4Ai+wRnvV7SBrL0KLoRlwaKVghOc7LQ5YkPLuX146b6Ydw== +iconv-lite@0.6.0: + version "0.6.0" + resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.0.tgz#66a93b80df0bd05d2a43a7426296b7f91073f125" + integrity sha512-43ZpGYZ9QtuutX5l6WC1DSO8ane9N+Ct5qPLF2OV7vM9abM69gnAbVkh66ibaZd3aOGkoP1ZmringlKhLBkw2Q== dependencies: safer-buffer ">= 2.1.2 < 3"