From 495112ded58c695aaa8c00115bdb5ea4b7d26876 Mon Sep 17 00:00:00 2001 From: Benjamin Pasero Date: Wed, 17 Jun 2020 09:09:59 +0200 Subject: [PATCH] Revert "decouple vs/base/node/encoding.ts from node streams for #79275 (#99413)" This reverts commit 1dbfecd37f0df32f7b3bde320a84a48db75fb7bd. --- build/package.json | 2 +- build/yarn.lock | 8 +- extensions/git/package.json | 2 +- extensions/git/yarn.lock | 8 +- package.json | 2 +- remote/package.json | 2 +- remote/yarn.lock | 8 +- src/vs/base/node/encoding.ts | 197 ++++++++---------- src/vs/base/node/stream.ts | 63 +++++- .../base/test/node/encoding/encoding.test.ts | 145 +++---------- .../electron-browser/nativeTextFileService.ts | 44 +++- yarn.lock | 8 +- 12 files changed, 244 insertions(+), 245 deletions(-) diff --git a/build/package.json b/build/package.json index 79862e3ce5d..3daba3dd482 100644 --- a/build/package.json +++ b/build/package.json @@ -38,7 +38,7 @@ "gulp-bom": "^1.0.0", "gulp-sourcemaps": "^1.11.0", "gulp-uglify": "^3.0.0", - "iconv-lite": "0.6.0", + "iconv-lite": "0.4.23", "mime": "^1.3.4", "minimatch": "3.0.4", "minimist": "^1.2.3", diff --git a/build/yarn.lock b/build/yarn.lock index 9bf1e1c7095..3f7d72a7438 100644 --- a/build/yarn.lock +++ b/build/yarn.lock @@ -1415,10 +1415,10 @@ http-signature@~1.2.0: jsprim "^1.2.2" sshpk "^1.7.0" -iconv-lite@0.6.0: - version "0.6.0" - resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.0.tgz#66a93b80df0bd05d2a43a7426296b7f91073f125" - integrity sha512-43ZpGYZ9QtuutX5l6WC1DSO8ane9N+Ct5qPLF2OV7vM9abM69gnAbVkh66ibaZd3aOGkoP1ZmringlKhLBkw2Q== +iconv-lite@0.4.23: + version "0.4.23" + resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.23.tgz#297871f63be507adcfbfca715d0cd0eed84e9a63" + integrity sha512-neyTUVFtahjf0mB3dZT77u+8O0QB89jFdnBkd5P1JgYPbPaia3gXXOVL2fq8VyU2gMMD7SaN7QukTB/pmXYvDA== dependencies: safer-buffer ">= 2.1.2 < 3" diff --git a/extensions/git/package.json b/extensions/git/package.json index 077e1d40600..2fc480eecaf 100644 --- a/extensions/git/package.json +++ b/extensions/git/package.json @@ -1878,7 +1878,7 @@ "dependencies": { "byline": "^5.0.0", "file-type": "^7.2.0", - "iconv-lite": "0.6.0", + "iconv-lite": "^0.4.24", "jschardet": "2.1.1", "vscode-extension-telemetry": "0.1.1", "vscode-nls": "^4.0.0", diff --git a/extensions/git/yarn.lock b/extensions/git/yarn.lock index f03bee0ad56..f72600de7bc 100644 --- a/extensions/git/yarn.lock +++ b/extensions/git/yarn.lock @@ -425,10 +425,10 @@ https-proxy-agent@^2.2.1: agent-base "^4.3.0" debug "^3.1.0" -iconv-lite@0.6.0: - version "0.6.0" - resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.0.tgz#66a93b80df0bd05d2a43a7426296b7f91073f125" - integrity sha512-43ZpGYZ9QtuutX5l6WC1DSO8ane9N+Ct5qPLF2OV7vM9abM69gnAbVkh66ibaZd3aOGkoP1ZmringlKhLBkw2Q== +iconv-lite@^0.4.24: + version "0.4.24" + resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.24.tgz#2022b4b25fbddc21d2f524974a474aafe733908b" + integrity sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA== dependencies: safer-buffer ">= 2.1.2 < 3" diff --git a/package.json b/package.json index 20f054a2a3f..458b3bffa52 100644 --- a/package.json +++ b/package.json @@ -41,7 +41,7 @@ "graceful-fs": "4.2.3", "http-proxy-agent": "^2.1.0", "https-proxy-agent": "^2.2.3", - "iconv-lite": "0.6.0", + "iconv-lite": "0.5.0", "jschardet": "2.1.1", "keytar": "^5.5.0", "minimist": "^1.2.5", diff --git a/remote/package.json b/remote/package.json index a1e1cd5d802..cba584d935d 100644 --- a/remote/package.json +++ b/remote/package.json @@ -8,7 +8,7 @@ "graceful-fs": "4.2.3", "http-proxy-agent": "^2.1.0", "https-proxy-agent": "^2.2.3", - "iconv-lite": "0.6.0", + "iconv-lite": "0.5.0", "jschardet": "2.1.1", "minimist": "^1.2.5", "native-watchdog": "1.3.0", diff --git a/remote/yarn.lock b/remote/yarn.lock index 4effe6f9fa0..7ba3b88854f 100644 --- a/remote/yarn.lock +++ b/remote/yarn.lock @@ -176,10 +176,10 @@ https-proxy-agent@^2.2.3: agent-base "^4.3.0" debug "^3.1.0" -iconv-lite@0.6.0: - version "0.6.0" - resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.0.tgz#66a93b80df0bd05d2a43a7426296b7f91073f125" - integrity sha512-43ZpGYZ9QtuutX5l6WC1DSO8ane9N+Ct5qPLF2OV7vM9abM69gnAbVkh66ibaZd3aOGkoP1ZmringlKhLBkw2Q== +iconv-lite@0.5.0: + version "0.5.0" + resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.5.0.tgz#59cdde0a2a297cc2aeb0c6445a195ee89f127550" + integrity sha512-NnEhI9hIEKHOzJ4f697DMz9IQEXr/MMJ5w64vN2/4Ai+wRnvV7SBrL0KLoRlwaKVghOc7LQ5YkPLuX146b6Ydw== dependencies: safer-buffer ">= 2.1.2 < 3" diff --git a/src/vs/base/node/encoding.ts b/src/vs/base/node/encoding.ts index a7034737930..ec3392a780e 100644 --- a/src/vs/base/node/encoding.ts +++ b/src/vs/base/node/encoding.ts @@ -4,9 +4,8 @@ *--------------------------------------------------------------------------------------------*/ import * as iconv from 'iconv-lite'; -import { Readable, ReadableStream, newWriteableStream } from 'vs/base/common/stream'; -import { isUndefinedOrNull, isUndefined, isNumber } from 'vs/base/common/types'; -import { VSBuffer, VSBufferReadable, VSBufferReadableStream } from 'vs/base/common/buffer'; +import { Readable, Writable } from 'stream'; +import { VSBuffer } from 'vs/base/common/buffer'; export const UTF8 = 'utf8'; export const UTF8_with_bom = 'utf8bom'; @@ -36,135 +35,121 @@ export interface IDecodeStreamOptions { } export interface IDecodeStreamResult { - stream: ReadableStream; + stream: NodeJS.ReadableStream; detected: IDetectedEncodingResult; } -export function toDecodeStream(source: VSBufferReadableStream, options: IDecodeStreamOptions): Promise { +export function toDecodeStream(readable: Readable, options: IDecodeStreamOptions): Promise { if (!options.minBytesRequiredForDetection) { options.minBytesRequiredForDetection = options.guessEncoding ? AUTO_ENCODING_GUESS_MIN_BYTES : NO_ENCODING_GUESS_MIN_BYTES; } return new Promise((resolve, reject) => { - const target = newWriteableStream(strings => strings.join('')); + const writer = new class extends Writable { + private decodeStream: NodeJS.ReadWriteStream | undefined; + private decodeStreamPromise: Promise | undefined; - const bufferedChunks: VSBuffer[] = []; - let bytesBuffered = 0; - let decoder: iconv.DecoderStream | null = null; + private bufferedChunks: Buffer[] = []; + private bytesBuffered = 0; + + _write(chunk: Buffer, encoding: string, callback: (error: Error | null | undefined) => void): void { + if (!Buffer.isBuffer(chunk)) { + return callback(new Error('toDecodeStream(): data must be a buffer')); + } + + // if the decode stream is ready, we just write directly + if (this.decodeStream) { + this.decodeStream.write(chunk, callback); + + return; + } + + // otherwise we need to buffer the data until the stream is ready + this.bufferedChunks.push(chunk); + this.bytesBuffered += chunk.byteLength; + + // waiting for the decoder to be ready + if (this.decodeStreamPromise) { + this.decodeStreamPromise.then(() => callback(null), error => callback(error)); + } + + // buffered enough data for encoding detection, create stream and forward data + else if (typeof options.minBytesRequiredForDetection === 'number' && this.bytesBuffered >= options.minBytesRequiredForDetection) { + this._startDecodeStream(callback); + } + + // only buffering until enough data for encoding detection is there + else { + callback(null); + } + } + + _startDecodeStream(callback: (error: Error | null | undefined) => void): void { + + // detect encoding from buffer + this.decodeStreamPromise = Promise.resolve(detectEncodingFromBuffer({ + buffer: Buffer.concat(this.bufferedChunks), + bytesRead: this.bytesBuffered + }, options.guessEncoding)).then(detected => { - const startDecodeStream = () => { - return Promise.resolve() - .then(() => - // detect encoding from buffer - detectEncodingFromBuffer({ - buffer: Buffer.from(VSBuffer.concat(bufferedChunks).buffer), - bytesRead: bytesBuffered - }, options.guessEncoding) - ) - .then(detected => { // ensure to respect overwrite of encoding detected.encoding = options.overwriteEncoding(detected.encoding); - // decode and write buffered content - decoder = iconv.getDecoder(toNodeEncoding(detected.encoding)); - const nodeBuffer = Buffer.from(VSBuffer.concat(bufferedChunks).buffer); - target.write(decoder.write(nodeBuffer)); - bufferedChunks.length = 0; + // decode and write buffer + this.decodeStream = decodeStream(detected.encoding); + this.decodeStream.write(Buffer.concat(this.bufferedChunks), callback); + this.bufferedChunks.length = 0; // signal to the outside our detected encoding // and final decoder stream - resolve({ - stream: target, - detected, - }); - }) - .catch(reject); - }; + resolve({ detected, stream: this.decodeStream }); + }, error => { + this.emit('error', error); - source.on('error', target.error); - source.on('data', (chunk) => { - // if the decoder is ready, we just write directly - if (!isUndefinedOrNull(decoder)) { - target.write(decoder.write(Buffer.from(chunk.buffer))); - return; - } - - // otherwise we need to buffer the data until the stream is ready - bufferedChunks.push(chunk); - bytesBuffered += chunk.byteLength; - - // buffered enough data for encoding detection, create stream and forward data - if (isNumber(options.minBytesRequiredForDetection) && bytesBuffered >= options.minBytesRequiredForDetection) { - startDecodeStream(); - } - }); - source.on('end', () => { - // normal finish - if (!isUndefinedOrNull(decoder)) { - target.end(decoder.end()); - } - - // we were still waiting for data to do the encoding - // detection. thus, wrap up starting the stream even - // without all the data to get things going - else { - startDecodeStream().then(() => { - target.end(decoder?.end()); + callback(error); }); } - }); + + _final(callback: () => void) { + + // normal finish + if (this.decodeStream) { + this.decodeStream.end(callback); + } + + // we were still waiting for data to do the encoding + // detection. thus, wrap up starting the stream even + // without all the data to get things going + else { + this._startDecodeStream(() => { + if (this.decodeStream) { + this.decodeStream.end(callback); + } + }); + } + } + }; + + // errors + readable.on('error', reject); + + // pipe through + readable.pipe(writer); }); } -export function toEncodeReadable(readable: Readable, encoding: string, options?: { addBOM?: boolean }): VSBufferReadable { - const encoder = iconv.getEncoder(toNodeEncoding(encoding), options); - let bytesRead = 0; - let done = false; - - return { - read() { - if (done) { - return null; - } - - const chunk = readable.read(); - if (isUndefinedOrNull(chunk)) { - done = true; - - // If we are instructed to add a BOM but we detect that no - // bytes have been read, we must ensure to return the BOM - // ourselves so that we comply with the contract. - if (bytesRead === 0 && options?.addBOM) { - switch (encoding) { - case UTF8: - case UTF8_with_bom: - return VSBuffer.wrap(Buffer.from(UTF8_BOM)); - case UTF16be: - return VSBuffer.wrap(Buffer.from(UTF16be_BOM)); - case UTF16le: - return VSBuffer.wrap(Buffer.from(UTF16le_BOM)); - } - } - - const leftovers = encoder.end(); - if (!isUndefined(leftovers) && leftovers.length > 0) { - return VSBuffer.wrap(leftovers); - } - - return null; - } - - bytesRead += chunk.length; - - return VSBuffer.wrap(encoder.write(chunk)); - } - }; -} - export function encodingExists(encoding: string): boolean { return iconv.encodingExists(toNodeEncoding(encoding)); } +function decodeStream(encoding: string | null): NodeJS.ReadWriteStream { + return iconv.decodeStream(toNodeEncoding(encoding)); +} + +export function encodeStream(encoding: string, options?: { addBOM?: boolean }): NodeJS.ReadWriteStream { + return iconv.encodeStream(toNodeEncoding(encoding), options); +} + export function toNodeEncoding(enc: string | null): string { if (enc === UTF8_with_bom || enc === null) { return UTF8; // iconv does not distinguish UTF 8 with or without BOM, so we need to help it diff --git a/src/vs/base/node/stream.ts b/src/vs/base/node/stream.ts index b280d2fb3ea..12ba5e5d929 100644 --- a/src/vs/base/node/stream.ts +++ b/src/vs/base/node/stream.ts @@ -3,8 +3,10 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { VSBufferReadableStream } from 'vs/base/common/buffer'; +import { VSBufferReadableStream, VSBufferReadable, VSBuffer } from 'vs/base/common/buffer'; import { Readable } from 'stream'; +import { isUndefinedOrNull } from 'vs/base/common/types'; +import { UTF8, UTF8_with_bom, UTF8_BOM, UTF16be, UTF16le_BOM, UTF16be_BOM, UTF16le, UTF_ENCODING } from 'vs/base/node/encoding'; export function streamToNodeReadable(stream: VSBufferReadableStream): Readable { return new class extends Readable { @@ -49,3 +51,62 @@ export function streamToNodeReadable(stream: VSBufferReadableStream): Readable { } }; } + +export function nodeReadableToString(stream: NodeJS.ReadableStream): Promise { + return new Promise((resolve, reject) => { + let result = ''; + + stream.on('data', chunk => result += chunk); + stream.on('error', reject); + stream.on('end', () => resolve(result)); + }); +} + +export function nodeStreamToVSBufferReadable(stream: NodeJS.ReadWriteStream, addBOM?: { encoding: UTF_ENCODING }): VSBufferReadable { + let bytesRead = 0; + let done = false; + + return { + read(): VSBuffer | null { + if (done) { + return null; + } + + const res = stream.read(); + if (isUndefinedOrNull(res)) { + done = true; + + // If we are instructed to add a BOM but we detect that no + // bytes have been read, we must ensure to return the BOM + // ourselves so that we comply with the contract. + if (bytesRead === 0 && addBOM) { + switch (addBOM.encoding) { + case UTF8: + case UTF8_with_bom: + return VSBuffer.wrap(Buffer.from(UTF8_BOM)); + case UTF16be: + return VSBuffer.wrap(Buffer.from(UTF16be_BOM)); + case UTF16le: + return VSBuffer.wrap(Buffer.from(UTF16le_BOM)); + } + } + + return null; + } + + // Handle String + if (typeof res === 'string') { + bytesRead += res.length; + + return VSBuffer.fromString(res); + } + + // Handle Buffer + else { + bytesRead += res.byteLength; + + return VSBuffer.wrap(res); + } + } + }; +} diff --git a/src/vs/base/test/node/encoding/encoding.test.ts b/src/vs/base/test/node/encoding/encoding.test.ts index 66eb9f3c013..024f7084d72 100644 --- a/src/vs/base/test/node/encoding/encoding.test.ts +++ b/src/vs/base/test/node/encoding/encoding.test.ts @@ -7,10 +7,9 @@ import * as assert from 'assert'; import * as fs from 'fs'; import * as encoding from 'vs/base/node/encoding'; import * as terminalEncoding from 'vs/base/node/terminalEncoding'; -import * as streams from 'vs/base/common/stream'; +import { Readable } from 'stream'; import * as iconv from 'iconv-lite'; import { getPathFromAmdModule } from 'vs/base/common/amd'; -import { newWriteableBufferStream, VSBuffer, VSBufferReadableStream, streamToBufferReadableStream } from 'vs/base/common/buffer'; export async function detectEncodingByBOM(file: string): Promise { try { @@ -232,32 +231,30 @@ suite('Encoding', () => { }); } - function newTestReadableStream(buffers: Buffer[]): VSBufferReadableStream { - const stream = newWriteableBufferStream(); - buffers - .map(VSBuffer.wrap) - .forEach(buffer => { - setTimeout(() => { - stream.write(buffer); - }); + async function readAllAsString(stream: NodeJS.ReadableStream) { + return new Promise((resolve, reject) => { + let all = ''; + stream.on('data', chunk => { + all += chunk; + assert.equal(typeof chunk, 'string'); }); - setTimeout(() => { - stream.end(); + stream.on('end', () => { + resolve(all); + }); + stream.on('error', reject); }); - return stream; - } - - async function readAllAsString(stream: streams.ReadableStream) { - return streams.consumeStream(stream, strings => strings.join('')); } test('toDecodeStream - some stream', async function () { - let source = newTestReadableStream([ - Buffer.from([65, 66, 67]), - Buffer.from([65, 66, 67]), - Buffer.from([65, 66, 67]), - ]); + let source = new Readable({ + read(size) { + this.push(Buffer.from([65, 66, 67])); + this.push(Buffer.from([65, 66, 67])); + this.push(Buffer.from([65, 66, 67])); + this.push(null); + } + }); let { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 4, guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 }); @@ -270,11 +267,14 @@ suite('Encoding', () => { test('toDecodeStream - some stream, expect too much data', async function () { - let source = newTestReadableStream([ - Buffer.from([65, 66, 67]), - Buffer.from([65, 66, 67]), - Buffer.from([65, 66, 67]), - ]); + let source = new Readable({ + read(size) { + this.push(Buffer.from([65, 66, 67])); + this.push(Buffer.from([65, 66, 67])); + this.push(Buffer.from([65, 66, 67])); + this.push(null); + } + }); let { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 64, guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 }); @@ -287,8 +287,11 @@ suite('Encoding', () => { test('toDecodeStream - some stream, no data', async function () { - let source = newWriteableBufferStream(); - source.end(); + let source = new Readable({ + read(size) { + this.push(null); // empty + } + }); let { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 512, guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 }); @@ -303,7 +306,7 @@ suite('Encoding', () => { test('toDecodeStream - encoding, utf16be', async function () { let path = getPathFromAmdModule(require, './fixtures/some_utf16be.css'); - let source = streamToBufferReadableStream(fs.createReadStream(path)); + let source = fs.createReadStream(path); let { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 64, guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 }); @@ -319,91 +322,11 @@ suite('Encoding', () => { test('toDecodeStream - empty file', async function () { let path = getPathFromAmdModule(require, './fixtures/empty.txt'); - let source = streamToBufferReadableStream(fs.createReadStream(path)); + let source = fs.createReadStream(path); let { detected, stream } = await encoding.toDecodeStream(source, { guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 }); let expected = await readAndDecodeFromDisk(path, detected.encoding); let actual = await readAllAsString(stream); assert.equal(actual, expected); }); - - test('toDecodeStream - decodes buffer entirely', async function () { - let emojis = Buffer.from('🖥️💻💾'); - let incompleteEmojis = emojis.slice(0, emojis.length - 1); - - let buffers = []; - for (let i = 0; i < incompleteEmojis.length; i++) { - buffers.push(incompleteEmojis.slice(i, i + 1)); - } - - const source = newTestReadableStream(buffers); - let { stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 4, guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 }); - - let expected = incompleteEmojis.toString(encoding.UTF8); - let actual = await readAllAsString(stream); - - assert.equal(actual, expected); - }); - - test('toEncodeReadable - encoding, utf16be', async function () { - - let path = getPathFromAmdModule(require, './fixtures/some_utf16be.css'); - let source = await readAndDecodeFromDisk(path, encoding.UTF16be); - - let expected = VSBuffer.wrap( - iconv.encode(source, encoding.toNodeEncoding(encoding.UTF16be)) - ).toString(); - let actual = streams.consumeReadable( - encoding.toEncodeReadable(streams.toReadable(source), encoding.UTF16be), - VSBuffer.concat - ).toString(); - - assert.equal(actual, expected); - }); - - test('toEncodeReadable - empty readable to utf8', async function () { - - const source: streams.Readable = { - read() { - return null; - } - }; - - let actual = streams.consumeReadable( - encoding.toEncodeReadable(source, encoding.UTF8), - VSBuffer.concat - ).toString(); - - assert.equal(actual, ''); - }); - - [{ - utfEncoding: encoding.UTF8, - relatedBom: encoding.UTF8_BOM - }, { - utfEncoding: encoding.UTF8_with_bom, - relatedBom: encoding.UTF8_BOM - }, { - utfEncoding: encoding.UTF16be, - relatedBom: encoding.UTF16be_BOM, - }, { - utfEncoding: encoding.UTF16le, - relatedBom: encoding.UTF16le_BOM - }].forEach(({ utfEncoding, relatedBom }) => { - test(`toEncodeReadable - empty readable to ${utfEncoding} with BOM`, async function () { - - const source: streams.Readable = { - read() { - return null; - } - }; - - let encodedReadable = encoding.toEncodeReadable(source, utfEncoding, { addBOM: true }); - - const expected = VSBuffer.wrap(Buffer.from(relatedBom)).toString(); - const actual = streams.consumeReadable(encodedReadable, VSBuffer.concat).toString(); - - assert.equal(actual, expected); - }); - }); }); diff --git a/src/vs/workbench/services/textfile/electron-browser/nativeTextFileService.ts b/src/vs/workbench/services/textfile/electron-browser/nativeTextFileService.ts index beecdcd2fbd..557622abf89 100644 --- a/src/vs/workbench/services/textfile/electron-browser/nativeTextFileService.ts +++ b/src/vs/workbench/services/textfile/electron-browser/nativeTextFileService.ts @@ -17,15 +17,16 @@ import { isMacintosh } from 'vs/base/common/platform'; import { IProductService } from 'vs/platform/product/common/productService'; import { ITextResourceConfigurationService } from 'vs/editor/common/services/textResourceConfigurationService'; import { IWorkspaceContextService } from 'vs/platform/workspace/common/workspace'; -import { UTF8, UTF8_with_bom, UTF16be, UTF16le, encodingExists, UTF8_BOM, toDecodeStream, toEncodeReadable, IDecodeStreamResult, detectEncodingByBOMFromBuffer } from 'vs/base/node/encoding'; +import { UTF8, UTF8_with_bom, UTF16be, UTF16le, encodingExists, encodeStream, UTF8_BOM, toDecodeStream, IDecodeStreamResult, detectEncodingByBOMFromBuffer, isUTFEncoding } from 'vs/base/node/encoding'; import { WORKSPACE_EXTENSION } from 'vs/platform/workspaces/common/workspaces'; import { joinPath, extname, isEqualOrParent } from 'vs/base/common/resources'; import { Disposable } from 'vs/base/common/lifecycle'; import { IEnvironmentService } from 'vs/platform/environment/common/environment'; -import { bufferToStream, VSBufferReadable } from 'vs/base/common/buffer'; +import { VSBufferReadable, bufferToStream } from 'vs/base/common/buffer'; +import { Readable } from 'stream'; import { createTextBufferFactoryFromStream } from 'vs/editor/common/model/textModel'; import { ITextSnapshot } from 'vs/editor/common/model'; -import { consumeStream } from 'vs/base/common/stream'; +import { nodeReadableToString, streamToNodeReadable, nodeStreamToVSBufferReadable } from 'vs/base/node/stream'; import { IUntitledTextEditorService } from 'vs/workbench/services/untitled/common/untitledTextEditorService'; import { ILifecycleService } from 'vs/platform/lifecycle/common/lifecycle'; import { IInstantiationService } from 'vs/platform/instantiation/common/instantiation'; @@ -88,7 +89,7 @@ export class NativeTextFileService extends AbstractTextFileService { return { ...bufferStream, encoding: decoder.detected.encoding || UTF8, - value: await consumeStream(decoder.stream, strings => strings.join('')) + value: await nodeReadableToString(decoder.stream) }; } @@ -120,7 +121,7 @@ export class NativeTextFileService extends AbstractTextFileService { } // read through encoding library - const decoder = await toDecodeStream(bufferStream.value, { + const decoder = await toDecodeStream(streamToNodeReadable(bufferStream.value), { guessEncoding: options?.autoGuessEncoding || this.textResourceConfigurationService.getValue(resource, 'files.autoGuessEncoding'), overwriteEncoding: detectedEncoding => this.encoding.getReadEncoding(resource, options, detectedEncoding) }); @@ -231,8 +232,37 @@ export class NativeTextFileService extends AbstractTextFileService { } private getEncodedReadable(value: string | ITextSnapshot, encoding: string, addBOM: boolean): VSBufferReadable { - const snapshot = typeof value === 'string' ? stringToSnapshot(value) : value; - return toEncodeReadable(snapshot, encoding, { addBOM }); + const readable = this.snapshotToNodeReadable(typeof value === 'string' ? stringToSnapshot(value) : value); + const encoder = encodeStream(encoding, { addBOM }); + + const encodedReadable = readable.pipe(encoder); + + return nodeStreamToVSBufferReadable(encodedReadable, addBOM && isUTFEncoding(encoding) ? { encoding } : undefined); + } + + private snapshotToNodeReadable(snapshot: ITextSnapshot): Readable { + return new Readable({ + read: function () { + try { + let chunk: string | null = null; + let canPush = true; + + // Push all chunks as long as we can push and as long as + // the underlying snapshot returns strings to us + while (canPush && typeof (chunk = snapshot.read()) === 'string') { + canPush = this.push(chunk); + } + + // Signal EOS by pushing NULL + if (typeof chunk !== 'string') { + this.push(null); + } + } catch (error) { + this.emit('error', error); + } + }, + encoding: UTF8 // very important, so that strings are passed around and not buffers! + }); } private async writeElevated(resource: URI, value: string | ITextSnapshot, options?: IWriteTextFileOptions): Promise { diff --git a/yarn.lock b/yarn.lock index 02d22b42d28..b14decdc251 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4579,10 +4579,10 @@ husky@^0.13.1: is-ci "^1.0.9" normalize-path "^1.0.0" -iconv-lite@0.6.0: - version "0.6.0" - resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.0.tgz#66a93b80df0bd05d2a43a7426296b7f91073f125" - integrity sha512-43ZpGYZ9QtuutX5l6WC1DSO8ane9N+Ct5qPLF2OV7vM9abM69gnAbVkh66ibaZd3aOGkoP1ZmringlKhLBkw2Q== +iconv-lite@0.5.0: + version "0.5.0" + resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.5.0.tgz#59cdde0a2a297cc2aeb0c6445a195ee89f127550" + integrity sha512-NnEhI9hIEKHOzJ4f697DMz9IQEXr/MMJ5w64vN2/4Ai+wRnvV7SBrL0KLoRlwaKVghOc7LQ5YkPLuX146b6Ydw== dependencies: safer-buffer ">= 2.1.2 < 3"