Revert "decouple vs/base/node/encoding.ts from node streams for #79275 (#99413)"

This reverts commit 1dbfecd37f.

build/package.json

@@ -38,7 +38,7 @@
     "gulp-bom": "^1.0.0",
     "gulp-sourcemaps": "^1.11.0",
     "gulp-uglify": "^3.0.0",
-    "iconv-lite": "0.6.0",
+    "iconv-lite": "0.4.23",
     "mime": "^1.3.4",
     "minimatch": "3.0.4",
     "minimist": "^1.2.3",

build/yarn.lock

@@ -1415,10 +1415,10 @@ http-signature@~1.2.0:
     jsprim "^1.2.2"
     sshpk "^1.7.0"
 
-iconv-lite@0.6.0:
+iconv-lite@0.4.23:
-  version "0.6.0"
+  version "0.4.23"
-  resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.0.tgz#66a93b80df0bd05d2a43a7426296b7f91073f125"
+  resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.23.tgz#297871f63be507adcfbfca715d0cd0eed84e9a63"
-  integrity sha512-43ZpGYZ9QtuutX5l6WC1DSO8ane9N+Ct5qPLF2OV7vM9abM69gnAbVkh66ibaZd3aOGkoP1ZmringlKhLBkw2Q==
+  integrity sha512-neyTUVFtahjf0mB3dZT77u+8O0QB89jFdnBkd5P1JgYPbPaia3gXXOVL2fq8VyU2gMMD7SaN7QukTB/pmXYvDA==
   dependencies:
     safer-buffer ">= 2.1.2 < 3"
 

extensions/git/package.json

@@ -1878,7 +1878,7 @@
   "dependencies": {
     "byline": "^5.0.0",
     "file-type": "^7.2.0",
-    "iconv-lite": "0.6.0",
+    "iconv-lite": "^0.4.24",
     "jschardet": "2.1.1",
     "vscode-extension-telemetry": "0.1.1",
     "vscode-nls": "^4.0.0",

extensions/git/yarn.lock

@@ -425,10 +425,10 @@ https-proxy-agent@^2.2.1:
     agent-base "^4.3.0"
     debug "^3.1.0"
 
-iconv-lite@0.6.0:
+iconv-lite@^0.4.24:
-  version "0.6.0"
+  version "0.4.24"
-  resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.0.tgz#66a93b80df0bd05d2a43a7426296b7f91073f125"
+  resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.24.tgz#2022b4b25fbddc21d2f524974a474aafe733908b"
-  integrity sha512-43ZpGYZ9QtuutX5l6WC1DSO8ane9N+Ct5qPLF2OV7vM9abM69gnAbVkh66ibaZd3aOGkoP1ZmringlKhLBkw2Q==
+  integrity sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==
   dependencies:
     safer-buffer ">= 2.1.2 < 3"
 

package.json

@@ -41,7 +41,7 @@
     "graceful-fs": "4.2.3",
     "http-proxy-agent": "^2.1.0",
     "https-proxy-agent": "^2.2.3",
-    "iconv-lite": "0.6.0",
+    "iconv-lite": "0.5.0",
     "jschardet": "2.1.1",
     "keytar": "^5.5.0",
     "minimist": "^1.2.5",

remote/package.json

@@ -8,7 +8,7 @@
     "graceful-fs": "4.2.3",
     "http-proxy-agent": "^2.1.0",
     "https-proxy-agent": "^2.2.3",
-    "iconv-lite": "0.6.0",
+    "iconv-lite": "0.5.0",
     "jschardet": "2.1.1",
     "minimist": "^1.2.5",
     "native-watchdog": "1.3.0",

remote/yarn.lock

@@ -176,10 +176,10 @@ https-proxy-agent@^2.2.3:
     agent-base "^4.3.0"
     debug "^3.1.0"
 
-iconv-lite@0.6.0:
+iconv-lite@0.5.0:
-  version "0.6.0"
+  version "0.5.0"
-  resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.0.tgz#66a93b80df0bd05d2a43a7426296b7f91073f125"
+  resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.5.0.tgz#59cdde0a2a297cc2aeb0c6445a195ee89f127550"
-  integrity sha512-43ZpGYZ9QtuutX5l6WC1DSO8ane9N+Ct5qPLF2OV7vM9abM69gnAbVkh66ibaZd3aOGkoP1ZmringlKhLBkw2Q==
+  integrity sha512-NnEhI9hIEKHOzJ4f697DMz9IQEXr/MMJ5w64vN2/4Ai+wRnvV7SBrL0KLoRlwaKVghOc7LQ5YkPLuX146b6Ydw==
   dependencies:
     safer-buffer ">= 2.1.2 < 3"
 

src/vs/base/node/encoding.ts

@@ -4,9 +4,8 @@
  *--------------------------------------------------------------------------------------------*/
 
 import * as iconv from 'iconv-lite';
-import { Readable, ReadableStream, newWriteableStream } from 'vs/base/common/stream';
+import { Readable, Writable } from 'stream';
-import { isUndefinedOrNull, isUndefined, isNumber } from 'vs/base/common/types';
+import { VSBuffer } from 'vs/base/common/buffer';
-import { VSBuffer, VSBufferReadable, VSBufferReadableStream } from 'vs/base/common/buffer';
 
 export const UTF8 = 'utf8';
 export const UTF8_with_bom = 'utf8bom';
@@ -36,135 +35,121 @@ export interface IDecodeStreamOptions {
 }
 
 export interface IDecodeStreamResult {
-	stream: ReadableStream<string>;
+	stream: NodeJS.ReadableStream;
 	detected: IDetectedEncodingResult;
 }
 
-export function toDecodeStream(source: VSBufferReadableStream, options: IDecodeStreamOptions): Promise<IDecodeStreamResult> {
+export function toDecodeStream(readable: Readable, options: IDecodeStreamOptions): Promise<IDecodeStreamResult> {
 	if (!options.minBytesRequiredForDetection) {
 		options.minBytesRequiredForDetection = options.guessEncoding ? AUTO_ENCODING_GUESS_MIN_BYTES : NO_ENCODING_GUESS_MIN_BYTES;
 	}
 
 	return new Promise<IDecodeStreamResult>((resolve, reject) => {
-		const target = newWriteableStream<string>(strings => strings.join(''));
+		const writer = new class extends Writable {
+			private decodeStream: NodeJS.ReadWriteStream | undefined;
+			private decodeStreamPromise: Promise<void> | undefined;
 
-		const bufferedChunks: VSBuffer[] = [];
+			private bufferedChunks: Buffer[] = [];
-		let bytesBuffered = 0;
+			private bytesBuffered = 0;
-		let decoder: iconv.DecoderStream | null = null;
+
+			_write(chunk: Buffer, encoding: string, callback: (error: Error | null | undefined) => void): void {
+				if (!Buffer.isBuffer(chunk)) {
+					return callback(new Error('toDecodeStream(): data must be a buffer'));
+				}
+
+				// if the decode stream is ready, we just write directly
+				if (this.decodeStream) {
+					this.decodeStream.write(chunk, callback);
+
+					return;
+				}
+
+				// otherwise we need to buffer the data until the stream is ready
+				this.bufferedChunks.push(chunk);
+				this.bytesBuffered += chunk.byteLength;
+
+				// waiting for the decoder to be ready
+				if (this.decodeStreamPromise) {
+					this.decodeStreamPromise.then(() => callback(null), error => callback(error));
+				}
+
+				// buffered enough data for encoding detection, create stream and forward data
+				else if (typeof options.minBytesRequiredForDetection === 'number' && this.bytesBuffered >= options.minBytesRequiredForDetection) {
+					this._startDecodeStream(callback);
+				}
+
+				// only buffering until enough data for encoding detection is there
+				else {
+					callback(null);
+				}
+			}
+
+			_startDecodeStream(callback: (error: Error | null | undefined) => void): void {
+
+				// detect encoding from buffer
+				this.decodeStreamPromise = Promise.resolve(detectEncodingFromBuffer({
+					buffer: Buffer.concat(this.bufferedChunks),
+					bytesRead: this.bytesBuffered
+				}, options.guessEncoding)).then(detected => {
 
-		const startDecodeStream = () => {
-			return Promise.resolve()
-				.then(() =>
-					// detect encoding from buffer
-					detectEncodingFromBuffer({
-						buffer: Buffer.from(VSBuffer.concat(bufferedChunks).buffer),
-						bytesRead: bytesBuffered
-					}, options.guessEncoding)
-				)
-				.then(detected => {
 					// ensure to respect overwrite of encoding
 					detected.encoding = options.overwriteEncoding(detected.encoding);
 
-					// decode and write buffered content
+					// decode and write buffer
-					decoder = iconv.getDecoder(toNodeEncoding(detected.encoding));
+					this.decodeStream = decodeStream(detected.encoding);
-					const nodeBuffer = Buffer.from(VSBuffer.concat(bufferedChunks).buffer);
+					this.decodeStream.write(Buffer.concat(this.bufferedChunks), callback);
-					target.write(decoder.write(nodeBuffer));
+					this.bufferedChunks.length = 0;
-					bufferedChunks.length = 0;
 
 					// signal to the outside our detected encoding
 					// and final decoder stream
-					resolve({
+					resolve({ detected, stream: this.decodeStream });
-						stream: target,
+				}, error => {
-						detected,
+					this.emit('error', error);
-					});
-				})
-				.catch(reject);
-		};
 
-		source.on('error', target.error);
+					callback(error);
-		source.on('data', (chunk) => {
-			// if the decoder is ready, we just write directly
-			if (!isUndefinedOrNull(decoder)) {
-				target.write(decoder.write(Buffer.from(chunk.buffer)));
-				return;
-			}
-
-			// otherwise we need to buffer the data until the stream is ready
-			bufferedChunks.push(chunk);
-			bytesBuffered += chunk.byteLength;
-
-			// buffered enough data for encoding detection, create stream and forward data
-			if (isNumber(options.minBytesRequiredForDetection) && bytesBuffered >= options.minBytesRequiredForDetection) {
-				startDecodeStream();
-			}
-		});
-		source.on('end', () => {
-			// normal finish
-			if (!isUndefinedOrNull(decoder)) {
-				target.end(decoder.end());
-			}
-
-			// we were still waiting for data to do the encoding
-			// detection. thus, wrap up starting the stream even
-			// without all the data to get things going
-			else {
-				startDecodeStream().then(() => {
-					target.end(decoder?.end());
 				});
 			}
-		});
+
+			_final(callback: () => void) {
+
+				// normal finish
+				if (this.decodeStream) {
+					this.decodeStream.end(callback);
+				}
+
+				// we were still waiting for data to do the encoding
+				// detection. thus, wrap up starting the stream even
+				// without all the data to get things going
+				else {
+					this._startDecodeStream(() => {
+						if (this.decodeStream) {
+							this.decodeStream.end(callback);
+						}
+					});
+				}
+			}
+		};
+
+		// errors
+		readable.on('error', reject);
+
+		// pipe through
+		readable.pipe(writer);
 	});
 }
 
-export function toEncodeReadable(readable: Readable<string>, encoding: string, options?: { addBOM?: boolean }): VSBufferReadable {
-	const encoder = iconv.getEncoder(toNodeEncoding(encoding), options);
-	let bytesRead = 0;
-	let done = false;
-
-	return {
-		read() {
-			if (done) {
-				return null;
-			}
-
-			const chunk = readable.read();
-			if (isUndefinedOrNull(chunk)) {
-				done = true;
-
-				// If we are instructed to add a BOM but we detect that no
-				// bytes have been read, we must ensure to return the BOM
-				// ourselves so that we comply with the contract.
-				if (bytesRead === 0 && options?.addBOM) {
-					switch (encoding) {
-						case UTF8:
-						case UTF8_with_bom:
-							return VSBuffer.wrap(Buffer.from(UTF8_BOM));
-						case UTF16be:
-							return VSBuffer.wrap(Buffer.from(UTF16be_BOM));
-						case UTF16le:
-							return VSBuffer.wrap(Buffer.from(UTF16le_BOM));
-					}
-				}
-
-				const leftovers = encoder.end();
-				if (!isUndefined(leftovers) && leftovers.length > 0) {
-					return VSBuffer.wrap(leftovers);
-				}
-
-				return null;
-			}
-
-			bytesRead += chunk.length;
-
-			return VSBuffer.wrap(encoder.write(chunk));
-		}
-	};
-}
-
 export function encodingExists(encoding: string): boolean {
 	return iconv.encodingExists(toNodeEncoding(encoding));
 }
 
+function decodeStream(encoding: string | null): NodeJS.ReadWriteStream {
+	return iconv.decodeStream(toNodeEncoding(encoding));
+}
+
+export function encodeStream(encoding: string, options?: { addBOM?: boolean }): NodeJS.ReadWriteStream {
+	return iconv.encodeStream(toNodeEncoding(encoding), options);
+}
+
 export function toNodeEncoding(enc: string | null): string {
 	if (enc === UTF8_with_bom || enc === null) {
 		return UTF8; // iconv does not distinguish UTF 8 with or without BOM, so we need to help it

src/vs/base/node/stream.ts

@@ -3,8 +3,10 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-import { VSBufferReadableStream } from 'vs/base/common/buffer';
+import { VSBufferReadableStream, VSBufferReadable, VSBuffer } from 'vs/base/common/buffer';
 import { Readable } from 'stream';
+import { isUndefinedOrNull } from 'vs/base/common/types';
+import { UTF8, UTF8_with_bom, UTF8_BOM, UTF16be, UTF16le_BOM, UTF16be_BOM, UTF16le, UTF_ENCODING } from 'vs/base/node/encoding';
 
 export function streamToNodeReadable(stream: VSBufferReadableStream): Readable {
 	return new class extends Readable {
@@ -49,3 +51,62 @@ export function streamToNodeReadable(stream: VSBufferReadableStream): Readable {
 		}
 	};
 }
+
+export function nodeReadableToString(stream: NodeJS.ReadableStream): Promise<string> {
+	return new Promise((resolve, reject) => {
+		let result = '';
+
+		stream.on('data', chunk => result += chunk);
+		stream.on('error', reject);
+		stream.on('end', () => resolve(result));
+	});
+}
+
+export function nodeStreamToVSBufferReadable(stream: NodeJS.ReadWriteStream, addBOM?: { encoding: UTF_ENCODING }): VSBufferReadable {
+	let bytesRead = 0;
+	let done = false;
+
+	return {
+		read(): VSBuffer | null {
+			if (done) {
+				return null;
+			}
+
+			const res = stream.read();
+			if (isUndefinedOrNull(res)) {
+				done = true;
+
+				// If we are instructed to add a BOM but we detect that no
+				// bytes have been read, we must ensure to return the BOM
+				// ourselves so that we comply with the contract.
+				if (bytesRead === 0 && addBOM) {
+					switch (addBOM.encoding) {
+						case UTF8:
+						case UTF8_with_bom:
+							return VSBuffer.wrap(Buffer.from(UTF8_BOM));
+						case UTF16be:
+							return VSBuffer.wrap(Buffer.from(UTF16be_BOM));
+						case UTF16le:
+							return VSBuffer.wrap(Buffer.from(UTF16le_BOM));
+					}
+				}
+
+				return null;
+			}
+
+			// Handle String
+			if (typeof res === 'string') {
+				bytesRead += res.length;
+
+				return VSBuffer.fromString(res);
+			}
+
+			// Handle Buffer
+			else {
+				bytesRead += res.byteLength;
+
+				return VSBuffer.wrap(res);
+			}
+		}
+	};
+}

src/vs/base/test/node/encoding/encoding.test.ts

@@ -7,10 +7,9 @@ import * as assert from 'assert';
 import * as fs from 'fs';
 import * as encoding from 'vs/base/node/encoding';
 import * as terminalEncoding from 'vs/base/node/terminalEncoding';
-import * as streams from 'vs/base/common/stream';
+import { Readable } from 'stream';
 import * as iconv from 'iconv-lite';
 import { getPathFromAmdModule } from 'vs/base/common/amd';
-import { newWriteableBufferStream, VSBuffer, VSBufferReadableStream, streamToBufferReadableStream } from 'vs/base/common/buffer';
 
 export async function detectEncodingByBOM(file: string): Promise<typeof encoding.UTF16be | typeof encoding.UTF16le | typeof encoding.UTF8_with_bom | null> {
 	try {
@@ -232,32 +231,30 @@ suite('Encoding', () => {
 		});
 	}
 
-	function newTestReadableStream(buffers: Buffer[]): VSBufferReadableStream {
+	async function readAllAsString(stream: NodeJS.ReadableStream) {
-		const stream = newWriteableBufferStream();
+		return new Promise<string>((resolve, reject) => {
-		buffers
+			let all = '';
-			.map(VSBuffer.wrap)
+			stream.on('data', chunk => {
-			.forEach(buffer => {
+				all += chunk;
-				setTimeout(() => {
+				assert.equal(typeof chunk, 'string');
-					stream.write(buffer);
-				});
 			});
-		setTimeout(() => {
+			stream.on('end', () => {
-			stream.end();
+				resolve(all);
+			});
+			stream.on('error', reject);
 		});
-		return stream;
-	}
-
-	async function readAllAsString(stream: streams.ReadableStream<string>) {
-		return streams.consumeStream(stream, strings => strings.join(''));
 	}
 
 	test('toDecodeStream - some stream', async function () {
 
-		let source = newTestReadableStream([
+		let source = new Readable({
-			Buffer.from([65, 66, 67]),
+			read(size) {
-			Buffer.from([65, 66, 67]),
+				this.push(Buffer.from([65, 66, 67]));
-			Buffer.from([65, 66, 67]),
+				this.push(Buffer.from([65, 66, 67]));
-		]);
+				this.push(Buffer.from([65, 66, 67]));
+				this.push(null);
+			}
+		});
 
 		let { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 4, guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 });
 
@@ -270,11 +267,14 @@ suite('Encoding', () => {
 
 	test('toDecodeStream - some stream, expect too much data', async function () {
 
-		let source = newTestReadableStream([
+		let source = new Readable({
-			Buffer.from([65, 66, 67]),
+			read(size) {
-			Buffer.from([65, 66, 67]),
+				this.push(Buffer.from([65, 66, 67]));
-			Buffer.from([65, 66, 67]),
+				this.push(Buffer.from([65, 66, 67]));
-		]);
+				this.push(Buffer.from([65, 66, 67]));
+				this.push(null);
+			}
+		});
 
 		let { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 64, guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 });
 
@@ -287,8 +287,11 @@ suite('Encoding', () => {
 
 	test('toDecodeStream - some stream, no data', async function () {
 
-		let source = newWriteableBufferStream();
+		let source = new Readable({
-		source.end();
+			read(size) {
+				this.push(null); // empty
+			}
+		});
 
 		let { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 512, guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 });
 
@@ -303,7 +306,7 @@ suite('Encoding', () => {
 	test('toDecodeStream - encoding, utf16be', async function () {
 
 		let path = getPathFromAmdModule(require, './fixtures/some_utf16be.css');
-		let source = streamToBufferReadableStream(fs.createReadStream(path));
+		let source = fs.createReadStream(path);
 
 		let { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 64, guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 });
 
@@ -319,91 +322,11 @@ suite('Encoding', () => {
 	test('toDecodeStream - empty file', async function () {
 
 		let path = getPathFromAmdModule(require, './fixtures/empty.txt');
-		let source = streamToBufferReadableStream(fs.createReadStream(path));
+		let source = fs.createReadStream(path);
 		let { detected, stream } = await encoding.toDecodeStream(source, { guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 });
 
 		let expected = await readAndDecodeFromDisk(path, detected.encoding);
 		let actual = await readAllAsString(stream);
 		assert.equal(actual, expected);
 	});
-
-	test('toDecodeStream - decodes buffer entirely', async function () {
-		let emojis = Buffer.from('🖥️💻💾');
-		let incompleteEmojis = emojis.slice(0, emojis.length - 1);
-
-		let buffers = [];
-		for (let i = 0; i < incompleteEmojis.length; i++) {
-			buffers.push(incompleteEmojis.slice(i, i + 1));
-		}
-
-		const source = newTestReadableStream(buffers);
-		let { stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 4, guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 });
-
-		let expected = incompleteEmojis.toString(encoding.UTF8);
-		let actual = await readAllAsString(stream);
-
-		assert.equal(actual, expected);
-	});
-
-	test('toEncodeReadable - encoding, utf16be', async function () {
-
-		let path = getPathFromAmdModule(require, './fixtures/some_utf16be.css');
-		let source = await readAndDecodeFromDisk(path, encoding.UTF16be);
-
-		let expected = VSBuffer.wrap(
-			iconv.encode(source, encoding.toNodeEncoding(encoding.UTF16be))
-		).toString();
-		let actual = streams.consumeReadable(
-			encoding.toEncodeReadable(streams.toReadable(source), encoding.UTF16be),
-			VSBuffer.concat
-		).toString();
-
-		assert.equal(actual, expected);
-	});
-
-	test('toEncodeReadable - empty readable to utf8', async function () {
-
-		const source: streams.Readable<string> = {
-			read() {
-				return null;
-			}
-		};
-
-		let actual = streams.consumeReadable(
-			encoding.toEncodeReadable(source, encoding.UTF8),
-			VSBuffer.concat
-		).toString();
-
-		assert.equal(actual, '');
-	});
-
-	[{
-		utfEncoding: encoding.UTF8,
-		relatedBom: encoding.UTF8_BOM
-	}, {
-		utfEncoding: encoding.UTF8_with_bom,
-		relatedBom: encoding.UTF8_BOM
-	}, {
-		utfEncoding: encoding.UTF16be,
-		relatedBom: encoding.UTF16be_BOM,
-	}, {
-		utfEncoding: encoding.UTF16le,
-		relatedBom: encoding.UTF16le_BOM
-	}].forEach(({ utfEncoding, relatedBom }) => {
-		test(`toEncodeReadable - empty readable to ${utfEncoding} with BOM`, async function () {
-
-			const source: streams.Readable<string> = {
-				read() {
-					return null;
-				}
-			};
-
-			let encodedReadable = encoding.toEncodeReadable(source, utfEncoding, { addBOM: true });
-
-			const expected = VSBuffer.wrap(Buffer.from(relatedBom)).toString();
-			const actual = streams.consumeReadable(encodedReadable, VSBuffer.concat).toString();
-
-			assert.equal(actual, expected);
-		});
-	});
 });

src/vs/workbench/services/textfile/electron-browser/nativeTextFileService.ts

@@ -17,15 +17,16 @@ import { isMacintosh } from 'vs/base/common/platform';
 import { IProductService } from 'vs/platform/product/common/productService';
 import { ITextResourceConfigurationService } from 'vs/editor/common/services/textResourceConfigurationService';
 import { IWorkspaceContextService } from 'vs/platform/workspace/common/workspace';
-import { UTF8, UTF8_with_bom, UTF16be, UTF16le, encodingExists, UTF8_BOM, toDecodeStream, toEncodeReadable, IDecodeStreamResult, detectEncodingByBOMFromBuffer } from 'vs/base/node/encoding';
+import { UTF8, UTF8_with_bom, UTF16be, UTF16le, encodingExists, encodeStream, UTF8_BOM, toDecodeStream, IDecodeStreamResult, detectEncodingByBOMFromBuffer, isUTFEncoding } from 'vs/base/node/encoding';
 import { WORKSPACE_EXTENSION } from 'vs/platform/workspaces/common/workspaces';
 import { joinPath, extname, isEqualOrParent } from 'vs/base/common/resources';
 import { Disposable } from 'vs/base/common/lifecycle';
 import { IEnvironmentService } from 'vs/platform/environment/common/environment';
-import { bufferToStream, VSBufferReadable } from 'vs/base/common/buffer';
+import { VSBufferReadable, bufferToStream } from 'vs/base/common/buffer';
+import { Readable } from 'stream';
 import { createTextBufferFactoryFromStream } from 'vs/editor/common/model/textModel';
 import { ITextSnapshot } from 'vs/editor/common/model';
-import { consumeStream } from 'vs/base/common/stream';
+import { nodeReadableToString, streamToNodeReadable, nodeStreamToVSBufferReadable } from 'vs/base/node/stream';
 import { IUntitledTextEditorService } from 'vs/workbench/services/untitled/common/untitledTextEditorService';
 import { ILifecycleService } from 'vs/platform/lifecycle/common/lifecycle';
 import { IInstantiationService } from 'vs/platform/instantiation/common/instantiation';
@@ -88,7 +89,7 @@ export class NativeTextFileService extends AbstractTextFileService {
 		return {
 			...bufferStream,
 			encoding: decoder.detected.encoding || UTF8,
-			value: await consumeStream(decoder.stream, strings => strings.join(''))
+			value: await nodeReadableToString(decoder.stream)
 		};
 	}
 
@@ -120,7 +121,7 @@ export class NativeTextFileService extends AbstractTextFileService {
 		}
 
 		// read through encoding library
-		const decoder = await toDecodeStream(bufferStream.value, {
+		const decoder = await toDecodeStream(streamToNodeReadable(bufferStream.value), {
 			guessEncoding: options?.autoGuessEncoding || this.textResourceConfigurationService.getValue(resource, 'files.autoGuessEncoding'),
 			overwriteEncoding: detectedEncoding => this.encoding.getReadEncoding(resource, options, detectedEncoding)
 		});
@@ -231,8 +232,37 @@ export class NativeTextFileService extends AbstractTextFileService {
 	}
 
 	private getEncodedReadable(value: string | ITextSnapshot, encoding: string, addBOM: boolean): VSBufferReadable {
-		const snapshot = typeof value === 'string' ? stringToSnapshot(value) : value;
+		const readable = this.snapshotToNodeReadable(typeof value === 'string' ? stringToSnapshot(value) : value);
-		return toEncodeReadable(snapshot, encoding, { addBOM });
+		const encoder = encodeStream(encoding, { addBOM });
+
+		const encodedReadable = readable.pipe(encoder);
+
+		return nodeStreamToVSBufferReadable(encodedReadable, addBOM && isUTFEncoding(encoding) ? { encoding } : undefined);
+	}
+
+	private snapshotToNodeReadable(snapshot: ITextSnapshot): Readable {
+		return new Readable({
+			read: function () {
+				try {
+					let chunk: string | null = null;
+					let canPush = true;
+
+					// Push all chunks as long as we can push and as long as
+					// the underlying snapshot returns strings to us
+					while (canPush && typeof (chunk = snapshot.read()) === 'string') {
+						canPush = this.push(chunk);
+					}
+
+					// Signal EOS by pushing NULL
+					if (typeof chunk !== 'string') {
+						this.push(null);
+					}
+				} catch (error) {
+					this.emit('error', error);
+				}
+			},
+			encoding: UTF8 // very important, so that strings are passed around and not buffers!
+		});
 	}
 
 	private async writeElevated(resource: URI, value: string | ITextSnapshot, options?: IWriteTextFileOptions): Promise<IFileStatWithMetadata> {

yarn.lock

@@ -4579,10 +4579,10 @@ husky@^0.13.1:
     is-ci "^1.0.9"
     normalize-path "^1.0.0"
 
-iconv-lite@0.6.0:
+iconv-lite@0.5.0:
-  version "0.6.0"
+  version "0.5.0"
-  resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.0.tgz#66a93b80df0bd05d2a43a7426296b7f91073f125"
+  resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.5.0.tgz#59cdde0a2a297cc2aeb0c6445a195ee89f127550"
-  integrity sha512-43ZpGYZ9QtuutX5l6WC1DSO8ane9N+Ct5qPLF2OV7vM9abM69gnAbVkh66ibaZd3aOGkoP1ZmringlKhLBkw2Q==
+  integrity sha512-NnEhI9hIEKHOzJ4f697DMz9IQEXr/MMJ5w64vN2/4Ai+wRnvV7SBrL0KLoRlwaKVghOc7LQ5YkPLuX146b6Ydw==
   dependencies:
     safer-buffer ">= 2.1.2 < 3"