development/vscode
development/vscode
1
0
Fork 0
mirror of https://github.com/Microsoft/vscode synced 2024-08-28 21:39:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

decouple vs/base/node/encoding.ts from node streams for #79275 (#99413)

Browse source
This commit is contained in:
Fedor Nezhivoi 2020-06-17 17:09:12 +10:00 committed by GitHub
parent e4a9ccdbdd
commit 1dbfecd37f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 244 additions and 243 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
build/package.json
View file

@ -38,7 +38,7 @@
"gulp-bom": "^1.0.0",
"gulp-sourcemaps": "^1.11.0",
"gulp-uglify": "^3.0.0",
"iconv-lite": "0.4.23",
"iconv-lite": "0.6.0",
"mime": "^1.3.4",
"minimatch": "3.0.4",
"minimist": "^1.2.3",

8
build/yarn.lock
View file

@ -1415,10 +1415,10 @@ http-signature@~1.2.0:
jsprim "^1.2.2"
sshpk "^1.7.0"
iconv-lite@0.4.23:
version "0.4.23"
resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.23.tgz#297871f63be507adcfbfca715d0cd0eed84e9a63"
integrity sha512-neyTUVFtahjf0mB3dZT77u+8O0QB89jFdnBkd5P1JgYPbPaia3gXXOVL2fq8VyU2gMMD7SaN7QukTB/pmXYvDA==
iconv-lite@0.6.0:
version "0.6.0"
resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.0.tgz#66a93b80df0bd05d2a43a7426296b7f91073f125"
integrity sha512-43ZpGYZ9QtuutX5l6WC1DSO8ane9N+Ct5qPLF2OV7vM9abM69gnAbVkh66ibaZd3aOGkoP1ZmringlKhLBkw2Q==
dependencies:
safer-buffer ">= 2.1.2 < 3"

2
extensions/git/package.json
View file

@ -1878,7 +1878,7 @@
"dependencies": {
"byline": "^5.0.0",
"file-type": "^7.2.0",
"iconv-lite": "^0.4.24",
"iconv-lite": "0.6.0",
"jschardet": "2.1.1",
"vscode-extension-telemetry": "0.1.1",
"vscode-nls": "^4.0.0",

8
extensions/git/yarn.lock
View file

@ -425,10 +425,10 @@ https-proxy-agent@^2.2.1:
agent-base "^4.3.0"
debug "^3.1.0"
iconv-lite@^0.4.24:
version "0.4.24"
resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.24.tgz#2022b4b25fbddc21d2f524974a474aafe733908b"
integrity sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==
iconv-lite@0.6.0:
version "0.6.0"
resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.0.tgz#66a93b80df0bd05d2a43a7426296b7f91073f125"
integrity sha512-43ZpGYZ9QtuutX5l6WC1DSO8ane9N+Ct5qPLF2OV7vM9abM69gnAbVkh66ibaZd3aOGkoP1ZmringlKhLBkw2Q==
dependencies:
safer-buffer ">= 2.1.2 < 3"

2
package.json
View file

@ -41,7 +41,7 @@
"graceful-fs": "4.2.3",
"http-proxy-agent": "^2.1.0",
"https-proxy-agent": "^2.2.3",
"iconv-lite": "0.5.0",
"iconv-lite": "0.6.0",
"jschardet": "2.1.1",
"keytar": "^5.5.0",
"minimist": "^1.2.5",

2
remote/package.json
View file

@ -8,7 +8,7 @@
"graceful-fs": "4.2.3",
"http-proxy-agent": "^2.1.0",
"https-proxy-agent": "^2.2.3",
"iconv-lite": "0.5.0",
"iconv-lite": "0.6.0",
"jschardet": "2.1.1",
"minimist": "^1.2.5",
"native-watchdog": "1.3.0",

8
remote/yarn.lock
View file

@ -176,10 +176,10 @@ https-proxy-agent@^2.2.3:
agent-base "^4.3.0"
debug "^3.1.0"
iconv-lite@0.5.0:
version "0.5.0"
resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.5.0.tgz#59cdde0a2a297cc2aeb0c6445a195ee89f127550"
integrity sha512-NnEhI9hIEKHOzJ4f697DMz9IQEXr/MMJ5w64vN2/4Ai+wRnvV7SBrL0KLoRlwaKVghOc7LQ5YkPLuX146b6Ydw==
iconv-lite@0.6.0:
version "0.6.0"
resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.0.tgz#66a93b80df0bd05d2a43a7426296b7f91073f125"
integrity sha512-43ZpGYZ9QtuutX5l6WC1DSO8ane9N+Ct5qPLF2OV7vM9abM69gnAbVkh66ibaZd3aOGkoP1ZmringlKhLBkw2Q==
dependencies:
safer-buffer ">= 2.1.2 < 3"

195
src/vs/base/node/encoding.ts
View file

@ -4,8 +4,9 @@
*--------------------------------------------------------------------------------------------*/
import * as iconv from 'iconv-lite';
import { Readable, Writable } from 'stream';
import { VSBuffer } from 'vs/base/common/buffer';
import { Readable, ReadableStream, newWriteableStream } from 'vs/base/common/stream';
import { isUndefinedOrNull, isUndefined, isNumber } from 'vs/base/common/types';
import { VSBuffer, VSBufferReadable, VSBufferReadableStream } from 'vs/base/common/buffer';
export const UTF8 = 'utf8';
export const UTF8_with_bom = 'utf8bom';
@ -35,121 +36,135 @@ export interface IDecodeStreamOptions {
}
export interface IDecodeStreamResult {
stream: NodeJS.ReadableStream;
stream: ReadableStream<string>;
detected: IDetectedEncodingResult;
}
export function toDecodeStream(readable: Readable, options: IDecodeStreamOptions): Promise<IDecodeStreamResult> {
export function toDecodeStream(source: VSBufferReadableStream, options: IDecodeStreamOptions): Promise<IDecodeStreamResult> {
if (!options.minBytesRequiredForDetection) {
options.minBytesRequiredForDetection = options.guessEncoding ? AUTO_ENCODING_GUESS_MIN_BYTES : NO_ENCODING_GUESS_MIN_BYTES;
}
return new Promise<IDecodeStreamResult>((resolve, reject) => {
const writer = new class extends Writable {
private decodeStream: NodeJS.ReadWriteStream | undefined;
private decodeStreamPromise: Promise<void> | undefined;
const target = newWriteableStream<string>(strings => strings.join(''));
private bufferedChunks: Buffer[] = [];
private bytesBuffered = 0;
_write(chunk: Buffer, encoding: string, callback: (error: Error | null | undefined) => void): void {
if (!Buffer.isBuffer(chunk)) {
return callback(new Error('toDecodeStream(): data must be a buffer'));
}
// if the decode stream is ready, we just write directly
if (this.decodeStream) {
this.decodeStream.write(chunk, callback);
return;
}
// otherwise we need to buffer the data until the stream is ready
this.bufferedChunks.push(chunk);
this.bytesBuffered += chunk.byteLength;
// waiting for the decoder to be ready
if (this.decodeStreamPromise) {
this.decodeStreamPromise.then(() => callback(null), error => callback(error));
}
// buffered enough data for encoding detection, create stream and forward data
else if (typeof options.minBytesRequiredForDetection === 'number' && this.bytesBuffered >= options.minBytesRequiredForDetection) {
this._startDecodeStream(callback);
}
// only buffering until enough data for encoding detection is there
else {
callback(null);
}
}
_startDecodeStream(callback: (error: Error | null | undefined) => void): void {
// detect encoding from buffer
this.decodeStreamPromise = Promise.resolve(detectEncodingFromBuffer({
buffer: Buffer.concat(this.bufferedChunks),
bytesRead: this.bytesBuffered
}, options.guessEncoding)).then(detected => {
const bufferedChunks: VSBuffer[] = [];
let bytesBuffered = 0;
let decoder: iconv.DecoderStream | null = null;
const startDecodeStream = () => {
return Promise.resolve()
.then(() =>
// detect encoding from buffer
detectEncodingFromBuffer({
buffer: Buffer.from(VSBuffer.concat(bufferedChunks).buffer),
bytesRead: bytesBuffered
}, options.guessEncoding)
)
.then(detected => {
// ensure to respect overwrite of encoding
detected.encoding = options.overwriteEncoding(detected.encoding);
// decode and write buffer
this.decodeStream = decodeStream(detected.encoding);
this.decodeStream.write(Buffer.concat(this.bufferedChunks), callback);
this.bufferedChunks.length = 0;
// decode and write buffered content
decoder = iconv.getDecoder(toNodeEncoding(detected.encoding));
const nodeBuffer = Buffer.from(VSBuffer.concat(bufferedChunks).buffer);
target.write(decoder.write(nodeBuffer));
bufferedChunks.length = 0;
// signal to the outside our detected encoding
// and final decoder stream
resolve({ detected, stream: this.decodeStream });
}, error => {
this.emit('error', error);
callback(error);
});
}
_final(callback: () => void) {
// normal finish
if (this.decodeStream) {
this.decodeStream.end(callback);
}
// we were still waiting for data to do the encoding
// detection. thus, wrap up starting the stream even
// without all the data to get things going
else {
this._startDecodeStream(() => {
if (this.decodeStream) {
this.decodeStream.end(callback);
}
resolve({
stream: target,
detected,
});
}
}
})
.catch(reject);
};
// errors
readable.on('error', reject);
source.on('error', target.error);
source.on('data', (chunk) => {
// if the decoder is ready, we just write directly
if (!isUndefinedOrNull(decoder)) {
target.write(decoder.write(Buffer.from(chunk.buffer)));
return;
}
// pipe through
readable.pipe(writer);
// otherwise we need to buffer the data until the stream is ready
bufferedChunks.push(chunk);
bytesBuffered += chunk.byteLength;
// buffered enough data for encoding detection, create stream and forward data
if (isNumber(options.minBytesRequiredForDetection) && bytesBuffered >= options.minBytesRequiredForDetection) {
startDecodeStream();
}
});
source.on('end', () => {
// normal finish
if (!isUndefinedOrNull(decoder)) {
target.end(decoder.end());
}
// we were still waiting for data to do the encoding
// detection. thus, wrap up starting the stream even
// without all the data to get things going
else {
startDecodeStream().then(() => {
target.end(decoder?.end());
});
}
});
});
}
export function toEncodeReadable(readable: Readable<string>, encoding: string, options?: { addBOM?: boolean }): VSBufferReadable {
const encoder = iconv.getEncoder(toNodeEncoding(encoding), options);
let bytesRead = 0;
let done = false;
return {
read() {
if (done) {
return null;
}
const chunk = readable.read();
if (isUndefinedOrNull(chunk)) {
done = true;
// If we are instructed to add a BOM but we detect that no
// bytes have been read, we must ensure to return the BOM
// ourselves so that we comply with the contract.
if (bytesRead === 0 && options?.addBOM) {
switch (encoding) {
case UTF8:
case UTF8_with_bom:
return VSBuffer.wrap(Buffer.from(UTF8_BOM));
case UTF16be:
return VSBuffer.wrap(Buffer.from(UTF16be_BOM));
case UTF16le:
return VSBuffer.wrap(Buffer.from(UTF16le_BOM));
}
}
const leftovers = encoder.end();
if (!isUndefined(leftovers) && leftovers.length > 0) {
return VSBuffer.wrap(leftovers);
}
return null;
}
bytesRead += chunk.length;
return VSBuffer.wrap(encoder.write(chunk));
}
};
}
export function encodingExists(encoding: string): boolean {
return iconv.encodingExists(toNodeEncoding(encoding));
}
function decodeStream(encoding: string | null): NodeJS.ReadWriteStream {
return iconv.decodeStream(toNodeEncoding(encoding));
}
export function encodeStream(encoding: string, options?: { addBOM?: boolean }): NodeJS.ReadWriteStream {
return iconv.encodeStream(toNodeEncoding(encoding), options);
}
export function toNodeEncoding(enc: string | null): string {
if (enc === UTF8_with_bom || enc === null) {
return UTF8; // iconv does not distinguish UTF 8 with or without BOM, so we need to help it

63
src/vs/base/node/stream.ts
View file

@ -3,10 +3,8 @@
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import { VSBufferReadableStream, VSBufferReadable, VSBuffer } from 'vs/base/common/buffer';
import { VSBufferReadableStream } from 'vs/base/common/buffer';
import { Readable } from 'stream';
import { isUndefinedOrNull } from 'vs/base/common/types';
import { UTF8, UTF8_with_bom, UTF8_BOM, UTF16be, UTF16le_BOM, UTF16be_BOM, UTF16le, UTF_ENCODING } from 'vs/base/node/encoding';
export function streamToNodeReadable(stream: VSBufferReadableStream): Readable {
return new class extends Readable {
@ -51,62 +49,3 @@ export function streamToNodeReadable(stream: VSBufferReadableStream): Readable {
}
};
}
export function nodeReadableToString(stream: NodeJS.ReadableStream): Promise<string> {
return new Promise((resolve, reject) => {
let result = '';
stream.on('data', chunk => result += chunk);
stream.on('error', reject);
stream.on('end', () => resolve(result));
});
}
export function nodeStreamToVSBufferReadable(stream: NodeJS.ReadWriteStream, addBOM?: { encoding: UTF_ENCODING }): VSBufferReadable {
let bytesRead = 0;
let done = false;
return {
read(): VSBuffer | null {
if (done) {
return null;
}
const res = stream.read();
if (isUndefinedOrNull(res)) {
done = true;
// If we are instructed to add a BOM but we detect that no
// bytes have been read, we must ensure to return the BOM
// ourselves so that we comply with the contract.
if (bytesRead === 0 && addBOM) {
switch (addBOM.encoding) {
case UTF8:
case UTF8_with_bom:
return VSBuffer.wrap(Buffer.from(UTF8_BOM));
case UTF16be:
return VSBuffer.wrap(Buffer.from(UTF16be_BOM));
case UTF16le:
return VSBuffer.wrap(Buffer.from(UTF16le_BOM));
}
}
return null;
}
// Handle String
if (typeof res === 'string') {
bytesRead += res.length;
return VSBuffer.fromString(res);
}
// Handle Buffer
else {
bytesRead += res.byteLength;
return VSBuffer.wrap(res);
}
}
};
}

145
src/vs/base/test/node/encoding/encoding.test.ts
View file

@ -7,9 +7,10 @@ import * as assert from 'assert';
import * as fs from 'fs';
import * as encoding from 'vs/base/node/encoding';
import * as terminalEncoding from 'vs/base/node/terminalEncoding';
import { Readable } from 'stream';
import * as streams from 'vs/base/common/stream';
import * as iconv from 'iconv-lite';
import { getPathFromAmdModule } from 'vs/base/common/amd';
import { newWriteableBufferStream, VSBuffer, VSBufferReadableStream, streamToBufferReadableStream } from 'vs/base/common/buffer';
export async function detectEncodingByBOM(file: string): Promise<typeof encoding.UTF16be | typeof encoding.UTF16le | typeof encoding.UTF8_with_bom | null> {
try {
@ -231,30 +232,32 @@ suite('Encoding', () => {
});
}
async function readAllAsString(stream: NodeJS.ReadableStream) {
return new Promise<string>((resolve, reject) => {
let all = '';
stream.on('data', chunk => {
all += chunk;
assert.equal(typeof chunk, 'string');
function newTestReadableStream(buffers: Buffer[]): VSBufferReadableStream {
const stream = newWriteableBufferStream();
buffers
.map(VSBuffer.wrap)
.forEach(buffer => {
setTimeout(() => {
stream.write(buffer);
});
});
stream.on('end', () => {
resolve(all);
});
stream.on('error', reject);
setTimeout(() => {
stream.end();
});
return stream;
}
async function readAllAsString(stream: streams.ReadableStream<string>) {
return streams.consumeStream(stream, strings => strings.join(''));
}
test('toDecodeStream - some stream', async function () {
let source = new Readable({
read(size) {
this.push(Buffer.from([65, 66, 67]));
this.push(Buffer.from([65, 66, 67]));
this.push(Buffer.from([65, 66, 67]));
this.push(null);
}
});
let source = newTestReadableStream([
Buffer.from([65, 66, 67]),
Buffer.from([65, 66, 67]),
Buffer.from([65, 66, 67]),
]);
let { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 4, guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 });
@ -267,14 +270,11 @@ suite('Encoding', () => {
test('toDecodeStream - some stream, expect too much data', async function () {
let source = new Readable({
read(size) {
this.push(Buffer.from([65, 66, 67]));
this.push(Buffer.from([65, 66, 67]));
this.push(Buffer.from([65, 66, 67]));
this.push(null);
}
});
let source = newTestReadableStream([
Buffer.from([65, 66, 67]),
Buffer.from([65, 66, 67]),
Buffer.from([65, 66, 67]),
]);
let { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 64, guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 });
@ -287,11 +287,8 @@ suite('Encoding', () => {
test('toDecodeStream - some stream, no data', async function () {
let source = new Readable({
read(size) {
this.push(null); // empty
}
});
let source = newWriteableBufferStream();
source.end();
let { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 512, guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 });
@ -306,7 +303,7 @@ suite('Encoding', () => {
test('toDecodeStream - encoding, utf16be', async function () {
let path = getPathFromAmdModule(require, './fixtures/some_utf16be.css');
let source = fs.createReadStream(path);
let source = streamToBufferReadableStream(fs.createReadStream(path));
let { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 64, guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 });
@ -322,11 +319,91 @@ suite('Encoding', () => {
test('toDecodeStream - empty file', async function () {
let path = getPathFromAmdModule(require, './fixtures/empty.txt');
let source = fs.createReadStream(path);
let source = streamToBufferReadableStream(fs.createReadStream(path));
let { detected, stream } = await encoding.toDecodeStream(source, { guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 });
let expected = await readAndDecodeFromDisk(path, detected.encoding);
let actual = await readAllAsString(stream);
assert.equal(actual, expected);
});
test('toDecodeStream - decodes buffer entirely', async function () {
let emojis = Buffer.from('🖥️💻💾');
let incompleteEmojis = emojis.slice(0, emojis.length - 1);
let buffers = [];
for (let i = 0; i < incompleteEmojis.length; i++) {
buffers.push(incompleteEmojis.slice(i, i + 1));
}
const source = newTestReadableStream(buffers);
let { stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 4, guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 });
let expected = incompleteEmojis.toString(encoding.UTF8);
let actual = await readAllAsString(stream);
assert.equal(actual, expected);
});
test('toEncodeReadable - encoding, utf16be', async function () {
let path = getPathFromAmdModule(require, './fixtures/some_utf16be.css');
let source = await readAndDecodeFromDisk(path, encoding.UTF16be);
let expected = VSBuffer.wrap(
iconv.encode(source, encoding.toNodeEncoding(encoding.UTF16be))
).toString();
let actual = streams.consumeReadable(
encoding.toEncodeReadable(streams.toReadable(source), encoding.UTF16be),
VSBuffer.concat
).toString();
assert.equal(actual, expected);
});
test('toEncodeReadable - empty readable to utf8', async function () {
const source: streams.Readable<string> = {
read() {
return null;
}
};
let actual = streams.consumeReadable(
encoding.toEncodeReadable(source, encoding.UTF8),
VSBuffer.concat
).toString();
assert.equal(actual, '');
});
[{
utfEncoding: encoding.UTF8,
relatedBom: encoding.UTF8_BOM
}, {
utfEncoding: encoding.UTF8_with_bom,
relatedBom: encoding.UTF8_BOM
}, {
utfEncoding: encoding.UTF16be,
relatedBom: encoding.UTF16be_BOM,
}, {
utfEncoding: encoding.UTF16le,
relatedBom: encoding.UTF16le_BOM
}].forEach(({ utfEncoding, relatedBom }) => {
test(`toEncodeReadable - empty readable to ${utfEncoding} with BOM`, async function () {
const source: streams.Readable<string> = {
read() {
return null;
}
};
let encodedReadable = encoding.toEncodeReadable(source, utfEncoding, { addBOM: true });
const expected = VSBuffer.wrap(Buffer.from(relatedBom)).toString();
const actual = streams.consumeReadable(encodedReadable, VSBuffer.concat).toString();
assert.equal(actual, expected);
});
});
});

44
src/vs/workbench/services/textfile/electron-browser/nativeTextFileService.ts
View file

@ -17,16 +17,15 @@ import { isMacintosh } from 'vs/base/common/platform';
import { IProductService } from 'vs/platform/product/common/productService';
import { ITextResourceConfigurationService } from 'vs/editor/common/services/textResourceConfigurationService';
import { IWorkspaceContextService } from 'vs/platform/workspace/common/workspace';
import { UTF8, UTF8_with_bom, UTF16be, UTF16le, encodingExists, encodeStream, UTF8_BOM, toDecodeStream, IDecodeStreamResult, detectEncodingByBOMFromBuffer, isUTFEncoding } from 'vs/base/node/encoding';
import { UTF8, UTF8_with_bom, UTF16be, UTF16le, encodingExists, UTF8_BOM, toDecodeStream, toEncodeReadable, IDecodeStreamResult, detectEncodingByBOMFromBuffer } from 'vs/base/node/encoding';
import { WORKSPACE_EXTENSION } from 'vs/platform/workspaces/common/workspaces';
import { joinPath, extname, isEqualOrParent } from 'vs/base/common/resources';
import { Disposable } from 'vs/base/common/lifecycle';
import { IEnvironmentService } from 'vs/platform/environment/common/environment';
import { VSBufferReadable, bufferToStream } from 'vs/base/common/buffer';
import { Readable } from 'stream';
import { bufferToStream, VSBufferReadable } from 'vs/base/common/buffer';
import { createTextBufferFactoryFromStream } from 'vs/editor/common/model/textModel';
import { ITextSnapshot } from 'vs/editor/common/model';
import { nodeReadableToString, streamToNodeReadable, nodeStreamToVSBufferReadable } from 'vs/base/node/stream';
import { consumeStream } from 'vs/base/common/stream';
import { IUntitledTextEditorService } from 'vs/workbench/services/untitled/common/untitledTextEditorService';
import { ILifecycleService } from 'vs/platform/lifecycle/common/lifecycle';
import { IInstantiationService } from 'vs/platform/instantiation/common/instantiation';
@ -89,7 +88,7 @@ export class NativeTextFileService extends AbstractTextFileService {
return {
...bufferStream,
encoding: decoder.detected.encoding || UTF8,
value: await nodeReadableToString(decoder.stream)
value: await consumeStream(decoder.stream, strings => strings.join(''))
};
}
@ -121,7 +120,7 @@ export class NativeTextFileService extends AbstractTextFileService {
}
// read through encoding library
const decoder = await toDecodeStream(streamToNodeReadable(bufferStream.value), {
const decoder = await toDecodeStream(bufferStream.value, {
guessEncoding: options?.autoGuessEncoding || this.textResourceConfigurationService.getValue(resource, 'files.autoGuessEncoding'),
overwriteEncoding: detectedEncoding => this.encoding.getReadEncoding(resource, options, detectedEncoding)
});
@ -232,37 +231,8 @@ export class NativeTextFileService extends AbstractTextFileService {
}
private getEncodedReadable(value: string | ITextSnapshot, encoding: string, addBOM: boolean): VSBufferReadable {
const readable = this.snapshotToNodeReadable(typeof value === 'string' ? stringToSnapshot(value) : value);
const encoder = encodeStream(encoding, { addBOM });
const encodedReadable = readable.pipe(encoder);
return nodeStreamToVSBufferReadable(encodedReadable, addBOM && isUTFEncoding(encoding) ? { encoding } : undefined);
}
private snapshotToNodeReadable(snapshot: ITextSnapshot): Readable {
return new Readable({
read: function () {
try {
let chunk: string | null = null;
let canPush = true;
// Push all chunks as long as we can push and as long as
// the underlying snapshot returns strings to us
while (canPush && typeof (chunk = snapshot.read()) === 'string') {
canPush = this.push(chunk);
}
// Signal EOS by pushing NULL
if (typeof chunk !== 'string') {
this.push(null);
}
} catch (error) {
this.emit('error', error);
}
},
encoding: UTF8 // very important, so that strings are passed around and not buffers!
});
const snapshot = typeof value === 'string' ? stringToSnapshot(value) : value;
return toEncodeReadable(snapshot, encoding, { addBOM });
}
private async writeElevated(resource: URI, value: string | ITextSnapshot, options?: IWriteTextFileOptions): Promise<IFileStatWithMetadata> {

8
yarn.lock
View file

@ -4579,10 +4579,10 @@ husky@^0.13.1:
is-ci "^1.0.9"
normalize-path "^1.0.0"
iconv-lite@0.5.0:
version "0.5.0"
resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.5.0.tgz#59cdde0a2a297cc2aeb0c6445a195ee89f127550"
integrity sha512-NnEhI9hIEKHOzJ4f697DMz9IQEXr/MMJ5w64vN2/4Ai+wRnvV7SBrL0KLoRlwaKVghOc7LQ5YkPLuX146b6Ydw==
iconv-lite@0.6.0:
version "0.6.0"
resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.0.tgz#66a93b80df0bd05d2a43a7426296b7f91073f125"
integrity sha512-43ZpGYZ9QtuutX5l6WC1DSO8ane9N+Ct5qPLF2OV7vM9abM69gnAbVkh66ibaZd3aOGkoP1ZmringlKhLBkw2Q==
dependencies:
safer-buffer ">= 2.1.2 < 3"