mirror of
https://github.com/Microsoft/vscode
synced 2024-10-30 13:43:07 +00:00
This reverts commit 1dbfecd37f
.
This commit is contained in:
parent
1dbfecd37f
commit
495112ded5
12 changed files with 244 additions and 245 deletions
|
@ -38,7 +38,7 @@
|
||||||
"gulp-bom": "^1.0.0",
|
"gulp-bom": "^1.0.0",
|
||||||
"gulp-sourcemaps": "^1.11.0",
|
"gulp-sourcemaps": "^1.11.0",
|
||||||
"gulp-uglify": "^3.0.0",
|
"gulp-uglify": "^3.0.0",
|
||||||
"iconv-lite": "0.6.0",
|
"iconv-lite": "0.4.23",
|
||||||
"mime": "^1.3.4",
|
"mime": "^1.3.4",
|
||||||
"minimatch": "3.0.4",
|
"minimatch": "3.0.4",
|
||||||
"minimist": "^1.2.3",
|
"minimist": "^1.2.3",
|
||||||
|
|
|
@ -1415,10 +1415,10 @@ http-signature@~1.2.0:
|
||||||
jsprim "^1.2.2"
|
jsprim "^1.2.2"
|
||||||
sshpk "^1.7.0"
|
sshpk "^1.7.0"
|
||||||
|
|
||||||
iconv-lite@0.6.0:
|
iconv-lite@0.4.23:
|
||||||
version "0.6.0"
|
version "0.4.23"
|
||||||
resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.0.tgz#66a93b80df0bd05d2a43a7426296b7f91073f125"
|
resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.23.tgz#297871f63be507adcfbfca715d0cd0eed84e9a63"
|
||||||
integrity sha512-43ZpGYZ9QtuutX5l6WC1DSO8ane9N+Ct5qPLF2OV7vM9abM69gnAbVkh66ibaZd3aOGkoP1ZmringlKhLBkw2Q==
|
integrity sha512-neyTUVFtahjf0mB3dZT77u+8O0QB89jFdnBkd5P1JgYPbPaia3gXXOVL2fq8VyU2gMMD7SaN7QukTB/pmXYvDA==
|
||||||
dependencies:
|
dependencies:
|
||||||
safer-buffer ">= 2.1.2 < 3"
|
safer-buffer ">= 2.1.2 < 3"
|
||||||
|
|
||||||
|
|
|
@ -1878,7 +1878,7 @@
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"byline": "^5.0.0",
|
"byline": "^5.0.0",
|
||||||
"file-type": "^7.2.0",
|
"file-type": "^7.2.0",
|
||||||
"iconv-lite": "0.6.0",
|
"iconv-lite": "^0.4.24",
|
||||||
"jschardet": "2.1.1",
|
"jschardet": "2.1.1",
|
||||||
"vscode-extension-telemetry": "0.1.1",
|
"vscode-extension-telemetry": "0.1.1",
|
||||||
"vscode-nls": "^4.0.0",
|
"vscode-nls": "^4.0.0",
|
||||||
|
|
|
@ -425,10 +425,10 @@ https-proxy-agent@^2.2.1:
|
||||||
agent-base "^4.3.0"
|
agent-base "^4.3.0"
|
||||||
debug "^3.1.0"
|
debug "^3.1.0"
|
||||||
|
|
||||||
iconv-lite@0.6.0:
|
iconv-lite@^0.4.24:
|
||||||
version "0.6.0"
|
version "0.4.24"
|
||||||
resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.0.tgz#66a93b80df0bd05d2a43a7426296b7f91073f125"
|
resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.24.tgz#2022b4b25fbddc21d2f524974a474aafe733908b"
|
||||||
integrity sha512-43ZpGYZ9QtuutX5l6WC1DSO8ane9N+Ct5qPLF2OV7vM9abM69gnAbVkh66ibaZd3aOGkoP1ZmringlKhLBkw2Q==
|
integrity sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==
|
||||||
dependencies:
|
dependencies:
|
||||||
safer-buffer ">= 2.1.2 < 3"
|
safer-buffer ">= 2.1.2 < 3"
|
||||||
|
|
||||||
|
|
|
@ -41,7 +41,7 @@
|
||||||
"graceful-fs": "4.2.3",
|
"graceful-fs": "4.2.3",
|
||||||
"http-proxy-agent": "^2.1.0",
|
"http-proxy-agent": "^2.1.0",
|
||||||
"https-proxy-agent": "^2.2.3",
|
"https-proxy-agent": "^2.2.3",
|
||||||
"iconv-lite": "0.6.0",
|
"iconv-lite": "0.5.0",
|
||||||
"jschardet": "2.1.1",
|
"jschardet": "2.1.1",
|
||||||
"keytar": "^5.5.0",
|
"keytar": "^5.5.0",
|
||||||
"minimist": "^1.2.5",
|
"minimist": "^1.2.5",
|
||||||
|
|
|
@ -8,7 +8,7 @@
|
||||||
"graceful-fs": "4.2.3",
|
"graceful-fs": "4.2.3",
|
||||||
"http-proxy-agent": "^2.1.0",
|
"http-proxy-agent": "^2.1.0",
|
||||||
"https-proxy-agent": "^2.2.3",
|
"https-proxy-agent": "^2.2.3",
|
||||||
"iconv-lite": "0.6.0",
|
"iconv-lite": "0.5.0",
|
||||||
"jschardet": "2.1.1",
|
"jschardet": "2.1.1",
|
||||||
"minimist": "^1.2.5",
|
"minimist": "^1.2.5",
|
||||||
"native-watchdog": "1.3.0",
|
"native-watchdog": "1.3.0",
|
||||||
|
|
|
@ -176,10 +176,10 @@ https-proxy-agent@^2.2.3:
|
||||||
agent-base "^4.3.0"
|
agent-base "^4.3.0"
|
||||||
debug "^3.1.0"
|
debug "^3.1.0"
|
||||||
|
|
||||||
iconv-lite@0.6.0:
|
iconv-lite@0.5.0:
|
||||||
version "0.6.0"
|
version "0.5.0"
|
||||||
resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.0.tgz#66a93b80df0bd05d2a43a7426296b7f91073f125"
|
resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.5.0.tgz#59cdde0a2a297cc2aeb0c6445a195ee89f127550"
|
||||||
integrity sha512-43ZpGYZ9QtuutX5l6WC1DSO8ane9N+Ct5qPLF2OV7vM9abM69gnAbVkh66ibaZd3aOGkoP1ZmringlKhLBkw2Q==
|
integrity sha512-NnEhI9hIEKHOzJ4f697DMz9IQEXr/MMJ5w64vN2/4Ai+wRnvV7SBrL0KLoRlwaKVghOc7LQ5YkPLuX146b6Ydw==
|
||||||
dependencies:
|
dependencies:
|
||||||
safer-buffer ">= 2.1.2 < 3"
|
safer-buffer ">= 2.1.2 < 3"
|
||||||
|
|
||||||
|
|
|
@ -4,9 +4,8 @@
|
||||||
*--------------------------------------------------------------------------------------------*/
|
*--------------------------------------------------------------------------------------------*/
|
||||||
|
|
||||||
import * as iconv from 'iconv-lite';
|
import * as iconv from 'iconv-lite';
|
||||||
import { Readable, ReadableStream, newWriteableStream } from 'vs/base/common/stream';
|
import { Readable, Writable } from 'stream';
|
||||||
import { isUndefinedOrNull, isUndefined, isNumber } from 'vs/base/common/types';
|
import { VSBuffer } from 'vs/base/common/buffer';
|
||||||
import { VSBuffer, VSBufferReadable, VSBufferReadableStream } from 'vs/base/common/buffer';
|
|
||||||
|
|
||||||
export const UTF8 = 'utf8';
|
export const UTF8 = 'utf8';
|
||||||
export const UTF8_with_bom = 'utf8bom';
|
export const UTF8_with_bom = 'utf8bom';
|
||||||
|
@ -36,135 +35,121 @@ export interface IDecodeStreamOptions {
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface IDecodeStreamResult {
|
export interface IDecodeStreamResult {
|
||||||
stream: ReadableStream<string>;
|
stream: NodeJS.ReadableStream;
|
||||||
detected: IDetectedEncodingResult;
|
detected: IDetectedEncodingResult;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function toDecodeStream(source: VSBufferReadableStream, options: IDecodeStreamOptions): Promise<IDecodeStreamResult> {
|
export function toDecodeStream(readable: Readable, options: IDecodeStreamOptions): Promise<IDecodeStreamResult> {
|
||||||
if (!options.minBytesRequiredForDetection) {
|
if (!options.minBytesRequiredForDetection) {
|
||||||
options.minBytesRequiredForDetection = options.guessEncoding ? AUTO_ENCODING_GUESS_MIN_BYTES : NO_ENCODING_GUESS_MIN_BYTES;
|
options.minBytesRequiredForDetection = options.guessEncoding ? AUTO_ENCODING_GUESS_MIN_BYTES : NO_ENCODING_GUESS_MIN_BYTES;
|
||||||
}
|
}
|
||||||
|
|
||||||
return new Promise<IDecodeStreamResult>((resolve, reject) => {
|
return new Promise<IDecodeStreamResult>((resolve, reject) => {
|
||||||
const target = newWriteableStream<string>(strings => strings.join(''));
|
const writer = new class extends Writable {
|
||||||
|
private decodeStream: NodeJS.ReadWriteStream | undefined;
|
||||||
|
private decodeStreamPromise: Promise<void> | undefined;
|
||||||
|
|
||||||
const bufferedChunks: VSBuffer[] = [];
|
private bufferedChunks: Buffer[] = [];
|
||||||
let bytesBuffered = 0;
|
private bytesBuffered = 0;
|
||||||
let decoder: iconv.DecoderStream | null = null;
|
|
||||||
|
_write(chunk: Buffer, encoding: string, callback: (error: Error | null | undefined) => void): void {
|
||||||
|
if (!Buffer.isBuffer(chunk)) {
|
||||||
|
return callback(new Error('toDecodeStream(): data must be a buffer'));
|
||||||
|
}
|
||||||
|
|
||||||
|
// if the decode stream is ready, we just write directly
|
||||||
|
if (this.decodeStream) {
|
||||||
|
this.decodeStream.write(chunk, callback);
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// otherwise we need to buffer the data until the stream is ready
|
||||||
|
this.bufferedChunks.push(chunk);
|
||||||
|
this.bytesBuffered += chunk.byteLength;
|
||||||
|
|
||||||
|
// waiting for the decoder to be ready
|
||||||
|
if (this.decodeStreamPromise) {
|
||||||
|
this.decodeStreamPromise.then(() => callback(null), error => callback(error));
|
||||||
|
}
|
||||||
|
|
||||||
|
// buffered enough data for encoding detection, create stream and forward data
|
||||||
|
else if (typeof options.minBytesRequiredForDetection === 'number' && this.bytesBuffered >= options.minBytesRequiredForDetection) {
|
||||||
|
this._startDecodeStream(callback);
|
||||||
|
}
|
||||||
|
|
||||||
|
// only buffering until enough data for encoding detection is there
|
||||||
|
else {
|
||||||
|
callback(null);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
_startDecodeStream(callback: (error: Error | null | undefined) => void): void {
|
||||||
|
|
||||||
|
// detect encoding from buffer
|
||||||
|
this.decodeStreamPromise = Promise.resolve(detectEncodingFromBuffer({
|
||||||
|
buffer: Buffer.concat(this.bufferedChunks),
|
||||||
|
bytesRead: this.bytesBuffered
|
||||||
|
}, options.guessEncoding)).then(detected => {
|
||||||
|
|
||||||
const startDecodeStream = () => {
|
|
||||||
return Promise.resolve()
|
|
||||||
.then(() =>
|
|
||||||
// detect encoding from buffer
|
|
||||||
detectEncodingFromBuffer({
|
|
||||||
buffer: Buffer.from(VSBuffer.concat(bufferedChunks).buffer),
|
|
||||||
bytesRead: bytesBuffered
|
|
||||||
}, options.guessEncoding)
|
|
||||||
)
|
|
||||||
.then(detected => {
|
|
||||||
// ensure to respect overwrite of encoding
|
// ensure to respect overwrite of encoding
|
||||||
detected.encoding = options.overwriteEncoding(detected.encoding);
|
detected.encoding = options.overwriteEncoding(detected.encoding);
|
||||||
|
|
||||||
// decode and write buffered content
|
// decode and write buffer
|
||||||
decoder = iconv.getDecoder(toNodeEncoding(detected.encoding));
|
this.decodeStream = decodeStream(detected.encoding);
|
||||||
const nodeBuffer = Buffer.from(VSBuffer.concat(bufferedChunks).buffer);
|
this.decodeStream.write(Buffer.concat(this.bufferedChunks), callback);
|
||||||
target.write(decoder.write(nodeBuffer));
|
this.bufferedChunks.length = 0;
|
||||||
bufferedChunks.length = 0;
|
|
||||||
|
|
||||||
// signal to the outside our detected encoding
|
// signal to the outside our detected encoding
|
||||||
// and final decoder stream
|
// and final decoder stream
|
||||||
resolve({
|
resolve({ detected, stream: this.decodeStream });
|
||||||
stream: target,
|
}, error => {
|
||||||
detected,
|
this.emit('error', error);
|
||||||
});
|
|
||||||
})
|
|
||||||
.catch(reject);
|
|
||||||
};
|
|
||||||
|
|
||||||
source.on('error', target.error);
|
callback(error);
|
||||||
source.on('data', (chunk) => {
|
|
||||||
// if the decoder is ready, we just write directly
|
|
||||||
if (!isUndefinedOrNull(decoder)) {
|
|
||||||
target.write(decoder.write(Buffer.from(chunk.buffer)));
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
// otherwise we need to buffer the data until the stream is ready
|
|
||||||
bufferedChunks.push(chunk);
|
|
||||||
bytesBuffered += chunk.byteLength;
|
|
||||||
|
|
||||||
// buffered enough data for encoding detection, create stream and forward data
|
|
||||||
if (isNumber(options.minBytesRequiredForDetection) && bytesBuffered >= options.minBytesRequiredForDetection) {
|
|
||||||
startDecodeStream();
|
|
||||||
}
|
|
||||||
});
|
|
||||||
source.on('end', () => {
|
|
||||||
// normal finish
|
|
||||||
if (!isUndefinedOrNull(decoder)) {
|
|
||||||
target.end(decoder.end());
|
|
||||||
}
|
|
||||||
|
|
||||||
// we were still waiting for data to do the encoding
|
|
||||||
// detection. thus, wrap up starting the stream even
|
|
||||||
// without all the data to get things going
|
|
||||||
else {
|
|
||||||
startDecodeStream().then(() => {
|
|
||||||
target.end(decoder?.end());
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
});
|
|
||||||
|
_final(callback: () => void) {
|
||||||
|
|
||||||
|
// normal finish
|
||||||
|
if (this.decodeStream) {
|
||||||
|
this.decodeStream.end(callback);
|
||||||
|
}
|
||||||
|
|
||||||
|
// we were still waiting for data to do the encoding
|
||||||
|
// detection. thus, wrap up starting the stream even
|
||||||
|
// without all the data to get things going
|
||||||
|
else {
|
||||||
|
this._startDecodeStream(() => {
|
||||||
|
if (this.decodeStream) {
|
||||||
|
this.decodeStream.end(callback);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// errors
|
||||||
|
readable.on('error', reject);
|
||||||
|
|
||||||
|
// pipe through
|
||||||
|
readable.pipe(writer);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
export function toEncodeReadable(readable: Readable<string>, encoding: string, options?: { addBOM?: boolean }): VSBufferReadable {
|
|
||||||
const encoder = iconv.getEncoder(toNodeEncoding(encoding), options);
|
|
||||||
let bytesRead = 0;
|
|
||||||
let done = false;
|
|
||||||
|
|
||||||
return {
|
|
||||||
read() {
|
|
||||||
if (done) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
const chunk = readable.read();
|
|
||||||
if (isUndefinedOrNull(chunk)) {
|
|
||||||
done = true;
|
|
||||||
|
|
||||||
// If we are instructed to add a BOM but we detect that no
|
|
||||||
// bytes have been read, we must ensure to return the BOM
|
|
||||||
// ourselves so that we comply with the contract.
|
|
||||||
if (bytesRead === 0 && options?.addBOM) {
|
|
||||||
switch (encoding) {
|
|
||||||
case UTF8:
|
|
||||||
case UTF8_with_bom:
|
|
||||||
return VSBuffer.wrap(Buffer.from(UTF8_BOM));
|
|
||||||
case UTF16be:
|
|
||||||
return VSBuffer.wrap(Buffer.from(UTF16be_BOM));
|
|
||||||
case UTF16le:
|
|
||||||
return VSBuffer.wrap(Buffer.from(UTF16le_BOM));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const leftovers = encoder.end();
|
|
||||||
if (!isUndefined(leftovers) && leftovers.length > 0) {
|
|
||||||
return VSBuffer.wrap(leftovers);
|
|
||||||
}
|
|
||||||
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
bytesRead += chunk.length;
|
|
||||||
|
|
||||||
return VSBuffer.wrap(encoder.write(chunk));
|
|
||||||
}
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
export function encodingExists(encoding: string): boolean {
|
export function encodingExists(encoding: string): boolean {
|
||||||
return iconv.encodingExists(toNodeEncoding(encoding));
|
return iconv.encodingExists(toNodeEncoding(encoding));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function decodeStream(encoding: string | null): NodeJS.ReadWriteStream {
|
||||||
|
return iconv.decodeStream(toNodeEncoding(encoding));
|
||||||
|
}
|
||||||
|
|
||||||
|
export function encodeStream(encoding: string, options?: { addBOM?: boolean }): NodeJS.ReadWriteStream {
|
||||||
|
return iconv.encodeStream(toNodeEncoding(encoding), options);
|
||||||
|
}
|
||||||
|
|
||||||
export function toNodeEncoding(enc: string | null): string {
|
export function toNodeEncoding(enc: string | null): string {
|
||||||
if (enc === UTF8_with_bom || enc === null) {
|
if (enc === UTF8_with_bom || enc === null) {
|
||||||
return UTF8; // iconv does not distinguish UTF 8 with or without BOM, so we need to help it
|
return UTF8; // iconv does not distinguish UTF 8 with or without BOM, so we need to help it
|
||||||
|
|
|
@ -3,8 +3,10 @@
|
||||||
* Licensed under the MIT License. See License.txt in the project root for license information.
|
* Licensed under the MIT License. See License.txt in the project root for license information.
|
||||||
*--------------------------------------------------------------------------------------------*/
|
*--------------------------------------------------------------------------------------------*/
|
||||||
|
|
||||||
import { VSBufferReadableStream } from 'vs/base/common/buffer';
|
import { VSBufferReadableStream, VSBufferReadable, VSBuffer } from 'vs/base/common/buffer';
|
||||||
import { Readable } from 'stream';
|
import { Readable } from 'stream';
|
||||||
|
import { isUndefinedOrNull } from 'vs/base/common/types';
|
||||||
|
import { UTF8, UTF8_with_bom, UTF8_BOM, UTF16be, UTF16le_BOM, UTF16be_BOM, UTF16le, UTF_ENCODING } from 'vs/base/node/encoding';
|
||||||
|
|
||||||
export function streamToNodeReadable(stream: VSBufferReadableStream): Readable {
|
export function streamToNodeReadable(stream: VSBufferReadableStream): Readable {
|
||||||
return new class extends Readable {
|
return new class extends Readable {
|
||||||
|
@ -49,3 +51,62 @@ export function streamToNodeReadable(stream: VSBufferReadableStream): Readable {
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function nodeReadableToString(stream: NodeJS.ReadableStream): Promise<string> {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
let result = '';
|
||||||
|
|
||||||
|
stream.on('data', chunk => result += chunk);
|
||||||
|
stream.on('error', reject);
|
||||||
|
stream.on('end', () => resolve(result));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function nodeStreamToVSBufferReadable(stream: NodeJS.ReadWriteStream, addBOM?: { encoding: UTF_ENCODING }): VSBufferReadable {
|
||||||
|
let bytesRead = 0;
|
||||||
|
let done = false;
|
||||||
|
|
||||||
|
return {
|
||||||
|
read(): VSBuffer | null {
|
||||||
|
if (done) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const res = stream.read();
|
||||||
|
if (isUndefinedOrNull(res)) {
|
||||||
|
done = true;
|
||||||
|
|
||||||
|
// If we are instructed to add a BOM but we detect that no
|
||||||
|
// bytes have been read, we must ensure to return the BOM
|
||||||
|
// ourselves so that we comply with the contract.
|
||||||
|
if (bytesRead === 0 && addBOM) {
|
||||||
|
switch (addBOM.encoding) {
|
||||||
|
case UTF8:
|
||||||
|
case UTF8_with_bom:
|
||||||
|
return VSBuffer.wrap(Buffer.from(UTF8_BOM));
|
||||||
|
case UTF16be:
|
||||||
|
return VSBuffer.wrap(Buffer.from(UTF16be_BOM));
|
||||||
|
case UTF16le:
|
||||||
|
return VSBuffer.wrap(Buffer.from(UTF16le_BOM));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Handle String
|
||||||
|
if (typeof res === 'string') {
|
||||||
|
bytesRead += res.length;
|
||||||
|
|
||||||
|
return VSBuffer.fromString(res);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Handle Buffer
|
||||||
|
else {
|
||||||
|
bytesRead += res.byteLength;
|
||||||
|
|
||||||
|
return VSBuffer.wrap(res);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
|
@ -7,10 +7,9 @@ import * as assert from 'assert';
|
||||||
import * as fs from 'fs';
|
import * as fs from 'fs';
|
||||||
import * as encoding from 'vs/base/node/encoding';
|
import * as encoding from 'vs/base/node/encoding';
|
||||||
import * as terminalEncoding from 'vs/base/node/terminalEncoding';
|
import * as terminalEncoding from 'vs/base/node/terminalEncoding';
|
||||||
import * as streams from 'vs/base/common/stream';
|
import { Readable } from 'stream';
|
||||||
import * as iconv from 'iconv-lite';
|
import * as iconv from 'iconv-lite';
|
||||||
import { getPathFromAmdModule } from 'vs/base/common/amd';
|
import { getPathFromAmdModule } from 'vs/base/common/amd';
|
||||||
import { newWriteableBufferStream, VSBuffer, VSBufferReadableStream, streamToBufferReadableStream } from 'vs/base/common/buffer';
|
|
||||||
|
|
||||||
export async function detectEncodingByBOM(file: string): Promise<typeof encoding.UTF16be | typeof encoding.UTF16le | typeof encoding.UTF8_with_bom | null> {
|
export async function detectEncodingByBOM(file: string): Promise<typeof encoding.UTF16be | typeof encoding.UTF16le | typeof encoding.UTF8_with_bom | null> {
|
||||||
try {
|
try {
|
||||||
|
@ -232,32 +231,30 @@ suite('Encoding', () => {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
function newTestReadableStream(buffers: Buffer[]): VSBufferReadableStream {
|
async function readAllAsString(stream: NodeJS.ReadableStream) {
|
||||||
const stream = newWriteableBufferStream();
|
return new Promise<string>((resolve, reject) => {
|
||||||
buffers
|
let all = '';
|
||||||
.map(VSBuffer.wrap)
|
stream.on('data', chunk => {
|
||||||
.forEach(buffer => {
|
all += chunk;
|
||||||
setTimeout(() => {
|
assert.equal(typeof chunk, 'string');
|
||||||
stream.write(buffer);
|
|
||||||
});
|
|
||||||
});
|
});
|
||||||
setTimeout(() => {
|
stream.on('end', () => {
|
||||||
stream.end();
|
resolve(all);
|
||||||
|
});
|
||||||
|
stream.on('error', reject);
|
||||||
});
|
});
|
||||||
return stream;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function readAllAsString(stream: streams.ReadableStream<string>) {
|
|
||||||
return streams.consumeStream(stream, strings => strings.join(''));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
test('toDecodeStream - some stream', async function () {
|
test('toDecodeStream - some stream', async function () {
|
||||||
|
|
||||||
let source = newTestReadableStream([
|
let source = new Readable({
|
||||||
Buffer.from([65, 66, 67]),
|
read(size) {
|
||||||
Buffer.from([65, 66, 67]),
|
this.push(Buffer.from([65, 66, 67]));
|
||||||
Buffer.from([65, 66, 67]),
|
this.push(Buffer.from([65, 66, 67]));
|
||||||
]);
|
this.push(Buffer.from([65, 66, 67]));
|
||||||
|
this.push(null);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
let { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 4, guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 });
|
let { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 4, guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 });
|
||||||
|
|
||||||
|
@ -270,11 +267,14 @@ suite('Encoding', () => {
|
||||||
|
|
||||||
test('toDecodeStream - some stream, expect too much data', async function () {
|
test('toDecodeStream - some stream, expect too much data', async function () {
|
||||||
|
|
||||||
let source = newTestReadableStream([
|
let source = new Readable({
|
||||||
Buffer.from([65, 66, 67]),
|
read(size) {
|
||||||
Buffer.from([65, 66, 67]),
|
this.push(Buffer.from([65, 66, 67]));
|
||||||
Buffer.from([65, 66, 67]),
|
this.push(Buffer.from([65, 66, 67]));
|
||||||
]);
|
this.push(Buffer.from([65, 66, 67]));
|
||||||
|
this.push(null);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
let { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 64, guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 });
|
let { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 64, guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 });
|
||||||
|
|
||||||
|
@ -287,8 +287,11 @@ suite('Encoding', () => {
|
||||||
|
|
||||||
test('toDecodeStream - some stream, no data', async function () {
|
test('toDecodeStream - some stream, no data', async function () {
|
||||||
|
|
||||||
let source = newWriteableBufferStream();
|
let source = new Readable({
|
||||||
source.end();
|
read(size) {
|
||||||
|
this.push(null); // empty
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
let { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 512, guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 });
|
let { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 512, guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 });
|
||||||
|
|
||||||
|
@ -303,7 +306,7 @@ suite('Encoding', () => {
|
||||||
test('toDecodeStream - encoding, utf16be', async function () {
|
test('toDecodeStream - encoding, utf16be', async function () {
|
||||||
|
|
||||||
let path = getPathFromAmdModule(require, './fixtures/some_utf16be.css');
|
let path = getPathFromAmdModule(require, './fixtures/some_utf16be.css');
|
||||||
let source = streamToBufferReadableStream(fs.createReadStream(path));
|
let source = fs.createReadStream(path);
|
||||||
|
|
||||||
let { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 64, guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 });
|
let { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 64, guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 });
|
||||||
|
|
||||||
|
@ -319,91 +322,11 @@ suite('Encoding', () => {
|
||||||
test('toDecodeStream - empty file', async function () {
|
test('toDecodeStream - empty file', async function () {
|
||||||
|
|
||||||
let path = getPathFromAmdModule(require, './fixtures/empty.txt');
|
let path = getPathFromAmdModule(require, './fixtures/empty.txt');
|
||||||
let source = streamToBufferReadableStream(fs.createReadStream(path));
|
let source = fs.createReadStream(path);
|
||||||
let { detected, stream } = await encoding.toDecodeStream(source, { guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 });
|
let { detected, stream } = await encoding.toDecodeStream(source, { guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 });
|
||||||
|
|
||||||
let expected = await readAndDecodeFromDisk(path, detected.encoding);
|
let expected = await readAndDecodeFromDisk(path, detected.encoding);
|
||||||
let actual = await readAllAsString(stream);
|
let actual = await readAllAsString(stream);
|
||||||
assert.equal(actual, expected);
|
assert.equal(actual, expected);
|
||||||
});
|
});
|
||||||
|
|
||||||
test('toDecodeStream - decodes buffer entirely', async function () {
|
|
||||||
let emojis = Buffer.from('🖥️💻💾');
|
|
||||||
let incompleteEmojis = emojis.slice(0, emojis.length - 1);
|
|
||||||
|
|
||||||
let buffers = [];
|
|
||||||
for (let i = 0; i < incompleteEmojis.length; i++) {
|
|
||||||
buffers.push(incompleteEmojis.slice(i, i + 1));
|
|
||||||
}
|
|
||||||
|
|
||||||
const source = newTestReadableStream(buffers);
|
|
||||||
let { stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 4, guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 });
|
|
||||||
|
|
||||||
let expected = incompleteEmojis.toString(encoding.UTF8);
|
|
||||||
let actual = await readAllAsString(stream);
|
|
||||||
|
|
||||||
assert.equal(actual, expected);
|
|
||||||
});
|
|
||||||
|
|
||||||
test('toEncodeReadable - encoding, utf16be', async function () {
|
|
||||||
|
|
||||||
let path = getPathFromAmdModule(require, './fixtures/some_utf16be.css');
|
|
||||||
let source = await readAndDecodeFromDisk(path, encoding.UTF16be);
|
|
||||||
|
|
||||||
let expected = VSBuffer.wrap(
|
|
||||||
iconv.encode(source, encoding.toNodeEncoding(encoding.UTF16be))
|
|
||||||
).toString();
|
|
||||||
let actual = streams.consumeReadable(
|
|
||||||
encoding.toEncodeReadable(streams.toReadable(source), encoding.UTF16be),
|
|
||||||
VSBuffer.concat
|
|
||||||
).toString();
|
|
||||||
|
|
||||||
assert.equal(actual, expected);
|
|
||||||
});
|
|
||||||
|
|
||||||
test('toEncodeReadable - empty readable to utf8', async function () {
|
|
||||||
|
|
||||||
const source: streams.Readable<string> = {
|
|
||||||
read() {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
let actual = streams.consumeReadable(
|
|
||||||
encoding.toEncodeReadable(source, encoding.UTF8),
|
|
||||||
VSBuffer.concat
|
|
||||||
).toString();
|
|
||||||
|
|
||||||
assert.equal(actual, '');
|
|
||||||
});
|
|
||||||
|
|
||||||
[{
|
|
||||||
utfEncoding: encoding.UTF8,
|
|
||||||
relatedBom: encoding.UTF8_BOM
|
|
||||||
}, {
|
|
||||||
utfEncoding: encoding.UTF8_with_bom,
|
|
||||||
relatedBom: encoding.UTF8_BOM
|
|
||||||
}, {
|
|
||||||
utfEncoding: encoding.UTF16be,
|
|
||||||
relatedBom: encoding.UTF16be_BOM,
|
|
||||||
}, {
|
|
||||||
utfEncoding: encoding.UTF16le,
|
|
||||||
relatedBom: encoding.UTF16le_BOM
|
|
||||||
}].forEach(({ utfEncoding, relatedBom }) => {
|
|
||||||
test(`toEncodeReadable - empty readable to ${utfEncoding} with BOM`, async function () {
|
|
||||||
|
|
||||||
const source: streams.Readable<string> = {
|
|
||||||
read() {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
let encodedReadable = encoding.toEncodeReadable(source, utfEncoding, { addBOM: true });
|
|
||||||
|
|
||||||
const expected = VSBuffer.wrap(Buffer.from(relatedBom)).toString();
|
|
||||||
const actual = streams.consumeReadable(encodedReadable, VSBuffer.concat).toString();
|
|
||||||
|
|
||||||
assert.equal(actual, expected);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
});
|
||||||
|
|
|
@ -17,15 +17,16 @@ import { isMacintosh } from 'vs/base/common/platform';
|
||||||
import { IProductService } from 'vs/platform/product/common/productService';
|
import { IProductService } from 'vs/platform/product/common/productService';
|
||||||
import { ITextResourceConfigurationService } from 'vs/editor/common/services/textResourceConfigurationService';
|
import { ITextResourceConfigurationService } from 'vs/editor/common/services/textResourceConfigurationService';
|
||||||
import { IWorkspaceContextService } from 'vs/platform/workspace/common/workspace';
|
import { IWorkspaceContextService } from 'vs/platform/workspace/common/workspace';
|
||||||
import { UTF8, UTF8_with_bom, UTF16be, UTF16le, encodingExists, UTF8_BOM, toDecodeStream, toEncodeReadable, IDecodeStreamResult, detectEncodingByBOMFromBuffer } from 'vs/base/node/encoding';
|
import { UTF8, UTF8_with_bom, UTF16be, UTF16le, encodingExists, encodeStream, UTF8_BOM, toDecodeStream, IDecodeStreamResult, detectEncodingByBOMFromBuffer, isUTFEncoding } from 'vs/base/node/encoding';
|
||||||
import { WORKSPACE_EXTENSION } from 'vs/platform/workspaces/common/workspaces';
|
import { WORKSPACE_EXTENSION } from 'vs/platform/workspaces/common/workspaces';
|
||||||
import { joinPath, extname, isEqualOrParent } from 'vs/base/common/resources';
|
import { joinPath, extname, isEqualOrParent } from 'vs/base/common/resources';
|
||||||
import { Disposable } from 'vs/base/common/lifecycle';
|
import { Disposable } from 'vs/base/common/lifecycle';
|
||||||
import { IEnvironmentService } from 'vs/platform/environment/common/environment';
|
import { IEnvironmentService } from 'vs/platform/environment/common/environment';
|
||||||
import { bufferToStream, VSBufferReadable } from 'vs/base/common/buffer';
|
import { VSBufferReadable, bufferToStream } from 'vs/base/common/buffer';
|
||||||
|
import { Readable } from 'stream';
|
||||||
import { createTextBufferFactoryFromStream } from 'vs/editor/common/model/textModel';
|
import { createTextBufferFactoryFromStream } from 'vs/editor/common/model/textModel';
|
||||||
import { ITextSnapshot } from 'vs/editor/common/model';
|
import { ITextSnapshot } from 'vs/editor/common/model';
|
||||||
import { consumeStream } from 'vs/base/common/stream';
|
import { nodeReadableToString, streamToNodeReadable, nodeStreamToVSBufferReadable } from 'vs/base/node/stream';
|
||||||
import { IUntitledTextEditorService } from 'vs/workbench/services/untitled/common/untitledTextEditorService';
|
import { IUntitledTextEditorService } from 'vs/workbench/services/untitled/common/untitledTextEditorService';
|
||||||
import { ILifecycleService } from 'vs/platform/lifecycle/common/lifecycle';
|
import { ILifecycleService } from 'vs/platform/lifecycle/common/lifecycle';
|
||||||
import { IInstantiationService } from 'vs/platform/instantiation/common/instantiation';
|
import { IInstantiationService } from 'vs/platform/instantiation/common/instantiation';
|
||||||
|
@ -88,7 +89,7 @@ export class NativeTextFileService extends AbstractTextFileService {
|
||||||
return {
|
return {
|
||||||
...bufferStream,
|
...bufferStream,
|
||||||
encoding: decoder.detected.encoding || UTF8,
|
encoding: decoder.detected.encoding || UTF8,
|
||||||
value: await consumeStream(decoder.stream, strings => strings.join(''))
|
value: await nodeReadableToString(decoder.stream)
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -120,7 +121,7 @@ export class NativeTextFileService extends AbstractTextFileService {
|
||||||
}
|
}
|
||||||
|
|
||||||
// read through encoding library
|
// read through encoding library
|
||||||
const decoder = await toDecodeStream(bufferStream.value, {
|
const decoder = await toDecodeStream(streamToNodeReadable(bufferStream.value), {
|
||||||
guessEncoding: options?.autoGuessEncoding || this.textResourceConfigurationService.getValue(resource, 'files.autoGuessEncoding'),
|
guessEncoding: options?.autoGuessEncoding || this.textResourceConfigurationService.getValue(resource, 'files.autoGuessEncoding'),
|
||||||
overwriteEncoding: detectedEncoding => this.encoding.getReadEncoding(resource, options, detectedEncoding)
|
overwriteEncoding: detectedEncoding => this.encoding.getReadEncoding(resource, options, detectedEncoding)
|
||||||
});
|
});
|
||||||
|
@ -231,8 +232,37 @@ export class NativeTextFileService extends AbstractTextFileService {
|
||||||
}
|
}
|
||||||
|
|
||||||
private getEncodedReadable(value: string | ITextSnapshot, encoding: string, addBOM: boolean): VSBufferReadable {
|
private getEncodedReadable(value: string | ITextSnapshot, encoding: string, addBOM: boolean): VSBufferReadable {
|
||||||
const snapshot = typeof value === 'string' ? stringToSnapshot(value) : value;
|
const readable = this.snapshotToNodeReadable(typeof value === 'string' ? stringToSnapshot(value) : value);
|
||||||
return toEncodeReadable(snapshot, encoding, { addBOM });
|
const encoder = encodeStream(encoding, { addBOM });
|
||||||
|
|
||||||
|
const encodedReadable = readable.pipe(encoder);
|
||||||
|
|
||||||
|
return nodeStreamToVSBufferReadable(encodedReadable, addBOM && isUTFEncoding(encoding) ? { encoding } : undefined);
|
||||||
|
}
|
||||||
|
|
||||||
|
private snapshotToNodeReadable(snapshot: ITextSnapshot): Readable {
|
||||||
|
return new Readable({
|
||||||
|
read: function () {
|
||||||
|
try {
|
||||||
|
let chunk: string | null = null;
|
||||||
|
let canPush = true;
|
||||||
|
|
||||||
|
// Push all chunks as long as we can push and as long as
|
||||||
|
// the underlying snapshot returns strings to us
|
||||||
|
while (canPush && typeof (chunk = snapshot.read()) === 'string') {
|
||||||
|
canPush = this.push(chunk);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Signal EOS by pushing NULL
|
||||||
|
if (typeof chunk !== 'string') {
|
||||||
|
this.push(null);
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
this.emit('error', error);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
encoding: UTF8 // very important, so that strings are passed around and not buffers!
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
private async writeElevated(resource: URI, value: string | ITextSnapshot, options?: IWriteTextFileOptions): Promise<IFileStatWithMetadata> {
|
private async writeElevated(resource: URI, value: string | ITextSnapshot, options?: IWriteTextFileOptions): Promise<IFileStatWithMetadata> {
|
||||||
|
|
|
@ -4579,10 +4579,10 @@ husky@^0.13.1:
|
||||||
is-ci "^1.0.9"
|
is-ci "^1.0.9"
|
||||||
normalize-path "^1.0.0"
|
normalize-path "^1.0.0"
|
||||||
|
|
||||||
iconv-lite@0.6.0:
|
iconv-lite@0.5.0:
|
||||||
version "0.6.0"
|
version "0.5.0"
|
||||||
resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.0.tgz#66a93b80df0bd05d2a43a7426296b7f91073f125"
|
resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.5.0.tgz#59cdde0a2a297cc2aeb0c6445a195ee89f127550"
|
||||||
integrity sha512-43ZpGYZ9QtuutX5l6WC1DSO8ane9N+Ct5qPLF2OV7vM9abM69gnAbVkh66ibaZd3aOGkoP1ZmringlKhLBkw2Q==
|
integrity sha512-NnEhI9hIEKHOzJ4f697DMz9IQEXr/MMJ5w64vN2/4Ai+wRnvV7SBrL0KLoRlwaKVghOc7LQ5YkPLuX146b6Ydw==
|
||||||
dependencies:
|
dependencies:
|
||||||
safer-buffer ">= 2.1.2 < 3"
|
safer-buffer ">= 2.1.2 < 3"
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue