mirror of
https://github.com/Microsoft/vscode
synced 2024-07-17 02:57:19 +00:00
Compress streams in notebook outputs (#160946)
* Revert "Compress notebook output streams before rendering (#160667)"
This reverts commit 4230c22a08.
* Compress stream output items
* Minor perf improvements
* Misc
* Comments
* Added tests
* Merge issue
* More merge issues
* Misc
* Address code review comments
This commit is contained in:
Don Jayamanne
GitHub
parent
4322170fd8
commit
43957ccfe1
|
|
@ -7,7 +7,6 @@ import type * as nbformat from '@jupyterlab/nbformat';
|
|||
import { NotebookCell, NotebookCellData, NotebookCellKind, NotebookCellOutput } from 'vscode';
|
||||
import { CellOutputMetadata } from './common';
|
||||
import { textMimeTypes } from './deserializers';
|
||||
import { compressOutputItemStreams } from './streamCompressor';
|
||||
|
||||
const textDecoder = new TextDecoder();
|
||||
|
||||
|
|
@ -277,17 +276,21 @@ type JupyterOutput =
|
|||
|
||||
function convertStreamOutput(output: NotebookCellOutput): JupyterOutput {
|
||||
const outputs: string[] = [];
|
||||
const compressedStream = output.items.length ? new TextDecoder().decode(compressOutputItemStreams(output.items[0].mime, output.items)) : '';
|
||||
// Ensure each line is a separate entry in an array (ending with \n).
|
||||
const lines = compressedStream.split('\n');
|
||||
// If the last item in `outputs` is not empty and the first item in `lines` is not empty, then concate them.
|
||||
// As they are part of the same line.
|
||||
if (outputs.length && lines.length && lines[0].length > 0) {
|
||||
outputs[outputs.length - 1] = `${outputs[outputs.length - 1]}${lines.shift()!}`;
|
||||
}
|
||||
for (const line of lines) {
|
||||
outputs.push(line);
|
||||
}
|
||||
output.items
|
||||
.filter((opit) => opit.mime === CellOutputMimeTypes.stderr || opit.mime === CellOutputMimeTypes.stdout)
|
||||
.map((opit) => textDecoder.decode(opit.data))
|
||||
.forEach(value => {
|
||||
// Ensure each line is a separate entry in an array (ending with \n).
|
||||
const lines = value.split('\n');
|
||||
// If the last item in `outputs` is not empty and the first item in `lines` is not empty, then concate them.
|
||||
// As they are part of the same line.
|
||||
if (outputs.length && lines.length && lines[0].length > 0) {
|
||||
outputs[outputs.length - 1] = `${outputs[outputs.length - 1]}${lines.shift()!}`;
|
||||
}
|
||||
for (const line of lines) {
|
||||
outputs.push(line);
|
||||
}
|
||||
});
|
||||
|
||||
for (let index = 0; index < (outputs.length - 1); index++) {
|
||||
outputs[index] = `${outputs[index]}\n`;
|
||||
|
|
|
|||
|
|
@ -1,63 +0,0 @@
|
|||
/*---------------------------------------------------------------------------------------------
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License. See License.txt in the project root for license information.
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
import type { NotebookCellOutputItem } from 'vscode';
|
||||
|
||||
|
||||
/**
|
||||
* Given a stream of individual stdout outputs, this function will return the compressed lines, escaping some of the common terminal escape codes.
|
||||
* E.g. some terminal escape codes would result in the previous line getting cleared, such if we had 3 lines and
|
||||
* last line contained such a code, then the result string would be just the first two lines.
|
||||
*/
|
||||
export function compressOutputItemStreams(mimeType: string, outputs: NotebookCellOutputItem[]) {
|
||||
// return outputs.find(op => op.mime === mimeType)!.data.buffer;
|
||||
|
||||
const buffers: Uint8Array[] = [];
|
||||
let startAppending = false;
|
||||
// Pick the first set of outputs with the same mime type.
|
||||
for (const output of outputs) {
|
||||
if (output.mime === mimeType) {
|
||||
if ((buffers.length === 0 || startAppending)) {
|
||||
buffers.push(output.data);
|
||||
startAppending = true;
|
||||
}
|
||||
} else if (startAppending) {
|
||||
startAppending = false;
|
||||
}
|
||||
}
|
||||
compressStreamBuffer(buffers);
|
||||
const totalBytes = buffers.reduce((p, c) => p + c.byteLength, 0);
|
||||
const combinedBuffer = new Uint8Array(totalBytes);
|
||||
let offset = 0;
|
||||
for (const buffer of buffers) {
|
||||
combinedBuffer.set(buffer, offset);
|
||||
offset = offset + buffer.byteLength;
|
||||
}
|
||||
return combinedBuffer;
|
||||
}
|
||||
const MOVE_CURSOR_1_LINE_COMMAND = `${String.fromCharCode(27)}[A`;
|
||||
const MOVE_CURSOR_1_LINE_COMMAND_BYTES = MOVE_CURSOR_1_LINE_COMMAND.split('').map(c => c.charCodeAt(0));
|
||||
const LINE_FEED = 10;
|
||||
function compressStreamBuffer(streams: Uint8Array[]) {
|
||||
streams.forEach((stream, index) => {
|
||||
if (index === 0 || stream.length < MOVE_CURSOR_1_LINE_COMMAND.length) {
|
||||
return;
|
||||
}
|
||||
|
||||
const previousStream = streams[index - 1];
|
||||
|
||||
// Remove the previous line if required.
|
||||
const command = stream.subarray(0, MOVE_CURSOR_1_LINE_COMMAND.length);
|
||||
if (command[0] === MOVE_CURSOR_1_LINE_COMMAND_BYTES[0] && command[1] === MOVE_CURSOR_1_LINE_COMMAND_BYTES[1] && command[2] === MOVE_CURSOR_1_LINE_COMMAND_BYTES[2]) {
|
||||
const lastIndexOfLineFeed = previousStream.lastIndexOf(LINE_FEED);
|
||||
if (lastIndexOfLineFeed === -1) {
|
||||
return;
|
||||
}
|
||||
streams[index - 1] = previousStream.subarray(0, lastIndexOfLineFeed);
|
||||
streams[index] = stream.subarray(MOVE_CURSOR_1_LINE_COMMAND.length);
|
||||
}
|
||||
});
|
||||
return streams;
|
||||
}
|
||||
|
|
@ -110,6 +110,32 @@ export class ExtHostCell {
|
|||
output.items.length = 0;
|
||||
}
|
||||
output.items.push(...newItems);
|
||||
|
||||
if (output.items.length > 1 && output.items.every(item => notebookCommon.isTextStreamMime(item.mime))) {
|
||||
// Look for the mimes in the items, and keep track of their order.
|
||||
// Merge the streams into one output item, per mime type.
|
||||
const mimeOutputs = new Map<string, Uint8Array[]>();
|
||||
const mimeTypes: string[] = [];
|
||||
output.items.forEach(item => {
|
||||
let items: Uint8Array[];
|
||||
if (mimeOutputs.has(item.mime)) {
|
||||
items = mimeOutputs.get(item.mime)!;
|
||||
} else {
|
||||
items = [];
|
||||
mimeOutputs.set(item.mime, items);
|
||||
mimeTypes.push(item.mime);
|
||||
}
|
||||
items.push(item.data);
|
||||
});
|
||||
output.items.length = 0;
|
||||
mimeTypes.forEach(mime => {
|
||||
const compressed = notebookCommon.compressOutputItemStreams(mimeOutputs.get(mime)!);
|
||||
output.items.push({
|
||||
mime,
|
||||
data: compressed.buffer
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import { TestRPCProtocol } from 'vs/workbench/api/test/common/testRPCProtocol';
|
|||
import { DisposableStore } from 'vs/base/common/lifecycle';
|
||||
import { NullLogService } from 'vs/platform/log/common/log';
|
||||
import { mock } from 'vs/base/test/common/mock';
|
||||
import { IModelAddedData, MainContext, MainThreadCommandsShape, MainThreadNotebookShape } from 'vs/workbench/api/common/extHost.protocol';
|
||||
import { IModelAddedData, MainContext, MainThreadCommandsShape, MainThreadNotebookShape, NotebookCellsChangedEventDto, NotebookOutputItemDto } from 'vs/workbench/api/common/extHost.protocol';
|
||||
import { ExtHostNotebookController } from 'vs/workbench/api/common/extHostNotebook';
|
||||
import { ExtHostNotebookDocument } from 'vs/workbench/api/common/extHostNotebookDocument';
|
||||
import { CellKind, CellUri, NotebookCellsChangeType } from 'vs/workbench/contrib/notebook/common/notebookCommon';
|
||||
|
|
@ -542,4 +542,77 @@ suite('NotebookCell#Document', function () {
|
|||
assert.ok(cellChange.metadata === undefined);
|
||||
assert.ok(cellChange.outputs === undefined);
|
||||
});
|
||||
|
||||
async function replaceOutputs(cellIndex: number, outputId: string, outputItems: NotebookOutputItemDto[]) {
|
||||
const changeEvent = Event.toPromise(extHostNotebookDocuments.onDidChangeNotebookDocument);
|
||||
extHostNotebookDocuments.$acceptModelChanged(notebook.uri, new SerializableObjectWithBuffers<NotebookCellsChangedEventDto>({
|
||||
versionId: notebook.apiNotebook.version + 1,
|
||||
rawEvents: [{
|
||||
kind: NotebookCellsChangeType.Output,
|
||||
index: cellIndex,
|
||||
outputs: [{ outputId, items: outputItems }]
|
||||
}]
|
||||
}), false);
|
||||
await changeEvent;
|
||||
}
|
||||
async function appendOutputItem(cellIndex: number, outputId: string, outputItems: NotebookOutputItemDto[]) {
|
||||
const changeEvent = Event.toPromise(extHostNotebookDocuments.onDidChangeNotebookDocument);
|
||||
extHostNotebookDocuments.$acceptModelChanged(notebook.uri, new SerializableObjectWithBuffers<NotebookCellsChangedEventDto>({
|
||||
versionId: notebook.apiNotebook.version + 1,
|
||||
rawEvents: [{
|
||||
kind: NotebookCellsChangeType.OutputItem,
|
||||
index: cellIndex,
|
||||
append: true,
|
||||
outputId,
|
||||
outputItems
|
||||
}]
|
||||
}), false);
|
||||
await changeEvent;
|
||||
}
|
||||
test('Append multiple text/plain output items', async function () {
|
||||
await replaceOutputs(1, '1', [{ mime: 'text/plain', valueBytes: VSBuffer.fromString('foo') }]);
|
||||
await appendOutputItem(1, '1', [{ mime: 'text/plain', valueBytes: VSBuffer.fromString('bar') }]);
|
||||
await appendOutputItem(1, '1', [{ mime: 'text/plain', valueBytes: VSBuffer.fromString('baz') }]);
|
||||
|
||||
|
||||
assert.strictEqual(notebook.apiNotebook.cellAt(1).outputs.length, 1);
|
||||
assert.strictEqual(notebook.apiNotebook.cellAt(1).outputs[0].items.length, 3);
|
||||
assert.strictEqual(notebook.apiNotebook.cellAt(1).outputs[0].items[0].mime, 'text/plain');
|
||||
assert.strictEqual(VSBuffer.wrap(notebook.apiNotebook.cellAt(1).outputs[0].items[0].data).toString(), 'foo');
|
||||
assert.strictEqual(notebook.apiNotebook.cellAt(1).outputs[0].items[1].mime, 'text/plain');
|
||||
assert.strictEqual(VSBuffer.wrap(notebook.apiNotebook.cellAt(1).outputs[0].items[1].data).toString(), 'bar');
|
||||
assert.strictEqual(notebook.apiNotebook.cellAt(1).outputs[0].items[2].mime, 'text/plain');
|
||||
assert.strictEqual(VSBuffer.wrap(notebook.apiNotebook.cellAt(1).outputs[0].items[2].data).toString(), 'baz');
|
||||
});
|
||||
test('Append multiple stdout stream output items to an output with another mime', async function () {
|
||||
await replaceOutputs(1, '1', [{ mime: 'text/plain', valueBytes: VSBuffer.fromString('foo') }]);
|
||||
await appendOutputItem(1, '1', [{ mime: 'application/vnd.code.notebook.stdout', valueBytes: VSBuffer.fromString('bar') }]);
|
||||
await appendOutputItem(1, '1', [{ mime: 'application/vnd.code.notebook.stdout', valueBytes: VSBuffer.fromString('baz') }]);
|
||||
|
||||
assert.strictEqual(notebook.apiNotebook.cellAt(1).outputs.length, 1);
|
||||
assert.strictEqual(notebook.apiNotebook.cellAt(1).outputs[0].items.length, 3);
|
||||
assert.strictEqual(notebook.apiNotebook.cellAt(1).outputs[0].items[0].mime, 'text/plain');
|
||||
assert.strictEqual(notebook.apiNotebook.cellAt(1).outputs[0].items[1].mime, 'application/vnd.code.notebook.stdout');
|
||||
assert.strictEqual(notebook.apiNotebook.cellAt(1).outputs[0].items[2].mime, 'application/vnd.code.notebook.stdout');
|
||||
});
|
||||
test('Compress multiple stdout stream output items', async function () {
|
||||
await replaceOutputs(1, '1', [{ mime: 'application/vnd.code.notebook.stdout', valueBytes: VSBuffer.fromString('foo') }]);
|
||||
await appendOutputItem(1, '1', [{ mime: 'application/vnd.code.notebook.stdout', valueBytes: VSBuffer.fromString('bar') }]);
|
||||
await appendOutputItem(1, '1', [{ mime: 'application/vnd.code.notebook.stdout', valueBytes: VSBuffer.fromString('baz') }]);
|
||||
|
||||
assert.strictEqual(notebook.apiNotebook.cellAt(1).outputs.length, 1);
|
||||
assert.strictEqual(notebook.apiNotebook.cellAt(1).outputs[0].items.length, 1);
|
||||
assert.strictEqual(notebook.apiNotebook.cellAt(1).outputs[0].items[0].mime, 'application/vnd.code.notebook.stdout');
|
||||
assert.strictEqual(VSBuffer.wrap(notebook.apiNotebook.cellAt(1).outputs[0].items[0].data).toString(), 'foobarbaz');
|
||||
});
|
||||
test('Compress multiple stderr stream output items', async function () {
|
||||
await replaceOutputs(1, '1', [{ mime: 'application/vnd.code.notebook.stderr', valueBytes: VSBuffer.fromString('foo') }]);
|
||||
await appendOutputItem(1, '1', [{ mime: 'application/vnd.code.notebook.stderr', valueBytes: VSBuffer.fromString('bar') }]);
|
||||
await appendOutputItem(1, '1', [{ mime: 'application/vnd.code.notebook.stderr', valueBytes: VSBuffer.fromString('baz') }]);
|
||||
|
||||
assert.strictEqual(notebook.apiNotebook.cellAt(1).outputs.length, 1);
|
||||
assert.strictEqual(notebook.apiNotebook.cellAt(1).outputs[0].items.length, 1);
|
||||
assert.strictEqual(notebook.apiNotebook.cellAt(1).outputs[0].items[0].mime, 'application/vnd.code.notebook.stderr');
|
||||
assert.strictEqual(VSBuffer.wrap(notebook.apiNotebook.cellAt(1).outputs[0].items[0].data).toString(), 'foobarbaz');
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ import { NOTEBOOK_WEBVIEW_BOUNDARY } from 'vs/workbench/contrib/notebook/browser
|
|||
import { preloadsScriptStr } from 'vs/workbench/contrib/notebook/browser/view/renderers/webviewPreloads';
|
||||
import { transformWebviewThemeVars } from 'vs/workbench/contrib/notebook/browser/view/renderers/webviewThemeMapping';
|
||||
import { MarkupCellViewModel } from 'vs/workbench/contrib/notebook/browser/viewModel/markupCellViewModel';
|
||||
import { CellUri, INotebookRendererInfo, isTextStreamMime, NotebookSetting, RendererMessagingSpec } from 'vs/workbench/contrib/notebook/common/notebookCommon';
|
||||
import { CellUri, INotebookRendererInfo, NotebookSetting, RendererMessagingSpec } from 'vs/workbench/contrib/notebook/common/notebookCommon';
|
||||
import { INotebookKernel } from 'vs/workbench/contrib/notebook/common/notebookKernelService';
|
||||
import { IScopedRendererMessaging } from 'vs/workbench/contrib/notebook/common/notebookRendererMessagingService';
|
||||
import { INotebookService } from 'vs/workbench/contrib/notebook/common/notebookService';
|
||||
|
|
@ -44,7 +44,6 @@ import { WebviewWindowDragMonitor } from 'vs/workbench/contrib/webview/browser/w
|
|||
import { IEditorGroupsService } from 'vs/workbench/services/editor/common/editorGroupsService';
|
||||
import { IWorkbenchEnvironmentService } from 'vs/workbench/services/environment/common/environmentService';
|
||||
import { FromWebviewMessage, IAckOutputHeight, IClickedDataUrlMessage, ICodeBlockHighlightRequest, IContentWidgetTopRequest, IControllerPreload, ICreationContent, ICreationRequestMessage, IFindMatch, IMarkupCellInitialization, RendererMetadata, ToWebviewMessage } from './webviewMessages';
|
||||
import { compressOutputItemStreams } from 'vs/workbench/contrib/notebook/browser/view/renderers/stdOutErrorPreProcessor';
|
||||
import { DeferredPromise } from 'vs/base/common/async';
|
||||
|
||||
export interface ICachedInset<K extends ICommonCellInfo> {
|
||||
|
|
@ -1277,14 +1276,12 @@ var requirejs = (function() {
|
|||
let updatedContent: ICreationContent | undefined = undefined;
|
||||
if (content.type === RenderOutputType.Extension) {
|
||||
const output = content.source.model;
|
||||
const firstBuffer = isTextStreamMime(content.mimeType) ?
|
||||
compressOutputItemStreams(content.mimeType, output.outputs) :
|
||||
output.outputs.find(op => op.mime === content.mimeType)!.data.buffer;
|
||||
const firstBuffer = output.outputs.find(op => op.mime === content.mimeType)!.data;
|
||||
updatedContent = {
|
||||
type: RenderOutputType.Extension,
|
||||
outputId: outputCache.outputId,
|
||||
mimeType: content.mimeType,
|
||||
valueBytes: firstBuffer,
|
||||
valueBytes: firstBuffer.buffer,
|
||||
metadata: output.metadata,
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,56 +0,0 @@
|
|||
/*---------------------------------------------------------------------------------------------
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License. See License.txt in the project root for license information.
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
import { VSBuffer } from 'vs/base/common/buffer';
|
||||
import type { IOutputItemDto } from 'vs/workbench/contrib/notebook/common/notebookCommon';
|
||||
|
||||
|
||||
/**
|
||||
* Given a stream of individual stdout outputs, this function will return the compressed lines, escaping some of the common terminal escape codes.
|
||||
* E.g. some terminal escape codes would result in the previous line getting cleared, such if we had 3 lines and
|
||||
* last line contained such a code, then the result string would be just the first two lines.
|
||||
*/
|
||||
export function compressOutputItemStreams(mimeType: string, outputs: IOutputItemDto[]) {
|
||||
const buffers: Uint8Array[] = [];
|
||||
let startAppending = false;
|
||||
|
||||
// Pick the first set of outputs with the same mime type.
|
||||
for (const output of outputs) {
|
||||
if (output.mime === mimeType) {
|
||||
if ((buffers.length === 0 || startAppending)) {
|
||||
buffers.push(output.data.buffer);
|
||||
startAppending = true;
|
||||
}
|
||||
} else if (startAppending) {
|
||||
startAppending = false;
|
||||
}
|
||||
}
|
||||
compressStreamBuffer(buffers);
|
||||
return VSBuffer.concat(buffers.map(buffer => VSBuffer.wrap(buffer))).buffer;
|
||||
}
|
||||
const MOVE_CURSOR_1_LINE_COMMAND = `${String.fromCharCode(27)}[A`;
|
||||
const MOVE_CURSOR_1_LINE_COMMAND_BYTES = MOVE_CURSOR_1_LINE_COMMAND.split('').map(c => c.charCodeAt(0));
|
||||
const LINE_FEED = 10;
|
||||
function compressStreamBuffer(streams: Uint8Array[]) {
|
||||
streams.forEach((stream, index) => {
|
||||
if (index === 0 || stream.length < MOVE_CURSOR_1_LINE_COMMAND.length) {
|
||||
return;
|
||||
}
|
||||
|
||||
const previousStream = streams[index - 1];
|
||||
|
||||
// Remove the previous line if required.
|
||||
const command = stream.subarray(0, MOVE_CURSOR_1_LINE_COMMAND.length);
|
||||
if (command[0] === MOVE_CURSOR_1_LINE_COMMAND_BYTES[0] && command[1] === MOVE_CURSOR_1_LINE_COMMAND_BYTES[1] && command[2] === MOVE_CURSOR_1_LINE_COMMAND_BYTES[2]) {
|
||||
const lastIndexOfLineFeed = previousStream.lastIndexOf(LINE_FEED);
|
||||
if (lastIndexOfLineFeed === -1) {
|
||||
return;
|
||||
}
|
||||
streams[index - 1] = previousStream.subarray(0, lastIndexOfLineFeed);
|
||||
streams[index] = stream.subarray(MOVE_CURSOR_1_LINE_COMMAND.length);
|
||||
}
|
||||
});
|
||||
return streams;
|
||||
}
|
||||
|
|
@ -16,7 +16,7 @@ import { TextModel } from 'vs/editor/common/model/textModel';
|
|||
import { PLAINTEXT_LANGUAGE_ID } from 'vs/editor/common/languages/modesRegistry';
|
||||
import { ILanguageService } from 'vs/editor/common/languages/language';
|
||||
import { NotebookCellOutputTextModel } from 'vs/workbench/contrib/notebook/common/model/notebookCellOutputTextModel';
|
||||
import { CellInternalMetadataChangedEvent, CellKind, ICell, ICellDto2, ICellOutput, IOutputDto, IOutputItemDto, NotebookCellCollapseState, NotebookCellInternalMetadata, NotebookCellMetadata, NotebookCellOutputsSplice, TransientOptions } from 'vs/workbench/contrib/notebook/common/notebookCommon';
|
||||
import { CellInternalMetadataChangedEvent, CellKind, compressOutputItemStreams, ICell, ICellDto2, ICellOutput, IOutputDto, IOutputItemDto, isTextStreamMime, NotebookCellCollapseState, NotebookCellInternalMetadata, NotebookCellMetadata, NotebookCellOutputsSplice, TransientOptions } from 'vs/workbench/contrib/notebook/common/notebookCommon';
|
||||
|
||||
export class NotebookCellTextModel extends Disposable implements ICell {
|
||||
private readonly _onDidChangeOutputs = this._register(new Emitter<NotebookCellOutputsSplice>());
|
||||
|
|
@ -297,6 +297,31 @@ export class NotebookCellTextModel extends Disposable implements ICell {
|
|||
} else {
|
||||
output.replaceData(items);
|
||||
}
|
||||
if (output.outputs.length > 1 && output.outputs.every(item => isTextStreamMime(item.mime))) {
|
||||
// Look for the mimes in the items, and keep track of their order.
|
||||
// Merge the streams into one output item, per mime type.
|
||||
const mimeOutputs = new Map<string, Uint8Array[]>();
|
||||
const mimeTypes: string[] = [];
|
||||
output.outputs.forEach(item => {
|
||||
let items: Uint8Array[];
|
||||
if (mimeOutputs.has(item.mime)) {
|
||||
items = mimeOutputs.get(item.mime)!;
|
||||
} else {
|
||||
items = [];
|
||||
mimeOutputs.set(item.mime, items);
|
||||
mimeTypes.push(item.mime);
|
||||
}
|
||||
items.push(item.data.buffer);
|
||||
});
|
||||
output.outputs.length = 0;
|
||||
mimeTypes.forEach(mime => {
|
||||
const compressed = compressOutputItemStreams(mimeOutputs.get(mime)!);
|
||||
output.outputs.push({
|
||||
mime,
|
||||
data: compressed
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
this._onDidChangeOutputItems.fire();
|
||||
return true;
|
||||
|
|
|
|||
|
|
@ -951,9 +951,98 @@ export interface NotebookExtensionDescription {
|
|||
}
|
||||
|
||||
/**
|
||||
* Whether the provided mime type is a text streamn like `stdout`, `stderr`.
|
||||
* Whether the provided mime type is a text stream like `stdout`, `stderr`.
|
||||
*/
|
||||
export function isTextStreamMime(mimeType: string) {
|
||||
return ['application/vnd.code.notebook.stdout', 'application/x.notebook.stdout', 'application/x.notebook.stream', 'application/vnd.code.notebook.stderr', 'application/x.notebook.stderr'].includes(mimeType);
|
||||
return ['application/vnd.code.notebook.stdout', 'application/vnd.code.notebook.stderr'].includes(mimeType);
|
||||
}
|
||||
|
||||
|
||||
const textDecoder = new TextDecoder();
|
||||
|
||||
/**
|
||||
* Given a stream of individual stdout outputs, this function will return the compressed lines, escaping some of the common terminal escape codes.
|
||||
* E.g. some terminal escape codes would result in the previous line getting cleared, such if we had 3 lines and
|
||||
* last line contained such a code, then the result string would be just the first two lines.
|
||||
*/
|
||||
export function compressOutputItemStreams(outputs: Uint8Array[]) {
|
||||
const buffers: Uint8Array[] = [];
|
||||
let startAppending = false;
|
||||
|
||||
// Pick the first set of outputs with the same mime type.
|
||||
for (const output of outputs) {
|
||||
if ((buffers.length === 0 || startAppending)) {
|
||||
buffers.push(output);
|
||||
startAppending = true;
|
||||
}
|
||||
}
|
||||
compressStreamBuffer(buffers);
|
||||
return formatStreamText(VSBuffer.concat(buffers.map(buffer => VSBuffer.wrap(buffer))));
|
||||
}
|
||||
const MOVE_CURSOR_1_LINE_COMMAND = `${String.fromCharCode(27)}[A`;
|
||||
const MOVE_CURSOR_1_LINE_COMMAND_BYTES = MOVE_CURSOR_1_LINE_COMMAND.split('').map(c => c.charCodeAt(0));
|
||||
const LINE_FEED = 10;
|
||||
function compressStreamBuffer(streams: Uint8Array[]) {
|
||||
streams.forEach((stream, index) => {
|
||||
if (index === 0 || stream.length < MOVE_CURSOR_1_LINE_COMMAND.length) {
|
||||
return;
|
||||
}
|
||||
|
||||
const previousStream = streams[index - 1];
|
||||
|
||||
// Remove the previous line if required.
|
||||
const command = stream.subarray(0, MOVE_CURSOR_1_LINE_COMMAND.length);
|
||||
if (command[0] === MOVE_CURSOR_1_LINE_COMMAND_BYTES[0] && command[1] === MOVE_CURSOR_1_LINE_COMMAND_BYTES[1] && command[2] === MOVE_CURSOR_1_LINE_COMMAND_BYTES[2]) {
|
||||
const lastIndexOfLineFeed = previousStream.lastIndexOf(LINE_FEED);
|
||||
if (lastIndexOfLineFeed === -1) {
|
||||
return;
|
||||
}
|
||||
streams[index - 1] = previousStream.subarray(0, lastIndexOfLineFeed);
|
||||
streams[index] = stream.subarray(MOVE_CURSOR_1_LINE_COMMAND.length);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Took this from jupyter/notebook
|
||||
* https://github.com/jupyter/notebook/blob/b8b66332e2023e83d2ee04f83d8814f567e01a4e/notebook/static/base/js/utils.js
|
||||
* Remove characters that are overridden by backspace characters
|
||||
*/
|
||||
function fixBackspace(txt: string) {
|
||||
let tmp = txt;
|
||||
do {
|
||||
txt = tmp;
|
||||
// Cancel out anything-but-newline followed by backspace
|
||||
tmp = txt.replace(/[^\n]\x08/gm, '');
|
||||
} while (tmp.length < txt.length);
|
||||
return txt;
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove chunks that should be overridden by the effect of carriage return characters
|
||||
* From https://github.com/jupyter/notebook/blob/master/notebook/static/base/js/utils.js
|
||||
*/
|
||||
function fixCarriageReturn(txt: string) {
|
||||
txt = txt.replace(/\r+\n/gm, '\n'); // \r followed by \n --> newline
|
||||
while (txt.search(/\r[^$]/g) > -1) {
|
||||
const base = txt.match(/^(.*)\r+/m)![1];
|
||||
let insert = txt.match(/\r+(.*)$/m)![1];
|
||||
insert = insert + base.slice(insert.length, base.length);
|
||||
txt = txt.replace(/\r+.*$/m, '\r').replace(/^.*\r/m, insert);
|
||||
}
|
||||
return txt;
|
||||
}
|
||||
|
||||
const BACKSPACE_CHARACTER = '\b'.charCodeAt(0);
|
||||
const CARRIAGE_RETURN_CHARACTER = '\r'.charCodeAt(0);
|
||||
function formatStreamText(buffer: VSBuffer): VSBuffer {
|
||||
// We have special handling for backspace and carriage return characters.
|
||||
// Don't unnecessary decode the bytes if we don't need to perform any processing.
|
||||
if (!buffer.buffer.includes(BACKSPACE_CHARACTER) && !buffer.buffer.includes(CARRIAGE_RETURN_CHARACTER)) {
|
||||
return buffer;
|
||||
}
|
||||
// Do the same thing jupyter is doing
|
||||
return VSBuffer.fromString(fixCarriageReturn(fixBackspace(textDecoder.decode(buffer.buffer))));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1043,4 +1043,110 @@ suite('NotebookTextModel', () => {
|
|||
assert.equal(model.cells[0].outputs[0].outputs[0].data.toString(), '_World_');
|
||||
});
|
||||
});
|
||||
test('Append multiple text/plain output items', async function () {
|
||||
await withTestNotebook([
|
||||
['var a = 1;', 'javascript', CellKind.Code, [{
|
||||
outputId: '1',
|
||||
outputs: [{ mime: 'text/plain', data: valueBytesFromString('foo') }]
|
||||
}], {}]
|
||||
], (editor) => {
|
||||
const model = editor.textModel;
|
||||
const edits: ICellEditOperation[] = [
|
||||
{
|
||||
editType: CellEditType.OutputItems,
|
||||
outputId: '1',
|
||||
append: true,
|
||||
items: [{ mime: 'text/plain', data: VSBuffer.fromString('bar') }, { mime: 'text/plain', data: VSBuffer.fromString('baz') }]
|
||||
}
|
||||
];
|
||||
model.applyEdits(edits, true, undefined, () => undefined, undefined, true);
|
||||
assert.equal(model.cells.length, 1);
|
||||
assert.equal(model.cells[0].outputs.length, 1);
|
||||
assert.equal(model.cells[0].outputs[0].outputs.length, 3);
|
||||
assert.equal(model.cells[0].outputs[0].outputs[0].mime, 'text/plain');
|
||||
assert.equal(model.cells[0].outputs[0].outputs[0].data.toString(), 'foo');
|
||||
assert.equal(model.cells[0].outputs[0].outputs[1].mime, 'text/plain');
|
||||
assert.equal(model.cells[0].outputs[0].outputs[1].data.toString(), 'bar');
|
||||
assert.equal(model.cells[0].outputs[0].outputs[2].mime, 'text/plain');
|
||||
assert.equal(model.cells[0].outputs[0].outputs[2].data.toString(), 'baz');
|
||||
});
|
||||
});
|
||||
test('Append multiple stdout stream output items to an output with another mime', async function () {
|
||||
await withTestNotebook([
|
||||
['var a = 1;', 'javascript', CellKind.Code, [{
|
||||
outputId: '1',
|
||||
outputs: [{ mime: 'text/plain', data: valueBytesFromString('foo') }]
|
||||
}], {}]
|
||||
], (editor) => {
|
||||
const model = editor.textModel;
|
||||
const edits: ICellEditOperation[] = [
|
||||
{
|
||||
editType: CellEditType.OutputItems,
|
||||
outputId: '1',
|
||||
append: true,
|
||||
items: [{ mime: 'application/vnd.code.notebook.stdout', data: VSBuffer.fromString('bar') }, { mime: 'application/vnd.code.notebook.stdout', data: VSBuffer.fromString('baz') }]
|
||||
}
|
||||
];
|
||||
model.applyEdits(edits, true, undefined, () => undefined, undefined, true);
|
||||
assert.equal(model.cells.length, 1);
|
||||
assert.equal(model.cells[0].outputs.length, 1);
|
||||
assert.equal(model.cells[0].outputs[0].outputs.length, 3);
|
||||
assert.equal(model.cells[0].outputs[0].outputs[0].mime, 'text/plain');
|
||||
assert.equal(model.cells[0].outputs[0].outputs[0].data.toString(), 'foo');
|
||||
assert.equal(model.cells[0].outputs[0].outputs[1].mime, 'application/vnd.code.notebook.stdout');
|
||||
assert.equal(model.cells[0].outputs[0].outputs[1].data.toString(), 'bar');
|
||||
assert.equal(model.cells[0].outputs[0].outputs[2].mime, 'application/vnd.code.notebook.stdout');
|
||||
assert.equal(model.cells[0].outputs[0].outputs[2].data.toString(), 'baz');
|
||||
});
|
||||
});
|
||||
test('Compress multiple stdout stream output items', async function () {
|
||||
await withTestNotebook([
|
||||
['var a = 1;', 'javascript', CellKind.Code, [{
|
||||
outputId: '1',
|
||||
outputs: [{ mime: 'application/vnd.code.notebook.stdout', data: valueBytesFromString('foo') }]
|
||||
}], {}]
|
||||
], (editor) => {
|
||||
const model = editor.textModel;
|
||||
const edits: ICellEditOperation[] = [
|
||||
{
|
||||
editType: CellEditType.OutputItems,
|
||||
outputId: '1',
|
||||
append: true,
|
||||
items: [{ mime: 'application/vnd.code.notebook.stdout', data: VSBuffer.fromString('bar') }, { mime: 'application/vnd.code.notebook.stdout', data: VSBuffer.fromString('baz') }]
|
||||
}
|
||||
];
|
||||
model.applyEdits(edits, true, undefined, () => undefined, undefined, true);
|
||||
assert.equal(model.cells.length, 1);
|
||||
assert.equal(model.cells[0].outputs.length, 1);
|
||||
assert.equal(model.cells[0].outputs[0].outputs.length, 1);
|
||||
assert.equal(model.cells[0].outputs[0].outputs[0].mime, 'application/vnd.code.notebook.stdout');
|
||||
assert.equal(model.cells[0].outputs[0].outputs[0].data.toString(), 'foobarbaz');
|
||||
});
|
||||
|
||||
});
|
||||
test('Compress multiple stderr stream output items', async function () {
|
||||
await withTestNotebook([
|
||||
['var a = 1;', 'javascript', CellKind.Code, [{
|
||||
outputId: '1',
|
||||
outputs: [{ mime: 'application/vnd.code.notebook.stderr', data: valueBytesFromString('foo') }]
|
||||
}], {}]
|
||||
], (editor) => {
|
||||
const model = editor.textModel;
|
||||
const edits: ICellEditOperation[] = [
|
||||
{
|
||||
editType: CellEditType.OutputItems,
|
||||
outputId: '1',
|
||||
append: true,
|
||||
items: [{ mime: 'application/vnd.code.notebook.stderr', data: VSBuffer.fromString('bar') }, { mime: 'application/vnd.code.notebook.stderr', data: VSBuffer.fromString('baz') }]
|
||||
}
|
||||
];
|
||||
model.applyEdits(edits, true, undefined, () => undefined, undefined, true);
|
||||
assert.equal(model.cells.length, 1);
|
||||
assert.equal(model.cells[0].outputs.length, 1);
|
||||
assert.equal(model.cells[0].outputs[0].outputs.length, 1);
|
||||
assert.equal(model.cells[0].outputs[0].outputs[0].mime, 'application/vnd.code.notebook.stderr');
|
||||
assert.equal(model.cells[0].outputs[0].outputs[0].data.toString(), 'foobarbaz');
|
||||
});
|
||||
|
||||
});
|
||||
});
|
||||
|
|
|
|||
Loading…
Reference in a new issue