From 43957ccfe1d7f387def153528812e5b513c7f11c Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Wed, 12 Oct 2022 08:43:46 +1100 Subject: [PATCH] Compress streams in notebook outputs (#160946) * Revert "Compress notebook output streams before rendering (#160667)" This reverts commit 4230c22a08a0438af4809d33e4cfd561f714b127. * Compress stream output items * Minor perf improvements * Misc * Comments * Added tests * Merge issue * More merge issues * Misc * Address code review comments --- extensions/ipynb/src/serializers.ts | 27 +++-- extensions/ipynb/src/streamCompressor.ts | 63 ----------- .../api/common/extHostNotebookDocument.ts | 26 +++++ .../api/test/browser/extHostNotebook.test.ts | 75 ++++++++++++- .../view/renderers/backLayerWebView.ts | 9 +- .../view/renderers/stdOutErrorPreProcessor.ts | 56 --------- .../common/model/notebookCellTextModel.ts | 27 ++++- .../contrib/notebook/common/notebookCommon.ts | 93 ++++++++++++++- .../test/browser/notebookTextModel.test.ts | 106 ++++++++++++++++++ 9 files changed, 341 insertions(+), 141 deletions(-) delete mode 100644 extensions/ipynb/src/streamCompressor.ts delete mode 100644 src/vs/workbench/contrib/notebook/browser/view/renderers/stdOutErrorPreProcessor.ts diff --git a/extensions/ipynb/src/serializers.ts b/extensions/ipynb/src/serializers.ts index f075737b4d8..96da8f2c617 100644 --- a/extensions/ipynb/src/serializers.ts +++ b/extensions/ipynb/src/serializers.ts @@ -7,7 +7,6 @@ import type * as nbformat from '@jupyterlab/nbformat'; import { NotebookCell, NotebookCellData, NotebookCellKind, NotebookCellOutput } from 'vscode'; import { CellOutputMetadata } from './common'; import { textMimeTypes } from './deserializers'; -import { compressOutputItemStreams } from './streamCompressor'; const textDecoder = new TextDecoder(); @@ -277,17 +276,21 @@ type JupyterOutput = function convertStreamOutput(output: NotebookCellOutput): JupyterOutput { const outputs: string[] = []; - const compressedStream = output.items.length ? new TextDecoder().decode(compressOutputItemStreams(output.items[0].mime, output.items)) : ''; - // Ensure each line is a separate entry in an array (ending with \n). - const lines = compressedStream.split('\n'); - // If the last item in `outputs` is not empty and the first item in `lines` is not empty, then concate them. - // As they are part of the same line. - if (outputs.length && lines.length && lines[0].length > 0) { - outputs[outputs.length - 1] = `${outputs[outputs.length - 1]}${lines.shift()!}`; - } - for (const line of lines) { - outputs.push(line); - } + output.items + .filter((opit) => opit.mime === CellOutputMimeTypes.stderr || opit.mime === CellOutputMimeTypes.stdout) + .map((opit) => textDecoder.decode(opit.data)) + .forEach(value => { + // Ensure each line is a separate entry in an array (ending with \n). + const lines = value.split('\n'); + // If the last item in `outputs` is not empty and the first item in `lines` is not empty, then concate them. + // As they are part of the same line. + if (outputs.length && lines.length && lines[0].length > 0) { + outputs[outputs.length - 1] = `${outputs[outputs.length - 1]}${lines.shift()!}`; + } + for (const line of lines) { + outputs.push(line); + } + }); for (let index = 0; index < (outputs.length - 1); index++) { outputs[index] = `${outputs[index]}\n`; diff --git a/extensions/ipynb/src/streamCompressor.ts b/extensions/ipynb/src/streamCompressor.ts deleted file mode 100644 index cea3184e16c..00000000000 --- a/extensions/ipynb/src/streamCompressor.ts +++ /dev/null @@ -1,63 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import type { NotebookCellOutputItem } from 'vscode'; - - -/** - * Given a stream of individual stdout outputs, this function will return the compressed lines, escaping some of the common terminal escape codes. - * E.g. some terminal escape codes would result in the previous line getting cleared, such if we had 3 lines and - * last line contained such a code, then the result string would be just the first two lines. - */ -export function compressOutputItemStreams(mimeType: string, outputs: NotebookCellOutputItem[]) { - // return outputs.find(op => op.mime === mimeType)!.data.buffer; - - const buffers: Uint8Array[] = []; - let startAppending = false; - // Pick the first set of outputs with the same mime type. - for (const output of outputs) { - if (output.mime === mimeType) { - if ((buffers.length === 0 || startAppending)) { - buffers.push(output.data); - startAppending = true; - } - } else if (startAppending) { - startAppending = false; - } - } - compressStreamBuffer(buffers); - const totalBytes = buffers.reduce((p, c) => p + c.byteLength, 0); - const combinedBuffer = new Uint8Array(totalBytes); - let offset = 0; - for (const buffer of buffers) { - combinedBuffer.set(buffer, offset); - offset = offset + buffer.byteLength; - } - return combinedBuffer; -} -const MOVE_CURSOR_1_LINE_COMMAND = `${String.fromCharCode(27)}[A`; -const MOVE_CURSOR_1_LINE_COMMAND_BYTES = MOVE_CURSOR_1_LINE_COMMAND.split('').map(c => c.charCodeAt(0)); -const LINE_FEED = 10; -function compressStreamBuffer(streams: Uint8Array[]) { - streams.forEach((stream, index) => { - if (index === 0 || stream.length < MOVE_CURSOR_1_LINE_COMMAND.length) { - return; - } - - const previousStream = streams[index - 1]; - - // Remove the previous line if required. - const command = stream.subarray(0, MOVE_CURSOR_1_LINE_COMMAND.length); - if (command[0] === MOVE_CURSOR_1_LINE_COMMAND_BYTES[0] && command[1] === MOVE_CURSOR_1_LINE_COMMAND_BYTES[1] && command[2] === MOVE_CURSOR_1_LINE_COMMAND_BYTES[2]) { - const lastIndexOfLineFeed = previousStream.lastIndexOf(LINE_FEED); - if (lastIndexOfLineFeed === -1) { - return; - } - streams[index - 1] = previousStream.subarray(0, lastIndexOfLineFeed); - streams[index] = stream.subarray(MOVE_CURSOR_1_LINE_COMMAND.length); - } - }); - return streams; -} diff --git a/src/vs/workbench/api/common/extHostNotebookDocument.ts b/src/vs/workbench/api/common/extHostNotebookDocument.ts index c3ebc53b3d6..38437605cec 100644 --- a/src/vs/workbench/api/common/extHostNotebookDocument.ts +++ b/src/vs/workbench/api/common/extHostNotebookDocument.ts @@ -110,6 +110,32 @@ export class ExtHostCell { output.items.length = 0; } output.items.push(...newItems); + + if (output.items.length > 1 && output.items.every(item => notebookCommon.isTextStreamMime(item.mime))) { + // Look for the mimes in the items, and keep track of their order. + // Merge the streams into one output item, per mime type. + const mimeOutputs = new Map(); + const mimeTypes: string[] = []; + output.items.forEach(item => { + let items: Uint8Array[]; + if (mimeOutputs.has(item.mime)) { + items = mimeOutputs.get(item.mime)!; + } else { + items = []; + mimeOutputs.set(item.mime, items); + mimeTypes.push(item.mime); + } + items.push(item.data); + }); + output.items.length = 0; + mimeTypes.forEach(mime => { + const compressed = notebookCommon.compressOutputItemStreams(mimeOutputs.get(mime)!); + output.items.push({ + mime, + data: compressed.buffer + }); + }); + } } } diff --git a/src/vs/workbench/api/test/browser/extHostNotebook.test.ts b/src/vs/workbench/api/test/browser/extHostNotebook.test.ts index 09e785c307b..60f453ec1f3 100644 --- a/src/vs/workbench/api/test/browser/extHostNotebook.test.ts +++ b/src/vs/workbench/api/test/browser/extHostNotebook.test.ts @@ -10,7 +10,7 @@ import { TestRPCProtocol } from 'vs/workbench/api/test/common/testRPCProtocol'; import { DisposableStore } from 'vs/base/common/lifecycle'; import { NullLogService } from 'vs/platform/log/common/log'; import { mock } from 'vs/base/test/common/mock'; -import { IModelAddedData, MainContext, MainThreadCommandsShape, MainThreadNotebookShape } from 'vs/workbench/api/common/extHost.protocol'; +import { IModelAddedData, MainContext, MainThreadCommandsShape, MainThreadNotebookShape, NotebookCellsChangedEventDto, NotebookOutputItemDto } from 'vs/workbench/api/common/extHost.protocol'; import { ExtHostNotebookController } from 'vs/workbench/api/common/extHostNotebook'; import { ExtHostNotebookDocument } from 'vs/workbench/api/common/extHostNotebookDocument'; import { CellKind, CellUri, NotebookCellsChangeType } from 'vs/workbench/contrib/notebook/common/notebookCommon'; @@ -542,4 +542,77 @@ suite('NotebookCell#Document', function () { assert.ok(cellChange.metadata === undefined); assert.ok(cellChange.outputs === undefined); }); + + async function replaceOutputs(cellIndex: number, outputId: string, outputItems: NotebookOutputItemDto[]) { + const changeEvent = Event.toPromise(extHostNotebookDocuments.onDidChangeNotebookDocument); + extHostNotebookDocuments.$acceptModelChanged(notebook.uri, new SerializableObjectWithBuffers({ + versionId: notebook.apiNotebook.version + 1, + rawEvents: [{ + kind: NotebookCellsChangeType.Output, + index: cellIndex, + outputs: [{ outputId, items: outputItems }] + }] + }), false); + await changeEvent; + } + async function appendOutputItem(cellIndex: number, outputId: string, outputItems: NotebookOutputItemDto[]) { + const changeEvent = Event.toPromise(extHostNotebookDocuments.onDidChangeNotebookDocument); + extHostNotebookDocuments.$acceptModelChanged(notebook.uri, new SerializableObjectWithBuffers({ + versionId: notebook.apiNotebook.version + 1, + rawEvents: [{ + kind: NotebookCellsChangeType.OutputItem, + index: cellIndex, + append: true, + outputId, + outputItems + }] + }), false); + await changeEvent; + } + test('Append multiple text/plain output items', async function () { + await replaceOutputs(1, '1', [{ mime: 'text/plain', valueBytes: VSBuffer.fromString('foo') }]); + await appendOutputItem(1, '1', [{ mime: 'text/plain', valueBytes: VSBuffer.fromString('bar') }]); + await appendOutputItem(1, '1', [{ mime: 'text/plain', valueBytes: VSBuffer.fromString('baz') }]); + + + assert.strictEqual(notebook.apiNotebook.cellAt(1).outputs.length, 1); + assert.strictEqual(notebook.apiNotebook.cellAt(1).outputs[0].items.length, 3); + assert.strictEqual(notebook.apiNotebook.cellAt(1).outputs[0].items[0].mime, 'text/plain'); + assert.strictEqual(VSBuffer.wrap(notebook.apiNotebook.cellAt(1).outputs[0].items[0].data).toString(), 'foo'); + assert.strictEqual(notebook.apiNotebook.cellAt(1).outputs[0].items[1].mime, 'text/plain'); + assert.strictEqual(VSBuffer.wrap(notebook.apiNotebook.cellAt(1).outputs[0].items[1].data).toString(), 'bar'); + assert.strictEqual(notebook.apiNotebook.cellAt(1).outputs[0].items[2].mime, 'text/plain'); + assert.strictEqual(VSBuffer.wrap(notebook.apiNotebook.cellAt(1).outputs[0].items[2].data).toString(), 'baz'); + }); + test('Append multiple stdout stream output items to an output with another mime', async function () { + await replaceOutputs(1, '1', [{ mime: 'text/plain', valueBytes: VSBuffer.fromString('foo') }]); + await appendOutputItem(1, '1', [{ mime: 'application/vnd.code.notebook.stdout', valueBytes: VSBuffer.fromString('bar') }]); + await appendOutputItem(1, '1', [{ mime: 'application/vnd.code.notebook.stdout', valueBytes: VSBuffer.fromString('baz') }]); + + assert.strictEqual(notebook.apiNotebook.cellAt(1).outputs.length, 1); + assert.strictEqual(notebook.apiNotebook.cellAt(1).outputs[0].items.length, 3); + assert.strictEqual(notebook.apiNotebook.cellAt(1).outputs[0].items[0].mime, 'text/plain'); + assert.strictEqual(notebook.apiNotebook.cellAt(1).outputs[0].items[1].mime, 'application/vnd.code.notebook.stdout'); + assert.strictEqual(notebook.apiNotebook.cellAt(1).outputs[0].items[2].mime, 'application/vnd.code.notebook.stdout'); + }); + test('Compress multiple stdout stream output items', async function () { + await replaceOutputs(1, '1', [{ mime: 'application/vnd.code.notebook.stdout', valueBytes: VSBuffer.fromString('foo') }]); + await appendOutputItem(1, '1', [{ mime: 'application/vnd.code.notebook.stdout', valueBytes: VSBuffer.fromString('bar') }]); + await appendOutputItem(1, '1', [{ mime: 'application/vnd.code.notebook.stdout', valueBytes: VSBuffer.fromString('baz') }]); + + assert.strictEqual(notebook.apiNotebook.cellAt(1).outputs.length, 1); + assert.strictEqual(notebook.apiNotebook.cellAt(1).outputs[0].items.length, 1); + assert.strictEqual(notebook.apiNotebook.cellAt(1).outputs[0].items[0].mime, 'application/vnd.code.notebook.stdout'); + assert.strictEqual(VSBuffer.wrap(notebook.apiNotebook.cellAt(1).outputs[0].items[0].data).toString(), 'foobarbaz'); + }); + test('Compress multiple stderr stream output items', async function () { + await replaceOutputs(1, '1', [{ mime: 'application/vnd.code.notebook.stderr', valueBytes: VSBuffer.fromString('foo') }]); + await appendOutputItem(1, '1', [{ mime: 'application/vnd.code.notebook.stderr', valueBytes: VSBuffer.fromString('bar') }]); + await appendOutputItem(1, '1', [{ mime: 'application/vnd.code.notebook.stderr', valueBytes: VSBuffer.fromString('baz') }]); + + assert.strictEqual(notebook.apiNotebook.cellAt(1).outputs.length, 1); + assert.strictEqual(notebook.apiNotebook.cellAt(1).outputs[0].items.length, 1); + assert.strictEqual(notebook.apiNotebook.cellAt(1).outputs[0].items[0].mime, 'application/vnd.code.notebook.stderr'); + assert.strictEqual(VSBuffer.wrap(notebook.apiNotebook.cellAt(1).outputs[0].items[0].data).toString(), 'foobarbaz'); + }); }); diff --git a/src/vs/workbench/contrib/notebook/browser/view/renderers/backLayerWebView.ts b/src/vs/workbench/contrib/notebook/browser/view/renderers/backLayerWebView.ts index a32ebc7dc1a..f01090a0561 100644 --- a/src/vs/workbench/contrib/notebook/browser/view/renderers/backLayerWebView.ts +++ b/src/vs/workbench/contrib/notebook/browser/view/renderers/backLayerWebView.ts @@ -35,7 +35,7 @@ import { NOTEBOOK_WEBVIEW_BOUNDARY } from 'vs/workbench/contrib/notebook/browser import { preloadsScriptStr } from 'vs/workbench/contrib/notebook/browser/view/renderers/webviewPreloads'; import { transformWebviewThemeVars } from 'vs/workbench/contrib/notebook/browser/view/renderers/webviewThemeMapping'; import { MarkupCellViewModel } from 'vs/workbench/contrib/notebook/browser/viewModel/markupCellViewModel'; -import { CellUri, INotebookRendererInfo, isTextStreamMime, NotebookSetting, RendererMessagingSpec } from 'vs/workbench/contrib/notebook/common/notebookCommon'; +import { CellUri, INotebookRendererInfo, NotebookSetting, RendererMessagingSpec } from 'vs/workbench/contrib/notebook/common/notebookCommon'; import { INotebookKernel } from 'vs/workbench/contrib/notebook/common/notebookKernelService'; import { IScopedRendererMessaging } from 'vs/workbench/contrib/notebook/common/notebookRendererMessagingService'; import { INotebookService } from 'vs/workbench/contrib/notebook/common/notebookService'; @@ -44,7 +44,6 @@ import { WebviewWindowDragMonitor } from 'vs/workbench/contrib/webview/browser/w import { IEditorGroupsService } from 'vs/workbench/services/editor/common/editorGroupsService'; import { IWorkbenchEnvironmentService } from 'vs/workbench/services/environment/common/environmentService'; import { FromWebviewMessage, IAckOutputHeight, IClickedDataUrlMessage, ICodeBlockHighlightRequest, IContentWidgetTopRequest, IControllerPreload, ICreationContent, ICreationRequestMessage, IFindMatch, IMarkupCellInitialization, RendererMetadata, ToWebviewMessage } from './webviewMessages'; -import { compressOutputItemStreams } from 'vs/workbench/contrib/notebook/browser/view/renderers/stdOutErrorPreProcessor'; import { DeferredPromise } from 'vs/base/common/async'; export interface ICachedInset { @@ -1277,14 +1276,12 @@ var requirejs = (function() { let updatedContent: ICreationContent | undefined = undefined; if (content.type === RenderOutputType.Extension) { const output = content.source.model; - const firstBuffer = isTextStreamMime(content.mimeType) ? - compressOutputItemStreams(content.mimeType, output.outputs) : - output.outputs.find(op => op.mime === content.mimeType)!.data.buffer; + const firstBuffer = output.outputs.find(op => op.mime === content.mimeType)!.data; updatedContent = { type: RenderOutputType.Extension, outputId: outputCache.outputId, mimeType: content.mimeType, - valueBytes: firstBuffer, + valueBytes: firstBuffer.buffer, metadata: output.metadata, }; } diff --git a/src/vs/workbench/contrib/notebook/browser/view/renderers/stdOutErrorPreProcessor.ts b/src/vs/workbench/contrib/notebook/browser/view/renderers/stdOutErrorPreProcessor.ts deleted file mode 100644 index 7f20c6316fc..00000000000 --- a/src/vs/workbench/contrib/notebook/browser/view/renderers/stdOutErrorPreProcessor.ts +++ /dev/null @@ -1,56 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { VSBuffer } from 'vs/base/common/buffer'; -import type { IOutputItemDto } from 'vs/workbench/contrib/notebook/common/notebookCommon'; - - -/** - * Given a stream of individual stdout outputs, this function will return the compressed lines, escaping some of the common terminal escape codes. - * E.g. some terminal escape codes would result in the previous line getting cleared, such if we had 3 lines and - * last line contained such a code, then the result string would be just the first two lines. - */ -export function compressOutputItemStreams(mimeType: string, outputs: IOutputItemDto[]) { - const buffers: Uint8Array[] = []; - let startAppending = false; - - // Pick the first set of outputs with the same mime type. - for (const output of outputs) { - if (output.mime === mimeType) { - if ((buffers.length === 0 || startAppending)) { - buffers.push(output.data.buffer); - startAppending = true; - } - } else if (startAppending) { - startAppending = false; - } - } - compressStreamBuffer(buffers); - return VSBuffer.concat(buffers.map(buffer => VSBuffer.wrap(buffer))).buffer; -} -const MOVE_CURSOR_1_LINE_COMMAND = `${String.fromCharCode(27)}[A`; -const MOVE_CURSOR_1_LINE_COMMAND_BYTES = MOVE_CURSOR_1_LINE_COMMAND.split('').map(c => c.charCodeAt(0)); -const LINE_FEED = 10; -function compressStreamBuffer(streams: Uint8Array[]) { - streams.forEach((stream, index) => { - if (index === 0 || stream.length < MOVE_CURSOR_1_LINE_COMMAND.length) { - return; - } - - const previousStream = streams[index - 1]; - - // Remove the previous line if required. - const command = stream.subarray(0, MOVE_CURSOR_1_LINE_COMMAND.length); - if (command[0] === MOVE_CURSOR_1_LINE_COMMAND_BYTES[0] && command[1] === MOVE_CURSOR_1_LINE_COMMAND_BYTES[1] && command[2] === MOVE_CURSOR_1_LINE_COMMAND_BYTES[2]) { - const lastIndexOfLineFeed = previousStream.lastIndexOf(LINE_FEED); - if (lastIndexOfLineFeed === -1) { - return; - } - streams[index - 1] = previousStream.subarray(0, lastIndexOfLineFeed); - streams[index] = stream.subarray(MOVE_CURSOR_1_LINE_COMMAND.length); - } - }); - return streams; -} diff --git a/src/vs/workbench/contrib/notebook/common/model/notebookCellTextModel.ts b/src/vs/workbench/contrib/notebook/common/model/notebookCellTextModel.ts index 7890d272892..e8576c8234a 100644 --- a/src/vs/workbench/contrib/notebook/common/model/notebookCellTextModel.ts +++ b/src/vs/workbench/contrib/notebook/common/model/notebookCellTextModel.ts @@ -16,7 +16,7 @@ import { TextModel } from 'vs/editor/common/model/textModel'; import { PLAINTEXT_LANGUAGE_ID } from 'vs/editor/common/languages/modesRegistry'; import { ILanguageService } from 'vs/editor/common/languages/language'; import { NotebookCellOutputTextModel } from 'vs/workbench/contrib/notebook/common/model/notebookCellOutputTextModel'; -import { CellInternalMetadataChangedEvent, CellKind, ICell, ICellDto2, ICellOutput, IOutputDto, IOutputItemDto, NotebookCellCollapseState, NotebookCellInternalMetadata, NotebookCellMetadata, NotebookCellOutputsSplice, TransientOptions } from 'vs/workbench/contrib/notebook/common/notebookCommon'; +import { CellInternalMetadataChangedEvent, CellKind, compressOutputItemStreams, ICell, ICellDto2, ICellOutput, IOutputDto, IOutputItemDto, isTextStreamMime, NotebookCellCollapseState, NotebookCellInternalMetadata, NotebookCellMetadata, NotebookCellOutputsSplice, TransientOptions } from 'vs/workbench/contrib/notebook/common/notebookCommon'; export class NotebookCellTextModel extends Disposable implements ICell { private readonly _onDidChangeOutputs = this._register(new Emitter()); @@ -297,6 +297,31 @@ export class NotebookCellTextModel extends Disposable implements ICell { } else { output.replaceData(items); } + if (output.outputs.length > 1 && output.outputs.every(item => isTextStreamMime(item.mime))) { + // Look for the mimes in the items, and keep track of their order. + // Merge the streams into one output item, per mime type. + const mimeOutputs = new Map(); + const mimeTypes: string[] = []; + output.outputs.forEach(item => { + let items: Uint8Array[]; + if (mimeOutputs.has(item.mime)) { + items = mimeOutputs.get(item.mime)!; + } else { + items = []; + mimeOutputs.set(item.mime, items); + mimeTypes.push(item.mime); + } + items.push(item.data.buffer); + }); + output.outputs.length = 0; + mimeTypes.forEach(mime => { + const compressed = compressOutputItemStreams(mimeOutputs.get(mime)!); + output.outputs.push({ + mime, + data: compressed + }); + }); + } this._onDidChangeOutputItems.fire(); return true; diff --git a/src/vs/workbench/contrib/notebook/common/notebookCommon.ts b/src/vs/workbench/contrib/notebook/common/notebookCommon.ts index ab73eaf88b9..0e2f1fe9ea6 100644 --- a/src/vs/workbench/contrib/notebook/common/notebookCommon.ts +++ b/src/vs/workbench/contrib/notebook/common/notebookCommon.ts @@ -951,9 +951,98 @@ export interface NotebookExtensionDescription { } /** - * Whether the provided mime type is a text streamn like `stdout`, `stderr`. + * Whether the provided mime type is a text stream like `stdout`, `stderr`. */ export function isTextStreamMime(mimeType: string) { - return ['application/vnd.code.notebook.stdout', 'application/x.notebook.stdout', 'application/x.notebook.stream', 'application/vnd.code.notebook.stderr', 'application/x.notebook.stderr'].includes(mimeType); + return ['application/vnd.code.notebook.stdout', 'application/vnd.code.notebook.stderr'].includes(mimeType); } + +const textDecoder = new TextDecoder(); + +/** + * Given a stream of individual stdout outputs, this function will return the compressed lines, escaping some of the common terminal escape codes. + * E.g. some terminal escape codes would result in the previous line getting cleared, such if we had 3 lines and + * last line contained such a code, then the result string would be just the first two lines. + */ +export function compressOutputItemStreams(outputs: Uint8Array[]) { + const buffers: Uint8Array[] = []; + let startAppending = false; + + // Pick the first set of outputs with the same mime type. + for (const output of outputs) { + if ((buffers.length === 0 || startAppending)) { + buffers.push(output); + startAppending = true; + } + } + compressStreamBuffer(buffers); + return formatStreamText(VSBuffer.concat(buffers.map(buffer => VSBuffer.wrap(buffer)))); +} +const MOVE_CURSOR_1_LINE_COMMAND = `${String.fromCharCode(27)}[A`; +const MOVE_CURSOR_1_LINE_COMMAND_BYTES = MOVE_CURSOR_1_LINE_COMMAND.split('').map(c => c.charCodeAt(0)); +const LINE_FEED = 10; +function compressStreamBuffer(streams: Uint8Array[]) { + streams.forEach((stream, index) => { + if (index === 0 || stream.length < MOVE_CURSOR_1_LINE_COMMAND.length) { + return; + } + + const previousStream = streams[index - 1]; + + // Remove the previous line if required. + const command = stream.subarray(0, MOVE_CURSOR_1_LINE_COMMAND.length); + if (command[0] === MOVE_CURSOR_1_LINE_COMMAND_BYTES[0] && command[1] === MOVE_CURSOR_1_LINE_COMMAND_BYTES[1] && command[2] === MOVE_CURSOR_1_LINE_COMMAND_BYTES[2]) { + const lastIndexOfLineFeed = previousStream.lastIndexOf(LINE_FEED); + if (lastIndexOfLineFeed === -1) { + return; + } + streams[index - 1] = previousStream.subarray(0, lastIndexOfLineFeed); + streams[index] = stream.subarray(MOVE_CURSOR_1_LINE_COMMAND.length); + } + }); +} + + + +/** + * Took this from jupyter/notebook + * https://github.com/jupyter/notebook/blob/b8b66332e2023e83d2ee04f83d8814f567e01a4e/notebook/static/base/js/utils.js + * Remove characters that are overridden by backspace characters + */ +function fixBackspace(txt: string) { + let tmp = txt; + do { + txt = tmp; + // Cancel out anything-but-newline followed by backspace + tmp = txt.replace(/[^\n]\x08/gm, ''); + } while (tmp.length < txt.length); + return txt; +} + +/** + * Remove chunks that should be overridden by the effect of carriage return characters + * From https://github.com/jupyter/notebook/blob/master/notebook/static/base/js/utils.js + */ +function fixCarriageReturn(txt: string) { + txt = txt.replace(/\r+\n/gm, '\n'); // \r followed by \n --> newline + while (txt.search(/\r[^$]/g) > -1) { + const base = txt.match(/^(.*)\r+/m)![1]; + let insert = txt.match(/\r+(.*)$/m)![1]; + insert = insert + base.slice(insert.length, base.length); + txt = txt.replace(/\r+.*$/m, '\r').replace(/^.*\r/m, insert); + } + return txt; +} + +const BACKSPACE_CHARACTER = '\b'.charCodeAt(0); +const CARRIAGE_RETURN_CHARACTER = '\r'.charCodeAt(0); +function formatStreamText(buffer: VSBuffer): VSBuffer { + // We have special handling for backspace and carriage return characters. + // Don't unnecessary decode the bytes if we don't need to perform any processing. + if (!buffer.buffer.includes(BACKSPACE_CHARACTER) && !buffer.buffer.includes(CARRIAGE_RETURN_CHARACTER)) { + return buffer; + } + // Do the same thing jupyter is doing + return VSBuffer.fromString(fixCarriageReturn(fixBackspace(textDecoder.decode(buffer.buffer)))); +} diff --git a/src/vs/workbench/contrib/notebook/test/browser/notebookTextModel.test.ts b/src/vs/workbench/contrib/notebook/test/browser/notebookTextModel.test.ts index 455d706b637..4da03a9f248 100644 --- a/src/vs/workbench/contrib/notebook/test/browser/notebookTextModel.test.ts +++ b/src/vs/workbench/contrib/notebook/test/browser/notebookTextModel.test.ts @@ -1043,4 +1043,110 @@ suite('NotebookTextModel', () => { assert.equal(model.cells[0].outputs[0].outputs[0].data.toString(), '_World_'); }); }); + test('Append multiple text/plain output items', async function () { + await withTestNotebook([ + ['var a = 1;', 'javascript', CellKind.Code, [{ + outputId: '1', + outputs: [{ mime: 'text/plain', data: valueBytesFromString('foo') }] + }], {}] + ], (editor) => { + const model = editor.textModel; + const edits: ICellEditOperation[] = [ + { + editType: CellEditType.OutputItems, + outputId: '1', + append: true, + items: [{ mime: 'text/plain', data: VSBuffer.fromString('bar') }, { mime: 'text/plain', data: VSBuffer.fromString('baz') }] + } + ]; + model.applyEdits(edits, true, undefined, () => undefined, undefined, true); + assert.equal(model.cells.length, 1); + assert.equal(model.cells[0].outputs.length, 1); + assert.equal(model.cells[0].outputs[0].outputs.length, 3); + assert.equal(model.cells[0].outputs[0].outputs[0].mime, 'text/plain'); + assert.equal(model.cells[0].outputs[0].outputs[0].data.toString(), 'foo'); + assert.equal(model.cells[0].outputs[0].outputs[1].mime, 'text/plain'); + assert.equal(model.cells[0].outputs[0].outputs[1].data.toString(), 'bar'); + assert.equal(model.cells[0].outputs[0].outputs[2].mime, 'text/plain'); + assert.equal(model.cells[0].outputs[0].outputs[2].data.toString(), 'baz'); + }); + }); + test('Append multiple stdout stream output items to an output with another mime', async function () { + await withTestNotebook([ + ['var a = 1;', 'javascript', CellKind.Code, [{ + outputId: '1', + outputs: [{ mime: 'text/plain', data: valueBytesFromString('foo') }] + }], {}] + ], (editor) => { + const model = editor.textModel; + const edits: ICellEditOperation[] = [ + { + editType: CellEditType.OutputItems, + outputId: '1', + append: true, + items: [{ mime: 'application/vnd.code.notebook.stdout', data: VSBuffer.fromString('bar') }, { mime: 'application/vnd.code.notebook.stdout', data: VSBuffer.fromString('baz') }] + } + ]; + model.applyEdits(edits, true, undefined, () => undefined, undefined, true); + assert.equal(model.cells.length, 1); + assert.equal(model.cells[0].outputs.length, 1); + assert.equal(model.cells[0].outputs[0].outputs.length, 3); + assert.equal(model.cells[0].outputs[0].outputs[0].mime, 'text/plain'); + assert.equal(model.cells[0].outputs[0].outputs[0].data.toString(), 'foo'); + assert.equal(model.cells[0].outputs[0].outputs[1].mime, 'application/vnd.code.notebook.stdout'); + assert.equal(model.cells[0].outputs[0].outputs[1].data.toString(), 'bar'); + assert.equal(model.cells[0].outputs[0].outputs[2].mime, 'application/vnd.code.notebook.stdout'); + assert.equal(model.cells[0].outputs[0].outputs[2].data.toString(), 'baz'); + }); + }); + test('Compress multiple stdout stream output items', async function () { + await withTestNotebook([ + ['var a = 1;', 'javascript', CellKind.Code, [{ + outputId: '1', + outputs: [{ mime: 'application/vnd.code.notebook.stdout', data: valueBytesFromString('foo') }] + }], {}] + ], (editor) => { + const model = editor.textModel; + const edits: ICellEditOperation[] = [ + { + editType: CellEditType.OutputItems, + outputId: '1', + append: true, + items: [{ mime: 'application/vnd.code.notebook.stdout', data: VSBuffer.fromString('bar') }, { mime: 'application/vnd.code.notebook.stdout', data: VSBuffer.fromString('baz') }] + } + ]; + model.applyEdits(edits, true, undefined, () => undefined, undefined, true); + assert.equal(model.cells.length, 1); + assert.equal(model.cells[0].outputs.length, 1); + assert.equal(model.cells[0].outputs[0].outputs.length, 1); + assert.equal(model.cells[0].outputs[0].outputs[0].mime, 'application/vnd.code.notebook.stdout'); + assert.equal(model.cells[0].outputs[0].outputs[0].data.toString(), 'foobarbaz'); + }); + + }); + test('Compress multiple stderr stream output items', async function () { + await withTestNotebook([ + ['var a = 1;', 'javascript', CellKind.Code, [{ + outputId: '1', + outputs: [{ mime: 'application/vnd.code.notebook.stderr', data: valueBytesFromString('foo') }] + }], {}] + ], (editor) => { + const model = editor.textModel; + const edits: ICellEditOperation[] = [ + { + editType: CellEditType.OutputItems, + outputId: '1', + append: true, + items: [{ mime: 'application/vnd.code.notebook.stderr', data: VSBuffer.fromString('bar') }, { mime: 'application/vnd.code.notebook.stderr', data: VSBuffer.fromString('baz') }] + } + ]; + model.applyEdits(edits, true, undefined, () => undefined, undefined, true); + assert.equal(model.cells.length, 1); + assert.equal(model.cells[0].outputs.length, 1); + assert.equal(model.cells[0].outputs[0].outputs.length, 1); + assert.equal(model.cells[0].outputs[0].outputs[0].mime, 'application/vnd.code.notebook.stderr'); + assert.equal(model.cells[0].outputs[0].outputs[0].data.toString(), 'foobarbaz'); + }); + + }); });