diff --git a/src/vs/workbench/contrib/chat/electron-sandbox/actions/voiceChatActions.ts b/src/vs/workbench/contrib/chat/electron-sandbox/actions/voiceChatActions.ts index e0a21d0ccb8..5e4802353de 100644 --- a/src/vs/workbench/contrib/chat/electron-sandbox/actions/voiceChatActions.ts +++ b/src/vs/workbench/contrib/chat/electron-sandbox/actions/voiceChatActions.ts @@ -45,7 +45,7 @@ import { IExtensionsWorkbenchService } from 'vs/workbench/contrib/extensions/com import { InlineChatController } from 'vs/workbench/contrib/inlineChat/browser/inlineChatController'; import { CTX_INLINE_CHAT_FOCUSED, CTX_INLINE_CHAT_HAS_ACTIVE_REQUEST } from 'vs/workbench/contrib/inlineChat/common/inlineChat'; import { NOTEBOOK_EDITOR_FOCUSED } from 'vs/workbench/contrib/notebook/common/notebookContextKeys'; -import { HasSpeechProvider, ISpeechService, ITextToSpeechSession, KeywordRecognitionStatus, SpeechToTextInProgress, SpeechToTextStatus, TextToSpeechStatus, TextToSpeechInProgress as GlobalTextToSpeechInProgress } from 'vs/workbench/contrib/speech/common/speechService'; +import { HasSpeechProvider, ISpeechService, KeywordRecognitionStatus, SpeechToTextInProgress, SpeechToTextStatus, TextToSpeechStatus, TextToSpeechInProgress as GlobalTextToSpeechInProgress } from 'vs/workbench/contrib/speech/common/speechService'; import { ITerminalService } from 'vs/workbench/contrib/terminal/browser/terminal'; import { TerminalChatContextKeys, TerminalChatController } from 'vs/workbench/contrib/terminal/browser/terminalContribExports'; import { IEditorService } from 'vs/workbench/services/editor/common/editorService'; @@ -55,6 +55,7 @@ import { IStatusbarEntry, IStatusbarEntryAccessor, IStatusbarService, StatusbarA import { IViewsService } from 'vs/workbench/services/views/common/viewsService'; import { IChatResponseModel } from 'vs/workbench/contrib/chat/common/chatModel'; import { IAccessibilityService } from 'vs/platform/accessibility/common/accessibility'; +import { renderStringAsPlaintext } from 'vs/base/browser/markdownRenderer'; //#region Speech to Text @@ -749,24 +750,12 @@ class ChatSynthesizerSessions { } })); - if (controller.response.isComplete) { - return this.synthesizeCompletedResponse(session, controller.response); - } else { - return this.synthesizePendingResponse(session, controller.response, activeSession.token); - } - } - - private synthesizeCompletedResponse(session: ITextToSpeechSession, response: IChatResponseModel): Promise { - return session.synthesize(response.response.asString()); - } - - private async synthesizePendingResponse(session: ITextToSpeechSession, response: IChatResponseModel, token: CancellationToken): Promise { - for await (const chunk of this.nextChatResponseChunk(response, token)) { - if (token.isCancellationRequested) { + for await (const chunk of this.nextChatResponseChunk(controller.response, activeSession.token)) { + if (activeSession.token.isCancellationRequested) { return; } - await raceCancellation(session.synthesize(chunk), token); + await raceCancellation(session.synthesize(chunk), activeSession.token); } } @@ -774,39 +763,43 @@ class ChatSynthesizerSessions { let totalOffset = 0; let complete = false; do { - const text = response.response.asString(); - const { chunks, offset, tail } = this.toChunks(text, totalOffset); + const responseLength = response.response.asString().length; + const { chunk, offset } = this.parseNextChatResponseChunk(response, totalOffset); totalOffset = offset; complete = response.isComplete; - for (const chunk of chunks) { + if (chunk) { yield chunk; - - if (token.isCancellationRequested) { - return; - } } - if (complete) { - yield tail; - } else if (text === response.response.asString()) { + if (token.isCancellationRequested) { + return; + } + + if (!complete && responseLength === response.response.asString().length) { await raceCancellation(Event.toPromise(response.onDidChange), token); // wait for the response to change } } while (!token.isCancellationRequested && !complete); } - private toChunks(text: string, offset: number): { readonly chunks: string[]; readonly offset: number; readonly tail: string } { - const chunks: string[] = []; + private parseNextChatResponseChunk(response: IChatResponseModel, offset: number): { readonly chunk: string | undefined; readonly offset: number } { + let chunk: string | undefined = undefined; - for (let i = offset; i < text.length; i++) { - const char = text[i]; - if (char === '.' || char === '!' || char === '?' || char === ':') { - chunks.push(text.substring(offset, i + 1)); - offset = i + 1; - } + const text = response.response.asString(); + + if (response.isComplete) { + chunk = text.substring(offset); + offset = text.length + 1; + } else { + const res = parseNextChatResponseChunk(text, offset); + chunk = res.chunk; + offset = res.offset; } - return { chunks, offset, tail: text.substring(offset) }; + return { + chunk: chunk ? renderStringAsPlaintext({ value: chunk }) : chunk, // convert markdown to plain text + offset + }; } stop(): void { @@ -815,6 +808,29 @@ class ChatSynthesizerSessions { } } +const sentenceDelimiter = ['.', '!', '?', ':']; +const lineDelimiter = '\n'; +const wordDelimiter = ' '; + +export function parseNextChatResponseChunk(text: string, offset: number): { readonly chunk: string | undefined; readonly offset: number } { + let chunk: string | undefined = undefined; + + for (let i = text.length - 1; i >= offset; i--) { // going from end to start to produce largest chunks + const cur = text[i]; + const next = text[i + 1]; + if ( + sentenceDelimiter.includes(cur) && next === wordDelimiter || // end of sentence + lineDelimiter === cur // end of line + ) { + chunk = text.substring(offset, i + 1).trim(); + offset = i + 1; + break; + } + } + + return { chunk, offset }; +} + export class ReadChatResponseAloud extends Action2 { constructor() { super({ diff --git a/src/vs/workbench/contrib/chat/test/electron-sandbox/voiceChatActions.test.ts b/src/vs/workbench/contrib/chat/test/electron-sandbox/voiceChatActions.test.ts new file mode 100644 index 00000000000..249fd8e457c --- /dev/null +++ b/src/vs/workbench/contrib/chat/test/electron-sandbox/voiceChatActions.test.ts @@ -0,0 +1,44 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as assert from 'assert'; +import { ensureNoDisposablesAreLeakedInTestSuite } from 'vs/base/test/common/utils'; +import { parseNextChatResponseChunk } from 'vs/workbench/contrib/chat/electron-sandbox/actions/voiceChatActions'; + +suite('VoiceChatActions', function () { + + function assertChunk(text: string, expected: string | undefined, offset: number): { chunk: string | undefined; offset: number } { + const res = parseNextChatResponseChunk(text, offset); + assert.strictEqual(res.chunk, expected); + + return res; + } + + test('parseNextChatResponseChunk', function () { + + // Simple, no offset + assertChunk('Hello World', undefined, 0); + assertChunk('Hello World.', undefined, 0); + assertChunk('Hello World. ', 'Hello World.', 0); + assertChunk('Hello World? ', 'Hello World?', 0); + assertChunk('Hello World! ', 'Hello World!', 0); + assertChunk('Hello World: ', 'Hello World:', 0); + + // Ensure chunks are parsed from the end, no offset + assertChunk('Hello World. How is your day? And more...', 'Hello World. How is your day?', 0); + + // Ensure chunks are parsed from the end, with offset + let offset = assertChunk('Hello World. How is your ', 'Hello World.', 0).offset; + offset = assertChunk('Hello World. How is your day? And more...', 'How is your day?', offset).offset; + offset = assertChunk('Hello World. How is your day? And more to come! ', 'And more to come!', offset).offset; + assertChunk('Hello World. How is your day? And more to come! ', undefined, offset); + + // Sparted by newlines + offset = assertChunk('Hello World.\nHow is your', 'Hello World.', 0).offset; + assertChunk('Hello World.\nHow is your day?\n', 'How is your day?', offset); + }); + + ensureNoDisposablesAreLeakedInTestSuite(); +});