Allow editor/terminal dictation to take over an active Chat dictation session (#328830)

* Initial plan

* Allow editor/terminal dictation to take over an active Chat session

Co-authored-by: meganrogge <29464607+meganrogge@users.noreply.github.com>

* Scope dictation UI to active surface

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* Serialize dictation session takeover

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: meganrogge <29464607+meganrogge@users.noreply.github.com>
Co-authored-by: meganrogge <merogge@microsoft.com>
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: 3 people

src/vs/sessions/contrib/chat/browser/newChatInput.ts

@@ -85,7 +85,7 @@ import { AGENT_SESSIONS_SCOPED_INPUT_HISTORY_SETTING } from './sessionsChatHisto
 import { IChatStatusItemService } from '../../../../workbench/contrib/chat/browser/chatStatus/chatStatusItemService.js';
 import { handleTerminalCommandPaste, isTerminalCommandInput } from '../../../../workbench/contrib/chat/browser/chatTerminalCommandPaste.js';
 import { getChatSessionType } from '../../../../workbench/contrib/chat/common/model/chatUri.js';
-import { ChatSpeechToTextState, DictationSettingId, IChatSpeechToTextService } from '../../../../workbench/contrib/chat/browser/speechToText/chatSpeechToTextService.js';
+import { ChatSpeechToTextState, DictationSettingId, IChatSpeechToTextService, isDictationActiveOnSurface } from '../../../../workbench/contrib/chat/browser/speechToText/chatSpeechToTextService.js';
 import { setupDictationMicGlow } from '../../../../workbench/contrib/chat/browser/speechToText/dictationMicGlow.js';
 import { IDictationOnboardingService } from '../../../../workbench/contrib/chat/browser/speechToText/dictationOnboarding.js';
 import { ChatVoiceInputModeAction, VoiceInputModeActionViewItem } from '../../../../workbench/contrib/chat/browser/voiceInputMode/voiceInputModeActionViewItem.js';
@@ -930,23 +930,23 @@ export class NewChatInputWidget extends Disposable implements IHistoryNavigation
 			// (which invites the user to click to cancel) so this composer matches
 			// the main chat toolbar affordance. Idle gets the richer description
 			// naming the configured dictation model.
-			content: sttService.isPreparingModel
+			content: sttService.currentSurface === 'chat' && sttService.isPreparingModel
 				? getDictationDownloadHoverMarkdown(sttService)
-				: (sttService.state !== ChatSpeechToTextState.Idle ? stopLabel : getDictationHoverMarkdown(micLabel, this.configurationService)),
+				: (isDictationActiveOnSurface(sttService, 'chat') ? stopLabel : getDictationHoverMarkdown(micLabel, this.configurationService)),
 			position: { hoverPosition: HoverPosition.BELOW },
 			appearance: { showPointer: true }
 		})));
 
 		const downloadRing = this._register(new MutableDisposable<DictationDownloadRing>());
 		const renderState = () => {
-			const preparing = sttService.isPreparingModel;
+			const active = isDictationActiveOnSurface(sttService, 'chat');
+			const preparing = active && sttService.isPreparingModel;
 			// Only the active Recording state should read as "recording" (filled
 			// mic). Once the user stops, the service enters Transcribing while it
 			// waits for the final transcript (up to a few seconds on the cloud
 			// backend); during that the mic must already read as idle, matching
 			// the chat toolbar which flips as soon as recording stops.
-			const recording = sttService.state === ChatSpeechToTextState.Recording;
-			const active = sttService.state !== ChatSpeechToTextState.Idle;
+			const recording = active && sttService.state === ChatSpeechToTextState.Recording;
 			dom.clearNode(button);
 			downloadRing.clear();
 			if (preparing) {

src/vs/workbench/contrib/chat/browser/speechToText/chatSpeechToTextService.ts

@@ -15,7 +15,7 @@ import { isDark } from '../../../../../platform/theme/common/theme.js';
 import { chatDictationActiveMicGlow } from '../../common/widget/chatColors.js';
 import { readVoiceGlowIntensity } from '../voiceClient/voiceGlow.js';
 import { createVoiceRimLight, IVoiceRimLight } from '../voiceClient/voiceGlowController.js';
-import { ChatSpeechToTextState, IChatSpeechToTextService } from './chatSpeechToTextService.js';
+import { ChatSpeechToTextState, IChatSpeechToTextService, isDictationActiveOnSurface } from './chatSpeechToTextService.js';
 
 export type DictationMicGlowPhase = 'off' | 'live' | 'settling';
 
@@ -134,6 +134,7 @@ export function setupDictationMicGlow(
 	};
 
 	const update = (active = isActive?.get() !== false) => {
+		active = active && isDictationActiveOnSurface(service, 'chat');
 		const phase = active ? getDictationMicGlowPhase(service.state, service.isPreparingModel) : 'off';
 		target.classList.toggle('dictation-mic-active', phase !== 'off');
 		target.classList.toggle('dictation-mic-settling', phase === 'settling');

src/vs/workbench/contrib/chat/browser/speechToText/dictationMicGlow.ts

@@ -315,7 +315,22 @@ export function activeDictationEditor(): ICodeEditor | undefined {
 
 /** Start dictating into `editor`, rendering the transcript live. */
 export async function startDictation(service: IChatSpeechToTextService, editor: ICodeEditor, window: Window & typeof globalThis, logService: ILogService, surface: ChatDictationSurface = 'chat'): Promise<void> {
-	if (_active || service.state !== ChatSpeechToTextState.Idle) {
+	// Already dictating into this exact editor: nothing to do (callers toggle
+	// stopping separately).
+	if (_active?.editor === editor) {
+		return;
+	}
+	// Only one surface can use the shared on-device engine at a time. If a
+	// dictation is already running — in the chat input, another editor, or the
+	// terminal — cancel it so this surface can take over. The previous surface
+	// clears its own state and UI when it observes the engine go Idle, keeping
+	// whatever transcript it had already inserted.
+	if (_active || service.isBusy) {
+		await service.cancel();
+	}
+	// If the engine did not return to Idle (an unexpected busy state), do not
+	// attach this surface's listeners to it.
+	if (service.state !== ChatSpeechToTextState.Idle) {
 		return;
 	}
 	const inserter = new LiveTranscriptInserter(editor, logService);
@@ -488,7 +503,7 @@ export function cancelDictation(): void {
 	// the input exactly as it was before dictation started.
 	active.inserter.revert();
 	active.disposables.dispose();
-	active.service.cancel();
+	void active.service.cancel();
 }
 
 /**

src/vs/workbench/contrib/chat/browser/speechToText/dictationSession.ts

@@ -14,7 +14,7 @@ import { ITextModel } from '../../../../../editor/common/model.js';
 import { createTestCodeEditor } from '../../../../../editor/test/browser/testCodeEditor.js';
 import { createTextModel } from '../../../../../editor/test/common/testTextModel.js';
 import { NullLogService } from '../../../../../platform/log/common/log.js';
-import { ChatSpeechToTextState, IChatDictationTranscript, IChatSpeechToTextService } from '../../browser/speechToText/chatSpeechToTextService.js';
+import { ChatDictationSurface, ChatSpeechToTextState, IChatDictationTranscript, IChatSpeechToTextService, isDictationActiveOnSurface } from '../../browser/speechToText/chatSpeechToTextService.js';
 import { isDictating, startDictation, stopDictation, stopDictationForEditor } from '../../browser/speechToText/dictationSession.js';
 
 suite('DictationSession', () => {
@@ -26,10 +26,12 @@ suite('DictationSession', () => {
 	 * the test drive interim updates through the returned emitter. `setTranscript`
 	 * updates what a subsequent `stopAndTranscribe` resolves with.
 	 */
-	function createService(transcript: string, showTranscriptWhileDictating: boolean): { service: IChatSpeechToTextService; onDidUpdateTranscript: Emitter<IChatDictationTranscript>; setTranscript(text: string): void; blockStop(): () => void } {
+	function createService(transcript: string, showTranscriptWhileDictating: boolean, cancelBarrier?: Promise<void>): { service: IChatSpeechToTextService; onDidUpdateTranscript: Emitter<IChatDictationTranscript>; setTranscript(text: string): void; starts: ChatDictationSurface[]; blockStop(): () => void } {
 		const onDidUpdateTranscript = store.add(new Emitter<IChatDictationTranscript>());
 		const onDidChangeState = store.add(new Emitter<ChatSpeechToTextState>());
+		const starts: ChatDictationSurface[] = [];
 		let state = ChatSpeechToTextState.Idle;
+		let currentSurface: ChatDictationSurface = 'chat';
 		let finalTranscript = transcript;
 		let stopBarrier: Promise<void> | undefined;
 		const service: IChatSpeechToTextService = {
@@ -40,6 +42,8 @@ suite('DictationSession', () => {
 			onDidChangeDownloadingModel: store.add(new Emitter<boolean>()).event,
 			onDidChangeModelDownloadProgress: store.add(new Emitter<void>()).event,
 			get state() { return state; },
+			get isBusy() { return state !== ChatSpeechToTextState.Idle; },
+			get currentSurface() { return currentSurface; },
 			get showTranscriptWhileDictating() { return showTranscriptWhileDictating; },
 			get analyserNode() { return undefined; },
 			get isConfigured() { return true; },
@@ -48,7 +52,9 @@ suite('DictationSession', () => {
 			get modelDownloadProgress() { return undefined; },
 			get currentBackend() { return 'mai' as const; },
 			async switchMicrophone() { return undefined; },
-			async start() {
+			async start(_window, surface = 'chat') {
+				currentSurface = surface;
+				starts.push(surface);
 				state = ChatSpeechToTextState.Recording;
 				onDidChangeState.fire(state);
 			},
@@ -59,13 +65,18 @@ suite('DictationSession', () => {
 				onDidChangeState.fire(state);
 				return finalTranscript;
 			},
-			cancel() { },
+			async cancel() {
+				state = ChatSpeechToTextState.Idle;
+				onDidChangeState.fire(state);
+				await cancelBarrier;
+			},
 			logDictationAccuracy() { },
 		};
 		return {
 			service,
 			onDidUpdateTranscript,
 			setTranscript: text => { finalTranscript = text; },
+			starts,
 			blockStop: () => {
 				const deferred = new DeferredPromise<void>();
 				stopBarrier = deferred.p;
@@ -216,4 +227,66 @@ suite('DictationSession', () => {
 
 		assert.deepStrictEqual([afterMore, editor.getValue()], ['one twox three', 'one twox three']);
 	});
+
+	test('starting dictation in another editor takes over the shared session', async () => {
+		const { service, onDidUpdateTranscript, setTranscript } = createService('hello', true);
+		const model1 = store.add(createTextModel(''));
+		const editor1 = store.add(createTestCodeEditor(model1));
+		const model2 = store.add(createTextModel(''));
+		const editor2 = store.add(createTestCodeEditor(model2));
+
+		await startDictation(service, editor1, mainWindow, new NullLogService());
+		onDidUpdateTranscript.fire({ text: 'hello', finalizedText: '' });
+		const editor1WhileDictating = editor1.getValue();
+		// Starting dictation in a second editor cancels the first session (keeping
+		// its already-inserted text) and takes over the shared engine.
+		await startDictation(service, editor2, mainWindow, new NullLogService());
+		onDidUpdateTranscript.fire({ text: 'world', finalizedText: '' });
+		const editor2WhileDictating = editor2.getValue();
+		setTranscript('world');
+		await stopDictation();
+
+		assert.deepStrictEqual(
+			[editor1WhileDictating, editor1.getValue(), editor2WhileDictating, editor2.getValue()],
+			['hello', 'hello', 'world', 'world'],
+		);
+	});
+
+	test('reports dictation activity only for the active surface', async () => {
+		const { service } = createService('', true);
+		const model = store.add(createTextModel(''));
+		const editor = store.add(createTestCodeEditor(model));
+
+		await startDictation(service, editor, mainWindow, new NullLogService(), 'editor');
+		const whileDictating = [
+			isDictationActiveOnSurface(service, 'chat'),
+			isDictationActiveOnSurface(service, 'editor'),
+			isDictationActiveOnSurface(service, 'terminal'),
+		];
+		await stopDictation();
+
+		assert.deepStrictEqual(
+			[whileDictating, isDictationActiveOnSurface(service, 'editor')],
+			[[false, true, false], false],
+		);
+	});
+
+	test('waits for cancellation to settle before starting a replacement', async () => {
+		const cancelBarrier = new DeferredPromise<void>();
+		const { service, starts } = createService('', true, cancelBarrier.p);
+		const firstModel = store.add(createTextModel(''));
+		const firstEditor = store.add(createTestCodeEditor(firstModel));
+		const secondModel = store.add(createTextModel(''));
+		const secondEditor = store.add(createTestCodeEditor(secondModel));
+
+		await startDictation(service, firstEditor, mainWindow, new NullLogService(), 'chat');
+		const replacement = startDictation(service, secondEditor, mainWindow, new NullLogService(), 'terminal');
+		assert.deepStrictEqual(starts, ['chat']);
+
+		cancelBarrier.complete();
+		await replacement;
+		await stopDictation();
+
+		assert.deepStrictEqual(starts, ['chat', 'terminal']);
+	});
 });

src/vs/workbench/contrib/chat/test/browser/dictationSession.test.ts

@@ -265,22 +265,25 @@ export class EditorDictation extends Disposable implements IEditorContribution {
 
 		disposables.add(this.editor.onDidChangeCursorPosition(() => this.widget.layout()));
 
-		// When the shared session ends on its own (final transcript applied, an
-		// error, or the model failing to load), tear down the editor-side UI.
-		disposables.add(this.chatSpeechToTextService.onDidChangeState(state => {
-			if (state === ChatSpeechToTextState.Idle) {
-				this.sessionDisposables.clear();
-			}
-		}));
-
 		const window = getWindow(this.editor.getDomNode()) ?? getActiveWindow();
 		await startDictation(this.chatSpeechToTextService, this.editor, window, this.logService, 'editor');
 
-		// If the session did not take (already dictating elsewhere, or start
-		// failed without a state transition), do not leave the widget stranded.
+		// If the session did not take (start failed without a state transition),
+		// do not leave the widget stranded.
 		if (activeDictationEditor() !== this.editor) {
 			this.sessionDisposables.clear();
+			return;
 		}
+
+		// When the shared session ends on its own (final transcript applied, an
+		// error, or the model failing to load), tear down the editor-side UI. This
+		// is registered only after the takeover in `startDictation` has settled so
+		// cancelling a previous surface's session cannot tear down this one.
+		disposables.add(this.chatSpeechToTextService.onDidChangeState(state => {
+			if (state === ChatSpeechToTextState.Idle) {
+				this.sessionDisposables.clear();
+			}
+		}));
 	}
 
 	private async startWithProvider(): Promise<void> {

src/vs/workbench/contrib/codeEditor/browser/dictation/editorDictation.ts

@@ -227,9 +227,14 @@ export class TerminalVoiceSession extends Disposable {
 
 		// Only one dictation can run at a time (the on-device engine is a shared
 		// singleton). If it is already recording elsewhere (chat input or an
-		// editor), `service.start()` would no-op while these listeners stayed
-		// attached and streamed that other surface's transcript into the
-		// terminal. Reject a non-idle engine before subscribing.
+		// editor), cancel that session so the terminal can take over — the other
+		// surface clears its own state and UI when it observes the engine go Idle.
+		// This runs before we attach our own listeners below, so it cannot tear
+		// down this new terminal session.
+		if (service.isBusy) {
+			await service.cancel();
+		}
+		// If the engine somehow stayed busy, bail rather than subscribing to it.
 		if (service.state !== ChatSpeechToTextState.Idle) {
 			this.stop();
 			return;
@@ -347,7 +352,7 @@ export class TerminalVoiceSession extends Disposable {
 		// Abort the on-device engine on teardown. On the accept path the engine
 		// has already finished via stopAndTranscribe(), so this is a no-op there.
 		if (this._usingBuiltin) {
-			this._chatSpeechToTextService.cancel();
+			void this._chatSpeechToTextService.cancel();
 		}
 		this._disposables.clear();
 		this._input = '';

src/vs/workbench/contrib/terminalContrib/voice/browser/terminalVoice.ts

@@ -103,7 +103,7 @@ suite('TerminalVoiceSession', () => {
 				stateEmitter.fire(state);
 				return finalText;
 			}
-			override cancel(): void { }
+			override async cancel(): Promise<void> { }
 		});
 	}