Merge pull request #185268 from microsoft/merogge/loop-cue

add audio cues for AI panel chat

src/vs/editor/standalone/browser/standaloneServices.ts

@@ -87,7 +87,7 @@ import { IQuickInputService } from 'vs/platform/quickinput/common/quickInput';
 import { IStorageService, InMemoryStorageService } from 'vs/platform/storage/common/storage';
 import { DefaultConfiguration } from 'vs/platform/configuration/common/configurations';
 import { WorkspaceEdit } from 'vs/editor/common/languages';
-import { AudioCue, IAudioCueService, Sound } from 'vs/platform/audioCues/browser/audioCueService';
+import { AudioCue, AudioCueGroupId, IAudioCueService, Sound } from 'vs/platform/audioCues/browser/audioCueService';
 import { LogService } from 'vs/platform/log/common/logService';
 import { getEditorFeatures } from 'vs/editor/common/editorFeatures';
 import { onUnexpectedError } from 'vs/base/common/errors';
@@ -1055,6 +1055,11 @@ class StandaloneAudioService implements IAudioCueService {
 
 	async playSound(cue: Sound, allowManyInParallel?: boolean | undefined): Promise<void> {
 	}
+	playAudioCueLoop(cue: AudioCue): IDisposable {
+		return toDisposable(() => { });
+	}
+	playRandomAudioCue(groupId: AudioCueGroupId, allowManyInParallel?: boolean): void {
+	}
 }
 
 export interface IEditorOverrideServices {

src/vs/platform/audioCues/browser/audioCueService.ts

@@ -3,7 +3,7 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-import { Disposable } from 'vs/base/common/lifecycle';
+import { Disposable, IDisposable, toDisposable } from 'vs/base/common/lifecycle';
 import { FileAccess } from 'vs/base/common/network';
 import { IAccessibilityService } from 'vs/platform/accessibility/common/accessibility';
 import { IConfigurationService } from 'vs/platform/configuration/common/configuration';
@@ -22,6 +22,8 @@ export interface IAudioCueService {
 	onEnabledChanged(cue: AudioCue): Event<void>;
 
 	playSound(cue: Sound, allowManyInParallel?: boolean): Promise<void>;
+	playAudioCueLoop(cue: AudioCue, milliseconds: number): IDisposable;
+	playRandomAudioCue(groupId: AudioCueGroupId, allowManyInParallel?: boolean): void;
 }
 
 export class AudioCueService extends Disposable implements IAudioCueService {
@@ -51,6 +53,12 @@ export class AudioCueService extends Disposable implements IAudioCueService {
 		await Promise.all(Array.from(sounds).map(sound => this.playSound(sound, true)));
 	}
 
+	public playRandomAudioCue(groupId: AudioCueGroupId, allowManyInParallel?: boolean): void {
+		const cues = AudioCue.allAudioCues.filter(cue => cue.groupId === groupId);
+		const index = Math.floor(Math.random() * cues.length);
+		this.playAudioCue(cues[index], allowManyInParallel);
+	}
+
 	private getVolumeInPercent(): number {
 		const volume = this.configurationService.getValue<number>('audioCues.volume');
 		if (typeof volume !== 'number') {
@@ -66,7 +74,6 @@ export class AudioCueService extends Disposable implements IAudioCueService {
 		if (!allowManyInParallel && this.playingSounds.has(sound)) {
 			return;
 		}
-
 		this.playingSounds.add(sound);
 		const url = FileAccess.asBrowserUri(`vs/platform/audioCues/browser/media/${sound.fileName}`).toString(true);
 
@@ -87,6 +94,23 @@ export class AudioCueService extends Disposable implements IAudioCueService {
 		}
 	}
 
+	public playAudioCueLoop(cue: AudioCue, milliseconds: number): IDisposable {
+		let playing = true;
+		const playSound = () => {
+			if (playing) {
+				this.playAudioCue(cue, true).finally(() => {
+					setTimeout(() => {
+						if (playing) {
+							playSound();
+						}
+					}, milliseconds);
+				});
+			}
+		};
+		playSound();
+		return toDisposable(() => playing = false);
+	}
+
 	private readonly obsoleteAudioCuesEnabled = observableFromEvent(
 		Event.filter(this.configurationService.onDidChangeConfiguration, (e) =>
 			e.affectsConfiguration('audioCues.enabled')
@@ -190,19 +214,30 @@ export class Sound {
 	public static readonly diffLineInserted = Sound.register({ fileName: 'diffLineInserted.mp3' });
 	public static readonly diffLineDeleted = Sound.register({ fileName: 'diffLineDeleted.mp3' });
 	public static readonly diffLineModified = Sound.register({ fileName: 'diffLineModified.mp3' });
+	public static readonly chatRequestSent = Sound.register({ fileName: 'chatRequestSent.mp3' });
+	public static readonly chatResponsePending = Sound.register({ fileName: 'chatResponsePending.mp3' });
+	public static readonly chatResponseReceived1 = Sound.register({ fileName: 'chatResponseReceived1.mp3' });
+	public static readonly chatResponseReceived2 = Sound.register({ fileName: 'chatResponseReceived2.mp3' });
+	public static readonly chatResponseReceived3 = Sound.register({ fileName: 'chatResponseReceived3.mp3' });
+	public static readonly chatResponseReceived4 = Sound.register({ fileName: 'chatResponseReceived4.mp3' });
+	public static readonly chatResponseReceived5 = Sound.register({ fileName: 'chatResponseReceived5.mp3' });
 
 	private constructor(public readonly fileName: string) { }
 }
 
+export const enum AudioCueGroupId {
+	chatResponseReceived = 'chatResponseReceived'
+}
+
 export class AudioCue {
 	private static _audioCues = new Set<AudioCue>();
-
 	private static register(options: {
 		name: string;
 		sound: Sound;
 		settingsKey: string;
+		groupId?: AudioCueGroupId;
 	}): AudioCue {
-		const audioCue = new AudioCue(options.sound, options.name, options.settingsKey);
+		const audioCue = new AudioCue(options.sound, options.name, options.settingsKey, options.groupId);
 		AudioCue._audioCues.add(audioCue);
 		return audioCue;
 	}
@@ -309,9 +344,53 @@ export class AudioCue {
 		settingsKey: 'audioCues.diffLineModified'
 	});
 
+	public static readonly chatRequestSent = AudioCue.register({
+		name: localize('audioCues.chatRequestSent', 'Chat Request Sent'),
+		sound: Sound.chatRequestSent,
+		settingsKey: 'audioCues.chatRequestSent'
+	});
+
+	public static readonly chatResponseReceived = {
+		name: localize('audioCues.chatResponseReceived', 'Chat Response Received'),
+		settingsKey: 'audioCues.chatResponseReceived',
+		groupId: AudioCueGroupId.chatResponseReceived
+	};
+
+	public static readonly chatResponseReceived1 = AudioCue.register({
+		sound: Sound.chatResponseReceived1,
+		...this.chatResponseReceived
+	});
+
+	public static readonly chatResponseReceived2 = AudioCue.register({
+		sound: Sound.chatResponseReceived2,
+		...this.chatResponseReceived
+	});
+
+	public static readonly chatResponseReceived3 = AudioCue.register({
+		sound: Sound.chatResponseReceived3,
+		...this.chatResponseReceived
+	});
+
+	public static readonly chatResponseReceived4 = AudioCue.register({
+		sound: Sound.chatResponseReceived4,
+		...this.chatResponseReceived
+	});
+
+	public static readonly chatResponseReceived5 = AudioCue.register({
+		sound: Sound.chatResponseReceived5,
+		...this.chatResponseReceived
+	});
+
+	public static readonly chatResponsePending = AudioCue.register({
+		name: localize('audioCues.chatResponsePending', 'Chat Response Pending'),
+		sound: Sound.chatResponsePending,
+		settingsKey: 'audioCues.chatResponsePending'
+	});
+
 	private constructor(
 		public readonly sound: Sound,
 		public readonly name: string,
 		public readonly settingsKey: string,
+		public readonly groupId?: string
 	) { }
 }

src/vs/workbench/contrib/audioCues/browser/audioCues.contribution.ts

@@ -117,6 +117,21 @@ Registry.as<IConfigurationRegistry>(ConfigurationExtensions.Configuration).regis
 			'description': localize('audioCues.notebookCellFailed', "Plays a sound when a notebook cell execution fails."),
 			...audioCueFeatureBase,
 		},
+		'audioCues.chatRequestSent': {
+			'description': localize('audioCues.chatRequestSent', "Plays a sound when a chat request is made."),
+			...audioCueFeatureBase,
+			default: 'off'
+		},
+		'audioCues.chatResponsePending': {
+			'description': localize('audioCues.chatResponsePending', "Plays a sound on loop while the response is pending."),
+			...audioCueFeatureBase,
+			default: 'off'
+		},
+		'audioCues.chatResponseReceived': {
+			'description': localize('audioCues.chatResponseReceived', "Plays a sound on loop while the response has been received."),
+			...audioCueFeatureBase,
+			default: 'off'
+		}
 	}
 });
 

src/vs/workbench/contrib/chat/browser/chat.contribution.ts

@@ -22,11 +22,11 @@ import { registerChatExecuteActions } from 'vs/workbench/contrib/chat/browser/ac
 import { registerChatQuickQuestionActions } from 'vs/workbench/contrib/chat/browser/actions/chatQuickInputActions';
 import { registerChatTitleActions } from 'vs/workbench/contrib/chat/browser/actions/chatTitleActions';
 import { registerChatExportActions } from 'vs/workbench/contrib/chat/browser/actions/chatImportExport';
-import { IChatWidgetService } from 'vs/workbench/contrib/chat/browser/chat';
+import { IChatAccessibilityService, IChatWidgetService } from 'vs/workbench/contrib/chat/browser/chat';
 import { ChatContributionService } from 'vs/workbench/contrib/chat/browser/chatContributionServiceImpl';
 import { ChatEditor, IChatEditorOptions } from 'vs/workbench/contrib/chat/browser/chatEditor';
 import { ChatEditorInput, ChatEditorInputSerializer } from 'vs/workbench/contrib/chat/browser/chatEditorInput';
-import { ChatWidgetService } from 'vs/workbench/contrib/chat/browser/chatWidget';
+import { ChatAccessibilityService, ChatWidgetService } from 'vs/workbench/contrib/chat/browser/chatWidget';
 import 'vs/workbench/contrib/chat/browser/contrib/chatInputEditorContrib';
 import { IChatContributionService } from 'vs/workbench/contrib/chat/common/chatContributionService';
 import { IChatService } from 'vs/workbench/contrib/chat/common/chatService';
@@ -136,5 +136,6 @@ registerClearActions();
 registerSingleton(IChatService, ChatService, InstantiationType.Delayed);
 registerSingleton(IChatContributionService, ChatContributionService, InstantiationType.Delayed);
 registerSingleton(IChatWidgetService, ChatWidgetService, InstantiationType.Delayed);
+registerSingleton(IChatAccessibilityService, ChatAccessibilityService, InstantiationType.Delayed);
 registerSingleton(IChatWidgetHistoryService, ChatWidgetHistoryService, InstantiationType.Delayed);
 

src/vs/workbench/contrib/chat/browser/chat.ts

@@ -11,6 +11,7 @@ import { URI } from 'vs/base/common/uri';
 import { createDecorator } from 'vs/platform/instantiation/common/instantiation';
 
 export const IChatWidgetService = createDecorator<IChatWidgetService>('chatWidgetService');
+export const IChatAccessibilityService = createDecorator<IChatAccessibilityService>('chatAccessibilityService');
 
 export interface IChatWidgetService {
 
@@ -29,6 +30,13 @@ export interface IChatWidgetService {
 	getWidgetByInputUri(uri: URI): IChatWidget | undefined;
 }
 
+
+export interface IChatAccessibilityService {
+	readonly _serviceBrand: undefined;
+	acceptRequest(): void;
+	acceptResponse(response?: IChatResponseViewModel): void;
+}
+
 export interface IChatCodeBlockInfo {
 	codeBlockIndex: number;
 	element: IChatResponseViewModel;

src/vs/workbench/contrib/chat/browser/chatWidget.ts

@@ -16,6 +16,7 @@ import 'vs/css!./media/chat';
 import { ICodeEditor } from 'vs/editor/browser/editorBrowser';
 import { localize } from 'vs/nls';
 import { MenuId } from 'vs/platform/actions/common/actions';
+import { AudioCue, AudioCueGroupId, IAudioCueService } from 'vs/platform/audioCues/browser/audioCueService';
 import { IContextKey, IContextKeyService } from 'vs/platform/contextkey/common/contextkey';
 import { IContextMenuService } from 'vs/platform/contextview/browser/contextView';
 import { IInstantiationService } from 'vs/platform/instantiation/common/instantiation';
@@ -23,7 +24,7 @@ import { ServiceCollection } from 'vs/platform/instantiation/common/serviceColle
 import { WorkbenchObjectTree } from 'vs/platform/list/browser/listService';
 import { IViewsService } from 'vs/workbench/common/views';
 import { clearChatSession } from 'vs/workbench/contrib/chat/browser/actions/chatClear';
-import { ChatTreeItem, IChatCodeBlockInfo, IChatWidget, IChatWidgetService, IChatWidgetViewContext } from 'vs/workbench/contrib/chat/browser/chat';
+import { ChatTreeItem, IChatAccessibilityService, IChatCodeBlockInfo, IChatWidget, IChatWidgetService, IChatWidgetViewContext } from 'vs/workbench/contrib/chat/browser/chat';
 import { ChatInputPart } from 'vs/workbench/contrib/chat/browser/chatInputPart';
 import { ChatAccessibilityProvider, ChatListDelegate, ChatListItemRenderer, IChatRendererDelegate } from 'vs/workbench/contrib/chat/browser/chatListRenderer';
 import { ChatEditorOptions } from 'vs/workbench/contrib/chat/browser/chatOptions';
@@ -115,6 +116,7 @@ export class ChatWidget extends Disposable implements IChatWidget {
 		@IChatService private readonly chatService: IChatService,
 		@IChatWidgetService chatWidgetService: IChatWidgetService,
 		@IContextMenuService private readonly contextMenuService: IContextMenuService,
+		@IChatAccessibilityService private readonly _chatAccessibilityService: IChatAccessibilityService
 	) {
 		super();
 		CONTEXT_IN_CHAT_SESSION.bindTo(contextKeyService).set(true);
@@ -388,18 +390,17 @@ export class ChatWidget extends Disposable implements IChatWidget {
 				this.instantiationService.invokeFunction(clearChatSession, this);
 				return;
 			}
-
+			this._chatAccessibilityService.acceptRequest();
 			const input = query ?? editorValue;
 			const result = await this.chatService.sendRequest(this.viewModel.sessionId, input);
+
 			if (result) {
 				this.inputPart.acceptInput(query);
-				result.responseCompletePromise.then(() => {
+				result.responseCompletePromise.then(async () => {
+
 					const responses = this.viewModel?.getItems().filter(isResponseVM);
 					const lastResponse = responses?.[responses.length - 1];
-					if (lastResponse) {
-						const errorDetails = lastResponse.errorDetails ? ` ${lastResponse.errorDetails.message}` : '';
-						alert(lastResponse.response.value + errorDetails);
-					}
+					this._chatAccessibilityService.acceptResponse(lastResponse);
 				});
 			}
 		}
@@ -505,3 +506,30 @@ export class ChatWidgetService implements IChatWidgetService {
 		);
 	}
 }
+
+
+const CHAT_RESPONSE_PENDING_AUDIO_CUE_LOOP_MS = 7000;
+export class ChatAccessibilityService extends Disposable implements IChatAccessibilityService {
+
+	declare readonly _serviceBrand: undefined;
+
+	private _responsePendingAudioCue: IDisposable | undefined;
+
+	constructor(@IAudioCueService private readonly _audioCueService: IAudioCueService) {
+		super();
+	}
+	acceptRequest(): void {
+		this._audioCueService.playAudioCue(AudioCue.chatRequestSent, true);
+		this._responsePendingAudioCue = this._audioCueService.playAudioCueLoop(AudioCue.chatResponsePending, CHAT_RESPONSE_PENDING_AUDIO_CUE_LOOP_MS);
+	}
+	acceptResponse(response?: IChatResponseViewModel): void {
+		this._responsePendingAudioCue?.dispose();
+		this._audioCueService.playRandomAudioCue(AudioCueGroupId.chatResponseReceived, true);
+		if (!response) {
+			return;
+		}
+		const errorDetails = response.errorDetails ? ` ${response.errorDetails.message}` : '';
+		alert(response.response.value + errorDetails);
+	}
+}
+