diff --git a/src/vs/base/common/event.ts b/src/vs/base/common/event.ts index e0070330e1d..3686b27d8e8 100644 --- a/src/vs/base/common/event.ts +++ b/src/vs/base/common/event.ts @@ -622,27 +622,6 @@ export namespace Event { return event(e => handler(e)); } - /** - * Adds a listener to an event and calls the listener immediately with undefined as the event object. A new - * {@link DisposableStore} is passed to the listener which is disposed when the returned disposable is disposed. - */ - export function runAndSubscribeWithStore(event: Event, handler: (e: T | undefined, disposableStore: DisposableStore) => any): IDisposable { - let store: DisposableStore | null = null; - - function run(e: T | undefined) { - store?.dispose(); - store = new DisposableStore(); - handler(e, store); - } - - run(undefined); - const disposable = event(e => run(e)); - return toDisposable(() => { - disposable.dispose(); - store?.dispose(); - }); - } - class EmitterObserver implements IObserver { readonly emitter: Emitter; diff --git a/src/vs/base/test/common/event.test.ts b/src/vs/base/test/common/event.test.ts index df7aac4363d..49962c89d5a 100644 --- a/src/vs/base/test/common/event.test.ts +++ b/src/vs/base/test/common/event.test.ts @@ -8,7 +8,7 @@ import { DeferredPromise, timeout } from 'vs/base/common/async'; import { CancellationToken } from 'vs/base/common/cancellation'; import { errorHandler, setUnexpectedErrorHandler } from 'vs/base/common/errors'; import { AsyncEmitter, DebounceEmitter, DynamicListEventMultiplexer, Emitter, Event, EventBufferer, EventMultiplexer, IWaitUntil, MicrotaskEmitter, PauseableEmitter, Relay, createEventDeliveryQueue } from 'vs/base/common/event'; -import { DisposableStore, IDisposable, isDisposable, setDisposableTracker, toDisposable, DisposableTracker } from 'vs/base/common/lifecycle'; +import { DisposableStore, IDisposable, isDisposable, setDisposableTracker, DisposableTracker } from 'vs/base/common/lifecycle'; import { observableValue, transaction } from 'vs/base/common/observable'; import { MicrotaskDelay } from 'vs/base/common/symbols'; import { runWithFakedTimers } from 'vs/base/test/common/timeTravelScheduler'; @@ -1272,36 +1272,6 @@ suite('Event utils', () => { }); }); - test('runAndSubscribeWithStore', () => { - const eventEmitter = ds.add(new Emitter()); - const event = eventEmitter.event; - - let i = 0; - const log = new Array(); - const disposable = Event.runAndSubscribeWithStore(event, (e, disposables) => { - const idx = i++; - log.push({ label: 'handleEvent', data: e || null, idx }); - disposables.add(toDisposable(() => { - log.push({ label: 'dispose', idx }); - })); - }); - - log.push({ label: 'fire' }); - eventEmitter.fire('someEventData'); - - log.push({ label: 'disposeAll' }); - disposable.dispose(); - - assert.deepStrictEqual(log, [ - { label: 'handleEvent', data: null, idx: 0 }, - { label: 'fire' }, - { label: 'dispose', idx: 0 }, - { label: 'handleEvent', data: 'someEventData', idx: 1 }, - { label: 'disposeAll' }, - { label: 'dispose', idx: 1 }, - ]); - }); - suite('accumulate', () => { test('should not fire after a listener is disposed with undefined or []', async () => { const eventEmitter = ds.add(new Emitter()); diff --git a/src/vs/workbench/api/browser/mainThreadSpeech.ts b/src/vs/workbench/api/browser/mainThreadSpeech.ts index d0c7acdc2b6..fa52270cdf0 100644 --- a/src/vs/workbench/api/browser/mainThreadSpeech.ts +++ b/src/vs/workbench/api/browser/mainThreadSpeech.ts @@ -8,20 +8,26 @@ import { Emitter } from 'vs/base/common/event'; import { Disposable, DisposableStore, IDisposable } from 'vs/base/common/lifecycle'; import { ILogService } from 'vs/platform/log/common/log'; import { ExtHostContext, ExtHostSpeechShape, MainContext, MainThreadSpeechShape } from 'vs/workbench/api/common/extHost.protocol'; -import { ISpeechProviderMetadata, ISpeechService, ISpeechToTextEvent } from 'vs/workbench/contrib/speech/common/speechService'; +import { IKeywordRecognitionEvent, ISpeechProviderMetadata, ISpeechService, ISpeechToTextEvent } from 'vs/workbench/contrib/speech/common/speechService'; import { IExtHostContext, extHostNamedCustomer } from 'vs/workbench/services/extensions/common/extHostCustomers'; type SpeechToTextSession = { readonly onDidChange: Emitter; }; +type KeywordRecognitionSession = { + readonly onDidChange: Emitter; +}; + @extHostNamedCustomer(MainContext.MainThreadSpeech) export class MainThreadSpeech extends Disposable implements MainThreadSpeechShape { private readonly proxy: ExtHostSpeechShape; private readonly providerRegistrations = new Map(); - private readonly providerSessions = new Map(); + + private readonly speechToTextSessions = new Map(); + private readonly keywordRecognitionSessions = new Map(); constructor( extHostContext: IExtHostContext, @@ -47,13 +53,33 @@ export class MainThreadSpeech extends Disposable implements MainThreadSpeechShap disposables.add(token.onCancellationRequested(() => this.proxy.$cancelSpeechToTextSession(session))); const onDidChange = disposables.add(new Emitter()); - this.providerSessions.set(session, { onDidChange }); + this.speechToTextSessions.set(session, { onDidChange }); return { onDidChange: onDidChange.event, dispose: () => { cts.dispose(true); - this.providerSessions.delete(session); + this.speechToTextSessions.delete(session); + disposables.dispose(); + } + }; + }, + createKeywordRecognitionSession: token => { + const disposables = new DisposableStore(); + const cts = new CancellationTokenSource(token); + const session = Math.random(); + + this.proxy.$createKeywordRecognitionSession(handle, session); + disposables.add(token.onCancellationRequested(() => this.proxy.$cancelKeywordRecognitionSession(session))); + + const onDidChange = disposables.add(new Emitter()); + this.keywordRecognitionSessions.set(session, { onDidChange }); + + return { + onDidChange: onDidChange.event, + dispose: () => { + cts.dispose(true); + this.keywordRecognitionSessions.delete(session); disposables.dispose(); } }; @@ -75,9 +101,12 @@ export class MainThreadSpeech extends Disposable implements MainThreadSpeechShap } $emitSpeechToTextEvent(session: number, event: ISpeechToTextEvent): void { - const providerSession = this.providerSessions.get(session); - if (providerSession) { - providerSession.onDidChange.fire(event); - } + const providerSession = this.speechToTextSessions.get(session); + providerSession?.onDidChange.fire(event); + } + + $emitKeywordRecognitionEvent(session: number, event: IKeywordRecognitionEvent): void { + const providerSession = this.keywordRecognitionSessions.get(session); + providerSession?.onDidChange.fire(event); } } diff --git a/src/vs/workbench/api/common/extHost.api.impl.ts b/src/vs/workbench/api/common/extHost.api.impl.ts index ee65e81f97f..dd36b43f34f 100644 --- a/src/vs/workbench/api/common/extHost.api.impl.ts +++ b/src/vs/workbench/api/common/extHost.api.impl.ts @@ -1627,7 +1627,8 @@ export function createApiFactoryAndRegisterActors(accessor: ServicesAccessor): I StackFrameFocus: extHostTypes.StackFrameFocus, ThreadFocus: extHostTypes.ThreadFocus, RelatedInformationType: extHostTypes.RelatedInformationType, - SpeechToTextStatus: extHostTypes.SpeechToTextStatus + SpeechToTextStatus: extHostTypes.SpeechToTextStatus, + KeywordRecognitionStatus: extHostTypes.KeywordRecognitionStatus }; }; } diff --git a/src/vs/workbench/api/common/extHost.protocol.ts b/src/vs/workbench/api/common/extHost.protocol.ts index 4f789f46598..978e0179e80 100644 --- a/src/vs/workbench/api/common/extHost.protocol.ts +++ b/src/vs/workbench/api/common/extHost.protocol.ts @@ -64,7 +64,7 @@ import { ICellRange } from 'vs/workbench/contrib/notebook/common/notebookRange'; import { InputValidationType } from 'vs/workbench/contrib/scm/common/scm'; import { IWorkspaceSymbol, NotebookPriorityInfo } from 'vs/workbench/contrib/search/common/search'; import { IRawClosedNotebookFileMatch } from 'vs/workbench/contrib/search/common/searchNotebookHelpers'; -import { ISpeechProviderMetadata, ISpeechToTextEvent } from 'vs/workbench/contrib/speech/common/speechService'; +import { IKeywordRecognitionEvent, ISpeechProviderMetadata, ISpeechToTextEvent } from 'vs/workbench/contrib/speech/common/speechService'; import { CoverageDetails, ExtensionRunTestsRequest, ICallProfileRunHandler, IFileCoverage, ISerializedTestResults, IStartControllerTests, ITestItem, ITestMessage, ITestRunProfile, ITestRunTask, ResolvedTestRunRequest, TestResultState, TestsDiffOp } from 'vs/workbench/contrib/testing/common/testTypes'; import { Timeline, TimelineChangeEvent, TimelineOptions, TimelineProviderDescriptor } from 'vs/workbench/contrib/timeline/common/timeline'; import { TypeHierarchyItem } from 'vs/workbench/contrib/typeHierarchy/common/typeHierarchy'; @@ -1159,11 +1159,15 @@ export interface MainThreadSpeechShape extends IDisposable { $unregisterProvider(handle: number): void; $emitSpeechToTextEvent(session: number, event: ISpeechToTextEvent): void; + $emitKeywordRecognitionEvent(session: number, event: IKeywordRecognitionEvent): void; } export interface ExtHostSpeechShape { $createSpeechToTextSession(handle: number, session: number): Promise; $cancelSpeechToTextSession(session: number): Promise; + + $createKeywordRecognitionSession(handle: number, session: number): Promise; + $cancelKeywordRecognitionSession(session: number): Promise; } export interface MainThreadChatProviderShape extends IDisposable { diff --git a/src/vs/workbench/api/common/extHostSpeech.ts b/src/vs/workbench/api/common/extHostSpeech.ts index 8207ab47a47..8d230fd19f2 100644 --- a/src/vs/workbench/api/common/extHostSpeech.ts +++ b/src/vs/workbench/api/common/extHostSpeech.ts @@ -52,6 +52,34 @@ export class ExtHostSpeech implements ExtHostSpeechShape { this.sessions.delete(session); } + async $createKeywordRecognitionSession(handle: number, session: number): Promise { + const provider = this.providers.get(handle); + if (!provider) { + return; + } + + const disposables = new DisposableStore(); + + const cts = new CancellationTokenSource(); + this.sessions.set(session, cts); + + const keywordRecognitionSession = disposables.add(provider.provideKeywordRecognitionSession(cts.token)); + disposables.add(keywordRecognitionSession.onDidChange(e => { + if (cts.token.isCancellationRequested) { + return; + } + + this.proxy.$emitKeywordRecognitionEvent(session, e); + })); + + disposables.add(cts.token.onCancellationRequested(() => disposables.dispose())); + } + + async $cancelKeywordRecognitionSession(session: number): Promise { + this.sessions.get(session)?.dispose(true); + this.sessions.delete(session); + } + registerProvider(extension: ExtensionIdentifier, identifier: string, provider: vscode.SpeechProvider): IDisposable { const handle = ExtHostSpeech.ID_POOL++; diff --git a/src/vs/workbench/api/common/extHostTypes.ts b/src/vs/workbench/api/common/extHostTypes.ts index 12fe0b4779d..4c45b385fa8 100644 --- a/src/vs/workbench/api/common/extHostTypes.ts +++ b/src/vs/workbench/api/common/extHostTypes.ts @@ -4205,4 +4205,9 @@ export enum SpeechToTextStatus { Stopped = 4 } +export enum KeywordRecognitionStatus { + Recognized = 1, + Stopped = 2 +} + //#endregion diff --git a/src/vs/workbench/contrib/accessibility/browser/accessibility.contribution.ts b/src/vs/workbench/contrib/accessibility/browser/accessibility.contribution.ts index 5df70bb0761..ac3d45e7a70 100644 --- a/src/vs/workbench/contrib/accessibility/browser/accessibility.contribution.ts +++ b/src/vs/workbench/contrib/accessibility/browser/accessibility.contribution.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { InstantiationType, registerSingleton } from 'vs/platform/instantiation/common/extensions'; -import { registerAccessibilityConfiguration } from 'vs/workbench/contrib/accessibility/browser/accessibilityConfiguration'; +import { DynamicSpeechAccessibilityConfiguration, registerAccessibilityConfiguration } from 'vs/workbench/contrib/accessibility/browser/accessibilityConfiguration'; import { IWorkbenchContributionsRegistry, Extensions as WorkbenchExtensions } from 'vs/workbench/common/contributions'; import { LifecyclePhase } from 'vs/workbench/services/lifecycle/common/lifecycle'; import { Registry } from 'vs/platform/registry/common/platform'; @@ -30,3 +30,4 @@ workbenchContributionsRegistry.registerWorkbenchContribution(NotificationAccessi workbenchContributionsRegistry.registerWorkbenchContribution(InlineCompletionsAccessibleViewContribution, LifecyclePhase.Eventually); workbenchContributionsRegistry.registerWorkbenchContribution(AccessibilityStatus, LifecyclePhase.Ready); workbenchContributionsRegistry.registerWorkbenchContribution(SaveAudioCueContribution, LifecyclePhase.Ready); +workbenchContributionsRegistry.registerWorkbenchContribution(DynamicSpeechAccessibilityConfiguration, LifecyclePhase.Ready); diff --git a/src/vs/workbench/contrib/accessibility/browser/accessibilityConfiguration.ts b/src/vs/workbench/contrib/accessibility/browser/accessibilityConfiguration.ts index 16b03b216cd..5c37fed8c8c 100644 --- a/src/vs/workbench/contrib/accessibility/browser/accessibilityConfiguration.ts +++ b/src/vs/workbench/contrib/accessibility/browser/accessibilityConfiguration.ts @@ -9,6 +9,10 @@ import { Registry } from 'vs/platform/registry/common/platform'; import { RawContextKey } from 'vs/platform/contextkey/common/contextkey'; import { workbenchConfigurationNodeBase } from 'vs/workbench/common/configuration'; import { AccessibilityAlertSettingId } from 'vs/platform/audioCues/browser/audioCueService'; +import { ISpeechService } from 'vs/workbench/contrib/speech/common/speechService'; +import { Disposable } from 'vs/base/common/lifecycle'; +import { IWorkbenchContribution } from 'vs/workbench/common/contributions'; +import { Event } from 'vs/base/common/event'; export const accessibilityHelpIsShown = new RawContextKey('accessibilityHelpIsShown', false, true); export const accessibleViewIsShown = new RawContextKey('accessibleViewIsShown', false, true); @@ -35,11 +39,6 @@ export const enum ViewDimUnfocusedOpacityProperties { Maximum = 1 } -export const enum AccessibilityVoiceSettingId { - SpeechTimeout = 'accessibility.voice.speechTimeout', -} -export const SpeechTimeoutDefault = 1200; - export const enum AccessibilityVerbositySettingId { Terminal = 'accessibility.verbosity.terminal', DiffEditor = 'accessibility.verbosity.diffEditor', @@ -77,10 +76,14 @@ const baseProperty: object = { tags: ['accessibility'] }; -const configuration: IConfigurationNode = { +export const accessibilityConfigurationNodeBase = Object.freeze({ id: 'accessibility', title: localize('accessibilityConfigurationTitle', "Accessibility"), - type: 'object', + type: 'object' +}); + +const configuration: IConfigurationNode = { + ...accessibilityConfigurationNodeBase, properties: { [AccessibilityVerbositySettingId.Terminal]: { description: localize('verbosity.terminal.description', 'Provide information about how to access the terminal accessibility help menu when the terminal is focused.'), @@ -251,13 +254,6 @@ const configuration: IConfigurationNode = { 'default': true, tags: ['accessibility'] }, - [AccessibilityVoiceSettingId.SpeechTimeout]: { - 'markdownDescription': localize('voice.speechTimeout', "The duration in milliseconds that voice speech recognition remains active after you stop speaking. For example in a chat session, the transcribed text is submitted automatically after the timeout is met. Set to `0` to disable this feature."), - 'type': 'number', - 'default': SpeechTimeoutDefault, - 'minimum': 0, - 'tags': ['accessibility'] - }, [AccessibilityWorkbenchSettingId.AccessibleViewCloseOnKeyPress]: { markdownDescription: localize('terminal.integrated.accessibleView.closeOnKeyPress', "On keypress, close the Accessible View and focus the element from which it was invoked."), type: 'boolean', @@ -298,3 +294,39 @@ export function registerAccessibilityConfiguration() { } }); } + +export const enum AccessibilityVoiceSettingId { + SpeechTimeout = 'accessibility.voice.speechTimeout' +} +export const SpeechTimeoutDefault = 1200; + +export class DynamicSpeechAccessibilityConfiguration extends Disposable implements IWorkbenchContribution { + + constructor( + @ISpeechService private readonly speechService: ISpeechService + ) { + super(); + + this._register(Event.runAndSubscribe(speechService.onDidRegisterSpeechProvider, () => this.updateConfiguration())); + } + + private updateConfiguration(): void { + if (!this.speechService.hasSpeechProvider) { + return; // these settings require a speech provider + } + + const registry = Registry.as(Extensions.Configuration); + registry.registerConfiguration({ + ...accessibilityConfigurationNodeBase, + properties: { + [AccessibilityVoiceSettingId.SpeechTimeout]: { + 'markdownDescription': localize('voice.speechTimeout', "The duration in milliseconds that voice speech recognition remains active after you stop speaking. For example in a chat session, the transcribed text is submitted automatically after the timeout is met. Set to `0` to disable this feature."), + 'type': 'number', + 'default': SpeechTimeoutDefault, + 'minimum': 0, + 'tags': ['accessibility'] + } + } + }); + } +} diff --git a/src/vs/workbench/contrib/chat/electron-sandbox/actions/voiceChatActions.ts b/src/vs/workbench/contrib/chat/electron-sandbox/actions/voiceChatActions.ts index 2a684fff64b..11dcef4882a 100644 --- a/src/vs/workbench/contrib/chat/electron-sandbox/actions/voiceChatActions.ts +++ b/src/vs/workbench/contrib/chat/electron-sandbox/actions/voiceChatActions.ts @@ -8,7 +8,7 @@ import { Event } from 'vs/base/common/event'; import { firstOrDefault } from 'vs/base/common/arrays'; import { CancellationTokenSource } from 'vs/base/common/cancellation'; import { Codicon } from 'vs/base/common/codicons'; -import { DisposableStore, toDisposable } from 'vs/base/common/lifecycle'; +import { Disposable, DisposableStore, MutableDisposable, toDisposable } from 'vs/base/common/lifecycle'; import { ServicesAccessor } from 'vs/editor/browser/editorExtensions'; import { localize } from 'vs/nls'; import { Action2, MenuId } from 'vs/platform/actions/common/actions'; @@ -23,14 +23,14 @@ import { CONTEXT_CHAT_REQUEST_IN_PROGRESS, CONTEXT_PROVIDER_EXISTS } from 'vs/wo import { InlineChatController } from 'vs/workbench/contrib/inlineChat/browser/inlineChatController'; import { IEditorService } from 'vs/workbench/services/editor/common/editorService'; import { getCodeEditor } from 'vs/editor/browser/editorBrowser'; -import { ICommandService } from 'vs/platform/commands/common/commands'; +import { CommandsRegistry, ICommandService } from 'vs/platform/commands/common/commands'; import { ActiveEditorContext } from 'vs/workbench/common/contextkeys'; import { IViewsService } from 'vs/workbench/services/views/common/viewsService'; import { IChatContributionService } from 'vs/workbench/contrib/chat/common/chatContributionService'; import { KeybindingWeight } from 'vs/platform/keybinding/common/keybindingsRegistry'; import { KeyCode } from 'vs/base/common/keyCodes'; import { IWorkbenchLayoutService, Parts } from 'vs/workbench/services/layout/browser/layoutService'; -import { HasSpeechProvider, ISpeechService, SpeechToTextStatus } from 'vs/workbench/contrib/speech/common/speechService'; +import { HasSpeechProvider, ISpeechService, KeywordRecognitionStatus, SpeechToTextStatus } from 'vs/workbench/contrib/speech/common/speechService'; import { RunOnceScheduler } from 'vs/base/common/async'; import { registerThemingParticipant } from 'vs/platform/theme/common/themeService'; import { ACTIVITY_BAR_BADGE_BACKGROUND } from 'vs/workbench/common/theme'; @@ -39,8 +39,13 @@ import { Color } from 'vs/base/common/color'; import { contrastBorder, focusBorder } from 'vs/platform/theme/common/colorRegistry'; import { IConfigurationService } from 'vs/platform/configuration/common/configuration'; import { isNumber } from 'vs/base/common/types'; -import { AccessibilityVoiceSettingId, SpeechTimeoutDefault } from 'vs/workbench/contrib/accessibility/browser/accessibilityConfiguration'; +import { AccessibilityVoiceSettingId, SpeechTimeoutDefault, accessibilityConfigurationNodeBase } from 'vs/workbench/contrib/accessibility/browser/accessibilityConfiguration'; import { IChatExecuteActionContext } from 'vs/workbench/contrib/chat/browser/actions/chatExecuteActions'; +import { IWorkbenchContribution } from 'vs/workbench/common/contributions'; +import { Registry } from 'vs/platform/registry/common/platform'; +import { IConfigurationRegistry, Extensions } from 'vs/platform/configuration/common/configurationRegistry'; +import { IStatusbarEntry, IStatusbarEntryAccessor, IStatusbarService, StatusbarAlignment } from 'vs/workbench/services/statusbar/browser/statusbar'; +import { IEditorGroupsService } from 'vs/workbench/services/editor/common/editorGroupsService'; const CONTEXT_VOICE_CHAT_GETTING_READY = new RawContextKey('voiceChatGettingReady', false, { type: 'boolean', description: localize('voiceChatGettingReady', "True when getting ready for receiving voice input from the microphone for voice chat.") }); const CONTEXT_VOICE_CHAT_IN_PROGRESS = new RawContextKey('voiceChatInProgress', false, { type: 'boolean', description: localize('voiceChatInProgress', "True when voice recording from microphone is in progress for voice chat.") }); @@ -747,3 +752,213 @@ registerThemingParticipant((theme, collector) => { } `); }); + +export class KeywordActivationContribution extends Disposable implements IWorkbenchContribution { + + static SETTINGS_ID = 'accessibility.voice.keywordActivation'; + + static SETTINGS_VALUE = { + OFF: 'off', + INLINE_CHAT: 'inlineChat', + QUICK_CHAT: 'quickChat', + VIEW_CHAT: 'chatInView', + CHAT_IN_CONTEXT: 'chatInContext' + }; + + private activeSession: CancellationTokenSource | undefined = undefined; + + constructor( + @ISpeechService private readonly speechService: ISpeechService, + @IConfigurationService private readonly configurationService: IConfigurationService, + @ICommandService private readonly commandService: ICommandService, + @IEditorGroupsService private readonly editorGroupService: IEditorGroupsService, + @IInstantiationService instantiationService: IInstantiationService + ) { + super(); + + this._register(instantiationService.createInstance(KeywordActivationStatusEntry)); + + this.registerListeners(); + } + + private registerListeners(): void { + this._register(Event.runAndSubscribe(this.speechService.onDidRegisterSpeechProvider, () => { + this.updateConfiguration(); + this.handleKeywordActivation(); + })); + + this._register(this.speechService.onDidStartSpeechToTextSession(() => this.handleKeywordActivation())); + this._register(this.speechService.onDidEndSpeechToTextSession(() => this.handleKeywordActivation())); + + this._register(this.configurationService.onDidChangeConfiguration(e => { + if (e.affectsConfiguration(KeywordActivationContribution.SETTINGS_ID)) { + this.handleKeywordActivation(); + } + })); + + this._register(this.editorGroupService.onDidCreateAuxiliaryEditorPart(({ instantiationService, disposables }) => { + disposables.add(instantiationService.createInstance(KeywordActivationStatusEntry)); + })); + } + + private updateConfiguration(): void { + if (!this.speechService.hasSpeechProvider) { + return; // these settings require a speech provider + } + + const registry = Registry.as(Extensions.Configuration); + registry.registerConfiguration({ + ...accessibilityConfigurationNodeBase, + properties: { + [KeywordActivationContribution.SETTINGS_ID]: { + 'type': 'string', + 'enum': [ + KeywordActivationContribution.SETTINGS_VALUE.OFF, + KeywordActivationContribution.SETTINGS_VALUE.VIEW_CHAT, + KeywordActivationContribution.SETTINGS_VALUE.QUICK_CHAT, + KeywordActivationContribution.SETTINGS_VALUE.INLINE_CHAT, + KeywordActivationContribution.SETTINGS_VALUE.CHAT_IN_CONTEXT + ], + 'enumDescriptions': [ + localize('voice.keywordActivation.off', "Keyword activation is disabled."), + localize('voice.keywordActivation.chatInView', "Keyword activation is enabled and listening for 'Hey Code' to start a voice chat session in the chat view."), + localize('voice.keywordActivation.quickChat', "Keyword activation is enabled and listening for 'Hey Code' to start a voice chat session in the quick chat."), + localize('voice.keywordActivation.inlineChat', "Keyword activation is enabled and listening for 'Hey Code' to start a voice chat session in the active editor."), + localize('voice.keywordActivation.chatInContext', "Keyword activation is enabled and listening for 'Hey Code' to start a voice chat session in the active editor or view depending on keyboard focus.") + ], + 'description': localize('voice.keywordActivation', "Controls whether the phrase 'Hey Code' should be speech recognized to start a voice chat session."), + 'default': 'off', + 'tags': ['accessibility'] + } + } + }); + } + + private handleKeywordActivation(): void { + const enabled = + this.speechService.hasSpeechProvider && + this.configurationService.getValue(KeywordActivationContribution.SETTINGS_ID) !== KeywordActivationContribution.SETTINGS_VALUE.OFF && + !this.speechService.hasActiveSpeechToTextSession; + if ( + (enabled && this.activeSession) || + (!enabled && !this.activeSession) + ) { + return; // already running or stopped + } + + // Start keyword activation + if (enabled) { + this.enableKeywordActivation(); + } + + // Stop keyword activation + else { + this.disableKeywordActivation(); + } + } + + private async enableKeywordActivation(): Promise { + const session = this.activeSession = new CancellationTokenSource(); + const result = await this.speechService.recognizeKeyword(session.token); + if (session.token.isCancellationRequested || session !== this.activeSession) { + return; // cancelled + } + + this.activeSession = undefined; + + if (result === KeywordRecognitionStatus.Recognized) { + this.commandService.executeCommand(this.getKeywordCommand()); + } + } + + private getKeywordCommand(): string { + const setting = this.configurationService.getValue(KeywordActivationContribution.SETTINGS_ID); + switch (setting) { + case KeywordActivationContribution.SETTINGS_VALUE.INLINE_CHAT: + return InlineVoiceChatAction.ID; + case KeywordActivationContribution.SETTINGS_VALUE.QUICK_CHAT: + return QuickVoiceChatAction.ID; + case KeywordActivationContribution.SETTINGS_VALUE.CHAT_IN_CONTEXT: + return StartVoiceChatAction.ID; + default: + return VoiceChatInChatViewAction.ID; + } + } + + private disableKeywordActivation(): void { + this.activeSession?.dispose(true); + this.activeSession = undefined; + } + + override dispose(): void { + this.activeSession?.dispose(); + + super.dispose(); + } +} + +class KeywordActivationStatusEntry extends Disposable { + + private readonly entry = this._register(new MutableDisposable()); + + private static STATUS_NAME = localize('keywordActivation.status.name', "Voice Keyword Activation"); + private static STATUS_COMMAND = 'keywordActivation.status.command'; + private static STATUS_ACTIVE = localize('keywordActivation.status.active', "Voice Keyword Activation: Active"); + private static STATUS_INACTIVE = localize('keywordActivation.status.inactive', "Voice Keyword Activation: Inactive"); + + constructor( + @ISpeechService private readonly speechService: ISpeechService, + @IStatusbarService private readonly statusbarService: IStatusbarService, + @ICommandService private readonly commandService: ICommandService, + @IConfigurationService private readonly configurationService: IConfigurationService, + ) { + super(); + + CommandsRegistry.registerCommand(KeywordActivationStatusEntry.STATUS_COMMAND, () => this.commandService.executeCommand('workbench.action.openSettings', KeywordActivationContribution.SETTINGS_ID)); + + this.registerListeners(); + this.updateStatusEntry(); + } + + private registerListeners(): void { + this._register(this.speechService.onDidStartKeywordRecognition(() => this.updateStatusEntry())); + this._register(this.speechService.onDidEndKeywordRecognition(() => this.updateStatusEntry())); + this._register(this.configurationService.onDidChangeConfiguration(e => { + if (e.affectsConfiguration(KeywordActivationContribution.SETTINGS_ID)) { + this.updateStatusEntry(); + } + })); + } + + private updateStatusEntry(): void { + const visible = this.configurationService.getValue(KeywordActivationContribution.SETTINGS_ID) !== KeywordActivationContribution.SETTINGS_VALUE.OFF; + if (visible) { + if (!this.entry.value) { + this.createStatusEntry(); + } + + this.updateStatusLabel(); + } else { + this.entry.clear(); + } + } + + private createStatusEntry() { + this.entry.value = this.statusbarService.addEntry(this.getStatusEntryProperties(), 'status.voiceKeywordActivation', StatusbarAlignment.RIGHT, 103); + } + + private getStatusEntryProperties(): IStatusbarEntry { + return { + name: KeywordActivationStatusEntry.STATUS_NAME, + text: '$(mic)', + tooltip: this.speechService.hasActiveKeywordRecognition ? KeywordActivationStatusEntry.STATUS_ACTIVE : KeywordActivationStatusEntry.STATUS_INACTIVE, + ariaLabel: this.speechService.hasActiveKeywordRecognition ? KeywordActivationStatusEntry.STATUS_ACTIVE : KeywordActivationStatusEntry.STATUS_INACTIVE, + command: KeywordActivationStatusEntry.STATUS_COMMAND, + kind: this.speechService.hasActiveKeywordRecognition ? 'prominent' : 'standard' + }; + } + + private updateStatusLabel(): void { + this.entry.value?.update(this.getStatusEntryProperties()); + } +} diff --git a/src/vs/workbench/contrib/chat/electron-sandbox/chat.contribution.ts b/src/vs/workbench/contrib/chat/electron-sandbox/chat.contribution.ts index 9ba5430737e..4d260fce917 100644 --- a/src/vs/workbench/contrib/chat/electron-sandbox/chat.contribution.ts +++ b/src/vs/workbench/contrib/chat/electron-sandbox/chat.contribution.ts @@ -3,8 +3,11 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { InlineVoiceChatAction, QuickVoiceChatAction, StartVoiceChatAction, StopListeningInInlineChatAction, StopListeningInQuickChatAction, StopListeningInChatEditorAction, StopListeningInChatViewAction, VoiceChatInChatViewAction, StopListeningAction, StopListeningAndSubmitAction } from 'vs/workbench/contrib/chat/electron-sandbox/actions/voiceChatActions'; +import { InlineVoiceChatAction, QuickVoiceChatAction, StartVoiceChatAction, StopListeningInInlineChatAction, StopListeningInQuickChatAction, StopListeningInChatEditorAction, StopListeningInChatViewAction, VoiceChatInChatViewAction, StopListeningAction, StopListeningAndSubmitAction, KeywordActivationContribution } from 'vs/workbench/contrib/chat/electron-sandbox/actions/voiceChatActions'; import { registerAction2 } from 'vs/platform/actions/common/actions'; +import { IWorkbenchContributionsRegistry, Extensions as WorkbenchExtensions } from 'vs/workbench/common/contributions'; +import { LifecyclePhase } from 'vs/workbench/services/lifecycle/common/lifecycle'; +import { Registry } from 'vs/platform/registry/common/platform'; registerAction2(StartVoiceChatAction); @@ -20,3 +23,5 @@ registerAction2(StopListeningInChatEditorAction); registerAction2(StopListeningInQuickChatAction); registerAction2(StopListeningInInlineChatAction); +const workbenchRegistry = Registry.as(WorkbenchExtensions.Workbench); +workbenchRegistry.registerWorkbenchContribution(KeywordActivationContribution, LifecyclePhase.Restored); diff --git a/src/vs/workbench/contrib/notebook/browser/services/notebookEditorServiceImpl.ts b/src/vs/workbench/contrib/notebook/browser/services/notebookEditorServiceImpl.ts index 46f2bd5e3bc..fddd0c63854 100644 --- a/src/vs/workbench/contrib/notebook/browser/services/notebookEditorServiceImpl.ts +++ b/src/vs/workbench/contrib/notebook/browser/services/notebookEditorServiceImpl.ts @@ -7,7 +7,7 @@ import { CodeWindow } from 'vs/base/browser/window'; import { ResourceMap } from 'vs/base/common/map'; import { getDefaultNotebookCreationOptions, NotebookEditorWidget } from 'vs/workbench/contrib/notebook/browser/notebookEditorWidget'; import { DisposableStore, IDisposable } from 'vs/base/common/lifecycle'; -import { IEditorGroupsService, IEditorGroup, IAuxiliaryEditorPart } from 'vs/workbench/services/editor/common/editorGroupsService'; +import { IEditorGroupsService, IEditorGroup } from 'vs/workbench/services/editor/common/editorGroupsService'; import { IInstantiationService, ServicesAccessor } from 'vs/platform/instantiation/common/instantiation'; import { isCompositeNotebookEditorInput, NotebookEditorInput } from 'vs/workbench/contrib/notebook/common/notebookEditorInput'; import { IBorrowValue, INotebookEditorService } from 'vs/workbench/contrib/notebook/browser/services/notebookEditorService'; @@ -130,7 +130,7 @@ export class NotebookEditorWidgetService implements INotebookEditorService { const sourcePart = this.editorGroupService.getPart(sourceID); const targetPart = this.editorGroupService.getPart(targetID); - if ((sourcePart as IAuxiliaryEditorPart).windowId !== (targetPart as IAuxiliaryEditorPart).windowId) { + if (sourcePart.windowId !== targetPart.windowId) { return; } diff --git a/src/vs/workbench/contrib/speech/common/speechService.ts b/src/vs/workbench/contrib/speech/common/speechService.ts index 13286662bb5..807dc5f7a95 100644 --- a/src/vs/workbench/contrib/speech/common/speechService.ts +++ b/src/vs/workbench/contrib/speech/common/speechService.ts @@ -7,7 +7,7 @@ import { localize } from 'vs/nls'; import { firstOrDefault } from 'vs/base/common/arrays'; import { CancellationToken } from 'vs/base/common/cancellation'; import { Emitter, Event } from 'vs/base/common/event'; -import { IDisposable, toDisposable } from 'vs/base/common/lifecycle'; +import { Disposable, DisposableStore, IDisposable, toDisposable } from 'vs/base/common/lifecycle'; import { IContextKeyService, RawContextKey } from 'vs/platform/contextkey/common/contextkey'; import { ExtensionIdentifier } from 'vs/platform/extensions/common/extensions'; import { createDecorator } from 'vs/platform/instantiation/common/instantiation'; @@ -34,14 +34,29 @@ export interface ISpeechToTextEvent { readonly text?: string; } +export interface ISpeechToTextSession extends IDisposable { + readonly onDidChange: Event; +} + +export enum KeywordRecognitionStatus { + Recognized = 1, + Stopped = 2 +} + +export interface IKeywordRecognitionEvent { + readonly status: KeywordRecognitionStatus; + readonly text?: string; +} + +export interface IKeywordRecognitionSession extends IDisposable { + readonly onDidChange: Event; +} + export interface ISpeechProvider { readonly metadata: ISpeechProviderMetadata; createSpeechToTextSession(token: CancellationToken): ISpeechToTextSession; -} - -export interface ISpeechToTextSession extends IDisposable { - readonly onDidChange: Event; + createKeywordRecognitionSession(token: CancellationToken): IKeywordRecognitionSession; } export interface ISpeechService { @@ -55,20 +70,41 @@ export interface ISpeechService { registerSpeechProvider(identifier: string, provider: ISpeechProvider): IDisposable; + readonly onDidStartSpeechToTextSession: Event; + readonly onDidEndSpeechToTextSession: Event; + + readonly hasActiveSpeechToTextSession: boolean; + + /** + * Starts to transcribe speech from the default microphone. The returned + * session object provides an event to subscribe for transcribed text. + */ createSpeechToTextSession(token: CancellationToken): ISpeechToTextSession; + + readonly onDidStartKeywordRecognition: Event; + readonly onDidEndKeywordRecognition: Event; + + readonly hasActiveKeywordRecognition: boolean; + + /** + * Starts to recognize a keyword from the default microphone. The returned + * status indicates if the keyword was recognized or if the session was + * stopped. + */ + recognizeKeyword(token: CancellationToken): Promise; } -export class SpeechService implements ISpeechService { +export class SpeechService extends Disposable implements ISpeechService { readonly _serviceBrand: undefined; - private readonly _onDidRegisterSpeechProvider = new Emitter(); + private readonly _onDidRegisterSpeechProvider = this._register(new Emitter()); readonly onDidRegisterSpeechProvider = this._onDidRegisterSpeechProvider.event; - private readonly _onDidUnregisterSpeechProvider = new Emitter(); + private readonly _onDidUnregisterSpeechProvider = this._register(new Emitter()); readonly onDidUnregisterSpeechProvider = this._onDidUnregisterSpeechProvider.event; - get hasSpeechProvider(): boolean { return this.providers.size > 0; } + get hasSpeechProvider() { return this.providers.size > 0; } private readonly providers = new Map(); @@ -78,6 +114,7 @@ export class SpeechService implements ISpeechService { @ILogService private readonly logService: ILogService, @IContextKeyService private readonly contextKeyService: IContextKeyService ) { + super(); } registerSpeechProvider(identifier: string, provider: ISpeechProvider): IDisposable { @@ -100,6 +137,15 @@ export class SpeechService implements ISpeechService { }); } + private readonly _onDidStartSpeechToTextSession = this._register(new Emitter()); + readonly onDidStartSpeechToTextSession = this._onDidStartSpeechToTextSession.event; + + private readonly _onDidEndSpeechToTextSession = this._register(new Emitter()); + readonly onDidEndSpeechToTextSession = this._onDidEndSpeechToTextSession.event; + + private _activeSpeechToTextSession: ISpeechToTextSession | undefined = undefined; + get hasActiveSpeechToTextSession() { return !!this._activeSpeechToTextSession; } + createSpeechToTextSession(token: CancellationToken): ISpeechToTextSession { const provider = firstOrDefault(Array.from(this.providers.values())); if (!provider) { @@ -108,6 +154,86 @@ export class SpeechService implements ISpeechService { this.logService.warn(`Multiple speech providers registered. Picking first one: ${provider.metadata.displayName}`); } - return provider.createSpeechToTextSession(token); + const session = this._activeSpeechToTextSession = provider.createSpeechToTextSession(token); + + const disposables = new DisposableStore(); + + const onSessionStoppedOrCanceled = () => { + if (session === this._activeSpeechToTextSession) { + this._activeSpeechToTextSession = undefined; + this._onDidEndSpeechToTextSession.fire(); + } + + disposables.dispose(); + }; + + disposables.add(token.onCancellationRequested(() => onSessionStoppedOrCanceled())); + if (token.isCancellationRequested) { + onSessionStoppedOrCanceled(); + } + + disposables.add(session.onDidChange(e => { + switch (e.status) { + case SpeechToTextStatus.Started: + if (session === this._activeSpeechToTextSession) { + this._onDidStartSpeechToTextSession.fire(); + } + break; + case SpeechToTextStatus.Stopped: + onSessionStoppedOrCanceled(); + break; + } + })); + + return session; + } + + private readonly _onDidStartKeywordRecognition = this._register(new Emitter()); + readonly onDidStartKeywordRecognition = this._onDidStartKeywordRecognition.event; + + private readonly _onDidEndKeywordRecognition = this._register(new Emitter()); + readonly onDidEndKeywordRecognition = this._onDidEndKeywordRecognition.event; + + private _activeKeywordRecognitionSession: IKeywordRecognitionSession | undefined = undefined; + get hasActiveKeywordRecognition() { return !!this._activeKeywordRecognitionSession; } + + async recognizeKeyword(token: CancellationToken): Promise { + const provider = firstOrDefault(Array.from(this.providers.values())); + if (!provider) { + throw new Error(`No Speech provider is registered.`); + } else if (this.providers.size > 1) { + this.logService.warn(`Multiple speech providers registered. Picking first one: ${provider.metadata.displayName}`); + } + + const session = this._activeKeywordRecognitionSession = provider.createKeywordRecognitionSession(token); + this._onDidStartKeywordRecognition.fire(); + + const disposables = new DisposableStore(); + + const onSessionStoppedOrCanceled = () => { + if (session === this._activeKeywordRecognitionSession) { + this._activeKeywordRecognitionSession = undefined; + this._onDidEndKeywordRecognition.fire(); + } + + disposables.dispose(); + }; + + disposables.add(token.onCancellationRequested(() => onSessionStoppedOrCanceled())); + if (token.isCancellationRequested) { + onSessionStoppedOrCanceled(); + } + + disposables.add(session.onDidChange(e => { + if (e.status === KeywordRecognitionStatus.Stopped) { + onSessionStoppedOrCanceled(); + } + })); + + try { + return (await Event.toPromise(session.onDidChange)).status; + } finally { + onSessionStoppedOrCanceled(); + } } } diff --git a/src/vs/workbench/services/editor/common/editorGroupsService.ts b/src/vs/workbench/services/editor/common/editorGroupsService.ts index bd39f81d3cc..0dd568153f3 100644 --- a/src/vs/workbench/services/editor/common/editorGroupsService.ts +++ b/src/vs/workbench/services/editor/common/editorGroupsService.ts @@ -433,6 +433,11 @@ export interface IEditorPart extends IEditorGroupsContainer { */ readonly onDidScroll: Event; + /** + * The identifier of the window the editor part is contained in. + */ + readonly windowId: number; + /** * The size of the editor part. */ @@ -461,11 +466,6 @@ export interface IEditorPart extends IEditorGroupsContainer { export interface IAuxiliaryEditorPart extends IEditorPart { - /** - * The identifier of the window the auxiliary editor part is contained in. - */ - readonly windowId: number; - /** * Close this auxiliary editor part after moving all * editors of all groups back to the main editor part. @@ -590,7 +590,7 @@ export interface IEditorGroup { readonly id: GroupIdentifier; /** - * The identifier of the `CodeWindow` this editor group is part of. + * The identifier of the window this editor group is part of. */ readonly windowId: number; diff --git a/src/vs/workbench/test/browser/workbenchTestServices.ts b/src/vs/workbench/test/browser/workbenchTestServices.ts index 0eacb21061c..dac55f97d95 100644 --- a/src/vs/workbench/test/browser/workbenchTestServices.ts +++ b/src/vs/workbench/test/browser/workbenchTestServices.ts @@ -828,6 +828,8 @@ export class TestEditorGroupsService implements IEditorGroupsService { readonly parts: readonly IEditorPart[] = [this]; + windowId = mainWindow.vscodeWindowId; + onDidCreateAuxiliaryEditorPart: Event = Event.None; onDidChangeActiveGroup: Event = Event.None; onDidActivateGroup: Event = Event.None; diff --git a/src/vscode-dts/vscode.proposed.speech.d.ts b/src/vscode-dts/vscode.proposed.speech.d.ts index 81c90e39bac..42fbbb03320 100644 --- a/src/vscode-dts/vscode.proposed.speech.d.ts +++ b/src/vscode-dts/vscode.proposed.speech.d.ts @@ -5,6 +5,8 @@ declare module 'vscode' { + // todo@bpasero work in progress speech API + export enum SpeechToTextStatus { Started = 1, Recognizing = 2, @@ -21,15 +23,27 @@ declare module 'vscode' { readonly onDidChange: Event; } + export enum KeywordRecognitionStatus { + Recognized = 1, + Stopped = 2 + } + + export interface KeywordRecognitionEvent { + readonly status: KeywordRecognitionStatus; + readonly text?: string; + } + + export interface KeywordRecognitionSession extends Disposable { + readonly onDidChange: Event; + } + export interface SpeechProvider { provideSpeechToTextSession(token: CancellationToken): SpeechToTextSession; + provideKeywordRecognitionSession(token: CancellationToken): KeywordRecognitionSession; } export namespace speech { - /** - * TODO@bpasero work in progress speech provider API - */ export function registerSpeechProvider(id: string, provider: SpeechProvider): Disposable; } }