diff --git a/extensions/copilot/src/extension/conversation/vscode-node/languageModelAccess.ts b/extensions/copilot/src/extension/conversation/vscode-node/languageModelAccess.ts index c7e1bb1e2af..cb04669e20b 100644 --- a/extensions/copilot/src/extension/conversation/vscode-node/languageModelAccess.ts +++ b/extensions/copilot/src/extension/conversation/vscode-node/languageModelAccess.ts @@ -156,7 +156,8 @@ function buildConfigurationSchema(endpoint: IChatEndpoint, autoTiersEnabled: boo return { configurationSchema: { properties } }; } -const utilityAliasFamilies: readonly ChatEndpointFamily[] = ['copilot-utility-small', 'copilot-utility']; +const DICTATION_CLEANUP_LUNA_ALIAS = 'copilot-dictation-cleanup-luna'; +const utilityAliasFamilies: readonly ChatEndpointFamily[] = ['copilot-utility-small', 'copilot-utility', DICTATION_CLEANUP_LUNA_ALIAS]; /** * Builds the {@link vscode.LanguageModelChatInformation} entry that publishes a @@ -295,6 +296,9 @@ export class LanguageModelAccess extends Disposable implements IExtensionContrib // honored while routing goes through `POST /auto`. this._onDidChange.fire(); })); + void this._refreshUtilityOverrides().catch(err => { + this._logService.warn(`[LanguageModelAccess] Failed to pre-resolve internal model aliases: ${err}`); + }); } private async _provideLanguageModelChatInfo(options: { silent: boolean }, token: vscode.CancellationToken): Promise { @@ -541,6 +545,9 @@ export class LanguageModelAccess extends Disposable implements IExtensionContrib progress: vscode.Progress, token: vscode.CancellationToken ): Promise { + if (model.id === DICTATION_CLEANUP_LUNA_ALIAS && options.requestInitiator !== 'core') { + throw new Error(`Model ${model.id} is only available to VS Code core.`); + } let endpoint = await this._getEndpointForModel(model, buildAutoRoutingContext(messages, options)); if (!endpoint) { throw new Error(`Endpoint not found for model ${model.id}`); diff --git a/extensions/copilot/src/extension/conversation/vscode-node/test/languageModelAccess.test.ts b/extensions/copilot/src/extension/conversation/vscode-node/test/languageModelAccess.test.ts index 0fd7d1f9909..1fbab195e98 100644 --- a/extensions/copilot/src/extension/conversation/vscode-node/test/languageModelAccess.test.ts +++ b/extensions/copilot/src/extension/conversation/vscode-node/test/languageModelAccess.test.ts @@ -434,6 +434,101 @@ suite('LanguageModelAccess model info', () => { await extensionContext.globalState.update(baseCountCacheKey, undefined); } }); + + test('publishes a core-only Luna alias for dictation cleanup without publishing hidden models directly', async () => { + const makeHiddenEndpoint = (model: string): IChatEndpoint => ({ + model, + name: model, + family: model, + version: '1', + modelProvider: 'copilot', + modelMaxPromptTokens: 128_000, + maxOutputTokens: 4_096, + supportsToolCalls: true, + supportsVision: false, + supportsPrediction: false, + showInModelPicker: false, + isFallback: false, + tokenizer: TokenizerType.O200K, + urlOrRequestMetadata: '', + } as unknown as IChatEndpoint); + const lunaEndpoint = makeHiddenEndpoint('gpt-5.6-luna'); + const otherEndpoint = makeHiddenEndpoint('some-hidden-model'); + const copilotToken = new CopilotToken(createTestExtendedTokenInfo({ token: 'token', username: 'fake', copilot_plan: 'unknown' })); + const testingServiceCollection = createExtensionTestingServices(); + testingServiceCollection.define(ICopilotTokenManager, { + _serviceBrand: undefined, + onDidCopilotTokenRefresh: Event.None, + getCopilotToken: async () => copilotToken, + resetCopilotToken: () => { }, + } as unknown as ICopilotTokenManager); + testingServiceCollection.define(IAutomodeService, { + _serviceBrand: undefined, + resolveAutoModeEndpoint: async () => lunaEndpoint, + resolveAutoModePickerEndpoint: async () => lunaEndpoint, + getAutoPickerMetadata: () => ({ discountRange: { low: 0, high: 0 } }), + areAutoModeTiersSupported: () => false, + onDidChangeAutoModeTierSupport: Event.None, + consumeLastRoutingDecision: () => undefined, + invalidateRouterCache: () => { }, + } as unknown as IAutomodeService); + testingServiceCollection.define(IEndpointProvider, { + _serviceBrand: undefined, + onDidModelsRefresh: Event.None, + getAllCompletionModels: async () => [], + getAllChatEndpoints: async () => [lunaEndpoint, otherEndpoint], + getChatEndpoint: async () => lunaEndpoint, + getEmbeddingsEndpoint: async () => { throw new Error('Not implemented in test'); }, + } as unknown as IEndpointProvider); + const accessor = testingServiceCollection.createTestingAccessor(); + const extensionContext = accessor.get(IVSCodeExtensionContext); + const version = accessor.get(IEnvService).getVersion(); + await extensionContext.globalState.update('lmBaseCount/gpt-5.6-luna', { extensionVersion: version, baseCount: 0 }); + await extensionContext.globalState.update('lmBaseCount/some-hidden-model', { extensionVersion: version, baseCount: 0 }); + const languageModelAccess = accessor.get(IInstantiationService).createInstance(LanguageModelAccess); + try { + const testAccess = languageModelAccess as unknown as { + _refreshUtilityOverrides(): Promise; + _provideLanguageModelChatInfo(options: { silent: boolean }, token: vscode.CancellationToken): Promise; + _provideLanguageModelChatResponse( + model: vscode.LanguageModelChatInformation, + messages: vscode.LanguageModelChatMessage[], + options: vscode.ProvideLanguageModelChatResponseOptions, + progress: vscode.Progress, + token: vscode.CancellationToken, + ): Promise; + }; + await testAccess._refreshUtilityOverrides(); + const modelInfo = await raceTimeout(testAccess._provideLanguageModelChatInfo({ silent: true }, CancellationToken.None), 2_000); + assert.ok(modelInfo, 'provideLanguageModelChatInfo did not resolve'); + const dictationAlias = modelInfo.find(m => m.id === 'copilot-dictation-cleanup-luna'); + assert.deepStrictEqual({ + dictationAliasPublished: Boolean(dictationAlias), + dictationAliasUserSelectable: dictationAlias?.isUserSelectable, + lunaPublishedDirectly: modelInfo.some(m => m.id === 'gpt-5.6-luna'), + otherPublished: modelInfo.some(m => m.id === 'some-hidden-model'), + }, { + dictationAliasPublished: true, + dictationAliasUserSelectable: false, + lunaPublishedDirectly: false, + otherPublished: false, + }); + await assert.rejects( + testAccess._provideLanguageModelChatResponse( + dictationAlias!, + [], + { requestInitiator: 'publisher.extension' } as vscode.ProvideLanguageModelChatResponseOptions, + { report: () => { } }, + CancellationToken.None, + ), + /only available to VS Code core/, + ); + } finally { + languageModelAccess.dispose(); + await extensionContext.globalState.update('lmBaseCount/gpt-5.6-luna', undefined); + await extensionContext.globalState.update('lmBaseCount/some-hidden-model', undefined); + } + }); }); suite('buildUtilityAliasModelInfo', () => { diff --git a/extensions/copilot/src/extension/prompt/vscode-node/endpointProviderImpl.ts b/extensions/copilot/src/extension/prompt/vscode-node/endpointProviderImpl.ts index 498fb7a2682..fe7e7ac710d 100644 --- a/extensions/copilot/src/extension/prompt/vscode-node/endpointProviderImpl.ts +++ b/extensions/copilot/src/extension/prompt/vscode-node/endpointProviderImpl.ts @@ -162,14 +162,17 @@ export class ProductionEndpointProvider extends Disposable implements IEndpointP /** * Resolves a chat endpoint from a family string. The internal utility - * families (`copilot-utility` / `copilot-utility-small`) are routed through - * their dedicated resolvers; any other value is treated as a CAPI model - * family (e.g. `gemini-3-flash`, `gpt-5-mini`) and resolved directly. This - * lets callers such as the execution and search subagents honor their - * `*.model` override settings rather than silently falling back to the - * parent model. + * aliases are routed through their dedicated resolvers; any other value is + * treated as a CAPI model family (e.g. `gemini-3-flash`, `gpt-5-mini`) and + * resolved directly. This lets callers such as the execution and search + * subagents honor their `*.model` override settings rather than silently + * falling back to the parent model. */ private async _resolveFamily(family: string): Promise { + if (family === 'copilot-dictation-cleanup-luna') { + const modelMetadata = await this._modelFetcher.getChatModelFromCapiFamily('gpt-5.6-luna'); + return this.getOrCreateChatEndpointInstance(modelMetadata); + } if (family === 'copilot-utility' || family === 'copilot-utility-small') { return this._resolveUtilityFamily(family); } diff --git a/extensions/copilot/src/platform/endpoint/common/endpointProvider.ts b/extensions/copilot/src/platform/endpoint/common/endpointProvider.ts index b58cf21cb76..4c1fbd6fa6e 100644 --- a/extensions/copilot/src/platform/endpoint/common/endpointProvider.ts +++ b/extensions/copilot/src/platform/endpoint/common/endpointProvider.ts @@ -180,14 +180,13 @@ export function isCompletionModelInformation(model: IModelAPIResponse): model is return model.capabilities.type === 'completion'; } -export type ChatEndpointFamily = 'copilot-utility' | 'copilot-utility-small'; +export type ChatEndpointFamily = 'copilot-utility' | 'copilot-utility-small' | 'copilot-dictation-cleanup-luna'; /** * A model family accepted by {@link IEndpointProvider.getChatEndpoint}: either - * an internal utility alias ({@link ChatEndpointFamily}) or any CAPI model - * family id (e.g. `gemini-3-flash`, `gpt-5-mini`). The utility literals are - * kept for editor autocomplete while still allowing arbitrary CAPI family - * strings. + * an internal model alias ({@link ChatEndpointFamily}) or any CAPI model family + * id (e.g. `gemini-3-flash`, `gpt-5-mini`). The internal literals are kept for + * editor autocomplete while still allowing arbitrary CAPI family strings. */ export type ChatModelFamily = ChatEndpointFamily | (string & {}); diff --git a/src/vs/workbench/contrib/chat/browser/chat.shared.contribution.ts b/src/vs/workbench/contrib/chat/browser/chat.shared.contribution.ts index eecff433c09..a92c30ca5cb 100644 --- a/src/vs/workbench/contrib/chat/browser/chat.shared.contribution.ts +++ b/src/vs/workbench/contrib/chat/browser/chat.shared.contribution.ts @@ -337,6 +337,13 @@ configurationRegistry.registerConfiguration({ default: true, tags: ['experimental'] }, + 'dictation.experimental.llmCleanupModel': { + type: 'string', + enum: ['auto', 'copilot-utility-small', 'gpt-5.6-luna'], + markdownDescription: nls.localize('dictation.experimental.llmCleanupModel', "Controls the language model used for experimental dictation cleanup. `auto` follows the active experiment treatment."), + default: 'auto', + tags: ['experimental'] + }, 'chat.editor.fontSize': { type: 'number', description: nls.localize('interactiveSession.editor.fontSize', "Controls the font size in pixels in chat codeblocks."), diff --git a/src/vs/workbench/contrib/chat/browser/speechToText/chatSpeechToTextService.ts b/src/vs/workbench/contrib/chat/browser/speechToText/chatSpeechToTextService.ts index 9bc53533ecc..7cfb35ba066 100644 --- a/src/vs/workbench/contrib/chat/browser/speechToText/chatSpeechToTextService.ts +++ b/src/vs/workbench/contrib/chat/browser/speechToText/chatSpeechToTextService.ts @@ -37,6 +37,7 @@ import { createPcmCaptureNode } from '../pcmCaptureWorklet.js'; import { getMediaCaptureWindow } from '../voiceClient/micCaptureService.js'; import { resolveDictationLanguage } from './dictationLanguage.js'; import { ChatEntitlement, IChatEntitlementService } from '../../../../services/chat/common/chatEntitlementService.js'; +import { IWorkbenchAssignmentService } from '../../../../services/assignment/common/assignmentService.js'; export const IChatSpeechToTextService = createDecorator('chatSpeechToTextService'); @@ -124,8 +125,15 @@ const LLM_CLEANUP_MAX_CHARS = 4000; /** Bounded deadline for cleanup, so a stalled provider does not make dictation feel stuck. */ const LLM_CLEANUP_TIMEOUT_MS = 1500; -/** Utility model used for transcript cleanup — a small, fast model in the spirit of gpt-4o-mini. */ -const LLM_CLEANUP_MODEL_SELECTOR = { vendor: 'copilot', id: 'copilot-utility-small' }; +/** Utility model used for transcript cleanup, currently backed by gpt-4o-mini. */ +const LLM_CLEANUP_MODEL_SELECTOR = { vendor: 'copilot', id: 'copilot-utility-small' } as const; + +const LLM_CLEANUP_MODEL_TREATMENT = 'dictationLlmCleanupModel'; +const LLM_CLEANUP_MODEL_SETTING = 'dictation.experimental.llmCleanupModel'; +const LLM_CLEANUP_LUNA_MODEL_ID = 'gpt-5.6-luna'; +const LLM_CLEANUP_LUNA_MODEL_SELECTOR = { vendor: 'copilot', id: 'copilot-dictation-cleanup-luna' } as const; + +type DictationCleanupModel = 'none' | 'copilot-utility-small' | 'gpt-5.6-luna'; /** * Which backend transcribes dictation audio: @@ -156,6 +164,7 @@ type SpeechToTextSessionEvent = { timeToFirstTranscriptMs: number; finalizeMs: number; errorCode: string; + cleanupModel: DictationCleanupModel; }; type SpeechToTextSessionClassification = { owner: 'meganrogge'; @@ -170,6 +179,7 @@ type SpeechToTextSessionClassification = { timeToFirstTranscriptMs: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; isMeasurement: true; comment: 'Milliseconds from the first streamed audio chunk to the first transcript update; the backend transcription latency (excludes mic acquisition and model download). -1 when no transcript arrived.' }; finalizeMs: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; isMeasurement: true; comment: 'Milliseconds from the user stopping recording until the final transcript resolved; the post-stop wait. -1 when not applicable.' }; errorCode: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'Short error identifier when the session failed, else empty.' }; + cleanupModel: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The language model used to attempt dictation cleanup, or none when no model request was made.' }; }; type SpeechToTextModelPrepareEvent = { @@ -493,6 +503,8 @@ export class ChatSpeechToTextService extends Disposable implements IChatSpeechTo private _firstTranscriptMs = 0; /** Milliseconds from stopping recording to the final transcript resolving; -1 until measured. */ private _finalizeMs = -1; + private _sessionCleanupModel: DictationCleanupModel = 'none'; + private _llmCleanupModelTreatment: string | undefined; /** Cancellation for the in-flight experimental LLM cleanup request, aborted when the session is cancelled or disposed. */ private readonly _cleanupCts = this._register(new MutableDisposable()); @@ -521,6 +533,7 @@ export class ChatSpeechToTextService extends Disposable implements IChatSpeechTo @ILanguageModelsService private readonly _languageModelsService: ILanguageModelsService, @IPromptsService private readonly _promptsService: IPromptsService, @IChatEntitlementService private readonly _chatEntitlementService: IChatEntitlementService, + @IWorkbenchAssignmentService private readonly _assignmentService: IWorkbenchAssignmentService, ) { super(); this._recordingContextKey = ChatContextKeys.speechToTextRecording.bindTo(contextKeyService); @@ -550,6 +563,26 @@ export class ChatSpeechToTextService extends Disposable implements IChatSpeechTo } }); })); + this._refreshLlmCleanupModelTreatment(); + this._register(this._assignmentService.onDidRefetchAssignments(() => this._refreshLlmCleanupModelTreatment())); + } + + private _refreshLlmCleanupModelTreatment(): void { + void this._assignmentService.getTreatment(LLM_CLEANUP_MODEL_TREATMENT).then(treatment => { + if (!this._store.isDisposed) { + this._llmCleanupModelTreatment = treatment; + } + }, err => this._logService.warn('[chat-stt] failed to resolve dictation cleanup model treatment', err)); + } + + private _getLlmCleanupModel(): Exclude { + const configuredModel = this._configurationService.getValue(LLM_CLEANUP_MODEL_SETTING); + if (configuredModel === LLM_CLEANUP_LUNA_MODEL_ID || configuredModel === LLM_CLEANUP_MODEL_SELECTOR.id) { + return configuredModel; + } + return this._llmCleanupModelTreatment === LLM_CLEANUP_LUNA_MODEL_ID + ? LLM_CLEANUP_LUNA_MODEL_ID + : LLM_CLEANUP_MODEL_SELECTOR.id; } /** Read the configured dictation backend, derived from the selected model. */ @@ -644,6 +677,7 @@ export class ChatSpeechToTextService extends Disposable implements IChatSpeechTo timeToFirstTranscriptMs, finalizeMs: this._finalizeMs, errorCode: this._sessionErrorCode, + cleanupModel: this._sessionCleanupModel, }); this._sessionStartMs = 0; } @@ -748,6 +782,7 @@ export class ChatSpeechToTextService extends Disposable implements IChatSpeechTo this._firstAudioMs = 0; this._firstTranscriptMs = 0; this._finalizeMs = -1; + this._sessionCleanupModel = 'none'; // Defensively clear any transcript left over from a previous session so a // new dictation never starts by re-emitting the prior transcript (teardown // already clears these, but a start without a clean teardown must not leak). @@ -1362,11 +1397,25 @@ export class ChatSpeechToTextService extends Disposable implements IChatSpeechTo cts.cancel(); }, LLM_CLEANUP_TIMEOUT_MS); try { - const models = await raceCancellation( - this._languageModelsService.selectLanguageModels(LLM_CLEANUP_MODEL_SELECTOR), + const cleanupModel = this._getLlmCleanupModel(); + const modelSelector = cleanupModel === LLM_CLEANUP_LUNA_MODEL_ID + ? LLM_CLEANUP_LUNA_MODEL_SELECTOR + : LLM_CLEANUP_MODEL_SELECTOR; + let models = await raceCancellation( + this._languageModelsService.selectLanguageModels(modelSelector), cts.token, [], ); + let selectedCleanupModel = cleanupModel; + if (!models.length && cleanupModel === LLM_CLEANUP_LUNA_MODEL_ID) { + this._logService.info('[chat-stt] Luna cleanup model unavailable; falling back to copilot-utility-small'); + models = await raceCancellation( + this._languageModelsService.selectLanguageModels(LLM_CLEANUP_MODEL_SELECTOR), + cts.token, + [], + ); + selectedCleanupModel = LLM_CLEANUP_MODEL_SELECTOR.id; + } if (!models.length) { this._logService.info('[chat-stt] skipped language model cleanup (reason=noModel); using raw transcript'); return undefined; @@ -1375,7 +1424,6 @@ export class ChatSpeechToTextService extends Disposable implements IChatSpeechTo this._logService.info(`[chat-stt] skipped language model cleanup (reason=${timedOut ? 'timeout' : 'cancelledBeforeRequest'}); using raw transcript`); return undefined; } - const dictationInstructions = await raceCancellation( this._promptsService.getDictationInstructions(cts.token), cts.token, @@ -1393,6 +1441,7 @@ export class ChatSpeechToTextService extends Disposable implements IChatSpeechTo '', ].join('\n'); + this._sessionCleanupModel = selectedCleanupModel; const response = await raceCancellation( this._languageModelsService.sendChatRequest( models[0], diff --git a/src/vs/workbench/contrib/chat/test/browser/chatSpeechToTextService.test.ts b/src/vs/workbench/contrib/chat/test/browser/chatSpeechToTextService.test.ts index 23ca11babba..e0df8a65050 100644 --- a/src/vs/workbench/contrib/chat/test/browser/chatSpeechToTextService.test.ts +++ b/src/vs/workbench/contrib/chat/test/browser/chatSpeechToTextService.test.ts @@ -10,12 +10,17 @@ import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../../base/tes import { ChatSpeechToTextService, createDictationCleanupSystemPrompt, isDictationEntitled, stripDictationFillers } from '../../browser/speechToText/chatSpeechToTextService.js'; import { resolveDictationLanguage } from '../../browser/speechToText/dictationLanguage.js'; import { ChatEntitlement } from '../../../../services/chat/common/chatEntitlementService.js'; +import { ILanguageModelChatSelector } from '../../common/languageModels.js'; type CleanupTestService = { + _configurationService: { + getValue: () => string; + }; _languageModelsService: { - selectLanguageModels: () => Promise; + selectLanguageModels: (selector: ILanguageModelChatSelector) => Promise; sendChatRequest: (...args: never[]) => Promise; }; + _llmCleanupModelTreatment: string | undefined; _promptsService: { getDictationInstructions: (token: CancellationToken) => Promise; }; @@ -155,6 +160,10 @@ suite('ChatSpeechToTextService', () => { const clock = sinon.useFakeTimers(); try { const service = Object.create(ChatSpeechToTextService.prototype) as CleanupTestService; + service._configurationService = { + getValue: () => 'auto', + }; + service._llmCleanupModelTreatment = undefined; service._languageModelsService = { selectLanguageModels: async () => ['test-model'], sendChatRequest: () => new Promise(() => { }), @@ -181,4 +190,47 @@ suite('ChatSpeechToTextService', () => { } }); + test('selects the configured or treated cleanup model and falls back when Luna is unavailable', async () => { + const selectors: ILanguageModelChatSelector[] = []; + const createService = (treatment: string | undefined, configuredModel = 'auto'): CleanupTestService => { + const service = Object.create(ChatSpeechToTextService.prototype) as CleanupTestService; + service._configurationService = { + getValue: () => configuredModel, + }; + service._llmCleanupModelTreatment = treatment; + service._languageModelsService = { + selectLanguageModels: async selector => { + selectors.push(selector); + return []; + }, + sendChatRequest: () => Promise.reject(new Error('Unexpected request')), + }; + service._promptsService = { + getDictationInstructions: async () => undefined, + }; + service._logService = { + info: () => { }, + warn: () => { }, + trace: () => { }, + }; + return service; + }; + + await createService(undefined)._cleanupWithLanguageModel('control transcript', CancellationToken.None); + await createService('gpt-5.6-luna')._cleanupWithLanguageModel('treatment transcript', CancellationToken.None); + await createService('unexpected-model')._cleanupWithLanguageModel('unknown treatment transcript', CancellationToken.None); + await createService(undefined, 'gpt-5.6-luna')._cleanupWithLanguageModel('configured Luna transcript', CancellationToken.None); + await createService('gpt-5.6-luna', 'copilot-utility-small')._cleanupWithLanguageModel('configured utility transcript', CancellationToken.None); + + assert.deepStrictEqual(selectors, [ + { vendor: 'copilot', id: 'copilot-utility-small' }, + { vendor: 'copilot', id: 'copilot-dictation-cleanup-luna' }, + { vendor: 'copilot', id: 'copilot-utility-small' }, + { vendor: 'copilot', id: 'copilot-utility-small' }, + { vendor: 'copilot', id: 'copilot-dictation-cleanup-luna' }, + { vendor: 'copilot', id: 'copilot-utility-small' }, + { vendor: 'copilot', id: 'copilot-utility-small' }, + ]); + }); + });