diff --git a/extensions/copilot/src/extension/prompt/node/chatMLFetcherTelemetry.ts b/extensions/copilot/src/extension/prompt/node/chatMLFetcherTelemetry.ts index ff35a497306..cabb725591e 100644 --- a/extensions/copilot/src/extension/prompt/node/chatMLFetcherTelemetry.ts +++ b/extensions/copilot/src/extension/prompt/node/chatMLFetcherTelemetry.ts @@ -9,7 +9,7 @@ import { getImageTelemetryEventMeasurements, type ImageTelemetryMeasurements } f import { FetcherId } from '../../../platform/networking/common/fetcherService'; import { IChatEndpoint, IChatRequestTelemetryProperties, IEndpointBody } from '../../../platform/networking/common/networking'; import { ChatCompletion } from '../../../platform/networking/common/openai'; -import { ITelemetryService } from '../../../platform/telemetry/common/telemetry'; +import { ITelemetryService, type TelemetryEventMeasurements, type TelemetryEventProperties } from '../../../platform/telemetry/common/telemetry'; import { TelemetryData } from '../../../platform/telemetry/common/telemetryData'; import { isBYOKModel } from '../../byok/node/openAIEndpoint'; @@ -99,6 +99,19 @@ function getTurnFromBaseTelemetry(baseTelemetry: TelemetryData): number | undefi return Number.isFinite(parsedTurnIndex) ? parsedTurnIndex : undefined; } +function sendResponseTelemetryEvent( + telemetryService: ITelemetryService, + eventName: string, + properties: TelemetryEventProperties, + measurements: TelemetryEventMeasurements, + imageTelemetryMeasurements: ImageTelemetryMeasurements, +): void { + telemetryService.sendTelemetryEvent(eventName, { github: true, microsoft: true }, properties, measurements); + if (imageTelemetryMeasurements.imageCount > 0) { + telemetryService.sendEnhancedGHTelemetryEvent(eventName, properties, measurements); + } +} + export class ChatMLFetcherTelemetrySender { public static sendSuccessTelemetry( @@ -196,7 +209,7 @@ export class ChatMLFetcherTelemetrySender { "iterationNumber": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Iteration number within the tool calling loop" } } */ - telemetryService.sendTelemetryEvent('response.success', { github: true, microsoft: true }, { + sendResponseTelemetryEvent(telemetryService, 'response.success', { reason: chatCompletion.finishReason, filterReason: chatCompletion.filterReason, source: baseTelemetry?.properties.messageSource ?? 'unknown', @@ -248,7 +261,7 @@ export class ChatMLFetcherTelemetrySender { bytesReceived, suspendEventSeen: suspendEventSeen ? 1 : 0, resumeEventSeen: resumeEventSeen ? 1 : 0, - }); + }, imageTelemetryMeasurements); } public static sendCancellationTelemetry( @@ -339,7 +352,7 @@ export class ChatMLFetcherTelemetrySender { "resumeEventSeen": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Whether a system resume event was seen during the request", "isMeasurement": true } } */ - telemetryService.sendTelemetryEvent('response.cancelled', { github: true, microsoft: true }, { + sendResponseTelemetryEvent(telemetryService, 'response.cancelled', { apiType, source, requestId, @@ -371,7 +384,7 @@ export class ChatMLFetcherTelemetrySender { bytesReceived, suspendEventSeen: suspendEventSeen ? 1 : 0, resumeEventSeen: resumeEventSeen ? 1 : 0, - }); + }, imageTelemetryMeasurements); } public static sendResponseErrorTelemetry( @@ -453,7 +466,7 @@ export class ChatMLFetcherTelemetrySender { "resumeEventSeen": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Whether a system resume event was seen during the request", "isMeasurement": true } } */ - telemetryService.sendTelemetryEvent('response.error', { github: true, microsoft: true }, { + sendResponseTelemetryEvent(telemetryService, 'response.error', { type: processed.type, reason: processed.reasonDetail || processed.reason, source: telemetryProperties?.messageSource ?? 'unknown', @@ -489,6 +502,6 @@ export class ChatMLFetcherTelemetrySender { bytesReceived, suspendEventSeen: suspendEventSeen ? 1 : 0, resumeEventSeen: resumeEventSeen ? 1 : 0, - }); + }, imageTelemetryMeasurements); } } diff --git a/extensions/copilot/src/platform/endpoint/node/automodeService.ts b/extensions/copilot/src/platform/endpoint/node/automodeService.ts index bc7c4682325..94bdd303809 100644 --- a/extensions/copilot/src/platform/endpoint/node/automodeService.ts +++ b/extensions/copilot/src/platform/endpoint/node/automodeService.ts @@ -13,7 +13,7 @@ import { ChatLocation } from '../../../vscodeTypes'; import { IAuthenticationService } from '../../authentication/common/authentication'; import { ConfigKey, IConfigurationService } from '../../configuration/common/configurationService'; import { IEnvService } from '../../env/common/envService'; -import { getImageTelemetryEventMeasurements, getImageTelemetryMeasurementsFromReferences } from '../../image/common/imageTelemetry'; +import { getImageTelemetryEventMeasurements, getImageTelemetryMeasurementsFromReferences, type ImageTelemetryMeasurements } from '../../image/common/imageTelemetry'; import { ILogService } from '../../log/common/logService'; import { createCapiClientFetchedValue } from '../../networking/common/capiClientFetchedValue'; import { isAbortError } from '../../networking/common/fetcherService'; @@ -191,12 +191,12 @@ export class AutomodeService extends Disposable implements IAutomodeService { if (entry?.needsReEval) { entry.needsReEval = false; } + const imageTelemetryMeasurements = getImageTelemetryMeasurementsFromReferences(chatRequest?.references); + const imageTelemetryEventMeasurements = getImageTelemetryEventMeasurements(imageTelemetryMeasurements); const routerResult = skipRouter ? { lastRoutedPrompt: chatRequest?.prompt?.trim() ?? entry?.lastRoutedPrompt } - : await this._tryRouterSelection(chatRequest, conversationId, entry, token, knownEndpoints); - const imageTelemetryMeasurements = getImageTelemetryMeasurementsFromReferences(chatRequest?.references); - const imageTelemetryEventMeasurements = getImageTelemetryEventMeasurements(imageTelemetryMeasurements); + : await this._tryRouterSelection(chatRequest, conversationId, entry, token, knownEndpoints, imageTelemetryEventMeasurements); let selectedModel = routerResult.selectedModel; const lastRoutedPrompt = routerResult.lastRoutedPrompt; const routerFallbackReason = routerResult.fallbackReason; @@ -314,6 +314,7 @@ export class AutomodeService extends Disposable implements IAutomodeService { entry: AutoModelCacheEntry | undefined, token: AutoModeAPIResponse, knownEndpoints: IChatEndpoint[], + imageTelemetryEventMeasurements: Partial, ): Promise<{ selectedModel?: IChatEndpoint; lastRoutedPrompt?: string; fallbackReason?: string; candidateModel?: string }> { const prompt = chatRequest?.prompt?.trim(); const lastRoutedPrompt = entry?.lastRoutedPrompt ?? prompt; @@ -360,7 +361,7 @@ export class AutomodeService extends Disposable implements IAutomodeService { this._logService.info(`[AutomodeService] Filtered ${droppedModels.length} unresolvable model(s) before routing: [${droppedModels.join(', ')}]`); } - const result = await this._routerDecisionFetcher.getRouterDecision(prompt, token.session_token, routableModels, undefined, contextSignals, conversationId, chatRequest?.id, routingMethod, hasImage(chatRequest)); + const result = await this._routerDecisionFetcher.getRouterDecision(prompt, token.session_token, routableModels, undefined, contextSignals, conversationId, chatRequest?.id, routingMethod, hasImage(chatRequest), imageTelemetryEventMeasurements); if (result.fallback) { this._logService.info(`[AutomodeService] Router signaled fallback: ${result.fallback_reason ?? 'unknown'}, routing_method=${result.routing_method ?? 'n/a'}`); diff --git a/extensions/copilot/src/platform/endpoint/node/routerDecisionFetcher.ts b/extensions/copilot/src/platform/endpoint/node/routerDecisionFetcher.ts index 65f95aaee5b..436ce8b5038 100644 --- a/extensions/copilot/src/platform/endpoint/node/routerDecisionFetcher.ts +++ b/extensions/copilot/src/platform/endpoint/node/routerDecisionFetcher.ts @@ -6,6 +6,7 @@ import { RequestType } from '@vscode/copilot-api'; import { Codicon } from '../../../util/vs/base/common/codicons'; import { IAuthenticationService } from '../../authentication/common/authentication'; +import type { ImageTelemetryMeasurements } from '../../image/common/imageTelemetry'; import { ILogService } from '../../log/common/logService'; import { Response } from '../../networking/common/fetcherService'; import { IRequestLogger, LoggedRequestKind } from '../../requestLogger/common/requestLogger'; @@ -66,7 +67,7 @@ export class RouterDecisionFetcher { ) { } - async getRouterDecision(query: string, autoModeToken: string, availableModels: string[], stickyThreshold?: number, contextSignals?: RoutingContextSignals, conversationId?: string, vscodeRequestId?: string, routingMethod?: string, hasImage?: boolean): Promise { + async getRouterDecision(query: string, autoModeToken: string, availableModels: string[], stickyThreshold?: number, contextSignals?: RoutingContextSignals, conversationId?: string, vscodeRequestId?: string, routingMethod?: string, hasImage?: boolean, imageTelemetryEventMeasurements?: Partial): Promise { const startTime = Date.now(); const requestBody: Record = { prompt: query, available_models: availableModels, ...contextSignals }; if (stickyThreshold !== undefined) { @@ -204,6 +205,7 @@ export class RouterDecisionFetcher { chosenShortfall: result.chosen_shortfall, scoreNeedsReasoning: result.scores.needs_reasoning, scoreNoReasoning: result.scores.no_reasoning, + ...imageTelemetryEventMeasurements, } ); diff --git a/extensions/copilot/src/platform/endpoint/node/test/automodeService.spec.ts b/extensions/copilot/src/platform/endpoint/node/test/automodeService.spec.ts index 1f2a640dfcc..fb5e98e155b 100644 --- a/extensions/copilot/src/platform/endpoint/node/test/automodeService.spec.ts +++ b/extensions/copilot/src/platform/endpoint/node/test/automodeService.spec.ts @@ -61,7 +61,7 @@ describe('AutomodeService', () => { let configurationService: IConfigurationService; let mockChatEndpoint: IChatEndpoint; let envService: NullEnvService; - let mockTelemetryService: ITelemetryService & { sendMSFTTelemetryEvent: ReturnType }; + let mockTelemetryService: ITelemetryService & { sendEnhancedGHTelemetryEvent: ReturnType; sendMSFTTelemetryEvent: ReturnType }; function createEndpoint(model: string, provider: string, overrides?: Partial): IChatEndpoint { return { @@ -154,7 +154,7 @@ describe('AutomodeService', () => { sendMSFTTelemetryErrorEvent: vi.fn(), sendSharedTelemetryEvent: vi.fn(), sendEnhancedGHTelemetryEvent: vi.fn(), - } as unknown as ITelemetryService & { sendMSFTTelemetryEvent: ReturnType }; + } as unknown as ITelemetryService & { sendEnhancedGHTelemetryEvent: ReturnType; sendMSFTTelemetryEvent: ReturnType }; }); afterEach(() => { @@ -1142,6 +1142,20 @@ describe('AutomodeService', () => { imagePngCount: 1, imageClipboardCount: 1, }); + + const restrictedEvent = mockTelemetryService.sendEnhancedGHTelemetryEvent.mock.calls.find((call: unknown[]) => call[0] === 'automode.routerDecisionRestricted'); + expect(restrictedEvent).toBeDefined(); + expect(restrictedEvent![2]).toMatchObject({ + imageCount: 1, + totalImageBytes: 24, + maxImageBytes: 24, + maxImageWidth: 7, + maxImageHeight: 11, + maxImagePixels: 77, + totalImagePixels: 77, + imagePngCount: 1, + imageClipboardCount: 1, + }); }); it('should not emit routerModelSelection when router fails', async () => {