From 988c65df74f34c194aec7a1e6b6d0728027e9939 Mon Sep 17 00:00:00 2001 From: Taylor Blair Date: Tue, 2 Jun 2026 12:20:59 -0700 Subject: [PATCH 1/2] Fix BYOK response success turn telemetry Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../byok/vscode-node/anthropicProvider.ts | 2 + .../byok/vscode-node/geminiNativeProvider.ts | 2 + .../test/geminiNativeProvider.spec.ts | 60 +++++++++++++++++++ .../node/defaultIntentRequestHandler.ts | 1 + .../endpoint/vscode-node/extChatEndpoint.ts | 12 ++++ .../vscode-node/test/extChatEndpoint.spec.ts | 54 +++++++++++++++++ .../platform/networking/common/networking.ts | 2 + .../src/platform/otel/common/otelService.ts | 1 + 8 files changed, 134 insertions(+) create mode 100644 extensions/copilot/src/platform/endpoint/vscode-node/test/extChatEndpoint.spec.ts diff --git a/extensions/copilot/src/extension/byok/vscode-node/anthropicProvider.ts b/extensions/copilot/src/extension/byok/vscode-node/anthropicProvider.ts index 95cc0d1df19..729dbbfc771 100644 --- a/extensions/copilot/src/extension/byok/vscode-node/anthropicProvider.ts +++ b/extensions/copilot/src/extension/byok/vscode-node/anthropicProvider.ts @@ -97,6 +97,7 @@ export class AnthropicLMProvider extends AbstractLanguageModelChatProvider { // This handles the case where AsyncLocalStorage context was lost crossing VS Code IPC. const correlationId = (options as { modelOptions?: OTelModelOptions }).modelOptions?._capturingTokenCorrelationId; const capturingToken = correlationId ? retrieveCapturingTokenByCorrelation(correlationId) : undefined; + const telemetryTurn = (options as { modelOptions?: OTelModelOptions }).modelOptions?._telemetryTurn; // Restore OTel trace context to link spans back to the agent trace const parentTraceContext = (options as { modelOptions?: OTelModelOptions }).modelOptions?._otelTraceContext ?? undefined; @@ -441,6 +442,7 @@ export class AnthropicLMProvider extends AbstractLanguageModelChatProvider { requestId, }, { totalTokenMax: model.maxInputTokens ?? -1, + turn: telemetryTurn, tokenCountMax: model.maxOutputTokens ?? -1, promptTokenCount: result.usage?.prompt_tokens, promptCacheTokenCount: result.usage?.prompt_tokens_details?.cached_tokens, diff --git a/extensions/copilot/src/extension/byok/vscode-node/geminiNativeProvider.ts b/extensions/copilot/src/extension/byok/vscode-node/geminiNativeProvider.ts index 4f1b2e2f7a4..ea71cfd93d5 100644 --- a/extensions/copilot/src/extension/byok/vscode-node/geminiNativeProvider.ts +++ b/extensions/copilot/src/extension/byok/vscode-node/geminiNativeProvider.ts @@ -82,6 +82,7 @@ export class GeminiNativeBYOKLMProvider extends AbstractLanguageModelChatProvide // This handles the case where AsyncLocalStorage context was lost crossing VS Code IPC. const correlationId = (options as { modelOptions?: OTelModelOptions }).modelOptions?._capturingTokenCorrelationId; const capturingToken = correlationId ? retrieveCapturingTokenByCorrelation(correlationId) : undefined; + const telemetryTurn = (options as { modelOptions?: OTelModelOptions }).modelOptions?._telemetryTurn; // Restore OTel trace context to link spans back to the agent trace const parentTraceContext = (options as { modelOptions?: OTelModelOptions }).modelOptions?._otelTraceContext ?? undefined; @@ -312,6 +313,7 @@ export class GeminiNativeBYOKLMProvider extends AbstractLanguageModelChatProvide requestId, }, { totalTokenMax: model.maxInputTokens ?? -1, + turn: telemetryTurn, tokenCountMax: model.maxOutputTokens ?? -1, promptTokenCount: result.usage?.prompt_tokens, promptCacheTokenCount: result.usage?.prompt_tokens_details?.cached_tokens, diff --git a/extensions/copilot/src/extension/byok/vscode-node/test/geminiNativeProvider.spec.ts b/extensions/copilot/src/extension/byok/vscode-node/test/geminiNativeProvider.spec.ts index fc3ae534a3e..2f2aef5af3b 100644 --- a/extensions/copilot/src/extension/byok/vscode-node/test/geminiNativeProvider.spec.ts +++ b/extensions/copilot/src/extension/byok/vscode-node/test/geminiNativeProvider.spec.ts @@ -9,6 +9,7 @@ import { NoopOTelService, resolveOTelConfig } from '../../../../platform/otel/co import type { CapturingToken } from '../../../../platform/requestLogger/common/capturingToken'; import type { IRequestLogger } from '../../../../platform/requestLogger/common/requestLogger'; import { NullTelemetryService } from '../../../../platform/telemetry/common/nullTelemetryService'; +import type { TelemetryDestination, TelemetryEventMeasurements, TelemetryEventProperties } from '../../../../platform/telemetry/common/telemetry'; import { TestLogService } from '../../../../platform/testing/common/testLogService'; import type { IBYOKStorageService } from '../byokStorageService'; @@ -63,6 +64,14 @@ class TestProgress implements vscode.Progress { } } +class RecordingTelemetryService extends NullTelemetryService { + public readonly events: { eventName: string; destination: TelemetryDestination; properties?: TelemetryEventProperties; measurements?: TelemetryEventMeasurements }[] = []; + + override sendTelemetryEvent(eventName: string, destination: TelemetryDestination, properties?: TelemetryEventProperties, measurements?: TelemetryEventMeasurements): void { + this.events.push({ eventName, destination, properties, measurements }); + } +} + function createStorageService(overrides?: Partial): IBYOKStorageService { return { getAPIKey: vi.fn().mockResolvedValue(undefined), @@ -102,6 +111,57 @@ describe('GeminiNativeBYOKLMProvider', () => { vi.clearAllMocks(); }); + it('emits response.success telemetry with the forwarded turn measurement', async () => { + const { GeminiNativeBYOKLMProvider } = await import('../geminiNativeProvider'); + const genai = await import('@google/genai'); + const MockGoogleGenAI = genai.GoogleGenAI as unknown as { streamChunks: any[] }; + MockGoogleGenAI.streamChunks.length = 0; + MockGoogleGenAI.streamChunks.push({ + candidates: [{ + content: { parts: [{ text: 'Hello from Gemini' }] } + }], + usageMetadata: { + promptTokenCount: 11, + candidatesTokenCount: 7, + totalTokenCount: 18, + cachedContentTokenCount: 2 + } + }); + + const telemetry = new RecordingTelemetryService(); + const provider = new GeminiNativeBYOKLMProvider(undefined, createStorageService(), new TestLogService(), createRequestLogger(), telemetry, new NoopOTelService(resolveOTelConfig({ env: {}, extensionVersion: '1.0.0', sessionId: 'test' }))); + const model = { + id: 'gemini-2.0-flash', + name: 'Gemini 2.0 Flash', + family: 'Gemini', + version: '1.0.0', + maxInputTokens: 1000, + maxOutputTokens: 1000, + capabilities: { toolCalling: false, imageInput: false }, + configuration: { apiKey: 'k_test' } + } as any; + const messages: vscode.LanguageModelChatMessage[] = [ + new vscode.LanguageModelChatMessage(vscode.LanguageModelChatMessageRole.User, 'hello') + ]; + + const tokenSource = new vscode.CancellationTokenSource(); + try { + await provider.provideLanguageModelChatResponse( + model, + messages, + { requestInitiator: 'test', tools: [], toolMode: vscode.LanguageModelChatToolMode.Auto, modelOptions: { _telemetryTurn: 3 } } as any, + new TestProgress(), + tokenSource.token + ); + } finally { + tokenSource.dispose(); + } + + const responseSuccessEvent = telemetry.events.find(event => event.eventName === 'response.success'); + expect(responseSuccessEvent).toBeDefined(); + expect(responseSuccessEvent?.measurements?.turn).toBe(3); + }, 30_000); + it.skip('throws a clear error when no API key is configured (no silent return)', async () => { const { GeminiNativeBYOKLMProvider } = await import('../geminiNativeProvider'); const storage = createStorageService({ getAPIKey: vi.fn().mockResolvedValue(undefined) }); diff --git a/extensions/copilot/src/extension/prompt/node/defaultIntentRequestHandler.ts b/extensions/copilot/src/extension/prompt/node/defaultIntentRequestHandler.ts index c17b227fccc..75e13e9a922 100644 --- a/extensions/copilot/src/extension/prompt/node/defaultIntentRequestHandler.ts +++ b/extensions/copilot/src/extension/prompt/node/defaultIntentRequestHandler.ts @@ -727,6 +727,7 @@ class DefaultToolCallingLoop extends ToolCallingLoop { messageSource: opts.isKeepAliveProbe ? 'chat.cacheKeepAlive' : this.options.intent?.id && this.options.intent.id !== UnknownIntent.ID ? `${messageSourcePrefix}.${this.options.intent.id}` : `${messageSourcePrefix}.user`, subType: this.options.request.subAgentInvocationId ? `subagent` : this.options.request.isSystemInitiated ? 'system-initiated' : undefined, parentRequestId: this.options.request.parentRequestId, + turnIndex: this.options.conversation.turns.length.toString(), iterationNumber: opts.iterationNumber.toString(), }, interactionTypeOverride: this.options.request.subAgentInvocationId ? 'conversation-subagent' : undefined, diff --git a/extensions/copilot/src/platform/endpoint/vscode-node/extChatEndpoint.ts b/extensions/copilot/src/platform/endpoint/vscode-node/extChatEndpoint.ts index 48bac293bf7..721253d5ece 100644 --- a/extensions/copilot/src/platform/endpoint/vscode-node/extChatEndpoint.ts +++ b/extensions/copilot/src/platform/endpoint/vscode-node/extChatEndpoint.ts @@ -167,6 +167,7 @@ export class ExtensionContributedChatEndpoint implements IChatEndpoint { finishedCb, location, source, + telemetryProperties, }: IMakeChatRequestOptions, token: CancellationToken): Promise { const vscodeMessages = convertToApiChatMessage(messages); const ourRequestId = generateUuid(); @@ -177,6 +178,7 @@ export class ExtensionContributedChatEndpoint implements IChatEndpoint { // - Anthropic: inside AnthropicLMProvider // - Gemini: inside GeminiNativeBYOKLMProvider const activeTraceCtx = this._otelService.getActiveTraceContext(); + const telemetryTurn = getTelemetryTurnFromProperties(telemetryProperties); const vscodeOptions: vscode.LanguageModelChatRequestOptions = { tools: ((requestOptions?.tools ?? []) as OpenAiFunctionTool[]).map(tool => ({ @@ -188,6 +190,7 @@ export class ExtensionContributedChatEndpoint implements IChatEndpoint { modelOptions: { _capturingTokenCorrelationId: ourRequestId, _otelTraceContext: activeTraceCtx ?? null, + ...(telemetryTurn !== undefined ? { _telemetryTurn: telemetryTurn } : {}), } }; @@ -307,6 +310,15 @@ export class ExtensionContributedChatEndpoint implements IChatEndpoint { } } +function getTelemetryTurnFromProperties(telemetryProperties: IMakeChatRequestOptions['telemetryProperties']): number | undefined { + if (typeof telemetryProperties?.turnIndex !== 'string') { + return undefined; + } + + const turn = Number(telemetryProperties.turnIndex); + return Number.isFinite(turn) ? turn : undefined; +} + export function convertToApiChatMessage(messages: Raw.ChatMessage[]): Array { const apiMessages: Array = []; for (const message of messages) { diff --git a/extensions/copilot/src/platform/endpoint/vscode-node/test/extChatEndpoint.spec.ts b/extensions/copilot/src/platform/endpoint/vscode-node/test/extChatEndpoint.spec.ts new file mode 100644 index 00000000000..afc030c1bb0 --- /dev/null +++ b/extensions/copilot/src/platform/endpoint/vscode-node/test/extChatEndpoint.spec.ts @@ -0,0 +1,54 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { Raw } from '@vscode/prompt-tsx'; +import { describe, expect, it, vi } from 'vitest'; +import * as vscode from 'vscode'; +import { ChatFetchResponseType, ChatLocation } from '../../../chat/common/commonTypes'; +import { NoopOTelService, resolveOTelConfig } from '../../../otel/common/index'; +import { ExtensionContributedChatEndpoint } from '../extChatEndpoint'; + +describe('ExtensionContributedChatEndpoint', () => { + it('forwards telemetry turn from request properties through model options', async () => { + let capturedOptions: vscode.LanguageModelChatRequestOptions | undefined; + const languageModel = { + id: 'test-model', + name: 'Test Model', + vendor: 'test-vendor', + family: 'test-family', + version: '1.0.0', + maxInputTokens: 1000, + capabilities: {}, + sendRequest: vi.fn(async (_messages, options) => { + capturedOptions = options; + return { + stream: (async function* () { + yield new vscode.LanguageModelTextPart('hello'); + })() + }; + }) + } as unknown as vscode.LanguageModelChat; + const endpoint = new ExtensionContributedChatEndpoint( + languageModel, + { createInstance: vi.fn() } as any, + new NoopOTelService(resolveOTelConfig({ env: {}, extensionVersion: '1.0.0', sessionId: 'test' })), + ); + + const result = await endpoint.makeChatRequest2({ + debugName: 'test', + messages: [{ + role: Raw.ChatRole.User, + content: [{ type: Raw.ChatCompletionContentPartKind.Text, text: 'hello' }] + }], + finishedCb: undefined, + location: ChatLocation.Panel, + requestOptions: {}, + telemetryProperties: { turnIndex: '5' } + }, new vscode.CancellationTokenSource().token); + + expect(result.type).toBe(ChatFetchResponseType.Success); + expect(capturedOptions?.modelOptions?._telemetryTurn).toBe(5); + }); +}); diff --git a/extensions/copilot/src/platform/networking/common/networking.ts b/extensions/copilot/src/platform/networking/common/networking.ts index e752a88eae7..398ea915b1d 100644 --- a/extensions/copilot/src/platform/networking/common/networking.ts +++ b/extensions/copilot/src/platform/networking/common/networking.ts @@ -251,6 +251,8 @@ export type IChatRequestTelemetryProperties = { parentHeaderRequestId?: string; /** For a subagent: The modelCallId from the parent agent's model call that triggered this subagent invocation. */ parentModelCallId?: string; + /** The conversation turn index, matching the panel.request turn measurement. */ + turnIndex?: string; /** The 0-based iteration number of the tool-calling loop that produced this request. */ iterationNumber?: string; }; diff --git a/extensions/copilot/src/platform/otel/common/otelService.ts b/extensions/copilot/src/platform/otel/common/otelService.ts index 34c35537706..177b0f4900c 100644 --- a/extensions/copilot/src/platform/otel/common/otelService.ts +++ b/extensions/copilot/src/platform/otel/common/otelService.ts @@ -160,6 +160,7 @@ export interface ISpanHandle { export interface OTelModelOptions { readonly _capturingTokenCorrelationId?: string; readonly _otelTraceContext?: TraceContext | null; + readonly _telemetryTurn?: number; } /** From c7ab05d2e5fa3a6362feb9ba897e8539cb39de28 Mon Sep 17 00:00:00 2001 From: Taylor Blair Date: Wed, 3 Jun 2026 10:54:35 -0700 Subject: [PATCH 2/2] Address BYOK telemetry review feedback Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../byok/vscode-node/anthropicProvider.ts | 2 +- .../byok/vscode-node/geminiNativeProvider.ts | 2 +- .../test/anthropicProvider.spec.ts | 195 ++++++++++++++++++ .../endpoint/vscode-node/extChatEndpoint.ts | 8 +- .../vscode-node/test/extChatEndpoint.spec.ts | 73 +++++-- 5 files changed, 258 insertions(+), 22 deletions(-) create mode 100644 extensions/copilot/src/extension/byok/vscode-node/test/anthropicProvider.spec.ts diff --git a/extensions/copilot/src/extension/byok/vscode-node/anthropicProvider.ts b/extensions/copilot/src/extension/byok/vscode-node/anthropicProvider.ts index 729dbbfc771..03c3e0a39b4 100644 --- a/extensions/copilot/src/extension/byok/vscode-node/anthropicProvider.ts +++ b/extensions/copilot/src/extension/byok/vscode-node/anthropicProvider.ts @@ -442,7 +442,7 @@ export class AnthropicLMProvider extends AbstractLanguageModelChatProvider { requestId, }, { totalTokenMax: model.maxInputTokens ?? -1, - turn: telemetryTurn, + ...(telemetryTurn !== undefined ? { turn: telemetryTurn } : {}), tokenCountMax: model.maxOutputTokens ?? -1, promptTokenCount: result.usage?.prompt_tokens, promptCacheTokenCount: result.usage?.prompt_tokens_details?.cached_tokens, diff --git a/extensions/copilot/src/extension/byok/vscode-node/geminiNativeProvider.ts b/extensions/copilot/src/extension/byok/vscode-node/geminiNativeProvider.ts index ea71cfd93d5..1c5971e6a74 100644 --- a/extensions/copilot/src/extension/byok/vscode-node/geminiNativeProvider.ts +++ b/extensions/copilot/src/extension/byok/vscode-node/geminiNativeProvider.ts @@ -313,7 +313,7 @@ export class GeminiNativeBYOKLMProvider extends AbstractLanguageModelChatProvide requestId, }, { totalTokenMax: model.maxInputTokens ?? -1, - turn: telemetryTurn, + ...(telemetryTurn !== undefined ? { turn: telemetryTurn } : {}), tokenCountMax: model.maxOutputTokens ?? -1, promptTokenCount: result.usage?.prompt_tokens, promptCacheTokenCount: result.usage?.prompt_tokens_details?.cached_tokens, diff --git a/extensions/copilot/src/extension/byok/vscode-node/test/anthropicProvider.spec.ts b/extensions/copilot/src/extension/byok/vscode-node/test/anthropicProvider.spec.ts new file mode 100644 index 00000000000..43652b748b3 --- /dev/null +++ b/extensions/copilot/src/extension/byok/vscode-node/test/anthropicProvider.spec.ts @@ -0,0 +1,195 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import * as vscode from 'vscode'; +import { DefaultsOnlyConfigurationService } from '../../../../platform/configuration/common/defaultsOnlyConfigurationService'; +import { IToolDeferralService } from '../../../../platform/networking/common/toolDeferralService'; +import { NoopOTelService, resolveOTelConfig } from '../../../../platform/otel/common/index'; +import type { CapturingToken } from '../../../../platform/requestLogger/common/capturingToken'; +import type { IRequestLogger } from '../../../../platform/requestLogger/common/requestLogger'; +import { NullExperimentationService } from '../../../../platform/telemetry/common/nullExperimentationService'; +import { NullTelemetryService } from '../../../../platform/telemetry/common/nullTelemetryService'; +import type { TelemetryDestination, TelemetryEventMeasurements, TelemetryEventProperties } from '../../../../platform/telemetry/common/telemetry'; +import { TestLogService } from '../../../../platform/testing/common/testLogService'; +import type { ExtendedLanguageModelChatInformation, LanguageModelChatConfiguration } from '../abstractLanguageModelChatProvider'; +import type { IBYOKStorageService } from '../byokStorageService'; + +type AnthropicStreamChunk = + | { type: 'message_start'; message: { usage: { input_tokens: number; cache_creation_input_tokens?: number; cache_read_input_tokens?: number } } } + | { type: 'content_block_delta'; delta: { type: 'text_delta'; text: string } } + | { type: 'message_delta'; usage: { output_tokens: number } }; + +type MockAnthropicConstructor = { + streamChunks: AnthropicStreamChunk[]; +}; + +vi.mock('@anthropic-ai/sdk', () => { + class MockAnthropic { + public static streamChunks: AnthropicStreamChunk[] = []; + + public readonly baseURL = 'https://api.anthropic.com'; + public readonly models = { + list: async () => ({ data: [] }), + }; + public readonly beta = { + messages: { + create: async () => (async function* () { + for (const chunk of MockAnthropic.streamChunks) { + yield chunk; + } + })() + } + }; + + constructor(_opts: { apiKey?: string }) { } + } + + return { + default: MockAnthropic, + }; +}); + +type ProgressItem = vscode.LanguageModelResponsePart2; + +class TestProgress implements vscode.Progress { + public readonly items: ProgressItem[] = []; + report(value: ProgressItem): void { + this.items.push(value); + } +} + +class RecordingTelemetryService extends NullTelemetryService { + public readonly events: { eventName: string; destination: TelemetryDestination; properties?: TelemetryEventProperties; measurements?: TelemetryEventMeasurements }[] = []; + + override sendTelemetryEvent(eventName: string, destination: TelemetryDestination, properties?: TelemetryEventProperties, measurements?: TelemetryEventMeasurements): void { + this.events.push({ eventName, destination, properties, measurements }); + } +} + +function createStorageService(overrides?: Partial): IBYOKStorageService { + return { + getAPIKey: vi.fn().mockResolvedValue(undefined), + storeAPIKey: vi.fn().mockResolvedValue(undefined), + deleteAPIKey: vi.fn().mockResolvedValue(undefined), + getStoredModelConfigs: vi.fn().mockResolvedValue({}), + saveModelConfig: vi.fn().mockResolvedValue(undefined), + removeModelConfig: vi.fn().mockResolvedValue(undefined), + ...overrides, + }; +} + +function createRequestLogger(): IRequestLogger { + const didChangeEmitter = new vscode.EventEmitter(); + return { + _serviceBrand: undefined, + promptRendererTracing: false, + captureInvocation: async (_request: CapturingToken, fn: () => Promise) => fn(), + logToolCall: () => undefined, + logModelListCall: () => undefined, + logChatRequest: () => ({ + markTimeToFirstToken: () => undefined, + resolveWithCancelation: () => undefined, + resolve: () => undefined, + }), + addPromptTrace: () => undefined, + addEntry: () => undefined, + onDidChangeRequests: didChangeEmitter.event, + getRequests: () => [], + enableWorkspaceEditTracing: () => undefined, + disableWorkspaceEditTracing: () => undefined, + } as unknown as IRequestLogger; +} + +function createModel(): ExtendedLanguageModelChatInformation { + return { + id: 'claude-sonnet-4-5', + name: 'Claude Sonnet 4.5', + family: 'Claude', + version: '1.0.0', + maxInputTokens: 1000, + maxOutputTokens: 1000, + capabilities: { toolCalling: false, imageInput: false }, + configuration: { apiKey: 'k_test' } + }; +} + +async function runAnthropicRequest(telemetryTurn?: number): Promise { + const { AnthropicLMProvider } = await import('../anthropicProvider'); + const MockAnthropic = (await import('@anthropic-ai/sdk')).default as unknown as MockAnthropicConstructor; + MockAnthropic.streamChunks = [ + { + type: 'message_start', + message: { + usage: { + input_tokens: 11, + cache_read_input_tokens: 2, + } + } + }, + { + type: 'content_block_delta', + delta: { type: 'text_delta', text: 'Hello from Anthropic' } + }, + { + type: 'message_delta', + usage: { output_tokens: 7 } + } + ]; + + const telemetry = new RecordingTelemetryService(); + const provider = new AnthropicLMProvider( + undefined, + createStorageService(), + new TestLogService(), + createRequestLogger(), + new DefaultsOnlyConfigurationService(), + new NullExperimentationService(), + telemetry, + new NoopOTelService(resolveOTelConfig({ env: {}, extensionVersion: '1.0.0', sessionId: 'test' })), + { _serviceBrand: undefined, isNonDeferredTool: () => true } satisfies IToolDeferralService, + ); + const messages: vscode.LanguageModelChatMessage[] = [ + new vscode.LanguageModelChatMessage(vscode.LanguageModelChatMessageRole.User, 'hello') + ]; + + const tokenSource = new vscode.CancellationTokenSource(); + try { + await provider.provideLanguageModelChatResponse( + createModel(), + messages, + { + requestInitiator: 'test', + tools: [], + toolMode: vscode.LanguageModelChatToolMode.Auto, + ...(telemetryTurn !== undefined ? { modelOptions: { _telemetryTurn: telemetryTurn } } : {}) + }, + new TestProgress(), + tokenSource.token + ); + } finally { + tokenSource.dispose(); + } + + return telemetry.events.find(event => event.eventName === 'response.success')?.measurements; +} + +describe('AnthropicLMProvider', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('emits response.success telemetry with the forwarded turn measurement', async () => { + const measurements = await runAnthropicRequest(3); + + expect(measurements?.turn).toBe(3); + }, 30_000); + + it('omits response.success turn telemetry when no turn is forwarded', async () => { + const measurements = await runAnthropicRequest(); + + expect(Object.prototype.hasOwnProperty.call(measurements ?? {}, 'turn')).toBe(false); + }, 30_000); +}); diff --git a/extensions/copilot/src/platform/endpoint/vscode-node/extChatEndpoint.ts b/extensions/copilot/src/platform/endpoint/vscode-node/extChatEndpoint.ts index 721253d5ece..eab63306a21 100644 --- a/extensions/copilot/src/platform/endpoint/vscode-node/extChatEndpoint.ts +++ b/extensions/copilot/src/platform/endpoint/vscode-node/extChatEndpoint.ts @@ -315,8 +315,12 @@ function getTelemetryTurnFromProperties(telemetryProperties: IMakeChatRequestOpt return undefined; } - const turn = Number(telemetryProperties.turnIndex); - return Number.isFinite(turn) ? turn : undefined; + if (!/^\d+$/.test(telemetryProperties.turnIndex)) { + return undefined; + } + + const turn = Number.parseInt(telemetryProperties.turnIndex, 10); + return Number.isSafeInteger(turn) ? turn : undefined; } export function convertToApiChatMessage(messages: Raw.ChatMessage[]): Array { diff --git a/extensions/copilot/src/platform/endpoint/vscode-node/test/extChatEndpoint.spec.ts b/extensions/copilot/src/platform/endpoint/vscode-node/test/extChatEndpoint.spec.ts index afc030c1bb0..fac1dd13c2d 100644 --- a/extensions/copilot/src/platform/endpoint/vscode-node/test/extChatEndpoint.spec.ts +++ b/extensions/copilot/src/platform/endpoint/vscode-node/test/extChatEndpoint.spec.ts @@ -8,31 +8,16 @@ import { describe, expect, it, vi } from 'vitest'; import * as vscode from 'vscode'; import { ChatFetchResponseType, ChatLocation } from '../../../chat/common/commonTypes'; import { NoopOTelService, resolveOTelConfig } from '../../../otel/common/index'; +import type { IInstantiationService } from '../../../../util/vs/platform/instantiation/common/instantiation'; import { ExtensionContributedChatEndpoint } from '../extChatEndpoint'; describe('ExtensionContributedChatEndpoint', () => { it('forwards telemetry turn from request properties through model options', async () => { let capturedOptions: vscode.LanguageModelChatRequestOptions | undefined; - const languageModel = { - id: 'test-model', - name: 'Test Model', - vendor: 'test-vendor', - family: 'test-family', - version: '1.0.0', - maxInputTokens: 1000, - capabilities: {}, - sendRequest: vi.fn(async (_messages, options) => { - capturedOptions = options; - return { - stream: (async function* () { - yield new vscode.LanguageModelTextPart('hello'); - })() - }; - }) - } as unknown as vscode.LanguageModelChat; + const languageModel = createLanguageModel(options => capturedOptions = options); const endpoint = new ExtensionContributedChatEndpoint( languageModel, - { createInstance: vi.fn() } as any, + createInstantiationService(), new NoopOTelService(resolveOTelConfig({ env: {}, extensionVersion: '1.0.0', sessionId: 'test' })), ); @@ -51,4 +36,56 @@ describe('ExtensionContributedChatEndpoint', () => { expect(result.type).toBe(ChatFetchResponseType.Success); expect(capturedOptions?.modelOptions?._telemetryTurn).toBe(5); }); + + it('only forwards telemetry turn for base-10 non-negative integer request properties', async () => { + const capturedOptions: vscode.LanguageModelChatRequestOptions[] = []; + const languageModel = createLanguageModel(options => capturedOptions.push(options)); + const endpoint = new ExtensionContributedChatEndpoint( + languageModel, + createInstantiationService(), + new NoopOTelService(resolveOTelConfig({ env: {}, extensionVersion: '1.0.0', sessionId: 'test' })), + ); + + for (const turnIndex of ['', ' ', '-1', '1e2', '3.14', 'abc']) { + const result = await endpoint.makeChatRequest2({ + debugName: 'test', + messages: [{ + role: Raw.ChatRole.User, + content: [{ type: Raw.ChatCompletionContentPartKind.Text, text: 'hello' }] + }], + finishedCb: undefined, + location: ChatLocation.Panel, + requestOptions: {}, + telemetryProperties: { turnIndex } + }, new vscode.CancellationTokenSource().token); + + expect(result.type).toBe(ChatFetchResponseType.Success); + } + + expect(capturedOptions.map(options => options.modelOptions?._telemetryTurn)).toEqual([undefined, undefined, undefined, undefined, undefined, undefined]); + }); }); + +function createLanguageModel(captureOptions: (options: vscode.LanguageModelChatRequestOptions) => void): vscode.LanguageModelChat { + return { + id: 'test-model', + name: 'Test Model', + vendor: 'test-vendor', + family: 'test-family', + version: '1.0.0', + maxInputTokens: 1000, + capabilities: {}, + sendRequest: vi.fn(async (_messages, options) => { + captureOptions(options); + return { + stream: (async function* () { + yield new vscode.LanguageModelTextPart('hello'); + })() + }; + }) + } as unknown as vscode.LanguageModelChat; +} + +function createInstantiationService(): IInstantiationService { + return { createInstance: vi.fn() } as unknown as IInstantiationService; +}