Merge pull request #319637 from tbogoodnews/main

Fix BYOK response success turn telemetry

extensions/copilot/src/extension/byok/vscode-node/anthropicProvider.ts

@@ -97,6 +97,7 @@ export class AnthropicLMProvider extends AbstractLanguageModelChatProvider {
 		// This handles the case where AsyncLocalStorage context was lost crossing VS Code IPC.
 		const correlationId = (options as { modelOptions?: OTelModelOptions }).modelOptions?._capturingTokenCorrelationId;
 		const capturingToken = correlationId ? retrieveCapturingTokenByCorrelation(correlationId) : undefined;
+		const telemetryTurn = (options as { modelOptions?: OTelModelOptions }).modelOptions?._telemetryTurn;
 
 		// Restore OTel trace context to link spans back to the agent trace
 		const parentTraceContext = (options as { modelOptions?: OTelModelOptions }).modelOptions?._otelTraceContext ?? undefined;
@@ -441,6 +442,7 @@ export class AnthropicLMProvider extends AbstractLanguageModelChatProvider {
 					requestId,
 				}, {
 					totalTokenMax: model.maxInputTokens ?? -1,
+					...(telemetryTurn !== undefined ? { turn: telemetryTurn } : {}),
 					tokenCountMax: model.maxOutputTokens ?? -1,
 					promptTokenCount: result.usage?.prompt_tokens,
 					promptCacheTokenCount: result.usage?.prompt_tokens_details?.cached_tokens,

extensions/copilot/src/extension/byok/vscode-node/geminiNativeProvider.ts

@@ -82,6 +82,7 @@ export class GeminiNativeBYOKLMProvider extends AbstractLanguageModelChatProvide
 		// This handles the case where AsyncLocalStorage context was lost crossing VS Code IPC.
 		const correlationId = (options as { modelOptions?: OTelModelOptions }).modelOptions?._capturingTokenCorrelationId;
 		const capturingToken = correlationId ? retrieveCapturingTokenByCorrelation(correlationId) : undefined;
+		const telemetryTurn = (options as { modelOptions?: OTelModelOptions }).modelOptions?._telemetryTurn;
 
 		// Restore OTel trace context to link spans back to the agent trace
 		const parentTraceContext = (options as { modelOptions?: OTelModelOptions }).modelOptions?._otelTraceContext ?? undefined;
@@ -312,6 +313,7 @@ export class GeminiNativeBYOKLMProvider extends AbstractLanguageModelChatProvide
 					requestId,
 				}, {
 					totalTokenMax: model.maxInputTokens ?? -1,
+					...(telemetryTurn !== undefined ? { turn: telemetryTurn } : {}),
 					tokenCountMax: model.maxOutputTokens ?? -1,
 					promptTokenCount: result.usage?.prompt_tokens,
 					promptCacheTokenCount: result.usage?.prompt_tokens_details?.cached_tokens,

extensions/copilot/src/extension/byok/vscode-node/test/anthropicProvider.spec.ts

@@ -0,0 +1,195 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import * as vscode from 'vscode';
+import { DefaultsOnlyConfigurationService } from '../../../../platform/configuration/common/defaultsOnlyConfigurationService';
+import { IToolDeferralService } from '../../../../platform/networking/common/toolDeferralService';
+import { NoopOTelService, resolveOTelConfig } from '../../../../platform/otel/common/index';
+import type { CapturingToken } from '../../../../platform/requestLogger/common/capturingToken';
+import type { IRequestLogger } from '../../../../platform/requestLogger/common/requestLogger';
+import { NullExperimentationService } from '../../../../platform/telemetry/common/nullExperimentationService';
+import { NullTelemetryService } from '../../../../platform/telemetry/common/nullTelemetryService';
+import type { TelemetryDestination, TelemetryEventMeasurements, TelemetryEventProperties } from '../../../../platform/telemetry/common/telemetry';
+import { TestLogService } from '../../../../platform/testing/common/testLogService';
+import type { ExtendedLanguageModelChatInformation, LanguageModelChatConfiguration } from '../abstractLanguageModelChatProvider';
+import type { IBYOKStorageService } from '../byokStorageService';
+
+type AnthropicStreamChunk =
+	| { type: 'message_start'; message: { usage: { input_tokens: number; cache_creation_input_tokens?: number; cache_read_input_tokens?: number } } }
+	| { type: 'content_block_delta'; delta: { type: 'text_delta'; text: string } }
+	| { type: 'message_delta'; usage: { output_tokens: number } };
+
+type MockAnthropicConstructor = {
+	streamChunks: AnthropicStreamChunk[];
+};
+
+vi.mock('@anthropic-ai/sdk', () => {
+	class MockAnthropic {
+		public static streamChunks: AnthropicStreamChunk[] = [];
+
+		public readonly baseURL = 'https://api.anthropic.com';
+		public readonly models = {
+			list: async () => ({ data: [] }),
+		};
+		public readonly beta = {
+			messages: {
+				create: async () => (async function* () {
+					for (const chunk of MockAnthropic.streamChunks) {
+						yield chunk;
+					}
+				})()
+			}
+		};
+
+		constructor(_opts: { apiKey?: string }) { }
+	}
+
+	return {
+		default: MockAnthropic,
+	};
+});
+
+type ProgressItem = vscode.LanguageModelResponsePart2;
+
+class TestProgress implements vscode.Progress<ProgressItem> {
+	public readonly items: ProgressItem[] = [];
+	report(value: ProgressItem): void {
+		this.items.push(value);
+	}
+}
+
+class RecordingTelemetryService extends NullTelemetryService {
+	public readonly events: { eventName: string; destination: TelemetryDestination; properties?: TelemetryEventProperties; measurements?: TelemetryEventMeasurements }[] = [];
+
+	override sendTelemetryEvent(eventName: string, destination: TelemetryDestination, properties?: TelemetryEventProperties, measurements?: TelemetryEventMeasurements): void {
+		this.events.push({ eventName, destination, properties, measurements });
+	}
+}
+
+function createStorageService(overrides?: Partial<IBYOKStorageService>): IBYOKStorageService {
+	return {
+		getAPIKey: vi.fn().mockResolvedValue(undefined),
+		storeAPIKey: vi.fn().mockResolvedValue(undefined),
+		deleteAPIKey: vi.fn().mockResolvedValue(undefined),
+		getStoredModelConfigs: vi.fn().mockResolvedValue({}),
+		saveModelConfig: vi.fn().mockResolvedValue(undefined),
+		removeModelConfig: vi.fn().mockResolvedValue(undefined),
+		...overrides,
+	};
+}
+
+function createRequestLogger(): IRequestLogger {
+	const didChangeEmitter = new vscode.EventEmitter<void>();
+	return {
+		_serviceBrand: undefined,
+		promptRendererTracing: false,
+		captureInvocation: async <T>(_request: CapturingToken, fn: () => Promise<T>) => fn(),
+		logToolCall: () => undefined,
+		logModelListCall: () => undefined,
+		logChatRequest: () => ({
+			markTimeToFirstToken: () => undefined,
+			resolveWithCancelation: () => undefined,
+			resolve: () => undefined,
+		}),
+		addPromptTrace: () => undefined,
+		addEntry: () => undefined,
+		onDidChangeRequests: didChangeEmitter.event,
+		getRequests: () => [],
+		enableWorkspaceEditTracing: () => undefined,
+		disableWorkspaceEditTracing: () => undefined,
+	} as unknown as IRequestLogger;
+}
+
+function createModel(): ExtendedLanguageModelChatInformation<LanguageModelChatConfiguration> {
+	return {
+		id: 'claude-sonnet-4-5',
+		name: 'Claude Sonnet 4.5',
+		family: 'Claude',
+		version: '1.0.0',
+		maxInputTokens: 1000,
+		maxOutputTokens: 1000,
+		capabilities: { toolCalling: false, imageInput: false },
+		configuration: { apiKey: 'k_test' }
+	};
+}
+
+async function runAnthropicRequest(telemetryTurn?: number): Promise<TelemetryEventMeasurements | undefined> {
+	const { AnthropicLMProvider } = await import('../anthropicProvider');
+	const MockAnthropic = (await import('@anthropic-ai/sdk')).default as unknown as MockAnthropicConstructor;
+	MockAnthropic.streamChunks = [
+		{
+			type: 'message_start',
+			message: {
+				usage: {
+					input_tokens: 11,
+					cache_read_input_tokens: 2,
+				}
+			}
+		},
+		{
+			type: 'content_block_delta',
+			delta: { type: 'text_delta', text: 'Hello from Anthropic' }
+		},
+		{
+			type: 'message_delta',
+			usage: { output_tokens: 7 }
+		}
+	];
+
+	const telemetry = new RecordingTelemetryService();
+	const provider = new AnthropicLMProvider(
+		undefined,
+		createStorageService(),
+		new TestLogService(),
+		createRequestLogger(),
+		new DefaultsOnlyConfigurationService(),
+		new NullExperimentationService(),
+		telemetry,
+		new NoopOTelService(resolveOTelConfig({ env: {}, extensionVersion: '1.0.0', sessionId: 'test' })),
+		{ _serviceBrand: undefined, isNonDeferredTool: () => true } satisfies IToolDeferralService,
+	);
+	const messages: vscode.LanguageModelChatMessage[] = [
+		new vscode.LanguageModelChatMessage(vscode.LanguageModelChatMessageRole.User, 'hello')
+	];
+
+	const tokenSource = new vscode.CancellationTokenSource();
+	try {
+		await provider.provideLanguageModelChatResponse(
+			createModel(),
+			messages,
+			{
+				requestInitiator: 'test',
+				tools: [],
+				toolMode: vscode.LanguageModelChatToolMode.Auto,
+				...(telemetryTurn !== undefined ? { modelOptions: { _telemetryTurn: telemetryTurn } } : {})
+			},
+			new TestProgress(),
+			tokenSource.token
+		);
+	} finally {
+		tokenSource.dispose();
+	}
+
+	return telemetry.events.find(event => event.eventName === 'response.success')?.measurements;
+}
+
+describe('AnthropicLMProvider', () => {
+	beforeEach(() => {
+		vi.clearAllMocks();
+	});
+
+	it('emits response.success telemetry with the forwarded turn measurement', async () => {
+		const measurements = await runAnthropicRequest(3);
+
+		expect(measurements?.turn).toBe(3);
+	}, 30_000);
+
+	it('omits response.success turn telemetry when no turn is forwarded', async () => {
+		const measurements = await runAnthropicRequest();
+
+		expect(Object.prototype.hasOwnProperty.call(measurements ?? {}, 'turn')).toBe(false);
+	}, 30_000);
+});

extensions/copilot/src/extension/byok/vscode-node/test/geminiNativeProvider.spec.ts

@@ -9,6 +9,7 @@ import { NoopOTelService, resolveOTelConfig } from '../../../../platform/otel/co
 import type { CapturingToken } from '../../../../platform/requestLogger/common/capturingToken';
 import type { IRequestLogger } from '../../../../platform/requestLogger/common/requestLogger';
 import { NullTelemetryService } from '../../../../platform/telemetry/common/nullTelemetryService';
+import type { TelemetryDestination, TelemetryEventMeasurements, TelemetryEventProperties } from '../../../../platform/telemetry/common/telemetry';
 import { TestLogService } from '../../../../platform/testing/common/testLogService';
 import type { IBYOKStorageService } from '../byokStorageService';
 
@@ -63,6 +64,14 @@ class TestProgress implements vscode.Progress<ProgressItem> {
 	}
 }
 
+class RecordingTelemetryService extends NullTelemetryService {
+	public readonly events: { eventName: string; destination: TelemetryDestination; properties?: TelemetryEventProperties; measurements?: TelemetryEventMeasurements }[] = [];
+
+	override sendTelemetryEvent(eventName: string, destination: TelemetryDestination, properties?: TelemetryEventProperties, measurements?: TelemetryEventMeasurements): void {
+		this.events.push({ eventName, destination, properties, measurements });
+	}
+}
+
 function createStorageService(overrides?: Partial<IBYOKStorageService>): IBYOKStorageService {
 	return {
 		getAPIKey: vi.fn().mockResolvedValue(undefined),
@@ -102,6 +111,57 @@ describe('GeminiNativeBYOKLMProvider', () => {
 		vi.clearAllMocks();
 	});
 
+	it('emits response.success telemetry with the forwarded turn measurement', async () => {
+		const { GeminiNativeBYOKLMProvider } = await import('../geminiNativeProvider');
+		const genai = await import('@google/genai');
+		const MockGoogleGenAI = genai.GoogleGenAI as unknown as { streamChunks: any[] };
+		MockGoogleGenAI.streamChunks.length = 0;
+		MockGoogleGenAI.streamChunks.push({
+			candidates: [{
+				content: { parts: [{ text: 'Hello from Gemini' }] }
+			}],
+			usageMetadata: {
+				promptTokenCount: 11,
+				candidatesTokenCount: 7,
+				totalTokenCount: 18,
+				cachedContentTokenCount: 2
+			}
+		});
+
+		const telemetry = new RecordingTelemetryService();
+		const provider = new GeminiNativeBYOKLMProvider(undefined, createStorageService(), new TestLogService(), createRequestLogger(), telemetry, new NoopOTelService(resolveOTelConfig({ env: {}, extensionVersion: '1.0.0', sessionId: 'test' })));
+		const model = {
+			id: 'gemini-2.0-flash',
+			name: 'Gemini 2.0 Flash',
+			family: 'Gemini',
+			version: '1.0.0',
+			maxInputTokens: 1000,
+			maxOutputTokens: 1000,
+			capabilities: { toolCalling: false, imageInput: false },
+			configuration: { apiKey: 'k_test' }
+		} as any;
+		const messages: vscode.LanguageModelChatMessage[] = [
+			new vscode.LanguageModelChatMessage(vscode.LanguageModelChatMessageRole.User, 'hello')
+		];
+
+		const tokenSource = new vscode.CancellationTokenSource();
+		try {
+			await provider.provideLanguageModelChatResponse(
+				model,
+				messages,
+				{ requestInitiator: 'test', tools: [], toolMode: vscode.LanguageModelChatToolMode.Auto, modelOptions: { _telemetryTurn: 3 } } as any,
+				new TestProgress(),
+				tokenSource.token
+			);
+		} finally {
+			tokenSource.dispose();
+		}
+
+		const responseSuccessEvent = telemetry.events.find(event => event.eventName === 'response.success');
+		expect(responseSuccessEvent).toBeDefined();
+		expect(responseSuccessEvent?.measurements?.turn).toBe(3);
+	}, 30_000);
+
 	it.skip('throws a clear error when no API key is configured (no silent return)', async () => {
 		const { GeminiNativeBYOKLMProvider } = await import('../geminiNativeProvider');
 		const storage = createStorageService({ getAPIKey: vi.fn().mockResolvedValue(undefined) });

extensions/copilot/src/extension/prompt/node/defaultIntentRequestHandler.ts

@@ -727,6 +727,7 @@ class DefaultToolCallingLoop extends ToolCallingLoop<IDefaultToolLoopOptions> {
 				messageSource: opts.isKeepAliveProbe ? 'chat.cacheKeepAlive' : this.options.intent?.id && this.options.intent.id !== UnknownIntent.ID ? `${messageSourcePrefix}.${this.options.intent.id}` : `${messageSourcePrefix}.user`,
 				subType: this.options.request.subAgentInvocationId ? `subagent` : this.options.request.isSystemInitiated ? 'system-initiated' : undefined,
 				parentRequestId: this.options.request.parentRequestId,
+				turnIndex: this.options.conversation.turns.length.toString(),
 				iterationNumber: opts.iterationNumber.toString(),
 			},
 			interactionTypeOverride: this.options.request.subAgentInvocationId ? 'conversation-subagent' : undefined,

extensions/copilot/src/platform/endpoint/vscode-node/extChatEndpoint.ts

@@ -169,6 +169,7 @@ export class ExtensionContributedChatEndpoint implements IChatEndpoint {
 		finishedCb,
 		location,
 		source,
+		telemetryProperties,
 	}: IMakeChatRequestOptions, token: CancellationToken): Promise<ChatResponse> {
 		const vscodeMessages = convertToApiChatMessage(messages);
 		const ourRequestId = generateUuid();
@@ -179,6 +180,7 @@ export class ExtensionContributedChatEndpoint implements IChatEndpoint {
 		// - Anthropic: inside AnthropicLMProvider
 		// - Gemini: inside GeminiNativeBYOKLMProvider
 		const activeTraceCtx = this._otelService.getActiveTraceContext();
+		const telemetryTurn = getTelemetryTurnFromProperties(telemetryProperties);
 
 		const vscodeOptions: vscode.LanguageModelChatRequestOptions = {
 			tools: ((requestOptions?.tools ?? []) as OpenAiFunctionTool[]).map(tool => ({
@@ -190,6 +192,7 @@ export class ExtensionContributedChatEndpoint implements IChatEndpoint {
 			modelOptions: {
 				_capturingTokenCorrelationId: ourRequestId,
 				_otelTraceContext: activeTraceCtx ?? null,
+				...(telemetryTurn !== undefined ? { _telemetryTurn: telemetryTurn } : {}),
 			}
 		};
 
@@ -309,6 +312,19 @@ export class ExtensionContributedChatEndpoint implements IChatEndpoint {
 	}
 }
 
+function getTelemetryTurnFromProperties(telemetryProperties: IMakeChatRequestOptions['telemetryProperties']): number | undefined {
+	if (typeof telemetryProperties?.turnIndex !== 'string') {
+		return undefined;
+	}
+
+	if (!/^\d+$/.test(telemetryProperties.turnIndex)) {
+		return undefined;
+	}
+
+	const turn = Number.parseInt(telemetryProperties.turnIndex, 10);
+	return Number.isSafeInteger(turn) ? turn : undefined;
+}
+
 export function convertToApiChatMessage(messages: Raw.ChatMessage[]): Array<vscode.LanguageModelChatMessage | vscode.LanguageModelChatMessage2> {
 	const apiMessages: Array<vscode.LanguageModelChatMessage | vscode.LanguageModelChatMessage2> = [];
 	for (const message of messages) {

extensions/copilot/src/platform/endpoint/vscode-node/test/extChatEndpoint.spec.ts

@@ -0,0 +1,91 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { Raw } from '@vscode/prompt-tsx';
+import { describe, expect, it, vi } from 'vitest';
+import * as vscode from 'vscode';
+import { ChatFetchResponseType, ChatLocation } from '../../../chat/common/commonTypes';
+import { NoopOTelService, resolveOTelConfig } from '../../../otel/common/index';
+import type { IInstantiationService } from '../../../../util/vs/platform/instantiation/common/instantiation';
+import { ExtensionContributedChatEndpoint } from '../extChatEndpoint';
+
+describe('ExtensionContributedChatEndpoint', () => {
+	it('forwards telemetry turn from request properties through model options', async () => {
+		let capturedOptions: vscode.LanguageModelChatRequestOptions | undefined;
+		const languageModel = createLanguageModel(options => capturedOptions = options);
+		const endpoint = new ExtensionContributedChatEndpoint(
+			languageModel,
+			createInstantiationService(),
+			new NoopOTelService(resolveOTelConfig({ env: {}, extensionVersion: '1.0.0', sessionId: 'test' })),
+		);
+
+		const result = await endpoint.makeChatRequest2({
+			debugName: 'test',
+			messages: [{
+				role: Raw.ChatRole.User,
+				content: [{ type: Raw.ChatCompletionContentPartKind.Text, text: 'hello' }]
+			}],
+			finishedCb: undefined,
+			location: ChatLocation.Panel,
+			requestOptions: {},
+			telemetryProperties: { turnIndex: '5' }
+		}, new vscode.CancellationTokenSource().token);
+
+		expect(result.type).toBe(ChatFetchResponseType.Success);
+		expect(capturedOptions?.modelOptions?._telemetryTurn).toBe(5);
+	});
+
+	it('only forwards telemetry turn for base-10 non-negative integer request properties', async () => {
+		const capturedOptions: vscode.LanguageModelChatRequestOptions[] = [];
+		const languageModel = createLanguageModel(options => capturedOptions.push(options));
+		const endpoint = new ExtensionContributedChatEndpoint(
+			languageModel,
+			createInstantiationService(),
+			new NoopOTelService(resolveOTelConfig({ env: {}, extensionVersion: '1.0.0', sessionId: 'test' })),
+		);
+
+		for (const turnIndex of ['', ' ', '-1', '1e2', '3.14', 'abc']) {
+			const result = await endpoint.makeChatRequest2({
+				debugName: 'test',
+				messages: [{
+					role: Raw.ChatRole.User,
+					content: [{ type: Raw.ChatCompletionContentPartKind.Text, text: 'hello' }]
+				}],
+				finishedCb: undefined,
+				location: ChatLocation.Panel,
+				requestOptions: {},
+				telemetryProperties: { turnIndex }
+			}, new vscode.CancellationTokenSource().token);
+
+			expect(result.type).toBe(ChatFetchResponseType.Success);
+		}
+
+		expect(capturedOptions.map(options => options.modelOptions?._telemetryTurn)).toEqual([undefined, undefined, undefined, undefined, undefined, undefined]);
+	});
+});
+
+function createLanguageModel(captureOptions: (options: vscode.LanguageModelChatRequestOptions) => void): vscode.LanguageModelChat {
+	return {
+		id: 'test-model',
+		name: 'Test Model',
+		vendor: 'test-vendor',
+		family: 'test-family',
+		version: '1.0.0',
+		maxInputTokens: 1000,
+		capabilities: {},
+		sendRequest: vi.fn(async (_messages, options) => {
+			captureOptions(options);
+			return {
+				stream: (async function* () {
+					yield new vscode.LanguageModelTextPart('hello');
+				})()
+			};
+		})
+	} as unknown as vscode.LanguageModelChat;
+}
+
+function createInstantiationService(): IInstantiationService {
+	return { createInstance: vi.fn() } as unknown as IInstantiationService;
+}

extensions/copilot/src/platform/networking/common/networking.ts

@@ -251,6 +251,8 @@ export type IChatRequestTelemetryProperties = {
 	parentHeaderRequestId?: string;
 	/** For a subagent: The modelCallId from the parent agent's model call that triggered this subagent invocation. */
 	parentModelCallId?: string;
+	/** The conversation turn index, matching the panel.request turn measurement. */
+	turnIndex?: string;
 	/** The 0-based iteration number of the tool-calling loop that produced this request. */
 	iterationNumber?: string;
 };

extensions/copilot/src/platform/otel/common/otelService.ts

@@ -160,6 +160,7 @@ export interface ISpanHandle {
 export interface OTelModelOptions {
 	readonly _capturingTokenCorrelationId?: string;
 	readonly _otelTraceContext?: TraceContext | null;
+	readonly _telemetryTurn?: number;
 }
 
 /**