From f487a678f3e4e6f2e48f87616ea6f74f5ea96064 Mon Sep 17 00:00:00 2001 From: Bhavya U Date: Sat, 14 Mar 2026 23:17:40 -0700 Subject: [PATCH] Track compaction summaries as an array with detailed metrics metadata (#4413) * Add Anthropic compaction metadata to IResultMetadata for evals Captures context editing compaction data from Anthropic Messages API and surfaces it through IResultMetadata so the evaluation system can access it. Changes: - Add anthropicCompaction field to IToolCallRound (parallels OpenAI compaction) - Capture Anthropic ContextManagementResponse deltas in tool calling loop - Aggregate compaction metrics (cleared tokens, tool uses, thinking turns) across rounds - Surface compactionMetrics on IResultMetadata via AnthropicCompactionMetadata - Merge into result in defaultIntentRequestHandler's resultWithMetadatas() * Add compaction metrics metadata for evals Surfaces background and foreground compaction (conversation summarization) metrics through IResultMetadata.compactionMetrics so the eval system can track when compaction is triggered and its cost. - Add compactionMetrics to IResultMetadata with type (foreground/background) and token usage - Create CompactionMetadata class on Turn - Set CompactionMetadata in agentIntent for both foreground and background paths - Merge CompactionMetadata into result in defaultIntentRequestHandler * Add durationMs to summarization metadata and update related logic * Refactor compaction metadata handling to use SummarizedConversationHistoryMetadata and remove CompactionMetadata references * Add usage metadata to AgentIntentInvocation and remove compaction metrics documentation * Enhance summary handling by replacing single summary with an array of summaries and updating related logic in conversation normalization * Refactor _persistSummaryOnTurn to use IBackgroundSummarizationResult for improved type safety * Simplify render result handling in AgentIntentInvocation by directly returning the await result * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> * Refactor SummarizedConversationHistoryMetadata to use options object for improved readability and maintainability --------- Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../src/extension/intents/node/agentIntent.ts | 89 ++++++++++++++-- .../extension/prompt/common/conversation.ts | 65 +++++++++--- .../node/defaultIntentRequestHandler.ts | 6 +- .../prompt/test/node/conversation.spec.ts | 47 ++++++++ .../node/agent/backgroundSummarizer.ts | 8 ++ .../agent/summarizedConversationHistory.tsx | 100 +++++++++++++++--- 6 files changed, 273 insertions(+), 42 deletions(-) diff --git a/extensions/copilot/src/extension/intents/node/agentIntent.ts b/extensions/copilot/src/extension/intents/node/agentIntent.ts index 7e02038be02..6c1e3992325 100644 --- a/extensions/copilot/src/extension/intents/node/agentIntent.ts +++ b/extensions/copilot/src/extension/intents/node/agentIntent.ts @@ -42,7 +42,7 @@ import { IDefaultIntentRequestHandlerOptions } from '../../prompt/node/defaultIn import { IDocumentContext } from '../../prompt/node/documentContext'; import { IBuildPromptResult, IIntent, IIntentInvocation } from '../../prompt/node/intents'; import { AgentPrompt, AgentPromptProps } from '../../prompts/node/agent/agentPrompt'; -import { BackgroundSummarizationState, BackgroundSummarizer } from '../../prompts/node/agent/backgroundSummarizer'; +import { BackgroundSummarizationState, BackgroundSummarizer, IBackgroundSummarizationResult } from '../../prompts/node/agent/backgroundSummarizer'; import { AgentPromptCustomizations, PromptRegistry } from '../../prompts/node/agent/promptRegistry'; import { SummarizedConversationHistory, SummarizedConversationHistoryMetadata, SummarizedConversationHistoryPropsBuilder } from '../../prompts/node/agent/summarizedConversationHistory'; import { PromptRenderer } from '../../prompts/node/base/promptRenderer'; @@ -445,7 +445,7 @@ export class AgentIntentInvocation extends EditCodeIntentInvocation implements I this.logService.debug(`[Agent] applying completed background summary (roundId=${bgResult.toolCallRoundId})`); progress.report(new ChatResponseProgressPart2(l10n.t('Compacted conversation'), async () => l10n.t('Compacted conversation'))); this._applySummaryToRounds(bgResult, promptContext); - this._persistSummaryOnTurn(bgResult, promptContext); + this._persistSummaryOnTurn(bgResult, promptContext, this._lastRenderTokenCount); this._sendBackgroundCompactionTelemetry('preRender', 'applied', contextRatio, promptContext); summaryAppliedThisIteration = true; } @@ -465,7 +465,7 @@ export class AgentIntentInvocation extends EditCodeIntentInvocation implements I if (bgResult) { this.logService.debug(`[Agent] background compaction completed — applying result (roundId=${bgResult.toolCallRoundId})`); this._applySummaryToRounds(bgResult, promptContext); - this._persistSummaryOnTurn(bgResult, promptContext); + this._persistSummaryOnTurn(bgResult, promptContext, this._lastRenderTokenCount); this._sendBackgroundCompactionTelemetry('preRenderBlocked', 'applied', contextRatio, promptContext); summaryAppliedThisIteration = true; } else { @@ -496,6 +496,19 @@ export class AgentIntentInvocation extends EditCodeIntentInvocation implements I */ this.telemetryService.sendMSFTTelemetryEvent('triggerSummarizeFailed', { errorKind, model: renderProps.endpoint.model }); + // Track failed foreground compaction + const turn = promptContext.conversation?.getLatestTurn(); + turn?.setMetadata(new SummarizedConversationHistoryMetadata( + '', // no toolCallRoundId for failures + '', // no summary text for failures + { + model: renderProps.endpoint.model, + source: 'foreground', + outcome: errorKind, + contextLengthBefore: this._lastRenderTokenCount, + }, + )); + // Something else went wrong, eg summarization failed, so render the prompt with no cache breakpoints, summarization, endpoint not reduced in size for tools or safety buffer const renderer = PromptRenderer.create(this.instantiationService, this.endpoint, this.prompt, { ...renderProps, @@ -515,6 +528,8 @@ export class AgentIntentInvocation extends EditCodeIntentInvocation implements I } }; + const contextLengthBefore = this._lastRenderTokenCount; + try { const renderer = PromptRenderer.create(this.instantiationService, endpoint, this.prompt, props); result = await renderer.render(progress, token); @@ -552,7 +567,7 @@ export class AgentIntentInvocation extends EditCodeIntentInvocation implements I if (bgResult) { this.logService.debug(`[Agent] background compaction applied after budget exceeded (roundId=${bgResult.toolCallRoundId})`); this._applySummaryToRounds(bgResult, promptContext); - this._persistSummaryOnTurn(bgResult, promptContext); + this._persistSummaryOnTurn(bgResult, promptContext, contextLengthBefore); this._sendBackgroundCompactionTelemetry(budgetExceededTrigger, 'applied', contextRatio, promptContext); summaryAppliedThisIteration = true; // Re-render with the compacted history @@ -574,6 +589,29 @@ export class AgentIntentInvocation extends EditCodeIntentInvocation implements I this._lastRenderTokenCount = result.tokenCount; + // Track foreground compaction if summarization happened during rendering + const summaryMeta = result.metadata.get(SummarizedConversationHistoryMetadata); + if (summaryMeta) { + const turn = promptContext.conversation?.getLatestTurn(); + turn?.setMetadata(new SummarizedConversationHistoryMetadata( + summaryMeta.toolCallRoundId, + summaryMeta.text, + { + thinking: summaryMeta.thinking, + usage: summaryMeta.usage, + promptTokenDetails: summaryMeta.promptTokenDetails, + model: summaryMeta.model, + summarizationMode: summaryMeta.summarizationMode, + numRounds: summaryMeta.numRounds, + numRoundsSinceLastSummarization: summaryMeta.numRoundsSinceLastSummarization, + durationMs: summaryMeta.durationMs, + source: 'foreground', + outcome: 'success', + contextLengthBefore, + }, + )); + } + // 3. Post-render background compaction checks. if (backgroundCompactionEnabled && backgroundSummarizer && !summaryAppliedThisIteration) { const postRenderRatio = budgetThreshold > 0 @@ -595,7 +633,7 @@ export class AgentIntentInvocation extends EditCodeIntentInvocation implements I if (bgResult) { this.logService.debug(`[Agent] post-render background compaction completed — applying result and re-rendering (roundId=${bgResult.toolCallRoundId})`); this._applySummaryToRounds(bgResult, promptContext); - this._persistSummaryOnTurn(bgResult, promptContext); + this._persistSummaryOnTurn(bgResult, promptContext, result.tokenCount); this._sendBackgroundCompactionTelemetry('postRenderBlocked', 'applied', postRenderRatio, promptContext); // Re-render with compacted history so the LLM receives the smaller prompt const reRenderer = PromptRenderer.create(this.instantiationService, endpoint, this.prompt, { ...props, promptContext }); @@ -686,6 +724,7 @@ export class AgentIntentInvocation extends EditCodeIntentInvocation implements I summarizationSource: 'background', }); const bgProgress: vscode.Progress = { report: () => { } }; + const bgStartTime = Date.now(); backgroundSummarizer.start(async bgToken => { try { const bgRenderResult = await bgRenderer.render(bgProgress, bgToken); @@ -694,7 +733,18 @@ export class AgentIntentInvocation extends EditCodeIntentInvocation implements I throw new Error('Background compaction produced no summary metadata'); } this.logService.debug(`[Agent] background compaction completed successfully (roundId=${summaryMetadata.toolCallRoundId})`); - return { summary: summaryMetadata.text, toolCallRoundId: summaryMetadata.toolCallRoundId }; + return { + summary: summaryMetadata.text, + toolCallRoundId: summaryMetadata.toolCallRoundId, + promptTokens: summaryMetadata.usage?.prompt_tokens, + promptCacheTokens: summaryMetadata.usage?.prompt_tokens_details?.cached_tokens, + outputTokens: summaryMetadata.usage?.completion_tokens, + durationMs: Date.now() - bgStartTime, + model: summaryMetadata.model, + summarizationMode: summaryMetadata.summarizationMode, + numRounds: summaryMetadata.numRounds, + numRoundsSinceLastSummarization: summaryMetadata.numRoundsSinceLastSummarization, + }; } catch (err) { this.logService.error(err, `[Agent] background compaction failed`); throw err; @@ -738,13 +788,34 @@ export class AgentIntentInvocation extends EditCodeIntentInvocation implements I * Persist the summary on the current turn's `resultMetadata` so that * `normalizeSummariesOnRounds` restores it on subsequent turns. */ - private _persistSummaryOnTurn(bgResult: { summary: string; toolCallRoundId: string }, promptContext: IBuildPromptContext): void { - const chatResult = promptContext.conversation?.getLatestTurn().responseChatResult; + private _persistSummaryOnTurn(bgResult: IBackgroundSummarizationResult, promptContext: IBuildPromptContext, contextLengthBefore?: number): void { + const turn = promptContext.conversation?.getLatestTurn(); + const chatResult = turn?.responseChatResult; if (chatResult) { const metadata = (chatResult.metadata ?? {}) as Record; - metadata['summary'] = { toolCallRoundId: bgResult.toolCallRoundId, text: bgResult.summary }; + const existingSummaries = (metadata['summaries'] as unknown[] ?? []); + existingSummaries.push({ toolCallRoundId: bgResult.toolCallRoundId, text: bgResult.summary }); + metadata['summaries'] = existingSummaries; (chatResult as { metadata: unknown }).metadata = metadata; } + const usage = bgResult.promptTokens !== undefined && bgResult.outputTokens !== undefined + ? { prompt_tokens: bgResult.promptTokens, completion_tokens: bgResult.outputTokens, total_tokens: bgResult.promptTokens + bgResult.outputTokens, ...(bgResult.promptCacheTokens !== undefined ? { prompt_tokens_details: { cached_tokens: bgResult.promptCacheTokens } } : {}) } + : undefined; + turn?.setMetadata(new SummarizedConversationHistoryMetadata( + bgResult.toolCallRoundId, + bgResult.summary, + { + usage, + model: bgResult.model, + summarizationMode: bgResult.summarizationMode, + numRounds: bgResult.numRounds, + numRoundsSinceLastSummarization: bgResult.numRoundsSinceLastSummarization, + durationMs: bgResult.durationMs, + source: 'background', + outcome: 'success', + contextLengthBefore, + }, + )); } private _sendBackgroundCompactionTelemetry( diff --git a/extensions/copilot/src/extension/prompt/common/conversation.ts b/extensions/copilot/src/extension/prompt/common/conversation.ts index 2f0a4cf414c..6a9a927c98a 100644 --- a/extensions/copilot/src/extension/prompt/common/conversation.ts +++ b/extensions/copilot/src/extension/prompt/common/conversation.ts @@ -198,19 +198,22 @@ export class Turn { */ export function normalizeSummariesOnRounds(turns: readonly Turn[]): void { for (const [idx, turn] of turns.entries()) { - const turnSummary = turn.resultMetadata?.summary; - if (turnSummary) { - const roundInTurn = turn.rounds.find(round => round.id === turnSummary.toolCallRoundId); - if (roundInTurn) { - roundInTurn.summary = turnSummary.text; - } else { - const previousTurns = turns.slice(0, idx); - for (const turn of previousTurns) { - const roundInPreviousTurn = turn.rounds.find(round => round.id === turnSummary.toolCallRoundId); - if (roundInPreviousTurn) { - roundInPreviousTurn.summary = turnSummary.text; - break; - } + const turnSummaries = turn.resultMetadata?.summaries ?? (turn.resultMetadata?.summary ? [turn.resultMetadata.summary] : []); + // Each summary supersedes all previous ones, so only the last one matters for restoration + const turnSummary = turnSummaries.at(-1); + if (!turnSummary) { + continue; + } + const roundInTurn = turn.rounds.find(round => round.id === turnSummary.toolCallRoundId); + if (roundInTurn) { + roundInTurn.summary = turnSummary.text; + } else { + const previousTurns = turns.slice(0, idx); + for (const turn of previousTurns) { + const roundInPreviousTurn = turn.rounds.find(round => round.id === turnSummary.toolCallRoundId); + if (roundInPreviousTurn) { + roundInPreviousTurn.summary = turnSummary.text; + break; } } } @@ -362,12 +365,40 @@ export interface IResultMetadata { toolCallRounds?: readonly IToolCallRound[]; toolCallResults?: Record; maxToolCallsExceeded?: boolean; - summary?: { toolCallRoundId: string; text: string }; - /** The actual model used to generate the response, which may differ from the requested model (e.g., when 'auto' resolves to a specific model) */ + /** + * @deprecated Use `summaries` instead. Kept for backward compatibility with + * persisted messages that were saved before `summaries` was introduced. + * `normalizeSummariesOnRounds` falls back to this field when `summaries` is absent. + * Safe to remove once all persisted conversations have migrated. + */ + summary?: { + toolCallRoundId: string; + text: string; + source?: 'foreground' | 'background'; + outcome?: string; + model?: string; + summarizationMode?: string; + durationMs?: number; + contextLengthBefore?: number; + numRounds?: number; + numRoundsSinceLastSummarization?: number; + usage?: { prompt_tokens: number; completion_tokens: number; prompt_tokens_details?: { cached_tokens?: number } }; + }; + summaries?: readonly { + toolCallRoundId: string; + text: string; + source?: 'foreground' | 'background'; + outcome?: string; + model?: string; + summarizationMode?: string; + durationMs?: number; + contextLengthBefore?: number; + numRounds?: number; + numRoundsSinceLastSummarization?: number; + usage?: { prompt_tokens: number; completion_tokens: number; prompt_tokens_details?: { cached_tokens?: number } }; + }[]; resolvedModel?: string; - /** Prompt tokens from the language model (e.g., Anthropic Messages API) */ promptTokens?: number; - /** Output tokens from the language model (e.g., Anthropic Messages API) */ outputTokens?: number; } diff --git a/extensions/copilot/src/extension/prompt/node/defaultIntentRequestHandler.ts b/extensions/copilot/src/extension/prompt/node/defaultIntentRequestHandler.ts index 920c330885b..ef975c4d47c 100644 --- a/extensions/copilot/src/extension/prompt/node/defaultIntentRequestHandler.ts +++ b/extensions/copilot/src/extension/prompt/node/defaultIntentRequestHandler.ts @@ -409,17 +409,17 @@ export class DefaultIntentRequestHandler { private resultWithMetadatas(chatResult: ChatResult | undefined): ChatResult | undefined { const codeBlocks = this.turn.getMetadata(CodeBlocksMetadata); - const summarizedConversationHistory = this.turn.getMetadata(SummarizedConversationHistoryMetadata); + const allSummarizedConversationHistory = this.turn.getAllMetadata(SummarizedConversationHistoryMetadata); const renderedUserMessageMetadata = this.turn.getMetadata(RenderedUserMessageMetadata); const globalContextMetadata = this.turn.getMetadata(GlobalContextMessageMetadata); const anthropicTokenUsageMetadata = this.turn.getMetadata(AnthropicTokenUsageMetadata); - return codeBlocks || summarizedConversationHistory || renderedUserMessageMetadata || globalContextMetadata || anthropicTokenUsageMetadata ? + return codeBlocks || allSummarizedConversationHistory?.length || renderedUserMessageMetadata || globalContextMetadata || anthropicTokenUsageMetadata ? { ...chatResult, metadata: { ...chatResult?.metadata, ...codeBlocks, - ...summarizedConversationHistory && { summary: summarizedConversationHistory }, + ...allSummarizedConversationHistory && allSummarizedConversationHistory.length > 0 && { summaries: allSummarizedConversationHistory }, ...renderedUserMessageMetadata, ...globalContextMetadata, ...anthropicTokenUsageMetadata, diff --git a/extensions/copilot/src/extension/prompt/test/node/conversation.spec.ts b/extensions/copilot/src/extension/prompt/test/node/conversation.spec.ts index 18e46c16b0f..950297f2941 100644 --- a/extensions/copilot/src/extension/prompt/test/node/conversation.spec.ts +++ b/extensions/copilot/src/extension/prompt/test/node/conversation.spec.ts @@ -59,6 +59,53 @@ describe('Turn', () => { expect(turn1.rounds[0].summary).to.equal('summary 1'); }); + it('should restore only the last summary from summaries array', () => { + const turn1 = new Turn('1', { type: 'user', message: 'Hello' }); + const turn1Meta: Partial = { + summaries: [ + { text: 'summary 1', toolCallRoundId: 'round1' }, + { text: 'summary 2', toolCallRoundId: 'round2' }, + ], + toolCallRounds: [ + new ToolCallRound('Hello', [genericToolCall], undefined, 'round1'), + new ToolCallRound('Hello', [genericToolCall], undefined, 'round2'), + new ToolCallRound('Hello', [], undefined, 'round3'), + ] + }; + turn1.setResponse(TurnStatus.Success, { type: 'model', message: 'Hi there!' }, undefined, { metadata: turn1Meta }); + normalizeSummariesOnRounds([turn1]); + expect(turn1.rounds[0].summary).to.be.undefined; + expect(turn1.rounds[1].summary).to.equal('summary 2'); + }); + + it('should restore only the last summary across turns', () => { + const turn1 = new Turn('1', { type: 'user', message: 'Hello' }); + const turn1Meta: Partial = { + toolCallRounds: [ + new ToolCallRound('Hello', [genericToolCall], undefined, 'round1'), + new ToolCallRound('Hello', [genericToolCall], undefined, 'round2'), + ] + }; + turn1.setResponse(TurnStatus.Success, { type: 'model', message: 'Hi there!' }, undefined, { metadata: turn1Meta }); + + const turn2 = new Turn('2', { type: 'user', message: 'Hello' }); + const turn2Meta: Partial = { + summaries: [ + { text: 'summary for round1', toolCallRoundId: 'round1' }, + { text: 'summary for round3', toolCallRoundId: 'round3' }, + ], + toolCallRounds: [ + new ToolCallRound('Hello', [genericToolCall], undefined, 'round3'), + new ToolCallRound('Hello', [], undefined, 'round4'), + ] + }; + turn2.setResponse(TurnStatus.Success, { type: 'model', message: 'Hi there!' }, undefined, { metadata: turn2Meta }); + + normalizeSummariesOnRounds([turn1, turn2]); + expect(turn1.rounds[0].summary).to.be.undefined; + expect(turn2.rounds[0].summary).to.equal('summary for round3'); + }); + it('should restore summaries from metadata to previous turns', () => { const turn1 = new Turn('1', { type: 'user', message: 'Hello' }); const turn1Meta: Partial = { diff --git a/extensions/copilot/src/extension/prompts/node/agent/backgroundSummarizer.ts b/extensions/copilot/src/extension/prompts/node/agent/backgroundSummarizer.ts index fd5ef7ae846..66492043a55 100644 --- a/extensions/copilot/src/extension/prompts/node/agent/backgroundSummarizer.ts +++ b/extensions/copilot/src/extension/prompts/node/agent/backgroundSummarizer.ts @@ -29,6 +29,14 @@ export const enum BackgroundSummarizationState { export interface IBackgroundSummarizationResult { readonly summary: string; readonly toolCallRoundId: string; + readonly promptTokens?: number; + readonly promptCacheTokens?: number; + readonly outputTokens?: number; + readonly durationMs?: number; + readonly model?: string; + readonly summarizationMode?: string; + readonly numRounds?: number; + readonly numRoundsSinceLastSummarization?: number; } /** diff --git a/extensions/copilot/src/extension/prompts/node/agent/summarizedConversationHistory.tsx b/extensions/copilot/src/extension/prompts/node/agent/summarizedConversationHistory.tsx index 21d25364ea2..e2221b4f5c9 100644 --- a/extensions/copilot/src/extension/prompts/node/agent/summarizedConversationHistory.tsx +++ b/extensions/copilot/src/extension/prompts/node/agent/summarizedConversationHistory.tsx @@ -341,15 +341,54 @@ class ConversationHistory extends PromptElement { } } +export interface ISummarizedConversationHistoryMetadataOptions { + readonly thinking?: ThinkingData; + readonly usage?: APIUsage; + readonly promptTokenDetails?: readonly ChatResultPromptTokenDetail[]; + readonly model?: string; + readonly summarizationMode?: string; + readonly numRounds?: number; + readonly numRoundsSinceLastSummarization?: number; + readonly durationMs?: number; + readonly source?: 'foreground' | 'background'; + readonly outcome?: string; + readonly contextLengthBefore?: number; +} + export class SummarizedConversationHistoryMetadata extends PromptMetadata { + public readonly toolCallRoundId: string; + public readonly text: string; + public readonly thinking?: ThinkingData; + public readonly usage?: APIUsage; + public readonly promptTokenDetails?: readonly ChatResultPromptTokenDetail[]; + public readonly model?: string; + public readonly summarizationMode?: string; + public readonly numRounds?: number; + public readonly numRoundsSinceLastSummarization?: number; + public readonly durationMs?: number; + public readonly source?: 'foreground' | 'background'; + public readonly outcome?: string; + public readonly contextLengthBefore?: number; + constructor( - public readonly toolCallRoundId: string, - public readonly text: string, - public readonly thinking?: ThinkingData, - public readonly usage?: APIUsage, - public readonly promptTokenDetails?: readonly ChatResultPromptTokenDetail[], + toolCallRoundId: string, + text: string, + options?: ISummarizedConversationHistoryMetadataOptions, ) { super(); + this.toolCallRoundId = toolCallRoundId; + this.text = text; + this.thinking = options?.thinking; + this.usage = options?.usage; + this.promptTokenDetails = options?.promptTokenDetails; + this.model = options?.model; + this.summarizationMode = options?.summarizationMode; + this.numRounds = options?.numRounds; + this.numRoundsSinceLastSummarization = options?.numRoundsSinceLastSummarization; + this.durationMs = options?.durationMs; + this.source = options?.source; + this.outcome = options?.outcome; + this.contextLengthBefore = options?.contextLengthBefore; } } @@ -389,7 +428,16 @@ export class SummarizedConversationHistory extends PromptElement; promptTokenDetails?: readonly ChatResultPromptTokenDetail[]; + model?: string; + summarizationMode?: string; + numRounds?: number; + numRoundsSinceLastSummarization?: number; + durationMs?: number; } class ConversationHistorySummarizer { @@ -451,7 +504,7 @@ class ConversationHistorySummarizer { @IChatHookService private readonly chatHookService: IChatHookService, ) { } - async summarizeHistory(): Promise<{ summary: string; toolCallRoundId: string; thinking?: ThinkingData; usage?: APIUsage; promptTokenDetails?: readonly ChatResultPromptTokenDetail[] }> { + async summarizeHistory(): Promise<{ summary: string; toolCallRoundId: string; thinking?: ThinkingData; usage?: APIUsage; promptTokenDetails?: readonly ChatResultPromptTokenDetail[]; model?: string; summarizationMode?: string; numRounds?: number; numRoundsSinceLastSummarization?: number; durationMs?: number }> { // Execute pre-compact hook before summarization to allow hooks to archive transcripts or perform cleanup await this.executePreCompactHook(); @@ -467,12 +520,18 @@ class ConversationHistorySummarizer { })); const summary = await summaryPromise; + const { numRounds, numRoundsSinceLastSummarization } = this.computeRoundCounts(); return { summary: summary.result.value, toolCallRoundId: propsInfo.summarizedToolCallRoundId, thinking: propsInfo.summarizedThinking, usage: summary.result.usage, promptTokenDetails: summary.promptTokenDetails, + model: summary.model, + summarizationMode: summary.summarizationMode, + numRounds, + numRoundsSinceLastSummarization, + durationMs: summary.durationMs, }; } @@ -608,9 +667,13 @@ class ConversationHistorySummarizer { totalPromptTokens: summaryResponse.type === ChatFetchResponseType.Success ? summaryResponse.usage?.prompt_tokens : undefined, }); + const durationMs = stopwatch.elapsed(); return { - result: await this.handleSummarizationResponse(summaryResponse, mode, stopwatch.elapsed()), + result: await this.handleSummarizationResponse(summaryResponse, mode, durationMs), promptTokenDetails, + model: endpoint.model, + summarizationMode: mode, + durationMs, }; } @@ -641,11 +704,7 @@ class ConversationHistorySummarizer { return response; } - /** - * Send telemetry for conversation summarization. - * @param success Whether the summarization was successful - */ - private sendSummarizationTelemetry(outcome: string, requestId: string, model: string, mode: SummaryMode, elapsedTime: number, usage: APIUsage | undefined, detailedOutcome?: string): void { + private computeRoundCounts(): { numRounds: number; numRoundsSinceLastSummarization: number } { const numRoundsInHistory = this.props.promptContext.history .map(turn => turn.rounds.length) .reduce((a, b) => a + b, 0); @@ -666,6 +725,21 @@ class ConversationHistorySummarizer { } } } + return { numRounds, numRoundsSinceLastSummarization }; + } + + /** + * Send telemetry for conversation summarization. + * @param outcome High-level result of the summarization (for example, 'success', 'too_large', or the ChatFetchResponseType value) + * @param requestId Unique identifier of the underlying chat request used for summarization + * @param model Identifier of the language model used to generate the summary + * @param mode Summarization mode indicating how the conversation was summarized + * @param elapsedTime Total time in milliseconds taken for the summarization request + * @param usage Token usage information for the summarization request, if available + * @param detailedOutcome Optional detailed reason for non-success outcomes (for example, error or cancellation reason) + */ + private sendSummarizationTelemetry(outcome: string, requestId: string, model: string, mode: SummaryMode, elapsedTime: number, usage: APIUsage | undefined, detailedOutcome?: string): void { + const { numRounds, numRoundsSinceLastSummarization } = this.computeRoundCounts(); const turnIndex = this.props.promptContext.history.length; const curTurnRoundIndex = this.props.promptContext.toolCallRounds?.length ?? 0;