From d02bfbbc5cebbd342d9c5bbc240ffe3edd5f4043 Mon Sep 17 00:00:00 2001 From: Bhavya U Date: Mon, 17 Aug 2026 14:11:41 -0700 Subject: [PATCH] Show Anthropic refusals correctly (#331315) * Fix Anthropic refusal error handling Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Handle refusal edge cases Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Simplify refusal error handling Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Avoid logging refusal explanations Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../extension/intents/node/toolCallingLoop.ts | 1 + .../node/toolCallingLoopAutopilot.spec.ts | 5 ++ .../extension/prompt/node/chatMLFetcher.ts | 7 ++ .../node/defaultIntentRequestHandler.ts | 6 ++ .../src/extension/xtab/node/xtabProvider.ts | 1 + .../src/platform/chat/common/commonTypes.ts | 11 +++ .../src/platform/endpoint/node/messagesApi.ts | 50 ++++++------ .../endpoint/test/node/messagesApi.spec.ts | 77 +++++++++++++++++-- .../src/platform/networking/common/openai.ts | 5 ++ 9 files changed, 129 insertions(+), 34 deletions(-) diff --git a/extensions/copilot/src/extension/intents/node/toolCallingLoop.ts b/extensions/copilot/src/extension/intents/node/toolCallingLoop.ts index ca068398332..9f770c47c25 100644 --- a/extensions/copilot/src/extension/intents/node/toolCallingLoop.ts +++ b/extensions/copilot/src/extension/intents/node/toolCallingLoop.ts @@ -994,6 +994,7 @@ export abstract class ToolCallingLoop { expect(loop.testShouldAutoRetry(mockResponse(ChatFetchResponseType.OffTopic))).toBe(false); }); + it('should not retry on Refusal', () => { + const loop = createLoop('autopilot'); + expect(loop.testShouldAutoRetry(mockResponse(ChatFetchResponseType.Refusal))).toBe(false); + }); + it('should not retry on Success', () => { const loop = createLoop('autoApprove'); expect(loop.testShouldAutoRetry(mockResponse(ChatFetchResponseType.Success))).toBe(false); diff --git a/extensions/copilot/src/extension/prompt/node/chatMLFetcher.ts b/extensions/copilot/src/extension/prompt/node/chatMLFetcher.ts index fe61da79064..73e58be2f1e 100644 --- a/extensions/copilot/src/extension/prompt/node/chatMLFetcher.ts +++ b/extensions/copilot/src/extension/prompt/node/chatMLFetcher.ts @@ -1918,6 +1918,13 @@ export class ChatMLFetcherImpl extends AbstractChatMLFetcher { requestId: requestId, serverRequestId: result.requestId.headerRequestId, }; + case FinishedCompletionReason.Refusal: + return { + type: ChatFetchResponseType.Refusal, + reason: 'Model declined to respond.', + requestId: requestId, + serverRequestId: result.requestId.headerRequestId, + }; case FinishedCompletionReason.Length: return { type: ChatFetchResponseType.Length, diff --git a/extensions/copilot/src/extension/prompt/node/defaultIntentRequestHandler.ts b/extensions/copilot/src/extension/prompt/node/defaultIntentRequestHandler.ts index 0e1b673b291..4793fbb3072 100644 --- a/extensions/copilot/src/extension/prompt/node/defaultIntentRequestHandler.ts +++ b/extensions/copilot/src/extension/prompt/node/defaultIntentRequestHandler.ts @@ -544,6 +544,12 @@ export class DefaultIntentRequestHandler { this.turn.setResponse(TurnStatus.Filtered, undefined, baseModelTelemetry.properties.messageId, chatResult); return chatResult; } + case ChatFetchResponseType.Refusal: { + const errorDetails = await this.getErrorDetails(fetchResult); + const chatResult = { errorDetails, metadata: metadataFragment }; + this.turn.setResponse(TurnStatus.Filtered, undefined, baseModelTelemetry.properties.messageId, chatResult); + return chatResult; + } case ChatFetchResponseType.PromptFiltered: { const errorDetails = await this.getErrorDetails(fetchResult); const chatResult = { errorDetails, metadata: { ...metadataFragment, filterReason: FilterReason.Prompt } }; diff --git a/extensions/copilot/src/extension/xtab/node/xtabProvider.ts b/extensions/copilot/src/extension/xtab/node/xtabProvider.ts index 196594e2419..daf5b42d560 100644 --- a/extensions/copilot/src/extension/xtab/node/xtabProvider.ts +++ b/extensions/copilot/src/extension/xtab/node/xtabProvider.ts @@ -1759,6 +1759,7 @@ export function mapChatFetcherErrorToNoNextEditReason(fetchError: ChatFetchError case ChatFetchResponseType.OffTopic: case ChatFetchResponseType.Filtered: case ChatFetchResponseType.PromptFiltered: + case ChatFetchResponseType.Refusal: case ChatFetchResponseType.Length: case ChatFetchResponseType.RateLimited: case ChatFetchResponseType.QuotaExceeded: diff --git a/extensions/copilot/src/platform/chat/common/commonTypes.ts b/extensions/copilot/src/platform/chat/common/commonTypes.ts index d3a200abfbb..c21c21f4350 100644 --- a/extensions/copilot/src/platform/chat/common/commonTypes.ts +++ b/extensions/copilot/src/platform/chat/common/commonTypes.ts @@ -98,6 +98,7 @@ export enum ChatFetchResponseType { Filtered = 'filtered', FilteredRetry = 'filteredRetry', PromptFiltered = 'promptFiltered', + Refusal = 'refusal', Length = 'length', RateLimited = 'rateLimited', QuotaExceeded = 'quotaExceeded', @@ -142,6 +143,10 @@ export type ChatFetchError = * We requested conversation, but the prompt was filtered by RAI. */ | { type: ChatFetchResponseType.PromptFiltered; reason: string; reasonDetail?: string; category: FilterReason; requestId: string; serverRequestId: string | undefined } + /** + * We requested conversation, but the model declined to answer. + */ + | { type: ChatFetchResponseType.Refusal; reason: string; reasonDetail?: string; requestId: string; serverRequestId: string | undefined } /** * We requested conversation, but the response was too long. */ @@ -452,6 +457,12 @@ function getErrorDetailsFromChatFetchErrorInner(fetchResult: ChatFetchError, cop level: ChatErrorLevel.Info, }; break; + case ChatFetchResponseType.Refusal: + details = { + message: l10n.t(`Sorry, the model declined to complete this request. Please rephrase your prompt.`), + level: ChatErrorLevel.Info, + }; + break; case ChatFetchResponseType.AgentUnauthorized: details = { message: l10n.t(`Sorry, something went wrong.`) }; break; diff --git a/extensions/copilot/src/platform/endpoint/node/messagesApi.ts b/extensions/copilot/src/platform/endpoint/node/messagesApi.ts index 109d8d622e4..a5ba356868c 100644 --- a/extensions/copilot/src/platform/endpoint/node/messagesApi.ts +++ b/extensions/copilot/src/platform/endpoint/node/messagesApi.ts @@ -3,7 +3,7 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { ContentBlockParam, DocumentBlockParam, ImageBlockParam, MessageParam, RedactedThinkingBlockParam, TextBlockParam, ThinkingBlockParam, ToolReferenceBlockParam, ToolResultBlockParam } from '@anthropic-ai/sdk/resources'; +import { ContentBlockParam, DocumentBlockParam, ImageBlockParam, MessageParam, RedactedThinkingBlockParam, RefusalStopDetails, TextBlockParam, ThinkingBlockParam, ToolReferenceBlockParam, ToolResultBlockParam } from '@anthropic-ai/sdk/resources'; import { Raw } from '@vscode/prompt-tsx'; import { Response } from '../../../platform/networking/common/fetcherService'; import { AsyncIterableObject } from '../../../util/vs/base/common/async'; @@ -137,11 +137,7 @@ interface AnthropicStreamEvent { signature?: string; stop_reason?: string; stop_sequence?: string; - stop_details?: { - category?: string; - explanation?: string; - type?: string; - }; + stop_details?: RefusalStopDetails | null; }; copilot_annotations?: { IPCodeCitations?: AnthropicIPCodeCitation[]; @@ -790,7 +786,7 @@ interface AnthropicCompletionState { function mapStopReason(stopReason: string | null | undefined): FinishedCompletionReason { switch (stopReason) { case 'refusal': - return FinishedCompletionReason.ClientDone; + return FinishedCompletionReason.Refusal; case 'max_tokens': case 'model_context_window_exceeded': return FinishedCompletionReason.Length; @@ -889,6 +885,7 @@ type AnthropicNonStreamingResponse = )[]; model: string; stop_reason: string | null; + stop_details?: RefusalStopDetails | null; usage: { input_tokens: number; output_tokens: number; @@ -987,23 +984,9 @@ export async function processNonStreamingResponseFromMessagesEndpoint( } } - // Report text and tool calls to finishedCb so callers that rely on - // the callback (e.g. for OTEL tracing, progress, langModelServer SSE - // forwarding) see the complete response — matching the streaming path. - const delta: IResponseDelta = { - text: textContent, - ...(toolCalls.length > 0 ? { - copilotToolCalls: toolCalls.map(tc => ({ - id: tc.id, - name: tc.name, - arguments: tc.arguments, - })), - } : {}), - }; - await finishCallback(textContent, 0, delta); - if (parsed.stop_reason === 'refusal') { - logService.warn(`[messagesAPI] non-streaming: Refusal received for model ${parsed.model}`); + const category = parsed.stop_details?.category ?? 'unknown'; + logService.warn(`[messagesAPI] non-streaming: Refusal received: category='${category}' for model ${parsed.model}`); /* __GDPR__ "messagesApi.refusal" : { @@ -1018,11 +1001,24 @@ export async function processNonStreamingResponseFromMessagesEndpoint( { requestId, model: parsed.model, - category: 'unknown', + category, } ); } + // There are no incremental deltas here, so callback-only consumers need the whole response. + const delta: IResponseDelta = { + text: textContent, + ...(toolCalls.length > 0 ? { + copilotToolCalls: toolCalls.map(tc => ({ + id: tc.id, + name: tc.name, + arguments: tc.arguments, + })), + } : {}), + }; + await finishCallback(textContent, 0, delta); + const usage = parsed.usage; const completion = buildAnthropicCompletion({ model: parsed.model, @@ -1091,7 +1087,7 @@ export class AnthropicMessagesProcessor { private copilotUsage?: { total_nano_aiu: number }; private contextManagementResponse?: ContextManagementResponse; private stopReason: string | undefined; - private stopDetails?: { category?: string; explanation?: string; type?: string }; + private stopDetails?: RefusalStopDetails; constructor( private readonly telemetryData: TelemetryData, @@ -1292,7 +1288,7 @@ export class AnthropicMessagesProcessor { if (chunk.context_management) { this.contextManagementResponse = chunk.context_management; // Report context management via delta so it gets logged to request logger - return onProgress({ + onProgress({ text: '', contextManagement: chunk.context_management }); @@ -1404,5 +1400,3 @@ export class AnthropicMessagesProcessor { } } } - - diff --git a/extensions/copilot/src/platform/endpoint/test/node/messagesApi.spec.ts b/extensions/copilot/src/platform/endpoint/test/node/messagesApi.spec.ts index bf22a589647..5ff085661f7 100644 --- a/extensions/copilot/src/platform/endpoint/test/node/messagesApi.spec.ts +++ b/extensions/copilot/src/platform/endpoint/test/node/messagesApi.spec.ts @@ -23,6 +23,14 @@ import { ConfigKey, IConfigurationService } from '../../../configuration/common/ import { IExperimentationService } from '../../../telemetry/common/nullExperimentationService'; import { InMemoryConfigurationService } from '../../../configuration/test/common/inMemoryConfigurationService'; +class RecordingLogService extends TestLogService { + readonly warnings: string[] = []; + + override warn(message: string): void { + this.warnings.push(message); + } +} + function assertContentArray(content: MessageParam['content']): ContentBlockParam[] { expect(Array.isArray(content)).toBe(true); return content as ContentBlockParam[]; @@ -1972,7 +1980,8 @@ suite('processNonStreamingResponseFromMessagesEndpoint', () => { expect(results[0].message.content).toHaveLength(0); }); - test('maps refusal stop_reason to ClientDone', async () => { + test('maps refusal stop_reason to Refusal, keeping any text the model did produce', async () => { + const explanation = 'API integrators: configure a fallback model.\n'; const response = createNonStreamingResponse({ id: 'msg_refusal', type: 'message', @@ -1980,21 +1989,34 @@ suite('processNonStreamingResponseFromMessagesEndpoint', () => { content: [{ type: 'text', text: 'refused' }], model: 'claude-sonnet-4-20250514', stop_reason: 'refusal', + stop_details: { type: 'refusal', category: 'cyber', explanation }, usage: { input_tokens: 10, output_tokens: 5 }, }); const telemetryData = TelemetryData.createAndMarkAsIssued(); + const deltas: IResponseDelta[] = []; + const logService = new RecordingLogService(); const completions = await processNonStreamingResponseFromMessagesEndpoint( new NullTelemetryService(), - new TestLogService(), + logService, response, - async () => undefined, + async (_text, _idx, delta) => { deltas.push(delta); return undefined; }, telemetryData, ); const results = []; for await (const c of completions) { results.push(c); } - expect(results[0].finishReason).toBe('DONE'); + expect({ + finishReason: results[0].finishReason, + content: results[0].message.content, + copilotErrors: deltas.flatMap(d => d.copilotErrors ?? []), + loggedExplanation: logService.warnings.some(message => message.includes(explanation)), + }).toEqual({ + finishReason: 'refusal', + content: [{ type: Raw.ChatCompletionContentPartKind.Text, text: 'refused' }], + copilotErrors: [], + loggedExplanation: false, + }); }); test('reports tool calls through finishCallback delta', async () => { @@ -2084,13 +2106,13 @@ suite('processResponseFromMessagesEndpoint routing', () => { }); suite('AnthropicMessagesProcessor streaming cache_creation', () => { - function makeProcessor(): AnthropicMessagesProcessor { + function makeProcessor(logService: TestLogService = new TestLogService()): AnthropicMessagesProcessor { return new AnthropicMessagesProcessor( TelemetryData.createAndMarkAsIssued(), 'req-1', 'gh-req-1', '', - new TestLogService(), + logService, new NullTelemetryService(), ); } @@ -2271,4 +2293,47 @@ suite('AnthropicMessagesProcessor streaming cache_creation', () => { expect(completion!.usage?.completion_tokens).toBe(2024); expect(completion!.usage?.completion_tokens_details?.reasoning_tokens).toBe(639); }); + + test('refusal stop_reason maps to Refusal even when it arrives alongside context management', () => { + const logService = new RecordingLogService(); + const processor = makeProcessor(logService); + const deltas: IResponseDelta[] = []; + const capture: FinishedCallback = async (_text, _idx, delta) => { deltas.push(delta); return undefined; }; + const explanation = 'API integrators: configure a fallback model.\n'; + + processor.push({ + type: 'message_start', + message: { + id: 'msg_refusal_stream', + type: 'message', + role: 'assistant', + content: [], + model: 'claude-sonnet-4-20250514', + stop_reason: null, + stop_sequence: null, + usage: { input_tokens: 5, output_tokens: 0 }, + }, + }, capture); + + processor.push({ + type: 'message_delta', + delta: { type: 'message_delta', stop_reason: 'refusal', stop_details: { type: 'refusal', category: 'cyber', explanation } }, + usage: { output_tokens: 0, input_tokens: 5 }, + context_management: { applied_edits: [] }, + }, capture); + + const completion = processor.push({ type: 'message_stop' }, capture); + + expect({ + finishReason: completion!.finishReason, + contextManagement: deltas.find(d => d.contextManagement)?.contextManagement, + copilotErrors: deltas.flatMap(d => d.copilotErrors ?? []), + loggedExplanation: logService.warnings.some(message => message.includes(explanation)), + }).toEqual({ + finishReason: 'refusal', + contextManagement: { applied_edits: [] }, + copilotErrors: [], + loggedExplanation: false, + }); + }); }); diff --git a/extensions/copilot/src/platform/networking/common/openai.ts b/extensions/copilot/src/platform/networking/common/openai.ts index 33faf490319..ccebb5cd499 100644 --- a/extensions/copilot/src/platform/networking/common/openai.ts +++ b/extensions/copilot/src/platform/networking/common/openai.ts @@ -249,6 +249,11 @@ export enum FinishedCompletionReason { * Reason generated by the server. See https://platform.openai.com/docs/guides/gpt/chat-completions-api */ ContentFilter = 'content_filter', + /** + * Reason generated by the server. The model itself declined, as opposed to {@link ContentFilter} + * where a separate system blocked the response. + */ + Refusal = 'refusal', /** * Reason generated by the server (CAPI). Happens when the stream cannot be completed and the server must terminate the response. */