From cbc648ad896620e76b1bc1668df365d9316a9d73 Mon Sep 17 00:00:00 2001 From: Peng Lyu Date: Thu, 19 Mar 2026 19:15:02 -0700 Subject: [PATCH] Support rendering image pills when thinking parts are collapsed. (#303363) * Support rendering image pills when thinking parts are collapsed. * :lipstick: --- .../chatResourceGroupWidget.ts | 5 ++ .../chatThinkingContentPart.ts | 41 +++++++++ .../chatThinkingExternalResourcesWidget.ts | 88 +++++++++++++++++++ .../media/chatThinkingContent.css | 5 ++ .../chatThinkingContentPart.test.ts | 63 +++++++++++++ 5 files changed, 202 insertions(+) create mode 100644 src/vs/workbench/contrib/chat/browser/widget/chatContentParts/chatThinkingExternalResourcesWidget.ts diff --git a/src/vs/workbench/contrib/chat/browser/widget/chatContentParts/chatResourceGroupWidget.ts b/src/vs/workbench/contrib/chat/browser/widget/chatContentParts/chatResourceGroupWidget.ts index dcd85778a0e..cfa95551f1a 100644 --- a/src/vs/workbench/contrib/chat/browser/widget/chatContentParts/chatResourceGroupWidget.ts +++ b/src/vs/workbench/contrib/chat/browser/widget/chatContentParts/chatResourceGroupWidget.ts @@ -7,6 +7,7 @@ import * as dom from '../../../../../../base/browser/dom.js'; import { disposableTimeout } from '../../../../../../base/common/async.js'; import { decodeBase64 } from '../../../../../../base/common/buffer.js'; import { Codicon } from '../../../../../../base/common/codicons.js'; +import { Emitter } from '../../../../../../base/common/event.js'; import { Disposable } from '../../../../../../base/common/lifecycle.js'; import { basename, joinPath } from '../../../../../../base/common/resources.js'; import { URI } from '../../../../../../base/common/uri.js'; @@ -47,6 +48,8 @@ const IMAGE_DECODE_DELAY_MS = 100; */ export class ChatResourceGroupWidget extends Disposable { public readonly domNode: HTMLElement; + private readonly _onDidChangeHeight = this._register(new Emitter()); + public readonly onDidChangeHeight = this._onDidChangeHeight.event; constructor( parts: IChatCollapsibleIODataPart[], @@ -124,6 +127,7 @@ export class ChatResourceGroupWidget extends Disposable { }; itemsContainer.appendChild(attachments.domNode!); + this._onDidChangeHeight.fire(); const toolbar = this._register(this._instantiationService.createInstance(MenuWorkbenchToolBar, actionsContainer, MenuId.ChatToolOutputResourceToolbar, { menuOptions: { @@ -146,6 +150,7 @@ export class ChatResourceGroupWidget extends Disposable { // Update attachments in place attachments.updateVariables(entries); + this._onDidChangeHeight.fire(); }, IMAGE_DECODE_DELAY_MS)); } } diff --git a/src/vs/workbench/contrib/chat/browser/widget/chatContentParts/chatThinkingContentPart.ts b/src/vs/workbench/contrib/chat/browser/widget/chatContentParts/chatThinkingContentPart.ts index 67e0bbf0ce1..00b9f678dab 100644 --- a/src/vs/workbench/contrib/chat/browser/widget/chatContentParts/chatThinkingContentPart.ts +++ b/src/vs/workbench/contrib/chat/browser/widget/chatContentParts/chatThinkingContentPart.ts @@ -34,6 +34,9 @@ import { IChatMarkdownAnchorService } from './chatMarkdownAnchorService.js'; import { ChatMessageRole, ILanguageModelsService } from '../../../common/languageModels.js'; import './media/chatThinkingContent.css'; import { IHoverService } from '../../../../../../platform/hover/browser/hover.js'; +import { extractImagesFromToolInvocationOutputDetails } from '../../../common/chatImageExtraction.js'; +import { IChatCollapsibleIODataPart } from './chatToolInputOutputContentPart.js'; +import { ChatThinkingExternalResourceWidget } from './chatThinkingExternalResourcesWidget.js'; function extractTextFromPart(content: IChatThinkingPart): string { @@ -233,6 +236,7 @@ export class ChatThinkingContentPart extends ChatCollapsibleContentPart implemen private lastKnownScrollTop: number = 0; private titleShimmerSpan: HTMLElement | undefined; private titleDetailContainer: HTMLElement | undefined; + private readonly _externalResourceWidget: ChatThinkingExternalResourceWidget; private readonly _titleDetailRendered = this._register(new MutableDisposable()); private getRandomWorkingMessage(category: WorkingMessageCategory = WorkingMessageCategory.Tool): string { @@ -313,6 +317,10 @@ export class ChatThinkingContentPart extends ChatCollapsibleContentPart implemen const node = this.domNode; node.classList.add('chat-thinking-box'); + this._externalResourceWidget = this._register(this.instantiationService.createInstance(ChatThinkingExternalResourceWidget)); + this._register(this._externalResourceWidget.onDidChangeHeight(() => this._onDidChangeHeight.fire())); + node.appendChild(this._externalResourceWidget.domNode); + if (!this.streamingCompleted && !this.element.isComplete) { if (!this.fixedScrollingMode) { node.classList.add('chat-thinking-active'); @@ -374,6 +382,8 @@ export class ChatThinkingContentPart extends ChatCollapsibleContentPart implemen return; } + this._externalResourceWidget.setCollapsed(!isExpanded); + // Fire when expanded/collapsed this._onDidChangeHeight.fire(); })); @@ -1232,6 +1242,8 @@ ${this.hookCount > 0 ? `EXAMPLES WITH BLOCKED CONTENT (from hooks): } this.toolLabelsByCallId.delete(toolCallId); + this._externalResourceWidget.removeToolInvocation(toolCallId); + this.updateDropdownClickability(); this._onDidChangeHeight.fire(); } @@ -1263,6 +1275,7 @@ ${this.hookCount > 0 ? `EXAMPLES WITH BLOCKED CONTENT (from hooks): // Use the tracked displayed label (which may differ from invocationMessage // for streaming edit tools that show "Editing files") const toolCallId = removedItem.toolInvocationOrMarkdown.toolCallId; + this._externalResourceWidget.removeToolInvocation(toolCallId); const label = this.toolLabelsByCallId.get(toolCallId); if (label) { const titleIndex = this.extractedTitles.indexOf(label); @@ -1356,6 +1369,7 @@ ${this.hookCount > 0 ? `EXAMPLES WITH BLOCKED CONTENT (from hooks): this.extractedTitles.splice(titleIndex, 1); } this.toolLabelsByCallId.delete(toolCallId); + this._externalResourceWidget.removeToolInvocation(toolCallId); this.updateDropdownClickability(); this._onDidChangeHeight.fire(); } @@ -1402,6 +1416,11 @@ ${this.hookCount > 0 ? `EXAMPLES WITH BLOCKED CONTENT (from hooks): const toolCallId = toolInvocationOrMarkdown.toolCallId; this.toolLabelsByCallId.set(toolCallId, toolCallLabel); + // Render external image pills for serialized (already-completed) tool invocations + if (toolInvocationOrMarkdown.kind === 'toolInvocationSerialized') { + this.updateExternalResourceParts(toolInvocationOrMarkdown); + } + // track state for live/still streaming tools, excluding serialized tools if (toolInvocationOrMarkdown.kind === 'toolInvocation') { let currentToolLabel = toolCallLabel; @@ -1462,6 +1481,12 @@ ${this.hookCount > 0 ? `EXAMPLES WITH BLOCKED CONTENT (from hooks): this.pendingRemovals.push({ toolCallId: toolInvocationOrMarkdown.toolCallId, toolLabel: currentToolLabel }); this.schedulePendingRemovalsFlush(); } + + // Render image pills outside the collapsible area for completed tools + if (currentState.type === IChatToolInvocation.StateKind.Completed) { + this.updateExternalResourceParts(toolInvocationOrMarkdown); + } + isComplete = true; return; } @@ -1526,6 +1551,22 @@ ${this.hookCount > 0 ? `EXAMPLES WITH BLOCKED CONTENT (from hooks): } } + private updateExternalResourceParts(toolInvocation: IChatToolInvocation | IChatToolInvocationSerialized): void { + const extractedImages = extractImagesFromToolInvocationOutputDetails(toolInvocation, this.element.sessionResource); + if (extractedImages.length === 0) { + return; + } + + const parts: IChatCollapsibleIODataPart[] = extractedImages.map(image => ({ + kind: 'data', + value: image.data.buffer, + mimeType: image.mimeType, + uri: image.uri, + })); + + this._externalResourceWidget.setToolInvocationParts(toolInvocation.toolCallId, parts); + } + private appendItemToDOM( content: HTMLElement, toolInvocationId?: string, diff --git a/src/vs/workbench/contrib/chat/browser/widget/chatContentParts/chatThinkingExternalResourcesWidget.ts b/src/vs/workbench/contrib/chat/browser/widget/chatContentParts/chatThinkingExternalResourcesWidget.ts new file mode 100644 index 00000000000..732ff270de5 --- /dev/null +++ b/src/vs/workbench/contrib/chat/browser/widget/chatContentParts/chatThinkingExternalResourcesWidget.ts @@ -0,0 +1,88 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { $, clearNode, hide, show } from '../../../../../../base/browser/dom.js'; +import { Emitter } from '../../../../../../base/common/event.js'; +import { Disposable, IDisposable, MutableDisposable } from '../../../../../../base/common/lifecycle.js'; +import { IInstantiationService } from '../../../../../../platform/instantiation/common/instantiation.js'; +import { ChatResourceGroupWidget } from './chatResourceGroupWidget.js'; +import { IChatCollapsibleIODataPart } from './chatToolInputOutputContentPart.js'; + +export class ChatThinkingExternalResourceWidget extends Disposable { + + public readonly domNode: HTMLElement; + private readonly _onDidChangeHeight = this._register(new Emitter()); + public readonly onDidChangeHeight = this._onDidChangeHeight.event; + + private readonly resourcePartsByToolCallId = new Map(); + private readonly resourceGroupWidget = this._register(new MutableDisposable()); + private readonly resourceGroupWidgetHeightListener = this._register(new MutableDisposable()); + private isCollapsed = true; + + constructor( + @IInstantiationService private readonly instantiationService: IInstantiationService, + ) { + super(); + this.domNode = $('.chat-thinking-external-resources'); + hide(this.domNode); + } + + public setToolInvocationParts(toolCallId: string, parts: IChatCollapsibleIODataPart[]): void { + if (parts.length === 0) { + return; + } + + this.resourcePartsByToolCallId.set(toolCallId, parts); + + this.rebuild(); + } + + public removeToolInvocation(toolCallId: string): void { + if (!this.resourcePartsByToolCallId.delete(toolCallId)) { + return; + } + + this.rebuild(); + } + + public setCollapsed(collapsed: boolean): void { + this.isCollapsed = collapsed; + + if (!this.resourceGroupWidget.value) { + hide(this.domNode); + return; + } + + if (this.isCollapsed) { + show(this.domNode); + } else { + hide(this.domNode); + } + } + + private rebuild(): void { + const allParts: IChatCollapsibleIODataPart[] = []; + for (const parts of this.resourcePartsByToolCallId.values()) { + allParts.push(...parts); + } + + this.resourceGroupWidgetHeightListener.clear(); + this.resourceGroupWidget.clear(); + clearNode(this.domNode); + + if (allParts.length === 0) { + hide(this.domNode); + this._onDidChangeHeight.fire(); + return; + } + + const widget = this.instantiationService.createInstance(ChatResourceGroupWidget, allParts); + this.resourceGroupWidgetHeightListener.value = widget.onDidChangeHeight(() => this._onDidChangeHeight.fire()); + this.resourceGroupWidget.value = widget; + this.domNode.appendChild(widget.domNode); + this.setCollapsed(this.isCollapsed); + this._onDidChangeHeight.fire(); + } +} diff --git a/src/vs/workbench/contrib/chat/browser/widget/chatContentParts/media/chatThinkingContent.css b/src/vs/workbench/contrib/chat/browser/widget/chatContentParts/media/chatThinkingContent.css index 28990fb01f8..19684ae349f 100644 --- a/src/vs/workbench/contrib/chat/browser/widget/chatContentParts/media/chatThinkingContent.css +++ b/src/vs/workbench/contrib/chat/browser/widget/chatContentParts/media/chatThinkingContent.css @@ -13,6 +13,11 @@ position: relative; color: var(--vscode-descriptionForeground); + .chat-thinking-external-resources { + margin-top: 4px; + margin-left: 5px; + } + .chat-used-context { margin: 0px; } diff --git a/src/vs/workbench/contrib/chat/test/browser/widget/chatContentParts/chatThinkingContentPart.test.ts b/src/vs/workbench/contrib/chat/test/browser/widget/chatContentParts/chatThinkingContentPart.test.ts index 1a76f109771..4d58c6c4339 100644 --- a/src/vs/workbench/contrib/chat/test/browser/widget/chatContentParts/chatThinkingContentPart.test.ts +++ b/src/vs/workbench/contrib/chat/test/browser/widget/chatContentParts/chatThinkingContentPart.test.ts @@ -1239,6 +1239,30 @@ suite('ChatThinkingContentPart', () => { } as IChatToolInvocation; } + function createMockSerializedImageToolInvocation(toolId: string, invocationMessage: string, toolCallId: string): IChatToolInvocationSerialized { + return { + kind: 'toolInvocationSerialized', + toolId, + toolCallId, + invocationMessage, + originMessage: undefined, + pastTenseMessage: undefined, + presentation: undefined, + resultDetails: { + output: { + type: 'data', + mimeType: 'image/png', + base64Data: 'AQID' + } + }, + isConfirmed: { type: 0 }, + isComplete: true, + source: ToolDataSource.Internal, + generatedTitle: undefined, + isAttachedToThinking: false, + }; + } + test('should show "Editing files" for streaming edit tools instead of generic display name', () => { const content = createThinkingPart('**Working**'); const context = createMockRenderContext(false); @@ -1364,5 +1388,44 @@ suite('ChatThinkingContentPart', () => { const labelText = button.querySelector('.icon-label')?.textContent ?? button.textContent ?? ''; assert.ok(labelText.includes('Creating newFile.ts'), `Title should contain "Creating newFile.ts" but got "${labelText}"`); }); + + test('should show external resources for serialized image tools when initially collapsed and hide them when expanded', () => { + const content = createThinkingPart('**Working**'); + const context = createMockRenderContext(false); + + const part = store.add(instantiationService.createInstance( + ChatThinkingContentPart, + content, + context, + mockMarkdownRenderer, + false + )); + + mainWindow.document.body.appendChild(part.domNode); + disposables.add(toDisposable(() => part.domNode.remove())); + + const serializedImageTool = createMockSerializedImageToolInvocation( + 'chat_screenshot', 'Captured screenshot', 'image-call-1' + ); + + part.appendItem(() => { + const div = $('div.test-item'); + div.textContent = 'Image tool'; + return { domNode: div }; + }, serializedImageTool.toolId, serializedImageTool); + + const externalResources = part.domNode.querySelector('.chat-thinking-external-resources') as HTMLElement; + assert.ok(externalResources, 'Should render external resources container'); + assert.notStrictEqual(externalResources.style.display, 'none', 'Should show external resources while initially collapsed'); + + const button = part.domNode.querySelector('.monaco-button') as HTMLElement; + assert.ok(button, 'Should have expand button'); + button.click(); + + assert.strictEqual(externalResources.style.display, 'none', 'Should hide external resources when expanded'); + + button.click(); + assert.notStrictEqual(externalResources.style.display, 'none', 'Should show external resources again after collapsing'); + }); }); });