Microsoft/vscode
1
0
Fork 0
mirror of https://github.com/microsoft/vscode.git synced 2026-04-02 00:09:30 +01:00
Code Issues Packages Projects Releases Wiki Activity

Link to browser editors in tool calls (#306909)

Browse Source 
* Link to browser editors in tool calls

* a11y, feedback
This commit is contained in:
Kyle Cutler
2026-03-31 14:37:23 -07:00
committed by GitHub
parent 6c3a002c90
commit 3111b55ca4
13 changed files with 81 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/vs/workbench/contrib/browserView/common/browserEditorInput.ts
View File

@@ -46,7 +46,7 @@ export interface IBrowserEditorInputData extends IBrowserEditorViewState {
export class BrowserEditorInput extends EditorInput {
static readonly ID = 'workbench.editorinputs.browser';
static readonly EDITOR_ID = 'workbench.editor.browser';
private static readonly DEFAULT_LABEL = localize('browser.editorLabel', "Browser");
static readonly DEFAULT_LABEL = localize('browser.editorLabel', "Browser");
private readonly _id: string;
private _initialData: IBrowserEditorInputData;

13
src/vs/workbench/contrib/browserView/electron-browser/tools/browserToolHelpers.ts
View File

@@ -3,12 +3,25 @@
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import { URI } from '../../../../../base/common/uri.js';
import { BrowserViewUri } from '../../../../../platform/browserView/common/browserViewUri.js';
import { IPlaywrightService } from '../../../../../platform/browserView/common/playwrightService.js';
import { IToolResult } from '../../../chat/common/tools/languageModelToolsService.js';
import { BrowserEditorInput } from '../../common/browserEditorInput.js';
// eslint-disable-next-line local/code-import-patterns
import type { Page } from 'playwright-core';
/**
* Creates a markdown link to a browser page.
*/
export function createBrowserPageLink(pageId: string | URI): string {
if (typeof pageId === 'string') {
pageId = BrowserViewUri.forId(pageId);
}
return `[${BrowserEditorInput.DEFAULT_LABEL}](${pageId.toString()}?vscodeLinkType=browser)`;
}
/**
* Shared helper for running a Playwright function against a page and returning its result.
*/

8
src/vs/workbench/contrib/browserView/electron-browser/tools/clickBrowserTool.ts
View File

@@ -5,10 +5,11 @@
import type { CancellationToken } from '../../../../../base/common/cancellation.js';
import { Codicon } from '../../../../../base/common/codicons.js';
import { MarkdownString } from '../../../../../base/common/htmlContent.js';
import { localize } from '../../../../../nls.js';
import { IPlaywrightService } from '../../../../../platform/browserView/common/playwrightService.js';
import { ToolDataSource, type CountTokensCallback, type IPreparedToolInvocation, type IToolData, type IToolImpl, type IToolInvocation, type IToolInvocationPreparationContext, type IToolResult, type ToolProgress } from '../../../chat/common/tools/languageModelToolsService.js';
import { errorResult, playwrightInvoke } from './browserToolHelpers.js';
import { createBrowserPageLink, errorResult, playwrightInvoke } from './browserToolHelpers.js';
import { OpenPageToolId } from './openBrowserTool.js';
export const ClickBrowserToolData: IToolData = {
@@ -62,9 +63,10 @@ export class ClickBrowserTool implements IToolImpl {
) { }
async prepareToolInvocation(_context: IToolInvocationPreparationContext, _token: CancellationToken): Promise<IPreparedToolInvocation | undefined> {
const link = createBrowserPageLink(_context.parameters.pageId);
return {
invocationMessage: localize('browser.click.invocation', "Clicking element in browser"),
pastTenseMessage: localize('browser.click.past', "Clicked element in browser"),
invocationMessage: new MarkdownString(localize('browser.click.invocation', "Clicking element in {0}", link)),
pastTenseMessage: new MarkdownString(localize('browser.click.past', "Clicked element in {0}", link)),
};
}

8
src/vs/workbench/contrib/browserView/electron-browser/tools/dragElementTool.ts
View File

@@ -5,10 +5,11 @@
import type { CancellationToken } from '../../../../../base/common/cancellation.js';
import { Codicon } from '../../../../../base/common/codicons.js';
import { MarkdownString } from '../../../../../base/common/htmlContent.js';
import { localize } from '../../../../../nls.js';
import { IPlaywrightService } from '../../../../../platform/browserView/common/playwrightService.js';
import { ToolDataSource, type CountTokensCallback, type IPreparedToolInvocation, type IToolData, type IToolImpl, type IToolInvocation, type IToolInvocationPreparationContext, type IToolResult, type ToolProgress } from '../../../chat/common/tools/languageModelToolsService.js';
import { errorResult, playwrightInvoke } from './browserToolHelpers.js';
import { createBrowserPageLink, errorResult, playwrightInvoke } from './browserToolHelpers.js';
import { OpenPageToolId } from './openBrowserTool.js';
export const DragElementToolData: IToolData = {
@@ -61,9 +62,10 @@ export class DragElementTool implements IToolImpl {
) { }
async prepareToolInvocation(_context: IToolInvocationPreparationContext, _token: CancellationToken): Promise<IPreparedToolInvocation | undefined> {
const link = createBrowserPageLink(_context.parameters.pageId);
return {
invocationMessage: localize('browser.drag.invocation', "Dragging element in browser"),
pastTenseMessage: localize('browser.drag.past', "Dragged element in browser"),
invocationMessage: new MarkdownString(localize('browser.drag.invocation', "Dragging element in {0}", link)),
pastTenseMessage: new MarkdownString(localize('browser.drag.past', "Dragged element in {0}", link)),
};
}

8
src/vs/workbench/contrib/browserView/electron-browser/tools/handleDialogBrowserTool.ts
View File

@@ -5,10 +5,11 @@
import type { CancellationToken } from '../../../../../base/common/cancellation.js';
import { Codicon } from '../../../../../base/common/codicons.js';
import { MarkdownString } from '../../../../../base/common/htmlContent.js';
import { localize } from '../../../../../nls.js';
import { IPlaywrightService } from '../../../../../platform/browserView/common/playwrightService.js';
import { ToolDataSource, type CountTokensCallback, type IPreparedToolInvocation, type IToolData, type IToolImpl, type IToolInvocation, type IToolInvocationPreparationContext, type IToolResult, type ToolProgress } from '../../../chat/common/tools/languageModelToolsService.js';
import { errorResult } from './browserToolHelpers.js';
import { createBrowserPageLink, errorResult } from './browserToolHelpers.js';
import { OpenPageToolId } from './openBrowserTool.js';
export const HandleDialogBrowserToolData: IToolData = {
@@ -57,9 +58,10 @@ export class HandleDialogBrowserTool implements IToolImpl {
) { }
async prepareToolInvocation(_context: IToolInvocationPreparationContext, _token: CancellationToken): Promise<IPreparedToolInvocation | undefined> {
const link = createBrowserPageLink(_context.parameters.pageId);
return {
invocationMessage: localize('browser.handleDialog.invocation', "Handling browser dialog"),
pastTenseMessage: localize('browser.handleDialog.past', "Handled browser dialog"),
invocationMessage: new MarkdownString(localize('browser.handleDialog.invocation', "Handling dialog in {0}", link)),
pastTenseMessage: new MarkdownString(localize('browser.handleDialog.past', "Handled dialog in {0}", link)),
};
}

8
src/vs/workbench/contrib/browserView/electron-browser/tools/hoverElementTool.ts
View File

@@ -5,10 +5,11 @@
import type { CancellationToken } from '../../../../../base/common/cancellation.js';
import { Codicon } from '../../../../../base/common/codicons.js';
import { MarkdownString } from '../../../../../base/common/htmlContent.js';
import { localize } from '../../../../../nls.js';
import { IPlaywrightService } from '../../../../../platform/browserView/common/playwrightService.js';
import { ToolDataSource, type CountTokensCallback, type IPreparedToolInvocation, type IToolData, type IToolImpl, type IToolInvocation, type IToolInvocationPreparationContext, type IToolResult, type ToolProgress } from '../../../chat/common/tools/languageModelToolsService.js';
import { errorResult, playwrightInvoke } from './browserToolHelpers.js';
import { createBrowserPageLink, errorResult, playwrightInvoke } from './browserToolHelpers.js';
import { OpenPageToolId } from './openBrowserTool.js';
export const HoverElementToolData: IToolData = {
@@ -51,9 +52,10 @@ export class HoverElementTool implements IToolImpl {
) { }
async prepareToolInvocation(_context: IToolInvocationPreparationContext, _token: CancellationToken): Promise<IPreparedToolInvocation | undefined> {
const link = createBrowserPageLink(_context.parameters.pageId);
return {
invocationMessage: localize('browser.hover.invocation', "Hovering over element in browser"),
pastTenseMessage: localize('browser.hover.past', "Hovered over element in browser"),
invocationMessage: new MarkdownString(localize('browser.hover.invocation', "Hovering over element in {0}", link)),
pastTenseMessage: new MarkdownString(localize('browser.hover.past', "Hovered over element in {0}", link)),
};
}

23
src/vs/workbench/contrib/browserView/electron-browser/tools/navigateBrowserTool.ts
View File

@@ -5,10 +5,11 @@
import type { CancellationToken } from '../../../../../base/common/cancellation.js';
import { Codicon } from '../../../../../base/common/codicons.js';
import { MarkdownString } from '../../../../../base/common/htmlContent.js';
import { localize } from '../../../../../nls.js';
import { IPlaywrightService } from '../../../../../platform/browserView/common/playwrightService.js';
import { ToolDataSource, type CountTokensCallback, type IPreparedToolInvocation, type IToolData, type IToolImpl, type IToolInvocation, type IToolInvocationPreparationContext, type IToolResult, type ToolProgress } from '../../../chat/common/tools/languageModelToolsService.js';
import { errorResult, playwrightInvoke } from './browserToolHelpers.js';
import { createBrowserPageLink, errorResult, playwrightInvoke } from './browserToolHelpers.js';
import { OpenPageToolId } from './openBrowserTool.js';
export const NavigateBrowserToolData: IToolData = {
@@ -53,21 +54,25 @@ export class NavigateBrowserTool implements IToolImpl {
async prepareToolInvocation(context: IToolInvocationPreparationContext, _token: CancellationToken): Promise<IPreparedToolInvocation | undefined> {
const params = context.parameters as INavigateBrowserToolParams;
const link = createBrowserPageLink(params.pageId);
switch (params.type) {
case 'reload':
return {
invocationMessage: localize('browser.reload.invocation', "Reloading browser page"),
pastTenseMessage: localize('browser.reload.past', "Reloaded browser page"),
invocationMessage: new MarkdownString(localize('browser.reload.invocation', "Reloading {0}", link)),
pastTenseMessage: new MarkdownString(localize('browser.reload.past', "Reloaded {0}", link)),
icon: Codicon.refresh,
};
case 'back':
return {
invocationMessage: localize('browser.goBack.invocation', "Going back in browser history"),
pastTenseMessage: localize('browser.goBack.past', "Went back in browser history"),
invocationMessage: new MarkdownString(localize('browser.goBack.invocation', "Navigating {0} backward", link)),
pastTenseMessage: new MarkdownString(localize('browser.goBack.past', "Navigated {0} backward", link)),
icon: Codicon.arrowLeft,
};
case 'forward':
return {
invocationMessage: localize('browser.goForward.invocation', "Going forward in browser history"),
pastTenseMessage: localize('browser.goForward.past', "Went forward in browser history"),
invocationMessage: new MarkdownString(localize('browser.goForward.invocation', "Navigating {0} forward", link)),
pastTenseMessage: new MarkdownString(localize('browser.goForward.past', "Navigated {0} forward", link)),
icon: Codicon.arrowRight,
};
default: {
if (!params.url) {
@@ -79,8 +84,8 @@ export class NavigateBrowserTool implements IToolImpl {
}
return {
invocationMessage: localize('browser.navigate.invocation', "Navigating browser to {0}", parsed.href),
pastTenseMessage: localize('browser.navigate.past', "Navigated browser to {0}", parsed.href),
invocationMessage: new MarkdownString(localize('browser.navigate.invocation', "Navigating {0} to {1}", link, parsed.href)),
pastTenseMessage: new MarkdownString(localize('browser.navigate.past', "Navigated {0} to {1}", link, parsed.href)),
confirmationMessages: {
title: localize('browser.navigate.confirmTitle', 'Navigate Browser?'),
message: localize('browser.navigate.confirmMessage', 'This will navigate the browser to {0} and allow the agent to access its contents.', parsed.href),

8
src/vs/workbench/contrib/browserView/electron-browser/tools/openBrowserTool.ts
View File

@@ -5,9 +5,11 @@
import type { CancellationToken } from '../../../../../base/common/cancellation.js';
import { Codicon } from '../../../../../base/common/codicons.js';
import { MarkdownString } from '../../../../../base/common/htmlContent.js';
import { localize } from '../../../../../nls.js';
import { IPlaywrightService } from '../../../../../platform/browserView/common/playwrightService.js';
import { ToolDataSource, type CountTokensCallback, type IPreparedToolInvocation, type IToolData, type IToolImpl, type IToolInvocation, type IToolInvocationPreparationContext, type IToolResult, type ToolProgress } from '../../../chat/common/tools/languageModelToolsService.js';
import { createBrowserPageLink } from './browserToolHelpers.js';
export const OpenPageToolId = 'open_browser_page';
@@ -69,8 +71,12 @@ export class OpenBrowserTool implements IToolImpl {
return {
content: [{
kind: 'text',
value: `Page ID: ${pageId}\n${summary}`,
value: `Page ID: ${pageId}\n\nSummary:\n`,
}, {
kind: 'text',
value: summary,
}],
toolResultMessage: new MarkdownString(localize('browser.open.result', "Opened {0}", createBrowserPageLink(pageId)))
};
}
}

5
src/vs/workbench/contrib/browserView/electron-browser/tools/openBrowserToolNonAgentic.ts
View File

@@ -12,6 +12,8 @@ import { ITelemetryService } from '../../../../../platform/telemetry/common/tele
import { IEditorService } from '../../../../services/editor/common/editorService.js';
import { type CountTokensCallback, type IPreparedToolInvocation, type IToolData, type IToolImpl, type IToolInvocation, type IToolInvocationPreparationContext, type IToolResult, type ToolProgress } from '../../../chat/common/tools/languageModelToolsService.js';
import { IOpenBrowserToolParams, OpenBrowserToolData } from './openBrowserTool.js';
import { MarkdownString } from '../../../../../base/common/htmlContent.js';
import { createBrowserPageLink } from './browserToolHelpers.js';
export const OpenBrowserToolNonAgenticData: IToolData = {
...OpenBrowserToolData,
@@ -58,7 +60,8 @@ export class OpenBrowserToolNonAgentic implements IToolImpl {
content: [{
kind: 'text',
value: `Page opened successfully. Note that you do not have access to the page contents unless the user enables agentic tools via the \`workbench.browser.enableChatTools\` setting.`,
}]
}],
toolResultMessage: new MarkdownString(localize('browser.open.nonAgentic.result', "Opened {0}", createBrowserPageLink(browserUri)))
};
}
}

5
src/vs/workbench/contrib/browserView/electron-browser/tools/readBrowserTool.ts
View File

@@ -7,10 +7,9 @@ import type { CancellationToken } from '../../../../../base/common/cancellation.
import { Codicon } from '../../../../../base/common/codicons.js';
import { MarkdownString } from '../../../../../base/common/htmlContent.js';
import { localize } from '../../../../../nls.js';
import { BrowserViewUri } from '../../../../../platform/browserView/common/browserViewUri.js';
import { IPlaywrightService } from '../../../../../platform/browserView/common/playwrightService.js';
import { ToolDataSource, type CountTokensCallback, type IPreparedToolInvocation, type IToolData, type IToolImpl, type IToolInvocation, type IToolInvocationPreparationContext, type IToolResult, type ToolProgress } from '../../../chat/common/tools/languageModelToolsService.js';
import { errorResult } from './browserToolHelpers.js';
import { createBrowserPageLink, errorResult } from './browserToolHelpers.js';
import { OpenPageToolId } from './openBrowserTool.js';
export const ReadBrowserToolData: IToolData = {
@@ -43,7 +42,7 @@ export class ReadBrowserTool implements IToolImpl {
) { }
async prepareToolInvocation(_context: IToolInvocationPreparationContext, _token: CancellationToken): Promise<IPreparedToolInvocation | undefined> {
const link = `[browser page](${BrowserViewUri.forId(_context.parameters.pageId).toString()})`;
const link = createBrowserPageLink(_context.parameters.pageId);
return {
invocationMessage: new MarkdownString(localize('browser.read.invocation', "Reading {0}", link)),
pastTenseMessage: new MarkdownString(localize('browser.read.past', "Read {0}", link)),

12
src/vs/workbench/contrib/browserView/electron-browser/tools/typeBrowserTool.ts
View File

@@ -5,10 +5,11 @@
import type { CancellationToken } from '../../../../../base/common/cancellation.js';
import { Codicon } from '../../../../../base/common/codicons.js';
import { MarkdownString } from '../../../../../base/common/htmlContent.js';
import { localize } from '../../../../../nls.js';
import { IPlaywrightService } from '../../../../../platform/browserView/common/playwrightService.js';
import { ToolDataSource, type CountTokensCallback, type IPreparedToolInvocation, type IToolData, type IToolImpl, type IToolInvocation, type IToolInvocationPreparationContext, type IToolResult, type ToolProgress } from '../../../chat/common/tools/languageModelToolsService.js';
import { errorResult, playwrightInvoke } from './browserToolHelpers.js';
import { createBrowserPageLink, errorResult, playwrightInvoke } from './browserToolHelpers.js';
import { OpenPageToolId } from './openBrowserTool.js';
export const TypeBrowserToolData: IToolData = {
@@ -62,15 +63,16 @@ export class TypeBrowserTool implements IToolImpl {
async prepareToolInvocation(context: IToolInvocationPreparationContext, _token: CancellationToken): Promise<IPreparedToolInvocation | undefined> {
const params = context.parameters as ITypeBrowserToolParams;
const link = createBrowserPageLink(params.pageId);
if (params.key) {
return {
invocationMessage: localize('browser.pressKey.invocation', "Pressing key {0} in browser", params.key),
pastTenseMessage: localize('browser.pressKey.past', "Pressed key {0} in browser", params.key),
invocationMessage: new MarkdownString(localize('browser.pressKey.invocation', "Pressing key `{0}` in {1}", params.key, link)),
pastTenseMessage: new MarkdownString(localize('browser.pressKey.past', "Pressed key `{0}` in {1}", params.key, link)),
};
}
return {
invocationMessage: localize('browser.type.invocation', "Typing text in browser"),
pastTenseMessage: localize('browser.type.past', "Typed text in browser"),
invocationMessage: new MarkdownString(localize('browser.type.invocation', "Typing text in {0}", link)),
pastTenseMessage: new MarkdownString(localize('browser.type.past', "Typed text in {0}", link)),
};
}

3
src/vs/workbench/contrib/chat/browser/widget/chatContentMarkdownRenderer.ts
View File

@@ -14,6 +14,7 @@ import { IConfigurationService } from '../../../../../platform/configuration/com
import { IHoverService } from '../../../../../platform/hover/browser/hover.js';
import { IOpenerService } from '../../../../../platform/opener/common/opener.js';
import product from '../../../../../platform/product/common/product.js';
import { Schemas } from '../../../../../base/common/network.js';
const _remoteImageDisallowed = () => false;
@@ -80,7 +81,7 @@ export class ChatContentMarkdownRenderer implements IMarkdownRenderer {
override: allowedChatMarkdownHtmlTags,
},
...options?.sanitizerConfig,
allowedLinkSchemes: { augment: [product.urlProtocol, 'copilot-skill'] },
allowedLinkSchemes: { augment: [product.urlProtocol, 'copilot-skill', Schemas.vscodeBrowser] },
remoteImageIsAllowed: _remoteImageDisallowed,
}
};

12
src/vs/workbench/contrib/chat/browser/widget/chatContentParts/chatInlineAnchorWidget.ts
View File

@@ -50,6 +50,10 @@ import { IChatMarkdownAnchorService } from './chatMarkdownAnchorService.js';
import { IConfigurationService } from '../../../../../../platform/configuration/common/configuration.js';
import { ChatConfiguration } from '../../../common/constants.js';
import { getMediaMime } from '../../../../../../base/common/mime.js';
import { Schemas } from '../../../../../../base/common/network.js';
import { Codicon } from '../../../../../../base/common/codicons.js';
import { ThemeIcon } from '../../../../../../base/common/themables.js';
import { BrowserEditorInput } from '../../../../browserView/common/browserEditorInput.js';
/**
* Returns the editor ID to use when opening a resource from chat pills (inline anchors), based on the
@@ -153,6 +157,7 @@ export class InlineAnchorWidget extends Disposable {
@IThemeService themeService: IThemeService,
@INotebookDocumentService private readonly notebookDocumentService: INotebookDocumentService,
@IOpenerService private readonly openerService: IOpenerService,
@IEditorService private readonly editorService: IEditorService,
) {
super();
@@ -183,6 +188,7 @@ export class InlineAnchorWidget extends Disposable {
location = this.data;
const filePathLabel = this.metadata?.linkText ?? labelService.getUriBasenameLabel(location.uri);
let defaultIcon: ThemeIcon | undefined;
if (location.range && this.data.kind !== 'symbol') {
const suffix = location.range.startLineNumber === location.range.endLineNumber
@@ -192,12 +198,16 @@ export class InlineAnchorWidget extends Disposable {
iconText = [filePathLabel, dom.$('span.label-suffix', undefined, suffix)];
} else if (location.uri.scheme === 'vscode-notebook-cell' && this.data.kind !== 'symbol') {
iconText = [`${filePathLabel} • cell${this.getCellIndex(location.uri)}`];
} else if (location.uri.scheme === Schemas.vscodeBrowser) {
defaultIcon = Codicon.globe;
const editorName = this.editorService.findEditors(location.uri)[0]?.editor?.getName() ?? BrowserEditorInput.DEFAULT_LABEL;
iconText = [editorName];
} else {
iconText = [filePathLabel];
}
let fileKind = location.uri.path.endsWith('/') ? FileKind.FOLDER : FileKind.FILE;
const recomputeIconClasses = () => getIconClasses(modelService, languageService, location.uri, fileKind, fileKind === FileKind.FOLDER && !themeService.getFileIconTheme().hasFolderIcons ? FolderThemeIcon : undefined);
const recomputeIconClasses = () => getIconClasses(modelService, languageService, location.uri, fileKind, fileKind === FileKind.FOLDER && !themeService.getFileIconTheme().hasFolderIcons ? FolderThemeIcon : defaultIcon);
iconClasses = recomputeIconClasses();