make integrated browser attach to chat feature kb accessible (#300465)

fix #300216

src/vs/platform/browserElements/common/browserElements.ts

@@ -56,6 +56,8 @@ export interface INativeBrowserElementsService {
 
 	getElementData(rect: IRectangle, token: CancellationToken, locator: IBrowserTargetLocator, cancellationId?: number): Promise<IElementData | undefined>;
 
+	getFocusedElementData(rect: IRectangle, token: CancellationToken, locator: IBrowserTargetLocator, cancellationId?: number): Promise<IElementData | undefined>;
+
 	startDebugSession(token: CancellationToken, locator: IBrowserTargetLocator, cancelAndDetachId?: number): Promise<void>;
 
 	startConsoleSession(token: CancellationToken, locator: IBrowserTargetLocator, cancelAndDetachId?: number): Promise<void>;

src/vs/platform/browserElements/electron-main/nativeBrowserElementsMainService.ts

@@ -431,6 +431,121 @@ export class NativeBrowserElementsMainService extends Disposable implements INat
 		};
 	}
 
+	async getFocusedElementData(windowId: number | undefined, rect: IRectangle, _token: CancellationToken, locator: IBrowserTargetLocator, _cancellationId?: number): Promise<IElementData | undefined> {
+		const window = this.windowById(windowId);
+		if (!window?.win) {
+			return undefined;
+		}
+
+		const allWebContents = webContents.getAllWebContents();
+		const simpleBrowserWebview = allWebContents.find(webContent => webContent.id === window.id);
+		if (!simpleBrowserWebview) {
+			return undefined;
+		}
+
+		const debuggers = simpleBrowserWebview.debugger;
+		if (!debuggers.isAttached()) {
+			debuggers.attach();
+		}
+
+		let sessionId: string | undefined;
+		try {
+			const targetId = await this.findWebviewTarget(debuggers, locator);
+			if (!targetId) {
+				return undefined;
+			}
+
+			const attach = await debuggers.sendCommand('Target.attachToTarget', { targetId, flatten: true });
+			sessionId = attach.sessionId;
+			await debuggers.sendCommand('Runtime.enable', {}, sessionId);
+
+			const { result } = await debuggers.sendCommand('Runtime.evaluate', {
+				expression: `(() => {
+					const el = document.activeElement;
+					if (!el || el.nodeType !== 1) {
+						return undefined;
+					}
+					const r = el.getBoundingClientRect();
+					const attrs = {};
+					for (let i = 0; i < el.attributes.length; i++) {
+						attrs[el.attributes[i].name] = el.attributes[i].value;
+					}
+					const ancestors = [];
+					let n = el;
+					while (n && n.nodeType === 1) {
+						const entry = { tagName: n.tagName.toLowerCase() };
+						if (n.id) {
+							entry.id = n.id;
+						}
+						if (typeof n.className === 'string' && n.className.trim().length > 0) {
+							entry.classNames = n.className.trim().split(/\\s+/).filter(Boolean);
+						}
+						ancestors.unshift(entry);
+						n = n.parentElement;
+					}
+					const css = getComputedStyle(el);
+					const computedStyles = {};
+					for (let i = 0; i < css.length; i++) {
+						const name = css[i];
+						computedStyles[name] = css.getPropertyValue(name);
+					}
+					const text = (el.innerText || '').trim();
+					return {
+						outerHTML: el.outerHTML,
+						computedStyle: '',
+						bounds: { x: r.x, y: r.y, width: r.width, height: r.height },
+						ancestors,
+						attributes: attrs,
+						computedStyles,
+						dimensions: { top: r.top, left: r.left, width: r.width, height: r.height },
+						innerText: text.length > 100 ? text.slice(0, 100) + '\\u2026' : text
+					};
+				})();`,
+				returnByValue: true
+			}, sessionId);
+
+			const focusedData = result?.value as NodeDataResponse | undefined;
+			if (!focusedData) {
+				return undefined;
+			}
+
+			const zoomFactor = simpleBrowserWebview.getZoomFactor();
+			const absoluteBounds = {
+				x: rect.x + focusedData.bounds.x,
+				y: rect.y + focusedData.bounds.y,
+				width: focusedData.bounds.width,
+				height: focusedData.bounds.height
+			};
+
+			const clippedBounds = {
+				x: Math.max(absoluteBounds.x, rect.x),
+				y: Math.max(absoluteBounds.y, rect.y),
+				width: Math.max(0, Math.min(absoluteBounds.x + absoluteBounds.width, rect.x + rect.width) - Math.max(absoluteBounds.x, rect.x)),
+				height: Math.max(0, Math.min(absoluteBounds.y + absoluteBounds.height, rect.y + rect.height) - Math.max(absoluteBounds.y, rect.y))
+			};
+
+			return {
+				outerHTML: focusedData.outerHTML,
+				computedStyle: focusedData.computedStyle,
+				bounds: {
+					x: clippedBounds.x * zoomFactor,
+					y: clippedBounds.y * zoomFactor,
+					width: clippedBounds.width * zoomFactor,
+					height: clippedBounds.height * zoomFactor
+				},
+				ancestors: focusedData.ancestors,
+				attributes: focusedData.attributes,
+				computedStyles: focusedData.computedStyles,
+				dimensions: focusedData.dimensions,
+				innerText: focusedData.innerText,
+			};
+		} finally {
+			if (debuggers.isAttached()) {
+				debuggers.detach();
+			}
+		}
+	}
+
 	async getNodeData(sessionId: string, debuggers: Electron.Debugger, window: BrowserWindow, cancellationId?: number): Promise<NodeDataResponse> {
 		return new Promise((resolve, reject) => {
 			const onMessage = async (event: Electron.Event, method: string, params: { backendNodeId: number }) => {

src/vs/platform/browserView/electron-main/browserView.ts

@@ -644,6 +644,7 @@ export class BrowserView extends Disposable implements ICDPTarget {
 
 		const isArrowKey = keyCode >= KeyCode.LeftArrow && keyCode <= KeyCode.DownArrow;
 		const isNonEditingKey =
+			keyCode === KeyCode.Enter ||
 			keyCode === KeyCode.Escape ||
 			keyCode >= KeyCode.F1 && keyCode <= KeyCode.F24 ||
 			keyCode >= KeyCode.AudioVolumeMute;

src/vs/workbench/contrib/browserView/electron-browser/browserEditor.ts

@@ -803,51 +803,7 @@ export class BrowserEditor extends EditorPane {
 				throw new Error('Element data not found');
 			}
 
-			const bounds = elementData.bounds;
-			const toAttach: IChatRequestVariableEntry[] = [];
-
-			// Prepare HTML/CSS context
-			const displayName = getDisplayNameFromOuterHTML(elementData.outerHTML);
-			const attachCss = this.configurationService.getValue<boolean>('chat.sendElementsToChat.attachCSS');
-			const value = this.createElementContextValue(elementData, displayName, attachCss);
-
-			toAttach.push({
-				id: 'element-' + Date.now(),
-				name: displayName,
-				fullName: displayName,
-				value: value,
-				modelDescription: attachCss
-					? 'Structured browser element context with HTML path, attributes, and computed styles.'
-					: 'Structured browser element context with HTML path and attributes.',
-				kind: 'element',
-				icon: ThemeIcon.fromId(Codicon.layout.id),
-				ancestors: elementData.ancestors,
-				attributes: elementData.attributes,
-				computedStyles: attachCss ? elementData.computedStyles : undefined,
-				dimensions: elementData.dimensions,
-				innerText: elementData.innerText,
-			});
-
-			// Attach screenshot if enabled
-			const attachImages = this.configurationService.getValue<boolean>('chat.sendElementsToChat.attachImages');
-			if (attachImages && this._model) {
-				const screenshotBuffer = await this._model.captureScreenshot({
-					quality: 90,
-					rect: bounds
-				});
-
-				toAttach.push({
-					id: 'element-screenshot-' + Date.now(),
-					name: 'Element Screenshot',
-					fullName: 'Element Screenshot',
-					kind: 'image',
-					value: screenshotBuffer.buffer
-				});
-			}
-
-			// Attach to chat widget
-			const widget = await this.chatWidgetService.revealWidget() ?? this.chatWidgetService.lastFocusedWidget;
-			widget?.attachmentModel?.addContext(...toAttach);
+			const { attachCss, attachImages } = await this.attachElementDataToChat(elementData);
 
 			type IntegratedBrowserAddElementToChatAddedEvent = {
 				attachCss: boolean;
@@ -992,6 +948,53 @@ export class BrowserEditor extends EditorPane {
 		return sections.join('\n\n');
 	}
 
+	private async attachElementDataToChat(elementData: IElementData): Promise<{ attachCss: boolean; attachImages: boolean }> {
+		const bounds = elementData.bounds;
+		const toAttach: IChatRequestVariableEntry[] = [];
+
+		const displayName = getDisplayNameFromOuterHTML(elementData.outerHTML);
+		const attachCss = this.configurationService.getValue<boolean>('chat.sendElementsToChat.attachCSS');
+		const value = this.createElementContextValue(elementData, displayName, attachCss);
+
+		toAttach.push({
+			id: 'element-' + Date.now(),
+			name: displayName,
+			fullName: displayName,
+			value: value,
+			modelDescription: attachCss
+				? 'Structured browser element context with HTML path, attributes, and computed styles.'
+				: 'Structured browser element context with HTML path and attributes.',
+			kind: 'element',
+			icon: ThemeIcon.fromId(Codicon.layout.id),
+			ancestors: elementData.ancestors,
+			attributes: elementData.attributes,
+			computedStyles: attachCss ? elementData.computedStyles : undefined,
+			dimensions: elementData.dimensions,
+			innerText: elementData.innerText,
+		});
+
+		const attachImages = this.configurationService.getValue<boolean>('chat.sendElementsToChat.attachImages');
+		if (attachImages && this._model) {
+			const screenshotBuffer = await this._model.captureScreenshot({
+				quality: 90,
+				rect: bounds
+			});
+
+			toAttach.push({
+				id: 'element-screenshot-' + Date.now(),
+				name: 'Element Screenshot',
+				fullName: 'Element Screenshot',
+				kind: 'image',
+				value: screenshotBuffer.buffer
+			});
+		}
+
+		const widget = await this.chatWidgetService.revealWidget() ?? this.chatWidgetService.lastFocusedWidget;
+		widget?.attachmentModel?.addContext(...toAttach);
+
+		return { attachCss, attachImages };
+	}
+
 	private formatElementPath(ancestors: readonly IElementAncestor[] | undefined): string | undefined {
 		if (!ancestors || ancestors.length === 0) {
 			return undefined;
@@ -1149,6 +1152,34 @@ export class BrowserEditor extends EditorPane {
 		this._currentKeyDownEvent = keyEvent;
 
 		try {
+			const isEnterKey =
+				keyEvent.code === 'Enter' ||
+				keyEvent.code === 'NumpadEnter' ||
+				keyEvent.key === 'Enter' ||
+				keyEvent.key === 'Return';
+			if (this._elementSelectionCts && isEnterKey) {
+				const cts = this._elementSelectionCts;
+				const resourceUri = this.input?.resource;
+				if (!resourceUri) {
+					return;
+				}
+
+				const locator: IBrowserTargetLocator = { browserViewId: BrowserViewUri.getId(resourceUri) };
+				const { width, height } = this._browserContainer.getBoundingClientRect();
+				const elementData = await this.browserElementsService.getFocusedElementData({ x: 0, y: 0, width, height }, cts.token, locator);
+				if (!elementData) {
+					return;
+				}
+
+				await this.attachElementDataToChat(elementData);
+				cts.dispose();
+				if (this._elementSelectionCts === cts) {
+					this._elementSelectionCts = undefined;
+					this._elementSelectionActiveContext.set(false);
+				}
+				return;
+			}
+
 			const syntheticEvent = new KeyboardEvent('keydown', keyEvent);
 			const standardEvent = new StandardKeyboardEvent(syntheticEvent);
 

src/vs/workbench/services/browserElements/browser/browserElementsService.ts

@@ -16,6 +16,8 @@ export interface IBrowserElementsService {
 	// no browser implementation yet
 	getElementData(rect: IRectangle, token: CancellationToken, locator: IBrowserTargetLocator | undefined): Promise<IElementData | undefined>;
 
+	getFocusedElementData(rect: IRectangle, token: CancellationToken, locator: IBrowserTargetLocator | undefined): Promise<IElementData | undefined>;
+
 	startDebugSession(token: CancellationToken, locator: IBrowserTargetLocator): Promise<void>;
 
 	startConsoleSession(token: CancellationToken, locator: IBrowserTargetLocator): Promise<void>;

src/vs/workbench/services/browserElements/browser/webBrowserElementsService.ts

@@ -18,6 +18,10 @@ class WebBrowserElementsService implements IBrowserElementsService {
 		throw new Error('Not implemented');
 	}
 
+	async getFocusedElementData(rect: IRectangle, token: CancellationToken, locator: IBrowserTargetLocator | undefined): Promise<IElementData | undefined> {
+		throw new Error('Not implemented');
+	}
+
 	async startDebugSession(token: CancellationToken, locator: IBrowserTargetLocator): Promise<void> {
 		throw new Error('Not implemented');
 	}

src/vs/workbench/services/browserElements/electron-browser/browserElementsService.ts

@@ -90,6 +90,22 @@ class WorkbenchBrowserElementsService implements IBrowserElementsService {
 			disposable.dispose();
 		}
 	}
+
+	async getFocusedElementData(rect: IRectangle, token: CancellationToken, locator: IBrowserTargetLocator | undefined): Promise<IElementData | undefined> {
+		if (!locator) {
+			return undefined;
+		}
+		const cancelSelectionId = cancelSelectionIdPool++;
+		const onCancelChannel = `vscode:cancelElementSelection${cancelSelectionId}`;
+		const disposable = token.onCancellationRequested(() => {
+			ipcRenderer.send(onCancelChannel, cancelSelectionId);
+		});
+		try {
+			return await this.simpleBrowser.getFocusedElementData(rect, token, locator, cancelSelectionId);
+		} finally {
+			disposable.dispose();
+		}
+	}
 }
 
 registerSingleton(IBrowserElementsService, WorkbenchBrowserElementsService, InstantiationType.Delayed);