ui element selection (#246643)

* ui element selection window hack

* add background

* target simple browser

* revert to non-simple browser attempt

* some saucy stuff

* saucy cleanup

* some additions:

* add better button, better listening, even saucier

* move to css and also make sure not to block elements during screenshot

* it's even saucier now

* remove browser id lookup

* fix merge conflicts and clean up

* make timeout 3 seconds

* some cleanup

* remove computed css

* use built in button instead

* address many comments :)

extensions/simple-browser/preview-src/index.ts

@@ -95,6 +95,8 @@ onceDocumentLoaded(() => {
 
 			// Try to bust the cache for the iframe
 			// There does not appear to be any way to reliably do this except modifying the url
+			const existing = new URLSearchParams(location.search);
+			url.searchParams.append('id', existing.get('id')!);
 			url.searchParams.append('vscodeBrowserReqId', Date.now().toString());
 
 			iframe.src = url.toString();

src/vs/platform/native/common/native.ts

@@ -4,6 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 
 import { VSBuffer } from '../../../base/common/buffer.js';
+import { CancellationToken } from '../../../base/common/cancellation.js';
 import { Event } from '../../../base/common/event.js';
 import { URI } from '../../../base/common/uri.js';
 import { MessageBoxOptions, MessageBoxReturnValue, OpenDevToolsOptions, OpenDialogOptions, OpenDialogReturnValue, SaveDialogOptions, SaveDialogReturnValue } from '../../../base/parts/sandbox/common/electronTypes.js';
@@ -38,6 +39,12 @@ export interface INativeHostOptions {
 	readonly targetWindowId?: number;
 }
 
+export interface IElementData {
+	readonly outerHTML: string;
+	readonly computedStyle: string;
+	readonly bounds: IRectangle;
+}
+
 export interface ICommonNativeHostService {
 
 	readonly _serviceBrand: undefined;
@@ -148,7 +155,8 @@ export interface ICommonNativeHostService {
 	hasWSLFeatureInstalled(): Promise<boolean>;
 
 	// Screenshots
-	getScreenshot(): Promise<VSBuffer | undefined>;
+	getScreenshot(rect?: IRectangle): Promise<VSBuffer | undefined>;
+	getElementData(offsetX: number, offsetY: number, token: CancellationToken): Promise<IElementData | undefined>;
 
 	// Process
 	getProcessId(): Promise<number | undefined>;

src/vs/platform/native/electron-main/nativeHostMainService.ts

@@ -28,7 +28,7 @@ import { IEnvironmentMainService } from '../../environment/electron-main/environ
 import { createDecorator, IInstantiationService } from '../../instantiation/common/instantiation.js';
 import { ILifecycleMainService, IRelaunchOptions } from '../../lifecycle/electron-main/lifecycleMainService.js';
 import { ILogService } from '../../log/common/log.js';
-import { ICommonNativeHostService, INativeHostOptions, IOSProperties, IOSStatistics } from '../common/native.js';
+import { ICommonNativeHostService, IElementData, INativeHostOptions, IOSProperties, IOSStatistics } from '../common/native.js';
 import { IProductService } from '../../product/common/productService.js';
 import { IPartsSplash } from '../../theme/common/themeService.js';
 import { IThemeMainService } from '../../theme/electron-main/themeMainService.js';
@@ -48,11 +48,18 @@ import { IConfigurationService } from '../../configuration/common/configuration.
 import { IProxyAuthService } from './auth.js';
 import { AuthInfo, Credentials, IRequestService } from '../../request/common/request.js';
 import { randomPath } from '../../../base/common/extpath.js';
+import { CancellationToken } from '../../../base/common/cancellation.js';
 
 export interface INativeHostMainService extends AddFirstParameterToFunctions<ICommonNativeHostService, Promise<unknown> /* only methods, not events */, number | undefined /* window ID */> { }
 
 export const INativeHostMainService = createDecorator<INativeHostMainService>('nativeHostMainService');
 
+interface NodeDataResponse {
+	outerHTML: string;
+	computedStyle: string;
+	bounds: IRectangle;
+}
+
 export class NativeHostMainService extends Disposable implements INativeHostMainService {
 
 	declare readonly _serviceBrand: undefined;
@@ -742,14 +749,302 @@ export class NativeHostMainService extends Disposable implements INativeHostMain
 
 	//#region Screenshots
 
-	async getScreenshot(windowId: number | undefined, options?: INativeHostOptions): Promise<VSBuffer | undefined> {
+	async getScreenshot(windowId: number | undefined, rect?: IRectangle, options?: INativeHostOptions): Promise<VSBuffer | undefined> {
 		const window = this.windowById(options?.targetWindowId, windowId);
-		const captured = await window?.win?.webContents.capturePage();
+		const captured = await window?.win?.webContents.capturePage(rect);
 
 		const buf = captured?.toJPEG(95);
 		return buf && VSBuffer.wrap(buf);
 	}
 
+	async getElementData(windowId: number | undefined, offsetX: number = 0, offsetY: number = 0, token: CancellationToken): Promise<IElementData | undefined> {
+		const window = this.windowById(windowId, windowId);
+		if (!window?.win) {
+			return undefined;
+		}
+
+		// Find the simple browser webview
+		const allWebContents = webContents.getAllWebContents();
+		const simpleBrowserWebview = allWebContents.find(webContent => webContent.getTitle().includes('Simple Browser'));
+
+		if (!simpleBrowserWebview) {
+			return undefined;
+		}
+
+		const debuggers = simpleBrowserWebview.debugger;
+		debuggers.attach();
+
+		const { targetInfos } = await debuggers.sendCommand('Target.getTargets');
+		let resultId: string | undefined = undefined;
+		let target: typeof targetInfos[number] | undefined = undefined;
+		let targetSessionId: number | undefined = undefined;
+		try {
+			// find parent id and extract id
+			const matchingTarget = targetInfos.find((targetInfo: { url: string }) => {
+				const url = new URL(targetInfo.url);
+				return url.searchParams.get('parentId') === window?.id.toString();
+			});
+
+			if (matchingTarget) {
+				const url = new URL(matchingTarget.url);
+				resultId = url.searchParams.get('id')!;
+			}
+
+			// use id to grab simple browser target
+			if (resultId) {
+				target = targetInfos.find((targetInfo: { url: string }) => {
+					const url = new URL(targetInfo.url);
+					return url.searchParams.get('id') === resultId && url.searchParams.get('vscodeBrowserReqId')!;
+				});
+			}
+
+			const { sessionId } = await debuggers.sendCommand('Target.attachToTarget', {
+				targetId: target.targetId,
+				flatten: true,
+			});
+
+			targetSessionId = sessionId;
+
+			await debuggers.sendCommand('DOM.enable', {}, sessionId);
+			await debuggers.sendCommand('CSS.enable', {}, sessionId);
+			await debuggers.sendCommand('Overlay.enable', {}, sessionId);
+			await debuggers.sendCommand('Debugger.enable', {}, sessionId);
+			await debuggers.sendCommand('Runtime.enable', {}, sessionId);
+
+			await debuggers.sendCommand('Runtime.evaluate', {
+				expression: `(function() {
+					const style = document.createElement('style');
+					style.id = '__pseudoBlocker__';
+					style.textContent = '*::before, *::after { pointer-events: none !important; }';
+					document.head.appendChild(style);
+				})();`,
+			}, sessionId);
+
+			// slightly changed default CDP debugger inspect colors
+			await debuggers.sendCommand('Overlay.setInspectMode', {
+				mode: 'searchForNode',
+				highlightConfig: {
+					showInfo: true,
+					showRulers: false,
+					showStyles: true,
+					showAccessibilityInfo: true,
+					showExtensionLines: false,
+					contrastAlgorithm: 'aa',
+					contentColor: { r: 173, g: 216, b: 255, a: 0.8 },
+					paddingColor: { r: 150, g: 200, b: 255, a: 0.5 },
+					borderColor: { r: 120, g: 180, b: 255, a: 0.7 },
+					marginColor: { r: 200, g: 220, b: 255, a: 0.4 },
+					eventTargetColor: { r: 130, g: 160, b: 255, a: 0.8 },
+					shapeColor: { r: 130, g: 160, b: 255, a: 0.8 },
+					shapeMarginColor: { r: 130, g: 160, b: 255, a: 0.5 },
+					gridHighlightConfig: {
+						rowGapColor: { r: 140, g: 190, b: 255, a: 0.3 },
+						rowHatchColor: { r: 140, g: 190, b: 255, a: 0.7 },
+						columnGapColor: { r: 140, g: 190, b: 255, a: 0.3 },
+						columnHatchColor: { r: 140, g: 190, b: 255, a: 0.7 },
+						rowLineColor: { r: 120, g: 180, b: 255 },
+						columnLineColor: { r: 120, g: 180, b: 255 },
+						rowLineDash: true,
+						columnLineDash: true
+					},
+					flexContainerHighlightConfig: {
+						containerBorder: {
+							color: { r: 120, g: 180, b: 255 },
+							pattern: 'solid'
+						},
+						itemSeparator: {
+							color: { r: 140, g: 190, b: 255 },
+							pattern: 'solid'
+						},
+						lineSeparator: {
+							color: { r: 140, g: 190, b: 255 },
+							pattern: 'solid'
+						},
+						mainDistributedSpace: {
+							hatchColor: { r: 140, g: 190, b: 255, a: 0.7 },
+							fillColor: { r: 140, g: 190, b: 255, a: 0.4 }
+						},
+						crossDistributedSpace: {
+							hatchColor: { r: 140, g: 190, b: 255, a: 0.7 },
+							fillColor: { r: 140, g: 190, b: 255, a: 0.4 }
+						},
+						rowGapSpace: {
+							hatchColor: { r: 140, g: 190, b: 255, a: 0.7 },
+							fillColor: { r: 140, g: 190, b: 255, a: 0.4 }
+						},
+						columnGapSpace: {
+							hatchColor: { r: 140, g: 190, b: 255, a: 0.7 },
+							fillColor: { r: 140, g: 190, b: 255, a: 0.4 }
+						}
+					},
+					flexItemHighlightConfig: {
+						baseSizeBox: {
+							hatchColor: { r: 130, g: 170, b: 255, a: 0.6 }
+						},
+						baseSizeBorder: {
+							color: { r: 120, g: 180, b: 255 },
+							pattern: 'solid'
+						},
+						flexibilityArrow: {
+							color: { r: 130, g: 190, b: 255 }
+						}
+					},
+				},
+			}, sessionId);
+		} catch (e) {
+			debuggers.detach();
+			throw new Error('No target found', e);
+		}
+
+		if (!targetSessionId) {
+			debuggers.detach();
+			throw new Error('No target session id found');
+		}
+
+		const nodeData = await this.getNodeData(targetSessionId, debuggers, window.win);
+		debuggers.detach();
+
+		const zoomFactor = simpleBrowserWebview.getZoomFactor();
+		const scaledBounds = {
+			x: (nodeData.bounds.x + offsetX) * zoomFactor,
+			y: (nodeData.bounds.y + offsetY) * zoomFactor,
+			width: nodeData.bounds.width * zoomFactor,
+			height: nodeData.bounds.height * zoomFactor
+		};
+
+		return { outerHTML: nodeData.outerHTML, computedStyle: nodeData.computedStyle, bounds: scaledBounds };
+	}
+
+	async getNodeData(sessionId: number, debuggers: any, window: BrowserWindow): Promise<NodeDataResponse> {
+		return new Promise((resolve, reject) => {
+			const onMessage = async (event: any, method: string, params: { backendNodeId: number }) => {
+				if (method === 'Overlay.inspectNodeRequested') {
+
+					await debuggers.sendCommand('Runtime.evaluate', {
+						expression: `(() => {
+								const style = document.getElementById('__pseudoBlocker__');
+								if (style) style.remove();
+							})();`,
+					}, sessionId);
+
+
+					this._register(debuggers.off('message', onMessage));
+					const backendNodeId = params?.backendNodeId;
+					if (!backendNodeId) {
+						throw new Error('Missing backendNodeId in inspectNodeRequested event');
+					}
+
+					try {
+						await debuggers.sendCommand('DOM.getDocument', {}, sessionId);
+						const { nodeIds } = await debuggers.sendCommand('DOM.pushNodesByBackendIdsToFrontend', { backendNodeIds: [backendNodeId] }, sessionId);
+						if (!nodeIds || nodeIds.length === 0) {
+							throw new Error('Failed to get node IDs.');
+						}
+						const nodeId = nodeIds[0];
+
+						const { model } = await debuggers.sendCommand('DOM.getBoxModel', { nodeId }, sessionId);
+						if (!model) {
+							throw new Error('Failed to get box model.');
+						}
+
+						const margin = model.margin;
+						const x = margin[0];
+						const y = margin[1] + 32.4 + 35; // 32.4 is height of the title bar, 35 is height of the tab bar
+						const width = margin[2] - margin[0];
+						const height = margin[5] - margin[1];
+
+						const matched = await debuggers.sendCommand('CSS.getMatchedStylesForNode', { nodeId }, sessionId);
+						if (!matched) {
+							throw new Error('Failed to get matched css.');
+						}
+
+						const formatted = this.formatMatchedStyles(matched);
+						const { outerHTML } = await debuggers.sendCommand('DOM.getOuterHTML', { nodeId }, sessionId);
+						if (!outerHTML) {
+							throw new Error('Failed to get outerHTML.');
+						}
+
+						resolve({
+							outerHTML,
+							computedStyle: formatted,
+							bounds: { x, y, width, height }
+						});
+					} catch (err) {
+						debuggers.detach();
+						reject(err);
+
+					}
+				}
+			};
+
+			window.webContents.on('ipc-message', async (event, channel) => {
+				if (channel === 'vscode:cancelElementSelection') {
+					this._register(debuggers.off('message', onMessage));
+					if (debuggers.isAttached()) {
+						debuggers.detach();
+					}
+				}
+			});
+
+			this._register(debuggers.on('message', onMessage));
+		});
+	}
+
+	formatMatchedStyles(matched: any): string {
+		const lines: string[] = [];
+
+		// inline
+		if (matched.inlineStyle?.cssProperties?.length) {
+			lines.push('/* Inline style */');
+			lines.push('element {');
+			for (const prop of matched.inlineStyle.cssProperties) {
+				if (prop.name && prop.value) {
+					lines.push(`  ${prop.name}: ${prop.value};`);
+				}
+			}
+			lines.push('}\n');
+		}
+
+		// matched
+		if (matched.matchedCSSRules?.length) {
+			for (const ruleEntry of matched.matchedCSSRules) {
+				const rule = ruleEntry.rule;
+				const selectors = rule.selectorList.selectors.map((s: any) => s.text).join(', ');
+				lines.push(`/* Matched Rule from ${rule.origin} */`);
+				lines.push(`${selectors} {`);
+				for (const prop of rule.style.cssProperties) {
+					if (prop.name && prop.value) {
+						lines.push(`  ${prop.name}: ${prop.value};`);
+					}
+				}
+				lines.push('}\n');
+			}
+		}
+
+		// inherited rules
+		if (matched.inherited?.length) {
+			let level = 1;
+			for (const inherited of matched.inherited) {
+				const rules = inherited.matchedCSSRules || [];
+				for (const ruleEntry of rules) {
+					const rule = ruleEntry.rule;
+					const selectors = rule.selectorList.selectors.map((s: any) => s.text).join(', ');
+					lines.push(`/* Inherited from ancestor level ${level} (${rule.origin}) */`);
+					lines.push(`${selectors} {`);
+					for (const prop of rule.style.cssProperties) {
+						if (prop.name && prop.value) {
+							lines.push(`  ${prop.name}: ${prop.value};`);
+						}
+					}
+					lines.push('}\n');
+				}
+				level++;
+			}
+		}
+
+		return '\n' + lines.join('\n');
+	}
+
 	//#endregion
 
 

src/vs/workbench/contrib/chat/browser/actions/chatContextActions.ts

@@ -488,7 +488,6 @@ export class AttachContextAction extends Action2 {
 		const fileService = accessor.get(IFileService);
 		const textModelService = accessor.get(ITextModelService);
 		const quickInputService = accessor.get(IQuickInputService);
-
 		const toAttach: IChatRequestVariableEntry[] = [];
 		for (const pick of picks) {
 
@@ -630,6 +629,7 @@ export class AttachContextAction extends Action2 {
 							fullName: pick.label,
 							value: resizedImage,
 							kind: 'image',
+							references: [{ reference: pick.resource, kind: 'reference' }]
 						});
 					}
 				} else {

src/vs/workbench/contrib/chat/browser/chat.contribution.ts

@@ -80,6 +80,7 @@ import { ChatEditingEditorAccessibility } from './chatEditing/chatEditingEditorA
 import { registerChatEditorActions } from './chatEditing/chatEditingEditorActions.js';
 import { ChatEditingEditorContextKeys } from './chatEditing/chatEditingEditorContextKeys.js';
 import { ChatEditingEditorOverlay } from './chatEditing/chatEditingEditorOverlay.js';
+import { SimpleBrowserOverlay } from './chatEditing/simpleBrowserEditorOverlay.js';
 import { ChatEditingService } from './chatEditing/chatEditingServiceImpl.js';
 import { ChatEditingNotebookFileSystemProviderContrib } from './chatEditing/notebook/chatEditingNotebookFileSystemProvider.js';
 import { ChatEditor, IChatEditorOptions } from './chatEditor.js';
@@ -640,6 +641,7 @@ registerWorkbenchContribution2(BuiltinToolsContribution.ID, BuiltinToolsContribu
 registerWorkbenchContribution2(ChatAgentSettingContribution.ID, ChatAgentSettingContribution, WorkbenchPhase.BlockRestore);
 registerWorkbenchContribution2(ChatEditingEditorAccessibility.ID, ChatEditingEditorAccessibility, WorkbenchPhase.AfterRestored);
 registerWorkbenchContribution2(ChatEditingEditorOverlay.ID, ChatEditingEditorOverlay, WorkbenchPhase.AfterRestored);
+registerWorkbenchContribution2(SimpleBrowserOverlay.ID, SimpleBrowserOverlay, WorkbenchPhase.AfterRestored);
 registerWorkbenchContribution2(ChatEditingEditorContextKeys.ID, ChatEditingEditorContextKeys, WorkbenchPhase.AfterRestored);
 registerWorkbenchContribution2(ChatTransferContribution.ID, ChatTransferContribution, WorkbenchPhase.BlockRestore);
 

src/vs/workbench/contrib/chat/browser/chatAttachmentWidgets.ts

@@ -25,7 +25,7 @@ import { IOpenerService, OpenInternalOptions } from '../../../../platform/opener
 import { IThemeService, FolderThemeIcon } from '../../../../platform/theme/common/themeService.js';
 import { IResourceLabel, ResourceLabels, IFileLabelOptions } from '../../../browser/labels.js';
 import { revealInSideBarCommand } from '../../files/browser/fileActions.contribution.js';
-import { IChatRequestPasteVariableEntry, IChatRequestVariableEntry, INotebookOutputVariableEntry, OmittedState } from '../common/chatModel.js';
+import { IChatRequestPasteVariableEntry, IChatRequestVariableEntry, IElementVariableEntry, INotebookOutputVariableEntry, OmittedState } from '../common/chatModel.js';
 import { ILanguageModelChatMetadataAndIdentifier, ILanguageModelsService } from '../common/languageModels.js';
 import { hookUpResourceAttachmentDragAndContextMenu, hookUpSymbolAttachmentDragAndContextMenu } from './chatContentParts/chatAttachmentsContentPart.js';
 import { KeyCode } from '../../../../base/common/keyCodes.js';
@@ -36,6 +36,7 @@ import { ITelemetryService } from '../../../../platform/telemetry/common/telemet
 import { INotebookService } from '../../notebook/common/notebookService.js';
 import { CellUri } from '../../notebook/common/notebookCommon.js';
 import { ThemeIcon } from '../../../../base/common/themables.js';
+import { IEditorService } from '../../../services/editor/common/editorService.js';
 
 abstract class AbstractChatAttachmentWidget extends Disposable {
 	public readonly element: HTMLElement;
@@ -542,3 +543,41 @@ export class NotebookCellOutputChatAttachmentWidget extends AbstractChatAttachme
 	}
 
 }
+
+export class ElementChatAttachmentWidget extends AbstractChatAttachmentWidget {
+	constructor(
+		attachment: IElementVariableEntry,
+		currentLanguageModel: ILanguageModelChatMetadataAndIdentifier | undefined,
+		shouldFocusClearButton: boolean,
+		container: HTMLElement,
+		contextResourceLabels: ResourceLabels,
+		hoverDelegate: IHoverDelegate,
+		@ICommandService commandService: ICommandService,
+		@IOpenerService openerService: IOpenerService,
+		@IEditorService editorService: IEditorService,
+	) {
+		super(attachment, shouldFocusClearButton, container, contextResourceLabels, hoverDelegate, currentLanguageModel, commandService, openerService);
+
+		const ariaLabel = localize('chat.elementAttachment', "Attached element, {0}", attachment.name);
+		this.element.ariaLabel = ariaLabel;
+
+		this.element.style.position = 'relative';
+		this.element.style.cursor = 'pointer';
+		const attachmentLabel = attachment.name;
+		const withIcon = attachment.icon?.id ? `$(${attachment.icon.id})\u00A0${attachmentLabel}` : attachmentLabel;
+		this.label.setLabel(withIcon, undefined, { title: localize('chat.clickToViewContents', "Click to view the contents of: {0}", attachmentLabel) });
+
+		this._register(dom.addDisposableListener(this.element, dom.EventType.CLICK, async () => {
+			const content = attachment.value?.toString() || '';
+			await editorService.openEditor({
+				resource: undefined,
+				contents: content,
+				options: {
+					pinned: true
+				}
+			});
+		}));
+
+		this.attachClearButton();
+	}
+}

src/vs/workbench/contrib/chat/browser/chatDragAndDrop.ts

@@ -285,7 +285,7 @@ export class ChatDragAndDrop extends Themable {
 			const resource = URI.parse(url);
 
 			if (IMAGE_DATA_REGEX.test(url)) {
-				return { data: await convertStringToUInt8Array(url), name: createDisplayName(), resource };
+				return { data: convertStringToUInt8Array(url), name: createDisplayName(), resource };
 			}
 
 			if (URL_REGEX.test(url)) {

src/vs/workbench/contrib/chat/browser/chatEditing/simpleBrowserEditorOverlay.ts

@@ -0,0 +1,295 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import '../media/simpleBrowserOverlay.css';
+import { combinedDisposable, DisposableMap, DisposableStore, toDisposable } from '../../../../../base/common/lifecycle.js';
+import { autorun, derivedOpts, observableFromEvent, observableSignalFromEvent } from '../../../../../base/common/observable.js';
+import { IInstantiationService } from '../../../../../platform/instantiation/common/instantiation.js';
+import { ThemeIcon } from '../../../../../base/common/themables.js';
+import { Codicon } from '../../../../../base/common/codicons.js';
+import { localize } from '../../../../../nls.js';
+import { IWorkbenchContribution } from '../../../../common/contributions.js';
+import { IEditorGroup, IEditorGroupsService } from '../../../../services/editor/common/editorGroupsService.js';
+import { EditorGroupView } from '../../../../browser/parts/editor/editorGroupView.js';
+import { Event } from '../../../../../base/common/event.js';
+import { ServiceCollection } from '../../../../../platform/instantiation/common/serviceCollection.js';
+import { IContextKeyService } from '../../../../../platform/contextkey/common/contextkey.js';
+import { EditorResourceAccessor, SideBySideEditor } from '../../../../common/editor.js';
+import { isEqual } from '../../../../../base/common/resources.js';
+import { CancellationTokenSource } from '../../../../../base/common/cancellation.js';
+import { IHostService } from '../../../../services/host/browser/host.js';
+import { IChatWidgetService, showChatView } from '../chat.js';
+import { IViewsService } from '../../../../services/views/common/viewsService.js';
+import { Button } from '../../../../../base/browser/ui/button/button.js';
+import { defaultButtonStyles } from '../../../../../platform/theme/browser/defaultStyles.js';
+import { addDisposableListener } from '../../../../../base/browser/dom.js';
+
+class SimpleBrowserOverlayWidget {
+
+	private readonly _domNode: HTMLElement;
+
+	private readonly _showStore = new DisposableStore();
+
+	constructor(
+		private readonly _editor: IEditorGroup,
+		private readonly _container: HTMLElement,
+		@IHostService private readonly _hostService: IHostService,
+		@IChatWidgetService private readonly _chatWidgetService: IChatWidgetService,
+		@IViewsService private readonly _viewService: IViewsService,
+	) {
+		this._domNode = document.createElement('div');
+		this._domNode.className = 'element-selection-message';
+
+		const message = document.createElement('span');
+		const startSelectionMessage = localize('elementSelectionMessage', 'Add UI element to chat.');
+		message.textContent = startSelectionMessage;
+		this._domNode.appendChild(message);
+
+		let cts: CancellationTokenSource;
+		const selectButton = new Button(this._domNode, { ...defaultButtonStyles, supportIcons: true, title: localize('selectAnElement', 'Click to select an element.') });
+		const cancelButton = new Button(this._domNode, { ...defaultButtonStyles, supportIcons: true, title: localize('cancelSelection', 'Click to cancel selection.') });
+
+		selectButton.element.className = 'element-selection-start';
+		selectButton.label = localize('startSelection', 'Start Selection');
+		cancelButton.element.className = 'element-selection-cancel';
+		cancelButton.label = localize('cancel', 'Cancel');
+
+		this.hideElement(cancelButton.element);
+
+		this._showStore.add(addDisposableListener(selectButton.element, 'click', async () => {
+			cts = new CancellationTokenSource();
+			this._editor.focus();
+
+			// start selection
+			message.textContent = localize('elementSelectionInProgress', 'Selection in progress...');
+			this.hideElement(selectButton.element);
+			this.showElement(cancelButton.element);
+
+			await this.addElementToChat(cts);
+			// stop selection
+			this.hideElement(cancelButton.element);
+			message.textContent = localize('elementSelectionComplete', 'Element added to chat.');
+
+			// wait 3 seconds before showing the start selection button again
+			setTimeout(() => {
+				message.textContent = startSelectionMessage;
+				this.showElement(selectButton.element);
+			}, 3000);
+		}));
+
+		this._showStore.add(addDisposableListener(cancelButton.element, 'click', () => {
+			cts.cancel();
+			this.hideElement(cancelButton.element);
+			message.textContent = localize('elementCancelMessage', 'Selection canceled');
+			setTimeout(() => {
+				message.textContent = startSelectionMessage;
+				this.showElement(selectButton.element);
+			}, 3000);
+		}));
+	}
+
+	hideElement(element: HTMLElement) {
+		element.classList.add('hidden');
+	}
+
+	showElement(element: HTMLElement) {
+		element.classList.remove('hidden');
+	}
+
+	async addElementToChat(cts: CancellationTokenSource) {
+		const rect = this._container.getBoundingClientRect();
+		const elementData = await this._hostService.getElementData(rect.x, rect.y, cts.token);
+		if (!elementData) {
+			throw new Error('Element data not found');
+		}
+		const bounds = elementData.bounds;
+
+		// remove container so we don't block anything on screenshot
+		this._domNode.style.display = 'none';
+
+		// Wait 1 extra frame to make sure overlay is gone
+		await new Promise(resolve => setTimeout(resolve, 100));
+
+		const screenshot = await this._hostService.getScreenshot(bounds);
+		if (!screenshot) {
+			throw new Error('Screenshot failed');
+		}
+		this._domNode.style.display = '';
+		const widget = this._chatWidgetService.lastFocusedWidget ?? await showChatView(this._viewService);
+
+		widget?.attachmentModel?.addContext({
+			id: 'element-' + Date.now(),
+			name: this.getDisplayNameFromOuterHTML(elementData.outerHTML),
+			fullName: this.getDisplayNameFromOuterHTML(elementData.outerHTML),
+			value: elementData.outerHTML + elementData.computedStyle,
+			kind: 'element',
+			icon: ThemeIcon.fromId(Codicon.layout.id),
+		}, {
+			id: 'element-screenshot-' + Date.now(),
+			name: 'Element Screenshot',
+			fullName: 'Element Screenshot',
+			kind: 'image',
+			value: screenshot.buffer
+		});
+	}
+
+
+	getDisplayNameFromOuterHTML(outerHTML: string): string {
+		const firstElementMatch = outerHTML.match(/^<(\w+)([^>]*?)>/);
+		if (!firstElementMatch) {
+			throw new Error('No outer element found');
+		}
+
+		const tagName = firstElementMatch[1];
+		const idMatch = firstElementMatch[2].match(/\s+id\s*=\s*["']([^"']+)["']/i);
+		const id = idMatch ? `#${idMatch[1]}` : '';
+		const classMatch = firstElementMatch[2].match(/\s+class\s*=\s*["']([^"']+)["']/i);
+		const className = classMatch ? `.${classMatch[1].replace(/\s+/g, '.')}` : '';
+		return `${tagName}${id}${className}`;
+	}
+
+	dispose() {
+		this.hide();
+		this._showStore.dispose();
+	}
+
+	getDomNode(): HTMLElement {
+		return this._domNode;
+	}
+
+	hide() {
+		this._showStore.clear();
+	}
+}
+
+class SimpleBrowserOverlayController {
+
+	private readonly _store = new DisposableStore();
+
+	private readonly _domNode = document.createElement('div');
+
+	constructor(
+		container: HTMLElement,
+		group: IEditorGroup,
+		@IInstantiationService instaService: IInstantiationService,
+	) {
+
+		this._domNode.classList.add('chat-editing-editor-overlay');
+		this._domNode.style.position = 'absolute';
+		this._domNode.style.bottom = `5px`;
+		this._domNode.style.right = `5px`;
+		this._domNode.style.zIndex = `100`;
+
+		const widget = instaService.createInstance(SimpleBrowserOverlayWidget, group, container);
+		this._domNode.appendChild(widget.getDomNode());
+		this._store.add(toDisposable(() => this._domNode.remove()));
+		this._store.add(widget);
+
+		const show = () => {
+			if (!container.contains(this._domNode)) {
+				container.appendChild(this._domNode);
+			}
+		};
+
+		const hide = () => {
+			if (container.contains(this._domNode)) {
+				widget.hide();
+				this._domNode.remove();
+			}
+		};
+
+		const activeEditorSignal = observableSignalFromEvent(this, Event.any(group.onDidActiveEditorChange, group.onDidModelChange));
+
+		const activeUriObs = derivedOpts({ equalsFn: isEqual }, r => {
+
+			activeEditorSignal.read(r); // signal
+
+			const editor = group.activeEditorPane;
+			if (editor?.input.editorId === 'mainThreadWebview-simpleBrowser.view') {
+				const uri = EditorResourceAccessor.getOriginalUri(editor?.input, { supportSideBySide: SideBySideEditor.PRIMARY });
+				return uri;
+			}
+			return undefined;
+		});
+
+		this._store.add(autorun(r => {
+
+			const data = activeUriObs.read(r);
+
+			if (!data) {
+				hide();
+				return;
+			}
+
+			// widget.show();
+			show();
+
+
+		}));
+	}
+
+	dispose(): void {
+		this._store.dispose();
+	}
+}
+
+export class SimpleBrowserOverlay implements IWorkbenchContribution {
+
+	static readonly ID = 'chat.simpleBrowser.overlay';
+
+	private readonly _store = new DisposableStore();
+
+	constructor(
+		@IEditorGroupsService editorGroupsService: IEditorGroupsService,
+		@IInstantiationService instantiationService: IInstantiationService,
+	) {
+
+		const editorGroups = observableFromEvent(
+			this,
+			Event.any(editorGroupsService.onDidAddGroup, editorGroupsService.onDidRemoveGroup),
+			() => editorGroupsService.groups
+		);
+
+		const overlayWidgets = new DisposableMap<IEditorGroup>();
+
+		this._store.add(autorun(r => {
+
+			const toDelete = new Set(overlayWidgets.keys());
+			const groups = editorGroups.read(r);
+
+
+			for (const group of groups) {
+
+				if (!(group instanceof EditorGroupView)) {
+					// TODO@jrieken better with https://github.com/microsoft/vscode/tree/ben/layout-group-container
+					continue;
+				}
+
+				toDelete.delete(group); // we keep the widget for this group!
+
+				if (!overlayWidgets.has(group)) {
+
+					const scopedInstaService = instantiationService.createChild(
+						new ServiceCollection([IContextKeyService, group.scopedContextKeyService])
+					);
+
+					const container = group.element;
+
+
+					const ctrl = scopedInstaService.createInstance(SimpleBrowserOverlayController, container, group);
+					overlayWidgets.set(group, combinedDisposable(ctrl, scopedInstaService));
+				}
+			}
+
+			for (const group of toDelete) {
+				overlayWidgets.deleteAndDispose(group);
+			}
+		}));
+	}
+
+	dispose(): void {
+		this._store.dispose();
+	}
+}

src/vs/workbench/contrib/chat/browser/chatInputPart.ts

@@ -71,7 +71,7 @@ import { getSimpleCodeEditorWidgetOptions, getSimpleEditorOptions, setupSimpleEd
 import { IChatAgentService } from '../common/chatAgents.js';
 import { ChatContextKeys } from '../common/chatContextKeys.js';
 import { IChatEditingSession } from '../common/chatEditingService.js';
-import { IChatRequestVariableEntry, isImageVariableEntry, isNotebookOutputVariableEntry, isPasteVariableEntry } from '../common/chatModel.js';
+import { IChatRequestVariableEntry, isElementVariableEntry, isImageVariableEntry, isNotebookOutputVariableEntry, isPasteVariableEntry } from '../common/chatModel.js';
 import { IChatFollowup, IChatService } from '../common/chatService.js';
 import { IChatVariablesService } from '../common/chatVariables.js';
 import { IChatResponseViewModel } from '../common/chatViewModel.js';
@@ -85,7 +85,7 @@ import { PromptInstructionsAttachmentsCollectionWidget } from './attachments/pro
 import { IChatWidget } from './chat.js';
 import { ChatAttachmentModel } from './chatAttachmentModel.js';
 import { toChatVariable } from './chatAttachmentModel/chatPromptAttachmentsCollection.js';
-import { DefaultChatAttachmentWidget, FileAttachmentWidget, ImageAttachmentWidget, NotebookCellOutputChatAttachmentWidget, PasteAttachmentWidget } from './chatAttachmentWidgets.js';
+import { DefaultChatAttachmentWidget, FileAttachmentWidget, ImageAttachmentWidget, NotebookCellOutputChatAttachmentWidget, PasteAttachmentWidget, ElementChatAttachmentWidget } from './chatAttachmentWidgets.js';
 import { IDisposableReference } from './chatContentParts/chatCollections.js';
 import { CollapsibleListPool, IChatCollapsibleListItem } from './chatContentParts/chatReferencesContentPart.js';
 import { ChatDragAndDrop } from './chatDragAndDrop.js';
@@ -1196,6 +1196,8 @@ export class ChatInputPart extends Disposable implements IHistoryNavigationWidge
 				attachmentWidget = this.instantiationService.createInstance(FileAttachmentWidget, resource, range, attachment, this._currentLanguageModel, shouldFocusClearButton, container, this._contextResourceLabels, hoverDelegate);
 			} else if (isImageVariableEntry(attachment)) {
 				attachmentWidget = this.instantiationService.createInstance(ImageAttachmentWidget, resource, attachment, this._currentLanguageModel, shouldFocusClearButton, container, this._contextResourceLabels, hoverDelegate);
+			} else if (isElementVariableEntry(attachment)) {
+				attachmentWidget = this.instantiationService.createInstance(ElementChatAttachmentWidget, attachment, this._currentLanguageModel, shouldFocusClearButton, container, this._contextResourceLabels, hoverDelegate);
 			} else if (isPasteVariableEntry(attachment)) {
 				attachmentWidget = this.instantiationService.createInstance(PasteAttachmentWidget, attachment, this._currentLanguageModel, shouldFocusClearButton, container, this._contextResourceLabels, hoverDelegate);
 			} else {

src/vs/workbench/contrib/chat/browser/media/chat.css

@@ -1713,6 +1713,9 @@ have to be updated for changes to the rules above, or to support more deeply nes
 	display: block;
 	max-height: 350px;
 	max-width: 100%;
+	min-width: 200px;
+	min-height: 200px;
+
 }
 
 .chat-attached-context-hover .chat-attached-context-url {
@@ -1757,6 +1760,7 @@ have to be updated for changes to the rules above, or to support more deeply nes
 	width: 14px;
 	height: 14px;
 	border-radius: 2px;
+	object-fit: cover;
 }
 
 .chat-attached-context-attachment .chat-attached-context-custom-text {

src/vs/workbench/contrib/chat/browser/media/simpleBrowserOverlay.css

@@ -0,0 +1,30 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+.element-selection-message {
+	position: absolute;
+	bottom: 10px;
+	right: 10px;
+	padding: 8px 10px;
+	background: var(--vscode-notifications-background);
+	color: var(--vscode-notifications-foreground);
+	border-radius: 4px;
+	font-size: 12px;
+	box-shadow: 0 2px 6px rgba(0, 0, 0, 0.2);
+	display: flex;
+	align-items: center;
+	gap: 8px;
+	width: max-content
+}
+
+.element-selection-cancel,
+.element-selection-start {
+	padding: 5px 10px;
+	width: fit-content;
+}
+
+.element-selection-message .hidden {
+	display: none;
+}

src/vs/workbench/contrib/chat/common/chatModel.ts

@@ -196,9 +196,13 @@ export interface IDiagnosticVariableEntry extends IBaseChatRequestVariableEntry,
 	readonly kind: 'diagnostic';
 }
 
+export interface IElementVariableEntry extends IBaseChatRequestVariableEntry {
+	readonly kind: 'element';
+}
+
 export type IChatRequestVariableEntry = IGenericChatRequestVariableEntry | IChatRequestImplicitVariableEntry | IChatRequestPasteVariableEntry
 	| ISymbolVariableEntry | ICommandResultVariableEntry | IDiagnosticVariableEntry | IImageVariableEntry | IChatRequestToolEntry
-	| IChatRequestDirectoryEntry | IChatRequestFileEntry | INotebookOutputVariableEntry;
+	| IChatRequestDirectoryEntry | IChatRequestFileEntry | INotebookOutputVariableEntry | IElementVariableEntry;
 
 export function isImplicitVariableEntry(obj: IChatRequestVariableEntry): obj is IChatRequestImplicitVariableEntry {
 	return obj.kind === 'implicit';
@@ -216,6 +220,10 @@ export function isNotebookOutputVariableEntry(obj: IChatRequestVariableEntry): o
 	return obj.kind === 'notebookOutput';
 }
 
+export function isElementVariableEntry(obj: IChatRequestVariableEntry): obj is IElementVariableEntry {
+	return obj.kind === 'element';
+}
+
 export function isDiagnosticsVariableEntry(obj: IChatRequestVariableEntry): obj is IDiagnosticVariableEntry {
 	return obj.kind === 'diagnostic';
 }

src/vs/workbench/contrib/webview/browser/webviewElement.ts

@@ -419,6 +419,7 @@ export class WebviewElement extends Disposable implements IWebview, WebviewFindD
 		// The extensionId and purpose in the URL are used for filtering in js-debug:
 		const params: { [key: string]: string } = {
 			id: this.id,
+			parentId: targetWindow.vscodeWindowId.toString(),
 			origin: this.origin,
 			swVersion: String(this._expectedServiceWorkerVersion),
 			extensionId: extension?.id.value ?? '',

src/vs/workbench/services/host/browser/browserHostService.ts

@@ -42,6 +42,7 @@ import { isIOS, isMacintosh } from '../../../../base/common/platform.js';
 import { IUserDataProfilesService } from '../../../../platform/userDataProfile/common/userDataProfile.js';
 import { URI } from '../../../../base/common/uri.js';
 import { VSBuffer } from '../../../../base/common/buffer.js';
+import { IElementData } from '../../../../platform/native/common/native.js';
 
 enum HostShutdownReason {
 
@@ -97,6 +98,7 @@ export class BrowserHostService extends Disposable implements IHostService {
 		this.registerListeners();
 	}
 
+
 	private registerListeners(): void {
 
 		// Veto shutdown depending on `window.confirmBeforeClose` setting
@@ -650,6 +652,14 @@ export class BrowserHostService extends Disposable implements IHostService {
 		}
 	}
 
+	async getElementData(): Promise<IElementData | undefined> {
+		return undefined;
+	}
+
+	async getBrowserId(): Promise<string | undefined> {
+		return undefined;
+	}
+
 	//#endregion
 
 	//#region Native Handle

src/vs/workbench/services/host/browser/host.ts

@@ -4,8 +4,10 @@
  *--------------------------------------------------------------------------------------------*/
 
 import { VSBuffer } from '../../../../base/common/buffer.js';
+import { CancellationToken } from '../../../../base/common/cancellation.js';
 import { Event } from '../../../../base/common/event.js';
 import { createDecorator } from '../../../../platform/instantiation/common/instantiation.js';
+import { IElementData } from '../../../../platform/native/common/native.js';
 import { IWindowOpenable, IOpenWindowOptions, IOpenEmptyWindowOptions, IPoint, IRectangle } from '../../../../platform/window/common/window.js';
 
 export const IHostService = createDecorator<IHostService>('hostService');
@@ -128,7 +130,9 @@ export interface IHostService {
 	/**
 	 * Captures a screenshot.
 	 */
-	getScreenshot(): Promise<VSBuffer | undefined>;
+	getScreenshot(rect?: IRectangle): Promise<VSBuffer | undefined>;
+
+	getElementData(offsetX: number, offsetY: number, token: CancellationToken): Promise<IElementData | undefined>;
 
 	//#endregion
 

src/vs/workbench/services/host/electron-sandbox/nativeHostService.ts

@@ -5,7 +5,7 @@
 
 import { Emitter, Event } from '../../../../base/common/event.js';
 import { IHostService } from '../browser/host.js';
-import { INativeHostService } from '../../../../platform/native/common/native.js';
+import { IElementData, INativeHostService } from '../../../../platform/native/common/native.js';
 import { InstantiationType, registerSingleton } from '../../../../platform/instantiation/common/extensions.js';
 import { ILabelService, Verbosity } from '../../../../platform/label/common/label.js';
 import { IWorkbenchEnvironmentService } from '../../environment/common/environmentService.js';
@@ -18,6 +18,8 @@ import { disposableWindowInterval, getActiveDocument, getWindowId, getWindowsCou
 import { memoize } from '../../../../base/common/decorators.js';
 import { isAuxiliaryWindow } from '../../../../base/browser/window.js';
 import { VSBuffer } from '../../../../base/common/buffer.js';
+import { CancellationToken } from '../../../../base/common/cancellation.js';
+import { ipcRenderer } from '../../../../base/parts/sandbox/electron-sandbox/globals.js';
 
 class WorkbenchNativeHostService extends NativeHostService {
 
@@ -193,8 +195,17 @@ class WorkbenchHostService extends Disposable implements IHostService {
 
 	//#region Screenshots
 
-	getScreenshot(): Promise<VSBuffer | undefined> {
+	getScreenshot(rect?: IRectangle): Promise<VSBuffer | undefined> {
-		return this.nativeHostService.getScreenshot();
+		return this.nativeHostService.getScreenshot(rect);
+	}
+
+	async getElementData(offsetX: number, offsetY: number, token: CancellationToken): Promise<IElementData | undefined> {
+		const disposable = token.onCancellationRequested(() => {
+			ipcRenderer.send('vscode:cancelElementSelection');
+		});
+		const elementData = this.nativeHostService.getElementData(offsetX, offsetY, token);
+		elementData.finally(() => disposable.dispose());
+		return elementData;
 	}
 
 	//#endregion

src/vs/workbench/test/browser/workbenchTestServices.ts

@@ -32,7 +32,7 @@ import { ILanguageService } from '../../../editor/common/languages/language.js';
 import { IHistoryService } from '../../services/history/common/history.js';
 import { IInstantiationService, ServiceIdentifier } from '../../../platform/instantiation/common/instantiation.js';
 import { TestConfigurationService } from '../../../platform/configuration/test/common/testConfigurationService.js';
-import { MenuBarVisibility, IWindowOpenable, IOpenWindowOptions, IOpenEmptyWindowOptions } from '../../../platform/window/common/window.js';
+import { MenuBarVisibility, IWindowOpenable, IOpenWindowOptions, IOpenEmptyWindowOptions, IRectangle } from '../../../platform/window/common/window.js';
 import { TestWorkspace } from '../../../platform/workspace/test/common/testWorkspace.js';
 import { IEnvironmentService } from '../../../platform/environment/common/environment.js';
 import { IThemeService } from '../../../platform/theme/common/themeService.js';
@@ -183,6 +183,7 @@ import { IHoverService } from '../../../platform/hover/browser/hover.js';
 import { NullHoverService } from '../../../platform/hover/test/browser/nullHoverService.js';
 import { IActionViewItemService, NullActionViewItemService } from '../../../platform/actions/browser/actionViewItemService.js';
 import { IMarkdownString } from '../../../base/common/htmlContent.js';
+import { IElementData } from '../../../platform/native/common/native.js';
 
 export function createFileEditorInput(instantiationService: IInstantiationService, resource: URI): FileEditorInput {
 	return instantiationService.createInstance(FileEditorInput, resource, undefined, undefined, undefined, undefined, undefined, undefined);
@@ -1577,7 +1578,8 @@ export class TestHostService implements IHostService {
 
 	async toggleFullScreen(): Promise<void> { }
 
-	async getScreenshot(): Promise<VSBuffer | undefined> { return undefined; }
+	async getScreenshot(rect?: IRectangle): Promise<VSBuffer | undefined> { return undefined; }
+	async getElementData(offsetX: number, offsetY: number, token: CancellationToken): Promise<IElementData | undefined> { return undefined; }
 
 	async getNativeWindowHandle(_windowId: number): Promise<VSBuffer | undefined> { return undefined; }
 

src/vs/workbench/test/electron-sandbox/workbenchTestServices.ts

@@ -6,7 +6,7 @@
 import { Event } from '../../../base/common/event.js';
 import { workbenchInstantiationService as browserWorkbenchInstantiationService, ITestInstantiationService, TestEncodingOracle, TestEnvironmentService, TestFileDialogService, TestFilesConfigurationService, TestFileService, TestLifecycleService, TestTextFileService } from '../browser/workbenchTestServices.js';
 import { ISharedProcessService } from '../../../platform/ipc/electron-sandbox/services.js';
-import { INativeHostService, INativeHostOptions, IOSProperties, IOSStatistics } from '../../../platform/native/common/native.js';
+import { INativeHostService, INativeHostOptions, IOSProperties, IOSStatistics, IElementData } from '../../../platform/native/common/native.js';
 import { VSBuffer, VSBufferReadable, VSBufferReadableStream } from '../../../base/common/buffer.js';
 import { DisposableStore, IDisposable } from '../../../base/common/lifecycle.js';
 import { URI } from '../../../base/common/uri.js';
@@ -166,7 +166,8 @@ export class TestNativeHostService implements INativeHostService {
 	async hasClipboard(format: string, type?: 'selection' | 'clipboard' | undefined): Promise<boolean> { return false; }
 	async windowsGetStringRegKey(hive: 'HKEY_CURRENT_USER' | 'HKEY_LOCAL_MACHINE' | 'HKEY_CLASSES_ROOT' | 'HKEY_USERS' | 'HKEY_CURRENT_CONFIG', path: string, name: string): Promise<string | undefined> { return undefined; }
 	async profileRenderer(): Promise<any> { throw new Error(); }
-	async getScreenshot(): Promise<VSBuffer | undefined> { return undefined; }
+	async getScreenshot(rect?: IRectangle): Promise<VSBuffer | undefined> { return undefined; }
+	async getElementData(offsetX: number, offsetY: number, token: CancellationToken): Promise<IElementData | undefined> { return undefined; }
 }
 
 export class TestExtensionTipsService extends AbstractNativeExtensionTipsService {