/*--------------------------------------------------------------------------------------------- * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ import assert from 'assert'; import path from 'path'; import { ToolName } from '../../src/extension/tools/common/toolNames'; import { deserializeWorkbenchState } from '../../src/platform/test/node/promptContextModel'; import { ssuite, stest } from '../base/stest'; import { generateToolTestRunner } from './toolSimTest'; import { shouldSkipAgentTests } from './tools.stest'; interface IFetchWebPageToolParams { urls: string[]; query?: string; } ssuite.optional(shouldSkipAgentTests, { title: 'fetchWebPageTool', subtitle: 'toolCalling', location: 'panel' }, () => { const scenarioFolder = path.join(__dirname, '..', 'test/scenarios/test-tools'); const getState = () => deserializeWorkbenchState(scenarioFolder, path.join(scenarioFolder, 'tools.state.json')); stest('proper URL validation and query handling', generateToolTestRunner({ question: 'fetch information about React hooks from https://react.dev/reference/react', scenarioFolderPath: '', getState, expectedToolCalls: ToolName.FetchWebPage, tools: { [ToolName.FetchWebPage]: true, [ToolName.FindFiles]: true, [ToolName.FindTextInFiles]: true, [ToolName.ReadFile]: true, [ToolName.EditFile]: true, [ToolName.Codebase]: true, [ToolName.ListDirectory]: true, [ToolName.SearchWorkspaceSymbols]: true, }, }, { allowParallelToolCalls: false, toolCallValidators: { [ToolName.FetchWebPage]: async (toolCalls) => { assert.strictEqual(toolCalls.length, 1, 'should make exactly one fetch webpage tool call'); const input = toolCalls[0].input as IFetchWebPageToolParams; // Should have exactly 1 URL assert.ok(Array.isArray(input.urls), 'urls should be an array'); assert.strictEqual(input.urls.length, 1, 'should have exactly 1 URL'); // Should be the exact URL from the question const expectedUrl = 'https://react.dev/reference/react'; assert.strictEqual(input.urls[0], expectedUrl, `should have the exact URL: ${expectedUrl}`); // Validate query parameter if present if (input.query !== undefined) { assert.ok(typeof input.query === 'string', 'query should be a string if provided'); assert.ok(input.query.length > 0, 'query should not be empty if provided'); } } } })); stest('multiple URLs handling', generateToolTestRunner({ question: 'get content from https://www.typescriptlang.org/docs/handbook/release-notes/typescript-5-0.html and https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/async_function about async/await', scenarioFolderPath: '', getState, expectedToolCalls: ToolName.FetchWebPage, tools: { [ToolName.FetchWebPage]: true, [ToolName.FindFiles]: true, [ToolName.FindTextInFiles]: true, [ToolName.ReadFile]: true, [ToolName.EditFile]: true, [ToolName.Codebase]: true, [ToolName.ListDirectory]: true, [ToolName.SearchWorkspaceSymbols]: true, }, }, { allowParallelToolCalls: true, toolCallValidators: { [ToolName.FetchWebPage]: (toolCalls) => { const expectedTypescriptUrl = 'https://www.typescriptlang.org/docs/handbook/release-notes/typescript-5-0.html'; const expectedMdnUrl = 'https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/async_function'; const expectedUrls = [expectedTypescriptUrl, expectedMdnUrl]; // Allow either 1 tool call with 2 URLs or 2 tool calls with 1 URL each assert.ok(toolCalls.length === 1 || toolCalls.length === 2, 'should make either 1 or 2 fetch webpage tool calls'); if (toolCalls.length === 1) { // Single tool call with multiple URLs const input = toolCalls[0].input as IFetchWebPageToolParams; // Should have exactly 2 URLs assert.ok(Array.isArray(input.urls), 'urls should be an array'); assert.strictEqual(input.urls.length, 2, 'should have exactly 2 URLs'); // Check that both expected URLs are present assert.ok(input.urls.includes(expectedTypescriptUrl), `should include the TypeScript URL: ${expectedTypescriptUrl}`); assert.ok(input.urls.includes(expectedMdnUrl), `should include the MDN URL: ${expectedMdnUrl}`); // Verify no unexpected URLs input.urls.forEach(url => { assert.ok(expectedUrls.includes(url), `unexpected URL found: ${url}`); }); } else { // Multiple parallel tool calls with one URL each const allUrls: string[] = []; toolCalls.forEach(toolCall => { const input = toolCall.input as IFetchWebPageToolParams; assert.ok(Array.isArray(input.urls), 'urls should be an array'); assert.strictEqual(input.urls.length, 1, 'each tool call should have exactly 1 URL'); allUrls.push(input.urls[0]); }); // Check that both expected URLs are present across all tool calls assert.ok(allUrls.includes(expectedTypescriptUrl), `should include the TypeScript URL: ${expectedTypescriptUrl}`); assert.ok(allUrls.includes(expectedMdnUrl), `should include the MDN URL: ${expectedMdnUrl}`); // Verify no unexpected URLs allUrls.forEach(url => { assert.ok(expectedUrls.includes(url), `unexpected URL found: ${url}`); }); } // Check query parameter for any tool call that has it toolCalls.forEach(toolCall => { const input = toolCall.input as IFetchWebPageToolParams; if (input.query) { assert.ok( input.query.toLowerCase().includes('async') || input.query.toLowerCase().includes('await'), 'query should relate to async/await when specifically requested' ); } }); } } })); stest('query parameter extraction', generateToolTestRunner({ question: 'find specific information about error handling patterns from https://nodejs.org/en/docs/guides/error-handling/', scenarioFolderPath: '', getState, expectedToolCalls: ToolName.FetchWebPage, tools: { [ToolName.FetchWebPage]: true, [ToolName.FindFiles]: true, [ToolName.FindTextInFiles]: true, [ToolName.ReadFile]: true, [ToolName.EditFile]: true, [ToolName.Codebase]: true, [ToolName.ListDirectory]: true, [ToolName.SearchWorkspaceSymbols]: true, }, }, { allowParallelToolCalls: false, toolCallValidators: { [ToolName.FetchWebPage]: async (toolCalls) => { assert.strictEqual(toolCalls.length, 1, 'should make exactly one fetch webpage tool call'); const input = toolCalls[0].input as IFetchWebPageToolParams; // Should have exactly 1 URL assert.ok(Array.isArray(input.urls), 'urls should be an array'); assert.strictEqual(input.urls.length, 1, 'should have exactly 1 URL'); // Should be the exact URL from the question const expectedUrl = 'https://nodejs.org/en/docs/guides/error-handling/'; assert.strictEqual(input.urls[0], expectedUrl, `should have the exact URL: ${expectedUrl}`); // Should extract meaningful query when user asks for specific information assert.ok(input.query !== undefined, 'should include a query when user asks for specific information'); assert.ok(typeof input.query === 'string', 'query should be a string'); assert.ok(input.query.length > 0, 'query should not be empty'); // Query should relate to error handling since that's what was requested const queryLower = input.query.toLowerCase(); assert.ok( queryLower.includes('error') || queryLower.includes('handling') || queryLower.includes('pattern'), 'query should relate to error handling patterns when specifically requested' ); } } })); stest('multiple URLs boundary test with 6 URLs', generateToolTestRunner({ question: 'gather information from these documentation sources: https://react.dev/learn/hooks-overview, https://vuejs.org/guide/essentials/reactivity-fundamentals.html, https://angular.io/guide/component-interaction, https://svelte.dev/docs/introduction, https://solid-js.com/guides/getting-started, and https://lit.dev/docs/ about component state management', scenarioFolderPath: '', getState, expectedToolCalls: ToolName.FetchWebPage, tools: { [ToolName.FetchWebPage]: true, [ToolName.FindFiles]: true, [ToolName.FindTextInFiles]: true, [ToolName.ReadFile]: true, [ToolName.EditFile]: true, [ToolName.Codebase]: true, [ToolName.ListDirectory]: true, [ToolName.SearchWorkspaceSymbols]: true, }, }, { allowParallelToolCalls: true, toolCallValidators: { [ToolName.FetchWebPage]: (toolCalls) => { const expectedUrls = [ 'https://react.dev/learn/hooks-overview', 'https://vuejs.org/guide/essentials/reactivity-fundamentals.html', 'https://angular.io/guide/component-interaction', 'https://svelte.dev/docs/introduction', 'https://solid-js.com/guides/getting-started', 'https://lit.dev/docs/' ]; // Allow anywhere from 1 to 6 tool calls assert.ok(toolCalls.length >= 1 && toolCalls.length <= 6, `should make between 1 and 6 fetch webpage tool calls, but got ${toolCalls.length}`); // Collect all URLs from all tool calls const allUrls: string[] = []; let totalUrlCount = 0; toolCalls.forEach((toolCall, index) => { const input = toolCall.input as IFetchWebPageToolParams; assert.ok(Array.isArray(input.urls), `tool call ${index + 1}: urls should be an array`); assert.ok(input.urls.length >= 1, `tool call ${index + 1}: should have at least 1 URL`); totalUrlCount += input.urls.length; allUrls.push(...input.urls); }); // Should have exactly 6 URLs total across all tool calls assert.strictEqual(totalUrlCount, 6, 'should have exactly 6 URLs total across all tool calls'); assert.strictEqual(allUrls.length, 6, 'collected URLs array should have exactly 6 URLs'); // Check that all expected URLs are present expectedUrls.forEach(expectedUrl => { assert.ok(allUrls.includes(expectedUrl), `should include the URL: ${expectedUrl}`); }); // Verify no unexpected URLs allUrls.forEach(url => { assert.ok(expectedUrls.includes(url), `unexpected URL found: ${url}`); }); // Verify no duplicate URLs const uniqueUrls = new Set(allUrls); assert.strictEqual(uniqueUrls.size, 6, 'should not have duplicate URLs'); // Check query parameter for any tool call that has it toolCalls.forEach((toolCall, index) => { const input = toolCall.input as IFetchWebPageToolParams; assert.ok(input.query, `tool call ${index + 1}: query should be defined if provided`); }); } } })); });