From 2d2f91445d19985640bd6ad68197067011ff8108 Mon Sep 17 00:00:00 2001 From: Matt Bierner Date: Thu, 7 Aug 2025 14:35:46 -0700 Subject: [PATCH 001/299] Make activationEventsGenerator a real generator Every time I add a new `activationEventsGenerator`, I wish it supported using a real generator --- .../common/implicitActivationEvents.ts | 4 ++-- src/vs/platform/extensions/common/extensionValidator.ts | 2 +- src/vs/platform/extensions/common/extensions.ts | 2 +- src/vs/workbench/api/browser/viewsExtensionPoint.ts | 4 ++-- .../api/test/common/extHostExtensionActivator.test.ts | 2 +- .../contrib/chat/browser/chatOutputItemRenderer.ts | 4 ++-- .../contrib/chat/browser/chatParticipant.contribution.ts | 4 ++-- .../contrib/chat/browser/chatSessions.contribution.ts | 4 ++-- src/vs/workbench/contrib/chat/common/languageModels.ts | 4 ++-- .../chat/common/tools/languageModelToolsContribution.ts | 4 ++-- .../contrib/customEditor/common/extensionPoint.ts | 4 ++-- src/vs/workbench/contrib/debug/common/debugVisualizers.ts | 4 ++-- src/vs/workbench/contrib/mcp/common/mcpConfiguration.ts | 4 ++-- .../contrib/notebook/browser/notebookExtensionPoint.ts | 8 ++++---- .../contrib/tasks/common/taskDefinitionRegistry.ts | 4 ++-- src/vs/workbench/contrib/terminal/common/terminal.ts | 4 ++-- .../quickFix/browser/terminalQuickFixService.ts | 4 ++-- .../browser/gettingStartedExtensionPoint.ts | 4 ++-- .../services/actions/common/menusExtensionPoint.ts | 4 ++-- .../authentication/browser/authenticationService.ts | 4 ++-- .../test/common/extensionDescriptionRegistry.test.ts | 2 +- .../workbench/services/language/common/languageService.ts | 4 ++-- 22 files changed, 42 insertions(+), 42 deletions(-) diff --git a/src/vs/platform/extensionManagement/common/implicitActivationEvents.ts b/src/vs/platform/extensionManagement/common/implicitActivationEvents.ts index b8a52b5e9aa..7fdeb16a04b 100644 --- a/src/vs/platform/extensionManagement/common/implicitActivationEvents.ts +++ b/src/vs/platform/extensionManagement/common/implicitActivationEvents.ts @@ -7,7 +7,7 @@ import { onUnexpectedError } from '../../../base/common/errors.js'; import { ExtensionIdentifier, IExtensionDescription } from '../../extensions/common/extensions.js'; export interface IActivationEventsGenerator { - (contributions: T[], result: { push(item: string): void }): void; + (contributions: readonly T[]): Iterable; } export class ImplicitActivationEventsImpl { @@ -73,7 +73,7 @@ export class ImplicitActivationEventsImpl { const contrib = (desc.contributes as any)[extPointName]; const contribArr = Array.isArray(contrib) ? contrib : [contrib]; try { - generator(contribArr, activationEvents); + activationEvents.push(...generator(contribArr)); } catch (err) { onUnexpectedError(err); } diff --git a/src/vs/platform/extensions/common/extensionValidator.ts b/src/vs/platform/extensions/common/extensionValidator.ts index 87a9288104d..0683f680cb7 100644 --- a/src/vs/platform/extensions/common/extensionValidator.ts +++ b/src/vs/platform/extensions/common/extensionValidator.ts @@ -417,7 +417,7 @@ function isVersionValid(currentVersion: string, date: ProductDate, requestedVers return true; } -function isStringArray(arr: string[]): boolean { +function isStringArray(arr: readonly string[]): boolean { if (!Array.isArray(arr)) { return false; } diff --git a/src/vs/platform/extensions/common/extensions.ts b/src/vs/platform/extensions/common/extensions.ts index bf4bb318095..107e54a8dc2 100644 --- a/src/vs/platform/extensions/common/extensions.ts +++ b/src/vs/platform/extensions/common/extensions.ts @@ -299,7 +299,7 @@ export interface IRelaxedExtensionManifest { icon?: string; categories?: string[]; keywords?: string[]; - activationEvents?: string[]; + activationEvents?: readonly string[]; extensionDependencies?: string[]; extensionPack?: string[]; extensionKind?: ExtensionKind | ExtensionKind[]; diff --git a/src/vs/workbench/api/browser/viewsExtensionPoint.ts b/src/vs/workbench/api/browser/viewsExtensionPoint.ts index fdf354d1f54..c98aada78a2 100644 --- a/src/vs/workbench/api/browser/viewsExtensionPoint.ts +++ b/src/vs/workbench/api/browser/viewsExtensionPoint.ts @@ -254,12 +254,12 @@ const viewsExtensionPoint: IExtensionPoint = ExtensionsR extensionPoint: 'views', deps: [viewsContainersExtensionPoint], jsonSchema: viewsContribution, - activationEventsGenerator: (viewExtensionPointTypeArray, result) => { + activationEventsGenerator: function* (viewExtensionPointTypeArray) { for (const viewExtensionPointType of viewExtensionPointTypeArray) { for (const viewDescriptors of Object.values(viewExtensionPointType)) { for (const viewDescriptor of viewDescriptors) { if (viewDescriptor.id) { - result.push(`onView:${viewDescriptor.id}`); + yield `onView:${viewDescriptor.id}`; } } } diff --git a/src/vs/workbench/api/test/common/extHostExtensionActivator.test.ts b/src/vs/workbench/api/test/common/extHostExtensionActivator.test.ts index d7a3c6f68e8..36ad00f99db 100644 --- a/src/vs/workbench/api/test/common/extHostExtensionActivator.test.ts +++ b/src/vs/workbench/api/test/common/extHostExtensionActivator.test.ts @@ -279,7 +279,7 @@ suite('ExtensionsActivator', () => { const basicActivationEventsReader: IActivationEventsReader = { readActivationEvents: (extensionDescription: IExtensionDescription): string[] => { - return extensionDescription.activationEvents ?? []; + return extensionDescription.activationEvents?.slice() ?? []; } }; diff --git a/src/vs/workbench/contrib/chat/browser/chatOutputItemRenderer.ts b/src/vs/workbench/contrib/chat/browser/chatOutputItemRenderer.ts index ecf3fea2687..b6f7ee9ffb6 100644 --- a/src/vs/workbench/contrib/chat/browser/chatOutputItemRenderer.ts +++ b/src/vs/workbench/contrib/chat/browser/chatOutputItemRenderer.ts @@ -178,9 +178,9 @@ interface IChatOutputRendererContribution { const chatOutputRenderContributionPoint = ExtensionsRegistry.registerExtensionPoint({ extensionPoint: 'chatOutputRenderers', - activationEventsGenerator: (contributions: IChatOutputRendererContribution[], result) => { + activationEventsGenerator: function* (contributions: readonly IChatOutputRendererContribution[]) { for (const contrib of contributions) { - result.push(`onChatOutputRenderer:${contrib.viewType}`); + yield `onChatOutputRenderer:${contrib.viewType}`; } }, jsonSchema: { diff --git a/src/vs/workbench/contrib/chat/browser/chatParticipant.contribution.ts b/src/vs/workbench/contrib/chat/browser/chatParticipant.contribution.ts index 42d8560ece4..aee5354bb5c 100644 --- a/src/vs/workbench/contrib/chat/browser/chatParticipant.contribution.ts +++ b/src/vs/workbench/contrib/chat/browser/chatParticipant.contribution.ts @@ -203,9 +203,9 @@ const chatParticipantExtensionPoint = extensionsRegistry.ExtensionsRegistry.regi } } }, - activationEventsGenerator: (contributions: IRawChatParticipantContribution[], result: { push(item: string): void }) => { + activationEventsGenerator: function* (contributions: readonly IRawChatParticipantContribution[]) { for (const contrib of contributions) { - result.push(`onChatParticipant:${contrib.id}`); + yield `onChatParticipant:${contrib.id}`; } }, }); diff --git a/src/vs/workbench/contrib/chat/browser/chatSessions.contribution.ts b/src/vs/workbench/contrib/chat/browser/chatSessions.contribution.ts index a9cb6a93f7f..69120bd3549 100644 --- a/src/vs/workbench/contrib/chat/browser/chatSessions.contribution.ts +++ b/src/vs/workbench/contrib/chat/browser/chatSessions.contribution.ts @@ -58,9 +58,9 @@ const extensionPoint = ExtensionsRegistry.registerExtensionPoint { + activationEventsGenerator: function* (contribs) { for (const contrib of contribs) { - results.push(`onChatSession:${contrib.type}`); + yield `onChatSession:${contrib.type}`; } } }); diff --git a/src/vs/workbench/contrib/chat/common/languageModels.ts b/src/vs/workbench/contrib/chat/common/languageModels.ts index d4678314284..b325f18c721 100644 --- a/src/vs/workbench/contrib/chat/common/languageModels.ts +++ b/src/vs/workbench/contrib/chat/common/languageModels.ts @@ -290,9 +290,9 @@ export const languageModelExtensionPoint = ExtensionsRegistry.registerExtensionP } ] }, - activationEventsGenerator: (contribs: IUserFriendlyLanguageModel[], result: { push(item: string): void }) => { + activationEventsGenerator: function* (contribs: readonly IUserFriendlyLanguageModel[]) { for (const contrib of contribs) { - result.push(`onLanguageModelChat:${contrib.vendor}`); + yield `onLanguageModelChat:${contrib.vendor}`; } } }); diff --git a/src/vs/workbench/contrib/chat/common/tools/languageModelToolsContribution.ts b/src/vs/workbench/contrib/chat/common/tools/languageModelToolsContribution.ts index ae6788c7a7e..c86f7b719f9 100644 --- a/src/vs/workbench/contrib/chat/common/tools/languageModelToolsContribution.ts +++ b/src/vs/workbench/contrib/chat/common/tools/languageModelToolsContribution.ts @@ -39,9 +39,9 @@ export interface IRawToolContribution { const languageModelToolsExtensionPoint = extensionsRegistry.ExtensionsRegistry.registerExtensionPoint({ extensionPoint: 'languageModelTools', - activationEventsGenerator: (contributions: IRawToolContribution[], result) => { + activationEventsGenerator: function* (contributions: readonly IRawToolContribution[]) { for (const contrib of contributions) { - result.push(`onLanguageModelTool:${contrib.name}`); + yield `onLanguageModelTool:${contrib.name}`; } }, jsonSchema: { diff --git a/src/vs/workbench/contrib/customEditor/common/extensionPoint.ts b/src/vs/workbench/contrib/customEditor/common/extensionPoint.ts index d2e06733cfb..68f480d0f55 100644 --- a/src/vs/workbench/contrib/customEditor/common/extensionPoint.ts +++ b/src/vs/workbench/contrib/customEditor/common/extensionPoint.ts @@ -96,11 +96,11 @@ export const customEditorsExtensionPoint = ExtensionsRegistry.registerExtensionP extensionPoint: 'customEditors', deps: [languagesExtPoint], jsonSchema: CustomEditorsContribution, - activationEventsGenerator: (contribs: ICustomEditorsExtensionPoint[], result: { push(item: string): void }) => { + activationEventsGenerator: function* (contribs: readonly ICustomEditorsExtensionPoint[]) { for (const contrib of contribs) { const viewType = contrib[Fields.viewType]; if (viewType) { - result.push(`onCustomEditor:${viewType}`); + yield `onCustomEditor:${viewType}`; } } }, diff --git a/src/vs/workbench/contrib/debug/common/debugVisualizers.ts b/src/vs/workbench/contrib/debug/common/debugVisualizers.ts index c607fbe166b..f3ae72a2703 100644 --- a/src/vs/workbench/contrib/debug/common/debugVisualizers.ts +++ b/src/vs/workbench/contrib/debug/common/debugVisualizers.ts @@ -291,10 +291,10 @@ const visualizersExtensionPoint = ExtensionsRegistry.registerExtensionPoint<{ id required: ['id', 'when'] } }, - activationEventsGenerator: (contribs, result: { push(item: string): void }) => { + activationEventsGenerator: function* (contribs) { for (const contrib of contribs) { if (contrib.id) { - result.push(`onDebugVisualizer:${contrib.id}`); + yield `onDebugVisualizer:${contrib.id}`; } } } diff --git a/src/vs/workbench/contrib/mcp/common/mcpConfiguration.ts b/src/vs/workbench/contrib/mcp/common/mcpConfiguration.ts index e1af22069c9..fb06e524c45 100644 --- a/src/vs/workbench/contrib/mcp/common/mcpConfiguration.ts +++ b/src/vs/workbench/contrib/mcp/common/mcpConfiguration.ts @@ -221,10 +221,10 @@ export const mcpServerSchema: IJSONSchema = { export const mcpContributionPoint: IExtensionPointDescriptor = { extensionPoint: 'mcpServerDefinitionProviders', - activationEventsGenerator(contribs, result) { + activationEventsGenerator: function* (contribs) { for (const contrib of contribs) { if (contrib.id) { - result.push(mcpActivationEvent(contrib.id)); + yield mcpActivationEvent(contrib.id); } } }, diff --git a/src/vs/workbench/contrib/notebook/browser/notebookExtensionPoint.ts b/src/vs/workbench/contrib/notebook/browser/notebookExtensionPoint.ts index 352dea887c8..6de1d23298e 100644 --- a/src/vs/workbench/contrib/notebook/browser/notebookExtensionPoint.ts +++ b/src/vs/workbench/contrib/notebook/browser/notebookExtensionPoint.ts @@ -245,10 +245,10 @@ const notebookPreloadContribution: IJSONSchema = { export const notebooksExtensionPoint = ExtensionsRegistry.registerExtensionPoint({ extensionPoint: 'notebooks', jsonSchema: notebookProviderContribution, - activationEventsGenerator: (contribs: INotebookEditorContribution[], result: { push(item: string): void }) => { + activationEventsGenerator: function* (contribs: readonly INotebookEditorContribution[]) { for (const contrib of contribs) { if (contrib.type) { - result.push(`onNotebookSerializer:${contrib.type}`); + yield `onNotebookSerializer:${contrib.type}`; } } } @@ -257,10 +257,10 @@ export const notebooksExtensionPoint = ExtensionsRegistry.registerExtensionPoint export const notebookRendererExtensionPoint = ExtensionsRegistry.registerExtensionPoint({ extensionPoint: 'notebookRenderer', jsonSchema: notebookRendererContribution, - activationEventsGenerator: (contribs: INotebookRendererContribution[], result: { push(item: string): void }) => { + activationEventsGenerator: function* (contribs: readonly INotebookRendererContribution[]) { for (const contrib of contribs) { if (contrib.id) { - result.push(`onRenderer:${contrib.id}`); + yield `onRenderer:${contrib.id}`; } } } diff --git a/src/vs/workbench/contrib/tasks/common/taskDefinitionRegistry.ts b/src/vs/workbench/contrib/tasks/common/taskDefinitionRegistry.ts index b44773c7dce..1605209c7f1 100644 --- a/src/vs/workbench/contrib/tasks/common/taskDefinitionRegistry.ts +++ b/src/vs/workbench/contrib/tasks/common/taskDefinitionRegistry.ts @@ -83,10 +83,10 @@ namespace Configuration { const taskDefinitionsExtPoint = ExtensionsRegistry.registerExtensionPoint({ extensionPoint: 'taskDefinitions', - activationEventsGenerator: (contributions: Configuration.ITaskDefinition[], result: { push(item: string): void }) => { + activationEventsGenerator: function* (contributions: readonly Configuration.ITaskDefinition[]) { for (const task of contributions) { if (task.type) { - result.push(`onTaskType:${task.type}`); + yield `onTaskType:${task.type}`; } } }, diff --git a/src/vs/workbench/contrib/terminal/common/terminal.ts b/src/vs/workbench/contrib/terminal/common/terminal.ts index 36d19fac59c..d5f17f977ed 100644 --- a/src/vs/workbench/contrib/terminal/common/terminal.ts +++ b/src/vs/workbench/contrib/terminal/common/terminal.ts @@ -634,10 +634,10 @@ export const DEFAULT_COMMANDS_TO_SKIP_SHELL: string[] = [ export const terminalContributionsDescriptor: IExtensionPointDescriptor = { extensionPoint: 'terminal', defaultExtensionKind: ['workspace'], - activationEventsGenerator: (contribs: ITerminalContributions[], result: { push(item: string): void }) => { + activationEventsGenerator: function* (contribs: readonly ITerminalContributions[]) { for (const contrib of contribs) { for (const profileContrib of (contrib.profiles ?? [])) { - result.push(`onTerminalProfile:${profileContrib.id}`); + yield `onTerminalProfile:${profileContrib.id}`; } } }, diff --git a/src/vs/workbench/contrib/terminalContrib/quickFix/browser/terminalQuickFixService.ts b/src/vs/workbench/contrib/terminalContrib/quickFix/browser/terminalQuickFixService.ts index 4a095d55e76..f3fed767041 100644 --- a/src/vs/workbench/contrib/terminalContrib/quickFix/browser/terminalQuickFixService.ts +++ b/src/vs/workbench/contrib/terminalContrib/quickFix/browser/terminalQuickFixService.ts @@ -94,9 +94,9 @@ export class TerminalQuickFixService implements ITerminalQuickFixService { const quickFixExtensionPoint = ExtensionsRegistry.registerExtensionPoint({ extensionPoint: 'terminalQuickFixes', defaultExtensionKind: ['workspace'], - activationEventsGenerator: (terminalQuickFixes: ITerminalCommandSelector[], result: { push(item: string): void }) => { + activationEventsGenerator: function* (terminalQuickFixes: readonly ITerminalCommandSelector[]) { for (const quickFixContrib of terminalQuickFixes ?? []) { - result.push(`onTerminalQuickFixRequest:${quickFixContrib.id}`); + yield `onTerminalQuickFixRequest:${quickFixContrib.id}`; } }, jsonSchema: { diff --git a/src/vs/workbench/contrib/welcomeGettingStarted/browser/gettingStartedExtensionPoint.ts b/src/vs/workbench/contrib/welcomeGettingStarted/browser/gettingStartedExtensionPoint.ts index 3f1c09885f9..297598efec2 100644 --- a/src/vs/workbench/contrib/welcomeGettingStarted/browser/gettingStartedExtensionPoint.ts +++ b/src/vs/workbench/contrib/welcomeGettingStarted/browser/gettingStartedExtensionPoint.ts @@ -218,10 +218,10 @@ export const walkthroughsExtensionPoint = ExtensionsRegistry.registerExtensionPo } } }, - activationEventsGenerator: (walkthroughContributions, result) => { + activationEventsGenerator: function* (walkthroughContributions) { for (const walkthroughContribution of walkthroughContributions) { if (walkthroughContribution.id) { - result.push(`onWalkthrough:${walkthroughContribution.id}`); + yield `onWalkthrough:${walkthroughContribution.id}`; } } } diff --git a/src/vs/workbench/services/actions/common/menusExtensionPoint.ts b/src/vs/workbench/services/actions/common/menusExtensionPoint.ts index 9d8ac1dd3e9..30d7fe62529 100644 --- a/src/vs/workbench/services/actions/common/menusExtensionPoint.ts +++ b/src/vs/workbench/services/actions/common/menusExtensionPoint.ts @@ -805,10 +805,10 @@ const _commandRegistrations = new DisposableStore(); export const commandsExtensionPoint = ExtensionsRegistry.registerExtensionPoint({ extensionPoint: 'commands', jsonSchema: schema.commandsContribution, - activationEventsGenerator: (contribs: schema.IUserFriendlyCommand[], result: { push(item: string): void }) => { + activationEventsGenerator: function* (contribs: readonly schema.IUserFriendlyCommand[]) { for (const contrib of contribs) { if (contrib.command) { - result.push(`onCommand:${contrib.command}`); + yield `onCommand:${contrib.command}`; } } } diff --git a/src/vs/workbench/services/authentication/browser/authenticationService.ts b/src/vs/workbench/services/authentication/browser/authenticationService.ts index fc0dac3399c..15e117de20a 100644 --- a/src/vs/workbench/services/authentication/browser/authenticationService.ts +++ b/src/vs/workbench/services/authentication/browser/authenticationService.ts @@ -81,10 +81,10 @@ const authenticationExtPoint = ExtensionsRegistry.registerExtensionPoint { + activationEventsGenerator: function* (authenticationProviders) { for (const authenticationProvider of authenticationProviders) { if (authenticationProvider.id) { - result.push(`onAuthenticationRequest:${authenticationProvider.id}`); + yield `onAuthenticationRequest:${authenticationProvider.id}`; } } } diff --git a/src/vs/workbench/services/extensions/test/common/extensionDescriptionRegistry.test.ts b/src/vs/workbench/services/extensions/test/common/extensionDescriptionRegistry.test.ts index 24348eb2260..ebc8563afc4 100644 --- a/src/vs/workbench/services/extensions/test/common/extensionDescriptionRegistry.test.ts +++ b/src/vs/workbench/services/extensions/test/common/extensionDescriptionRegistry.test.ts @@ -20,7 +20,7 @@ suite('ExtensionDescriptionRegistry', () => { const basicActivationEventsReader: IActivationEventsReader = { readActivationEvents: (extensionDescription: IExtensionDescription): string[] => { - return extensionDescription.activationEvents ?? []; + return extensionDescription.activationEvents?.slice() ?? []; } }; diff --git a/src/vs/workbench/services/language/common/languageService.ts b/src/vs/workbench/services/language/common/languageService.ts index 1ab9a920338..f25eaf50de3 100644 --- a/src/vs/workbench/services/language/common/languageService.ts +++ b/src/vs/workbench/services/language/common/languageService.ts @@ -112,10 +112,10 @@ export const languagesExtPoint: IExtensionPoint = } } }, - activationEventsGenerator: (languageContributions, result) => { + activationEventsGenerator: function* (languageContributions) { for (const languageContribution of languageContributions) { if (languageContribution.id && languageContribution.configuration) { - result.push(`onLanguage:${languageContribution.id}`); + yield `onLanguage:${languageContribution.id}`; } } } From 01b144481d168b122a3f702e3f66afde3033e582 Mon Sep 17 00:00:00 2001 From: Andrew Howson Date: Fri, 8 Aug 2025 14:18:38 +0100 Subject: [PATCH 002/299] When suggest box is too tall, ensure max height and position use larger of above and below spaces --- src/vs/editor/contrib/suggest/browser/suggestWidget.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vs/editor/contrib/suggest/browser/suggestWidget.ts b/src/vs/editor/contrib/suggest/browser/suggestWidget.ts index 59f517623a2..fac1fb1de21 100644 --- a/src/vs/editor/contrib/suggest/browser/suggestWidget.ts +++ b/src/vs/editor/contrib/suggest/browser/suggestWidget.ts @@ -873,7 +873,7 @@ export class SuggestWidget implements IDisposable { } const forceRenderingAboveRequiredSpace = 150; - if (height > maxHeightBelow || (this._forceRenderingAbove && availableSpaceAbove > forceRenderingAboveRequiredSpace)) { + if ((height > maxHeightBelow && maxHeightAbove > maxHeightBelow) || (this._forceRenderingAbove && availableSpaceAbove > forceRenderingAboveRequiredSpace)) { this._contentWidget.setPreference(ContentWidgetPositionPreference.ABOVE); this.element.enableSashes(true, true, false, false); maxHeight = maxHeightAbove; From 6a45a1ad8354fc9efd024094a1a66d2c2d49a2b2 Mon Sep 17 00:00:00 2001 From: Martin Aeschlimann Date: Thu, 28 Aug 2025 16:54:08 +0200 Subject: [PATCH 003/299] new prompt parser --- src/vs/base/common/yaml.ts | 829 +++++++++++++ src/vs/base/test/common/yaml.test.ts | 1083 +++++++++++++++++ .../promptSyntax/service/newPromptsParser.ts | 94 ++ .../promptSyntax/service/promptsService.ts | 10 +- .../service/promptsServiceImpl.ts | 4 +- 5 files changed, 2016 insertions(+), 4 deletions(-) create mode 100644 src/vs/base/common/yaml.ts create mode 100644 src/vs/base/test/common/yaml.test.ts create mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/service/newPromptsParser.ts diff --git a/src/vs/base/common/yaml.ts b/src/vs/base/common/yaml.ts new file mode 100644 index 00000000000..b6b977e5897 --- /dev/null +++ b/src/vs/base/common/yaml.ts @@ -0,0 +1,829 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +/** + * Parses a simplified YAML-like input from an iterable of strings (lines). + * Supports objects, arrays, primitive types (string, number, boolean, null). + * Tracks positions for error reporting and node locations. + * + * Limitations: + * - No multi-line strings or block literals + * - No anchors or references + * - No complex types (dates, binary) + * - No special handling for escape sequences in strings + * - Indentation must be consistent (spaces only, no tabs) + * + * @param input Iterable of strings representing lines of the YAML-like input + * @param errors Array to collect parsing errors + * @param options Parsing options + * @returns The parsed representation (ObjectNode, ArrayNode, or primitive node) + */ +export function parse(input: Iterable, errors: YamlParseError[] = [], options: ParseOptions = {}): YamlNode | undefined { + const lines = Array.from(input); + const parser = new YamlParser(lines, errors, options); + return parser.parse(); +} + +export interface YamlParseError { + readonly message: string; + readonly start: Position; + readonly end: Position; + readonly code: string; +} + +export interface ParseOptions { + readonly allowDuplicateKeys?: boolean; +} + +export interface Position { + readonly line: number; + readonly character: number; +} + +export interface YamlStringNode { + readonly type: 'string'; + readonly value: string; + readonly start: Position; + readonly end: Position; +} + +export interface YamlNumberNode { + readonly type: 'number'; + readonly value: number; + readonly start: Position; + readonly end: Position; +} + +export interface YamlBooleanNode { + readonly type: 'boolean'; + readonly value: boolean; + readonly start: Position; + readonly end: Position; +} + +export interface YamlNullNode { + readonly type: 'null'; + readonly value: null; + readonly start: Position; + readonly end: Position; +} + +export interface YamlObjectNode { + readonly type: 'object'; + readonly properties: { key: YamlStringNode; value: YamlNode }[]; + readonly start: Position; + readonly end: Position; +} + +export interface YamlArrayNode { + readonly type: 'array'; + readonly items: YamlNode[]; + readonly start: Position; + readonly end: Position; +} + +export type YamlNode = YamlStringNode | YamlNumberNode | YamlBooleanNode | YamlNullNode | YamlObjectNode | YamlArrayNode; + +// Helper functions for position and node creation +function createPosition(line: number, character: number): Position { + return { line, character }; +} + +// Specialized node creation functions using a more concise approach +function createStringNode(value: string, start: Position, end: Position): YamlStringNode { + return { type: 'string', value, start, end }; +} + +function createNumberNode(value: number, start: Position, end: Position): YamlNumberNode { + return { type: 'number', value, start, end }; +} + +function createBooleanNode(value: boolean, start: Position, end: Position): YamlBooleanNode { + return { type: 'boolean', value, start, end }; +} + +function createNullNode(start: Position, end: Position): YamlNullNode { + return { type: 'null', value: null, start, end }; +} + +function createObjectNode(properties: { key: YamlStringNode; value: YamlNode }[], start: Position, end: Position): YamlObjectNode { + return { type: 'object', start, end, properties }; +} + +function createArrayNode(items: YamlNode[], start: Position, end: Position): YamlArrayNode { + return { type: 'array', start, end, items }; +} + +// Utility functions for parsing +function isWhitespace(char: string): boolean { + return char === ' ' || char === '\t'; +} + +// Simplified number validation using regex +function isValidNumber(value: string): boolean { + return /^-?\d*\.?\d+$/.test(value); +} + +// Lexer/Tokenizer for YAML content +class YamlLexer { + private lines: string[]; + private currentLine: number = 0; + private currentChar: number = 0; + + constructor(lines: string[]) { + this.lines = lines; + } + + getCurrentPosition(): Position { + return createPosition(this.currentLine, this.currentChar); + } + + getCurrentLineNumber(): number { + return this.currentLine; + } + + getCurrentCharNumber(): number { + return this.currentChar; + } + + getCurrentLineText(): string { + return this.currentLine < this.lines.length ? this.lines[this.currentLine] : ''; + } + + savePosition(): { line: number; char: number } { + return { line: this.currentLine, char: this.currentChar }; + } + + restorePosition(pos: { line: number; char: number }): void { + this.currentLine = pos.line; + this.currentChar = pos.char; + } + + isAtEnd(): boolean { + return this.currentLine >= this.lines.length; + } + + getCurrentChar(): string { + if (this.isAtEnd() || this.currentChar >= this.lines[this.currentLine].length) { + return ''; + } + return this.lines[this.currentLine][this.currentChar]; + } + + peek(offset: number = 1): string { + const newChar = this.currentChar + offset; + if (this.currentLine >= this.lines.length || newChar >= this.lines[this.currentLine].length) { + return ''; + } + return this.lines[this.currentLine][newChar]; + } + + advance(): string { + const char = this.getCurrentChar(); + if (this.currentChar >= this.lines[this.currentLine].length && this.currentLine < this.lines.length - 1) { + this.currentLine++; + this.currentChar = 0; + } else { + this.currentChar++; + } + return char; + } + + advanceLine(): void { + this.currentLine++; + this.currentChar = 0; + } + + skipWhitespace(): void { + while (!this.isAtEnd() && this.currentChar < this.lines[this.currentLine].length && isWhitespace(this.getCurrentChar())) { + this.advance(); + } + } + + skipToEndOfLine(): void { + this.currentChar = this.lines[this.currentLine].length; + } + + getIndentation(): number { + if (this.isAtEnd()) { + return 0; + } + let indent = 0; + for (let i = 0; i < this.lines[this.currentLine].length; i++) { + if (this.lines[this.currentLine][i] === ' ') { + indent++; + } else if (this.lines[this.currentLine][i] === '\t') { + indent += 4; // Treat tab as 4 spaces + } else { + break; + } + } + return indent; + } + + moveToNextNonEmptyLine(): void { + while (this.currentLine < this.lines.length) { + // First check current line from current position + if (this.currentChar < this.lines[this.currentLine].length) { + const remainingLine = this.lines[this.currentLine].substring(this.currentChar).trim(); + if (remainingLine.length > 0 && !remainingLine.startsWith('#')) { + this.skipWhitespace(); + return; + } + } + + // Move to next line and check from beginning + this.currentLine++; + this.currentChar = 0; + + if (this.currentLine < this.lines.length) { + const line = this.lines[this.currentLine].trim(); + if (line.length > 0 && !line.startsWith('#')) { + this.skipWhitespace(); + return; + } + } + } + } +} + +// Parser class for handling YAML parsing +class YamlParser { + private lexer: YamlLexer; + private errors: YamlParseError[]; + private options: ParseOptions; + + constructor(lines: string[], errors: YamlParseError[], options: ParseOptions) { + this.lexer = new YamlLexer(lines); + this.errors = errors; + this.options = options; + } + + addError(message: string, code: string, start: Position, end: Position): void { + this.errors.push({ message, code, start, end }); + } + + parseValue(expectedIndent?: number): YamlNode { + this.lexer.skipWhitespace(); + + if (this.lexer.isAtEnd()) { + const pos = this.lexer.getCurrentPosition(); + return createStringNode('', pos, pos); + } + + const char = this.lexer.getCurrentChar(); + + // Handle quoted strings + if (char === '"' || char === `'`) { + return this.parseQuotedString(char); + } + + // Handle inline arrays + if (char === '[') { + return this.parseInlineArray(); + } + + // Handle inline objects + if (char === '{') { + return this.parseInlineObject(); + } + + // Handle unquoted values + return this.parseUnquotedValue(); + } + + parseQuotedString(quote: string): YamlNode { + const start = this.lexer.getCurrentPosition(); + this.lexer.advance(); // Skip opening quote + + let value = ''; + while (!this.lexer.isAtEnd() && this.lexer.getCurrentChar() !== '' && this.lexer.getCurrentChar() !== quote) { + value += this.lexer.advance(); + } + + if (this.lexer.getCurrentChar() === quote) { + this.lexer.advance(); // Skip closing quote + } + + const end = this.lexer.getCurrentPosition(); + return createStringNode(value, start, end); + } + + parseUnquotedValue(): YamlNode { + const start = this.lexer.getCurrentPosition(); + let value = ''; + let endPos = start; + + // Helper function to check for value terminators + const isTerminator = (char: string): boolean => + char === '#' || char === ',' || char === ']' || char === '}'; + + // Handle opening quote that might not be closed + const firstChar = this.lexer.getCurrentChar(); + if (firstChar === '"' || firstChar === `'`) { + value += this.lexer.advance(); + endPos = this.lexer.getCurrentPosition(); + + // Continue until we find closing quote or terminator + while (!this.lexer.isAtEnd() && this.lexer.getCurrentChar() !== '') { + const char = this.lexer.getCurrentChar(); + + if (char === firstChar || isTerminator(char)) { + break; + } + + value += this.lexer.advance(); + endPos = this.lexer.getCurrentPosition(); + } + } else { + // Regular unquoted value + while (!this.lexer.isAtEnd() && this.lexer.getCurrentChar() !== '') { + const char = this.lexer.getCurrentChar(); + + if (isTerminator(char)) { + break; + } + + value += this.lexer.advance(); + endPos = this.lexer.getCurrentPosition(); + } + } + + value = value.trim(); + + // Adjust end position for trimmed value + if (value.length === 0) { + endPos = start; + } else { + endPos = createPosition(start.line, start.character + value.length); + } + + // Return appropriate node type based on value + return this.createValueNode(value, start, endPos); + } + + private createValueNode(value: string, start: Position, end: Position): YamlNode { + if (value === '') { + return createStringNode('', start, start); + } + + // Boolean values + if (value === 'true') { + return createBooleanNode(true, start, end); + } + if (value === 'false') { + return createBooleanNode(false, start, end); + } + + // Null values + if (value === 'null' || value === '~') { + return createNullNode(start, end); + } + + // Number values + const numberValue = Number(value); + if (!isNaN(numberValue) && isFinite(numberValue) && isValidNumber(value)) { + return createNumberNode(numberValue, start, end); + } + + // Default to string + return createStringNode(value, start, end); + } + + parseInlineArray(): YamlArrayNode { + const start = this.lexer.getCurrentPosition(); + this.lexer.advance(); // Skip '[' + + const items: YamlNode[] = []; + + while (!this.lexer.isAtEnd()) { + this.lexer.skipWhitespace(); + + // Handle end of array + if (this.lexer.getCurrentChar() === ']') { + this.lexer.advance(); + break; + } + + // Handle end of line - continue to next line for multi-line arrays + if (this.lexer.getCurrentChar() === '') { + this.lexer.advanceLine(); + continue; + } + + // Parse array item + const item = this.parseValue(); + items.push(item); + + this.lexer.skipWhitespace(); + + // Handle comma separator + if (this.lexer.getCurrentChar() === ',') { + this.lexer.advance(); + } + } + + const end = this.lexer.getCurrentPosition(); + return createArrayNode(items, start, end); + } + + parseInlineObject(): YamlObjectNode { + const start = this.lexer.getCurrentPosition(); + this.lexer.advance(); // Skip '{' + + const properties: { key: YamlStringNode; value: YamlNode }[] = []; + + while (!this.lexer.isAtEnd()) { + this.lexer.skipWhitespace(); + + // Handle end of object + if (this.lexer.getCurrentChar() === '}') { + this.lexer.advance(); + break; + } + + // Parse key - read until colon + const keyStart = this.lexer.getCurrentPosition(); + let keyValue = ''; + + // Handle quoted keys + if (this.lexer.getCurrentChar() === '"' || this.lexer.getCurrentChar() === `'`) { + const quote = this.lexer.getCurrentChar(); + this.lexer.advance(); // Skip opening quote + + while (!this.lexer.isAtEnd() && this.lexer.getCurrentChar() !== '' && this.lexer.getCurrentChar() !== quote) { + keyValue += this.lexer.advance(); + } + + if (this.lexer.getCurrentChar() === quote) { + this.lexer.advance(); // Skip closing quote + } + } else { + // Handle unquoted keys - read until colon + while (!this.lexer.isAtEnd() && this.lexer.getCurrentChar() !== '' && this.lexer.getCurrentChar() !== ':') { + keyValue += this.lexer.advance(); + } + } + + keyValue = keyValue.trim(); + const keyEnd = this.lexer.getCurrentPosition(); + const key = createStringNode(keyValue, keyStart, keyEnd); + + this.lexer.skipWhitespace(); + + // Expect colon + if (this.lexer.getCurrentChar() === ':') { + this.lexer.advance(); + } + + this.lexer.skipWhitespace(); + + // Parse value + const value = this.parseValue(); + + properties.push({ key, value }); + + this.lexer.skipWhitespace(); + + // Handle comma separator + if (this.lexer.getCurrentChar() === ',') { + this.lexer.advance(); + } + } + + const end = this.lexer.getCurrentPosition(); + return createObjectNode(properties, start, end); + } + + parseBlockArray(baseIndent: number): YamlArrayNode { + const start = this.lexer.getCurrentPosition(); + const items: YamlNode[] = []; + + while (!this.lexer.isAtEnd()) { + this.lexer.moveToNextNonEmptyLine(); + + if (this.lexer.isAtEnd()) { + break; + } + + const currentIndent = this.lexer.getIndentation(); + + // If indentation is less than expected, we're done with this array + if (currentIndent < baseIndent) { + break; + } + + this.lexer.skipWhitespace(); + + // Check for array item marker + if (this.lexer.getCurrentChar() === '-') { + this.lexer.advance(); // Skip '-' + this.lexer.skipWhitespace(); + + const itemStart = this.lexer.getCurrentPosition(); + + // Check if this is a nested structure + if (this.lexer.getCurrentChar() === '' || this.lexer.getCurrentChar() === '#') { + // Empty item - check if next lines form a nested structure + this.lexer.advanceLine(); + + if (!this.lexer.isAtEnd()) { + const nextIndent = this.lexer.getIndentation(); + + if (nextIndent > currentIndent) { + // Check if the next line starts with a dash (nested array) or has properties (nested object) + this.lexer.skipWhitespace(); + if (this.lexer.getCurrentChar() === '-') { + // It's a nested array + const nestedArray = this.parseBlockArray(nextIndent); + items.push(nestedArray); + } else { + // Check if it looks like an object property (has a colon) + const currentLine = this.lexer.getCurrentLineText(); + const currentPos = this.lexer.getCurrentCharNumber(); + const remainingLine = currentLine.substring(currentPos); + + if (remainingLine.includes(':') && !remainingLine.trim().startsWith('#')) { + // It's a nested object + const nestedObject = this.parseBlockObject(nextIndent, this.lexer.getCurrentCharNumber()); + items.push(nestedObject); + } else { + // Not a nested structure, create empty string + items.push(createStringNode('', itemStart, itemStart)); + } + } + } else { + // No nested content, empty item + items.push(createStringNode('', itemStart, itemStart)); + } + } else { + // End of input, empty item + items.push(createStringNode('', itemStart, itemStart)); + } + } else { + // Parse the item value + // Check if this is a multi-line object by looking for a colon and checking next lines + const currentLine = this.lexer.getCurrentLineText(); + const currentPos = this.lexer.getCurrentCharNumber(); + const remainingLine = currentLine.substring(currentPos); + + // Check if there's a colon on this line (indicating object properties) + const hasColon = remainingLine.includes(':'); + + if (hasColon) { + // Any line with a colon should be treated as an object + // Parse as an object with the current item's indentation as the base + const item = this.parseBlockObject(itemStart.character, itemStart.character); + items.push(item); + } else { + // No colon, parse as regular value + const item = this.parseValue(); + items.push(item); + + // Skip to end of line + while (!this.lexer.isAtEnd() && this.lexer.getCurrentChar() !== '' && this.lexer.getCurrentChar() !== '#') { + this.lexer.advance(); + } + this.lexer.advanceLine(); + } + } + } else { + // No dash found at expected indent level, break + break; + } + } + + // Calculate end position based on the last item + let end = start; + if (items.length > 0) { + const lastItem = items[items.length - 1]; + end = lastItem.end; + } else { + // If no items, end is right after the start + end = createPosition(start.line, start.character + 1); + } + + return createArrayNode(items, start, end); + } + + parseBlockObject(baseIndent: number, baseCharPosition?: number): YamlObjectNode { + const start = this.lexer.getCurrentPosition(); + const properties: { key: YamlStringNode; value: YamlNode }[] = []; + const localKeysSeen = new Set(); + + // For parsing from current position (inline object parsing) + const fromCurrentPosition = baseCharPosition !== undefined; + let firstIteration = true; + + while (!this.lexer.isAtEnd()) { + if (!firstIteration || !fromCurrentPosition) { + this.lexer.moveToNextNonEmptyLine(); + } + firstIteration = false; + + if (this.lexer.isAtEnd()) { + break; + } + + const currentIndent = this.lexer.getIndentation(); + + if (fromCurrentPosition) { + // For current position parsing, check character position alignment + this.lexer.skipWhitespace(); + const currentCharPosition = this.lexer.getCurrentCharNumber(); + + if (currentCharPosition < baseCharPosition) { + break; + } + } else { + // For normal block parsing, check indentation level + if (currentIndent < baseIndent) { + break; + } + + // Check for incorrect indentation + if (currentIndent > baseIndent) { + const lineStart = createPosition(this.lexer.getCurrentLineNumber(), 0); + const lineEnd = createPosition(this.lexer.getCurrentLineNumber(), this.lexer.getCurrentLineText().length); + this.addError('Unexpected indentation', 'indentation', lineStart, lineEnd); + + // Try to recover by treating it as a property anyway + this.lexer.skipWhitespace(); + } else { + this.lexer.skipWhitespace(); + } + } + + // Parse key + const keyStart = this.lexer.getCurrentPosition(); + let keyValue = ''; + + while (!this.lexer.isAtEnd() && this.lexer.getCurrentChar() !== '' && this.lexer.getCurrentChar() !== ':') { + keyValue += this.lexer.advance(); + } + + keyValue = keyValue.trim(); + const keyEnd = this.lexer.getCurrentPosition(); + const key = createStringNode(keyValue, keyStart, keyEnd); + + // Check for duplicate keys + if (!this.options.allowDuplicateKeys && localKeysSeen.has(keyValue)) { + this.addError(`Duplicate key '${keyValue}'`, 'duplicateKey', keyStart, keyEnd); + } + localKeysSeen.add(keyValue); + + // Expect colon + if (this.lexer.getCurrentChar() === ':') { + this.lexer.advance(); + } + + this.lexer.skipWhitespace(); + + // Determine if value is on same line or next line(s) + let value: YamlNode; + const valueStart = this.lexer.getCurrentPosition(); + + if (this.lexer.getCurrentChar() === '' || this.lexer.getCurrentChar() === '#') { + // Value is on next line(s) or empty + this.lexer.advanceLine(); + + // Check next line for nested content + if (!this.lexer.isAtEnd()) { + const nextIndent = this.lexer.getIndentation(); + + if (nextIndent > currentIndent) { + // Nested content - determine if it's an object or array + this.lexer.skipWhitespace(); + + if (this.lexer.getCurrentChar() === '-') { + value = this.parseBlockArray(nextIndent); + } else { + value = this.parseBlockObject(nextIndent); + } + } else if (!fromCurrentPosition && nextIndent === currentIndent) { + // Same indentation level - check if it's an array item + this.lexer.skipWhitespace(); + + if (this.lexer.getCurrentChar() === '-') { + value = this.parseBlockArray(currentIndent); + } else { + value = createStringNode('', valueStart, valueStart); + } + } else { + value = createStringNode('', valueStart, valueStart); + } + } else { + value = createStringNode('', valueStart, valueStart); + } + } else { + // Value is on the same line + value = this.parseValue(); + + // Skip any remaining content on this line (comments, etc.) + while (!this.lexer.isAtEnd() && this.lexer.getCurrentChar() !== '' && this.lexer.getCurrentChar() !== '#') { + if (isWhitespace(this.lexer.getCurrentChar())) { + this.lexer.advance(); + } else { + break; + } + } + + // Skip to end of line if we hit a comment + if (this.lexer.getCurrentChar() === '#') { + this.lexer.skipToEndOfLine(); + } + + // Move to next line for next iteration + if (!this.lexer.isAtEnd() && this.lexer.getCurrentChar() === '') { + this.lexer.advanceLine(); + } + } + + properties.push({ key, value }); + } + + // Calculate the end position based on the last property + let end = start; + if (properties.length > 0) { + const lastProperty = properties[properties.length - 1]; + end = lastProperty.value.end; + } + + return createObjectNode(properties, start, end); + } + + parse(): YamlNode | undefined { + if (this.lexer.isAtEnd()) { + return undefined; + } + + this.lexer.moveToNextNonEmptyLine(); + + if (this.lexer.isAtEnd()) { + return undefined; + } + + // Determine the root structure type + this.lexer.skipWhitespace(); + + if (this.lexer.getCurrentChar() === '-') { + // Check if this is an array item or a negative number + // Look at the character after the dash + const nextChar = this.lexer.peek(); + if (nextChar === ' ' || nextChar === '\t' || nextChar === '' || nextChar === '#') { + // It's an array item (dash followed by whitespace/end/comment) + return this.parseBlockArray(0); + } else { + // It's likely a negative number or other value, treat as single value + return this.parseValue(); + } + } else if (this.lexer.getCurrentChar() === '[') { + // Root is an inline array + return this.parseInlineArray(); + } else if (this.lexer.getCurrentChar() === '{') { + // Root is an inline object + return this.parseInlineObject(); + } else { + // Check if this looks like a key-value pair by looking for a colon + // For single values, there shouldn't be a colon + const currentLine = this.lexer.getCurrentLineText(); + const currentPos = this.lexer.getCurrentCharNumber(); + const remainingLine = currentLine.substring(currentPos); + + // Check if there's a colon that's not inside quotes + let hasColon = false; + let inQuotes = false; + let quoteChar = ''; + + for (let i = 0; i < remainingLine.length; i++) { + const char = remainingLine[i]; + + if (!inQuotes && (char === '"' || char === `'`)) { + inQuotes = true; + quoteChar = char; + } else if (inQuotes && char === quoteChar) { + inQuotes = false; + quoteChar = ''; + } else if (!inQuotes && char === ':') { + hasColon = true; + break; + } else if (!inQuotes && char === '#') { + // Comment starts, stop looking + break; + } + } + + if (hasColon) { + // Root is an object + return this.parseBlockObject(0); + } else { + // Root is a single value + return this.parseValue(); + } + } + } +} + + diff --git a/src/vs/base/test/common/yaml.test.ts b/src/vs/base/test/common/yaml.test.ts new file mode 100644 index 00000000000..49ddbd2596f --- /dev/null +++ b/src/vs/base/test/common/yaml.test.ts @@ -0,0 +1,1083 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { deepStrictEqual, strictEqual, ok } from 'assert'; +import { ensureNoDisposablesAreLeakedInTestSuite } from './utils.js'; +import { parse, ParseOptions, YamlParseError, Position, YamlNode } from '../../common/yaml.js'; + + +function assertValidParse(input: string[], expected: YamlNode, expectedErrors: YamlParseError[], options?: ParseOptions): void { + const errors: YamlParseError[] = []; + const actual1 = parse(input, errors, options); + deepStrictEqual(actual1, expected); + deepStrictEqual(errors, expectedErrors); +} + +function pos(line: number, character: number): Position { + return { line, character }; +} + +suite('YAML Parser', () => { + + ensureNoDisposablesAreLeakedInTestSuite(); + + suite('scalars', () => { + + test('numbers', () => { + assertValidParse(['1'], { type: 'number', start: pos(0, 0), end: pos(0, 1), value: 1 }, []); + assertValidParse(['1.234'], { type: 'number', start: pos(0, 0), end: pos(0, 5), value: 1.234 }, []); + assertValidParse(['-42'], { type: 'number', start: pos(0, 0), end: pos(0, 3), value: -42 }, []); + }); + + test('boolean', () => { + assertValidParse(['true'], { type: 'boolean', start: pos(0, 0), end: pos(0, 4), value: true }, []); + assertValidParse(['false'], { type: 'boolean', start: pos(0, 0), end: pos(0, 5), value: false }, []); + }); + + test('null', () => { + assertValidParse(['null'], { type: 'null', start: pos(0, 0), end: pos(0, 4), value: null }, []); + assertValidParse(['~'], { type: 'null', start: pos(0, 0), end: pos(0, 1), value: null }, []); + }); + + test('string', () => { + assertValidParse(['A Developer'], { type: 'string', start: pos(0, 0), end: pos(0, 11), value: 'A Developer' }, []); + assertValidParse(['\'A Developer\''], { type: 'string', start: pos(0, 0), end: pos(0, 13), value: 'A Developer' }, []); + assertValidParse(['"A Developer"'], { type: 'string', start: pos(0, 0), end: pos(0, 13), value: 'A Developer' }, []); + }); + }); + + suite('objects', () => { + + test('simple properties', () => { + assertValidParse(['name: John Doe'], { + type: 'object', start: pos(0, 0), end: pos(0, 14), properties: [ + { + key: { type: 'string', start: pos(0, 0), end: pos(0, 4), value: 'name' }, + value: { type: 'string', start: pos(0, 6), end: pos(0, 14), value: 'John Doe' } + } + ] + }, []); + assertValidParse(['age: 30'], { + type: 'object', start: pos(0, 0), end: pos(0, 7), properties: [ + { + key: { type: 'string', start: pos(0, 0), end: pos(0, 3), value: 'age' }, + value: { type: 'number', start: pos(0, 5), end: pos(0, 7), value: 30 } + } + ] + }, []); + assertValidParse(['active: true'], { + type: 'object', start: pos(0, 0), end: pos(0, 12), properties: [ + { + key: { type: 'string', start: pos(0, 0), end: pos(0, 6), value: 'active' }, + value: { type: 'boolean', start: pos(0, 8), end: pos(0, 12), value: true } + } + ] + }, []); + assertValidParse(['value: null'], { + type: 'object', start: pos(0, 0), end: pos(0, 11), properties: [ + { + key: { type: 'string', start: pos(0, 0), end: pos(0, 5), value: 'value' }, + value: { type: 'null', start: pos(0, 7), end: pos(0, 11), value: null } + } + ] + }, []); + }); + + test('multiple properties', () => { + assertValidParse( + [ + 'name: John Doe', + 'age: 30' + ], + { + type: 'object', start: pos(0, 0), end: pos(1, 7), properties: [ + { + key: { type: 'string', start: pos(0, 0), end: pos(0, 4), value: 'name' }, + value: { type: 'string', start: pos(0, 6), end: pos(0, 14), value: 'John Doe' } + }, + { + key: { type: 'string', start: pos(1, 0), end: pos(1, 3), value: 'age' }, + value: { type: 'number', start: pos(1, 5), end: pos(1, 7), value: 30 } + } + ] + }, + [] + ); + }); + + test('nested object', () => { + assertValidParse( + [ + 'person:', + ' name: John Doe', + ' age: 30' + ], + { + type: 'object', start: pos(0, 0), end: pos(2, 9), properties: [ + { + key: { type: 'string', start: pos(0, 0), end: pos(0, 6), value: 'person' }, + value: { + type: 'object', start: pos(1, 2), end: pos(2, 9), properties: [ + { + key: { type: 'string', start: pos(1, 2), end: pos(1, 6), value: 'name' }, + value: { type: 'string', start: pos(1, 8), end: pos(1, 16), value: 'John Doe' } + }, + { + key: { type: 'string', start: pos(2, 2), end: pos(2, 5), value: 'age' }, + value: { type: 'number', start: pos(2, 7), end: pos(2, 9), value: 30 } + } + ] + } + } + ] + + }, + [] + ); + }); + + + test('nested objects with address', () => { + assertValidParse( + [ + 'person:', + ' name: John Doe', + ' age: 30', + ' address:', + ' street: 123 Main St', + ' city: Example City' + ], + { + type: 'object', start: pos(0, 0), end: pos(5, 22), properties: [ + { + key: { type: 'string', start: pos(0, 0), end: pos(0, 6), value: 'person' }, + value: { + type: 'object', start: pos(1, 2), end: pos(5, 22), + properties: [ + { + key: { type: 'string', start: pos(1, 2), end: pos(1, 6), value: 'name' }, + value: { type: 'string', start: pos(1, 8), end: pos(1, 16), value: 'John Doe' } + }, + { + key: { type: 'string', start: pos(2, 2), end: pos(2, 5), value: 'age' }, + value: { type: 'number', start: pos(2, 7), end: pos(2, 9), value: 30 } + }, + { + key: { type: 'string', start: pos(3, 2), end: pos(3, 9), value: 'address' }, + value: { + type: 'object', start: pos(4, 4), end: pos(5, 22), properties: [ + { + key: { type: 'string', start: pos(4, 4), end: pos(4, 10), value: 'street' }, + value: { type: 'string', start: pos(4, 12), end: pos(4, 23), value: '123 Main St' } + }, + { + key: { type: 'string', start: pos(5, 4), end: pos(5, 8), value: 'city' }, + value: { type: 'string', start: pos(5, 10), end: pos(5, 22), value: 'Example City' } + } + ] + } + } + ] + } + } + ] + }, + [] + ); + }); + + test('properties without space after colon', () => { + assertValidParse( + ['name:John'], + { + type: 'object', start: pos(0, 0), end: pos(0, 9), properties: [ + { + key: { type: 'string', start: pos(0, 0), end: pos(0, 4), value: 'name' }, + value: { type: 'string', start: pos(0, 5), end: pos(0, 9), value: 'John' } + } + ] + }, + [] + ); + + // Test mixed: some properties with space, some without + assertValidParse( + [ + 'config:', + ' database:', + ' host:localhost', + ' port: 5432', + ' credentials:', + ' username:admin', + ' password: secret123' + ], + { + type: 'object', start: pos(0, 0), end: pos(6, 25), properties: [ + { + key: { type: 'string', start: pos(0, 0), end: pos(0, 6), value: 'config' }, + value: { + type: 'object', start: pos(1, 2), end: pos(6, 25), properties: [ + { + key: { type: 'string', start: pos(1, 2), end: pos(1, 10), value: 'database' }, + value: { + type: 'object', start: pos(2, 4), end: pos(6, 25), properties: [ + { + key: { type: 'string', start: pos(2, 4), end: pos(2, 8), value: 'host' }, + value: { type: 'string', start: pos(2, 9), end: pos(2, 18), value: 'localhost' } + }, + { + key: { type: 'string', start: pos(3, 4), end: pos(3, 8), value: 'port' }, + value: { type: 'number', start: pos(3, 10), end: pos(3, 14), value: 5432 } + }, + { + key: { type: 'string', start: pos(4, 4), end: pos(4, 15), value: 'credentials' }, + value: { + type: 'object', start: pos(5, 6), end: pos(6, 25), properties: [ + { + key: { type: 'string', start: pos(5, 6), end: pos(5, 14), value: 'username' }, + value: { type: 'string', start: pos(5, 15), end: pos(5, 20), value: 'admin' } + }, + { + key: { type: 'string', start: pos(6, 6), end: pos(6, 14), value: 'password' }, + value: { type: 'string', start: pos(6, 16), end: pos(6, 25), value: 'secret123' } + } + ] + } + } + ] + } + } + ] + } + } + ] + }, + [] + ); + }); + + test('inline objects', () => { + assertValidParse( + ['{name: John, age: 30}'], + { + type: 'object', start: pos(0, 0), end: pos(0, 21), properties: [ + { + key: { type: 'string', start: pos(0, 1), end: pos(0, 5), value: 'name' }, + value: { type: 'string', start: pos(0, 7), end: pos(0, 11), value: 'John' } + }, + { + key: { type: 'string', start: pos(0, 13), end: pos(0, 16), value: 'age' }, + value: { type: 'number', start: pos(0, 18), end: pos(0, 20), value: 30 } + } + ] + }, + [] + ); + + // Test with different data types + assertValidParse( + ['{active: true, score: 85.5, role: null}'], + { + type: 'object', start: pos(0, 0), end: pos(0, 39), properties: [ + { + key: { type: 'string', start: pos(0, 1), end: pos(0, 7), value: 'active' }, + value: { type: 'boolean', start: pos(0, 9), end: pos(0, 13), value: true } + }, + { + key: { type: 'string', start: pos(0, 15), end: pos(0, 20), value: 'score' }, + value: { type: 'number', start: pos(0, 22), end: pos(0, 26), value: 85.5 } + }, + { + key: { type: 'string', start: pos(0, 28), end: pos(0, 32), value: 'role' }, + value: { type: 'null', start: pos(0, 34), end: pos(0, 38), value: null } + } + ] + }, + [] + ); + + // Test empty inline object + assertValidParse( + ['{}'], + { + type: 'object', start: pos(0, 0), end: pos(0, 2), properties: [] + }, + [] + ); + + // Test inline object with quoted keys and values + assertValidParse( + ['{"name": "John Doe", "age": 30}'], + { + type: 'object', start: pos(0, 0), end: pos(0, 31), properties: [ + { + key: { type: 'string', start: pos(0, 1), end: pos(0, 7), value: 'name' }, + value: { type: 'string', start: pos(0, 9), end: pos(0, 19), value: 'John Doe' } + }, + { + key: { type: 'string', start: pos(0, 21), end: pos(0, 26), value: 'age' }, + value: { type: 'number', start: pos(0, 28), end: pos(0, 30), value: 30 } + } + ] + }, + [] + ); + + // Test inline object without spaces + assertValidParse( + ['{name:John,age:30}'], + { + type: 'object', start: pos(0, 0), end: pos(0, 18), properties: [ + { + key: { type: 'string', start: pos(0, 1), end: pos(0, 5), value: 'name' }, + value: { type: 'string', start: pos(0, 6), end: pos(0, 10), value: 'John' } + }, + { + key: { type: 'string', start: pos(0, 11), end: pos(0, 14), value: 'age' }, + value: { type: 'number', start: pos(0, 15), end: pos(0, 17), value: 30 } + } + ] + }, + [] + ); + }); + + test('special characters in values', () => { + // Test values with special characters + assertValidParse( + [`key: value with \t special chars`], + { + type: 'object', start: pos(0, 0), end: pos(0, 31), properties: [ + { + key: { type: 'string', start: pos(0, 0), end: pos(0, 3), value: 'key' }, + value: { type: 'string', start: pos(0, 5), end: pos(0, 31), value: `value with \t special chars` } + } + ] + }, + [] + ); + }); + + test('various whitespace types', () => { + // Test different types of whitespace + assertValidParse( + [`key:\t \t \t value`], + { + type: 'object', start: pos(0, 0), end: pos(0, 15), properties: [ + { + key: { type: 'string', start: pos(0, 0), end: pos(0, 3), value: 'key' }, + value: { type: 'string', start: pos(0, 10), end: pos(0, 15), value: 'value' } + } + ] + }, + [] + ); + }); + }); + + suite('arrays', () => { + + + test('arrays', () => { + assertValidParse( + [ + '- Boston Red Sox', + '- Detroit Tigers', + '- New York Yankees' + ], + { + type: 'array', start: pos(0, 0), end: pos(2, 18), items: [ + { type: 'string', start: pos(0, 2), end: pos(0, 16), value: 'Boston Red Sox' }, + { type: 'string', start: pos(1, 2), end: pos(1, 16), value: 'Detroit Tigers' }, + { type: 'string', start: pos(2, 2), end: pos(2, 18), value: 'New York Yankees' } + ] + + }, + [] + ); + }); + + + test('inline arrays', () => { + assertValidParse( + ['[Apple, Banana, Cherry]'], + { + type: 'array', start: pos(0, 0), end: pos(0, 23), items: [ + { type: 'string', start: pos(0, 1), end: pos(0, 6), value: 'Apple' }, + { type: 'string', start: pos(0, 8), end: pos(0, 14), value: 'Banana' }, + { type: 'string', start: pos(0, 16), end: pos(0, 22), value: 'Cherry' } + ] + + }, + [] + ); + }); + + test('multi-line inline arrays', () => { + assertValidParse( + [ + '[', + ' geen, ', + ' yello, red]' + ], + { + type: 'array', start: pos(0, 0), end: pos(2, 15), items: [ + { type: 'string', start: pos(1, 4), end: pos(1, 8), value: 'geen' }, + { type: 'string', start: pos(2, 4), end: pos(2, 9), value: 'yello' }, + { type: 'string', start: pos(2, 11), end: pos(2, 14), value: 'red' } + ] + }, + [] + ); + }); + + test('arrays of arrays', () => { + assertValidParse( + [ + '-', + ' - Apple', + ' - Banana', + ' - Cherry' + ], + { + type: 'array', start: pos(0, 0), end: pos(3, 10), items: [ + { + type: 'array', start: pos(1, 2), end: pos(3, 10), items: [ + { type: 'string', start: pos(1, 4), end: pos(1, 9), value: 'Apple' }, + { type: 'string', start: pos(2, 4), end: pos(2, 10), value: 'Banana' }, + { type: 'string', start: pos(3, 4), end: pos(3, 10), value: 'Cherry' } + ] + } + ] + }, + [] + ); + }); + + test('inline arrays of inline arrays', () => { + assertValidParse( + [ + '[', + ' [ee], [ff, gg]', + ']', + ], + { + type: 'array', start: pos(0, 0), end: pos(2, 1), items: [ + { + type: 'array', start: pos(1, 2), end: pos(1, 6), items: [ + { type: 'string', start: pos(1, 3), end: pos(1, 5), value: 'ee' }, + ], + }, + { + type: 'array', start: pos(1, 8), end: pos(1, 16), items: [ + { type: 'string', start: pos(1, 9), end: pos(1, 11), value: 'ff' }, + { type: 'string', start: pos(1, 13), end: pos(1, 15), value: 'gg' }, + ], + } + ] + }, + [] + ); + }); + + test('object with array containing single object', () => { + assertValidParse( + [ + 'items:', + '- name: John', + ' age: 30' + ], + { + type: 'object', start: pos(0, 0), end: pos(2, 9), properties: [ + { + key: { type: 'string', start: pos(0, 0), end: pos(0, 5), value: 'items' }, + value: { + type: 'array', start: pos(1, 0), end: pos(2, 9), items: [ + { + type: 'object', start: pos(1, 2), end: pos(2, 9), properties: [ + { + key: { type: 'string', start: pos(1, 2), end: pos(1, 6), value: 'name' }, + value: { type: 'string', start: pos(1, 8), end: pos(1, 12), value: 'John' } + }, + { + key: { type: 'string', start: pos(2, 2), end: pos(2, 5), value: 'age' }, + value: { type: 'number', start: pos(2, 7), end: pos(2, 9), value: 30 } + } + ] + } + ] + } + } + ] + }, + [] + ); + }); + + test('arrays of objects', () => { + assertValidParse( + [ + '-', + ' name: one', + '- name: two', + '-', + ' name: three' + ], + { + type: 'array', start: pos(0, 0), end: pos(4, 13), items: [ + { + type: 'object', start: pos(1, 2), end: pos(1, 11), properties: [ + { + key: { type: 'string', start: pos(1, 2), end: pos(1, 6), value: 'name' }, + value: { type: 'string', start: pos(1, 8), end: pos(1, 11), value: 'one' } + } + ] + }, + { + type: 'object', start: pos(2, 2), end: pos(2, 11), properties: [ + { + key: { type: 'string', start: pos(2, 2), end: pos(2, 6), value: 'name' }, + value: { type: 'string', start: pos(2, 8), end: pos(2, 11), value: 'two' } + } + ] + }, + { + type: 'object', start: pos(4, 2), end: pos(4, 13), properties: [ + { + key: { type: 'string', start: pos(4, 2), end: pos(4, 6), value: 'name' }, + value: { type: 'string', start: pos(4, 8), end: pos(4, 13), value: 'three' } + } + ] + } + ] + }, + [] + ); + }); + }); + + suite('complex structures', () => { + + test('array of objects', () => { + assertValidParse( + [ + 'products:', + ' - name: Laptop', + ' price: 999.99', + ' in_stock: true', + ' - name: Mouse', + ' price: 25.50', + ' in_stock: false' + ], + { + type: 'object', start: pos(0, 0), end: pos(6, 19), properties: [ + { + key: { type: 'string', start: pos(0, 0), end: pos(0, 8), value: 'products' }, + value: { + type: 'array', start: pos(1, 2), end: pos(6, 19), items: [ + { + type: 'object', start: pos(1, 4), end: pos(3, 18), properties: [ + { + key: { type: 'string', start: pos(1, 4), end: pos(1, 8), value: 'name' }, + value: { type: 'string', start: pos(1, 10), end: pos(1, 16), value: 'Laptop' } + }, + { + key: { type: 'string', start: pos(2, 4), end: pos(2, 9), value: 'price' }, + value: { type: 'number', start: pos(2, 11), end: pos(2, 17), value: 999.99 } + }, + { + key: { type: 'string', start: pos(3, 4), end: pos(3, 12), value: 'in_stock' }, + value: { type: 'boolean', start: pos(3, 14), end: pos(3, 18), value: true } + } + ] + }, + { + type: 'object', start: pos(4, 4), end: pos(6, 19), properties: [ + { + key: { type: 'string', start: pos(4, 4), end: pos(4, 8), value: 'name' }, + value: { type: 'string', start: pos(4, 10), end: pos(4, 15), value: 'Mouse' } + }, + { + key: { type: 'string', start: pos(5, 4), end: pos(5, 9), value: 'price' }, + value: { type: 'number', start: pos(5, 11), end: pos(5, 16), value: 25.50 } + }, + { + key: { type: 'string', start: pos(6, 4), end: pos(6, 12), value: 'in_stock' }, + value: { type: 'boolean', start: pos(6, 14), end: pos(6, 19), value: false } + } + ] + } + ] + } + } + ] + }, + [] + ); + }); + + test('inline array mixed primitives', () => { + assertValidParse( + ['vals: [1, true, null, "str"]'], + { + type: 'object', start: pos(0, 0), end: pos(0, 28), properties: [ + { + key: { type: 'string', start: pos(0, 0), end: pos(0, 4), value: 'vals' }, + value: { + type: 'array', start: pos(0, 6), end: pos(0, 28), items: [ + { type: 'number', start: pos(0, 7), end: pos(0, 8), value: 1 }, + { type: 'boolean', start: pos(0, 10), end: pos(0, 14), value: true }, + { type: 'null', start: pos(0, 16), end: pos(0, 20), value: null }, + { type: 'string', start: pos(0, 22), end: pos(0, 27), value: 'str' } + ] + } + } + ] + }, + [] + ); + }); + + test('mixed inline structures', () => { + assertValidParse( + ['config: {env: "prod", settings: [true, 42], debug: false}'], + { + type: 'object', start: pos(0, 0), end: pos(0, 57), properties: [ + { + key: { type: 'string', start: pos(0, 0), end: pos(0, 6), value: 'config' }, + value: { + type: 'object', start: pos(0, 8), end: pos(0, 57), properties: [ + { + key: { type: 'string', start: pos(0, 9), end: pos(0, 12), value: 'env' }, + value: { type: 'string', start: pos(0, 14), end: pos(0, 20), value: 'prod' } + }, + { + key: { type: 'string', start: pos(0, 22), end: pos(0, 30), value: 'settings' }, + value: { + type: 'array', start: pos(0, 32), end: pos(0, 42), items: [ + { type: 'boolean', start: pos(0, 33), end: pos(0, 37), value: true }, + { type: 'number', start: pos(0, 39), end: pos(0, 41), value: 42 } + ] + } + }, + { + key: { type: 'string', start: pos(0, 44), end: pos(0, 49), value: 'debug' }, + value: { type: 'boolean', start: pos(0, 51), end: pos(0, 56), value: false } + } + ] + } + } + ] + }, + [] + ); + }); + + test('with comments', () => { + assertValidParse( + [ + `# This is a comment`, + 'name: John Doe # inline comment', + 'age: 30' + ], + { + type: 'object', start: pos(1, 0), end: pos(2, 7), properties: [ + { + key: { type: 'string', start: pos(1, 0), end: pos(1, 4), value: 'name' }, + value: { type: 'string', start: pos(1, 6), end: pos(1, 14), value: 'John Doe' } + }, + { + key: { type: 'string', start: pos(2, 0), end: pos(2, 3), value: 'age' }, + value: { type: 'number', start: pos(2, 5), end: pos(2, 7), value: 30 } + } + ] + }, + [] + ); + }); + }); + + suite('edge cases and error handling', () => { + + + // Edge cases + test('duplicate keys error', () => { + assertValidParse( + [ + 'key: 1', + 'key: 2' + ], + { + type: 'object', start: pos(0, 0), end: pos(1, 6), properties: [ + { + key: { type: 'string', start: pos(0, 0), end: pos(0, 3), value: 'key' }, + value: { type: 'number', start: pos(0, 5), end: pos(0, 6), value: 1 } + }, + { + key: { type: 'string', start: pos(1, 0), end: pos(1, 3), value: 'key' }, + value: { type: 'number', start: pos(1, 5), end: pos(1, 6), value: 2 } + } + ] + }, + [ + { + message: "Duplicate key 'key'", + code: 'duplicateKey', + start: pos(1, 0), + end: pos(1, 3) + } + ] + ); + }); + + test('duplicate keys allowed with option', () => { + assertValidParse( + [ + 'key: 1', + 'key: 2' + ], + { + type: 'object', start: pos(0, 0), end: pos(1, 6), properties: [ + { + key: { type: 'string', start: pos(0, 0), end: pos(0, 3), value: 'key' }, + value: { type: 'number', start: pos(0, 5), end: pos(0, 6), value: 1 } + }, + { + key: { type: 'string', start: pos(1, 0), end: pos(1, 3), value: 'key' }, + value: { type: 'number', start: pos(1, 5), end: pos(1, 6), value: 2 } + } + ] + }, + [], + { allowDuplicateKeys: true } + ); + }); + + test('unexpected indentation error with recovery', () => { + // Parser reports error but still captures the over-indented property. + assertValidParse( + [ + 'key: 1', + ' stray: value' + ], + { + type: 'object', start: pos(0, 0), end: pos(1, 16), properties: [ + { + key: { type: 'string', start: pos(0, 0), end: pos(0, 3), value: 'key' }, + value: { type: 'number', start: pos(0, 5), end: pos(0, 6), value: 1 } + }, + { + key: { type: 'string', start: pos(1, 4), end: pos(1, 9), value: 'stray' }, + value: { type: 'string', start: pos(1, 11), end: pos(1, 16), value: 'value' } + } + ] + }, + [ + { + message: 'Unexpected indentation', + code: 'indentation', + start: pos(1, 0), + end: pos(1, 16) + } + ] + ); + }); + + test('empty values and inline empty array', () => { + assertValidParse( + [ + 'empty:', + 'array: []' + ], + { + type: 'object', start: pos(0, 0), end: pos(1, 9), properties: [ + { + key: { type: 'string', start: pos(0, 0), end: pos(0, 5), value: 'empty' }, + value: { type: 'string', start: pos(0, 6), end: pos(0, 6), value: '' } + }, + { + key: { type: 'string', start: pos(1, 0), end: pos(1, 5), value: 'array' }, + value: { type: 'array', start: pos(1, 7), end: pos(1, 9), items: [] } + } + ] + }, + [] + ); + }); + + + + test('nested empty objects', () => { + // Parser should create nodes for both parent and child, with child having empty string value. + assertValidParse( + [ + 'parent:', + ' child:' + ], + { + type: 'object', start: pos(0, 0), end: pos(1, 8), properties: [ + { + key: { type: 'string', start: pos(0, 0), end: pos(0, 6), value: 'parent' }, + value: { + type: 'object', start: pos(1, 2), end: pos(1, 8), properties: [ + { + key: { type: 'string', start: pos(1, 2), end: pos(1, 7), value: 'child' }, + value: { type: 'string', start: pos(1, 8), end: pos(1, 8), value: '' } + } + ] + } + } + ] + }, + [] + ); + }); + + test('empty object with only colons', () => { + // Test object with empty values + assertValidParse( + ["key1:", "key2:", "key3:"], + { + type: 'object', start: pos(0, 0), end: pos(2, 5), properties: [ + { + key: { type: 'string', start: pos(0, 0), end: pos(0, 4), value: 'key1' }, + value: { type: 'string', start: pos(0, 5), end: pos(0, 5), value: '' } + }, + { + key: { type: 'string', start: pos(1, 0), end: pos(1, 4), value: 'key2' }, + value: { type: 'string', start: pos(1, 5), end: pos(1, 5), value: '' } + }, + { + key: { type: 'string', start: pos(2, 0), end: pos(2, 4), value: 'key3' }, + value: { type: 'string', start: pos(2, 5), end: pos(2, 5), value: '' } + } + ] + }, + [] + ); + }); + + test('large input performance', () => { + // Test that large inputs are handled efficiently + const input = Array.from({ length: 1000 }, (_, i) => `key${i}: value${i}`); + const expectedProperties = Array.from({ length: 1000 }, (_, i) => ({ + key: { type: 'string' as const, start: pos(i, 0), end: pos(i, `key${i}`.length), value: `key${i}` }, + value: { type: 'string' as const, start: pos(i, `key${i}: `.length), end: pos(i, `key${i}: value${i}`.length), value: `value${i}` } + })); + + const start = Date.now(); + assertValidParse( + input, + { + type: 'object', + start: pos(0, 0), + end: pos(999, 'key999: value999'.length), + properties: expectedProperties + }, + [] + ); + const duration = Date.now() - start; + + ok(duration < 100, `Parsing took ${duration}ms, expected < 100ms`); + }); + + test('deeply nested structure performance', () => { + // Test that deeply nested structures are handled efficiently + const lines = []; + for (let i = 0; i < 50; i++) { + const indent = ' '.repeat(i); + lines.push(`${indent}level${i}:`); + } + lines.push(' '.repeat(50) + 'deepValue: reached'); + + const start = Date.now(); + const errors: YamlParseError[] = []; + const result = parse(lines, errors); + const duration = Date.now() - start; + + ok(result); + strictEqual(result.type, 'object'); + strictEqual(errors.length, 0); + ok(duration < 100, `Parsing took ${duration}ms, expected < 100ms`); + }); + + test('malformed array with position issues', () => { + // Test malformed arrays that might cause position advancement issues + assertValidParse( + [ + "key: [", + "", + "", + "", + "" + ], + { + type: 'object', start: pos(0, 0), end: pos(5, 0), properties: [ + { + key: { type: 'string', start: pos(0, 0), end: pos(0, 3), value: 'key' }, + value: { type: 'array', start: pos(0, 5), end: pos(5, 0), items: [] } + } + ] + }, + [] + ); + }); + + test('self-referential like structure', () => { + // Test structures that might appear self-referential + assertValidParse( + [ + "a:", + " b:", + " a:", + " b:", + " value: test" + ], + { + type: 'object', start: pos(0, 0), end: pos(4, 19), properties: [ + { + key: { type: 'string', start: pos(0, 0), end: pos(0, 1), value: 'a' }, + value: { + type: 'object', start: pos(1, 2), end: pos(4, 19), properties: [ + { + key: { type: 'string', start: pos(1, 2), end: pos(1, 3), value: 'b' }, + value: { + type: 'object', start: pos(2, 4), end: pos(4, 19), properties: [ + { + key: { type: 'string', start: pos(2, 4), end: pos(2, 5), value: 'a' }, + value: { + type: 'object', start: pos(3, 6), end: pos(4, 19), properties: [ + { + key: { type: 'string', start: pos(3, 6), end: pos(3, 7), value: 'b' }, + value: { + type: 'object', start: pos(4, 8), end: pos(4, 19), properties: [ + { + key: { type: 'string', start: pos(4, 8), end: pos(4, 13), value: 'value' }, + value: { type: 'string', start: pos(4, 15), end: pos(4, 19), value: 'test' } + } + ] + } + } + ] + } + } + ] + } + } + ] + } + } + ] + }, + [] + ); + }); + + test('array with empty lines', () => { + // Test arrays spanning multiple lines with empty lines + assertValidParse( + ["arr: [", "", "item1,", "", "item2", "", "]"], + { + type: 'object', start: pos(0, 0), end: pos(6, 1), properties: [ + { + key: { type: 'string', start: pos(0, 0), end: pos(0, 3), value: 'arr' }, + value: { + type: 'array', start: pos(0, 5), end: pos(6, 1), items: [ + { type: 'string', start: pos(2, 0), end: pos(2, 5), value: 'item1' }, + { type: 'string', start: pos(4, 0), end: pos(4, 5), value: 'item2' } + ] + } + } + ] + }, + [] + ); + }); + + test('whitespace advancement robustness', () => { + // Test that whitespace advancement works correctly + assertValidParse( + [`key: value`], + { + type: 'object', start: pos(0, 0), end: pos(0, 15), properties: [ + { + key: { type: 'string', start: pos(0, 0), end: pos(0, 3), value: 'key' }, + value: { type: 'string', start: pos(0, 10), end: pos(0, 15), value: 'value' } + } + ] + }, + [] + ); + }); + + + test('missing end quote in string values', () => { + // Test unclosed double quote - parser treats it as bare string with quote included + assertValidParse( + ['name: "John'], + { + type: 'object', start: pos(0, 0), end: pos(0, 11), properties: [ + { + key: { type: 'string', start: pos(0, 0), end: pos(0, 4), value: 'name' }, + value: { type: 'string', start: pos(0, 6), end: pos(0, 11), value: 'John' } + } + ] + }, + [] + ); + + // Test unclosed single quote - parser treats it as bare string with quote included + assertValidParse( + ['description: \'Hello world'], + { + type: 'object', start: pos(0, 0), end: pos(0, 25), properties: [ + { + key: { type: 'string', start: pos(0, 0), end: pos(0, 11), value: 'description' }, + value: { type: 'string', start: pos(0, 13), end: pos(0, 25), value: 'Hello world' } + } + ] + }, + [] + ); + + // Test unclosed quote in multi-line context + assertValidParse( + [ + 'data: "incomplete', + 'next: value' + ], + { + type: 'object', start: pos(0, 0), end: pos(1, 11), properties: [ + { + key: { type: 'string', start: pos(0, 0), end: pos(0, 4), value: 'data' }, + value: { type: 'string', start: pos(0, 6), end: pos(0, 17), value: 'incomplete' } + }, + { + key: { type: 'string', start: pos(1, 0), end: pos(1, 4), value: 'next' }, + value: { type: 'string', start: pos(1, 6), end: pos(1, 11), value: 'value' } + } + ] + }, + [] + ); + + // Test properly quoted strings for comparison + assertValidParse( + ['name: "John"'], + { + type: 'object', start: pos(0, 0), end: pos(0, 12), properties: [ + { + key: { type: 'string', start: pos(0, 0), end: pos(0, 4), value: 'name' }, + value: { type: 'string', start: pos(0, 6), end: pos(0, 12), value: 'John' } + } + ] + }, + [] + ); + }); + + + }); + +}); diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/service/newPromptsParser.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/service/newPromptsParser.ts new file mode 100644 index 00000000000..15c2ec6a24b --- /dev/null +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/service/newPromptsParser.ts @@ -0,0 +1,94 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { splitLines } from '../../../../../../base/common/strings.js'; +import { URI } from '../../../../../../base/common/uri.js'; +import { parse, YamlNode, YamlParseError } from '../../../../../../base/common/yaml.js'; +import { IModelService } from '../../../../../../editor/common/services/model.js'; +import { IFileService } from '../../../../../../platform/files/common/files.js'; +import { IPromptParserResult } from './promptsService.js'; + +export class NewPromptsParser { + constructor( + private readonly modelService: IModelService, + private readonly fileService: IFileService, + ) { + // TODO + } + + public async parse(uri: URI): Promise { + const content = await this.getContents(uri); + if (!content) { + return; + } + const lines = splitLines(content); + if (lines.length === 0) { + return createResult(uri, undefined, []); + } + let header: PromptHeader | undefined = undefined; + let body: { references: URI[] } | undefined = undefined; + let bodyStart = 0; + if (lines[0] === '---') { + let headerEnd = lines.indexOf('---', 1); + if (headerEnd === -1) { + headerEnd = lines.length; + bodyStart = lines.length; + } else { + bodyStart = headerEnd + 1; + } + header = this.parseHeader(lines.slice(1, headerEnd !== -1 ? headerEnd : lines.length)); + } + if (bodyStart < lines.length) { + body = this.parseBody(lines.slice(bodyStart)); + } + return createResult(uri, header, body?.references ?? []); + } + + private parseBody(lines: string[]): { references: URI[] } { + const references: URI[] = []; + for (const line of lines) { + const match = line.match(/\[(.+?)\]\((.+?)\)/); + if (match) { + const [, _text, uri] = match; + references.push(URI.file(uri)); + } + } + return { references }; + } + + private parseHeader(lines: string[]): PromptHeader { + const errors: YamlParseError[] = []; + const node = parse(lines, errors); + return new PromptHeader(node, errors); + } + + private async getContents(uri: URI): Promise { + const model = this.modelService.getModel(uri); + if (model) { + return model.getValue(); + } + const content = await this.fileService.readFile(uri); + if (content) { + return content.value.toString(); + } + return undefined; + } +} + +function createResult(uri: URI, header: PromptHeader | undefined, references: URI[]): IPromptParserResult { + return { + uri, + header, + references, + metadata: null + }; +} + +export class PromptHeader { + constructor(public readonly node: YamlNode | undefined, public readonly errors: YamlParseError[]) { + + } +} + diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsService.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsService.ts index 577a9e4d1df..21709075896 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsService.ts +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsService.ts @@ -13,7 +13,7 @@ import { TextModelPromptParser } from '../parsers/textModelPromptParser.js'; import { CancellationToken } from '../../../../../../base/common/cancellation.js'; import { PromptsType } from '../promptTypes.js'; import { createDecorator } from '../../../../../../platform/instantiation/common/instantiation.js'; -import { ITopError } from '../parsers/types.js'; +import { YamlNode, YamlParseError } from '../../../../../../base/common/yaml.js'; /** * Provides prompt services. @@ -217,6 +217,12 @@ export interface IChatPromptSlashCommand { export interface IPromptParserResult { readonly uri: URI; readonly metadata: TMetadata | null; - readonly topError: ITopError | undefined; readonly references: readonly URI[]; + readonly header?: IPromptHeader; } + +export interface IPromptHeader { + readonly node: YamlNode | undefined; + readonly errors: YamlParseError[]; +} + diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts index 662aa6673b6..746e9d52b75 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts @@ -270,8 +270,8 @@ export class PromptsService extends Disposable implements IPromptsService { return { uri: parser.uri, metadata: parser.metadata, - topError: parser.topError, - references: parser.references.map(ref => ref.uri) + references: parser.references.map(ref => ref.uri), + header: undefined }; } finally { parser?.dispose(); From f999ab90e129f11afae946b258b79c8173252a0a Mon Sep 17 00:00:00 2001 From: Martin Aeschlimann Date: Thu, 28 Aug 2025 17:05:21 +0200 Subject: [PATCH 004/299] fix tests --- .../common/promptSyntax/service/promptsService.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/promptsService.test.ts b/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/promptsService.test.ts index f8db1ba18f0..7bba2c0cfbc 100644 --- a/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/promptsService.test.ts +++ b/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/promptsService.test.ts @@ -679,7 +679,7 @@ suite('PromptsService', () => { tools: ['my-tool1'], mode: 'agent', }, - topError: undefined, + header: undefined, references: [file3, file4] }); @@ -690,7 +690,7 @@ suite('PromptsService', () => { promptType: PromptsType.prompt, mode: 'edit', }, - topError: undefined, + header: undefined, references: [nonExistingFolder, yetAnotherFile] }); @@ -702,7 +702,7 @@ suite('PromptsService', () => { description: 'Another file description.', applyTo: '**/*.tsx', }, - topError: undefined, + header: undefined, references: [someOtherFolder, someOtherFolderFile] }); @@ -713,7 +713,7 @@ suite('PromptsService', () => { promptType: PromptsType.instructions, description: 'File 4 splendid description.', }, - topError: undefined, + header: undefined, references: [ URI.joinPath(rootFolderUri, '/folder1/some-other-folder/some-non-existing/file.prompt.md'), URI.joinPath(rootFolderUri, '/folder1/some-other-folder/some-non-prompt-file.md'), From a86f0c916b2f434fbe252bff4680f90ebacd24b9 Mon Sep 17 00:00:00 2001 From: Martin Aeschlimann Date: Fri, 5 Sep 2025 22:31:54 +0200 Subject: [PATCH 005/299] update --- .../promptSyntax/service/newPromptsParser.ts | 221 ++++++++++++------ .../service/promptsServiceImpl.ts | 15 ++ .../service/newPromptsParser.test.ts | 48 ++++ 3 files changed, 211 insertions(+), 73 deletions(-) create mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/service/newPromptsParser.test.ts diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/service/newPromptsParser.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/service/newPromptsParser.ts index c6d368c54dd..3f21c87cb43 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/service/newPromptsParser.ts +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/service/newPromptsParser.ts @@ -3,98 +3,173 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { splitLines } from '../../../../../../base/common/strings.js'; +import { Iterable } from '../../../../../../base/common/iterator.js'; +import { dirname, resolvePath } from '../../../../../../base/common/resources.js'; +import { splitLinesIncludeSeparators } from '../../../../../../base/common/strings.js'; import { URI } from '../../../../../../base/common/uri.js'; import { parse, YamlNode, YamlParseError } from '../../../../../../base/common/yaml.js'; -import { IModelService } from '../../../../../../editor/common/services/model.js'; -import { IFileService } from '../../../../../../platform/files/common/files.js'; -import { IVariableReference } from '../../chatModes.js'; -import { IPromptParserResult } from './promptsService.js'; +import { Range } from '../../../../../../editor/common/core/range.js'; +import { chatVariableLeader } from '../../chatParserTypes.js'; export class NewPromptsParser { - constructor( - private readonly modelService: IModelService, - private readonly fileService: IFileService, - ) { - // TODO + constructor() { } - public async parse(uri: URI): Promise { - const content = await this.getContents(uri); - if (!content) { - return; - } - const lines = splitLines(content); - if (lines.length === 0) { - return createResult(uri, undefined, [], []); + public parse(uri: URI, content: string): ParsedPromptFile { + const linesWithEOL = splitLinesIncludeSeparators(content); + if (linesWithEOL.length === 0) { + return new ParsedPromptFile(uri, undefined, undefined); } let header: PromptHeader | undefined = undefined; - - let body: { fileReferences: URI[]; variableReferences: IVariableReference[] } | undefined = undefined; - let bodyStart = 0; - if (lines[0] === '---') { - let headerEnd = lines.indexOf('---', 1); - if (headerEnd === -1) { - headerEnd = lines.length; - bodyStart = lines.length; + let body: PromptBody | undefined = undefined; + let bodyStartLine = 0; + if (linesWithEOL[0].match(/^---[\s\r\n]*$/)) { + let headerEndLine = linesWithEOL.findIndex((line, index) => index > 0 && line.match(/^---[\s\r\n]*$/)); + if (headerEndLine === -1) { + headerEndLine = linesWithEOL.length; + bodyStartLine = linesWithEOL.length; } else { - bodyStart = headerEnd + 1; + bodyStartLine = headerEndLine + 1; } - header = this.parseHeader(lines.slice(1, headerEnd !== -1 ? headerEnd : lines.length)); + // range starts on the line after the ---, and ends at the beginning of the line that has the closing --- + const range = new Range(2, 1, headerEndLine + 1, 1); + header = new PromptHeader(range, linesWithEOL); } - if (bodyStart < lines.length) { - body = this.parseBody(lines.slice(bodyStart)); - } else { - body = { fileReferences: [], variableReferences: [] }; + if (bodyStartLine < linesWithEOL.length) { + // range starts on the line after the ---, and ends at the beginning of line after the last line + const range = new Range(bodyStartLine + 1, 1, linesWithEOL.length + 1, 1); + body = new PromptBody(range, linesWithEOL, uri); } - return createResult(uri, header, body.fileReferences, body.variableReferences); - } - - private parseBody(lines: string[]): { fileReferences: URI[]; variableReferences: IVariableReference[] } { - const fileReferences: URI[] = []; - const variableReferences: IVariableReference[] = []; - for (const line of lines) { - const match = line.match(/\[(.+?)\]\((.+?)\)/); - if (match) { - const [, _text, uri] = match; - fileReferences.push(URI.file(uri)); - } - } - return { fileReferences, variableReferences }; - } - - private parseHeader(lines: string[]): PromptHeader { - const errors: YamlParseError[] = []; - const node = parse(lines, errors); - return new PromptHeader(node, errors); - } - - private async getContents(uri: URI): Promise { - const model = this.modelService.getModel(uri); - if (model) { - return model.getValue(); - } - const content = await this.fileService.readFile(uri); - if (content) { - return content.value.toString(); - } - return undefined; + return new ParsedPromptFile(uri, header, body); } } -function createResult(uri: URI, header: PromptHeader | undefined, fileReferences: URI[], variableReferences: IVariableReference[]): IPromptParserResult { - return { - uri, - header, - fileReferences, - variableReferences, - metadata: null - }; + +export class ParsedPromptFile { + constructor(public readonly uri: URI, public readonly header?: PromptHeader, public readonly body?: PromptBody) { + } +} + +interface ParsedHeader { + readonly node: YamlNode | undefined; + readonly errors: YamlParseError[]; + readonly attributes: IHeaderAttribute[]; } export class PromptHeader { - constructor(public readonly node: YamlNode | undefined, public readonly errors: YamlParseError[]) { + private _parsed: ParsedHeader | undefined; + constructor(public readonly range: Range, private readonly linesWithEOL: string[]) { + } + + public getParsedHeader(): ParsedHeader { + if (this._parsed === undefined) { + const errors: YamlParseError[] = []; + const lines = Iterable.map(Iterable.slice(this.linesWithEOL, this.range.startLineNumber - 1, this.range.endLineNumber - 1), line => line.replace(/[\r\n]+$/, '')); + const node = parse(lines, errors); + const attributes = []; + if (node?.type === 'object') { + for (const property of node.properties) { + attributes.push({ + key: property.key.value, + range: new Range(this.range.startLineNumber + property.key.start.line, property.key.start.character + 1, this.range.startLineNumber + property.value.end.line, property.value.end.character + 1) + }); + } + } + this._parsed = { node, attributes, errors }; + } + return this._parsed; + } + + public get attributes(): IHeaderAttribute[] { + return this.getParsedHeader().attributes; + } +} +interface IHeaderAttribute { + readonly range: Range; + readonly key: string; +} + + +interface ParsedBody { + readonly fileReferences: readonly IBodyFileReference[]; + readonly variableReferences: readonly IBodyVariableReference[]; +} + +export class PromptBody { + private _parsed: ParsedBody | undefined; + + constructor(public readonly range: Range, private readonly linesWithEOL: string[], public readonly uri: URI) { + } + + public get fileReferences(): readonly IBodyFileReference[] { + return this.getParsedBody().fileReferences; + } + + public get variableReferences(): readonly IBodyVariableReference[] { + return this.getParsedBody().variableReferences; + } + + private getParsedBody(): ParsedBody { + if (this._parsed === undefined) { + const fileReferences: IBodyFileReference[] = []; + const variableReferences: IBodyVariableReference[] = []; + for (let i = this.range.startLineNumber - 1; i < this.range.endLineNumber - 1; i++) { + const line = this.linesWithEOL[i]; + const linkMatch = line.matchAll(/\[(.+?)\]\((.+?)\)/g); + for (const match of linkMatch) { + const linkEndOffset = match.index + match[0].length - 1; // before the parenthesis + const linkStartOffset = match.index + match[0].length - match[2].length - 1; + const range = new Range(i + 1, linkStartOffset + 1, i + 1, linkEndOffset + 1); + fileReferences.push({ content: match[2], range }); + } + const reg = new RegExp(`${chatVariableLeader}([\\w]+:)?([^\\s#]*)`, 'g'); + const matches = line.matchAll(reg); + for (const match of matches) { + const varType = match[1]; + if (varType) { + if (varType === 'file:') { + const linkStartOffset = match.index + match[0].length - match[2].length; + const linkEndOffset = match.index + match[0].length; + const range = new Range(i + 1, linkStartOffset + 1, i + 1, linkEndOffset + 1); + fileReferences.push({ content: match[2], range }); + } + } else { + const contentStartOffset = match.index + 1; // after the # + const contentEndOffset = match.index + match[0].length; + const range = new Range(i + 1, contentStartOffset + 1, i + 1, contentEndOffset + 1); + variableReferences.push({ content: match[2], range }); + } + } + } + this._parsed = { fileReferences: fileReferences.sort((a, b) => Range.compareRangesUsingStarts(a.range, b.range)), variableReferences }; + } + return this._parsed; + } + + public resolveFilePath(path: string): URI | undefined { + try { + if (path.startsWith('/')) { + return URI.file(path); + } else if (path.match(/^[a-zA-Z]:\\/)) { + return URI.parse(path); + } else { + return resolvePath(dirname(this.uri), path); + } + } catch { + return undefined; + } } } +interface IBodyFileReference { + content: string; + range: Range; +} + +interface IBodyVariableReference { + content: string; + range: Range; +} + + diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts index 14eb712360e..45b976408d9 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts @@ -27,6 +27,8 @@ import { ILanguageService } from '../../../../../../editor/common/languages/lang import { PromptsConfig } from '../config/config.js'; import { IConfigurationService } from '../../../../../../platform/configuration/common/configuration.js'; import { PositionOffsetTransformer } from '../../../../../../editor/common/core/text/positionToOffset.js'; +import { NewPromptsParser, ParsedPromptFile } from './newPromptsParser.js'; +import { IFileService } from '../../../../../../platform/files/common/files.js'; /** * Provides prompt services. @@ -62,6 +64,7 @@ export class PromptsService extends Disposable implements IPromptsService { @IUserDataProfileService private readonly userDataService: IUserDataProfileService, @ILanguageService private readonly languageService: ILanguageService, @IConfigurationService private readonly configurationService: IConfigurationService, + @IFileService private readonly fileService: IFileService, ) { super(); @@ -302,6 +305,18 @@ export class PromptsService extends Disposable implements IPromptsService { parser?.dispose(); } } + + public async parseNew(uri: URI): Promise { + let content: string | undefined; + const model = this.modelService.getModel(uri); + if (model) { + content = model.getValue(); + } else { + const fileContent = await this.fileService.readFile(uri); + content = fileContent.value.toString(); + } + return new NewPromptsParser().parse(uri, content); + } } export function getPromptCommandName(path: string): string { diff --git a/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/newPromptsParser.test.ts b/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/newPromptsParser.test.ts new file mode 100644 index 00000000000..1281fcfeef9 --- /dev/null +++ b/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/newPromptsParser.test.ts @@ -0,0 +1,48 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'assert'; + +import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../../../../base/test/common/utils.js'; +import { Range } from '../../../../../../../editor/common/core/range.js'; +import { URI } from '../../../../../../../base/common/uri.js'; +import { NewPromptsParser } from '../../../../common/promptSyntax/service/newPromptsParser.js'; + +suite('NewPromptsParser', () => { + ensureNoDisposablesAreLeakedInTestSuite(); + + + test('provides cached parser instance', async () => { + const uri = URI.parse('file:///test/prompt1.md'); + const content = [ + /* 01 */"---", + /* 02 */`description: "Agent mode test"`, + /* 03 */"mode: agent", + /* 04 */"tools: ['tool1', 'tool2']", + /* 05 */"---", + /* 06 */"This is a builtin agent mode test.", + /* 07 */"Here is a #tool1 variable and a #file:./reference1.md as well as a [reference](./reference2.md).", + ].join('\n'); + const result = new NewPromptsParser().parse(uri, content); + assert.deepEqual(result.uri, uri); + assert.deepEqual(result.header?.range, { startLineNumber: 2, startColumn: 1, endLineNumber: 5, endColumn: 1 }); + assert.deepEqual(result.header?.attributes, [ + { key: 'description', range: new Range(2, 1, 2, 31) }, + { key: 'mode', range: new Range(3, 1, 3, 12) }, + { key: 'tools', range: new Range(4, 1, 4, 26) }, + ]); + + + assert.deepEqual(result.body?.range, { startLineNumber: 6, startColumn: 1, endLineNumber: 8, endColumn: 1 }); + assert.deepEqual(result.body?.fileReferences, [ + { range: new Range(7, 39, 7, 54), content: './reference1.md' }, + { range: new Range(7, 80, 7, 95), content: './reference2.md' } + ]); + assert.deepEqual(result.body?.variableReferences, [ + { range: new Range(7, 12, 7, 17), content: 'tool1' } + ]); + }); + +}); From 6439bdd441671f4457676d1386e5ff35d12409f7 Mon Sep 17 00:00:00 2001 From: Martin Aeschlimann Date: Mon, 8 Sep 2025 08:15:12 +0200 Subject: [PATCH 006/299] update --- .../promptSyntax/service/newPromptsParser.ts | 108 +++++++++++++++++- .../service/newPromptsParser.test.ts | 65 ++++++++--- 2 files changed, 156 insertions(+), 17 deletions(-) diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/service/newPromptsParser.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/service/newPromptsParser.ts index 3f21c87cb43..7a5396f47af 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/service/newPromptsParser.ts +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/service/newPromptsParser.ts @@ -7,7 +7,7 @@ import { Iterable } from '../../../../../../base/common/iterator.js'; import { dirname, resolvePath } from '../../../../../../base/common/resources.js'; import { splitLinesIncludeSeparators } from '../../../../../../base/common/strings.js'; import { URI } from '../../../../../../base/common/uri.js'; -import { parse, YamlNode, YamlParseError } from '../../../../../../base/common/yaml.js'; +import { parse, YamlNode, YamlParseError, Position as YamlPosition } from '../../../../../../base/common/yaml.js'; import { Range } from '../../../../../../editor/common/core/range.js'; import { chatVariableLeader } from '../../chatParserTypes.js'; @@ -72,7 +72,8 @@ export class PromptHeader { for (const property of node.properties) { attributes.push({ key: property.key.value, - range: new Range(this.range.startLineNumber + property.key.start.line, property.key.start.character + 1, this.range.startLineNumber + property.value.end.line, property.value.end.character + 1) + range: this.asRange({ start: property.key.start, end: property.value.end }), + value: this.asValue(property.value) }); } } @@ -81,15 +82,112 @@ export class PromptHeader { return this._parsed; } + private asRange({ start, end }: { start: YamlPosition; end: YamlPosition }): Range { + return new Range(this.range.startLineNumber + start.line, start.character + 1, this.range.startLineNumber + end.line, end.character + 1); + } + + private asValue(node: YamlNode): IValue { + switch (node.type) { + case 'string': + return { type: 'string', value: node.value, range: this.asRange(node) }; + case 'number': + return { type: 'number', value: node.value, range: this.asRange(node) }; + case 'boolean': + return { type: 'boolean', value: node.value, range: this.asRange(node) }; + case 'null': + return { type: 'null', value: node.value, range: this.asRange(node) }; + case 'array': + return { type: 'array', items: node.items.map(item => this.asValue(item)), range: this.asRange(node) }; + case 'object': { + const properties = node.properties.map(property => ({ key: this.asValue(property.key) as IStringValue, value: this.asValue(property.value) })); + return { type: 'object', properties, range: this.asRange(node) }; + } + } + } + public get attributes(): IHeaderAttribute[] { return this.getParsedHeader().attributes; } + + private getStringAttribute(key: string): string | undefined { + const attribute = this.getParsedHeader().attributes.find(attr => attr.key === key); + if (attribute?.value.type === 'string') { + return attribute.value.value; + } + return undefined; + } + + public get description(): string | undefined { + return this.getStringAttribute('description'); + } + + public get mode(): string | undefined { + return this.getStringAttribute('mode'); + } + + public get model(): string | undefined { + return this.getStringAttribute('model'); + } + + public get applyTo(): string | undefined { + return this.getStringAttribute('applyTo'); + } + + public get tools(): Map | undefined { + const toolsAttribute = this.getParsedHeader().attributes.find(attr => attr.key === 'tools'); + if (!toolsAttribute) { + return undefined; + } + if (toolsAttribute.value.type === 'array') { + const tools = new Map; + for (const item of toolsAttribute.value.items) { + if (item.type === 'string') { + tools.set(item.value, true); + } + } + return tools; + } else if (toolsAttribute.value.type === 'object') { + const tools = new Map; + const collectLeafs = ({ key, value }: { key: IStringValue; value: IValue }) => { + if (value.type === 'boolean') { + tools.set(key.value, value.value); + } else if (value.type === 'object') { + value.properties.forEach(collectLeafs); + } + }; + toolsAttribute.value.properties.forEach(collectLeafs); + return tools; + } + return undefined; + } + } + interface IHeaderAttribute { readonly range: Range; readonly key: string; + readonly value: IValue; } +export interface IStringValue { readonly type: 'string'; readonly value: string; readonly range: Range } +export interface INumberValue { readonly type: 'number'; readonly value: number; readonly range: Range } +export interface INullValue { readonly type: 'null'; readonly value: null; readonly range: Range } +export interface IBooleanValue { readonly type: 'boolean'; readonly value: boolean; readonly range: Range } + +export interface IArrayValue { + readonly type: 'array'; + readonly items: readonly IValue[]; + readonly range: Range; +} + +export interface IObjectValue { + readonly type: 'object'; + readonly properties: { key: IStringValue; value: IValue }[]; + readonly range: Range; +} + +export type IValue = IStringValue | INumberValue | IBooleanValue | IArrayValue | IObjectValue | INullValue; + interface ParsedBody { readonly fileReferences: readonly IBodyFileReference[]; @@ -112,6 +210,7 @@ export class PromptBody { private getParsedBody(): ParsedBody { if (this._parsed === undefined) { + const markdownLinkRanges: Range[] = []; const fileReferences: IBodyFileReference[] = []; const variableReferences: IBodyVariableReference[] = []; for (let i = this.range.startLineNumber - 1; i < this.range.endLineNumber - 1; i++) { @@ -122,10 +221,15 @@ export class PromptBody { const linkStartOffset = match.index + match[0].length - match[2].length - 1; const range = new Range(i + 1, linkStartOffset + 1, i + 1, linkEndOffset + 1); fileReferences.push({ content: match[2], range }); + markdownLinkRanges.push(new Range(i + 1, match.index + 1, i + 1, match.index + match[0].length + 1)); } const reg = new RegExp(`${chatVariableLeader}([\\w]+:)?([^\\s#]*)`, 'g'); const matches = line.matchAll(reg); for (const match of matches) { + const fullRange = new Range(i + 1, match.index + 1, i + 1, match.index + match[0].length + 1); + if (markdownLinkRanges.some(mdRange => Range.areIntersectingOrTouching(mdRange, fullRange))) { + continue; + } const varType = match[1]; if (varType) { if (varType === 'file:') { diff --git a/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/newPromptsParser.test.ts b/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/newPromptsParser.test.ts index 1281fcfeef9..97fc5649a94 100644 --- a/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/newPromptsParser.test.ts +++ b/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/newPromptsParser.test.ts @@ -13,36 +13,71 @@ import { NewPromptsParser } from '../../../../common/promptSyntax/service/newPro suite('NewPromptsParser', () => { ensureNoDisposablesAreLeakedInTestSuite(); - - test('provides cached parser instance', async () => { - const uri = URI.parse('file:///test/prompt1.md'); + test('mode', async () => { + const uri = URI.parse('file:///test/chatmode.md'); const content = [ /* 01 */"---", /* 02 */`description: "Agent mode test"`, - /* 03 */"mode: agent", + /* 03 */"model: GPT 4.1", /* 04 */"tools: ['tool1', 'tool2']", /* 05 */"---", - /* 06 */"This is a builtin agent mode test.", + /* 06 */"This is a chat mode test.", /* 07 */"Here is a #tool1 variable and a #file:./reference1.md as well as a [reference](./reference2.md).", ].join('\n'); const result = new NewPromptsParser().parse(uri, content); assert.deepEqual(result.uri, uri); - assert.deepEqual(result.header?.range, { startLineNumber: 2, startColumn: 1, endLineNumber: 5, endColumn: 1 }); - assert.deepEqual(result.header?.attributes, [ - { key: 'description', range: new Range(2, 1, 2, 31) }, - { key: 'mode', range: new Range(3, 1, 3, 12) }, - { key: 'tools', range: new Range(4, 1, 4, 26) }, + assert.ok(result.header); + assert.ok(result.body); + assert.deepEqual(result.header.range, { startLineNumber: 2, startColumn: 1, endLineNumber: 5, endColumn: 1 }); + assert.deepEqual(result.header.attributes, [ + { key: 'description', range: new Range(2, 1, 2, 31), value: { type: 'string', value: 'Agent mode test', range: new Range(2, 14, 2, 31) } }, + { key: 'model', range: new Range(3, 1, 3, 15), value: { type: 'string', value: 'GPT 4.1', range: new Range(3, 8, 3, 15) } }, + { + key: 'tools', range: new Range(4, 1, 4, 26), value: { + type: 'array', + items: [{ type: 'string', value: 'tool1', range: new Range(4, 9, 4, 16) }, { type: 'string', value: 'tool2', range: new Range(4, 18, 4, 25) }], + range: new Range(4, 8, 4, 26) + } + }, ]); - - - assert.deepEqual(result.body?.range, { startLineNumber: 6, startColumn: 1, endLineNumber: 8, endColumn: 1 }); - assert.deepEqual(result.body?.fileReferences, [ + assert.deepEqual(result.body.range, { startLineNumber: 6, startColumn: 1, endLineNumber: 8, endColumn: 1 }); + assert.deepEqual(result.body.fileReferences, [ { range: new Range(7, 39, 7, 54), content: './reference1.md' }, { range: new Range(7, 80, 7, 95), content: './reference2.md' } ]); - assert.deepEqual(result.body?.variableReferences, [ + assert.deepEqual(result.body.variableReferences, [ { range: new Range(7, 12, 7, 17), content: 'tool1' } ]); + assert.deepEqual(result.header.description, 'Agent mode test'); + assert.deepEqual(result.header.model, 'GPT 4.1'); + assert.ok(result.header.tools); + assert.deepEqual([...result.header.tools.entries()], [['tool1', true], ['tool2', true]]); }); + test('instructions', async () => { + const uri = URI.parse('file:///test/prompt1.md'); + const content = [ + /* 01 */"---", + /* 02 */`description: "Code style instructions for TypeScript"`, + /* 03 */"applyTo: *.ts", + /* 04 */"---", + /* 05 */"Follow my companies coding guidlines at [mycomp-ts-guidelines](https://mycomp/guidelines#typescript.md)", + ].join('\n'); + const result = new NewPromptsParser().parse(uri, content); + assert.deepEqual(result.uri, uri); + assert.ok(result.header); + assert.ok(result.body); + assert.deepEqual(result.header.range, { startLineNumber: 2, startColumn: 1, endLineNumber: 4, endColumn: 1 }); + assert.deepEqual(result.header.attributes, [ + { key: 'description', range: new Range(2, 1, 2, 54), value: { type: 'string', value: 'Code style instructions for TypeScript', range: new Range(2, 14, 2, 54) } }, + { key: 'applyTo', range: new Range(3, 1, 3, 14), value: { type: 'string', value: '*.ts', range: new Range(3, 10, 3, 14) } }, + ]); + assert.deepEqual(result.body.range, { startLineNumber: 5, startColumn: 1, endLineNumber: 6, endColumn: 1 }); + assert.deepEqual(result.body.fileReferences, [ + { range: new Range(5, 64, 5, 103), content: 'https://mycomp/guidelines#typescript.md' }, + ]); + assert.deepEqual(result.body.variableReferences, []); + assert.deepEqual(result.header.description, 'Code style instructions for TypeScript'); + assert.deepEqual(result.header.applyTo, '*.ts'); + }); }); From 67e8752460ab5162d6e3bc81bbfd7cec6a18fc4a Mon Sep 17 00:00:00 2001 From: Martin Aeschlimann Date: Tue, 9 Sep 2025 16:25:46 +0200 Subject: [PATCH 007/299] update --- src/vs/base/common/yaml.ts | 51 +++++++++++++++------------- src/vs/base/test/common/yaml.test.ts | 9 ++--- 2 files changed, 32 insertions(+), 28 deletions(-) diff --git a/src/vs/base/common/yaml.ts b/src/vs/base/common/yaml.ts index b6b977e5897..490d068e0f0 100644 --- a/src/vs/base/common/yaml.ts +++ b/src/vs/base/common/yaml.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ /** - * Parses a simplified YAML-like input from an iterable of strings (lines). + * Parses a simplified YAML-like input from a single string. * Supports objects, arrays, primitive types (string, number, boolean, null). * Tracks positions for error reporting and node locations. * @@ -15,13 +15,18 @@ * - No special handling for escape sequences in strings * - Indentation must be consistent (spaces only, no tabs) * - * @param input Iterable of strings representing lines of the YAML-like input + * Notes: + * - New line separators can be either "\n" or "\r\n". The input string is split into lines internally. + * + * @param input A string containing the YAML-like input * @param errors Array to collect parsing errors * @param options Parsing options * @returns The parsed representation (ObjectNode, ArrayNode, or primitive node) */ -export function parse(input: Iterable, errors: YamlParseError[] = [], options: ParseOptions = {}): YamlNode | undefined { - const lines = Array.from(input); +export function parse(input: string, errors: YamlParseError[] = [], options: ParseOptions = {}): YamlNode | undefined { + // Normalize both LF and CRLF by splitting on either; CR characters are not retained as part of line text. + // This keeps the existing line/character based lexer logic intact. + const lines = input.length === 0 ? [] : input.split(/\r\n|\n/); const parser = new YamlParser(lines, errors, options); return parser.parse(); } @@ -254,6 +259,8 @@ class YamlParser { private lexer: YamlLexer; private errors: YamlParseError[]; private options: ParseOptions; + // Track nesting level of flow (inline) collections '[' ']' '{' '}' + private flowLevel: number = 0; constructor(lines: string[], errors: YamlParseError[], options: ParseOptions) { this.lexer = new YamlLexer(lines); @@ -317,51 +324,43 @@ class YamlParser { let endPos = start; // Helper function to check for value terminators - const isTerminator = (char: string): boolean => - char === '#' || char === ',' || char === ']' || char === '}'; + const isTerminator = (char: string): boolean => { + if (char === '#') { return true; } + // Comma, ']' and '}' only terminate inside flow collections + if (this.flowLevel > 0 && (char === ',' || char === ']' || char === '}')) { return true; } + return false; + }; // Handle opening quote that might not be closed const firstChar = this.lexer.getCurrentChar(); if (firstChar === '"' || firstChar === `'`) { value += this.lexer.advance(); endPos = this.lexer.getCurrentPosition(); - - // Continue until we find closing quote or terminator while (!this.lexer.isAtEnd() && this.lexer.getCurrentChar() !== '') { const char = this.lexer.getCurrentChar(); - if (char === firstChar || isTerminator(char)) { break; } - value += this.lexer.advance(); endPos = this.lexer.getCurrentPosition(); } } else { - // Regular unquoted value while (!this.lexer.isAtEnd() && this.lexer.getCurrentChar() !== '') { const char = this.lexer.getCurrentChar(); - if (isTerminator(char)) { break; } - value += this.lexer.advance(); endPos = this.lexer.getCurrentPosition(); } } - - value = value.trim(); - - // Adjust end position for trimmed value - if (value.length === 0) { - endPos = start; - } else { - endPos = createPosition(start.line, start.character + value.length); + const trimmed = value.trimEnd(); + const diff = value.length - trimmed.length; + if (diff) { + endPos = createPosition(start.line, endPos.character - diff); } - - // Return appropriate node type based on value - return this.createValueNode(value, start, endPos); + const finalValue = (firstChar === '"' || firstChar === `'`) ? trimmed.substring(1) : trimmed; + return this.createValueNode(finalValue, start, endPos); } private createValueNode(value: string, start: Position, end: Position): YamlNode { @@ -395,6 +394,7 @@ class YamlParser { parseInlineArray(): YamlArrayNode { const start = this.lexer.getCurrentPosition(); this.lexer.advance(); // Skip '[' + this.flowLevel++; const items: YamlNode[] = []; @@ -426,12 +426,14 @@ class YamlParser { } const end = this.lexer.getCurrentPosition(); + this.flowLevel--; return createArrayNode(items, start, end); } parseInlineObject(): YamlObjectNode { const start = this.lexer.getCurrentPosition(); this.lexer.advance(); // Skip '{' + this.flowLevel++; const properties: { key: YamlStringNode; value: YamlNode }[] = []; @@ -494,6 +496,7 @@ class YamlParser { } const end = this.lexer.getCurrentPosition(); + this.flowLevel--; return createObjectNode(properties, start, end); } diff --git a/src/vs/base/test/common/yaml.test.ts b/src/vs/base/test/common/yaml.test.ts index 49ddbd2596f..cba621bf023 100644 --- a/src/vs/base/test/common/yaml.test.ts +++ b/src/vs/base/test/common/yaml.test.ts @@ -3,13 +3,13 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ import { deepStrictEqual, strictEqual, ok } from 'assert'; -import { ensureNoDisposablesAreLeakedInTestSuite } from './utils.js'; import { parse, ParseOptions, YamlParseError, Position, YamlNode } from '../../common/yaml.js'; - +import { ensureNoDisposablesAreLeakedInTestSuite } from './utils.js'; function assertValidParse(input: string[], expected: YamlNode, expectedErrors: YamlParseError[], options?: ParseOptions): void { const errors: YamlParseError[] = []; - const actual1 = parse(input, errors, options); + const text = input.join('\n'); + const actual1 = parse(text, errors, options); deepStrictEqual(actual1, expected); deepStrictEqual(errors, expectedErrors); } @@ -44,6 +44,7 @@ suite('YAML Parser', () => { assertValidParse(['A Developer'], { type: 'string', start: pos(0, 0), end: pos(0, 11), value: 'A Developer' }, []); assertValidParse(['\'A Developer\''], { type: 'string', start: pos(0, 0), end: pos(0, 13), value: 'A Developer' }, []); assertValidParse(['"A Developer"'], { type: 'string', start: pos(0, 0), end: pos(0, 13), value: 'A Developer' }, []); + assertValidParse(['*.js,*.ts'], { type: 'string', start: pos(0, 0), end: pos(0, 9), value: '*.js,*.ts' }, []); }); }); @@ -893,7 +894,7 @@ suite('YAML Parser', () => { const start = Date.now(); const errors: YamlParseError[] = []; - const result = parse(lines, errors); + const result = parse(lines.join('\n'), errors); const duration = Date.now() - start; ok(result); From 9a87c8e34e7d5b8c1224ca4cbe25293c2d4dd9e6 Mon Sep 17 00:00:00 2001 From: Martin Aeschlimann Date: Tue, 9 Sep 2025 16:29:32 +0200 Subject: [PATCH 008/299] update --- .../chat/common/promptSyntax/service/newPromptsParser.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/service/newPromptsParser.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/service/newPromptsParser.ts index 7a5396f47af..716494a36c3 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/service/newPromptsParser.ts +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/service/newPromptsParser.ts @@ -3,7 +3,6 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { Iterable } from '../../../../../../base/common/iterator.js'; import { dirname, resolvePath } from '../../../../../../base/common/resources.js'; import { splitLinesIncludeSeparators } from '../../../../../../base/common/strings.js'; import { URI } from '../../../../../../base/common/uri.js'; @@ -65,7 +64,7 @@ export class PromptHeader { public getParsedHeader(): ParsedHeader { if (this._parsed === undefined) { const errors: YamlParseError[] = []; - const lines = Iterable.map(Iterable.slice(this.linesWithEOL, this.range.startLineNumber - 1, this.range.endLineNumber - 1), line => line.replace(/[\r\n]+$/, '')); + const lines = this.linesWithEOL.slice(this.range.startLineNumber - 1, this.range.endLineNumber - 1).join(''); const node = parse(lines, errors); const attributes = []; if (node?.type === 'object') { From ff5c6f710106e6b81e05dbaabfe9ea0b30a43c62 Mon Sep 17 00:00:00 2001 From: Martin Aeschlimann Date: Wed, 10 Sep 2025 16:58:40 +0200 Subject: [PATCH 009/299] update --- .../promptSyntax/promptFileContributions.ts | 13 +- .../promptSyntax/service/newPromptsParser.ts | 21 +- .../promptSyntax/service/promptValidator.ts | 352 ++++++++++++++++++ .../service/newPromptsParser.test.ts | 112 ++++++ 4 files changed, 483 insertions(+), 15 deletions(-) create mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/service/promptValidator.ts diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/promptFileContributions.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/promptFileContributions.ts index 63c908abeeb..71d350b20fa 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/promptFileContributions.ts +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/promptFileContributions.ts @@ -8,12 +8,11 @@ import { Registry } from '../../../../../platform/registry/common/platform.js'; import { LifecyclePhase } from '../../../../services/lifecycle/common/lifecycle.js'; import { IWorkbenchContributionsRegistry, Extensions, IWorkbenchContribution } from '../../../../common/contributions.js'; import { PromptLinkProvider } from './languageProviders/promptLinkProvider.js'; -import { PromptLinkDiagnosticsInstanceManager } from './languageProviders/promptLinkDiagnosticsProvider.js'; -import { PromptHeaderDiagnosticsInstanceManager } from './languageProviders/promptHeaderDiagnosticsProvider.js'; import { PromptBodyAutocompletion } from './languageProviders/promptBodyAutocompletion.js'; import { PromptHeaderAutocompletion } from './languageProviders/promptHeaderAutocompletion.js'; import { PromptHeaderHoverProvider } from './languageProviders/promptHeaderHovers.js'; import { PromptHeaderDefinitionProvider } from './languageProviders/PromptHeaderDefinitionProvider.js'; +import { PromptValidatorContribution } from './service/promptValidator.js'; /** @@ -24,15 +23,7 @@ export function registerPromptFileContributions(): void { // all language constributions registerContribution(PromptLinkProvider); - registerContribution(PromptLinkDiagnosticsInstanceManager); - registerContribution(PromptHeaderDiagnosticsInstanceManager); - /** - * PromptDecorationsProviderInstanceManager is currently disabled because the only currently - * available decoration is the Front Matter header, which we decided to disable for now. - * Add it back when more decorations are needed. - */ - // registerContribution(PromptDecorationsProviderInstanceManager); , - + registerContribution(PromptValidatorContribution); registerContribution(PromptBodyAutocompletion); registerContribution(PromptHeaderAutocompletion); diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/service/newPromptsParser.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/service/newPromptsParser.ts index 716494a36c3..f0c88fcb01d 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/service/newPromptsParser.ts +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/service/newPromptsParser.ts @@ -49,9 +49,15 @@ export class ParsedPromptFile { } } +export interface ParseError { + readonly message: string; + readonly range: Range; + readonly code: string; +} + interface ParsedHeader { readonly node: YamlNode | undefined; - readonly errors: YamlParseError[]; + readonly errors: ParseError[]; readonly attributes: IHeaderAttribute[]; } @@ -63,10 +69,11 @@ export class PromptHeader { public getParsedHeader(): ParsedHeader { if (this._parsed === undefined) { - const errors: YamlParseError[] = []; + const yamlErrors: YamlParseError[] = []; const lines = this.linesWithEOL.slice(this.range.startLineNumber - 1, this.range.endLineNumber - 1).join(''); - const node = parse(lines, errors); + const node = parse(lines, yamlErrors); const attributes = []; + const errors: ParseError[] = yamlErrors.map(err => ({ message: err.message, range: this.asRange(err), code: err.code })); if (node?.type === 'object') { for (const property of node.properties) { attributes.push({ @@ -75,6 +82,8 @@ export class PromptHeader { value: this.asValue(property.value) }); } + } else { + errors.push({ message: 'Invalid header, expecting pairs', range: this.range, code: 'INVALID_YAML' }); } this._parsed = { node, attributes, errors }; } @@ -108,6 +117,10 @@ export class PromptHeader { return this.getParsedHeader().attributes; } + public get errors(): ParseError[] { + return this.getParsedHeader().errors; + } + private getStringAttribute(key: string): string | undefined { const attribute = this.getParsedHeader().attributes.find(attr => attr.key === key); if (attribute?.value.type === 'string') { @@ -162,7 +175,7 @@ export class PromptHeader { } -interface IHeaderAttribute { +export interface IHeaderAttribute { readonly range: Range; readonly key: string; readonly value: IValue; diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptValidator.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptValidator.ts new file mode 100644 index 00000000000..488f956733b --- /dev/null +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptValidator.ts @@ -0,0 +1,352 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { isEmptyPattern, parse, splitGlobAware } from '../../../../../../base/common/glob.js'; +import { Iterable } from '../../../../../../base/common/iterator.js'; +import { Range } from '../../../../../../editor/common/core/range.js'; +import { ITextModel } from '../../../../../../editor/common/model.js'; +import { IModelService } from '../../../../../../editor/common/services/model.js'; +import { localize } from '../../../../../../nls.js'; +import { IConfigurationService } from '../../../../../../platform/configuration/common/configuration.js'; +import { IInstantiationService } from '../../../../../../platform/instantiation/common/instantiation.js'; +import { IMarkerData, IMarkerService, MarkerSeverity } from '../../../../../../platform/markers/common/markers.js'; +import { IChatMode, IChatModeService } from '../../chatModes.js'; +import { ChatModeKind } from '../../constants.js'; +import { ILanguageModelChatMetadata, ILanguageModelsService } from '../../languageModels.js'; +import { ILanguageModelToolsService } from '../../languageModelToolsService.js'; +import { getPromptsTypeForLanguageId, PromptsType } from '../promptTypes.js'; +import { IHeaderAttribute, NewPromptsParser, ParsedPromptFile } from './newPromptsParser.js'; +import { PromptsConfig } from '../config/config.js'; +import { Disposable, DisposableStore, toDisposable } from '../../../../../../base/common/lifecycle.js'; +import { Delayer } from '../../../../../../base/common/async.js'; +import { ResourceMap } from '../../../../../../base/common/map.js'; + +const MARKERS_OWNER_ID = 'prompts-diagnostics-provider'; + +export class PromptValidator { + constructor( + @ILanguageModelsService private readonly languageModelsService: ILanguageModelsService, + @ILanguageModelToolsService private readonly languageModelToolsService: ILanguageModelToolsService, + @IChatModeService private readonly chatModeService: IChatModeService, + ) { } + + public validate(promptAST: ParsedPromptFile, model: ITextModel, promptType: PromptsType): IMarkerData[] { + const markers: IMarkerData[] = []; + promptAST.header?.errors.forEach(error => { + markers.push(toMarker(error.message, error.range, MarkerSeverity.Error)); + }); + this.validateHeader(promptAST, model, promptType, markers); + return markers; + } + + private validateHeader(promptAST: ParsedPromptFile, model: ITextModel, promptType: PromptsType, result: IMarkerData[]): void { + const header = promptAST.header; + if (!header) { + return; + } + const validAttributeNames = getValidAttributeNames(promptType); + const attributes = header.attributes; + for (const attribute of attributes) { + if (!validAttributeNames.includes(attribute.key)) { + switch (promptType) { + case PromptsType.prompt: + result.push(toMarker(localize('promptValidator.unknownAttribute.prompt', "Attribute '{0}' is not supported in prompt files. Supported: {1}", attribute.key, validAttributeNames.join(', ')), attribute.range, MarkerSeverity.Warning)); + break; + case PromptsType.mode: + result.push(toMarker(localize('promptValidator.unknownAttribute.mode', "Attribute '{0}' is not supported in mode files. Supported: {1}", attribute.key, validAttributeNames.join(', ')), attribute.range, MarkerSeverity.Warning)); + break; + case PromptsType.instructions: + result.push(toMarker(localize('promptValidator.unknownAttribute.instructions', "Attribute '{0}' is not supported in instructions files. Supported: {1}", attribute.key, validAttributeNames.join(', ')), attribute.range, MarkerSeverity.Warning)); + break; + } + } + } + this.validateDescription(attributes, result); + switch (promptType) { + case PromptsType.prompt: { + const mode = this.validateMode(attributes, result); + this.validateTools(attributes, mode?.kind ?? ChatModeKind.Agent, result); + this.validateModel(attributes, mode?.kind ?? ChatModeKind.Agent, result); + break; + } + case PromptsType.instructions: + this.validateApplyTo(attributes, result); + break; + + case PromptsType.mode: + this.validateTools(attributes, ChatModeKind.Agent, result); + this.validateModel(attributes, ChatModeKind.Agent, result); + break; + + } + } + + private validateDescription(attributes: IHeaderAttribute[], markers: IMarkerData[]): void { + const descriptionAttribute = attributes.find(attr => attr.key === 'description'); + if (!descriptionAttribute) { + return; + } + if (descriptionAttribute.value.type !== 'string') { + markers.push(toMarker(localize('promptValidator.descriptionMustBeString', "The 'description' attribute must be a string."), descriptionAttribute.range, MarkerSeverity.Error)); + return; + } + if (descriptionAttribute.value.value.trim().length === 0) { + markers.push(toMarker(localize('promptValidator.descriptionShouldNotBeEmpty', "The 'description' attribute should not be empty."), descriptionAttribute.value.range, MarkerSeverity.Error)); + return; + } + } + + + private validateModel(attributes: IHeaderAttribute[], modeKind: ChatModeKind, markers: IMarkerData[]): void { + const attribute = attributes.find(attr => attr.key === 'model'); + if (!attribute) { + return; + } + if (attribute.value.type !== 'string') { + markers.push(toMarker(localize('promptValidator.modelMustBeString', "The 'model' attribute must be a string."), attribute.value.range, MarkerSeverity.Error)); + return; + } + const modelName = attribute.value.value.trim(); + if (modelName.length === 0) { + markers.push(toMarker(localize('promptValidator.modelMustBeNonEmpty', "The 'model' attribute must be a non-empty string."), attribute.value.range, MarkerSeverity.Error)); + return; + } + + const languageModes = this.languageModelsService.getLanguageModelIds(); + if (languageModes.length === 0) { + // likely the service is not initialized yet + return; + } + const modelMetadata = this.findModelByName(languageModes, modelName); + if (!modelMetadata) { + markers.push(toMarker(localize('promptValidator.modelNotFound', "Unknown model '{0}'", modelName), attribute.value.range, MarkerSeverity.Warning)); + + } else if (modeKind === ChatModeKind.Agent && !ILanguageModelChatMetadata.suitableForAgentMode(modelMetadata)) { + markers.push(toMarker(localize('promptValidator.modelNotSuited', "Model '{0}' is not suited for agent mode", modelName), attribute.value.range, MarkerSeverity.Warning)); + } + } + + findModelByName(languageModes: string[], modelName: string): ILanguageModelChatMetadata | undefined { + for (const model of languageModes) { + const metadata = this.languageModelsService.lookupLanguageModel(model); + if (metadata && metadata.isUserSelectable !== false && ILanguageModelChatMetadata.matchesQualifiedName(modelName, metadata)) { + return metadata; + } + } + return undefined; + } + + private validateMode(attributes: IHeaderAttribute[], markers: IMarkerData[]): IChatMode | undefined { + const attribute = attributes.find(attr => attr.key === 'mode'); + if (!attribute) { + return undefined; // default mode for prompts is Agent + } + if (attribute.value.type !== 'string') { + markers.push(toMarker(localize('promptValidator.modeMustBeString', "The 'mode' attribute must be a string."), attribute.value.range, MarkerSeverity.Error)); + return undefined; + } + const modeValue = attribute.value.value; + if (modeValue.trim().length === 0) { + markers.push(toMarker(localize('promptValidator.modeMustBeNonEmpty', "The 'mode' attribute must be a non-empty string."), attribute.value.range, MarkerSeverity.Error)); + return undefined; + } + + const modes = this.chatModeService.getModes(); + const availableModes = []; + + // Check if mode exists in builtin or custom modes + for (const mode of Iterable.concat(modes.builtin, modes.custom)) { + if (mode.name === modeValue) { + return mode; + } + availableModes.push(mode.name); // collect all available mode names + } + + const errorMessage = localize('promptValidator.modeNotFound', "Unknown mode '{0}'. Available modes: {1}", modeValue, availableModes.join(', ')); + markers.push(toMarker(errorMessage, attribute.value.range, MarkerSeverity.Warning)); + return undefined; + } + + private validateTools(attributes: IHeaderAttribute[], modeKind: ChatModeKind, markers: IMarkerData[]): undefined { + const attribute = attributes.find(attr => attr.key === 'tools'); + if (!attribute) { + return; + } + if (modeKind !== ChatModeKind.Agent) { + markers.push(toMarker(localize('promptValidator.toolsOnlyInAgent', "The 'tools' attribute is only supported in agent mode. Attribute will be ignored."), attribute.range, MarkerSeverity.Warning)); + + } + if (attribute.value.type !== 'array') { + markers.push(toMarker(localize('promptValidator.toolsMustBeArray', "The 'tools' attribute must be an array."), attribute.value.range, MarkerSeverity.Error)); + return; + } + + const toolNames = new Map(); + for (const item of attribute.value.items) { + if (item.type !== 'string') { + markers.push(toMarker(localize('promptValidator.eachToolMustBeString', "Each tool name in the 'tools' attribute must be a string."), item.range, MarkerSeverity.Error)); + } else { + toolNames.set(item.value, item.range); + } + } + if (toolNames.size === 0) { + return; + } + for (const tool of this.languageModelToolsService.getTools()) { + toolNames.delete(tool.toolReferenceName ?? tool.displayName); + } + for (const toolSet of this.languageModelToolsService.toolSets.get()) { + toolNames.delete(toolSet.referenceName); + } + + for (const [toolName, range] of toolNames) { + markers.push(toMarker(localize('promptValidator.toolNotFound', "Unknown tool '{0}'", toolName), range, MarkerSeverity.Warning)); + } + } + + private validateApplyTo(attributes: IHeaderAttribute[], markers: IMarkerData[]): undefined { + const attribute = attributes.find(attr => attr.key === 'applyTo'); + if (!attribute) { + return; + } + if (attribute.value.type !== 'string') { + markers.push(toMarker(localize('promptValidator.applyToMustBeString', "The 'applyTo' attribute must be a string."), attribute.value.range, MarkerSeverity.Error)); + return; + } + const pattern = attribute.value.value; + try { + const patterns = splitGlobAware(pattern, ','); + if (patterns.length === 0) { + markers.push(toMarker(localize('promptValidator.applyToMustBeValidGlob', "The 'applyTo' attribute must be a valid glob pattern."), attribute.value.range, MarkerSeverity.Error)); + return; + } + for (const pattern of patterns) { + const globPattern = parse(pattern); + if (isEmptyPattern(globPattern)) { + markers.push(toMarker(localize('promptValidator.applyToMustBeValidGlob', "The 'applyTo' attribute must be a valid glob pattern."), attribute.value.range, MarkerSeverity.Error)); + return; + } + } + } catch (_error) { + markers.push(toMarker(localize('promptValidator.applyToMustBeValidGlob', "The 'applyTo' attribute must be a valid glob pattern."), attribute.value.range, MarkerSeverity.Error)); + } + } +} + +function getValidAttributeNames(promptType: PromptsType): string[] { + switch (promptType) { + case PromptsType.prompt: + return ['description', 'model', 'tools', 'mode']; + case PromptsType.instructions: + return ['description', 'applyTo']; + case PromptsType.mode: + return ['description', 'model', 'tools']; + } +} + +function toMarker(message: string, range: Range, severity = MarkerSeverity.Error): IMarkerData { + return { severity, message, ...range }; +} + +export class PromptValidatorContribution extends Disposable { + + private readonly validator: PromptValidator; + private readonly promptParser: NewPromptsParser; + private readonly localDisposables = this._register(new DisposableStore()); + + constructor( + @IModelService private modelService: IModelService, + @IInstantiationService instantiationService: IInstantiationService, + @IConfigurationService private configService: IConfigurationService, + @IMarkerService private readonly markerService: IMarkerService, + ) { + super(); + this.validator = instantiationService.createInstance(PromptValidator); + this.promptParser = instantiationService.createInstance(NewPromptsParser); + + this.updateRegistration(); + this._register(this.configService.onDidChangeConfiguration(e => { + if (e.affectsConfiguration(PromptsConfig.KEY)) { + this.updateRegistration(); + } + })); + } + + updateRegistration(): void { + this.localDisposables.clear(); + if (!PromptsConfig.enabled(this.configService)) { + return; + } + const trackers = new ResourceMap(); + this.localDisposables.add(toDisposable(() => { + trackers.forEach(tracker => tracker.dispose()); + })); + this.modelService.getModels().forEach(model => { + const promptType = getPromptsTypeForLanguageId(model.getLanguageId()); + if (promptType) { + trackers.set(model.uri, new ModelTracker(model, promptType, this.validator, this.promptParser, this.markerService)); + } + }); + this.localDisposables.add(this.modelService.onModelAdded((model) => { + const promptType = getPromptsTypeForLanguageId(model.getLanguageId()); + if (promptType) { + trackers.set(model.uri, new ModelTracker(model, promptType, this.validator, this.promptParser, this.markerService)); + } + })); + this.localDisposables.add(this.modelService.onModelRemoved((model) => { + const promptType = getPromptsTypeForLanguageId(model.getLanguageId()); + if (promptType) { + const tracker = trackers.get(model.uri); + if (tracker) { + tracker.dispose(); + trackers.delete(model.uri); + } + } + })); + this.localDisposables.add(this.modelService.onModelLanguageChanged((event) => { + const { model } = event; + const tracker = trackers.get(model.uri); + if (tracker) { + tracker.dispose(); + trackers.delete(model.uri); + } + const promptType = getPromptsTypeForLanguageId(model.getLanguageId()); + if (promptType) { + trackers.set(model.uri, new ModelTracker(model, promptType, this.validator, this.promptParser, this.markerService)); + } + })); + } +} + +class ModelTracker extends Disposable { + + private readonly delayer: Delayer; + + constructor( + private readonly textModel: ITextModel, + private readonly promptType: PromptsType, + private readonly validator: PromptValidator, + private readonly promptParser: NewPromptsParser, + @IMarkerService private readonly markerService: IMarkerService, + ) { + super(); + this.delayer = this._register(new Delayer(200)); + this._register(textModel.onDidChangeContent(() => this.validate())); + this.validate(); + } + + private validate(): void { + this.delayer.trigger(() => { + const ast = this.promptParser.parse(this.textModel.uri, this.textModel.getValue()); + const markers = this.validator.validate(ast, this.textModel, this.promptType); + this.markerService.changeOne(MARKERS_OWNER_ID, this.textModel.uri, markers); + }); + } + + public override dispose() { + this.markerService.remove(MARKERS_OWNER_ID, [this.textModel.uri]); + super.dispose(); + } +} diff --git a/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/newPromptsParser.test.ts b/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/newPromptsParser.test.ts index 97fc5649a94..57352939405 100644 --- a/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/newPromptsParser.test.ts +++ b/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/newPromptsParser.test.ts @@ -80,4 +80,116 @@ suite('NewPromptsParser', () => { assert.deepEqual(result.header.description, 'Code style instructions for TypeScript'); assert.deepEqual(result.header.applyTo, '*.ts'); }); + + test('prompt file', async () => { + const uri = URI.parse('file:///test/prompt2.md'); + const content = [ + /* 01 */"---", + /* 02 */`description: "General purpose coding assistant"`, + /* 03 */"mode: agent", + /* 04 */"model: GPT 4.1", + /* 05 */"tools: ['search', 'terminal']", + /* 06 */"---", + /* 07 */"This is a prompt file body referencing #search and [docs](https://example.com/docs).", + ].join('\n'); + const result = new NewPromptsParser().parse(uri, content); + assert.deepEqual(result.uri, uri); + assert.ok(result.header); + assert.ok(result.body); + assert.deepEqual(result.header.range, { startLineNumber: 2, startColumn: 1, endLineNumber: 6, endColumn: 1 }); + assert.deepEqual(result.header.attributes, [ + { key: 'description', range: new Range(2, 1, 2, 48), value: { type: 'string', value: 'General purpose coding assistant', range: new Range(2, 14, 2, 48) } }, + { key: 'mode', range: new Range(3, 1, 3, 13), value: { type: 'string', value: 'agent', range: new Range(3, 7, 3, 12) } }, + { key: 'model', range: new Range(4, 1, 4, 15), value: { type: 'string', value: 'GPT 4.1', range: new Range(4, 8, 4, 15) } }, + { + key: 'tools', range: new Range(5, 1, 5, 30), value: { + type: 'array', + items: [{ type: 'string', value: 'search', range: new Range(5, 9, 5, 17) }, { type: 'string', value: 'terminal', range: new Range(5, 19, 5, 29) }], + range: new Range(5, 8, 5, 30) + } + }, + ]); + assert.deepEqual(result.body.range, { startLineNumber: 7, startColumn: 1, endLineNumber: 8, endColumn: 1 }); + assert.deepEqual(result.body.fileReferences, [ + { range: new Range(7, 59, 7, 83), content: 'https://example.com/docs' }, + ]); + assert.deepEqual(result.body.variableReferences, [ + { range: new Range(7, 41, 7, 47), content: 'search' } + ]); + assert.deepEqual(result.header.description, 'General purpose coding assistant'); + assert.deepEqual(result.header.mode, 'prompt'); + assert.deepEqual(result.header.model, 'GPT 4.1'); + assert.ok(result.header.tools); + assert.deepEqual([...result.header.tools.entries()], [['search', true], ['terminal', true]]); + }); + + test('prompt file tools as map', async () => { + const uri = URI.parse('file:///test/prompt2.md'); + const content = [ + /* 01 */"---", + /* 02 */"tools:", + /* 03 */" built-in: true", + /* 04 */" mcp:", + /* 05 */" vscode-playright-mcp:", + /* 06 */" browser-click: true", + /* 07 */" extensions:", + /* 08 */" github.vscode-pull-request-github:", + /* 09 */" openPullRequest: true", + /* 10 */" copilotCodingAgent: false", + /* 11 */"---", + ].join('\n'); + const result = new NewPromptsParser().parse(uri, content); + assert.deepEqual(result.uri, uri); + assert.ok(result.header); + assert.ok(!result.body); + assert.deepEqual(result.header.range, { startLineNumber: 2, startColumn: 1, endLineNumber: 11, endColumn: 1 }); + assert.deepEqual(result.header.attributes, [ + { + key: 'tools', range: new Range(2, 1, 10, 32), value: { + type: 'object', + properties: [ + { + "key": { type: 'string', value: 'built-in', range: new Range(3, 3, 3, 11) }, + "value": { type: 'boolean', value: true, range: new Range(3, 13, 3, 17) } + }, + { + "key": { type: 'string', value: 'mcp', range: new Range(4, 3, 4, 6) }, + "value": { + type: 'object', range: new Range(5, 5, 6, 26), properties: [ + { + "key": { type: 'string', value: 'vscode-playright-mcp', range: new Range(5, 5, 5, 25) }, "value": { + type: 'object', range: new Range(6, 7, 6, 26), properties: [ + { "key": { type: 'string', value: 'browser-click', range: new Range(6, 7, 6, 20) }, "value": { type: 'boolean', value: true, range: new Range(6, 22, 6, 26) } } + ] + } + } + ] + } + }, + { + "key": { type: 'string', value: 'extensions', range: new Range(7, 3, 7, 13) }, + "value": { + type: 'object', range: new Range(8, 5, 10, 32), properties: [ + { + "key": { type: 'string', value: 'github.vscode-pull-request-github', range: new Range(8, 5, 8, 38) }, "value": { + type: 'object', range: new Range(9, 7, 10, 32), properties: [ + { "key": { type: 'string', value: 'openPullRequest', range: new Range(9, 7, 9, 22) }, "value": { type: 'boolean', value: true, range: new Range(9, 24, 9, 28) } }, + { "key": { type: 'string', value: 'copilotCodingAgent', range: new Range(10, 7, 10, 25) }, "value": { type: 'boolean', value: false, range: new Range(10, 27, 10, 32) } } + ] + } + } + ] + } + }, + ], + range: new Range(3, 3, 10, 32) + }, + } + ]); + assert.deepEqual(result.header.description, undefined); + assert.deepEqual(result.header.mode, undefined); + assert.deepEqual(result.header.model, undefined); + assert.ok(result.header.tools); + assert.deepEqual([...result.header.tools.entries()], [['built-in', true], ['browser-click', true], ['openPullRequest', true], ['copilotCodingAgent', false]]); + }); }); From a2b3aee71293c653a9fc868e05e4a1bbaebc7cad Mon Sep 17 00:00:00 2001 From: Martin Aeschlimann Date: Wed, 10 Sep 2025 18:43:31 +0200 Subject: [PATCH 010/299] update --- .../promptSyntax/service/promptValidator.ts | 18 +- .../chat/test/common/mockChatModeService.ts | 4 +- .../service/newPromptsParser.test.ts | 4 +- .../service/promptValidator.test.ts | 275 ++++++++++++++++++ .../service/promptsService.test.ts | 1 + 5 files changed, 289 insertions(+), 13 deletions(-) create mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/service/promptValidator.test.ts diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptValidator.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptValidator.ts index 488f956733b..288c26c6c53 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptValidator.ts +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptValidator.ts @@ -32,16 +32,16 @@ export class PromptValidator { @IChatModeService private readonly chatModeService: IChatModeService, ) { } - public validate(promptAST: ParsedPromptFile, model: ITextModel, promptType: PromptsType): IMarkerData[] { + public validate(promptAST: ParsedPromptFile, promptType: PromptsType): IMarkerData[] { const markers: IMarkerData[] = []; promptAST.header?.errors.forEach(error => { markers.push(toMarker(error.message, error.range, MarkerSeverity.Error)); }); - this.validateHeader(promptAST, model, promptType, markers); + this.validateHeader(promptAST, promptType, markers); return markers; } - private validateHeader(promptAST: ParsedPromptFile, model: ITextModel, promptType: PromptsType, result: IMarkerData[]): void { + private validateHeader(promptAST: ParsedPromptFile, promptType: PromptsType, result: IMarkerData[]): void { const header = promptAST.header; if (!header) { return; @@ -121,14 +121,14 @@ export class PromptValidator { } const modelMetadata = this.findModelByName(languageModes, modelName); if (!modelMetadata) { - markers.push(toMarker(localize('promptValidator.modelNotFound', "Unknown model '{0}'", modelName), attribute.value.range, MarkerSeverity.Warning)); + markers.push(toMarker(localize('promptValidator.modelNotFound', "Unknown model '{0}'.", modelName), attribute.value.range, MarkerSeverity.Warning)); } else if (modeKind === ChatModeKind.Agent && !ILanguageModelChatMetadata.suitableForAgentMode(modelMetadata)) { - markers.push(toMarker(localize('promptValidator.modelNotSuited', "Model '{0}' is not suited for agent mode", modelName), attribute.value.range, MarkerSeverity.Warning)); + markers.push(toMarker(localize('promptValidator.modelNotSuited', "Model '{0}' is not suited for agent mode.", modelName), attribute.value.range, MarkerSeverity.Warning)); } } - findModelByName(languageModes: string[], modelName: string): ILanguageModelChatMetadata | undefined { + private findModelByName(languageModes: string[], modelName: string): ILanguageModelChatMetadata | undefined { for (const model of languageModes) { const metadata = this.languageModelsService.lookupLanguageModel(model); if (metadata && metadata.isUserSelectable !== false && ILanguageModelChatMetadata.matchesQualifiedName(modelName, metadata)) { @@ -164,7 +164,7 @@ export class PromptValidator { availableModes.push(mode.name); // collect all available mode names } - const errorMessage = localize('promptValidator.modeNotFound', "Unknown mode '{0}'. Available modes: {1}", modeValue, availableModes.join(', ')); + const errorMessage = localize('promptValidator.modeNotFound', "Unknown mode '{0}'. Available modes: {1}.", modeValue, availableModes.join(', ')); markers.push(toMarker(errorMessage, attribute.value.range, MarkerSeverity.Warning)); return undefined; } @@ -202,7 +202,7 @@ export class PromptValidator { } for (const [toolName, range] of toolNames) { - markers.push(toMarker(localize('promptValidator.toolNotFound', "Unknown tool '{0}'", toolName), range, MarkerSeverity.Warning)); + markers.push(toMarker(localize('promptValidator.toolNotFound', "Unknown tool '{0}'.", toolName), range, MarkerSeverity.Warning)); } } @@ -340,7 +340,7 @@ class ModelTracker extends Disposable { private validate(): void { this.delayer.trigger(() => { const ast = this.promptParser.parse(this.textModel.uri, this.textModel.getValue()); - const markers = this.validator.validate(ast, this.textModel, this.promptType); + const markers = this.validator.validate(ast, this.promptType); this.markerService.changeOne(MARKERS_OWNER_ID, this.textModel.uri, markers); }); } diff --git a/src/vs/workbench/contrib/chat/test/common/mockChatModeService.ts b/src/vs/workbench/contrib/chat/test/common/mockChatModeService.ts index 300b1f202fc..fc3cdb78daf 100644 --- a/src/vs/workbench/contrib/chat/test/common/mockChatModeService.ts +++ b/src/vs/workbench/contrib/chat/test/common/mockChatModeService.ts @@ -10,10 +10,10 @@ import { ChatMode, IChatMode, IChatModeService } from '../../common/chatModes.js export class MockChatModeService implements IChatModeService { readonly _serviceBrand: undefined; - private _modes: { builtin: readonly IChatMode[]; custom: readonly IChatMode[] } = { builtin: [ChatMode.Ask], custom: [] }; - public readonly onDidChangeChatModes = Event.None; + constructor(private readonly _modes: { builtin: readonly IChatMode[]; custom: readonly IChatMode[] } = { builtin: [ChatMode.Ask], custom: [] }) { } + getModes(): { builtin: readonly IChatMode[]; custom: readonly IChatMode[] } { return this._modes; } diff --git a/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/newPromptsParser.test.ts b/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/newPromptsParser.test.ts index 57352939405..62ee58be278 100644 --- a/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/newPromptsParser.test.ts +++ b/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/newPromptsParser.test.ts @@ -99,7 +99,7 @@ suite('NewPromptsParser', () => { assert.deepEqual(result.header.range, { startLineNumber: 2, startColumn: 1, endLineNumber: 6, endColumn: 1 }); assert.deepEqual(result.header.attributes, [ { key: 'description', range: new Range(2, 1, 2, 48), value: { type: 'string', value: 'General purpose coding assistant', range: new Range(2, 14, 2, 48) } }, - { key: 'mode', range: new Range(3, 1, 3, 13), value: { type: 'string', value: 'agent', range: new Range(3, 7, 3, 12) } }, + { key: 'mode', range: new Range(3, 1, 3, 12), value: { type: 'string', value: 'agent', range: new Range(3, 7, 3, 12) } }, { key: 'model', range: new Range(4, 1, 4, 15), value: { type: 'string', value: 'GPT 4.1', range: new Range(4, 8, 4, 15) } }, { key: 'tools', range: new Range(5, 1, 5, 30), value: { @@ -117,7 +117,7 @@ suite('NewPromptsParser', () => { { range: new Range(7, 41, 7, 47), content: 'search' } ]); assert.deepEqual(result.header.description, 'General purpose coding assistant'); - assert.deepEqual(result.header.mode, 'prompt'); + assert.deepEqual(result.header.mode, 'agent'); assert.deepEqual(result.header.model, 'GPT 4.1'); assert.ok(result.header.tools); assert.deepEqual([...result.header.tools.entries()], [['search', true], ['terminal', true]]); diff --git a/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/promptValidator.test.ts b/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/promptValidator.test.ts new file mode 100644 index 00000000000..58231ccc889 --- /dev/null +++ b/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/promptValidator.test.ts @@ -0,0 +1,275 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'assert'; + +import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../../../../base/test/common/utils.js'; +import { URI } from '../../../../../../../base/common/uri.js'; +import { NewPromptsParser } from '../../../../common/promptSyntax/service/newPromptsParser.js'; +import { PromptValidator } from '../../../../common/promptSyntax/service/promptValidator.js'; +import { TestInstantiationService } from '../../../../../../../platform/instantiation/test/common/instantiationServiceMock.js'; +import { TestConfigurationService } from '../../../../../../../platform/configuration/test/common/testConfigurationService.js'; +import { PromptsConfig } from '../../../../common/promptSyntax/config/config.js'; +import { IConfigurationService } from '../../../../../../../platform/configuration/common/configuration.js'; +import { ILanguageModelToolsService, IToolData, ToolDataSource, ToolSet } from '../../../../common/languageModelToolsService.js'; +import { ObservableSet } from '../../../../../../../base/common/observable.js'; +import { ILanguageModelChatMetadata, ILanguageModelsService } from '../../../../common/languageModels.js'; +import { ExtensionIdentifier } from '../../../../../../../platform/extensions/common/extensions.js'; +import { ChatMode, CustomChatMode, IChatModeService } from '../../../../common/chatModes.js'; +import { MockChatModeService } from '../../mockChatModeService.js'; +import { PromptsType } from '../../../../common/promptSyntax/promptTypes.js'; +import { IMarkerData, MarkerSeverity } from '../../../../../../../platform/markers/common/markers.js'; +import { getPromptFileExtension } from '../../../../common/promptSyntax/config/promptFileLocations.js'; + +suite('PromptValidator', () => { + const disposables = ensureNoDisposablesAreLeakedInTestSuite(); + + let instaService: TestInstantiationService; + + setup(async () => { + instaService = disposables.add(new TestInstantiationService()); + + const testConfigService = new TestConfigurationService(); + testConfigService.setUserConfiguration(PromptsConfig.KEY, true); + + instaService.stub(IConfigurationService, testConfigService); + + const testTool1 = { id: 'testTool1', displayName: 'tool1', canBeReferencedInPrompt: true, modelDescription: 'Test Tool 1', source: ToolDataSource.External, inputSchema: {} } satisfies IToolData; + const testTool2 = { id: 'testTool2', displayName: 'tool2', canBeReferencedInPrompt: true, toolReferenceName: 'tool2', modelDescription: 'Test Tool 2', source: ToolDataSource.External, inputSchema: {} } satisfies IToolData; + + instaService.stub(ILanguageModelToolsService, { + getTools() { return [testTool1, testTool2]; }, + toolSets: new ObservableSet().observable + }); + + const testModels: ILanguageModelChatMetadata[] = [ + { id: 'mae-4', name: 'MAE 4', vendor: 'olama', version: '1.0', family: 'mae', modelPickerCategory: undefined, extension: new ExtensionIdentifier('a.b'), isUserSelectable: true, maxInputTokens: 8192, maxOutputTokens: 1024, capabilities: { agentMode: true, toolCalling: true } } satisfies ILanguageModelChatMetadata, + { id: 'mae-4.1', name: 'MAE 4.1', vendor: 'copilot', version: '1.0', family: 'mae', modelPickerCategory: undefined, extension: new ExtensionIdentifier('a.b'), isUserSelectable: true, maxInputTokens: 8192, maxOutputTokens: 1024, capabilities: { agentMode: true, toolCalling: true } } satisfies ILanguageModelChatMetadata, + { id: 'mae-3.5-turbo', name: 'MAE 3.5 Turbo', vendor: 'copilot', version: '1.0', family: 'mae', modelPickerCategory: undefined, extension: new ExtensionIdentifier('a.b'), isUserSelectable: true, maxInputTokens: 8192, maxOutputTokens: 1024 } satisfies ILanguageModelChatMetadata + ]; + + instaService.stub(ILanguageModelsService, { + getLanguageModelIds() { return testModels.map(m => m.id); }, + lookupLanguageModel(name: string) { + return testModels.find(m => m.id === name); + } + }); + + const customChatMode = new CustomChatMode({ uri: URI.file('/test/chatmode.md'), name: 'BeastMode', body: '', variableReferences: [] }); + instaService.stub(IChatModeService, new MockChatModeService({ builtin: [ChatMode.Agent, ChatMode.Ask, ChatMode.Edit], custom: [customChatMode] })); + }); + + function validate(code: string, promptType: PromptsType): IMarkerData[] { + const uri = URI.parse('file:///test/chatmode' + getPromptFileExtension(promptType)); + const result = new NewPromptsParser().parse(uri, code); + const validator = instaService.createInstance(PromptValidator); + return validator.validate(result, promptType); + } + suite('modes', () => { + + test('correct mode', async () => { + const content = [ + /* 01 */"---", + /* 02 */`description: "Agent mode test"`, + /* 03 */"model: MAE 4.1", + /* 04 */"tools: ['tool1', 'tool2']", + /* 05 */"---", + /* 06 */"This is a chat mode test.", + /* 07 */"Here is a #tool1 variable and a #file:./reference1.md as well as a [reference](./reference2.md).", + ].join('\n'); + const markers = validate(content, PromptsType.mode); + assert.deepStrictEqual(markers, []); + }); + + test('mode with errors (empty description, unknown tool & model)', async () => { + const content = [ + /* 01 */"---", + /* 02 */`description: ""`, // empty description -> error + /* 03 */"model: MAE 4.2", // unknown model -> warning + /* 04 */"tools: ['tool1', 'tool2', 'tool3']", // tool3 unknown -> warning + /* 05 */"---", + /* 06 */"Body", + ].join('\n'); + const markers = validate(content, PromptsType.mode); + assert.strictEqual(markers.length, 3, 'Expected 3 validation issues'); + assert.deepStrictEqual( + markers.map(m => ({ severity: m.severity, message: m.message })), + [ + { severity: MarkerSeverity.Error, message: "The 'description' attribute should not be empty." }, + { severity: MarkerSeverity.Warning, message: "Unknown tool 'tool3'." }, + { severity: MarkerSeverity.Warning, message: "Unknown model 'MAE 4.2'." }, + ] + ); + }); + + test('tools must be array', async () => { + const content = [ + "---", + "description: \"Test\"", + "tools: 'tool1'", + "---", + ].join('\n'); + const markers = validate(content, PromptsType.mode); + assert.strictEqual(markers.length, 1); + assert.deepStrictEqual(markers.map(m => m.message), ["The 'tools' attribute must be an array."]); + }); + + test('each tool must be string', async () => { + const content = [ + "---", + "description: \"Test\"", + "tools: ['tool1', 2]", + "---", + ].join('\n'); + const markers = validate(content, PromptsType.mode); + assert.strictEqual(markers.length, 1); + assert.strictEqual(markers[0].message, "Each tool name in the 'tools' attribute must be a string."); + }); + + test('unknown attribute in mode file', async () => { + const content = [ + "---", + "description: \"Test\"", + "applyTo: '*.ts'", // not allowed in mode file + "---", + ].join('\n'); + const markers = validate(content, PromptsType.mode); + assert.strictEqual(markers.length, 1); + assert.strictEqual(markers[0].severity, MarkerSeverity.Warning); + assert.ok(markers[0].message.startsWith("Attribute 'applyTo' is not supported in mode files.")); + }); + }); + + suite('instructions', () => { + + test('instructions valid', async () => { + const content = [ + "---", + "description: \"Instr\"", + "applyTo: *.ts,*.js", + "---", + ].join('\n'); + const markers = validate(content, PromptsType.instructions); + assert.deepEqual(markers, []); + }); + + test('instructions invalid applyTo type', async () => { + const content = [ + "---", + "description: \"Instr\"", + "applyTo: 5", + "---", + ].join('\n'); + const markers = validate(content, PromptsType.instructions); + assert.strictEqual(markers.length, 1); + assert.strictEqual(markers[0].message, "The 'applyTo' attribute must be a string."); + }); + + test('instructions invalid applyTo glob & unknown attribute', async () => { + const content = [ + "---", + "description: \"Instr\"", + "applyTo: ''", // empty -> invalid glob + "model: mae-4", // model not allowed in instructions + "---", + ].join('\n'); + const markers = validate(content, PromptsType.instructions); + assert.strictEqual(markers.length, 2); + // Order: unknown attribute warnings first (attribute iteration) then applyTo validation + assert.strictEqual(markers[0].severity, MarkerSeverity.Warning); + assert.ok(markers[0].message.startsWith("Attribute 'model' is not supported in instructions files.")); + assert.strictEqual(markers[1].message, "The 'applyTo' attribute must be a valid glob pattern."); + }); + + test('invalid header structure (YAML array)', async () => { + const content = [ + "---", + "- item1", + "---", + "Body", + ].join('\n'); + const markers = validate(content, PromptsType.instructions); + assert.strictEqual(markers.length, 1); + assert.strictEqual(markers[0].message, 'Invalid header, expecting pairs'); + }); + }); + + suite('prompts', () => { + + test('prompt valid with agent mode (default) and tools and a BYO model', async () => { + // mode omitted -> defaults to Agent; tools+model should validate; model MAE 4 is agent capable + const content = [ + '---', + 'description: "Prompt with tools"', + "model: MAE 4 (olama)", + "tools: ['tool1','tool2']", + '---', + 'Body' + ].join('\n'); + const markers = validate(content, PromptsType.prompt); + assert.deepStrictEqual(markers, []); + }); + + test('prompt model not suited for agent mode', async () => { + // MAE 3.5 Turbo lacks agentMode capability -> warning when used in agent (default) mode + const content = [ + '---', + 'description: "Prompt with unsuitable model"', + "model: MAE 3.5 Turbo", + '---', + 'Body' + ].join('\n'); + const markers = validate(content, PromptsType.prompt); + assert.strictEqual(markers.length, 1, 'Expected one warning about unsuitable model'); + assert.strictEqual(markers[0].severity, MarkerSeverity.Warning); + assert.strictEqual(markers[0].message, "Model 'MAE 3.5 Turbo' is not suited for agent mode."); + }); + + test('prompt with custom mode BeastMode and tools', async () => { + // Explicit custom mode should be recognized; BeastMode kind comes from setup; ensure tools accepted + const content = [ + '---', + 'description: "Prompt custom mode"', + 'mode: BeastMode', + "tools: ['tool1']", + '---', + 'Body' + ].join('\n'); + const markers = validate(content, PromptsType.prompt); + assert.deepStrictEqual(markers, []); + }); + + test('prompt with mode Ask and tools warns', async () => { + const content = [ + '---', + 'description: "Prompt ask mode with tools"', + 'mode: Ask', + "tools: ['tool1','tool2']", + '---', + 'Body' + ].join('\n'); + const markers = validate(content, PromptsType.prompt); + assert.strictEqual(markers.length, 1, 'Expected one warning about unknown mode'); + assert.strictEqual(markers[0].severity, MarkerSeverity.Warning); + assert.strictEqual(markers[0].message, "Unknown mode 'Ask'. Available modes: agent, ask, edit, BeastMode."); + }); + + test('prompt with mode edit', async () => { + const content = [ + '---', + 'description: "Prompt edit mode with tool"', + 'mode: edit', + "tools: ['tool1']", + '---', + 'Body' + ].join('\n'); + const markers = validate(content, PromptsType.prompt); + assert.strictEqual(markers.length, 1); + assert.strictEqual(markers[0].severity, MarkerSeverity.Warning); + assert.strictEqual(markers[0].message, "The 'tools' attribute is only supported in agent mode. Attribute will be ignored."); + }); + }); + +}); diff --git a/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/promptsService.test.ts b/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/promptsService.test.ts index ffb62a68d0d..69ea302d960 100644 --- a/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/promptsService.test.ts +++ b/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/promptsService.test.ts @@ -699,6 +699,7 @@ suite('PromptsService', () => { const result1 = await service.parse(rootFileUri, PromptsType.prompt, CancellationToken.None); assert.deepEqual(result1, { uri: rootFileUri, + header: undefined, metadata: { promptType: PromptsType.prompt, description: 'Root prompt description.', From 7c1912cde1b6f9519197f9238353db1d7074055b Mon Sep 17 00:00:00 2001 From: Martin Aeschlimann Date: Wed, 10 Sep 2025 20:29:52 +0200 Subject: [PATCH 011/299] update --- .../promptSyntax/service/promptValidator.ts | 162 +++++++++++------- .../service/promptValidator.test.ts | 98 ++++++++--- 2 files changed, 179 insertions(+), 81 deletions(-) diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptValidator.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptValidator.ts index 288c26c6c53..fe9cfa3f8c7 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptValidator.ts +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptValidator.ts @@ -22,6 +22,7 @@ import { PromptsConfig } from '../config/config.js'; import { Disposable, DisposableStore, toDisposable } from '../../../../../../base/common/lifecycle.js'; import { Delayer } from '../../../../../../base/common/async.js'; import { ResourceMap } from '../../../../../../base/common/map.js'; +import { IFileService } from '../../../../../../platform/files/common/files.js'; const MARKERS_OWNER_ID = 'prompts-diagnostics-provider'; @@ -30,18 +31,55 @@ export class PromptValidator { @ILanguageModelsService private readonly languageModelsService: ILanguageModelsService, @ILanguageModelToolsService private readonly languageModelToolsService: ILanguageModelToolsService, @IChatModeService private readonly chatModeService: IChatModeService, + @IFileService private readonly fileService: IFileService, ) { } - public validate(promptAST: ParsedPromptFile, promptType: PromptsType): IMarkerData[] { - const markers: IMarkerData[] = []; - promptAST.header?.errors.forEach(error => { - markers.push(toMarker(error.message, error.range, MarkerSeverity.Error)); - }); - this.validateHeader(promptAST, promptType, markers); - return markers; + public async validate(promptAST: ParsedPromptFile, promptType: PromptsType, report: (markers: IMarkerData) => void): Promise { + promptAST.header?.errors.forEach(error => report(toMarker(error.message, error.range, MarkerSeverity.Error))); + this.validateHeader(promptAST, promptType, report); + await this.validateBody(promptAST, report); } - private validateHeader(promptAST: ParsedPromptFile, promptType: PromptsType, result: IMarkerData[]): void { + private async validateBody(promptAST: ParsedPromptFile, report: (markers: IMarkerData) => void): Promise { + const body = promptAST.body; + if (!body) { + return; + } + + // Validate file references + const fileReferenceChecks: Promise[] = []; + for (const ref of body.fileReferences) { + const resolved = body.resolveFilePath(ref.content); + if (!resolved) { + report(toMarker(localize('promptValidator.invalidFileReference', "Invalid file reference '{0}'.", ref.content), ref.range, MarkerSeverity.Warning)); + continue; + } + fileReferenceChecks.push((async () => { + try { + const exists = await this.fileService.exists(resolved); + if (!exists) { + report(toMarker(localize('promptValidator.fileNotFound', "File '{0}' not found.", ref.content), ref.range, MarkerSeverity.Warning)); + } + } catch { + report(toMarker(localize('promptValidator.fileNotFound', "File '{0}' not found.", ref.content), ref.range, MarkerSeverity.Warning)); + } + })()); + } + + // Validate variable references (tool or toolset names) + if (body.variableReferences.length) { + const available = this.getAvailableToolAndToolSetNames(); + for (const variable of body.variableReferences) { + if (!available.has(variable.content)) { + report(toMarker(localize('promptValidator.unknownVariableReference', "Unknown tool or toolset '{0}'.", variable.content), variable.range, MarkerSeverity.Warning)); + } + } + } + + await Promise.all(fileReferenceChecks); + } + + private validateHeader(promptAST: ParsedPromptFile, promptType: PromptsType, report: (markers: IMarkerData) => void): void { const header = promptAST.header; if (!header) { return; @@ -52,65 +90,65 @@ export class PromptValidator { if (!validAttributeNames.includes(attribute.key)) { switch (promptType) { case PromptsType.prompt: - result.push(toMarker(localize('promptValidator.unknownAttribute.prompt', "Attribute '{0}' is not supported in prompt files. Supported: {1}", attribute.key, validAttributeNames.join(', ')), attribute.range, MarkerSeverity.Warning)); + report(toMarker(localize('promptValidator.unknownAttribute.prompt', "Attribute '{0}' is not supported in prompt files. Supported: {1}", attribute.key, validAttributeNames.join(', ')), attribute.range, MarkerSeverity.Warning)); break; case PromptsType.mode: - result.push(toMarker(localize('promptValidator.unknownAttribute.mode', "Attribute '{0}' is not supported in mode files. Supported: {1}", attribute.key, validAttributeNames.join(', ')), attribute.range, MarkerSeverity.Warning)); + report(toMarker(localize('promptValidator.unknownAttribute.mode', "Attribute '{0}' is not supported in mode files. Supported: {1}", attribute.key, validAttributeNames.join(', ')), attribute.range, MarkerSeverity.Warning)); break; case PromptsType.instructions: - result.push(toMarker(localize('promptValidator.unknownAttribute.instructions', "Attribute '{0}' is not supported in instructions files. Supported: {1}", attribute.key, validAttributeNames.join(', ')), attribute.range, MarkerSeverity.Warning)); + report(toMarker(localize('promptValidator.unknownAttribute.instructions', "Attribute '{0}' is not supported in instructions files. Supported: {1}", attribute.key, validAttributeNames.join(', ')), attribute.range, MarkerSeverity.Warning)); break; } } } - this.validateDescription(attributes, result); + this.validateDescription(attributes, report); switch (promptType) { case PromptsType.prompt: { - const mode = this.validateMode(attributes, result); - this.validateTools(attributes, mode?.kind ?? ChatModeKind.Agent, result); - this.validateModel(attributes, mode?.kind ?? ChatModeKind.Agent, result); + const mode = this.validateMode(attributes, report); + this.validateTools(attributes, mode?.kind ?? ChatModeKind.Agent, report); + this.validateModel(attributes, mode?.kind ?? ChatModeKind.Agent, report); break; } case PromptsType.instructions: - this.validateApplyTo(attributes, result); + this.validateApplyTo(attributes, report); break; case PromptsType.mode: - this.validateTools(attributes, ChatModeKind.Agent, result); - this.validateModel(attributes, ChatModeKind.Agent, result); + this.validateTools(attributes, ChatModeKind.Agent, report); + this.validateModel(attributes, ChatModeKind.Agent, report); break; } } - private validateDescription(attributes: IHeaderAttribute[], markers: IMarkerData[]): void { + private validateDescription(attributes: IHeaderAttribute[], report: (markers: IMarkerData) => void): void { const descriptionAttribute = attributes.find(attr => attr.key === 'description'); if (!descriptionAttribute) { return; } if (descriptionAttribute.value.type !== 'string') { - markers.push(toMarker(localize('promptValidator.descriptionMustBeString', "The 'description' attribute must be a string."), descriptionAttribute.range, MarkerSeverity.Error)); + report(toMarker(localize('promptValidator.descriptionMustBeString', "The 'description' attribute must be a string."), descriptionAttribute.range, MarkerSeverity.Error)); return; } if (descriptionAttribute.value.value.trim().length === 0) { - markers.push(toMarker(localize('promptValidator.descriptionShouldNotBeEmpty', "The 'description' attribute should not be empty."), descriptionAttribute.value.range, MarkerSeverity.Error)); + report(toMarker(localize('promptValidator.descriptionShouldNotBeEmpty', "The 'description' attribute should not be empty."), descriptionAttribute.value.range, MarkerSeverity.Error)); return; } } - private validateModel(attributes: IHeaderAttribute[], modeKind: ChatModeKind, markers: IMarkerData[]): void { + private validateModel(attributes: IHeaderAttribute[], modeKind: ChatModeKind, report: (markers: IMarkerData) => void): void { const attribute = attributes.find(attr => attr.key === 'model'); if (!attribute) { return; } if (attribute.value.type !== 'string') { - markers.push(toMarker(localize('promptValidator.modelMustBeString', "The 'model' attribute must be a string."), attribute.value.range, MarkerSeverity.Error)); + report(toMarker(localize('promptValidator.modelMustBeString', "The 'model' attribute must be a string."), attribute.value.range, MarkerSeverity.Error)); return; } const modelName = attribute.value.value.trim(); if (modelName.length === 0) { - markers.push(toMarker(localize('promptValidator.modelMustBeNonEmpty', "The 'model' attribute must be a non-empty string."), attribute.value.range, MarkerSeverity.Error)); + report(toMarker(localize('promptValidator.modelMustBeNonEmpty', "The 'model' attribute must be a non-empty string."), attribute.value.range, MarkerSeverity.Error)); return; } @@ -121,10 +159,10 @@ export class PromptValidator { } const modelMetadata = this.findModelByName(languageModes, modelName); if (!modelMetadata) { - markers.push(toMarker(localize('promptValidator.modelNotFound', "Unknown model '{0}'.", modelName), attribute.value.range, MarkerSeverity.Warning)); + report(toMarker(localize('promptValidator.modelNotFound', "Unknown model '{0}'.", modelName), attribute.value.range, MarkerSeverity.Warning)); } else if (modeKind === ChatModeKind.Agent && !ILanguageModelChatMetadata.suitableForAgentMode(modelMetadata)) { - markers.push(toMarker(localize('promptValidator.modelNotSuited', "Model '{0}' is not suited for agent mode.", modelName), attribute.value.range, MarkerSeverity.Warning)); + report(toMarker(localize('promptValidator.modelNotSuited', "Model '{0}' is not suited for agent mode.", modelName), attribute.value.range, MarkerSeverity.Warning)); } } @@ -138,18 +176,18 @@ export class PromptValidator { return undefined; } - private validateMode(attributes: IHeaderAttribute[], markers: IMarkerData[]): IChatMode | undefined { + private validateMode(attributes: IHeaderAttribute[], report: (markers: IMarkerData) => void): IChatMode | undefined { const attribute = attributes.find(attr => attr.key === 'mode'); if (!attribute) { return undefined; // default mode for prompts is Agent } if (attribute.value.type !== 'string') { - markers.push(toMarker(localize('promptValidator.modeMustBeString', "The 'mode' attribute must be a string."), attribute.value.range, MarkerSeverity.Error)); + report(toMarker(localize('promptValidator.modeMustBeString', "The 'mode' attribute must be a string."), attribute.value.range, MarkerSeverity.Error)); return undefined; } const modeValue = attribute.value.value; if (modeValue.trim().length === 0) { - markers.push(toMarker(localize('promptValidator.modeMustBeNonEmpty', "The 'mode' attribute must be a non-empty string."), attribute.value.range, MarkerSeverity.Error)); + report(toMarker(localize('promptValidator.modeMustBeNonEmpty', "The 'mode' attribute must be a non-empty string."), attribute.value.range, MarkerSeverity.Error)); return undefined; } @@ -165,72 +203,77 @@ export class PromptValidator { } const errorMessage = localize('promptValidator.modeNotFound', "Unknown mode '{0}'. Available modes: {1}.", modeValue, availableModes.join(', ')); - markers.push(toMarker(errorMessage, attribute.value.range, MarkerSeverity.Warning)); + report(toMarker(errorMessage, attribute.value.range, MarkerSeverity.Warning)); return undefined; } - private validateTools(attributes: IHeaderAttribute[], modeKind: ChatModeKind, markers: IMarkerData[]): undefined { + private validateTools(attributes: IHeaderAttribute[], modeKind: ChatModeKind, report: (markers: IMarkerData) => void): undefined { const attribute = attributes.find(attr => attr.key === 'tools'); if (!attribute) { return; } if (modeKind !== ChatModeKind.Agent) { - markers.push(toMarker(localize('promptValidator.toolsOnlyInAgent', "The 'tools' attribute is only supported in agent mode. Attribute will be ignored."), attribute.range, MarkerSeverity.Warning)); + report(toMarker(localize('promptValidator.toolsOnlyInAgent', "The 'tools' attribute is only supported in agent mode. Attribute will be ignored."), attribute.range, MarkerSeverity.Warning)); } if (attribute.value.type !== 'array') { - markers.push(toMarker(localize('promptValidator.toolsMustBeArray', "The 'tools' attribute must be an array."), attribute.value.range, MarkerSeverity.Error)); + report(toMarker(localize('promptValidator.toolsMustBeArray', "The 'tools' attribute must be an array."), attribute.value.range, MarkerSeverity.Error)); return; } - const toolNames = new Map(); - for (const item of attribute.value.items) { - if (item.type !== 'string') { - markers.push(toMarker(localize('promptValidator.eachToolMustBeString', "Each tool name in the 'tools' attribute must be a string."), item.range, MarkerSeverity.Error)); - } else { - toolNames.set(item.value, item.range); + if (attribute.value.items.length > 0) { + const available = this.getAvailableToolAndToolSetNames(); + for (const item of attribute.value.items) { + if (item.type !== 'string') { + report(toMarker(localize('promptValidator.eachToolMustBeString', "Each tool name in the 'tools' attribute must be a string."), item.range, MarkerSeverity.Error)); + } else if (!available.has(item.value)) { + report(toMarker(localize('promptValidator.toolNotFound', "Unknown tool '{0}'.", item.value), item.range, MarkerSeverity.Warning)); + } } } - if (toolNames.size === 0) { - return; - } - for (const tool of this.languageModelToolsService.getTools()) { - toolNames.delete(tool.toolReferenceName ?? tool.displayName); - } - for (const toolSet of this.languageModelToolsService.toolSets.get()) { - toolNames.delete(toolSet.referenceName); - } - - for (const [toolName, range] of toolNames) { - markers.push(toMarker(localize('promptValidator.toolNotFound', "Unknown tool '{0}'.", toolName), range, MarkerSeverity.Warning)); - } } - private validateApplyTo(attributes: IHeaderAttribute[], markers: IMarkerData[]): undefined { + private getAvailableToolAndToolSetNames(): Set { + const available = new Set(); + for (const tool of this.languageModelToolsService.getTools()) { + if (tool.canBeReferencedInPrompt) { + available.add(tool.toolReferenceName ?? tool.displayName); + } + } + for (const toolSet of this.languageModelToolsService.toolSets.get()) { + available.add(toolSet.referenceName); + for (const tool of toolSet.getTools()) { + available.add(tool.toolReferenceName ?? tool.displayName); + } + } + return available; + } + + private validateApplyTo(attributes: IHeaderAttribute[], report: (markers: IMarkerData) => void): undefined { const attribute = attributes.find(attr => attr.key === 'applyTo'); if (!attribute) { return; } if (attribute.value.type !== 'string') { - markers.push(toMarker(localize('promptValidator.applyToMustBeString', "The 'applyTo' attribute must be a string."), attribute.value.range, MarkerSeverity.Error)); + report(toMarker(localize('promptValidator.applyToMustBeString', "The 'applyTo' attribute must be a string."), attribute.value.range, MarkerSeverity.Error)); return; } const pattern = attribute.value.value; try { const patterns = splitGlobAware(pattern, ','); if (patterns.length === 0) { - markers.push(toMarker(localize('promptValidator.applyToMustBeValidGlob', "The 'applyTo' attribute must be a valid glob pattern."), attribute.value.range, MarkerSeverity.Error)); + report(toMarker(localize('promptValidator.applyToMustBeValidGlob', "The 'applyTo' attribute must be a valid glob pattern."), attribute.value.range, MarkerSeverity.Error)); return; } for (const pattern of patterns) { const globPattern = parse(pattern); if (isEmptyPattern(globPattern)) { - markers.push(toMarker(localize('promptValidator.applyToMustBeValidGlob', "The 'applyTo' attribute must be a valid glob pattern."), attribute.value.range, MarkerSeverity.Error)); + report(toMarker(localize('promptValidator.applyToMustBeValidGlob', "The 'applyTo' attribute must be a valid glob pattern."), attribute.value.range, MarkerSeverity.Error)); return; } } } catch (_error) { - markers.push(toMarker(localize('promptValidator.applyToMustBeValidGlob', "The 'applyTo' attribute must be a valid glob pattern."), attribute.value.range, MarkerSeverity.Error)); + report(toMarker(localize('promptValidator.applyToMustBeValidGlob', "The 'applyTo' attribute must be a valid glob pattern."), attribute.value.range, MarkerSeverity.Error)); } } } @@ -338,9 +381,10 @@ class ModelTracker extends Disposable { } private validate(): void { - this.delayer.trigger(() => { + this.delayer.trigger(async () => { + const markers: IMarkerData[] = []; const ast = this.promptParser.parse(this.textModel.uri, this.textModel.getValue()); - const markers = this.validator.validate(ast, this.promptType); + await this.validator.validate(ast, this.promptType, m => markers.push(m)); this.markerService.changeOne(MARKERS_OWNER_ID, this.textModel.uri, markers); }); } diff --git a/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/promptValidator.test.ts b/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/promptValidator.test.ts index 58231ccc889..397ec504af4 100644 --- a/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/promptValidator.test.ts +++ b/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/promptValidator.test.ts @@ -22,6 +22,8 @@ import { MockChatModeService } from '../../mockChatModeService.js'; import { PromptsType } from '../../../../common/promptSyntax/promptTypes.js'; import { IMarkerData, MarkerSeverity } from '../../../../../../../platform/markers/common/markers.js'; import { getPromptFileExtension } from '../../../../common/promptSyntax/config/promptFileLocations.js'; +import { IFileService } from '../../../../../../../platform/files/common/files.js'; +import { ResourceSet } from '../../../../../../../base/common/map.js'; suite('PromptValidator', () => { const disposables = ensureNoDisposablesAreLeakedInTestSuite(); @@ -57,15 +59,25 @@ suite('PromptValidator', () => { } }); - const customChatMode = new CustomChatMode({ uri: URI.file('/test/chatmode.md'), name: 'BeastMode', body: '', variableReferences: [] }); + const customChatMode = new CustomChatMode({ uri: URI.parse('myFs://test/test/chatmode.md'), name: 'BeastMode', body: '', variableReferences: [] }); instaService.stub(IChatModeService, new MockChatModeService({ builtin: [ChatMode.Agent, ChatMode.Ask, ChatMode.Edit], custom: [customChatMode] })); + + + const existingFiles = new ResourceSet([URI.parse('myFs://test/reference1.md'), URI.parse('myFs://test/reference2.md')]); + instaService.stub(IFileService, { + exists(uri: URI) { + return Promise.resolve(existingFiles.has(uri)); + } + }); }); - function validate(code: string, promptType: PromptsType): IMarkerData[] { - const uri = URI.parse('file:///test/chatmode' + getPromptFileExtension(promptType)); + async function validate(code: string, promptType: PromptsType): Promise { + const uri = URI.parse('myFs://test/testFile' + getPromptFileExtension(promptType)); const result = new NewPromptsParser().parse(uri, code); const validator = instaService.createInstance(PromptValidator); - return validator.validate(result, promptType); + const markers: IMarkerData[] = []; + await validator.validate(result, promptType, m => markers.push(m)); + return markers; } suite('modes', () => { @@ -79,7 +91,7 @@ suite('PromptValidator', () => { /* 06 */"This is a chat mode test.", /* 07 */"Here is a #tool1 variable and a #file:./reference1.md as well as a [reference](./reference2.md).", ].join('\n'); - const markers = validate(content, PromptsType.mode); + const markers = await validate(content, PromptsType.mode); assert.deepStrictEqual(markers, []); }); @@ -92,7 +104,7 @@ suite('PromptValidator', () => { /* 05 */"---", /* 06 */"Body", ].join('\n'); - const markers = validate(content, PromptsType.mode); + const markers = await validate(content, PromptsType.mode); assert.strictEqual(markers.length, 3, 'Expected 3 validation issues'); assert.deepStrictEqual( markers.map(m => ({ severity: m.severity, message: m.message })), @@ -111,7 +123,7 @@ suite('PromptValidator', () => { "tools: 'tool1'", "---", ].join('\n'); - const markers = validate(content, PromptsType.mode); + const markers = await validate(content, PromptsType.mode); assert.strictEqual(markers.length, 1); assert.deepStrictEqual(markers.map(m => m.message), ["The 'tools' attribute must be an array."]); }); @@ -123,7 +135,7 @@ suite('PromptValidator', () => { "tools: ['tool1', 2]", "---", ].join('\n'); - const markers = validate(content, PromptsType.mode); + const markers = await validate(content, PromptsType.mode); assert.strictEqual(markers.length, 1); assert.strictEqual(markers[0].message, "Each tool name in the 'tools' attribute must be a string."); }); @@ -135,7 +147,7 @@ suite('PromptValidator', () => { "applyTo: '*.ts'", // not allowed in mode file "---", ].join('\n'); - const markers = validate(content, PromptsType.mode); + const markers = await validate(content, PromptsType.mode); assert.strictEqual(markers.length, 1); assert.strictEqual(markers[0].severity, MarkerSeverity.Warning); assert.ok(markers[0].message.startsWith("Attribute 'applyTo' is not supported in mode files.")); @@ -151,7 +163,7 @@ suite('PromptValidator', () => { "applyTo: *.ts,*.js", "---", ].join('\n'); - const markers = validate(content, PromptsType.instructions); + const markers = await validate(content, PromptsType.instructions); assert.deepEqual(markers, []); }); @@ -162,7 +174,7 @@ suite('PromptValidator', () => { "applyTo: 5", "---", ].join('\n'); - const markers = validate(content, PromptsType.instructions); + const markers = await validate(content, PromptsType.instructions); assert.strictEqual(markers.length, 1); assert.strictEqual(markers[0].message, "The 'applyTo' attribute must be a string."); }); @@ -175,7 +187,7 @@ suite('PromptValidator', () => { "model: mae-4", // model not allowed in instructions "---", ].join('\n'); - const markers = validate(content, PromptsType.instructions); + const markers = await validate(content, PromptsType.instructions); assert.strictEqual(markers.length, 2); // Order: unknown attribute warnings first (attribute iteration) then applyTo validation assert.strictEqual(markers[0].severity, MarkerSeverity.Warning); @@ -190,7 +202,7 @@ suite('PromptValidator', () => { "---", "Body", ].join('\n'); - const markers = validate(content, PromptsType.instructions); + const markers = await validate(content, PromptsType.instructions); assert.strictEqual(markers.length, 1); assert.strictEqual(markers[0].message, 'Invalid header, expecting pairs'); }); @@ -203,12 +215,12 @@ suite('PromptValidator', () => { const content = [ '---', 'description: "Prompt with tools"', - "model: MAE 4 (olama)", + "model: MAE 4.1", "tools: ['tool1','tool2']", '---', 'Body' ].join('\n'); - const markers = validate(content, PromptsType.prompt); + const markers = await validate(content, PromptsType.prompt); assert.deepStrictEqual(markers, []); }); @@ -221,7 +233,7 @@ suite('PromptValidator', () => { '---', 'Body' ].join('\n'); - const markers = validate(content, PromptsType.prompt); + const markers = await validate(content, PromptsType.prompt); assert.strictEqual(markers.length, 1, 'Expected one warning about unsuitable model'); assert.strictEqual(markers[0].severity, MarkerSeverity.Warning); assert.strictEqual(markers[0].message, "Model 'MAE 3.5 Turbo' is not suited for agent mode."); @@ -237,21 +249,21 @@ suite('PromptValidator', () => { '---', 'Body' ].join('\n'); - const markers = validate(content, PromptsType.prompt); + const markers = await validate(content, PromptsType.prompt); assert.deepStrictEqual(markers, []); }); - test('prompt with mode Ask and tools warns', async () => { + test('prompt with unknown mode Ask', async () => { const content = [ '---', - 'description: "Prompt ask mode with tools"', + 'description: "Prompt unknown mode Ask"', 'mode: Ask', "tools: ['tool1','tool2']", '---', 'Body' ].join('\n'); - const markers = validate(content, PromptsType.prompt); - assert.strictEqual(markers.length, 1, 'Expected one warning about unknown mode'); + const markers = await validate(content, PromptsType.prompt); + assert.strictEqual(markers.length, 1, 'Expected one warning about tools in non-agent mode'); assert.strictEqual(markers[0].severity, MarkerSeverity.Warning); assert.strictEqual(markers[0].message, "Unknown mode 'Ask'. Available modes: agent, ask, edit, BeastMode."); }); @@ -265,11 +277,53 @@ suite('PromptValidator', () => { '---', 'Body' ].join('\n'); - const markers = validate(content, PromptsType.prompt); + const markers = await validate(content, PromptsType.prompt); assert.strictEqual(markers.length, 1); assert.strictEqual(markers[0].severity, MarkerSeverity.Warning); assert.strictEqual(markers[0].message, "The 'tools' attribute is only supported in agent mode. Attribute will be ignored."); }); }); + suite('body', () => { + test('body with existing file references and known tools has no markers', async () => { + const content = [ + '---', + 'description: "Refs"', + '---', + 'Here is a #file:./reference1.md and a markdown [reference](./reference2.md) plus variables #tool1 and #tool2' + ].join('\n'); + const markers = await validate(content, PromptsType.prompt); + assert.deepStrictEqual(markers, [], 'Expected no validation issues'); + }); + + test('body with missing file references reports warnings', async () => { + const content = [ + '---', + 'description: "Missing Refs"', + '---', + 'Here is a #file:./missing1.md and a markdown [missing link](./missing2.md).' + ].join('\n'); + const markers = await validate(content, PromptsType.prompt); + const messages = markers.map(m => m.message).sort(); + assert.deepStrictEqual(messages, [ + "File './missing1.md' not found.", + "File './missing2.md' not found." + ]); + }); + + test('body with unknown tool variable reference warns', async () => { + const content = [ + '---', + 'description: "Unknown tool var"', + '---', + 'This line references known #tool1 and unknown #toolX' + ].join('\n'); + const markers = await validate(content, PromptsType.prompt); + assert.strictEqual(markers.length, 1, 'Expected one warning for unknown tool variable'); + assert.strictEqual(markers[0].severity, MarkerSeverity.Warning); + assert.strictEqual(markers[0].message, "Unknown tool or toolset 'toolX'."); + }); + + }); + }); From 99646f23fe0051a1bc1f53f003166826e1dca6cb Mon Sep 17 00:00:00 2001 From: Martin Aeschlimann Date: Wed, 10 Sep 2025 22:21:59 +0200 Subject: [PATCH 012/299] IPromptsService.getParsedPromptFile --- .../promptSyntax/service/promptValidator.ts | 16 +++++----- .../promptSyntax/service/promptsService.ts | 7 +++++ .../service/promptsServiceImpl.ts | 29 +++++++++++++++---- .../chat/test/common/mockPromptsService.ts | 3 ++ 4 files changed, 41 insertions(+), 14 deletions(-) diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptValidator.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptValidator.ts index fe9cfa3f8c7..8f6604b33fd 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptValidator.ts +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptValidator.ts @@ -17,12 +17,13 @@ import { ChatModeKind } from '../../constants.js'; import { ILanguageModelChatMetadata, ILanguageModelsService } from '../../languageModels.js'; import { ILanguageModelToolsService } from '../../languageModelToolsService.js'; import { getPromptsTypeForLanguageId, PromptsType } from '../promptTypes.js'; -import { IHeaderAttribute, NewPromptsParser, ParsedPromptFile } from './newPromptsParser.js'; +import { IHeaderAttribute, ParsedPromptFile } from './newPromptsParser.js'; import { PromptsConfig } from '../config/config.js'; import { Disposable, DisposableStore, toDisposable } from '../../../../../../base/common/lifecycle.js'; import { Delayer } from '../../../../../../base/common/async.js'; import { ResourceMap } from '../../../../../../base/common/map.js'; import { IFileService } from '../../../../../../platform/files/common/files.js'; +import { IPromptsService } from './promptsService.js'; const MARKERS_OWNER_ID = 'prompts-diagnostics-provider'; @@ -296,7 +297,6 @@ function toMarker(message: string, range: Range, severity = MarkerSeverity.Error export class PromptValidatorContribution extends Disposable { private readonly validator: PromptValidator; - private readonly promptParser: NewPromptsParser; private readonly localDisposables = this._register(new DisposableStore()); constructor( @@ -304,10 +304,10 @@ export class PromptValidatorContribution extends Disposable { @IInstantiationService instantiationService: IInstantiationService, @IConfigurationService private configService: IConfigurationService, @IMarkerService private readonly markerService: IMarkerService, + @IPromptsService private readonly promptsService: IPromptsService, ) { super(); this.validator = instantiationService.createInstance(PromptValidator); - this.promptParser = instantiationService.createInstance(NewPromptsParser); this.updateRegistration(); this._register(this.configService.onDidChangeConfiguration(e => { @@ -329,13 +329,13 @@ export class PromptValidatorContribution extends Disposable { this.modelService.getModels().forEach(model => { const promptType = getPromptsTypeForLanguageId(model.getLanguageId()); if (promptType) { - trackers.set(model.uri, new ModelTracker(model, promptType, this.validator, this.promptParser, this.markerService)); + trackers.set(model.uri, new ModelTracker(model, promptType, this.validator, this.promptsService, this.markerService)); } }); this.localDisposables.add(this.modelService.onModelAdded((model) => { const promptType = getPromptsTypeForLanguageId(model.getLanguageId()); if (promptType) { - trackers.set(model.uri, new ModelTracker(model, promptType, this.validator, this.promptParser, this.markerService)); + trackers.set(model.uri, new ModelTracker(model, promptType, this.validator, this.promptsService, this.markerService)); } })); this.localDisposables.add(this.modelService.onModelRemoved((model) => { @@ -357,7 +357,7 @@ export class PromptValidatorContribution extends Disposable { } const promptType = getPromptsTypeForLanguageId(model.getLanguageId()); if (promptType) { - trackers.set(model.uri, new ModelTracker(model, promptType, this.validator, this.promptParser, this.markerService)); + trackers.set(model.uri, new ModelTracker(model, promptType, this.validator, this.promptsService, this.markerService)); } })); } @@ -371,7 +371,7 @@ class ModelTracker extends Disposable { private readonly textModel: ITextModel, private readonly promptType: PromptsType, private readonly validator: PromptValidator, - private readonly promptParser: NewPromptsParser, + @IPromptsService private readonly promptsService: IPromptsService, @IMarkerService private readonly markerService: IMarkerService, ) { super(); @@ -383,7 +383,7 @@ class ModelTracker extends Disposable { private validate(): void { this.delayer.trigger(async () => { const markers: IMarkerData[] = []; - const ast = this.promptParser.parse(this.textModel.uri, this.textModel.getValue()); + const ast = this.promptsService.getParsedPromptFile(this.textModel); await this.validator.validate(ast, this.promptType, m => markers.push(m)); this.markerService.changeOne(MARKERS_OWNER_ID, this.textModel.uri, markers); }); diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsService.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsService.ts index e287f55e851..ca85e45152f 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsService.ts +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsService.ts @@ -15,6 +15,7 @@ import { PromptsType } from '../promptTypes.js'; import { createDecorator } from '../../../../../../platform/instantiation/common/instantiation.js'; import { YamlNode, YamlParseError } from '../../../../../../base/common/yaml.js'; import { IVariableReference } from '../../chatModes.js'; +import { ParsedPromptFile } from './newPromptsParser.js'; /** * Provides prompt services. @@ -164,6 +165,12 @@ export interface IPromptsService extends IDisposable { */ getSyntaxParserFor(model: ITextModel): TSharedPrompt & { isDisposed: false }; + /** + * The parsed prompt file for the provided text model. + * @param textModel Returns the parsed prompt file. + */ + getParsedPromptFile(textModel: ITextModel): ParsedPromptFile; + /** * List all available prompt files. */ diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts index 45b976408d9..68217698a22 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts @@ -29,6 +29,7 @@ import { IConfigurationService } from '../../../../../../platform/configuration/ import { PositionOffsetTransformer } from '../../../../../../editor/common/core/text/positionToOffset.js'; import { NewPromptsParser, ParsedPromptFile } from './newPromptsParser.js'; import { IFileService } from '../../../../../../platform/files/common/files.js'; +import { ResourceMap } from '../../../../../../base/common/map.js'; /** * Provides prompt services. @@ -51,6 +52,9 @@ export class PromptsService extends Disposable implements IPromptsService { */ private cachedCustomChatModes: Promise | undefined; + + private parsedPromptFileCache = new ResourceMap<[number, ParsedPromptFile]>(); + /** * Lazily created event that is fired when the custom chat modes change. */ @@ -99,6 +103,10 @@ export class PromptsService extends Disposable implements IPromptsService { return parser; }) ); + + this._register(this.modelService.onModelRemoved((model) => { + this.parsedPromptFileCache.delete(model.uri); + })); } /** @@ -136,6 +144,18 @@ export class PromptsService extends Disposable implements IPromptsService { return this.cache.get(model); } + public getParsedPromptFile(textModel: ITextModel): ParsedPromptFile { + const cached = this.parsedPromptFileCache.get(textModel.uri); + if (cached && cached[0] === textModel.getVersionId()) { + return cached[1]; + } + const ast = new NewPromptsParser().parse(textModel.uri, textModel.getValue()); + if (!cached || cached[0] < textModel.getVersionId()) { + this.parsedPromptFileCache.set(textModel.uri, [textModel.getVersionId(), ast]); + } + return ast; + } + public async listPromptFiles(type: PromptsType, token: CancellationToken): Promise { if (!PromptsConfig.enabled(this.configurationService)) { return []; @@ -307,15 +327,12 @@ export class PromptsService extends Disposable implements IPromptsService { } public async parseNew(uri: URI): Promise { - let content: string | undefined; const model = this.modelService.getModel(uri); if (model) { - content = model.getValue(); - } else { - const fileContent = await this.fileService.readFile(uri); - content = fileContent.value.toString(); + return this.getParsedPromptFile(model); } - return new NewPromptsParser().parse(uri, content); + const fileContent = await this.fileService.readFile(uri); + return new NewPromptsParser().parse(uri, fileContent.value.toString()); } } diff --git a/src/vs/workbench/contrib/chat/test/common/mockPromptsService.ts b/src/vs/workbench/contrib/chat/test/common/mockPromptsService.ts index 97394b57d90..6728e53f6a6 100644 --- a/src/vs/workbench/contrib/chat/test/common/mockPromptsService.ts +++ b/src/vs/workbench/contrib/chat/test/common/mockPromptsService.ts @@ -6,6 +6,8 @@ import { CancellationToken } from '../../../../../base/common/cancellation.js'; import { Emitter } from '../../../../../base/common/event.js'; import { URI } from '../../../../../base/common/uri.js'; +import { ITextModel } from '../../../../../editor/common/model.js'; +import { ParsedPromptFile } from '../../common/promptSyntax/service/newPromptsParser.js'; import { ICustomChatMode, IPromptsService } from '../../common/promptSyntax/service/promptsService.js'; export class MockPromptsService implements IPromptsService { @@ -34,5 +36,6 @@ export class MockPromptsService implements IPromptsService { findPromptSlashCommands(): Promise { throw new Error('Not implemented'); } parse(_uri: URI, _type: any, _token: CancellationToken): Promise { throw new Error('Not implemented'); } getPromptFileType(_resource: URI): any { return undefined; } + getParsedPromptFile(textModel: ITextModel): ParsedPromptFile { throw new Error('Not implemented'); } dispose(): void { } } From c7e7a779e896b5e48861ce3d5f1120dada5259c3 Mon Sep 17 00:00:00 2001 From: Matt Bierner <12821956+mjbvz@users.noreply.github.com> Date: Wed, 10 Sep 2025 18:22:39 -0700 Subject: [PATCH 013/299] Add basic mermaid rendering support in core For #257761 Ports over extension sample + a few improvements to core --- .eslint-ignore | 1 + build/gulpfile.extensions.js | 1 + build/lib/extensions.ts | 3 +- build/npm/dirs.js | 1 + eslint.config.js | 5 + extensions/mermaid-chat-features/.gitignore | 1 + extensions/mermaid-chat-features/.npmrc | 2 + .../mermaid-chat-features/.vscodeignore | 8 + extensions/mermaid-chat-features/README.md | 5 + .../mermaid-chat-features/cgmanifest.json | 4 + .../chat-webview-src/index.ts | 73 + .../chat-webview-src/tsconfig.json | 12 + .../esbuild-chat-webview.mjs | 18 + .../extension-browser.webpack.config.js | 13 + .../extension.webpack.config.js | 16 + .../mermaid-chat-features/package-lock.json | 1857 +++++++++++++++++ extensions/mermaid-chat-features/package.json | 75 + .../mermaid-chat-features/package.nls.json | 6 + .../mermaid-chat-features/src/extension.ts | 244 +++ .../mermaid-chat-features/tsconfig.json | 15 + extensions/simple-browser/package.json | 6 +- 21 files changed, 2362 insertions(+), 4 deletions(-) create mode 100644 extensions/mermaid-chat-features/.gitignore create mode 100644 extensions/mermaid-chat-features/.npmrc create mode 100644 extensions/mermaid-chat-features/.vscodeignore create mode 100644 extensions/mermaid-chat-features/README.md create mode 100644 extensions/mermaid-chat-features/cgmanifest.json create mode 100644 extensions/mermaid-chat-features/chat-webview-src/index.ts create mode 100644 extensions/mermaid-chat-features/chat-webview-src/tsconfig.json create mode 100644 extensions/mermaid-chat-features/esbuild-chat-webview.mjs create mode 100644 extensions/mermaid-chat-features/extension-browser.webpack.config.js create mode 100644 extensions/mermaid-chat-features/extension.webpack.config.js create mode 100644 extensions/mermaid-chat-features/package-lock.json create mode 100644 extensions/mermaid-chat-features/package.json create mode 100644 extensions/mermaid-chat-features/package.nls.json create mode 100644 extensions/mermaid-chat-features/src/extension.ts create mode 100644 extensions/mermaid-chat-features/tsconfig.json diff --git a/.eslint-ignore b/.eslint-ignore index e493198185e..6f55ce40f69 100644 --- a/.eslint-ignore +++ b/.eslint-ignore @@ -10,6 +10,7 @@ **/extensions/markdown-language-features/media/** **/extensions/markdown-language-features/notebook-out/** **/extensions/markdown-math/notebook-out/** +**/extensions/mermaid-chat-features/chat-webview-out/** **/extensions/notebook-renderers/renderer-out/index.js **/extensions/simple-browser/media/index.js **/extensions/terminal-suggest/src/completions/upstream/** diff --git a/build/gulpfile.extensions.js b/build/gulpfile.extensions.js index 687bd16fd8c..7826f48490b 100644 --- a/build/gulpfile.extensions.js +++ b/build/gulpfile.extensions.js @@ -51,6 +51,7 @@ const compilations = [ 'extensions/markdown-math/tsconfig.json', 'extensions/media-preview/tsconfig.json', 'extensions/merge-conflict/tsconfig.json', + 'extensions/mermaid-chat-features/tsconfig.json', 'extensions/terminal-suggest/tsconfig.json', 'extensions/microsoft-authentication/tsconfig.json', 'extensions/notebook-renderers/tsconfig.json', diff --git a/build/lib/extensions.ts b/build/lib/extensions.ts index b997fe4046e..9e7cf9f954a 100644 --- a/build/lib/extensions.ts +++ b/build/lib/extensions.ts @@ -559,11 +559,12 @@ const extensionsPath = path.join(root, 'extensions'); // Additional projects to run esbuild on. These typically build code for webviews const esbuildMediaScripts = [ + 'ipynb/esbuild.mjs', 'markdown-language-features/esbuild-notebook.mjs', 'markdown-language-features/esbuild-preview.mjs', 'markdown-math/esbuild.mjs', + 'mermaid-chat-features/esbuild-chat-webview.mjs', 'notebook-renderers/esbuild.mjs', - 'ipynb/esbuild.mjs', 'simple-browser/esbuild-preview.mjs', ]; diff --git a/build/npm/dirs.js b/build/npm/dirs.js index a7ee7c3a6c7..33c18196dd4 100644 --- a/build/npm/dirs.js +++ b/build/npm/dirs.js @@ -33,6 +33,7 @@ const dirs = [ 'extensions/markdown-math', 'extensions/media-preview', 'extensions/merge-conflict', + 'extensions/mermaid-chat-features', 'extensions/microsoft-authentication', 'extensions/notebook-renderers', 'extensions/npm', diff --git a/eslint.config.js b/eslint.config.js index d7b29f29cc0..0435f80cce3 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -1410,6 +1410,7 @@ export default tseslint.config( { files: [ 'extensions/markdown-language-features/**/*.ts', + 'extensions/mermaid-chat-features/**/*.ts', 'extensions/media-preview/**/*.ts', 'extensions/simple-browser/**/*.ts', 'extensions/typescript-language-features/**/*.ts', @@ -1430,6 +1431,10 @@ export default tseslint.config( 'extensions/simple-browser/tsconfig.json', 'extensions/simple-browser/preview-src/tsconfig.json', + // Mermaid chat features + 'extensions/mermaid-chat-features/tsconfig.json', + 'extensions/mermaid-chat-features/chat-webview-src/tsconfig.json', + // TypeScript 'extensions/typescript-language-features/tsconfig.json', 'extensions/typescript-language-features/web/tsconfig.json', diff --git a/extensions/mermaid-chat-features/.gitignore b/extensions/mermaid-chat-features/.gitignore new file mode 100644 index 00000000000..2877bd189bb --- /dev/null +++ b/extensions/mermaid-chat-features/.gitignore @@ -0,0 +1 @@ +chat-webview-out diff --git a/extensions/mermaid-chat-features/.npmrc b/extensions/mermaid-chat-features/.npmrc new file mode 100644 index 00000000000..a9c57709666 --- /dev/null +++ b/extensions/mermaid-chat-features/.npmrc @@ -0,0 +1,2 @@ +legacy-peer-deps="true" +timeout=180000 diff --git a/extensions/mermaid-chat-features/.vscodeignore b/extensions/mermaid-chat-features/.vscodeignore new file mode 100644 index 00000000000..4722e586990 --- /dev/null +++ b/extensions/mermaid-chat-features/.vscodeignore @@ -0,0 +1,8 @@ +src/** +extension.webpack.config.js +esbuild.* +cgmanifest.json +package-lock.json +webpack.config.js +tsconfig*.json +.gitignore diff --git a/extensions/mermaid-chat-features/README.md b/extensions/mermaid-chat-features/README.md new file mode 100644 index 00000000000..09a73bc1b1a --- /dev/null +++ b/extensions/mermaid-chat-features/README.md @@ -0,0 +1,5 @@ +# Markdown Math + +**Notice:** This extension is bundled with Visual Studio Code. It can be disabled but not uninstalled. + +Adds math rendering using [KaTeX](https://katex.org) to VS Code's built-in markdown preview and markdown cells in notebooks. diff --git a/extensions/mermaid-chat-features/cgmanifest.json b/extensions/mermaid-chat-features/cgmanifest.json new file mode 100644 index 00000000000..0c39c97297b --- /dev/null +++ b/extensions/mermaid-chat-features/cgmanifest.json @@ -0,0 +1,4 @@ +{ + "registrations": [], + "version": 1 +} diff --git a/extensions/mermaid-chat-features/chat-webview-src/index.ts b/extensions/mermaid-chat-features/chat-webview-src/index.ts new file mode 100644 index 00000000000..9b3c9df71b6 --- /dev/null +++ b/extensions/mermaid-chat-features/chat-webview-src/index.ts @@ -0,0 +1,73 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import mermaid, { MermaidConfig } from 'mermaid'; + +function getMermaidTheme() { + return document.body.classList.contains('vscode-dark') || document.body.classList.contains('vscode-high-contrast') + ? 'dark' + : 'default'; +} + +type State = { + readonly diagramText: string; + readonly theme: 'dark' | 'default'; +}; + +let state: State | undefined = undefined; + +function init() { + const diagram = document.querySelector('.mermaid'); + if (!diagram) { + return; + } + + const theme = getMermaidTheme(); + state = { + diagramText: diagram.textContent ?? '', + theme + }; + + const config: MermaidConfig = { + startOnLoad: true, + theme, + }; + mermaid.initialize(config); +} + +function tryUpdate() { + const newTheme = getMermaidTheme(); + if (state?.theme === newTheme) { + return; + } + + const diagramNode = document.querySelector('.mermaid'); + if (!diagramNode || !(diagramNode instanceof HTMLElement)) { + return; + } + + state = { + diagramText: state?.diagramText ?? '', + theme: newTheme + }; + + // Re-render + diagramNode.textContent = state?.diagramText ?? ''; + delete diagramNode.dataset.processed; + + mermaid.initialize({ + theme: newTheme, + }); + mermaid.run({ + nodes: [diagramNode] + }); +} + +// Update when theme changes +new MutationObserver(() => { + tryUpdate(); +}).observe(document.body, { attributes: true, attributeFilter: ['class'] }); + +init(); + diff --git a/extensions/mermaid-chat-features/chat-webview-src/tsconfig.json b/extensions/mermaid-chat-features/chat-webview-src/tsconfig.json new file mode 100644 index 00000000000..72282fb0c7d --- /dev/null +++ b/extensions/mermaid-chat-features/chat-webview-src/tsconfig.json @@ -0,0 +1,12 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist/", + "jsx": "react", + "lib": [ + "ES2024", + "DOM", + "DOM.Iterable" + ] + } +} diff --git a/extensions/mermaid-chat-features/esbuild-chat-webview.mjs b/extensions/mermaid-chat-features/esbuild-chat-webview.mjs new file mode 100644 index 00000000000..b23de5746fa --- /dev/null +++ b/extensions/mermaid-chat-features/esbuild-chat-webview.mjs @@ -0,0 +1,18 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +// @ts-check +import path from 'path'; +import { run } from '../esbuild-webview-common.mjs'; + +const srcDir = path.join(import.meta.dirname, 'chat-webview-src'); +const outDir = path.join(import.meta.dirname, 'chat-webview-out'); + +run({ + entryPoints: [ + path.join(srcDir, 'index.ts'), + ], + srcDir, + outdir: outDir, +}, process.argv); diff --git a/extensions/mermaid-chat-features/extension-browser.webpack.config.js b/extensions/mermaid-chat-features/extension-browser.webpack.config.js new file mode 100644 index 00000000000..b758f2d8155 --- /dev/null +++ b/extensions/mermaid-chat-features/extension-browser.webpack.config.js @@ -0,0 +1,13 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +// @ts-check +import { browser as withBrowserDefaults } from '../shared.webpack.config.mjs'; + +export default withBrowserDefaults({ + context: import.meta.dirname, + entry: { + extension: './src/extension.ts' + } +}); diff --git a/extensions/mermaid-chat-features/extension.webpack.config.js b/extensions/mermaid-chat-features/extension.webpack.config.js new file mode 100644 index 00000000000..4928186ae55 --- /dev/null +++ b/extensions/mermaid-chat-features/extension.webpack.config.js @@ -0,0 +1,16 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +// @ts-check +import withDefaults from '../shared.webpack.config.mjs'; + +export default withDefaults({ + context: import.meta.dirname, + resolve: { + mainFields: ['module', 'main'] + }, + entry: { + extension: './src/extension.ts', + } +}); diff --git a/extensions/mermaid-chat-features/package-lock.json b/extensions/mermaid-chat-features/package-lock.json new file mode 100644 index 00000000000..a9de7747f05 --- /dev/null +++ b/extensions/mermaid-chat-features/package-lock.json @@ -0,0 +1,1857 @@ +{ + "name": "marmaid-chat-features", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "marmaid-chat-features", + "version": "1.0.0", + "license": "MIT", + "dependencies": { + "dompurify": "^3.2.6", + "jsdom": "^26.1.0", + "mermaid": "^11.11.0" + }, + "devDependencies": { + "@types/jsdom": "^21.1.7", + "@types/node": "^22" + }, + "engines": { + "vscode": "^1.104.0" + } + }, + "node_modules/@antfu/install-pkg": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@antfu/install-pkg/-/install-pkg-1.1.0.tgz", + "integrity": "sha512-MGQsmw10ZyI+EJo45CdSER4zEb+p31LpDAFp2Z3gkSd1yqVZGi0Ebx++YTEMonJy4oChEMLsxZ64j8FH6sSqtQ==", + "license": "MIT", + "dependencies": { + "package-manager-detector": "^1.3.0", + "tinyexec": "^1.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/@antfu/utils": { + "version": "9.2.0", + "resolved": "https://registry.npmjs.org/@antfu/utils/-/utils-9.2.0.tgz", + "integrity": "sha512-Oq1d9BGZakE/FyoEtcNeSwM7MpDO2vUBi11RWBZXf75zPsbUVWmUs03EqkRFrcgbXyKTas0BdZWC1wcuSoqSAw==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/@asamuzakjp/css-color": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@asamuzakjp/css-color/-/css-color-3.2.0.tgz", + "integrity": "sha512-K1A6z8tS3XsmCMM86xoWdn7Fkdn9m6RSVtocUrJYIwZnFVkng/PvkEoWtOWmP+Scc6saYWHWZYbndEEXxl24jw==", + "license": "MIT", + "dependencies": { + "@csstools/css-calc": "^2.1.3", + "@csstools/css-color-parser": "^3.0.9", + "@csstools/css-parser-algorithms": "^3.0.4", + "@csstools/css-tokenizer": "^3.0.3", + "lru-cache": "^10.4.3" + } + }, + "node_modules/@braintree/sanitize-url": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/@braintree/sanitize-url/-/sanitize-url-7.1.1.tgz", + "integrity": "sha512-i1L7noDNxtFyL5DmZafWy1wRVhGehQmzZaz1HiN5e7iylJMSZR7ekOV7NsIqa5qBldlLrsKv4HbgFUVlQrz8Mw==", + "license": "MIT" + }, + "node_modules/@chevrotain/cst-dts-gen": { + "version": "11.0.3", + "resolved": "https://registry.npmjs.org/@chevrotain/cst-dts-gen/-/cst-dts-gen-11.0.3.tgz", + "integrity": "sha512-BvIKpRLeS/8UbfxXxgC33xOumsacaeCKAjAeLyOn7Pcp95HiRbrpl14S+9vaZLolnbssPIUuiUd8IvgkRyt6NQ==", + "license": "Apache-2.0", + "dependencies": { + "@chevrotain/gast": "11.0.3", + "@chevrotain/types": "11.0.3", + "lodash-es": "4.17.21" + } + }, + "node_modules/@chevrotain/gast": { + "version": "11.0.3", + "resolved": "https://registry.npmjs.org/@chevrotain/gast/-/gast-11.0.3.tgz", + "integrity": "sha512-+qNfcoNk70PyS/uxmj3li5NiECO+2YKZZQMbmjTqRI3Qchu8Hig/Q9vgkHpI3alNjr7M+a2St5pw5w5F6NL5/Q==", + "license": "Apache-2.0", + "dependencies": { + "@chevrotain/types": "11.0.3", + "lodash-es": "4.17.21" + } + }, + "node_modules/@chevrotain/regexp-to-ast": { + "version": "11.0.3", + "resolved": "https://registry.npmjs.org/@chevrotain/regexp-to-ast/-/regexp-to-ast-11.0.3.tgz", + "integrity": "sha512-1fMHaBZxLFvWI067AVbGJav1eRY7N8DDvYCTwGBiE/ytKBgP8azTdgyrKyWZ9Mfh09eHWb5PgTSO8wi7U824RA==", + "license": "Apache-2.0" + }, + "node_modules/@chevrotain/types": { + "version": "11.0.3", + "resolved": "https://registry.npmjs.org/@chevrotain/types/-/types-11.0.3.tgz", + "integrity": "sha512-gsiM3G8b58kZC2HaWR50gu6Y1440cHiJ+i3JUvcp/35JchYejb2+5MVeJK0iKThYpAa/P2PYFV4hoi44HD+aHQ==", + "license": "Apache-2.0" + }, + "node_modules/@chevrotain/utils": { + "version": "11.0.3", + "resolved": "https://registry.npmjs.org/@chevrotain/utils/-/utils-11.0.3.tgz", + "integrity": "sha512-YslZMgtJUyuMbZ+aKvfF3x1f5liK4mWNxghFRv7jqRR9C3R3fAOGTTKvxXDa2Y1s9zSbcpuO0cAxDYsc9SrXoQ==", + "license": "Apache-2.0" + }, + "node_modules/@csstools/color-helpers": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/@csstools/color-helpers/-/color-helpers-5.1.0.tgz", + "integrity": "sha512-S11EXWJyy0Mz5SYvRmY8nJYTFFd1LCNV+7cXyAgQtOOuzb4EsgfqDufL+9esx72/eLhsRdGZwaldu/h+E4t4BA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT-0", + "engines": { + "node": ">=18" + } + }, + "node_modules/@csstools/css-calc": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/@csstools/css-calc/-/css-calc-2.1.4.tgz", + "integrity": "sha512-3N8oaj+0juUw/1H3YwmDDJXCgTB1gKU6Hc/bB502u9zR0q2vd786XJH9QfrKIEgFlZmhZiq6epXl4rHqhzsIgQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@csstools/css-parser-algorithms": "^3.0.5", + "@csstools/css-tokenizer": "^3.0.4" + } + }, + "node_modules/@csstools/css-color-parser": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@csstools/css-color-parser/-/css-color-parser-3.1.0.tgz", + "integrity": "sha512-nbtKwh3a6xNVIp/VRuXV64yTKnb1IjTAEEh3irzS+HkKjAOYLTGNb9pmVNntZ8iVBHcWDA2Dof0QtPgFI1BaTA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "dependencies": { + "@csstools/color-helpers": "^5.1.0", + "@csstools/css-calc": "^2.1.4" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@csstools/css-parser-algorithms": "^3.0.5", + "@csstools/css-tokenizer": "^3.0.4" + } + }, + "node_modules/@csstools/css-parser-algorithms": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@csstools/css-parser-algorithms/-/css-parser-algorithms-3.0.5.tgz", + "integrity": "sha512-DaDeUkXZKjdGhgYaHNJTV9pV7Y9B3b644jCLs9Upc3VeNGg6LWARAT6O+Q+/COo+2gg/bM5rhpMAtf70WqfBdQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@csstools/css-tokenizer": "^3.0.4" + } + }, + "node_modules/@csstools/css-tokenizer": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@csstools/css-tokenizer/-/css-tokenizer-3.0.4.tgz", + "integrity": "sha512-Vd/9EVDiu6PPJt9yAh6roZP6El1xHrdvIVGjyBsHR0RYwNHgL7FJPyIIW4fANJNG6FtyZfvlRPpFI4ZM/lubvw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/csstools" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/csstools" + } + ], + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@iconify/types": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@iconify/types/-/types-2.0.0.tgz", + "integrity": "sha512-+wluvCrRhXrhyOmRDJ3q8mux9JkKy5SJ/v8ol2tu4FVjyYvtEzkc/3pK15ET6RKg4b4w4BmTk1+gsCUhf21Ykg==", + "license": "MIT" + }, + "node_modules/@iconify/utils": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@iconify/utils/-/utils-3.0.1.tgz", + "integrity": "sha512-A78CUEnFGX8I/WlILxJCuIJXloL0j/OJ9PSchPAfCargEIKmUBWvvEMmKWB5oONwiUqlNt+5eRufdkLxeHIWYw==", + "license": "MIT", + "dependencies": { + "@antfu/install-pkg": "^1.1.0", + "@antfu/utils": "^9.2.0", + "@iconify/types": "^2.0.0", + "debug": "^4.4.1", + "globals": "^15.15.0", + "kolorist": "^1.8.0", + "local-pkg": "^1.1.1", + "mlly": "^1.7.4" + } + }, + "node_modules/@iconify/utils/node_modules/globals": { + "version": "15.15.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-15.15.0.tgz", + "integrity": "sha512-7ACyT3wmyp3I61S4fG682L0VA2RGD9otkqGJIwNUMF1SWUombIIk+af1unuDYgMm082aHYwD+mzJvv9Iu8dsgg==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@mermaid-js/parser": { + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/@mermaid-js/parser/-/parser-0.6.2.tgz", + "integrity": "sha512-+PO02uGF6L6Cs0Bw8RpGhikVvMWEysfAyl27qTlroUB8jSWr1lL0Sf6zi78ZxlSnmgSY2AMMKVgghnN9jTtwkQ==", + "license": "MIT", + "dependencies": { + "langium": "3.3.1" + } + }, + "node_modules/@types/d3": { + "version": "7.4.3", + "resolved": "https://registry.npmjs.org/@types/d3/-/d3-7.4.3.tgz", + "integrity": "sha512-lZXZ9ckh5R8uiFVt8ogUNf+pIrK4EsWrx2Np75WvF/eTpJ0FMHNhjXk8CKEx/+gpHbNQyJWehbFaTvqmHWB3ww==", + "license": "MIT", + "dependencies": { + "@types/d3-array": "*", + "@types/d3-axis": "*", + "@types/d3-brush": "*", + "@types/d3-chord": "*", + "@types/d3-color": "*", + "@types/d3-contour": "*", + "@types/d3-delaunay": "*", + "@types/d3-dispatch": "*", + "@types/d3-drag": "*", + "@types/d3-dsv": "*", + "@types/d3-ease": "*", + "@types/d3-fetch": "*", + "@types/d3-force": "*", + "@types/d3-format": "*", + "@types/d3-geo": "*", + "@types/d3-hierarchy": "*", + "@types/d3-interpolate": "*", + "@types/d3-path": "*", + "@types/d3-polygon": "*", + "@types/d3-quadtree": "*", + "@types/d3-random": "*", + "@types/d3-scale": "*", + "@types/d3-scale-chromatic": "*", + "@types/d3-selection": "*", + "@types/d3-shape": "*", + "@types/d3-time": "*", + "@types/d3-time-format": "*", + "@types/d3-timer": "*", + "@types/d3-transition": "*", + "@types/d3-zoom": "*" + } + }, + "node_modules/@types/d3-array": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/@types/d3-array/-/d3-array-3.2.1.tgz", + "integrity": "sha512-Y2Jn2idRrLzUfAKV2LyRImR+y4oa2AntrgID95SHJxuMUrkNXmanDSed71sRNZysveJVt1hLLemQZIady0FpEg==", + "license": "MIT" + }, + "node_modules/@types/d3-axis": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@types/d3-axis/-/d3-axis-3.0.6.tgz", + "integrity": "sha512-pYeijfZuBd87T0hGn0FO1vQ/cgLk6E1ALJjfkC0oJ8cbwkZl3TpgS8bVBLZN+2jjGgg38epgxb2zmoGtSfvgMw==", + "license": "MIT", + "dependencies": { + "@types/d3-selection": "*" + } + }, + "node_modules/@types/d3-brush": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@types/d3-brush/-/d3-brush-3.0.6.tgz", + "integrity": "sha512-nH60IZNNxEcrh6L1ZSMNA28rj27ut/2ZmI3r96Zd+1jrZD++zD3LsMIjWlvg4AYrHn/Pqz4CF3veCxGjtbqt7A==", + "license": "MIT", + "dependencies": { + "@types/d3-selection": "*" + } + }, + "node_modules/@types/d3-chord": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@types/d3-chord/-/d3-chord-3.0.6.tgz", + "integrity": "sha512-LFYWWd8nwfwEmTZG9PfQxd17HbNPksHBiJHaKuY1XeqscXacsS2tyoo6OdRsjf+NQYeB6XrNL3a25E3gH69lcg==", + "license": "MIT" + }, + "node_modules/@types/d3-color": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/@types/d3-color/-/d3-color-3.1.3.tgz", + "integrity": "sha512-iO90scth9WAbmgv7ogoq57O9YpKmFBbmoEoCHDB2xMBY0+/KVrqAaCDyCE16dUspeOvIxFFRI+0sEtqDqy2b4A==", + "license": "MIT" + }, + "node_modules/@types/d3-contour": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@types/d3-contour/-/d3-contour-3.0.6.tgz", + "integrity": "sha512-BjzLgXGnCWjUSYGfH1cpdo41/hgdWETu4YxpezoztawmqsvCeep+8QGfiY6YbDvfgHz/DkjeIkkZVJavB4a3rg==", + "license": "MIT", + "dependencies": { + "@types/d3-array": "*", + "@types/geojson": "*" + } + }, + "node_modules/@types/d3-delaunay": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/@types/d3-delaunay/-/d3-delaunay-6.0.4.tgz", + "integrity": "sha512-ZMaSKu4THYCU6sV64Lhg6qjf1orxBthaC161plr5KuPHo3CNm8DTHiLw/5Eq2b6TsNP0W0iJrUOFscY6Q450Hw==", + "license": "MIT" + }, + "node_modules/@types/d3-dispatch": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/@types/d3-dispatch/-/d3-dispatch-3.0.7.tgz", + "integrity": "sha512-5o9OIAdKkhN1QItV2oqaE5KMIiXAvDWBDPrD85e58Qlz1c1kI/J0NcqbEG88CoTwJrYe7ntUCVfeUl2UJKbWgA==", + "license": "MIT" + }, + "node_modules/@types/d3-drag": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/@types/d3-drag/-/d3-drag-3.0.7.tgz", + "integrity": "sha512-HE3jVKlzU9AaMazNufooRJ5ZpWmLIoc90A37WU2JMmeq28w1FQqCZswHZ3xR+SuxYftzHq6WU6KJHvqxKzTxxQ==", + "license": "MIT", + "dependencies": { + "@types/d3-selection": "*" + } + }, + "node_modules/@types/d3-dsv": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/@types/d3-dsv/-/d3-dsv-3.0.7.tgz", + "integrity": "sha512-n6QBF9/+XASqcKK6waudgL0pf/S5XHPPI8APyMLLUHd8NqouBGLsU8MgtO7NINGtPBtk9Kko/W4ea0oAspwh9g==", + "license": "MIT" + }, + "node_modules/@types/d3-ease": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/d3-ease/-/d3-ease-3.0.2.tgz", + "integrity": "sha512-NcV1JjO5oDzoK26oMzbILE6HW7uVXOHLQvHshBUW4UMdZGfiY6v5BeQwh9a9tCzv+CeefZQHJt5SRgK154RtiA==", + "license": "MIT" + }, + "node_modules/@types/d3-fetch": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/@types/d3-fetch/-/d3-fetch-3.0.7.tgz", + "integrity": "sha512-fTAfNmxSb9SOWNB9IoG5c8Hg6R+AzUHDRlsXsDZsNp6sxAEOP0tkP3gKkNSO/qmHPoBFTxNrjDprVHDQDvo5aA==", + "license": "MIT", + "dependencies": { + "@types/d3-dsv": "*" + } + }, + "node_modules/@types/d3-force": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@types/d3-force/-/d3-force-3.0.10.tgz", + "integrity": "sha512-ZYeSaCF3p73RdOKcjj+swRlZfnYpK1EbaDiYICEEp5Q6sUiqFaFQ9qgoshp5CzIyyb/yD09kD9o2zEltCexlgw==", + "license": "MIT" + }, + "node_modules/@types/d3-format": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/d3-format/-/d3-format-3.0.4.tgz", + "integrity": "sha512-fALi2aI6shfg7vM5KiR1wNJnZ7r6UuggVqtDA+xiEdPZQwy/trcQaHnwShLuLdta2rTymCNpxYTiMZX/e09F4g==", + "license": "MIT" + }, + "node_modules/@types/d3-geo": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@types/d3-geo/-/d3-geo-3.1.0.tgz", + "integrity": "sha512-856sckF0oP/diXtS4jNsiQw/UuK5fQG8l/a9VVLeSouf1/PPbBE1i1W852zVwKwYCBkFJJB7nCFTbk6UMEXBOQ==", + "license": "MIT", + "dependencies": { + "@types/geojson": "*" + } + }, + "node_modules/@types/d3-hierarchy": { + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/@types/d3-hierarchy/-/d3-hierarchy-3.1.7.tgz", + "integrity": "sha512-tJFtNoYBtRtkNysX1Xq4sxtjK8YgoWUNpIiUee0/jHGRwqvzYxkq0hGVbbOGSz+JgFxxRu4K8nb3YpG3CMARtg==", + "license": "MIT" + }, + "node_modules/@types/d3-interpolate": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/d3-interpolate/-/d3-interpolate-3.0.4.tgz", + "integrity": "sha512-mgLPETlrpVV1YRJIglr4Ez47g7Yxjl1lj7YKsiMCb27VJH9W8NVM6Bb9d8kkpG/uAQS5AmbA48q2IAolKKo1MA==", + "license": "MIT", + "dependencies": { + "@types/d3-color": "*" + } + }, + "node_modules/@types/d3-path": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/@types/d3-path/-/d3-path-3.1.1.tgz", + "integrity": "sha512-VMZBYyQvbGmWyWVea0EHs/BwLgxc+MKi1zLDCONksozI4YJMcTt8ZEuIR4Sb1MMTE8MMW49v0IwI5+b7RmfWlg==", + "license": "MIT" + }, + "node_modules/@types/d3-polygon": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/d3-polygon/-/d3-polygon-3.0.2.tgz", + "integrity": "sha512-ZuWOtMaHCkN9xoeEMr1ubW2nGWsp4nIql+OPQRstu4ypeZ+zk3YKqQT0CXVe/PYqrKpZAi+J9mTs05TKwjXSRA==", + "license": "MIT" + }, + "node_modules/@types/d3-quadtree": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@types/d3-quadtree/-/d3-quadtree-3.0.6.tgz", + "integrity": "sha512-oUzyO1/Zm6rsxKRHA1vH0NEDG58HrT5icx/azi9MF1TWdtttWl0UIUsjEQBBh+SIkrpd21ZjEv7ptxWys1ncsg==", + "license": "MIT" + }, + "node_modules/@types/d3-random": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/d3-random/-/d3-random-3.0.3.tgz", + "integrity": "sha512-Imagg1vJ3y76Y2ea0871wpabqp613+8/r0mCLEBfdtqC7xMSfj9idOnmBYyMoULfHePJyxMAw3nWhJxzc+LFwQ==", + "license": "MIT" + }, + "node_modules/@types/d3-scale": { + "version": "4.0.9", + "resolved": "https://registry.npmjs.org/@types/d3-scale/-/d3-scale-4.0.9.tgz", + "integrity": "sha512-dLmtwB8zkAeO/juAMfnV+sItKjlsw2lKdZVVy6LRr0cBmegxSABiLEpGVmSJJ8O08i4+sGR6qQtb6WtuwJdvVw==", + "license": "MIT", + "dependencies": { + "@types/d3-time": "*" + } + }, + "node_modules/@types/d3-scale-chromatic": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@types/d3-scale-chromatic/-/d3-scale-chromatic-3.1.0.tgz", + "integrity": "sha512-iWMJgwkK7yTRmWqRB5plb1kadXyQ5Sj8V/zYlFGMUBbIPKQScw+Dku9cAAMgJG+z5GYDoMjWGLVOvjghDEFnKQ==", + "license": "MIT" + }, + "node_modules/@types/d3-selection": { + "version": "3.0.11", + "resolved": "https://registry.npmjs.org/@types/d3-selection/-/d3-selection-3.0.11.tgz", + "integrity": "sha512-bhAXu23DJWsrI45xafYpkQ4NtcKMwWnAC/vKrd2l+nxMFuvOT3XMYTIj2opv8vq8AO5Yh7Qac/nSeP/3zjTK0w==", + "license": "MIT" + }, + "node_modules/@types/d3-shape": { + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/@types/d3-shape/-/d3-shape-3.1.7.tgz", + "integrity": "sha512-VLvUQ33C+3J+8p+Daf+nYSOsjB4GXp19/S/aGo60m9h1v6XaxjiT82lKVWJCfzhtuZ3yD7i/TPeC/fuKLLOSmg==", + "license": "MIT", + "dependencies": { + "@types/d3-path": "*" + } + }, + "node_modules/@types/d3-time": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/d3-time/-/d3-time-3.0.4.tgz", + "integrity": "sha512-yuzZug1nkAAaBlBBikKZTgzCeA+k1uy4ZFwWANOfKw5z5LRhV0gNA7gNkKm7HoK+HRN0wX3EkxGk0fpbWhmB7g==", + "license": "MIT" + }, + "node_modules/@types/d3-time-format": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/@types/d3-time-format/-/d3-time-format-4.0.3.tgz", + "integrity": "sha512-5xg9rC+wWL8kdDj153qZcsJ0FWiFt0J5RB6LYUNZjwSnesfblqrI/bJ1wBdJ8OQfncgbJG5+2F+qfqnqyzYxyg==", + "license": "MIT" + }, + "node_modules/@types/d3-timer": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/d3-timer/-/d3-timer-3.0.2.tgz", + "integrity": "sha512-Ps3T8E8dZDam6fUyNiMkekK3XUsaUEik+idO9/YjPtfj2qruF8tFBXS7XhtE4iIXBLxhmLjP3SXpLhVf21I9Lw==", + "license": "MIT" + }, + "node_modules/@types/d3-transition": { + "version": "3.0.9", + "resolved": "https://registry.npmjs.org/@types/d3-transition/-/d3-transition-3.0.9.tgz", + "integrity": "sha512-uZS5shfxzO3rGlu0cC3bjmMFKsXv+SmZZcgp0KD22ts4uGXp5EVYGzu/0YdwZeKmddhcAccYtREJKkPfXkZuCg==", + "license": "MIT", + "dependencies": { + "@types/d3-selection": "*" + } + }, + "node_modules/@types/d3-zoom": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/@types/d3-zoom/-/d3-zoom-3.0.8.tgz", + "integrity": "sha512-iqMC4/YlFCSlO8+2Ii1GGGliCAY4XdeG748w5vQUbevlbDu0zSjH/+jojorQVBK/se0j6DUFNPBGSqD3YWYnDw==", + "license": "MIT", + "dependencies": { + "@types/d3-interpolate": "*", + "@types/d3-selection": "*" + } + }, + "node_modules/@types/geojson": { + "version": "7946.0.16", + "resolved": "https://registry.npmjs.org/@types/geojson/-/geojson-7946.0.16.tgz", + "integrity": "sha512-6C8nqWur3j98U6+lXDfTUWIfgvZU+EumvpHKcYjujKH7woYyLj2sUmff0tRhrqM7BohUw7Pz3ZB1jj2gW9Fvmg==", + "license": "MIT" + }, + "node_modules/@types/jsdom": { + "version": "21.1.7", + "resolved": "https://registry.npmjs.org/@types/jsdom/-/jsdom-21.1.7.tgz", + "integrity": "sha512-yOriVnggzrnQ3a9OKOCxaVuSug3w3/SbOj5i7VwXWZEyUNl3bLF9V3MfxGbZKuwqJOQyRfqXyROBB1CoZLFWzA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "@types/tough-cookie": "*", + "parse5": "^7.0.0" + } + }, + "node_modules/@types/node": { + "version": "22.18.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.18.1.tgz", + "integrity": "sha512-rzSDyhn4cYznVG+PCzGe1lwuMYJrcBS1fc3JqSa2PvtABwWo+dZ1ij5OVok3tqfpEBCBoaR4d7upFJk73HRJDw==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@types/tough-cookie": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz", + "integrity": "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/trusted-types": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz", + "integrity": "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==", + "license": "MIT", + "optional": true + }, + "node_modules/acorn": { + "version": "8.15.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", + "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/chevrotain": { + "version": "11.0.3", + "resolved": "https://registry.npmjs.org/chevrotain/-/chevrotain-11.0.3.tgz", + "integrity": "sha512-ci2iJH6LeIkvP9eJW6gpueU8cnZhv85ELY8w8WiFtNjMHA5ad6pQLaJo9mEly/9qUyCpvqX8/POVUTf18/HFdw==", + "license": "Apache-2.0", + "dependencies": { + "@chevrotain/cst-dts-gen": "11.0.3", + "@chevrotain/gast": "11.0.3", + "@chevrotain/regexp-to-ast": "11.0.3", + "@chevrotain/types": "11.0.3", + "@chevrotain/utils": "11.0.3", + "lodash-es": "4.17.21" + } + }, + "node_modules/chevrotain-allstar": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/chevrotain-allstar/-/chevrotain-allstar-0.3.1.tgz", + "integrity": "sha512-b7g+y9A0v4mxCW1qUhf3BSVPg+/NvGErk/dOkrDaHA0nQIQGAtrOjlX//9OQtRlSCy+x9rfB5N8yC71lH1nvMw==", + "license": "MIT", + "dependencies": { + "lodash-es": "^4.17.21" + }, + "peerDependencies": { + "chevrotain": "^11.0.0" + } + }, + "node_modules/commander": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-8.3.0.tgz", + "integrity": "sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/confbox": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/confbox/-/confbox-0.2.2.tgz", + "integrity": "sha512-1NB+BKqhtNipMsov4xI/NnhCKp9XG9NamYp5PVm9klAT0fsrNPjaFICsCFhNhwZJKNh7zB/3q8qXz0E9oaMNtQ==", + "license": "MIT" + }, + "node_modules/cose-base": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/cose-base/-/cose-base-1.0.3.tgz", + "integrity": "sha512-s9whTXInMSgAp/NVXVNuVxVKzGH2qck3aQlVHxDCdAEPgtMKwc4Wq6/QKhgdEdgbLSi9rBTAcPoRa6JpiG4ksg==", + "license": "MIT", + "dependencies": { + "layout-base": "^1.0.0" + } + }, + "node_modules/cssstyle": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-4.6.0.tgz", + "integrity": "sha512-2z+rWdzbbSZv6/rhtvzvqeZQHrBaqgogqt85sqFNbabZOuFbCVFb8kPeEtZjiKkbrm395irpNKiYeFeLiQnFPg==", + "license": "MIT", + "dependencies": { + "@asamuzakjp/css-color": "^3.2.0", + "rrweb-cssom": "^0.8.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/cytoscape": { + "version": "3.33.1", + "resolved": "https://registry.npmjs.org/cytoscape/-/cytoscape-3.33.1.tgz", + "integrity": "sha512-iJc4TwyANnOGR1OmWhsS9ayRS3s+XQ185FmuHObThD+5AeJCakAAbWv8KimMTt08xCCLNgneQwFp+JRJOr9qGQ==", + "license": "MIT", + "engines": { + "node": ">=0.10" + } + }, + "node_modules/cytoscape-cose-bilkent": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/cytoscape-cose-bilkent/-/cytoscape-cose-bilkent-4.1.0.tgz", + "integrity": "sha512-wgQlVIUJF13Quxiv5e1gstZ08rnZj2XaLHGoFMYXz7SkNfCDOOteKBE6SYRfA9WxxI/iBc3ajfDoc6hb/MRAHQ==", + "license": "MIT", + "dependencies": { + "cose-base": "^1.0.0" + }, + "peerDependencies": { + "cytoscape": "^3.2.0" + } + }, + "node_modules/cytoscape-fcose": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/cytoscape-fcose/-/cytoscape-fcose-2.2.0.tgz", + "integrity": "sha512-ki1/VuRIHFCzxWNrsshHYPs6L7TvLu3DL+TyIGEsRcvVERmxokbf5Gdk7mFxZnTdiGtnA4cfSmjZJMviqSuZrQ==", + "license": "MIT", + "dependencies": { + "cose-base": "^2.2.0" + }, + "peerDependencies": { + "cytoscape": "^3.2.0" + } + }, + "node_modules/cytoscape-fcose/node_modules/cose-base": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/cose-base/-/cose-base-2.2.0.tgz", + "integrity": "sha512-AzlgcsCbUMymkADOJtQm3wO9S3ltPfYOFD5033keQn9NJzIbtnZj+UdBJe7DYml/8TdbtHJW3j58SOnKhWY/5g==", + "license": "MIT", + "dependencies": { + "layout-base": "^2.0.0" + } + }, + "node_modules/cytoscape-fcose/node_modules/layout-base": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/layout-base/-/layout-base-2.0.1.tgz", + "integrity": "sha512-dp3s92+uNI1hWIpPGH3jK2kxE2lMjdXdr+DH8ynZHpd6PUlH6x6cbuXnoMmiNumznqaNO31xu9e79F0uuZ0JFg==", + "license": "MIT" + }, + "node_modules/d3": { + "version": "7.9.0", + "resolved": "https://registry.npmjs.org/d3/-/d3-7.9.0.tgz", + "integrity": "sha512-e1U46jVP+w7Iut8Jt8ri1YsPOvFpg46k+K8TpCb0P+zjCkjkPnV7WzfDJzMHy1LnA+wj5pLT1wjO901gLXeEhA==", + "license": "ISC", + "dependencies": { + "d3-array": "3", + "d3-axis": "3", + "d3-brush": "3", + "d3-chord": "3", + "d3-color": "3", + "d3-contour": "4", + "d3-delaunay": "6", + "d3-dispatch": "3", + "d3-drag": "3", + "d3-dsv": "3", + "d3-ease": "3", + "d3-fetch": "3", + "d3-force": "3", + "d3-format": "3", + "d3-geo": "3", + "d3-hierarchy": "3", + "d3-interpolate": "3", + "d3-path": "3", + "d3-polygon": "3", + "d3-quadtree": "3", + "d3-random": "3", + "d3-scale": "4", + "d3-scale-chromatic": "3", + "d3-selection": "3", + "d3-shape": "3", + "d3-time": "3", + "d3-time-format": "4", + "d3-timer": "3", + "d3-transition": "3", + "d3-zoom": "3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-array": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-3.2.4.tgz", + "integrity": "sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg==", + "license": "ISC", + "dependencies": { + "internmap": "1 - 2" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-axis": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-axis/-/d3-axis-3.0.0.tgz", + "integrity": "sha512-IH5tgjV4jE/GhHkRV0HiVYPDtvfjHQlQfJHs0usq7M30XcSBvOotpmH1IgkcXsO/5gEQZD43B//fc7SRT5S+xw==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-brush": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-brush/-/d3-brush-3.0.0.tgz", + "integrity": "sha512-ALnjWlVYkXsVIGlOsuWH1+3udkYFI48Ljihfnh8FZPF2QS9o+PzGLBslO0PjzVoHLZ2KCVgAM8NVkXPJB2aNnQ==", + "license": "ISC", + "dependencies": { + "d3-dispatch": "1 - 3", + "d3-drag": "2 - 3", + "d3-interpolate": "1 - 3", + "d3-selection": "3", + "d3-transition": "3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-chord": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-chord/-/d3-chord-3.0.1.tgz", + "integrity": "sha512-VE5S6TNa+j8msksl7HwjxMHDM2yNK3XCkusIlpX5kwauBfXuyLAtNg9jCp/iHH61tgI4sb6R/EIMWCqEIdjT/g==", + "license": "ISC", + "dependencies": { + "d3-path": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-color": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-color/-/d3-color-3.1.0.tgz", + "integrity": "sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-contour": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/d3-contour/-/d3-contour-4.0.2.tgz", + "integrity": "sha512-4EzFTRIikzs47RGmdxbeUvLWtGedDUNkTcmzoeyg4sP/dvCexO47AaQL7VKy/gul85TOxw+IBgA8US2xwbToNA==", + "license": "ISC", + "dependencies": { + "d3-array": "^3.2.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-delaunay": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/d3-delaunay/-/d3-delaunay-6.0.4.tgz", + "integrity": "sha512-mdjtIZ1XLAM8bm/hx3WwjfHt6Sggek7qH043O8KEjDXN40xi3vx/6pYSVTwLjEgiXQTbvaouWKynLBiUZ6SK6A==", + "license": "ISC", + "dependencies": { + "delaunator": "5" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-dispatch": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-dispatch/-/d3-dispatch-3.0.1.tgz", + "integrity": "sha512-rzUyPU/S7rwUflMyLc1ETDeBj0NRuHKKAcvukozwhshr6g6c5d8zh4c2gQjY2bZ0dXeGLWc1PF174P2tVvKhfg==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-drag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-drag/-/d3-drag-3.0.0.tgz", + "integrity": "sha512-pWbUJLdETVA8lQNJecMxoXfH6x+mO2UQo8rSmZ+QqxcbyA3hfeprFgIT//HW2nlHChWeIIMwS2Fq+gEARkhTkg==", + "license": "ISC", + "dependencies": { + "d3-dispatch": "1 - 3", + "d3-selection": "3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-dsv": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-dsv/-/d3-dsv-3.0.1.tgz", + "integrity": "sha512-UG6OvdI5afDIFP9w4G0mNq50dSOsXHJaRE8arAS5o9ApWnIElp8GZw1Dun8vP8OyHOZ/QJUKUJwxiiCCnUwm+Q==", + "license": "ISC", + "dependencies": { + "commander": "7", + "iconv-lite": "0.6", + "rw": "1" + }, + "bin": { + "csv2json": "bin/dsv2json.js", + "csv2tsv": "bin/dsv2dsv.js", + "dsv2dsv": "bin/dsv2dsv.js", + "dsv2json": "bin/dsv2json.js", + "json2csv": "bin/json2dsv.js", + "json2dsv": "bin/json2dsv.js", + "json2tsv": "bin/json2dsv.js", + "tsv2csv": "bin/dsv2dsv.js", + "tsv2json": "bin/dsv2json.js" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-dsv/node_modules/commander": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-7.2.0.tgz", + "integrity": "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==", + "license": "MIT", + "engines": { + "node": ">= 10" + } + }, + "node_modules/d3-ease": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-ease/-/d3-ease-3.0.1.tgz", + "integrity": "sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-fetch": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-fetch/-/d3-fetch-3.0.1.tgz", + "integrity": "sha512-kpkQIM20n3oLVBKGg6oHrUchHM3xODkTzjMoj7aWQFq5QEM+R6E4WkzT5+tojDY7yjez8KgCBRoj4aEr99Fdqw==", + "license": "ISC", + "dependencies": { + "d3-dsv": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-force": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-force/-/d3-force-3.0.0.tgz", + "integrity": "sha512-zxV/SsA+U4yte8051P4ECydjD/S+qeYtnaIyAs9tgHCqfguma/aAQDjo85A9Z6EKhBirHRJHXIgJUlffT4wdLg==", + "license": "ISC", + "dependencies": { + "d3-dispatch": "1 - 3", + "d3-quadtree": "1 - 3", + "d3-timer": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-format": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-format/-/d3-format-3.1.0.tgz", + "integrity": "sha512-YyUI6AEuY/Wpt8KWLgZHsIU86atmikuoOmCfommt0LYHiQSPjvX2AcFc38PX0CBpr2RCyZhjex+NS/LPOv6YqA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-geo": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/d3-geo/-/d3-geo-3.1.1.tgz", + "integrity": "sha512-637ln3gXKXOwhalDzinUgY83KzNWZRKbYubaG+fGVuc/dxO64RRljtCTnf5ecMyE1RIdtqpkVcq0IbtU2S8j2Q==", + "license": "ISC", + "dependencies": { + "d3-array": "2.5.0 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-hierarchy": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/d3-hierarchy/-/d3-hierarchy-3.1.2.tgz", + "integrity": "sha512-FX/9frcub54beBdugHjDCdikxThEqjnR93Qt7PvQTOHxyiNCAlvMrHhclk3cD5VeAaq9fxmfRp+CnWw9rEMBuA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-interpolate": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-interpolate/-/d3-interpolate-3.0.1.tgz", + "integrity": "sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==", + "license": "ISC", + "dependencies": { + "d3-color": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-path": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-path/-/d3-path-3.1.0.tgz", + "integrity": "sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-polygon": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-polygon/-/d3-polygon-3.0.1.tgz", + "integrity": "sha512-3vbA7vXYwfe1SYhED++fPUQlWSYTTGmFmQiany/gdbiWgU/iEyQzyymwL9SkJjFFuCS4902BSzewVGsHHmHtXg==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-quadtree": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-quadtree/-/d3-quadtree-3.0.1.tgz", + "integrity": "sha512-04xDrxQTDTCFwP5H6hRhsRcb9xxv2RzkcsygFzmkSIOJy3PeRJP7sNk3VRIbKXcog561P9oU0/rVH6vDROAgUw==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-random": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-random/-/d3-random-3.0.1.tgz", + "integrity": "sha512-FXMe9GfxTxqd5D6jFsQ+DJ8BJS4E/fT5mqqdjovykEB2oFbTMDVdg1MGFxfQW+FBOGoB++k8swBrgwSHT1cUXQ==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-sankey": { + "version": "0.12.3", + "resolved": "https://registry.npmjs.org/d3-sankey/-/d3-sankey-0.12.3.tgz", + "integrity": "sha512-nQhsBRmM19Ax5xEIPLMY9ZmJ/cDvd1BG3UVvt5h3WRxKg5zGRbvnteTyWAbzeSvlh3tW7ZEmq4VwR5mB3tutmQ==", + "license": "BSD-3-Clause", + "dependencies": { + "d3-array": "1 - 2", + "d3-shape": "^1.2.0" + } + }, + "node_modules/d3-sankey/node_modules/d3-array": { + "version": "2.12.1", + "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-2.12.1.tgz", + "integrity": "sha512-B0ErZK/66mHtEsR1TkPEEkwdy+WDesimkM5gpZr5Dsg54BiTA5RXtYW5qTLIAcekaS9xfZrzBLF/OAkB3Qn1YQ==", + "license": "BSD-3-Clause", + "dependencies": { + "internmap": "^1.0.0" + } + }, + "node_modules/d3-sankey/node_modules/d3-path": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/d3-path/-/d3-path-1.0.9.tgz", + "integrity": "sha512-VLaYcn81dtHVTjEHd8B+pbe9yHWpXKZUC87PzoFmsFrJqgFwDe/qxfp5MlfsfM1V5E/iVt0MmEbWQ7FVIXh/bg==", + "license": "BSD-3-Clause" + }, + "node_modules/d3-sankey/node_modules/d3-shape": { + "version": "1.3.7", + "resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-1.3.7.tgz", + "integrity": "sha512-EUkvKjqPFUAZyOlhY5gzCxCeI0Aep04LwIRpsZ/mLFelJiUfnK56jo5JMDSE7yyP2kLSb6LtF+S5chMk7uqPqw==", + "license": "BSD-3-Clause", + "dependencies": { + "d3-path": "1" + } + }, + "node_modules/d3-sankey/node_modules/internmap": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/internmap/-/internmap-1.0.1.tgz", + "integrity": "sha512-lDB5YccMydFBtasVtxnZ3MRBHuaoE8GKsppq+EchKL2U4nK/DmEpPHNH8MZe5HkMtpSiTSOZwfN0tzYjO/lJEw==", + "license": "ISC" + }, + "node_modules/d3-scale": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/d3-scale/-/d3-scale-4.0.2.tgz", + "integrity": "sha512-GZW464g1SH7ag3Y7hXjf8RoUuAFIqklOAq3MRl4OaWabTFJY9PN/E1YklhXLh+OQ3fM9yS2nOkCoS+WLZ6kvxQ==", + "license": "ISC", + "dependencies": { + "d3-array": "2.10.0 - 3", + "d3-format": "1 - 3", + "d3-interpolate": "1.2.0 - 3", + "d3-time": "2.1.1 - 3", + "d3-time-format": "2 - 4" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-scale-chromatic": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-scale-chromatic/-/d3-scale-chromatic-3.1.0.tgz", + "integrity": "sha512-A3s5PWiZ9YCXFye1o246KoscMWqf8BsD9eRiJ3He7C9OBaxKhAd5TFCdEx/7VbKtxxTsu//1mMJFrEt572cEyQ==", + "license": "ISC", + "dependencies": { + "d3-color": "1 - 3", + "d3-interpolate": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-selection": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz", + "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-shape": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-3.2.0.tgz", + "integrity": "sha512-SaLBuwGm3MOViRq2ABk3eLoxwZELpH6zhl3FbAoJ7Vm1gofKx6El1Ib5z23NUEhF9AsGl7y+dzLe5Cw2AArGTA==", + "license": "ISC", + "dependencies": { + "d3-path": "^3.1.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-time": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-time/-/d3-time-3.1.0.tgz", + "integrity": "sha512-VqKjzBLejbSMT4IgbmVgDjpkYrNWUYJnbCGo874u7MMKIWsILRX+OpX/gTk8MqjpT1A/c6HY2dCA77ZN0lkQ2Q==", + "license": "ISC", + "dependencies": { + "d3-array": "2 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-time-format": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/d3-time-format/-/d3-time-format-4.1.0.tgz", + "integrity": "sha512-dJxPBlzC7NugB2PDLwo9Q8JiTR3M3e4/XANkreKSUxF8vvXKqm1Yfq4Q5dl8budlunRVlUUaDUgFt7eA8D6NLg==", + "license": "ISC", + "dependencies": { + "d3-time": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-timer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-timer/-/d3-timer-3.0.1.tgz", + "integrity": "sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-transition": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-transition/-/d3-transition-3.0.1.tgz", + "integrity": "sha512-ApKvfjsSR6tg06xrL434C0WydLr7JewBB3V+/39RMHsaXTOG0zmt/OAXeng5M5LBm0ojmxJrpomQVZ1aPvBL4w==", + "license": "ISC", + "dependencies": { + "d3-color": "1 - 3", + "d3-dispatch": "1 - 3", + "d3-ease": "1 - 3", + "d3-interpolate": "1 - 3", + "d3-timer": "1 - 3" + }, + "engines": { + "node": ">=12" + }, + "peerDependencies": { + "d3-selection": "2 - 3" + } + }, + "node_modules/d3-zoom": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-zoom/-/d3-zoom-3.0.0.tgz", + "integrity": "sha512-b8AmV3kfQaqWAuacbPuNbL6vahnOJflOhexLzMMNLga62+/nh0JzvJ0aO/5a5MVgUFGS7Hu1P9P03o3fJkDCyw==", + "license": "ISC", + "dependencies": { + "d3-dispatch": "1 - 3", + "d3-drag": "2 - 3", + "d3-interpolate": "1 - 3", + "d3-selection": "2 - 3", + "d3-transition": "2 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/dagre-d3-es": { + "version": "7.0.11", + "resolved": "https://registry.npmjs.org/dagre-d3-es/-/dagre-d3-es-7.0.11.tgz", + "integrity": "sha512-tvlJLyQf834SylNKax8Wkzco/1ias1OPw8DcUMDE7oUIoSEW25riQVuiu/0OWEFqT0cxHT3Pa9/D82Jr47IONw==", + "license": "MIT", + "dependencies": { + "d3": "^7.9.0", + "lodash-es": "^4.17.21" + } + }, + "node_modules/data-urls": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-5.0.0.tgz", + "integrity": "sha512-ZYP5VBHshaDAiVZxjbRVcFJpc+4xGgT0bK3vzy1HLN8jTO975HEbuYzZJcHoQEY5K1a0z8YayJkyVETa08eNTg==", + "license": "MIT", + "dependencies": { + "whatwg-mimetype": "^4.0.0", + "whatwg-url": "^14.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/dayjs": { + "version": "1.11.18", + "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.18.tgz", + "integrity": "sha512-zFBQ7WFRvVRhKcWoUh+ZA1g2HVgUbsZm9sbddh8EC5iv93sui8DVVz1Npvz+r6meo9VKfa8NyLWBsQK1VvIKPA==", + "license": "MIT" + }, + "node_modules/debug": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz", + "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/decimal.js": { + "version": "10.6.0", + "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.6.0.tgz", + "integrity": "sha512-YpgQiITW3JXGntzdUmyUR1V812Hn8T1YVXhCu+wO3OpS4eU9l4YdD3qjyiKdV6mvV29zapkMeD390UVEf2lkUg==", + "license": "MIT" + }, + "node_modules/delaunator": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/delaunator/-/delaunator-5.0.1.tgz", + "integrity": "sha512-8nvh+XBe96aCESrGOqMp/84b13H9cdKbG5P2ejQCh4d4sK9RL4371qou9drQjMhvnPmhWl5hnmqbEE0fXr9Xnw==", + "license": "ISC", + "dependencies": { + "robust-predicates": "^3.0.2" + } + }, + "node_modules/dompurify": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.2.6.tgz", + "integrity": "sha512-/2GogDQlohXPZe6D6NOgQvXLPSYBqIWMnZ8zzOhn09REE4eyAzb+Hed3jhoM9OkuaJ8P6ZGTTVWQKAi8ieIzfQ==", + "license": "(MPL-2.0 OR Apache-2.0)", + "optionalDependencies": { + "@types/trusted-types": "^2.0.7" + } + }, + "node_modules/entities": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz", + "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/exsolve": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/exsolve/-/exsolve-1.0.7.tgz", + "integrity": "sha512-VO5fQUzZtI6C+vx4w/4BWJpg3s/5l+6pRQEHzFRM8WFi4XffSP1Z+4qi7GbjWbvRQEbdIco5mIMq+zX4rPuLrw==", + "license": "MIT" + }, + "node_modules/hachure-fill": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/hachure-fill/-/hachure-fill-0.5.2.tgz", + "integrity": "sha512-3GKBOn+m2LX9iq+JC1064cSFprJY4jL1jCXTcpnfER5HYE2l/4EfWSGzkPa/ZDBmYI0ZOEj5VHV/eKnPGkHuOg==", + "license": "MIT" + }, + "node_modules/html-encoding-sniffer": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-4.0.0.tgz", + "integrity": "sha512-Y22oTqIU4uuPgEemfz7NDJz6OeKf12Lsu+QC+s3BVpda64lTiMYCyGwg5ki4vFxkMwQdeZDl2adZoqUgdFuTgQ==", + "license": "MIT", + "dependencies": { + "whatwg-encoding": "^3.1.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/internmap": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/internmap/-/internmap-2.0.3.tgz", + "integrity": "sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/is-potential-custom-element-name": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz", + "integrity": "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==", + "license": "MIT" + }, + "node_modules/jsdom": { + "version": "26.1.0", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-26.1.0.tgz", + "integrity": "sha512-Cvc9WUhxSMEo4McES3P7oK3QaXldCfNWp7pl2NNeiIFlCoLr3kfq9kb1fxftiwk1FLV7CvpvDfonxtzUDeSOPg==", + "license": "MIT", + "dependencies": { + "cssstyle": "^4.2.1", + "data-urls": "^5.0.0", + "decimal.js": "^10.5.0", + "html-encoding-sniffer": "^4.0.0", + "http-proxy-agent": "^7.0.2", + "https-proxy-agent": "^7.0.6", + "is-potential-custom-element-name": "^1.0.1", + "nwsapi": "^2.2.16", + "parse5": "^7.2.1", + "rrweb-cssom": "^0.8.0", + "saxes": "^6.0.0", + "symbol-tree": "^3.2.4", + "tough-cookie": "^5.1.1", + "w3c-xmlserializer": "^5.0.0", + "webidl-conversions": "^7.0.0", + "whatwg-encoding": "^3.1.1", + "whatwg-mimetype": "^4.0.0", + "whatwg-url": "^14.1.1", + "ws": "^8.18.0", + "xml-name-validator": "^5.0.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "canvas": "^3.0.0" + }, + "peerDependenciesMeta": { + "canvas": { + "optional": true + } + } + }, + "node_modules/katex": { + "version": "0.16.22", + "resolved": "https://registry.npmjs.org/katex/-/katex-0.16.22.tgz", + "integrity": "sha512-XCHRdUw4lf3SKBaJe4EvgqIuWwkPSo9XoeO8GjQW94Bp7TWv9hNhzZjZ+OH9yf1UmLygb7DIT5GSFQiyt16zYg==", + "funding": [ + "https://opencollective.com/katex", + "https://github.com/sponsors/katex" + ], + "license": "MIT", + "dependencies": { + "commander": "^8.3.0" + }, + "bin": { + "katex": "cli.js" + } + }, + "node_modules/khroma": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/khroma/-/khroma-2.1.0.tgz", + "integrity": "sha512-Ls993zuzfayK269Svk9hzpeGUKob/sIgZzyHYdjQoAdQetRKpOLj+k/QQQ/6Qi0Yz65mlROrfd+Ev+1+7dz9Kw==" + }, + "node_modules/kolorist": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/kolorist/-/kolorist-1.8.0.tgz", + "integrity": "sha512-Y+60/zizpJ3HRH8DCss+q95yr6145JXZo46OTpFvDZWLfRCE4qChOyk1b26nMaNpfHHgxagk9dXT5OP0Tfe+dQ==", + "license": "MIT" + }, + "node_modules/langium": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/langium/-/langium-3.3.1.tgz", + "integrity": "sha512-QJv/h939gDpvT+9SiLVlY7tZC3xB2qK57v0J04Sh9wpMb6MP1q8gB21L3WIo8T5P1MSMg3Ep14L7KkDCFG3y4w==", + "license": "MIT", + "dependencies": { + "chevrotain": "~11.0.3", + "chevrotain-allstar": "~0.3.0", + "vscode-languageserver": "~9.0.1", + "vscode-languageserver-textdocument": "~1.0.11", + "vscode-uri": "~3.0.8" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/layout-base": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/layout-base/-/layout-base-1.0.2.tgz", + "integrity": "sha512-8h2oVEZNktL4BH2JCOI90iD1yXwL6iNW7KcCKT2QZgQJR2vbqDsldCTPRU9NifTCqHZci57XvQQ15YTu+sTYPg==", + "license": "MIT" + }, + "node_modules/local-pkg": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/local-pkg/-/local-pkg-1.1.2.tgz", + "integrity": "sha512-arhlxbFRmoQHl33a0Zkle/YWlmNwoyt6QNZEIJcqNbdrsix5Lvc4HyyI3EnwxTYlZYc32EbYrQ8SzEZ7dqgg9A==", + "license": "MIT", + "dependencies": { + "mlly": "^1.7.4", + "pkg-types": "^2.3.0", + "quansync": "^0.2.11" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/antfu" + } + }, + "node_modules/lodash-es": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash-es/-/lodash-es-4.17.21.tgz", + "integrity": "sha512-mKnC+QJ9pWVzv+C4/U3rRsHapFfHvQFoFB92e52xeyGMcX6/OlIl78je1u8vePzYZSkkogMPJ2yjxxsb89cxyw==", + "license": "MIT" + }, + "node_modules/lru-cache": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", + "license": "ISC" + }, + "node_modules/marked": { + "version": "15.0.12", + "resolved": "https://registry.npmjs.org/marked/-/marked-15.0.12.tgz", + "integrity": "sha512-8dD6FusOQSrpv9Z1rdNMdlSgQOIP880DHqnohobOmYLElGEqAL/JvxvuxZO16r4HtjTlfPRDC1hbvxC9dPN2nA==", + "license": "MIT", + "bin": { + "marked": "bin/marked.js" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/mermaid": { + "version": "11.11.0", + "resolved": "https://registry.npmjs.org/mermaid/-/mermaid-11.11.0.tgz", + "integrity": "sha512-9lb/VNkZqWTRjVgCV+l1N+t4kyi94y+l5xrmBmbbxZYkfRl5hEDaTPMOcaWKCl1McG8nBEaMlWwkcAEEgjhBgg==", + "license": "MIT", + "dependencies": { + "@braintree/sanitize-url": "^7.0.4", + "@iconify/utils": "^3.0.1", + "@mermaid-js/parser": "^0.6.2", + "@types/d3": "^7.4.3", + "cytoscape": "^3.29.3", + "cytoscape-cose-bilkent": "^4.1.0", + "cytoscape-fcose": "^2.2.0", + "d3": "^7.9.0", + "d3-sankey": "^0.12.3", + "dagre-d3-es": "7.0.11", + "dayjs": "^1.11.13", + "dompurify": "^3.2.5", + "katex": "^0.16.22", + "khroma": "^2.1.0", + "lodash-es": "^4.17.21", + "marked": "^15.0.7", + "roughjs": "^4.6.6", + "stylis": "^4.3.6", + "ts-dedent": "^2.2.0", + "uuid": "^11.1.0" + } + }, + "node_modules/mlly": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/mlly/-/mlly-1.8.0.tgz", + "integrity": "sha512-l8D9ODSRWLe2KHJSifWGwBqpTZXIXTeo8mlKjY+E2HAakaTeNpqAyBZ8GSqLzHgw4XmHmC8whvpjJNMbFZN7/g==", + "license": "MIT", + "dependencies": { + "acorn": "^8.15.0", + "pathe": "^2.0.3", + "pkg-types": "^1.3.1", + "ufo": "^1.6.1" + } + }, + "node_modules/mlly/node_modules/confbox": { + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/confbox/-/confbox-0.1.8.tgz", + "integrity": "sha512-RMtmw0iFkeR4YV+fUOSucriAQNb9g8zFR52MWCtl+cCZOFRNL6zeB395vPzFhEjjn4fMxXudmELnl/KF/WrK6w==", + "license": "MIT" + }, + "node_modules/mlly/node_modules/pkg-types": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/pkg-types/-/pkg-types-1.3.1.tgz", + "integrity": "sha512-/Jm5M4RvtBFVkKWRu2BLUTNP8/M2a+UwuAX+ae4770q1qVGtfjG+WTCupoZixokjmHiry8uI+dlY8KXYV5HVVQ==", + "license": "MIT", + "dependencies": { + "confbox": "^0.1.8", + "mlly": "^1.7.4", + "pathe": "^2.0.1" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/nwsapi": { + "version": "2.2.22", + "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.22.tgz", + "integrity": "sha512-ujSMe1OWVn55euT1ihwCI1ZcAaAU3nxUiDwfDQldc51ZXaB9m2AyOn6/jh1BLe2t/G8xd6uKG1UBF2aZJeg2SQ==", + "license": "MIT" + }, + "node_modules/package-manager-detector": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/package-manager-detector/-/package-manager-detector-1.3.0.tgz", + "integrity": "sha512-ZsEbbZORsyHuO00lY1kV3/t72yp6Ysay6Pd17ZAlNGuGwmWDLCJxFpRs0IzfXfj1o4icJOkUEioexFHzyPurSQ==", + "license": "MIT" + }, + "node_modules/parse5": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.3.0.tgz", + "integrity": "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==", + "license": "MIT", + "dependencies": { + "entities": "^6.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/path-data-parser": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/path-data-parser/-/path-data-parser-0.1.0.tgz", + "integrity": "sha512-NOnmBpt5Y2RWbuv0LMzsayp3lVylAHLPUTut412ZA3l+C4uw4ZVkQbjShYCQ8TCpUMdPapr4YjUqLYD6v68j+w==", + "license": "MIT" + }, + "node_modules/pathe": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", + "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", + "license": "MIT" + }, + "node_modules/pkg-types": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/pkg-types/-/pkg-types-2.3.0.tgz", + "integrity": "sha512-SIqCzDRg0s9npO5XQ3tNZioRY1uK06lA41ynBC1YmFTmnY6FjUjVt6s4LoADmwoig1qqD0oK8h1p/8mlMx8Oig==", + "license": "MIT", + "dependencies": { + "confbox": "^0.2.2", + "exsolve": "^1.0.7", + "pathe": "^2.0.3" + } + }, + "node_modules/points-on-curve": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/points-on-curve/-/points-on-curve-0.2.0.tgz", + "integrity": "sha512-0mYKnYYe9ZcqMCWhUjItv/oHjvgEsfKvnUTg8sAtnHr3GVy7rGkXCb6d5cSyqrWqL4k81b9CPg3urd+T7aop3A==", + "license": "MIT" + }, + "node_modules/points-on-path": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/points-on-path/-/points-on-path-0.2.1.tgz", + "integrity": "sha512-25ClnWWuw7JbWZcgqY/gJ4FQWadKxGWk+3kR/7kD0tCaDtPPMj7oHu2ToLaVhfpnHrZzYby2w6tUA0eOIuUg8g==", + "license": "MIT", + "dependencies": { + "path-data-parser": "0.1.0", + "points-on-curve": "0.2.0" + } + }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/quansync": { + "version": "0.2.11", + "resolved": "https://registry.npmjs.org/quansync/-/quansync-0.2.11.tgz", + "integrity": "sha512-AifT7QEbW9Nri4tAwR5M/uzpBuqfZf+zwaEM/QkzEjj7NBuFD2rBuy0K3dE+8wltbezDV7JMA0WfnCPYRSYbXA==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/antfu" + }, + { + "type": "individual", + "url": "https://github.com/sponsors/sxzz" + } + ], + "license": "MIT" + }, + "node_modules/robust-predicates": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/robust-predicates/-/robust-predicates-3.0.2.tgz", + "integrity": "sha512-IXgzBWvWQwE6PrDI05OvmXUIruQTcoMDzRsOd5CDvHCVLcLHMTSYvOK5Cm46kWqlV3yAbuSpBZdJ5oP5OUoStg==", + "license": "Unlicense" + }, + "node_modules/roughjs": { + "version": "4.6.6", + "resolved": "https://registry.npmjs.org/roughjs/-/roughjs-4.6.6.tgz", + "integrity": "sha512-ZUz/69+SYpFN/g/lUlo2FXcIjRkSu3nDarreVdGGndHEBJ6cXPdKguS8JGxwj5HA5xIbVKSmLgr5b3AWxtRfvQ==", + "license": "MIT", + "dependencies": { + "hachure-fill": "^0.5.2", + "path-data-parser": "^0.1.0", + "points-on-curve": "^0.2.0", + "points-on-path": "^0.2.1" + } + }, + "node_modules/rrweb-cssom": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/rrweb-cssom/-/rrweb-cssom-0.8.0.tgz", + "integrity": "sha512-guoltQEx+9aMf2gDZ0s62EcV8lsXR+0w8915TC3ITdn2YueuNjdAYh/levpU9nFaoChh9RUS5ZdQMrKfVEN9tw==", + "license": "MIT" + }, + "node_modules/rw": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/rw/-/rw-1.3.3.tgz", + "integrity": "sha512-PdhdWy89SiZogBLaw42zdeqtRJ//zFd2PgQavcICDUgJT5oW10QCRKbJ6bg4r0/UY2M6BWd5tkxuGFRvCkgfHQ==", + "license": "BSD-3-Clause" + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/saxes": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/saxes/-/saxes-6.0.0.tgz", + "integrity": "sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==", + "license": "ISC", + "dependencies": { + "xmlchars": "^2.2.0" + }, + "engines": { + "node": ">=v12.22.7" + } + }, + "node_modules/stylis": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/stylis/-/stylis-4.3.6.tgz", + "integrity": "sha512-yQ3rwFWRfwNUY7H5vpU0wfdkNSnvnJinhF9830Swlaxl03zsOjCfmX0ugac+3LtK0lYSgwL/KXc8oYL3mG4YFQ==", + "license": "MIT" + }, + "node_modules/symbol-tree": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz", + "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==", + "license": "MIT" + }, + "node_modules/tinyexec": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.0.1.tgz", + "integrity": "sha512-5uC6DDlmeqiOwCPmK9jMSdOuZTh8bU39Ys6yidB+UTt5hfZUPGAypSgFRiEp+jbi9qH40BLDvy85jIU88wKSqw==", + "license": "MIT" + }, + "node_modules/tldts": { + "version": "6.1.86", + "resolved": "https://registry.npmjs.org/tldts/-/tldts-6.1.86.tgz", + "integrity": "sha512-WMi/OQ2axVTf/ykqCQgXiIct+mSQDFdH2fkwhPwgEwvJ1kSzZRiinb0zF2Xb8u4+OqPChmyI6MEu4EezNJz+FQ==", + "license": "MIT", + "dependencies": { + "tldts-core": "^6.1.86" + }, + "bin": { + "tldts": "bin/cli.js" + } + }, + "node_modules/tldts-core": { + "version": "6.1.86", + "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-6.1.86.tgz", + "integrity": "sha512-Je6p7pkk+KMzMv2XXKmAE3McmolOQFdxkKw0R8EYNr7sELW46JqnNeTX8ybPiQgvg1ymCoF8LXs5fzFaZvJPTA==", + "license": "MIT" + }, + "node_modules/tough-cookie": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-5.1.2.tgz", + "integrity": "sha512-FVDYdxtnj0G6Qm/DhNPSb8Ju59ULcup3tuJxkFb5K8Bv2pUXILbf0xZWU8PX8Ov19OXljbUyveOFwRMwkXzO+A==", + "license": "BSD-3-Clause", + "dependencies": { + "tldts": "^6.1.32" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/tr46": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-5.1.1.tgz", + "integrity": "sha512-hdF5ZgjTqgAntKkklYw0R03MG2x/bSzTtkxmIRw/sTNV8YXsCJ1tfLAX23lhxhHJlEf3CRCOCGGWw3vI3GaSPw==", + "license": "MIT", + "dependencies": { + "punycode": "^2.3.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/ts-dedent": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/ts-dedent/-/ts-dedent-2.2.0.tgz", + "integrity": "sha512-q5W7tVM71e2xjHZTlgfTDoPF/SmqKG5hddq9SzR49CH2hayqRKJtQ4mtRlSxKaJlR/+9rEM+mnBHf7I2/BQcpQ==", + "license": "MIT", + "engines": { + "node": ">=6.10" + } + }, + "node_modules/ufo": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/ufo/-/ufo-1.6.1.tgz", + "integrity": "sha512-9a4/uxlTWJ4+a5i0ooc1rU7C7YOw3wT+UGqdeNNHWnOF9qcMBgLRS+4IYUqbczewFx4mLEig6gawh7X6mFlEkA==", + "license": "MIT" + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/uuid": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-11.1.0.tgz", + "integrity": "sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/esm/bin/uuid" + } + }, + "node_modules/vscode-jsonrpc": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/vscode-jsonrpc/-/vscode-jsonrpc-8.2.0.tgz", + "integrity": "sha512-C+r0eKJUIfiDIfwJhria30+TYWPtuHJXHtI7J0YlOmKAo7ogxP20T0zxB7HZQIFhIyvoBPwWskjxrvAtfjyZfA==", + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/vscode-languageserver": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/vscode-languageserver/-/vscode-languageserver-9.0.1.tgz", + "integrity": "sha512-woByF3PDpkHFUreUa7Hos7+pUWdeWMXRd26+ZX2A8cFx6v/JPTtd4/uN0/jB6XQHYaOlHbio03NTHCqrgG5n7g==", + "license": "MIT", + "dependencies": { + "vscode-languageserver-protocol": "3.17.5" + }, + "bin": { + "installServerIntoExtension": "bin/installServerIntoExtension" + } + }, + "node_modules/vscode-languageserver-protocol": { + "version": "3.17.5", + "resolved": "https://registry.npmjs.org/vscode-languageserver-protocol/-/vscode-languageserver-protocol-3.17.5.tgz", + "integrity": "sha512-mb1bvRJN8SVznADSGWM9u/b07H7Ecg0I3OgXDuLdn307rl/J3A9YD6/eYOssqhecL27hK1IPZAsaqh00i/Jljg==", + "license": "MIT", + "dependencies": { + "vscode-jsonrpc": "8.2.0", + "vscode-languageserver-types": "3.17.5" + } + }, + "node_modules/vscode-languageserver-textdocument": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/vscode-languageserver-textdocument/-/vscode-languageserver-textdocument-1.0.12.tgz", + "integrity": "sha512-cxWNPesCnQCcMPeenjKKsOCKQZ/L6Tv19DTRIGuLWe32lyzWhihGVJ/rcckZXJxfdKCFvRLS3fpBIsV/ZGX4zA==", + "license": "MIT" + }, + "node_modules/vscode-languageserver-types": { + "version": "3.17.5", + "resolved": "https://registry.npmjs.org/vscode-languageserver-types/-/vscode-languageserver-types-3.17.5.tgz", + "integrity": "sha512-Ld1VelNuX9pdF39h2Hgaeb5hEZM2Z3jUrrMgWQAu82jMtZp7p3vJT3BzToKtZI7NgQssZje5o0zryOrhQvzQAg==", + "license": "MIT" + }, + "node_modules/vscode-uri": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/vscode-uri/-/vscode-uri-3.0.8.tgz", + "integrity": "sha512-AyFQ0EVmsOZOlAnxoFOGOq1SQDWAB7C6aqMGS23svWAllfOaxbuFvcT8D1i8z3Gyn8fraVeZNNmN6e9bxxXkKw==", + "license": "MIT" + }, + "node_modules/w3c-xmlserializer": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-5.0.0.tgz", + "integrity": "sha512-o8qghlI8NZHU1lLPrpi2+Uq7abh4GGPpYANlalzWxyWteJOCsr/P+oPBA49TOLu5FTZO4d3F9MnWJfiMo4BkmA==", + "license": "MIT", + "dependencies": { + "xml-name-validator": "^5.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/webidl-conversions": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", + "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + } + }, + "node_modules/whatwg-encoding": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-3.1.1.tgz", + "integrity": "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==", + "license": "MIT", + "dependencies": { + "iconv-lite": "0.6.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/whatwg-mimetype": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-4.0.0.tgz", + "integrity": "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/whatwg-url": { + "version": "14.2.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-14.2.0.tgz", + "integrity": "sha512-De72GdQZzNTUBBChsXueQUnPKDkg/5A5zp7pFDuQAj5UFoENpiACU0wlCvzpAGnTkj++ihpKwKyYewn/XNUbKw==", + "license": "MIT", + "dependencies": { + "tr46": "^5.1.0", + "webidl-conversions": "^7.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/ws": { + "version": "8.18.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.3.tgz", + "integrity": "sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/xml-name-validator": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-5.0.0.tgz", + "integrity": "sha512-EvGK8EJ3DhaHfbRlETOWAS5pO9MZITeauHKJyb8wyajUfQUenkIg2MvLDTZ4T/TgIcm3HU0TFBgWWboAZ30UHg==", + "license": "Apache-2.0", + "engines": { + "node": ">=18" + } + }, + "node_modules/xmlchars": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", + "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==", + "license": "MIT" + } + } +} diff --git a/extensions/mermaid-chat-features/package.json b/extensions/mermaid-chat-features/package.json new file mode 100644 index 00000000000..1276397b807 --- /dev/null +++ b/extensions/mermaid-chat-features/package.json @@ -0,0 +1,75 @@ +{ + "name": "marmaid-chat-features", + "displayName": "%displayName%", + "description": "%description%", + "version": "1.0.0", + "publisher": "vscode", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/microsoft/vscode.git" + }, + "aiKey": "0c6ae279ed8443289764825290e4f9e2-1a736e7c-1324-4338-be46-fc2a58ae4d14-7255", + "engines": { + "vscode": "^1.104.0" + }, + "enabledApiProposals": [ + "chatOutputRenderer" + ], + "capabilities": { + "virtualWorkspaces": true, + "untrustedWorkspaces": { + "supported": true + } + }, + "main": "./out/extension", + "browser": "./dist/browser/extension", + "activationEvents": [], + "contributes": { + "chatOutputRenderers": [ + { + "viewType": "vscode.chatMermaidDiagram", + "mimeTypes": [ + "text/vnd.mermaid" + ] + } + ], + "languageModelTools": [ + { + "name": "renderMermaidDiagram", + "displayName": "Mermaid Renderer", + "toolReferenceName": "renderMermaidDiagram", + "canBeReferencedInPrompt": true, + "modelDescription": "Renders a Mermaid diagram from Mermaid.js markup.", + "userDescription": "Render a Mermaid.js diagrams from markup.", + "inputSchema": { + "type": "object", + "properties": { + "markup": { + "type": "string", + "description": "The mermaid diagram markup to render as a Mermaid diagram. This should only be the markup of the diagram. Do not include a wrapping code block." + } + } + } + } + ] + }, + "scripts": { + "compile": "gulp compile-extension:mermaid-chat-features && npm run build-chat-webview", + "watch": "npm run build-chat-webview && gulp watch-extension:mermaid-chat-features", + "vscode:prepublish": "npm run build-ext && npm run build-chat-webview", + "build-ext": "node ../../node_modules/gulp/bin/gulp.js --gulpfile ../../build/gulpfile.extensions.js compile-extension:mermaid-chat-features ./tsconfig.json", + "build-chat-webview": "node ./esbuild-chat-webview.mjs", + "compile-web": "npx webpack-cli --config extension-browser.webpack.config --mode none", + "watch-web": "npx webpack-cli --config extension-browser.webpack.config --mode none --watch --info-verbosity verbose" + }, + "devDependencies": { + "@types/jsdom": "^21.1.7", + "@types/node": "^22" + }, + "dependencies": { + "dompurify": "^3.2.6", + "jsdom": "^26.1.0", + "mermaid": "^11.11.0" + } +} diff --git a/extensions/mermaid-chat-features/package.nls.json b/extensions/mermaid-chat-features/package.nls.json new file mode 100644 index 00000000000..8e95dac52bb --- /dev/null +++ b/extensions/mermaid-chat-features/package.nls.json @@ -0,0 +1,6 @@ +{ + "displayName": "Markdown Math", + "description": "Adds math support to Markdown in notebooks.", + "config.markdown.math.enabled": "Enable/disable rendering math in the built-in Markdown preview.", + "config.markdown.math.macros": "A collection of custom macros. Each macro is a key-value pair where the key is a new command name and the value is the expansion of the macro." +} diff --git a/extensions/mermaid-chat-features/src/extension.ts b/extensions/mermaid-chat-features/src/extension.ts new file mode 100644 index 00000000000..f52d203f078 --- /dev/null +++ b/extensions/mermaid-chat-features/src/extension.ts @@ -0,0 +1,244 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import DOMPurify from 'dompurify'; +import { JSDOM } from 'jsdom'; +import * as vscode from 'vscode'; + +/** + * View type that uniquely identifies the Mermaid chat output renderer. + */ +const viewType = 'vscode.chatMermaidDiagram'; + +/** + * Mime type used to identify Mermaid diagram data in chat output. + */ +const mime = 'text/vnd.mermaid'; + +const maxFixAttempts = 3; + +export function activate(context: vscode.ExtensionContext) { + + // Register tools + context.subscriptions.push( + vscode.lm.registerTool<{ markup: string }>('renderMermaidDiagram', { + invoke: async (options, token) => { + let sourceCode = options.input.markup; + sourceCode = await runMermaidMarkupFixLoop(sourceCode, token); + return writeMermaidToolOutput(sourceCode); + }, + }) + ); + + // Register the chat output renderer for Mermaid diagrams. + // This will be invoked with the data generated by the tools. + // It can also be invoked when rendering old Mermaid diagrams in the chat history. + context.subscriptions.push( + vscode.chat.registerChatOutputRenderer(viewType, { + async renderChatOutput({ value }, webview, _ctx, _token) { + const mermaidSource = new TextDecoder().decode(value); + + // Set the options for the webview + const mediaRoot = vscode.Uri.joinPath(context.extensionUri, 'chat-webview-out'); + webview.options = { + enableScripts: true, + localResourceRoots: [mediaRoot], + }; + + // Set the HTML content for the webview + const nonce = getNonce(); + const mermaidScript = vscode.Uri.joinPath(mediaRoot, 'index.js'); + + webview.html = ` + + + + + + + Mermaid Diagram + + + + + 
+							${escapeHtmlText(mermaidSource)}
+
+ + + + `; + }, + })); +} + +/** + * Lazily load mermaid + */ +const getMermaidInstance = (() => { + const createMermaidInstance = async () => { + // Patch the global window object for mermaid + + const { window } = new JSDOM(''); + (global as any).window = window; + (global as any).DOMPurify = DOMPurify(window); + return import('mermaid'); + }; + + let cached: Promise | undefined; + return async (): Promise => { + cached ??= createMermaidInstance(); + return (await cached).default; + }; +})(); + +/** + * Tries to fix mermaid syntax errors in a set number of attempts. + * + * @returns The best effort to fix the Mermaid markup. + */ +async function runMermaidMarkupFixLoop(sourceCode: string, token: vscode.CancellationToken): Promise { + let attempt = 0; + while (attempt < maxFixAttempts) { + const result = await validateMermaidMarkup(sourceCode); + if (token.isCancellationRequested) { + throw new Error('Operation cancelled'); + } + + if (result.type === 'success') { + return sourceCode; + } + + attempt++; + + sourceCode = await tryFixingUpMermaidMarkup(sourceCode, result.message, token); + if (token.isCancellationRequested) { + throw new Error('Operation cancelled'); + } + } + + // Return whatever we have after max attempts + return sourceCode; +} + +/** + * Validates the syntax of the provided Mermaid markup. + */ +async function validateMermaidMarkup(sourceCode: string): Promise<{ type: 'success' } | { type: 'error'; message: string }> { + try { + const mermaid = await getMermaidInstance(); + await mermaid.parse(sourceCode); + return { type: 'success' }; + } catch (error) { + if (!(error instanceof Error)) { + throw error; + } + + return { type: 'error', message: error.message }; + } +} + +/** + * Uses a language model to try to fix Mermaid markup based on an error message. + */ +async function tryFixingUpMermaidMarkup(sourceCode: string, errorMessage: string, token: vscode.CancellationToken): Promise { + const model = await getPreferredLm(); + if (!model) { + console.warn('No suitable model found for fixing Mermaid markup'); + return sourceCode; + } + + if (token.isCancellationRequested) { + throw new Error('Operation cancelled'); + } + + const completion = await model.sendRequest([ + vscode.LanguageModelChatMessage.Assistant(joinLines( + `The user will provide you with the source code for the Mermaid diagram and an error message.`, + `Your task is to fix the Mermaid source code based on the error message.`, + `Please return the fixed Mermaid source code inside a \`mermaid\` fenced code block. Do not add any comments or explanation.`, + `Make sure to return the entire source code.` + )), + vscode.LanguageModelChatMessage.User(joinLines( + `Here is my Mermaid source code:`, + ``, + `\`\`\`mermaid`, + `${sourceCode}`, + `\`\`\``, + ``, + `And here is the mermaid error message:`, + ``, + errorMessage, + )), + ], {}, token); + + return await parseMermaidMarkupFromChatResponse(completion, token) ?? sourceCode; +} + +async function parseMermaidMarkupFromChatResponse(chatResponse: vscode.LanguageModelChatResponse, token: vscode.CancellationToken): Promise { + const parts: string[] = []; + for await (const line of chatResponse.text) { + if (token.isCancellationRequested) { + throw new Error('Operation cancelled'); + } + + parts.push(line); + } + + const response = parts.join(''); + const lines = response.split('\n'); + if (!lines.at(0)?.startsWith('```') || !lines.at(-1)?.endsWith('```')) { + console.warn('Invalid response format from model, expected fenced code block'); + return undefined; + } + + return lines.slice(1, -1).join('\n').trim(); +} + + +async function getPreferredLm(): Promise { + return (await vscode.lm.selectChatModels({ family: 'gpt-4o-mini' })).at(0) + ?? (await vscode.lm.selectChatModels({ family: 'gpt-4o' })).at(0) + ?? (await vscode.lm.selectChatModels({})).at(0); +} + +function writeMermaidToolOutput(sourceCode: string): vscode.LanguageModelToolResult { + // Expose the source code as a tool result for the LM + const result = new vscode.LanguageModelToolResult([ + new vscode.LanguageModelTextPart(sourceCode) + ]); + + // And store custom data in the tool result details to indicate that a custom renderer should be used for it. + // In this case we just store the source code as binary data. + + // Add cast to use proposed API + (result as vscode.ExtendedLanguageModelToolResult2).toolResultDetails2 = { + mime, + value: new TextEncoder().encode(sourceCode), + }; + + return result; +} + +function joinLines(...lines: string[]): string { + return lines.join('\n'); +} + +function escapeHtmlText(str: string): string { + return str + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, '''); +} + +function getNonce() { + let text = ''; + const possible = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'; + for (let i = 0; i < 64; i++) { + text += possible.charAt(Math.floor(Math.random() * possible.length)); + } + return text; +} diff --git a/extensions/mermaid-chat-features/tsconfig.json b/extensions/mermaid-chat-features/tsconfig.json new file mode 100644 index 00000000000..daa4f6357b3 --- /dev/null +++ b/extensions/mermaid-chat-features/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "../tsconfig.base.json", + "compilerOptions": { + "outDir": "./out", + "types": [] + }, + "include": [ + "src/**/*", + "../../src/vscode-dts/vscode.d.ts", + "../../src/vscode-dts/vscode.proposed.chatOutputRenderer.d.ts", + "../../src/vscode-dts/vscode.proposed.chatParticipantAdditions.d.ts", + "../../src/vscode-dts/vscode.proposed.languageModelThinkingPart.d.ts", + "../../src/vscode-dts/vscode.proposed.chatParticipantPrivate.d.ts" + ] +} diff --git a/extensions/simple-browser/package.json b/extensions/simple-browser/package.json index 789de38deb3..5e081c4bbd2 100644 --- a/extensions/simple-browser/package.json +++ b/extensions/simple-browser/package.json @@ -57,10 +57,10 @@ ] }, "scripts": { - "compile": "gulp compile-extension:markdown-language-features && npm run build-preview", - "watch": "npm run build-preview && gulp watch-extension:markdown-language-features", + "compile": "gulp compile-extension:simple-browser && npm run build-preview", + "watch": "npm run build-preview && gulp watch-extension:simple-browser", "vscode:prepublish": "npm run build-ext && npm run build-preview", - "build-ext": "node ../../node_modules/gulp/bin/gulp.js --gulpfile ../../build/gulpfile.extensions.js compile-extension:markdown-language-features ./tsconfig.json", + "build-ext": "node ../../node_modules/gulp/bin/gulp.js --gulpfile ../../build/gulpfile.extensions.js compile-extension:simple-browser ./tsconfig.json", "build-preview": "node ./esbuild-preview.mjs", "compile-web": "npx webpack-cli --config extension-browser.webpack.config --mode none", "watch-web": "npx webpack-cli --config extension-browser.webpack.config --mode none --watch --info-verbosity verbose" From 2e32e7fc92032f48ef9b39de8dd11e74ef56d027 Mon Sep 17 00:00:00 2001 From: Matt Bierner <12821956+mjbvz@users.noreply.github.com> Date: Wed, 10 Sep 2025 18:24:08 -0700 Subject: [PATCH 014/299] Update strings --- extensions/mermaid-chat-features/package.nls.json | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/extensions/mermaid-chat-features/package.nls.json b/extensions/mermaid-chat-features/package.nls.json index 8e95dac52bb..882e5d59f01 100644 --- a/extensions/mermaid-chat-features/package.nls.json +++ b/extensions/mermaid-chat-features/package.nls.json @@ -1,6 +1,4 @@ { - "displayName": "Markdown Math", - "description": "Adds math support to Markdown in notebooks.", - "config.markdown.math.enabled": "Enable/disable rendering math in the built-in Markdown preview.", - "config.markdown.math.macros": "A collection of custom macros. Each macro is a key-value pair where the key is a new command name and the value is the expansion of the macro." + "displayName": "Mermaid Chat Features", + "description": "Adds Mermaid diagram support to built-in chats." } From b07db7a40a428535ffba8422f86d3a21d2138ccd Mon Sep 17 00:00:00 2001 From: Matt Bierner <12821956+mjbvz@users.noreply.github.com> Date: Wed, 10 Sep 2025 18:31:55 -0700 Subject: [PATCH 015/299] Update build dir --- build/lib/extensions.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build/lib/extensions.js b/build/lib/extensions.js index aeea722d372..c80a1be1a84 100644 --- a/build/lib/extensions.js +++ b/build/lib/extensions.js @@ -510,11 +510,12 @@ function translatePackageJSON(packageJSON, packageNLSPath) { const extensionsPath = path_1.default.join(root, 'extensions'); // Additional projects to run esbuild on. These typically build code for webviews const esbuildMediaScripts = [ + 'ipynb/esbuild.mjs', 'markdown-language-features/esbuild-notebook.mjs', 'markdown-language-features/esbuild-preview.mjs', 'markdown-math/esbuild.mjs', + 'mermaid-chat-features/esbuild-chat-webview.mjs', 'notebook-renderers/esbuild.mjs', - 'ipynb/esbuild.mjs', 'simple-browser/esbuild-preview.mjs', ]; async function webpackExtensions(taskName, isWatch, webpackConfigLocations) { From 4b3b3c9507c3a9e13a21859ea70684bd5fd3df33 Mon Sep 17 00:00:00 2001 From: Martin Aeschlimann Date: Thu, 11 Sep 2025 15:42:40 +0200 Subject: [PATCH 016/299] adopt code completion --- .../promptHeaderAutocompletion.ts | 56 ++++++------------- 1 file changed, 18 insertions(+), 38 deletions(-) diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/promptHeaderAutocompletion.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/promptHeaderAutocompletion.ts index 92c425f90bb..12644cb61eb 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/promptHeaderAutocompletion.ts +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/promptHeaderAutocompletion.ts @@ -14,13 +14,10 @@ import { ILanguageFeaturesService } from '../../../../../../editor/common/servic import { ILanguageModelChatMetadata, ILanguageModelsService } from '../../languageModels.js'; import { ILanguageModelToolsService } from '../../languageModelToolsService.js'; import { IChatModeService } from '../../chatModes.js'; -import { InstructionsHeader } from '../parsers/promptHeader/instructionsHeader.js'; -import { PromptToolsMetadata } from '../parsers/promptHeader/metadata/tools.js'; -import { ModeHeader } from '../parsers/promptHeader/modeHeader.js'; -import { PromptHeader } from '../parsers/promptHeader/promptHeader.js'; import { ALL_PROMPTS_LANGUAGE_SELECTOR, getPromptsTypeForLanguageId, PromptsType } from '../promptTypes.js'; import { IPromptsService } from '../service/promptsService.js'; import { Iterable } from '../../../../../../base/common/iterator.js'; +import { PromptHeader } from '../service/newPromptsParser.js'; export class PromptHeaderAutocompletion extends Disposable implements CompletionItemProvider { /** @@ -62,27 +59,14 @@ export class PromptHeaderAutocompletion extends Disposable implements Completion return undefined; } - const parser = this.promptsService.getSyntaxParserFor(model); - await parser.start(token).settled(); - - if (token.isCancellationRequested) { - return undefined; - } - + const parser = this.promptsService.getParsedPromptFile(model); const header = parser.header; if (!header) { return undefined; } - const completed = await header.settled; - if (!completed || token.isCancellationRequested) { - return undefined; - } - - const fullHeaderRange = parser.header.range; - const headerRange = new Range(fullHeaderRange.startLineNumber + 1, 0, fullHeaderRange.endLineNumber - 1, model.getLineMaxColumn(fullHeaderRange.endLineNumber - 1),); - - if (!headerRange.containsPosition(position)) { + const headerRange = parser.header.range; + if (position.lineNumber < headerRange.startLineNumber || position.lineNumber >= headerRange.endLineNumber) { // if the position is not inside the header, we don't provide any completions return undefined; } @@ -103,7 +87,7 @@ export class PromptHeaderAutocompletion extends Disposable implements Completion position: Position, headerRange: Range, colonPosition: Position | undefined, - promptType: string, + promptType: PromptsType, ): Promise { const suggestions: CompletionItem[] = []; @@ -140,9 +124,9 @@ export class PromptHeaderAutocompletion extends Disposable implements Completion private async provideValueCompletions( model: ITextModel, position: Position, - header: PromptHeader | ModeHeader | InstructionsHeader, + header: PromptHeader, colonPosition: Position, - promptType: string, + promptType: PromptsType, ): Promise { const suggestions: CompletionItem[] = []; @@ -153,14 +137,11 @@ export class PromptHeaderAutocompletion extends Disposable implements Completion return undefined; } - if (header instanceof PromptHeader || header instanceof ModeHeader) { - const tools = header.metadataUtility.tools; - if (tools) { - // if the position is inside the tools metadata, we provide tool name completions - const result = this.provideToolCompletions(model, position, tools); - if (result) { - return result; - } + if (promptType === PromptsType.prompt || promptType === PromptsType.mode) { + // if the position is inside the tools metadata, we provide tool name completions + const result = this.provideToolCompletions(model, position, header); + if (result) { + return result; } } @@ -244,9 +225,9 @@ export class PromptHeaderAutocompletion extends Disposable implements Completion return result; } - private provideToolCompletions(model: ITextModel, position: Position, node: PromptToolsMetadata): CompletionList | undefined { - const tools = node.value; - if (!tools || !node.range.containsPosition(position)) { + private provideToolCompletions(model: ITextModel, position: Position, header: PromptHeader): CompletionList | undefined { + const toolsAttr = header.attributes.find(attr => attr.key === 'tools'); + if (!toolsAttr || toolsAttr.value.type !== 'array' || !toolsAttr.range.containsPosition(position)) { return undefined; } const getSuggestions = (toolRange: Range) => { @@ -278,11 +259,10 @@ export class PromptHeaderAutocompletion extends Disposable implements Completion return { suggestions }; }; - for (const tool of tools) { - const toolRange = node.getToolRange(tool); - if (toolRange?.containsPosition(position)) { + for (const toolNameNode of toolsAttr.value.items) { + if (toolNameNode.range.containsPosition(position)) { // if the position is inside a tool range, we provide tool name completions - return getSuggestions(toolRange); + return getSuggestions(toolNameNode.range); } } const prefix = model.getValueInRange(new Range(position.lineNumber, 1, position.lineNumber, position.column)); From f9145afad26457527dd50d8101f1b7f3f91612f3 Mon Sep 17 00:00:00 2001 From: Martin Aeschlimann Date: Thu, 11 Sep 2025 17:07:46 +0200 Subject: [PATCH 017/299] adopt getParsedPromptFile --- .../promptSyntax/promptFileRewriter.ts | 24 ++---- .../promptToolsCodeLensProvider.ts | 39 ++++------ .../PromptHeaderDefinitionProvider.ts | 32 +------- .../promptHeaderAutocompletion.ts | 2 +- .../languageProviders/promptHeaderHovers.ts | 76 +++++++------------ .../languageProviders/promptLinkProvider.ts | 65 +++++----------- .../promptSyntax/service/newPromptsParser.ts | 9 ++- .../promptSyntax/service/promptsService.ts | 6 ++ .../service/promptsServiceImpl.ts | 6 +- .../service/newPromptsParser.test.ts | 8 +- 10 files changed, 96 insertions(+), 171 deletions(-) diff --git a/src/vs/workbench/contrib/chat/browser/promptSyntax/promptFileRewriter.ts b/src/vs/workbench/contrib/chat/browser/promptSyntax/promptFileRewriter.ts index c87e8d9c7e3..21072444d07 100644 --- a/src/vs/workbench/contrib/chat/browser/promptSyntax/promptFileRewriter.ts +++ b/src/vs/workbench/contrib/chat/browser/promptSyntax/promptFileRewriter.ts @@ -26,30 +26,20 @@ export class PromptFileRewriter { } const model = editor.getModel(); - const parser = this._promptsService.getSyntaxParserFor(model); - await parser.start(token).settled(); - const { header } = parser; - if (header === undefined) { + const parser = this._promptsService.getParsedPromptFile(model); + if (!parser.header) { return undefined; } - const completed = await header.settled; - if (!completed || token.isCancellationRequested) { - return; + const toolsAttr = parser.header.getAttribute('tools'); + if (!toolsAttr) { + return undefined; } - if (('tools' in header.metadataUtility) === false) { - return undefined; - } - const { tools } = header.metadataUtility; - if (tools === undefined) { - return undefined; - } - editor.setSelection(tools.range); - this.rewriteTools(model, newTools, tools.range); + editor.setSelection(toolsAttr.range); + this.rewriteTools(model, newTools, toolsAttr.range); } - public rewriteTools(model: ITextModel, newTools: IToolAndToolSetEnablementMap | undefined, range: Range): void { const newString = newTools === undefined ? '' : `tools: ${this.getNewValueString(newTools)}`; model.pushStackElement(); diff --git a/src/vs/workbench/contrib/chat/browser/promptSyntax/promptToolsCodeLensProvider.ts b/src/vs/workbench/contrib/chat/browser/promptSyntax/promptToolsCodeLensProvider.ts index 904ff2dff6f..f06236bb2a6 100644 --- a/src/vs/workbench/contrib/chat/browser/promptSyntax/promptToolsCodeLensProvider.ts +++ b/src/vs/workbench/contrib/chat/browser/promptSyntax/promptToolsCodeLensProvider.ts @@ -15,10 +15,10 @@ import { IInstantiationService } from '../../../../../platform/instantiation/com import { showToolsPicker } from '../actions/chatToolPicker.js'; import { ILanguageModelToolsService } from '../../common/languageModelToolsService.js'; import { ALL_PROMPTS_LANGUAGE_SELECTOR } from '../../common/promptSyntax/promptTypes.js'; -import { PromptToolsMetadata } from '../../common/promptSyntax/parsers/promptHeader/metadata/tools.js'; import { IPromptsService } from '../../common/promptSyntax/service/promptsService.js'; import { registerEditorFeature } from '../../../../../editor/common/editorFeatures.js'; import { PromptFileRewriter } from './promptFileRewriter.js'; +import { Range } from '../../../../../editor/common/core/range.js'; class PromptToolsCodeLensProvider extends Disposable implements CodeLensProvider { @@ -37,56 +37,47 @@ class PromptToolsCodeLensProvider extends Disposable implements CodeLensProvider this._register(this.languageService.codeLensProvider.register(ALL_PROMPTS_LANGUAGE_SELECTOR, this)); this._register(CommandsRegistry.registerCommand(this.cmdId, (_accessor, ...args) => { - const [first, second] = args; - if (isITextModel(first) && second instanceof PromptToolsMetadata) { - this.updateTools(first, second); + const [first, second, third] = args; + if (isITextModel(first) && Range.isIRange(second) && Array.isArray(third)) { + this.updateTools(first, Range.lift(second), third); } })); } async provideCodeLenses(model: ITextModel, token: CancellationToken): Promise { - const parser = this.promptsService.getSyntaxParserFor(model); - - await parser.start(token).settled(); - const { header } = parser; - if (!header) { + const parser = this.promptsService.getParsedPromptFile(model); + if (!parser.header) { return undefined; } - const completed = await header.settled; - if (!completed || token.isCancellationRequested) { - return undefined; - } - if (('tools' in header.metadataUtility) === false) { - return undefined; - } - - const { tools } = header.metadataUtility; - if (tools === undefined) { + const toolsAttr = parser.header.getAttribute('tools'); + if (!toolsAttr || toolsAttr.value.type !== 'array') { return undefined; } + const items = toolsAttr.value.items; + const selectedTools = items.filter(item => item.type === 'string').map(item => item.value); const codeLens: CodeLens = { - range: tools.range.collapseToStart(), + range: toolsAttr.range.collapseToStart(), command: { title: localize('configure-tools.capitalized.ellipsis', "Configure Tools..."), id: this.cmdId, - arguments: [model, tools] + arguments: [model, toolsAttr.range, selectedTools] } }; return { lenses: [codeLens] }; } - private async updateTools(model: ITextModel, tools: PromptToolsMetadata) { + private async updateTools(model: ITextModel, range: Range, selectedTools: readonly string[]) { - const selectedToolsNow = tools.value ? this.languageModelToolsService.toToolAndToolSetEnablementMap(tools.value) : new Map(); + const selectedToolsNow = this.languageModelToolsService.toToolAndToolSetEnablementMap(selectedTools); const newSelectedAfter = await this.instantiationService.invokeFunction(showToolsPicker, localize('placeholder', "Select tools"), undefined, selectedToolsNow); if (!newSelectedAfter) { return; } - await this.instantiationService.createInstance(PromptFileRewriter).rewriteTools(model, newSelectedAfter, tools.range); + await this.instantiationService.createInstance(PromptFileRewriter).rewriteTools(model, newSelectedAfter, range); } } diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/PromptHeaderDefinitionProvider.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/PromptHeaderDefinitionProvider.ts index 785b407da9c..f9afca76373 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/PromptHeaderDefinitionProvider.ts +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/PromptHeaderDefinitionProvider.ts @@ -13,8 +13,6 @@ import { ILanguageFeaturesService } from '../../../../../../editor/common/servic import { IChatModeService } from '../../chatModes.js'; import { ALL_PROMPTS_LANGUAGE_SELECTOR, getPromptsTypeForLanguageId } from '../promptTypes.js'; import { IPromptsService } from '../service/promptsService.js'; -import { PromptModeMetadata } from '../parsers/promptHeader/metadata/mode.js'; -import { PromptHeader } from '../parsers/promptHeader/promptHeader.js'; export class PromptHeaderDefinitionProvider extends Disposable implements DefinitionProvider { /** @@ -38,37 +36,15 @@ export class PromptHeaderDefinitionProvider extends Disposable implements Defini return undefined; } - const parser = this.promptsService.getSyntaxParserFor(model); - await parser.start(token).settled(); - - if (token.isCancellationRequested) { - return undefined; - } - + const parser = this.promptsService.getParsedPromptFile(model); const header = parser.header; if (!header) { return undefined; } - const completed = await header.settled; - if (!completed || token.isCancellationRequested) { - return undefined; - } - - if (header instanceof PromptHeader) { - const mode = header.metadataUtility.mode; - if (mode?.range.containsPosition(position)) { - return this.getModeDefinition(mode, position); - } - } - return undefined; - } - - - private getModeDefinition(mode: PromptModeMetadata, position: Position): Definition | undefined { - const value = mode.value; - if (value && mode.valueRange?.containsPosition(position)) { - const mode = this.chatModeService.findModeByName(value); + const modeAttr = header.getAttribute('mode'); + if (modeAttr && modeAttr.value.type === 'string' && modeAttr.range.containsPosition(position)) { + const mode = this.chatModeService.findModeByName(modeAttr.value.value); if (mode && mode.uri) { return { uri: mode.uri.get(), diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/promptHeaderAutocompletion.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/promptHeaderAutocompletion.ts index 12644cb61eb..61bbbcc7754 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/promptHeaderAutocompletion.ts +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/promptHeaderAutocompletion.ts @@ -226,7 +226,7 @@ export class PromptHeaderAutocompletion extends Disposable implements Completion } private provideToolCompletions(model: ITextModel, position: Position, header: PromptHeader): CompletionList | undefined { - const toolsAttr = header.attributes.find(attr => attr.key === 'tools'); + const toolsAttr = header.getAttribute('tools'); if (!toolsAttr || toolsAttr.value.type !== 'array' || !toolsAttr.range.containsPosition(position)) { return undefined; } diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/promptHeaderHovers.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/promptHeaderHovers.ts index 9072b2bf21f..050ccbebe23 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/promptHeaderHovers.ts +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/promptHeaderHovers.ts @@ -15,13 +15,9 @@ import { localize } from '../../../../../../nls.js'; import { ILanguageModelChatMetadata, ILanguageModelsService } from '../../languageModels.js'; import { ILanguageModelToolsService, ToolSet } from '../../languageModelToolsService.js'; import { IChatModeService, isBuiltinChatMode } from '../../chatModes.js'; -import { InstructionsHeader } from '../parsers/promptHeader/instructionsHeader.js'; -import { PromptModelMetadata } from '../parsers/promptHeader/metadata/model.js'; -import { PromptToolsMetadata } from '../parsers/promptHeader/metadata/tools.js'; -import { ModeHeader } from '../parsers/promptHeader/modeHeader.js'; -import { ALL_PROMPTS_LANGUAGE_SELECTOR, getPromptsTypeForLanguageId } from '../promptTypes.js'; +import { ALL_PROMPTS_LANGUAGE_SELECTOR, getPromptsTypeForLanguageId, PromptsType } from '../promptTypes.js'; import { IPromptsService } from '../service/promptsService.js'; -import { PromptModeMetadata } from '../parsers/promptHeader/metadata/mode.js'; +import { IHeaderAttribute } from '../service/newPromptsParser.js'; export class PromptHeaderHoverProvider extends Disposable implements HoverProvider { /** @@ -61,60 +57,49 @@ export class PromptHeaderHoverProvider extends Disposable implements HoverProvid return undefined; } - const parser = this.promptsService.getSyntaxParserFor(model); - await parser.start(token).settled(); - - if (token.isCancellationRequested) { - return undefined; - } - + const parser = this.promptsService.getParsedPromptFile(model); const header = parser.header; if (!header) { return undefined; } - const completed = await header.settled; - if (!completed || token.isCancellationRequested) { - return undefined; - } - - if (header instanceof InstructionsHeader) { - const descriptionRange = header.metadataUtility.description?.range; + if (promptType === PromptsType.instructions) { + const descriptionRange = header.getAttribute('description')?.range; if (descriptionRange?.containsPosition(position)) { return this.createHover(localize('promptHeader.instructions.description', 'The description of the instruction file. It can be used to provide additional context or information about the instructions and is passed to the language model as part of the prompt.'), descriptionRange); } - const applyToRange = header.metadataUtility.applyTo?.range; + const applyToRange = header.getAttribute('applyTo')?.range; if (applyToRange?.containsPosition(position)) { return this.createHover(localize('promptHeader.instructions.applyToRange', 'One or more glob pattern (separated by comma) that describe for which files the instructions apply to. Based on these patterns, the file is automatically included in the prompt, when the context contains a file that matches one or more of these patterns. Use `**` when you want this file to always be added.\nExample: **/*.ts, **/*.js, client/**'), applyToRange); } - } else if (header instanceof ModeHeader) { - const descriptionRange = header.metadataUtility.description?.range; + } else if (promptType === PromptsType.mode) { + const descriptionRange = header.getAttribute('description')?.range; if (descriptionRange?.containsPosition(position)) { return this.createHover(localize('promptHeader.mode.description', 'The description of the mode file. It can be used to provide additional context or information about the mode to the mode author.'), descriptionRange); } - const model = header.metadataUtility.model; + const model = header.getAttribute('model'); if (model?.range.containsPosition(position)) { return this.getModelHover(model, model.range, localize('promptHeader.mode.model', 'The model to use in this mode.')); } - const tools = header.metadataUtility.tools; + const tools = header.getAttribute('tools'); if (tools?.range.containsPosition(position)) { return this.getToolHover(tools, position, localize('promptHeader.mode.tools', 'The tools to use in this mode.')); } } else { - const descriptionRange = header.metadataUtility.description?.range; + const descriptionRange = header.getAttribute('description')?.range; if (descriptionRange?.containsPosition(position)) { return this.createHover(localize('promptHeader.prompt.description', 'The description of the prompt file. It can be used to provide additional context or information about the prompt to the prompt author.'), descriptionRange); } - const model = header.metadataUtility.model; + const model = header.getAttribute('model'); if (model?.range.containsPosition(position)) { return this.getModelHover(model, model.range, localize('promptHeader.prompt.model', 'The model to use in this prompt.')); } - const tools = header.metadataUtility.tools; + const tools = header.getAttribute('tools'); if (tools?.range.containsPosition(position)) { return this.getToolHover(tools, position, localize('promptHeader.prompt.tools', 'The tools to use in this prompt.')); } - const mode = header.metadataUtility.mode; + const mode = header.getAttribute('mode'); if (mode?.range.containsPosition(position)) { return this.getModeHover(mode, position, localize('promptHeader.prompt.mode', 'The mode to use in this prompt.')); } @@ -122,19 +107,17 @@ export class PromptHeaderHoverProvider extends Disposable implements HoverProvid return undefined; } - private getToolHover(node: PromptToolsMetadata, position: Position, baseMessage: string): Hover | undefined { - if (node.value) { - - for (const toolName of node.value) { - const toolRange = node.getToolRange(toolName); - if (toolRange?.containsPosition(position)) { - const tool = this.languageModelToolsService.getToolByName(toolName); + private getToolHover(node: IHeaderAttribute, position: Position, baseMessage: string): Hover | undefined { + if (node.value.type === 'array') { + for (const toolName of node.value.items) { + if (toolName.type === 'string' && toolName.range.containsPosition(position)) { + const tool = this.languageModelToolsService.getToolByName(toolName.value); if (tool) { - return this.createHover(tool.modelDescription, toolRange); + return this.createHover(tool.modelDescription, toolName.range); } - const toolSet = this.languageModelToolsService.getToolSetByName(toolName); + const toolSet = this.languageModelToolsService.getToolSetByName(toolName.value); if (toolSet) { - return this.getToolsetHover(toolSet, toolRange); + return this.getToolsetHover(toolSet, toolName.range); } } } @@ -154,12 +137,11 @@ export class PromptHeaderHoverProvider extends Disposable implements HoverProvid return this.createHover(lines.join('\n'), range); } - private getModelHover(node: PromptModelMetadata, range: Range, baseMessage: string): Hover | undefined { - const modelName = node.value; - if (modelName) { + private getModelHover(node: IHeaderAttribute, range: Range, baseMessage: string): Hover | undefined { + if (node.value.type === 'string') { for (const id of this.languageModelsService.getLanguageModelIds()) { const meta = this.languageModelsService.lookupLanguageModel(id); - if (meta && ILanguageModelChatMetadata.matchesQualifiedName(modelName, meta)) { + if (meta && ILanguageModelChatMetadata.matchesQualifiedName(node.value.value, meta)) { const lines: string[] = []; lines.push(baseMessage + '\n'); lines.push(localize('modelName', '- Name: {0}', meta.name)); @@ -175,13 +157,11 @@ export class PromptHeaderHoverProvider extends Disposable implements HoverProvid return this.createHover(baseMessage, range); } - private getModeHover(mode: PromptModeMetadata, position: Position, baseMessage: string): Hover | undefined { + private getModeHover(mode: IHeaderAttribute, position: Position, baseMessage: string): Hover | undefined { const lines: string[] = []; - - const value = mode.value; - if (value && mode.valueRange?.containsPosition(position)) { - const mode = this.chatModeService.findModeByName(value); + if (value.type === 'string' && value.range.containsPosition(position)) { + const mode = this.chatModeService.findModeByName(value.value); if (mode) { const description = mode.description.get() || (isBuiltinChatMode(mode) ? localize('promptHeader.prompt.mode.builtInDesc', 'Built-in chat mode') : localize('promptHeader.prompt.mode.customDesc', 'Custom chat mode')); lines.push(`\`${mode.name}\`: ${description}`); diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/promptLinkProvider.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/promptLinkProvider.ts index 22b498d8b3a..53e408c38dc 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/promptLinkProvider.ts +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/promptLinkProvider.ts @@ -4,67 +4,40 @@ *--------------------------------------------------------------------------------------------*/ import { IPromptsService } from '../service/promptsService.js'; -import { assert } from '../../../../../../base/common/assert.js'; import { ITextModel } from '../../../../../../editor/common/model.js'; -import { assertDefined } from '../../../../../../base/common/types.js'; -import { CancellationError } from '../../../../../../base/common/errors.js'; import { CancellationToken } from '../../../../../../base/common/cancellation.js'; import { ILink, ILinksList, LinkProvider } from '../../../../../../editor/common/languages.js'; +import { ALL_PROMPTS_LANGUAGE_SELECTOR } from '../promptTypes.js'; +import { Disposable } from '../../../../../../base/common/lifecycle.js'; +import { ILanguageFeaturesService } from '../../../../../../editor/common/services/languageFeatures.js'; /** * Provides link references for prompt files. */ -export class PromptLinkProvider implements LinkProvider { +export class PromptLinkProvider extends Disposable implements LinkProvider { constructor( + @ILanguageFeaturesService languageService: ILanguageFeaturesService, @IPromptsService private readonly promptsService: IPromptsService, ) { + super(); + this._register(languageService.linkProvider.register(ALL_PROMPTS_LANGUAGE_SELECTOR, this)); } /** * Provide list of links for the provided text model. */ - public async provideLinks( - model: ITextModel, - token: CancellationToken, - ): Promise { - assert( - !token.isCancellationRequested, - new CancellationError(), - ); - - const parser = this.promptsService.getSyntaxParserFor(model); - assert( - parser.isDisposed === false, - 'Prompt parser must not be disposed.', - ); - - // start the parser in case it was not started yet, - // and wait for it to settle to a final result - const completed = await parser.start(token).settled(); - if (!completed || token.isCancellationRequested) { - return undefined; + public async provideLinks(model: ITextModel, token: CancellationToken): Promise { + const parser = this.promptsService.getParsedPromptFile(model); + if (!parser.body) { + return; } - const { references } = parser; - - // filter out references that are not valid links - const links: ILink[] = references - .map((reference) => { - const { uri, linkRange } = reference; - - // must always be true because of the filter above - assertDefined( - linkRange, - 'Link range must be defined.', - ); - - return { - range: linkRange, - url: uri, - }; - }); - - return { - links, - }; + const links: ILink[] = []; + for (const ref of parser.body.fileReferences) { + const url = parser.body.resolveFilePath(ref.content); + if (url) { + links.push({ range: ref.range, url }); + } + } + return { links }; } } diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/service/newPromptsParser.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/service/newPromptsParser.ts index f0c88fcb01d..bf35d5da09e 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/service/newPromptsParser.ts +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/service/newPromptsParser.ts @@ -117,6 +117,10 @@ export class PromptHeader { return this.getParsedHeader().attributes; } + public getAttribute(key: string): IHeaderAttribute | undefined { + return this.getParsedHeader().attributes.find(attr => attr.key === key); + } + public get errors(): ParseError[] { return this.getParsedHeader().errors; } @@ -232,7 +236,7 @@ export class PromptBody { const linkEndOffset = match.index + match[0].length - 1; // before the parenthesis const linkStartOffset = match.index + match[0].length - match[2].length - 1; const range = new Range(i + 1, linkStartOffset + 1, i + 1, linkEndOffset + 1); - fileReferences.push({ content: match[2], range }); + fileReferences.push({ content: match[2], range, isMarkdownLink: true }); markdownLinkRanges.push(new Range(i + 1, match.index + 1, i + 1, match.index + match[0].length + 1)); } const reg = new RegExp(`${chatVariableLeader}([\\w]+:)?([^\\s#]*)`, 'g'); @@ -248,7 +252,7 @@ export class PromptBody { const linkStartOffset = match.index + match[0].length - match[2].length; const linkEndOffset = match.index + match[0].length; const range = new Range(i + 1, linkStartOffset + 1, i + 1, linkEndOffset + 1); - fileReferences.push({ content: match[2], range }); + fileReferences.push({ content: match[2], range, isMarkdownLink: false }); } } else { const contentStartOffset = match.index + 1; // after the # @@ -281,6 +285,7 @@ export class PromptBody { interface IBodyFileReference { content: string; range: Range; + isMarkdownLink: boolean; } interface IBodyVariableReference { diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsService.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsService.ts index ca85e45152f..5363d277802 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsService.ts +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsService.ts @@ -213,6 +213,12 @@ export interface IPromptsService extends IDisposable { */ parse(uri: URI, type: PromptsType, token: CancellationToken): Promise; + /** + * Parses the provided URI + * @param uris + */ + parseNew(uri: URI, token: CancellationToken): Promise; + /** * Returns the prompt file type for the given URI. * @param resource the URI of the resource diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts index 68217698a22..36a75f027a2 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts @@ -30,6 +30,7 @@ import { PositionOffsetTransformer } from '../../../../../../editor/common/core/ import { NewPromptsParser, ParsedPromptFile } from './newPromptsParser.js'; import { IFileService } from '../../../../../../platform/files/common/files.js'; import { ResourceMap } from '../../../../../../base/common/map.js'; +import { CancellationError } from '../../../../../../base/common/errors.js'; /** * Provides prompt services. @@ -326,12 +327,15 @@ export class PromptsService extends Disposable implements IPromptsService { } } - public async parseNew(uri: URI): Promise { + public async parseNew(uri: URI, token: CancellationToken): Promise { const model = this.modelService.getModel(uri); if (model) { return this.getParsedPromptFile(model); } const fileContent = await this.fileService.readFile(uri); + if (token.isCancellationRequested) { + throw new CancellationError(); + } return new NewPromptsParser().parse(uri, fileContent.value.toString()); } } diff --git a/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/newPromptsParser.test.ts b/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/newPromptsParser.test.ts index 62ee58be278..2353b60db80 100644 --- a/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/newPromptsParser.test.ts +++ b/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/newPromptsParser.test.ts @@ -42,8 +42,8 @@ suite('NewPromptsParser', () => { ]); assert.deepEqual(result.body.range, { startLineNumber: 6, startColumn: 1, endLineNumber: 8, endColumn: 1 }); assert.deepEqual(result.body.fileReferences, [ - { range: new Range(7, 39, 7, 54), content: './reference1.md' }, - { range: new Range(7, 80, 7, 95), content: './reference2.md' } + { range: new Range(7, 39, 7, 54), content: './reference1.md', isMarkdownLink: false }, + { range: new Range(7, 80, 7, 95), content: './reference2.md', isMarkdownLink: true } ]); assert.deepEqual(result.body.variableReferences, [ { range: new Range(7, 12, 7, 17), content: 'tool1' } @@ -74,7 +74,7 @@ suite('NewPromptsParser', () => { ]); assert.deepEqual(result.body.range, { startLineNumber: 5, startColumn: 1, endLineNumber: 6, endColumn: 1 }); assert.deepEqual(result.body.fileReferences, [ - { range: new Range(5, 64, 5, 103), content: 'https://mycomp/guidelines#typescript.md' }, + { range: new Range(5, 64, 5, 103), content: 'https://mycomp/guidelines#typescript.md', isMarkdownLink: true }, ]); assert.deepEqual(result.body.variableReferences, []); assert.deepEqual(result.header.description, 'Code style instructions for TypeScript'); @@ -111,7 +111,7 @@ suite('NewPromptsParser', () => { ]); assert.deepEqual(result.body.range, { startLineNumber: 7, startColumn: 1, endLineNumber: 8, endColumn: 1 }); assert.deepEqual(result.body.fileReferences, [ - { range: new Range(7, 59, 7, 83), content: 'https://example.com/docs' }, + { range: new Range(7, 59, 7, 83), content: 'https://example.com/docs', isMarkdownLink: true }, ]); assert.deepEqual(result.body.variableReferences, [ { range: new Range(7, 41, 7, 47), content: 'search' } From 58f906bc23d12b4b8df371c2a2e4fbc3c5f15e33 Mon Sep 17 00:00:00 2001 From: Martin Aeschlimann Date: Thu, 11 Sep 2025 17:09:01 +0200 Subject: [PATCH 018/299] update --- src/vs/workbench/contrib/chat/test/common/mockPromptsService.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/vs/workbench/contrib/chat/test/common/mockPromptsService.ts b/src/vs/workbench/contrib/chat/test/common/mockPromptsService.ts index 6728e53f6a6..93709eefffc 100644 --- a/src/vs/workbench/contrib/chat/test/common/mockPromptsService.ts +++ b/src/vs/workbench/contrib/chat/test/common/mockPromptsService.ts @@ -35,6 +35,7 @@ export class MockPromptsService implements IPromptsService { resolvePromptSlashCommand(_data: any, _token: CancellationToken): Promise { throw new Error('Not implemented'); } findPromptSlashCommands(): Promise { throw new Error('Not implemented'); } parse(_uri: URI, _type: any, _token: CancellationToken): Promise { throw new Error('Not implemented'); } + parseNew(_uri: URI, _token: CancellationToken): Promise { throw new Error('Not implemented'); } getPromptFileType(_resource: URI): any { return undefined; } getParsedPromptFile(textModel: ITextModel): ParsedPromptFile { throw new Error('Not implemented'); } dispose(): void { } From 2a101129a39587b597187a5ae1dc6d6aadd5b2d9 Mon Sep 17 00:00:00 2001 From: Martin Aeschlimann Date: Thu, 11 Sep 2025 17:26:07 +0200 Subject: [PATCH 019/299] fix tests --- .../chat/common/promptSyntax/service/promptsServiceImpl.ts | 1 - .../test/common/promptSyntax/service/promptsService.test.ts | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts index 36a75f027a2..779960cf9f9 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts @@ -320,7 +320,6 @@ export class PromptsService extends Disposable implements IPromptsService { metadata: parser.metadata, variableReferences, fileReferences: parser.references.map(ref => ref.uri), - header: undefined }; } finally { parser?.dispose(); diff --git a/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/promptsService.test.ts b/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/promptsService.test.ts index 69ea302d960..e2a7e64aa27 100644 --- a/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/promptsService.test.ts +++ b/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/promptsService.test.ts @@ -42,6 +42,7 @@ import { testWorkspace } from '../../../../../../../platform/workspace/test/comm import { IUserDataProfileService } from '../../../../../../services/userDataProfile/common/userDataProfile.js'; import { ITelemetryService } from '../../../../../../../platform/telemetry/common/telemetry.js'; import { NullTelemetryService } from '../../../../../../../platform/telemetry/common/telemetryUtils.js'; +import { Event } from '../../../../../../../base/common/event.js'; /** * Helper class to assert the properties of a link. @@ -139,7 +140,7 @@ suite('PromptsService', () => { const fileService = disposables.add(instaService.createInstance(FileService)); instaService.stub(IFileService, fileService); - instaService.stub(IModelService, { getModel() { return null; } }); + instaService.stub(IModelService, { getModel() { return null; }, onModelRemoved: Event.None }); instaService.stub(ILanguageService, { guessLanguageIdByFilepathOrFirstLine(uri: URI) { if (uri.path.endsWith(PROMPT_FILE_EXTENSION)) { @@ -699,7 +700,6 @@ suite('PromptsService', () => { const result1 = await service.parse(rootFileUri, PromptsType.prompt, CancellationToken.None); assert.deepEqual(result1, { uri: rootFileUri, - header: undefined, metadata: { promptType: PromptsType.prompt, description: 'Root prompt description.', From 795cd6297edf1a16ed9875408e0dd53f491a4a9f Mon Sep 17 00:00:00 2001 From: Martin Aeschlimann Date: Thu, 11 Sep 2025 17:38:28 +0200 Subject: [PATCH 020/299] revalidate on tool/mode or model change --- .../promptSyntax/service/promptValidator.ts | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptValidator.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptValidator.ts index 8f6604b33fd..3158901db24 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptValidator.ts +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptValidator.ts @@ -305,6 +305,9 @@ export class PromptValidatorContribution extends Disposable { @IConfigurationService private configService: IConfigurationService, @IMarkerService private readonly markerService: IMarkerService, @IPromptsService private readonly promptsService: IPromptsService, + @ILanguageModelsService private readonly languageModelsService: ILanguageModelsService, + @ILanguageModelToolsService private readonly languageModelToolsService: ILanguageModelToolsService, + @IChatModeService private readonly chatModeService: IChatModeService, ) { super(); this.validator = instantiationService.createInstance(PromptValidator); @@ -326,12 +329,18 @@ export class PromptValidatorContribution extends Disposable { this.localDisposables.add(toDisposable(() => { trackers.forEach(tracker => tracker.dispose()); })); - this.modelService.getModels().forEach(model => { - const promptType = getPromptsTypeForLanguageId(model.getLanguageId()); - if (promptType) { - trackers.set(model.uri, new ModelTracker(model, promptType, this.validator, this.promptsService, this.markerService)); - } - }); + + const validateAllDelayer = this._register(new Delayer(200)); + const validateAll = (): void => { + validateAllDelayer.trigger(async () => { + this.modelService.getModels().forEach(model => { + const promptType = getPromptsTypeForLanguageId(model.getLanguageId()); + if (promptType) { + trackers.set(model.uri, new ModelTracker(model, promptType, this.validator, this.promptsService, this.markerService)); + } + }); + }); + }; this.localDisposables.add(this.modelService.onModelAdded((model) => { const promptType = getPromptsTypeForLanguageId(model.getLanguageId()); if (promptType) { @@ -360,6 +369,10 @@ export class PromptValidatorContribution extends Disposable { trackers.set(model.uri, new ModelTracker(model, promptType, this.validator, this.promptsService, this.markerService)); } })); + this.localDisposables.add(this.languageModelToolsService.onDidChangeTools(() => validateAll())); + this.localDisposables.add(this.chatModeService.onDidChangeChatModes(() => validateAll())); + this.localDisposables.add(this.languageModelsService.onDidChangeLanguageModels(() => validateAll())); + validateAll(); } } From 9cfa06cf36cdb2c21b12ff7f621a2b686a6e88f0 Mon Sep 17 00:00:00 2001 From: Martin Aeschlimann Date: Thu, 11 Sep 2025 17:59:27 +0200 Subject: [PATCH 021/299] adopt parseNew --- .../contrib/chat/browser/chatWidget.ts | 27 ++++++-------- .../computeAutomaticInstructions.ts | 35 +++++++++---------- .../promptSyntax/service/newPromptsParser.ts | 18 +++++----- .../promptSyntax/service/promptValidator.ts | 4 +-- .../promptSyntax/service/promptsService.ts | 2 +- .../service/promptsServiceImpl.ts | 4 +-- .../service/newPromptsParser.test.ts | 6 ++-- 7 files changed, 43 insertions(+), 53 deletions(-) diff --git a/src/vs/workbench/contrib/chat/browser/chatWidget.ts b/src/vs/workbench/contrib/chat/browser/chatWidget.ts index 652a7929c43..d83b1b8ce7b 100644 --- a/src/vs/workbench/contrib/chat/browser/chatWidget.ts +++ b/src/vs/workbench/contrib/chat/browser/chatWidget.ts @@ -64,9 +64,8 @@ import { ChatAgentLocation, ChatConfiguration, ChatModeKind, TodoListWidgetPosit import { ILanguageModelToolsService, IToolData, ToolSet } from '../common/languageModelToolsService.js'; import { ComputeAutomaticInstructions } from '../common/promptSyntax/computeAutomaticInstructions.js'; import { PromptsConfig } from '../common/promptSyntax/config/config.js'; -import { type TPromptMetadata } from '../common/promptSyntax/parsers/promptHeader/promptHeader.js'; import { PromptsType } from '../common/promptSyntax/promptTypes.js'; -import { IPromptParserResult, IPromptsService } from '../common/promptSyntax/service/promptsService.js'; +import { IPromptsService } from '../common/promptSyntax/service/promptsService.js'; import { handleModeSwitch } from './actions/chatActions.js'; import { ChatTreeItem, IChatAcceptInputOptions, IChatAccessibilityService, IChatCodeBlockInfo, IChatFileTreeInfo, IChatListItemRendererOptions, IChatWidget, IChatWidgetService, IChatWidgetViewContext, IChatWidgetViewOptions, ChatViewId } from './chat.js'; import { ChatAccessibilityProvider } from './chatAccessibilityProvider.js'; @@ -83,6 +82,7 @@ import { ChatViewPane } from './chatViewPane.js'; import { IViewsService } from '../../../services/views/common/viewsService.js'; import { ICommandService } from '../../../../platform/commands/common/commands.js'; import { IHoverService } from '../../../../platform/hover/browser/hover.js'; +import { ParsedPromptFile, PromptHeader } from '../common/promptSyntax/service/newPromptsParser.js'; const $ = dom.$; @@ -2145,13 +2145,13 @@ export class ChatWidget extends Disposable implements IChatWidget { return undefined; } - private async _applyPromptFileIfSet(requestInput: IChatRequestInputOptions): Promise { + private async _applyPromptFileIfSet(requestInput: IChatRequestInputOptions): Promise { if (!PromptsConfig.enabled(this.configurationService)) { // if prompts are not enabled, we don't need to do anything return undefined; } - let parseResult: IPromptParserResult | undefined; + let parseResult: ParsedPromptFile | undefined; // first check if the input has a prompt slash command const agentSlashPromptPart = this.parsedInput.parts.find((r): r is ChatRequestSlashPromptPart => r instanceof ChatRequestSlashPromptPart); @@ -2159,7 +2159,7 @@ export class ChatWidget extends Disposable implements IChatWidget { parseResult = await this.promptsService.resolvePromptSlashCommand(agentSlashPromptPart.slashPromptCommand, CancellationToken.None); if (parseResult) { // add the prompt file to the context, but not sticky - const toolReferences = this.toolsService.toToolReferences(parseResult.variableReferences); + const toolReferences = this.toolsService.toToolReferences([]); // TODO: this.toolsService.toToolReferences(parseResult.body?.variableReferences ?? []); requestInput.attachedContext.insertFirst(toPromptFileVariableEntry(parseResult.uri, PromptFileVariableKind.PromptFile, undefined, true, toolReferences)); // remove the slash command from the input @@ -2170,7 +2170,7 @@ export class ChatWidget extends Disposable implements IChatWidget { const uri = this._findPromptFileInContext(requestInput.attachedContext); if (uri) { try { - parseResult = await this.promptsService.parse(uri, PromptsType.prompt, CancellationToken.None); + parseResult = await this.promptsService.parseNew(uri, CancellationToken.None); } catch (error) { this.logService.error(`[_applyPromptFileIfSet] Failed to parse prompt file: ${uri}`, error); } @@ -2180,10 +2180,6 @@ export class ChatWidget extends Disposable implements IChatWidget { if (!parseResult) { return undefined; } - const meta = parseResult.metadata; - if (meta?.promptType !== PromptsType.prompt) { - return undefined; - } const input = requestInput.input.trim(); requestInput.input = `Follow instructions in [${basename(parseResult.uri)}](${parseResult.uri.toString()}).`; @@ -2191,10 +2187,9 @@ export class ChatWidget extends Disposable implements IChatWidget { // if the input is not empty, append it to the prompt requestInput.input += `\n${input}`; } - - await this._applyPromptMetadata(meta, requestInput); - - return parseResult; + if (parseResult.header) { + await this._applyPromptMetadata(parseResult.header, requestInput); + } } private async _acceptInput(query: { query: string } | undefined, options?: IChatAcceptInputOptions): Promise { @@ -2533,9 +2528,7 @@ export class ChatWidget extends Disposable implements IChatWidget { this.agentInInput.set(!!currentAgent); } - private async _applyPromptMetadata(metadata: TPromptMetadata, requestInput: IChatRequestInputOptions): Promise { - - const { mode, tools, model } = metadata; + private async _applyPromptMetadata({ mode, tools, model }: PromptHeader, requestInput: IChatRequestInputOptions): Promise { const currentMode = this.input.currentModeObs.get(); diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/computeAutomaticInstructions.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/computeAutomaticInstructions.ts index 51129a130f1..9aa457af4be 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/computeAutomaticInstructions.ts +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/computeAutomaticInstructions.ts @@ -21,7 +21,8 @@ import { IToolData } from '../languageModelToolsService.js'; import { PromptsConfig } from './config/config.js'; import { COPILOT_CUSTOM_INSTRUCTIONS_FILENAME, isPromptOrInstructionsFile } from './config/promptFileLocations.js'; import { PromptsType } from './promptTypes.js'; -import { IPromptParserResult, IPromptPath, IPromptsService } from './service/promptsService.js'; +import { ParsedPromptFile } from './service/newPromptsParser.js'; +import { IPromptPath, IPromptsService } from './service/promptsService.js'; export type InstructionsCollectionEvent = { applyingInstructionsCount: number; @@ -46,7 +47,7 @@ type InstructionsCollectionClassification = { export class ComputeAutomaticInstructions { - private _parseResults: ResourceMap = new ResourceMap(); + private _parseResults: ResourceMap = new ResourceMap(); constructor( private readonly _readFileTool: IToolData | undefined, @@ -60,12 +61,12 @@ export class ComputeAutomaticInstructions { ) { } - private async _parseInstructionsFile(uri: URI, token: CancellationToken): Promise { + private async _parseInstructionsFile(uri: URI, token: CancellationToken): Promise { if (this._parseResults.has(uri)) { return this._parseResults.get(uri)!; } try { - const result = await this._promptsService.parse(uri, PromptsType.instructions, token); + const result = await this._promptsService.parseNew(uri, token); this._parseResults.set(uri, result); return result; } catch (error) { @@ -125,11 +126,7 @@ export class ComputeAutomaticInstructions { continue; } - if (parsedFile.metadata?.promptType !== PromptsType.instructions) { - this._logService.trace(`[InstructionsContextComputer] Not an instruction file: ${uri}`); - continue; - } - const applyTo = parsedFile.metadata.applyTo; + const applyTo = parsedFile.header?.applyTo; if (!applyTo) { this._logService.trace(`[InstructionsContextComputer] No 'applyTo' found: ${uri}`); @@ -262,12 +259,11 @@ export class ComputeAutomaticInstructions { const entries: string[] = []; for (const { uri } of instructionFiles) { const parsedFile = await this._parseInstructionsFile(uri, token); - if (parsedFile?.metadata?.promptType !== PromptsType.instructions) { - continue; + if (parsedFile) { + const applyTo = parsedFile.header?.applyTo ?? '**/*'; + const description = parsedFile.header?.description ?? ''; + entries.push(`| '${getFilePath(uri)}' | ${applyTo} | ${description} |`); } - const applyTo = parsedFile.metadata.applyTo ?? '**/*'; - const description = parsedFile.metadata.description ?? ''; - entries.push(`| '${getFilePath(uri)}' | ${applyTo} | ${description} |`); } if (entries.length === 0) { return entries; @@ -299,13 +295,14 @@ export class ComputeAutomaticInstructions { let next = todo.pop(); while (next) { const result = await this._parseInstructionsFile(next, token); - if (result) { + if (result && result.body) { const refsToCheck: { resource: URI }[] = []; - for (const ref of result.fileReferences) { - if (!seen.has(ref) && (isPromptOrInstructionsFile(ref) || this._workspaceService.getWorkspaceFolder(ref) !== undefined)) { + for (const ref of result.body.fileReferences) { + const url = result.body.resolveFilePath(ref.content); + if (url && !seen.has(url) && (isPromptOrInstructionsFile(url) || this._workspaceService.getWorkspaceFolder(url) !== undefined)) { // only add references that are either prompt or instruction files or are part of the workspace - refsToCheck.push({ resource: ref }); - seen.add(ref); + refsToCheck.push({ resource: url }); + seen.add(url); } } if (refsToCheck.length > 0) { diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/service/newPromptsParser.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/service/newPromptsParser.ts index bf35d5da09e..51d6dfdfcf4 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/service/newPromptsParser.ts +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/service/newPromptsParser.ts @@ -149,24 +149,24 @@ export class PromptHeader { return this.getStringAttribute('applyTo'); } - public get tools(): Map | undefined { + public get tools(): string[] | undefined { const toolsAttribute = this.getParsedHeader().attributes.find(attr => attr.key === 'tools'); if (!toolsAttribute) { return undefined; } if (toolsAttribute.value.type === 'array') { - const tools = new Map; + const tools: string[] = []; for (const item of toolsAttribute.value.items) { if (item.type === 'string') { - tools.set(item.value, true); + tools.push(item.value); } } return tools; } else if (toolsAttribute.value.type === 'object') { - const tools = new Map; + const tools: string[] = []; const collectLeafs = ({ key, value }: { key: IStringValue; value: IValue }) => { if (value.type === 'boolean') { - tools.set(key.value, value.value); + tools.push(key.value); } else if (value.type === 'object') { value.properties.forEach(collectLeafs); } @@ -258,7 +258,7 @@ export class PromptBody { const contentStartOffset = match.index + 1; // after the # const contentEndOffset = match.index + match[0].length; const range = new Range(i + 1, contentStartOffset + 1, i + 1, contentEndOffset + 1); - variableReferences.push({ content: match[2], range }); + variableReferences.push({ name: match[2], range }); } } } @@ -282,14 +282,14 @@ export class PromptBody { } } -interface IBodyFileReference { +export interface IBodyFileReference { content: string; range: Range; isMarkdownLink: boolean; } -interface IBodyVariableReference { - content: string; +export interface IBodyVariableReference { + name: string; range: Range; } diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptValidator.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptValidator.ts index 3158901db24..511e1a1e28a 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptValidator.ts +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptValidator.ts @@ -71,8 +71,8 @@ export class PromptValidator { if (body.variableReferences.length) { const available = this.getAvailableToolAndToolSetNames(); for (const variable of body.variableReferences) { - if (!available.has(variable.content)) { - report(toMarker(localize('promptValidator.unknownVariableReference', "Unknown tool or toolset '{0}'.", variable.content), variable.range, MarkerSeverity.Warning)); + if (!available.has(variable.name)) { + report(toMarker(localize('promptValidator.unknownVariableReference', "Unknown tool or toolset '{0}'.", variable.name), variable.range, MarkerSeverity.Warning)); } } } diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsService.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsService.ts index 5363d277802..ab3b4887aa4 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsService.ts +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsService.ts @@ -190,7 +190,7 @@ export interface IPromptsService extends IDisposable { /** * Gets the prompt file for a slash command. */ - resolvePromptSlashCommand(data: IChatPromptSlashCommand, _token: CancellationToken): Promise; + resolvePromptSlashCommand(data: IChatPromptSlashCommand, _token: CancellationToken): Promise; /** * Returns a prompt command if the command name is valid. diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts index 779960cf9f9..df8191d491b 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts @@ -195,13 +195,13 @@ export class PromptsService extends Disposable implements IPromptsService { return undefined; } - public async resolvePromptSlashCommand(data: IChatPromptSlashCommand, token: CancellationToken): Promise { + public async resolvePromptSlashCommand(data: IChatPromptSlashCommand, token: CancellationToken): Promise { const promptUri = await this.getPromptPath(data); if (!promptUri) { return undefined; } try { - return await this.parse(promptUri, PromptsType.prompt, token); + return await this.parseNew(promptUri, token); } catch (error) { this.logger.error(`[resolvePromptSlashCommand] Failed to parse prompt file: ${promptUri}`, error); return undefined; diff --git a/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/newPromptsParser.test.ts b/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/newPromptsParser.test.ts index 2353b60db80..93b4d30b495 100644 --- a/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/newPromptsParser.test.ts +++ b/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/newPromptsParser.test.ts @@ -51,7 +51,7 @@ suite('NewPromptsParser', () => { assert.deepEqual(result.header.description, 'Agent mode test'); assert.deepEqual(result.header.model, 'GPT 4.1'); assert.ok(result.header.tools); - assert.deepEqual([...result.header.tools.entries()], [['tool1', true], ['tool2', true]]); + assert.deepEqual(result.header.tools, ['tool1', 'tool2']); }); test('instructions', async () => { @@ -120,7 +120,7 @@ suite('NewPromptsParser', () => { assert.deepEqual(result.header.mode, 'agent'); assert.deepEqual(result.header.model, 'GPT 4.1'); assert.ok(result.header.tools); - assert.deepEqual([...result.header.tools.entries()], [['search', true], ['terminal', true]]); + assert.deepEqual(result.header.tools, ['search', 'terminal']); }); test('prompt file tools as map', async () => { @@ -190,6 +190,6 @@ suite('NewPromptsParser', () => { assert.deepEqual(result.header.mode, undefined); assert.deepEqual(result.header.model, undefined); assert.ok(result.header.tools); - assert.deepEqual([...result.header.tools.entries()], [['built-in', true], ['browser-click', true], ['openPullRequest', true], ['copilotCodingAgent', false]]); + assert.deepEqual(result.header.tools, ['built-in', 'browser-click', 'openPullRequest', 'copilotCodingAgent']); }); }); From e11146a36965ff44ff3e37f20686af8dd041fb05 Mon Sep 17 00:00:00 2001 From: Martin Aeschlimann Date: Thu, 11 Sep 2025 18:02:58 +0200 Subject: [PATCH 022/299] fix tests --- .../test/common/promptSyntax/service/newPromptsParser.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/newPromptsParser.test.ts b/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/newPromptsParser.test.ts index 93b4d30b495..22409da076b 100644 --- a/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/newPromptsParser.test.ts +++ b/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/newPromptsParser.test.ts @@ -46,7 +46,7 @@ suite('NewPromptsParser', () => { { range: new Range(7, 80, 7, 95), content: './reference2.md', isMarkdownLink: true } ]); assert.deepEqual(result.body.variableReferences, [ - { range: new Range(7, 12, 7, 17), content: 'tool1' } + { range: new Range(7, 12, 7, 17), name: 'tool1' } ]); assert.deepEqual(result.header.description, 'Agent mode test'); assert.deepEqual(result.header.model, 'GPT 4.1'); @@ -114,7 +114,7 @@ suite('NewPromptsParser', () => { { range: new Range(7, 59, 7, 83), content: 'https://example.com/docs', isMarkdownLink: true }, ]); assert.deepEqual(result.body.variableReferences, [ - { range: new Range(7, 41, 7, 47), content: 'search' } + { range: new Range(7, 41, 7, 47), name: 'search' } ]); assert.deepEqual(result.header.description, 'General purpose coding assistant'); assert.deepEqual(result.header.mode, 'agent'); From a7bf33c840f6cfd9afcc736b78a4e58f7480287d Mon Sep 17 00:00:00 2001 From: Martin Aeschlimann Date: Thu, 11 Sep 2025 18:07:53 +0200 Subject: [PATCH 023/299] update --- src/vs/workbench/contrib/chat/browser/chatWidget.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/vs/workbench/contrib/chat/browser/chatWidget.ts b/src/vs/workbench/contrib/chat/browser/chatWidget.ts index d83b1b8ce7b..9070071c397 100644 --- a/src/vs/workbench/contrib/chat/browser/chatWidget.ts +++ b/src/vs/workbench/contrib/chat/browser/chatWidget.ts @@ -1339,13 +1339,13 @@ export class ChatWidget extends Disposable implements IChatWidget { if (promptNames.includes(promptCommand.command)) { try { if (promptCommand.promptPath) { - const parseResult = await this.promptsService.parse( + const parseResult = await this.promptsService.parseNew( promptCommand.promptPath.uri, - promptCommand.promptPath.type, CancellationToken.None ); - if (parseResult.metadata?.description) { - this.promptDescriptionsCache.set(promptCommand.command, parseResult.metadata.description); + const description = parseResult.header?.description; + if (description) { + this.promptDescriptionsCache.set(promptCommand.command, description); } else { // Set empty string to indicate we've checked this prompt this.promptDescriptionsCache.set(promptCommand.command, ''); From 8fcd9147be876d78c3ec5c747eba2a48d52a3502 Mon Sep 17 00:00:00 2001 From: Martin Aeschlimann Date: Thu, 11 Sep 2025 22:40:47 +0200 Subject: [PATCH 024/299] adopt promptFileReference.test --- .../contrib/chat/browser/chatWidget.ts | 4 + .../languageProviders/promptLinkProvider.ts | 8 +- .../promptSyntax/promptFileReference.test.ts | 238 ++++++++---------- .../promptSyntax/testUtils/mockFilesystem.ts | 4 +- 4 files changed, 109 insertions(+), 145 deletions(-) diff --git a/src/vs/workbench/contrib/chat/browser/chatWidget.ts b/src/vs/workbench/contrib/chat/browser/chatWidget.ts index 9070071c397..eba8f37399a 100644 --- a/src/vs/workbench/contrib/chat/browser/chatWidget.ts +++ b/src/vs/workbench/contrib/chat/browser/chatWidget.ts @@ -2532,6 +2532,10 @@ export class ChatWidget extends Disposable implements IChatWidget { const currentMode = this.input.currentModeObs.get(); + if (tools !== undefined && !mode && currentMode.kind !== ChatModeKind.Agent) { + mode = ChatModeKind.Agent; + } + // switch to appropriate chat mode if needed if (mode && mode !== currentMode.name) { // Find the mode object to get its kind diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/promptLinkProvider.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/promptLinkProvider.ts index 53e408c38dc..094e4619f21 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/promptLinkProvider.ts +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/promptLinkProvider.ts @@ -33,9 +33,11 @@ export class PromptLinkProvider extends Disposable implements LinkProvider { } const links: ILink[] = []; for (const ref of parser.body.fileReferences) { - const url = parser.body.resolveFilePath(ref.content); - if (url) { - links.push({ range: ref.range, url }); + if (!ref.isMarkdownLink) { + const url = parser.body.resolveFilePath(ref.content); + if (url) { + links.push({ range: ref.range, url }); + } } } return { links }; diff --git a/src/vs/workbench/contrib/chat/test/common/promptSyntax/promptFileReference.test.ts b/src/vs/workbench/contrib/chat/test/common/promptSyntax/promptFileReference.test.ts index c1cfdd00f0b..928bcfc6f81 100644 --- a/src/vs/workbench/contrib/chat/test/common/promptSyntax/promptFileReference.test.ts +++ b/src/vs/workbench/contrib/chat/test/common/promptSyntax/promptFileReference.test.ts @@ -4,11 +4,8 @@ *--------------------------------------------------------------------------------------------*/ import assert from 'assert'; -import { timeout } from '../../../../../../base/common/async.js'; import { Disposable } from '../../../../../../base/common/lifecycle.js'; -import { Schemas } from '../../../../../../base/common/network.js'; import { URI } from '../../../../../../base/common/uri.js'; -import { randomBoolean } from '../../../../../../base/test/common/testUtils.js'; import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../../../base/test/common/utils.js'; import { Range } from '../../../../../../editor/common/core/range.js'; import { ILanguageService } from '../../../../../../editor/common/languages/language.js'; @@ -23,14 +20,10 @@ import { TestInstantiationService } from '../../../../../../platform/instantiati import { ILogService, NullLogService } from '../../../../../../platform/log/common/log.js'; import { NullPolicyService } from '../../../../../../platform/policy/common/policy.js'; import { ChatModeKind } from '../../../common/constants.js'; -import { MarkdownLink } from '../../../common/promptSyntax/codecs/base/markdownCodec/tokens/markdownLink.js'; -import { FileReference } from '../../../common/promptSyntax/codecs/tokens/fileReference.js'; import { getPromptFileType } from '../../../common/promptSyntax/config/promptFileLocations.js'; import { PromptsType } from '../../../common/promptSyntax/promptTypes.js'; -import { type TErrorCondition } from '../../../common/promptSyntax/parsers/basePromptParser.js'; -import { FilePromptParser } from '../../../common/promptSyntax/parsers/filePromptParser.js'; -import { type TPromptReference } from '../../../common/promptSyntax/parsers/types.js'; import { IMockFolder, MockFilesystem } from './testUtils/mockFilesystem.js'; +import { IBodyFileReference, NewPromptsParser } from '../../../common/promptSyntax/service/newPromptsParser.js'; /** * Represents a file reference with an expected @@ -44,19 +37,18 @@ class ExpectedReference { constructor( dirname: URI, - public readonly linkToken: FileReference | MarkdownLink, - public readonly errorCondition?: TErrorCondition, + public readonly ref: IBodyFileReference, ) { - this.uri = (linkToken.path.startsWith('/')) - ? URI.file(linkToken.path) - : URI.joinPath(dirname, linkToken.path); + this.uri = (ref.content.startsWith('/')) + ? URI.file(ref.content) + : URI.joinPath(dirname, ref.content); } /** * Range of the underlying file reference token. */ public get range(): Range { - return this.linkToken.range; + return this.ref.range; } /** @@ -67,6 +59,10 @@ class ExpectedReference { } } +function toUri(filePath: string): URI { + return URI.parse('testFs://' + filePath); +} + /** * A reusable test utility to test the `PromptFileReference` class. */ @@ -82,80 +78,32 @@ class TestPromptFileReference extends Disposable { // create in-memory file system const fileSystemProvider = this._register(new InMemoryFileSystemProvider()); - this._register(this.fileService.registerProvider(Schemas.file, fileSystemProvider)); + this._register(this.fileService.registerProvider('testFs', fileSystemProvider)); } /** * Run the test. */ - public async run( - ): Promise { + public async run(): Promise { // create the files structure on the disk - await (this.instantiationService.createInstance(MockFilesystem, this.fileStructure)).mock(); + await (this.instantiationService.createInstance(MockFilesystem, this.fileStructure)).mock(toUri('/')); - // randomly test with and without delay to ensure that the file - // reference resolution is not susceptible to race conditions - if (randomBoolean()) { - await timeout(5); - } + const content = await this.fileService.readFile(this.rootFileUri); - // start resolving references for the specified root file - const rootReference = this._register( - this.instantiationService.createInstance( - FilePromptParser, - this.rootFileUri, - { allowNonPromptFiles: true, languageId: undefined, updateOnChange: true }, - ), - ).start(); - - // wait until entire prompts tree is resolved - await rootReference.settled(); + const ast = new NewPromptsParser().parse(this.rootFileUri, content.value.toString()); + assert(ast.body, 'Prompt file must have a body'); // resolve the root file reference including all nested references - const resolvedReferences: readonly (TPromptReference | undefined)[] = rootReference.references; + const resolvedReferences = ast.body.fileReferences ?? []; for (let i = 0; i < this.expectedReferences.length; i++) { const expectedReference = this.expectedReferences[i]; const resolvedReference = resolvedReferences[i]; - if (expectedReference.linkToken instanceof MarkdownLink) { - assert( - resolvedReference?.subtype === 'markdown', - [ - `Expected ${i}th resolved reference to be a markdown link`, - `got '${resolvedReference}'.`, - ].join(', '), - ); - } - - if (expectedReference.linkToken instanceof FileReference) { - assert( - resolvedReference?.subtype === 'prompt', - [ - `Expected ${i}th resolved reference to be a #file: link`, - `got '${resolvedReference}'.`, - ].join(', '), - ); - } - - assert( - (resolvedReference) && - (resolvedReference.uri.toString() === expectedReference.uri.toString()), - [ - `Expected ${i}th resolved reference URI to be '${expectedReference.uri}'`, - `got '${resolvedReference?.uri}'.`, - ].join(', '), - ); - - assert( - (resolvedReference) && - (resolvedReference.range.equalsRange(expectedReference.range)), - [ - `Expected ${i}th resolved reference range to be '${expectedReference.range}'`, - `got '${resolvedReference?.range}'.`, - ].join(', '), - ); + const resolvedUri = ast.body.resolveFilePath(resolvedReference.content); + assert.equal(resolvedUri?.fsPath, expectedReference.uri.fsPath); + assert.deepStrictEqual(resolvedReference.range, expectedReference.range); } assert.strictEqual( @@ -167,7 +115,16 @@ class TestPromptFileReference extends Disposable { ].join('\n'), ); - return rootReference; + const result: any = {}; + result.promptType = getPromptFileType(this.rootFileUri); + if (ast.header) { + for (const key of ['tools', 'model', 'mode', 'applyTo', 'description'] as const) { + if (ast.header[key]) { + result[key] = ast.header[key]; + } + } + } + return result; } } @@ -180,19 +137,34 @@ class TestPromptFileReference extends Disposable { * @param lineNumber The expected line number of the file reference. * @param startColumnNumber The expected start column number of the file reference. */ -function createTestFileReference( - filePath: string, - lineNumber: number, - startColumnNumber: number, -): FileReference { +function createFileReference(filePath: string, lineNumber: number, startColumnNumber: number): IBodyFileReference { const range = new Range( lineNumber, - startColumnNumber, + startColumnNumber + '#file:'.length, lineNumber, - startColumnNumber + `#file:${filePath}`.length, + startColumnNumber + '#file:'.length + filePath.length, ); - return new FileReference(range, filePath); + return { + range, + content: filePath, + isMarkdownLink: false, + }; +} + +function createMarkdownReference(lineNumber: number, startColumnNumber: number, firstSeg: string, secondSeg: string): IBodyFileReference { + const range = new Range( + lineNumber, + startColumnNumber + firstSeg.length + 1, + lineNumber, + startColumnNumber + firstSeg.length + secondSeg.length - 1, + ); + + return { + range, + content: secondSeg.substring(1, secondSeg.length - 1), + isMarkdownLink: true, + }; } suite('PromptFileReference', function () { @@ -225,7 +197,7 @@ suite('PromptFileReference', function () { test('resolves nested file references', async function () { const rootFolderName = 'resolves-nested-file-references'; const rootFolder = `/${rootFolderName}`; - const rootUri = URI.file(rootFolder); + const rootUri = toUri(rootFolder); const test = testDisposables.add(instantiationService.createInstance(TestPromptFileReference, /** @@ -282,18 +254,18 @@ suite('PromptFileReference', function () { /** * The root file path to start the resolve process from. */ - URI.file(`/${rootFolderName}/file2.prompt.md`), + toUri(`/${rootFolderName}/file2.prompt.md`), /** * The expected references to be resolved. */ [ new ExpectedReference( rootUri, - createTestFileReference('folder1/file3.prompt.md', 2, 14), + createFileReference('folder1/file3.prompt.md', 2, 14), ), new ExpectedReference( rootUri, - new MarkdownLink( + createMarkdownReference( 3, 14, '[file4.prompt.md]', '(./folder1/some-other-folder/file4.prompt.md)', ), @@ -309,7 +281,7 @@ suite('PromptFileReference', function () { test('tools', async function () { const rootFolderName = 'resolves-nested-file-references'; const rootFolder = `/${rootFolderName}`; - const rootUri = URI.file(rootFolder); + const rootUri = toUri(rootFolder); const test = testDisposables.add(instantiationService.createInstance(TestPromptFileReference, /** @@ -405,18 +377,18 @@ suite('PromptFileReference', function () { /** * The root file path to start the resolve process from. */ - URI.file(`/${rootFolderName}/file2.prompt.md`), + toUri(`/${rootFolderName}/file2.prompt.md`), /** * The expected references to be resolved. */ [ new ExpectedReference( rootUri, - createTestFileReference('folder1/file3.prompt.md', 7, 14), + createFileReference('folder1/file3.prompt.md', 7, 14), ), new ExpectedReference( rootUri, - new MarkdownLink( + createMarkdownReference( 8, 14, '[file4.prompt.md]', '(./folder1/some-other-folder/file4.prompt.md)', ), @@ -424,9 +396,7 @@ suite('PromptFileReference', function () { ] )); - const rootReference = await test.run(); - - const { metadata } = rootReference; + const metadata = await test.run(); assert.deepStrictEqual( metadata, @@ -445,7 +415,7 @@ suite('PromptFileReference', function () { test('prompt language', async function () { const rootFolderName = 'resolves-nested-file-references'; const rootFolder = `/${rootFolderName}`; - const rootUri = URI.file(rootFolder); + const rootUri = toUri(rootFolder); const test = testDisposables.add(instantiationService.createInstance(TestPromptFileReference, /** @@ -513,18 +483,18 @@ suite('PromptFileReference', function () { /** * The root file path to start the resolve process from. */ - URI.file(`/${rootFolderName}/file2.prompt.md`), + toUri(`/${rootFolderName}/file2.prompt.md`), /** * The expected references to be resolved. */ [ new ExpectedReference( rootUri, - createTestFileReference('folder1/file3.prompt.md', 7, 14), + createFileReference('folder1/file3.prompt.md', 7, 14), ), new ExpectedReference( rootUri, - new MarkdownLink( + createMarkdownReference( 8, 14, '[file4.prompt.md]', '(./folder1/some-other-folder/file4.prompt.md)', ), @@ -532,17 +502,15 @@ suite('PromptFileReference', function () { ] )); - const rootReference = await test.run(); - - const { metadata } = rootReference; + const metadata = await test.run(); assert.deepStrictEqual( metadata, { promptType: PromptsType.prompt, - mode: ChatModeKind.Agent, description: 'Description of my prompt.', tools: ['my-tool12'], + applyTo: '**/*', }, 'Must have correct metadata.', ); @@ -553,7 +521,7 @@ suite('PromptFileReference', function () { test('instructions language', async function () { const rootFolderName = 'resolves-nested-file-references'; const rootFolder = `/${rootFolderName}`; - const rootUri = URI.file(rootFolder); + const rootUri = toUri(rootFolder); const test = testDisposables.add(instantiationService.createInstance(TestPromptFileReference, /** @@ -621,18 +589,18 @@ suite('PromptFileReference', function () { /** * The root file path to start the resolve process from. */ - URI.file(`/${rootFolderName}/file2.instructions.md`), + toUri(`/${rootFolderName}/file2.instructions.md`), /** * The expected references to be resolved. */ [ new ExpectedReference( rootUri, - createTestFileReference('folder1/file3.prompt.md', 7, 14), + createFileReference('folder1/file3.prompt.md', 7, 14), ), new ExpectedReference( rootUri, - new MarkdownLink( + createMarkdownReference( 8, 14, '[file4.prompt.md]', '(./folder1/some-other-folder/file4.prompt.md)', ), @@ -640,9 +608,7 @@ suite('PromptFileReference', function () { ] )); - const rootReference = await test.run(); - - const { metadata } = rootReference; + const metadata = await test.run(); assert.deepStrictEqual( metadata, @@ -650,6 +616,7 @@ suite('PromptFileReference', function () { promptType: PromptsType.instructions, applyTo: '**/*', description: 'Description of my instructions file.', + tools: ['my-tool12'], }, 'Must have correct metadata.', ); @@ -657,10 +624,10 @@ suite('PromptFileReference', function () { }); suite('tools and mode compatibility', () => { - test('tools are ignored if root prompt is in the ask mode', async function () { + test('ask mode', async function () { const rootFolderName = 'resolves-nested-file-references'; const rootFolder = `/${rootFolderName}`; - const rootUri = URI.file(rootFolder); + const rootUri = toUri(rootFolder); const test = testDisposables.add(instantiationService.createInstance(TestPromptFileReference, /** @@ -728,18 +695,18 @@ suite('PromptFileReference', function () { /** * The root file path to start the resolve process from. */ - URI.file(`/${rootFolderName}/file2.prompt.md`), + toUri(`/${rootFolderName}/file2.prompt.md`), /** * The expected references to be resolved. */ [ new ExpectedReference( rootUri, - createTestFileReference('folder1/file3.prompt.md', 6, 14), + createFileReference('folder1/file3.prompt.md', 6, 14), ), new ExpectedReference( rootUri, - new MarkdownLink( + createMarkdownReference( 7, 14, '[file4.prompt.md]', '(./folder1/some-other-folder/file4.prompt.md)', ), @@ -747,9 +714,7 @@ suite('PromptFileReference', function () { ] )); - const rootReference = await test.run(); - - const { metadata } = rootReference; + const metadata = await test.run(); assert.deepStrictEqual( metadata, @@ -762,10 +727,10 @@ suite('PromptFileReference', function () { ); }); - test('tools are ignored if root prompt is in the edit mode', async function () { + test('edit mode', async function () { const rootFolderName = 'resolves-nested-file-references'; const rootFolder = `/${rootFolderName}`; - const rootUri = URI.file(rootFolder); + const rootUri = toUri(rootFolder); const test = testDisposables.add(instantiationService.createInstance(TestPromptFileReference, /** @@ -832,18 +797,18 @@ suite('PromptFileReference', function () { /** * The root file path to start the resolve process from. */ - URI.file(`/${rootFolderName}/file2.prompt.md`), + toUri(`/${rootFolderName}/file2.prompt.md`), /** * The expected references to be resolved. */ [ new ExpectedReference( rootUri, - createTestFileReference('folder1/file3.prompt.md', 6, 14), + createFileReference('folder1/file3.prompt.md', 6, 14), ), new ExpectedReference( rootUri, - new MarkdownLink( + createMarkdownReference( 7, 14, '[file4.prompt.md]', '(./folder1/some-other-folder/file4.prompt.md)', ), @@ -851,9 +816,7 @@ suite('PromptFileReference', function () { ] )); - const rootReference = await test.run(); - - const { metadata } = rootReference; + const metadata = await test.run(); assert.deepStrictEqual( metadata, @@ -867,10 +830,10 @@ suite('PromptFileReference', function () { }); - test('tools are not ignored if root prompt is in the agent mode', async function () { + test('agent mode', async function () { const rootFolderName = 'resolves-nested-file-references'; const rootFolder = `/${rootFolderName}`; - const rootUri = URI.file(rootFolder); + const rootUri = toUri(rootFolder); const test = testDisposables.add(instantiationService.createInstance(TestPromptFileReference, /** @@ -937,18 +900,18 @@ suite('PromptFileReference', function () { /** * The root file path to start the resolve process from. */ - URI.file(`/${rootFolderName}/file2.prompt.md`), + toUri(`/${rootFolderName}/file2.prompt.md`), /** * The expected references to be resolved. */ [ new ExpectedReference( rootUri, - createTestFileReference('folder1/file3.prompt.md', 6, 14), + createFileReference('folder1/file3.prompt.md', 6, 14), ), new ExpectedReference( rootUri, - new MarkdownLink( + createMarkdownReference( 7, 14, '[file4.prompt.md]', '(./folder1/some-other-folder/file4.prompt.md)', ), @@ -956,9 +919,7 @@ suite('PromptFileReference', function () { ] )); - const rootReference = await test.run(); - - const { metadata } = rootReference; + const metadata = await test.run(); assert.deepStrictEqual( metadata, @@ -972,10 +933,10 @@ suite('PromptFileReference', function () { }); - test('tools are not ignored if root prompt implicitly in the agent mode', async function () { + test('no mode', async function () { const rootFolderName = 'resolves-nested-file-references'; const rootFolder = `/${rootFolderName}`; - const rootUri = URI.file(rootFolder); + const rootUri = toUri(rootFolder); const test = testDisposables.add(instantiationService.createInstance(TestPromptFileReference, /** @@ -1042,18 +1003,18 @@ suite('PromptFileReference', function () { /** * The root file path to start the resolve process from. */ - URI.file(`/${rootFolderName}/file2.prompt.md`), + toUri(`/${rootFolderName}/file2.prompt.md`), /** * The expected references to be resolved. */ [ new ExpectedReference( rootUri, - createTestFileReference('folder1/file3.prompt.md', 6, 14), + createFileReference('folder1/file3.prompt.md', 6, 14), ), new ExpectedReference( rootUri, - new MarkdownLink( + createMarkdownReference( 7, 14, '[file4.prompt.md]', '(./folder1/some-other-folder/file4.prompt.md)', ), @@ -1061,15 +1022,12 @@ suite('PromptFileReference', function () { ] )); - const rootReference = await test.run(); - - const { metadata, } = rootReference; + const metadata = await test.run(); assert.deepStrictEqual( metadata, { promptType: PromptsType.prompt, - mode: ChatModeKind.Agent, tools: ['my-tool12'], description: 'Description of the prompt file.', }, diff --git a/src/vs/workbench/contrib/chat/test/common/promptSyntax/testUtils/mockFilesystem.ts b/src/vs/workbench/contrib/chat/test/common/promptSyntax/testUtils/mockFilesystem.ts index dc0a44da5c7..d08b9685ae8 100644 --- a/src/vs/workbench/contrib/chat/test/common/promptSyntax/testUtils/mockFilesystem.ts +++ b/src/vs/workbench/contrib/chat/test/common/promptSyntax/testUtils/mockFilesystem.ts @@ -47,11 +47,11 @@ export class MockFilesystem { /** * Starts the mock process. */ - public async mock(): Promise[]> { + public async mock(parentFolder?: URI): Promise[]> { const result = await Promise.all( this.folders .map((folder) => { - return this.mockFolder(folder); + return this.mockFolder(folder, parentFolder); }), ); From bf29996207b43bc12aa91fe0102c9dd9a2a53749 Mon Sep 17 00:00:00 2001 From: Martin Aeschlimann Date: Fri, 12 Sep 2025 12:37:11 +0200 Subject: [PATCH 025/299] adopt promptService.test --- .../promptSyntax/service/newPromptsParser.ts | 6 +- .../promptSyntax/service/promptValidator.ts | 2 +- .../service/promptsService.test.ts | 105 ++++++++---------- 3 files changed, 50 insertions(+), 63 deletions(-) diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/service/newPromptsParser.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/service/newPromptsParser.ts index 51d6dfdfcf4..fb0112acae5 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/service/newPromptsParser.ts +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/service/newPromptsParser.ts @@ -157,7 +157,7 @@ export class PromptHeader { if (toolsAttribute.value.type === 'array') { const tools: string[] = []; for (const item of toolsAttribute.value.items) { - if (item.type === 'string') { + if (item.type === 'string' && item.value) { tools.push(item.value); } } @@ -231,7 +231,7 @@ export class PromptBody { const variableReferences: IBodyVariableReference[] = []; for (let i = this.range.startLineNumber - 1; i < this.range.endLineNumber - 1; i++) { const line = this.linesWithEOL[i]; - const linkMatch = line.matchAll(/\[(.+?)\]\((.+?)\)/g); + const linkMatch = line.matchAll(/\[(.*?)\]\((.+?)\)/g); for (const match of linkMatch) { const linkEndOffset = match.index + match[0].length - 1; // before the parenthesis const linkStartOffset = match.index + match[0].length - match[2].length - 1; @@ -239,7 +239,7 @@ export class PromptBody { fileReferences.push({ content: match[2], range, isMarkdownLink: true }); markdownLinkRanges.push(new Range(i + 1, match.index + 1, i + 1, match.index + match[0].length + 1)); } - const reg = new RegExp(`${chatVariableLeader}([\\w]+:)?([^\\s#]*)`, 'g'); + const reg = new RegExp(`${chatVariableLeader}([\\w]+:)?([^\\s#]+)`, 'g'); const matches = line.matchAll(reg); for (const match of matches) { const fullRange = new Range(i + 1, match.index + 1, i + 1, match.index + match[0].length + 1); diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptValidator.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptValidator.ts index 511e1a1e28a..ead5fc60b43 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptValidator.ts +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptValidator.ts @@ -227,7 +227,7 @@ export class PromptValidator { for (const item of attribute.value.items) { if (item.type !== 'string') { report(toMarker(localize('promptValidator.eachToolMustBeString', "Each tool name in the 'tools' attribute must be a string."), item.range, MarkerSeverity.Error)); - } else if (!available.has(item.value)) { + } else if (item.value && !available.has(item.value)) { report(toMarker(localize('promptValidator.toolNotFound', "Unknown tool '{0}'.", item.value), item.range, MarkerSeverity.Warning)); } } diff --git a/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/promptsService.test.ts b/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/promptsService.test.ts index e2a7e64aa27..0c453e00b46 100644 --- a/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/promptsService.test.ts +++ b/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/promptsService.test.ts @@ -27,7 +27,7 @@ import { INSTRUCTIONS_LANGUAGE_ID, PROMPT_LANGUAGE_ID, PromptsType } from '../.. import { TextModelPromptParser } from '../../../../common/promptSyntax/parsers/textModelPromptParser.js'; import { IPromptFileReference } from '../../../../common/promptSyntax/parsers/types.js'; import { PromptsService } from '../../../../common/promptSyntax/service/promptsServiceImpl.js'; -import { ICustomChatMode, IPromptParserResult, IPromptsService } from '../../../../common/promptSyntax/service/promptsService.js'; +import { ICustomChatMode, IPromptsService } from '../../../../common/promptSyntax/service/promptsService.js'; import { MockFilesystem } from '../testUtils/mockFilesystem.js'; import { ILabelService } from '../../../../../../../platform/label/common/label.js'; import { ComputeAutomaticInstructions, newInstructionsCollectionEvent } from '../../../../common/promptSyntax/computeAutomaticInstructions.js'; @@ -697,70 +697,57 @@ suite('PromptsService', () => { const yetAnotherFile = URI.joinPath(rootFolderUri, 'folder1/some-other-folder/yetAnotherFolder🤭/another-file.instructions.md'); - const result1 = await service.parse(rootFileUri, PromptsType.prompt, CancellationToken.None); - assert.deepEqual(result1, { - uri: rootFileUri, - metadata: { - promptType: PromptsType.prompt, - description: 'Root prompt description.', - tools: ['my-tool1'], - mode: 'agent', - }, - fileReferences: [file3, file4], - variableReferences: [ - { - name: "my-other-tool", - range: { - endExclusive: 265, - start: 251 - } - }, - { - name: "my-tool", - range: { - start: 239, - endExclusive: 247, - } - }], - } satisfies IPromptParserResult); + const result1 = await service.parseNew(rootFileUri, CancellationToken.None); + assert.deepEqual(result1.uri, rootFileUri); + assert.deepEqual(result1.header?.description, 'Root prompt description.'); + assert.deepEqual(result1.header?.tools, ['my-tool1']); + assert.deepEqual(result1.header?.mode, 'agent'); + assert.ok(result1.body); + assert.deepEqual( + result1.body.fileReferences.map(r => result1.body?.resolveFilePath(r.content)), + [file3, file4], + ); + assert.deepEqual( + result1.body.variableReferences, + [ + { name: "my-tool", range: new Range(10, 5, 10, 12) }, + { name: "my-other-tool", range: new Range(11, 5, 11, 18) }, + ] + ); - const result2 = await service.parse(file3, PromptsType.prompt, CancellationToken.None); - assert.deepEqual(result2, { - uri: file3, - metadata: { - promptType: PromptsType.prompt, - mode: 'edit', - }, - fileReferences: [nonExistingFolder, yetAnotherFile], - variableReferences: [] - } satisfies IPromptParserResult); + const result2 = await service.parseNew(file3, CancellationToken.None); + assert.deepEqual(result2.uri, file3); + assert.deepEqual(result2.header?.mode, 'edit'); + assert.ok(result2.body); + assert.deepEqual( + result2.body.fileReferences.map(r => result2.body?.resolveFilePath(r.content)), + [nonExistingFolder, yetAnotherFile], + ); - const result3 = await service.parse(yetAnotherFile, PromptsType.instructions, CancellationToken.None); - assert.deepEqual(result3, { - uri: yetAnotherFile, - metadata: { - promptType: PromptsType.instructions, - description: 'Another file description.', - applyTo: '**/*.tsx', - }, - fileReferences: [someOtherFolder, someOtherFolderFile], - variableReferences: [] - } satisfies IPromptParserResult); + const result3 = await service.parseNew(yetAnotherFile, CancellationToken.None); + assert.deepEqual(result3.uri, yetAnotherFile); + assert.deepEqual(result3.header?.description, 'Another file description.'); + assert.deepEqual(result3.header?.applyTo, '**/*.tsx'); + assert.ok(result3.body); + assert.deepEqual( + result3.body.fileReferences.map(r => result3.body?.resolveFilePath(r.content)), + [someOtherFolder, someOtherFolderFile], + ); + assert.deepEqual(result3.body.variableReferences, []); - const result4 = await service.parse(file4, PromptsType.instructions, CancellationToken.None); - assert.deepEqual(result4, { - uri: file4, - metadata: { - promptType: PromptsType.instructions, - description: 'File 4 splendid description.', - }, - fileReferences: [ + const result4 = await service.parseNew(file4, CancellationToken.None); + assert.deepEqual(result4.uri, file4); + assert.deepEqual(result4.header?.description, 'File 4 splendid description.'); + assert.ok(result4.body); + assert.deepEqual( + result4.body.fileReferences.map(r => result4.body?.resolveFilePath(r.content)), + [ URI.joinPath(rootFolderUri, '/folder1/some-other-folder/some-non-existing/file.prompt.md'), URI.joinPath(rootFolderUri, '/folder1/some-other-folder/some-non-prompt-file.md'), - URI.joinPath(rootFolderUri, '/folder1/'), + URI.joinPath(rootFolderUri, '/folder1'), ], - variableReferences: [] - } satisfies IPromptParserResult); + ); + assert.deepEqual(result4.body.variableReferences, []); }); }); From a650fa7a1468782714372c0f0f2477f0dfc0442b Mon Sep 17 00:00:00 2001 From: Martin Aeschlimann Date: Fri, 12 Sep 2025 14:26:19 +0200 Subject: [PATCH 026/299] adopt computeCustomChatModes, IVariableReferences --- .../contrib/chat/browser/chatWidget.ts | 6 +- .../promptSyntax/service/newPromptsParser.ts | 31 ++++++---- .../service/promptsServiceImpl.ts | 56 ++++++++----------- .../service/newPromptsParser.test.ts | 12 +++- .../service/promptsService.test.ts | 9 +-- 5 files changed, 60 insertions(+), 54 deletions(-) diff --git a/src/vs/workbench/contrib/chat/browser/chatWidget.ts b/src/vs/workbench/contrib/chat/browser/chatWidget.ts index e864ff2dc3a..66279d3061a 100644 --- a/src/vs/workbench/contrib/chat/browser/chatWidget.ts +++ b/src/vs/workbench/contrib/chat/browser/chatWidget.ts @@ -83,6 +83,7 @@ import { IViewsService } from '../../../services/views/common/viewsService.js'; import { ICommandService } from '../../../../platform/commands/common/commands.js'; import { IHoverService } from '../../../../platform/hover/browser/hover.js'; import { ParsedPromptFile, PromptHeader } from '../common/promptSyntax/service/newPromptsParser.js'; +import { OffsetRange } from '../../../../editor/common/core/ranges/offsetRange.js'; const $ = dom.$; @@ -2157,8 +2158,9 @@ export class ChatWidget extends Disposable implements IChatWidget { if (agentSlashPromptPart) { parseResult = await this.promptsService.resolvePromptSlashCommand(agentSlashPromptPart.slashPromptCommand, CancellationToken.None); if (parseResult) { - // add the prompt file to the context, but not sticky - const toolReferences = this.toolsService.toToolReferences([]); // TODO: this.toolsService.toToolReferences(parseResult.body?.variableReferences ?? []); + // add the prompt file to the context + const refs = parseResult.body?.variableReferences.map(({ name, offset }) => ({ name, range: new OffsetRange(offset, offset + name.length + 1) })) ?? []; + const toolReferences = this.toolsService.toToolReferences(refs); requestInput.attachedContext.insertFirst(toPromptFileVariableEntry(parseResult.uri, PromptFileVariableKind.PromptFile, undefined, true, toolReferences)); // remove the slash command from the input diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/service/newPromptsParser.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/service/newPromptsParser.ts index fb0112acae5..fd08120c4ce 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/service/newPromptsParser.ts +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/service/newPromptsParser.ts @@ -3,6 +3,7 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ +import { Iterable } from '../../../../../../base/common/iterator.js'; import { dirname, resolvePath } from '../../../../../../base/common/resources.js'; import { splitLinesIncludeSeparators } from '../../../../../../base/common/strings.js'; import { URI } from '../../../../../../base/common/uri.js'; @@ -208,6 +209,7 @@ export type IValue = IStringValue | INumberValue | IBooleanValue | IArrayValue | interface ParsedBody { readonly fileReferences: readonly IBodyFileReference[]; readonly variableReferences: readonly IBodyVariableReference[]; + readonly bodyOffset: number; } export class PromptBody { @@ -224,12 +226,17 @@ export class PromptBody { return this.getParsedBody().variableReferences; } + public get offset(): number { + return this.getParsedBody().bodyOffset; + } + private getParsedBody(): ParsedBody { if (this._parsed === undefined) { const markdownLinkRanges: Range[] = []; const fileReferences: IBodyFileReference[] = []; const variableReferences: IBodyVariableReference[] = []; - for (let i = this.range.startLineNumber - 1; i < this.range.endLineNumber - 1; i++) { + const bodyOffset = Iterable.reduce(Iterable.slice(this.linesWithEOL, 0, this.range.startLineNumber - 1), (len, line) => line.length + len, 0); + for (let i = this.range.startLineNumber - 1, lineStartOffset = bodyOffset; i < this.range.endLineNumber - 1; i++) { const line = this.linesWithEOL[i]; const linkMatch = line.matchAll(/\[(.*?)\]\((.+?)\)/g); for (const match of linkMatch) { @@ -258,15 +265,20 @@ export class PromptBody { const contentStartOffset = match.index + 1; // after the # const contentEndOffset = match.index + match[0].length; const range = new Range(i + 1, contentStartOffset + 1, i + 1, contentEndOffset + 1); - variableReferences.push({ name: match[2], range }); + variableReferences.push({ name: match[2], range, offset: lineStartOffset + match.index }); } } + lineStartOffset += line.length; } - this._parsed = { fileReferences: fileReferences.sort((a, b) => Range.compareRangesUsingStarts(a.range, b.range)), variableReferences }; + this._parsed = { fileReferences: fileReferences.sort((a, b) => Range.compareRangesUsingStarts(a.range, b.range)), variableReferences, bodyOffset }; } return this._parsed; } + public getContent(): string { + return this.linesWithEOL.slice(this.range.startLineNumber - 1, this.range.endLineNumber - 1).join(''); + } + public resolveFilePath(path: string): URI | undefined { try { if (path.startsWith('/')) { @@ -283,14 +295,13 @@ export class PromptBody { } export interface IBodyFileReference { - content: string; - range: Range; - isMarkdownLink: boolean; + readonly content: string; + readonly range: Range; + readonly isMarkdownLink: boolean; } export interface IBodyVariableReference { - name: string; - range: Range; + readonly name: string; + readonly range: Range; + readonly offset: number; } - - diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts index df8191d491b..07579a4d003 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { localize } from '../../../../../../nls.js'; -import { getLanguageIdForPromptsType, getPromptsTypeForLanguageId, MODE_LANGUAGE_ID, PROMPT_LANGUAGE_ID, PromptsType } from '../promptTypes.js'; +import { getLanguageIdForPromptsType, getPromptsTypeForLanguageId, PROMPT_LANGUAGE_ID, PromptsType } from '../promptTypes.js'; import { PromptParser } from '../parsers/promptParser.js'; import { type URI } from '../../../../../../base/common/uri.js'; import { assert } from '../../../../../../base/common/assert.js'; @@ -31,6 +31,8 @@ import { NewPromptsParser, ParsedPromptFile } from './newPromptsParser.js'; import { IFileService } from '../../../../../../platform/files/common/files.js'; import { ResourceMap } from '../../../../../../base/common/map.js'; import { CancellationError } from '../../../../../../base/common/errors.js'; +import { OffsetRange } from '../../../../../../editor/common/core/ranges/offsetRange.js'; +import { IVariableReference } from '../../chatModes.js'; /** * Provides prompt services. @@ -255,42 +257,28 @@ export class PromptsService extends Disposable implements IPromptsService { const metadataList = await Promise.all( modeFiles.map(async ({ uri }): Promise => { - let parser: PromptParser | undefined; - try { - // Note! this can be (and should be) improved by using shared parser instances - // that the `getSyntaxParserFor` method provides for opened documents. - parser = this.instantiationService.createInstance( - PromptParser, - uri, - { allowNonPromptFiles: true, languageId: MODE_LANGUAGE_ID, updateOnChange: false }, - ).start(token); + const ast = await this.parseNew(uri, token); - const completed = await parser.settled(); - if (!completed) { - throw new Error(localize('promptParser.notCompleted', "Prompt parser for {0} did not complete.", uri.toString())); + const variableReferences: IVariableReference[] = []; + let body = ''; + if (ast.body) { + const bodyOffset = ast.body.offset; + const bodyVarRefs = ast.body.variableReferences; + for (let i = bodyVarRefs.length - 1; i >= 0; i--) { // in reverse order + const { name, offset } = bodyVarRefs[i]; + const range = new OffsetRange(offset - bodyOffset, offset - bodyOffset + name.length + 1); + variableReferences.push({ name, range }); } - - const body = await parser.getBody(); - const nHeaderLines = parser.header?.range.endLineNumber ?? 0; - const transformer = new PositionOffsetTransformer(body); - const variableReferences = parser.variableReferences.map(ref => { - return { - name: ref.name, - range: transformer.getOffsetRange(ref.range.delta(-nHeaderLines)) - }; - }).sort((a, b) => b.range.start - a.range.start); // in reverse order - - const name = getCleanPromptName(uri); - - const metadata = parser.metadata; - if (metadata?.promptType !== PromptsType.mode) { - return { uri, name, body, variableReferences }; - } - const { description, model, tools } = metadata; - return { uri, name, description, model, tools, body, variableReferences }; - } finally { - parser?.dispose(); + body = ast.body.getContent(); } + + const name = getCleanPromptName(uri); + if (!ast.header) { + return { uri, name, body, variableReferences }; + } + const { description, model, tools } = ast.header; + return { uri, name, description, model, tools, body, variableReferences }; + }) ); diff --git a/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/newPromptsParser.test.ts b/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/newPromptsParser.test.ts index 22409da076b..89dac4ac4db 100644 --- a/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/newPromptsParser.test.ts +++ b/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/newPromptsParser.test.ts @@ -41,12 +41,15 @@ suite('NewPromptsParser', () => { }, ]); assert.deepEqual(result.body.range, { startLineNumber: 6, startColumn: 1, endLineNumber: 8, endColumn: 1 }); + assert.equal(result.body.offset, 80); + assert.equal(result.body.getContent(), 'This is a chat mode test.\nHere is a #tool1 variable and a #file:./reference1.md as well as a [reference](./reference2.md).'); + assert.deepEqual(result.body.fileReferences, [ { range: new Range(7, 39, 7, 54), content: './reference1.md', isMarkdownLink: false }, { range: new Range(7, 80, 7, 95), content: './reference2.md', isMarkdownLink: true } ]); assert.deepEqual(result.body.variableReferences, [ - { range: new Range(7, 12, 7, 17), name: 'tool1' } + { range: new Range(7, 12, 7, 17), name: 'tool1', offset: 116 } ]); assert.deepEqual(result.header.description, 'Agent mode test'); assert.deepEqual(result.header.model, 'GPT 4.1'); @@ -73,6 +76,9 @@ suite('NewPromptsParser', () => { { key: 'applyTo', range: new Range(3, 1, 3, 14), value: { type: 'string', value: '*.ts', range: new Range(3, 10, 3, 14) } }, ]); assert.deepEqual(result.body.range, { startLineNumber: 5, startColumn: 1, endLineNumber: 6, endColumn: 1 }); + assert.equal(result.body.offset, 76); + assert.equal(result.body.getContent(), 'Follow my companies coding guidlines at [mycomp-ts-guidelines](https://mycomp/guidelines#typescript.md)'); + assert.deepEqual(result.body.fileReferences, [ { range: new Range(5, 64, 5, 103), content: 'https://mycomp/guidelines#typescript.md', isMarkdownLink: true }, ]); @@ -110,11 +116,13 @@ suite('NewPromptsParser', () => { }, ]); assert.deepEqual(result.body.range, { startLineNumber: 7, startColumn: 1, endLineNumber: 8, endColumn: 1 }); + assert.equal(result.body.offset, 113); + assert.equal(result.body.getContent(), 'This is a prompt file body referencing #search and [docs](https://example.com/docs).'); assert.deepEqual(result.body.fileReferences, [ { range: new Range(7, 59, 7, 83), content: 'https://example.com/docs', isMarkdownLink: true }, ]); assert.deepEqual(result.body.variableReferences, [ - { range: new Range(7, 41, 7, 47), name: 'search' } + { range: new Range(7, 41, 7, 47), name: 'search', offset: 152 } ]); assert.deepEqual(result.header.description, 'General purpose coding assistant'); assert.deepEqual(result.header.mode, 'agent'); diff --git a/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/promptsService.test.ts b/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/promptsService.test.ts index 0c453e00b46..21f60a3aae0 100644 --- a/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/promptsService.test.ts +++ b/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/promptsService.test.ts @@ -710,8 +710,8 @@ suite('PromptsService', () => { assert.deepEqual( result1.body.variableReferences, [ - { name: "my-tool", range: new Range(10, 5, 10, 12) }, - { name: "my-other-tool", range: new Range(11, 5, 11, 18) }, + { name: "my-tool", range: new Range(10, 5, 10, 12), offset: 239 }, + { name: "my-other-tool", range: new Range(11, 5, 11, 18), offset: 251 }, ] ); @@ -1275,18 +1275,15 @@ suite('PromptsService', () => { }, { name: 'mode2', - description: undefined, - tools: undefined, body: 'First use #tool2\nThen use #tool1', variableReferences: [{ name: 'tool1', range: { start: 26, endExclusive: 32 } }, { name: 'tool2', range: { start: 10, endExclusive: 16 } }], - model: undefined, uri: URI.joinPath(rootFolderUri, '.github/chatmodes/mode2.instructions.md'), } ]; assert.deepEqual( - expected, result, + expected, 'Must get custom chat modes.', ); }); From d9832733879901756c13cf6cb2368a4b46f4ef81 Mon Sep 17 00:00:00 2001 From: Martin Aeschlimann Date: Fri, 12 Sep 2025 14:26:19 +0200 Subject: [PATCH 027/299] adopt computeCustomChatModes, IVariableReferences --- .../contrib/chat/browser/chatWidget.ts | 6 +- .../promptSyntax/service/newPromptsParser.ts | 31 ++++++---- .../service/promptsServiceImpl.ts | 56 ++++++++----------- .../service/newPromptsParser.test.ts | 12 +++- .../service/promptsService.test.ts | 9 +-- 5 files changed, 60 insertions(+), 54 deletions(-) diff --git a/src/vs/workbench/contrib/chat/browser/chatWidget.ts b/src/vs/workbench/contrib/chat/browser/chatWidget.ts index e864ff2dc3a..66279d3061a 100644 --- a/src/vs/workbench/contrib/chat/browser/chatWidget.ts +++ b/src/vs/workbench/contrib/chat/browser/chatWidget.ts @@ -83,6 +83,7 @@ import { IViewsService } from '../../../services/views/common/viewsService.js'; import { ICommandService } from '../../../../platform/commands/common/commands.js'; import { IHoverService } from '../../../../platform/hover/browser/hover.js'; import { ParsedPromptFile, PromptHeader } from '../common/promptSyntax/service/newPromptsParser.js'; +import { OffsetRange } from '../../../../editor/common/core/ranges/offsetRange.js'; const $ = dom.$; @@ -2157,8 +2158,9 @@ export class ChatWidget extends Disposable implements IChatWidget { if (agentSlashPromptPart) { parseResult = await this.promptsService.resolvePromptSlashCommand(agentSlashPromptPart.slashPromptCommand, CancellationToken.None); if (parseResult) { - // add the prompt file to the context, but not sticky - const toolReferences = this.toolsService.toToolReferences([]); // TODO: this.toolsService.toToolReferences(parseResult.body?.variableReferences ?? []); + // add the prompt file to the context + const refs = parseResult.body?.variableReferences.map(({ name, offset }) => ({ name, range: new OffsetRange(offset, offset + name.length + 1) })) ?? []; + const toolReferences = this.toolsService.toToolReferences(refs); requestInput.attachedContext.insertFirst(toPromptFileVariableEntry(parseResult.uri, PromptFileVariableKind.PromptFile, undefined, true, toolReferences)); // remove the slash command from the input diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/service/newPromptsParser.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/service/newPromptsParser.ts index fb0112acae5..fd08120c4ce 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/service/newPromptsParser.ts +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/service/newPromptsParser.ts @@ -3,6 +3,7 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ +import { Iterable } from '../../../../../../base/common/iterator.js'; import { dirname, resolvePath } from '../../../../../../base/common/resources.js'; import { splitLinesIncludeSeparators } from '../../../../../../base/common/strings.js'; import { URI } from '../../../../../../base/common/uri.js'; @@ -208,6 +209,7 @@ export type IValue = IStringValue | INumberValue | IBooleanValue | IArrayValue | interface ParsedBody { readonly fileReferences: readonly IBodyFileReference[]; readonly variableReferences: readonly IBodyVariableReference[]; + readonly bodyOffset: number; } export class PromptBody { @@ -224,12 +226,17 @@ export class PromptBody { return this.getParsedBody().variableReferences; } + public get offset(): number { + return this.getParsedBody().bodyOffset; + } + private getParsedBody(): ParsedBody { if (this._parsed === undefined) { const markdownLinkRanges: Range[] = []; const fileReferences: IBodyFileReference[] = []; const variableReferences: IBodyVariableReference[] = []; - for (let i = this.range.startLineNumber - 1; i < this.range.endLineNumber - 1; i++) { + const bodyOffset = Iterable.reduce(Iterable.slice(this.linesWithEOL, 0, this.range.startLineNumber - 1), (len, line) => line.length + len, 0); + for (let i = this.range.startLineNumber - 1, lineStartOffset = bodyOffset; i < this.range.endLineNumber - 1; i++) { const line = this.linesWithEOL[i]; const linkMatch = line.matchAll(/\[(.*?)\]\((.+?)\)/g); for (const match of linkMatch) { @@ -258,15 +265,20 @@ export class PromptBody { const contentStartOffset = match.index + 1; // after the # const contentEndOffset = match.index + match[0].length; const range = new Range(i + 1, contentStartOffset + 1, i + 1, contentEndOffset + 1); - variableReferences.push({ name: match[2], range }); + variableReferences.push({ name: match[2], range, offset: lineStartOffset + match.index }); } } + lineStartOffset += line.length; } - this._parsed = { fileReferences: fileReferences.sort((a, b) => Range.compareRangesUsingStarts(a.range, b.range)), variableReferences }; + this._parsed = { fileReferences: fileReferences.sort((a, b) => Range.compareRangesUsingStarts(a.range, b.range)), variableReferences, bodyOffset }; } return this._parsed; } + public getContent(): string { + return this.linesWithEOL.slice(this.range.startLineNumber - 1, this.range.endLineNumber - 1).join(''); + } + public resolveFilePath(path: string): URI | undefined { try { if (path.startsWith('/')) { @@ -283,14 +295,13 @@ export class PromptBody { } export interface IBodyFileReference { - content: string; - range: Range; - isMarkdownLink: boolean; + readonly content: string; + readonly range: Range; + readonly isMarkdownLink: boolean; } export interface IBodyVariableReference { - name: string; - range: Range; + readonly name: string; + readonly range: Range; + readonly offset: number; } - - diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts index df8191d491b..07579a4d003 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { localize } from '../../../../../../nls.js'; -import { getLanguageIdForPromptsType, getPromptsTypeForLanguageId, MODE_LANGUAGE_ID, PROMPT_LANGUAGE_ID, PromptsType } from '../promptTypes.js'; +import { getLanguageIdForPromptsType, getPromptsTypeForLanguageId, PROMPT_LANGUAGE_ID, PromptsType } from '../promptTypes.js'; import { PromptParser } from '../parsers/promptParser.js'; import { type URI } from '../../../../../../base/common/uri.js'; import { assert } from '../../../../../../base/common/assert.js'; @@ -31,6 +31,8 @@ import { NewPromptsParser, ParsedPromptFile } from './newPromptsParser.js'; import { IFileService } from '../../../../../../platform/files/common/files.js'; import { ResourceMap } from '../../../../../../base/common/map.js'; import { CancellationError } from '../../../../../../base/common/errors.js'; +import { OffsetRange } from '../../../../../../editor/common/core/ranges/offsetRange.js'; +import { IVariableReference } from '../../chatModes.js'; /** * Provides prompt services. @@ -255,42 +257,28 @@ export class PromptsService extends Disposable implements IPromptsService { const metadataList = await Promise.all( modeFiles.map(async ({ uri }): Promise => { - let parser: PromptParser | undefined; - try { - // Note! this can be (and should be) improved by using shared parser instances - // that the `getSyntaxParserFor` method provides for opened documents. - parser = this.instantiationService.createInstance( - PromptParser, - uri, - { allowNonPromptFiles: true, languageId: MODE_LANGUAGE_ID, updateOnChange: false }, - ).start(token); + const ast = await this.parseNew(uri, token); - const completed = await parser.settled(); - if (!completed) { - throw new Error(localize('promptParser.notCompleted', "Prompt parser for {0} did not complete.", uri.toString())); + const variableReferences: IVariableReference[] = []; + let body = ''; + if (ast.body) { + const bodyOffset = ast.body.offset; + const bodyVarRefs = ast.body.variableReferences; + for (let i = bodyVarRefs.length - 1; i >= 0; i--) { // in reverse order + const { name, offset } = bodyVarRefs[i]; + const range = new OffsetRange(offset - bodyOffset, offset - bodyOffset + name.length + 1); + variableReferences.push({ name, range }); } - - const body = await parser.getBody(); - const nHeaderLines = parser.header?.range.endLineNumber ?? 0; - const transformer = new PositionOffsetTransformer(body); - const variableReferences = parser.variableReferences.map(ref => { - return { - name: ref.name, - range: transformer.getOffsetRange(ref.range.delta(-nHeaderLines)) - }; - }).sort((a, b) => b.range.start - a.range.start); // in reverse order - - const name = getCleanPromptName(uri); - - const metadata = parser.metadata; - if (metadata?.promptType !== PromptsType.mode) { - return { uri, name, body, variableReferences }; - } - const { description, model, tools } = metadata; - return { uri, name, description, model, tools, body, variableReferences }; - } finally { - parser?.dispose(); + body = ast.body.getContent(); } + + const name = getCleanPromptName(uri); + if (!ast.header) { + return { uri, name, body, variableReferences }; + } + const { description, model, tools } = ast.header; + return { uri, name, description, model, tools, body, variableReferences }; + }) ); diff --git a/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/newPromptsParser.test.ts b/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/newPromptsParser.test.ts index 22409da076b..89dac4ac4db 100644 --- a/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/newPromptsParser.test.ts +++ b/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/newPromptsParser.test.ts @@ -41,12 +41,15 @@ suite('NewPromptsParser', () => { }, ]); assert.deepEqual(result.body.range, { startLineNumber: 6, startColumn: 1, endLineNumber: 8, endColumn: 1 }); + assert.equal(result.body.offset, 80); + assert.equal(result.body.getContent(), 'This is a chat mode test.\nHere is a #tool1 variable and a #file:./reference1.md as well as a [reference](./reference2.md).'); + assert.deepEqual(result.body.fileReferences, [ { range: new Range(7, 39, 7, 54), content: './reference1.md', isMarkdownLink: false }, { range: new Range(7, 80, 7, 95), content: './reference2.md', isMarkdownLink: true } ]); assert.deepEqual(result.body.variableReferences, [ - { range: new Range(7, 12, 7, 17), name: 'tool1' } + { range: new Range(7, 12, 7, 17), name: 'tool1', offset: 116 } ]); assert.deepEqual(result.header.description, 'Agent mode test'); assert.deepEqual(result.header.model, 'GPT 4.1'); @@ -73,6 +76,9 @@ suite('NewPromptsParser', () => { { key: 'applyTo', range: new Range(3, 1, 3, 14), value: { type: 'string', value: '*.ts', range: new Range(3, 10, 3, 14) } }, ]); assert.deepEqual(result.body.range, { startLineNumber: 5, startColumn: 1, endLineNumber: 6, endColumn: 1 }); + assert.equal(result.body.offset, 76); + assert.equal(result.body.getContent(), 'Follow my companies coding guidlines at [mycomp-ts-guidelines](https://mycomp/guidelines#typescript.md)'); + assert.deepEqual(result.body.fileReferences, [ { range: new Range(5, 64, 5, 103), content: 'https://mycomp/guidelines#typescript.md', isMarkdownLink: true }, ]); @@ -110,11 +116,13 @@ suite('NewPromptsParser', () => { }, ]); assert.deepEqual(result.body.range, { startLineNumber: 7, startColumn: 1, endLineNumber: 8, endColumn: 1 }); + assert.equal(result.body.offset, 113); + assert.equal(result.body.getContent(), 'This is a prompt file body referencing #search and [docs](https://example.com/docs).'); assert.deepEqual(result.body.fileReferences, [ { range: new Range(7, 59, 7, 83), content: 'https://example.com/docs', isMarkdownLink: true }, ]); assert.deepEqual(result.body.variableReferences, [ - { range: new Range(7, 41, 7, 47), name: 'search' } + { range: new Range(7, 41, 7, 47), name: 'search', offset: 152 } ]); assert.deepEqual(result.header.description, 'General purpose coding assistant'); assert.deepEqual(result.header.mode, 'agent'); diff --git a/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/promptsService.test.ts b/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/promptsService.test.ts index 0c453e00b46..21f60a3aae0 100644 --- a/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/promptsService.test.ts +++ b/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/promptsService.test.ts @@ -710,8 +710,8 @@ suite('PromptsService', () => { assert.deepEqual( result1.body.variableReferences, [ - { name: "my-tool", range: new Range(10, 5, 10, 12) }, - { name: "my-other-tool", range: new Range(11, 5, 11, 18) }, + { name: "my-tool", range: new Range(10, 5, 10, 12), offset: 239 }, + { name: "my-other-tool", range: new Range(11, 5, 11, 18), offset: 251 }, ] ); @@ -1275,18 +1275,15 @@ suite('PromptsService', () => { }, { name: 'mode2', - description: undefined, - tools: undefined, body: 'First use #tool2\nThen use #tool1', variableReferences: [{ name: 'tool1', range: { start: 26, endExclusive: 32 } }, { name: 'tool2', range: { start: 10, endExclusive: 16 } }], - model: undefined, uri: URI.joinPath(rootFolderUri, '.github/chatmodes/mode2.instructions.md'), } ]; assert.deepEqual( - expected, result, + expected, 'Must get custom chat modes.', ); }); From 609aaecdfb02ab4157004f22a9073dbdca47fd1c Mon Sep 17 00:00:00 2001 From: Martin Aeschlimann Date: Fri, 12 Sep 2025 15:28:13 +0200 Subject: [PATCH 028/299] remove old prompt parser --- src/vs/base/common/numbers.ts | 59 - src/vs/base/test/common/numbers.test.ts | 175 +- .../promptSyntax/codecs/base/asyncDecoder.ts | 93 - .../promptSyntax/codecs/base/baseDecoder.ts | 361 --- .../promptSyntax/codecs/base/baseToken.ts | 150 -- .../codecs/base/compositeToken.ts | 63 - .../codecs/base/frontMatterCodec/constants.ts | 16 - .../frontMatterCodec/frontMatterDecoder.ts | 156 -- .../parsers/frontMatterArray.ts | 197 -- .../parsers/frontMatterParserFactory.ts | 41 - .../frontMatterRecord/frontMatterRecord.ts | 210 -- .../frontMatterRecordName.ts | 74 - .../frontMatterRecordNameWithDelimiter.ts | 105 - .../parsers/frontMatterSequence.ts | 78 - .../parsers/frontMatterString.ts | 69 - .../parsers/frontMatterValue.ts | 184 -- .../tokens/frontMatterArray.ts | 43 - .../tokens/frontMatterBoolean.ts | 88 - .../tokens/frontMatterRecord.ts | 118 - .../tokens/frontMatterSequence.ts | 79 - .../tokens/frontMatterString.ts | 39 - .../tokens/frontMatterToken.ts | 33 - .../base/frontMatterCodec/tokens/index.ts | 15 - .../codecs/base/linesCodec/linesDecoder.ts | 237 -- .../base/linesCodec/tokens/carriageReturn.ts | 44 - .../codecs/base/linesCodec/tokens/line.ts | 48 - .../codecs/base/linesCodec/tokens/newLine.ts | 44 - .../base/markdownCodec/markdownDecoder.ts | 136 -- .../markdownCodec/parsers/markdownComment.ts | 173 -- .../markdownCodec/parsers/markdownImage.ts | 99 - .../markdownCodec/parsers/markdownLink.ts | 211 -- .../markdownCodec/tokens/markdownComment.ts | 40 - .../markdownCodec/tokens/markdownImage.ts | 125 -- .../base/markdownCodec/tokens/markdownLink.ts | 120 - .../markdownCodec/tokens/markdownToken.ts | 12 - .../markdownExtensionsDecoder.ts | 119 - .../parsers/frontMatterHeader.ts | 345 --- .../tokens/frontMatterHeader.ts | 79 - .../tokens/frontMatterMarker.ts | 59 - .../tokens/markdownExtensionsToken.ts | 11 - .../codecs/base/simpleCodec/parserBase.ts | 137 -- .../codecs/base/simpleCodec/simpleDecoder.ts | 132 -- .../base/simpleCodec/tokens/angleBrackets.ts | 61 - .../codecs/base/simpleCodec/tokens/at.ts | 31 - .../base/simpleCodec/tokens/brackets.ts | 61 - .../codecs/base/simpleCodec/tokens/colon.ts | 31 - .../codecs/base/simpleCodec/tokens/comma.ts | 31 - .../base/simpleCodec/tokens/curlyBraces.ts | 61 - .../codecs/base/simpleCodec/tokens/dash.ts | 31 - .../base/simpleCodec/tokens/dollarSign.ts | 31 - .../base/simpleCodec/tokens/doubleQuote.ts | 40 - .../simpleCodec/tokens/exclamationMark.ts | 31 - .../base/simpleCodec/tokens/formFeed.ts | 31 - .../codecs/base/simpleCodec/tokens/hash.ts | 31 - .../base/simpleCodec/tokens/parentheses.ts | 61 - .../codecs/base/simpleCodec/tokens/quote.ts | 40 - .../base/simpleCodec/tokens/simpleToken.ts | 59 - .../codecs/base/simpleCodec/tokens/slash.ts | 31 - .../codecs/base/simpleCodec/tokens/space.ts | 31 - .../codecs/base/simpleCodec/tokens/tab.ts | 31 - .../codecs/base/simpleCodec/tokens/tokens.ts | 25 - .../base/simpleCodec/tokens/verticalTab.ts | 31 - .../codecs/base/simpleCodec/tokens/word.ts | 60 - .../promptSyntax/codecs/base/textToken.ts | 19 - .../codecs/base/utils/objectStream.ts | 224 -- .../base/utils/objectStreamFromTextModel.ts | 46 - .../promptSyntax/codecs/chatPromptCodec.ts | 73 - .../promptSyntax/codecs/chatPromptDecoder.ts | 202 -- .../codecs/parsers/promptAtMentionParser.ts | 121 -- .../parsers/promptSlashCommandParser.ts | 122 -- .../parsers/promptTemplateVariableParser.ts | 148 -- .../codecs/parsers/promptVariableParser.ts | 252 --- .../codecs/tokens/fileReference.ts | 50 - .../codecs/tokens/promptAtMention.ts | 52 - .../codecs/tokens/promptSlashCommand.ts | 42 - .../codecs/tokens/promptTemplateVariable.ts | 44 - .../promptSyntax/codecs/tokens/promptToken.ts | 11 - .../codecs/tokens/promptVariable.ts | 103 - .../filePromptContentsProvider.ts | 166 -- .../promptContentsProviderBase.ts | 196 -- .../textModelContentsProvider.ts | 93 - .../promptSyntax/contentProviders/types.ts | 70 - .../decorations/frontMatterDecoration.ts | 120 - .../frontMatterMarkerDecoration.ts | 56 - .../decorations/utils/decorationBase.ts | 127 -- .../utils/reactiveDecorationBase.ts | 162 -- .../decorations/utils/types.ts | 56 - .../promptDecorationsProvider.ts | 205 -- .../decorationsProvider/types.ts | 48 - .../promptHeaderDiagnosticsProvider.ts | 234 -- .../promptLinkDiagnosticsProvider.ts | 98 - .../languageProviders/providerInstanceBase.ts | 54 - .../providerInstanceManagerBase.ts | 176 -- .../promptSyntax/parsers/basePromptParser.ts | 728 ------- .../promptSyntax/parsers/filePromptParser.ts | 37 - .../parsers/promptHeader/diagnostics.ts | 47 - .../parsers/promptHeader/headerBase.ts | 264 --- .../promptHeader/instructionsHeader.ts | 44 - .../parsers/promptHeader/metadata/applyTo.ts | 122 -- .../promptHeader/metadata/base/enum.ts | 84 - .../promptHeader/metadata/base/record.ts | 108 - .../promptHeader/metadata/base/string.ts | 73 - .../promptHeader/metadata/description.ts | 46 - .../parsers/promptHeader/metadata/mode.ts | 47 - .../parsers/promptHeader/metadata/model.ts | 41 - .../parsers/promptHeader/metadata/tools.ts | 182 -- .../parsers/promptHeader/modeHeader.ts | 56 - .../parsers/promptHeader/promptHeader.ts | 103 - .../promptSyntax/parsers/promptParser.ts | 72 - .../parsers/textModelPromptParser.ts | 42 - .../common/promptSyntax/parsers/topError.ts | 102 - .../chat/common/promptSyntax/parsers/types.ts | 116 - .../promptSyntax/service/promptsService.ts | 49 - .../service/promptsServiceImpl.ts | 93 +- .../common/promptSyntax/utils/objectCache.ts | 153 -- .../utils/observableDisposable.ts | 89 - .../frontMatterBoolean.test.ts | 317 --- .../frontMatterDecoder.test.ts | 415 ---- .../frontMatterRecord.test.ts | 183 -- .../frontMatterSequence.test.ts | 106 - .../codecs/base/linesDecoder.test.ts | 256 --- .../codecs/base/markdownDecoder.test.ts | 937 -------- .../codecs/base/simpleDecoder.test.ts | 238 -- .../codecs/base/testUtils/randomRange.ts | 58 - .../codecs/base/testUtils/randomTokens.ts | 146 -- .../codecs/base/tokens/baseToken.test.ts | 516 ----- .../codecs/base/tokens/compositeToken.test.ts | 250 --- .../codecs/base/tokens/simpleToken.test.ts | 66 - .../codecs/base/utils/objectStream.test.ts | 180 -- .../codecs/base/utils/testDecoder.ts | 258 --- .../codecs/chatPromptCodec.test.ts | 136 -- .../codecs/chatPromptDecoder.test.ts | 428 ---- .../codecs/markdownExtensionsDecoder.test.ts | 445 ---- .../codecs/tokens/fileReference.test.ts | 106 - .../codecs/tokens/markdownLink.test.ts | 98 - .../common/promptSyntax/config/config.test.ts | 15 +- .../promptSyntax/config/constants.test.ts | 7 +- .../filePromptContentsProvider.test.ts | 210 -- .../parsers/textModelPromptParser.test.ts | 1928 ----------------- .../service/promptsService.test.ts | 484 ----- .../testUtils/expectedDiagnostic.ts | 90 - .../testUtils/expectedReference.ts | 129 -- .../common/promptSyntax/utils/mock.test.ts | 5 +- .../promptSyntax/utils/objectCache.test.ts | 332 --- .../utils/observableDisposable.test.ts | 368 ---- 145 files changed, 17 insertions(+), 20215 deletions(-) delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/asyncDecoder.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/baseDecoder.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/baseToken.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/compositeToken.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/constants.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/frontMatterDecoder.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterArray.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterParserFactory.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterRecord/frontMatterRecord.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterRecord/frontMatterRecordName.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterRecord/frontMatterRecordNameWithDelimiter.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterSequence.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterString.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterValue.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterArray.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterBoolean.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterRecord.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterSequence.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterString.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterToken.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/index.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/linesCodec/linesDecoder.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/linesCodec/tokens/carriageReturn.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/linesCodec/tokens/line.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/linesCodec/tokens/newLine.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/markdownDecoder.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/parsers/markdownComment.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/parsers/markdownImage.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/parsers/markdownLink.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/tokens/markdownComment.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/tokens/markdownImage.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/tokens/markdownLink.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/tokens/markdownToken.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/markdownExtensionsDecoder.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/parsers/frontMatterHeader.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/tokens/frontMatterHeader.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/tokens/frontMatterMarker.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/tokens/markdownExtensionsToken.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/parserBase.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/simpleDecoder.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/angleBrackets.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/at.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/brackets.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/colon.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/comma.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/curlyBraces.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/dash.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/dollarSign.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/doubleQuote.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/exclamationMark.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/formFeed.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/hash.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/parentheses.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/quote.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/simpleToken.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/slash.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/space.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/tab.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/tokens.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/verticalTab.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/word.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/textToken.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/utils/objectStream.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/utils/objectStreamFromTextModel.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/chatPromptCodec.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/chatPromptDecoder.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/parsers/promptAtMentionParser.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/parsers/promptSlashCommandParser.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/parsers/promptTemplateVariableParser.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/parsers/promptVariableParser.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/fileReference.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptAtMention.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptSlashCommand.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptTemplateVariable.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptToken.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptVariable.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/contentProviders/filePromptContentsProvider.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/contentProviders/promptContentsProviderBase.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/contentProviders/textModelContentsProvider.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/contentProviders/types.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/frontMatterDecoration.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/frontMatterMarkerDecoration.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/utils/decorationBase.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/utils/reactiveDecorationBase.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/utils/types.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/promptDecorationsProvider.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/types.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/promptHeaderDiagnosticsProvider.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/promptLinkDiagnosticsProvider.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/providerInstanceBase.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/providerInstanceManagerBase.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/parsers/basePromptParser.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/parsers/filePromptParser.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/diagnostics.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/headerBase.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/instructionsHeader.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/applyTo.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/base/enum.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/base/record.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/base/string.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/description.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/mode.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/model.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/tools.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/modeHeader.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/promptHeader.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptParser.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/parsers/textModelPromptParser.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/parsers/topError.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/parsers/types.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/utils/objectCache.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/utils/observableDisposable.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/frontMatterDecoder/frontMatterBoolean.test.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/frontMatterDecoder/frontMatterDecoder.test.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/frontMatterDecoder/frontMatterRecord.test.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/frontMatterDecoder/frontMatterSequence.test.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/linesDecoder.test.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/markdownDecoder.test.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/simpleDecoder.test.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/testUtils/randomRange.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/testUtils/randomTokens.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/tokens/baseToken.test.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/tokens/compositeToken.test.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/tokens/simpleToken.test.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/utils/objectStream.test.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/utils/testDecoder.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/chatPromptCodec.test.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/chatPromptDecoder.test.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/markdownExtensionsDecoder.test.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/tokens/fileReference.test.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/tokens/markdownLink.test.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/contentProviders/filePromptContentsProvider.test.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/parsers/textModelPromptParser.test.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/testUtils/expectedDiagnostic.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/testUtils/expectedReference.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/utils/objectCache.test.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/utils/observableDisposable.test.ts diff --git a/src/vs/base/common/numbers.ts b/src/vs/base/common/numbers.ts index 89c9f183e6d..326a9b2f2c3 100644 --- a/src/vs/base/common/numbers.ts +++ b/src/vs/base/common/numbers.ts @@ -99,65 +99,6 @@ export function isPointWithinTriangle( return u >= 0 && v >= 0 && u + v < 1; } -/** - * Function to get a (pseudo)random integer from a provided `max`...[`min`] range. - * Both `min` and `max` values are inclusive. The `min` value is optional (defaults to `0`). - * - * @throws in the next cases: - * - if provided `min` or `max` is not a number - * - if provided `min` or `max` is not finite - * - if provided `min` is larger than `max` value - * - * ## Examples - * - * Specifying a `max` value only uses `0` as the `min` value by default: - * - * ```typescript - * // get a random integer between 0 and 10 - * const randomInt = randomInt(10); - * - * assert( - * randomInt >= 0, - * 'Should be greater than or equal to 0.', - * ); - * - * assert( - * randomInt <= 10, - * 'Should be less than or equal to 10.', - * ); - * ``` - * * Specifying both `max` and `min` values: - * - * ```typescript - * // get a random integer between 5 and 8 - * const randomInt = randomInt(8, 5); - * - * assert( - * randomInt >= 5, - * 'Should be greater than or equal to 5.', - * ); - * - * assert( - * randomInt <= 8, - * 'Should be less than or equal to 8.', - * ); - * ``` - */ -export function randomInt(max: number, min: number = 0): number { - assert(!isNaN(min), '"min" param is not a number.'); - assert(!isNaN(max), '"max" param is not a number.'); - - assert(isFinite(max), '"max" param is not finite.'); - assert(isFinite(min), '"min" param is not finite.'); - - assert(max > min, `"max"(${max}) param should be greater than "min"(${min}).`); - - const delta = max - min; - const randomFloat = delta * Math.random(); - - return Math.round(min + randomFloat); -} - export function randomChance(p: number): boolean { assert(p >= 0 && p <= 1, 'p must be between 0 and 1'); return Math.random() < p; diff --git a/src/vs/base/test/common/numbers.test.ts b/src/vs/base/test/common/numbers.test.ts index 94c07090585..e21844eea98 100644 --- a/src/vs/base/test/common/numbers.test.ts +++ b/src/vs/base/test/common/numbers.test.ts @@ -5,7 +5,7 @@ import assert from 'assert'; import { ensureNoDisposablesAreLeakedInTestSuite } from './utils.js'; -import { isPointWithinTriangle, randomInt } from '../../common/numbers.js'; +import { isPointWithinTriangle } from '../../common/numbers.js'; suite('isPointWithinTriangle', () => { ensureNoDisposablesAreLeakedInTestSuite(); @@ -25,176 +25,3 @@ suite('isPointWithinTriangle', () => { assert.ok(result); }); }); - -suite('randomInt', () => { - ensureNoDisposablesAreLeakedInTestSuite(); - - /** - * Test helper that allows to run a test on the `randomInt()` - * utility with specified `max` and `min` values. - */ - const testRandomIntUtil = (max: number, min: number | undefined, testName: string) => { - suite(testName, () => { - let i = 0; - while (++i < 5) { - test(`should generate random boolean attempt#${i}`, async () => { - let iterations = 100; - while (iterations-- > 0) { - const int = randomInt(max, min); - - assert( - int <= max, - `Expected ${int} to be less than or equal to ${max}.` - ); - assert( - int >= (min ?? 0), - `Expected ${int} to be greater than or equal to ${min ?? 0}.`, - ); - } - }); - } - - test('should include min and max', async () => { - let iterations = 125; - const results = []; - while (iterations-- > 0) { - results.push(randomInt(max, min)); - } - - assert( - results.includes(max), - `Expected ${results} to include ${max}.`, - ); - assert( - results.includes(min ?? 0), - `Expected ${results} to include ${min ?? 0}.`, - ); - }); - }); - }; - - suite('positive numbers', () => { - testRandomIntUtil(4, 2, 'max: 4, min: 2'); - testRandomIntUtil(4, 0, 'max: 4, min: 0'); - testRandomIntUtil(4, undefined, 'max: 4, min: undefined'); - testRandomIntUtil(1, 0, 'max: 0, min: 0'); - }); - - suite('negative numbers', () => { - testRandomIntUtil(-2, -5, 'max: -2, min: -5'); - testRandomIntUtil(0, -5, 'max: 0, min: -5'); - testRandomIntUtil(0, -1, 'max: 0, min: -1'); - }); - - suite('split numbers', () => { - testRandomIntUtil(3, -1, 'max: 3, min: -1'); - testRandomIntUtil(2, -2, 'max: 2, min: -2'); - testRandomIntUtil(1, -3, 'max: 2, min: -2'); - }); - - suite('errors', () => { - test('should throw if "min" is == "max" #1', () => { - assert.throws(() => { - randomInt(200, 200); - }, `"max"(200) param should be greater than "min"(200)."`); - }); - - test('should throw if "min" is == "max" #2', () => { - assert.throws(() => { - randomInt(2, 2); - }, `"max"(2) param should be greater than "min"(2)."`); - }); - - test('should throw if "min" is == "max" #3', () => { - assert.throws(() => { - randomInt(0); - }, `"max"(0) param should be greater than "min"(0)."`); - }); - - test('should throw if "min" is > "max" #1', () => { - assert.throws(() => { - randomInt(2, 3); - }, `"max"(2) param should be greater than "min"(3)."`); - }); - - test('should throw if "min" is > "max" #2', () => { - assert.throws(() => { - randomInt(999, 2000); - }, `"max"(999) param should be greater than "min"(2000)."`); - }); - - test('should throw if "min" is > "max" #3', () => { - assert.throws(() => { - randomInt(0, 1); - }, `"max"(0) param should be greater than "min"(1)."`); - }); - - test('should throw if "min" is > "max" #4', () => { - assert.throws(() => { - randomInt(-5, 2); - }, `"max"(-5) param should be greater than "min"(2)."`); - }); - - test('should throw if "min" is > "max" #5', () => { - assert.throws(() => { - randomInt(-4, 0); - }, `"max"(-4) param should be greater than "min"(0)."`); - }); - - test('should throw if "min" is > "max" #6', () => { - assert.throws(() => { - randomInt(-4); - }, `"max"(-4) param should be greater than "min"(0)."`); - }); - - test('should throw if "max" is `NaN`', () => { - assert.throws(() => { - randomInt(NaN); - }, `"max" param is not a number."`); - }); - - test('should throw if "min" is `NaN`', () => { - assert.throws(() => { - randomInt(4, NaN); - }, `"min" param is not a number."`); - }); - - suite('infinite arguments', () => { - test('should throw if "max" is infinite [Infinity]', () => { - assert.throws(() => { - randomInt(Infinity); - }, `"max" param is not finite."`); - }); - - test('should throw if "max" is infinite [-Infinity]', () => { - assert.throws(() => { - randomInt(-Infinity); - }, `"max" param is not finite."`); - }); - - test('should throw if "max" is infinite [+Infinity]', () => { - assert.throws(() => { - randomInt(+Infinity); - }, `"max" param is not finite."`); - }); - - test('should throw if "min" is infinite [Infinity]', () => { - assert.throws(() => { - randomInt(Infinity, Infinity); - }, `"max" param is not finite."`); - }); - - test('should throw if "min" is infinite [-Infinity]', () => { - assert.throws(() => { - randomInt(Infinity, -Infinity); - }, `"max" param is not finite."`); - }); - - test('should throw if "min" is infinite [+Infinity]', () => { - assert.throws(() => { - randomInt(Infinity, +Infinity); - }, `"max" param is not finite."`); - }); - }); - }); -}); diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/asyncDecoder.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/asyncDecoder.ts deleted file mode 100644 index be10b50ebba..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/asyncDecoder.ts +++ /dev/null @@ -1,93 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { Disposable } from '../../../../../../../base/common/lifecycle.js'; -import { BaseDecoder } from './baseDecoder.js'; - -/** - * Asynchronous iterator wrapper for a decoder. - */ -export class AsyncDecoder, K extends NonNullable = NonNullable> extends Disposable { - // Buffer of messages that have been decoded but not yet consumed. - private readonly messages: T[] = []; - - /** - * A transient promise that is resolved when a new event - * is received. Used in the situation when there is no new - * data available and decoder stream did not finish yet, - * hence we need to wait until new event is received. - */ - private resolveOnNewEvent?: (value: void) => void; - - /** - * @param decoder The decoder instance to wrap. - * - * Note! Assumes ownership of the `decoder` object, hence will `dispose` - * it when the decoder stream is ended. - */ - constructor( - private readonly decoder: BaseDecoder, - ) { - super(); - - this._register(decoder); - } - - /** - * Async iterator implementation. - */ - async *[Symbol.asyncIterator](): AsyncIterator { - // callback is called when `data` or `end` event is received - const callback = (data?: T) => { - if (data !== undefined) { - this.messages.push(data); - } else { - this.decoder.removeListener('data', callback); - this.decoder.removeListener('end', callback); - } - - // is the promise resolve callback is present, - // then call it and remove the reference - if (this.resolveOnNewEvent) { - this.resolveOnNewEvent(); - delete this.resolveOnNewEvent; - } - }; - - /** - * !NOTE! The order of event subscriptions below is critical here because - * the `data` event is also starts the stream, hence changing - * the order of event subscriptions can lead to race conditions. - * See {@link ReadableStreamEvents} for more info. - */ - - this.decoder.on('end', callback); - this.decoder.on('data', callback); - - // start flowing the decoder stream - this.decoder.start(); - - while (true) { - const maybeMessage = this.messages.shift(); - if (maybeMessage !== undefined) { - yield maybeMessage; - continue; - } - - // if no data available and stream ended, we're done - if (this.decoder.ended) { - this.dispose(); - - return null; - } - - // stream isn't ended so wait for the new - // `data` or `end` event to be received - await new Promise((resolve) => { - this.resolveOnNewEvent = resolve; - }); - } - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/baseDecoder.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/baseDecoder.ts deleted file mode 100644 index 76d1745d3c6..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/baseDecoder.ts +++ /dev/null @@ -1,361 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { Emitter } from '../../../../../../../base/common/event.js'; -import { ReadableStream } from '../../../../../../../base/common/stream.js'; -import { DeferredPromise } from '../../../../../../../base/common/async.js'; -import { AsyncDecoder } from './asyncDecoder.js'; -import { assert, assertNever } from '../../../../../../../base/common/assert.js'; -import { DisposableMap, IDisposable } from '../../../../../../../base/common/lifecycle.js'; -import { ObservableDisposable } from '../../utils/observableDisposable.js'; - -/** - * Event names of {@link ReadableStream} stream. - */ -export type TStreamListenerNames = 'data' | 'error' | 'end'; - -/** - * Base decoder class that can be used to convert stream messages data type - * from one type to another. For instance, a stream of binary data can be - * "decoded" into a stream of well defined objects. - * Intended to be a part of "codec" implementation rather than used directly. - */ -export abstract class BaseDecoder< - T extends NonNullable, - K extends NonNullable = NonNullable, -> extends ObservableDisposable implements ReadableStream { - /** - * Private attribute to track if the stream has ended. - */ - private _ended = false; - - protected readonly _onData = this._register(new Emitter()); - private readonly _onEnd = this._register(new Emitter()); - private readonly _onError = this._register(new Emitter()); - - /** - * A store of currently registered event listeners. - */ - private readonly _listeners: DisposableMap< - TStreamListenerNames, - DisposableMap - > = this._register(new DisposableMap()); - - /** - * This method is called when a new incoming data - * is received from the input stream. - */ - protected abstract onStreamData(data: K): void; - - /** - * @param stream The input stream to decode. - */ - constructor( - protected readonly stream: ReadableStream, - ) { - super(); - } - - /** - * Private attribute to track if the stream has started. - */ - private started = false; - - /** - * Promise that resolves when the stream has ended, either by - * receiving the `end` event or by a disposal, but not when - * the `error` event is received alone. - * The promise is true if the stream has ended, and false - * if the stream has been disposed without ending. - */ - private settledPromise = new DeferredPromise(); - - /** - * Promise that resolves when the stream has ended, either by - * receiving the `end` event or by a disposal, but not when - * the `error` event is received alone. - * The promise is true if the stream has ended, and false - * if the stream has been disposed without ending. - * - * @throws If the stream was not yet started to prevent this - * promise to block the consumer calls indefinitely. - */ - public get settled(): Promise { - // if the stream has not started yet, the promise might - // block the consumer calls indefinitely if they forget - // to call the `start()` method, or if the call happens - // after await on the `settled` promise; to forbid this - // confusion, we require the stream to be started first - assert( - this.started, - [ - 'Cannot get `settled` promise of a stream that has not been started.', - 'Please call `start()` first.', - ].join(' '), - ); - - return this.settledPromise.p; - } - - /** - * Start receiving data from the stream. - * @throws if the decoder stream has already ended. - */ - public start(): this { - assert( - this._ended === false, - 'Cannot start stream that has already ended.', - ); - assert( - this.isDisposed === false, - 'Cannot start stream that has already disposed.', - ); - - // if already started, nothing to do - if (this.started) { - return this; - } - this.started = true; - - /** - * !NOTE! The order of event subscriptions is critical here because - * the `data` event is also starts the stream, hence changing - * the order of event subscriptions can lead to race conditions. - * See {@link ReadableStreamEvents} for more info. - */ - this.stream.on('end', this.onStreamEnd.bind(this)); - this.stream.on('error', this.onStreamError.bind(this)); - this.stream.on('data', this.tryOnStreamData.bind(this)); - - // this allows to compose decoders together, - if a decoder - // instance is passed as a readable stream to this decoder, - // then we need to call `start` on it too - if (this.stream instanceof BaseDecoder) { - this.stream.start(); - } - - return this; - } - - /** - * Check if the decoder has been ended hence has - * no more data to produce. - */ - public get ended(): boolean { - return this._ended; - } - - /** - * Automatically catch and dispatch errors thrown inside `onStreamData`. - */ - private tryOnStreamData(data: K): void { - try { - this.onStreamData(data); - } catch (error) { - this.onStreamError(error); - } - } - - public on(event: 'data', callback: (data: T) => void): void; - public on(event: 'error', callback: (err: Error) => void): void; - public on(event: 'end', callback: () => void): void; - public on(event: TStreamListenerNames, callback: unknown): void { - if (event === 'data') { - return this.onData(callback as (data: T) => void); - } - - if (event === 'error') { - return this.onError(callback as (error: Error) => void); - } - - if (event === 'end') { - return this.onEnd(callback as () => void); - } - - assertNever(event, `Invalid event name '${event}'`); - } - - /** - * Add listener for the `data` event. - * @throws if the decoder stream has already ended. - */ - public onData(callback: (data: T) => void): void { - assert( - !this.ended, - 'Cannot subscribe to the `data` event because the decoder stream has already ended.', - ); - - let currentListeners = this._listeners.get('data'); - - if (!currentListeners) { - currentListeners = new DisposableMap(); - this._listeners.set('data', currentListeners); - } - - currentListeners.set(callback, this._onData.event(callback)); - } - - /** - * Add listener for the `error` event. - * @throws if the decoder stream has already ended. - */ - public onError(callback: (error: Error) => void): void { - assert( - !this.ended, - 'Cannot subscribe to the `error` event because the decoder stream has already ended.', - ); - - let currentListeners = this._listeners.get('error'); - - if (!currentListeners) { - currentListeners = new DisposableMap(); - this._listeners.set('error', currentListeners); - } - - currentListeners.set(callback, this._onError.event(callback)); - } - - /** - * Add listener for the `end` event. - * @throws if the decoder stream has already ended. - */ - public onEnd(callback: () => void): void { - assert( - !this.ended, - 'Cannot subscribe to the `end` event because the decoder stream has already ended.', - ); - - let currentListeners = this._listeners.get('end'); - - if (!currentListeners) { - currentListeners = new DisposableMap(); - this._listeners.set('end', currentListeners); - } - - currentListeners.set(callback, this._onEnd.event(callback)); - } - - /** - * Pauses the stream. - */ - public pause(): void { - this.stream.pause(); - } - - /** - * Resumes the stream if it has been paused. - * @throws if the decoder stream has already ended. - */ - public resume(): void { - assert( - this.ended === false, - 'Cannot resume the stream because it has already ended.', - ); - - this.stream.resume(); - } - - /** - * Destroys(disposes) the stream. - */ - public destroy(): void { - this.dispose(); - } - - /** - * Removes a previously-registered event listener for a specified event. - * - * Note! - * - the callback function must be the same as the one that was used when - * registering the event listener as it is used as an identifier to - * remove the listener - * - this method is idempotent and results in no-op if the listener is - * not found, therefore passing incorrect `callback` function may - * result in silent unexpected behavior - */ - public removeListener(eventName: TStreamListenerNames, callback: Function): void { - const listeners = this._listeners.get(eventName); - if (listeners === undefined) { - return; - } - - for (const [listener] of listeners) { - if (listener !== callback) { - continue; - } - - listeners.deleteAndDispose(listener); - } - } - - /** - * This method is called when the input stream ends. - */ - protected onStreamEnd(): void { - if (this._ended) { - return; - } - - this._ended = true; - this._onEnd.fire(); - this.settledPromise.complete(this._ended); - } - - /** - * This method is called when the input stream emits an error. - * We re-emit the error here by default, but subclasses can - * override this method to handle the error differently. - */ - private onStreamError(error: Error): void { - this._onError.fire(error); - } - - /** - * Consume all messages from the stream, blocking until the stream finishes. - * @throws if the decoder stream has already ended. - */ - public async consumeAll(): Promise { - assert( - !this._ended, - 'Cannot consume all messages of the stream that has already ended.', - ); - - const messages = []; - - for await (const maybeMessage of this) { - if (maybeMessage === null) { - break; - } - - messages.push(maybeMessage); - } - - return messages; - } - - /** - * Async iterator interface for the decoder. - * @throws if the decoder stream has already ended. - */ - [Symbol.asyncIterator](): AsyncIterator { - assert( - !this._ended, - 'Cannot iterate on messages of the stream that has already ended.', - ); - - const asyncDecoder = this._register(new AsyncDecoder(this)); - - return asyncDecoder[Symbol.asyncIterator](); - } - - public override dispose(): void { - this.settledPromise.complete(this.ended); - - this._listeners.clearAndDisposeAll(); - this.stream.destroy(); - - super.dispose(); - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/baseToken.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/baseToken.ts deleted file mode 100644 index ba81a78738e..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/baseToken.ts +++ /dev/null @@ -1,150 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { assert } from '../../../../../../../base/common/assert.js'; -import { IRange, Range } from '../../../../../../../editor/common/core/range.js'; - -/** - * Base class for all tokens with a `range` that reflects - * token position in the original text. - */ -export abstract class BaseToken { - constructor( - private tokenRange: Range, - ) { } - - /** - * Range of the token in the original text. - */ - public get range(): Range { - return this.tokenRange; - } - - /** - * Text representation of the token. - */ - public abstract get text(): TText; - - /** - * Check if this token has the same range as another one. - */ - public sameRange(other: Range): boolean { - return this.range.equalsRange(other); - } - - /** - * Returns a string representation of the token. - */ - public abstract toString(): string; - - /** - * Check if this token is equal to another one. - */ - public equals(other: BaseToken): other is typeof this { - if (other.constructor !== this.constructor) { - return false; - } - - if (this.text.length !== other.text.length) { - return false; - } - - if (this.text !== other.text) { - return false; - } - - return this.sameRange(other.range); - } - - /** - * Change `range` of the token with provided range components. - */ - public withRange(components: Partial): this { - this.tokenRange = new Range( - components.startLineNumber ?? this.range.startLineNumber, - components.startColumn ?? this.range.startColumn, - components.endLineNumber ?? this.range.endLineNumber, - components.endColumn ?? this.range.endColumn, - ); - - return this; - } - - /** - * Collapse range of the token to its start position. - * See {@link Range.collapseToStart} for more details. - */ - public collapseRangeToStart(): this { - this.tokenRange = this.tokenRange.collapseToStart(); - - return this; - } - - /** - * Render a list of tokens into a string. - */ - public static render( - tokens: readonly BaseToken[], - delimiter: string = '', - ): string { - return tokens.map(token => token.text).join(delimiter); - } - - /** - * Returns the full range of a list of tokens in which the first token is - * used as the start of a tokens sequence and the last token reflects the end. - * - * @throws if: - * - provided {@link tokens} list is empty - * - the first token start number is greater than the start line of the last token - * - if the first and last token are on the same line, the first token start column must - * be smaller than the start column of the last token - */ - public static fullRange(tokens: readonly BaseToken[]): Range { - assert( - tokens.length > 0, - 'Cannot get full range for an empty list of tokens.', - ); - - const firstToken = tokens[0]; - const lastToken = tokens[tokens.length - 1]; - - // sanity checks for the full range we would construct - assert( - firstToken.range.startLineNumber <= lastToken.range.startLineNumber, - 'First token must start on previous or the same line as the last token.', - ); - - if ((firstToken !== lastToken) && (firstToken.range.startLineNumber === lastToken.range.startLineNumber)) { - assert( - firstToken.range.endColumn <= lastToken.range.startColumn, - [ - 'First token must end at least on previous or the same column as the last token.', - `First token: ${firstToken}; Last token: ${lastToken}.`, - ].join('\n'), - ); - } - - return new Range( - firstToken.range.startLineNumber, - firstToken.range.startColumn, - lastToken.range.endLineNumber, - lastToken.range.endColumn, - ); - } - - /** - * Shorten version of the {@link text} property. - */ - public shortText( - maxLength: number = 32, - ): string { - if (this.text.length <= maxLength) { - return this.text; - } - - return `${this.text.slice(0, maxLength - 1)}...`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/compositeToken.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/compositeToken.ts deleted file mode 100644 index e4eb1470a03..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/compositeToken.ts +++ /dev/null @@ -1,63 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from './baseToken.js'; - -/** - * Composite token consists of a list of other tokens. - * Composite token consists of a list of other tokens. - */ -export abstract class CompositeToken< - TTokens extends readonly BaseToken[], -> extends BaseToken { - /** - * Reference to the list of child tokens. - */ - protected readonly childTokens: [...TTokens]; - - constructor( - tokens: TTokens, - ) { - super(BaseToken.fullRange(tokens)); - - this.childTokens = [...tokens]; - } - - public override get text(): string { - return BaseToken.render(this.childTokens); - } - - /** - * Tokens that this composite token consists of. - */ - public get children(): TTokens { - return this.childTokens; - } - - /** - * Check if this token is equal to another one, - * including all of its child tokens. - */ - public override equals(other: BaseToken): other is typeof this { - if (super.equals(other) === false) { - return false; - } - - if (this.children.length !== other.children.length) { - return false; - } - - for (let i = 0; i < this.children.length; i++) { - const childToken = this.children[i]; - const otherChildToken = other.children[i]; - - if (childToken.equals(otherChildToken) === false) { - return false; - } - } - - return true; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/constants.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/constants.ts deleted file mode 100644 index 779e53a1ef1..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/constants.ts +++ /dev/null @@ -1,16 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { NewLine } from '../linesCodec/tokens/newLine.js'; -import { CarriageReturn } from '../linesCodec/tokens/carriageReturn.js'; -import { FormFeed, SpacingToken } from '../simpleCodec/tokens/tokens.js'; - -/** - * List of valid "space" tokens that are valid between different - * records of a Front Matter header. - */ -export const VALID_INTER_RECORD_SPACING_TOKENS = Object.freeze([ - SpacingToken, CarriageReturn, NewLine, FormFeed, -]); diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/frontMatterDecoder.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/frontMatterDecoder.ts deleted file mode 100644 index 704e10f1754..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/frontMatterDecoder.ts +++ /dev/null @@ -1,156 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { Word } from '../simpleCodec/tokens/tokens.js'; -import { assert } from '../../../../../../../../base/common/assert.js'; -import { VSBuffer } from '../../../../../../../../base/common/buffer.js'; -import { VALID_INTER_RECORD_SPACING_TOKENS } from './constants.js'; -import { ReadableStream } from '../../../../../../../../base/common/stream.js'; -import { FrontMatterToken, FrontMatterRecord } from './tokens/index.js'; -import { BaseDecoder } from '../baseDecoder.js'; -import { SimpleDecoder, type TSimpleDecoderToken } from '../simpleCodec/simpleDecoder.js'; -import { ObjectStream } from '../utils/objectStream.js'; -import { PartialFrontMatterRecordNameWithDelimiter } from './parsers/frontMatterRecord/frontMatterRecordNameWithDelimiter.js'; -import { PartialFrontMatterRecord } from './parsers/frontMatterRecord/frontMatterRecord.js'; -import { PartialFrontMatterRecordName } from './parsers/frontMatterRecord/frontMatterRecordName.js'; -import { FrontMatterParserFactory } from './parsers/frontMatterParserFactory.js'; - -/** - * Tokens produced by this decoder. - */ -export type TFrontMatterToken = FrontMatterRecord | TSimpleDecoderToken; - -/** - * Decoder capable of parsing Front Matter contents from a sequence of simple tokens. - */ -export class FrontMatterDecoder extends BaseDecoder { - /** - * Current parser reference responsible for parsing a specific sequence - * of tokens into a standalone token. - */ - private current?: PartialFrontMatterRecordName | PartialFrontMatterRecordNameWithDelimiter | PartialFrontMatterRecord; - - private readonly parserFactory: FrontMatterParserFactory; - - constructor( - stream: ReadableStream | ObjectStream, - ) { - if (stream instanceof ObjectStream) { - super(stream); - } else { - super(new SimpleDecoder(stream)); - } - this.parserFactory = new FrontMatterParserFactory(); - } - - protected override onStreamData(token: TSimpleDecoderToken): void { - if (this.current !== undefined) { - const acceptResult = this.current.accept(token); - const { result, wasTokenConsumed } = acceptResult; - - if (result === 'failure') { - this.reEmitCurrentTokens(); - - if (wasTokenConsumed === false) { - this._onData.fire(token); - } - - delete this.current; - return; - } - - const { nextParser } = acceptResult; - - if (nextParser instanceof FrontMatterToken) { - // front matter record token is the spacial case - because it can - // contain trailing space tokens, we want to emit "trimmed" record - // token and the trailing spaces tokens separately - const trimmedTokens = (nextParser instanceof FrontMatterRecord) - ? nextParser.trimValueEnd() - : []; - - this._onData.fire(nextParser); - - // re-emit all trailing space tokens if present - for (const trimmedToken of trimmedTokens) { - this._onData.fire(trimmedToken); - } - - if (wasTokenConsumed === false) { - this._onData.fire(token); - } - - delete this.current; - return; - } - - this.current = nextParser; - if (wasTokenConsumed === false) { - this._onData.fire(token); - } - - return; - } - - // a word token starts a new record - if (token instanceof Word) { - this.current = this.parserFactory.createRecordName(token); - return; - } - - // re-emit all "space" tokens immediately as all of them - // are valid while we are not in the "record parsing" mode - for (const ValidToken of VALID_INTER_RECORD_SPACING_TOKENS) { - if (token instanceof ValidToken) { - this._onData.fire(token); - return; - } - } - - // unexpected token type, re-emit existing tokens and continue - this.reEmitCurrentTokens(); - } - - protected override onStreamEnd(): void { - try { - if (this.current === undefined) { - return; - } - - assert( - this.current instanceof PartialFrontMatterRecord, - 'Only partial front matter records can be processed on stream end.', - ); - - const record = this.current.asRecordToken(); - const trimmedTokens = record.trimValueEnd(); - - this._onData.fire(record); - - for (const trimmedToken of trimmedTokens) { - this._onData.fire(trimmedToken); - } - } catch (_error) { - this.reEmitCurrentTokens(); - } finally { - delete this.current; - super.onStreamEnd(); - } - } - - /** - * Re-emit tokens accumulated so far in the current parser object. - */ - protected reEmitCurrentTokens(): void { - if (this.current === undefined) { - return; - } - - for (const token of this.current.tokens) { - this._onData.fire(token); - } - delete this.current; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterArray.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterArray.ts deleted file mode 100644 index df84fa012c7..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterArray.ts +++ /dev/null @@ -1,197 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { assert } from '../../../../../../../../../base/common/assert.js'; -import { type PartialFrontMatterValue } from './frontMatterValue.js'; -import { FrontMatterArray } from '../tokens/frontMatterArray.js'; -import { assertDefined } from '../../../../../../../../../base/common/types.js'; -import { VALID_INTER_RECORD_SPACING_TOKENS } from '../constants.js'; -import { FrontMatterValueToken } from '../tokens/frontMatterToken.js'; -import { FrontMatterSequence } from '../tokens/frontMatterSequence.js'; -import { TSimpleDecoderToken } from '../../simpleCodec/simpleDecoder.js'; -import { Comma, LeftBracket, RightBracket } from '../../simpleCodec/tokens/tokens.js'; -import { assertNotConsumed, ParserBase, TAcceptTokenResult } from '../../simpleCodec/parserBase.js'; -import { type FrontMatterParserFactory } from './frontMatterParserFactory.js'; - -/** - * List of tokens that can go in-between array items - * and array brackets. - */ -const VALID_DELIMITER_TOKENS = Object.freeze([ - ...VALID_INTER_RECORD_SPACING_TOKENS, - Comma, -]); - -/** - * Responsible for parsing an array syntax (or "inline sequence" - * in YAML terms), e.g. `[1, '2', true, 2.54]` -*/ -export class PartialFrontMatterArray extends ParserBase { - /** - * Current parser reference responsible for parsing an array "value". - */ - private currentValueParser?: PartialFrontMatterValue; - - /** - * Whether an array item is allowed in the current position of the token - * sequence. E.g., items are allowed after a command or a open bracket, - * but not immediately after another item in the array. - */ - private arrayItemAllowed = true; - - constructor( - private readonly factory: FrontMatterParserFactory, - private readonly startToken: LeftBracket, - ) { - super([startToken]); - } - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - if (this.currentValueParser !== undefined) { - const acceptResult = this.currentValueParser.accept(token); - const { result, wasTokenConsumed } = acceptResult; - - if (result === 'failure') { - this.isConsumed = true; - - return { - result: 'failure', - wasTokenConsumed, - }; - } - - const { nextParser } = acceptResult; - - if (nextParser instanceof FrontMatterValueToken) { - this.currentTokens.push(nextParser); - delete this.currentValueParser; - - // if token was not consume, call the `accept()` method - // recursively so that the current parser can re-process - // the token (e.g., a comma or a closing square bracket) - if (wasTokenConsumed === false) { - return this.accept(token); - } - - return { - result: 'success', - nextParser: this, - wasTokenConsumed, - }; - } - - this.currentValueParser = nextParser; - return { - result: 'success', - nextParser: this, - wasTokenConsumed, - }; - } - - if (token instanceof RightBracket) { - // sanity check in case this block moves around - // to a different place in the code - assert( - this.currentValueParser === undefined, - `Unexpected end of array. Last value is not finished.`, - ); - - this.currentTokens.push(token); - - this.isConsumed = true; - return { - result: 'success', - nextParser: this.asArrayToken(), - wasTokenConsumed: true, - }; - } - - // iterate until a valid value start token is found - for (const ValidToken of VALID_DELIMITER_TOKENS) { - if (token instanceof ValidToken) { - this.currentTokens.push(token); - - if ((this.arrayItemAllowed === false) && token instanceof Comma) { - this.arrayItemAllowed = true; - } - - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } - } - - // is an array item value is allowed at this position, create a new - // value parser and start the value parsing process using it - if (this.arrayItemAllowed === true) { - this.currentValueParser = this.factory.createValue( - (currentToken) => { - // comma or a closing square bracket must stop the parsing - // process of the value represented by a generic sequence of tokens - return ( - (currentToken instanceof RightBracket) - || (currentToken instanceof Comma) - ); - }, - ); - this.arrayItemAllowed = false; - - return this.accept(token); - } - - // in all other cases fail because of the unexpected token type - this.isConsumed = true; - return { - result: 'failure', - wasTokenConsumed: false, - }; - } - - /** - * Convert current parser into a {@link FrontMatterArray} token, - * if possible. - * - * @throws if the last token in the accumulated token list - * is not a closing bracket ({@link RightBracket}). - */ - public asArrayToken(): FrontMatterArray { - const endToken = this.currentTokens[this.currentTokens.length - 1]; - - assertDefined( - endToken, - 'No tokens found.', - ); - - assert( - endToken instanceof RightBracket, - 'Cannot find a closing bracket of the array.', - ); - - const valueTokens: FrontMatterValueToken[] = []; - for (const currentToken of this.currentTokens) { - if ((currentToken instanceof FrontMatterValueToken) === false) { - continue; - } - - // the generic sequence tokens can have trailing spacing tokens, - // hence trim them to ensure the array contains only "clean" values - if (currentToken instanceof FrontMatterSequence) { - currentToken.trimEnd(); - } - - valueTokens.push(currentToken); - } - - this.isConsumed = true; - return new FrontMatterArray([ - this.startToken, - ...valueTokens, - endToken, - ]); - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterParserFactory.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterParserFactory.ts deleted file mode 100644 index 0be65371449..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterParserFactory.ts +++ /dev/null @@ -1,41 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../../baseToken.js'; -import { LeftBracket } from '../../simpleCodec/tokens/brackets.js'; -import { Word } from '../../simpleCodec/tokens/word.js'; -import { FrontMatterRecordDelimiter, FrontMatterRecordName } from '../tokens/frontMatterRecord.js'; -import { TQuoteToken } from '../tokens/frontMatterString.js'; -import { PartialFrontMatterArray } from './frontMatterArray.js'; -import { PartialFrontMatterRecord } from './frontMatterRecord/frontMatterRecord.js'; -import { PartialFrontMatterRecordName } from './frontMatterRecord/frontMatterRecordName.js'; -import { PartialFrontMatterRecordNameWithDelimiter, TNameStopToken } from './frontMatterRecord/frontMatterRecordNameWithDelimiter.js'; -import { PartialFrontMatterSequence } from './frontMatterSequence.js'; -import { PartialFrontMatterString } from './frontMatterString.js'; -import { PartialFrontMatterValue } from './frontMatterValue.js'; - -export class FrontMatterParserFactory { - createRecord(tokens: [FrontMatterRecordName, FrontMatterRecordDelimiter]): PartialFrontMatterRecord { - return new PartialFrontMatterRecord(this, tokens); - } - createRecordName(startToken: Word): PartialFrontMatterRecordName { - return new PartialFrontMatterRecordName(this, startToken); - } - createRecordNameWithDelimiter(tokens: readonly [FrontMatterRecordName, TNameStopToken]): PartialFrontMatterRecordNameWithDelimiter { - return new PartialFrontMatterRecordNameWithDelimiter(this, tokens); - } - createArray(startToken: LeftBracket) { - return new PartialFrontMatterArray(this, startToken); - } - createValue(shouldStop: (token: BaseToken) => boolean): PartialFrontMatterValue { - return new PartialFrontMatterValue(this, shouldStop); - } - createString(startToken: TQuoteToken): PartialFrontMatterString { - return new PartialFrontMatterString(startToken); - } - createSequence(shouldStop: (token: BaseToken) => boolean): PartialFrontMatterSequence { - return new PartialFrontMatterSequence(shouldStop); - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterRecord/frontMatterRecord.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterRecord/frontMatterRecord.ts deleted file mode 100644 index e4afee055b5..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterRecord/frontMatterRecord.ts +++ /dev/null @@ -1,210 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../../../baseToken.js'; -import { NewLine } from '../../../linesCodec/tokens/newLine.js'; -import { PartialFrontMatterValue } from '../frontMatterValue.js'; -import { assertNever } from '../../../../../../../../../../base/common/assert.js'; -import { assertDefined } from '../../../../../../../../../../base/common/types.js'; -import { PartialFrontMatterSequence } from '../frontMatterSequence.js'; -import { CarriageReturn } from '../../../linesCodec/tokens/carriageReturn.js'; -import { type TSimpleDecoderToken } from '../../../simpleCodec/simpleDecoder.js'; -import { Word, FormFeed, SpacingToken } from '../../../simpleCodec/tokens/tokens.js'; -import { assertNotConsumed, ParserBase, type TAcceptTokenResult } from '../../../simpleCodec/parserBase.js'; -import { FrontMatterValueToken, FrontMatterRecordName, FrontMatterRecordDelimiter, FrontMatterRecord } from '../../tokens/index.js'; -import { type FrontMatterParserFactory } from '../frontMatterParserFactory.js'; - -/** - * Type of a next parser that can be returned by {@link PartialFrontMatterRecord}. - */ -type TNextParser = PartialFrontMatterRecord | FrontMatterRecord; - -/** - * Parser for a `record` inside a Front Matter header. - * - * * E.g., `name: 'value'` in the example below: - * - * ``` - * --- - * name: 'value' - * isExample: true - * --- - * ``` - */ -export class PartialFrontMatterRecord extends ParserBase { - /** - * Token that represents the 'name' part of the record. - */ - private readonly recordNameToken: FrontMatterRecordName; - - /** - * Token that represents the 'delimiter' part of the record. - */ - private readonly recordDelimiterToken: FrontMatterRecordDelimiter; - - constructor( - private readonly factory: FrontMatterParserFactory, - tokens: [FrontMatterRecordName, FrontMatterRecordDelimiter], - ) { - super(tokens); - this.recordNameToken = tokens[0]; - this.recordDelimiterToken = tokens[1]; - } - - /** - * Current parser reference responsible for parsing the "value" part of the record. - */ - private valueParser?: PartialFrontMatterValue | PartialFrontMatterSequence; - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - if (this.valueParser !== undefined) { - const acceptResult = this.valueParser.accept(token); - const { result, wasTokenConsumed } = acceptResult; - - if (result === 'failure') { - this.isConsumed = true; - - return { - result: 'failure', - wasTokenConsumed, - }; - } - - const { nextParser } = acceptResult; - - if (nextParser instanceof FrontMatterValueToken) { - this.currentTokens.push(nextParser); - delete this.valueParser; - - this.isConsumed = true; - try { - return { - result: 'success', - nextParser: new FrontMatterRecord([ - this.recordNameToken, - this.recordDelimiterToken, - nextParser, - ]), - wasTokenConsumed, - }; - } catch (_error) { - return { - result: 'failure', - wasTokenConsumed, - }; - } - } - - this.valueParser = nextParser; - return { - result: 'success', - nextParser: this, - wasTokenConsumed, - }; - } - - // iterate until the first non-space token is found - if (token instanceof SpacingToken) { - this.currentTokens.push(token); - - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } - - // if token can start a "value" sequence, parse the value - if (PartialFrontMatterValue.isValueStartToken(token)) { - this.valueParser = this.factory.createValue(shouldEndTokenSequence); - - return this.accept(token); - } - - // in all other cases, collect all the subsequent tokens into - // a "sequence of tokens" until a new line is found - this.valueParser = this.factory.createSequence( - shouldEndTokenSequence, - ); - - // if we reached this "generic sequence" parser point, but the current token is - // already of a type that stops such sequence, we must have accumulated some - // spacing tokens, hence pass those to the parser and end the sequence immediately - if (shouldEndTokenSequence(token)) { - const spaceTokens = this.currentTokens - .slice(this.startTokensCount); - - // if no space tokens accumulated at all, create an "empty" one this is needed - // to ensure that the parser always has at least one token hence it can have - // a valid range and can be interpreted as a real "value" token of the record - if (spaceTokens.length === 0) { - spaceTokens.push( - Word.newOnLine( - '', - token.range.startLineNumber, - token.range.startColumn, - ), - ); - } - - this.valueParser.addTokens(spaceTokens); - - return { - result: 'success', - nextParser: this.asRecordToken(), - wasTokenConsumed: false, - }; - } - - // otherwise use the "generic sequence" parser moving on - return this.accept(token); - } - - /** - * Convert current parser into a {@link FrontMatterRecord} token. - * - * @throws if no current parser is present, or it is not of the {@link PartialFrontMatterValue} - * or {@link PartialFrontMatterSequence} types - */ - public asRecordToken(): FrontMatterRecord { - assertDefined( - this.valueParser, - 'Current value parser must be defined.' - ); - - if ( - (this.valueParser instanceof PartialFrontMatterValue) - || (this.valueParser instanceof PartialFrontMatterSequence) - ) { - const valueToken = this.valueParser.asSequenceToken(); - this.currentTokens.push(valueToken); - - this.isConsumed = true; - return new FrontMatterRecord([ - this.recordNameToken, - this.recordDelimiterToken, - valueToken, - ]); - } - - assertNever( - this.valueParser, - `Unexpected value parser '${this.valueParser}'.`, - ); - } -} - -/** - * Callback to check if a current token should end a - * record value that is a generic sequence of tokens. - */ -function shouldEndTokenSequence(token: BaseToken): token is (NewLine | CarriageReturn | FormFeed) { - return ( - (token instanceof NewLine) - || (token instanceof CarriageReturn) - || (token instanceof FormFeed) - ); -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterRecord/frontMatterRecordName.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterRecord/frontMatterRecordName.ts deleted file mode 100644 index 84c94b75211..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterRecord/frontMatterRecordName.ts +++ /dev/null @@ -1,74 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { type TSimpleDecoderToken } from '../../../simpleCodec/simpleDecoder.js'; -import { FrontMatterRecordName, type TRecordNameToken } from '../../tokens/index.js'; -import { Colon, Word, Dash, SpacingToken } from '../../../simpleCodec/tokens/tokens.js'; -import { type PartialFrontMatterRecordNameWithDelimiter } from './frontMatterRecordNameWithDelimiter.js'; -import { assertNotConsumed, ParserBase, type TAcceptTokenResult } from '../../../simpleCodec/parserBase.js'; -import { type FrontMatterParserFactory } from '../frontMatterParserFactory.js'; - -/** - * Tokens that can be used inside a record name. - */ -const VALID_NAME_TOKENS = [Word, Dash]; - -/** - * Type of a next parser that can be returned by {@link PartialFrontMatterRecordName}. - */ -type TNextParser = PartialFrontMatterRecordName | PartialFrontMatterRecordNameWithDelimiter; - -/** - * Parser for a `name` part of a Front Matter record. - * - * E.g., `'name'` in the example below: - * - * ``` - * name: 'value' - * ``` - */ -export class PartialFrontMatterRecordName extends ParserBase { - constructor( - private readonly factory: FrontMatterParserFactory, - startToken: Word, - ) { - super([startToken]); - } - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - for (const ValidToken of VALID_NAME_TOKENS) { - if (token instanceof ValidToken) { - this.currentTokens.push(token); - - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } - } - - // once name is followed by a "space" token or a "colon", we have the full - // record name hence can transition to the next parser - if ((token instanceof Colon) || (token instanceof SpacingToken)) { - const recordName = new FrontMatterRecordName(this.currentTokens); - - this.isConsumed = true; - return { - result: 'success', - nextParser: this.factory.createRecordNameWithDelimiter([recordName, token]), - wasTokenConsumed: true, - }; - } - - // in all other cases fail due to the unexpected token type for a record name - this.isConsumed = true; - return { - result: 'failure', - wasTokenConsumed: false, - }; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterRecord/frontMatterRecordNameWithDelimiter.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterRecord/frontMatterRecordNameWithDelimiter.ts deleted file mode 100644 index 59735860631..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterRecord/frontMatterRecordNameWithDelimiter.ts +++ /dev/null @@ -1,105 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { assert } from '../../../../../../../../../../base/common/assert.js'; -import { type PartialFrontMatterRecord } from './frontMatterRecord.js'; -import { Colon, SpacingToken } from '../../../simpleCodec/tokens/tokens.js'; -import { type TSimpleDecoderToken } from '../../../simpleCodec/simpleDecoder.js'; -import { FrontMatterRecordName, FrontMatterRecordDelimiter } from '../../tokens/index.js'; -import { assertNotConsumed, ParserBase, type TAcceptTokenResult } from '../../../simpleCodec/parserBase.js'; -import { type FrontMatterParserFactory } from '../frontMatterParserFactory.js'; - -/** - * Type for tokens that stop a front matter record name sequence. - */ -export type TNameStopToken = Colon | SpacingToken; - -/** - * Type for the next parser that can be returned by {@link PartialFrontMatterRecordNameWithDelimiter}. - */ -type TNextParser = PartialFrontMatterRecordNameWithDelimiter | PartialFrontMatterRecord; - -/** - * Parser for a record `name` with the `: ` delimiter. - * - * * E.g., `name:` in the example below: - * - * ``` - * name: 'value' - * ``` - */ -export class PartialFrontMatterRecordNameWithDelimiter extends ParserBase< - FrontMatterRecordName | TNameStopToken, - TNextParser -> { - constructor( - private readonly factory: FrontMatterParserFactory, - tokens: readonly [FrontMatterRecordName, TNameStopToken], - ) { - super([...tokens]); - } - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - const previousToken = this.currentTokens[this.currentTokens.length - 1]; - const isSpacingToken = (token instanceof SpacingToken); - - // delimiter must always be a `:` followed by a "space" character - // once we encounter that sequence, we can transition to the next parser - if (isSpacingToken && (previousToken instanceof Colon)) { - const recordDelimiter = new FrontMatterRecordDelimiter([ - previousToken, - token, - ]); - - const recordName = this.currentTokens[0]; - - // sanity check - assert( - recordName instanceof FrontMatterRecordName, - `Expected a front matter record name, got '${recordName}'.`, - ); - - this.isConsumed = true; - return { - result: 'success', - nextParser: this.factory.createRecord( - [recordName, recordDelimiter], - ), - wasTokenConsumed: true, - }; - } - - // allow some spacing before the colon delimiter - if (token instanceof SpacingToken) { - this.currentTokens.push(token); - - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } - - // include the colon delimiter - if (token instanceof Colon) { - this.currentTokens.push(token); - - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } - - // otherwise fail due to the unexpected token type between - // record name and record name delimiter tokens - this.isConsumed = true; - return { - result: 'failure', - wasTokenConsumed: false, - }; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterSequence.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterSequence.ts deleted file mode 100644 index 0d91c32528c..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterSequence.ts +++ /dev/null @@ -1,78 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../../baseToken.js'; -import { FrontMatterSequence } from '../tokens/frontMatterSequence.js'; -import { TSimpleDecoderToken } from '../../simpleCodec/simpleDecoder.js'; -import { assertNotConsumed, ParserBase, TAcceptTokenResult } from '../../simpleCodec/parserBase.js'; - -/** - * Parser responsible for parsing a "generic sequence of tokens" - * of an arbitrary length in a Front Matter header. - */ -export class PartialFrontMatterSequence extends ParserBase< - TSimpleDecoderToken, - PartialFrontMatterSequence | FrontMatterSequence -> { - constructor( - /** - * Callback function that is called to check if the current token - * should stop the parsing process of the current generic "value" - * sequence of arbitrary tokens by returning `true`. - * - * When this happens, the parser *will not consume* the token that - * was passed to the `shouldStop` callback or to its `accept` method. - * On the other hand, the parser will be "consumed" hence using it - * to process other tokens will yield an error. - */ - private readonly shouldStop: (token: BaseToken) => boolean, - ) { - super([]); - } - - @assertNotConsumed - public accept( - token: TSimpleDecoderToken, - ): TAcceptTokenResult { - - // collect all tokens until an end of the sequence is found - if (this.shouldStop(token)) { - this.isConsumed = true; - - return { - result: 'success', - nextParser: this.asSequenceToken(), - wasTokenConsumed: false, - }; - } - - this.currentTokens.push(token); - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } - - /** - * Add provided tokens to the list of the current parsed tokens. - */ - public addTokens( - tokens: readonly TSimpleDecoderToken[], - ): this { - this.currentTokens.push(...tokens); - - return this; - } - - /** - * Convert the current parser into a {@link FrontMatterSequence} token. - */ - public asSequenceToken(): FrontMatterSequence { - this.isConsumed = true; - - return new FrontMatterSequence(this.currentTokens); - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterString.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterString.ts deleted file mode 100644 index d0ed92746d6..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterString.ts +++ /dev/null @@ -1,69 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { assert } from '../../../../../../../../../base/common/assert.js'; -import { SimpleToken } from '../../simpleCodec/tokens/tokens.js'; -import { assertDefined } from '../../../../../../../../../base/common/types.js'; -import { TSimpleDecoderToken } from '../../simpleCodec/simpleDecoder.js'; -import { FrontMatterString, type TQuoteToken } from '../tokens/frontMatterString.js'; -import { assertNotConsumed, ParserBase, type TAcceptTokenResult } from '../../simpleCodec/parserBase.js'; - -/** - * Parser responsible for parsing a string value. - */ -export class PartialFrontMatterString extends ParserBase> { - constructor( - private readonly startToken: TQuoteToken, - ) { - super([startToken]); - } - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult> { - this.currentTokens.push(token); - - // iterate until a `matching end quote` is found - if ((token instanceof SimpleToken) && (this.startToken.sameType(token))) { - return { - result: 'success', - nextParser: this.asStringToken(), - wasTokenConsumed: true, - }; - } - - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } - - /** - * Convert the current parser into a {@link FrontMatterString} token, - * if possible. - * - * @throws if the first and last tokens are not quote tokens of the same type. - */ - public asStringToken(): FrontMatterString { - const endToken = this.currentTokens[this.currentTokens.length - 1]; - - assertDefined( - endToken, - `No matching end token found.`, - ); - - assert( - this.startToken.sameType(endToken), - `String starts with \`${this.startToken.text}\`, but ends with \`${endToken.text}\`.`, - ); - - return new FrontMatterString([ - this.startToken, - ...this.currentTokens - .slice(1, this.currentTokens.length - 1), - endToken, - ]); - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterValue.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterValue.ts deleted file mode 100644 index 710767aed40..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterValue.ts +++ /dev/null @@ -1,184 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../../baseToken.js'; -import { type PartialFrontMatterArray } from './frontMatterArray.js'; -import { type PartialFrontMatterString } from './frontMatterString.js'; -import { asBoolean, FrontMatterBoolean } from '../tokens/frontMatterBoolean.js'; -import { FrontMatterValueToken } from '../tokens/frontMatterToken.js'; -import { PartialFrontMatterSequence } from './frontMatterSequence.js'; -import { FrontMatterSequence } from '../tokens/frontMatterSequence.js'; -import { TSimpleDecoderToken } from '../../simpleCodec/simpleDecoder.js'; -import { Word, Quote, DoubleQuote, LeftBracket } from '../../simpleCodec/tokens/tokens.js'; -import { assertNotConsumed, ParserBase, TAcceptTokenResult } from '../../simpleCodec/parserBase.js'; -import { type FrontMatterParserFactory } from './frontMatterParserFactory.js'; - -/** - * List of tokens that can start a "value" sequence. - * - * - {@link Word} - can be a `boolean` value - * - {@link Quote}, {@link DoubleQuote} - can start a `string` value - * - {@link LeftBracket} - can start an `array` value - */ -export const VALID_VALUE_START_TOKENS = Object.freeze([ - Quote, - DoubleQuote, - LeftBracket, -]); - -/** - * Type alias for a token that can start a "value" sequence. - */ -type TValueStartToken = InstanceType; - -/** - * Parser responsible for parsing a "value" sequence in a Front Matter header. - */ -export class PartialFrontMatterValue extends ParserBase { - /** - * Current parser reference responsible for parsing - * a specific "value" sequence. - */ - private currentValueParser?: PartialFrontMatterString | PartialFrontMatterArray | PartialFrontMatterSequence; - - /** - * Get the tokens that were accumulated so far. - */ - public override get tokens(): readonly TSimpleDecoderToken[] { - if (this.currentValueParser === undefined) { - return []; - } - - return this.currentValueParser.tokens; - } - - constructor( - private readonly factory: FrontMatterParserFactory, - /** - * Callback function to pass to the {@link PartialFrontMatterSequence} - * if the current "value" sequence is not of a specific type. - */ - private readonly shouldStop: (token: BaseToken) => boolean, - ) { - super(); - } - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - if (this.currentValueParser !== undefined) { - const acceptResult = this.currentValueParser.accept(token); - const { result, wasTokenConsumed } = acceptResult; - - // current value parser is consumed with its child value parser - this.isConsumed = this.currentValueParser.consumed; - - if (result === 'success') { - const { nextParser } = acceptResult; - - if (nextParser instanceof FrontMatterValueToken) { - return { - result: 'success', - nextParser, - wasTokenConsumed, - }; - } - - this.currentValueParser = nextParser; - return { - result: 'success', - nextParser: this, - wasTokenConsumed, - }; - } - - return { - result: 'failure', - wasTokenConsumed, - }; - } - - // if the first token represents a `quote` character, try to parse a string value - if ((token instanceof Quote) || (token instanceof DoubleQuote)) { - this.currentValueParser = this.factory.createString(token); - - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } - - // if the first token represents a `[` character, try to parse an array value - if (token instanceof LeftBracket) { - this.currentValueParser = this.factory.createArray(token); - - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } - - // if the first token represents a `word` try to parse a boolean - const maybeBoolean = FrontMatterBoolean.tryFromToken(token); - if (maybeBoolean !== null) { - this.isConsumed = true; - - return { - result: 'success', - nextParser: maybeBoolean, - wasTokenConsumed: true, - }; - } - - // in all other cases, collect all the subsequent tokens into - // a generic sequence of tokens until stopped by the `this.shouldStop` - // callback or the call to the 'this.asSequenceToken' method - this.currentValueParser = this.factory.createSequence(this.shouldStop); - - return this.accept(token); - } - - /** - * Check if provided token can be a start of a "value" sequence. - * See {@link VALID_VALUE_START_TOKENS} for the list of valid tokens. - */ - public static isValueStartToken( - token: BaseToken, - ): token is TValueStartToken | Word<'true' | 'false'> { - for (const ValidToken of VALID_VALUE_START_TOKENS) { - if (token instanceof ValidToken) { - return true; - } - } - - if ((token instanceof Word) && (asBoolean(token) !== null)) { - return true; - } - - return false; - } - - /** - * Check if the current 'value' sequence does not have a specific type - * and is represented by a generic sequence of tokens ({@link PartialFrontMatterSequence}). - */ - public get isSequence(): boolean { - if (this.currentValueParser === undefined) { - return false; - } - - return (this.currentValueParser instanceof PartialFrontMatterSequence); - } - - /** - * Convert current parser into a generic sequence of tokens. - */ - public asSequenceToken(): FrontMatterSequence { - this.isConsumed = true; - - return new FrontMatterSequence(this.tokens); - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterArray.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterArray.ts deleted file mode 100644 index b8b01e42c79..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterArray.ts +++ /dev/null @@ -1,43 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../../baseToken.js'; -import { LeftBracket, RightBracket } from '../../simpleCodec/tokens/tokens.js'; -import { FrontMatterValueToken, type TValueTypeName } from './frontMatterToken.js'; - -/** - * Token that represents an `array` value in a Front Matter header. - */ -export class FrontMatterArray extends FrontMatterValueToken<'array', [ - LeftBracket, - ...FrontMatterValueToken[], - RightBracket, -]> { - /** - * Name of the `array` value type. - */ - public override readonly valueTypeName = 'array'; - - /** - * List of the array items. - */ - public get items(): readonly FrontMatterValueToken[] { - const result = []; - - for (const token of this.children) { - if (token instanceof FrontMatterValueToken) { - result.push(token); - } - } - - return result; - } - - public override toString(): string { - const itemsString = BaseToken.render(this.items, ', '); - - return `front-matter-array(${itemsString})${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterBoolean.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterBoolean.ts deleted file mode 100644 index 3eae6d114ed..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterBoolean.ts +++ /dev/null @@ -1,88 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../../baseToken.js'; -import { Word } from '../../simpleCodec/tokens/tokens.js'; -import { FrontMatterValueToken } from './frontMatterToken.js'; -import { assertDefined } from '../../../../../../../../../base/common/types.js'; - -/** - * Token that represents a `boolean` value in a Front Matter header. - */ -export class FrontMatterBoolean extends FrontMatterValueToken<'boolean', readonly [Word]> { - /** - * Name of the `boolean` value type. - */ - public override readonly valueTypeName = 'boolean'; - - /** - * Value of the `boolean` token. - */ - public readonly value: boolean; - - /** - * @throws if provided {@link Word} cannot be converted to a `boolean` value. - */ - constructor(token: Word) { - const value = asBoolean(token); - assertDefined( - value, - `Cannot convert '${token}' to a boolean value.`, - ); - - super([token]); - - this.value = value; - } - - /** - * Try creating a {@link FrontMatterBoolean} out of provided token. - * Unlike the constructor, this method does not throw, returning - * a 'null' value on failure instead. - */ - public static tryFromToken( - token: BaseToken, - ): FrontMatterBoolean | null { - if (token instanceof Word === false) { - return null; - } - - try { - return new FrontMatterBoolean(token); - } catch (_error) { - // noop - return null; - } - } - - public override equals(other: BaseToken): other is typeof this { - if (super.equals(other) === false) { - return false; - } - - return this.value === other.value; - } - - public override toString(): string { - return `front-matter-boolean(${this.shortText()})${this.range}`; - } -} - -/** - * Try to convert a {@link Word} token to a `boolean` value. - */ -export function asBoolean( - token: Word, -): boolean | null { - if (token.text.toLowerCase() === 'true') { - return true; - } - - if (token.text.toLowerCase() === 'false') { - return false; - } - - return null; -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterRecord.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterRecord.ts deleted file mode 100644 index dc498c6d667..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterRecord.ts +++ /dev/null @@ -1,118 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../../baseToken.js'; -import { FrontMatterSequence } from './frontMatterSequence.js'; -import { Colon, Word, Dash, SpacingToken } from '../../simpleCodec/tokens/tokens.js'; -import { FrontMatterToken, FrontMatterValueToken, type TValueTypeName } from './frontMatterToken.js'; - -/** - * Type for tokens that can be used inside a record name. - */ -export type TNameToken = Word | Dash; - -/** - * Token representing a `record name` inside a Front Matter record. - * - * E.g., `name` in the example below: - * - * ``` - * --- - * name: 'value' - * --- - * ``` - */ -export class FrontMatterRecordName extends FrontMatterToken { - public override toString(): string { - return `front-matter-record-name(${this.shortText()})${this.range}`; - } -} - -/** - * Token representing a delimiter of a record inside a Front Matter header. - * - * E.g., `: ` in the example below: - * - * ``` - * --- - * name: 'value' - * --- - * ``` - */ -export class FrontMatterRecordDelimiter extends FrontMatterToken { - public override toString(): string { - return `front-matter-delimiter(${this.shortText()})${this.range}`; - } -} - -/** - * Token representing a `record` inside a Front Matter header. - * - * E.g., `name: 'value'` in the example below: - * - * ``` - * --- - * name: 'value' - * --- - * ``` - */ -export class FrontMatterRecord extends FrontMatterToken< - readonly [FrontMatterRecordName, FrontMatterRecordDelimiter, FrontMatterValueToken] -> { - /** - * Token that represent `name` of the record. - * - * E.g., `tools` in the example below: - * - * ``` - * --- - * tools: ['value'] - * --- - * ``` - */ - public get nameToken(): FrontMatterRecordName { - return this.children[0]; - } - - /** - * Token that represent `value` of the record. - * - * E.g., `['value']` in the example below: - * - * ``` - * --- - * tools: ['value'] - * --- - * ``` - */ - public get valueToken(): FrontMatterValueToken { - return this.children[2]; - } - - /** - * Trim spacing tokens at the end of the record. - */ - public trimValueEnd(): readonly SpacingToken[] { - const { valueToken } = this; - - // only the "generic sequence" value tokens can hold - // some spacing tokens at the end of them - if ((valueToken instanceof FrontMatterSequence) === false) { - return []; - } - - const trimmedTokens = valueToken.trimEnd(); - // update the current range to reflect the current trimmed value - this.withRange( - BaseToken.fullRange(this.children), - ); - - return trimmedTokens; - } - - public override toString(): string { - return `front-matter-record(${this.shortText()})${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterSequence.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterSequence.ts deleted file mode 100644 index dcb6f70d5a3..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterSequence.ts +++ /dev/null @@ -1,79 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../../baseToken.js'; -import { FrontMatterValueToken } from './frontMatterToken.js'; -import { Word, SpacingToken } from '../../simpleCodec/tokens/tokens.js'; -import { type TSimpleDecoderToken } from '../../simpleCodec/simpleDecoder.js'; - - -/** - * Token represents a generic sequence of tokens in a Front Matter header. - */ -export class FrontMatterSequence extends FrontMatterValueToken { - /** - * @override Because this token represent a generic sequence of tokens, - * the type name is represented by the sequence of tokens itself - */ - public override get valueTypeName(): this { - return this; - } - - /** - * Text of the sequence value. The method exists to provide a - * consistent interface with {@link FrontMatterString} token. - * - * Note! that this method does not automatically trim spacing tokens - * in the sequence. If you need to get a trimmed value, call - * {@link trimEnd} method first. - */ - public get cleanText(): string { - return this.text; - } - - /** - * Trim spacing tokens at the end of the sequence. - */ - public trimEnd(): readonly SpacingToken[] { - const trimmedTokens = []; - - // iterate the tokens list from the end to the start, collecting - // all the spacing tokens we encounter until we reach a non-spacing token - let lastNonSpace = this.childTokens.length - 1; - while (lastNonSpace >= 0) { - const token = this.childTokens[lastNonSpace]; - - if (token instanceof SpacingToken) { - trimmedTokens.push(token); - lastNonSpace--; - - continue; - } - - break; - } - this.childTokens.length = lastNonSpace + 1; - - // if there are only spacing tokens were present add a single - // empty token to the sequence, so it has something to work with - if (this.childTokens.length === 0) { - this.collapseRangeToStart(); - this.childTokens.push(new Word(this.range, '')); - } - - // update the current range to reflect the current trimmed value - this.withRange( - BaseToken.fullRange(this.childTokens), - ); - - // trimmed tokens are collected starting from the end, - // moving to the start, hence reverse them before returning - return trimmedTokens.reverse(); - } - - public override toString(): string { - return `front-matter-sequence(${this.shortText()})${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterString.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterString.ts deleted file mode 100644 index 8a7283d1098..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterString.ts +++ /dev/null @@ -1,39 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../../baseToken.js'; -import { FrontMatterValueToken } from './frontMatterToken.js'; -import { Quote, DoubleQuote } from '../../simpleCodec/tokens/tokens.js'; - -/** - * Type for any quote token that can be used to wrap a string. - */ -export type TQuoteToken = Quote | DoubleQuote; - -/** - * Token that represents a string value in a Front Matter header. - */ -export class FrontMatterString extends FrontMatterValueToken< - 'quoted-string', - readonly [TQuote, ...BaseToken[], TQuote] -> { - /** - * Name of the `string` value type. - */ - public override readonly valueTypeName = 'quoted-string'; - - /** - * Text of the string value without the wrapping quotes. - */ - public get cleanText(): string { - return BaseToken.render( - this.children.slice(1, this.children.length - 1), - ); - } - - public override toString(): string { - return `front-matter-string(${this.shortText()})${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterToken.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterToken.ts deleted file mode 100644 index d85f85230d7..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterToken.ts +++ /dev/null @@ -1,33 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../../baseToken.js'; -import { CompositeToken } from '../../compositeToken.js'; -import { FrontMatterSequence } from './frontMatterSequence.js'; - -/** - * Base class for all tokens inside a Front Matter header. - */ -export abstract class FrontMatterToken< - TTokens extends readonly BaseToken[] = readonly BaseToken[], -> extends CompositeToken { } - -/** - * List of all currently supported value types. - */ -export type TValueTypeName = 'quoted-string' | 'boolean' | 'array' | FrontMatterSequence; - -/** - * Base class for all tokens that represent a `value` inside a Front Matter header. - */ -export abstract class FrontMatterValueToken< - TTypeName extends TValueTypeName = TValueTypeName, - TTokens extends readonly BaseToken[] = readonly BaseToken[], -> extends FrontMatterToken { - /** - * Type name of the `value` represented by this token. - */ - public abstract readonly valueTypeName: TTypeName; -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/index.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/index.ts deleted file mode 100644 index 033a1250fa5..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/index.ts +++ /dev/null @@ -1,15 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -export { FrontMatterArray } from './frontMatterArray.js'; -export { FrontMatterString } from './frontMatterString.js'; -export { FrontMatterBoolean } from './frontMatterBoolean.js'; -export { FrontMatterToken, FrontMatterValueToken } from './frontMatterToken.js'; -export { - FrontMatterRecordName, - FrontMatterRecordDelimiter, - FrontMatterRecord, - type TNameToken as TRecordNameToken, -} from './frontMatterRecord.js'; diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/linesCodec/linesDecoder.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/linesCodec/linesDecoder.ts deleted file mode 100644 index 5d9ad249d6f..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/linesCodec/linesDecoder.ts +++ /dev/null @@ -1,237 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { Line } from './tokens/line.js'; -import { Range } from '../../../../../../../../editor/common/core/range.js'; -import { NewLine } from './tokens/newLine.js'; -import { assert } from '../../../../../../../../base/common/assert.js'; -import { CarriageReturn } from './tokens/carriageReturn.js'; -import { VSBuffer } from '../../../../../../../../base/common/buffer.js'; -import { assertDefined } from '../../../../../../../../base/common/types.js'; -import { BaseDecoder } from '../baseDecoder.js'; - -/** - * Any line break token type. - */ -export type TLineBreakToken = CarriageReturn | NewLine; - -/** - * Tokens produced by the {@link LinesDecoder}. - */ -export type TLineToken = Line | TLineBreakToken; - -/** - * The `decoder` part of the `LinesCodec` and is able to transform - * data from a binary stream into a stream of text lines(`Line`). - */ -export class LinesDecoder extends BaseDecoder { - /** - * Buffered received data yet to be processed. - */ - private buffer: VSBuffer = VSBuffer.alloc(0); - - /** - * The last emitted `Line` token, if any. The value is used - * to correctly emit remaining line range in the `onStreamEnd` - * method when underlying input stream ends and `buffer` still - * contains some data that must be emitted as the last line. - */ - private lastEmittedLine?: Line; - - /** - * Process data received from the input stream. - */ - protected override onStreamData(chunk: VSBuffer): void { - this.buffer = VSBuffer.concat([this.buffer, chunk]); - - this.processData(false); - } - - /** - * Process buffered data. - * - * @param streamEnded Flag that indicates if the input stream has ended, - * which means that is the last call of this method. - * @throws If internal logic implementation error is detected. - */ - private processData( - streamEnded: boolean, - ): void { - // iterate over each line of the data buffer, emitting each line - // as a `Line` token followed by a `NewLine` token, if applies - while (this.buffer.byteLength > 0) { - // get line number based on a previously emitted line, if any - const lineNumber = this.lastEmittedLine - ? this.lastEmittedLine.range.startLineNumber + 1 - : 1; - - // find the `\r`, `\n`, or `\r\n` tokens in the data - const endOfLineTokens = this.findEndOfLineTokens( - lineNumber, - streamEnded, - ); - const firstToken: (NewLine | CarriageReturn | undefined) = endOfLineTokens[0]; - - // if no end-of-the-line tokens found, stop the current processing - // attempt because we either (1) need more data to be received or - // (2) the stream has ended; in the case (2) remaining data must - // be emitted as the last line - if (firstToken === undefined) { - // (2) if `streamEnded`, we need to emit the whole remaining - // data as the last line immediately - if (streamEnded) { - this.emitLine(lineNumber, this.buffer.slice(0)); - } - - break; - } - - // emit the line found in the data as the `Line` token - this.emitLine(lineNumber, this.buffer.slice(0, firstToken.range.startColumn - 1)); - - // must always hold true as the `emitLine` above sets this - assertDefined( - this.lastEmittedLine, - 'No last emitted line found.', - ); - - // Note! A standalone `\r` token case is not a well-defined case, and - // was primarily used by old Mac OSx systems which treated it as - // a line ending (same as `\n`). Hence for backward compatibility - // with those systems, we treat it as a new line token as well. - // We do that by replacing standalone `\r` token with `\n` one. - if ((endOfLineTokens.length === 1) && (firstToken instanceof CarriageReturn)) { - endOfLineTokens.splice(0, 1, new NewLine(firstToken.range)); - } - - // emit the end-of-the-line tokens - let startColumn = this.lastEmittedLine.range.endColumn; - for (const token of endOfLineTokens) { - const byteLength = token.byte.byteLength; - const endColumn = startColumn + byteLength; - // emit the token updating its column start/end numbers based on - // the emitted line text length and previous end-of-the-line token - this._onData.fire(token.withRange({ startColumn, endColumn })); - // shorten the data buffer by the length of the token - this.buffer = this.buffer.slice(byteLength); - // update the start column for the next token - startColumn = endColumn; - } - } - - // if the stream has ended, assert that the input data buffer is now empty - // otherwise we have a logic error and leaving some buffered data behind - if (streamEnded) { - assert( - this.buffer.byteLength === 0, - 'Expected the input data buffer to be empty when the stream ends.', - ); - } - } - - /** - * Find the end of line tokens in the data buffer. - * Can return: - * - [`\r`, `\n`] tokens if the sequence is found - * - [`\r`] token if only the carriage return is found - * - [`\n`] token if only the newline is found - * - an `empty array` if no end of line tokens found - */ - private findEndOfLineTokens( - lineNumber: number, - streamEnded: boolean, - ): (CarriageReturn | NewLine)[] { - const result = []; - - // find the first occurrence of the carriage return and newline tokens - const carriageReturnIndex = this.buffer.indexOf(CarriageReturn.byte); - const newLineIndex = this.buffer.indexOf(NewLine.byte); - - // if the `\r` comes before the `\n`(if `\n` present at all) - if (carriageReturnIndex >= 0 && ((carriageReturnIndex < newLineIndex) || (newLineIndex === -1))) { - // add the carriage return token first - result.push( - new CarriageReturn(new Range( - lineNumber, - (carriageReturnIndex + 1), - lineNumber, - (carriageReturnIndex + 1) + CarriageReturn.byte.byteLength, - )), - ); - - // if the `\r\n` sequence - if (newLineIndex === carriageReturnIndex + 1) { - // add the newline token to the result - result.push( - new NewLine(new Range( - lineNumber, - (newLineIndex + 1), - lineNumber, - (newLineIndex + 1) + NewLine.byte.byteLength, - )), - ); - } - - // either `\r` or `\r\n` cases found; if we have the `\r` token, we can return - // the end-of-line tokens only, if the `\r` is followed by at least one more - // character (it could be a `\n` or any other character), or if the stream has - // ended (which means the `\r` is at the end of the line) - if ((this.buffer.byteLength > carriageReturnIndex + 1) || streamEnded) { - return result; - } - - // in all other cases, return the empty array (no lend-of-line tokens found) - return []; - } - - // no `\r`, but there is `\n` - if (newLineIndex >= 0) { - result.push( - new NewLine(new Range( - lineNumber, - (newLineIndex + 1), - lineNumber, - (newLineIndex + 1) + NewLine.byte.byteLength, - )), - ); - } - - // neither `\r` nor `\n` found, no end of line found at all - return result; - } - - /** - * Emit a provided line as the `Line` token to the output stream. - */ - private emitLine( - lineNumber: number, // Note! 1-based indexing - lineBytes: VSBuffer, - ): void { - - const line = new Line(lineNumber, lineBytes.toString()); - this._onData.fire(line); - - // store the last emitted line so we can use it when we need - // to send the remaining line in the `onStreamEnd` method - this.lastEmittedLine = line; - - // shorten the data buffer by the length of the line emitted - this.buffer = this.buffer.slice(lineBytes.byteLength); - } - - /** - * Handle the end of the input stream - if the buffer still has some data, - * emit it as the last available line token before firing the `onEnd` event. - */ - protected override onStreamEnd(): void { - // if the input data buffer is not empty when the input stream ends, emit - // the remaining data as the last line before firing the `onEnd` event - if (this.buffer.byteLength > 0) { - this.processData(true); - } - - super.onStreamEnd(); - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/linesCodec/tokens/carriageReturn.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/linesCodec/tokens/carriageReturn.ts deleted file mode 100644 index 46b3031829e..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/linesCodec/tokens/carriageReturn.ts +++ /dev/null @@ -1,44 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { VSBuffer } from '../../../../../../../../../base/common/buffer.js'; -import { SimpleToken } from '../../simpleCodec/tokens/simpleToken.js'; - -/** - * Token that represent a `carriage return` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class CarriageReturn extends SimpleToken<'\r'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '\r' = '\r'; - - /** - * The byte representation of the {@link symbol}. - */ - public static readonly byte = VSBuffer.fromString(CarriageReturn.symbol); - - /** - * The byte representation of the token. - */ - public get byte(): VSBuffer { - return CarriageReturn.byte; - } - - /** - * Return text representation of the token. - */ - public override get text(): '\r' { - return CarriageReturn.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `CR${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/linesCodec/tokens/line.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/linesCodec/tokens/line.ts deleted file mode 100644 index 2fa0afef6ff..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/linesCodec/tokens/line.ts +++ /dev/null @@ -1,48 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../../baseToken.js'; -import { assert } from '../../../../../../../../../base/common/assert.js'; -import { Range } from '../../../../../../../../../editor/common/core/range.js'; - -/** - * Token representing a line of text with a `range` which - * reflects the line's position in the original data. - */ -export class Line extends BaseToken { - constructor( - // the line index - // Note! 1-based indexing - lineNumber: number, - // the line contents - public readonly text: string, - ) { - assert( - !isNaN(lineNumber), - `The line number must not be a NaN.`, - ); - - assert( - lineNumber > 0, - `The line number must be >= 1, got "${lineNumber}".`, - ); - - super( - new Range( - lineNumber, - 1, - lineNumber, - text.length + 1, - ), - ); - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `line("${this.shortText()}")${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/linesCodec/tokens/newLine.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/linesCodec/tokens/newLine.ts deleted file mode 100644 index 6d95751721d..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/linesCodec/tokens/newLine.ts +++ /dev/null @@ -1,44 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { VSBuffer } from '../../../../../../../../../base/common/buffer.js'; -import { SimpleToken } from '../../simpleCodec/tokens/simpleToken.js'; - -/** - * A token that represent a `new line` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class NewLine extends SimpleToken<'\n'> { - /** - * The underlying symbol of the `NewLine` token. - */ - public static override readonly symbol: '\n' = '\n'; - - /** - * The byte representation of the {@link symbol}. - */ - public static readonly byte = VSBuffer.fromString(NewLine.symbol); - - /** - * Return text representation of the token. - */ - public override get text(): '\n' { - return NewLine.symbol; - } - - /** - * The byte representation of the token. - */ - public get byte(): VSBuffer { - return NewLine.byte; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `newline${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/markdownDecoder.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/markdownDecoder.ts deleted file mode 100644 index 67bbda1f2ec..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/markdownDecoder.ts +++ /dev/null @@ -1,136 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { MarkdownToken } from './tokens/markdownToken.js'; -import { VSBuffer } from '../../../../../../../../base/common/buffer.js'; -import { LeftBracket } from '../simpleCodec/tokens/brackets.js'; -import { PartialMarkdownImage } from './parsers/markdownImage.js'; -import { ReadableStream } from '../../../../../../../../base/common/stream.js'; -import { TSimpleDecoderToken } from '../simpleCodec/simpleDecoder.js'; -import { LeftAngleBracket } from '../simpleCodec/tokens/angleBrackets.js'; -import { ExclamationMark } from '../simpleCodec/tokens/exclamationMark.js'; -import { BaseDecoder } from '../baseDecoder.js'; -import { MarkdownCommentStart, PartialMarkdownCommentStart } from './parsers/markdownComment.js'; -import { MarkdownExtensionsDecoder } from '../markdownExtensionsCodec/markdownExtensionsDecoder.js'; -import { MarkdownLinkCaption, PartialMarkdownLink, PartialMarkdownLinkCaption } from './parsers/markdownLink.js'; - -/** - * Tokens produced by this decoder. - */ -export type TMarkdownToken = MarkdownToken | TSimpleDecoderToken; - -/** - * Decoder capable of parsing markdown entities (e.g., links) from a sequence of simple tokens. - */ -export class MarkdownDecoder extends BaseDecoder { - /** - * Current parser object that is responsible for parsing a sequence of tokens into - * some markdown entity. Set to `undefined` when no parsing is in progress at the moment. - */ - private current?: - PartialMarkdownLinkCaption | MarkdownLinkCaption | PartialMarkdownLink | - PartialMarkdownCommentStart | MarkdownCommentStart | - PartialMarkdownImage; - - constructor( - stream: ReadableStream, - ) { - super(new MarkdownExtensionsDecoder(stream)); - } - - protected override onStreamData(token: TSimpleDecoderToken): void { - // `markdown links` start with `[` character, so here we can - // initiate the process of parsing a markdown link - if (token instanceof LeftBracket && !this.current) { - this.current = new PartialMarkdownLinkCaption(token); - - return; - } - - // `markdown comments` start with `<` character, so here we can - // initiate the process of parsing a markdown comment - if (token instanceof LeftAngleBracket && !this.current) { - this.current = new PartialMarkdownCommentStart(token); - - return; - } - - // `markdown image links` start with `!` character, so here we can - // initiate the process of parsing a markdown image - if (token instanceof ExclamationMark && !this.current) { - this.current = new PartialMarkdownImage(token); - - return; - } - - // if current parser was not initiated before, - we are not inside a sequence - // of tokens we care about, therefore re-emit the token immediately and continue - if (!this.current) { - this._onData.fire(token); - return; - } - - // if there is a current parser object, submit the token to it - // so it can progress with parsing the tokens sequence - const parseResult = this.current.accept(token); - if (parseResult.result === 'success') { - const { nextParser } = parseResult; - - // if got a fully parsed out token back, emit it and reset - // the current parser object so a new parsing process can start - if (nextParser instanceof MarkdownToken) { - this._onData.fire(nextParser); - delete this.current; - } else { - // otherwise, update the current parser object - this.current = nextParser; - } - } else { - // if failed to parse a sequence of a tokens as a single markdown - // entity (e.g., a link), re-emit the tokens accumulated so far - // then reset the current parser object - for (const currentToken of this.current.tokens) { - this._onData.fire(currentToken); - } - - delete this.current; - } - - // if token was not consumed by the parser, call `onStreamData` again - // so the token is properly handled by the decoder in the case when a - // new sequence starts with this token - if (!parseResult.wasTokenConsumed) { - this.onStreamData(token); - } - } - - protected override onStreamEnd(): void { - // if the stream has ended and there is a current incomplete parser - // object present, handle the remaining parser object - if (this.current) { - // if a `markdown comment` does not have an end marker `-->` - // it is still a comment that extends to the end of the file - // so re-emit the current parser as a comment token - if (this.current instanceof MarkdownCommentStart) { - this._onData.fire(this.current.asMarkdownComment()); - delete this.current; - this.onStreamEnd(); - - return; - } - - // in all other cases, re-emit existing parser tokens - const { tokens } = this.current; - - for (const token of [...tokens]) { - this._onData.fire(token); - } - - delete this.current; - } - - super.onStreamEnd(); - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/parsers/markdownComment.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/parsers/markdownComment.ts deleted file mode 100644 index a017649f8b8..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/parsers/markdownComment.ts +++ /dev/null @@ -1,173 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { Range } from '../../../../../../../../../editor/common/core/range.js'; -import { BaseToken } from '../../baseToken.js'; -import { Dash } from '../../simpleCodec/tokens/dash.js'; -import { assert } from '../../../../../../../../../base/common/assert.js'; -import { MarkdownComment } from '../tokens/markdownComment.js'; -import { TSimpleDecoderToken } from '../../simpleCodec/simpleDecoder.js'; -import { ExclamationMark } from '../../simpleCodec/tokens/exclamationMark.js'; -import { LeftAngleBracket, RightAngleBracket } from '../../simpleCodec/tokens/angleBrackets.js'; -import { assertNotConsumed, ParserBase, TAcceptTokenResult } from '../../simpleCodec/parserBase.js'; - -/** - * The parser responsible for parsing the ``. If it does, - * then the parser transitions to the {@link MarkdownComment} token. - */ -export class MarkdownCommentStart extends ParserBase { - constructor(tokens: [LeftAngleBracket, ExclamationMark, Dash, Dash]) { - super(tokens); - } - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - // if received `>` while current token sequence ends with `--`, - // then this is the end of the comment sequence - if (token instanceof RightAngleBracket && this.endsWithDashes) { - this.currentTokens.push(token); - - return { - result: 'success', - nextParser: this.asMarkdownComment(), - wasTokenConsumed: true, - }; - } - - this.currentTokens.push(token); - - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } - - /** - * Convert the current token sequence into a {@link MarkdownComment} token. - * - * Note! that this method marks the current parser object as "consumed" - * hence it should not be used after this method is called. - */ - public asMarkdownComment(): MarkdownComment { - this.isConsumed = true; - - return new MarkdownComment( - this.range, - BaseToken.render(this.currentTokens), - ); - } - - /** - * Get range of current token sequence. - */ - private get range(): Range { - const firstToken = this.currentTokens[0]; - const lastToken = this.currentTokens[this.currentTokens.length - 1]; - - const range = new Range( - firstToken.range.startLineNumber, - firstToken.range.startColumn, - lastToken.range.endLineNumber, - lastToken.range.endColumn, - ); - - return range; - } - - /** - * Whether the current token sequence ends with two dashes. - */ - private get endsWithDashes(): boolean { - const lastToken = this.currentTokens[this.currentTokens.length - 1]; - if (!(lastToken instanceof Dash)) { - return false; - } - - const secondLastToken = this.currentTokens[this.currentTokens.length - 2]; - if (!(secondLastToken instanceof Dash)) { - return false; - } - - return true; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/parsers/markdownImage.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/parsers/markdownImage.ts deleted file mode 100644 index 5684f48de4a..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/parsers/markdownImage.ts +++ /dev/null @@ -1,99 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { MarkdownLink } from '../tokens/markdownLink.js'; -import { MarkdownImage } from '../tokens/markdownImage.js'; -import { LeftBracket } from '../../simpleCodec/tokens/brackets.js'; -import { TSimpleDecoderToken } from '../../simpleCodec/simpleDecoder.js'; -import { ExclamationMark } from '../../simpleCodec/tokens/exclamationMark.js'; -import { assertNotConsumed, ParserBase, TAcceptTokenResult } from '../../simpleCodec/parserBase.js'; -import { MarkdownLinkCaption, PartialMarkdownLink, PartialMarkdownLinkCaption } from './markdownLink.js'; - -/** - * The parser responsible for parsing the `markdown image` sequence of characters. - * E.g., `![alt text](./path/to/image.jpeg)` syntax. - */ -export class PartialMarkdownImage extends ParserBase { - /** - * Current active parser instance, if in the mode of actively parsing the markdown link sequence. - */ - private markdownLinkParser: PartialMarkdownLinkCaption | MarkdownLinkCaption | PartialMarkdownLink | undefined; - - constructor(token: ExclamationMark) { - super([token]); - } - - /** - * Get all currently available tokens of the `markdown link` sequence. - */ - public override get tokens(): readonly TSimpleDecoderToken[] { - const linkTokens = this.markdownLinkParser?.tokens ?? []; - - return [ - ...this.currentTokens, - ...linkTokens, - ]; - } - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - // on the first call we expect a character that begins `markdown link` sequence - // hence we initiate the markdown link parsing process, otherwise we fail - if (!this.markdownLinkParser) { - if (token instanceof LeftBracket) { - this.markdownLinkParser = new PartialMarkdownLinkCaption(token); - - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } - - return { - result: 'failure', - wasTokenConsumed: false, - }; - } - - // handle subsequent tokens next - - const acceptResult = this.markdownLinkParser.accept(token); - const { result, wasTokenConsumed } = acceptResult; - - if (result === 'success') { - const { nextParser } = acceptResult; - - // if full markdown link was parsed out, the process completes - if (nextParser instanceof MarkdownLink) { - this.isConsumed = true; - - const firstToken = this.currentTokens[0]; - return { - result, - wasTokenConsumed, - nextParser: new MarkdownImage( - firstToken.range.startLineNumber, - firstToken.range.startColumn, - `${firstToken.text}${nextParser.caption}`, - nextParser.reference, - ), - }; - } - - // otherwise save new link parser reference and continue - this.markdownLinkParser = nextParser; - return { - result, - wasTokenConsumed, - nextParser: this, - }; - } - - // return the failure result - this.isConsumed = true; - return acceptResult; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/parsers/markdownLink.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/parsers/markdownLink.ts deleted file mode 100644 index 641ddd14b9b..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/parsers/markdownLink.ts +++ /dev/null @@ -1,211 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../../baseToken.js'; -import { MarkdownLink } from '../tokens/markdownLink.js'; -import { NewLine } from '../../linesCodec/tokens/newLine.js'; -import { assert } from '../../../../../../../../../base/common/assert.js'; -import { FormFeed } from '../../simpleCodec/tokens/formFeed.js'; -import { VerticalTab } from '../../simpleCodec/tokens/verticalTab.js'; -import { TSimpleDecoderToken } from '../../simpleCodec/simpleDecoder.js'; -import { CarriageReturn } from '../../linesCodec/tokens/carriageReturn.js'; -import { LeftBracket, RightBracket } from '../../simpleCodec/tokens/brackets.js'; -import { ParserBase, TAcceptTokenResult } from '../../simpleCodec/parserBase.js'; -import { LeftParenthesis, RightParenthesis } from '../../simpleCodec/tokens/parentheses.js'; - -/** - * List of characters that are not allowed in links so stop a markdown link sequence abruptly. - */ -const MARKDOWN_LINK_STOP_CHARACTERS: readonly string[] = [CarriageReturn, NewLine, VerticalTab, FormFeed] - .map((token) => { return token.symbol; }); - -/** - * The parser responsible for parsing a `markdown link caption` part of a markdown - * link (e.g., the `[caption text]` part of the `[caption text](./some/path)` link). - * - * The parsing process starts with single `[` token and collects all tokens until - * the first `]` token is encountered. In this successful case, the parser transitions - * into the {@link MarkdownLinkCaption} parser type which continues the general - * parsing process of the markdown link. - * - * Otherwise, if one of the stop characters defined in the {@link MARKDOWN_LINK_STOP_CHARACTERS} - * is encountered before the `]` token, the parsing process is aborted which is communicated to - * the caller by returning a `failure` result. In this case, the caller is assumed to be responsible - * for re-emitting the {@link tokens} accumulated so far as standalone entities since they are no - * longer represent a coherent token entity of a larger size. - */ -export class PartialMarkdownLinkCaption extends ParserBase { - constructor(token: LeftBracket) { - super([token]); - } - - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - // any of stop characters is are breaking a markdown link caption sequence - if (MARKDOWN_LINK_STOP_CHARACTERS.includes(token.text)) { - return { - result: 'failure', - wasTokenConsumed: false, - }; - } - - // the `]` character ends the caption of a markdown link - if (token instanceof RightBracket) { - return { - result: 'success', - nextParser: new MarkdownLinkCaption([...this.tokens, token]), - wasTokenConsumed: true, - }; - } - - // otherwise, include the token in the sequence - // and keep the current parser object instance - this.currentTokens.push(token); - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } -} - -/** - * The parser responsible for transitioning from a {@link PartialMarkdownLinkCaption} - * parser to the {@link PartialMarkdownLink} one, therefore serves a parser glue between - * the `[caption]` and the `(./some/path)` parts of the `[caption](./some/path)` link. - * - * The only successful case of this parser is the `(` token that initiated the process - * of parsing the `reference` part of a markdown link and in this case the parser - * transitions into the `PartialMarkdownLink` parser type. - * - * Any other character is considered a failure result. In this case, the caller is assumed - * to be responsible for re-emitting the {@link tokens} accumulated so far as standalone - * entities since they are no longer represent a coherent token entity of a larger size. - */ -export class MarkdownLinkCaption extends ParserBase { - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - // the `(` character starts the link part of a markdown link - // that is the only character that can follow the caption - if (token instanceof LeftParenthesis) { - return { - result: 'success', - wasTokenConsumed: true, - nextParser: new PartialMarkdownLink([...this.tokens], token), - }; - } - - return { - result: 'failure', - wasTokenConsumed: false, - }; - } -} - -/** - * The parser responsible for parsing a `link reference` part of a markdown link - * (e.g., the `(./some/path)` part of the `[caption text](./some/path)` link). - * - * The parsing process starts with tokens that represent the `[caption]` part of a markdown - * link, followed by the `(` token. The parser collects all subsequent tokens until final closing - * parenthesis (`)`) is encountered (*\*see [1] below*). In this successful case, the parser object - * transitions into the {@link MarkdownLink} token type which signifies the end of the entire - * parsing process of the link text. - * - * Otherwise, if one of the stop characters defined in the {@link MARKDOWN_LINK_STOP_CHARACTERS} - * is encountered before the final `)` token, the parsing process is aborted which is communicated to - * the caller by returning a `failure` result. In this case, the caller is assumed to be responsible - * for re-emitting the {@link tokens} accumulated so far as standalone entities since they are no - * longer represent a coherent token entity of a larger size. - * - * `[1]` The `reference` part of the markdown link can contain any number of nested parenthesis, e.g., - * `[caption](/some/p(th/file.md)` is a valid markdown link and a valid folder name, hence number - * of open parenthesis must match the number of closing ones and the path sequence is considered - * to be complete as soon as this requirement is met. Therefore the `final` word is used in - * the description comments above to highlight this important detail. - */ -export class PartialMarkdownLink extends ParserBase { - /** - * Number of open parenthesis in the sequence. - * See comment in the {@link accept} method for more details. - */ - private openParensCount: number = 1; - - constructor( - protected readonly captionTokens: TSimpleDecoderToken[], - token: LeftParenthesis, - ) { - super([token]); - } - - public override get tokens(): readonly TSimpleDecoderToken[] { - return [...this.captionTokens, ...this.currentTokens]; - } - - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - // markdown links allow for nested parenthesis inside the link reference part, but - // the number of open parenthesis must match the number of closing parenthesis, e.g.: - // - `[caption](/some/p()th/file.md)` is a valid markdown link - // - `[caption](/some/p(th/file.md)` is an invalid markdown link - // hence we use the `openParensCount` variable to keep track of the number of open - // parenthesis encountered so far; then upon encountering a closing parenthesis we - // decrement the `openParensCount` and if it reaches 0 - we consider the link reference - // to be complete - - if (token instanceof LeftParenthesis) { - this.openParensCount += 1; - } - - if (token instanceof RightParenthesis) { - this.openParensCount -= 1; - - // sanity check! this must alway hold true because we return a complete markdown - // link as soon as we encounter matching number of closing parenthesis, hence - // we must never have `openParensCount` that is less than 0 - assert( - this.openParensCount >= 0, - `Unexpected right parenthesis token encountered: '${token}'.`, - ); - - // the markdown link is complete as soon as we get the same number of closing parenthesis - if (this.openParensCount === 0) { - const { startLineNumber, startColumn } = this.captionTokens[0].range; - - // create link caption string - const caption = BaseToken.render(this.captionTokens); - - // create link reference string - this.currentTokens.push(token); - const reference = BaseToken.render(this.currentTokens); - - // return complete markdown link object - return { - result: 'success', - wasTokenConsumed: true, - nextParser: new MarkdownLink( - startLineNumber, - startColumn, - caption, - reference, - ), - }; - } - } - - // any of stop characters is are breaking a markdown link reference sequence - if (MARKDOWN_LINK_STOP_CHARACTERS.includes(token.text)) { - return { - result: 'failure', - wasTokenConsumed: false, - }; - } - - // the rest of the tokens can be included in the sequence - this.currentTokens.push(token); - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/tokens/markdownComment.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/tokens/markdownComment.ts deleted file mode 100644 index 8eadff3589a..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/tokens/markdownComment.ts +++ /dev/null @@ -1,40 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { Range } from '../../../../../../../../../editor/common/core/range.js'; -import { MarkdownToken } from './markdownToken.js'; -import { assert } from '../../../../../../../../../base/common/assert.js'; - -/** - * A token that represent a `markdown comment` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class MarkdownComment extends MarkdownToken { - constructor( - range: Range, - public readonly text: string, - ) { - assert( - text.startsWith('`. - */ - public get hasEndMarker(): boolean { - return this.text.endsWith('-->'); - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `md-comment("${this.shortText()}")${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/tokens/markdownImage.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/tokens/markdownImage.ts deleted file mode 100644 index cab120f53db..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/tokens/markdownImage.ts +++ /dev/null @@ -1,125 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { MarkdownToken } from './markdownToken.js'; -import { IRange, Range } from '../../../../../../../../../editor/common/core/range.js'; -import { assert } from '../../../../../../../../../base/common/assert.js'; - -/** - * A token that represent a `markdown image` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class MarkdownImage extends MarkdownToken { - /** - * Check if this `markdown image link` points to a valid URL address. - */ - public readonly isURL: boolean; - - constructor( - /** - * The starting line number of the image (1-based indexing). - */ - lineNumber: number, - /** - * The starting column number of the image (1-based indexing). - */ - columnNumber: number, - /** - * The caption of the image, including the `!` and `square brackets`. - */ - private readonly caption: string, - /** - * The reference of the image, including the parentheses. - */ - private readonly reference: string, - ) { - assert( - !isNaN(lineNumber), - `The line number must not be a NaN.`, - ); - - assert( - lineNumber > 0, - `The line number must be >= 1, got "${lineNumber}".`, - ); - - assert( - columnNumber > 0, - `The column number must be >= 1, got "${columnNumber}".`, - ); - - assert( - caption[0] === '!', - `The caption must start with '!' character, got "${caption}".`, - ); - - assert( - caption[1] === '[' && caption[caption.length - 1] === ']', - `The caption must be enclosed in square brackets, got "${caption}".`, - ); - - assert( - reference[0] === '(' && reference[reference.length - 1] === ')', - `The reference must be enclosed in parentheses, got "${reference}".`, - ); - - super( - new Range( - lineNumber, - columnNumber, - lineNumber, - columnNumber + caption.length + reference.length, - ), - ); - - // set up the `isURL` flag based on the current - try { - new URL(this.path); - this.isURL = true; - } catch { - this.isURL = false; - } - } - - public override get text(): string { - return `${this.caption}${this.reference}`; - } - - /** - * Returns the `reference` part of the link without enclosing parentheses. - */ - public get path(): string { - return this.reference.slice(1, this.reference.length - 1); - } - - /** - * Get the range of the `link part` of the token. - */ - public get linkRange(): IRange | undefined { - if (this.path.length === 0) { - return undefined; - } - - const { range } = this; - - // note! '+1' for openning `(` of the link - const startColumn = range.startColumn + this.caption.length + 1; - const endColumn = startColumn + this.path.length; - - return new Range( - range.startLineNumber, - startColumn, - range.endLineNumber, - endColumn, - ); - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `md-image("${this.shortText()}")${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/tokens/markdownLink.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/tokens/markdownLink.ts deleted file mode 100644 index a963628675d..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/tokens/markdownLink.ts +++ /dev/null @@ -1,120 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { MarkdownToken } from './markdownToken.js'; -import { IRange, Range } from '../../../../../../../../../editor/common/core/range.js'; -import { assert } from '../../../../../../../../../base/common/assert.js'; - -/** - * A token that represent a `markdown link` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class MarkdownLink extends MarkdownToken { - /** - * Check if this `markdown link` points to a valid URL address. - */ - public readonly isURL: boolean; - - constructor( - /** - * The starting line number of the link (1-based indexing). - */ - lineNumber: number, - /** - * The starting column number of the link (1-based indexing). - */ - columnNumber: number, - /** - * The caption of the original link, including the square brackets. - */ - public readonly caption: string, - /** - * The reference of the original link, including the parentheses. - */ - public readonly reference: string, - ) { - assert( - !isNaN(lineNumber), - `The line number must not be a NaN.`, - ); - - assert( - lineNumber > 0, - `The line number must be >= 1, got "${lineNumber}".`, - ); - - assert( - columnNumber > 0, - `The column number must be >= 1, got "${columnNumber}".`, - ); - - assert( - caption[0] === '[' && caption[caption.length - 1] === ']', - `The caption must be enclosed in square brackets, got "${caption}".`, - ); - - assert( - reference[0] === '(' && reference[reference.length - 1] === ')', - `The reference must be enclosed in parentheses, got "${reference}".`, - ); - - super( - new Range( - lineNumber, - columnNumber, - lineNumber, - columnNumber + caption.length + reference.length, - ), - ); - - // set up the `isURL` flag based on the current - try { - new URL(this.path); - this.isURL = true; - } catch { - this.isURL = false; - } - } - - public override get text(): string { - return `${this.caption}${this.reference}`; - } - - /** - * Returns the `reference` part of the link without enclosing parentheses. - */ - public get path(): string { - return this.reference.slice(1, this.reference.length - 1); - } - - /** - * Get the range of the `link part` of the token. - */ - public get linkRange(): IRange | undefined { - if (this.path.length === 0) { - return undefined; - } - - const { range } = this; - - // note! '+1' for opening `(` of the link - const startColumn = range.startColumn + this.caption.length + 1; - const endColumn = startColumn + this.path.length; - - return new Range( - range.startLineNumber, - startColumn, - range.endLineNumber, - endColumn, - ); - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `md-link("${this.shortText()}")${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/tokens/markdownToken.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/tokens/markdownToken.ts deleted file mode 100644 index fc1935d081b..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/tokens/markdownToken.ts +++ /dev/null @@ -1,12 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../../baseToken.js'; - -/** - * Common base token that all `markdown` tokens should - * inherit from. - */ -export abstract class MarkdownToken extends BaseToken { } diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/markdownExtensionsDecoder.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/markdownExtensionsDecoder.ts deleted file mode 100644 index b5ff3d2b9f9..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/markdownExtensionsDecoder.ts +++ /dev/null @@ -1,119 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { VSBuffer } from '../../../../../../../../base/common/buffer.js'; -import { ReadableStream } from '../../../../../../../../base/common/stream.js'; -import { BaseDecoder } from '../baseDecoder.js'; -import { MarkdownExtensionsToken } from './tokens/markdownExtensionsToken.js'; -import { SimpleDecoder, TSimpleDecoderToken } from '../simpleCodec/simpleDecoder.js'; -import { PartialFrontMatterHeader, PartialFrontMatterStartMarker } from './parsers/frontMatterHeader.js'; - -/** - * Tokens produced by this decoder. - */ -export type TMarkdownExtensionsToken = MarkdownExtensionsToken | TSimpleDecoderToken; - -/** - * Decoder responsible for decoding extensions of markdown syntax, - * e.g., a `Front Matter` header, etc. - */ -export class MarkdownExtensionsDecoder extends BaseDecoder { - /** - * Current parser object that is responsible for parsing a sequence of tokens into - * some markdown entity. Set to `undefined` when no parsing is in progress at the moment. - */ - private current?: PartialFrontMatterStartMarker | PartialFrontMatterHeader; - - constructor( - stream: ReadableStream, - ) { - super(new SimpleDecoder(stream)); - } - - protected override onStreamData(token: TSimpleDecoderToken): void { - // front matter headers start with a `-` at the first column of the first line - if ((this.current === undefined) && PartialFrontMatterStartMarker.mayStartHeader(token)) { - this.current = new PartialFrontMatterStartMarker(token); - - return; - } - - // if current parser is not initiated, - we are not inside a sequence of tokens - // we care about, therefore re-emit the token immediately and continue - if (this.current === undefined) { - this._onData.fire(token); - return; - } - - // if there is a current parser object, submit the token to it - // so it can progress with parsing the tokens sequence - const parseResult = this.current.accept(token); - if (parseResult.result === 'success') { - const { nextParser } = parseResult; - - // if got a fully parsed out token back, emit it and reset - // the current parser object so a new parsing process can start - if (nextParser instanceof MarkdownExtensionsToken) { - this._onData.fire(nextParser); - delete this.current; - } else { - // otherwise, update the current parser object - this.current = nextParser; - } - } else { - // if failed to parse a sequence of a tokens as a single markdown - // entity (e.g., a link), re-emit the tokens accumulated so far - // then reset the currently initialized parser object - this.reEmitCurrentTokens(); - } - - // if token was not consumed by the parser, call `onStreamData` again - // so the token is properly handled by the decoder in the case when a - // new sequence starts with this token - if (!parseResult.wasTokenConsumed) { - this.onStreamData(token); - } - } - - protected override onStreamEnd(): void { - try { - if (this.current === undefined) { - return; - } - - // if current parser can be converted into a valid Front Matter - // header, then emit it and reset the current parser object - if (this.current instanceof PartialFrontMatterHeader) { - this._onData.fire( - this.current.asFrontMatterHeader(), - ); - delete this.current; - return; - } - - } catch { - // if failed to convert current parser object to a token, - // re-emit the tokens accumulated so far - this.reEmitCurrentTokens(); - } finally { - delete this.current; - super.onStreamEnd(); - } - } - - /** - * Re-emit tokens accumulated so far in the current parser object. - */ - protected reEmitCurrentTokens(): void { - if (this.current === undefined) { - return; - } - - for (const token of this.current.tokens) { - this._onData.fire(token); - } - delete this.current; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/parsers/frontMatterHeader.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/parsers/frontMatterHeader.ts deleted file mode 100644 index a63c911174c..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/parsers/frontMatterHeader.ts +++ /dev/null @@ -1,345 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { Dash } from '../../simpleCodec/tokens/dash.js'; -import { NewLine } from '../../linesCodec/tokens/newLine.js'; -import { FrontMatterHeader } from '../tokens/frontMatterHeader.js'; -import { assertDefined } from '../../../../../../../../../base/common/types.js'; -import { TSimpleDecoderToken } from '../../simpleCodec/simpleDecoder.js'; -import { assert, assertNever } from '../../../../../../../../../base/common/assert.js'; -import { CarriageReturn } from '../../linesCodec/tokens/carriageReturn.js'; -import { FrontMatterMarker, TMarkerToken } from '../tokens/frontMatterMarker.js'; -import { assertNotConsumed, IAcceptTokenSuccess, ParserBase, TAcceptTokenResult } from '../../simpleCodec/parserBase.js'; - -/** - * Parses the start marker of a Front Matter header. - */ -export class PartialFrontMatterStartMarker extends ParserBase { - constructor(token: Dash) { - const { range } = token; - - assert( - range.startLineNumber === 1, - `Front Matter header must start at the first line, but it starts at line #${range.startLineNumber}.`, - ); - - assert( - range.startColumn === 1, - `Front Matter header must start at the beginning of the line, but it starts at ${range.startColumn}.`, - ); - - super([token]); - } - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - const previousToken = this.currentTokens[this.currentTokens.length - 1]; - - // collect a sequence of dash tokens that may end with a CR token - if ((token instanceof Dash) || (token instanceof CarriageReturn)) { - // a dash or CR tokens can go only after another dash token - if ((previousToken instanceof Dash) === false) { - this.isConsumed = true; - - return { - result: 'failure', - wasTokenConsumed: false, - }; - } - - this.currentTokens.push(token); - - return { - result: 'success', - wasTokenConsumed: true, - nextParser: this, - }; - } - - // stop collecting dash tokens when a new line token is encountered - if (token instanceof NewLine) { - this.isConsumed = true; - - return { - result: 'success', - wasTokenConsumed: true, - nextParser: new PartialFrontMatterHeader( - FrontMatterMarker.fromTokens([ - ...this.currentTokens, - token, - ]), - ), - }; - } - - // any other token is invalid for the `start marker` - this.isConsumed = true; - return { - result: 'failure', - wasTokenConsumed: false, - }; - } - - /** - * Check if provided dash token can be a start of a Front Matter header. - */ - public static mayStartHeader(token: TSimpleDecoderToken): token is Dash { - return (token instanceof Dash) - && (token.range.startLineNumber === 1) - && (token.range.startColumn === 1); - } -} - -/** - * Parses a Front Matter header that already has a start marker - * and possibly some content that follows. - */ -export class PartialFrontMatterHeader extends ParserBase { - /** - * Parser instance for the end marker of the Front Matter header. - */ - private maybeEndMarker?: PartialFrontMatterEndMarker; - - constructor( - public readonly startMarker: FrontMatterMarker, - ) { - super([]); - } - - public override get tokens(): readonly TSimpleDecoderToken[] { - const endMarkerTokens = (this.maybeEndMarker !== undefined) - ? this.maybeEndMarker.tokens - : []; - - return [ - ...this.startMarker.tokens, - ...this.currentTokens, - ...endMarkerTokens, - ]; - } - - /** - * Convert the current token sequence into a {@link FrontMatterHeader} token. - * - * Note! that this method marks the current parser object as "consumed" - * hence it should not be used after this method is called. - */ - public asFrontMatterHeader(): FrontMatterHeader { - assertDefined( - this.maybeEndMarker, - 'Cannot convert to Front Matter header token without an end marker.', - ); - - assert( - this.maybeEndMarker.dashCount === this.startMarker.dashTokens.length, - [ - 'Start and end markers must have the same number of dashes', - `, got ${this.startMarker.dashTokens.length} / ${this.maybeEndMarker.dashCount}.`, - ].join(''), - ); - - this.isConsumed = true; - - return FrontMatterHeader.fromTokens( - this.startMarker.tokens, - this.currentTokens, - this.maybeEndMarker.tokens, - ); - } - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - // if in the mode of parsing the end marker sequence, forward - // the token to the current end marker parser instance - if (this.maybeEndMarker !== undefined) { - return this.acceptEndMarkerToken(token); - } - - // collect all tokens until a `dash token at the beginning of a line` is found - if (((token instanceof Dash) === false) || (token.range.startColumn !== 1)) { - this.currentTokens.push(token); - - return { - result: 'success', - wasTokenConsumed: true, - nextParser: this, - }; - } - - // a dash token at the beginning of the line might be a start of the `end marker` - // sequence of the front matter header, hence initialize appropriate parser object - assert( - this.maybeEndMarker === undefined, - `End marker parser must not be present.`, - ); - this.maybeEndMarker = new PartialFrontMatterEndMarker(token); - - return { - result: 'success', - wasTokenConsumed: true, - nextParser: this, - }; - } - - /** - * When a end marker parser is present, we pass all tokens to it - * until it is completes the parsing process(either success or failure). - */ - private acceptEndMarkerToken( - token: TSimpleDecoderToken, - ): TAcceptTokenResult { - assertDefined( - this.maybeEndMarker, - `Partial end marker parser must be initialized.`, - ); - - // if we have a partial end marker, we are in the process of parsing - // the end marker, so just pass the token to it and return - const acceptResult = this.maybeEndMarker.accept(token); - const { result, wasTokenConsumed } = acceptResult; - - if (result === 'success') { - const { nextParser } = acceptResult; - const endMarkerParsingComplete = (nextParser instanceof FrontMatterMarker); - - if (endMarkerParsingComplete === false) { - return { - result: 'success', - wasTokenConsumed, - nextParser: this, - }; - } - - const endMarker = nextParser; - - // start and end markers must have the same number of dashes, hence - // if they don't match, we would like to continue parsing the header - // until we find an end marker with the same number of dashes - if (endMarker.dashTokens.length !== this.startMarker.dashTokens.length) { - return this.handleEndMarkerParsingFailure( - endMarker.tokens, - wasTokenConsumed, - ); - } - - this.isConsumed = true; - return { - result: 'success', - wasTokenConsumed: true, - nextParser: FrontMatterHeader.fromTokens( - this.startMarker.tokens, - this.currentTokens, - this.maybeEndMarker.tokens, - ), - }; - } - - // if failed to parse the end marker, we would like to continue parsing - // the header until we find a valid end marker - if (result === 'failure') { - return this.handleEndMarkerParsingFailure( - this.maybeEndMarker.tokens, - wasTokenConsumed, - ); - } - - assertNever( - result, - `Unexpected result '${result}' while parsing the end marker.`, - ); - } - - /** - * On failure to parse the end marker, we need to continue parsing - * the header because there might be another valid end marker in - * the stream of tokens. Therefore we copy over the end marker tokens - * into the list of "content" tokens and reset the end marker parser. - */ - private handleEndMarkerParsingFailure( - tokens: readonly TSimpleDecoderToken[], - wasTokenConsumed: boolean, - ): IAcceptTokenSuccess { - this.currentTokens.push(...tokens); - delete this.maybeEndMarker; - - return { - result: 'success', - wasTokenConsumed, - nextParser: this, - }; - } -} - -/** - * Parser the end marker sequence of a Front Matter header. - */ -class PartialFrontMatterEndMarker extends ParserBase { - constructor(token: Dash) { - const { range } = token; - - assert( - range.startColumn === 1, - `Front Matter header must start at the beginning of the line, but it starts at ${range.startColumn}.`, - ); - - super([token]); - } - - /** - * Number of dashes in the marker. - */ - public get dashCount(): number { - return this.tokens - .filter((token) => { return token instanceof Dash; }) - .length; - } - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - const previousToken = this.currentTokens[this.currentTokens.length - 1]; - - // collect a sequence of dash tokens that may end with a CR token - if ((token instanceof Dash) || (token instanceof CarriageReturn)) { - // a dash or CR tokens can go only after another dash token - if ((previousToken instanceof Dash) === false) { - this.isConsumed = true; - - return { - result: 'failure', - wasTokenConsumed: false, - }; - } - - this.currentTokens.push(token); - - return { - result: 'success', - wasTokenConsumed: true, - nextParser: this, - }; - } - - // stop collecting dash tokens when a new line token is encountered - if (token instanceof NewLine) { - this.isConsumed = true; - this.currentTokens.push(token); - - return { - result: 'success', - wasTokenConsumed: true, - nextParser: FrontMatterMarker.fromTokens([ - ...this.currentTokens, - ]), - }; - } - - // any other token is invalid for the `start marker` - this.isConsumed = true; - return { - result: 'failure', - wasTokenConsumed: false, - }; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/tokens/frontMatterHeader.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/tokens/frontMatterHeader.ts deleted file mode 100644 index b5b3903b20a..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/tokens/frontMatterHeader.ts +++ /dev/null @@ -1,79 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { Text } from '../../textToken.js'; -import { Range } from '../../../../../../../../../editor/common/core/range.js'; -import { BaseToken } from '../../baseToken.js'; -import { MarkdownExtensionsToken } from './markdownExtensionsToken.js'; -import { TSimpleDecoderToken } from '../../simpleCodec/simpleDecoder.js'; -import { FrontMatterMarker, TMarkerToken } from './frontMatterMarker.js'; - -/** - * Token that represents a `Front Matter` header in a text. - */ -export class FrontMatterHeader extends MarkdownExtensionsToken { - constructor( - range: Range, - public readonly startMarker: FrontMatterMarker, - public readonly content: Text, - public readonly endMarker: FrontMatterMarker, - ) { - super(range); - } - - /** - * Return complete text representation of the token. - */ - public get text(): string { - const text: string[] = [ - this.startMarker.text, - this.content.text, - this.endMarker.text, - ]; - - return text.join(''); - } - - /** - * Range of the content of the Front Matter header. - */ - public get contentRange(): Range { - return this.content.range; - } - - /** - * Content token of the Front Matter header. - */ - public get contentToken(): Text { - return this.content; - } - - /** - * Create new instance of the token from the given tokens. - */ - public static fromTokens( - startMarkerTokens: readonly TMarkerToken[], - contentTokens: readonly TSimpleDecoderToken[], - endMarkerTokens: readonly TMarkerToken[], - ): FrontMatterHeader { - const range = BaseToken.fullRange( - [...startMarkerTokens, ...endMarkerTokens], - ); - - return new FrontMatterHeader( - range, - FrontMatterMarker.fromTokens(startMarkerTokens), - new Text(contentTokens), - FrontMatterMarker.fromTokens(endMarkerTokens), - ); - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `frontmatter("${this.shortText()}")${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/tokens/frontMatterMarker.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/tokens/frontMatterMarker.ts deleted file mode 100644 index 14813b3116b..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/tokens/frontMatterMarker.ts +++ /dev/null @@ -1,59 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { Range } from '../../../../../../../../../editor/common/core/range.js'; -import { BaseToken } from '../../baseToken.js'; -import { Dash } from '../../simpleCodec/tokens/dash.js'; -import { NewLine } from '../../linesCodec/tokens/newLine.js'; -import { MarkdownExtensionsToken } from './markdownExtensionsToken.js'; -import { CarriageReturn } from '../../linesCodec/tokens/carriageReturn.js'; - -/** - * Type for tokens inside a Front Matter header marker. - */ -export type TMarkerToken = Dash | CarriageReturn | NewLine; - -/** - * Marker for the start and end of a Front Matter header. - */ -export class FrontMatterMarker extends MarkdownExtensionsToken { - /** - * Returns complete text representation of the token. - */ - public get text(): string { - return BaseToken.render(this.tokens); - } - - /** - * List of {@link Dash} tokens in the marker. - */ - public get dashTokens(): readonly Dash[] { - return this.tokens - .filter((token) => { return token instanceof Dash; }); - } - - constructor( - range: Range, - public readonly tokens: readonly TMarkerToken[], - ) { - super(range); - } - - /** - * Create new instance of the token from a provided - * list of tokens. - */ - public static fromTokens( - tokens: readonly TMarkerToken[], - ): FrontMatterMarker { - const range = BaseToken.fullRange(tokens); - - return new FrontMatterMarker(range, tokens); - } - - public toString(): string { - return `frontmatter-marker(${this.dashTokens.length}:${this.range})`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/tokens/markdownExtensionsToken.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/tokens/markdownExtensionsToken.ts deleted file mode 100644 index 82046eb2b4d..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/tokens/markdownExtensionsToken.ts +++ /dev/null @@ -1,11 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { MarkdownToken } from '../../markdownCodec/tokens/markdownToken.js'; - -/** - * Base class for all tokens produced by the `MarkdownExtensionsDecoder`. - */ -export abstract class MarkdownExtensionsToken extends MarkdownToken { } diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/parserBase.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/parserBase.ts deleted file mode 100644 index 84515dc1d6c..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/parserBase.ts +++ /dev/null @@ -1,137 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../baseToken.js'; -import { assert } from '../../../../../../../../base/common/assert.js'; - -/** - * Common interface for a result of accepting a next token - * in a sequence. - */ -export interface IAcceptTokenResult { - /** - * The result type of accepting a next token in a sequence. - */ - result: 'success' | 'failure'; - - /** - * Whether the token to accept was consumed by the parser - * during the accept operation. - */ - wasTokenConsumed: boolean; -} - -/** - * Successful result of accepting a next token in a sequence. - */ -export interface IAcceptTokenSuccess extends IAcceptTokenResult { - result: 'success'; - nextParser: T; -} - -/** - * Failure result of accepting a next token in a sequence. - */ -export interface IAcceptTokenFailure extends IAcceptTokenResult { - result: 'failure'; -} - -/** - * The result of operation of accepting a next token in a sequence. - */ -export type TAcceptTokenResult = IAcceptTokenSuccess | IAcceptTokenFailure; - -/** - * An abstract parser class that is able to parse a sequence of - * tokens into a new single entity. - */ -export abstract class ParserBase { - /** - * Whether the parser object was "consumed" and should not be used anymore. - */ - protected isConsumed: boolean = false; - - /** - * Whether the parser object was "consumed" hence must not be used anymore. - */ - public get consumed(): boolean { - return this.isConsumed; - } - - /** - * Number of tokens at the initialization of the current parser. - */ - protected readonly startTokensCount: number; - - constructor( - /** - * Set of tokens that were accumulated so far. - */ - protected readonly currentTokens: TToken[] = [], - ) { - this.startTokensCount = this.currentTokens.length; - } - - /** - * Get the tokens that were accumulated so far. - */ - public get tokens(): readonly TToken[] { - return this.currentTokens; - } - - /** - * Accept a new token returning parsing result: - * - successful result must include the next parser object or a fully parsed out token - * - failure result must indicate that the token was not consumed - * - * @param token The token to accept. - * @returns The parsing result. - */ - public abstract accept(token: TToken): TAcceptTokenResult; - - /** - * A helper method that validates that the current parser object was not yet consumed, - * hence can still be used to accept new tokens in the parsing process. - * - * @throws if the parser object is already consumed. - */ - protected assertNotConsumed(): void { - assert( - this.isConsumed === false, - `The parser object is already consumed and should not be used anymore.`, - ); - } -} - -/** - * Decorator that validates that the current parser object was not yet consumed, - * hence can still be used to accept new tokens in the parsing process. - * - * @throws the resulting decorated method throws if the parser object was already consumed. - */ -export function assertNotConsumed>( - _target: T, - propertyKey: 'accept', - descriptor: PropertyDescriptor, -): PropertyDescriptor { - // store the original method reference - const originalMethod = descriptor.value; - - // validate that the current parser object was not yet consumed - // before invoking the original accept method - descriptor.value = function ( - this: T, - ...args: Parameters - ): ReturnType { - assert( - this.isConsumed === false, - `The parser object is already consumed and should not be used anymore.`, - ); - - return originalMethod.apply(this, args); - }; - - return descriptor; -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/simpleDecoder.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/simpleDecoder.ts deleted file mode 100644 index 9addf4df2ca..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/simpleDecoder.ts +++ /dev/null @@ -1,132 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { NewLine } from '../linesCodec/tokens/newLine.js'; -import { CarriageReturn } from '../linesCodec/tokens/carriageReturn.js'; -import { LinesDecoder, TLineBreakToken, TLineToken } from '../linesCodec/linesDecoder.js'; -import { - At, - Tab, - Word, - Hash, - Dash, - Colon, - Slash, - Space, - Quote, - Comma, - FormFeed, - DollarSign, - DoubleQuote, - VerticalTab, - type TBracket, - LeftBracket, - RightBracket, - type TCurlyBrace, - LeftCurlyBrace, - RightCurlyBrace, - ExclamationMark, - type TParenthesis, - LeftParenthesis, - RightParenthesis, - type TAngleBracket, - LeftAngleBracket, - RightAngleBracket, -} from './tokens/tokens.js'; -import { ISimpleTokenClass, SimpleToken } from './tokens/simpleToken.js'; -import { VSBuffer } from '../../../../../../../../base/common/buffer.js'; -import { BaseDecoder } from '../baseDecoder.js'; -import { ReadableStream } from '../../../../../../../../base/common/stream.js'; - -/** - * Type for all simple tokens. - */ -export type TSimpleToken = Space | Tab | VerticalTab | At | Quote | DoubleQuote - | CarriageReturn | NewLine | FormFeed | TBracket | TAngleBracket | TCurlyBrace - | TParenthesis | Colon | Hash | Dash | ExclamationMark | Slash | DollarSign | Comma - | TLineBreakToken; - -/** -* Type of tokens emitted by this decoder. -*/ -export type TSimpleDecoderToken = TSimpleToken | Word; - -/** - * List of well-known distinct tokens that this decoder emits (excluding - * the word stop characters defined below). Everything else is considered - * an arbitrary "text" sequence and is emitted as a single {@link Word} token. - */ -export const WELL_KNOWN_TOKENS: readonly ISimpleTokenClass[] = Object.freeze([ - LeftParenthesis, RightParenthesis, LeftBracket, RightBracket, LeftCurlyBrace, RightCurlyBrace, - LeftAngleBracket, RightAngleBracket, Space, Tab, VerticalTab, FormFeed, Colon, Hash, Dash, - ExclamationMark, At, Slash, DollarSign, Quote, DoubleQuote, Comma, -]); - -/** - * A {@link Word} sequence stops when one of the well-known tokens are encountered. - * Note! the `\r` and `\n` are excluded from the list because this decoder based on - * the {@link LinesDecoder} which emits {@link Line} tokens without them. - */ -const WORD_STOP_CHARACTERS: readonly string[] = Object.freeze( - WELL_KNOWN_TOKENS.map(token => token.symbol), -); - -/** - * A decoder that can decode a stream of `Line`s into a stream - * of simple token, - `Word`, `Space`, `Tab`, `NewLine`, etc. - */ -export class SimpleDecoder extends BaseDecoder { - constructor( - stream: ReadableStream, - ) { - super(new LinesDecoder(stream)); - } - - protected override onStreamData(line: TLineToken): void { - // re-emit new line tokens immediately - if (line instanceof CarriageReturn || line instanceof NewLine) { - this._onData.fire(line); - - return; - } - - // loop through the text separating it into `Word` and `well-known` tokens - const lineText = line.text.split(''); - let i = 0; - while (i < lineText.length) { - // index is 0-based, but column numbers are 1-based - const columnNumber = i + 1; - const character = lineText[i]; - - // check if the current character is a well-known token - const tokenConstructor = WELL_KNOWN_TOKENS - .find((wellKnownToken) => { - return wellKnownToken.symbol === character; - }); - - // if it is a well-known token, emit it and continue to the next one - if (tokenConstructor) { - this._onData.fire(SimpleToken.newOnLine(line, columnNumber, tokenConstructor)); - - i++; - continue; - } - - // otherwise, it is an arbitrary "text" sequence of characters, - // that needs to be collected into a single `Word` token, hence - // read all the characters until a stop character is encountered - let word = ''; - while (i < lineText.length && !(WORD_STOP_CHARACTERS.includes(lineText[i]))) { - word += lineText[i]; - i++; - } - - // emit a "text" sequence of characters as a single `Word` token - this._onData.fire( - Word.newOnLine(word, line, columnNumber), - ); - } - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/angleBrackets.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/angleBrackets.ts deleted file mode 100644 index d57acedc1c3..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/angleBrackets.ts +++ /dev/null @@ -1,61 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { SimpleToken } from './simpleToken.js'; - -/** - * A token that represent a `<` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class LeftAngleBracket extends SimpleToken<'<'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '<' = '<'; - - /** - * Return text representation of the token. - */ - public override get text(): '<' { - return LeftAngleBracket.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `left-angle-bracket${this.range}`; - } -} - -/** - * A token that represent a `>` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class RightAngleBracket extends SimpleToken<'>'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '>' = '>'; - - /** - * Return text representation of the token. - */ - public override get text(): '>' { - return RightAngleBracket.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `right-angle-bracket${this.range}`; - } -} - -/** - * General angle bracket token type. - */ -export type TAngleBracket = LeftAngleBracket | RightAngleBracket; diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/at.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/at.ts deleted file mode 100644 index 25c44433afd..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/at.ts +++ /dev/null @@ -1,31 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { SimpleToken } from './simpleToken.js'; - -/** - * A token that represent a `@` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class At extends SimpleToken<'@'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '@' = '@'; - - /** - * Return text representation of the token. - */ - public override get text(): '@' { - return At.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `at${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/brackets.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/brackets.ts deleted file mode 100644 index 491cb2a588c..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/brackets.ts +++ /dev/null @@ -1,61 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { SimpleToken } from './simpleToken.js'; - -/** - * A token that represent a `[` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class LeftBracket extends SimpleToken<'['> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '[' = '['; - - /** - * Return text representation of the token. - */ - public override get text(): '[' { - return LeftBracket.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `left-bracket${this.range}`; - } -} - -/** - * A token that represent a `]` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class RightBracket extends SimpleToken<']'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: ']' = ']'; - - /** - * Return text representation of the token. - */ - public override get text(): ']' { - return RightBracket.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `right-bracket${this.range}`; - } -} - -/** - * General bracket token type. - */ -export type TBracket = LeftBracket | RightBracket; diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/colon.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/colon.ts deleted file mode 100644 index a04b3e853df..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/colon.ts +++ /dev/null @@ -1,31 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { SimpleToken } from './simpleToken.js'; - -/** - * A token that represent a `:` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class Colon extends SimpleToken<':'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: ':' = ':'; - - /** - * Return text representation of the token. - */ - public override get text(): ':' { - return Colon.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `colon${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/comma.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/comma.ts deleted file mode 100644 index ce5df1748ca..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/comma.ts +++ /dev/null @@ -1,31 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { SimpleToken } from './simpleToken.js'; - -/** - * A token that represent a `,` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class Comma extends SimpleToken<','> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: ',' = ','; - - /** - * Return text representation of the token. - */ - public override get text(): ',' { - return Comma.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `comma${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/curlyBraces.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/curlyBraces.ts deleted file mode 100644 index 9c13e501d27..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/curlyBraces.ts +++ /dev/null @@ -1,61 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { SimpleToken } from './simpleToken.js'; - -/** - * A token that represent a `{` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class LeftCurlyBrace extends SimpleToken<'{'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '{' = '{'; - - /** - * Return text representation of the token. - */ - public override get text(): '{' { - return LeftCurlyBrace.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `left-curly-brace${this.range}`; - } -} - -/** - * A token that represent a `}` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class RightCurlyBrace extends SimpleToken<'}'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '}' = '}'; - - /** - * Return text representation of the token. - */ - public override get text(): '}' { - return RightCurlyBrace.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `right-curly-brace${this.range}`; - } -} - -/** - * General curly brace token type. - */ -export type TCurlyBrace = LeftCurlyBrace | RightCurlyBrace; diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/dash.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/dash.ts deleted file mode 100644 index a77b441ad26..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/dash.ts +++ /dev/null @@ -1,31 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { SimpleToken } from './simpleToken.js'; - -/** - * A token that represent a `-` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class Dash extends SimpleToken<'-'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '-' = '-'; - - /** - * Return text representation of the token. - */ - public override get text(): '-' { - return Dash.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `dash${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/dollarSign.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/dollarSign.ts deleted file mode 100644 index 5b81d0afda8..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/dollarSign.ts +++ /dev/null @@ -1,31 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { SimpleToken } from './simpleToken.js'; - -/** - * A token that represent a `$` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class DollarSign extends SimpleToken<'$'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '$' = '$'; - - /** - * Return text representation of the token. - */ - public override get text(): '$' { - return DollarSign.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `dollarSign${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/doubleQuote.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/doubleQuote.ts deleted file mode 100644 index ba041a03ed5..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/doubleQuote.ts +++ /dev/null @@ -1,40 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../../baseToken.js'; -import { SimpleToken } from './simpleToken.js'; - -/** - * A token that represent a `"` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class DoubleQuote extends SimpleToken<'"'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '"' = '"'; - - /** - * Return text representation of the token. - */ - public override get text(): '"' { - return DoubleQuote.symbol; - } - - /** - * Checks if the provided token is of the same type - * as the current one. - */ - public sameType(other: BaseToken): other is typeof this { - return (other instanceof this.constructor); - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `double-quote${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/exclamationMark.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/exclamationMark.ts deleted file mode 100644 index 32675fdf8a3..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/exclamationMark.ts +++ /dev/null @@ -1,31 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { SimpleToken } from './simpleToken.js'; - -/** - * A token that represent a `!` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class ExclamationMark extends SimpleToken<'!'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '!' = '!'; - - /** - * Return text representation of the token. - */ - public override get text(): '!' { - return ExclamationMark.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `exclamation-mark${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/formFeed.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/formFeed.ts deleted file mode 100644 index df5b8b0d446..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/formFeed.ts +++ /dev/null @@ -1,31 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { SimpleToken } from './simpleToken.js'; - -/** - * Token that represent a `form feed` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class FormFeed extends SimpleToken<'\f'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '\f' = '\f'; - - /** - * Return text representation of the token. - */ - public override get text(): '\f' { - return FormFeed.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `formfeed${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/hash.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/hash.ts deleted file mode 100644 index f499085859d..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/hash.ts +++ /dev/null @@ -1,31 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { SimpleToken } from './simpleToken.js'; - -/** - * A token that represent a `#` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class Hash extends SimpleToken<'#'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '#' = '#'; - - /** - * Return text representation of the token. - */ - public override get text(): '#' { - return Hash.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `hash${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/parentheses.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/parentheses.ts deleted file mode 100644 index b7ef4f5bc2a..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/parentheses.ts +++ /dev/null @@ -1,61 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { SimpleToken } from './simpleToken.js'; - -/** - * A token that represent a `(` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class LeftParenthesis extends SimpleToken<'('> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '(' = '('; - - /** - * Return text representation of the token. - */ - public override get text(): '(' { - return LeftParenthesis.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `left-parenthesis${this.range}`; - } -} - -/** - * A token that represent a `)` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class RightParenthesis extends SimpleToken<')'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: ')' = ')'; - - /** - * Return text representation of the token. - */ - public override get text(): ')' { - return RightParenthesis.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `right-parenthesis${this.range}`; - } -} - -/** - * General parenthesis token type. - */ -export type TParenthesis = LeftParenthesis | RightParenthesis; diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/quote.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/quote.ts deleted file mode 100644 index 3c262f67b49..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/quote.ts +++ /dev/null @@ -1,40 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../../baseToken.js'; -import { SimpleToken } from './simpleToken.js'; - -/** - * A token that represent a `'` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class Quote extends SimpleToken<`'`> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '\'' = '\''; - - /** - * Return text representation of the token. - */ - public override get text(): '\'' { - return Quote.symbol; - } - - /** - * Checks if the provided token is of the same type - * as the current one. - */ - public sameType(other: BaseToken): other is Quote { - return (other instanceof this.constructor); - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `quote${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/simpleToken.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/simpleToken.ts deleted file mode 100644 index 5a611bb1ddd..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/simpleToken.ts +++ /dev/null @@ -1,59 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { Range } from '../../../../../../../../../editor/common/core/range.js'; -import { BaseToken } from '../../baseToken.js'; -import { Line } from '../../linesCodec/tokens/line.js'; - -/** - * Interface for a class that can be instantiated into a {@link SimpleToken}. - */ -export interface ISimpleTokenClass> { - /** - * Character representing the token. - */ - readonly symbol: string; - - /** - * Constructor for the token. - */ - new(...args: any[]): TSimpleToken; -} - -/** - * Base class for all "simple" tokens with a `range`. - * A simple token is the one that represents a single character. - */ -export abstract class SimpleToken extends BaseToken { - /** - * The underlying symbol of the token. - */ - public static readonly symbol: string; - - /** - * Create new token instance with range inside - * the given `Line` at the given `column number`. - */ - public static newOnLine>( - line: Line, - atColumnNumber: number, - Constructor: ISimpleTokenClass, - ): TSimpleToken { - const { range } = line; - - return new Constructor(new Range( - range.startLineNumber, - atColumnNumber, - range.startLineNumber, - atColumnNumber + Constructor.symbol.length, - )); - } -} - -/** - * Base class for all tokens that represent some form of - * a spacing character, e.g. 'space', 'tab', etc. - */ -export abstract class SpacingToken extends SimpleToken { } diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/slash.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/slash.ts deleted file mode 100644 index 04d382bc89f..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/slash.ts +++ /dev/null @@ -1,31 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { SimpleToken } from './simpleToken.js'; - -/** - * A token that represent a `/` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class Slash extends SimpleToken<'/'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '/' = '/'; - - /** - * Return text representation of the token. - */ - public override get text(): '/' { - return Slash.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `slash${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/space.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/space.ts deleted file mode 100644 index 07eed9f58b2..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/space.ts +++ /dev/null @@ -1,31 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { SpacingToken } from './simpleToken.js'; - -/** - * A token that represent a `space` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class Space extends SpacingToken<' '> { - /** - * The underlying symbol of the `Space` token. - */ - public static override readonly symbol: ' ' = ' '; - - /** - * Return text representation of the token. - */ - public override get text(): ' ' { - return Space.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `space${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/tab.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/tab.ts deleted file mode 100644 index 718a01df813..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/tab.ts +++ /dev/null @@ -1,31 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { SpacingToken } from './simpleToken.js'; - -/** - * A token that represent a `tab` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class Tab extends SpacingToken<'\t'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '\t' = '\t'; - - /** - * Return text representation of the token. - */ - public override get text(): '\t' { - return Tab.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `tab${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/tokens.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/tokens.ts deleted file mode 100644 index ee25fb83ddf..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/tokens.ts +++ /dev/null @@ -1,25 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -export { At } from './at.js'; -export { Tab } from './tab.js'; -export { Dash } from './dash.js'; -export { Hash } from './hash.js'; -export { Word } from './word.js'; -export { Colon } from './colon.js'; -export { Quote } from './quote.js'; -export { Slash } from './slash.js'; -export { Space } from './space.js'; -export { Comma } from './comma.js'; -export { FormFeed } from './formFeed.js'; -export { DollarSign } from './dollarSign.js'; -export { VerticalTab } from './verticalTab.js'; -export { DoubleQuote } from './doubleQuote.js'; -export { ExclamationMark } from './exclamationMark.js'; -export { SimpleToken, SpacingToken } from './simpleToken.js'; -export { type TBracket, LeftBracket, RightBracket } from './brackets.js'; -export { type TCurlyBrace, LeftCurlyBrace, RightCurlyBrace } from './curlyBraces.js'; -export { type TParenthesis, LeftParenthesis, RightParenthesis } from './parentheses.js'; -export { type TAngleBracket, LeftAngleBracket, RightAngleBracket } from './angleBrackets.js'; diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/verticalTab.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/verticalTab.ts deleted file mode 100644 index 4b7b4241433..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/verticalTab.ts +++ /dev/null @@ -1,31 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { SpacingToken } from './simpleToken.js'; - -/** - * Token that represent a `vertical tab` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class VerticalTab extends SpacingToken<'\v'> { - /** - * The underlying symbol of the `VerticalTab` token. - */ - public static override readonly symbol: '\v' = '\v'; - - /** - * Return text representation of the token. - */ - public override get text(): '\v' { - return VerticalTab.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `vtab${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/word.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/word.ts deleted file mode 100644 index bea656895fc..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/word.ts +++ /dev/null @@ -1,60 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../../baseToken.js'; -import { Line } from '../../linesCodec/tokens/line.js'; -import { Range } from '../../../../../../../../../editor/common/core/range.js'; - -/** - * A token that represent a word - a set of continuous - * characters without stop characters, like a `space`, - * a `tab`, or a `new line`. - */ -export class Word extends BaseToken { - constructor( - /** - * The word range. - */ - range: Range, - - /** - * The string value of the word. - */ - public readonly text: TText, - ) { - super(range); - } - - /** - * Create new `Word` token with the given `text` and the range - * inside the given `Line` at the specified `column number`. - */ - public static newOnLine( - text: string, - line: Line | number, - atColumnNumber: number, - ): Word { - const startLineNumber = (typeof line === 'number') - ? line - : line.range.startLineNumber; - - const range = new Range( - startLineNumber, atColumnNumber, - startLineNumber, atColumnNumber + text.length - ); - - return new Word( - range, - text, - ); - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `word("${this.shortText()}")${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/textToken.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/textToken.ts deleted file mode 100644 index ecb4c61c94e..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/textToken.ts +++ /dev/null @@ -1,19 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { type BaseToken } from './baseToken.js'; -import { CompositeToken } from './compositeToken.js'; - -/** - * Tokens that represent a sequence of tokens that does not - * hold an additional meaning in the text. - */ -export class Text< - TTokens extends readonly BaseToken[] = readonly BaseToken[], -> extends CompositeToken { - public override toString(): string { - return `text(${this.shortText()})${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/utils/objectStream.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/utils/objectStream.ts deleted file mode 100644 index 3c8476c2d56..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/utils/objectStream.ts +++ /dev/null @@ -1,224 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { assertNever } from '../../../../../../../../base/common/assert.js'; -import { CancellationToken } from '../../../../../../../../base/common/cancellation.js'; -import { ObservableDisposable } from '../../../utils/observableDisposable.js'; -import { newWriteableStream, ReadableStream, WriteableStream } from '../../../../../../../../base/common/stream.js'; - - -/** - * A readable stream of provided objects. - */ -export class ObjectStream extends ObservableDisposable implements ReadableStream { - /** - * Flag that indicates whether the stream has ended. - */ - private ended: boolean = false; - - /** - * Underlying writable stream instance. - */ - private readonly stream: WriteableStream; - - /** - * Interval reference that is used to periodically send - * objects to the stream in the background. - */ - private timeoutHandle: Timeout | undefined; - - constructor( - private readonly data: Generator, - private readonly cancellationToken?: CancellationToken, - ) { - super(); - - this.stream = newWriteableStream(null); - - if (cancellationToken?.isCancellationRequested) { - this.end(); - return; - } - - // send a first batch of data immediately - this.send(true); - } - - /** - * Starts process of sending data to the stream. - * - * @param stopAfterFirstSend whether to continue sending data to the stream - * or stop sending after the first batch of data is sent instead - */ - public send( - stopAfterFirstSend: boolean = false, - ): void { - // this method can be called asynchronously by the `setTimeout` utility below, hence - // the state of the cancellation token or the stream itself might have changed by that time - if (this.cancellationToken?.isCancellationRequested || this.ended) { - this.end(); - - return; - } - - this.sendData() - .then(() => { - if (this.cancellationToken?.isCancellationRequested || this.ended) { - this.end(); - - return; - } - - if (stopAfterFirstSend === true) { - this.stopStream(); - return; - } - - this.timeoutHandle = setTimeout(this.send.bind(this)); - }) - .catch((error) => { - this.stream.error(error); - this.dispose(); - }); - } - - /** - * Stop the data sending loop. - */ - public stopStream(): this { - if (this.timeoutHandle === undefined) { - return this; - } - - clearTimeout(this.timeoutHandle); - this.timeoutHandle = undefined; - - return this; - } - - /** - * Sends a provided number of objects to the stream. - */ - private async sendData( - objectsCount: number = 25, - ): Promise { - // send up to 'objectsCount' objects at a time - while (objectsCount > 0) { - try { - const next = this.data.next(); - if (next.done || this.cancellationToken?.isCancellationRequested) { - this.end(); - - return; - } - - await this.stream.write(next.value); - objectsCount--; - } catch (error) { - this.stream.error(error); - this.dispose(); - return; - } - } - } - - /** - * Ends the stream and stops sending data objects. - */ - private end(): this { - if (this.ended) { - return this; - } - this.ended = true; - - this.stopStream(); - this.stream.end(); - return this; - } - - public pause(): void { - this.stopStream(); - this.stream.pause(); - - return; - } - - public resume(): void { - this.send(); - this.stream.resume(); - - return; - } - - public destroy(): void { - this.dispose(); - } - - public removeListener(event: string, callback: (...args: any[]) => void): void { - this.stream.removeListener(event, callback); - - return; - } - - public on(event: 'data', callback: (data: T) => void): void; - public on(event: 'error', callback: (err: Error) => void): void; - public on(event: 'end', callback: () => void): void; - public on(event: 'data' | 'error' | 'end', callback: (...args: any[]) => void): void { - if (event === 'data') { - this.stream.on(event, callback); - // this is the convention of the readable stream, - when - // the `data` event is registered, the stream is started - this.send(); - - return; - } - - if (event === 'error') { - this.stream.on(event, callback); - return; - } - - if (event === 'end') { - this.stream.on(event, callback); - return; - } - - assertNever( - event, - `Unexpected event name '${event}'.`, - ); - } - - /** - * Cleanup send interval and destroy the stream. - */ - public override dispose(): void { - this.stopStream(); - this.stream.destroy(); - - super.dispose(); - } - - /** - * Create new instance of the stream from a provided array. - */ - public static fromArray( - array: T[], - cancellationToken?: CancellationToken, - ): ObjectStream { - return new ObjectStream(arrayToGenerator(array), cancellationToken); - } -} - -/** - * Create a generator out of a provided array. - */ -export function arrayToGenerator>(array: T[]): Generator { - return (function* (): Generator { - for (const item of array) { - yield item; - } - })(); -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/utils/objectStreamFromTextModel.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/utils/objectStreamFromTextModel.ts deleted file mode 100644 index 2a9b2173b5d..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/utils/objectStreamFromTextModel.ts +++ /dev/null @@ -1,46 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { ITextModel } from '../../../../../../../../editor/common/model.js'; -import { ObjectStream } from './objectStream.js'; -import { VSBuffer } from '../../../../../../../../base/common/buffer.js'; -import { CancellationToken } from '../../../../../../../../base/common/cancellation.js'; - -/** - * Create new instance of the stream from a provided text model. - */ -export function objectStreamFromTextModel( - model: ITextModel, - cancellationToken?: CancellationToken, -): ObjectStream { - return new ObjectStream(modelToGenerator(model), cancellationToken); -} - -/** - * Create a generator out of a provided text model. - */ -function modelToGenerator(model: ITextModel): Generator { - return (function* (): Generator { - const totalLines = model.getLineCount(); - let currentLine = 1; - - while (currentLine <= totalLines) { - if (model.isDisposed()) { - return undefined; - } - - yield VSBuffer.fromString( - model.getLineContent(currentLine), - ); - if (currentLine !== totalLines) { - yield VSBuffer.fromString( - model.getEOL(), - ); - } - - currentLine++; - } - })(); -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/chatPromptCodec.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/chatPromptCodec.ts deleted file mode 100644 index da27950bf92..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/chatPromptCodec.ts +++ /dev/null @@ -1,73 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { VSBuffer } from '../../../../../../base/common/buffer.js'; -import { ReadableStream } from '../../../../../../base/common/stream.js'; -import { ChatPromptDecoder, TChatPromptToken } from './chatPromptDecoder.js'; - -/** - * A codec is an object capable of encoding/decoding a stream of data transforming its messages. - * Useful for abstracting a data transfer or protocol logic on top of a stream of bytes. - * - * For instance, if protocol messages need to be transferred over `TCP` connection, a codec that - * encodes the messages into a sequence of bytes before sending it to a network socket. Likewise, - * on the other end of the connection, the same codec can decode the sequence of bytes back into - * a sequence of the protocol messages. - */ -export interface ICodec { - /** - * Encode a stream of `K`s into a stream of `T`s. - */ - encode: (value: ReadableStream) => ReadableStream; - - /** - * Decode a stream of `T`s into a stream of `K`s. - */ - decode: (value: ReadableStream) => ReadableStream; -} - - -/** - * `ChatPromptCodec` type is a `ICodec` with specific types for - * stream messages and return types of the `encode`/`decode` functions. - * @see {@link ICodec} - */ -interface IChatPromptCodec extends ICodec { - /** - * Decode a stream of `VSBuffer`s into a stream of `TChatPromptToken`s. - * - * @see {@link TChatPromptToken} - * @see {@link VSBuffer} - * @see {@link ChatPromptDecoder} - */ - decode: (value: ReadableStream) => ChatPromptDecoder; -} - -/** - * Codec that is capable to encode and decode tokens of an AI chatbot prompt message. - */ -export const ChatPromptCodec: IChatPromptCodec = Object.freeze({ - /** - * Encode a stream of `TChatPromptToken`s into a stream of `VSBuffer`s. - * - * @see {@link ReadableStream} - * @see {@link VSBuffer} - */ - encode: (_stream: ReadableStream): ReadableStream => { - throw new Error('The `encode` method is not implemented.'); - }, - - /** - * Decode a of `VSBuffer`s into a readable of `TChatPromptToken`s. - * - * @see {@link TChatPromptToken} - * @see {@link VSBuffer} - * @see {@link ChatPromptDecoder} - * @see {@link ReadableStream} - */ - decode: (stream: ReadableStream): ChatPromptDecoder => { - return new ChatPromptDecoder(stream); - }, -}); diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/chatPromptDecoder.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/chatPromptDecoder.ts deleted file mode 100644 index 5ac4a138fbe..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/chatPromptDecoder.ts +++ /dev/null @@ -1,202 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { PromptToken } from './tokens/promptToken.js'; -import { PromptAtMention } from './tokens/promptAtMention.js'; -import { VSBuffer } from '../../../../../../base/common/buffer.js'; -import { PromptSlashCommand } from './tokens/promptSlashCommand.js'; -import { ReadableStream } from '../../../../../../base/common/stream.js'; -import { PartialPromptAtMention } from './parsers/promptAtMentionParser.js'; -import { PromptTemplateVariable } from './tokens/promptTemplateVariable.js'; -import { assert, assertNever } from '../../../../../../base/common/assert.js'; -import { PartialPromptSlashCommand } from './parsers/promptSlashCommandParser.js'; -import { BaseDecoder } from './base/baseDecoder.js'; -import { PromptVariable, PromptVariableWithData } from './tokens/promptVariable.js'; -import { At } from './base/simpleCodec/tokens/at.js'; -import { Hash } from './base/simpleCodec/tokens/hash.js'; -import { Slash } from './base/simpleCodec/tokens/slash.js'; -import { DollarSign } from './base/simpleCodec/tokens/dollarSign.js'; -import { PartialPromptVariableName, PartialPromptVariableWithData } from './parsers/promptVariableParser.js'; -import { MarkdownDecoder, TMarkdownToken } from './base/markdownCodec/markdownDecoder.js'; -import { PartialPromptTemplateVariable, PartialPromptTemplateVariableStart, TPromptTemplateVariableParser } from './parsers/promptTemplateVariableParser.js'; - -/** - * Tokens produced by this decoder. - */ -export type TChatPromptToken = TMarkdownToken | (PromptVariable | PromptVariableWithData) - | PromptAtMention | PromptSlashCommand | PromptTemplateVariable; - -/** - * Decoder for the common chatbot prompt message syntax. - * For instance, the file references `#file:./path/file.md` are handled by this decoder. - */ -export class ChatPromptDecoder extends BaseDecoder { - /** - * Currently active parser object that is used to parse a well-known sequence of - * tokens, for instance, a `#file:/path/to/file.md` link that consists of `hash`, - * `word`, and `colon` tokens sequence plus the `file path` part that follows. - */ - private current?: (PartialPromptVariableName | PartialPromptVariableWithData) - | PartialPromptAtMention | PartialPromptSlashCommand - | TPromptTemplateVariableParser; - - constructor( - stream: ReadableStream, - ) { - super(new MarkdownDecoder(stream)); - } - - protected override onStreamData(token: TMarkdownToken): void { - // prompt `#variables` always start with the `#` character, hence - // initiate a parser object if we encounter respective token and - // there is no active parser object present at the moment - if ((token instanceof Hash) && !this.current) { - this.current = new PartialPromptVariableName(token); - - return; - } - - // prompt `@mentions` always start with the `@` character, hence - // initiate a parser object if we encounter respective token and - // there is no active parser object present at the moment - if ((token instanceof At) && !this.current) { - this.current = new PartialPromptAtMention(token); - - return; - } - - // prompt `/commands` always start with the `/` character, hence - // initiate a parser object if we encounter respective token and - // there is no active parser object present at the moment - if ((token instanceof Slash) && !this.current) { - this.current = new PartialPromptSlashCommand(token); - - return; - } - - // prompt `${template:variables}` always start with the `$` character, - // hence initiate a parser object if we encounter respective token and - // there is no active parser object present at the moment - if ((token instanceof DollarSign) && !this.current) { - this.current = new PartialPromptTemplateVariableStart(token); - - return; - } - - // if current parser was not yet initiated, - we are in the general "text" - // parsing mode, therefore re-emit the token immediately and continue - if (!this.current) { - this._onData.fire(token); - return; - } - - // if there is a current parser object, submit the token to it - // so it can progress with parsing the tokens sequence - const parseResult = this.current.accept(token); - - // process the parse result next - switch (parseResult.result) { - // in the case of success there might be 2 cases: - // 1) parsing fully completed and an instance of `PromptToken` is returned back, - // in this case, emit the parsed token (e.g., a `link`) and reset the current - // parser object reference so a new parsing process can be initiated next - // 2) parsing is still in progress and the next parser object is returned, hence - // we need to replace the current parser object with a new one and continue - case 'success': { - const { nextParser } = parseResult; - - if (nextParser instanceof PromptToken) { - this._onData.fire(nextParser); - delete this.current; - } else { - this.current = nextParser; - } - - break; - } - // in the case of failure, reset the current parser object - case 'failure': { - // if failed to parse a sequence of a tokens, re-emit the tokens accumulated - // so far then reset the current parser object - this.reEmitCurrentTokens(); - break; - } - } - - // if token was not consumed by the parser, call `onStreamData` again - // so the token is properly handled by the decoder in the case when a - // new sequence starts with this token - if (!parseResult.wasTokenConsumed) { - this.onStreamData(token); - } - } - - protected override onStreamEnd(): void { - try { - // if there is no currently active parser object present, nothing to do - if (this.current === undefined) { - return; - } - - // otherwise try to convert unfinished parser object to a token - - if (this.current instanceof PartialPromptVariableName) { - this._onData.fire(this.current.asPromptVariable()); - return; - } - - if (this.current instanceof PartialPromptVariableWithData) { - this._onData.fire(this.current.asPromptVariableWithData()); - return; - } - - if (this.current instanceof PartialPromptAtMention) { - this._onData.fire(this.current.asPromptAtMention()); - return; - } - - if (this.current instanceof PartialPromptSlashCommand) { - this._onData.fire(this.current.asPromptSlashCommand()); - return; - } - - assert( - (this.current instanceof PartialPromptTemplateVariableStart) === false, - 'Incomplete template variable token.', - ); - - if (this.current instanceof PartialPromptTemplateVariable) { - this._onData.fire(this.current.asPromptTemplateVariable()); - return; - } - - assertNever( - this.current, - `Unknown parser object '${this.current}'`, - ); - } catch (_error) { - // if failed to convert current parser object to a token, - // re-emit the tokens accumulated so far - this.reEmitCurrentTokens(); - } finally { - delete this.current; - super.onStreamEnd(); - } - } - - /** - * Re-emit tokens accumulated so far in the current parser object. - */ - protected reEmitCurrentTokens(): void { - if (this.current === undefined) { - return; - } - - for (const token of this.current.tokens) { - this._onData.fire(token); - } - delete this.current; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/parsers/promptAtMentionParser.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/parsers/promptAtMentionParser.ts deleted file mode 100644 index 2173ba438ad..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/parsers/promptAtMentionParser.ts +++ /dev/null @@ -1,121 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { PromptAtMention } from '../tokens/promptAtMention.js'; -import { assert } from '../../../../../../../base/common/assert.js'; -import { Range } from '../../../../../../../editor/common/core/range.js'; -import { BaseToken } from '../base/baseToken.js'; -import { At } from '../base/simpleCodec/tokens/at.js'; -import { Tab } from '../base/simpleCodec/tokens/tab.js'; -import { Hash } from '../base/simpleCodec/tokens/hash.js'; -import { Space } from '../base/simpleCodec/tokens/space.js'; -import { Colon } from '../base/simpleCodec/tokens/colon.js'; -import { NewLine } from '../base/linesCodec/tokens/newLine.js'; -import { FormFeed } from '../base/simpleCodec/tokens/formFeed.js'; -import { VerticalTab } from '../base/simpleCodec/tokens/verticalTab.js'; -import { TSimpleDecoderToken } from '../base/simpleCodec/simpleDecoder.js'; -import { CarriageReturn } from '../base/linesCodec/tokens/carriageReturn.js'; -import { ExclamationMark } from '../base/simpleCodec/tokens/exclamationMark.js'; -import { LeftBracket, RightBracket } from '../base/simpleCodec/tokens/brackets.js'; -import { LeftAngleBracket, RightAngleBracket } from '../base/simpleCodec/tokens/angleBrackets.js'; -import { assertNotConsumed, ParserBase, TAcceptTokenResult } from '../base/simpleCodec/parserBase.js'; - -/** - * List of characters that terminate the prompt at-mention sequence. - */ -export const STOP_CHARACTERS: readonly string[] = [Space, Tab, NewLine, CarriageReturn, VerticalTab, FormFeed, At, Colon, Hash] - .map((token) => { return token.symbol; }); - -/** - * List of characters that cannot be in an at-mention name (excluding the {@link STOP_CHARACTERS}). - */ -export const INVALID_NAME_CHARACTERS: readonly string[] = [ExclamationMark, LeftAngleBracket, RightAngleBracket, LeftBracket, RightBracket] - .map((token) => { return token.symbol; }); - -/** - * The parser responsible for parsing a `prompt @mention` sequences. - * E.g., `@workspace` or `@github` participant mention. - */ -export class PartialPromptAtMention extends ParserBase { - constructor(token: At) { - super([token]); - } - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - // if a `stop` character is encountered, finish the parsing process - if (STOP_CHARACTERS.includes(token.text)) { - try { - // if it is possible to convert current parser to `PromptAtMention`, return success result - return { - result: 'success', - nextParser: this.asPromptAtMention(), - wasTokenConsumed: false, - }; - } catch (error) { - // otherwise fail - return { - result: 'failure', - wasTokenConsumed: false, - }; - } finally { - // in any case this is an end of the parsing process - this.isConsumed = true; - } - } - - // variables cannot have {@link INVALID_NAME_CHARACTERS} in their names - if (INVALID_NAME_CHARACTERS.includes(token.text)) { - this.isConsumed = true; - - return { - result: 'failure', - wasTokenConsumed: false, - }; - } - - // otherwise it is a valid name character, so add it to the list of - // the current tokens and continue the parsing process - this.currentTokens.push(token); - - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } - - /** - * Try to convert current parser instance into a fully-parsed {@link PromptAtMention} token. - * - * @throws if sequence of tokens received so far do not constitute a valid prompt variable, - * for instance, if there is only `1` starting `@` token is available. - */ - public asPromptAtMention(): PromptAtMention { - // if there is only one token before the stop character - // must be the starting `@` one), then fail - assert( - this.currentTokens.length > 1, - 'Cannot create a prompt @mention out of incomplete token sequence.', - ); - - const firstToken = this.currentTokens[0]; - const lastToken = this.currentTokens[this.currentTokens.length - 1]; - - // render the characters above into strings, excluding the starting `@` character - const nameTokens = this.currentTokens.slice(1); - const atMentionName = BaseToken.render(nameTokens); - - return new PromptAtMention( - new Range( - firstToken.range.startLineNumber, - firstToken.range.startColumn, - lastToken.range.endLineNumber, - lastToken.range.endColumn, - ), - atMentionName, - ); - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/parsers/promptSlashCommandParser.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/parsers/promptSlashCommandParser.ts deleted file mode 100644 index 557f5e74379..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/parsers/promptSlashCommandParser.ts +++ /dev/null @@ -1,122 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { assert } from '../../../../../../../base/common/assert.js'; -import { PromptSlashCommand } from '../tokens/promptSlashCommand.js'; -import { Range } from '../../../../../../../editor/common/core/range.js'; -import { BaseToken } from '../base/baseToken.js'; -import { At } from '../base/simpleCodec/tokens/at.js'; -import { Tab } from '../base/simpleCodec/tokens/tab.js'; -import { Hash } from '../base/simpleCodec/tokens/hash.js'; -import { Slash } from '../base/simpleCodec/tokens/slash.js'; -import { Space } from '../base/simpleCodec/tokens/space.js'; -import { Colon } from '../base/simpleCodec/tokens/colon.js'; -import { NewLine } from '../base/linesCodec/tokens/newLine.js'; -import { FormFeed } from '../base/simpleCodec/tokens/formFeed.js'; -import { VerticalTab } from '../base/simpleCodec/tokens/verticalTab.js'; -import { TSimpleDecoderToken } from '../base/simpleCodec/simpleDecoder.js'; -import { CarriageReturn } from '../base/linesCodec/tokens/carriageReturn.js'; -import { ExclamationMark } from '../base/simpleCodec/tokens/exclamationMark.js'; -import { LeftBracket, RightBracket } from '../base/simpleCodec/tokens/brackets.js'; -import { LeftAngleBracket, RightAngleBracket } from '../base/simpleCodec/tokens/angleBrackets.js'; -import { assertNotConsumed, ParserBase, TAcceptTokenResult } from '../base/simpleCodec/parserBase.js'; - -/** - * List of characters that terminate the prompt at-mention sequence. - */ -export const STOP_CHARACTERS: readonly string[] = [Space, Tab, NewLine, CarriageReturn, VerticalTab, FormFeed, Colon, At, Hash, Slash] - .map((token) => { return token.symbol; }); - -/** - * List of characters that cannot be in an at-mention name (excluding the {@link STOP_CHARACTERS}). - */ -export const INVALID_NAME_CHARACTERS: readonly string[] = [ExclamationMark, LeftAngleBracket, RightAngleBracket, LeftBracket, RightBracket] - .map((token) => { return token.symbol; }); - -/** - * The parser responsible for parsing a `prompt /command` sequences. - * E.g., `/search` or `/explain` command. - */ -export class PartialPromptSlashCommand extends ParserBase { - constructor(token: Slash) { - super([token]); - } - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - // if a `stop` character is encountered, finish the parsing process - if (STOP_CHARACTERS.includes(token.text)) { - try { - // if it is possible to convert current parser to `PromptSlashCommand`, return success result - return { - result: 'success', - nextParser: this.asPromptSlashCommand(), - wasTokenConsumed: false, - }; - } catch (error) { - // otherwise fail - return { - result: 'failure', - wasTokenConsumed: false, - }; - } finally { - // in any case this is an end of the parsing process - this.isConsumed = true; - } - } - - // variables cannot have {@link INVALID_NAME_CHARACTERS} in their names - if (INVALID_NAME_CHARACTERS.includes(token.text)) { - this.isConsumed = true; - - return { - result: 'failure', - wasTokenConsumed: false, - }; - } - - // otherwise it is a valid name character, so add it to the list of - // the current tokens and continue the parsing process - this.currentTokens.push(token); - - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } - - /** - * Try to convert current parser instance into a fully-parsed {@link PromptSlashCommand} token. - * - * @throws if sequence of tokens received so far do not constitute a valid prompt variable, - * for instance, if there is only `1` starting `/` token is available. - */ - public asPromptSlashCommand(): PromptSlashCommand { - // if there is only one token before the stop character - // must be the starting `/` one), then fail - assert( - this.currentTokens.length > 1, - 'Cannot create a prompt /command out of incomplete token sequence.', - ); - - const firstToken = this.currentTokens[0]; - const lastToken = this.currentTokens[this.currentTokens.length - 1]; - - // render the characters above into strings, excluding the starting `/` character - const nameTokens = this.currentTokens.slice(1); - const atMentionName = BaseToken.render(nameTokens); - - return new PromptSlashCommand( - new Range( - firstToken.range.startLineNumber, - firstToken.range.startColumn, - lastToken.range.endLineNumber, - lastToken.range.endColumn, - ), - atMentionName, - ); - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/parsers/promptTemplateVariableParser.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/parsers/promptTemplateVariableParser.ts deleted file mode 100644 index b97ad0fd479..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/parsers/promptTemplateVariableParser.ts +++ /dev/null @@ -1,148 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { assert } from '../../../../../../../base/common/assert.js'; -import { PromptTemplateVariable } from '../tokens/promptTemplateVariable.js'; -import { BaseToken } from '../base/baseToken.js'; -import { TSimpleDecoderToken } from '../base/simpleCodec/simpleDecoder.js'; -import { DollarSign, LeftCurlyBrace, RightCurlyBrace } from '../base/simpleCodec/tokens/tokens.js'; -import { assertNotConsumed, ParserBase, TAcceptTokenResult } from '../base/simpleCodec/parserBase.js'; - -/** - * Parsers of the `${variable}` token sequence in a prompt text. - */ -export type TPromptTemplateVariableParser = PartialPromptTemplateVariableStart | PartialPromptTemplateVariable; - -/** - * Parser that handles start sequence of a `${variable}` token sequence in - * a prompt text. Transitions to {@link PartialPromptTemplateVariable} parser - * as soon as the `${` character sequence is found. - */ -export class PartialPromptTemplateVariableStart extends ParserBase { - constructor(token: DollarSign) { - super([token]); - } - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - if (token instanceof LeftCurlyBrace) { - this.currentTokens.push(token); - - this.isConsumed = true; - return { - result: 'success', - nextParser: new PartialPromptTemplateVariable(this.currentTokens), - wasTokenConsumed: true, - }; - } - - return { - result: 'failure', - wasTokenConsumed: false, - }; - } -} - -/** - * Parser that handles a partial `${variable}` token sequence in a prompt text. - */ -export class PartialPromptTemplateVariable extends ParserBase { - constructor(tokens: (DollarSign | LeftCurlyBrace)[]) { - super(tokens); - } - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - // template variables are terminated by the `}` character - if (token instanceof RightCurlyBrace) { - this.currentTokens.push(token); - - this.isConsumed = true; - return { - result: 'success', - nextParser: this.asPromptTemplateVariable(), - wasTokenConsumed: true, - }; - } - - // otherwise it is a valid name character, so add it to the list of - // the current tokens and continue the parsing process - this.currentTokens.push(token); - - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } - - /** - * Returns a string representation of the prompt template variable - * contents, if any is present. - */ - private get contents(): string { - const contentTokens: TSimpleDecoderToken[] = []; - - // template variables are surrounded by `${}`, hence we need to have - // at least `${` plus one character for the contents to be non-empty - if (this.currentTokens.length < 3) { - return ''; - } - - // collect all tokens besides the first two (`${`) and a possible `}` at the end - for (let i = 2; i < this.currentTokens.length; i++) { - const token = this.currentTokens[i]; - const isLastToken = (i === this.currentTokens.length - 1); - - if ((token instanceof RightCurlyBrace) && (isLastToken === true)) { - break; - } - - contentTokens.push(token); - } - - return BaseToken.render(contentTokens); - } - - /** - * Try to convert current parser instance into a {@link PromptTemplateVariable} token. - * - * @throws if: - * - current tokens sequence cannot be converted to a valid template variable token - */ - public asPromptTemplateVariable(): PromptTemplateVariable { - const firstToken = this.currentTokens[0]; - const secondToken = this.currentTokens[1]; - const lastToken = this.currentTokens[this.currentTokens.length - 1]; - - // template variables are surrounded by `${}`, hence we need - // to have at least 3 tokens in the list for a valid one - assert( - this.currentTokens.length >= 3, - 'Prompt template variable should have at least 3 tokens.', - ); - - // a complete template variable must end with a `}` - assert( - lastToken instanceof RightCurlyBrace, - 'Last token is not a "}".', - ); - - // sanity checks of the first and second tokens - assert( - firstToken instanceof DollarSign, - 'First token must be a "$".', - ); - assert( - secondToken instanceof LeftCurlyBrace, - 'Second token must be a "{".', - ); - - return new PromptTemplateVariable( - BaseToken.fullRange(this.currentTokens), - this.contents, - ); - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/parsers/promptVariableParser.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/parsers/promptVariableParser.ts deleted file mode 100644 index ef68e78ec2d..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/parsers/promptVariableParser.ts +++ /dev/null @@ -1,252 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { assert } from '../../../../../../../base/common/assert.js'; -import { Range } from '../../../../../../../editor/common/core/range.js'; -import { BaseToken } from '../base/baseToken.js'; -import { PromptVariable, PromptVariableWithData } from '../tokens/promptVariable.js'; -import { At } from '../base/simpleCodec/tokens/at.js'; -import { Tab } from '../base/simpleCodec/tokens/tab.js'; -import { Hash } from '../base/simpleCodec/tokens/hash.js'; -import { Space } from '../base/simpleCodec/tokens/space.js'; -import { Colon } from '../base/simpleCodec/tokens/colon.js'; -import { NewLine } from '../base/linesCodec/tokens/newLine.js'; -import { FormFeed } from '../base/simpleCodec/tokens/formFeed.js'; -import { VerticalTab } from '../base/simpleCodec/tokens/verticalTab.js'; -import { TSimpleDecoderToken } from '../base/simpleCodec/simpleDecoder.js'; -import { CarriageReturn } from '../base/linesCodec/tokens/carriageReturn.js'; -import { ExclamationMark } from '../base/simpleCodec/tokens/exclamationMark.js'; -import { LeftBracket, RightBracket } from '../base/simpleCodec/tokens/brackets.js'; -import { LeftAngleBracket, RightAngleBracket } from '../base/simpleCodec/tokens/angleBrackets.js'; -import { assertNotConsumed, ParserBase, TAcceptTokenResult } from '../base/simpleCodec/parserBase.js'; - -/** - * List of characters that terminate the prompt variable sequence. - */ -export const STOP_CHARACTERS: readonly string[] = [Space, Tab, NewLine, CarriageReturn, VerticalTab, FormFeed, Hash, At] - .map((token) => { return token.symbol; }); - -/** - * List of characters that cannot be in a variable name (excluding the {@link STOP_CHARACTERS}). - */ -export const INVALID_NAME_CHARACTERS: readonly string[] = [Hash, Colon, ExclamationMark, LeftAngleBracket, RightAngleBracket, LeftBracket, RightBracket] - .map((token) => { return token.symbol; }); - -/** - * The parser responsible for parsing a `prompt variable name`. - * E.g., `#selection` or `#codebase` variable. If the `:` character follows - * the variable name, the parser transitions to {@link PartialPromptVariableWithData} - * that is also able to parse the `data` part of the variable. E.g., the `#file` part - * of the `#file:/path/to/something.md` sequence. - */ -export class PartialPromptVariableName extends ParserBase { - constructor(token: Hash) { - super([token]); - } - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - // if a `stop` character is encountered, finish the parsing process - if (STOP_CHARACTERS.includes(token.text)) { - try { - // if it is possible to convert current parser to `PromptVariable`, return success result - return { - result: 'success', - nextParser: this.asPromptVariable(), - wasTokenConsumed: false, - }; - } catch (error) { - // otherwise fail - return { - result: 'failure', - wasTokenConsumed: false, - }; - } finally { - // in any case this is an end of the parsing process - this.isConsumed = true; - } - } - - // if a `:` character is encountered, we might transition to {@link PartialPromptVariableWithData} - if (token instanceof Colon) { - this.isConsumed = true; - - // if there is only one token before the `:` character, it must be the starting - // `#` symbol, therefore fail because there is no variable name present - if (this.currentTokens.length <= 1) { - return { - result: 'failure', - wasTokenConsumed: false, - }; - } - - // otherwise, if there are more characters after `#` available, - // we have a variable name, so we can transition to {@link PromptVariableWithData} - return { - result: 'success', - nextParser: new PartialPromptVariableWithData([...this.currentTokens, token]), - wasTokenConsumed: true, - }; - } - - // variables cannot have {@link INVALID_NAME_CHARACTERS} in their names - if (INVALID_NAME_CHARACTERS.includes(token.text)) { - this.isConsumed = true; - - return { - result: 'failure', - wasTokenConsumed: false, - }; - } - - // otherwise, a valid name character, so add it to the list of - // the current tokens and continue the parsing process - this.currentTokens.push(token); - - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } - - /** - * Try to convert current parser instance into a fully-parsed {@link PromptVariable} token. - * - * @throws if sequence of tokens received so far do not constitute a valid prompt variable, - * for instance, if there is only `1` starting `#` token is available. - */ - public asPromptVariable(): PromptVariable { - // if there is only one token before the stop character - // must be the starting `#` one), then fail - assert( - this.currentTokens.length > 1, - 'Cannot create a prompt variable out of incomplete token sequence.', - ); - - const firstToken = this.currentTokens[0]; - const lastToken = this.currentTokens[this.currentTokens.length - 1]; - - // render the characters above into strings, excluding the starting `#` character - const variableNameTokens = this.currentTokens.slice(1); - const variableName = BaseToken.render(variableNameTokens); - - return new PromptVariable( - new Range( - firstToken.range.startLineNumber, - firstToken.range.startColumn, - lastToken.range.endLineNumber, - lastToken.range.endColumn, - ), - variableName, - ); - } -} - -/** - * The parser responsible for parsing a `prompt variable name` with `data`. - * E.g., the `/path/to/something.md` part of the `#file:/path/to/something.md` sequence. - */ -export class PartialPromptVariableWithData extends ParserBase { - - constructor(tokens: readonly TSimpleDecoderToken[]) { - const firstToken = tokens[0]; - const lastToken = tokens[tokens.length - 1]; - - // sanity checks of our expectations about the tokens list - assert( - tokens.length > 2, - `Tokens list must contain at least 3 items, got '${tokens.length}'.`, - ); - assert( - firstToken instanceof Hash, - `The first token must be a '#', got '${firstToken} '.`, - ); - assert( - lastToken instanceof Colon, - `The last token must be a ':', got '${lastToken} '.`, - ); - - super([...tokens]); - } - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - // if a `stop` character is encountered, finish the parsing process - if (STOP_CHARACTERS.includes(token.text)) { - // in any case, success of failure below, this is an end of the parsing process - this.isConsumed = true; - - const firstToken = this.currentTokens[0]; - const lastToken = this.currentTokens[this.currentTokens.length - 1]; - - // tokens representing variable name without the `#` character at the start and - // the `:` data separator character at the end - const variableNameTokens = this.currentTokens.slice(1, this.startTokensCount - 1); - // tokens representing variable data without the `:` separator character at the start - const variableDataTokens = this.currentTokens.slice(this.startTokensCount); - // compute the full range of the variable token - const fullRange = new Range( - firstToken.range.startLineNumber, - firstToken.range.startColumn, - lastToken.range.endLineNumber, - lastToken.range.endColumn, - ); - - // render the characters above into strings - const variableName = BaseToken.render(variableNameTokens); - const variableData = BaseToken.render(variableDataTokens); - - return { - result: 'success', - nextParser: new PromptVariableWithData( - fullRange, - variableName, - variableData, - ), - wasTokenConsumed: false, - }; - } - - // otherwise, token is a valid data character - the data can contain almost any character, - // including `:` and `#`, hence add it to the list of the current tokens and continue - this.currentTokens.push(token); - - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } - - /** - * Try to convert current parser instance into a fully-parsed {@link asPromptVariableWithData} token. - */ - public asPromptVariableWithData(): PromptVariableWithData { - // tokens representing variable name without the `#` character at the start and - // the `:` data separator character at the end - const variableNameTokens = this.currentTokens.slice(1, this.startTokensCount - 1); - // tokens representing variable data without the `:` separator character at the start - const variableDataTokens = this.currentTokens.slice(this.startTokensCount); - - // render the characters above into strings - const variableName = BaseToken.render(variableNameTokens); - const variableData = BaseToken.render(variableDataTokens); - - const firstToken = this.currentTokens[0]; - const lastToken = this.currentTokens[this.currentTokens.length - 1]; - - return new PromptVariableWithData( - new Range( - firstToken.range.startLineNumber, - firstToken.range.startColumn, - lastToken.range.endLineNumber, - lastToken.range.endColumn, - ), - variableName, - variableData, - ); - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/fileReference.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/fileReference.ts deleted file mode 100644 index 78c4f7cb6ea..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/fileReference.ts +++ /dev/null @@ -1,50 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - - -import { PromptVariableWithData } from './promptVariable.js'; -import { assert } from '../../../../../../../base/common/assert.js'; -import { IRange, Range } from '../../../../../../../editor/common/core/range.js'; - -/** - * Name of the variable. - */ -const VARIABLE_NAME: string = 'file'; - -/** - * Object represents a file reference token inside a chatbot prompt. - */ -export class FileReference extends PromptVariableWithData { - constructor( - range: Range, - public readonly path: string, - ) { - super(range, VARIABLE_NAME, path); - } - - /** - * Create a {@link FileReference} from a {@link PromptVariableWithData} instance. - * @throws if variable name is not equal to {@link VARIABLE_NAME}. - */ - public static from(variable: PromptVariableWithData): FileReference { - assert( - variable.name === VARIABLE_NAME, - `Variable name must be '${VARIABLE_NAME}', got '${variable.name}'.`, - ); - - return new FileReference( - variable.range, - variable.data, - ); - } - - /** - * Get the range of the `link` part of the token (e.g., - * the `/path/to/file.md` part of `#file:/path/to/file.md`). - */ - public get linkRange(): IRange | undefined { - return super.dataRange; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptAtMention.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptAtMention.ts deleted file mode 100644 index 43bd8005235..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptAtMention.ts +++ /dev/null @@ -1,52 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { PromptToken } from './promptToken.js'; -import { assert } from '../../../../../../../base/common/assert.js'; -import { Range } from '../../../../../../../editor/common/core/range.js'; -import { INVALID_NAME_CHARACTERS, STOP_CHARACTERS } from '../parsers/promptVariableParser.js'; - -/** - * All prompt at-mentions start with `@` character. - */ -const START_CHARACTER: string = '@'; - -/** - * Represents a `@mention` token in a prompt text. - */ -export class PromptAtMention extends PromptToken { - constructor( - range: Range, - /** - * The name of a mention, excluding the `@` character at the start. - */ - public readonly name: string, - ) { - // sanity check of characters used in the provided mention name - for (const character of name) { - assert( - (INVALID_NAME_CHARACTERS.includes(character) === false) && - (STOP_CHARACTERS.includes(character) === false), - `Mention 'name' cannot contain character '${character}', got '${name}'.`, - ); - } - - super(range); - } - - /** - * Get full text of the token. - */ - public get text(): string { - return `${START_CHARACTER}${this.name}`; - } - - /** - * Return a string representation of the token. - */ - public override toString(): string { - return `${this.text}${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptSlashCommand.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptSlashCommand.ts deleted file mode 100644 index 1069d4a7a89..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptSlashCommand.ts +++ /dev/null @@ -1,42 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { PromptToken } from './promptToken.js'; -import { Range } from '../../../../../../../editor/common/core/range.js'; - -/** - * All prompt at-mentions start with `/` character. - */ -const START_CHARACTER: string = '/'; - -/** - * Represents a `/command` token in a prompt text. - */ -export class PromptSlashCommand extends PromptToken { - constructor( - range: Range, - /** - * The name of a command, excluding the `/` character at the start. - */ - public readonly name: string, - ) { - - super(range); - } - - /** - * Get full text of the token. - */ - public get text(): string { - return `${START_CHARACTER}${this.name}`; - } - - /** - * Return a string representation of the token. - */ - public override toString(): string { - return `${this.text}${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptTemplateVariable.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptTemplateVariable.ts deleted file mode 100644 index be3051f2c88..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptTemplateVariable.ts +++ /dev/null @@ -1,44 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { PromptToken } from './promptToken.js'; -import { Range } from '../../../../../../../editor/common/core/range.js'; -import { DollarSign } from '../base/simpleCodec/tokens/dollarSign.js'; -import { LeftCurlyBrace, RightCurlyBrace } from '../base/simpleCodec/tokens/curlyBraces.js'; - -/** - * Represents a `${variable}` token in a prompt text. - */ -export class PromptTemplateVariable extends PromptToken { - constructor( - range: Range, - /** - * The contents of the template variable, excluding - * the surrounding `${}` characters. - */ - public readonly contents: string, - ) { - super(range); - } - - /** - * Get full text of the token. - */ - public get text(): string { - return [ - DollarSign.symbol, - LeftCurlyBrace.symbol, - this.contents, - RightCurlyBrace.symbol, - ].join(''); - } - - /** - * Return a string representation of the token. - */ - public override toString(): string { - return `${this.text}${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptToken.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptToken.ts deleted file mode 100644 index 021b8d5a425..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptToken.ts +++ /dev/null @@ -1,11 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../base/baseToken.js'; - -/** - * Common base token that all chatbot `prompt` tokens should inherit from. - */ -export abstract class PromptToken extends BaseToken { } diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptVariable.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptVariable.ts deleted file mode 100644 index ec5dd2e7a36..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptVariable.ts +++ /dev/null @@ -1,103 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { PromptToken } from './promptToken.js'; -import { IRange, Range } from '../../../../../../../editor/common/core/range.js'; - -/** - * All prompt variables start with `#` character. - */ -const START_CHARACTER: string = '#'; - -/** - * Character that separates name of a prompt variable from its data. - */ -const DATA_SEPARATOR: string = ':'; - -/** - * Represents a `#variable` token in a prompt text. - */ -export class PromptVariable extends PromptToken { - constructor( - range: Range, - /** - * The name of a prompt variable, excluding the `#` character at the start. - */ - public readonly name: string, - ) { - - super(range); - } - - /** - * Get full text of the token. - */ - public get text(): string { - return `${START_CHARACTER}${this.name}`; - } - - /** - * Return a string representation of the token. - */ - public override toString(): string { - return `${this.text}${this.range}`; - } -} - -/** - * Represents a {@link PromptVariable} with additional data token in a prompt text. - * (e.g., `#variable:/path/to/file.md`) - */ -export class PromptVariableWithData extends PromptVariable { - constructor( - fullRange: Range, - /** - * The name of the variable, excluding the starting `#` character. - */ - name: string, - - /** - * The data of the variable, excluding the starting {@link DATA_SEPARATOR} character. - */ - public readonly data: string, - ) { - super(fullRange, name); - } - - /** - * Get full text of the token. - */ - public override get text(): string { - return `${START_CHARACTER}${this.name}${DATA_SEPARATOR}${this.data}`; - } - - /** - * Range of the `data` part of the variable. - */ - public get dataRange(): IRange | undefined { - const { range } = this; - - // calculate the start column number of the `data` part of the variable - const dataStartColumn = range.startColumn + - START_CHARACTER.length + this.name.length + - DATA_SEPARATOR.length; - - // create `range` of the `data` part of the variable - const result = new Range( - range.startLineNumber, - dataStartColumn, - range.endLineNumber, - range.endColumn, - ); - - // if the resulting range is empty, return `undefined` - // because there is no `data` part present in the variable - if (result.isEmpty()) { - return undefined; - } - - return result; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/contentProviders/filePromptContentsProvider.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/contentProviders/filePromptContentsProvider.ts deleted file mode 100644 index 47705213282..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/contentProviders/filePromptContentsProvider.ts +++ /dev/null @@ -1,166 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { PROMPT_LANGUAGE_ID } from '../promptTypes.js'; -import { IPromptContentsProvider } from './types.js'; -import { URI } from '../../../../../../base/common/uri.js'; -import { assert } from '../../../../../../base/common/assert.js'; -import { CancellationError } from '../../../../../../base/common/errors.js'; -import { VSBufferReadableStream } from '../../../../../../base/common/buffer.js'; -import { CancellationToken } from '../../../../../../base/common/cancellation.js'; -import { IModelService } from '../../../../../../editor/common/services/model.js'; -import { ILanguageService } from '../../../../../../editor/common/languages/language.js'; -import { isPromptOrInstructionsFile } from '../config/promptFileLocations.js'; -import { IPromptContentsProviderOptions, PromptContentsProviderBase } from './promptContentsProviderBase.js'; -import { OpenFailed, NotPromptFile, ResolveError, FolderReference } from '../../promptFileReferenceErrors.js'; -import { FileChangesEvent, FileChangeType, IFileService } from '../../../../../../platform/files/common/files.js'; - -/** - * Prompt contents provider for a file on the disk referenced by - * a provided {@link URI}. - */ -export class FilePromptContentProvider extends PromptContentsProviderBase implements IPromptContentsProvider { - public override get sourceName(): string { - return 'file'; - } - - public override get languageId(): string { - if (this.options.languageId) { - return this.options.languageId; - } - - const model = this.modelService.getModel(this.uri); - - if (model !== null) { - return model.getLanguageId(); - } - - const inferredId = this.languageService - .guessLanguageIdByFilepathOrFirstLine(this.uri); - - if (inferredId !== null) { - return inferredId; - } - - // fallback to the default prompt language ID - return PROMPT_LANGUAGE_ID; - } - - constructor( - public readonly uri: URI, - options: IPromptContentsProviderOptions, - @IFileService private readonly fileService: IFileService, - @IModelService private readonly modelService: IModelService, - @ILanguageService private readonly languageService: ILanguageService, - ) { - super(options); - - if (options.updateOnChange) { - // make sure the object is updated on file changes - this._register( - this.fileService.onDidFilesChange((event) => { - // if file was added or updated, forward the event to - // the `getContentsStream()` produce a new stream for file contents - if (event.contains(this.uri, FileChangeType.ADDED, FileChangeType.UPDATED)) { - // we support only full file parsing right now because - // the event doesn't contain a list of changed lines - this.onChangeEmitter.fire('full'); - return; - } - - // if file was deleted, forward the event to - // the `getContentsStream()` produce an error - if (event.contains(this.uri, FileChangeType.DELETED)) { - this.onChangeEmitter.fire(event); - return; - } - }), - ); - } - } - - /** - * Creates a stream of lines from the file based on the changes listed in - * the provided event. - * - * @param event - event that describes the changes in the file; `'full'` is - * the special value that means that all contents have changed - * @param cancellationToken - token that cancels this operation - */ - protected async getContentsStream( - _event: FileChangesEvent | 'full', - cancellationToken?: CancellationToken, - ): Promise { - assert( - !cancellationToken?.isCancellationRequested, - new CancellationError(), - ); - - // get the binary stream of the file contents - let fileStream; - try { - // ensure that the referenced URI points to a file before - // trying to get a stream for its contents - const info = await this.fileService.resolve(this.uri); - - // validate that the cancellation was not yet requested - assert( - !cancellationToken?.isCancellationRequested, - new CancellationError(), - ); - - assert( - info.isFile, - new FolderReference(this.uri), - ); - - const { allowNonPromptFiles } = this.options; - - // if URI doesn't point to a prompt file, don't try to resolve it, - // unless the `allowNonPromptFiles` option is set to `true` - if ((allowNonPromptFiles !== true) && (isPromptOrInstructionsFile(this.uri) === false)) { - throw new NotPromptFile(this.uri); - } - - fileStream = await this.fileService.readFileStream(this.uri); - - // after the promise above complete, this object can be already disposed or - // the cancellation could be requested, in that case destroy the stream and - // throw cancellation error - if (this.isDisposed || cancellationToken?.isCancellationRequested) { - fileStream.value.destroy(); - throw new CancellationError(); - } - - return fileStream.value; - } catch (error) { - if ((error instanceof ResolveError) || (error instanceof CancellationError)) { - throw error; - } - - throw new OpenFailed(this.uri, error); - } - } - - public override createNew( - promptContentsSource: { uri: URI }, - options: IPromptContentsProviderOptions, - ): IPromptContentsProvider { - return new FilePromptContentProvider( - promptContentsSource.uri, - options, - this.fileService, - this.modelService, - this.languageService, - ); - } - - /** - * String representation of this object. - */ - public override toString(): string { - return `file-prompt-contents-provider:${this.uri.path}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/contentProviders/promptContentsProviderBase.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/contentProviders/promptContentsProviderBase.ts deleted file mode 100644 index 7c9e58dc3d3..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/contentProviders/promptContentsProviderBase.ts +++ /dev/null @@ -1,196 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { assert } from '../../../../../../base/common/assert.js'; -import { VSBufferReadableStream } from '../../../../../../base/common/buffer.js'; -import { CancellationToken } from '../../../../../../base/common/cancellation.js'; -import { cancelPreviousCalls } from '../../../../../../base/common/decorators/cancelPreviousCalls.js'; -import { CancellationError } from '../../../../../../base/common/errors.js'; -import { Emitter } from '../../../../../../base/common/event.js'; -import { URI } from '../../../../../../base/common/uri.js'; -import { FailedToResolveContentsStream, ResolveError } from '../../promptFileReferenceErrors.js'; -import { INSTRUCTIONS_LANGUAGE_ID, MODE_LANGUAGE_ID, PROMPT_LANGUAGE_ID, PromptsType } from '../promptTypes.js'; -import { ObservableDisposable } from '../utils/observableDisposable.js'; -import { IPromptContentsProvider } from './types.js'; - -/** - * Options of the {@link PromptContentsProviderBase} class. - */ -export interface IPromptContentsProviderOptions { - /** - * Whether to allow files that don't have usual prompt - * file extension to be treated as a prompt file. - */ - readonly allowNonPromptFiles: boolean; - - /** - * Language ID to use for the prompt contents. If not set, the language ID will be inferred from the file. - */ - readonly languageId: string | undefined; - - /** - * If set to `true`, the contents provider will listen for updates and retrigger a parse. - */ - readonly updateOnChange: boolean; -} - - -/** - * Base class for prompt contents providers. Classes that extend this one are responsible to: - * - * - implement the {@link getContentsStream} method to provide the contents stream - * of a prompt; this method should throw a `ResolveError` or its derivative if the contents - * cannot be parsed for any reason - * - fire a {@link TChangeEvent} event on the {@link onChangeEmitter} event when - * prompt contents change - * - misc: - * - provide the {@link uri} property that represents the URI of a prompt that - * the contents are for - * - implement the {@link toString} method to return a string representation of this - * provider type to aid with debugging/tracing - */ -export abstract class PromptContentsProviderBase< - TChangeEvent extends NonNullable, -> extends ObservableDisposable implements IPromptContentsProvider { - public abstract readonly uri: URI; - public abstract createNew(promptContentsSource: { uri: URI }, options: IPromptContentsProviderOptions): IPromptContentsProvider; - public abstract override toString(): string; - public abstract get languageId(): string; - public abstract get sourceName(): string; - - /** - * Prompt contents stream. - */ - public get contents(): Promise { - return this.getContentsStream('full'); - } - - /** - * Prompt type used to determine how to interpret file contents. - */ - public get promptType(): PromptsType | 'non-prompt' { - const { languageId } = this; - - if (languageId === PROMPT_LANGUAGE_ID) { - return PromptsType.prompt; - } - - if (languageId === INSTRUCTIONS_LANGUAGE_ID) { - return PromptsType.instructions; - } - - if (languageId === MODE_LANGUAGE_ID) { - return PromptsType.mode; - } - - return 'non-prompt'; - } - - /** - * Function to get contents stream for the provider. This function should - * throw a `ResolveError` or its derivative if the contents cannot be parsed. - * - * @param changesEvent The event that triggered the change. The special - * `'full'` value means that everything has changed hence entire prompt - * contents need to be re-parsed from scratch. - */ - protected abstract getContentsStream( - changesEvent: TChangeEvent | 'full', - cancellationToken?: CancellationToken, - ): Promise; - - /** - * Internal event emitter for the prompt contents change event. Classes that extend - * this abstract class are responsible to use this emitter to fire the contents change - * event when the prompt contents get modified. - */ - protected readonly onChangeEmitter = this._register(new Emitter()); - - /** - * Options passed to the constructor - */ - protected readonly options: IPromptContentsProviderOptions; - - constructor( - options: IPromptContentsProviderOptions, - ) { - super(); - - this.options = options; - } - - /** - * Event emitter for the prompt contents change event. - * See {@link onContentChanged} for more details. - */ - private readonly onContentChangedEmitter = this._register(new Emitter()); - - /** - * Event that fires when the prompt contents change. The event is either - * a `VSBufferReadableStream` stream with changed contents or an instance of - * the `ResolveError` class representing a parsing failure case. - * - * `Note!` this field is meant to be used by the external consumers of the prompt - * contents provider that the classes that extend this abstract class. - * Please use the {@link onChangeEmitter} event to provide a change - * event in your prompt contents implementation instead. - */ - public readonly onContentChanged = this.onContentChangedEmitter.event; - - /** - * Internal common implementation of the event that should be fired when - * prompt contents change. - */ - @cancelPreviousCalls - private onContentsChanged( - event: TChangeEvent | 'full', - cancellationToken?: CancellationToken, - ): this { - const promise = (cancellationToken?.isCancellationRequested) - ? Promise.reject(new CancellationError()) - : this.getContentsStream(event, cancellationToken); - - promise - .then((stream) => { - if (cancellationToken?.isCancellationRequested || this.isDisposed) { - stream.destroy(); - throw new CancellationError(); - } - - this.onContentChangedEmitter.fire(stream); - }) - .catch((error) => { - if (error instanceof ResolveError) { - this.onContentChangedEmitter.fire(error); - - return; - } - - this.onContentChangedEmitter.fire( - new FailedToResolveContentsStream(this.uri, error), - ); - }); - - return this; - } - - /** - * Start producing the prompt contents data. - */ - public start(token?: CancellationToken): this { - assert( - !this.isDisposed, - 'Cannot start contents provider that was already disposed.', - ); - - // `'full'` means "everything has changed" - this.onContentsChanged('full', token); - - // subscribe to the change event emitted by a child class - this._register(this.onChangeEmitter.event(this.onContentsChanged, this)); - - return this; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/contentProviders/textModelContentsProvider.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/contentProviders/textModelContentsProvider.ts deleted file mode 100644 index aea65908ccf..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/contentProviders/textModelContentsProvider.ts +++ /dev/null @@ -1,93 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { VSBufferReadableStream } from '../../../../../../base/common/buffer.js'; -import { CancellationToken } from '../../../../../../base/common/cancellation.js'; -import { URI } from '../../../../../../base/common/uri.js'; -import { ITextModel } from '../../../../../../editor/common/model.js'; -import { TextModel } from '../../../../../../editor/common/model/textModel.js'; -import { IModelContentChangedEvent } from '../../../../../../editor/common/textModelEvents.js'; -import { IInstantiationService } from '../../../../../../platform/instantiation/common/instantiation.js'; -import { objectStreamFromTextModel } from '../codecs/base/utils/objectStreamFromTextModel.js'; -import { FilePromptContentProvider } from './filePromptContentsProvider.js'; -import { IPromptContentsProviderOptions, PromptContentsProviderBase } from './promptContentsProviderBase.js'; -import { IPromptContentsProvider } from './types.js'; - -/** - * Prompt contents provider for a {@link ITextModel} instance. - */ -export class TextModelContentsProvider extends PromptContentsProviderBase { - /** - * URI component of the prompt associated with this contents provider. - */ - public get uri(): URI { - return this.model.uri; - } - - public override get sourceName(): string { - return 'text-model'; - } - - public override get languageId(): string { - return this.options.languageId ?? this.model.getLanguageId(); - } - - constructor( - private readonly model: ITextModel, - options: IPromptContentsProviderOptions, - @IInstantiationService private readonly instantiationService: IInstantiationService, - ) { - super(options); - - this._register(this.model.onWillDispose(this.dispose.bind(this))); - if (options.updateOnChange) { - this._register(this.model.onDidChangeContent(this.onChangeEmitter.fire.bind(this.onChangeEmitter))); - } - } - - /** - * Creates a stream of binary data from the text model based on the changes - * listed in the provided event. - * - * Note! this method implements a basic logic which does not take into account - * the `_event` argument for incremental updates. This needs to be improved. - * - * @param _event - event that describes the changes in the text model; `'full'` is - * the special value that means that all contents have changed - * @param cancellationToken - token that cancels this operation - */ - protected override async getContentsStream( - _event: IModelContentChangedEvent | 'full', - cancellationToken?: CancellationToken, - ): Promise { - return objectStreamFromTextModel(this.model, cancellationToken); - } - - public override createNew( - promptContentsSource: TextModel | { uri: URI }, - options: IPromptContentsProviderOptions, - ): IPromptContentsProvider { - if (promptContentsSource instanceof TextModel) { - return this.instantiationService.createInstance( - TextModelContentsProvider, - promptContentsSource, - options, - ); - } - - return this.instantiationService.createInstance( - FilePromptContentProvider, - promptContentsSource.uri, - options, - ); - } - - /** - * String representation of this object. - */ - public override toString(): string { - return `text-model-prompt-contents-provider:${this.uri.path}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/contentProviders/types.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/contentProviders/types.ts deleted file mode 100644 index a26eaa43330..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/contentProviders/types.ts +++ /dev/null @@ -1,70 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { URI } from '../../../../../../base/common/uri.js'; -import { Event } from '../../../../../../base/common/event.js'; -import { ResolveError } from '../../promptFileReferenceErrors.js'; -import { IDisposable } from '../../../../../../base/common/lifecycle.js'; -import { VSBufferReadableStream } from '../../../../../../base/common/buffer.js'; -import { PromptsType } from '../promptTypes.js'; -import { CancellationToken } from '../../../../../../base/common/cancellation.js'; -import { IPromptContentsProviderOptions } from './promptContentsProviderBase.js'; - -/** - * Interface for a prompt contents provider. Prompt contents providers are - * responsible for providing contents of a prompt as a byte streams and - * allow to subscribe to the change events of the prompt contents. - */ -export interface IPromptContentsProvider extends IDisposable { - /** - * URI component of the prompt associated with this contents provider. - */ - readonly uri: URI; - - /** - * Language ID of the prompt contents. - */ - readonly languageId: string; - - /** - * Prompt type used to determine how to interpret file contents. - */ - readonly promptType: PromptsType | 'non-prompt'; - - /** - * Prompt contents stream. - */ - readonly contents: Promise; - - /** - * Prompt contents source name. - */ - readonly sourceName: string; - - /** - * Event that fires when the prompt contents change. The event is either a - * {@linkcode VSBufferReadableStream} stream with changed contents or - * an instance of the {@linkcode ResolveError} error. - */ - readonly onContentChanged: Event; - - /** - * Subscribe to `onDispose` event of the contents provider. - */ - readonly onDispose: Event; - - /** - * Start the contents provider to produce the underlying contents. - */ - start(token?: CancellationToken): this; - - /** - * Create a new instance of prompt contents provider. - */ - createNew( - promptContentsSource: { uri: URI }, - options: IPromptContentsProviderOptions, - ): IPromptContentsProvider; -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/frontMatterDecoration.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/frontMatterDecoration.ts deleted file mode 100644 index d07720eff5b..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/frontMatterDecoration.ts +++ /dev/null @@ -1,120 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { Position } from '../../../../../../../../editor/common/core/position.js'; -import { localize } from '../../../../../../../../nls.js'; -import { contrastBorder, editorBackground } from '../../../../../../../../platform/theme/common/colorRegistry.js'; -import { asCssVariable, ColorIdentifier, darken, registerColor } from '../../../../../../../../platform/theme/common/colorUtils.js'; -import { BaseToken } from '../../../codecs/base/baseToken.js'; -import { FrontMatterHeader } from '../../../codecs/base/markdownExtensionsCodec/tokens/frontMatterHeader.js'; -import { CssClassModifiers } from '../types.js'; -import { FrontMatterMarkerDecoration } from './frontMatterMarkerDecoration.js'; -import { ReactiveDecorationBase } from './utils/reactiveDecorationBase.js'; -import { IReactiveDecorationClassNames, TAddAccessor, TDecorationStyles } from './utils/types.js'; - -/** - * Decoration CSS class names. - */ -export enum CssClassNames { - Main = '.prompt-front-matter-decoration', - Inline = '.prompt-front-matter-decoration-inline', - MainInactive = `${CssClassNames.Main}${CssClassModifiers.Inactive}`, - InlineInactive = `${CssClassNames.Inline}${CssClassModifiers.Inactive}`, -} - -/** - * Main background color of `active` Front Matter header block. - */ -export const BACKGROUND_COLOR: ColorIdentifier = registerColor( - 'prompt.frontMatter.background', - { dark: darken(editorBackground, 0.2), light: darken(editorBackground, 0.05), hcDark: contrastBorder, hcLight: contrastBorder }, - localize('chat.prompt.frontMatter.background.description', "Background color of a Front Matter header block."), -); - -/** - * Background color of `inactive` Front Matter header block. - */ -export const INACTIVE_BACKGROUND_COLOR: ColorIdentifier = registerColor( - 'prompt.frontMatter.inactiveBackground', - { dark: darken(editorBackground, 0.1), light: darken(editorBackground, 0.025), hcDark: contrastBorder, hcLight: contrastBorder }, - localize('chat.prompt.frontMatter.inactiveBackground.description', "Background color of an inactive Front Matter header block."), -); - -/** - * CSS styles for the decoration. - */ -export const CSS_STYLES = { - [CssClassNames.Main]: [ - `background-color: ${asCssVariable(BACKGROUND_COLOR)};`, - 'z-index: -1;', // this is required to allow for selections to appear above the decoration background - ], - [CssClassNames.MainInactive]: [ - `background-color: ${asCssVariable(INACTIVE_BACKGROUND_COLOR)};`, - ], - [CssClassNames.InlineInactive]: [ - 'color: var(--vscode-disabledForeground);', - ], - ...FrontMatterMarkerDecoration.cssStyles, -}; - -/** - * Editor decoration for the Front Matter header token inside a prompt. - */ -export class FrontMatterDecoration extends ReactiveDecorationBase { - constructor( - accessor: TAddAccessor, - token: FrontMatterHeader, - ) { - super(accessor, token); - - this.childDecorators.push( - new FrontMatterMarkerDecoration(accessor, token.startMarker), - new FrontMatterMarkerDecoration(accessor, token.endMarker), - ); - } - - public override setCursorPosition( - position: Position | null | undefined, - ): this is { readonly changed: true } { - const result = super.setCursorPosition(position); - - for (const marker of this.childDecorators) { - if ((marker instanceof FrontMatterMarkerDecoration) === false) { - continue; - } - - // activate/deactivate markers based on the active state - // of the main Front Matter header decoration - marker.activate(this.active); - } - - return result; - } - - protected override get classNames(): IReactiveDecorationClassNames { - return CssClassNames; - } - - protected override get isWholeLine(): boolean { - return true; - } - - protected override get description(): string { - return 'Front Matter header decoration.'; - } - - public static get cssStyles(): TDecorationStyles { - return CSS_STYLES; - } - - /** - * Whether current decoration class can decorate provided token. - */ - public static handles( - token: BaseToken, - ): token is FrontMatterHeader { - return token instanceof FrontMatterHeader; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/frontMatterMarkerDecoration.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/frontMatterMarkerDecoration.ts deleted file mode 100644 index 2a4a174c9f4..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/frontMatterMarkerDecoration.ts +++ /dev/null @@ -1,56 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { CssClassModifiers } from '../types.js'; -import { TDecorationStyles, IReactiveDecorationClassNames } from './utils/types.js'; -import { FrontMatterMarker } from '../../../codecs/base/markdownExtensionsCodec/tokens/frontMatterMarker.js'; -import { ReactiveDecorationBase } from './utils/reactiveDecorationBase.js'; - -/** - * Decoration CSS class names. - */ -export enum CssClassNames { - Main = '.prompt-front-matter-decoration-marker', - Inline = '.prompt-front-matter-decoration-marker-inline', - MainInactive = `${CssClassNames.Main}${CssClassModifiers.Inactive}`, - InlineInactive = `${CssClassNames.Inline}${CssClassModifiers.Inactive}`, -} - -/** - * Editor decoration for a `marker` token of a Front Matter header. - */ -export class FrontMatterMarkerDecoration extends ReactiveDecorationBase { - /** - * Activate/deactivate the decoration. - */ - public activate(state: boolean): this { - const position = (state === true) - ? this.token.range.getStartPosition() - : null; - - this.setCursorPosition(position); - - return this; - } - - protected override get classNames(): IReactiveDecorationClassNames { - return CssClassNames; - } - - protected override get description(): string { - return 'Marker decoration of a Front Matter header.'; - } - - public static get cssStyles(): TDecorationStyles { - return { - [CssClassNames.Inline]: [ - 'color: var(--vscode-disabledForeground);', - ], - [CssClassNames.InlineInactive]: [ - 'opacity: 0.25;', - ], - }; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/utils/decorationBase.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/utils/decorationBase.ts deleted file mode 100644 index 8f28ec33247..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/utils/decorationBase.ts +++ /dev/null @@ -1,127 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { Range } from '../../../../../../../../../editor/common/core/range.js'; -import { IMarkdownString } from '../../../../../../../../../base/common/htmlContent.js'; -import { BaseToken } from '../../../../codecs/base/baseToken.js'; -import { TrackedRangeStickiness } from '../../../../../../../../../editor/common/model.js'; -import type { TAddAccessor, TChangeAccessor, TDecorationStyles, TRemoveAccessor } from './types.js'; -import { ModelDecorationOptions } from '../../../../../../../../../editor/common/model/textModel.js'; - -/** - * Base class for all editor decorations. - */ -export abstract class DecorationBase< - TPromptToken extends BaseToken, - TCssClassName extends string = string, -> { - /** - * Description of the decoration. - */ - protected abstract get description(): string; - - /** - * Default CSS class name of the decoration. - */ - protected abstract get className(): TCssClassName; - - /** - * Inline CSS class name of the decoration. - */ - protected abstract get inlineClassName(): TCssClassName; - - /** - * Indicates whether the decoration spans the whole line(s). - */ - protected get isWholeLine(): boolean { - return false; - } - - /** - * Hover message of the decoration. - */ - protected get hoverMessage(): IMarkdownString | IMarkdownString[] | null { - return null; - } - - /** - * ID of editor decoration it was registered with. - */ - public readonly id: string; - - constructor( - accessor: TAddAccessor, - protected readonly token: TPromptToken, - ) { - this.id = accessor.addDecoration(this.range, this.decorationOptions); - } - - /** - * Range of the decoration. - */ - public get range(): Range { - return this.token.range; - } - - /** - * Changes the decoration in the editor. - */ - public change( - accessor: TChangeAccessor, - ): this { - accessor.changeDecorationOptions( - this.id, - this.decorationOptions, - ); - - return this; - } - - /** - * Removes associated editor decoration(s). - */ - public remove( - accessor: TRemoveAccessor, - ): this { - accessor.removeDecoration(this.id); - - return this; - } - - /** - * Get editor decoration options for this decorator. - */ - private get decorationOptions(): ModelDecorationOptions { - return ModelDecorationOptions.createDynamic({ - description: this.description, - hoverMessage: this.hoverMessage, - className: this.className, - inlineClassName: this.inlineClassName, - isWholeLine: this.isWholeLine, - stickiness: TrackedRangeStickiness.NeverGrowsWhenTypingAtEdges, - shouldFillLineOnLineBreak: true, - }); - } -} - -/** - * Type of a generic decoration class. - */ -export type TDecorationClass = { - new( - accessor: TAddAccessor, - token: TPromptToken, - ): DecorationBase; - - /** - * CSS styles for the decoration. - */ - readonly cssStyles: TDecorationStyles; - - /** - * Whether the decoration class handles the provided token. - */ - handles(token: BaseToken): token is TPromptToken; -}; diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/utils/reactiveDecorationBase.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/utils/reactiveDecorationBase.ts deleted file mode 100644 index 42533aea5fe..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/utils/reactiveDecorationBase.ts +++ /dev/null @@ -1,162 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { DecorationBase } from './decorationBase.js'; -import { Position } from '../../../../../../../../../editor/common/core/position.js'; -import { BaseToken } from '../../../../codecs/base/baseToken.js'; -import type { IReactiveDecorationClassNames, TAddAccessor, TChangeAccessor, TRemoveAccessor } from './types.js'; - -/** - * Base class for all reactive editor decorations. A reactive decoration - * is a decoration that can change its appearance based on current cursor - * position in the editor, hence can "react" to the user's actions. - */ -export abstract class ReactiveDecorationBase< - TPromptToken extends BaseToken, - TCssClassName extends string = string, -> extends DecorationBase { - /** - * CSS class names of the decoration. - */ - protected abstract get classNames(): IReactiveDecorationClassNames; - - /** - * A list of child decorators that are part of this decoration. - * For instance a Front Matter header decoration can have child - * decorators for each of the header's `---` markers. - */ - protected readonly childDecorators: DecorationBase[]; - - /** - * Whether the decoration has changed since the last {@link change}. - */ - public get changed(): boolean { - // if any of the child decorators changed, this object is also - // considered to be changed - for (const marker of this.childDecorators) { - if ((marker instanceof ReactiveDecorationBase) === false) { - continue; - } - - if (marker.changed === true) { - return true; - } - } - - return this.didChange; - } - - constructor( - accessor: TAddAccessor, - token: TPromptToken, - ) { - super(accessor, token); - - this.childDecorators = []; - } - - /** - * Current position of cursor in the editor. - */ - private cursorPosition?: Position | null; - - /** - * Private field for the {@link changed} property. - */ - private didChange = true; - - /** - * Whether cursor is currently inside the decoration range. - */ - protected get active(): boolean { - return true; - - /** - * Temporarily disable until we have a proper way to get - * the cursor position inside active editor. - */ - /** - * if (!this.cursorPosition) { - * return false; - * } - * - * // when cursor is at the end of a range, the range considered to - * // not contain the position, but we want to include it - * const atEnd = (this.range.endLineNumber === this.cursorPosition.lineNumber) - * && (this.range.endColumn === this.cursorPosition.column); - * - * return atEnd || this.range.containsPosition(this.cursorPosition); - */ - } - - /** - * Set cursor position and update {@link changed} property if needed. - */ - public setCursorPosition( - position: Position | null | undefined, - ): this is { readonly changed: true } { - if (this.cursorPosition === position) { - return false; - } - - if (this.cursorPosition && position) { - if (this.cursorPosition.equals(position)) { - return false; - } - } - - const wasActive = this.active; - this.cursorPosition = position; - this.didChange = (wasActive !== this.active); - - return this.changed; - } - - public override change( - accessor: TChangeAccessor, - ): this { - if (this.didChange === false) { - return this; - } - - super.change(accessor); - this.didChange = false; - - for (const marker of this.childDecorators) { - marker.change(accessor); - } - - return this; - } - - public override remove( - accessor: TRemoveAccessor, - ): this { - super.remove(accessor); - - for (const marker of this.childDecorators) { - marker.remove(accessor); - } - - return this; - } - - protected override get className(): TCssClassName { - return (this.active) - ? this.classNames.Main - : this.classNames.MainInactive; - } - - protected override get inlineClassName(): TCssClassName { - return (this.active) - ? this.classNames.Inline - : this.classNames.InlineInactive; - } -} - -/** - * Type for a decorator with {@link ReactiveDecorationBase.changed changed} property set to `true`. - */ -export type TChangedDecorator = ReactiveDecorationBase & { readonly changed: true }; diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/utils/types.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/utils/types.ts deleted file mode 100644 index 1ca7e270d3f..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/utils/types.ts +++ /dev/null @@ -1,56 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { IModelDecorationsChangeAccessor } from '../../../../../../../../../editor/common/model.js'; - -/** - * CSS class names of a `reactive` decoration. - */ -export interface IReactiveDecorationClassNames { - /** - * Main, default CSS class name of the decoration. - */ - readonly Main: T; - - /** - * CSS class name of the decoration for the `inline`(text) styles. - */ - readonly Inline: T; - - /** - * main CSS class name of the decoration for the `inactive` - * decoration state. - */ - readonly MainInactive: T; - - /** - * CSS class name of the decoration for the `inline`(text) - * styles when decoration is in the `inactive` state. - */ - readonly InlineInactive: T; -} - -/** - * CSS styles for a decoration to be registered with editor. - */ -export type TDecorationStyles = { - readonly [key in TClassNames]: readonly string[]; -}; - -/** - * A model decorations accessor that can be used to `add` a decoration. - */ -export type TAddAccessor = Pick; - -/** - * A model decorations accessor that can be used to `change` a decoration. - */ -export type TChangeAccessor = Pick; - -/** - * A model decorations accessor that can be used to `remove` a decoration. - */ -export type TRemoveAccessor = Pick; - diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/promptDecorationsProvider.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/promptDecorationsProvider.ts deleted file mode 100644 index d7a9d9b235f..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/promptDecorationsProvider.ts +++ /dev/null @@ -1,205 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { IPromptsService } from '../../service/promptsService.js'; -import { ProviderInstanceBase } from '../providerInstanceBase.js'; -import { ITextModel } from '../../../../../../../editor/common/model.js'; -import { FrontMatterDecoration } from './decorations/frontMatterDecoration.js'; -import { toDisposable } from '../../../../../../../base/common/lifecycle.js'; -import { Position } from '../../../../../../../editor/common/core/position.js'; -import { BaseToken } from '../../codecs/base/baseToken.js'; -import { ProviderInstanceManagerBase, TProviderClass } from '../providerInstanceManagerBase.js'; -import { registerThemingParticipant } from '../../../../../../../platform/theme/common/themeService.js'; -import { FrontMatterHeader } from '../../codecs/base/markdownExtensionsCodec/tokens/frontMatterHeader.js'; -import { ReactiveDecorationBase, TChangedDecorator } from './decorations/utils/reactiveDecorationBase.js'; -import { DecorationBase, TDecorationClass } from './decorations/utils/decorationBase.js'; - -/** - * Prompt tokens that are decorated by this provider. - */ -type TDecoratedToken = FrontMatterHeader; - -/** - * List of all supported decorations. - */ -const SUPPORTED_DECORATIONS: readonly TDecorationClass[] = Object.freeze([ - FrontMatterDecoration, -]); - -/** - * Prompt syntax decorations provider for text models. - */ -export class PromptDecorator extends ProviderInstanceBase { - /** - * Currently active decorations. - */ - private readonly decorations: DecorationBase[] = []; - - constructor( - model: ITextModel, - @IPromptsService promptsService: IPromptsService, - ) { - super(model, promptsService); - - this.watchCursorPosition(); - } - - protected override async onPromptSettled( - _error?: Error, - ): Promise { - // by the time the promise above completes, either this object - // or the text model might be already has been disposed - if (this.isDisposed || this.model.isDisposed()) { - return; - } - - this.addDecorations(); - - return; - } - - /** - * Get the current cursor position inside an active editor. - * Note! Currently not implemented because the provider is disabled, and - * we need to do some refactoring to get accurate cursor position. - */ - private get cursorPosition(): Position | null { - if (this.model.isDisposed()) { - return null; - } - - return null; - } - - /** - * Watch editor cursor position and update reactive decorations accordingly. - */ - private watchCursorPosition(): this { - const interval = setInterval(() => { - const { cursorPosition } = this; - - const changedDecorations: TChangedDecorator[] = []; - for (const decoration of this.decorations) { - if ((decoration instanceof ReactiveDecorationBase) === false) { - continue; - } - - if (decoration.setCursorPosition(cursorPosition) === true) { - changedDecorations.push(decoration); - } - } - - if (changedDecorations.length === 0) { - return; - } - - this.changeModelDecorations(changedDecorations); - }, 25); - - this._register(toDisposable(() => { - clearInterval(interval); - })); - - return this; - } - - /** - * Update existing decorations. - */ - private changeModelDecorations( - decorations: readonly TChangedDecorator[], - ): this { - this.model.changeDecorations((accessor) => { - for (const decoration of decorations) { - decoration.change(accessor); - } - }); - - return this; - } - - /** - * Add decorations for all prompt tokens. - */ - private addDecorations(): this { - this.model.changeDecorations((accessor) => { - const { tokens } = this.parser; - - // remove all existing decorations - for (const decoration of this.decorations.splice(0)) { - decoration.remove(accessor); - } - - // then add new decorations based on the current tokens - for (const token of tokens) { - for (const Decoration of SUPPORTED_DECORATIONS) { - if (Decoration.handles(token) === false) { - continue; - } - - this.decorations.push( - new Decoration(accessor, token), - ); - break; - } - } - }); - - return this; - } - - /** - * Remove all existing decorations. - */ - private removeAllDecorations(): this { - if (this.decorations.length === 0) { - return this; - } - - this.model.changeDecorations((accessor) => { - for (const decoration of this.decorations.splice(0)) { - decoration.remove(accessor); - } - }); - - return this; - } - - public override dispose(): void { - if (this.isDisposed) { - return; - } - - this.removeAllDecorations(); - super.dispose(); - } - - /** - * Returns a string representation of this object. - */ - public override toString(): string { - return `text-model-prompt-decorator:${this.model.uri.path}`; - } -} - -/** - * Register CSS styles of the supported decorations. - */ -registerThemingParticipant((_theme, collector) => { - for (const Decoration of SUPPORTED_DECORATIONS) { - for (const [className, styles] of Object.entries(Decoration.cssStyles)) { - collector.addRule(`.monaco-editor ${className} { ${styles.join(' ')} }`); - } - } -}); - -/** - * Provider for prompt syntax decorators on text models. - */ -export class PromptDecorationsProviderInstanceManager extends ProviderInstanceManagerBase { - protected override get InstanceClass(): TProviderClass { - return PromptDecorator; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/types.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/types.ts deleted file mode 100644 index fb3ef81b122..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/types.ts +++ /dev/null @@ -1,48 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { IRange } from '../../../../../../../editor/common/core/range.js'; -import { ModelDecorationOptions } from '../../../../../../../editor/common/model/textModel.js'; - -/** - * Decoration object. - */ -export interface ITextModelDecoration { - /** - * Range of the decoration. - */ - range: IRange; - - /** - * Associated decoration options. - */ - options: ModelDecorationOptions; -} - -/** - * Decoration CSS class names. - */ -export enum DecorationClassNames { - /** - * CSS class name for `default` prompt syntax decoration. - */ - Default = 'prompt-decoration', - - /** - * CSS class name for `file reference` prompt syntax decoration. - */ - FileReference = DecorationClassNames.Default, -} - -/** - * Decoration CSS class modifiers. - */ -export enum CssClassModifiers { - /** - * CSS class modifier for `active` state of - * a `reactive` prompt syntax decoration. - */ - Inactive = '.prompt-decoration-inactive', -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/promptHeaderDiagnosticsProvider.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/promptHeaderDiagnosticsProvider.ts deleted file mode 100644 index b8bca008829..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/promptHeaderDiagnosticsProvider.ts +++ /dev/null @@ -1,234 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { IPromptsService } from '../service/promptsService.js'; -import { ProviderInstanceBase } from './providerInstanceBase.js'; -import { ITextModel } from '../../../../../../editor/common/model.js'; -import { assertNever } from '../../../../../../base/common/assert.js'; -import { CancellationToken } from '../../../../../../base/common/cancellation.js'; -import { ProviderInstanceManagerBase, TProviderClass } from './providerInstanceManagerBase.js'; -import { TDiagnostic, PromptMetadataError, PromptMetadataWarning } from '../parsers/promptHeader/diagnostics.js'; -import { IMarkerData, IMarkerService, MarkerSeverity } from '../../../../../../platform/markers/common/markers.js'; -import { PromptHeader } from '../parsers/promptHeader/promptHeader.js'; -import { PromptToolsMetadata } from '../parsers/promptHeader/metadata/tools.js'; -import { PromptModelMetadata } from '../parsers/promptHeader/metadata/model.js'; -import { ModeHeader } from '../parsers/promptHeader/modeHeader.js'; -import { ILanguageModelChatMetadata, ILanguageModelsService } from '../../languageModels.js'; -import { ILanguageModelToolsService } from '../../languageModelToolsService.js'; -import { localize } from '../../../../../../nls.js'; -import { ChatModeKind } from '../../constants.js'; -import { IChatMode, IChatModeService } from '../../chatModes.js'; -import { PromptModeMetadata } from '../parsers/promptHeader/metadata/mode.js'; -import { Iterable } from '../../../../../../base/common/iterator.js'; - -/** - * Unique ID of the markers provider class. - */ -const MARKERS_OWNER_ID = 'prompts-header-diagnostics-provider'; - -/** - * Prompt header diagnostics provider for an individual text model - * of a prompt file. - */ -class PromptHeaderDiagnosticsProvider extends ProviderInstanceBase { - constructor( - model: ITextModel, - @IPromptsService promptsService: IPromptsService, - @IMarkerService private readonly markerService: IMarkerService, - @ILanguageModelsService private readonly languageModelsService: ILanguageModelsService, - @ILanguageModelToolsService private readonly languageModelToolsService: ILanguageModelToolsService, - @IChatModeService private readonly chatModeService: IChatModeService, - ) { - super(model, promptsService); - this._register(languageModelsService.onDidChangeLanguageModels(() => { - this.onPromptSettled(undefined, CancellationToken.None); - })); - this._register(languageModelToolsService.onDidChangeTools(() => { - this.onPromptSettled(undefined, CancellationToken.None); - })); - this._register(chatModeService.onDidChangeChatModes(() => { - this.onPromptSettled(undefined, CancellationToken.None); - })); - } - - /** - * Update diagnostic markers for the current editor. - */ - protected override async onPromptSettled( - _error: Error | undefined, - token: CancellationToken, - ): Promise { - - const { header } = this.parser; - if (header === undefined) { - this.markerService.remove(MARKERS_OWNER_ID, [this.model.uri]); - return; - } - - // header parsing process is separate from the prompt parsing one, hence - // apply markers only after the header is settled and so has diagnostics - const completed = await header.settled; - if (!completed || token.isCancellationRequested) { - return; - } - - const markers: IMarkerData[] = []; - for (const diagnostic of header.diagnostics) { - markers.push(toMarker(diagnostic)); - } - - if (header instanceof PromptHeader) { - const mode = this.validateMode(header.metadataUtility.mode, markers); - this.validateTools(header.metadataUtility.tools, mode?.kind, markers); - this.validateModel(header.metadataUtility.model, mode?.kind, markers); - } else if (header instanceof ModeHeader) { - this.validateTools(header.metadataUtility.tools, ChatModeKind.Agent, markers); - this.validateModel(header.metadataUtility.model, ChatModeKind.Agent, markers); - - } - - if (markers.length === 0) { - this.markerService.remove(MARKERS_OWNER_ID, [this.model.uri]); - return; - } - - this.markerService.changeOne( - MARKERS_OWNER_ID, - this.model.uri, - markers, - ); - return; - } - validateModel(modelNode: PromptModelMetadata | undefined, modeKind: string | ChatModeKind | undefined, markers: IMarkerData[]) { - if (!modelNode || modelNode.value === undefined) { - return; - } - const languageModes = this.languageModelsService.getLanguageModelIds(); - if (languageModes.length === 0) { - // likely the service is not initialized yet - return; - } - const modelMetadata = this.findModelByName(languageModes, modelNode.value); - if (!modelMetadata) { - markers.push({ - message: localize('promptHeaderDiagnosticsProvider.modelNotFound', "Unknown model '{0}'", modelNode.value), - severity: MarkerSeverity.Warning, - ...modelNode.range, - }); - } else if (modeKind === ChatModeKind.Agent && !ILanguageModelChatMetadata.suitableForAgentMode(modelMetadata)) { - markers.push({ - message: localize('promptHeaderDiagnosticsProvider.modelNotSuited', "Model '{0}' is not suited for agent mode", modelNode.value), - severity: MarkerSeverity.Warning, - ...modelNode.range, - }); - } - - } - findModelByName(languageModes: string[], modelName: string): ILanguageModelChatMetadata | undefined { - for (const model of languageModes) { - const metadata = this.languageModelsService.lookupLanguageModel(model); - if (metadata && metadata.isUserSelectable !== false && ILanguageModelChatMetadata.matchesQualifiedName(modelName, metadata)) { - return metadata; - } - } - return undefined; - } - - validateTools(tools: PromptToolsMetadata | undefined, modeKind: string | ChatModeKind | undefined, markers: IMarkerData[]) { - if (!tools || tools.value === undefined || modeKind === ChatModeKind.Ask || modeKind === ChatModeKind.Edit) { - return; - } - const toolNames = new Set(tools.value); - if (toolNames.size === 0) { - return; - } - for (const tool of this.languageModelToolsService.getTools()) { - toolNames.delete(tool.toolReferenceName ?? tool.displayName); - } - for (const toolSet of this.languageModelToolsService.toolSets.get()) { - toolNames.delete(toolSet.referenceName); - } - - for (const toolName of toolNames) { - const range = tools.getToolRange(toolName); - if (range) { - markers.push({ - message: localize('promptHeaderDiagnosticsProvider.toolNotFound', "Unknown tool '{0}'", toolName), - severity: MarkerSeverity.Warning, - ...range, - }); - } - } - } - - validateMode(modeNode: PromptModeMetadata | undefined, markers: IMarkerData[]): IChatMode | undefined { - if (!modeNode || modeNode.value === undefined) { - return; - } - - const modeValue = modeNode.value; - const modes = this.chatModeService.getModes(); - const availableModes = []; - - // Check if mode exists in builtin or custom modes - for (const mode of Iterable.concat(modes.builtin, modes.custom)) { - if (mode.name === modeValue) { - return mode; - } - availableModes.push(mode.name); // collect all available mode names - } - - markers.push({ - message: localize('promptHeaderDiagnosticsProvider.modeNotFound', "Unknown mode '{0}'. Available modes: {1}", modeValue, availableModes.join(', ')), - severity: MarkerSeverity.Warning, - ...modeNode.range, - }); - return undefined; - - } - - /** - * Returns a string representation of this object. - */ - public override toString(): string { - return `prompt-header-diagnostics:${this.model.uri.path}`; - } -} - -/** - * Convert a provided diagnostic object into a marker data object. - */ -function toMarker(diagnostic: TDiagnostic): IMarkerData { - if (diagnostic instanceof PromptMetadataWarning) { - return { - message: diagnostic.message, - severity: MarkerSeverity.Warning, - ...diagnostic.range, - }; - } - - if (diagnostic instanceof PromptMetadataError) { - return { - message: diagnostic.message, - severity: MarkerSeverity.Error, - ...diagnostic.range, - }; - } - - assertNever( - diagnostic, - `Unknown prompt metadata diagnostic type '${diagnostic}'.`, - ); -} - -/** - * The class that manages creation and disposal of {@link PromptHeaderDiagnosticsProvider} - * classes for each specific editor text model. - */ -export class PromptHeaderDiagnosticsInstanceManager extends ProviderInstanceManagerBase { - protected override get InstanceClass(): TProviderClass { - return PromptHeaderDiagnosticsProvider; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/promptLinkDiagnosticsProvider.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/promptLinkDiagnosticsProvider.ts deleted file mode 100644 index 3809f67ca8f..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/promptLinkDiagnosticsProvider.ts +++ /dev/null @@ -1,98 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { IPromptsService } from '../service/promptsService.js'; -import { IPromptFileReference } from '../parsers/types.js'; -import { ProviderInstanceBase } from './providerInstanceBase.js'; -import { ITextModel } from '../../../../../../editor/common/model.js'; -import { assertDefined } from '../../../../../../base/common/types.js'; -import { ProviderInstanceManagerBase, TProviderClass } from './providerInstanceManagerBase.js'; -import { IMarkerData, IMarkerService, MarkerSeverity } from '../../../../../../platform/markers/common/markers.js'; -import { IFileService } from '../../../../../../platform/files/common/files.js'; -import { localize } from '../../../../../../nls.js'; - -/** - * Unique ID of the markers provider class. - */ -const MARKERS_OWNER_ID = 'prompt-link-diagnostics-provider'; - -/** - * Prompt links diagnostics provider for a single text model. - */ -class PromptLinkDiagnosticsProvider extends ProviderInstanceBase { - constructor( - model: ITextModel, - @IPromptsService promptsService: IPromptsService, - @IMarkerService private readonly markerService: IMarkerService, - @IFileService private readonly fileService: IFileService - ) { - super(model, promptsService); - } - - /** - * Update diagnostic markers for the current editor. - */ - protected override async onPromptSettled(): Promise { - // clean up all previously added markers - this.markerService.remove(MARKERS_OWNER_ID, [this.model.uri]); - - const markers: IMarkerData[] = []; - - const stats = await this.fileService.resolveAll(this.parser.references.map(ref => ({ resource: ref.uri }))); - for (let i = 0; i < stats.length; i++) { - if (!stats[i].success) { - markers.push(toMarker(this.parser.references[i], localize('fileNotFound', 'File not found.'))); - } - } - - this.markerService.changeOne( - MARKERS_OWNER_ID, - this.model.uri, - markers, - ); - } - - /** - * Returns a string representation of this object. - */ - public override toString(): string { - return `prompt-link-diagnostics:${this.model.uri.path}`; - } -} - -/** - * Convert a prompt link with an issue to a marker data. - * - * @throws - * - if there is no link issue (e.g., `topError` undefined) - * - if there is no link range to highlight (e.g., `linkRange` undefined) - * - if the original error is of `NotPromptFile` type - we don't want to - * show diagnostic markers for non-prompt file links in the prompts - */ -function toMarker(link: IPromptFileReference, message: string): IMarkerData { - const { linkRange } = link; - - assertDefined( - linkRange, - 'Link range must to be defined.', - ); - - - return { - message: message, - severity: MarkerSeverity.Warning, - ...linkRange, - }; -} - -/** - * The class that manages creation and disposal of {@link PromptLinkDiagnosticsProvider} - * classes for each specific editor text model. - */ -export class PromptLinkDiagnosticsInstanceManager extends ProviderInstanceManagerBase { - protected override get InstanceClass(): TProviderClass { - return PromptLinkDiagnosticsProvider; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/providerInstanceBase.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/providerInstanceBase.ts deleted file mode 100644 index da51052ab58..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/providerInstanceBase.ts +++ /dev/null @@ -1,54 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { IPromptsService, TSharedPrompt } from '../service/promptsService.js'; -import { ITextModel } from '../../../../../../editor/common/model.js'; -import { ObservableDisposable } from '../utils/observableDisposable.js'; -import { CancellationToken, CancellationTokenSource } from '../../../../../../base/common/cancellation.js'; - -/** - * Abstract base class for all reusable prompt file providers. - */ -export abstract class ProviderInstanceBase extends ObservableDisposable { - /** - * Function that is called when the prompt parser is settled. - */ - protected abstract onPromptSettled(error: Error | undefined, token: CancellationToken): Promise; - - /** - * Returns a string representation of this object. - */ - public abstract override toString(): string; - - /** - * The prompt parser instance. - */ - protected readonly parser: TSharedPrompt; - - constructor( - protected readonly model: ITextModel, - @IPromptsService promptsService: IPromptsService, - ) { - super(); - - this.parser = promptsService.getSyntaxParserFor(model); - - this._register( - this.parser.onDispose(this.dispose.bind(this)), - ); - - let cancellationSource = new CancellationTokenSource(); - this._register( - this.parser.onSettled((error) => { - cancellationSource.dispose(true); - cancellationSource = new CancellationTokenSource(); - - this.onPromptSettled(error, cancellationSource.token); - }), - ); - - this.parser.start(); - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/providerInstanceManagerBase.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/providerInstanceManagerBase.ts deleted file mode 100644 index 36ff2339b49..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/providerInstanceManagerBase.ts +++ /dev/null @@ -1,176 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { ProviderInstanceBase } from './providerInstanceBase.js'; -import { assert } from '../../../../../../base/common/assert.js'; -import { ITextModel } from '../../../../../../editor/common/model.js'; -import { assertDefined } from '../../../../../../base/common/types.js'; -import { Disposable } from '../../../../../../base/common/lifecycle.js'; -import { ObjectCache } from '../utils/objectCache.js'; -import { INSTRUCTIONS_LANGUAGE_ID, MODE_LANGUAGE_ID, PROMPT_LANGUAGE_ID } from '../promptTypes.js'; -import { IModelService } from '../../../../../../editor/common/services/model.js'; -import { PromptsConfig } from '../config/config.js'; -import { IEditorService } from '../../../../../services/editor/common/editorService.js'; -import { IDiffEditor, IEditor, IEditorModel } from '../../../../../../editor/common/editorCommon.js'; -import { IInstantiationService } from '../../../../../../platform/instantiation/common/instantiation.js'; -import { IConfigurationService } from '../../../../../../platform/configuration/common/configuration.js'; - -/** - * Type for a text editor that is used for reusable prompt files. - */ -export interface IPromptFileEditor extends IEditor { - readonly getModel: () => ITextModel; -} - -/** - * Type for a class that can create a new provider instance. - */ -export type TProviderClass = new (editor: ITextModel, ...args: any[]) => TInstance; - -/** - * A generic base class that manages creation and disposal of {@link TInstance} - * objects for each specific editor object that is used for reusable prompt files. - */ -export abstract class ProviderInstanceManagerBase extends Disposable { - /** - * Currently available {@link TInstance} instances. - */ - private readonly instances: ObjectCache; - - /** - * Class object of the managed {@link TInstance}. - */ - protected abstract get InstanceClass(): TProviderClass; - - constructor( - @IModelService modelService: IModelService, - @IEditorService editorService: IEditorService, - @IInstantiationService instantiationService: IInstantiationService, - @IConfigurationService configService: IConfigurationService, - ) { - super(); - - // cache of managed instances - this.instances = this._register( - new ObjectCache((model: ITextModel) => { - assert( - model.isDisposed() === false, - 'Text model must not be disposed.', - ); - - // sanity check - the new TS/JS discrepancies regarding fields initialization - // logic mean that this can be `undefined` during runtime while defined in TS - assertDefined( - this.InstanceClass, - 'Instance class field must be defined.', - ); - - const instance: TInstance = instantiationService.createInstance( - this.InstanceClass, - model, - ); - - // this is a sanity check and the contract of the object cache, - // we must return a non-disposed object from this factory function - instance.assertNotDisposed( - 'Created instance must not be disposed.', - ); - - return instance; - }), - ); - - // if the feature is disabled, do not create any providers - if (PromptsConfig.enabled(configService) === false) { - return; - } - - // subscribe to changes of the active editor - this._register(editorService.onDidActiveEditorChange(() => { - const { activeTextEditorControl } = editorService; - if (activeTextEditorControl === undefined) { - return; - } - - this.handleNewEditor(activeTextEditorControl); - })); - - // handle existing visible text editors - editorService - .visibleTextEditorControls - .forEach(this.handleNewEditor.bind(this)); - - // subscribe to "language change" events for all models - this._register( - modelService.onModelLanguageChanged((event) => { - const { model, oldLanguageId } = event; - - // if language is set to `prompt` or `instructions` language, handle that model - if (isPromptFileModel(model)) { - this.instances.get(model); - return; - } - - // if the language is changed away from `prompt` or `instructions`, - // remove and dispose provider for this model - if (isPromptFile(oldLanguageId)) { - this.instances.remove(model, true); - return; - } - }), - ); - } - - /** - * Initialize a new {@link TInstance} for the given editor. - */ - private handleNewEditor(editor: IEditor | IDiffEditor): this { - const model = editor.getModel(); - if (model === null) { - return this; - } - - if (isPromptFileModel(model) === false) { - return this; - } - - // note! calling `get` also creates a provider if it does not exist; - // and the provider is auto-removed when the editor is disposed - this.instances.get(model); - - return this; - } -} - -/** - * Check if provided language ID is one of the prompt file languages. - */ -function isPromptFile(languageId: string): boolean { - return [ - PROMPT_LANGUAGE_ID, - INSTRUCTIONS_LANGUAGE_ID, - MODE_LANGUAGE_ID, - ].includes(languageId); -} - -/** - * Check if a provided model is used for prompt files. - */ -function isPromptFileModel(model: IEditorModel): model is ITextModel { - // we support only `text editors` for now so filter out `diff` ones - if ('modified' in model || 'model' in model) { - return false; - } - - if (model.isDisposed()) { - return false; - } - - if (isPromptFile(model.getLanguageId()) === false) { - return false; - } - - return true; -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/basePromptParser.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/basePromptParser.ts deleted file mode 100644 index 9fe72885d03..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/basePromptParser.ts +++ /dev/null @@ -1,728 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { TopError } from './topError.js'; -import { ChatModeKind } from '../../constants.js'; -import { TMetadata } from './promptHeader/headerBase.js'; -import { ModeHeader } from './promptHeader/modeHeader.js'; -import { URI } from '../../../../../../base/common/uri.js'; -import { PromptToken } from '../codecs/tokens/promptToken.js'; -import * as path from '../../../../../../base/common/path.js'; -import { ChatPromptCodec } from '../codecs/chatPromptCodec.js'; -import { FileReference } from '../codecs/tokens/fileReference.js'; -import { ChatPromptDecoder } from '../codecs/chatPromptDecoder.js'; -import { assertDefined } from '../../../../../../base/common/types.js'; -import { Emitter } from '../../../../../../base/common/event.js'; -import { DeferredPromise } from '../../../../../../base/common/async.js'; -import { InstructionsHeader } from './promptHeader/instructionsHeader.js'; -import { ILogService } from '../../../../../../platform/log/common/log.js'; -import { PromptVariable, PromptVariableWithData } from '../codecs/tokens/promptVariable.js'; -import type { IPromptContentsProvider } from '../contentProviders/types.js'; -import type { TPromptReference, ITopError, TVariableReference } from './types.js'; -import { type IDisposable } from '../../../../../../base/common/lifecycle.js'; -import { assert, assertNever } from '../../../../../../base/common/assert.js'; -import { basename, dirname, joinPath } from '../../../../../../base/common/resources.js'; -import { BaseToken } from '../codecs/base/baseToken.js'; -import { VSBufferReadableStream } from '../../../../../../base/common/buffer.js'; -import { type IRange, Range } from '../../../../../../editor/common/core/range.js'; -import { PromptHeader, type TPromptMetadata } from './promptHeader/promptHeader.js'; -import { ObservableDisposable } from '../utils/observableDisposable.js'; -import { INSTRUCTIONS_LANGUAGE_ID, MODE_LANGUAGE_ID, PROMPT_LANGUAGE_ID } from '../promptTypes.js'; -import { LinesDecoder } from '../codecs/base/linesCodec/linesDecoder.js'; -import { IInstantiationService } from '../../../../../../platform/instantiation/common/instantiation.js'; -import { MarkdownLink } from '../codecs/base/markdownCodec/tokens/markdownLink.js'; -import { MarkdownToken } from '../codecs/base/markdownCodec/tokens/markdownToken.js'; -import { FrontMatterHeader } from '../codecs/base/markdownExtensionsCodec/tokens/frontMatterHeader.js'; -import { OpenFailed, NotPromptFile, RecursiveReference, FolderReference, ResolveError } from '../../promptFileReferenceErrors.js'; -import { type IPromptContentsProviderOptions } from '../contentProviders/promptContentsProviderBase.js'; -import { CancellationToken } from '../../../../../../base/common/cancellation.js'; -import { IWorkbenchEnvironmentService } from '../../../../../services/environment/common/environmentService.js'; -import { Schemas } from '../../../../../../base/common/network.js'; - -/** - * Options of the {@link BasePromptParser} class. - */ -export interface IBasePromptParserOptions { -} - -export type IPromptParserOptions = IBasePromptParserOptions & IPromptContentsProviderOptions; - - -/** - * Error conditions that may happen during the file reference resolution. - */ -export type TErrorCondition = OpenFailed | RecursiveReference | FolderReference | NotPromptFile; - -/** - * Base prompt parser class that provides a common interface for all - * prompt parsers that are responsible for parsing chat prompt syntax. - */ -export class BasePromptParser extends ObservableDisposable { - /** - * Options passed to the constructor. - */ - protected readonly options: IBasePromptParserOptions; - - /** - * List of all tokens that were parsed from the prompt contents so far. - */ - public get tokens(): readonly BaseToken[] { - return [...this.receivedTokens]; - } - /** - * Private field behind the readonly {@link tokens} property. - */ - private receivedTokens: BaseToken[] = []; - - /** - * List of file references in the prompt file. - */ - private readonly _references: TPromptReference[] = []; - - /** - * List of variable references in the prompt file. - */ - private readonly _variableReferences: TVariableReference[] = []; - - /** - * Reference to the prompt header object that holds metadata associated - * with the prompt. - */ - private promptHeader?: PromptHeader | InstructionsHeader | ModeHeader | undefined; - - /** - * Reference to the prompt header object that holds metadata associated - * with the prompt. - */ - public get header(): PromptHeader | InstructionsHeader | ModeHeader | undefined { - return this.promptHeader; - } - - /** - * Get contents of the prompt body. - */ - public async getBody(): Promise { - const startLineNumber = (this.header !== undefined) - ? this.header.range.endLineNumber + 1 - : 1; - - const decoder = new LinesDecoder( - await this.promptContentsProvider.contents, - ); - - const tokens = (await decoder.consumeAll()) - .filter(({ range }) => { - return (range.startLineNumber >= startLineNumber); - }); - - return BaseToken.render(tokens); - } - - /** - * Get the full contents of the prompt, including the header - */ - public async getFullContent(): Promise { - const decoder = new LinesDecoder( - await this.promptContentsProvider.contents, - ); - const tokens = await decoder.consumeAll(); - return BaseToken.render(tokens); - } - - /** - * The event is fired when lines or their content change. - */ - private readonly _onUpdate = this._register(new Emitter()); - /** - * Subscribe to the event that is fired the parser state or contents - * changes, including changes in the possible prompt child references. - */ - public readonly onUpdate = this._onUpdate.event; - - /** - * Event that is fired when the current prompt parser is settled. - */ - private readonly _onSettled = this._register(new Emitter()); - - /** - * Event that is fired when the current prompt parser is settled. - */ - public onSettled( - callback: (error?: Error) => void, - ): IDisposable { - const disposable = this._onSettled.event(callback); - const streamEnded = (this.stream?.ended && (this.stream.isDisposed === false)); - - // if already in the error state or stream has already ended, - // invoke the callback immediately but asynchronously - if (streamEnded || this.errorCondition) { - setTimeout(callback.bind(undefined, this.errorCondition)); - - return disposable; - } - - return disposable; - } - - /** - * If failed to parse prompt contents, this property has - * an error object that describes the failure reason. - */ - private _errorCondition?: ResolveError; - - /** - * If file reference resolution fails, this attribute will be set - * to an error instance that describes the error condition. - */ - public get errorCondition(): ResolveError | undefined { - return this._errorCondition; - } - - /** - * Whether file references resolution failed. - * Set to `undefined` if the `resolve` method hasn't been ever called yet. - */ - public get resolveFailed(): boolean | undefined { - if (!this.firstParseResult.gotFirstResult) { - return undefined; - } - - return !!this._errorCondition; - } - - /** - * The promise is resolved when at least one parse result (a stream or - * an error) has been received from the prompt contents provider. - */ - private readonly firstParseResult = new FirstParseResult(); - - /** - * Returned promise is resolved when the parser process is settled. - * The settled state means that the prompt parser stream exists and - * has ended, or an error condition has been set in case of failure. - * - * Furthermore, this function can be called multiple times and will - * block until the latest prompt contents parsing logic is settled - * (e.g., for every `onContentChanged` event of the prompt source). - */ - public async settled(): Promise { - assert( - this.started, - 'Cannot wait on the parser that did not start yet.', - ); - - await this.firstParseResult.promise; - - if (this.errorCondition) { - return false; - } - - // by the time when the `firstParseResult` promise is resolved, - // this object may have been already disposed, hence noop - if (this.isDisposed) { - return false; - } - - assertDefined( - this.stream, - 'No stream reference found.', - ); - - const completed = await this.stream.settled; - - // if prompt header exists, also wait for it to be settled - if (this.promptHeader) { - const headerCompleted = await this.promptHeader.settled; - if (!headerCompleted) { - return false; - } - } - - return completed; - } - - constructor( - private readonly promptContentsProvider: TContentsProvider, - options: IBasePromptParserOptions, - @IInstantiationService protected readonly instantiationService: IInstantiationService, - @IWorkbenchEnvironmentService private readonly envService: IWorkbenchEnvironmentService, - @ILogService protected readonly logService: ILogService, - ) { - super(); - - this.options = options; - - this._register( - this.promptContentsProvider.onContentChanged((streamOrError) => { - // process the received message - this.onContentsChanged(streamOrError); - - // indicate that we've received at least one `onContentChanged` event - this.firstParseResult.end(); - }), - ); - - // dispose self when contents provider is disposed - this._register( - this.promptContentsProvider.onDispose(this.dispose.bind(this)), - ); - } - - /** - * The latest received stream of prompt tokens, if any. - */ - private stream: ChatPromptDecoder | undefined; - - /** - * Handler the event event that is triggered when prompt contents change. - * - * @param streamOrError Either a binary stream of file contents, or an error object - * that was generated during the reference resolve attempt. - * @param seenReferences List of parent references that we've have already seen - * during the process of traversing the references tree. It's - * used to prevent the tree navigation to fall into an infinite - * references recursion. - */ - private onContentsChanged( - streamOrError: VSBufferReadableStream | ResolveError - ): void { - // dispose and cleanup the previously received stream - // object or an error condition, if any received yet - this.stream?.dispose(); - delete this.stream; - delete this._errorCondition; - this.receivedTokens = []; - - // cleanup current prompt header object - this.promptHeader?.dispose(); - delete this.promptHeader; - - // dispose all currently existing references - this.disposeReferences(); - - // if an error received, set up the error condition and stop - if (streamOrError instanceof ResolveError) { - this._errorCondition = streamOrError; - this._onUpdate.fire(); - - // when error received fire the 'onSettled' event immediately - this._onSettled.fire(streamOrError); - - return; - } - - // decode the byte stream to a stream of prompt tokens - this.stream = ChatPromptCodec.decode(streamOrError); - - /** - * !NOTE! The order of event subscriptions below is critical here because - * the `data` event is also starts the stream, hence changing - * the order of event subscriptions can lead to race conditions. - * See {@link ReadableStreamEvents} for more info. - */ - - // on error or stream end, dispose the stream and fire the update event - this.stream.on('error', this.onStreamEnd.bind(this, this.stream)); - this.stream.on('end', this.onStreamEnd.bind(this, this.stream)); - - // when some tokens received, process and store the references - this.stream.on('data', (token) => { - // store all markdown and prompt token references - if ((token instanceof MarkdownToken) || (token instanceof PromptToken)) { - this.receivedTokens.push(token); - } - - // if a prompt header token received, create a new prompt header instance - if (token instanceof FrontMatterHeader) { - return this.createHeader(token); - } - - // try to convert a prompt variable with data token into a file reference - if (token instanceof PromptVariableWithData) { - try { - if (token.name === 'file') { - this.handleLinkToken(FileReference.from(token)); - } - } catch (error) { - // the `FileReference.from` call might throw if the `PromptVariableWithData` token - // can not be converted into a valid `#file` reference, hence we ignore the error - } - } else if (token instanceof PromptVariable) { - this.handleVariableToken(token); - } - - // note! the `isURL` is a simple check and needs to be improved to truly - // handle only file references, ignoring broken URLs or references - if (token instanceof MarkdownLink && !token.isURL) { - this.handleLinkToken(token); - } - }); - - // calling `start` on a disposed stream throws, so we warn and return instead - if (this.stream.isDisposed) { - this.logService.warn( - `[prompt parser][${basename(this.uri)}] cannot start stream that has been already disposed, aborting`, - ); - - return; - } - - // start receiving data on the stream - this.stream.start(); - } - - /** - * Create header object base on the target prompt file language ID. - * The language ID is important here, because it defines what type - * of metadata is valid for a prompt file and what type of related - * diagnostics we would show to the user. - */ - private createHeader(headerToken: FrontMatterHeader): void { - const { languageId } = this.promptContentsProvider; - - if (languageId === PROMPT_LANGUAGE_ID) { - this.promptHeader = new PromptHeader(headerToken, languageId); - } - - if (languageId === INSTRUCTIONS_LANGUAGE_ID) { - this.promptHeader = new InstructionsHeader(headerToken, languageId); - } - - if (languageId === MODE_LANGUAGE_ID) { - this.promptHeader = new ModeHeader(headerToken, languageId); - } - - this.promptHeader?.start(); - } - - /** - * Handle a new reference token inside prompt contents. - */ - private handleLinkToken(token: FileReference | MarkdownLink): this { - - let referenceUri: URI; - if (path.isAbsolute(token.path)) { - referenceUri = URI.file(token.path); - if (this.envService.remoteAuthority) { - referenceUri = referenceUri.with({ - scheme: Schemas.vscodeRemote, - authority: this.envService.remoteAuthority, - }); - } - } else { - referenceUri = joinPath(dirname(this.uri), token.path); - } - this._references.push(new PromptReference(referenceUri, token)); - - this._onUpdate.fire(); - - return this; - } - - private handleVariableToken(token: PromptVariable): this { - - this._variableReferences.push({ name: token.name, range: token.range }); - - this._onUpdate.fire(); - - return this; - } - - /** - * Handle the `stream` end event. - * - * @param stream The stream that has ended. - * @param error Optional error object if stream ended with an error. - */ - private onStreamEnd( - stream: ChatPromptDecoder, - error?: Error, - ): this { - // decoders can fire the 'end' event also when they are get disposed, - // but because we dispose them when a new stream is received, we can - // safely ignore the event in this case - if (stream.isDisposed === true) { - return this; - } - - if (error) { - this.logService.warn( - `[prompt parser][${basename(this.uri)}] received an error on the chat prompt decoder stream: ${error}`, - ); - } - - this._onUpdate.fire(); - this._onSettled.fire(error); - - return this; - } - - - private disposeReferences(): void { - - - this._references.length = 0; - this._variableReferences.length = 0; - } - - /** - * Private attribute to track if the {@link start} - * method has been already called at least once. - */ - private started: boolean = false; - - /** - * Start the prompt parser. - */ - public start(token?: CancellationToken): this { - // if already started, nothing to do - if (this.started) { - return this; - } - this.started = true; - - - // if already in the error state that could be set - // in the constructor, then nothing to do - if (this.errorCondition) { - return this; - } - - this.promptContentsProvider.start(token); - return this; - } - - /** - * Associated URI of the prompt. - */ - public get uri(): URI { - return this.promptContentsProvider.uri; - } - - /** - * Get a list of immediate child references of the prompt. - */ - public get references(): readonly TPromptReference[] { - return [...this._references]; - } - - /** - * Get a list of variable references of the prompt. - */ - public get variableReferences(): readonly TVariableReference[] { - return [...this._variableReferences]; - } - - /** - * Valid metadata records defined in the prompt header. - */ - public get metadata(): TMetadata | null { - const { promptType } = this.promptContentsProvider; - if (promptType === 'non-prompt') { - return null; - } - - if (this.header === undefined) { - return { promptType }; - } - - if (this.header instanceof InstructionsHeader || this.header instanceof ModeHeader) { - return { promptType, ...this.header.metadata }; - } - - const { tools, mode, description, model } = this.header.metadata; - - const result: Partial = {}; - - if (description !== undefined) { - result.description = description; - } - - if (tools !== undefined && mode !== ChatModeKind.Ask && mode !== ChatModeKind.Edit) { - result.tools = tools; - // Preserve custom mode if specified, otherwise default to Agent - result.mode = mode || ChatModeKind.Agent; - } else if (mode !== undefined) { - result.mode = mode; - } - - if (model !== undefined) { - result.model = model; - } - - return { promptType, ...result }; - } - - /** - * The top most error of the current reference or any of its - * possible child reference errors. - */ - public get topError(): ITopError | undefined { - if (this.errorCondition) { - return new TopError({ - errorSubject: 'root', - errorsCount: 1, - originalError: this.errorCondition, - }); - } - - return undefined; - } - - /** - * Returns a string representation of this object. - */ - public override toString(): string { - return `prompt:${this.uri.path}`; - } - - /** - * @inheritdoc - */ - public override dispose(): void { - if (this.isDisposed) { - return; - } - - this.disposeReferences(); - - this.stream?.dispose(); - delete this.stream; - - this.promptHeader?.dispose(); - delete this.promptHeader; - - super.dispose(); - } -} - -/** - * Prompt reference object represents any reference inside prompt text - * contents. For instance the file variable(`#file:/path/to/file.md`) or - * a markdown link(`[#file:file.md](/path/to/file.md)`). - */ -export class PromptReference implements TPromptReference { - - - constructor( - public readonly uri: URI, - public readonly token: FileReference | MarkdownLink, - ) { - } - - /** - * Get the range of the `link` part of the reference. - */ - public get linkRange(): IRange | undefined { - // `#file:` references - if (this.token instanceof FileReference) { - return this.token.dataRange; - } - - // `markdown link` references - if (this.token instanceof MarkdownLink) { - return this.token.linkRange; - } - - return undefined; - } - - /** - * Type of the reference, - either a prompt `#file` variable, - * or a `markdown link` reference (`[caption](/path/to/file.md)`). - */ - public get type(): 'file' { - if (this.token instanceof FileReference) { - return 'file'; - } - - if (this.token instanceof MarkdownLink) { - return 'file'; - } - - assertNever( - this.token, - `Unknown token type '${this.token}'.`, - ); - } - - /** - * Subtype of the reference, - either a prompt `#file` variable, - * or a `markdown link` reference (`[caption](/path/to/file.md)`). - */ - public get subtype(): 'prompt' | 'markdown' { - if (this.token instanceof FileReference) { - return 'prompt'; - } - - if (this.token instanceof MarkdownLink) { - return 'markdown'; - } - - assertNever( - this.token, - `Unknown token type '${this.token}'.`, - ); - } - - public get range(): Range { - return this.token.range; - } - - public get path(): string { - return this.token.path; - } - - public get text(): string { - return this.token.text; - } - - /** - * Returns a string representation of this object. - */ - public toString(): string { - return `prompt-reference/${this.type}:${this.subtype}/${this.token}`; - } -} - -/** - * A tiny utility object that helps us to track existence - * of at least one parse result from the content provider. - */ -class FirstParseResult extends DeferredPromise { - /** - * Private attribute to track if we have - * received at least one result. - */ - private _gotResult = false; - - /** - * Whether we've received at least one result. - */ - public get gotFirstResult(): boolean { - return this._gotResult; - } - - /** - * Get underlying promise reference. - */ - public get promise(): Promise { - return this.p; - } - - /** - * Complete the underlying promise. - */ - public end(): void { - this._gotResult = true; - super.complete(void 0) - .catch(() => { - // the complete method is never fails - // so we can ignore the error here - }); - - return; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/filePromptParser.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/filePromptParser.ts deleted file mode 100644 index c73efe10884..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/filePromptParser.ts +++ /dev/null @@ -1,37 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { URI } from '../../../../../../base/common/uri.js'; -import { ILogService } from '../../../../../../platform/log/common/log.js'; -import { BasePromptParser, IPromptParserOptions } from './basePromptParser.js'; -import { FilePromptContentProvider } from '../contentProviders/filePromptContentsProvider.js'; -import { IInstantiationService } from '../../../../../../platform/instantiation/common/instantiation.js'; -import { IWorkbenchEnvironmentService } from '../../../../../services/environment/common/environmentService.js'; - -/** - * Class capable of parsing prompt syntax out of a provided file, - * including all the nested child file references it may have. - */ -export class FilePromptParser extends BasePromptParser { - constructor( - uri: URI, - options: IPromptParserOptions, - @IInstantiationService instantiationService: IInstantiationService, - @IWorkbenchEnvironmentService envService: IWorkbenchEnvironmentService, - @ILogService logService: ILogService, - ) { - const contentsProvider = instantiationService.createInstance(FilePromptContentProvider, uri, options); - super(contentsProvider, options, instantiationService, envService, logService); - - this._register(contentsProvider); - } - - /** - * Returns a string representation of this object. - */ - public override toString(): string { - return `file-prompt:${this.uri.path}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/diagnostics.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/diagnostics.ts deleted file mode 100644 index 754293f6dcf..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/diagnostics.ts +++ /dev/null @@ -1,47 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { Range } from '../../../../../../../editor/common/core/range.js'; - -/** - * List of all currently supported diagnostic types. - */ -export type TDiagnostic = PromptMetadataWarning | PromptMetadataError; - -/** - * Diagnostics object that hold information about some issue - * related to the prompt header metadata. - */ -export abstract class PromptMetadataDiagnostic { - constructor( - public readonly range: Range, - public readonly message: string, - ) { } - - /** - * String representation of the diagnostic object. - */ - public abstract toString(): string; -} - -/** - * Diagnostics object that hold information about some - * non-fatal issue related to the prompt header metadata. - */ -export class PromptMetadataWarning extends PromptMetadataDiagnostic { - public override toString(): string { - return `warning(${this.message})${this.range}`; - } -} - -/** - * Diagnostics object that hold information about some - * fatal issue related to the prompt header metadata. - */ -export class PromptMetadataError extends PromptMetadataDiagnostic { - public override toString(): string { - return `error(${this.message})${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/headerBase.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/headerBase.ts deleted file mode 100644 index eedebeabeed..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/headerBase.ts +++ /dev/null @@ -1,264 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { type TModeMetadata } from './modeHeader.js'; -import { localize } from '../../../../../../../nls.js'; -import { type TPromptMetadata } from './promptHeader.js'; -import { type IMetadataRecord } from './metadata/base/record.js'; -import { type TInstructionsMetadata } from './instructionsHeader.js'; -import { Range } from '../../../../../../../editor/common/core/range.js'; -import { Disposable } from '../../../../../../../base/common/lifecycle.js'; -import { ObjectStream } from '../../codecs/base/utils/objectStream.js'; -import { PromptMetadataError, PromptMetadataWarning, type TDiagnostic } from './diagnostics.js'; -import { SimpleToken } from '../../codecs/base/simpleCodec/tokens/tokens.js'; -import { FrontMatterRecord } from '../../codecs/base/frontMatterCodec/tokens/index.js'; -import { FrontMatterHeader } from '../../codecs/base/markdownExtensionsCodec/tokens/frontMatterHeader.js'; -import { FrontMatterDecoder, type TFrontMatterToken } from '../../codecs/base/frontMatterCodec/frontMatterDecoder.js'; -import { PromptDescriptionMetadata } from './metadata/description.js'; - -/** - * A metadata utility class "dehydrated" into a plain data object with - * semi-primitive record values (string, boolean, string[], boolean[], etc.). - */ -export type TDehydrated = { - [K in keyof T]: T[K] extends IMetadataRecord ? (U extends undefined ? undefined : NonNullable) : undefined; -}; - -/** - * Metadata defined in the prompt header. - */ -export interface IHeaderMetadata { - /** - * Description metadata in the prompt header. - */ - description: PromptDescriptionMetadata; -} - -/** - * Metadata for prompt/instruction/mode files. - */ -export type THeaderMetadata = Partial>; - -/** - * Metadata defined in the header of prompt/instruction/mode files. - */ -export type TMetadata = TPromptMetadata | TModeMetadata | TInstructionsMetadata; - -/** - * Base class for prompt/instruction/mode headers. - */ -export abstract class HeaderBase< - TMetadata extends IHeaderMetadata, -> extends Disposable { - /** - * Underlying decoder for a Front Matter header. - */ - private readonly stream: FrontMatterDecoder; - - /** - * Metadata records. - */ - protected readonly meta: Partial; - - /** - * Data object with all header's metadata records. - */ - public get metadata(): Partial> { - const result: Partial> = {}; - - for (const [entryName, entryValue] of Object.entries(this.meta)) { - if (entryValue?.value === undefined) { - continue; - } - - // note! we have to resort to `Object.assign()` here because - // the `Object.entries()` call looses type information - Object.assign(result, { - [entryName]: entryValue.value, - }); - } - - return result; - } - - /** - * A copy of metadata object with utility classes as values - * for each of prompt header's record. - * - * Please use {@link metadata} instead if all you need to read is - * the plain "data" object representation of valid metadata records. - */ - public get metadataUtility(): Partial { - return { ...this.meta }; - } - - /** - * List of all unique metadata record names. - */ - private readonly recordNames: Set; - - /** - * List of all issues found while parsing the prompt header. - */ - protected readonly issues: TDiagnostic[]; - - /** - * List of all diagnostic issues found while parsing - * the prompt header. - */ - public get diagnostics(): readonly TDiagnostic[] { - return this.issues; - } - - /** - * Full range of the header in the original document. - */ - public get range(): Range { - return this.token.range; - } - - constructor( - public readonly token: FrontMatterHeader, - public readonly languageId: string, - ) { - super(); - - this.issues = []; - this.meta = {}; - this.recordNames = new Set(); - - this.stream = this._register( - new FrontMatterDecoder( - ObjectStream.fromArray([...token.contentToken.children]), - ), - ); - this.stream.onData(this.onData.bind(this)); - this.stream.onError(this.onError.bind(this)); - } - - /** - * Process a front matter record token, which includes: - * - validation of the record and whether it is compatible with other header records - * - adding validation-related diagnostic messages to the {@link issues} list - * - setting associated utility class for the record on the {@link meta} object - * - * @returns a boolean flag that indicates whether the token was handled and therefore - * should not be processed any further. - */ - protected abstract handleToken( - token: FrontMatterRecord, - ): boolean; - - /** - * Process front matter tokens, converting them into - * well-known prompt metadata records. - */ - private onData(token: TFrontMatterToken): void { - // we currently expect only front matter 'records' for - // the prompt metadata, hence add diagnostics for all - // other tokens and ignore them - if ((token instanceof FrontMatterRecord) === false) { - // unless its a simple token, in which case we just ignore it - if (token instanceof SimpleToken) { - return; - } - - this.issues.push( - new PromptMetadataError( - token.range, - localize( - 'prompt.header.diagnostics.unexpected-token', - "Unexpected token '{0}'.", - token.text, - ), - ), - ); - - return; - } - - const recordName = token.nameToken.text; - - // if we already have a record with this name, - // add a warning diagnostic and ignore it - if (this.recordNames.has(recordName)) { - this.issues.push( - new PromptMetadataWarning( - token.range, - localize( - 'prompt.header.metadata.diagnostics.duplicate-record', - "Duplicate property '{0}' will be ignored.", - recordName, - ), - ), - ); - - return; - } - this.recordNames.add(recordName); - - // if the record might be a "description" metadata - // add it to the list of parsed metadata records - if (PromptDescriptionMetadata.isDescriptionRecord(token)) { - const metadata = new PromptDescriptionMetadata(token, this.languageId); - - this.issues.push(...metadata.validate()); - this.meta.description = metadata; - this.recordNames.add(recordName); - return; - } - - // pipe the token to the actual implementation class - // that might to handle it based on the token type - if (this.handleToken(token)) { - return; - } - - // all other records are "unknown" ones - this.issues.push( - new PromptMetadataWarning( - token.range, - localize( - 'prompt.header.metadata.diagnostics.unknown-record', - "Unknown property '{0}' will be ignored.", - recordName, - ), - ), - ); - } - - /** - * Process errors from the underlying front matter decoder. - */ - private onError(error: Error): void { - this.issues.push( - new PromptMetadataError( - this.token.range, - localize( - 'prompt.header.diagnostics.parsing-error', - "Failed to parse prompt header: {0}", - error.message, - ), - ), - ); - } - - /** - * Promise that resolves when parsing process of - * the prompt header completes. - */ - public get settled(): Promise { - return this.stream.settled; - } - - /** - * Starts the parsing process of the prompt header. - */ - public start(): this { - this.stream.start(); - - return this; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/instructionsHeader.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/instructionsHeader.ts deleted file mode 100644 index 1f13901f960..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/instructionsHeader.ts +++ /dev/null @@ -1,44 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { PromptApplyToMetadata } from './metadata/applyTo.js'; -import { HeaderBase, IHeaderMetadata, type TDehydrated } from './headerBase.js'; -import { PromptsType } from '../../promptTypes.js'; -import { FrontMatterRecord } from '../../codecs/base/frontMatterCodec/tokens/index.js'; - -/** - * Metadata utility object for instruction files. - */ -interface IInstructionsMetadata extends IHeaderMetadata { - /** - * Chat 'applyTo' metadata in the prompt header. - */ - applyTo: PromptApplyToMetadata; -} - -/** - * Metadata for instruction files. - */ -export type TInstructionsMetadata = Partial> & { promptType: PromptsType.instructions }; - -/** - * Header object for instruction files. - */ -export class InstructionsHeader extends HeaderBase { - protected override handleToken(token: FrontMatterRecord): boolean { - // if the record might be a "applyTo" metadata - // add it to the list of parsed metadata records - if (PromptApplyToMetadata.isApplyToRecord(token)) { - const metadata = new PromptApplyToMetadata(token, this.languageId); - - this.issues.push(...metadata.validate()); - this.meta.applyTo = metadata; - - return true; - } - - return false; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/applyTo.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/applyTo.ts deleted file mode 100644 index 7f374c0f108..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/applyTo.ts +++ /dev/null @@ -1,122 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { PromptStringMetadata } from './base/string.js'; -import { localize } from '../../../../../../../../nls.js'; -import { INSTRUCTIONS_LANGUAGE_ID } from '../../../promptTypes.js'; -import { isEmptyPattern, parse, splitGlobAware } from '../../../../../../../../base/common/glob.js'; -import { PromptMetadataDiagnostic, PromptMetadataError, PromptMetadataWarning } from '../diagnostics.js'; -import { FrontMatterRecord, FrontMatterToken } from '../../../codecs/base/frontMatterCodec/tokens/index.js'; - -/** - * Name of the metadata record in the prompt header. - */ -const RECORD_NAME = 'applyTo'; - -/** - * Prompt `applyTo` metadata record inside the prompt header. - */ -export class PromptApplyToMetadata extends PromptStringMetadata { - constructor( - recordToken: FrontMatterRecord, - languageId: string, - ) { - super(RECORD_NAME, recordToken, languageId); - } - - public override get recordName(): string { - return RECORD_NAME; - } - - public override validate(): readonly PromptMetadataDiagnostic[] { - super.validate(); - - // if we don't have a value token, validation must - // has failed already so nothing to do more - if (this.valueToken === undefined) { - return this.issues; - } - - // the applyTo metadata makes sense only for 'instruction' prompts - if (this.languageId !== INSTRUCTIONS_LANGUAGE_ID) { - this.issues.push( - new PromptMetadataError( - this.range, - localize( - 'prompt.header.metadata.string.diagnostics.invalid-language', - "The '{0}' header property is only valid in instruction files.", - this.recordName, - ), - ), - ); - - delete this.valueToken; - return this.issues; - } - - const { cleanText } = this.valueToken; - - // warn user if specified glob pattern is not valid - if (this.isValidGlob(cleanText) === false) { - this.issues.push( - new PromptMetadataWarning( - this.valueToken.range, - localize( - 'prompt.header.metadata.applyTo.diagnostics.non-valid-glob', - "Invalid glob pattern '{0}'.", - cleanText, - ), - ), - ); - - delete this.valueToken; - return this.issues; - } - - return this.issues; - } - - /** - * Check if a provided string contains a valid glob pattern. - */ - private isValidGlob( - pattern: string, - ): boolean { - try { - const patterns = splitGlobAware(pattern, ','); - if (patterns.length === 0) { - return false; - } - for (const pattern of patterns) { - - const globPattern = parse(pattern); - if (isEmptyPattern(globPattern)) { - return false; - } - } - return true; - } catch (_error) { - return false; - } - } - - /** - * Check if a provided front matter token is a metadata record - * with name equal to `applyTo`. - */ - public static isApplyToRecord( - token: FrontMatterToken, - ): boolean { - if ((token instanceof FrontMatterRecord) === false) { - return false; - } - - if (token.nameToken.text === RECORD_NAME) { - return true; - } - - return false; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/base/enum.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/base/enum.ts deleted file mode 100644 index 71b7d9e7a4f..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/base/enum.ts +++ /dev/null @@ -1,84 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { PromptStringMetadata } from './string.js'; -import { localize } from '../../../../../../../../../nls.js'; -import { assert } from '../../../../../../../../../base/common/assert.js'; -import { isOneOf } from '../../../../../../../../../base/common/types.js'; -import { PromptMetadataDiagnostic, PromptMetadataError } from '../../diagnostics.js'; -import { FrontMatterSequence } from '../../../../codecs/base/frontMatterCodec/tokens/frontMatterSequence.js'; -import { FrontMatterRecord, FrontMatterString } from '../../../../codecs/base/frontMatterCodec/tokens/index.js'; - -/** - * Enum type is the special case of the {@link PromptStringMetadata string} - * type that can take only a well-defined set of {@link validValues}. - */ -export abstract class PromptEnumMetadata< - TValidValues extends string = string, -> extends PromptStringMetadata { - constructor( - private readonly validValues: readonly TValidValues[], - expectedRecordName: string, - recordToken: FrontMatterRecord, - languageId: string, - ) { - super(expectedRecordName, recordToken, languageId); - } - - /** - * Valid enum value or 'undefined'. - */ - private enumValue: TValidValues | undefined; - /** - * Valid enum value or 'undefined'. - */ - public override get value(): TValidValues | undefined { - return this.enumValue; - } - - /** - * Validate the metadata record has an allowed value. - */ - public override validate(): readonly PromptMetadataDiagnostic[] { - super.validate(); - - if (this.valueToken === undefined) { - return this.issues; - } - - // sanity check for our expectations about the validate call - assert( - this.valueToken instanceof FrontMatterString - || this.valueToken instanceof FrontMatterSequence, - `Record token must be 'string', got '${this.valueToken}'.`, - ); - - const { cleanText } = this.valueToken; - if (isOneOf(cleanText, this.validValues)) { - this.enumValue = cleanText; - - return this.issues; - } - - this.issues.push( - new PromptMetadataError( - this.valueToken.range, - localize( - 'prompt.header.metadata.enum.diagnostics.invalid-value', - "The property '{0}' must be one of {1}, got '{2}'.", - this.recordName, - this.validValues - .map((value) => { - return `'${value}'`; - }).join(' | '), - cleanText, - ), - ), - ); - - delete this.valueToken; - return this.issues; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/base/record.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/base/record.ts deleted file mode 100644 index aa7a4660f66..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/base/record.ts +++ /dev/null @@ -1,108 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { assert } from '../../../../../../../../../base/common/assert.js'; -import { Range } from '../../../../../../../../../editor/common/core/range.js'; -import { PromptMetadataDiagnostic, PromptMetadataError, PromptMetadataWarning } from '../../diagnostics.js'; -import { FrontMatterRecord } from '../../../../codecs/base/frontMatterCodec/tokens/index.js'; - -/** - * Supported primitive types for metadata values in a prompt header. - */ -type TMetadataPrimitive = string | boolean; - -/** - * Supported metadata values in a prompt header. - */ -type TMetadataValue = TMetadataPrimitive | TMetadataPrimitive[]; - -/** - * Interface for a generic metadata record in the prompt header. - */ -export interface IMetadataRecord { - /** - * Value of a metadata record. If the value is not defined, it usually - * means that a record is present but its value is not set or valid. - */ - readonly value: T | undefined; -} - -/** - * Abstract class for all metadata records in the prompt header. - */ -export abstract class PromptMetadataRecord implements IMetadataRecord { - /** - * Private field for tracking all diagnostic issues - * related to this metadata record. - */ - protected readonly issues: PromptMetadataDiagnostic[]; - - /** - * Full range of the metadata's record text in the prompt header. - */ - public get range(): Range { - return this.recordToken.range; - } - - constructor( - protected readonly expectedRecordName: string, - protected readonly recordToken: FrontMatterRecord, - protected readonly languageId: string, - ) { - // validate that the record name has the expected name - const recordName = recordToken.nameToken.text; - assert( - recordName === expectedRecordName, - `Record name must be '${expectedRecordName}', got '${recordName}'.`, - ); - - this.issues = []; - } - - /** - * Name of the metadata record. - */ - public get recordName(): string { - return this.recordToken.nameToken.text; - } - - /** - * Validate the metadata record and collect all issues - * related to its content. - */ - public abstract validate(): readonly PromptMetadataDiagnostic[]; - - /** - * List of all diagnostic issues related to this metadata record. - */ - public get diagnostics(): readonly PromptMetadataDiagnostic[] { - return this.issues; - } - - /** - * Get the value of the metadata record. - */ - public abstract get value(): TValue | undefined; - - /** - * List of all `error` issue diagnostics. - */ - public get errorDiagnostics(): readonly PromptMetadataError[] { - return this.diagnostics - .filter((diagnostic) => { - return (diagnostic instanceof PromptMetadataError); - }); - } - - /** - * List of all `warning` issue diagnostics. - */ - public get warningDiagnostics(): readonly PromptMetadataWarning[] { - return this.diagnostics - .filter((diagnostic) => { - return (diagnostic instanceof PromptMetadataWarning); - }); - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/base/string.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/base/string.ts deleted file mode 100644 index 87423ab25b2..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/base/string.ts +++ /dev/null @@ -1,73 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { PromptMetadataRecord } from './record.js'; -import { localize } from '../../../../../../../../../nls.js'; -import { PromptMetadataDiagnostic, PromptMetadataError } from '../../diagnostics.js'; -import { FrontMatterSequence } from '../../../../codecs/base/frontMatterCodec/tokens/frontMatterSequence.js'; -import { FrontMatterRecord, FrontMatterString } from '../../../../codecs/base/frontMatterCodec/tokens/index.js'; -import { Range } from '../../../../../../../../../editor/common/core/range.js'; - - -/** - * Base class for all metadata records with a `string` value. - */ -export abstract class PromptStringMetadata extends PromptMetadataRecord { - /** - * Value token reference of the record. - */ - protected valueToken: FrontMatterString | FrontMatterSequence | undefined; - - /** - * String value of a metadata record. - */ - public override get value(): string | undefined { - return this.valueToken?.cleanText; - } - - public get valueRange(): Range | undefined { - return this.valueToken?.range; - } - - constructor( - expectedRecordName: string, - recordToken: FrontMatterRecord, - languageId: string, - ) { - super(expectedRecordName, recordToken, languageId); - } - - /** - * Validate the metadata record has a 'string' value. - */ - public override validate(): readonly PromptMetadataDiagnostic[] { - const { valueToken } = this.recordToken; - - // validate that the record value is a string or a generic sequence - // of tokens that can be interpreted as a string without quotes - const isString = (valueToken instanceof FrontMatterString); - const isSequence = (valueToken instanceof FrontMatterSequence); - if (isString || isSequence) { - this.valueToken = valueToken; - return this.issues; - } - - this.issues.push( - new PromptMetadataError( - valueToken.range, - localize( - 'prompt.header.metadata.string.diagnostics.invalid-value-type', - "The property '{0}' must be of type '{1}', got '{2}'.", - this.recordName, - 'string', - valueToken.valueTypeName.toString(), - ), - ), - ); - - delete this.valueToken; - return this.issues; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/description.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/description.ts deleted file mode 100644 index aaa8fa3a640..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/description.ts +++ /dev/null @@ -1,46 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { PromptStringMetadata } from './base/string.js'; -import { FrontMatterRecord, FrontMatterToken } from '../../../codecs/base/frontMatterCodec/tokens/index.js'; - -/** - * Name of the metadata record in the prompt header. - */ -const RECORD_NAME = 'description'; - -/** - * Prompt `description` metadata record inside the prompt header. - */ -export class PromptDescriptionMetadata extends PromptStringMetadata { - public override get recordName(): string { - return RECORD_NAME; - } - - constructor( - recordToken: FrontMatterRecord, - languageId: string, - ) { - super(RECORD_NAME, recordToken, languageId); - } - - /** - * Check if a provided front matter token is a metadata record - * with name equal to `description`. - */ - public static isDescriptionRecord( - token: FrontMatterToken, - ): boolean { - if ((token instanceof FrontMatterRecord) === false) { - return false; - } - - if (token.nameToken.text === RECORD_NAME) { - return true; - } - - return false; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/mode.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/mode.ts deleted file mode 100644 index b12728f7e75..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/mode.ts +++ /dev/null @@ -1,47 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { PromptStringMetadata } from './base/string.js'; -import { FrontMatterRecord, FrontMatterToken } from '../../../codecs/base/frontMatterCodec/tokens/index.js'; - -/** - * Name of the metadata record in the prompt header. - */ -const RECORD_NAME = 'mode'; - -/** - * Prompt `mode` metadata record inside the prompt header. - * Now supports both built-in modes (ask, edit, agent) and custom mode IDs. - */ -export class PromptModeMetadata extends PromptStringMetadata { - constructor( - recordToken: FrontMatterRecord, - languageId: string, - ) { - super( - RECORD_NAME, - recordToken, - languageId, - ); - } - - /** - * Check if a provided front matter token is a metadata record - * with name equal to `mode`. - */ - public static isModeRecord( - token: FrontMatterToken, - ): boolean { - if ((token instanceof FrontMatterRecord) === false) { - return false; - } - - if (token.nameToken.text === RECORD_NAME) { - return true; - } - - return false; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/model.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/model.ts deleted file mode 100644 index aa559e57fc0..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/model.ts +++ /dev/null @@ -1,41 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { FrontMatterRecord, FrontMatterToken } from '../../../codecs/base/frontMatterCodec/tokens/index.js'; -import { PromptStringMetadata } from './base/string.js'; - -/** - * Name of the metadata record in the prompt header. - */ -const RECORD_NAME = 'model'; - -export class PromptModelMetadata extends PromptStringMetadata { - public override get recordName(): string { - return RECORD_NAME; - } - - constructor( - recordToken: FrontMatterRecord, - languageId: string, - ) { - super(RECORD_NAME, recordToken, languageId); - } - - /** - * Check if a provided front matter token is a metadata record - * with name equal to `description`. - */ - public static isModelRecord(token: FrontMatterToken): boolean { - if ((token instanceof FrontMatterRecord) === false) { - return false; - } - - if (token.nameToken.text === RECORD_NAME) { - return true; - } - - return false; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/tools.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/tools.ts deleted file mode 100644 index ad52e5faebe..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/tools.ts +++ /dev/null @@ -1,182 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { PromptMetadataRecord } from './base/record.js'; -import { localize } from '../../../../../../../../nls.js'; -import { PromptMetadataDiagnostic, PromptMetadataError, PromptMetadataWarning } from '../diagnostics.js'; -import { FrontMatterSequence } from '../../../codecs/base/frontMatterCodec/tokens/frontMatterSequence.js'; -import { FrontMatterArray, FrontMatterRecord, FrontMatterString, FrontMatterToken, FrontMatterValueToken } from '../../../codecs/base/frontMatterCodec/tokens/index.js'; -import { Range } from '../../../../../../../../editor/common/core/range.js'; - -/** - * Name of the metadata record in the prompt header. - */ -const RECORD_NAME = 'tools'; - -/** - * Prompt `tools` metadata record inside the prompt header. - */ -export class PromptToolsMetadata extends PromptMetadataRecord { - - /** - * List of all valid tool names that were found in - * this metadata record. - */ - public override get value(): string[] | undefined { - if (this.validToolNames === undefined) { - return []; - } - - return [...this.validToolNames.keys()]; - } - - public override get recordName(): string { - return RECORD_NAME; - } - - /** - * Value token reference of the record. - */ - protected valueToken: FrontMatterArray | undefined; - - /** - * List of all valid tool names that were found in - * this metadata record. - */ - private validToolNames: Map | undefined; - - - - constructor( - recordToken: FrontMatterRecord, - languageId: string, - ) { - super(RECORD_NAME, recordToken, languageId); - } - - /** - * Validate the metadata record and collect all issues - * related to its content. - */ - public override validate(): readonly PromptMetadataDiagnostic[] { - const { valueToken } = this.recordToken; - - // validate that the record value is an array - if ((valueToken instanceof FrontMatterArray) === false) { - this.issues.push( - new PromptMetadataError( - valueToken.range, - localize( - 'prompt.header.metadata.tools.diagnostics.invalid-value-type', - "Must be an array of tool names, got '{0}'.", - valueToken.valueTypeName.toString(), - ), - ), - ); - - delete this.valueToken; - return this.issues; - } - - this.valueToken = valueToken; - - // validate that all array items - this.validToolNames = new Map(); - for (const item of this.valueToken.items) { - this.issues.push( - ...this.validateToolName(item, this.validToolNames), - ); - } - - return this.issues; - } - - public getToolRange(toolName: string): Range | undefined { - return this.validToolNames?.get(toolName); - } - - /** - * Validate an individual provided value token that is used - * for a tool name. - */ - private validateToolName( - valueToken: FrontMatterValueToken, - validToolNames: Map, - ): readonly PromptMetadataDiagnostic[] { - const issues: PromptMetadataDiagnostic[] = []; - - // tool name must be a quoted or an unquoted 'string' - if ( - (valueToken instanceof FrontMatterString) === false && - (valueToken instanceof FrontMatterSequence) === false - ) { - issues.push( - new PromptMetadataWarning( - valueToken.range, - localize( - 'prompt.header.metadata.tools.diagnostics.invalid-tool-name-type', - "Unexpected tool name '{0}', expected a string literal.", - valueToken.text - ), - ), - ); - - return issues; - } - - const cleanToolName = valueToken.cleanText.trim(); - // the tool name should not be empty - if (cleanToolName.length === 0) { - issues.push( - new PromptMetadataWarning( - valueToken.range, - localize( - 'prompt.header.metadata.tools.diagnostics.empty-tool-name', - "Tool name cannot be empty.", - ), - ), - ); - - return issues; - } - - // the tool name should not be duplicated - if (validToolNames.has(cleanToolName)) { - issues.push( - new PromptMetadataWarning( - valueToken.range, - localize( - 'prompt.header.metadata.tools.diagnostics.duplicate-tool-name', - "Duplicate tool name '{0}'.", - cleanToolName, - ), - ), - ); - - return issues; - } - - validToolNames.set(cleanToolName, valueToken.range); - return issues; - } - - /** - * Check if a provided front matter token is a metadata record - * with name equal to `tools`. - */ - public static isToolsRecord( - token: FrontMatterToken, - ): boolean { - if ((token instanceof FrontMatterRecord) === false) { - return false; - } - - if (token.nameToken.text === RECORD_NAME) { - return true; - } - - return false; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/modeHeader.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/modeHeader.ts deleted file mode 100644 index 94307163bf6..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/modeHeader.ts +++ /dev/null @@ -1,56 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { HeaderBase, IHeaderMetadata, type TDehydrated } from './headerBase.js'; -import { PromptsType } from '../../promptTypes.js'; -import { FrontMatterRecord } from '../../codecs/base/frontMatterCodec/tokens/index.js'; -import { PromptModelMetadata } from './metadata/model.js'; -import { PromptToolsMetadata } from './metadata/tools.js'; - -/** - * Metadata utility object for mode files. - */ -interface IModeMetadata extends IHeaderMetadata { - /** - * Tools metadata in the mode header. - */ - tools: PromptToolsMetadata; - - /** - * Chat model metadata in the mode header. - */ - model: PromptModelMetadata; -} - -/** - * Metadata for mode files. - */ -export type TModeMetadata = Partial> & { promptType: PromptsType.mode }; - -/** - * Header object for mode files. - */ -export class ModeHeader extends HeaderBase { - protected override handleToken(token: FrontMatterRecord): boolean { - // if the record might be a "tools" metadata - // add it to the list of parsed metadata records - if (PromptToolsMetadata.isToolsRecord(token)) { - const metadata = new PromptToolsMetadata(token, this.languageId); - - this.issues.push(...metadata.validate()); - this.meta.tools = metadata; - return true; - } - if (PromptModelMetadata.isModelRecord(token)) { - const metadata = new PromptModelMetadata(token, this.languageId); - - this.issues.push(...metadata.validate()); - this.meta.model = metadata; - - return true; - } - return false; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/promptHeader.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/promptHeader.ts deleted file mode 100644 index 250e8382372..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/promptHeader.ts +++ /dev/null @@ -1,103 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { ChatModeKind } from '../../../constants.js'; -import { localize } from '../../../../../../../nls.js'; -import { PromptMetadataWarning } from './diagnostics.js'; -import { HeaderBase, IHeaderMetadata, type TDehydrated } from './headerBase.js'; -import { PromptsType } from '../../promptTypes.js'; -import { FrontMatterRecord } from '../../codecs/base/frontMatterCodec/tokens/index.js'; -import { PromptModelMetadata } from './metadata/model.js'; -import { PromptToolsMetadata } from './metadata/tools.js'; -import { PromptModeMetadata } from './metadata/mode.js'; - -/** - * Metadata utility object for prompt files. - */ -export interface IPromptMetadata extends IHeaderMetadata { - /** - * Tools metadata in the prompt header. - */ - tools: PromptToolsMetadata; - - /** - * Chat mode metadata in the prompt header. - */ - mode: PromptModeMetadata; - - /** - * Chat model metadata in the prompt header. - */ - model: PromptModelMetadata; -} - -/** - * Metadata for prompt files. - */ -export type TPromptMetadata = Partial> & { promptType: PromptsType.prompt }; - -/** - * Header object for prompt files. - */ -export class PromptHeader extends HeaderBase { - protected override handleToken(token: FrontMatterRecord): boolean { - // if the record might be a "tools" metadata - // add it to the list of parsed metadata records - if (PromptToolsMetadata.isToolsRecord(token)) { - const metadata = new PromptToolsMetadata(token, this.languageId); - - this.issues.push(...metadata.validate()); - this.meta.tools = metadata; - - this.validateToolsAndModeCompatibility(); - return true; - } - - // if the record might be a "mode" metadata - // add it to the list of parsed metadata records - if (PromptModeMetadata.isModeRecord(token)) { - const metadata = new PromptModeMetadata(token, this.languageId); - - this.issues.push(...metadata.validate()); - this.meta.mode = metadata; - - this.validateToolsAndModeCompatibility(); - return true; - } - - if (PromptModelMetadata.isModelRecord(token)) { - const metadata = new PromptModelMetadata(token, this.languageId); - - this.issues.push(...metadata.validate()); - this.meta.model = metadata; - - return true; - } - - return false; - } - - /** - * Validate that the `tools` and `mode` metadata are compatible - * with each other. If not, add a warning diagnostic. - */ - private validateToolsAndModeCompatibility(): void { - const { tools, mode } = this.meta; - const modeValue = mode?.value; - - if (tools !== undefined && (modeValue === ChatModeKind.Edit || modeValue === ChatModeKind.Ask)) { - this.issues.push( - new PromptMetadataWarning( - tools.range, - localize( - 'prompt.header.metadata.mode.diagnostics.incompatible-with-tools', - "Tools can not be used in '{0}' mode and will be ignored.", - modeValue - ), - ), - ); - } - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptParser.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptParser.ts deleted file mode 100644 index 99df26fd5e9..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptParser.ts +++ /dev/null @@ -1,72 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { URI } from '../../../../../../base/common/uri.js'; -import { IPromptContentsProvider } from '../contentProviders/types.js'; -import { ILogService } from '../../../../../../platform/log/common/log.js'; -import { BasePromptParser, IPromptParserOptions } from './basePromptParser.js'; -import { IModelService } from '../../../../../../editor/common/services/model.js'; -import { TextModelContentsProvider } from '../contentProviders/textModelContentsProvider.js'; -import { FilePromptContentProvider } from '../contentProviders/filePromptContentsProvider.js'; -import { IInstantiationService } from '../../../../../../platform/instantiation/common/instantiation.js'; -import { IPromptContentsProviderOptions } from '../contentProviders/promptContentsProviderBase.js'; -import { IWorkbenchEnvironmentService } from '../../../../../services/environment/common/environmentService.js'; - -/** - * Get prompt contents provider object based on the prompt type. - */ -function getContentsProvider( - uri: URI, - options: IPromptContentsProviderOptions, - modelService: IModelService, - instaService: IInstantiationService -): IPromptContentsProvider { - const model = modelService.getModel(uri); - if (model) { - return instaService.createInstance(TextModelContentsProvider, model, options); - } - return instaService.createInstance(FilePromptContentProvider, uri, options); -} - -/** - * General prompt parser class that automatically infers a prompt - * contents provider type by the type of provided prompt URI. - */ -export class PromptParser extends BasePromptParser { - /** - * Underlying prompt contents provider instance. - */ - private readonly contentsProvider: IPromptContentsProvider; - - constructor( - uri: URI, - options: IPromptParserOptions, - @ILogService logService: ILogService, - @IModelService modelService: IModelService, - @IInstantiationService instaService: IInstantiationService, - @IWorkbenchEnvironmentService envService: IWorkbenchEnvironmentService, - ) { - const contentsProvider = getContentsProvider(uri, options, modelService, instaService); - - super( - contentsProvider, - options, - instaService, - envService, - logService, - ); - - this.contentsProvider = this._register(contentsProvider); - } - - /** - * Returns a string representation of this object. - */ - public override toString(): string { - const { sourceName } = this.contentsProvider; - - return `prompt-parser:${sourceName}:${this.uri.path}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/textModelPromptParser.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/textModelPromptParser.ts deleted file mode 100644 index bb2b7060abb..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/textModelPromptParser.ts +++ /dev/null @@ -1,42 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { ITextModel } from '../../../../../../editor/common/model.js'; -import { ILogService } from '../../../../../../platform/log/common/log.js'; -import { BasePromptParser, IPromptParserOptions } from './basePromptParser.js'; -import { TextModelContentsProvider } from '../contentProviders/textModelContentsProvider.js'; -import { IInstantiationService } from '../../../../../../platform/instantiation/common/instantiation.js'; -import { IWorkbenchEnvironmentService } from '../../../../../services/environment/common/environmentService.js'; - -/** - * Class capable of parsing prompt syntax out of a provided text model, - * including all the nested child file references it may have. - */ -export class TextModelPromptParser extends BasePromptParser { - constructor( - model: ITextModel, - options: IPromptParserOptions, - @IInstantiationService instantiationService: IInstantiationService, - @IWorkbenchEnvironmentService envService: IWorkbenchEnvironmentService, - @ILogService logService: ILogService, - ) { - const contentsProvider = instantiationService.createInstance( - TextModelContentsProvider, - model, - options, - ); - - super(contentsProvider, options, instantiationService, envService, logService); - - this._register(contentsProvider); - } - - /** - * Returns a string representation of this object. - */ - public override toString(): string { - return `text-model-prompt:${this.uri.path}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/topError.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/topError.ts deleted file mode 100644 index b6281d3f42b..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/topError.ts +++ /dev/null @@ -1,102 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { ITopError } from './types.js'; -import { localize } from '../../../../../../nls.js'; -import { assert } from '../../../../../../base/common/assert.js'; -import { assertDefined } from '../../../../../../base/common/types.js'; -import { OpenFailed, RecursiveReference, FailedToResolveContentsStream } from '../../promptFileReferenceErrors.js'; - -/** - * The top-most error of the reference tree. - */ -export class TopError implements ITopError { - public readonly originalError: ITopError['originalError']; - public readonly errorSubject: ITopError['errorSubject']; - public readonly errorsCount: ITopError['errorsCount']; - public readonly parentUri: ITopError['parentUri']; - - constructor( - options: Omit, - ) { - this.originalError = options.originalError; - this.errorSubject = options.errorSubject; - this.errorsCount = options.errorsCount; - this.parentUri = options.parentUri; - } - - public get localizedMessage(): string { - const { originalError, parentUri, errorSubject: subject, errorsCount } = this; - - assert( - errorsCount >= 1, - `Error count must be at least 1, got '${errorsCount}'.`, - ); - - // a note about how many more link issues are there - const moreIssuesLabel = (errorsCount > 1) - ? localize('workbench.reusable-prompts.top-error.more-issues-label', "\n(+{0} more issues)", errorsCount - 1) - : ''; - - if (subject === 'root') { - if (originalError instanceof OpenFailed) { - return localize( - 'workbench.reusable-prompts.top-error.open-failed', - "Cannot open '{0}'.{1}", - originalError.uri.path, - moreIssuesLabel, - ); - } - - if (originalError instanceof FailedToResolveContentsStream) { - return localize( - 'workbench.reusable-prompts.top-error.cannot-read', - "Cannot read '{0}'.{1}", - originalError.uri.path, - moreIssuesLabel, - ); - } - - if (originalError instanceof RecursiveReference) { - return localize( - 'workbench.reusable-prompts.top-error.recursive-reference', - "Recursion to itself.", - ); - } - - return originalError.message + moreIssuesLabel; - } - - // a sanity check - because the error subject is not `root`, the parent must set - assertDefined( - parentUri, - 'Parent URI must be defined for error of non-root link.', - ); - - const errorMessageStart = (subject === 'child') - ? localize( - 'workbench.reusable-prompts.top-error.child.direct', - "Contains", - ) - : localize( - 'workbench.reusable-prompts.top-error.child.indirect', - "Indirectly referenced prompt '{0}' contains", - parentUri.path, - ); - - const linkIssueName = (originalError instanceof RecursiveReference) - ? localize('recursive', "recursive") - : localize('broken', "broken"); - - return localize( - 'workbench.reusable-prompts.top-error.child.final-message', - "{0} a {1} link to '{2}' that will be ignored.{3}", - errorMessageStart, - linkIssueName, - originalError.uri.path, - moreIssuesLabel, - ); - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/types.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/types.ts deleted file mode 100644 index 0a3d693c61d..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/types.ts +++ /dev/null @@ -1,116 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { URI } from '../../../../../../base/common/uri.js'; -import { ResolveError } from '../../promptFileReferenceErrors.js'; -import { IRange, Range } from '../../../../../../editor/common/core/range.js'; - -/** - * A resolve error with a parent prompt URI, if any. - */ -export interface IResolveError { - /** - * Original error instance. - */ - readonly originalError: ResolveError; - - /** - * URI of the parent that references this error. - */ - readonly parentUri?: URI; -} - -/** - * Top most error of the reference tree. - */ -export interface ITopError extends IResolveError { - /** - * Where does the error belong to: - * - * - `root` - the error is the top most error of the entire tree - * - `child` - the error is a child of the root error - * - `indirect-child` - the error is a child of a child of the root error - */ - readonly errorSubject: 'root' | 'child' | 'indirect-child'; - - /** - * Total number of all errors in the references tree, including the error - * of the current reference and all possible errors of its children. - */ - readonly errorsCount: number; - - /** - * Localized error message. - */ - readonly localizedMessage: string; -} - -/** - * Base interface for a generic prompt reference. - */ -interface IPromptReferenceBase { - /** - * Type of the prompt reference. E.g., `file`, `http`, `image`, etc. - */ - readonly type: string; - - /** - * Subtype of the prompt reference. For instance a `file` reference - * can be a `markdown link` or a prompt `#file:` variable reference. - */ - readonly subtype: string; - - /** - * URI component of the associated with this reference. - */ - readonly uri: URI; - - /** - * The full range of the prompt reference in the source text, - * including the {@link linkRange} and any additional - * parts the reference may contain (e.g., the `#file:` prefix). - */ - readonly range: Range; - - /** - * Range of the link part that the reference points to. - */ - readonly linkRange: IRange | undefined; - - /** - * Text of the reference as it appears in the source. - */ - readonly text: string; - - /** - * Original link path as it appears in the source. - */ - readonly path: string; - -} - -/** - * The special case of the {@link IPromptReferenceBase} that pertains - * to a file resource on the disk. - */ -export interface IPromptFileReference extends IPromptReferenceBase { - readonly type: 'file'; - - /** - * Subtype of a file reference, - either a prompt `#file` variable, - * or a `markdown link` (e.g., `[caption](/path/to/file.md)`). - */ - readonly subtype: 'prompt' | 'markdown'; -} - -/** - * List of all known prompt reference types. - */ -export type TPromptReference = IPromptFileReference; - -export type TVariableReference = { - readonly name: string; - readonly range: Range; -}; diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsService.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsService.ts index ab3b4887aa4..f1eef78e834 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsService.ts +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsService.ts @@ -6,10 +6,8 @@ import { ChatModeKind } from '../../constants.js'; import { URI } from '../../../../../../base/common/uri.js'; import { Event } from '../../../../../../base/common/event.js'; -import { TMetadata } from '../parsers/promptHeader/headerBase.js'; import { ITextModel } from '../../../../../../editor/common/model.js'; import { IDisposable } from '../../../../../../base/common/lifecycle.js'; -import { TextModelPromptParser } from '../parsers/textModelPromptParser.js'; import { CancellationToken } from '../../../../../../base/common/cancellation.js'; import { PromptsType } from '../promptTypes.js'; import { createDecorator } from '../../../../../../platform/instantiation/common/instantiation.js'; @@ -48,32 +46,6 @@ export interface IPromptPath { readonly type: PromptsType; } -/** - * Type for a shared prompt parser instance returned by the {@link IPromptsService}. - * Because the parser is shared, we omit the `dispose` method from - * the original type so the caller cannot dispose it prematurely - */ -export type TSharedPrompt = Omit; - -/** - * Metadata node object in a hierarchical tree of prompt references. - */ -export interface IMetadata { - /** - * URI of a prompt file. - */ - readonly uri: URI; - - /** - * Metadata of the prompt file. - */ - readonly metadata: TMetadata | null; - - /** - * List of metadata for each valid child prompt reference. - */ - readonly children?: readonly IMetadata[]; -} export interface ICustomChatMode { /** @@ -159,12 +131,6 @@ export type TCombinedToolsMetadata = ICombinedAgentToolsMetadata | ICombinedNonA export interface IPromptsService extends IDisposable { readonly _serviceBrand: undefined; - /** - * Get a prompt syntax parser for the provided text model. - * See {@link TextModelPromptParser} for more info on the parser API. - */ - getSyntaxParserFor(model: ITextModel): TSharedPrompt & { isDisposed: false }; - /** * The parsed prompt file for the provided text model. * @param textModel Returns the parsed prompt file. @@ -207,12 +173,6 @@ export interface IPromptsService extends IDisposable { */ getCustomChatModes(token: CancellationToken): Promise; - /** - * Parses the provided URI - * @param uris - */ - parse(uri: URI, type: PromptsType, token: CancellationToken): Promise; - /** * Parses the provided URI * @param uris @@ -232,15 +192,6 @@ export interface IChatPromptSlashCommand { readonly promptPath?: IPromptPath; } - -export interface IPromptParserResult { - readonly uri: URI; - readonly metadata: TMetadata | null; - readonly fileReferences: readonly URI[]; - readonly variableReferences: readonly IVariableReference[]; - readonly header?: IPromptHeader; -} - export interface IPromptHeader { readonly node: YamlNode | undefined; readonly errors: YamlParseError[]; diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts index 07579a4d003..50032a655c1 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts @@ -4,29 +4,24 @@ *--------------------------------------------------------------------------------------------*/ import { localize } from '../../../../../../nls.js'; -import { getLanguageIdForPromptsType, getPromptsTypeForLanguageId, PROMPT_LANGUAGE_ID, PromptsType } from '../promptTypes.js'; -import { PromptParser } from '../parsers/promptParser.js'; +import { getPromptsTypeForLanguageId, PROMPT_LANGUAGE_ID, PromptsType } from '../promptTypes.js'; import { type URI } from '../../../../../../base/common/uri.js'; -import { assert } from '../../../../../../base/common/assert.js'; import { basename } from '../../../../../../base/common/path.js'; import { PromptFilesLocator } from '../utils/promptFilesLocator.js'; import { Disposable } from '../../../../../../base/common/lifecycle.js'; import { Event } from '../../../../../../base/common/event.js'; import { type ITextModel } from '../../../../../../editor/common/model.js'; -import { ObjectCache } from '../utils/objectCache.js'; import { ILogService } from '../../../../../../platform/log/common/log.js'; -import { TextModelPromptParser } from '../parsers/textModelPromptParser.js'; import { ILabelService } from '../../../../../../platform/label/common/label.js'; import { IModelService } from '../../../../../../editor/common/services/model.js'; import { CancellationToken } from '../../../../../../base/common/cancellation.js'; import { IInstantiationService } from '../../../../../../platform/instantiation/common/instantiation.js'; import { IUserDataProfileService } from '../../../../../services/userDataProfile/common/userDataProfile.js'; -import type { IChatPromptSlashCommand, ICustomChatMode, IPromptParserResult, IPromptPath, IPromptsService, TPromptsStorage } from './promptsService.js'; +import type { IChatPromptSlashCommand, ICustomChatMode, IPromptPath, IPromptsService, TPromptsStorage } from './promptsService.js'; import { getCleanPromptName, PROMPT_FILE_EXTENSION } from '../config/promptFileLocations.js'; import { ILanguageService } from '../../../../../../editor/common/languages/language.js'; import { PromptsConfig } from '../config/config.js'; import { IConfigurationService } from '../../../../../../platform/configuration/common/configuration.js'; -import { PositionOffsetTransformer } from '../../../../../../editor/common/core/text/positionToOffset.js'; import { NewPromptsParser, ParsedPromptFile } from './newPromptsParser.js'; import { IFileService } from '../../../../../../platform/files/common/files.js'; import { ResourceMap } from '../../../../../../base/common/map.js'; @@ -40,11 +35,6 @@ import { IVariableReference } from '../../chatModes.js'; export class PromptsService extends Disposable implements IPromptsService { public declare readonly _serviceBrand: undefined; - /** - * Cache of text model content prompt parsers. - */ - private readonly cache: ObjectCache; - /** * Prompt files locator utility. */ @@ -77,36 +67,6 @@ export class PromptsService extends Disposable implements IPromptsService { this.fileLocator = this._register(this.instantiationService.createInstance(PromptFilesLocator)); - // the factory function below creates a new prompt parser object - // for the provided model, if no active non-disposed parser exists - this.cache = this._register( - new ObjectCache((model) => { - assert( - model.isDisposed() === false, - 'Text model must not be disposed.', - ); - - /** - * Note! When/if shared with "file" prompts, the `seenReferences` array below must be taken into account. - * Otherwise consumers will either see incorrect failing or incorrect successful results, based on their - * use case, timing of their calls to the {@link getSyntaxParserFor} function, and state of this service. - */ - const parser: TextModelPromptParser = instantiationService.createInstance( - TextModelPromptParser, - model, - { allowNonPromptFiles: true, languageId: undefined, updateOnChange: true }, - ).start(); - - // this is a sanity check and the contract of the object cache, - // we must return a non-disposed object from this factory function - parser.assertNotDisposed( - 'Created prompt parser must not be disposed.', - ); - - return parser; - }) - ); - this._register(this.modelService.onModelRemoved((model) => { this.parsedPromptFileCache.delete(model.uri); })); @@ -132,21 +92,6 @@ export class PromptsService extends Disposable implements IPromptsService { } - /** - * @throws {Error} if: - * - the provided model is disposed - * - newly created parser is disposed immediately on initialization. - * See factory function in the {@link constructor} for more info. - */ - public getSyntaxParserFor(model: ITextModel): TextModelPromptParser & { isDisposed: false } { - assert( - model.isDisposed() === false, - 'Cannot create a prompt syntax parser for a disposed model.', - ); - - return this.cache.get(model); - } - public getParsedPromptFile(textModel: ITextModel): ParsedPromptFile { const cached = this.parsedPromptFileCache.get(textModel.uri); if (cached && cached[0] === textModel.getVersionId()) { @@ -255,7 +200,7 @@ export class PromptsService extends Disposable implements IPromptsService { private async computeCustomChatModes(token: CancellationToken): Promise { const modeFiles = await this.listPromptFiles(PromptsType.mode, token); - const metadataList = await Promise.all( + const customChatModes = await Promise.all( modeFiles.map(async ({ uri }): Promise => { const ast = await this.parseNew(uri, token); @@ -281,37 +226,7 @@ export class PromptsService extends Disposable implements IPromptsService { }) ); - - return metadataList; - } - - public async parse(uri: URI, type: PromptsType, token: CancellationToken): Promise { - let parser: PromptParser | undefined; - try { - const languageId = getLanguageIdForPromptsType(type); - parser = this.instantiationService.createInstance(PromptParser, uri, { allowNonPromptFiles: true, languageId, updateOnChange: false }).start(token); - const completed = await parser.settled(); - if (!completed) { - throw new Error(localize('promptParser.notCompleted', "Prompt parser for {0} did not complete.", uri.toString())); - } - const fullContent = await parser.getFullContent(); - const transformer = new PositionOffsetTransformer(fullContent); - const variableReferences = parser.variableReferences.map(ref => { - return { - name: ref.name, - range: transformer.getOffsetRange(ref.range) - }; - }).sort((a, b) => b.range.start - a.range.start); // in reverse order - // make a copy, to avoid leaking the parser instance - return { - uri: parser.uri, - metadata: parser.metadata, - variableReferences, - fileReferences: parser.references.map(ref => ref.uri), - }; - } finally { - parser?.dispose(); - } + return customChatModes; } public async parseNew(uri: URI, token: CancellationToken): Promise { diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/utils/objectCache.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/utils/objectCache.ts deleted file mode 100644 index 443d6e44c94..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/utils/objectCache.ts +++ /dev/null @@ -1,153 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { Disposable, DisposableMap } from '../../../../../../base/common/lifecycle.js'; -import { ObservableDisposable, assertNotDisposed } from './observableDisposable.js'; - -/** - * Generic cache for object instances. Guarantees to return only non-disposed - * objects from the {@linkcode get} method. If a requested object is not yet - * in the cache or is disposed already, the {@linkcode factory} callback is - * called to create a new object. - * - * @throws if {@linkcode factory} callback returns a disposed object. - * - * ## Examples - * - * ```typescript - * // a class that will be used as a cache key; the key can be of any - * // non-nullable type, including primitives like `string` or `number`, - * // but in this case we use an object pointer as a key - * class KeyObject {} - * - * // a class for testing purposes - * class TestObject extends ObservableDisposable { - * constructor( - * public readonly id: KeyObject, - * ) {} - * }; - * - * // create an object cache instance providing it a factory function that - * // is responsible for creating new objects based on the provided key if - * // the cache does not contain the requested object yet or an existing - * // object is already disposed - * const cache = new ObjectCache((key) => { - * // create a new test object based on the provided key - * return new TestObject(key); - * }); - * - * // create two keys - * const key1 = new KeyObject(); - * const key2 = new KeyObject(); - * - * // get an object from the cache by its key - * const object1 = cache.get(key1); // returns a new test object - * - * // validate that the new object has the correct key - * assert( - * object1.id === key1, - * 'Object 1 must have correct ID.', - * ); - * - * // returns the same cached test object - * const object2 = cache.get(key1); - * - * // validate that the same exact object is returned from the cache - * assert( - * object1 === object2, - * 'Object 2 the same cached object as object 1.', - * ); - * - * // returns a new test object - * const object3 = cache.get(key2); - * - * // validate that the new object has the correct key - * assert( - * object3.id === key2, - * 'Object 3 must have correct ID.', - * ); - * - * assert( - * object3 !== object1, - * 'Object 3 must be a new object.', - * ); - * ``` - */ -export class ObjectCache< - TValue extends ObservableDisposable, - TKey extends NonNullable = string, -> extends Disposable { - private readonly cache: DisposableMap = - this._register(new DisposableMap()); - - constructor( - private readonly factory: (key: TKey) => TValue & { isDisposed: false }, - ) { - super(); - } - - /** - * Get an existing object from the cache. If a requested object is not yet - * in the cache or is disposed already, the {@linkcode factory} callback is - * called to create a new object. - * - * @throws if {@linkcode factory} callback returns a disposed object. - * @param key - ID of the object in the cache - */ - public get(key: TKey): TValue & { isDisposed: false } { - let object = this.cache.get(key); - - // if object is already disposed, remove it from the cache - if (object?.isDisposed) { - this.cache.deleteAndLeak(key); - object = undefined; - } - - // if object exists and is not disposed, return it - if (object) { - // must always hold true due to the check above - assertNotDisposed( - object, - 'Object must not be disposed.', - ); - - return object; - } - - // create a new object by calling the factory - object = this.factory(key); - - // newly created object must not be disposed - assertNotDisposed( - object, - 'Newly created object must not be disposed.', - ); - - // remove it from the cache automatically on dispose - object.addDisposables( - object.onDispose(() => { - this.cache.deleteAndLeak(key); - })); - this.cache.set(key, object); - - return object; - } - - /** - * Remove an object from the cache by its key. - * - * @param key ID of the object to remove. - * @param dispose Whether the removed object must be disposed. - */ - public remove(key: TKey, dispose: boolean): this { - if (dispose) { - this.cache.deleteAndDispose(key); - return this; - } - - this.cache.deleteAndLeak(key); - return this; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/utils/observableDisposable.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/utils/observableDisposable.ts deleted file mode 100644 index a0e6a70e733..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/utils/observableDisposable.ts +++ /dev/null @@ -1,89 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { Disposable, DisposableStore, IDisposable, toDisposable } from '../../../../../../base/common/lifecycle.js'; - -/** -* @deprecated do not use this, https://github.com/microsoft/vscode/issues/248366 - */ -export abstract class ObservableDisposable extends Disposable { - /** - * Underlying disposables store this object relies on. - */ - private readonly store = this._register(new DisposableStore()); - - /** - * Check if the current object is already has been disposed. - */ - public get isDisposed(): boolean { - return this.store.isDisposed; - } - - /** - * The event is fired when this object is disposed. - * Note! Executes the callback immediately if already disposed. - * - * @param callback The callback function to be called on updates. - */ - public onDispose(callback: () => void): IDisposable { - // if already disposed, execute the callback immediately - if (this.isDisposed) { - const timeoutHandle = setTimeout(callback); - - return toDisposable(() => { - clearTimeout(timeoutHandle); - }); - } - - return this.store.add(toDisposable(callback)); - } - - /** - * Adds disposable object(s) to the list of disposables - * that will be disposed with this object. - */ - public addDisposables(...disposables: IDisposable[]): this { - for (const disposable of disposables) { - this.store.add(disposable); - } - - return this; - } - - /** - * Assert that the current object was not yet disposed. - * - * @throws If the current object was already disposed. - * @param error Error message or error object to throw if assertion fails. - */ - public assertNotDisposed( - error: string | Error, - ): asserts this is TNotDisposed { - assertNotDisposed(this, error); - } -} - -/** - * @deprecated do not use this, https://github.com/microsoft/vscode/issues/248366 - */ -type TNotDisposed = TObject & { isDisposed: false }; - -/** - * @deprecated do not use this, https://github.com/microsoft/vscode/issues/248366 - */ -export function assertNotDisposed( - object: TObject, - error: string | Error, -): asserts object is TNotDisposed { - if (!object.isDisposed) { - return; - } - - const errorToThrow = typeof error === 'string' - ? new Error(error) - : error; - - throw errorToThrow; -} diff --git a/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/frontMatterDecoder/frontMatterBoolean.test.ts b/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/frontMatterDecoder/frontMatterBoolean.test.ts deleted file mode 100644 index 8d32c30d851..00000000000 --- a/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/frontMatterDecoder/frontMatterBoolean.test.ts +++ /dev/null @@ -1,317 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import assert from 'assert'; -import { Range } from '../../../../../../../../../editor/common/core/range.js'; -import { Word } from '../../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/tokens.js'; -import { randomBoolean } from '../../../../../../../../../base/test/common/testUtils.js'; -import { FrontMatterBoolean } from '../../../../../../common/promptSyntax/codecs/base/frontMatterCodec/tokens/index.js'; -import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../../../../../../base/test/common/utils.js'; -import { FrontMatterSequence } from '../../../../../../common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterSequence.js'; - -suite('FrontMatterBoolean', () => { - ensureNoDisposablesAreLeakedInTestSuite(); - - suite('equals()', () => { - suite('base case', () => { - test('true', () => { - // both values should yield the same result - const booleanText = (randomBoolean()) - ? 'true' - : 'TRUE'; - - const boolean = new FrontMatterBoolean( - new Word( - new Range(1, 1, 1, 5), - booleanText, - ), - ); - - const other = new FrontMatterBoolean( - new Word( - new Range(1, 1, 1, 5), - booleanText, - ), - ); - - assert.strictEqual( - boolean.value, - true, - 'Must have correct boolean value.', - ); - - assert( - boolean.equals(other), - 'Booleans must be equal.', - ); - }); - - test('false', () => { - // both values should yield the same result - const booleanText = (randomBoolean()) - ? 'false' - : 'FALSE'; - - const boolean = new FrontMatterBoolean( - new Word( - new Range(5, 15, 5, 15 + 6), - booleanText, - ), - ); - - const other = new FrontMatterBoolean( - new Word( - new Range(5, 15, 5, 15 + 6), - booleanText, - ), - ); - - assert.strictEqual( - boolean.value, - false, - 'Must have correct boolean value.', - ); - - assert( - boolean.equals(other), - 'Booleans must be equal.', - ); - }); - }); - - suite('non-boolean token', () => { - suite('word token', () => { - test('true', () => { - // both values should yield the same result - const booleanText = (randomBoolean()) - ? 'true' - : 'TRUE'; - - const boolean = new FrontMatterBoolean( - new Word( - new Range(1, 1, 1, 5), - booleanText, - ), - ); - - const other = new Word( - new Range(1, 1, 1, 5), - booleanText, - ); - - assert( - boolean.equals(other) === false, - 'Booleans must not be equal.', - ); - }); - - test('false', () => { - // both values should yield the same result - const booleanText = (randomBoolean()) - ? 'false' - : 'FALSE'; - - const boolean = new FrontMatterBoolean( - new Word( - new Range(1, 2, 1, 2 + 6), - booleanText, - ), - ); - - const other = new Word( - new Range(1, 2, 1, 2 + 6), - booleanText, - ); - - assert( - boolean.equals(other) === false, - 'Booleans must not be equal.', - ); - }); - }); - - suite('sequence token', () => { - test('true', () => { - // both values should yield the same result - const booleanText = (randomBoolean()) - ? 'true' - : 'TRUE'; - - const boolean = new FrontMatterBoolean( - new Word( - new Range(1, 1, 1, 5), - booleanText, - ), - ); - - const other = new FrontMatterSequence([ - new Word( - new Range(1, 1, 1, 5), - booleanText, - ), - ]); - - assert( - boolean.equals(other) === false, - 'Booleans must not be equal.', - ); - }); - - test('false', () => { - // both values should yield the same result - const booleanText = (randomBoolean()) - ? 'false' - : 'FALSE'; - - const boolean = new FrontMatterBoolean( - new Word( - new Range(1, 2, 1, 2 + 6), - booleanText, - ), - ); - - const other = new FrontMatterSequence([ - new Word( - new Range(1, 2, 1, 2 + 6), - booleanText, - ), - ]); - - assert( - boolean.equals(other) === false, - 'Booleans must not be equal.', - ); - }); - }); - }); - - suite('different range', () => { - test('true', () => { - // both values should yield the same result - const booleanText = (randomBoolean()) - ? 'true' - : 'TRUE'; - - const boolean = new FrontMatterBoolean( - new Word( - new Range(1, 2, 1, 2 + 4), - booleanText, - ), - ); - - const other = new FrontMatterBoolean( - new Word( - new Range(3, 2, 3, 2 + 4), - booleanText, - ), - ); - - assert( - boolean.equals(other) === false, - 'Booleans must not be equal.', - ); - }); - - test('false', () => { - // both values should yield the same result - const booleanText = (randomBoolean()) - ? 'false' - : 'FALSE'; - - const boolean = new FrontMatterBoolean( - new Word( - new Range(5, 15, 5, 15 + 5), - booleanText, - ), - ); - - const other = new FrontMatterBoolean( - new Word( - new Range(4, 15, 4, 15 + 5), - booleanText, - ), - ); - - assert( - boolean.equals(other) === false, - 'Booleans must not be equal.', - ); - }); - }); - - suite('different text', () => { - test('true', () => { - const boolean = new FrontMatterBoolean( - new Word( - new Range(1, 1, 1, 5), - 'true', - ), - ); - - const other = new FrontMatterBoolean( - new Word( - new Range(1, 1, 1, 5), - 'True', - ), - ); - - assert( - boolean.equals(other) === false, - 'Booleans must not be equal.', - ); - }); - - test('false', () => { - const boolean = new FrontMatterBoolean( - new Word( - new Range(5, 15, 5, 15 + 6), - 'FALSE', - ), - ); - - const other = new FrontMatterBoolean( - new Word( - new Range(5, 15, 5, 15 + 6), - 'false', - ), - ); - - assert( - boolean.equals(other) === false, - 'Booleans must not be equal.', - ); - }); - }); - - test('throws if cannot be converted to a boolean', () => { - assert.throws(() => { - new FrontMatterBoolean( - new Word( - new Range(1, 1, 1, 5), - 'true1', - ), - ); - }); - - assert.throws(() => { - new FrontMatterBoolean( - new Word( - new Range(2, 5, 2, 5 + 6), - 'fal se', - ), - ); - }); - - assert.throws(() => { - new FrontMatterBoolean( - new Word( - new Range(20, 4, 20, 4 + 1), - '1', - ), - ); - }); - }); - }); -}); diff --git a/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/frontMatterDecoder/frontMatterDecoder.test.ts b/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/frontMatterDecoder/frontMatterDecoder.test.ts deleted file mode 100644 index b4f78b9d19b..00000000000 --- a/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/frontMatterDecoder/frontMatterDecoder.test.ts +++ /dev/null @@ -1,415 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - - -import { Range } from '../../../../../../../../../editor/common/core/range.js'; -import { TestDecoder } from '../utils/testDecoder.js'; -import { VSBuffer } from '../../../../../../../../../base/common/buffer.js'; -import { newWriteableStream } from '../../../../../../../../../base/common/stream.js'; -import { NewLine } from '../../../../../../common/promptSyntax/codecs/base/linesCodec/tokens/newLine.js'; -import { DoubleQuote } from '../../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/doubleQuote.js'; -import { type TSimpleDecoderToken } from '../../../../../../common/promptSyntax/codecs/base/simpleCodec/simpleDecoder.js'; -import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../../../../../../base/test/common/utils.js'; -import { LeftBracket, RightBracket } from '../../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/brackets.js'; -import { FrontMatterDecoder } from '../../../../../../common/promptSyntax/codecs/base/frontMatterCodec/frontMatterDecoder.js'; -import { FrontMatterSequence } from '../../../../../../common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterSequence.js'; -import { ExclamationMark, Quote, Tab, Word, Space, Colon, VerticalTab, Comma, Dash } from '../../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/tokens.js'; -import { FrontMatterBoolean, FrontMatterString, FrontMatterArray, FrontMatterRecord, FrontMatterRecordDelimiter, FrontMatterRecordName } from '../../../../../../common/promptSyntax/codecs/base/frontMatterCodec/tokens/index.js'; - -/** - * Front Matter decoder for testing purposes. - */ -export class TestFrontMatterDecoder extends TestDecoder { - constructor() { - const stream = newWriteableStream(null); - const decoder = new FrontMatterDecoder(stream); - - super(stream, decoder); - } -} - -suite('FrontMatterDecoder', () => { - const disposables = ensureNoDisposablesAreLeakedInTestSuite(); - - test('produces expected tokens', async () => { - const test = disposables.add(new TestFrontMatterDecoder()); - - await test.run( - [ - 'just: "write some yaml "', - 'write-some :\t[ \' just\t \', "yaml!", true, , ,]', - 'anotherField \t\t\t : FALSE ', - ], - [ - // first record - new FrontMatterRecord([ - new FrontMatterRecordName([ - new Word(new Range(1, 1, 1, 1 + 4), 'just'), - ]), - new FrontMatterRecordDelimiter([ - new Colon(new Range(1, 5, 1, 6)), - new Space(new Range(1, 6, 1, 7)), - ]), - new FrontMatterString([ - new DoubleQuote(new Range(1, 7, 1, 8)), - new Word(new Range(1, 8, 1, 8 + 5), 'write'), - new Space(new Range(1, 13, 1, 14)), - new Word(new Range(1, 14, 1, 14 + 4), 'some'), - new Space(new Range(1, 18, 1, 19)), - new Word(new Range(1, 19, 1, 19 + 4), 'yaml'), - new Space(new Range(1, 23, 1, 24)), - new DoubleQuote(new Range(1, 24, 1, 25)), - ]), - ]), - new NewLine(new Range(1, 25, 1, 26)), - // second record - new FrontMatterRecord([ - new FrontMatterRecordName([ - new Word(new Range(2, 1, 2, 1 + 5), 'write'), - new Dash(new Range(2, 6, 2, 7)), - new Word(new Range(2, 7, 2, 7 + 4), 'some'), - ]), - new FrontMatterRecordDelimiter([ - new Colon(new Range(2, 12, 2, 13)), - new Tab(new Range(2, 13, 2, 14)), - ]), - new FrontMatterArray([ - new LeftBracket(new Range(2, 14, 2, 15)), - new FrontMatterString([ - new Quote(new Range(2, 16, 2, 17)), - new Space(new Range(2, 17, 2, 18)), - new Word(new Range(2, 18, 2, 18 + 4), 'just'), - new Tab(new Range(2, 22, 2, 23)), - new Space(new Range(2, 23, 2, 24)), - new Quote(new Range(2, 24, 2, 25)), - ]), - new FrontMatterString([ - new DoubleQuote(new Range(2, 28, 2, 29)), - new Word(new Range(2, 29, 2, 29 + 4), 'yaml'), - new ExclamationMark(new Range(2, 33, 2, 34)), - new DoubleQuote(new Range(2, 34, 2, 35)), - ]), - new FrontMatterBoolean( - new Word(new Range(2, 37, 2, 37 + 4), 'true'), - ), - new RightBracket(new Range(2, 46, 2, 47)), - ]), - ]), - new NewLine(new Range(2, 47, 2, 48)), - // third record - new FrontMatterRecord([ - new FrontMatterRecordName([ - new Word(new Range(3, 1, 3, 1 + 12), 'anotherField'), - ]), - new FrontMatterRecordDelimiter([ - new Colon(new Range(3, 19, 3, 20)), - new Space(new Range(3, 20, 3, 21)), - ]), - new FrontMatterBoolean( - new Word(new Range(3, 22, 3, 22 + 5), 'FALSE'), - ), - ]), - new Space(new Range(3, 27, 3, 28)), - ]); - }); - - suite('record', () => { - suite('values', () => { - test('unquoted string', async () => { - const test = disposables.add(new TestFrontMatterDecoder()); - - await test.run( - [ - 'just: write some yaml ', - 'anotherField \t\t : fal\v \t', - ], - [ - // first record - new FrontMatterRecord([ - new FrontMatterRecordName([ - new Word(new Range(1, 1, 1, 1 + 4), 'just'), - ]), - new FrontMatterRecordDelimiter([ - new Colon(new Range(1, 5, 1, 6)), - new Space(new Range(1, 6, 1, 7)), - ]), - new FrontMatterSequence([ - new Word(new Range(1, 7, 1, 7 + 5), 'write'), - new Space(new Range(1, 12, 1, 13)), - new Word(new Range(1, 13, 1, 13 + 4), 'some'), - new Space(new Range(1, 17, 1, 18)), - new Word(new Range(1, 18, 1, 18 + 4), 'yaml'), - ]), - ]), - new Space(new Range(1, 22, 1, 23)), - new NewLine(new Range(1, 23, 1, 24)), - // second record - new FrontMatterRecord([ - new FrontMatterRecordName([ - new Word(new Range(2, 1, 2, 1 + 12), 'anotherField'), - ]), - new FrontMatterRecordDelimiter([ - new Colon(new Range(2, 17, 2, 18)), - new Space(new Range(2, 18, 2, 19)), - ]), - new FrontMatterSequence([ - new Word(new Range(2, 20, 2, 20 + 3), 'fal'), - ]), - ]), - new VerticalTab(new Range(2, 23, 2, 24)), - new Space(new Range(2, 24, 2, 25)), - new Tab(new Range(2, 25, 2, 26)), - ]); - }); - - test('quoted string', async () => { - const test = disposables.add(new TestFrontMatterDecoder()); - - await test.run( - [ - `just\t:\t'\vdo\tsome\ntesting, please\v' `, - 'anotherField \t\t :\v\v"fal\nse"', - ], - [ - // first record - new FrontMatterRecord([ - new FrontMatterRecordName([ - new Word(new Range(1, 1, 1, 1 + 4), 'just'), - ]), - new FrontMatterRecordDelimiter([ - new Colon(new Range(1, 6, 1, 7)), - new Tab(new Range(1, 7, 1, 8)), - ]), - new FrontMatterString([ - new Quote(new Range(1, 8, 1, 9)), - new VerticalTab(new Range(1, 9, 1, 10)), - new Word(new Range(1, 10, 1, 10 + 2), 'do'), - new Tab(new Range(1, 12, 1, 13)), - new Word(new Range(1, 13, 1, 13 + 4), 'some'), - new NewLine(new Range(1, 17, 1, 18)), - new Word(new Range(2, 1, 2, 1 + 7), 'testing'), - new Comma(new Range(2, 8, 2, 9)), - new Space(new Range(2, 9, 2, 10)), - new Word(new Range(2, 10, 2, 10 + 6), 'please'), - new VerticalTab(new Range(2, 16, 2, 17)), - new Quote(new Range(2, 17, 2, 18)), - ]), - ]), - new Space(new Range(2, 18, 2, 19)), - new NewLine(new Range(2, 19, 2, 20)), - // second record - new FrontMatterRecord([ - new FrontMatterRecordName([ - new Word(new Range(3, 1, 3, 1 + 12), 'anotherField'), - ]), - new FrontMatterRecordDelimiter([ - new Colon(new Range(3, 17, 3, 18)), - new VerticalTab(new Range(3, 18, 3, 19)), - ]), - new FrontMatterString([ - new DoubleQuote(new Range(3, 20, 3, 21)), - new Word(new Range(3, 21, 3, 21 + 3), 'fal'), - new NewLine(new Range(3, 24, 3, 25)), - new Word(new Range(4, 1, 4, 1 + 2), 'se'), - new DoubleQuote(new Range(4, 3, 4, 4)), - ]), - ]), - ]); - }); - - test('boolean', async () => { - const test = disposables.add(new TestFrontMatterDecoder()); - - await test.run( - [ - 'anotherField \t\t : FALSE ', - 'my-field: true\t ', - ], - [ - // first record - new FrontMatterRecord([ - new FrontMatterRecordName([ - new Word(new Range(1, 1, 1, 1 + 12), 'anotherField'), - ]), - new FrontMatterRecordDelimiter([ - new Colon(new Range(1, 17, 1, 18)), - new Space(new Range(1, 18, 1, 19)), - ]), - new FrontMatterBoolean( - new Word( - new Range(1, 20, 1, 20 + 5), - 'FALSE', - ), - ), - ]), - new Space(new Range(1, 25, 1, 26)), - new NewLine(new Range(1, 26, 1, 27)), - // second record - new FrontMatterRecord([ - new FrontMatterRecordName([ - new Word(new Range(2, 1, 2, 1 + 2), 'my'), - new Dash(new Range(2, 3, 2, 4)), - new Word(new Range(2, 4, 2, 4 + 5), 'field'), - ]), - new FrontMatterRecordDelimiter([ - new Colon(new Range(2, 9, 2, 10)), - new Space(new Range(2, 10, 2, 11)), - ]), - new FrontMatterBoolean( - new Word( - new Range(2, 11, 2, 11 + 4), - 'true', - ), - ), - ]), - new Tab(new Range(2, 15, 2, 16)), - new Space(new Range(2, 16, 2, 17)), - ]); - }); - - suite('array', () => { - test('empty', async () => { - const test = disposables.add(new TestFrontMatterDecoder()); - - await test.run( - [ - `tools\v:\t []`, - 'anotherField \t\t :\v\v"fal\nse"', - ], - [ - // first record - new FrontMatterRecord([ - new FrontMatterRecordName([ - new Word(new Range(1, 1, 1, 1 + 5), 'tools'), - ]), - new FrontMatterRecordDelimiter([ - new Colon(new Range(1, 7, 1, 8)), - new Tab(new Range(1, 8, 1, 9)), - ]), - new FrontMatterArray([ - new LeftBracket(new Range(1, 10, 1, 11)), - new RightBracket(new Range(1, 11, 1, 12)), - ]), - ]), - new NewLine(new Range(1, 12, 1, 13)), - // second record - new FrontMatterRecord([ - new FrontMatterRecordName([ - new Word(new Range(2, 1, 2, 1 + 12), 'anotherField'), - ]), - new FrontMatterRecordDelimiter([ - new Colon(new Range(2, 17, 2, 18)), - new VerticalTab(new Range(2, 18, 2, 19)), - ]), - new FrontMatterString([ - new DoubleQuote(new Range(2, 20, 2, 21)), - new Word(new Range(2, 21, 2, 21 + 3), 'fal'), - new NewLine(new Range(2, 24, 2, 25)), - new Word(new Range(3, 1, 3, 1 + 2), 'se'), - new DoubleQuote(new Range(3, 3, 3, 4)), - ]), - ]), - ]); - }); - - test('mixed values', async () => { - const test = disposables.add(new TestFrontMatterDecoder()); - - await test.run( - [ - `tools\v:\t [true , 'toolName', some-tool]`, - ], - [ - // first record - new FrontMatterRecord([ - new FrontMatterRecordName([ - new Word(new Range(1, 1, 1, 1 + 5), 'tools'), - ]), - new FrontMatterRecordDelimiter([ - new Colon(new Range(1, 7, 1, 8)), - new Tab(new Range(1, 8, 1, 9)), - ]), - new FrontMatterArray([ - new LeftBracket(new Range(1, 10, 1, 11)), - // first array value - new FrontMatterBoolean( - new Word( - new Range(1, 11, 1, 11 + 4), - 'true', - ), - ), - // second array value - new FrontMatterString([ - new Quote(new Range(1, 18, 1, 19)), - new Word(new Range(1, 19, 1, 19 + 8), 'toolName'), - new Quote(new Range(1, 27, 1, 28)), - ]), - // third array value - new FrontMatterSequence([ - new Word(new Range(1, 30, 1, 30 + 4), 'some'), - new Dash(new Range(1, 34, 1, 35)), - new Word(new Range(1, 35, 1, 35 + 4), 'tool'), - ]), - new RightBracket(new Range(1, 39, 1, 40)), - ]), - ]), - ]); - }); - - test('redundant commas', async () => { - const test = disposables.add(new TestFrontMatterDecoder()); - - await test.run( - [ - `tools\v:\t [true ,, 'toolName', , , some-tool ,]`, - ], - [ - // first record - new FrontMatterRecord([ - new FrontMatterRecordName([ - new Word(new Range(1, 1, 1, 1 + 5), 'tools'), - ]), - new FrontMatterRecordDelimiter([ - new Colon(new Range(1, 7, 1, 8)), - new Tab(new Range(1, 8, 1, 9)), - ]), - new FrontMatterArray([ - new LeftBracket(new Range(1, 10, 1, 11)), - // first array value - new FrontMatterBoolean( - new Word( - new Range(1, 11, 1, 11 + 4), - 'true', - ), - ), - // second array value - new FrontMatterString([ - new Quote(new Range(1, 19, 1, 20)), - new Word(new Range(1, 20, 1, 20 + 8), 'toolName'), - new Quote(new Range(1, 28, 1, 29)), - ]), - // third array value - new FrontMatterSequence([ - new Word(new Range(1, 35, 1, 35 + 4), 'some'), - new Dash(new Range(1, 39, 1, 40)), - new Word(new Range(1, 40, 1, 40 + 4), 'tool'), - ]), - new RightBracket(new Range(1, 47, 1, 48)), - ]), - ]), - ]); - }); - }); - }); - }); - - test('empty', async () => { - const test = disposables.add( - new TestFrontMatterDecoder(), - ); - - await test.run('', []); - }); -}); diff --git a/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/frontMatterDecoder/frontMatterRecord.test.ts b/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/frontMatterDecoder/frontMatterRecord.test.ts deleted file mode 100644 index fba57e45209..00000000000 --- a/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/frontMatterDecoder/frontMatterRecord.test.ts +++ /dev/null @@ -1,183 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import assert from 'assert'; -import { Range } from '../../../../../../../../../editor/common/core/range.js'; -import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../../../../../../base/test/common/utils.js'; -import { FrontMatterSequence } from '../../../../../../common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterSequence.js'; -import { Colon, LeftBracket, Quote, RightBracket, Space, Tab, VerticalTab, Word } from '../../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/tokens.js'; -import { FrontMatterArray, FrontMatterBoolean, FrontMatterRecord, FrontMatterRecordDelimiter, FrontMatterRecordName, FrontMatterString } from '../../../../../../common/promptSyntax/codecs/base/frontMatterCodec/tokens/index.js'; - -suite('FrontMatterBoolean', () => { - ensureNoDisposablesAreLeakedInTestSuite(); - - suite('trimValueEnd()', () => { - test('trims space tokens at the end of record\'s value', () => { - const recordName = new FrontMatterRecordName([ - new Word( - new Range(4, 10, 4, 10 + 3), - 'key', - ), - ]); - - const recordDelimiter = new FrontMatterRecordDelimiter([ - new Colon(new Range(4, 14, 4, 15)), - new VerticalTab(new Range(4, 15, 4, 16)), - ]); - - const recordValue = new FrontMatterSequence([ - new Word(new Range(4, 18, 4, 18 + 10), 'some-value'), - new VerticalTab(new Range(4, 28, 4, 29)), - new Tab(new Range(4, 29, 4, 30)), - new Space(new Range(4, 30, 4, 31)), - new Tab(new Range(4, 31, 4, 32)), - ]); - - const record = new FrontMatterRecord([ - recordName, recordDelimiter, recordValue, - ]); - - const trimmed = record.trimValueEnd(); - assert.deepStrictEqual( - trimmed, - [ - new VerticalTab(new Range(4, 28, 4, 29)), - new Tab(new Range(4, 29, 4, 30)), - new Space(new Range(4, 30, 4, 31)), - new Tab(new Range(4, 31, 4, 32)), - ], - 'Must return correct trimmed list of spacing tokens.', - ); - - assert( - record.range.equalsRange( - new Range(4, 10, 4, 28), - ), - 'Must correctly update token range.', - ); - }); - - suite('does not trim non-sequence value tokens', () => { - test('boolean', () => { - const recordName = new FrontMatterRecordName([ - new Word( - new Range(4, 10, 4, 10 + 3), - 'yke', - ), - ]); - - const recordDelimiter = new FrontMatterRecordDelimiter([ - new Colon(new Range(4, 14, 4, 15)), - new VerticalTab(new Range(4, 15, 4, 16)), - ]); - - const recordValue = new FrontMatterBoolean( - new Word(new Range(4, 18, 4, 18 + 4), 'true'), - ); - - const record = new FrontMatterRecord([ - recordName, recordDelimiter, recordValue, - ]); - - const trimmed = record.trimValueEnd(); - assert.deepStrictEqual( - trimmed, - [], - 'Must return empty list of trimmed spacing tokens.', - ); - - assert( - record.range.equalsRange( - new Range(4, 10, 4, 22), - ), - 'Must not update token range.', - ); - }); - - test('quoted string', () => { - const recordName = new FrontMatterRecordName([ - new Word( - new Range(4, 10, 4, 10 + 3), - 'eyk', - ), - ]); - - const recordDelimiter = new FrontMatterRecordDelimiter([ - new Colon(new Range(4, 14, 4, 15)), - new VerticalTab(new Range(4, 15, 4, 16)), - ]); - - const recordValue = new FrontMatterString([ - new Quote(new Range(4, 18, 4, 19)), - new Word(new Range(4, 19, 4, 19 + 10), 'some text'), - new Quote(new Range(4, 29, 4, 30)), - ]); - - const record = new FrontMatterRecord([ - recordName, recordDelimiter, recordValue, - ]); - - const trimmed = record.trimValueEnd(); - assert.deepStrictEqual( - trimmed, - [], - 'Must return empty list of trimmed spacing tokens.', - ); - - assert( - record.range.equalsRange( - new Range(4, 10, 4, 30), - ), - 'Must not update token range.', - ); - }); - - test('array', () => { - const recordName = new FrontMatterRecordName([ - new Word( - new Range(4, 10, 4, 10 + 3), - 'yek', - ), - ]); - - const recordDelimiter = new FrontMatterRecordDelimiter([ - new Colon(new Range(4, 14, 4, 15)), - new VerticalTab(new Range(4, 15, 4, 16)), - ]); - - const recordValue = new FrontMatterArray([ - new LeftBracket(new Range(4, 18, 4, 19)), - new FrontMatterString([ - new Quote(new Range(4, 18, 4, 19)), - new Word(new Range(4, 19, 4, 19 + 10), 'some text'), - new Quote(new Range(4, 29, 4, 30)), - ]), - new FrontMatterBoolean( - new Word(new Range(4, 34, 4, 34 + 4), 'true'), - ), - new RightBracket(new Range(4, 38, 4, 39)), - ]); - - const record = new FrontMatterRecord([ - recordName, recordDelimiter, recordValue, - ]); - - const trimmed = record.trimValueEnd(); - assert.deepStrictEqual( - trimmed, - [], - 'Must return empty list of trimmed spacing tokens.', - ); - - assert( - record.range.equalsRange( - new Range(4, 10, 4, 39), - ), - 'Must not update token range.', - ); - }); - }); - }); -}); diff --git a/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/frontMatterDecoder/frontMatterSequence.test.ts b/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/frontMatterDecoder/frontMatterSequence.test.ts deleted file mode 100644 index 68602622bba..00000000000 --- a/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/frontMatterDecoder/frontMatterSequence.test.ts +++ /dev/null @@ -1,106 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import assert from 'assert'; -import { Range } from '../../../../../../../../../editor/common/core/range.js'; -import { FrontMatterValueToken } from '../../../../../../common/promptSyntax/codecs/base/frontMatterCodec/tokens/index.js'; -import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../../../../../../base/test/common/utils.js'; -import { Space, Tab, VerticalTab, Word } from '../../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/tokens.js'; -import { FrontMatterSequence } from '../../../../../../common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterSequence.js'; - -suite('FrontMatterSequence', () => { - ensureNoDisposablesAreLeakedInTestSuite(); - - test('extends \'FrontMatterValueToken\'', () => { - const sequence = new FrontMatterSequence([ - new Word( - new Range(1, 1, 1, 5), - 'test', - ), - ]); - - assert( - sequence instanceof FrontMatterValueToken, - 'Must extend FrontMatterValueToken class.', - ); - }); - - suite('trimEnd()', () => { - test('trims space tokens at the end of the sequence', () => { - const sequence = new FrontMatterSequence([ - new Word(new Range(4, 18, 4, 18 + 10), 'some-value'), - new Space(new Range(4, 28, 4, 29)), - new Space(new Range(4, 29, 4, 30)), - new VerticalTab(new Range(4, 30, 4, 31)), - new Tab(new Range(4, 31, 4, 32)), - new Space(new Range(4, 32, 4, 33)), - ]); - - const trimmed = sequence.trimEnd(); - assert.deepStrictEqual( - trimmed, - [ - new Space(new Range(4, 28, 4, 29)), - new Space(new Range(4, 29, 4, 30)), - new VerticalTab(new Range(4, 30, 4, 31)), - new Tab(new Range(4, 31, 4, 32)), - new Space(new Range(4, 32, 4, 33)), - ], - 'Must return correct trimmed list of spacing tokens.', - ); - - assert( - sequence.range.equalsRange( - new Range(4, 18, 4, 28), - ), - 'Must correctly update token range.', - ); - }); - - test('remains functional if only spacing tokens were present', () => { - const sequence = new FrontMatterSequence([ - new Space(new Range(4, 28, 4, 29)), - new Space(new Range(4, 29, 4, 30)), - new VerticalTab(new Range(4, 30, 4, 31)), - new Tab(new Range(4, 31, 4, 32)), - new Space(new Range(4, 32, 4, 33)), - ]); - - const trimmed = sequence.trimEnd(); - assert.deepStrictEqual( - trimmed, - [ - new Space(new Range(4, 28, 4, 29)), - new Space(new Range(4, 29, 4, 30)), - new VerticalTab(new Range(4, 30, 4, 31)), - new Tab(new Range(4, 31, 4, 32)), - new Space(new Range(4, 32, 4, 33)), - ], - 'Must return correct trimmed list of spacing tokens.', - ); - - assert( - sequence.range.equalsRange( - new Range(4, 28, 4, 28), - ), - 'Must correctly update token range.', - ); - - assert.deepStrictEqual( - sequence.children, - [ - new Word(new Range(4, 28, 4, 28), ''), - ], - 'Must contain a single empty token.', - ); - }); - }); - - test('throws if no tokens provided', () => { - assert.throws(() => { - new FrontMatterSequence([]); - }); - }); -}); diff --git a/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/linesDecoder.test.ts b/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/linesDecoder.test.ts deleted file mode 100644 index b9ac4916923..00000000000 --- a/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/linesDecoder.test.ts +++ /dev/null @@ -1,256 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import assert from 'assert'; -import { Range } from '../../../../../../../../editor/common/core/range.js'; -import { VSBuffer } from '../../../../../../../../base/common/buffer.js'; -import { DisposableStore } from '../../../../../../../../base/common/lifecycle.js'; -import { Line } from '../../../../../common/promptSyntax/codecs/base/linesCodec/tokens/line.js'; -import { TestDecoder, TTokensConsumeMethod } from './utils/testDecoder.js'; -import { NewLine } from '../../../../../common/promptSyntax/codecs/base/linesCodec/tokens/newLine.js'; -import { newWriteableStream, WriteableStream } from '../../../../../../../../base/common/stream.js'; -import { CarriageReturn } from '../../../../../common/promptSyntax/codecs/base/linesCodec/tokens/carriageReturn.js'; -import { LinesDecoder, TLineToken } from '../../../../../common/promptSyntax/codecs/base/linesCodec/linesDecoder.js'; -import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../../../../../base/test/common/utils.js'; - -/** - * Note! This decoder is also often used to test common logic of abstract {@link BaseDecoder} - * class, because the {@link LinesDecoder} is one of the simplest non-abstract decoders we have. - */ -suite('LinesDecoder', () => { - const disposables = ensureNoDisposablesAreLeakedInTestSuite(); - - /** - * Test the core logic with specific method of consuming - * tokens that are produced by a lines decoder instance. - */ - suite('core logic', () => { - testLinesDecoder('async-generator', disposables); - testLinesDecoder('consume-all-method', disposables); - testLinesDecoder('on-data-event', disposables); - }); - - suite('settled promise', () => { - test('throws if accessed on not-yet-started decoder instance', () => { - const test = disposables.add(new TestLinesDecoder()); - - assert.throws( - () => { - // testing the field access that throws here, so - // its OK to not use the returned value afterwards - // eslint-disable-next-line local/code-no-unused-expressions - test.decoder.settled; - }, - [ - 'Cannot get `settled` promise of a stream that has not been started.', - 'Please call `start()` first.', - ].join(' '), - ); - }); - }); - - suite('start', () => { - test('throws if the decoder object is already `disposed`', () => { - const test = disposables.add(new TestLinesDecoder()); - const { decoder } = test; - decoder.dispose(); - - assert.throws( - decoder.start.bind(decoder), - 'Cannot start stream that has already disposed.', - ); - }); - - test('throws if the decoder object is already `ended`', async () => { - const inputStream = newWriteableStream(null); - const test = disposables.add(new TestLinesDecoder(inputStream)); - const { decoder } = test; - - setTimeout(() => { - test.sendData([ - 'hello', - 'world :wave:', - ]); - }, 5); - - const receivedTokens = await decoder.start() - .consumeAll(); - - // a basic sanity check for received tokens - assert.strictEqual( - receivedTokens.length, - 3, - 'Must produce the correct number of tokens.', - ); - - // validate that calling `start()` after stream has ended throws - assert.throws( - decoder.start.bind(decoder), - 'Cannot start stream that has already ended.', - ); - }); - }); -}); - - -/** - * A reusable test utility that asserts that a `LinesDecoder` instance - * correctly decodes `inputData` into a stream of `TLineToken` tokens. - * - * ## Examples - * - * ```typescript - * // create a new test utility instance - * const test = disposables.add(new TestLinesDecoder()); - * - * // run the test - * await test.run( - * ' hello world\n', - * [ - * new Line(1, ' hello world'), - * new NewLine(new Range(1, 13, 1, 14)), - * ], - * ); - */ -export class TestLinesDecoder extends TestDecoder { - constructor( - inputStream?: WriteableStream, - ) { - const stream = (inputStream) - ? inputStream - : newWriteableStream(null); - - const decoder = new LinesDecoder(stream); - - super(stream, decoder); - } -} - -/** - * Common reusable test utility to validate {@link LinesDecoder} logic with - * the provided {@link tokensConsumeMethod} way of consuming decoder-produced tokens. - * - * @throws if a test fails, please see thrown error for failure details. - * @param tokensConsumeMethod The way to consume tokens produced by the decoder. - * @param disposables Test disposables store. - */ -function testLinesDecoder( - tokensConsumeMethod: TTokensConsumeMethod, - disposables: Pick, -) { - suite(tokensConsumeMethod, () => { - suite('produces expected tokens', () => { - test('input starts with line data', async () => { - const test = disposables.add(new TestLinesDecoder()); - - await test.run( - ' hello world\nhow are you doing?\n\n 😊 \r', - [ - new Line(1, ' hello world'), - new NewLine(new Range(1, 13, 1, 14)), - new Line(2, 'how are you doing?'), - new NewLine(new Range(2, 19, 2, 20)), - new Line(3, ''), - new NewLine(new Range(3, 1, 3, 2)), - new Line(4, ' 😊 '), - new NewLine(new Range(4, 5, 4, 6)), - ], - ); - }); - - test('standalone \\r is treated as new line', async () => { - const test = disposables.add(new TestLinesDecoder()); - - await test.run( - ' hello world\nhow are you doing?\n\n 😊 \r ', - [ - new Line(1, ' hello world'), - new NewLine(new Range(1, 13, 1, 14)), - new Line(2, 'how are you doing?'), - new NewLine(new Range(2, 19, 2, 20)), - new Line(3, ''), - new NewLine(new Range(3, 1, 3, 2)), - new Line(4, ' 😊 '), - new NewLine(new Range(4, 5, 4, 6)), - new Line(5, ' '), - ], - ); - }); - - test('input starts with a new line', async () => { - const test = disposables.add(new TestLinesDecoder()); - - await test.run( - '\nsome text on this line\n\n\nanother 💬 on this line\r\n🤫\n', - [ - new Line(1, ''), - new NewLine(new Range(1, 1, 1, 2)), - new Line(2, 'some text on this line'), - new NewLine(new Range(2, 23, 2, 24)), - new Line(3, ''), - new NewLine(new Range(3, 1, 3, 2)), - new Line(4, ''), - new NewLine(new Range(4, 1, 4, 2)), - new Line(5, 'another 💬 on this line'), - new CarriageReturn(new Range(5, 24, 5, 25)), - new NewLine(new Range(5, 25, 5, 26)), - new Line(6, '🤫'), - new NewLine(new Range(6, 3, 6, 4)), - ], - ); - }); - - test('input starts and ends with multiple new lines', async () => { - const test = disposables.add(new TestLinesDecoder()); - - await test.run( - '\n\n\r\nciao! 🗯️\t💭 💥 come\tva?\n\n\n\n\n', - [ - new Line(1, ''), - new NewLine(new Range(1, 1, 1, 2)), - new Line(2, ''), - new NewLine(new Range(2, 1, 2, 2)), - new Line(3, ''), - new CarriageReturn(new Range(3, 1, 3, 2)), - new NewLine(new Range(3, 2, 3, 3)), - new Line(4, 'ciao! 🗯️\t💭 💥 come\tva?'), - new NewLine(new Range(4, 25, 4, 26)), - new Line(5, ''), - new NewLine(new Range(5, 1, 5, 2)), - new Line(6, ''), - new NewLine(new Range(6, 1, 6, 2)), - new Line(7, ''), - new NewLine(new Range(7, 1, 7, 2)), - new Line(8, ''), - new NewLine(new Range(8, 1, 8, 2)), - ], - ); - }); - - test('single carriage return is treated as new line', async () => { - const test = disposables.add(new TestLinesDecoder()); - - await test.run( - '\r\rhaalo! 💥💥 how\'re you?\r ?!\r\n\r\n ', - [ - new Line(1, ''), - new NewLine(new Range(1, 1, 1, 2)), - new Line(2, ''), - new NewLine(new Range(2, 1, 2, 2)), - new Line(3, 'haalo! 💥💥 how\'re you?'), - new NewLine(new Range(3, 24, 3, 25)), - new Line(4, ' ?!'), - new CarriageReturn(new Range(4, 4, 4, 5)), - new NewLine(new Range(4, 5, 4, 6)), - new Line(5, ''), - new CarriageReturn(new Range(5, 1, 5, 2)), - new NewLine(new Range(5, 2, 5, 3)), - new Line(6, ' '), - ], - ); - }); - }); - }); -} diff --git a/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/markdownDecoder.test.ts b/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/markdownDecoder.test.ts deleted file mode 100644 index 37a3331f796..00000000000 --- a/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/markdownDecoder.test.ts +++ /dev/null @@ -1,937 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import assert from 'assert'; -import { TestDecoder } from './utils/testDecoder.js'; -import { Range } from '../../../../../../../../editor/common/core/range.js'; -import { VSBuffer } from '../../../../../../../../base/common/buffer.js'; -import { newWriteableStream } from '../../../../../../../../base/common/stream.js'; -import { Tab } from '../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/tab.js'; -import { Word } from '../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/word.js'; -import { Dash } from '../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/dash.js'; -import { Space } from '../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/space.js'; -import { Slash } from '../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/slash.js'; -import { NewLine } from '../../../../../common/promptSyntax/codecs/base/linesCodec/tokens/newLine.js'; -import { FormFeed } from '../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/formFeed.js'; -import { VerticalTab } from '../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/verticalTab.js'; -import { MarkdownLink } from '../../../../../common/promptSyntax/codecs/base/markdownCodec/tokens/markdownLink.js'; -import { CarriageReturn } from '../../../../../common/promptSyntax/codecs/base/linesCodec/tokens/carriageReturn.js'; -import { MarkdownImage } from '../../../../../common/promptSyntax/codecs/base/markdownCodec/tokens/markdownImage.js'; -import { ExclamationMark } from '../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/exclamationMark.js'; -import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../../../../../base/test/common/utils.js'; -import { MarkdownComment } from '../../../../../common/promptSyntax/codecs/base/markdownCodec/tokens/markdownComment.js'; -import { LeftBracket, RightBracket } from '../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/brackets.js'; -import { MarkdownDecoder, TMarkdownToken } from '../../../../../common/promptSyntax/codecs/base/markdownCodec/markdownDecoder.js'; -import { LeftParenthesis, RightParenthesis } from '../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/parentheses.js'; -import { LeftAngleBracket, RightAngleBracket } from '../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/angleBrackets.js'; - -/** - * A reusable test utility that asserts that a `TestMarkdownDecoder` instance - * correctly decodes `inputData` into a stream of `TMarkdownToken` tokens. - * - * ## Examples - * - * ```typescript - * // create a new test utility instance - * const test = testDisposables.add(new TestMarkdownDecoder()); - * - * // run the test - * await test.run( - * ' hello [world](/etc/hosts)!', - * [ - * new Space(new Range(1, 1, 1, 2)), - * new Word(new Range(1, 2, 1, 7), 'hello'), - * new Space(new Range(1, 7, 1, 8)), - * new MarkdownLink(1, 8, '[world]', '(/etc/hosts)'), - * new Word(new Range(1, 27, 1, 28), '!'), - * new NewLine(new Range(1, 28, 1, 29)), - * ], - * ); - */ -export class TestMarkdownDecoder extends TestDecoder { - constructor() { - const stream = newWriteableStream(null); - - super(stream, new MarkdownDecoder(stream)); - } -} - -suite('MarkdownDecoder', () => { - const testDisposables = ensureNoDisposablesAreLeakedInTestSuite(); - - suite('general', () => { - test('base cases', async () => { - const test = testDisposables.add( - new TestMarkdownDecoder(), - ); - - await test.run( - [ - // basic text - ' hello world', - // text with markdown link and special characters in the filename - 'how are\t you [caption text](./some/file/path/refer🎨nce.md)?\v', - // empty line - '', - // markdown link with special characters in the link caption and path - '[(example!)](another/path/with[-and-]-chars/folder)\t ', - // markdown link `#file` variable in the caption and with absolute path - '\t[#file:something.txt](/absolute/path/to/something.txt)', - // text with a commented out markdown link - '\v\f machines must suffer', - ], - [ - // first line - new Space(new Range(1, 1, 1, 2)), - new Word(new Range(1, 2, 1, 7), 'hello'), - new Space(new Range(1, 7, 1, 8)), - new Word(new Range(1, 8, 1, 13), 'world'), - new NewLine(new Range(1, 13, 1, 14)), - // second line - new Word(new Range(2, 1, 2, 4), 'how'), - new Space(new Range(2, 4, 2, 5)), - new Word(new Range(2, 5, 2, 8), 'are'), - new Tab(new Range(2, 8, 2, 9)), - new Space(new Range(2, 9, 2, 10)), - new Word(new Range(2, 10, 2, 13), 'you'), - new Space(new Range(2, 13, 2, 14)), - new MarkdownLink(2, 14, '[caption text]', '(./some/file/path/refer🎨nce.md)'), - new Word(new Range(2, 60, 2, 61), '?'), - new VerticalTab(new Range(2, 61, 2, 62)), - new NewLine(new Range(2, 62, 2, 63)), - // third line - new NewLine(new Range(3, 1, 3, 2)), - // fourth line - new MarkdownLink(4, 1, '[(example!)]', '(another/path/with[-and-]-chars/folder)'), - new Tab(new Range(4, 52, 4, 53)), - new Space(new Range(4, 53, 4, 54)), - new NewLine(new Range(4, 54, 4, 55)), - // fifth line - new Tab(new Range(5, 1, 5, 2)), - new MarkdownLink(5, 2, '[#file:something.txt]', '(/absolute/path/to/something.txt)'), - new NewLine(new Range(5, 56, 5, 57)), - // sixth line - new VerticalTab(new Range(6, 1, 6, 2)), - new FormFeed(new Range(6, 2, 6, 3)), - new Space(new Range(6, 3, 6, 4)), - new Word(new Range(6, 4, 6, 12), 'machines'), - new Space(new Range(6, 12, 6, 13)), - new Word(new Range(6, 13, 6, 17), 'must'), - new Space(new Range(6, 17, 6, 18)), - new MarkdownComment(new Range(6, 18, 6, 18 + 41), ''), - new Space(new Range(6, 59, 6, 60)), - new Word(new Range(6, 60, 6, 66), 'suffer'), - ], - ); - }); - - test('nuanced', async () => { - const test = testDisposables.add( - new TestMarkdownDecoder(), - ); - - const inputLines = [ - // tests that the link caption contain a chat prompt `#file:` reference, while - // the file path can contain other `graphical characters` - '\v\t[#file:./another/path/to/file.txt](./real/file!path/file◆name.md)', - // tests that the link file path contain a chat prompt `#file:` reference, - // `spaces`, `emojies`, and other `graphical characters` - ' [reference ∘ label](/absolute/pa th/to-#file:file.txt/f🥸⚡️le.md)', - // tests that link caption and file path can contain `parentheses`, `spaces`, and - // `emojies` - '\f[!(hello)!](./w(())rld/nice-🦚-filen(a).git))\n\t', - // tests that the link caption can be empty, while the file path can contain `square brackets` - '[](./s[]me/pa[h!) ', - ]; - - await test.run( - inputLines, - [ - // `1st` line - new VerticalTab(new Range(1, 1, 1, 2)), - new Tab(new Range(1, 2, 1, 3)), - new MarkdownLink(1, 3, '[#file:./another/path/to/file.txt]', '(./real/file!path/file◆name.md)'), - new NewLine(new Range(1, 68, 1, 69)), - // `2nd` line - new Space(new Range(2, 1, 2, 2)), - new MarkdownLink(2, 2, '[reference ∘ label]', '(/absolute/pa th/to-#file:file.txt/f🥸⚡️le.md)'), - new NewLine(new Range(2, 67, 2, 68)), - // `3rd` line - new FormFeed(new Range(3, 1, 3, 2)), - new MarkdownLink(3, 2, '[!(hello)!]', '(./w(())rld/nice-🦚-filen(a).git)'), - new RightParenthesis(new Range(3, 50, 3, 51)), - new NewLine(new Range(3, 51, 3, 52)), - // `4th` line - new Tab(new Range(4, 1, 4, 2)), - new NewLine(new Range(4, 2, 4, 3)), - // `5th` line - new MarkdownLink(5, 1, '[]', '(./s[]me/pa[h!)'), - new Space(new Range(5, 24, 5, 25)), - ], - ); - }); - }); - - suite('links', () => { - suite('broken', () => { - test('invalid', async () => { - const test = testDisposables.add( - new TestMarkdownDecoder(), - ); - - const inputLines = [ - // incomplete link reference with empty caption - '[ ](./real/file path/file⇧name.md', - // space between caption and reference is disallowed - '[link text] (./file path/name.txt)', - ]; - - await test.run( - inputLines, - [ - // `1st` line - new LeftBracket(new Range(1, 1, 1, 2)), - new Space(new Range(1, 2, 1, 3)), - new RightBracket(new Range(1, 3, 1, 4)), - new LeftParenthesis(new Range(1, 4, 1, 5)), - new Word(new Range(1, 5, 1, 5 + 1), '.'), - new Slash(new Range(1, 6, 1, 7)), - new Word(new Range(1, 7, 1, 7 + 4), 'real'), - new Slash(new Range(1, 11, 1, 12)), - new Word(new Range(1, 12, 1, 12 + 4), 'file'), - new Space(new Range(1, 16, 1, 17)), - new Word(new Range(1, 17, 1, 17 + 4), 'path'), - new Slash(new Range(1, 21, 1, 22)), - new Word(new Range(1, 22, 1, 22 + 12), 'file⇧name.md'), - new NewLine(new Range(1, 34, 1, 35)), - // `2nd` line - new LeftBracket(new Range(2, 1, 2, 2)), - new Word(new Range(2, 2, 2, 2 + 4), 'link'), - new Space(new Range(2, 6, 2, 7)), - new Word(new Range(2, 7, 2, 7 + 4), 'text'), - new RightBracket(new Range(2, 11, 2, 12)), - new Space(new Range(2, 12, 2, 13)), - new LeftParenthesis(new Range(2, 13, 2, 14)), - new Word(new Range(2, 14, 2, 14 + 1), '.'), - new Slash(new Range(2, 15, 2, 16)), - new Word(new Range(2, 16, 2, 16 + 4), 'file'), - new Space(new Range(2, 20, 2, 21)), - new Word(new Range(2, 21, 2, 21 + 4), 'path'), - new Slash(new Range(2, 25, 2, 26)), - new Word(new Range(2, 26, 2, 26 + 8), 'name.txt'), - new RightParenthesis(new Range(2, 34, 2, 35)), - ], - ); - }); - - suite('stop characters inside caption/reference (new lines)', () => { - for (const StopCharacter of [CarriageReturn, NewLine]) { - let characterName = ''; - - if (StopCharacter === CarriageReturn) { - characterName = '\\r'; - } - if (StopCharacter === NewLine) { - characterName = '\\n'; - } - - assert( - characterName !== '', - 'The "characterName" must be set, got "empty line".', - ); - - test(`stop character - "${characterName}"`, async () => { - const test = testDisposables.add( - new TestMarkdownDecoder(), - ); - - const inputLines = [ - // stop character inside link caption - `[haa${StopCharacter.symbol}loů](./real/💁/name.txt)`, - // stop character inside link reference - `[ref text](/etc/pat${StopCharacter.symbol}h/to/file.md)`, - // stop character between line caption and link reference is disallowed - `[text]${StopCharacter.symbol}(/etc/ path/main.mdc)`, - ]; - - - await test.run( - inputLines, - [ - // `1st` input line - new LeftBracket(new Range(1, 1, 1, 2)), - new Word(new Range(1, 2, 1, 2 + 3), 'haa'), - new NewLine(new Range(1, 5, 1, 6)), // a single CR token is treated as a `new line` - new Word(new Range(2, 1, 2, 1 + 3), 'loů'), - new RightBracket(new Range(2, 4, 2, 5)), - new LeftParenthesis(new Range(2, 5, 2, 6)), - new Word(new Range(2, 6, 2, 6 + 1), '.'), - new Slash(new Range(2, 7, 2, 8)), - new Word(new Range(2, 8, 2, 8 + 4), 'real'), - new Slash(new Range(2, 12, 2, 13)), - new Word(new Range(2, 13, 2, 13 + 2), '💁'), - new Slash(new Range(2, 15, 2, 16)), - new Word(new Range(2, 16, 2, 16 + 8), 'name.txt'), - new RightParenthesis(new Range(2, 24, 2, 25)), - new NewLine(new Range(2, 25, 2, 26)), - // `2nd` input line - new LeftBracket(new Range(3, 1, 3, 2)), - new Word(new Range(3, 2, 3, 2 + 3), 'ref'), - new Space(new Range(3, 5, 3, 6)), - new Word(new Range(3, 6, 3, 6 + 4), 'text'), - new RightBracket(new Range(3, 10, 3, 11)), - new LeftParenthesis(new Range(3, 11, 3, 12)), - new Slash(new Range(3, 12, 3, 13)), - new Word(new Range(3, 13, 3, 13 + 3), 'etc'), - new Slash(new Range(3, 16, 3, 17)), - new Word(new Range(3, 17, 3, 17 + 3), 'pat'), - new NewLine(new Range(3, 20, 3, 21)), // a single CR token is treated as a `new line` - new Word(new Range(4, 1, 4, 1 + 1), 'h'), - new Slash(new Range(4, 2, 4, 3)), - new Word(new Range(4, 3, 4, 3 + 2), 'to'), - new Slash(new Range(4, 5, 4, 6)), - new Word(new Range(4, 6, 4, 6 + 7), 'file.md'), - new RightParenthesis(new Range(4, 13, 4, 14)), - new NewLine(new Range(4, 14, 4, 15)), - // `3nd` input line - new LeftBracket(new Range(5, 1, 5, 2)), - new Word(new Range(5, 2, 5, 2 + 4), 'text'), - new RightBracket(new Range(5, 6, 5, 7)), - new NewLine(new Range(5, 7, 5, 8)), // a single CR token is treated as a `new line` - new LeftParenthesis(new Range(6, 1, 6, 2)), - new Slash(new Range(6, 2, 6, 3)), - new Word(new Range(6, 3, 6, 3 + 3), 'etc'), - new Slash(new Range(6, 6, 6, 7)), - new Space(new Range(6, 7, 6, 8)), - new Word(new Range(6, 8, 6, 8 + 4), 'path'), - new Slash(new Range(6, 12, 6, 13)), - new Word(new Range(6, 13, 6, 13 + 8), 'main.mdc'), - new RightParenthesis(new Range(6, 21, 6, 22)), - ], - ); - }); - } - }); - - /** - * Same as above but these stop characters do not move the caret to the next line. - */ - suite('stop characters inside caption/reference (same line)', () => { - for (const StopCharacter of [VerticalTab, FormFeed]) { - let characterName = ''; - - if (StopCharacter === VerticalTab) { - characterName = '\\v'; - } - if (StopCharacter === FormFeed) { - characterName = '\\f'; - } - - assert( - characterName !== '', - 'The "characterName" must be set, got "empty line".', - ); - - test(`stop character - "${characterName}"`, async () => { - const test = testDisposables.add( - new TestMarkdownDecoder(), - ); - - const inputLines = [ - // stop character inside link caption - `[haa${StopCharacter.symbol}loů](./real/💁/name.txt)`, - // stop character inside link reference - `[ref text](/etc/pat${StopCharacter.symbol}h/to/file.md)`, - // stop character between line caption and link reference is disallowed - `[text]${StopCharacter.symbol}(/etc/ path/file.md)`, - ]; - - - await test.run( - inputLines, - [ - // `1st` input line - new LeftBracket(new Range(1, 1, 1, 2)), - new Word(new Range(1, 2, 1, 2 + 3), 'haa'), - new StopCharacter(new Range(1, 5, 1, 6)), // <- stop character - new Word(new Range(1, 6, 1, 6 + 3), 'loů'), - new RightBracket(new Range(1, 9, 1, 10)), - new LeftParenthesis(new Range(1, 10, 1, 11)), - new Word(new Range(1, 11, 1, 11 + 1), '.'), - new Slash(new Range(1, 12, 1, 13)), - new Word(new Range(1, 13, 1, 13 + 4), 'real'), - new Slash(new Range(1, 17, 1, 18)), - new Word(new Range(1, 18, 1, 18 + 2), '💁'), - new Slash(new Range(1, 20, 1, 21)), - new Word(new Range(1, 21, 1, 21 + 8), 'name.txt'), - new RightParenthesis(new Range(1, 29, 1, 30)), - new NewLine(new Range(1, 30, 1, 31)), - // `2nd` input line - new LeftBracket(new Range(2, 1, 2, 2)), - new Word(new Range(2, 2, 2, 2 + 3), 'ref'), - new Space(new Range(2, 5, 2, 6)), - new Word(new Range(2, 6, 2, 6 + 4), 'text'), - new RightBracket(new Range(2, 10, 2, 11)), - new LeftParenthesis(new Range(2, 11, 2, 12)), - new Slash(new Range(2, 12, 2, 13)), - new Word(new Range(2, 13, 2, 13 + 3), 'etc'), - new Slash(new Range(2, 16, 2, 17)), - new Word(new Range(2, 17, 2, 17 + 3), 'pat'), - new StopCharacter(new Range(2, 20, 2, 21)), // <- stop character - new Word(new Range(2, 21, 2, 21 + 1), 'h'), - new Slash(new Range(2, 22, 2, 23)), - new Word(new Range(2, 23, 2, 23 + 2), 'to'), - new Slash(new Range(2, 25, 2, 26)), - new Word(new Range(2, 26, 2, 26 + 7), 'file.md'), - new RightParenthesis(new Range(2, 33, 2, 34)), - new NewLine(new Range(2, 34, 2, 35)), - // `3nd` input line - new LeftBracket(new Range(3, 1, 3, 2)), - new Word(new Range(3, 2, 3, 2 + 4), 'text'), - new RightBracket(new Range(3, 6, 3, 7)), - new StopCharacter(new Range(3, 7, 3, 8)), // <- stop character - new LeftParenthesis(new Range(3, 8, 3, 9)), - new Slash(new Range(3, 9, 3, 10)), - new Word(new Range(3, 10, 3, 10 + 3), 'etc'), - new Slash(new Range(3, 13, 3, 14)), - new Space(new Range(3, 14, 3, 15)), - new Word(new Range(3, 15, 3, 15 + 4), 'path'), - new Slash(new Range(3, 19, 3, 20)), - new Word(new Range(3, 20, 3, 20 + 7), 'file.md'), - new RightParenthesis(new Range(3, 27, 3, 28)), - ], - ); - }); - } - }); - }); - }); - - - suite('images', () => { - suite('general', () => { - test('base cases', async () => { - const test = testDisposables.add( - new TestMarkdownDecoder(), - ); - - const inputData = [ - '\t![alt text](./some/path/to/file.jpg) ', - 'plain text \f![label](./image.png)\v and more text', - '![](/var/images/default) following text', - ]; - - await test.run( - inputData, - [ - // `1st` - new Tab(new Range(1, 1, 1, 2)), - new MarkdownImage(1, 2, '![alt text]', '(./some/path/to/file.jpg)'), - new Space(new Range(1, 38, 1, 39)), - new NewLine(new Range(1, 39, 1, 40)), - // `2nd` - new Word(new Range(2, 1, 2, 6), 'plain'), - new Space(new Range(2, 6, 2, 7)), - new Word(new Range(2, 7, 2, 11), 'text'), - new Space(new Range(2, 11, 2, 12)), - new FormFeed(new Range(2, 12, 2, 13)), - new MarkdownImage(2, 13, '![label]', '(./image.png)'), - new VerticalTab(new Range(2, 34, 2, 35)), - new Space(new Range(2, 35, 2, 36)), - new Word(new Range(2, 36, 2, 39), 'and'), - new Space(new Range(2, 39, 2, 40)), - new Word(new Range(2, 40, 2, 44), 'more'), - new Space(new Range(2, 44, 2, 45)), - new Word(new Range(2, 45, 2, 49), 'text'), - new NewLine(new Range(2, 49, 2, 50)), - // `3rd` - new MarkdownImage(3, 1, '![]', '(/var/images/default)'), - new Space(new Range(3, 25, 3, 26)), - new Word(new Range(3, 26, 3, 35), 'following'), - new Space(new Range(3, 35, 3, 36)), - new Word(new Range(3, 36, 3, 40), 'text'), - ], - ); - }); - - test('nuanced', async () => { - const test = testDisposables.add( - new TestMarkdownDecoder(), - ); - - const inputData = [ - '\t![](./s☻me/path/to/file.jpeg) ', - 'raw text \f![(/1.png)](./image-🥸.png)\v and more text', - // '![](/var/images/default) following text', - ]; - - await test.run( - inputData, - [ - // `1st` - new Tab(new Range(1, 1, 1, 2)), - new MarkdownImage(1, 2, '![]', '(./s☻me/path/to/file.jpeg)'), - new Space(new Range(1, 47, 1, 48)), - new NewLine(new Range(1, 48, 1, 49)), - // `2nd` - new Word(new Range(2, 1, 2, 4), 'raw'), - new Space(new Range(2, 4, 2, 5)), - new Word(new Range(2, 5, 2, 9), 'text'), - new Space(new Range(2, 9, 2, 10)), - new FormFeed(new Range(2, 10, 2, 11)), - new MarkdownImage(2, 11, '![(/1.png)]', '(./image-🥸.png)'), - new VerticalTab(new Range(2, 38, 2, 39)), - new Space(new Range(2, 39, 2, 40)), - new Word(new Range(2, 40, 2, 43), 'and'), - new Space(new Range(2, 43, 2, 44)), - new Word(new Range(2, 44, 2, 48), 'more'), - new Space(new Range(2, 48, 2, 49)), - new Word(new Range(2, 49, 2, 53), 'text'), - ], - ); - }); - }); - - suite('broken', () => { - test('invalid', async () => { - const test = testDisposables.add( - new TestMarkdownDecoder(), - ); - - const inputLines = [ - // incomplete link reference with empty caption - '![ ](./real/file path/file★name.webp', - // space between caption and reference is disallowed - '\f![link text] (./file path/name.jpg)', - // new line inside the link reference - '\v![ ](./file\npath/name.jpeg )', - ]; - - await test.run( - inputLines, - [ - // `1st` line - new ExclamationMark(new Range(1, 1, 1, 2)), - new LeftBracket(new Range(1, 2, 1, 3)), - new Space(new Range(1, 3, 1, 4)), - new RightBracket(new Range(1, 4, 1, 5)), - new LeftParenthesis(new Range(1, 5, 1, 6)), - new Word(new Range(1, 6, 1, 6 + 1), '.'), - new Slash(new Range(1, 7, 1, 8)), - new Word(new Range(1, 8, 1, 8 + 4), 'real'), - new Slash(new Range(1, 12, 1, 13)), - new Word(new Range(1, 13, 1, 13 + 4), 'file'), - new Space(new Range(1, 17, 1, 18)), - new Word(new Range(1, 18, 1, 18 + 4), 'path'), - new Slash(new Range(1, 22, 1, 23)), - new Word(new Range(1, 23, 1, 23 + 14), 'file★name.webp'), - new NewLine(new Range(1, 37, 1, 38)), - // `2nd` line - new FormFeed(new Range(2, 1, 2, 2)), - new ExclamationMark(new Range(2, 2, 2, 3)), - new LeftBracket(new Range(2, 3, 2, 4)), - new Word(new Range(2, 4, 2, 4 + 4), 'link'), - new Space(new Range(2, 8, 2, 9)), - new Word(new Range(2, 9, 2, 9 + 4), 'text'), - new RightBracket(new Range(2, 13, 2, 14)), - new Space(new Range(2, 14, 2, 15)), - new LeftParenthesis(new Range(2, 15, 2, 16)), - new Word(new Range(2, 16, 2, 16 + 1), '.'), - new Slash(new Range(2, 17, 2, 18)), - new Word(new Range(2, 18, 2, 18 + 4), 'file'), - new Space(new Range(2, 22, 2, 23)), - new Word(new Range(2, 23, 2, 23 + 4), 'path'), - new Slash(new Range(2, 27, 2, 28)), - new Word(new Range(2, 28, 2, 28 + 8), 'name.jpg'), - new RightParenthesis(new Range(2, 36, 2, 37)), - new NewLine(new Range(2, 37, 2, 38)), - // `3rd` line - new VerticalTab(new Range(3, 1, 3, 2)), - new ExclamationMark(new Range(3, 2, 3, 3)), - new LeftBracket(new Range(3, 3, 3, 4)), - new Space(new Range(3, 4, 3, 5)), - new RightBracket(new Range(3, 5, 3, 6)), - new LeftParenthesis(new Range(3, 6, 3, 7)), - new Word(new Range(3, 7, 3, 7 + 1), '.'), - new Slash(new Range(3, 8, 3, 9)), - new Word(new Range(3, 9, 3, 9 + 4), 'file'), - new NewLine(new Range(3, 13, 3, 14)), - new Word(new Range(4, 1, 4, 1 + 4), 'path'), - new Slash(new Range(4, 5, 4, 6)), - new Word(new Range(4, 6, 4, 6 + 9), 'name.jpeg'), - new Space(new Range(4, 15, 4, 16)), - new RightParenthesis(new Range(4, 16, 4, 17)), - ], - ); - }); - - suite('stop characters inside caption/reference (new lines)', () => { - for (const StopCharacter of [CarriageReturn, NewLine]) { - let characterName = ''; - - if (StopCharacter === CarriageReturn) { - characterName = '\\r'; - } - if (StopCharacter === NewLine) { - characterName = '\\n'; - } - - assert( - characterName !== '', - 'The "characterName" must be set, got "empty line".', - ); - - test(`stop character - "${characterName}"`, async () => { - const test = testDisposables.add( - new TestMarkdownDecoder(), - ); - - const inputLines = [ - // stop character inside link caption - `![haa${StopCharacter.symbol}loů](./real/💁/name.png)`, - // stop character inside link reference - `![ref text](/etc/pat${StopCharacter.symbol}h/to/file.webp)`, - // stop character between line caption and link reference is disallowed - `![text]${StopCharacter.symbol}(/etc/ path/file.jpeg)`, - ]; - - - await test.run( - inputLines, - [ - // `1st` input line - new ExclamationMark(new Range(1, 1, 1, 2)), - new LeftBracket(new Range(1, 2, 1, 3)), - new Word(new Range(1, 3, 1, 3 + 3), 'haa'), - new NewLine(new Range(1, 6, 1, 7)), // a single CR token is treated as a `new line` - new Word(new Range(2, 1, 2, 1 + 3), 'loů'), - new RightBracket(new Range(2, 4, 2, 5)), - new LeftParenthesis(new Range(2, 5, 2, 6)), - new Word(new Range(2, 6, 2, 6 + 1), '.'), - new Slash(new Range(2, 7, 2, 8)), - new Word(new Range(2, 8, 2, 8 + 4), 'real'), - new Slash(new Range(2, 12, 2, 13)), - new Word(new Range(2, 13, 2, 13 + 2), '💁'), - new Slash(new Range(2, 15, 2, 16)), - new Word(new Range(2, 16, 2, 16 + 8), 'name.png'), - new RightParenthesis(new Range(2, 24, 2, 25)), - new NewLine(new Range(2, 25, 2, 26)), - // `2nd` input line - new ExclamationMark(new Range(3, 1, 3, 2)), - new LeftBracket(new Range(3, 2, 3, 3)), - new Word(new Range(3, 3, 3, 3 + 3), 'ref'), - new Space(new Range(3, 6, 3, 7)), - new Word(new Range(3, 7, 3, 7 + 4), 'text'), - new RightBracket(new Range(3, 11, 3, 12)), - new LeftParenthesis(new Range(3, 12, 3, 13)), - new Slash(new Range(3, 13, 3, 14)), - new Word(new Range(3, 14, 3, 14 + 3), 'etc'), - new Slash(new Range(3, 17, 3, 18)), - new Word(new Range(3, 18, 3, 18 + 3), 'pat'), - new NewLine(new Range(3, 21, 3, 22)), // a single CR token is treated as a `new line` - new Word(new Range(4, 1, 4, 1 + 1), 'h'), - new Slash(new Range(4, 2, 4, 3)), - new Word(new Range(4, 3, 4, 3 + 2), 'to'), - new Slash(new Range(4, 5, 4, 6)), - new Word(new Range(4, 6, 4, 6 + 9), 'file.webp'), - new RightParenthesis(new Range(4, 15, 4, 16)), - new NewLine(new Range(4, 16, 4, 17)), - // `3nd` input line - new ExclamationMark(new Range(5, 1, 5, 2)), - new LeftBracket(new Range(5, 2, 5, 3)), - new Word(new Range(5, 3, 5, 3 + 4), 'text'), - new RightBracket(new Range(5, 7, 5, 8)), - new NewLine(new Range(5, 8, 5, 9)), // a single CR token is treated as a `new line` - new LeftParenthesis(new Range(6, 1, 6, 2)), - new Slash(new Range(6, 2, 6, 3)), - new Word(new Range(6, 3, 6, 3 + 3), 'etc'), - new Slash(new Range(6, 6, 6, 7)), - new Space(new Range(6, 7, 6, 8)), - new Word(new Range(6, 8, 6, 8 + 4), 'path'), - new Slash(new Range(6, 12, 6, 13)), - new Word(new Range(6, 13, 6, 13 + 9), 'file.jpeg'), - new RightParenthesis(new Range(6, 22, 6, 23)), - ], - ); - }); - } - }); - - /** - * Same as above but these stop characters do not move the caret to the next line. - */ - suite('stop characters inside caption/reference (same line)', () => { - for (const stopCharacter of [VerticalTab, FormFeed]) { - let characterName = ''; - - if (stopCharacter === VerticalTab) { - characterName = '\\v'; - } - if (stopCharacter === FormFeed) { - characterName = '\\f'; - } - - assert( - characterName !== '', - 'The "characterName" must be set, got "empty line".', - ); - - test(`stop character - "${characterName}"`, async () => { - const test = testDisposables.add( - new TestMarkdownDecoder(), - ); - - const inputLines = [ - // stop character inside link caption - `![haa${stopCharacter.symbol}loů](./real/💁/name)`, - // stop character inside link reference - `![ref text](/etc/pat${stopCharacter.symbol}h/to/file.webp)`, - // stop character between line caption and link reference is disallowed - `![text]${stopCharacter.symbol}(/etc/ path/image.gif)`, - ]; - - - await test.run( - inputLines, - [ - // `1st` input line - new ExclamationMark(new Range(1, 1, 1, 2)), - new LeftBracket(new Range(1, 2, 1, 3)), - new Word(new Range(1, 3, 1, 3 + 3), 'haa'), - new stopCharacter(new Range(1, 6, 1, 7)), // <- stop character - new Word(new Range(1, 7, 1, 7 + 3), 'loů'), - new RightBracket(new Range(1, 10, 1, 11)), - new LeftParenthesis(new Range(1, 11, 1, 12)), - new Word(new Range(1, 12, 1, 12 + 1), '.'), - new Slash(new Range(1, 13, 1, 14)), - new Word(new Range(1, 14, 1, 14 + 4), 'real'), - new Slash(new Range(1, 18, 1, 19)), - new Word(new Range(1, 19, 1, 19 + 2), '💁'), - new Slash(new Range(1, 21, 1, 22)), - new Word(new Range(1, 22, 1, 22 + 4), 'name'), - new RightParenthesis(new Range(1, 26, 1, 27)), - new NewLine(new Range(1, 27, 1, 28)), - // `2nd` input line - new ExclamationMark(new Range(2, 1, 2, 2)), - new LeftBracket(new Range(2, 2, 2, 3)), - new Word(new Range(2, 3, 2, 3 + 3), 'ref'), - new Space(new Range(2, 6, 2, 7)), - new Word(new Range(2, 7, 2, 7 + 4), 'text'), - new RightBracket(new Range(2, 11, 2, 12)), - new LeftParenthesis(new Range(2, 12, 2, 13)), - new Slash(new Range(2, 13, 2, 14)), - new Word(new Range(2, 14, 2, 14 + 3), 'etc'), - new Slash(new Range(2, 17, 2, 18)), - new Word(new Range(2, 18, 2, 18 + 3), 'pat'), - new stopCharacter(new Range(2, 21, 2, 22)), // <- stop character - new Word(new Range(2, 22, 2, 22 + 1), 'h'), - new Slash(new Range(2, 23, 2, 24)), - new Word(new Range(2, 24, 2, 24 + 2), 'to'), - new Slash(new Range(2, 26, 2, 27)), - new Word(new Range(2, 27, 2, 27 + 9), 'file.webp'), - new RightParenthesis(new Range(2, 36, 2, 37)), - new NewLine(new Range(2, 37, 2, 38)), - // `3nd` input line - new ExclamationMark(new Range(3, 1, 3, 2)), - new LeftBracket(new Range(3, 2, 3, 3)), - new Word(new Range(3, 3, 3, 3 + 4), 'text'), - new RightBracket(new Range(3, 7, 3, 8)), - new stopCharacter(new Range(3, 8, 3, 9)), // <- stop character - new LeftParenthesis(new Range(3, 9, 3, 10)), - new Slash(new Range(3, 10, 3, 11)), - new Word(new Range(3, 11, 3, 11 + 3), 'etc'), - new Slash(new Range(3, 14, 3, 15)), - new Space(new Range(3, 15, 3, 16)), - new Word(new Range(3, 16, 3, 16 + 4), 'path'), - new Slash(new Range(3, 20, 3, 21)), - new Word(new Range(3, 21, 3, 21 + 9), 'image.gif'), - new RightParenthesis(new Range(3, 30, 3, 31)), - ], - ); - }); - } - }); - }); - }); - - suite('comments', () => { - suite('general', () => { - test('base cases', async () => { - const test = testDisposables.add( - new TestMarkdownDecoder(), - ); - - const inputData = [ - // comment with text inside it - '\t', - // comment with a link inside - 'some text and more text ', - // comment new lines inside it - ' usual text follows', - // an empty comment - '\t\t', - // comment that was not closed properly - 'haalo\t'), - new NewLine(new Range(1, 22, 1, 23)), - // `2nd` - new Word(new Range(2, 1, 2, 5), 'some'), - new Space(new Range(2, 5, 2, 6)), - new Word(new Range(2, 6, 2, 10), 'text'), - new MarkdownComment(new Range(2, 10, 2, 10 + 46), ''), - new Space(new Range(2, 56, 2, 57)), - new Word(new Range(2, 57, 2, 60), 'and'), - new Space(new Range(2, 60, 2, 61)), - new Word(new Range(2, 61, 2, 65), 'more'), - new Space(new Range(2, 65, 2, 66)), - new Word(new Range(2, 66, 2, 70), 'text'), - new Space(new Range(2, 70, 2, 71)), - new NewLine(new Range(2, 71, 2, 72)), - // `3rd` - new MarkdownComment(new Range(3, 1, 3 + 3, 1 + 13), ''), - new Space(new Range(6, 14, 6, 15)), - new Word(new Range(6, 15, 6, 15 + 5), 'usual'), - new Space(new Range(6, 20, 6, 21)), - new Word(new Range(6, 21, 6, 21 + 4), 'text'), - new Space(new Range(6, 25, 6, 26)), - new Word(new Range(6, 26, 6, 26 + 7), 'follows'), - new NewLine(new Range(6, 33, 6, 34)), - // `4rd` - new Tab(new Range(7, 1, 7, 2)), - new MarkdownComment(new Range(7, 2, 7, 2 + 7), ''), - new Tab(new Range(7, 9, 7, 10)), - new NewLine(new Range(7, 10, 7, 11)), - // `5th` - new Word(new Range(8, 1, 8, 6), 'haalo'), - new Tab(new Range(8, 6, 8, 7)), - new MarkdownComment(new Range(8, 7, 8, 7 + 40), '>', - // comment contains `<[]>` brackets and `!` - '\t\t', - // comment contains `\t\t', - // comment contains `'), - new RightAngleBracket(new Range(1, 19, 1, 20)), - new NewLine(new Range(1, 20, 1, 21)), - // `2nd` - new MarkdownComment(new Range(2, 1, 2, 1 + 21), ''), - new Tab(new Range(2, 22, 2, 23)), - new Tab(new Range(2, 23, 2, 24)), - new NewLine(new Range(2, 24, 2, 25)), - // `3rd` - new VerticalTab(new Range(3, 1, 3, 2)), - new MarkdownComment(new Range(3, 2, 3 + 3, 1 + 7), ''), - new Tab(new Range(6, 8, 6, 9)), - new Tab(new Range(6, 9, 6, 10)), - new NewLine(new Range(6, 10, 6, 11)), - // `4rd` - new Space(new Range(7, 1, 7, 2)), - // note! comment does not have correct closing `-->`, hence the comment extends - // to the end of the text, and therefore includes the \t\v\f and space at the end - new MarkdownComment(new Range(7, 2, 8, 1 + 12), ' ', - ' < !-- світ -->\t', - '\v\f', - '`, hence the comment extends - // to the end of the text, and therefore includes the `space` at the end - new MarkdownComment(new Range(4, 1, 4, 1 + 15), '