From d9832733879901756c13cf6cb2368a4b46f4ef81 Mon Sep 17 00:00:00 2001 From: Martin Aeschlimann Date: Fri, 12 Sep 2025 14:26:19 +0200 Subject: [PATCH 01/75] adopt computeCustomChatModes, IVariableReferences --- .../contrib/chat/browser/chatWidget.ts | 6 +- .../promptSyntax/service/newPromptsParser.ts | 31 ++++++---- .../service/promptsServiceImpl.ts | 56 ++++++++----------- .../service/newPromptsParser.test.ts | 12 +++- .../service/promptsService.test.ts | 9 +-- 5 files changed, 60 insertions(+), 54 deletions(-) diff --git a/src/vs/workbench/contrib/chat/browser/chatWidget.ts b/src/vs/workbench/contrib/chat/browser/chatWidget.ts index e864ff2dc3a..66279d3061a 100644 --- a/src/vs/workbench/contrib/chat/browser/chatWidget.ts +++ b/src/vs/workbench/contrib/chat/browser/chatWidget.ts @@ -83,6 +83,7 @@ import { IViewsService } from '../../../services/views/common/viewsService.js'; import { ICommandService } from '../../../../platform/commands/common/commands.js'; import { IHoverService } from '../../../../platform/hover/browser/hover.js'; import { ParsedPromptFile, PromptHeader } from '../common/promptSyntax/service/newPromptsParser.js'; +import { OffsetRange } from '../../../../editor/common/core/ranges/offsetRange.js'; const $ = dom.$; @@ -2157,8 +2158,9 @@ export class ChatWidget extends Disposable implements IChatWidget { if (agentSlashPromptPart) { parseResult = await this.promptsService.resolvePromptSlashCommand(agentSlashPromptPart.slashPromptCommand, CancellationToken.None); if (parseResult) { - // add the prompt file to the context, but not sticky - const toolReferences = this.toolsService.toToolReferences([]); // TODO: this.toolsService.toToolReferences(parseResult.body?.variableReferences ?? []); + // add the prompt file to the context + const refs = parseResult.body?.variableReferences.map(({ name, offset }) => ({ name, range: new OffsetRange(offset, offset + name.length + 1) })) ?? []; + const toolReferences = this.toolsService.toToolReferences(refs); requestInput.attachedContext.insertFirst(toPromptFileVariableEntry(parseResult.uri, PromptFileVariableKind.PromptFile, undefined, true, toolReferences)); // remove the slash command from the input diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/service/newPromptsParser.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/service/newPromptsParser.ts index fb0112acae5..fd08120c4ce 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/service/newPromptsParser.ts +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/service/newPromptsParser.ts @@ -3,6 +3,7 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ +import { Iterable } from '../../../../../../base/common/iterator.js'; import { dirname, resolvePath } from '../../../../../../base/common/resources.js'; import { splitLinesIncludeSeparators } from '../../../../../../base/common/strings.js'; import { URI } from '../../../../../../base/common/uri.js'; @@ -208,6 +209,7 @@ export type IValue = IStringValue | INumberValue | IBooleanValue | IArrayValue | interface ParsedBody { readonly fileReferences: readonly IBodyFileReference[]; readonly variableReferences: readonly IBodyVariableReference[]; + readonly bodyOffset: number; } export class PromptBody { @@ -224,12 +226,17 @@ export class PromptBody { return this.getParsedBody().variableReferences; } + public get offset(): number { + return this.getParsedBody().bodyOffset; + } + private getParsedBody(): ParsedBody { if (this._parsed === undefined) { const markdownLinkRanges: Range[] = []; const fileReferences: IBodyFileReference[] = []; const variableReferences: IBodyVariableReference[] = []; - for (let i = this.range.startLineNumber - 1; i < this.range.endLineNumber - 1; i++) { + const bodyOffset = Iterable.reduce(Iterable.slice(this.linesWithEOL, 0, this.range.startLineNumber - 1), (len, line) => line.length + len, 0); + for (let i = this.range.startLineNumber - 1, lineStartOffset = bodyOffset; i < this.range.endLineNumber - 1; i++) { const line = this.linesWithEOL[i]; const linkMatch = line.matchAll(/\[(.*?)\]\((.+?)\)/g); for (const match of linkMatch) { @@ -258,15 +265,20 @@ export class PromptBody { const contentStartOffset = match.index + 1; // after the # const contentEndOffset = match.index + match[0].length; const range = new Range(i + 1, contentStartOffset + 1, i + 1, contentEndOffset + 1); - variableReferences.push({ name: match[2], range }); + variableReferences.push({ name: match[2], range, offset: lineStartOffset + match.index }); } } + lineStartOffset += line.length; } - this._parsed = { fileReferences: fileReferences.sort((a, b) => Range.compareRangesUsingStarts(a.range, b.range)), variableReferences }; + this._parsed = { fileReferences: fileReferences.sort((a, b) => Range.compareRangesUsingStarts(a.range, b.range)), variableReferences, bodyOffset }; } return this._parsed; } + public getContent(): string { + return this.linesWithEOL.slice(this.range.startLineNumber - 1, this.range.endLineNumber - 1).join(''); + } + public resolveFilePath(path: string): URI | undefined { try { if (path.startsWith('/')) { @@ -283,14 +295,13 @@ export class PromptBody { } export interface IBodyFileReference { - content: string; - range: Range; - isMarkdownLink: boolean; + readonly content: string; + readonly range: Range; + readonly isMarkdownLink: boolean; } export interface IBodyVariableReference { - name: string; - range: Range; + readonly name: string; + readonly range: Range; + readonly offset: number; } - - diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts index df8191d491b..07579a4d003 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { localize } from '../../../../../../nls.js'; -import { getLanguageIdForPromptsType, getPromptsTypeForLanguageId, MODE_LANGUAGE_ID, PROMPT_LANGUAGE_ID, PromptsType } from '../promptTypes.js'; +import { getLanguageIdForPromptsType, getPromptsTypeForLanguageId, PROMPT_LANGUAGE_ID, PromptsType } from '../promptTypes.js'; import { PromptParser } from '../parsers/promptParser.js'; import { type URI } from '../../../../../../base/common/uri.js'; import { assert } from '../../../../../../base/common/assert.js'; @@ -31,6 +31,8 @@ import { NewPromptsParser, ParsedPromptFile } from './newPromptsParser.js'; import { IFileService } from '../../../../../../platform/files/common/files.js'; import { ResourceMap } from '../../../../../../base/common/map.js'; import { CancellationError } from '../../../../../../base/common/errors.js'; +import { OffsetRange } from '../../../../../../editor/common/core/ranges/offsetRange.js'; +import { IVariableReference } from '../../chatModes.js'; /** * Provides prompt services. @@ -255,42 +257,28 @@ export class PromptsService extends Disposable implements IPromptsService { const metadataList = await Promise.all( modeFiles.map(async ({ uri }): Promise => { - let parser: PromptParser | undefined; - try { - // Note! this can be (and should be) improved by using shared parser instances - // that the `getSyntaxParserFor` method provides for opened documents. - parser = this.instantiationService.createInstance( - PromptParser, - uri, - { allowNonPromptFiles: true, languageId: MODE_LANGUAGE_ID, updateOnChange: false }, - ).start(token); + const ast = await this.parseNew(uri, token); - const completed = await parser.settled(); - if (!completed) { - throw new Error(localize('promptParser.notCompleted', "Prompt parser for {0} did not complete.", uri.toString())); + const variableReferences: IVariableReference[] = []; + let body = ''; + if (ast.body) { + const bodyOffset = ast.body.offset; + const bodyVarRefs = ast.body.variableReferences; + for (let i = bodyVarRefs.length - 1; i >= 0; i--) { // in reverse order + const { name, offset } = bodyVarRefs[i]; + const range = new OffsetRange(offset - bodyOffset, offset - bodyOffset + name.length + 1); + variableReferences.push({ name, range }); } - - const body = await parser.getBody(); - const nHeaderLines = parser.header?.range.endLineNumber ?? 0; - const transformer = new PositionOffsetTransformer(body); - const variableReferences = parser.variableReferences.map(ref => { - return { - name: ref.name, - range: transformer.getOffsetRange(ref.range.delta(-nHeaderLines)) - }; - }).sort((a, b) => b.range.start - a.range.start); // in reverse order - - const name = getCleanPromptName(uri); - - const metadata = parser.metadata; - if (metadata?.promptType !== PromptsType.mode) { - return { uri, name, body, variableReferences }; - } - const { description, model, tools } = metadata; - return { uri, name, description, model, tools, body, variableReferences }; - } finally { - parser?.dispose(); + body = ast.body.getContent(); } + + const name = getCleanPromptName(uri); + if (!ast.header) { + return { uri, name, body, variableReferences }; + } + const { description, model, tools } = ast.header; + return { uri, name, description, model, tools, body, variableReferences }; + }) ); diff --git a/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/newPromptsParser.test.ts b/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/newPromptsParser.test.ts index 22409da076b..89dac4ac4db 100644 --- a/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/newPromptsParser.test.ts +++ b/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/newPromptsParser.test.ts @@ -41,12 +41,15 @@ suite('NewPromptsParser', () => { }, ]); assert.deepEqual(result.body.range, { startLineNumber: 6, startColumn: 1, endLineNumber: 8, endColumn: 1 }); + assert.equal(result.body.offset, 80); + assert.equal(result.body.getContent(), 'This is a chat mode test.\nHere is a #tool1 variable and a #file:./reference1.md as well as a [reference](./reference2.md).'); + assert.deepEqual(result.body.fileReferences, [ { range: new Range(7, 39, 7, 54), content: './reference1.md', isMarkdownLink: false }, { range: new Range(7, 80, 7, 95), content: './reference2.md', isMarkdownLink: true } ]); assert.deepEqual(result.body.variableReferences, [ - { range: new Range(7, 12, 7, 17), name: 'tool1' } + { range: new Range(7, 12, 7, 17), name: 'tool1', offset: 116 } ]); assert.deepEqual(result.header.description, 'Agent mode test'); assert.deepEqual(result.header.model, 'GPT 4.1'); @@ -73,6 +76,9 @@ suite('NewPromptsParser', () => { { key: 'applyTo', range: new Range(3, 1, 3, 14), value: { type: 'string', value: '*.ts', range: new Range(3, 10, 3, 14) } }, ]); assert.deepEqual(result.body.range, { startLineNumber: 5, startColumn: 1, endLineNumber: 6, endColumn: 1 }); + assert.equal(result.body.offset, 76); + assert.equal(result.body.getContent(), 'Follow my companies coding guidlines at [mycomp-ts-guidelines](https://mycomp/guidelines#typescript.md)'); + assert.deepEqual(result.body.fileReferences, [ { range: new Range(5, 64, 5, 103), content: 'https://mycomp/guidelines#typescript.md', isMarkdownLink: true }, ]); @@ -110,11 +116,13 @@ suite('NewPromptsParser', () => { }, ]); assert.deepEqual(result.body.range, { startLineNumber: 7, startColumn: 1, endLineNumber: 8, endColumn: 1 }); + assert.equal(result.body.offset, 113); + assert.equal(result.body.getContent(), 'This is a prompt file body referencing #search and [docs](https://example.com/docs).'); assert.deepEqual(result.body.fileReferences, [ { range: new Range(7, 59, 7, 83), content: 'https://example.com/docs', isMarkdownLink: true }, ]); assert.deepEqual(result.body.variableReferences, [ - { range: new Range(7, 41, 7, 47), name: 'search' } + { range: new Range(7, 41, 7, 47), name: 'search', offset: 152 } ]); assert.deepEqual(result.header.description, 'General purpose coding assistant'); assert.deepEqual(result.header.mode, 'agent'); diff --git a/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/promptsService.test.ts b/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/promptsService.test.ts index 0c453e00b46..21f60a3aae0 100644 --- a/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/promptsService.test.ts +++ b/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/promptsService.test.ts @@ -710,8 +710,8 @@ suite('PromptsService', () => { assert.deepEqual( result1.body.variableReferences, [ - { name: "my-tool", range: new Range(10, 5, 10, 12) }, - { name: "my-other-tool", range: new Range(11, 5, 11, 18) }, + { name: "my-tool", range: new Range(10, 5, 10, 12), offset: 239 }, + { name: "my-other-tool", range: new Range(11, 5, 11, 18), offset: 251 }, ] ); @@ -1275,18 +1275,15 @@ suite('PromptsService', () => { }, { name: 'mode2', - description: undefined, - tools: undefined, body: 'First use #tool2\nThen use #tool1', variableReferences: [{ name: 'tool1', range: { start: 26, endExclusive: 32 } }, { name: 'tool2', range: { start: 10, endExclusive: 16 } }], - model: undefined, uri: URI.joinPath(rootFolderUri, '.github/chatmodes/mode2.instructions.md'), } ]; assert.deepEqual( - expected, result, + expected, 'Must get custom chat modes.', ); }); From 609aaecdfb02ab4157004f22a9073dbdca47fd1c Mon Sep 17 00:00:00 2001 From: Martin Aeschlimann Date: Fri, 12 Sep 2025 15:28:13 +0200 Subject: [PATCH 02/75] remove old prompt parser --- src/vs/base/common/numbers.ts | 59 - src/vs/base/test/common/numbers.test.ts | 175 +- .../promptSyntax/codecs/base/asyncDecoder.ts | 93 - .../promptSyntax/codecs/base/baseDecoder.ts | 361 --- .../promptSyntax/codecs/base/baseToken.ts | 150 -- .../codecs/base/compositeToken.ts | 63 - .../codecs/base/frontMatterCodec/constants.ts | 16 - .../frontMatterCodec/frontMatterDecoder.ts | 156 -- .../parsers/frontMatterArray.ts | 197 -- .../parsers/frontMatterParserFactory.ts | 41 - .../frontMatterRecord/frontMatterRecord.ts | 210 -- .../frontMatterRecordName.ts | 74 - .../frontMatterRecordNameWithDelimiter.ts | 105 - .../parsers/frontMatterSequence.ts | 78 - .../parsers/frontMatterString.ts | 69 - .../parsers/frontMatterValue.ts | 184 -- .../tokens/frontMatterArray.ts | 43 - .../tokens/frontMatterBoolean.ts | 88 - .../tokens/frontMatterRecord.ts | 118 - .../tokens/frontMatterSequence.ts | 79 - .../tokens/frontMatterString.ts | 39 - .../tokens/frontMatterToken.ts | 33 - .../base/frontMatterCodec/tokens/index.ts | 15 - .../codecs/base/linesCodec/linesDecoder.ts | 237 -- .../base/linesCodec/tokens/carriageReturn.ts | 44 - .../codecs/base/linesCodec/tokens/line.ts | 48 - .../codecs/base/linesCodec/tokens/newLine.ts | 44 - .../base/markdownCodec/markdownDecoder.ts | 136 -- .../markdownCodec/parsers/markdownComment.ts | 173 -- .../markdownCodec/parsers/markdownImage.ts | 99 - .../markdownCodec/parsers/markdownLink.ts | 211 -- .../markdownCodec/tokens/markdownComment.ts | 40 - .../markdownCodec/tokens/markdownImage.ts | 125 -- .../base/markdownCodec/tokens/markdownLink.ts | 120 - .../markdownCodec/tokens/markdownToken.ts | 12 - .../markdownExtensionsDecoder.ts | 119 - .../parsers/frontMatterHeader.ts | 345 --- .../tokens/frontMatterHeader.ts | 79 - .../tokens/frontMatterMarker.ts | 59 - .../tokens/markdownExtensionsToken.ts | 11 - .../codecs/base/simpleCodec/parserBase.ts | 137 -- .../codecs/base/simpleCodec/simpleDecoder.ts | 132 -- .../base/simpleCodec/tokens/angleBrackets.ts | 61 - .../codecs/base/simpleCodec/tokens/at.ts | 31 - .../base/simpleCodec/tokens/brackets.ts | 61 - .../codecs/base/simpleCodec/tokens/colon.ts | 31 - .../codecs/base/simpleCodec/tokens/comma.ts | 31 - .../base/simpleCodec/tokens/curlyBraces.ts | 61 - .../codecs/base/simpleCodec/tokens/dash.ts | 31 - .../base/simpleCodec/tokens/dollarSign.ts | 31 - .../base/simpleCodec/tokens/doubleQuote.ts | 40 - .../simpleCodec/tokens/exclamationMark.ts | 31 - .../base/simpleCodec/tokens/formFeed.ts | 31 - .../codecs/base/simpleCodec/tokens/hash.ts | 31 - .../base/simpleCodec/tokens/parentheses.ts | 61 - .../codecs/base/simpleCodec/tokens/quote.ts | 40 - .../base/simpleCodec/tokens/simpleToken.ts | 59 - .../codecs/base/simpleCodec/tokens/slash.ts | 31 - .../codecs/base/simpleCodec/tokens/space.ts | 31 - .../codecs/base/simpleCodec/tokens/tab.ts | 31 - .../codecs/base/simpleCodec/tokens/tokens.ts | 25 - .../base/simpleCodec/tokens/verticalTab.ts | 31 - .../codecs/base/simpleCodec/tokens/word.ts | 60 - .../promptSyntax/codecs/base/textToken.ts | 19 - .../codecs/base/utils/objectStream.ts | 224 -- .../base/utils/objectStreamFromTextModel.ts | 46 - .../promptSyntax/codecs/chatPromptCodec.ts | 73 - .../promptSyntax/codecs/chatPromptDecoder.ts | 202 -- .../codecs/parsers/promptAtMentionParser.ts | 121 -- .../parsers/promptSlashCommandParser.ts | 122 -- .../parsers/promptTemplateVariableParser.ts | 148 -- .../codecs/parsers/promptVariableParser.ts | 252 --- .../codecs/tokens/fileReference.ts | 50 - .../codecs/tokens/promptAtMention.ts | 52 - .../codecs/tokens/promptSlashCommand.ts | 42 - .../codecs/tokens/promptTemplateVariable.ts | 44 - .../promptSyntax/codecs/tokens/promptToken.ts | 11 - .../codecs/tokens/promptVariable.ts | 103 - .../filePromptContentsProvider.ts | 166 -- .../promptContentsProviderBase.ts | 196 -- .../textModelContentsProvider.ts | 93 - .../promptSyntax/contentProviders/types.ts | 70 - .../decorations/frontMatterDecoration.ts | 120 - .../frontMatterMarkerDecoration.ts | 56 - .../decorations/utils/decorationBase.ts | 127 -- .../utils/reactiveDecorationBase.ts | 162 -- .../decorations/utils/types.ts | 56 - .../promptDecorationsProvider.ts | 205 -- .../decorationsProvider/types.ts | 48 - .../promptHeaderDiagnosticsProvider.ts | 234 -- .../promptLinkDiagnosticsProvider.ts | 98 - .../languageProviders/providerInstanceBase.ts | 54 - .../providerInstanceManagerBase.ts | 176 -- .../promptSyntax/parsers/basePromptParser.ts | 728 ------- .../promptSyntax/parsers/filePromptParser.ts | 37 - .../parsers/promptHeader/diagnostics.ts | 47 - .../parsers/promptHeader/headerBase.ts | 264 --- .../promptHeader/instructionsHeader.ts | 44 - .../parsers/promptHeader/metadata/applyTo.ts | 122 -- .../promptHeader/metadata/base/enum.ts | 84 - .../promptHeader/metadata/base/record.ts | 108 - .../promptHeader/metadata/base/string.ts | 73 - .../promptHeader/metadata/description.ts | 46 - .../parsers/promptHeader/metadata/mode.ts | 47 - .../parsers/promptHeader/metadata/model.ts | 41 - .../parsers/promptHeader/metadata/tools.ts | 182 -- .../parsers/promptHeader/modeHeader.ts | 56 - .../parsers/promptHeader/promptHeader.ts | 103 - .../promptSyntax/parsers/promptParser.ts | 72 - .../parsers/textModelPromptParser.ts | 42 - .../common/promptSyntax/parsers/topError.ts | 102 - .../chat/common/promptSyntax/parsers/types.ts | 116 - .../promptSyntax/service/promptsService.ts | 49 - .../service/promptsServiceImpl.ts | 93 +- .../common/promptSyntax/utils/objectCache.ts | 153 -- .../utils/observableDisposable.ts | 89 - .../frontMatterBoolean.test.ts | 317 --- .../frontMatterDecoder.test.ts | 415 ---- .../frontMatterRecord.test.ts | 183 -- .../frontMatterSequence.test.ts | 106 - .../codecs/base/linesDecoder.test.ts | 256 --- .../codecs/base/markdownDecoder.test.ts | 937 -------- .../codecs/base/simpleDecoder.test.ts | 238 -- .../codecs/base/testUtils/randomRange.ts | 58 - .../codecs/base/testUtils/randomTokens.ts | 146 -- .../codecs/base/tokens/baseToken.test.ts | 516 ----- .../codecs/base/tokens/compositeToken.test.ts | 250 --- .../codecs/base/tokens/simpleToken.test.ts | 66 - .../codecs/base/utils/objectStream.test.ts | 180 -- .../codecs/base/utils/testDecoder.ts | 258 --- .../codecs/chatPromptCodec.test.ts | 136 -- .../codecs/chatPromptDecoder.test.ts | 428 ---- .../codecs/markdownExtensionsDecoder.test.ts | 445 ---- .../codecs/tokens/fileReference.test.ts | 106 - .../codecs/tokens/markdownLink.test.ts | 98 - .../common/promptSyntax/config/config.test.ts | 15 +- .../promptSyntax/config/constants.test.ts | 7 +- .../filePromptContentsProvider.test.ts | 210 -- .../parsers/textModelPromptParser.test.ts | 1928 ----------------- .../service/promptsService.test.ts | 484 ----- .../testUtils/expectedDiagnostic.ts | 90 - .../testUtils/expectedReference.ts | 129 -- .../common/promptSyntax/utils/mock.test.ts | 5 +- .../promptSyntax/utils/objectCache.test.ts | 332 --- .../utils/observableDisposable.test.ts | 368 ---- 145 files changed, 17 insertions(+), 20215 deletions(-) delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/asyncDecoder.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/baseDecoder.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/baseToken.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/compositeToken.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/constants.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/frontMatterDecoder.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterArray.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterParserFactory.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterRecord/frontMatterRecord.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterRecord/frontMatterRecordName.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterRecord/frontMatterRecordNameWithDelimiter.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterSequence.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterString.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterValue.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterArray.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterBoolean.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterRecord.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterSequence.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterString.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterToken.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/index.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/linesCodec/linesDecoder.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/linesCodec/tokens/carriageReturn.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/linesCodec/tokens/line.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/linesCodec/tokens/newLine.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/markdownDecoder.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/parsers/markdownComment.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/parsers/markdownImage.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/parsers/markdownLink.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/tokens/markdownComment.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/tokens/markdownImage.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/tokens/markdownLink.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/tokens/markdownToken.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/markdownExtensionsDecoder.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/parsers/frontMatterHeader.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/tokens/frontMatterHeader.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/tokens/frontMatterMarker.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/tokens/markdownExtensionsToken.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/parserBase.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/simpleDecoder.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/angleBrackets.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/at.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/brackets.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/colon.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/comma.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/curlyBraces.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/dash.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/dollarSign.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/doubleQuote.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/exclamationMark.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/formFeed.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/hash.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/parentheses.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/quote.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/simpleToken.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/slash.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/space.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/tab.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/tokens.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/verticalTab.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/word.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/textToken.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/utils/objectStream.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/utils/objectStreamFromTextModel.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/chatPromptCodec.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/chatPromptDecoder.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/parsers/promptAtMentionParser.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/parsers/promptSlashCommandParser.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/parsers/promptTemplateVariableParser.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/parsers/promptVariableParser.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/fileReference.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptAtMention.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptSlashCommand.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptTemplateVariable.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptToken.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptVariable.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/contentProviders/filePromptContentsProvider.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/contentProviders/promptContentsProviderBase.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/contentProviders/textModelContentsProvider.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/contentProviders/types.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/frontMatterDecoration.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/frontMatterMarkerDecoration.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/utils/decorationBase.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/utils/reactiveDecorationBase.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/utils/types.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/promptDecorationsProvider.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/types.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/promptHeaderDiagnosticsProvider.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/promptLinkDiagnosticsProvider.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/providerInstanceBase.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/providerInstanceManagerBase.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/parsers/basePromptParser.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/parsers/filePromptParser.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/diagnostics.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/headerBase.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/instructionsHeader.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/applyTo.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/base/enum.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/base/record.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/base/string.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/description.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/mode.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/model.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/tools.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/modeHeader.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/promptHeader.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptParser.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/parsers/textModelPromptParser.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/parsers/topError.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/parsers/types.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/utils/objectCache.ts delete mode 100644 src/vs/workbench/contrib/chat/common/promptSyntax/utils/observableDisposable.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/frontMatterDecoder/frontMatterBoolean.test.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/frontMatterDecoder/frontMatterDecoder.test.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/frontMatterDecoder/frontMatterRecord.test.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/frontMatterDecoder/frontMatterSequence.test.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/linesDecoder.test.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/markdownDecoder.test.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/simpleDecoder.test.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/testUtils/randomRange.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/testUtils/randomTokens.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/tokens/baseToken.test.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/tokens/compositeToken.test.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/tokens/simpleToken.test.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/utils/objectStream.test.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/utils/testDecoder.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/chatPromptCodec.test.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/chatPromptDecoder.test.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/markdownExtensionsDecoder.test.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/tokens/fileReference.test.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/tokens/markdownLink.test.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/contentProviders/filePromptContentsProvider.test.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/parsers/textModelPromptParser.test.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/testUtils/expectedDiagnostic.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/testUtils/expectedReference.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/utils/objectCache.test.ts delete mode 100644 src/vs/workbench/contrib/chat/test/common/promptSyntax/utils/observableDisposable.test.ts diff --git a/src/vs/base/common/numbers.ts b/src/vs/base/common/numbers.ts index 89c9f183e6d..326a9b2f2c3 100644 --- a/src/vs/base/common/numbers.ts +++ b/src/vs/base/common/numbers.ts @@ -99,65 +99,6 @@ export function isPointWithinTriangle( return u >= 0 && v >= 0 && u + v < 1; } -/** - * Function to get a (pseudo)random integer from a provided `max`...[`min`] range. - * Both `min` and `max` values are inclusive. The `min` value is optional (defaults to `0`). - * - * @throws in the next cases: - * - if provided `min` or `max` is not a number - * - if provided `min` or `max` is not finite - * - if provided `min` is larger than `max` value - * - * ## Examples - * - * Specifying a `max` value only uses `0` as the `min` value by default: - * - * ```typescript - * // get a random integer between 0 and 10 - * const randomInt = randomInt(10); - * - * assert( - * randomInt >= 0, - * 'Should be greater than or equal to 0.', - * ); - * - * assert( - * randomInt <= 10, - * 'Should be less than or equal to 10.', - * ); - * ``` - * * Specifying both `max` and `min` values: - * - * ```typescript - * // get a random integer between 5 and 8 - * const randomInt = randomInt(8, 5); - * - * assert( - * randomInt >= 5, - * 'Should be greater than or equal to 5.', - * ); - * - * assert( - * randomInt <= 8, - * 'Should be less than or equal to 8.', - * ); - * ``` - */ -export function randomInt(max: number, min: number = 0): number { - assert(!isNaN(min), '"min" param is not a number.'); - assert(!isNaN(max), '"max" param is not a number.'); - - assert(isFinite(max), '"max" param is not finite.'); - assert(isFinite(min), '"min" param is not finite.'); - - assert(max > min, `"max"(${max}) param should be greater than "min"(${min}).`); - - const delta = max - min; - const randomFloat = delta * Math.random(); - - return Math.round(min + randomFloat); -} - export function randomChance(p: number): boolean { assert(p >= 0 && p <= 1, 'p must be between 0 and 1'); return Math.random() < p; diff --git a/src/vs/base/test/common/numbers.test.ts b/src/vs/base/test/common/numbers.test.ts index 94c07090585..e21844eea98 100644 --- a/src/vs/base/test/common/numbers.test.ts +++ b/src/vs/base/test/common/numbers.test.ts @@ -5,7 +5,7 @@ import assert from 'assert'; import { ensureNoDisposablesAreLeakedInTestSuite } from './utils.js'; -import { isPointWithinTriangle, randomInt } from '../../common/numbers.js'; +import { isPointWithinTriangle } from '../../common/numbers.js'; suite('isPointWithinTriangle', () => { ensureNoDisposablesAreLeakedInTestSuite(); @@ -25,176 +25,3 @@ suite('isPointWithinTriangle', () => { assert.ok(result); }); }); - -suite('randomInt', () => { - ensureNoDisposablesAreLeakedInTestSuite(); - - /** - * Test helper that allows to run a test on the `randomInt()` - * utility with specified `max` and `min` values. - */ - const testRandomIntUtil = (max: number, min: number | undefined, testName: string) => { - suite(testName, () => { - let i = 0; - while (++i < 5) { - test(`should generate random boolean attempt#${i}`, async () => { - let iterations = 100; - while (iterations-- > 0) { - const int = randomInt(max, min); - - assert( - int <= max, - `Expected ${int} to be less than or equal to ${max}.` - ); - assert( - int >= (min ?? 0), - `Expected ${int} to be greater than or equal to ${min ?? 0}.`, - ); - } - }); - } - - test('should include min and max', async () => { - let iterations = 125; - const results = []; - while (iterations-- > 0) { - results.push(randomInt(max, min)); - } - - assert( - results.includes(max), - `Expected ${results} to include ${max}.`, - ); - assert( - results.includes(min ?? 0), - `Expected ${results} to include ${min ?? 0}.`, - ); - }); - }); - }; - - suite('positive numbers', () => { - testRandomIntUtil(4, 2, 'max: 4, min: 2'); - testRandomIntUtil(4, 0, 'max: 4, min: 0'); - testRandomIntUtil(4, undefined, 'max: 4, min: undefined'); - testRandomIntUtil(1, 0, 'max: 0, min: 0'); - }); - - suite('negative numbers', () => { - testRandomIntUtil(-2, -5, 'max: -2, min: -5'); - testRandomIntUtil(0, -5, 'max: 0, min: -5'); - testRandomIntUtil(0, -1, 'max: 0, min: -1'); - }); - - suite('split numbers', () => { - testRandomIntUtil(3, -1, 'max: 3, min: -1'); - testRandomIntUtil(2, -2, 'max: 2, min: -2'); - testRandomIntUtil(1, -3, 'max: 2, min: -2'); - }); - - suite('errors', () => { - test('should throw if "min" is == "max" #1', () => { - assert.throws(() => { - randomInt(200, 200); - }, `"max"(200) param should be greater than "min"(200)."`); - }); - - test('should throw if "min" is == "max" #2', () => { - assert.throws(() => { - randomInt(2, 2); - }, `"max"(2) param should be greater than "min"(2)."`); - }); - - test('should throw if "min" is == "max" #3', () => { - assert.throws(() => { - randomInt(0); - }, `"max"(0) param should be greater than "min"(0)."`); - }); - - test('should throw if "min" is > "max" #1', () => { - assert.throws(() => { - randomInt(2, 3); - }, `"max"(2) param should be greater than "min"(3)."`); - }); - - test('should throw if "min" is > "max" #2', () => { - assert.throws(() => { - randomInt(999, 2000); - }, `"max"(999) param should be greater than "min"(2000)."`); - }); - - test('should throw if "min" is > "max" #3', () => { - assert.throws(() => { - randomInt(0, 1); - }, `"max"(0) param should be greater than "min"(1)."`); - }); - - test('should throw if "min" is > "max" #4', () => { - assert.throws(() => { - randomInt(-5, 2); - }, `"max"(-5) param should be greater than "min"(2)."`); - }); - - test('should throw if "min" is > "max" #5', () => { - assert.throws(() => { - randomInt(-4, 0); - }, `"max"(-4) param should be greater than "min"(0)."`); - }); - - test('should throw if "min" is > "max" #6', () => { - assert.throws(() => { - randomInt(-4); - }, `"max"(-4) param should be greater than "min"(0)."`); - }); - - test('should throw if "max" is `NaN`', () => { - assert.throws(() => { - randomInt(NaN); - }, `"max" param is not a number."`); - }); - - test('should throw if "min" is `NaN`', () => { - assert.throws(() => { - randomInt(4, NaN); - }, `"min" param is not a number."`); - }); - - suite('infinite arguments', () => { - test('should throw if "max" is infinite [Infinity]', () => { - assert.throws(() => { - randomInt(Infinity); - }, `"max" param is not finite."`); - }); - - test('should throw if "max" is infinite [-Infinity]', () => { - assert.throws(() => { - randomInt(-Infinity); - }, `"max" param is not finite."`); - }); - - test('should throw if "max" is infinite [+Infinity]', () => { - assert.throws(() => { - randomInt(+Infinity); - }, `"max" param is not finite."`); - }); - - test('should throw if "min" is infinite [Infinity]', () => { - assert.throws(() => { - randomInt(Infinity, Infinity); - }, `"max" param is not finite."`); - }); - - test('should throw if "min" is infinite [-Infinity]', () => { - assert.throws(() => { - randomInt(Infinity, -Infinity); - }, `"max" param is not finite."`); - }); - - test('should throw if "min" is infinite [+Infinity]', () => { - assert.throws(() => { - randomInt(Infinity, +Infinity); - }, `"max" param is not finite."`); - }); - }); - }); -}); diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/asyncDecoder.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/asyncDecoder.ts deleted file mode 100644 index be10b50ebba..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/asyncDecoder.ts +++ /dev/null @@ -1,93 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { Disposable } from '../../../../../../../base/common/lifecycle.js'; -import { BaseDecoder } from './baseDecoder.js'; - -/** - * Asynchronous iterator wrapper for a decoder. - */ -export class AsyncDecoder, K extends NonNullable = NonNullable> extends Disposable { - // Buffer of messages that have been decoded but not yet consumed. - private readonly messages: T[] = []; - - /** - * A transient promise that is resolved when a new event - * is received. Used in the situation when there is no new - * data available and decoder stream did not finish yet, - * hence we need to wait until new event is received. - */ - private resolveOnNewEvent?: (value: void) => void; - - /** - * @param decoder The decoder instance to wrap. - * - * Note! Assumes ownership of the `decoder` object, hence will `dispose` - * it when the decoder stream is ended. - */ - constructor( - private readonly decoder: BaseDecoder, - ) { - super(); - - this._register(decoder); - } - - /** - * Async iterator implementation. - */ - async *[Symbol.asyncIterator](): AsyncIterator { - // callback is called when `data` or `end` event is received - const callback = (data?: T) => { - if (data !== undefined) { - this.messages.push(data); - } else { - this.decoder.removeListener('data', callback); - this.decoder.removeListener('end', callback); - } - - // is the promise resolve callback is present, - // then call it and remove the reference - if (this.resolveOnNewEvent) { - this.resolveOnNewEvent(); - delete this.resolveOnNewEvent; - } - }; - - /** - * !NOTE! The order of event subscriptions below is critical here because - * the `data` event is also starts the stream, hence changing - * the order of event subscriptions can lead to race conditions. - * See {@link ReadableStreamEvents} for more info. - */ - - this.decoder.on('end', callback); - this.decoder.on('data', callback); - - // start flowing the decoder stream - this.decoder.start(); - - while (true) { - const maybeMessage = this.messages.shift(); - if (maybeMessage !== undefined) { - yield maybeMessage; - continue; - } - - // if no data available and stream ended, we're done - if (this.decoder.ended) { - this.dispose(); - - return null; - } - - // stream isn't ended so wait for the new - // `data` or `end` event to be received - await new Promise((resolve) => { - this.resolveOnNewEvent = resolve; - }); - } - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/baseDecoder.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/baseDecoder.ts deleted file mode 100644 index 76d1745d3c6..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/baseDecoder.ts +++ /dev/null @@ -1,361 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { Emitter } from '../../../../../../../base/common/event.js'; -import { ReadableStream } from '../../../../../../../base/common/stream.js'; -import { DeferredPromise } from '../../../../../../../base/common/async.js'; -import { AsyncDecoder } from './asyncDecoder.js'; -import { assert, assertNever } from '../../../../../../../base/common/assert.js'; -import { DisposableMap, IDisposable } from '../../../../../../../base/common/lifecycle.js'; -import { ObservableDisposable } from '../../utils/observableDisposable.js'; - -/** - * Event names of {@link ReadableStream} stream. - */ -export type TStreamListenerNames = 'data' | 'error' | 'end'; - -/** - * Base decoder class that can be used to convert stream messages data type - * from one type to another. For instance, a stream of binary data can be - * "decoded" into a stream of well defined objects. - * Intended to be a part of "codec" implementation rather than used directly. - */ -export abstract class BaseDecoder< - T extends NonNullable, - K extends NonNullable = NonNullable, -> extends ObservableDisposable implements ReadableStream { - /** - * Private attribute to track if the stream has ended. - */ - private _ended = false; - - protected readonly _onData = this._register(new Emitter()); - private readonly _onEnd = this._register(new Emitter()); - private readonly _onError = this._register(new Emitter()); - - /** - * A store of currently registered event listeners. - */ - private readonly _listeners: DisposableMap< - TStreamListenerNames, - DisposableMap - > = this._register(new DisposableMap()); - - /** - * This method is called when a new incoming data - * is received from the input stream. - */ - protected abstract onStreamData(data: K): void; - - /** - * @param stream The input stream to decode. - */ - constructor( - protected readonly stream: ReadableStream, - ) { - super(); - } - - /** - * Private attribute to track if the stream has started. - */ - private started = false; - - /** - * Promise that resolves when the stream has ended, either by - * receiving the `end` event or by a disposal, but not when - * the `error` event is received alone. - * The promise is true if the stream has ended, and false - * if the stream has been disposed without ending. - */ - private settledPromise = new DeferredPromise(); - - /** - * Promise that resolves when the stream has ended, either by - * receiving the `end` event or by a disposal, but not when - * the `error` event is received alone. - * The promise is true if the stream has ended, and false - * if the stream has been disposed without ending. - * - * @throws If the stream was not yet started to prevent this - * promise to block the consumer calls indefinitely. - */ - public get settled(): Promise { - // if the stream has not started yet, the promise might - // block the consumer calls indefinitely if they forget - // to call the `start()` method, or if the call happens - // after await on the `settled` promise; to forbid this - // confusion, we require the stream to be started first - assert( - this.started, - [ - 'Cannot get `settled` promise of a stream that has not been started.', - 'Please call `start()` first.', - ].join(' '), - ); - - return this.settledPromise.p; - } - - /** - * Start receiving data from the stream. - * @throws if the decoder stream has already ended. - */ - public start(): this { - assert( - this._ended === false, - 'Cannot start stream that has already ended.', - ); - assert( - this.isDisposed === false, - 'Cannot start stream that has already disposed.', - ); - - // if already started, nothing to do - if (this.started) { - return this; - } - this.started = true; - - /** - * !NOTE! The order of event subscriptions is critical here because - * the `data` event is also starts the stream, hence changing - * the order of event subscriptions can lead to race conditions. - * See {@link ReadableStreamEvents} for more info. - */ - this.stream.on('end', this.onStreamEnd.bind(this)); - this.stream.on('error', this.onStreamError.bind(this)); - this.stream.on('data', this.tryOnStreamData.bind(this)); - - // this allows to compose decoders together, - if a decoder - // instance is passed as a readable stream to this decoder, - // then we need to call `start` on it too - if (this.stream instanceof BaseDecoder) { - this.stream.start(); - } - - return this; - } - - /** - * Check if the decoder has been ended hence has - * no more data to produce. - */ - public get ended(): boolean { - return this._ended; - } - - /** - * Automatically catch and dispatch errors thrown inside `onStreamData`. - */ - private tryOnStreamData(data: K): void { - try { - this.onStreamData(data); - } catch (error) { - this.onStreamError(error); - } - } - - public on(event: 'data', callback: (data: T) => void): void; - public on(event: 'error', callback: (err: Error) => void): void; - public on(event: 'end', callback: () => void): void; - public on(event: TStreamListenerNames, callback: unknown): void { - if (event === 'data') { - return this.onData(callback as (data: T) => void); - } - - if (event === 'error') { - return this.onError(callback as (error: Error) => void); - } - - if (event === 'end') { - return this.onEnd(callback as () => void); - } - - assertNever(event, `Invalid event name '${event}'`); - } - - /** - * Add listener for the `data` event. - * @throws if the decoder stream has already ended. - */ - public onData(callback: (data: T) => void): void { - assert( - !this.ended, - 'Cannot subscribe to the `data` event because the decoder stream has already ended.', - ); - - let currentListeners = this._listeners.get('data'); - - if (!currentListeners) { - currentListeners = new DisposableMap(); - this._listeners.set('data', currentListeners); - } - - currentListeners.set(callback, this._onData.event(callback)); - } - - /** - * Add listener for the `error` event. - * @throws if the decoder stream has already ended. - */ - public onError(callback: (error: Error) => void): void { - assert( - !this.ended, - 'Cannot subscribe to the `error` event because the decoder stream has already ended.', - ); - - let currentListeners = this._listeners.get('error'); - - if (!currentListeners) { - currentListeners = new DisposableMap(); - this._listeners.set('error', currentListeners); - } - - currentListeners.set(callback, this._onError.event(callback)); - } - - /** - * Add listener for the `end` event. - * @throws if the decoder stream has already ended. - */ - public onEnd(callback: () => void): void { - assert( - !this.ended, - 'Cannot subscribe to the `end` event because the decoder stream has already ended.', - ); - - let currentListeners = this._listeners.get('end'); - - if (!currentListeners) { - currentListeners = new DisposableMap(); - this._listeners.set('end', currentListeners); - } - - currentListeners.set(callback, this._onEnd.event(callback)); - } - - /** - * Pauses the stream. - */ - public pause(): void { - this.stream.pause(); - } - - /** - * Resumes the stream if it has been paused. - * @throws if the decoder stream has already ended. - */ - public resume(): void { - assert( - this.ended === false, - 'Cannot resume the stream because it has already ended.', - ); - - this.stream.resume(); - } - - /** - * Destroys(disposes) the stream. - */ - public destroy(): void { - this.dispose(); - } - - /** - * Removes a previously-registered event listener for a specified event. - * - * Note! - * - the callback function must be the same as the one that was used when - * registering the event listener as it is used as an identifier to - * remove the listener - * - this method is idempotent and results in no-op if the listener is - * not found, therefore passing incorrect `callback` function may - * result in silent unexpected behavior - */ - public removeListener(eventName: TStreamListenerNames, callback: Function): void { - const listeners = this._listeners.get(eventName); - if (listeners === undefined) { - return; - } - - for (const [listener] of listeners) { - if (listener !== callback) { - continue; - } - - listeners.deleteAndDispose(listener); - } - } - - /** - * This method is called when the input stream ends. - */ - protected onStreamEnd(): void { - if (this._ended) { - return; - } - - this._ended = true; - this._onEnd.fire(); - this.settledPromise.complete(this._ended); - } - - /** - * This method is called when the input stream emits an error. - * We re-emit the error here by default, but subclasses can - * override this method to handle the error differently. - */ - private onStreamError(error: Error): void { - this._onError.fire(error); - } - - /** - * Consume all messages from the stream, blocking until the stream finishes. - * @throws if the decoder stream has already ended. - */ - public async consumeAll(): Promise { - assert( - !this._ended, - 'Cannot consume all messages of the stream that has already ended.', - ); - - const messages = []; - - for await (const maybeMessage of this) { - if (maybeMessage === null) { - break; - } - - messages.push(maybeMessage); - } - - return messages; - } - - /** - * Async iterator interface for the decoder. - * @throws if the decoder stream has already ended. - */ - [Symbol.asyncIterator](): AsyncIterator { - assert( - !this._ended, - 'Cannot iterate on messages of the stream that has already ended.', - ); - - const asyncDecoder = this._register(new AsyncDecoder(this)); - - return asyncDecoder[Symbol.asyncIterator](); - } - - public override dispose(): void { - this.settledPromise.complete(this.ended); - - this._listeners.clearAndDisposeAll(); - this.stream.destroy(); - - super.dispose(); - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/baseToken.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/baseToken.ts deleted file mode 100644 index ba81a78738e..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/baseToken.ts +++ /dev/null @@ -1,150 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { assert } from '../../../../../../../base/common/assert.js'; -import { IRange, Range } from '../../../../../../../editor/common/core/range.js'; - -/** - * Base class for all tokens with a `range` that reflects - * token position in the original text. - */ -export abstract class BaseToken { - constructor( - private tokenRange: Range, - ) { } - - /** - * Range of the token in the original text. - */ - public get range(): Range { - return this.tokenRange; - } - - /** - * Text representation of the token. - */ - public abstract get text(): TText; - - /** - * Check if this token has the same range as another one. - */ - public sameRange(other: Range): boolean { - return this.range.equalsRange(other); - } - - /** - * Returns a string representation of the token. - */ - public abstract toString(): string; - - /** - * Check if this token is equal to another one. - */ - public equals(other: BaseToken): other is typeof this { - if (other.constructor !== this.constructor) { - return false; - } - - if (this.text.length !== other.text.length) { - return false; - } - - if (this.text !== other.text) { - return false; - } - - return this.sameRange(other.range); - } - - /** - * Change `range` of the token with provided range components. - */ - public withRange(components: Partial): this { - this.tokenRange = new Range( - components.startLineNumber ?? this.range.startLineNumber, - components.startColumn ?? this.range.startColumn, - components.endLineNumber ?? this.range.endLineNumber, - components.endColumn ?? this.range.endColumn, - ); - - return this; - } - - /** - * Collapse range of the token to its start position. - * See {@link Range.collapseToStart} for more details. - */ - public collapseRangeToStart(): this { - this.tokenRange = this.tokenRange.collapseToStart(); - - return this; - } - - /** - * Render a list of tokens into a string. - */ - public static render( - tokens: readonly BaseToken[], - delimiter: string = '', - ): string { - return tokens.map(token => token.text).join(delimiter); - } - - /** - * Returns the full range of a list of tokens in which the first token is - * used as the start of a tokens sequence and the last token reflects the end. - * - * @throws if: - * - provided {@link tokens} list is empty - * - the first token start number is greater than the start line of the last token - * - if the first and last token are on the same line, the first token start column must - * be smaller than the start column of the last token - */ - public static fullRange(tokens: readonly BaseToken[]): Range { - assert( - tokens.length > 0, - 'Cannot get full range for an empty list of tokens.', - ); - - const firstToken = tokens[0]; - const lastToken = tokens[tokens.length - 1]; - - // sanity checks for the full range we would construct - assert( - firstToken.range.startLineNumber <= lastToken.range.startLineNumber, - 'First token must start on previous or the same line as the last token.', - ); - - if ((firstToken !== lastToken) && (firstToken.range.startLineNumber === lastToken.range.startLineNumber)) { - assert( - firstToken.range.endColumn <= lastToken.range.startColumn, - [ - 'First token must end at least on previous or the same column as the last token.', - `First token: ${firstToken}; Last token: ${lastToken}.`, - ].join('\n'), - ); - } - - return new Range( - firstToken.range.startLineNumber, - firstToken.range.startColumn, - lastToken.range.endLineNumber, - lastToken.range.endColumn, - ); - } - - /** - * Shorten version of the {@link text} property. - */ - public shortText( - maxLength: number = 32, - ): string { - if (this.text.length <= maxLength) { - return this.text; - } - - return `${this.text.slice(0, maxLength - 1)}...`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/compositeToken.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/compositeToken.ts deleted file mode 100644 index e4eb1470a03..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/compositeToken.ts +++ /dev/null @@ -1,63 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from './baseToken.js'; - -/** - * Composite token consists of a list of other tokens. - * Composite token consists of a list of other tokens. - */ -export abstract class CompositeToken< - TTokens extends readonly BaseToken[], -> extends BaseToken { - /** - * Reference to the list of child tokens. - */ - protected readonly childTokens: [...TTokens]; - - constructor( - tokens: TTokens, - ) { - super(BaseToken.fullRange(tokens)); - - this.childTokens = [...tokens]; - } - - public override get text(): string { - return BaseToken.render(this.childTokens); - } - - /** - * Tokens that this composite token consists of. - */ - public get children(): TTokens { - return this.childTokens; - } - - /** - * Check if this token is equal to another one, - * including all of its child tokens. - */ - public override equals(other: BaseToken): other is typeof this { - if (super.equals(other) === false) { - return false; - } - - if (this.children.length !== other.children.length) { - return false; - } - - for (let i = 0; i < this.children.length; i++) { - const childToken = this.children[i]; - const otherChildToken = other.children[i]; - - if (childToken.equals(otherChildToken) === false) { - return false; - } - } - - return true; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/constants.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/constants.ts deleted file mode 100644 index 779e53a1ef1..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/constants.ts +++ /dev/null @@ -1,16 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { NewLine } from '../linesCodec/tokens/newLine.js'; -import { CarriageReturn } from '../linesCodec/tokens/carriageReturn.js'; -import { FormFeed, SpacingToken } from '../simpleCodec/tokens/tokens.js'; - -/** - * List of valid "space" tokens that are valid between different - * records of a Front Matter header. - */ -export const VALID_INTER_RECORD_SPACING_TOKENS = Object.freeze([ - SpacingToken, CarriageReturn, NewLine, FormFeed, -]); diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/frontMatterDecoder.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/frontMatterDecoder.ts deleted file mode 100644 index 704e10f1754..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/frontMatterDecoder.ts +++ /dev/null @@ -1,156 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { Word } from '../simpleCodec/tokens/tokens.js'; -import { assert } from '../../../../../../../../base/common/assert.js'; -import { VSBuffer } from '../../../../../../../../base/common/buffer.js'; -import { VALID_INTER_RECORD_SPACING_TOKENS } from './constants.js'; -import { ReadableStream } from '../../../../../../../../base/common/stream.js'; -import { FrontMatterToken, FrontMatterRecord } from './tokens/index.js'; -import { BaseDecoder } from '../baseDecoder.js'; -import { SimpleDecoder, type TSimpleDecoderToken } from '../simpleCodec/simpleDecoder.js'; -import { ObjectStream } from '../utils/objectStream.js'; -import { PartialFrontMatterRecordNameWithDelimiter } from './parsers/frontMatterRecord/frontMatterRecordNameWithDelimiter.js'; -import { PartialFrontMatterRecord } from './parsers/frontMatterRecord/frontMatterRecord.js'; -import { PartialFrontMatterRecordName } from './parsers/frontMatterRecord/frontMatterRecordName.js'; -import { FrontMatterParserFactory } from './parsers/frontMatterParserFactory.js'; - -/** - * Tokens produced by this decoder. - */ -export type TFrontMatterToken = FrontMatterRecord | TSimpleDecoderToken; - -/** - * Decoder capable of parsing Front Matter contents from a sequence of simple tokens. - */ -export class FrontMatterDecoder extends BaseDecoder { - /** - * Current parser reference responsible for parsing a specific sequence - * of tokens into a standalone token. - */ - private current?: PartialFrontMatterRecordName | PartialFrontMatterRecordNameWithDelimiter | PartialFrontMatterRecord; - - private readonly parserFactory: FrontMatterParserFactory; - - constructor( - stream: ReadableStream | ObjectStream, - ) { - if (stream instanceof ObjectStream) { - super(stream); - } else { - super(new SimpleDecoder(stream)); - } - this.parserFactory = new FrontMatterParserFactory(); - } - - protected override onStreamData(token: TSimpleDecoderToken): void { - if (this.current !== undefined) { - const acceptResult = this.current.accept(token); - const { result, wasTokenConsumed } = acceptResult; - - if (result === 'failure') { - this.reEmitCurrentTokens(); - - if (wasTokenConsumed === false) { - this._onData.fire(token); - } - - delete this.current; - return; - } - - const { nextParser } = acceptResult; - - if (nextParser instanceof FrontMatterToken) { - // front matter record token is the spacial case - because it can - // contain trailing space tokens, we want to emit "trimmed" record - // token and the trailing spaces tokens separately - const trimmedTokens = (nextParser instanceof FrontMatterRecord) - ? nextParser.trimValueEnd() - : []; - - this._onData.fire(nextParser); - - // re-emit all trailing space tokens if present - for (const trimmedToken of trimmedTokens) { - this._onData.fire(trimmedToken); - } - - if (wasTokenConsumed === false) { - this._onData.fire(token); - } - - delete this.current; - return; - } - - this.current = nextParser; - if (wasTokenConsumed === false) { - this._onData.fire(token); - } - - return; - } - - // a word token starts a new record - if (token instanceof Word) { - this.current = this.parserFactory.createRecordName(token); - return; - } - - // re-emit all "space" tokens immediately as all of them - // are valid while we are not in the "record parsing" mode - for (const ValidToken of VALID_INTER_RECORD_SPACING_TOKENS) { - if (token instanceof ValidToken) { - this._onData.fire(token); - return; - } - } - - // unexpected token type, re-emit existing tokens and continue - this.reEmitCurrentTokens(); - } - - protected override onStreamEnd(): void { - try { - if (this.current === undefined) { - return; - } - - assert( - this.current instanceof PartialFrontMatterRecord, - 'Only partial front matter records can be processed on stream end.', - ); - - const record = this.current.asRecordToken(); - const trimmedTokens = record.trimValueEnd(); - - this._onData.fire(record); - - for (const trimmedToken of trimmedTokens) { - this._onData.fire(trimmedToken); - } - } catch (_error) { - this.reEmitCurrentTokens(); - } finally { - delete this.current; - super.onStreamEnd(); - } - } - - /** - * Re-emit tokens accumulated so far in the current parser object. - */ - protected reEmitCurrentTokens(): void { - if (this.current === undefined) { - return; - } - - for (const token of this.current.tokens) { - this._onData.fire(token); - } - delete this.current; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterArray.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterArray.ts deleted file mode 100644 index df84fa012c7..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterArray.ts +++ /dev/null @@ -1,197 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { assert } from '../../../../../../../../../base/common/assert.js'; -import { type PartialFrontMatterValue } from './frontMatterValue.js'; -import { FrontMatterArray } from '../tokens/frontMatterArray.js'; -import { assertDefined } from '../../../../../../../../../base/common/types.js'; -import { VALID_INTER_RECORD_SPACING_TOKENS } from '../constants.js'; -import { FrontMatterValueToken } from '../tokens/frontMatterToken.js'; -import { FrontMatterSequence } from '../tokens/frontMatterSequence.js'; -import { TSimpleDecoderToken } from '../../simpleCodec/simpleDecoder.js'; -import { Comma, LeftBracket, RightBracket } from '../../simpleCodec/tokens/tokens.js'; -import { assertNotConsumed, ParserBase, TAcceptTokenResult } from '../../simpleCodec/parserBase.js'; -import { type FrontMatterParserFactory } from './frontMatterParserFactory.js'; - -/** - * List of tokens that can go in-between array items - * and array brackets. - */ -const VALID_DELIMITER_TOKENS = Object.freeze([ - ...VALID_INTER_RECORD_SPACING_TOKENS, - Comma, -]); - -/** - * Responsible for parsing an array syntax (or "inline sequence" - * in YAML terms), e.g. `[1, '2', true, 2.54]` -*/ -export class PartialFrontMatterArray extends ParserBase { - /** - * Current parser reference responsible for parsing an array "value". - */ - private currentValueParser?: PartialFrontMatterValue; - - /** - * Whether an array item is allowed in the current position of the token - * sequence. E.g., items are allowed after a command or a open bracket, - * but not immediately after another item in the array. - */ - private arrayItemAllowed = true; - - constructor( - private readonly factory: FrontMatterParserFactory, - private readonly startToken: LeftBracket, - ) { - super([startToken]); - } - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - if (this.currentValueParser !== undefined) { - const acceptResult = this.currentValueParser.accept(token); - const { result, wasTokenConsumed } = acceptResult; - - if (result === 'failure') { - this.isConsumed = true; - - return { - result: 'failure', - wasTokenConsumed, - }; - } - - const { nextParser } = acceptResult; - - if (nextParser instanceof FrontMatterValueToken) { - this.currentTokens.push(nextParser); - delete this.currentValueParser; - - // if token was not consume, call the `accept()` method - // recursively so that the current parser can re-process - // the token (e.g., a comma or a closing square bracket) - if (wasTokenConsumed === false) { - return this.accept(token); - } - - return { - result: 'success', - nextParser: this, - wasTokenConsumed, - }; - } - - this.currentValueParser = nextParser; - return { - result: 'success', - nextParser: this, - wasTokenConsumed, - }; - } - - if (token instanceof RightBracket) { - // sanity check in case this block moves around - // to a different place in the code - assert( - this.currentValueParser === undefined, - `Unexpected end of array. Last value is not finished.`, - ); - - this.currentTokens.push(token); - - this.isConsumed = true; - return { - result: 'success', - nextParser: this.asArrayToken(), - wasTokenConsumed: true, - }; - } - - // iterate until a valid value start token is found - for (const ValidToken of VALID_DELIMITER_TOKENS) { - if (token instanceof ValidToken) { - this.currentTokens.push(token); - - if ((this.arrayItemAllowed === false) && token instanceof Comma) { - this.arrayItemAllowed = true; - } - - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } - } - - // is an array item value is allowed at this position, create a new - // value parser and start the value parsing process using it - if (this.arrayItemAllowed === true) { - this.currentValueParser = this.factory.createValue( - (currentToken) => { - // comma or a closing square bracket must stop the parsing - // process of the value represented by a generic sequence of tokens - return ( - (currentToken instanceof RightBracket) - || (currentToken instanceof Comma) - ); - }, - ); - this.arrayItemAllowed = false; - - return this.accept(token); - } - - // in all other cases fail because of the unexpected token type - this.isConsumed = true; - return { - result: 'failure', - wasTokenConsumed: false, - }; - } - - /** - * Convert current parser into a {@link FrontMatterArray} token, - * if possible. - * - * @throws if the last token in the accumulated token list - * is not a closing bracket ({@link RightBracket}). - */ - public asArrayToken(): FrontMatterArray { - const endToken = this.currentTokens[this.currentTokens.length - 1]; - - assertDefined( - endToken, - 'No tokens found.', - ); - - assert( - endToken instanceof RightBracket, - 'Cannot find a closing bracket of the array.', - ); - - const valueTokens: FrontMatterValueToken[] = []; - for (const currentToken of this.currentTokens) { - if ((currentToken instanceof FrontMatterValueToken) === false) { - continue; - } - - // the generic sequence tokens can have trailing spacing tokens, - // hence trim them to ensure the array contains only "clean" values - if (currentToken instanceof FrontMatterSequence) { - currentToken.trimEnd(); - } - - valueTokens.push(currentToken); - } - - this.isConsumed = true; - return new FrontMatterArray([ - this.startToken, - ...valueTokens, - endToken, - ]); - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterParserFactory.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterParserFactory.ts deleted file mode 100644 index 0be65371449..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterParserFactory.ts +++ /dev/null @@ -1,41 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../../baseToken.js'; -import { LeftBracket } from '../../simpleCodec/tokens/brackets.js'; -import { Word } from '../../simpleCodec/tokens/word.js'; -import { FrontMatterRecordDelimiter, FrontMatterRecordName } from '../tokens/frontMatterRecord.js'; -import { TQuoteToken } from '../tokens/frontMatterString.js'; -import { PartialFrontMatterArray } from './frontMatterArray.js'; -import { PartialFrontMatterRecord } from './frontMatterRecord/frontMatterRecord.js'; -import { PartialFrontMatterRecordName } from './frontMatterRecord/frontMatterRecordName.js'; -import { PartialFrontMatterRecordNameWithDelimiter, TNameStopToken } from './frontMatterRecord/frontMatterRecordNameWithDelimiter.js'; -import { PartialFrontMatterSequence } from './frontMatterSequence.js'; -import { PartialFrontMatterString } from './frontMatterString.js'; -import { PartialFrontMatterValue } from './frontMatterValue.js'; - -export class FrontMatterParserFactory { - createRecord(tokens: [FrontMatterRecordName, FrontMatterRecordDelimiter]): PartialFrontMatterRecord { - return new PartialFrontMatterRecord(this, tokens); - } - createRecordName(startToken: Word): PartialFrontMatterRecordName { - return new PartialFrontMatterRecordName(this, startToken); - } - createRecordNameWithDelimiter(tokens: readonly [FrontMatterRecordName, TNameStopToken]): PartialFrontMatterRecordNameWithDelimiter { - return new PartialFrontMatterRecordNameWithDelimiter(this, tokens); - } - createArray(startToken: LeftBracket) { - return new PartialFrontMatterArray(this, startToken); - } - createValue(shouldStop: (token: BaseToken) => boolean): PartialFrontMatterValue { - return new PartialFrontMatterValue(this, shouldStop); - } - createString(startToken: TQuoteToken): PartialFrontMatterString { - return new PartialFrontMatterString(startToken); - } - createSequence(shouldStop: (token: BaseToken) => boolean): PartialFrontMatterSequence { - return new PartialFrontMatterSequence(shouldStop); - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterRecord/frontMatterRecord.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterRecord/frontMatterRecord.ts deleted file mode 100644 index e4afee055b5..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterRecord/frontMatterRecord.ts +++ /dev/null @@ -1,210 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../../../baseToken.js'; -import { NewLine } from '../../../linesCodec/tokens/newLine.js'; -import { PartialFrontMatterValue } from '../frontMatterValue.js'; -import { assertNever } from '../../../../../../../../../../base/common/assert.js'; -import { assertDefined } from '../../../../../../../../../../base/common/types.js'; -import { PartialFrontMatterSequence } from '../frontMatterSequence.js'; -import { CarriageReturn } from '../../../linesCodec/tokens/carriageReturn.js'; -import { type TSimpleDecoderToken } from '../../../simpleCodec/simpleDecoder.js'; -import { Word, FormFeed, SpacingToken } from '../../../simpleCodec/tokens/tokens.js'; -import { assertNotConsumed, ParserBase, type TAcceptTokenResult } from '../../../simpleCodec/parserBase.js'; -import { FrontMatterValueToken, FrontMatterRecordName, FrontMatterRecordDelimiter, FrontMatterRecord } from '../../tokens/index.js'; -import { type FrontMatterParserFactory } from '../frontMatterParserFactory.js'; - -/** - * Type of a next parser that can be returned by {@link PartialFrontMatterRecord}. - */ -type TNextParser = PartialFrontMatterRecord | FrontMatterRecord; - -/** - * Parser for a `record` inside a Front Matter header. - * - * * E.g., `name: 'value'` in the example below: - * - * ``` - * --- - * name: 'value' - * isExample: true - * --- - * ``` - */ -export class PartialFrontMatterRecord extends ParserBase { - /** - * Token that represents the 'name' part of the record. - */ - private readonly recordNameToken: FrontMatterRecordName; - - /** - * Token that represents the 'delimiter' part of the record. - */ - private readonly recordDelimiterToken: FrontMatterRecordDelimiter; - - constructor( - private readonly factory: FrontMatterParserFactory, - tokens: [FrontMatterRecordName, FrontMatterRecordDelimiter], - ) { - super(tokens); - this.recordNameToken = tokens[0]; - this.recordDelimiterToken = tokens[1]; - } - - /** - * Current parser reference responsible for parsing the "value" part of the record. - */ - private valueParser?: PartialFrontMatterValue | PartialFrontMatterSequence; - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - if (this.valueParser !== undefined) { - const acceptResult = this.valueParser.accept(token); - const { result, wasTokenConsumed } = acceptResult; - - if (result === 'failure') { - this.isConsumed = true; - - return { - result: 'failure', - wasTokenConsumed, - }; - } - - const { nextParser } = acceptResult; - - if (nextParser instanceof FrontMatterValueToken) { - this.currentTokens.push(nextParser); - delete this.valueParser; - - this.isConsumed = true; - try { - return { - result: 'success', - nextParser: new FrontMatterRecord([ - this.recordNameToken, - this.recordDelimiterToken, - nextParser, - ]), - wasTokenConsumed, - }; - } catch (_error) { - return { - result: 'failure', - wasTokenConsumed, - }; - } - } - - this.valueParser = nextParser; - return { - result: 'success', - nextParser: this, - wasTokenConsumed, - }; - } - - // iterate until the first non-space token is found - if (token instanceof SpacingToken) { - this.currentTokens.push(token); - - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } - - // if token can start a "value" sequence, parse the value - if (PartialFrontMatterValue.isValueStartToken(token)) { - this.valueParser = this.factory.createValue(shouldEndTokenSequence); - - return this.accept(token); - } - - // in all other cases, collect all the subsequent tokens into - // a "sequence of tokens" until a new line is found - this.valueParser = this.factory.createSequence( - shouldEndTokenSequence, - ); - - // if we reached this "generic sequence" parser point, but the current token is - // already of a type that stops such sequence, we must have accumulated some - // spacing tokens, hence pass those to the parser and end the sequence immediately - if (shouldEndTokenSequence(token)) { - const spaceTokens = this.currentTokens - .slice(this.startTokensCount); - - // if no space tokens accumulated at all, create an "empty" one this is needed - // to ensure that the parser always has at least one token hence it can have - // a valid range and can be interpreted as a real "value" token of the record - if (spaceTokens.length === 0) { - spaceTokens.push( - Word.newOnLine( - '', - token.range.startLineNumber, - token.range.startColumn, - ), - ); - } - - this.valueParser.addTokens(spaceTokens); - - return { - result: 'success', - nextParser: this.asRecordToken(), - wasTokenConsumed: false, - }; - } - - // otherwise use the "generic sequence" parser moving on - return this.accept(token); - } - - /** - * Convert current parser into a {@link FrontMatterRecord} token. - * - * @throws if no current parser is present, or it is not of the {@link PartialFrontMatterValue} - * or {@link PartialFrontMatterSequence} types - */ - public asRecordToken(): FrontMatterRecord { - assertDefined( - this.valueParser, - 'Current value parser must be defined.' - ); - - if ( - (this.valueParser instanceof PartialFrontMatterValue) - || (this.valueParser instanceof PartialFrontMatterSequence) - ) { - const valueToken = this.valueParser.asSequenceToken(); - this.currentTokens.push(valueToken); - - this.isConsumed = true; - return new FrontMatterRecord([ - this.recordNameToken, - this.recordDelimiterToken, - valueToken, - ]); - } - - assertNever( - this.valueParser, - `Unexpected value parser '${this.valueParser}'.`, - ); - } -} - -/** - * Callback to check if a current token should end a - * record value that is a generic sequence of tokens. - */ -function shouldEndTokenSequence(token: BaseToken): token is (NewLine | CarriageReturn | FormFeed) { - return ( - (token instanceof NewLine) - || (token instanceof CarriageReturn) - || (token instanceof FormFeed) - ); -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterRecord/frontMatterRecordName.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterRecord/frontMatterRecordName.ts deleted file mode 100644 index 84c94b75211..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterRecord/frontMatterRecordName.ts +++ /dev/null @@ -1,74 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { type TSimpleDecoderToken } from '../../../simpleCodec/simpleDecoder.js'; -import { FrontMatterRecordName, type TRecordNameToken } from '../../tokens/index.js'; -import { Colon, Word, Dash, SpacingToken } from '../../../simpleCodec/tokens/tokens.js'; -import { type PartialFrontMatterRecordNameWithDelimiter } from './frontMatterRecordNameWithDelimiter.js'; -import { assertNotConsumed, ParserBase, type TAcceptTokenResult } from '../../../simpleCodec/parserBase.js'; -import { type FrontMatterParserFactory } from '../frontMatterParserFactory.js'; - -/** - * Tokens that can be used inside a record name. - */ -const VALID_NAME_TOKENS = [Word, Dash]; - -/** - * Type of a next parser that can be returned by {@link PartialFrontMatterRecordName}. - */ -type TNextParser = PartialFrontMatterRecordName | PartialFrontMatterRecordNameWithDelimiter; - -/** - * Parser for a `name` part of a Front Matter record. - * - * E.g., `'name'` in the example below: - * - * ``` - * name: 'value' - * ``` - */ -export class PartialFrontMatterRecordName extends ParserBase { - constructor( - private readonly factory: FrontMatterParserFactory, - startToken: Word, - ) { - super([startToken]); - } - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - for (const ValidToken of VALID_NAME_TOKENS) { - if (token instanceof ValidToken) { - this.currentTokens.push(token); - - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } - } - - // once name is followed by a "space" token or a "colon", we have the full - // record name hence can transition to the next parser - if ((token instanceof Colon) || (token instanceof SpacingToken)) { - const recordName = new FrontMatterRecordName(this.currentTokens); - - this.isConsumed = true; - return { - result: 'success', - nextParser: this.factory.createRecordNameWithDelimiter([recordName, token]), - wasTokenConsumed: true, - }; - } - - // in all other cases fail due to the unexpected token type for a record name - this.isConsumed = true; - return { - result: 'failure', - wasTokenConsumed: false, - }; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterRecord/frontMatterRecordNameWithDelimiter.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterRecord/frontMatterRecordNameWithDelimiter.ts deleted file mode 100644 index 59735860631..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterRecord/frontMatterRecordNameWithDelimiter.ts +++ /dev/null @@ -1,105 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { assert } from '../../../../../../../../../../base/common/assert.js'; -import { type PartialFrontMatterRecord } from './frontMatterRecord.js'; -import { Colon, SpacingToken } from '../../../simpleCodec/tokens/tokens.js'; -import { type TSimpleDecoderToken } from '../../../simpleCodec/simpleDecoder.js'; -import { FrontMatterRecordName, FrontMatterRecordDelimiter } from '../../tokens/index.js'; -import { assertNotConsumed, ParserBase, type TAcceptTokenResult } from '../../../simpleCodec/parserBase.js'; -import { type FrontMatterParserFactory } from '../frontMatterParserFactory.js'; - -/** - * Type for tokens that stop a front matter record name sequence. - */ -export type TNameStopToken = Colon | SpacingToken; - -/** - * Type for the next parser that can be returned by {@link PartialFrontMatterRecordNameWithDelimiter}. - */ -type TNextParser = PartialFrontMatterRecordNameWithDelimiter | PartialFrontMatterRecord; - -/** - * Parser for a record `name` with the `: ` delimiter. - * - * * E.g., `name:` in the example below: - * - * ``` - * name: 'value' - * ``` - */ -export class PartialFrontMatterRecordNameWithDelimiter extends ParserBase< - FrontMatterRecordName | TNameStopToken, - TNextParser -> { - constructor( - private readonly factory: FrontMatterParserFactory, - tokens: readonly [FrontMatterRecordName, TNameStopToken], - ) { - super([...tokens]); - } - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - const previousToken = this.currentTokens[this.currentTokens.length - 1]; - const isSpacingToken = (token instanceof SpacingToken); - - // delimiter must always be a `:` followed by a "space" character - // once we encounter that sequence, we can transition to the next parser - if (isSpacingToken && (previousToken instanceof Colon)) { - const recordDelimiter = new FrontMatterRecordDelimiter([ - previousToken, - token, - ]); - - const recordName = this.currentTokens[0]; - - // sanity check - assert( - recordName instanceof FrontMatterRecordName, - `Expected a front matter record name, got '${recordName}'.`, - ); - - this.isConsumed = true; - return { - result: 'success', - nextParser: this.factory.createRecord( - [recordName, recordDelimiter], - ), - wasTokenConsumed: true, - }; - } - - // allow some spacing before the colon delimiter - if (token instanceof SpacingToken) { - this.currentTokens.push(token); - - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } - - // include the colon delimiter - if (token instanceof Colon) { - this.currentTokens.push(token); - - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } - - // otherwise fail due to the unexpected token type between - // record name and record name delimiter tokens - this.isConsumed = true; - return { - result: 'failure', - wasTokenConsumed: false, - }; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterSequence.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterSequence.ts deleted file mode 100644 index 0d91c32528c..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterSequence.ts +++ /dev/null @@ -1,78 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../../baseToken.js'; -import { FrontMatterSequence } from '../tokens/frontMatterSequence.js'; -import { TSimpleDecoderToken } from '../../simpleCodec/simpleDecoder.js'; -import { assertNotConsumed, ParserBase, TAcceptTokenResult } from '../../simpleCodec/parserBase.js'; - -/** - * Parser responsible for parsing a "generic sequence of tokens" - * of an arbitrary length in a Front Matter header. - */ -export class PartialFrontMatterSequence extends ParserBase< - TSimpleDecoderToken, - PartialFrontMatterSequence | FrontMatterSequence -> { - constructor( - /** - * Callback function that is called to check if the current token - * should stop the parsing process of the current generic "value" - * sequence of arbitrary tokens by returning `true`. - * - * When this happens, the parser *will not consume* the token that - * was passed to the `shouldStop` callback or to its `accept` method. - * On the other hand, the parser will be "consumed" hence using it - * to process other tokens will yield an error. - */ - private readonly shouldStop: (token: BaseToken) => boolean, - ) { - super([]); - } - - @assertNotConsumed - public accept( - token: TSimpleDecoderToken, - ): TAcceptTokenResult { - - // collect all tokens until an end of the sequence is found - if (this.shouldStop(token)) { - this.isConsumed = true; - - return { - result: 'success', - nextParser: this.asSequenceToken(), - wasTokenConsumed: false, - }; - } - - this.currentTokens.push(token); - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } - - /** - * Add provided tokens to the list of the current parsed tokens. - */ - public addTokens( - tokens: readonly TSimpleDecoderToken[], - ): this { - this.currentTokens.push(...tokens); - - return this; - } - - /** - * Convert the current parser into a {@link FrontMatterSequence} token. - */ - public asSequenceToken(): FrontMatterSequence { - this.isConsumed = true; - - return new FrontMatterSequence(this.currentTokens); - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterString.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterString.ts deleted file mode 100644 index d0ed92746d6..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterString.ts +++ /dev/null @@ -1,69 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { assert } from '../../../../../../../../../base/common/assert.js'; -import { SimpleToken } from '../../simpleCodec/tokens/tokens.js'; -import { assertDefined } from '../../../../../../../../../base/common/types.js'; -import { TSimpleDecoderToken } from '../../simpleCodec/simpleDecoder.js'; -import { FrontMatterString, type TQuoteToken } from '../tokens/frontMatterString.js'; -import { assertNotConsumed, ParserBase, type TAcceptTokenResult } from '../../simpleCodec/parserBase.js'; - -/** - * Parser responsible for parsing a string value. - */ -export class PartialFrontMatterString extends ParserBase> { - constructor( - private readonly startToken: TQuoteToken, - ) { - super([startToken]); - } - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult> { - this.currentTokens.push(token); - - // iterate until a `matching end quote` is found - if ((token instanceof SimpleToken) && (this.startToken.sameType(token))) { - return { - result: 'success', - nextParser: this.asStringToken(), - wasTokenConsumed: true, - }; - } - - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } - - /** - * Convert the current parser into a {@link FrontMatterString} token, - * if possible. - * - * @throws if the first and last tokens are not quote tokens of the same type. - */ - public asStringToken(): FrontMatterString { - const endToken = this.currentTokens[this.currentTokens.length - 1]; - - assertDefined( - endToken, - `No matching end token found.`, - ); - - assert( - this.startToken.sameType(endToken), - `String starts with \`${this.startToken.text}\`, but ends with \`${endToken.text}\`.`, - ); - - return new FrontMatterString([ - this.startToken, - ...this.currentTokens - .slice(1, this.currentTokens.length - 1), - endToken, - ]); - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterValue.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterValue.ts deleted file mode 100644 index 710767aed40..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterValue.ts +++ /dev/null @@ -1,184 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../../baseToken.js'; -import { type PartialFrontMatterArray } from './frontMatterArray.js'; -import { type PartialFrontMatterString } from './frontMatterString.js'; -import { asBoolean, FrontMatterBoolean } from '../tokens/frontMatterBoolean.js'; -import { FrontMatterValueToken } from '../tokens/frontMatterToken.js'; -import { PartialFrontMatterSequence } from './frontMatterSequence.js'; -import { FrontMatterSequence } from '../tokens/frontMatterSequence.js'; -import { TSimpleDecoderToken } from '../../simpleCodec/simpleDecoder.js'; -import { Word, Quote, DoubleQuote, LeftBracket } from '../../simpleCodec/tokens/tokens.js'; -import { assertNotConsumed, ParserBase, TAcceptTokenResult } from '../../simpleCodec/parserBase.js'; -import { type FrontMatterParserFactory } from './frontMatterParserFactory.js'; - -/** - * List of tokens that can start a "value" sequence. - * - * - {@link Word} - can be a `boolean` value - * - {@link Quote}, {@link DoubleQuote} - can start a `string` value - * - {@link LeftBracket} - can start an `array` value - */ -export const VALID_VALUE_START_TOKENS = Object.freeze([ - Quote, - DoubleQuote, - LeftBracket, -]); - -/** - * Type alias for a token that can start a "value" sequence. - */ -type TValueStartToken = InstanceType; - -/** - * Parser responsible for parsing a "value" sequence in a Front Matter header. - */ -export class PartialFrontMatterValue extends ParserBase { - /** - * Current parser reference responsible for parsing - * a specific "value" sequence. - */ - private currentValueParser?: PartialFrontMatterString | PartialFrontMatterArray | PartialFrontMatterSequence; - - /** - * Get the tokens that were accumulated so far. - */ - public override get tokens(): readonly TSimpleDecoderToken[] { - if (this.currentValueParser === undefined) { - return []; - } - - return this.currentValueParser.tokens; - } - - constructor( - private readonly factory: FrontMatterParserFactory, - /** - * Callback function to pass to the {@link PartialFrontMatterSequence} - * if the current "value" sequence is not of a specific type. - */ - private readonly shouldStop: (token: BaseToken) => boolean, - ) { - super(); - } - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - if (this.currentValueParser !== undefined) { - const acceptResult = this.currentValueParser.accept(token); - const { result, wasTokenConsumed } = acceptResult; - - // current value parser is consumed with its child value parser - this.isConsumed = this.currentValueParser.consumed; - - if (result === 'success') { - const { nextParser } = acceptResult; - - if (nextParser instanceof FrontMatterValueToken) { - return { - result: 'success', - nextParser, - wasTokenConsumed, - }; - } - - this.currentValueParser = nextParser; - return { - result: 'success', - nextParser: this, - wasTokenConsumed, - }; - } - - return { - result: 'failure', - wasTokenConsumed, - }; - } - - // if the first token represents a `quote` character, try to parse a string value - if ((token instanceof Quote) || (token instanceof DoubleQuote)) { - this.currentValueParser = this.factory.createString(token); - - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } - - // if the first token represents a `[` character, try to parse an array value - if (token instanceof LeftBracket) { - this.currentValueParser = this.factory.createArray(token); - - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } - - // if the first token represents a `word` try to parse a boolean - const maybeBoolean = FrontMatterBoolean.tryFromToken(token); - if (maybeBoolean !== null) { - this.isConsumed = true; - - return { - result: 'success', - nextParser: maybeBoolean, - wasTokenConsumed: true, - }; - } - - // in all other cases, collect all the subsequent tokens into - // a generic sequence of tokens until stopped by the `this.shouldStop` - // callback or the call to the 'this.asSequenceToken' method - this.currentValueParser = this.factory.createSequence(this.shouldStop); - - return this.accept(token); - } - - /** - * Check if provided token can be a start of a "value" sequence. - * See {@link VALID_VALUE_START_TOKENS} for the list of valid tokens. - */ - public static isValueStartToken( - token: BaseToken, - ): token is TValueStartToken | Word<'true' | 'false'> { - for (const ValidToken of VALID_VALUE_START_TOKENS) { - if (token instanceof ValidToken) { - return true; - } - } - - if ((token instanceof Word) && (asBoolean(token) !== null)) { - return true; - } - - return false; - } - - /** - * Check if the current 'value' sequence does not have a specific type - * and is represented by a generic sequence of tokens ({@link PartialFrontMatterSequence}). - */ - public get isSequence(): boolean { - if (this.currentValueParser === undefined) { - return false; - } - - return (this.currentValueParser instanceof PartialFrontMatterSequence); - } - - /** - * Convert current parser into a generic sequence of tokens. - */ - public asSequenceToken(): FrontMatterSequence { - this.isConsumed = true; - - return new FrontMatterSequence(this.tokens); - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterArray.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterArray.ts deleted file mode 100644 index b8b01e42c79..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterArray.ts +++ /dev/null @@ -1,43 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../../baseToken.js'; -import { LeftBracket, RightBracket } from '../../simpleCodec/tokens/tokens.js'; -import { FrontMatterValueToken, type TValueTypeName } from './frontMatterToken.js'; - -/** - * Token that represents an `array` value in a Front Matter header. - */ -export class FrontMatterArray extends FrontMatterValueToken<'array', [ - LeftBracket, - ...FrontMatterValueToken[], - RightBracket, -]> { - /** - * Name of the `array` value type. - */ - public override readonly valueTypeName = 'array'; - - /** - * List of the array items. - */ - public get items(): readonly FrontMatterValueToken[] { - const result = []; - - for (const token of this.children) { - if (token instanceof FrontMatterValueToken) { - result.push(token); - } - } - - return result; - } - - public override toString(): string { - const itemsString = BaseToken.render(this.items, ', '); - - return `front-matter-array(${itemsString})${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterBoolean.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterBoolean.ts deleted file mode 100644 index 3eae6d114ed..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterBoolean.ts +++ /dev/null @@ -1,88 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../../baseToken.js'; -import { Word } from '../../simpleCodec/tokens/tokens.js'; -import { FrontMatterValueToken } from './frontMatterToken.js'; -import { assertDefined } from '../../../../../../../../../base/common/types.js'; - -/** - * Token that represents a `boolean` value in a Front Matter header. - */ -export class FrontMatterBoolean extends FrontMatterValueToken<'boolean', readonly [Word]> { - /** - * Name of the `boolean` value type. - */ - public override readonly valueTypeName = 'boolean'; - - /** - * Value of the `boolean` token. - */ - public readonly value: boolean; - - /** - * @throws if provided {@link Word} cannot be converted to a `boolean` value. - */ - constructor(token: Word) { - const value = asBoolean(token); - assertDefined( - value, - `Cannot convert '${token}' to a boolean value.`, - ); - - super([token]); - - this.value = value; - } - - /** - * Try creating a {@link FrontMatterBoolean} out of provided token. - * Unlike the constructor, this method does not throw, returning - * a 'null' value on failure instead. - */ - public static tryFromToken( - token: BaseToken, - ): FrontMatterBoolean | null { - if (token instanceof Word === false) { - return null; - } - - try { - return new FrontMatterBoolean(token); - } catch (_error) { - // noop - return null; - } - } - - public override equals(other: BaseToken): other is typeof this { - if (super.equals(other) === false) { - return false; - } - - return this.value === other.value; - } - - public override toString(): string { - return `front-matter-boolean(${this.shortText()})${this.range}`; - } -} - -/** - * Try to convert a {@link Word} token to a `boolean` value. - */ -export function asBoolean( - token: Word, -): boolean | null { - if (token.text.toLowerCase() === 'true') { - return true; - } - - if (token.text.toLowerCase() === 'false') { - return false; - } - - return null; -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterRecord.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterRecord.ts deleted file mode 100644 index dc498c6d667..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterRecord.ts +++ /dev/null @@ -1,118 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../../baseToken.js'; -import { FrontMatterSequence } from './frontMatterSequence.js'; -import { Colon, Word, Dash, SpacingToken } from '../../simpleCodec/tokens/tokens.js'; -import { FrontMatterToken, FrontMatterValueToken, type TValueTypeName } from './frontMatterToken.js'; - -/** - * Type for tokens that can be used inside a record name. - */ -export type TNameToken = Word | Dash; - -/** - * Token representing a `record name` inside a Front Matter record. - * - * E.g., `name` in the example below: - * - * ``` - * --- - * name: 'value' - * --- - * ``` - */ -export class FrontMatterRecordName extends FrontMatterToken { - public override toString(): string { - return `front-matter-record-name(${this.shortText()})${this.range}`; - } -} - -/** - * Token representing a delimiter of a record inside a Front Matter header. - * - * E.g., `: ` in the example below: - * - * ``` - * --- - * name: 'value' - * --- - * ``` - */ -export class FrontMatterRecordDelimiter extends FrontMatterToken { - public override toString(): string { - return `front-matter-delimiter(${this.shortText()})${this.range}`; - } -} - -/** - * Token representing a `record` inside a Front Matter header. - * - * E.g., `name: 'value'` in the example below: - * - * ``` - * --- - * name: 'value' - * --- - * ``` - */ -export class FrontMatterRecord extends FrontMatterToken< - readonly [FrontMatterRecordName, FrontMatterRecordDelimiter, FrontMatterValueToken] -> { - /** - * Token that represent `name` of the record. - * - * E.g., `tools` in the example below: - * - * ``` - * --- - * tools: ['value'] - * --- - * ``` - */ - public get nameToken(): FrontMatterRecordName { - return this.children[0]; - } - - /** - * Token that represent `value` of the record. - * - * E.g., `['value']` in the example below: - * - * ``` - * --- - * tools: ['value'] - * --- - * ``` - */ - public get valueToken(): FrontMatterValueToken { - return this.children[2]; - } - - /** - * Trim spacing tokens at the end of the record. - */ - public trimValueEnd(): readonly SpacingToken[] { - const { valueToken } = this; - - // only the "generic sequence" value tokens can hold - // some spacing tokens at the end of them - if ((valueToken instanceof FrontMatterSequence) === false) { - return []; - } - - const trimmedTokens = valueToken.trimEnd(); - // update the current range to reflect the current trimmed value - this.withRange( - BaseToken.fullRange(this.children), - ); - - return trimmedTokens; - } - - public override toString(): string { - return `front-matter-record(${this.shortText()})${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterSequence.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterSequence.ts deleted file mode 100644 index dcb6f70d5a3..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterSequence.ts +++ /dev/null @@ -1,79 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../../baseToken.js'; -import { FrontMatterValueToken } from './frontMatterToken.js'; -import { Word, SpacingToken } from '../../simpleCodec/tokens/tokens.js'; -import { type TSimpleDecoderToken } from '../../simpleCodec/simpleDecoder.js'; - - -/** - * Token represents a generic sequence of tokens in a Front Matter header. - */ -export class FrontMatterSequence extends FrontMatterValueToken { - /** - * @override Because this token represent a generic sequence of tokens, - * the type name is represented by the sequence of tokens itself - */ - public override get valueTypeName(): this { - return this; - } - - /** - * Text of the sequence value. The method exists to provide a - * consistent interface with {@link FrontMatterString} token. - * - * Note! that this method does not automatically trim spacing tokens - * in the sequence. If you need to get a trimmed value, call - * {@link trimEnd} method first. - */ - public get cleanText(): string { - return this.text; - } - - /** - * Trim spacing tokens at the end of the sequence. - */ - public trimEnd(): readonly SpacingToken[] { - const trimmedTokens = []; - - // iterate the tokens list from the end to the start, collecting - // all the spacing tokens we encounter until we reach a non-spacing token - let lastNonSpace = this.childTokens.length - 1; - while (lastNonSpace >= 0) { - const token = this.childTokens[lastNonSpace]; - - if (token instanceof SpacingToken) { - trimmedTokens.push(token); - lastNonSpace--; - - continue; - } - - break; - } - this.childTokens.length = lastNonSpace + 1; - - // if there are only spacing tokens were present add a single - // empty token to the sequence, so it has something to work with - if (this.childTokens.length === 0) { - this.collapseRangeToStart(); - this.childTokens.push(new Word(this.range, '')); - } - - // update the current range to reflect the current trimmed value - this.withRange( - BaseToken.fullRange(this.childTokens), - ); - - // trimmed tokens are collected starting from the end, - // moving to the start, hence reverse them before returning - return trimmedTokens.reverse(); - } - - public override toString(): string { - return `front-matter-sequence(${this.shortText()})${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterString.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterString.ts deleted file mode 100644 index 8a7283d1098..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterString.ts +++ /dev/null @@ -1,39 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../../baseToken.js'; -import { FrontMatterValueToken } from './frontMatterToken.js'; -import { Quote, DoubleQuote } from '../../simpleCodec/tokens/tokens.js'; - -/** - * Type for any quote token that can be used to wrap a string. - */ -export type TQuoteToken = Quote | DoubleQuote; - -/** - * Token that represents a string value in a Front Matter header. - */ -export class FrontMatterString extends FrontMatterValueToken< - 'quoted-string', - readonly [TQuote, ...BaseToken[], TQuote] -> { - /** - * Name of the `string` value type. - */ - public override readonly valueTypeName = 'quoted-string'; - - /** - * Text of the string value without the wrapping quotes. - */ - public get cleanText(): string { - return BaseToken.render( - this.children.slice(1, this.children.length - 1), - ); - } - - public override toString(): string { - return `front-matter-string(${this.shortText()})${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterToken.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterToken.ts deleted file mode 100644 index d85f85230d7..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterToken.ts +++ /dev/null @@ -1,33 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../../baseToken.js'; -import { CompositeToken } from '../../compositeToken.js'; -import { FrontMatterSequence } from './frontMatterSequence.js'; - -/** - * Base class for all tokens inside a Front Matter header. - */ -export abstract class FrontMatterToken< - TTokens extends readonly BaseToken[] = readonly BaseToken[], -> extends CompositeToken { } - -/** - * List of all currently supported value types. - */ -export type TValueTypeName = 'quoted-string' | 'boolean' | 'array' | FrontMatterSequence; - -/** - * Base class for all tokens that represent a `value` inside a Front Matter header. - */ -export abstract class FrontMatterValueToken< - TTypeName extends TValueTypeName = TValueTypeName, - TTokens extends readonly BaseToken[] = readonly BaseToken[], -> extends FrontMatterToken { - /** - * Type name of the `value` represented by this token. - */ - public abstract readonly valueTypeName: TTypeName; -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/index.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/index.ts deleted file mode 100644 index 033a1250fa5..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/index.ts +++ /dev/null @@ -1,15 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -export { FrontMatterArray } from './frontMatterArray.js'; -export { FrontMatterString } from './frontMatterString.js'; -export { FrontMatterBoolean } from './frontMatterBoolean.js'; -export { FrontMatterToken, FrontMatterValueToken } from './frontMatterToken.js'; -export { - FrontMatterRecordName, - FrontMatterRecordDelimiter, - FrontMatterRecord, - type TNameToken as TRecordNameToken, -} from './frontMatterRecord.js'; diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/linesCodec/linesDecoder.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/linesCodec/linesDecoder.ts deleted file mode 100644 index 5d9ad249d6f..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/linesCodec/linesDecoder.ts +++ /dev/null @@ -1,237 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { Line } from './tokens/line.js'; -import { Range } from '../../../../../../../../editor/common/core/range.js'; -import { NewLine } from './tokens/newLine.js'; -import { assert } from '../../../../../../../../base/common/assert.js'; -import { CarriageReturn } from './tokens/carriageReturn.js'; -import { VSBuffer } from '../../../../../../../../base/common/buffer.js'; -import { assertDefined } from '../../../../../../../../base/common/types.js'; -import { BaseDecoder } from '../baseDecoder.js'; - -/** - * Any line break token type. - */ -export type TLineBreakToken = CarriageReturn | NewLine; - -/** - * Tokens produced by the {@link LinesDecoder}. - */ -export type TLineToken = Line | TLineBreakToken; - -/** - * The `decoder` part of the `LinesCodec` and is able to transform - * data from a binary stream into a stream of text lines(`Line`). - */ -export class LinesDecoder extends BaseDecoder { - /** - * Buffered received data yet to be processed. - */ - private buffer: VSBuffer = VSBuffer.alloc(0); - - /** - * The last emitted `Line` token, if any. The value is used - * to correctly emit remaining line range in the `onStreamEnd` - * method when underlying input stream ends and `buffer` still - * contains some data that must be emitted as the last line. - */ - private lastEmittedLine?: Line; - - /** - * Process data received from the input stream. - */ - protected override onStreamData(chunk: VSBuffer): void { - this.buffer = VSBuffer.concat([this.buffer, chunk]); - - this.processData(false); - } - - /** - * Process buffered data. - * - * @param streamEnded Flag that indicates if the input stream has ended, - * which means that is the last call of this method. - * @throws If internal logic implementation error is detected. - */ - private processData( - streamEnded: boolean, - ): void { - // iterate over each line of the data buffer, emitting each line - // as a `Line` token followed by a `NewLine` token, if applies - while (this.buffer.byteLength > 0) { - // get line number based on a previously emitted line, if any - const lineNumber = this.lastEmittedLine - ? this.lastEmittedLine.range.startLineNumber + 1 - : 1; - - // find the `\r`, `\n`, or `\r\n` tokens in the data - const endOfLineTokens = this.findEndOfLineTokens( - lineNumber, - streamEnded, - ); - const firstToken: (NewLine | CarriageReturn | undefined) = endOfLineTokens[0]; - - // if no end-of-the-line tokens found, stop the current processing - // attempt because we either (1) need more data to be received or - // (2) the stream has ended; in the case (2) remaining data must - // be emitted as the last line - if (firstToken === undefined) { - // (2) if `streamEnded`, we need to emit the whole remaining - // data as the last line immediately - if (streamEnded) { - this.emitLine(lineNumber, this.buffer.slice(0)); - } - - break; - } - - // emit the line found in the data as the `Line` token - this.emitLine(lineNumber, this.buffer.slice(0, firstToken.range.startColumn - 1)); - - // must always hold true as the `emitLine` above sets this - assertDefined( - this.lastEmittedLine, - 'No last emitted line found.', - ); - - // Note! A standalone `\r` token case is not a well-defined case, and - // was primarily used by old Mac OSx systems which treated it as - // a line ending (same as `\n`). Hence for backward compatibility - // with those systems, we treat it as a new line token as well. - // We do that by replacing standalone `\r` token with `\n` one. - if ((endOfLineTokens.length === 1) && (firstToken instanceof CarriageReturn)) { - endOfLineTokens.splice(0, 1, new NewLine(firstToken.range)); - } - - // emit the end-of-the-line tokens - let startColumn = this.lastEmittedLine.range.endColumn; - for (const token of endOfLineTokens) { - const byteLength = token.byte.byteLength; - const endColumn = startColumn + byteLength; - // emit the token updating its column start/end numbers based on - // the emitted line text length and previous end-of-the-line token - this._onData.fire(token.withRange({ startColumn, endColumn })); - // shorten the data buffer by the length of the token - this.buffer = this.buffer.slice(byteLength); - // update the start column for the next token - startColumn = endColumn; - } - } - - // if the stream has ended, assert that the input data buffer is now empty - // otherwise we have a logic error and leaving some buffered data behind - if (streamEnded) { - assert( - this.buffer.byteLength === 0, - 'Expected the input data buffer to be empty when the stream ends.', - ); - } - } - - /** - * Find the end of line tokens in the data buffer. - * Can return: - * - [`\r`, `\n`] tokens if the sequence is found - * - [`\r`] token if only the carriage return is found - * - [`\n`] token if only the newline is found - * - an `empty array` if no end of line tokens found - */ - private findEndOfLineTokens( - lineNumber: number, - streamEnded: boolean, - ): (CarriageReturn | NewLine)[] { - const result = []; - - // find the first occurrence of the carriage return and newline tokens - const carriageReturnIndex = this.buffer.indexOf(CarriageReturn.byte); - const newLineIndex = this.buffer.indexOf(NewLine.byte); - - // if the `\r` comes before the `\n`(if `\n` present at all) - if (carriageReturnIndex >= 0 && ((carriageReturnIndex < newLineIndex) || (newLineIndex === -1))) { - // add the carriage return token first - result.push( - new CarriageReturn(new Range( - lineNumber, - (carriageReturnIndex + 1), - lineNumber, - (carriageReturnIndex + 1) + CarriageReturn.byte.byteLength, - )), - ); - - // if the `\r\n` sequence - if (newLineIndex === carriageReturnIndex + 1) { - // add the newline token to the result - result.push( - new NewLine(new Range( - lineNumber, - (newLineIndex + 1), - lineNumber, - (newLineIndex + 1) + NewLine.byte.byteLength, - )), - ); - } - - // either `\r` or `\r\n` cases found; if we have the `\r` token, we can return - // the end-of-line tokens only, if the `\r` is followed by at least one more - // character (it could be a `\n` or any other character), or if the stream has - // ended (which means the `\r` is at the end of the line) - if ((this.buffer.byteLength > carriageReturnIndex + 1) || streamEnded) { - return result; - } - - // in all other cases, return the empty array (no lend-of-line tokens found) - return []; - } - - // no `\r`, but there is `\n` - if (newLineIndex >= 0) { - result.push( - new NewLine(new Range( - lineNumber, - (newLineIndex + 1), - lineNumber, - (newLineIndex + 1) + NewLine.byte.byteLength, - )), - ); - } - - // neither `\r` nor `\n` found, no end of line found at all - return result; - } - - /** - * Emit a provided line as the `Line` token to the output stream. - */ - private emitLine( - lineNumber: number, // Note! 1-based indexing - lineBytes: VSBuffer, - ): void { - - const line = new Line(lineNumber, lineBytes.toString()); - this._onData.fire(line); - - // store the last emitted line so we can use it when we need - // to send the remaining line in the `onStreamEnd` method - this.lastEmittedLine = line; - - // shorten the data buffer by the length of the line emitted - this.buffer = this.buffer.slice(lineBytes.byteLength); - } - - /** - * Handle the end of the input stream - if the buffer still has some data, - * emit it as the last available line token before firing the `onEnd` event. - */ - protected override onStreamEnd(): void { - // if the input data buffer is not empty when the input stream ends, emit - // the remaining data as the last line before firing the `onEnd` event - if (this.buffer.byteLength > 0) { - this.processData(true); - } - - super.onStreamEnd(); - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/linesCodec/tokens/carriageReturn.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/linesCodec/tokens/carriageReturn.ts deleted file mode 100644 index 46b3031829e..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/linesCodec/tokens/carriageReturn.ts +++ /dev/null @@ -1,44 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { VSBuffer } from '../../../../../../../../../base/common/buffer.js'; -import { SimpleToken } from '../../simpleCodec/tokens/simpleToken.js'; - -/** - * Token that represent a `carriage return` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class CarriageReturn extends SimpleToken<'\r'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '\r' = '\r'; - - /** - * The byte representation of the {@link symbol}. - */ - public static readonly byte = VSBuffer.fromString(CarriageReturn.symbol); - - /** - * The byte representation of the token. - */ - public get byte(): VSBuffer { - return CarriageReturn.byte; - } - - /** - * Return text representation of the token. - */ - public override get text(): '\r' { - return CarriageReturn.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `CR${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/linesCodec/tokens/line.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/linesCodec/tokens/line.ts deleted file mode 100644 index 2fa0afef6ff..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/linesCodec/tokens/line.ts +++ /dev/null @@ -1,48 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../../baseToken.js'; -import { assert } from '../../../../../../../../../base/common/assert.js'; -import { Range } from '../../../../../../../../../editor/common/core/range.js'; - -/** - * Token representing a line of text with a `range` which - * reflects the line's position in the original data. - */ -export class Line extends BaseToken { - constructor( - // the line index - // Note! 1-based indexing - lineNumber: number, - // the line contents - public readonly text: string, - ) { - assert( - !isNaN(lineNumber), - `The line number must not be a NaN.`, - ); - - assert( - lineNumber > 0, - `The line number must be >= 1, got "${lineNumber}".`, - ); - - super( - new Range( - lineNumber, - 1, - lineNumber, - text.length + 1, - ), - ); - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `line("${this.shortText()}")${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/linesCodec/tokens/newLine.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/linesCodec/tokens/newLine.ts deleted file mode 100644 index 6d95751721d..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/linesCodec/tokens/newLine.ts +++ /dev/null @@ -1,44 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { VSBuffer } from '../../../../../../../../../base/common/buffer.js'; -import { SimpleToken } from '../../simpleCodec/tokens/simpleToken.js'; - -/** - * A token that represent a `new line` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class NewLine extends SimpleToken<'\n'> { - /** - * The underlying symbol of the `NewLine` token. - */ - public static override readonly symbol: '\n' = '\n'; - - /** - * The byte representation of the {@link symbol}. - */ - public static readonly byte = VSBuffer.fromString(NewLine.symbol); - - /** - * Return text representation of the token. - */ - public override get text(): '\n' { - return NewLine.symbol; - } - - /** - * The byte representation of the token. - */ - public get byte(): VSBuffer { - return NewLine.byte; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `newline${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/markdownDecoder.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/markdownDecoder.ts deleted file mode 100644 index 67bbda1f2ec..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/markdownDecoder.ts +++ /dev/null @@ -1,136 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { MarkdownToken } from './tokens/markdownToken.js'; -import { VSBuffer } from '../../../../../../../../base/common/buffer.js'; -import { LeftBracket } from '../simpleCodec/tokens/brackets.js'; -import { PartialMarkdownImage } from './parsers/markdownImage.js'; -import { ReadableStream } from '../../../../../../../../base/common/stream.js'; -import { TSimpleDecoderToken } from '../simpleCodec/simpleDecoder.js'; -import { LeftAngleBracket } from '../simpleCodec/tokens/angleBrackets.js'; -import { ExclamationMark } from '../simpleCodec/tokens/exclamationMark.js'; -import { BaseDecoder } from '../baseDecoder.js'; -import { MarkdownCommentStart, PartialMarkdownCommentStart } from './parsers/markdownComment.js'; -import { MarkdownExtensionsDecoder } from '../markdownExtensionsCodec/markdownExtensionsDecoder.js'; -import { MarkdownLinkCaption, PartialMarkdownLink, PartialMarkdownLinkCaption } from './parsers/markdownLink.js'; - -/** - * Tokens produced by this decoder. - */ -export type TMarkdownToken = MarkdownToken | TSimpleDecoderToken; - -/** - * Decoder capable of parsing markdown entities (e.g., links) from a sequence of simple tokens. - */ -export class MarkdownDecoder extends BaseDecoder { - /** - * Current parser object that is responsible for parsing a sequence of tokens into - * some markdown entity. Set to `undefined` when no parsing is in progress at the moment. - */ - private current?: - PartialMarkdownLinkCaption | MarkdownLinkCaption | PartialMarkdownLink | - PartialMarkdownCommentStart | MarkdownCommentStart | - PartialMarkdownImage; - - constructor( - stream: ReadableStream, - ) { - super(new MarkdownExtensionsDecoder(stream)); - } - - protected override onStreamData(token: TSimpleDecoderToken): void { - // `markdown links` start with `[` character, so here we can - // initiate the process of parsing a markdown link - if (token instanceof LeftBracket && !this.current) { - this.current = new PartialMarkdownLinkCaption(token); - - return; - } - - // `markdown comments` start with `<` character, so here we can - // initiate the process of parsing a markdown comment - if (token instanceof LeftAngleBracket && !this.current) { - this.current = new PartialMarkdownCommentStart(token); - - return; - } - - // `markdown image links` start with `!` character, so here we can - // initiate the process of parsing a markdown image - if (token instanceof ExclamationMark && !this.current) { - this.current = new PartialMarkdownImage(token); - - return; - } - - // if current parser was not initiated before, - we are not inside a sequence - // of tokens we care about, therefore re-emit the token immediately and continue - if (!this.current) { - this._onData.fire(token); - return; - } - - // if there is a current parser object, submit the token to it - // so it can progress with parsing the tokens sequence - const parseResult = this.current.accept(token); - if (parseResult.result === 'success') { - const { nextParser } = parseResult; - - // if got a fully parsed out token back, emit it and reset - // the current parser object so a new parsing process can start - if (nextParser instanceof MarkdownToken) { - this._onData.fire(nextParser); - delete this.current; - } else { - // otherwise, update the current parser object - this.current = nextParser; - } - } else { - // if failed to parse a sequence of a tokens as a single markdown - // entity (e.g., a link), re-emit the tokens accumulated so far - // then reset the current parser object - for (const currentToken of this.current.tokens) { - this._onData.fire(currentToken); - } - - delete this.current; - } - - // if token was not consumed by the parser, call `onStreamData` again - // so the token is properly handled by the decoder in the case when a - // new sequence starts with this token - if (!parseResult.wasTokenConsumed) { - this.onStreamData(token); - } - } - - protected override onStreamEnd(): void { - // if the stream has ended and there is a current incomplete parser - // object present, handle the remaining parser object - if (this.current) { - // if a `markdown comment` does not have an end marker `-->` - // it is still a comment that extends to the end of the file - // so re-emit the current parser as a comment token - if (this.current instanceof MarkdownCommentStart) { - this._onData.fire(this.current.asMarkdownComment()); - delete this.current; - this.onStreamEnd(); - - return; - } - - // in all other cases, re-emit existing parser tokens - const { tokens } = this.current; - - for (const token of [...tokens]) { - this._onData.fire(token); - } - - delete this.current; - } - - super.onStreamEnd(); - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/parsers/markdownComment.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/parsers/markdownComment.ts deleted file mode 100644 index a017649f8b8..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/parsers/markdownComment.ts +++ /dev/null @@ -1,173 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { Range } from '../../../../../../../../../editor/common/core/range.js'; -import { BaseToken } from '../../baseToken.js'; -import { Dash } from '../../simpleCodec/tokens/dash.js'; -import { assert } from '../../../../../../../../../base/common/assert.js'; -import { MarkdownComment } from '../tokens/markdownComment.js'; -import { TSimpleDecoderToken } from '../../simpleCodec/simpleDecoder.js'; -import { ExclamationMark } from '../../simpleCodec/tokens/exclamationMark.js'; -import { LeftAngleBracket, RightAngleBracket } from '../../simpleCodec/tokens/angleBrackets.js'; -import { assertNotConsumed, ParserBase, TAcceptTokenResult } from '../../simpleCodec/parserBase.js'; - -/** - * The parser responsible for parsing the ``. If it does, - * then the parser transitions to the {@link MarkdownComment} token. - */ -export class MarkdownCommentStart extends ParserBase { - constructor(tokens: [LeftAngleBracket, ExclamationMark, Dash, Dash]) { - super(tokens); - } - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - // if received `>` while current token sequence ends with `--`, - // then this is the end of the comment sequence - if (token instanceof RightAngleBracket && this.endsWithDashes) { - this.currentTokens.push(token); - - return { - result: 'success', - nextParser: this.asMarkdownComment(), - wasTokenConsumed: true, - }; - } - - this.currentTokens.push(token); - - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } - - /** - * Convert the current token sequence into a {@link MarkdownComment} token. - * - * Note! that this method marks the current parser object as "consumed" - * hence it should not be used after this method is called. - */ - public asMarkdownComment(): MarkdownComment { - this.isConsumed = true; - - return new MarkdownComment( - this.range, - BaseToken.render(this.currentTokens), - ); - } - - /** - * Get range of current token sequence. - */ - private get range(): Range { - const firstToken = this.currentTokens[0]; - const lastToken = this.currentTokens[this.currentTokens.length - 1]; - - const range = new Range( - firstToken.range.startLineNumber, - firstToken.range.startColumn, - lastToken.range.endLineNumber, - lastToken.range.endColumn, - ); - - return range; - } - - /** - * Whether the current token sequence ends with two dashes. - */ - private get endsWithDashes(): boolean { - const lastToken = this.currentTokens[this.currentTokens.length - 1]; - if (!(lastToken instanceof Dash)) { - return false; - } - - const secondLastToken = this.currentTokens[this.currentTokens.length - 2]; - if (!(secondLastToken instanceof Dash)) { - return false; - } - - return true; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/parsers/markdownImage.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/parsers/markdownImage.ts deleted file mode 100644 index 5684f48de4a..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/parsers/markdownImage.ts +++ /dev/null @@ -1,99 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { MarkdownLink } from '../tokens/markdownLink.js'; -import { MarkdownImage } from '../tokens/markdownImage.js'; -import { LeftBracket } from '../../simpleCodec/tokens/brackets.js'; -import { TSimpleDecoderToken } from '../../simpleCodec/simpleDecoder.js'; -import { ExclamationMark } from '../../simpleCodec/tokens/exclamationMark.js'; -import { assertNotConsumed, ParserBase, TAcceptTokenResult } from '../../simpleCodec/parserBase.js'; -import { MarkdownLinkCaption, PartialMarkdownLink, PartialMarkdownLinkCaption } from './markdownLink.js'; - -/** - * The parser responsible for parsing the `markdown image` sequence of characters. - * E.g., `![alt text](./path/to/image.jpeg)` syntax. - */ -export class PartialMarkdownImage extends ParserBase { - /** - * Current active parser instance, if in the mode of actively parsing the markdown link sequence. - */ - private markdownLinkParser: PartialMarkdownLinkCaption | MarkdownLinkCaption | PartialMarkdownLink | undefined; - - constructor(token: ExclamationMark) { - super([token]); - } - - /** - * Get all currently available tokens of the `markdown link` sequence. - */ - public override get tokens(): readonly TSimpleDecoderToken[] { - const linkTokens = this.markdownLinkParser?.tokens ?? []; - - return [ - ...this.currentTokens, - ...linkTokens, - ]; - } - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - // on the first call we expect a character that begins `markdown link` sequence - // hence we initiate the markdown link parsing process, otherwise we fail - if (!this.markdownLinkParser) { - if (token instanceof LeftBracket) { - this.markdownLinkParser = new PartialMarkdownLinkCaption(token); - - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } - - return { - result: 'failure', - wasTokenConsumed: false, - }; - } - - // handle subsequent tokens next - - const acceptResult = this.markdownLinkParser.accept(token); - const { result, wasTokenConsumed } = acceptResult; - - if (result === 'success') { - const { nextParser } = acceptResult; - - // if full markdown link was parsed out, the process completes - if (nextParser instanceof MarkdownLink) { - this.isConsumed = true; - - const firstToken = this.currentTokens[0]; - return { - result, - wasTokenConsumed, - nextParser: new MarkdownImage( - firstToken.range.startLineNumber, - firstToken.range.startColumn, - `${firstToken.text}${nextParser.caption}`, - nextParser.reference, - ), - }; - } - - // otherwise save new link parser reference and continue - this.markdownLinkParser = nextParser; - return { - result, - wasTokenConsumed, - nextParser: this, - }; - } - - // return the failure result - this.isConsumed = true; - return acceptResult; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/parsers/markdownLink.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/parsers/markdownLink.ts deleted file mode 100644 index 641ddd14b9b..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/parsers/markdownLink.ts +++ /dev/null @@ -1,211 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../../baseToken.js'; -import { MarkdownLink } from '../tokens/markdownLink.js'; -import { NewLine } from '../../linesCodec/tokens/newLine.js'; -import { assert } from '../../../../../../../../../base/common/assert.js'; -import { FormFeed } from '../../simpleCodec/tokens/formFeed.js'; -import { VerticalTab } from '../../simpleCodec/tokens/verticalTab.js'; -import { TSimpleDecoderToken } from '../../simpleCodec/simpleDecoder.js'; -import { CarriageReturn } from '../../linesCodec/tokens/carriageReturn.js'; -import { LeftBracket, RightBracket } from '../../simpleCodec/tokens/brackets.js'; -import { ParserBase, TAcceptTokenResult } from '../../simpleCodec/parserBase.js'; -import { LeftParenthesis, RightParenthesis } from '../../simpleCodec/tokens/parentheses.js'; - -/** - * List of characters that are not allowed in links so stop a markdown link sequence abruptly. - */ -const MARKDOWN_LINK_STOP_CHARACTERS: readonly string[] = [CarriageReturn, NewLine, VerticalTab, FormFeed] - .map((token) => { return token.symbol; }); - -/** - * The parser responsible for parsing a `markdown link caption` part of a markdown - * link (e.g., the `[caption text]` part of the `[caption text](./some/path)` link). - * - * The parsing process starts with single `[` token and collects all tokens until - * the first `]` token is encountered. In this successful case, the parser transitions - * into the {@link MarkdownLinkCaption} parser type which continues the general - * parsing process of the markdown link. - * - * Otherwise, if one of the stop characters defined in the {@link MARKDOWN_LINK_STOP_CHARACTERS} - * is encountered before the `]` token, the parsing process is aborted which is communicated to - * the caller by returning a `failure` result. In this case, the caller is assumed to be responsible - * for re-emitting the {@link tokens} accumulated so far as standalone entities since they are no - * longer represent a coherent token entity of a larger size. - */ -export class PartialMarkdownLinkCaption extends ParserBase { - constructor(token: LeftBracket) { - super([token]); - } - - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - // any of stop characters is are breaking a markdown link caption sequence - if (MARKDOWN_LINK_STOP_CHARACTERS.includes(token.text)) { - return { - result: 'failure', - wasTokenConsumed: false, - }; - } - - // the `]` character ends the caption of a markdown link - if (token instanceof RightBracket) { - return { - result: 'success', - nextParser: new MarkdownLinkCaption([...this.tokens, token]), - wasTokenConsumed: true, - }; - } - - // otherwise, include the token in the sequence - // and keep the current parser object instance - this.currentTokens.push(token); - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } -} - -/** - * The parser responsible for transitioning from a {@link PartialMarkdownLinkCaption} - * parser to the {@link PartialMarkdownLink} one, therefore serves a parser glue between - * the `[caption]` and the `(./some/path)` parts of the `[caption](./some/path)` link. - * - * The only successful case of this parser is the `(` token that initiated the process - * of parsing the `reference` part of a markdown link and in this case the parser - * transitions into the `PartialMarkdownLink` parser type. - * - * Any other character is considered a failure result. In this case, the caller is assumed - * to be responsible for re-emitting the {@link tokens} accumulated so far as standalone - * entities since they are no longer represent a coherent token entity of a larger size. - */ -export class MarkdownLinkCaption extends ParserBase { - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - // the `(` character starts the link part of a markdown link - // that is the only character that can follow the caption - if (token instanceof LeftParenthesis) { - return { - result: 'success', - wasTokenConsumed: true, - nextParser: new PartialMarkdownLink([...this.tokens], token), - }; - } - - return { - result: 'failure', - wasTokenConsumed: false, - }; - } -} - -/** - * The parser responsible for parsing a `link reference` part of a markdown link - * (e.g., the `(./some/path)` part of the `[caption text](./some/path)` link). - * - * The parsing process starts with tokens that represent the `[caption]` part of a markdown - * link, followed by the `(` token. The parser collects all subsequent tokens until final closing - * parenthesis (`)`) is encountered (*\*see [1] below*). In this successful case, the parser object - * transitions into the {@link MarkdownLink} token type which signifies the end of the entire - * parsing process of the link text. - * - * Otherwise, if one of the stop characters defined in the {@link MARKDOWN_LINK_STOP_CHARACTERS} - * is encountered before the final `)` token, the parsing process is aborted which is communicated to - * the caller by returning a `failure` result. In this case, the caller is assumed to be responsible - * for re-emitting the {@link tokens} accumulated so far as standalone entities since they are no - * longer represent a coherent token entity of a larger size. - * - * `[1]` The `reference` part of the markdown link can contain any number of nested parenthesis, e.g., - * `[caption](/some/p(th/file.md)` is a valid markdown link and a valid folder name, hence number - * of open parenthesis must match the number of closing ones and the path sequence is considered - * to be complete as soon as this requirement is met. Therefore the `final` word is used in - * the description comments above to highlight this important detail. - */ -export class PartialMarkdownLink extends ParserBase { - /** - * Number of open parenthesis in the sequence. - * See comment in the {@link accept} method for more details. - */ - private openParensCount: number = 1; - - constructor( - protected readonly captionTokens: TSimpleDecoderToken[], - token: LeftParenthesis, - ) { - super([token]); - } - - public override get tokens(): readonly TSimpleDecoderToken[] { - return [...this.captionTokens, ...this.currentTokens]; - } - - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - // markdown links allow for nested parenthesis inside the link reference part, but - // the number of open parenthesis must match the number of closing parenthesis, e.g.: - // - `[caption](/some/p()th/file.md)` is a valid markdown link - // - `[caption](/some/p(th/file.md)` is an invalid markdown link - // hence we use the `openParensCount` variable to keep track of the number of open - // parenthesis encountered so far; then upon encountering a closing parenthesis we - // decrement the `openParensCount` and if it reaches 0 - we consider the link reference - // to be complete - - if (token instanceof LeftParenthesis) { - this.openParensCount += 1; - } - - if (token instanceof RightParenthesis) { - this.openParensCount -= 1; - - // sanity check! this must alway hold true because we return a complete markdown - // link as soon as we encounter matching number of closing parenthesis, hence - // we must never have `openParensCount` that is less than 0 - assert( - this.openParensCount >= 0, - `Unexpected right parenthesis token encountered: '${token}'.`, - ); - - // the markdown link is complete as soon as we get the same number of closing parenthesis - if (this.openParensCount === 0) { - const { startLineNumber, startColumn } = this.captionTokens[0].range; - - // create link caption string - const caption = BaseToken.render(this.captionTokens); - - // create link reference string - this.currentTokens.push(token); - const reference = BaseToken.render(this.currentTokens); - - // return complete markdown link object - return { - result: 'success', - wasTokenConsumed: true, - nextParser: new MarkdownLink( - startLineNumber, - startColumn, - caption, - reference, - ), - }; - } - } - - // any of stop characters is are breaking a markdown link reference sequence - if (MARKDOWN_LINK_STOP_CHARACTERS.includes(token.text)) { - return { - result: 'failure', - wasTokenConsumed: false, - }; - } - - // the rest of the tokens can be included in the sequence - this.currentTokens.push(token); - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/tokens/markdownComment.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/tokens/markdownComment.ts deleted file mode 100644 index 8eadff3589a..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/tokens/markdownComment.ts +++ /dev/null @@ -1,40 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { Range } from '../../../../../../../../../editor/common/core/range.js'; -import { MarkdownToken } from './markdownToken.js'; -import { assert } from '../../../../../../../../../base/common/assert.js'; - -/** - * A token that represent a `markdown comment` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class MarkdownComment extends MarkdownToken { - constructor( - range: Range, - public readonly text: string, - ) { - assert( - text.startsWith('`. - */ - public get hasEndMarker(): boolean { - return this.text.endsWith('-->'); - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `md-comment("${this.shortText()}")${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/tokens/markdownImage.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/tokens/markdownImage.ts deleted file mode 100644 index cab120f53db..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/tokens/markdownImage.ts +++ /dev/null @@ -1,125 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { MarkdownToken } from './markdownToken.js'; -import { IRange, Range } from '../../../../../../../../../editor/common/core/range.js'; -import { assert } from '../../../../../../../../../base/common/assert.js'; - -/** - * A token that represent a `markdown image` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class MarkdownImage extends MarkdownToken { - /** - * Check if this `markdown image link` points to a valid URL address. - */ - public readonly isURL: boolean; - - constructor( - /** - * The starting line number of the image (1-based indexing). - */ - lineNumber: number, - /** - * The starting column number of the image (1-based indexing). - */ - columnNumber: number, - /** - * The caption of the image, including the `!` and `square brackets`. - */ - private readonly caption: string, - /** - * The reference of the image, including the parentheses. - */ - private readonly reference: string, - ) { - assert( - !isNaN(lineNumber), - `The line number must not be a NaN.`, - ); - - assert( - lineNumber > 0, - `The line number must be >= 1, got "${lineNumber}".`, - ); - - assert( - columnNumber > 0, - `The column number must be >= 1, got "${columnNumber}".`, - ); - - assert( - caption[0] === '!', - `The caption must start with '!' character, got "${caption}".`, - ); - - assert( - caption[1] === '[' && caption[caption.length - 1] === ']', - `The caption must be enclosed in square brackets, got "${caption}".`, - ); - - assert( - reference[0] === '(' && reference[reference.length - 1] === ')', - `The reference must be enclosed in parentheses, got "${reference}".`, - ); - - super( - new Range( - lineNumber, - columnNumber, - lineNumber, - columnNumber + caption.length + reference.length, - ), - ); - - // set up the `isURL` flag based on the current - try { - new URL(this.path); - this.isURL = true; - } catch { - this.isURL = false; - } - } - - public override get text(): string { - return `${this.caption}${this.reference}`; - } - - /** - * Returns the `reference` part of the link without enclosing parentheses. - */ - public get path(): string { - return this.reference.slice(1, this.reference.length - 1); - } - - /** - * Get the range of the `link part` of the token. - */ - public get linkRange(): IRange | undefined { - if (this.path.length === 0) { - return undefined; - } - - const { range } = this; - - // note! '+1' for openning `(` of the link - const startColumn = range.startColumn + this.caption.length + 1; - const endColumn = startColumn + this.path.length; - - return new Range( - range.startLineNumber, - startColumn, - range.endLineNumber, - endColumn, - ); - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `md-image("${this.shortText()}")${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/tokens/markdownLink.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/tokens/markdownLink.ts deleted file mode 100644 index a963628675d..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/tokens/markdownLink.ts +++ /dev/null @@ -1,120 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { MarkdownToken } from './markdownToken.js'; -import { IRange, Range } from '../../../../../../../../../editor/common/core/range.js'; -import { assert } from '../../../../../../../../../base/common/assert.js'; - -/** - * A token that represent a `markdown link` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class MarkdownLink extends MarkdownToken { - /** - * Check if this `markdown link` points to a valid URL address. - */ - public readonly isURL: boolean; - - constructor( - /** - * The starting line number of the link (1-based indexing). - */ - lineNumber: number, - /** - * The starting column number of the link (1-based indexing). - */ - columnNumber: number, - /** - * The caption of the original link, including the square brackets. - */ - public readonly caption: string, - /** - * The reference of the original link, including the parentheses. - */ - public readonly reference: string, - ) { - assert( - !isNaN(lineNumber), - `The line number must not be a NaN.`, - ); - - assert( - lineNumber > 0, - `The line number must be >= 1, got "${lineNumber}".`, - ); - - assert( - columnNumber > 0, - `The column number must be >= 1, got "${columnNumber}".`, - ); - - assert( - caption[0] === '[' && caption[caption.length - 1] === ']', - `The caption must be enclosed in square brackets, got "${caption}".`, - ); - - assert( - reference[0] === '(' && reference[reference.length - 1] === ')', - `The reference must be enclosed in parentheses, got "${reference}".`, - ); - - super( - new Range( - lineNumber, - columnNumber, - lineNumber, - columnNumber + caption.length + reference.length, - ), - ); - - // set up the `isURL` flag based on the current - try { - new URL(this.path); - this.isURL = true; - } catch { - this.isURL = false; - } - } - - public override get text(): string { - return `${this.caption}${this.reference}`; - } - - /** - * Returns the `reference` part of the link without enclosing parentheses. - */ - public get path(): string { - return this.reference.slice(1, this.reference.length - 1); - } - - /** - * Get the range of the `link part` of the token. - */ - public get linkRange(): IRange | undefined { - if (this.path.length === 0) { - return undefined; - } - - const { range } = this; - - // note! '+1' for opening `(` of the link - const startColumn = range.startColumn + this.caption.length + 1; - const endColumn = startColumn + this.path.length; - - return new Range( - range.startLineNumber, - startColumn, - range.endLineNumber, - endColumn, - ); - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `md-link("${this.shortText()}")${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/tokens/markdownToken.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/tokens/markdownToken.ts deleted file mode 100644 index fc1935d081b..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/tokens/markdownToken.ts +++ /dev/null @@ -1,12 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../../baseToken.js'; - -/** - * Common base token that all `markdown` tokens should - * inherit from. - */ -export abstract class MarkdownToken extends BaseToken { } diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/markdownExtensionsDecoder.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/markdownExtensionsDecoder.ts deleted file mode 100644 index b5ff3d2b9f9..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/markdownExtensionsDecoder.ts +++ /dev/null @@ -1,119 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { VSBuffer } from '../../../../../../../../base/common/buffer.js'; -import { ReadableStream } from '../../../../../../../../base/common/stream.js'; -import { BaseDecoder } from '../baseDecoder.js'; -import { MarkdownExtensionsToken } from './tokens/markdownExtensionsToken.js'; -import { SimpleDecoder, TSimpleDecoderToken } from '../simpleCodec/simpleDecoder.js'; -import { PartialFrontMatterHeader, PartialFrontMatterStartMarker } from './parsers/frontMatterHeader.js'; - -/** - * Tokens produced by this decoder. - */ -export type TMarkdownExtensionsToken = MarkdownExtensionsToken | TSimpleDecoderToken; - -/** - * Decoder responsible for decoding extensions of markdown syntax, - * e.g., a `Front Matter` header, etc. - */ -export class MarkdownExtensionsDecoder extends BaseDecoder { - /** - * Current parser object that is responsible for parsing a sequence of tokens into - * some markdown entity. Set to `undefined` when no parsing is in progress at the moment. - */ - private current?: PartialFrontMatterStartMarker | PartialFrontMatterHeader; - - constructor( - stream: ReadableStream, - ) { - super(new SimpleDecoder(stream)); - } - - protected override onStreamData(token: TSimpleDecoderToken): void { - // front matter headers start with a `-` at the first column of the first line - if ((this.current === undefined) && PartialFrontMatterStartMarker.mayStartHeader(token)) { - this.current = new PartialFrontMatterStartMarker(token); - - return; - } - - // if current parser is not initiated, - we are not inside a sequence of tokens - // we care about, therefore re-emit the token immediately and continue - if (this.current === undefined) { - this._onData.fire(token); - return; - } - - // if there is a current parser object, submit the token to it - // so it can progress with parsing the tokens sequence - const parseResult = this.current.accept(token); - if (parseResult.result === 'success') { - const { nextParser } = parseResult; - - // if got a fully parsed out token back, emit it and reset - // the current parser object so a new parsing process can start - if (nextParser instanceof MarkdownExtensionsToken) { - this._onData.fire(nextParser); - delete this.current; - } else { - // otherwise, update the current parser object - this.current = nextParser; - } - } else { - // if failed to parse a sequence of a tokens as a single markdown - // entity (e.g., a link), re-emit the tokens accumulated so far - // then reset the currently initialized parser object - this.reEmitCurrentTokens(); - } - - // if token was not consumed by the parser, call `onStreamData` again - // so the token is properly handled by the decoder in the case when a - // new sequence starts with this token - if (!parseResult.wasTokenConsumed) { - this.onStreamData(token); - } - } - - protected override onStreamEnd(): void { - try { - if (this.current === undefined) { - return; - } - - // if current parser can be converted into a valid Front Matter - // header, then emit it and reset the current parser object - if (this.current instanceof PartialFrontMatterHeader) { - this._onData.fire( - this.current.asFrontMatterHeader(), - ); - delete this.current; - return; - } - - } catch { - // if failed to convert current parser object to a token, - // re-emit the tokens accumulated so far - this.reEmitCurrentTokens(); - } finally { - delete this.current; - super.onStreamEnd(); - } - } - - /** - * Re-emit tokens accumulated so far in the current parser object. - */ - protected reEmitCurrentTokens(): void { - if (this.current === undefined) { - return; - } - - for (const token of this.current.tokens) { - this._onData.fire(token); - } - delete this.current; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/parsers/frontMatterHeader.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/parsers/frontMatterHeader.ts deleted file mode 100644 index a63c911174c..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/parsers/frontMatterHeader.ts +++ /dev/null @@ -1,345 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { Dash } from '../../simpleCodec/tokens/dash.js'; -import { NewLine } from '../../linesCodec/tokens/newLine.js'; -import { FrontMatterHeader } from '../tokens/frontMatterHeader.js'; -import { assertDefined } from '../../../../../../../../../base/common/types.js'; -import { TSimpleDecoderToken } from '../../simpleCodec/simpleDecoder.js'; -import { assert, assertNever } from '../../../../../../../../../base/common/assert.js'; -import { CarriageReturn } from '../../linesCodec/tokens/carriageReturn.js'; -import { FrontMatterMarker, TMarkerToken } from '../tokens/frontMatterMarker.js'; -import { assertNotConsumed, IAcceptTokenSuccess, ParserBase, TAcceptTokenResult } from '../../simpleCodec/parserBase.js'; - -/** - * Parses the start marker of a Front Matter header. - */ -export class PartialFrontMatterStartMarker extends ParserBase { - constructor(token: Dash) { - const { range } = token; - - assert( - range.startLineNumber === 1, - `Front Matter header must start at the first line, but it starts at line #${range.startLineNumber}.`, - ); - - assert( - range.startColumn === 1, - `Front Matter header must start at the beginning of the line, but it starts at ${range.startColumn}.`, - ); - - super([token]); - } - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - const previousToken = this.currentTokens[this.currentTokens.length - 1]; - - // collect a sequence of dash tokens that may end with a CR token - if ((token instanceof Dash) || (token instanceof CarriageReturn)) { - // a dash or CR tokens can go only after another dash token - if ((previousToken instanceof Dash) === false) { - this.isConsumed = true; - - return { - result: 'failure', - wasTokenConsumed: false, - }; - } - - this.currentTokens.push(token); - - return { - result: 'success', - wasTokenConsumed: true, - nextParser: this, - }; - } - - // stop collecting dash tokens when a new line token is encountered - if (token instanceof NewLine) { - this.isConsumed = true; - - return { - result: 'success', - wasTokenConsumed: true, - nextParser: new PartialFrontMatterHeader( - FrontMatterMarker.fromTokens([ - ...this.currentTokens, - token, - ]), - ), - }; - } - - // any other token is invalid for the `start marker` - this.isConsumed = true; - return { - result: 'failure', - wasTokenConsumed: false, - }; - } - - /** - * Check if provided dash token can be a start of a Front Matter header. - */ - public static mayStartHeader(token: TSimpleDecoderToken): token is Dash { - return (token instanceof Dash) - && (token.range.startLineNumber === 1) - && (token.range.startColumn === 1); - } -} - -/** - * Parses a Front Matter header that already has a start marker - * and possibly some content that follows. - */ -export class PartialFrontMatterHeader extends ParserBase { - /** - * Parser instance for the end marker of the Front Matter header. - */ - private maybeEndMarker?: PartialFrontMatterEndMarker; - - constructor( - public readonly startMarker: FrontMatterMarker, - ) { - super([]); - } - - public override get tokens(): readonly TSimpleDecoderToken[] { - const endMarkerTokens = (this.maybeEndMarker !== undefined) - ? this.maybeEndMarker.tokens - : []; - - return [ - ...this.startMarker.tokens, - ...this.currentTokens, - ...endMarkerTokens, - ]; - } - - /** - * Convert the current token sequence into a {@link FrontMatterHeader} token. - * - * Note! that this method marks the current parser object as "consumed" - * hence it should not be used after this method is called. - */ - public asFrontMatterHeader(): FrontMatterHeader { - assertDefined( - this.maybeEndMarker, - 'Cannot convert to Front Matter header token without an end marker.', - ); - - assert( - this.maybeEndMarker.dashCount === this.startMarker.dashTokens.length, - [ - 'Start and end markers must have the same number of dashes', - `, got ${this.startMarker.dashTokens.length} / ${this.maybeEndMarker.dashCount}.`, - ].join(''), - ); - - this.isConsumed = true; - - return FrontMatterHeader.fromTokens( - this.startMarker.tokens, - this.currentTokens, - this.maybeEndMarker.tokens, - ); - } - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - // if in the mode of parsing the end marker sequence, forward - // the token to the current end marker parser instance - if (this.maybeEndMarker !== undefined) { - return this.acceptEndMarkerToken(token); - } - - // collect all tokens until a `dash token at the beginning of a line` is found - if (((token instanceof Dash) === false) || (token.range.startColumn !== 1)) { - this.currentTokens.push(token); - - return { - result: 'success', - wasTokenConsumed: true, - nextParser: this, - }; - } - - // a dash token at the beginning of the line might be a start of the `end marker` - // sequence of the front matter header, hence initialize appropriate parser object - assert( - this.maybeEndMarker === undefined, - `End marker parser must not be present.`, - ); - this.maybeEndMarker = new PartialFrontMatterEndMarker(token); - - return { - result: 'success', - wasTokenConsumed: true, - nextParser: this, - }; - } - - /** - * When a end marker parser is present, we pass all tokens to it - * until it is completes the parsing process(either success or failure). - */ - private acceptEndMarkerToken( - token: TSimpleDecoderToken, - ): TAcceptTokenResult { - assertDefined( - this.maybeEndMarker, - `Partial end marker parser must be initialized.`, - ); - - // if we have a partial end marker, we are in the process of parsing - // the end marker, so just pass the token to it and return - const acceptResult = this.maybeEndMarker.accept(token); - const { result, wasTokenConsumed } = acceptResult; - - if (result === 'success') { - const { nextParser } = acceptResult; - const endMarkerParsingComplete = (nextParser instanceof FrontMatterMarker); - - if (endMarkerParsingComplete === false) { - return { - result: 'success', - wasTokenConsumed, - nextParser: this, - }; - } - - const endMarker = nextParser; - - // start and end markers must have the same number of dashes, hence - // if they don't match, we would like to continue parsing the header - // until we find an end marker with the same number of dashes - if (endMarker.dashTokens.length !== this.startMarker.dashTokens.length) { - return this.handleEndMarkerParsingFailure( - endMarker.tokens, - wasTokenConsumed, - ); - } - - this.isConsumed = true; - return { - result: 'success', - wasTokenConsumed: true, - nextParser: FrontMatterHeader.fromTokens( - this.startMarker.tokens, - this.currentTokens, - this.maybeEndMarker.tokens, - ), - }; - } - - // if failed to parse the end marker, we would like to continue parsing - // the header until we find a valid end marker - if (result === 'failure') { - return this.handleEndMarkerParsingFailure( - this.maybeEndMarker.tokens, - wasTokenConsumed, - ); - } - - assertNever( - result, - `Unexpected result '${result}' while parsing the end marker.`, - ); - } - - /** - * On failure to parse the end marker, we need to continue parsing - * the header because there might be another valid end marker in - * the stream of tokens. Therefore we copy over the end marker tokens - * into the list of "content" tokens and reset the end marker parser. - */ - private handleEndMarkerParsingFailure( - tokens: readonly TSimpleDecoderToken[], - wasTokenConsumed: boolean, - ): IAcceptTokenSuccess { - this.currentTokens.push(...tokens); - delete this.maybeEndMarker; - - return { - result: 'success', - wasTokenConsumed, - nextParser: this, - }; - } -} - -/** - * Parser the end marker sequence of a Front Matter header. - */ -class PartialFrontMatterEndMarker extends ParserBase { - constructor(token: Dash) { - const { range } = token; - - assert( - range.startColumn === 1, - `Front Matter header must start at the beginning of the line, but it starts at ${range.startColumn}.`, - ); - - super([token]); - } - - /** - * Number of dashes in the marker. - */ - public get dashCount(): number { - return this.tokens - .filter((token) => { return token instanceof Dash; }) - .length; - } - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - const previousToken = this.currentTokens[this.currentTokens.length - 1]; - - // collect a sequence of dash tokens that may end with a CR token - if ((token instanceof Dash) || (token instanceof CarriageReturn)) { - // a dash or CR tokens can go only after another dash token - if ((previousToken instanceof Dash) === false) { - this.isConsumed = true; - - return { - result: 'failure', - wasTokenConsumed: false, - }; - } - - this.currentTokens.push(token); - - return { - result: 'success', - wasTokenConsumed: true, - nextParser: this, - }; - } - - // stop collecting dash tokens when a new line token is encountered - if (token instanceof NewLine) { - this.isConsumed = true; - this.currentTokens.push(token); - - return { - result: 'success', - wasTokenConsumed: true, - nextParser: FrontMatterMarker.fromTokens([ - ...this.currentTokens, - ]), - }; - } - - // any other token is invalid for the `start marker` - this.isConsumed = true; - return { - result: 'failure', - wasTokenConsumed: false, - }; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/tokens/frontMatterHeader.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/tokens/frontMatterHeader.ts deleted file mode 100644 index b5b3903b20a..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/tokens/frontMatterHeader.ts +++ /dev/null @@ -1,79 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { Text } from '../../textToken.js'; -import { Range } from '../../../../../../../../../editor/common/core/range.js'; -import { BaseToken } from '../../baseToken.js'; -import { MarkdownExtensionsToken } from './markdownExtensionsToken.js'; -import { TSimpleDecoderToken } from '../../simpleCodec/simpleDecoder.js'; -import { FrontMatterMarker, TMarkerToken } from './frontMatterMarker.js'; - -/** - * Token that represents a `Front Matter` header in a text. - */ -export class FrontMatterHeader extends MarkdownExtensionsToken { - constructor( - range: Range, - public readonly startMarker: FrontMatterMarker, - public readonly content: Text, - public readonly endMarker: FrontMatterMarker, - ) { - super(range); - } - - /** - * Return complete text representation of the token. - */ - public get text(): string { - const text: string[] = [ - this.startMarker.text, - this.content.text, - this.endMarker.text, - ]; - - return text.join(''); - } - - /** - * Range of the content of the Front Matter header. - */ - public get contentRange(): Range { - return this.content.range; - } - - /** - * Content token of the Front Matter header. - */ - public get contentToken(): Text { - return this.content; - } - - /** - * Create new instance of the token from the given tokens. - */ - public static fromTokens( - startMarkerTokens: readonly TMarkerToken[], - contentTokens: readonly TSimpleDecoderToken[], - endMarkerTokens: readonly TMarkerToken[], - ): FrontMatterHeader { - const range = BaseToken.fullRange( - [...startMarkerTokens, ...endMarkerTokens], - ); - - return new FrontMatterHeader( - range, - FrontMatterMarker.fromTokens(startMarkerTokens), - new Text(contentTokens), - FrontMatterMarker.fromTokens(endMarkerTokens), - ); - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `frontmatter("${this.shortText()}")${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/tokens/frontMatterMarker.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/tokens/frontMatterMarker.ts deleted file mode 100644 index 14813b3116b..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/tokens/frontMatterMarker.ts +++ /dev/null @@ -1,59 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { Range } from '../../../../../../../../../editor/common/core/range.js'; -import { BaseToken } from '../../baseToken.js'; -import { Dash } from '../../simpleCodec/tokens/dash.js'; -import { NewLine } from '../../linesCodec/tokens/newLine.js'; -import { MarkdownExtensionsToken } from './markdownExtensionsToken.js'; -import { CarriageReturn } from '../../linesCodec/tokens/carriageReturn.js'; - -/** - * Type for tokens inside a Front Matter header marker. - */ -export type TMarkerToken = Dash | CarriageReturn | NewLine; - -/** - * Marker for the start and end of a Front Matter header. - */ -export class FrontMatterMarker extends MarkdownExtensionsToken { - /** - * Returns complete text representation of the token. - */ - public get text(): string { - return BaseToken.render(this.tokens); - } - - /** - * List of {@link Dash} tokens in the marker. - */ - public get dashTokens(): readonly Dash[] { - return this.tokens - .filter((token) => { return token instanceof Dash; }); - } - - constructor( - range: Range, - public readonly tokens: readonly TMarkerToken[], - ) { - super(range); - } - - /** - * Create new instance of the token from a provided - * list of tokens. - */ - public static fromTokens( - tokens: readonly TMarkerToken[], - ): FrontMatterMarker { - const range = BaseToken.fullRange(tokens); - - return new FrontMatterMarker(range, tokens); - } - - public toString(): string { - return `frontmatter-marker(${this.dashTokens.length}:${this.range})`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/tokens/markdownExtensionsToken.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/tokens/markdownExtensionsToken.ts deleted file mode 100644 index 82046eb2b4d..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/tokens/markdownExtensionsToken.ts +++ /dev/null @@ -1,11 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { MarkdownToken } from '../../markdownCodec/tokens/markdownToken.js'; - -/** - * Base class for all tokens produced by the `MarkdownExtensionsDecoder`. - */ -export abstract class MarkdownExtensionsToken extends MarkdownToken { } diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/parserBase.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/parserBase.ts deleted file mode 100644 index 84515dc1d6c..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/parserBase.ts +++ /dev/null @@ -1,137 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../baseToken.js'; -import { assert } from '../../../../../../../../base/common/assert.js'; - -/** - * Common interface for a result of accepting a next token - * in a sequence. - */ -export interface IAcceptTokenResult { - /** - * The result type of accepting a next token in a sequence. - */ - result: 'success' | 'failure'; - - /** - * Whether the token to accept was consumed by the parser - * during the accept operation. - */ - wasTokenConsumed: boolean; -} - -/** - * Successful result of accepting a next token in a sequence. - */ -export interface IAcceptTokenSuccess extends IAcceptTokenResult { - result: 'success'; - nextParser: T; -} - -/** - * Failure result of accepting a next token in a sequence. - */ -export interface IAcceptTokenFailure extends IAcceptTokenResult { - result: 'failure'; -} - -/** - * The result of operation of accepting a next token in a sequence. - */ -export type TAcceptTokenResult = IAcceptTokenSuccess | IAcceptTokenFailure; - -/** - * An abstract parser class that is able to parse a sequence of - * tokens into a new single entity. - */ -export abstract class ParserBase { - /** - * Whether the parser object was "consumed" and should not be used anymore. - */ - protected isConsumed: boolean = false; - - /** - * Whether the parser object was "consumed" hence must not be used anymore. - */ - public get consumed(): boolean { - return this.isConsumed; - } - - /** - * Number of tokens at the initialization of the current parser. - */ - protected readonly startTokensCount: number; - - constructor( - /** - * Set of tokens that were accumulated so far. - */ - protected readonly currentTokens: TToken[] = [], - ) { - this.startTokensCount = this.currentTokens.length; - } - - /** - * Get the tokens that were accumulated so far. - */ - public get tokens(): readonly TToken[] { - return this.currentTokens; - } - - /** - * Accept a new token returning parsing result: - * - successful result must include the next parser object or a fully parsed out token - * - failure result must indicate that the token was not consumed - * - * @param token The token to accept. - * @returns The parsing result. - */ - public abstract accept(token: TToken): TAcceptTokenResult; - - /** - * A helper method that validates that the current parser object was not yet consumed, - * hence can still be used to accept new tokens in the parsing process. - * - * @throws if the parser object is already consumed. - */ - protected assertNotConsumed(): void { - assert( - this.isConsumed === false, - `The parser object is already consumed and should not be used anymore.`, - ); - } -} - -/** - * Decorator that validates that the current parser object was not yet consumed, - * hence can still be used to accept new tokens in the parsing process. - * - * @throws the resulting decorated method throws if the parser object was already consumed. - */ -export function assertNotConsumed>( - _target: T, - propertyKey: 'accept', - descriptor: PropertyDescriptor, -): PropertyDescriptor { - // store the original method reference - const originalMethod = descriptor.value; - - // validate that the current parser object was not yet consumed - // before invoking the original accept method - descriptor.value = function ( - this: T, - ...args: Parameters - ): ReturnType { - assert( - this.isConsumed === false, - `The parser object is already consumed and should not be used anymore.`, - ); - - return originalMethod.apply(this, args); - }; - - return descriptor; -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/simpleDecoder.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/simpleDecoder.ts deleted file mode 100644 index 9addf4df2ca..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/simpleDecoder.ts +++ /dev/null @@ -1,132 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { NewLine } from '../linesCodec/tokens/newLine.js'; -import { CarriageReturn } from '../linesCodec/tokens/carriageReturn.js'; -import { LinesDecoder, TLineBreakToken, TLineToken } from '../linesCodec/linesDecoder.js'; -import { - At, - Tab, - Word, - Hash, - Dash, - Colon, - Slash, - Space, - Quote, - Comma, - FormFeed, - DollarSign, - DoubleQuote, - VerticalTab, - type TBracket, - LeftBracket, - RightBracket, - type TCurlyBrace, - LeftCurlyBrace, - RightCurlyBrace, - ExclamationMark, - type TParenthesis, - LeftParenthesis, - RightParenthesis, - type TAngleBracket, - LeftAngleBracket, - RightAngleBracket, -} from './tokens/tokens.js'; -import { ISimpleTokenClass, SimpleToken } from './tokens/simpleToken.js'; -import { VSBuffer } from '../../../../../../../../base/common/buffer.js'; -import { BaseDecoder } from '../baseDecoder.js'; -import { ReadableStream } from '../../../../../../../../base/common/stream.js'; - -/** - * Type for all simple tokens. - */ -export type TSimpleToken = Space | Tab | VerticalTab | At | Quote | DoubleQuote - | CarriageReturn | NewLine | FormFeed | TBracket | TAngleBracket | TCurlyBrace - | TParenthesis | Colon | Hash | Dash | ExclamationMark | Slash | DollarSign | Comma - | TLineBreakToken; - -/** -* Type of tokens emitted by this decoder. -*/ -export type TSimpleDecoderToken = TSimpleToken | Word; - -/** - * List of well-known distinct tokens that this decoder emits (excluding - * the word stop characters defined below). Everything else is considered - * an arbitrary "text" sequence and is emitted as a single {@link Word} token. - */ -export const WELL_KNOWN_TOKENS: readonly ISimpleTokenClass[] = Object.freeze([ - LeftParenthesis, RightParenthesis, LeftBracket, RightBracket, LeftCurlyBrace, RightCurlyBrace, - LeftAngleBracket, RightAngleBracket, Space, Tab, VerticalTab, FormFeed, Colon, Hash, Dash, - ExclamationMark, At, Slash, DollarSign, Quote, DoubleQuote, Comma, -]); - -/** - * A {@link Word} sequence stops when one of the well-known tokens are encountered. - * Note! the `\r` and `\n` are excluded from the list because this decoder based on - * the {@link LinesDecoder} which emits {@link Line} tokens without them. - */ -const WORD_STOP_CHARACTERS: readonly string[] = Object.freeze( - WELL_KNOWN_TOKENS.map(token => token.symbol), -); - -/** - * A decoder that can decode a stream of `Line`s into a stream - * of simple token, - `Word`, `Space`, `Tab`, `NewLine`, etc. - */ -export class SimpleDecoder extends BaseDecoder { - constructor( - stream: ReadableStream, - ) { - super(new LinesDecoder(stream)); - } - - protected override onStreamData(line: TLineToken): void { - // re-emit new line tokens immediately - if (line instanceof CarriageReturn || line instanceof NewLine) { - this._onData.fire(line); - - return; - } - - // loop through the text separating it into `Word` and `well-known` tokens - const lineText = line.text.split(''); - let i = 0; - while (i < lineText.length) { - // index is 0-based, but column numbers are 1-based - const columnNumber = i + 1; - const character = lineText[i]; - - // check if the current character is a well-known token - const tokenConstructor = WELL_KNOWN_TOKENS - .find((wellKnownToken) => { - return wellKnownToken.symbol === character; - }); - - // if it is a well-known token, emit it and continue to the next one - if (tokenConstructor) { - this._onData.fire(SimpleToken.newOnLine(line, columnNumber, tokenConstructor)); - - i++; - continue; - } - - // otherwise, it is an arbitrary "text" sequence of characters, - // that needs to be collected into a single `Word` token, hence - // read all the characters until a stop character is encountered - let word = ''; - while (i < lineText.length && !(WORD_STOP_CHARACTERS.includes(lineText[i]))) { - word += lineText[i]; - i++; - } - - // emit a "text" sequence of characters as a single `Word` token - this._onData.fire( - Word.newOnLine(word, line, columnNumber), - ); - } - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/angleBrackets.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/angleBrackets.ts deleted file mode 100644 index d57acedc1c3..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/angleBrackets.ts +++ /dev/null @@ -1,61 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { SimpleToken } from './simpleToken.js'; - -/** - * A token that represent a `<` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class LeftAngleBracket extends SimpleToken<'<'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '<' = '<'; - - /** - * Return text representation of the token. - */ - public override get text(): '<' { - return LeftAngleBracket.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `left-angle-bracket${this.range}`; - } -} - -/** - * A token that represent a `>` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class RightAngleBracket extends SimpleToken<'>'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '>' = '>'; - - /** - * Return text representation of the token. - */ - public override get text(): '>' { - return RightAngleBracket.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `right-angle-bracket${this.range}`; - } -} - -/** - * General angle bracket token type. - */ -export type TAngleBracket = LeftAngleBracket | RightAngleBracket; diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/at.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/at.ts deleted file mode 100644 index 25c44433afd..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/at.ts +++ /dev/null @@ -1,31 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { SimpleToken } from './simpleToken.js'; - -/** - * A token that represent a `@` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class At extends SimpleToken<'@'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '@' = '@'; - - /** - * Return text representation of the token. - */ - public override get text(): '@' { - return At.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `at${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/brackets.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/brackets.ts deleted file mode 100644 index 491cb2a588c..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/brackets.ts +++ /dev/null @@ -1,61 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { SimpleToken } from './simpleToken.js'; - -/** - * A token that represent a `[` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class LeftBracket extends SimpleToken<'['> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '[' = '['; - - /** - * Return text representation of the token. - */ - public override get text(): '[' { - return LeftBracket.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `left-bracket${this.range}`; - } -} - -/** - * A token that represent a `]` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class RightBracket extends SimpleToken<']'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: ']' = ']'; - - /** - * Return text representation of the token. - */ - public override get text(): ']' { - return RightBracket.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `right-bracket${this.range}`; - } -} - -/** - * General bracket token type. - */ -export type TBracket = LeftBracket | RightBracket; diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/colon.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/colon.ts deleted file mode 100644 index a04b3e853df..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/colon.ts +++ /dev/null @@ -1,31 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { SimpleToken } from './simpleToken.js'; - -/** - * A token that represent a `:` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class Colon extends SimpleToken<':'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: ':' = ':'; - - /** - * Return text representation of the token. - */ - public override get text(): ':' { - return Colon.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `colon${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/comma.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/comma.ts deleted file mode 100644 index ce5df1748ca..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/comma.ts +++ /dev/null @@ -1,31 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { SimpleToken } from './simpleToken.js'; - -/** - * A token that represent a `,` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class Comma extends SimpleToken<','> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: ',' = ','; - - /** - * Return text representation of the token. - */ - public override get text(): ',' { - return Comma.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `comma${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/curlyBraces.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/curlyBraces.ts deleted file mode 100644 index 9c13e501d27..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/curlyBraces.ts +++ /dev/null @@ -1,61 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { SimpleToken } from './simpleToken.js'; - -/** - * A token that represent a `{` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class LeftCurlyBrace extends SimpleToken<'{'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '{' = '{'; - - /** - * Return text representation of the token. - */ - public override get text(): '{' { - return LeftCurlyBrace.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `left-curly-brace${this.range}`; - } -} - -/** - * A token that represent a `}` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class RightCurlyBrace extends SimpleToken<'}'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '}' = '}'; - - /** - * Return text representation of the token. - */ - public override get text(): '}' { - return RightCurlyBrace.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `right-curly-brace${this.range}`; - } -} - -/** - * General curly brace token type. - */ -export type TCurlyBrace = LeftCurlyBrace | RightCurlyBrace; diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/dash.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/dash.ts deleted file mode 100644 index a77b441ad26..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/dash.ts +++ /dev/null @@ -1,31 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { SimpleToken } from './simpleToken.js'; - -/** - * A token that represent a `-` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class Dash extends SimpleToken<'-'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '-' = '-'; - - /** - * Return text representation of the token. - */ - public override get text(): '-' { - return Dash.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `dash${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/dollarSign.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/dollarSign.ts deleted file mode 100644 index 5b81d0afda8..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/dollarSign.ts +++ /dev/null @@ -1,31 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { SimpleToken } from './simpleToken.js'; - -/** - * A token that represent a `$` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class DollarSign extends SimpleToken<'$'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '$' = '$'; - - /** - * Return text representation of the token. - */ - public override get text(): '$' { - return DollarSign.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `dollarSign${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/doubleQuote.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/doubleQuote.ts deleted file mode 100644 index ba041a03ed5..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/doubleQuote.ts +++ /dev/null @@ -1,40 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../../baseToken.js'; -import { SimpleToken } from './simpleToken.js'; - -/** - * A token that represent a `"` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class DoubleQuote extends SimpleToken<'"'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '"' = '"'; - - /** - * Return text representation of the token. - */ - public override get text(): '"' { - return DoubleQuote.symbol; - } - - /** - * Checks if the provided token is of the same type - * as the current one. - */ - public sameType(other: BaseToken): other is typeof this { - return (other instanceof this.constructor); - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `double-quote${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/exclamationMark.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/exclamationMark.ts deleted file mode 100644 index 32675fdf8a3..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/exclamationMark.ts +++ /dev/null @@ -1,31 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { SimpleToken } from './simpleToken.js'; - -/** - * A token that represent a `!` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class ExclamationMark extends SimpleToken<'!'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '!' = '!'; - - /** - * Return text representation of the token. - */ - public override get text(): '!' { - return ExclamationMark.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `exclamation-mark${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/formFeed.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/formFeed.ts deleted file mode 100644 index df5b8b0d446..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/formFeed.ts +++ /dev/null @@ -1,31 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { SimpleToken } from './simpleToken.js'; - -/** - * Token that represent a `form feed` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class FormFeed extends SimpleToken<'\f'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '\f' = '\f'; - - /** - * Return text representation of the token. - */ - public override get text(): '\f' { - return FormFeed.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `formfeed${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/hash.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/hash.ts deleted file mode 100644 index f499085859d..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/hash.ts +++ /dev/null @@ -1,31 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { SimpleToken } from './simpleToken.js'; - -/** - * A token that represent a `#` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class Hash extends SimpleToken<'#'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '#' = '#'; - - /** - * Return text representation of the token. - */ - public override get text(): '#' { - return Hash.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `hash${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/parentheses.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/parentheses.ts deleted file mode 100644 index b7ef4f5bc2a..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/parentheses.ts +++ /dev/null @@ -1,61 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { SimpleToken } from './simpleToken.js'; - -/** - * A token that represent a `(` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class LeftParenthesis extends SimpleToken<'('> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '(' = '('; - - /** - * Return text representation of the token. - */ - public override get text(): '(' { - return LeftParenthesis.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `left-parenthesis${this.range}`; - } -} - -/** - * A token that represent a `)` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class RightParenthesis extends SimpleToken<')'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: ')' = ')'; - - /** - * Return text representation of the token. - */ - public override get text(): ')' { - return RightParenthesis.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `right-parenthesis${this.range}`; - } -} - -/** - * General parenthesis token type. - */ -export type TParenthesis = LeftParenthesis | RightParenthesis; diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/quote.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/quote.ts deleted file mode 100644 index 3c262f67b49..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/quote.ts +++ /dev/null @@ -1,40 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../../baseToken.js'; -import { SimpleToken } from './simpleToken.js'; - -/** - * A token that represent a `'` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class Quote extends SimpleToken<`'`> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '\'' = '\''; - - /** - * Return text representation of the token. - */ - public override get text(): '\'' { - return Quote.symbol; - } - - /** - * Checks if the provided token is of the same type - * as the current one. - */ - public sameType(other: BaseToken): other is Quote { - return (other instanceof this.constructor); - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `quote${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/simpleToken.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/simpleToken.ts deleted file mode 100644 index 5a611bb1ddd..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/simpleToken.ts +++ /dev/null @@ -1,59 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { Range } from '../../../../../../../../../editor/common/core/range.js'; -import { BaseToken } from '../../baseToken.js'; -import { Line } from '../../linesCodec/tokens/line.js'; - -/** - * Interface for a class that can be instantiated into a {@link SimpleToken}. - */ -export interface ISimpleTokenClass> { - /** - * Character representing the token. - */ - readonly symbol: string; - - /** - * Constructor for the token. - */ - new(...args: any[]): TSimpleToken; -} - -/** - * Base class for all "simple" tokens with a `range`. - * A simple token is the one that represents a single character. - */ -export abstract class SimpleToken extends BaseToken { - /** - * The underlying symbol of the token. - */ - public static readonly symbol: string; - - /** - * Create new token instance with range inside - * the given `Line` at the given `column number`. - */ - public static newOnLine>( - line: Line, - atColumnNumber: number, - Constructor: ISimpleTokenClass, - ): TSimpleToken { - const { range } = line; - - return new Constructor(new Range( - range.startLineNumber, - atColumnNumber, - range.startLineNumber, - atColumnNumber + Constructor.symbol.length, - )); - } -} - -/** - * Base class for all tokens that represent some form of - * a spacing character, e.g. 'space', 'tab', etc. - */ -export abstract class SpacingToken extends SimpleToken { } diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/slash.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/slash.ts deleted file mode 100644 index 04d382bc89f..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/slash.ts +++ /dev/null @@ -1,31 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { SimpleToken } from './simpleToken.js'; - -/** - * A token that represent a `/` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class Slash extends SimpleToken<'/'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '/' = '/'; - - /** - * Return text representation of the token. - */ - public override get text(): '/' { - return Slash.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `slash${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/space.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/space.ts deleted file mode 100644 index 07eed9f58b2..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/space.ts +++ /dev/null @@ -1,31 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { SpacingToken } from './simpleToken.js'; - -/** - * A token that represent a `space` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class Space extends SpacingToken<' '> { - /** - * The underlying symbol of the `Space` token. - */ - public static override readonly symbol: ' ' = ' '; - - /** - * Return text representation of the token. - */ - public override get text(): ' ' { - return Space.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `space${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/tab.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/tab.ts deleted file mode 100644 index 718a01df813..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/tab.ts +++ /dev/null @@ -1,31 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { SpacingToken } from './simpleToken.js'; - -/** - * A token that represent a `tab` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class Tab extends SpacingToken<'\t'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '\t' = '\t'; - - /** - * Return text representation of the token. - */ - public override get text(): '\t' { - return Tab.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `tab${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/tokens.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/tokens.ts deleted file mode 100644 index ee25fb83ddf..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/tokens.ts +++ /dev/null @@ -1,25 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -export { At } from './at.js'; -export { Tab } from './tab.js'; -export { Dash } from './dash.js'; -export { Hash } from './hash.js'; -export { Word } from './word.js'; -export { Colon } from './colon.js'; -export { Quote } from './quote.js'; -export { Slash } from './slash.js'; -export { Space } from './space.js'; -export { Comma } from './comma.js'; -export { FormFeed } from './formFeed.js'; -export { DollarSign } from './dollarSign.js'; -export { VerticalTab } from './verticalTab.js'; -export { DoubleQuote } from './doubleQuote.js'; -export { ExclamationMark } from './exclamationMark.js'; -export { SimpleToken, SpacingToken } from './simpleToken.js'; -export { type TBracket, LeftBracket, RightBracket } from './brackets.js'; -export { type TCurlyBrace, LeftCurlyBrace, RightCurlyBrace } from './curlyBraces.js'; -export { type TParenthesis, LeftParenthesis, RightParenthesis } from './parentheses.js'; -export { type TAngleBracket, LeftAngleBracket, RightAngleBracket } from './angleBrackets.js'; diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/verticalTab.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/verticalTab.ts deleted file mode 100644 index 4b7b4241433..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/verticalTab.ts +++ /dev/null @@ -1,31 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { SpacingToken } from './simpleToken.js'; - -/** - * Token that represent a `vertical tab` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class VerticalTab extends SpacingToken<'\v'> { - /** - * The underlying symbol of the `VerticalTab` token. - */ - public static override readonly symbol: '\v' = '\v'; - - /** - * Return text representation of the token. - */ - public override get text(): '\v' { - return VerticalTab.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `vtab${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/word.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/word.ts deleted file mode 100644 index bea656895fc..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/word.ts +++ /dev/null @@ -1,60 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../../baseToken.js'; -import { Line } from '../../linesCodec/tokens/line.js'; -import { Range } from '../../../../../../../../../editor/common/core/range.js'; - -/** - * A token that represent a word - a set of continuous - * characters without stop characters, like a `space`, - * a `tab`, or a `new line`. - */ -export class Word extends BaseToken { - constructor( - /** - * The word range. - */ - range: Range, - - /** - * The string value of the word. - */ - public readonly text: TText, - ) { - super(range); - } - - /** - * Create new `Word` token with the given `text` and the range - * inside the given `Line` at the specified `column number`. - */ - public static newOnLine( - text: string, - line: Line | number, - atColumnNumber: number, - ): Word { - const startLineNumber = (typeof line === 'number') - ? line - : line.range.startLineNumber; - - const range = new Range( - startLineNumber, atColumnNumber, - startLineNumber, atColumnNumber + text.length - ); - - return new Word( - range, - text, - ); - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `word("${this.shortText()}")${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/textToken.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/textToken.ts deleted file mode 100644 index ecb4c61c94e..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/textToken.ts +++ /dev/null @@ -1,19 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { type BaseToken } from './baseToken.js'; -import { CompositeToken } from './compositeToken.js'; - -/** - * Tokens that represent a sequence of tokens that does not - * hold an additional meaning in the text. - */ -export class Text< - TTokens extends readonly BaseToken[] = readonly BaseToken[], -> extends CompositeToken { - public override toString(): string { - return `text(${this.shortText()})${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/utils/objectStream.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/utils/objectStream.ts deleted file mode 100644 index 3c8476c2d56..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/utils/objectStream.ts +++ /dev/null @@ -1,224 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { assertNever } from '../../../../../../../../base/common/assert.js'; -import { CancellationToken } from '../../../../../../../../base/common/cancellation.js'; -import { ObservableDisposable } from '../../../utils/observableDisposable.js'; -import { newWriteableStream, ReadableStream, WriteableStream } from '../../../../../../../../base/common/stream.js'; - - -/** - * A readable stream of provided objects. - */ -export class ObjectStream extends ObservableDisposable implements ReadableStream { - /** - * Flag that indicates whether the stream has ended. - */ - private ended: boolean = false; - - /** - * Underlying writable stream instance. - */ - private readonly stream: WriteableStream; - - /** - * Interval reference that is used to periodically send - * objects to the stream in the background. - */ - private timeoutHandle: Timeout | undefined; - - constructor( - private readonly data: Generator, - private readonly cancellationToken?: CancellationToken, - ) { - super(); - - this.stream = newWriteableStream(null); - - if (cancellationToken?.isCancellationRequested) { - this.end(); - return; - } - - // send a first batch of data immediately - this.send(true); - } - - /** - * Starts process of sending data to the stream. - * - * @param stopAfterFirstSend whether to continue sending data to the stream - * or stop sending after the first batch of data is sent instead - */ - public send( - stopAfterFirstSend: boolean = false, - ): void { - // this method can be called asynchronously by the `setTimeout` utility below, hence - // the state of the cancellation token or the stream itself might have changed by that time - if (this.cancellationToken?.isCancellationRequested || this.ended) { - this.end(); - - return; - } - - this.sendData() - .then(() => { - if (this.cancellationToken?.isCancellationRequested || this.ended) { - this.end(); - - return; - } - - if (stopAfterFirstSend === true) { - this.stopStream(); - return; - } - - this.timeoutHandle = setTimeout(this.send.bind(this)); - }) - .catch((error) => { - this.stream.error(error); - this.dispose(); - }); - } - - /** - * Stop the data sending loop. - */ - public stopStream(): this { - if (this.timeoutHandle === undefined) { - return this; - } - - clearTimeout(this.timeoutHandle); - this.timeoutHandle = undefined; - - return this; - } - - /** - * Sends a provided number of objects to the stream. - */ - private async sendData( - objectsCount: number = 25, - ): Promise { - // send up to 'objectsCount' objects at a time - while (objectsCount > 0) { - try { - const next = this.data.next(); - if (next.done || this.cancellationToken?.isCancellationRequested) { - this.end(); - - return; - } - - await this.stream.write(next.value); - objectsCount--; - } catch (error) { - this.stream.error(error); - this.dispose(); - return; - } - } - } - - /** - * Ends the stream and stops sending data objects. - */ - private end(): this { - if (this.ended) { - return this; - } - this.ended = true; - - this.stopStream(); - this.stream.end(); - return this; - } - - public pause(): void { - this.stopStream(); - this.stream.pause(); - - return; - } - - public resume(): void { - this.send(); - this.stream.resume(); - - return; - } - - public destroy(): void { - this.dispose(); - } - - public removeListener(event: string, callback: (...args: any[]) => void): void { - this.stream.removeListener(event, callback); - - return; - } - - public on(event: 'data', callback: (data: T) => void): void; - public on(event: 'error', callback: (err: Error) => void): void; - public on(event: 'end', callback: () => void): void; - public on(event: 'data' | 'error' | 'end', callback: (...args: any[]) => void): void { - if (event === 'data') { - this.stream.on(event, callback); - // this is the convention of the readable stream, - when - // the `data` event is registered, the stream is started - this.send(); - - return; - } - - if (event === 'error') { - this.stream.on(event, callback); - return; - } - - if (event === 'end') { - this.stream.on(event, callback); - return; - } - - assertNever( - event, - `Unexpected event name '${event}'.`, - ); - } - - /** - * Cleanup send interval and destroy the stream. - */ - public override dispose(): void { - this.stopStream(); - this.stream.destroy(); - - super.dispose(); - } - - /** - * Create new instance of the stream from a provided array. - */ - public static fromArray( - array: T[], - cancellationToken?: CancellationToken, - ): ObjectStream { - return new ObjectStream(arrayToGenerator(array), cancellationToken); - } -} - -/** - * Create a generator out of a provided array. - */ -export function arrayToGenerator>(array: T[]): Generator { - return (function* (): Generator { - for (const item of array) { - yield item; - } - })(); -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/utils/objectStreamFromTextModel.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/utils/objectStreamFromTextModel.ts deleted file mode 100644 index 2a9b2173b5d..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/utils/objectStreamFromTextModel.ts +++ /dev/null @@ -1,46 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { ITextModel } from '../../../../../../../../editor/common/model.js'; -import { ObjectStream } from './objectStream.js'; -import { VSBuffer } from '../../../../../../../../base/common/buffer.js'; -import { CancellationToken } from '../../../../../../../../base/common/cancellation.js'; - -/** - * Create new instance of the stream from a provided text model. - */ -export function objectStreamFromTextModel( - model: ITextModel, - cancellationToken?: CancellationToken, -): ObjectStream { - return new ObjectStream(modelToGenerator(model), cancellationToken); -} - -/** - * Create a generator out of a provided text model. - */ -function modelToGenerator(model: ITextModel): Generator { - return (function* (): Generator { - const totalLines = model.getLineCount(); - let currentLine = 1; - - while (currentLine <= totalLines) { - if (model.isDisposed()) { - return undefined; - } - - yield VSBuffer.fromString( - model.getLineContent(currentLine), - ); - if (currentLine !== totalLines) { - yield VSBuffer.fromString( - model.getEOL(), - ); - } - - currentLine++; - } - })(); -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/chatPromptCodec.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/chatPromptCodec.ts deleted file mode 100644 index da27950bf92..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/chatPromptCodec.ts +++ /dev/null @@ -1,73 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { VSBuffer } from '../../../../../../base/common/buffer.js'; -import { ReadableStream } from '../../../../../../base/common/stream.js'; -import { ChatPromptDecoder, TChatPromptToken } from './chatPromptDecoder.js'; - -/** - * A codec is an object capable of encoding/decoding a stream of data transforming its messages. - * Useful for abstracting a data transfer or protocol logic on top of a stream of bytes. - * - * For instance, if protocol messages need to be transferred over `TCP` connection, a codec that - * encodes the messages into a sequence of bytes before sending it to a network socket. Likewise, - * on the other end of the connection, the same codec can decode the sequence of bytes back into - * a sequence of the protocol messages. - */ -export interface ICodec { - /** - * Encode a stream of `K`s into a stream of `T`s. - */ - encode: (value: ReadableStream) => ReadableStream; - - /** - * Decode a stream of `T`s into a stream of `K`s. - */ - decode: (value: ReadableStream) => ReadableStream; -} - - -/** - * `ChatPromptCodec` type is a `ICodec` with specific types for - * stream messages and return types of the `encode`/`decode` functions. - * @see {@link ICodec} - */ -interface IChatPromptCodec extends ICodec { - /** - * Decode a stream of `VSBuffer`s into a stream of `TChatPromptToken`s. - * - * @see {@link TChatPromptToken} - * @see {@link VSBuffer} - * @see {@link ChatPromptDecoder} - */ - decode: (value: ReadableStream) => ChatPromptDecoder; -} - -/** - * Codec that is capable to encode and decode tokens of an AI chatbot prompt message. - */ -export const ChatPromptCodec: IChatPromptCodec = Object.freeze({ - /** - * Encode a stream of `TChatPromptToken`s into a stream of `VSBuffer`s. - * - * @see {@link ReadableStream} - * @see {@link VSBuffer} - */ - encode: (_stream: ReadableStream): ReadableStream => { - throw new Error('The `encode` method is not implemented.'); - }, - - /** - * Decode a of `VSBuffer`s into a readable of `TChatPromptToken`s. - * - * @see {@link TChatPromptToken} - * @see {@link VSBuffer} - * @see {@link ChatPromptDecoder} - * @see {@link ReadableStream} - */ - decode: (stream: ReadableStream): ChatPromptDecoder => { - return new ChatPromptDecoder(stream); - }, -}); diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/chatPromptDecoder.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/chatPromptDecoder.ts deleted file mode 100644 index 5ac4a138fbe..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/chatPromptDecoder.ts +++ /dev/null @@ -1,202 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { PromptToken } from './tokens/promptToken.js'; -import { PromptAtMention } from './tokens/promptAtMention.js'; -import { VSBuffer } from '../../../../../../base/common/buffer.js'; -import { PromptSlashCommand } from './tokens/promptSlashCommand.js'; -import { ReadableStream } from '../../../../../../base/common/stream.js'; -import { PartialPromptAtMention } from './parsers/promptAtMentionParser.js'; -import { PromptTemplateVariable } from './tokens/promptTemplateVariable.js'; -import { assert, assertNever } from '../../../../../../base/common/assert.js'; -import { PartialPromptSlashCommand } from './parsers/promptSlashCommandParser.js'; -import { BaseDecoder } from './base/baseDecoder.js'; -import { PromptVariable, PromptVariableWithData } from './tokens/promptVariable.js'; -import { At } from './base/simpleCodec/tokens/at.js'; -import { Hash } from './base/simpleCodec/tokens/hash.js'; -import { Slash } from './base/simpleCodec/tokens/slash.js'; -import { DollarSign } from './base/simpleCodec/tokens/dollarSign.js'; -import { PartialPromptVariableName, PartialPromptVariableWithData } from './parsers/promptVariableParser.js'; -import { MarkdownDecoder, TMarkdownToken } from './base/markdownCodec/markdownDecoder.js'; -import { PartialPromptTemplateVariable, PartialPromptTemplateVariableStart, TPromptTemplateVariableParser } from './parsers/promptTemplateVariableParser.js'; - -/** - * Tokens produced by this decoder. - */ -export type TChatPromptToken = TMarkdownToken | (PromptVariable | PromptVariableWithData) - | PromptAtMention | PromptSlashCommand | PromptTemplateVariable; - -/** - * Decoder for the common chatbot prompt message syntax. - * For instance, the file references `#file:./path/file.md` are handled by this decoder. - */ -export class ChatPromptDecoder extends BaseDecoder { - /** - * Currently active parser object that is used to parse a well-known sequence of - * tokens, for instance, a `#file:/path/to/file.md` link that consists of `hash`, - * `word`, and `colon` tokens sequence plus the `file path` part that follows. - */ - private current?: (PartialPromptVariableName | PartialPromptVariableWithData) - | PartialPromptAtMention | PartialPromptSlashCommand - | TPromptTemplateVariableParser; - - constructor( - stream: ReadableStream, - ) { - super(new MarkdownDecoder(stream)); - } - - protected override onStreamData(token: TMarkdownToken): void { - // prompt `#variables` always start with the `#` character, hence - // initiate a parser object if we encounter respective token and - // there is no active parser object present at the moment - if ((token instanceof Hash) && !this.current) { - this.current = new PartialPromptVariableName(token); - - return; - } - - // prompt `@mentions` always start with the `@` character, hence - // initiate a parser object if we encounter respective token and - // there is no active parser object present at the moment - if ((token instanceof At) && !this.current) { - this.current = new PartialPromptAtMention(token); - - return; - } - - // prompt `/commands` always start with the `/` character, hence - // initiate a parser object if we encounter respective token and - // there is no active parser object present at the moment - if ((token instanceof Slash) && !this.current) { - this.current = new PartialPromptSlashCommand(token); - - return; - } - - // prompt `${template:variables}` always start with the `$` character, - // hence initiate a parser object if we encounter respective token and - // there is no active parser object present at the moment - if ((token instanceof DollarSign) && !this.current) { - this.current = new PartialPromptTemplateVariableStart(token); - - return; - } - - // if current parser was not yet initiated, - we are in the general "text" - // parsing mode, therefore re-emit the token immediately and continue - if (!this.current) { - this._onData.fire(token); - return; - } - - // if there is a current parser object, submit the token to it - // so it can progress with parsing the tokens sequence - const parseResult = this.current.accept(token); - - // process the parse result next - switch (parseResult.result) { - // in the case of success there might be 2 cases: - // 1) parsing fully completed and an instance of `PromptToken` is returned back, - // in this case, emit the parsed token (e.g., a `link`) and reset the current - // parser object reference so a new parsing process can be initiated next - // 2) parsing is still in progress and the next parser object is returned, hence - // we need to replace the current parser object with a new one and continue - case 'success': { - const { nextParser } = parseResult; - - if (nextParser instanceof PromptToken) { - this._onData.fire(nextParser); - delete this.current; - } else { - this.current = nextParser; - } - - break; - } - // in the case of failure, reset the current parser object - case 'failure': { - // if failed to parse a sequence of a tokens, re-emit the tokens accumulated - // so far then reset the current parser object - this.reEmitCurrentTokens(); - break; - } - } - - // if token was not consumed by the parser, call `onStreamData` again - // so the token is properly handled by the decoder in the case when a - // new sequence starts with this token - if (!parseResult.wasTokenConsumed) { - this.onStreamData(token); - } - } - - protected override onStreamEnd(): void { - try { - // if there is no currently active parser object present, nothing to do - if (this.current === undefined) { - return; - } - - // otherwise try to convert unfinished parser object to a token - - if (this.current instanceof PartialPromptVariableName) { - this._onData.fire(this.current.asPromptVariable()); - return; - } - - if (this.current instanceof PartialPromptVariableWithData) { - this._onData.fire(this.current.asPromptVariableWithData()); - return; - } - - if (this.current instanceof PartialPromptAtMention) { - this._onData.fire(this.current.asPromptAtMention()); - return; - } - - if (this.current instanceof PartialPromptSlashCommand) { - this._onData.fire(this.current.asPromptSlashCommand()); - return; - } - - assert( - (this.current instanceof PartialPromptTemplateVariableStart) === false, - 'Incomplete template variable token.', - ); - - if (this.current instanceof PartialPromptTemplateVariable) { - this._onData.fire(this.current.asPromptTemplateVariable()); - return; - } - - assertNever( - this.current, - `Unknown parser object '${this.current}'`, - ); - } catch (_error) { - // if failed to convert current parser object to a token, - // re-emit the tokens accumulated so far - this.reEmitCurrentTokens(); - } finally { - delete this.current; - super.onStreamEnd(); - } - } - - /** - * Re-emit tokens accumulated so far in the current parser object. - */ - protected reEmitCurrentTokens(): void { - if (this.current === undefined) { - return; - } - - for (const token of this.current.tokens) { - this._onData.fire(token); - } - delete this.current; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/parsers/promptAtMentionParser.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/parsers/promptAtMentionParser.ts deleted file mode 100644 index 2173ba438ad..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/parsers/promptAtMentionParser.ts +++ /dev/null @@ -1,121 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { PromptAtMention } from '../tokens/promptAtMention.js'; -import { assert } from '../../../../../../../base/common/assert.js'; -import { Range } from '../../../../../../../editor/common/core/range.js'; -import { BaseToken } from '../base/baseToken.js'; -import { At } from '../base/simpleCodec/tokens/at.js'; -import { Tab } from '../base/simpleCodec/tokens/tab.js'; -import { Hash } from '../base/simpleCodec/tokens/hash.js'; -import { Space } from '../base/simpleCodec/tokens/space.js'; -import { Colon } from '../base/simpleCodec/tokens/colon.js'; -import { NewLine } from '../base/linesCodec/tokens/newLine.js'; -import { FormFeed } from '../base/simpleCodec/tokens/formFeed.js'; -import { VerticalTab } from '../base/simpleCodec/tokens/verticalTab.js'; -import { TSimpleDecoderToken } from '../base/simpleCodec/simpleDecoder.js'; -import { CarriageReturn } from '../base/linesCodec/tokens/carriageReturn.js'; -import { ExclamationMark } from '../base/simpleCodec/tokens/exclamationMark.js'; -import { LeftBracket, RightBracket } from '../base/simpleCodec/tokens/brackets.js'; -import { LeftAngleBracket, RightAngleBracket } from '../base/simpleCodec/tokens/angleBrackets.js'; -import { assertNotConsumed, ParserBase, TAcceptTokenResult } from '../base/simpleCodec/parserBase.js'; - -/** - * List of characters that terminate the prompt at-mention sequence. - */ -export const STOP_CHARACTERS: readonly string[] = [Space, Tab, NewLine, CarriageReturn, VerticalTab, FormFeed, At, Colon, Hash] - .map((token) => { return token.symbol; }); - -/** - * List of characters that cannot be in an at-mention name (excluding the {@link STOP_CHARACTERS}). - */ -export const INVALID_NAME_CHARACTERS: readonly string[] = [ExclamationMark, LeftAngleBracket, RightAngleBracket, LeftBracket, RightBracket] - .map((token) => { return token.symbol; }); - -/** - * The parser responsible for parsing a `prompt @mention` sequences. - * E.g., `@workspace` or `@github` participant mention. - */ -export class PartialPromptAtMention extends ParserBase { - constructor(token: At) { - super([token]); - } - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - // if a `stop` character is encountered, finish the parsing process - if (STOP_CHARACTERS.includes(token.text)) { - try { - // if it is possible to convert current parser to `PromptAtMention`, return success result - return { - result: 'success', - nextParser: this.asPromptAtMention(), - wasTokenConsumed: false, - }; - } catch (error) { - // otherwise fail - return { - result: 'failure', - wasTokenConsumed: false, - }; - } finally { - // in any case this is an end of the parsing process - this.isConsumed = true; - } - } - - // variables cannot have {@link INVALID_NAME_CHARACTERS} in their names - if (INVALID_NAME_CHARACTERS.includes(token.text)) { - this.isConsumed = true; - - return { - result: 'failure', - wasTokenConsumed: false, - }; - } - - // otherwise it is a valid name character, so add it to the list of - // the current tokens and continue the parsing process - this.currentTokens.push(token); - - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } - - /** - * Try to convert current parser instance into a fully-parsed {@link PromptAtMention} token. - * - * @throws if sequence of tokens received so far do not constitute a valid prompt variable, - * for instance, if there is only `1` starting `@` token is available. - */ - public asPromptAtMention(): PromptAtMention { - // if there is only one token before the stop character - // must be the starting `@` one), then fail - assert( - this.currentTokens.length > 1, - 'Cannot create a prompt @mention out of incomplete token sequence.', - ); - - const firstToken = this.currentTokens[0]; - const lastToken = this.currentTokens[this.currentTokens.length - 1]; - - // render the characters above into strings, excluding the starting `@` character - const nameTokens = this.currentTokens.slice(1); - const atMentionName = BaseToken.render(nameTokens); - - return new PromptAtMention( - new Range( - firstToken.range.startLineNumber, - firstToken.range.startColumn, - lastToken.range.endLineNumber, - lastToken.range.endColumn, - ), - atMentionName, - ); - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/parsers/promptSlashCommandParser.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/parsers/promptSlashCommandParser.ts deleted file mode 100644 index 557f5e74379..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/parsers/promptSlashCommandParser.ts +++ /dev/null @@ -1,122 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { assert } from '../../../../../../../base/common/assert.js'; -import { PromptSlashCommand } from '../tokens/promptSlashCommand.js'; -import { Range } from '../../../../../../../editor/common/core/range.js'; -import { BaseToken } from '../base/baseToken.js'; -import { At } from '../base/simpleCodec/tokens/at.js'; -import { Tab } from '../base/simpleCodec/tokens/tab.js'; -import { Hash } from '../base/simpleCodec/tokens/hash.js'; -import { Slash } from '../base/simpleCodec/tokens/slash.js'; -import { Space } from '../base/simpleCodec/tokens/space.js'; -import { Colon } from '../base/simpleCodec/tokens/colon.js'; -import { NewLine } from '../base/linesCodec/tokens/newLine.js'; -import { FormFeed } from '../base/simpleCodec/tokens/formFeed.js'; -import { VerticalTab } from '../base/simpleCodec/tokens/verticalTab.js'; -import { TSimpleDecoderToken } from '../base/simpleCodec/simpleDecoder.js'; -import { CarriageReturn } from '../base/linesCodec/tokens/carriageReturn.js'; -import { ExclamationMark } from '../base/simpleCodec/tokens/exclamationMark.js'; -import { LeftBracket, RightBracket } from '../base/simpleCodec/tokens/brackets.js'; -import { LeftAngleBracket, RightAngleBracket } from '../base/simpleCodec/tokens/angleBrackets.js'; -import { assertNotConsumed, ParserBase, TAcceptTokenResult } from '../base/simpleCodec/parserBase.js'; - -/** - * List of characters that terminate the prompt at-mention sequence. - */ -export const STOP_CHARACTERS: readonly string[] = [Space, Tab, NewLine, CarriageReturn, VerticalTab, FormFeed, Colon, At, Hash, Slash] - .map((token) => { return token.symbol; }); - -/** - * List of characters that cannot be in an at-mention name (excluding the {@link STOP_CHARACTERS}). - */ -export const INVALID_NAME_CHARACTERS: readonly string[] = [ExclamationMark, LeftAngleBracket, RightAngleBracket, LeftBracket, RightBracket] - .map((token) => { return token.symbol; }); - -/** - * The parser responsible for parsing a `prompt /command` sequences. - * E.g., `/search` or `/explain` command. - */ -export class PartialPromptSlashCommand extends ParserBase { - constructor(token: Slash) { - super([token]); - } - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - // if a `stop` character is encountered, finish the parsing process - if (STOP_CHARACTERS.includes(token.text)) { - try { - // if it is possible to convert current parser to `PromptSlashCommand`, return success result - return { - result: 'success', - nextParser: this.asPromptSlashCommand(), - wasTokenConsumed: false, - }; - } catch (error) { - // otherwise fail - return { - result: 'failure', - wasTokenConsumed: false, - }; - } finally { - // in any case this is an end of the parsing process - this.isConsumed = true; - } - } - - // variables cannot have {@link INVALID_NAME_CHARACTERS} in their names - if (INVALID_NAME_CHARACTERS.includes(token.text)) { - this.isConsumed = true; - - return { - result: 'failure', - wasTokenConsumed: false, - }; - } - - // otherwise it is a valid name character, so add it to the list of - // the current tokens and continue the parsing process - this.currentTokens.push(token); - - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } - - /** - * Try to convert current parser instance into a fully-parsed {@link PromptSlashCommand} token. - * - * @throws if sequence of tokens received so far do not constitute a valid prompt variable, - * for instance, if there is only `1` starting `/` token is available. - */ - public asPromptSlashCommand(): PromptSlashCommand { - // if there is only one token before the stop character - // must be the starting `/` one), then fail - assert( - this.currentTokens.length > 1, - 'Cannot create a prompt /command out of incomplete token sequence.', - ); - - const firstToken = this.currentTokens[0]; - const lastToken = this.currentTokens[this.currentTokens.length - 1]; - - // render the characters above into strings, excluding the starting `/` character - const nameTokens = this.currentTokens.slice(1); - const atMentionName = BaseToken.render(nameTokens); - - return new PromptSlashCommand( - new Range( - firstToken.range.startLineNumber, - firstToken.range.startColumn, - lastToken.range.endLineNumber, - lastToken.range.endColumn, - ), - atMentionName, - ); - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/parsers/promptTemplateVariableParser.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/parsers/promptTemplateVariableParser.ts deleted file mode 100644 index b97ad0fd479..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/parsers/promptTemplateVariableParser.ts +++ /dev/null @@ -1,148 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { assert } from '../../../../../../../base/common/assert.js'; -import { PromptTemplateVariable } from '../tokens/promptTemplateVariable.js'; -import { BaseToken } from '../base/baseToken.js'; -import { TSimpleDecoderToken } from '../base/simpleCodec/simpleDecoder.js'; -import { DollarSign, LeftCurlyBrace, RightCurlyBrace } from '../base/simpleCodec/tokens/tokens.js'; -import { assertNotConsumed, ParserBase, TAcceptTokenResult } from '../base/simpleCodec/parserBase.js'; - -/** - * Parsers of the `${variable}` token sequence in a prompt text. - */ -export type TPromptTemplateVariableParser = PartialPromptTemplateVariableStart | PartialPromptTemplateVariable; - -/** - * Parser that handles start sequence of a `${variable}` token sequence in - * a prompt text. Transitions to {@link PartialPromptTemplateVariable} parser - * as soon as the `${` character sequence is found. - */ -export class PartialPromptTemplateVariableStart extends ParserBase { - constructor(token: DollarSign) { - super([token]); - } - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - if (token instanceof LeftCurlyBrace) { - this.currentTokens.push(token); - - this.isConsumed = true; - return { - result: 'success', - nextParser: new PartialPromptTemplateVariable(this.currentTokens), - wasTokenConsumed: true, - }; - } - - return { - result: 'failure', - wasTokenConsumed: false, - }; - } -} - -/** - * Parser that handles a partial `${variable}` token sequence in a prompt text. - */ -export class PartialPromptTemplateVariable extends ParserBase { - constructor(tokens: (DollarSign | LeftCurlyBrace)[]) { - super(tokens); - } - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - // template variables are terminated by the `}` character - if (token instanceof RightCurlyBrace) { - this.currentTokens.push(token); - - this.isConsumed = true; - return { - result: 'success', - nextParser: this.asPromptTemplateVariable(), - wasTokenConsumed: true, - }; - } - - // otherwise it is a valid name character, so add it to the list of - // the current tokens and continue the parsing process - this.currentTokens.push(token); - - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } - - /** - * Returns a string representation of the prompt template variable - * contents, if any is present. - */ - private get contents(): string { - const contentTokens: TSimpleDecoderToken[] = []; - - // template variables are surrounded by `${}`, hence we need to have - // at least `${` plus one character for the contents to be non-empty - if (this.currentTokens.length < 3) { - return ''; - } - - // collect all tokens besides the first two (`${`) and a possible `}` at the end - for (let i = 2; i < this.currentTokens.length; i++) { - const token = this.currentTokens[i]; - const isLastToken = (i === this.currentTokens.length - 1); - - if ((token instanceof RightCurlyBrace) && (isLastToken === true)) { - break; - } - - contentTokens.push(token); - } - - return BaseToken.render(contentTokens); - } - - /** - * Try to convert current parser instance into a {@link PromptTemplateVariable} token. - * - * @throws if: - * - current tokens sequence cannot be converted to a valid template variable token - */ - public asPromptTemplateVariable(): PromptTemplateVariable { - const firstToken = this.currentTokens[0]; - const secondToken = this.currentTokens[1]; - const lastToken = this.currentTokens[this.currentTokens.length - 1]; - - // template variables are surrounded by `${}`, hence we need - // to have at least 3 tokens in the list for a valid one - assert( - this.currentTokens.length >= 3, - 'Prompt template variable should have at least 3 tokens.', - ); - - // a complete template variable must end with a `}` - assert( - lastToken instanceof RightCurlyBrace, - 'Last token is not a "}".', - ); - - // sanity checks of the first and second tokens - assert( - firstToken instanceof DollarSign, - 'First token must be a "$".', - ); - assert( - secondToken instanceof LeftCurlyBrace, - 'Second token must be a "{".', - ); - - return new PromptTemplateVariable( - BaseToken.fullRange(this.currentTokens), - this.contents, - ); - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/parsers/promptVariableParser.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/parsers/promptVariableParser.ts deleted file mode 100644 index ef68e78ec2d..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/parsers/promptVariableParser.ts +++ /dev/null @@ -1,252 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { assert } from '../../../../../../../base/common/assert.js'; -import { Range } from '../../../../../../../editor/common/core/range.js'; -import { BaseToken } from '../base/baseToken.js'; -import { PromptVariable, PromptVariableWithData } from '../tokens/promptVariable.js'; -import { At } from '../base/simpleCodec/tokens/at.js'; -import { Tab } from '../base/simpleCodec/tokens/tab.js'; -import { Hash } from '../base/simpleCodec/tokens/hash.js'; -import { Space } from '../base/simpleCodec/tokens/space.js'; -import { Colon } from '../base/simpleCodec/tokens/colon.js'; -import { NewLine } from '../base/linesCodec/tokens/newLine.js'; -import { FormFeed } from '../base/simpleCodec/tokens/formFeed.js'; -import { VerticalTab } from '../base/simpleCodec/tokens/verticalTab.js'; -import { TSimpleDecoderToken } from '../base/simpleCodec/simpleDecoder.js'; -import { CarriageReturn } from '../base/linesCodec/tokens/carriageReturn.js'; -import { ExclamationMark } from '../base/simpleCodec/tokens/exclamationMark.js'; -import { LeftBracket, RightBracket } from '../base/simpleCodec/tokens/brackets.js'; -import { LeftAngleBracket, RightAngleBracket } from '../base/simpleCodec/tokens/angleBrackets.js'; -import { assertNotConsumed, ParserBase, TAcceptTokenResult } from '../base/simpleCodec/parserBase.js'; - -/** - * List of characters that terminate the prompt variable sequence. - */ -export const STOP_CHARACTERS: readonly string[] = [Space, Tab, NewLine, CarriageReturn, VerticalTab, FormFeed, Hash, At] - .map((token) => { return token.symbol; }); - -/** - * List of characters that cannot be in a variable name (excluding the {@link STOP_CHARACTERS}). - */ -export const INVALID_NAME_CHARACTERS: readonly string[] = [Hash, Colon, ExclamationMark, LeftAngleBracket, RightAngleBracket, LeftBracket, RightBracket] - .map((token) => { return token.symbol; }); - -/** - * The parser responsible for parsing a `prompt variable name`. - * E.g., `#selection` or `#codebase` variable. If the `:` character follows - * the variable name, the parser transitions to {@link PartialPromptVariableWithData} - * that is also able to parse the `data` part of the variable. E.g., the `#file` part - * of the `#file:/path/to/something.md` sequence. - */ -export class PartialPromptVariableName extends ParserBase { - constructor(token: Hash) { - super([token]); - } - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - // if a `stop` character is encountered, finish the parsing process - if (STOP_CHARACTERS.includes(token.text)) { - try { - // if it is possible to convert current parser to `PromptVariable`, return success result - return { - result: 'success', - nextParser: this.asPromptVariable(), - wasTokenConsumed: false, - }; - } catch (error) { - // otherwise fail - return { - result: 'failure', - wasTokenConsumed: false, - }; - } finally { - // in any case this is an end of the parsing process - this.isConsumed = true; - } - } - - // if a `:` character is encountered, we might transition to {@link PartialPromptVariableWithData} - if (token instanceof Colon) { - this.isConsumed = true; - - // if there is only one token before the `:` character, it must be the starting - // `#` symbol, therefore fail because there is no variable name present - if (this.currentTokens.length <= 1) { - return { - result: 'failure', - wasTokenConsumed: false, - }; - } - - // otherwise, if there are more characters after `#` available, - // we have a variable name, so we can transition to {@link PromptVariableWithData} - return { - result: 'success', - nextParser: new PartialPromptVariableWithData([...this.currentTokens, token]), - wasTokenConsumed: true, - }; - } - - // variables cannot have {@link INVALID_NAME_CHARACTERS} in their names - if (INVALID_NAME_CHARACTERS.includes(token.text)) { - this.isConsumed = true; - - return { - result: 'failure', - wasTokenConsumed: false, - }; - } - - // otherwise, a valid name character, so add it to the list of - // the current tokens and continue the parsing process - this.currentTokens.push(token); - - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } - - /** - * Try to convert current parser instance into a fully-parsed {@link PromptVariable} token. - * - * @throws if sequence of tokens received so far do not constitute a valid prompt variable, - * for instance, if there is only `1` starting `#` token is available. - */ - public asPromptVariable(): PromptVariable { - // if there is only one token before the stop character - // must be the starting `#` one), then fail - assert( - this.currentTokens.length > 1, - 'Cannot create a prompt variable out of incomplete token sequence.', - ); - - const firstToken = this.currentTokens[0]; - const lastToken = this.currentTokens[this.currentTokens.length - 1]; - - // render the characters above into strings, excluding the starting `#` character - const variableNameTokens = this.currentTokens.slice(1); - const variableName = BaseToken.render(variableNameTokens); - - return new PromptVariable( - new Range( - firstToken.range.startLineNumber, - firstToken.range.startColumn, - lastToken.range.endLineNumber, - lastToken.range.endColumn, - ), - variableName, - ); - } -} - -/** - * The parser responsible for parsing a `prompt variable name` with `data`. - * E.g., the `/path/to/something.md` part of the `#file:/path/to/something.md` sequence. - */ -export class PartialPromptVariableWithData extends ParserBase { - - constructor(tokens: readonly TSimpleDecoderToken[]) { - const firstToken = tokens[0]; - const lastToken = tokens[tokens.length - 1]; - - // sanity checks of our expectations about the tokens list - assert( - tokens.length > 2, - `Tokens list must contain at least 3 items, got '${tokens.length}'.`, - ); - assert( - firstToken instanceof Hash, - `The first token must be a '#', got '${firstToken} '.`, - ); - assert( - lastToken instanceof Colon, - `The last token must be a ':', got '${lastToken} '.`, - ); - - super([...tokens]); - } - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - // if a `stop` character is encountered, finish the parsing process - if (STOP_CHARACTERS.includes(token.text)) { - // in any case, success of failure below, this is an end of the parsing process - this.isConsumed = true; - - const firstToken = this.currentTokens[0]; - const lastToken = this.currentTokens[this.currentTokens.length - 1]; - - // tokens representing variable name without the `#` character at the start and - // the `:` data separator character at the end - const variableNameTokens = this.currentTokens.slice(1, this.startTokensCount - 1); - // tokens representing variable data without the `:` separator character at the start - const variableDataTokens = this.currentTokens.slice(this.startTokensCount); - // compute the full range of the variable token - const fullRange = new Range( - firstToken.range.startLineNumber, - firstToken.range.startColumn, - lastToken.range.endLineNumber, - lastToken.range.endColumn, - ); - - // render the characters above into strings - const variableName = BaseToken.render(variableNameTokens); - const variableData = BaseToken.render(variableDataTokens); - - return { - result: 'success', - nextParser: new PromptVariableWithData( - fullRange, - variableName, - variableData, - ), - wasTokenConsumed: false, - }; - } - - // otherwise, token is a valid data character - the data can contain almost any character, - // including `:` and `#`, hence add it to the list of the current tokens and continue - this.currentTokens.push(token); - - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } - - /** - * Try to convert current parser instance into a fully-parsed {@link asPromptVariableWithData} token. - */ - public asPromptVariableWithData(): PromptVariableWithData { - // tokens representing variable name without the `#` character at the start and - // the `:` data separator character at the end - const variableNameTokens = this.currentTokens.slice(1, this.startTokensCount - 1); - // tokens representing variable data without the `:` separator character at the start - const variableDataTokens = this.currentTokens.slice(this.startTokensCount); - - // render the characters above into strings - const variableName = BaseToken.render(variableNameTokens); - const variableData = BaseToken.render(variableDataTokens); - - const firstToken = this.currentTokens[0]; - const lastToken = this.currentTokens[this.currentTokens.length - 1]; - - return new PromptVariableWithData( - new Range( - firstToken.range.startLineNumber, - firstToken.range.startColumn, - lastToken.range.endLineNumber, - lastToken.range.endColumn, - ), - variableName, - variableData, - ); - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/fileReference.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/fileReference.ts deleted file mode 100644 index 78c4f7cb6ea..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/fileReference.ts +++ /dev/null @@ -1,50 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - - -import { PromptVariableWithData } from './promptVariable.js'; -import { assert } from '../../../../../../../base/common/assert.js'; -import { IRange, Range } from '../../../../../../../editor/common/core/range.js'; - -/** - * Name of the variable. - */ -const VARIABLE_NAME: string = 'file'; - -/** - * Object represents a file reference token inside a chatbot prompt. - */ -export class FileReference extends PromptVariableWithData { - constructor( - range: Range, - public readonly path: string, - ) { - super(range, VARIABLE_NAME, path); - } - - /** - * Create a {@link FileReference} from a {@link PromptVariableWithData} instance. - * @throws if variable name is not equal to {@link VARIABLE_NAME}. - */ - public static from(variable: PromptVariableWithData): FileReference { - assert( - variable.name === VARIABLE_NAME, - `Variable name must be '${VARIABLE_NAME}', got '${variable.name}'.`, - ); - - return new FileReference( - variable.range, - variable.data, - ); - } - - /** - * Get the range of the `link` part of the token (e.g., - * the `/path/to/file.md` part of `#file:/path/to/file.md`). - */ - public get linkRange(): IRange | undefined { - return super.dataRange; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptAtMention.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptAtMention.ts deleted file mode 100644 index 43bd8005235..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptAtMention.ts +++ /dev/null @@ -1,52 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { PromptToken } from './promptToken.js'; -import { assert } from '../../../../../../../base/common/assert.js'; -import { Range } from '../../../../../../../editor/common/core/range.js'; -import { INVALID_NAME_CHARACTERS, STOP_CHARACTERS } from '../parsers/promptVariableParser.js'; - -/** - * All prompt at-mentions start with `@` character. - */ -const START_CHARACTER: string = '@'; - -/** - * Represents a `@mention` token in a prompt text. - */ -export class PromptAtMention extends PromptToken { - constructor( - range: Range, - /** - * The name of a mention, excluding the `@` character at the start. - */ - public readonly name: string, - ) { - // sanity check of characters used in the provided mention name - for (const character of name) { - assert( - (INVALID_NAME_CHARACTERS.includes(character) === false) && - (STOP_CHARACTERS.includes(character) === false), - `Mention 'name' cannot contain character '${character}', got '${name}'.`, - ); - } - - super(range); - } - - /** - * Get full text of the token. - */ - public get text(): string { - return `${START_CHARACTER}${this.name}`; - } - - /** - * Return a string representation of the token. - */ - public override toString(): string { - return `${this.text}${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptSlashCommand.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptSlashCommand.ts deleted file mode 100644 index 1069d4a7a89..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptSlashCommand.ts +++ /dev/null @@ -1,42 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { PromptToken } from './promptToken.js'; -import { Range } from '../../../../../../../editor/common/core/range.js'; - -/** - * All prompt at-mentions start with `/` character. - */ -const START_CHARACTER: string = '/'; - -/** - * Represents a `/command` token in a prompt text. - */ -export class PromptSlashCommand extends PromptToken { - constructor( - range: Range, - /** - * The name of a command, excluding the `/` character at the start. - */ - public readonly name: string, - ) { - - super(range); - } - - /** - * Get full text of the token. - */ - public get text(): string { - return `${START_CHARACTER}${this.name}`; - } - - /** - * Return a string representation of the token. - */ - public override toString(): string { - return `${this.text}${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptTemplateVariable.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptTemplateVariable.ts deleted file mode 100644 index be3051f2c88..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptTemplateVariable.ts +++ /dev/null @@ -1,44 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { PromptToken } from './promptToken.js'; -import { Range } from '../../../../../../../editor/common/core/range.js'; -import { DollarSign } from '../base/simpleCodec/tokens/dollarSign.js'; -import { LeftCurlyBrace, RightCurlyBrace } from '../base/simpleCodec/tokens/curlyBraces.js'; - -/** - * Represents a `${variable}` token in a prompt text. - */ -export class PromptTemplateVariable extends PromptToken { - constructor( - range: Range, - /** - * The contents of the template variable, excluding - * the surrounding `${}` characters. - */ - public readonly contents: string, - ) { - super(range); - } - - /** - * Get full text of the token. - */ - public get text(): string { - return [ - DollarSign.symbol, - LeftCurlyBrace.symbol, - this.contents, - RightCurlyBrace.symbol, - ].join(''); - } - - /** - * Return a string representation of the token. - */ - public override toString(): string { - return `${this.text}${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptToken.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptToken.ts deleted file mode 100644 index 021b8d5a425..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptToken.ts +++ /dev/null @@ -1,11 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../base/baseToken.js'; - -/** - * Common base token that all chatbot `prompt` tokens should inherit from. - */ -export abstract class PromptToken extends BaseToken { } diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptVariable.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptVariable.ts deleted file mode 100644 index ec5dd2e7a36..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptVariable.ts +++ /dev/null @@ -1,103 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { PromptToken } from './promptToken.js'; -import { IRange, Range } from '../../../../../../../editor/common/core/range.js'; - -/** - * All prompt variables start with `#` character. - */ -const START_CHARACTER: string = '#'; - -/** - * Character that separates name of a prompt variable from its data. - */ -const DATA_SEPARATOR: string = ':'; - -/** - * Represents a `#variable` token in a prompt text. - */ -export class PromptVariable extends PromptToken { - constructor( - range: Range, - /** - * The name of a prompt variable, excluding the `#` character at the start. - */ - public readonly name: string, - ) { - - super(range); - } - - /** - * Get full text of the token. - */ - public get text(): string { - return `${START_CHARACTER}${this.name}`; - } - - /** - * Return a string representation of the token. - */ - public override toString(): string { - return `${this.text}${this.range}`; - } -} - -/** - * Represents a {@link PromptVariable} with additional data token in a prompt text. - * (e.g., `#variable:/path/to/file.md`) - */ -export class PromptVariableWithData extends PromptVariable { - constructor( - fullRange: Range, - /** - * The name of the variable, excluding the starting `#` character. - */ - name: string, - - /** - * The data of the variable, excluding the starting {@link DATA_SEPARATOR} character. - */ - public readonly data: string, - ) { - super(fullRange, name); - } - - /** - * Get full text of the token. - */ - public override get text(): string { - return `${START_CHARACTER}${this.name}${DATA_SEPARATOR}${this.data}`; - } - - /** - * Range of the `data` part of the variable. - */ - public get dataRange(): IRange | undefined { - const { range } = this; - - // calculate the start column number of the `data` part of the variable - const dataStartColumn = range.startColumn + - START_CHARACTER.length + this.name.length + - DATA_SEPARATOR.length; - - // create `range` of the `data` part of the variable - const result = new Range( - range.startLineNumber, - dataStartColumn, - range.endLineNumber, - range.endColumn, - ); - - // if the resulting range is empty, return `undefined` - // because there is no `data` part present in the variable - if (result.isEmpty()) { - return undefined; - } - - return result; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/contentProviders/filePromptContentsProvider.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/contentProviders/filePromptContentsProvider.ts deleted file mode 100644 index 47705213282..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/contentProviders/filePromptContentsProvider.ts +++ /dev/null @@ -1,166 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { PROMPT_LANGUAGE_ID } from '../promptTypes.js'; -import { IPromptContentsProvider } from './types.js'; -import { URI } from '../../../../../../base/common/uri.js'; -import { assert } from '../../../../../../base/common/assert.js'; -import { CancellationError } from '../../../../../../base/common/errors.js'; -import { VSBufferReadableStream } from '../../../../../../base/common/buffer.js'; -import { CancellationToken } from '../../../../../../base/common/cancellation.js'; -import { IModelService } from '../../../../../../editor/common/services/model.js'; -import { ILanguageService } from '../../../../../../editor/common/languages/language.js'; -import { isPromptOrInstructionsFile } from '../config/promptFileLocations.js'; -import { IPromptContentsProviderOptions, PromptContentsProviderBase } from './promptContentsProviderBase.js'; -import { OpenFailed, NotPromptFile, ResolveError, FolderReference } from '../../promptFileReferenceErrors.js'; -import { FileChangesEvent, FileChangeType, IFileService } from '../../../../../../platform/files/common/files.js'; - -/** - * Prompt contents provider for a file on the disk referenced by - * a provided {@link URI}. - */ -export class FilePromptContentProvider extends PromptContentsProviderBase implements IPromptContentsProvider { - public override get sourceName(): string { - return 'file'; - } - - public override get languageId(): string { - if (this.options.languageId) { - return this.options.languageId; - } - - const model = this.modelService.getModel(this.uri); - - if (model !== null) { - return model.getLanguageId(); - } - - const inferredId = this.languageService - .guessLanguageIdByFilepathOrFirstLine(this.uri); - - if (inferredId !== null) { - return inferredId; - } - - // fallback to the default prompt language ID - return PROMPT_LANGUAGE_ID; - } - - constructor( - public readonly uri: URI, - options: IPromptContentsProviderOptions, - @IFileService private readonly fileService: IFileService, - @IModelService private readonly modelService: IModelService, - @ILanguageService private readonly languageService: ILanguageService, - ) { - super(options); - - if (options.updateOnChange) { - // make sure the object is updated on file changes - this._register( - this.fileService.onDidFilesChange((event) => { - // if file was added or updated, forward the event to - // the `getContentsStream()` produce a new stream for file contents - if (event.contains(this.uri, FileChangeType.ADDED, FileChangeType.UPDATED)) { - // we support only full file parsing right now because - // the event doesn't contain a list of changed lines - this.onChangeEmitter.fire('full'); - return; - } - - // if file was deleted, forward the event to - // the `getContentsStream()` produce an error - if (event.contains(this.uri, FileChangeType.DELETED)) { - this.onChangeEmitter.fire(event); - return; - } - }), - ); - } - } - - /** - * Creates a stream of lines from the file based on the changes listed in - * the provided event. - * - * @param event - event that describes the changes in the file; `'full'` is - * the special value that means that all contents have changed - * @param cancellationToken - token that cancels this operation - */ - protected async getContentsStream( - _event: FileChangesEvent | 'full', - cancellationToken?: CancellationToken, - ): Promise { - assert( - !cancellationToken?.isCancellationRequested, - new CancellationError(), - ); - - // get the binary stream of the file contents - let fileStream; - try { - // ensure that the referenced URI points to a file before - // trying to get a stream for its contents - const info = await this.fileService.resolve(this.uri); - - // validate that the cancellation was not yet requested - assert( - !cancellationToken?.isCancellationRequested, - new CancellationError(), - ); - - assert( - info.isFile, - new FolderReference(this.uri), - ); - - const { allowNonPromptFiles } = this.options; - - // if URI doesn't point to a prompt file, don't try to resolve it, - // unless the `allowNonPromptFiles` option is set to `true` - if ((allowNonPromptFiles !== true) && (isPromptOrInstructionsFile(this.uri) === false)) { - throw new NotPromptFile(this.uri); - } - - fileStream = await this.fileService.readFileStream(this.uri); - - // after the promise above complete, this object can be already disposed or - // the cancellation could be requested, in that case destroy the stream and - // throw cancellation error - if (this.isDisposed || cancellationToken?.isCancellationRequested) { - fileStream.value.destroy(); - throw new CancellationError(); - } - - return fileStream.value; - } catch (error) { - if ((error instanceof ResolveError) || (error instanceof CancellationError)) { - throw error; - } - - throw new OpenFailed(this.uri, error); - } - } - - public override createNew( - promptContentsSource: { uri: URI }, - options: IPromptContentsProviderOptions, - ): IPromptContentsProvider { - return new FilePromptContentProvider( - promptContentsSource.uri, - options, - this.fileService, - this.modelService, - this.languageService, - ); - } - - /** - * String representation of this object. - */ - public override toString(): string { - return `file-prompt-contents-provider:${this.uri.path}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/contentProviders/promptContentsProviderBase.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/contentProviders/promptContentsProviderBase.ts deleted file mode 100644 index 7c9e58dc3d3..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/contentProviders/promptContentsProviderBase.ts +++ /dev/null @@ -1,196 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { assert } from '../../../../../../base/common/assert.js'; -import { VSBufferReadableStream } from '../../../../../../base/common/buffer.js'; -import { CancellationToken } from '../../../../../../base/common/cancellation.js'; -import { cancelPreviousCalls } from '../../../../../../base/common/decorators/cancelPreviousCalls.js'; -import { CancellationError } from '../../../../../../base/common/errors.js'; -import { Emitter } from '../../../../../../base/common/event.js'; -import { URI } from '../../../../../../base/common/uri.js'; -import { FailedToResolveContentsStream, ResolveError } from '../../promptFileReferenceErrors.js'; -import { INSTRUCTIONS_LANGUAGE_ID, MODE_LANGUAGE_ID, PROMPT_LANGUAGE_ID, PromptsType } from '../promptTypes.js'; -import { ObservableDisposable } from '../utils/observableDisposable.js'; -import { IPromptContentsProvider } from './types.js'; - -/** - * Options of the {@link PromptContentsProviderBase} class. - */ -export interface IPromptContentsProviderOptions { - /** - * Whether to allow files that don't have usual prompt - * file extension to be treated as a prompt file. - */ - readonly allowNonPromptFiles: boolean; - - /** - * Language ID to use for the prompt contents. If not set, the language ID will be inferred from the file. - */ - readonly languageId: string | undefined; - - /** - * If set to `true`, the contents provider will listen for updates and retrigger a parse. - */ - readonly updateOnChange: boolean; -} - - -/** - * Base class for prompt contents providers. Classes that extend this one are responsible to: - * - * - implement the {@link getContentsStream} method to provide the contents stream - * of a prompt; this method should throw a `ResolveError` or its derivative if the contents - * cannot be parsed for any reason - * - fire a {@link TChangeEvent} event on the {@link onChangeEmitter} event when - * prompt contents change - * - misc: - * - provide the {@link uri} property that represents the URI of a prompt that - * the contents are for - * - implement the {@link toString} method to return a string representation of this - * provider type to aid with debugging/tracing - */ -export abstract class PromptContentsProviderBase< - TChangeEvent extends NonNullable, -> extends ObservableDisposable implements IPromptContentsProvider { - public abstract readonly uri: URI; - public abstract createNew(promptContentsSource: { uri: URI }, options: IPromptContentsProviderOptions): IPromptContentsProvider; - public abstract override toString(): string; - public abstract get languageId(): string; - public abstract get sourceName(): string; - - /** - * Prompt contents stream. - */ - public get contents(): Promise { - return this.getContentsStream('full'); - } - - /** - * Prompt type used to determine how to interpret file contents. - */ - public get promptType(): PromptsType | 'non-prompt' { - const { languageId } = this; - - if (languageId === PROMPT_LANGUAGE_ID) { - return PromptsType.prompt; - } - - if (languageId === INSTRUCTIONS_LANGUAGE_ID) { - return PromptsType.instructions; - } - - if (languageId === MODE_LANGUAGE_ID) { - return PromptsType.mode; - } - - return 'non-prompt'; - } - - /** - * Function to get contents stream for the provider. This function should - * throw a `ResolveError` or its derivative if the contents cannot be parsed. - * - * @param changesEvent The event that triggered the change. The special - * `'full'` value means that everything has changed hence entire prompt - * contents need to be re-parsed from scratch. - */ - protected abstract getContentsStream( - changesEvent: TChangeEvent | 'full', - cancellationToken?: CancellationToken, - ): Promise; - - /** - * Internal event emitter for the prompt contents change event. Classes that extend - * this abstract class are responsible to use this emitter to fire the contents change - * event when the prompt contents get modified. - */ - protected readonly onChangeEmitter = this._register(new Emitter()); - - /** - * Options passed to the constructor - */ - protected readonly options: IPromptContentsProviderOptions; - - constructor( - options: IPromptContentsProviderOptions, - ) { - super(); - - this.options = options; - } - - /** - * Event emitter for the prompt contents change event. - * See {@link onContentChanged} for more details. - */ - private readonly onContentChangedEmitter = this._register(new Emitter()); - - /** - * Event that fires when the prompt contents change. The event is either - * a `VSBufferReadableStream` stream with changed contents or an instance of - * the `ResolveError` class representing a parsing failure case. - * - * `Note!` this field is meant to be used by the external consumers of the prompt - * contents provider that the classes that extend this abstract class. - * Please use the {@link onChangeEmitter} event to provide a change - * event in your prompt contents implementation instead. - */ - public readonly onContentChanged = this.onContentChangedEmitter.event; - - /** - * Internal common implementation of the event that should be fired when - * prompt contents change. - */ - @cancelPreviousCalls - private onContentsChanged( - event: TChangeEvent | 'full', - cancellationToken?: CancellationToken, - ): this { - const promise = (cancellationToken?.isCancellationRequested) - ? Promise.reject(new CancellationError()) - : this.getContentsStream(event, cancellationToken); - - promise - .then((stream) => { - if (cancellationToken?.isCancellationRequested || this.isDisposed) { - stream.destroy(); - throw new CancellationError(); - } - - this.onContentChangedEmitter.fire(stream); - }) - .catch((error) => { - if (error instanceof ResolveError) { - this.onContentChangedEmitter.fire(error); - - return; - } - - this.onContentChangedEmitter.fire( - new FailedToResolveContentsStream(this.uri, error), - ); - }); - - return this; - } - - /** - * Start producing the prompt contents data. - */ - public start(token?: CancellationToken): this { - assert( - !this.isDisposed, - 'Cannot start contents provider that was already disposed.', - ); - - // `'full'` means "everything has changed" - this.onContentsChanged('full', token); - - // subscribe to the change event emitted by a child class - this._register(this.onChangeEmitter.event(this.onContentsChanged, this)); - - return this; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/contentProviders/textModelContentsProvider.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/contentProviders/textModelContentsProvider.ts deleted file mode 100644 index aea65908ccf..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/contentProviders/textModelContentsProvider.ts +++ /dev/null @@ -1,93 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { VSBufferReadableStream } from '../../../../../../base/common/buffer.js'; -import { CancellationToken } from '../../../../../../base/common/cancellation.js'; -import { URI } from '../../../../../../base/common/uri.js'; -import { ITextModel } from '../../../../../../editor/common/model.js'; -import { TextModel } from '../../../../../../editor/common/model/textModel.js'; -import { IModelContentChangedEvent } from '../../../../../../editor/common/textModelEvents.js'; -import { IInstantiationService } from '../../../../../../platform/instantiation/common/instantiation.js'; -import { objectStreamFromTextModel } from '../codecs/base/utils/objectStreamFromTextModel.js'; -import { FilePromptContentProvider } from './filePromptContentsProvider.js'; -import { IPromptContentsProviderOptions, PromptContentsProviderBase } from './promptContentsProviderBase.js'; -import { IPromptContentsProvider } from './types.js'; - -/** - * Prompt contents provider for a {@link ITextModel} instance. - */ -export class TextModelContentsProvider extends PromptContentsProviderBase { - /** - * URI component of the prompt associated with this contents provider. - */ - public get uri(): URI { - return this.model.uri; - } - - public override get sourceName(): string { - return 'text-model'; - } - - public override get languageId(): string { - return this.options.languageId ?? this.model.getLanguageId(); - } - - constructor( - private readonly model: ITextModel, - options: IPromptContentsProviderOptions, - @IInstantiationService private readonly instantiationService: IInstantiationService, - ) { - super(options); - - this._register(this.model.onWillDispose(this.dispose.bind(this))); - if (options.updateOnChange) { - this._register(this.model.onDidChangeContent(this.onChangeEmitter.fire.bind(this.onChangeEmitter))); - } - } - - /** - * Creates a stream of binary data from the text model based on the changes - * listed in the provided event. - * - * Note! this method implements a basic logic which does not take into account - * the `_event` argument for incremental updates. This needs to be improved. - * - * @param _event - event that describes the changes in the text model; `'full'` is - * the special value that means that all contents have changed - * @param cancellationToken - token that cancels this operation - */ - protected override async getContentsStream( - _event: IModelContentChangedEvent | 'full', - cancellationToken?: CancellationToken, - ): Promise { - return objectStreamFromTextModel(this.model, cancellationToken); - } - - public override createNew( - promptContentsSource: TextModel | { uri: URI }, - options: IPromptContentsProviderOptions, - ): IPromptContentsProvider { - if (promptContentsSource instanceof TextModel) { - return this.instantiationService.createInstance( - TextModelContentsProvider, - promptContentsSource, - options, - ); - } - - return this.instantiationService.createInstance( - FilePromptContentProvider, - promptContentsSource.uri, - options, - ); - } - - /** - * String representation of this object. - */ - public override toString(): string { - return `text-model-prompt-contents-provider:${this.uri.path}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/contentProviders/types.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/contentProviders/types.ts deleted file mode 100644 index a26eaa43330..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/contentProviders/types.ts +++ /dev/null @@ -1,70 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { URI } from '../../../../../../base/common/uri.js'; -import { Event } from '../../../../../../base/common/event.js'; -import { ResolveError } from '../../promptFileReferenceErrors.js'; -import { IDisposable } from '../../../../../../base/common/lifecycle.js'; -import { VSBufferReadableStream } from '../../../../../../base/common/buffer.js'; -import { PromptsType } from '../promptTypes.js'; -import { CancellationToken } from '../../../../../../base/common/cancellation.js'; -import { IPromptContentsProviderOptions } from './promptContentsProviderBase.js'; - -/** - * Interface for a prompt contents provider. Prompt contents providers are - * responsible for providing contents of a prompt as a byte streams and - * allow to subscribe to the change events of the prompt contents. - */ -export interface IPromptContentsProvider extends IDisposable { - /** - * URI component of the prompt associated with this contents provider. - */ - readonly uri: URI; - - /** - * Language ID of the prompt contents. - */ - readonly languageId: string; - - /** - * Prompt type used to determine how to interpret file contents. - */ - readonly promptType: PromptsType | 'non-prompt'; - - /** - * Prompt contents stream. - */ - readonly contents: Promise; - - /** - * Prompt contents source name. - */ - readonly sourceName: string; - - /** - * Event that fires when the prompt contents change. The event is either a - * {@linkcode VSBufferReadableStream} stream with changed contents or - * an instance of the {@linkcode ResolveError} error. - */ - readonly onContentChanged: Event; - - /** - * Subscribe to `onDispose` event of the contents provider. - */ - readonly onDispose: Event; - - /** - * Start the contents provider to produce the underlying contents. - */ - start(token?: CancellationToken): this; - - /** - * Create a new instance of prompt contents provider. - */ - createNew( - promptContentsSource: { uri: URI }, - options: IPromptContentsProviderOptions, - ): IPromptContentsProvider; -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/frontMatterDecoration.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/frontMatterDecoration.ts deleted file mode 100644 index d07720eff5b..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/frontMatterDecoration.ts +++ /dev/null @@ -1,120 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { Position } from '../../../../../../../../editor/common/core/position.js'; -import { localize } from '../../../../../../../../nls.js'; -import { contrastBorder, editorBackground } from '../../../../../../../../platform/theme/common/colorRegistry.js'; -import { asCssVariable, ColorIdentifier, darken, registerColor } from '../../../../../../../../platform/theme/common/colorUtils.js'; -import { BaseToken } from '../../../codecs/base/baseToken.js'; -import { FrontMatterHeader } from '../../../codecs/base/markdownExtensionsCodec/tokens/frontMatterHeader.js'; -import { CssClassModifiers } from '../types.js'; -import { FrontMatterMarkerDecoration } from './frontMatterMarkerDecoration.js'; -import { ReactiveDecorationBase } from './utils/reactiveDecorationBase.js'; -import { IReactiveDecorationClassNames, TAddAccessor, TDecorationStyles } from './utils/types.js'; - -/** - * Decoration CSS class names. - */ -export enum CssClassNames { - Main = '.prompt-front-matter-decoration', - Inline = '.prompt-front-matter-decoration-inline', - MainInactive = `${CssClassNames.Main}${CssClassModifiers.Inactive}`, - InlineInactive = `${CssClassNames.Inline}${CssClassModifiers.Inactive}`, -} - -/** - * Main background color of `active` Front Matter header block. - */ -export const BACKGROUND_COLOR: ColorIdentifier = registerColor( - 'prompt.frontMatter.background', - { dark: darken(editorBackground, 0.2), light: darken(editorBackground, 0.05), hcDark: contrastBorder, hcLight: contrastBorder }, - localize('chat.prompt.frontMatter.background.description', "Background color of a Front Matter header block."), -); - -/** - * Background color of `inactive` Front Matter header block. - */ -export const INACTIVE_BACKGROUND_COLOR: ColorIdentifier = registerColor( - 'prompt.frontMatter.inactiveBackground', - { dark: darken(editorBackground, 0.1), light: darken(editorBackground, 0.025), hcDark: contrastBorder, hcLight: contrastBorder }, - localize('chat.prompt.frontMatter.inactiveBackground.description', "Background color of an inactive Front Matter header block."), -); - -/** - * CSS styles for the decoration. - */ -export const CSS_STYLES = { - [CssClassNames.Main]: [ - `background-color: ${asCssVariable(BACKGROUND_COLOR)};`, - 'z-index: -1;', // this is required to allow for selections to appear above the decoration background - ], - [CssClassNames.MainInactive]: [ - `background-color: ${asCssVariable(INACTIVE_BACKGROUND_COLOR)};`, - ], - [CssClassNames.InlineInactive]: [ - 'color: var(--vscode-disabledForeground);', - ], - ...FrontMatterMarkerDecoration.cssStyles, -}; - -/** - * Editor decoration for the Front Matter header token inside a prompt. - */ -export class FrontMatterDecoration extends ReactiveDecorationBase { - constructor( - accessor: TAddAccessor, - token: FrontMatterHeader, - ) { - super(accessor, token); - - this.childDecorators.push( - new FrontMatterMarkerDecoration(accessor, token.startMarker), - new FrontMatterMarkerDecoration(accessor, token.endMarker), - ); - } - - public override setCursorPosition( - position: Position | null | undefined, - ): this is { readonly changed: true } { - const result = super.setCursorPosition(position); - - for (const marker of this.childDecorators) { - if ((marker instanceof FrontMatterMarkerDecoration) === false) { - continue; - } - - // activate/deactivate markers based on the active state - // of the main Front Matter header decoration - marker.activate(this.active); - } - - return result; - } - - protected override get classNames(): IReactiveDecorationClassNames { - return CssClassNames; - } - - protected override get isWholeLine(): boolean { - return true; - } - - protected override get description(): string { - return 'Front Matter header decoration.'; - } - - public static get cssStyles(): TDecorationStyles { - return CSS_STYLES; - } - - /** - * Whether current decoration class can decorate provided token. - */ - public static handles( - token: BaseToken, - ): token is FrontMatterHeader { - return token instanceof FrontMatterHeader; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/frontMatterMarkerDecoration.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/frontMatterMarkerDecoration.ts deleted file mode 100644 index 2a4a174c9f4..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/frontMatterMarkerDecoration.ts +++ /dev/null @@ -1,56 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { CssClassModifiers } from '../types.js'; -import { TDecorationStyles, IReactiveDecorationClassNames } from './utils/types.js'; -import { FrontMatterMarker } from '../../../codecs/base/markdownExtensionsCodec/tokens/frontMatterMarker.js'; -import { ReactiveDecorationBase } from './utils/reactiveDecorationBase.js'; - -/** - * Decoration CSS class names. - */ -export enum CssClassNames { - Main = '.prompt-front-matter-decoration-marker', - Inline = '.prompt-front-matter-decoration-marker-inline', - MainInactive = `${CssClassNames.Main}${CssClassModifiers.Inactive}`, - InlineInactive = `${CssClassNames.Inline}${CssClassModifiers.Inactive}`, -} - -/** - * Editor decoration for a `marker` token of a Front Matter header. - */ -export class FrontMatterMarkerDecoration extends ReactiveDecorationBase { - /** - * Activate/deactivate the decoration. - */ - public activate(state: boolean): this { - const position = (state === true) - ? this.token.range.getStartPosition() - : null; - - this.setCursorPosition(position); - - return this; - } - - protected override get classNames(): IReactiveDecorationClassNames { - return CssClassNames; - } - - protected override get description(): string { - return 'Marker decoration of a Front Matter header.'; - } - - public static get cssStyles(): TDecorationStyles { - return { - [CssClassNames.Inline]: [ - 'color: var(--vscode-disabledForeground);', - ], - [CssClassNames.InlineInactive]: [ - 'opacity: 0.25;', - ], - }; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/utils/decorationBase.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/utils/decorationBase.ts deleted file mode 100644 index 8f28ec33247..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/utils/decorationBase.ts +++ /dev/null @@ -1,127 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { Range } from '../../../../../../../../../editor/common/core/range.js'; -import { IMarkdownString } from '../../../../../../../../../base/common/htmlContent.js'; -import { BaseToken } from '../../../../codecs/base/baseToken.js'; -import { TrackedRangeStickiness } from '../../../../../../../../../editor/common/model.js'; -import type { TAddAccessor, TChangeAccessor, TDecorationStyles, TRemoveAccessor } from './types.js'; -import { ModelDecorationOptions } from '../../../../../../../../../editor/common/model/textModel.js'; - -/** - * Base class for all editor decorations. - */ -export abstract class DecorationBase< - TPromptToken extends BaseToken, - TCssClassName extends string = string, -> { - /** - * Description of the decoration. - */ - protected abstract get description(): string; - - /** - * Default CSS class name of the decoration. - */ - protected abstract get className(): TCssClassName; - - /** - * Inline CSS class name of the decoration. - */ - protected abstract get inlineClassName(): TCssClassName; - - /** - * Indicates whether the decoration spans the whole line(s). - */ - protected get isWholeLine(): boolean { - return false; - } - - /** - * Hover message of the decoration. - */ - protected get hoverMessage(): IMarkdownString | IMarkdownString[] | null { - return null; - } - - /** - * ID of editor decoration it was registered with. - */ - public readonly id: string; - - constructor( - accessor: TAddAccessor, - protected readonly token: TPromptToken, - ) { - this.id = accessor.addDecoration(this.range, this.decorationOptions); - } - - /** - * Range of the decoration. - */ - public get range(): Range { - return this.token.range; - } - - /** - * Changes the decoration in the editor. - */ - public change( - accessor: TChangeAccessor, - ): this { - accessor.changeDecorationOptions( - this.id, - this.decorationOptions, - ); - - return this; - } - - /** - * Removes associated editor decoration(s). - */ - public remove( - accessor: TRemoveAccessor, - ): this { - accessor.removeDecoration(this.id); - - return this; - } - - /** - * Get editor decoration options for this decorator. - */ - private get decorationOptions(): ModelDecorationOptions { - return ModelDecorationOptions.createDynamic({ - description: this.description, - hoverMessage: this.hoverMessage, - className: this.className, - inlineClassName: this.inlineClassName, - isWholeLine: this.isWholeLine, - stickiness: TrackedRangeStickiness.NeverGrowsWhenTypingAtEdges, - shouldFillLineOnLineBreak: true, - }); - } -} - -/** - * Type of a generic decoration class. - */ -export type TDecorationClass = { - new( - accessor: TAddAccessor, - token: TPromptToken, - ): DecorationBase; - - /** - * CSS styles for the decoration. - */ - readonly cssStyles: TDecorationStyles; - - /** - * Whether the decoration class handles the provided token. - */ - handles(token: BaseToken): token is TPromptToken; -}; diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/utils/reactiveDecorationBase.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/utils/reactiveDecorationBase.ts deleted file mode 100644 index 42533aea5fe..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/utils/reactiveDecorationBase.ts +++ /dev/null @@ -1,162 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { DecorationBase } from './decorationBase.js'; -import { Position } from '../../../../../../../../../editor/common/core/position.js'; -import { BaseToken } from '../../../../codecs/base/baseToken.js'; -import type { IReactiveDecorationClassNames, TAddAccessor, TChangeAccessor, TRemoveAccessor } from './types.js'; - -/** - * Base class for all reactive editor decorations. A reactive decoration - * is a decoration that can change its appearance based on current cursor - * position in the editor, hence can "react" to the user's actions. - */ -export abstract class ReactiveDecorationBase< - TPromptToken extends BaseToken, - TCssClassName extends string = string, -> extends DecorationBase { - /** - * CSS class names of the decoration. - */ - protected abstract get classNames(): IReactiveDecorationClassNames; - - /** - * A list of child decorators that are part of this decoration. - * For instance a Front Matter header decoration can have child - * decorators for each of the header's `---` markers. - */ - protected readonly childDecorators: DecorationBase[]; - - /** - * Whether the decoration has changed since the last {@link change}. - */ - public get changed(): boolean { - // if any of the child decorators changed, this object is also - // considered to be changed - for (const marker of this.childDecorators) { - if ((marker instanceof ReactiveDecorationBase) === false) { - continue; - } - - if (marker.changed === true) { - return true; - } - } - - return this.didChange; - } - - constructor( - accessor: TAddAccessor, - token: TPromptToken, - ) { - super(accessor, token); - - this.childDecorators = []; - } - - /** - * Current position of cursor in the editor. - */ - private cursorPosition?: Position | null; - - /** - * Private field for the {@link changed} property. - */ - private didChange = true; - - /** - * Whether cursor is currently inside the decoration range. - */ - protected get active(): boolean { - return true; - - /** - * Temporarily disable until we have a proper way to get - * the cursor position inside active editor. - */ - /** - * if (!this.cursorPosition) { - * return false; - * } - * - * // when cursor is at the end of a range, the range considered to - * // not contain the position, but we want to include it - * const atEnd = (this.range.endLineNumber === this.cursorPosition.lineNumber) - * && (this.range.endColumn === this.cursorPosition.column); - * - * return atEnd || this.range.containsPosition(this.cursorPosition); - */ - } - - /** - * Set cursor position and update {@link changed} property if needed. - */ - public setCursorPosition( - position: Position | null | undefined, - ): this is { readonly changed: true } { - if (this.cursorPosition === position) { - return false; - } - - if (this.cursorPosition && position) { - if (this.cursorPosition.equals(position)) { - return false; - } - } - - const wasActive = this.active; - this.cursorPosition = position; - this.didChange = (wasActive !== this.active); - - return this.changed; - } - - public override change( - accessor: TChangeAccessor, - ): this { - if (this.didChange === false) { - return this; - } - - super.change(accessor); - this.didChange = false; - - for (const marker of this.childDecorators) { - marker.change(accessor); - } - - return this; - } - - public override remove( - accessor: TRemoveAccessor, - ): this { - super.remove(accessor); - - for (const marker of this.childDecorators) { - marker.remove(accessor); - } - - return this; - } - - protected override get className(): TCssClassName { - return (this.active) - ? this.classNames.Main - : this.classNames.MainInactive; - } - - protected override get inlineClassName(): TCssClassName { - return (this.active) - ? this.classNames.Inline - : this.classNames.InlineInactive; - } -} - -/** - * Type for a decorator with {@link ReactiveDecorationBase.changed changed} property set to `true`. - */ -export type TChangedDecorator = ReactiveDecorationBase & { readonly changed: true }; diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/utils/types.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/utils/types.ts deleted file mode 100644 index 1ca7e270d3f..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/utils/types.ts +++ /dev/null @@ -1,56 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { IModelDecorationsChangeAccessor } from '../../../../../../../../../editor/common/model.js'; - -/** - * CSS class names of a `reactive` decoration. - */ -export interface IReactiveDecorationClassNames { - /** - * Main, default CSS class name of the decoration. - */ - readonly Main: T; - - /** - * CSS class name of the decoration for the `inline`(text) styles. - */ - readonly Inline: T; - - /** - * main CSS class name of the decoration for the `inactive` - * decoration state. - */ - readonly MainInactive: T; - - /** - * CSS class name of the decoration for the `inline`(text) - * styles when decoration is in the `inactive` state. - */ - readonly InlineInactive: T; -} - -/** - * CSS styles for a decoration to be registered with editor. - */ -export type TDecorationStyles = { - readonly [key in TClassNames]: readonly string[]; -}; - -/** - * A model decorations accessor that can be used to `add` a decoration. - */ -export type TAddAccessor = Pick; - -/** - * A model decorations accessor that can be used to `change` a decoration. - */ -export type TChangeAccessor = Pick; - -/** - * A model decorations accessor that can be used to `remove` a decoration. - */ -export type TRemoveAccessor = Pick; - diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/promptDecorationsProvider.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/promptDecorationsProvider.ts deleted file mode 100644 index d7a9d9b235f..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/promptDecorationsProvider.ts +++ /dev/null @@ -1,205 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { IPromptsService } from '../../service/promptsService.js'; -import { ProviderInstanceBase } from '../providerInstanceBase.js'; -import { ITextModel } from '../../../../../../../editor/common/model.js'; -import { FrontMatterDecoration } from './decorations/frontMatterDecoration.js'; -import { toDisposable } from '../../../../../../../base/common/lifecycle.js'; -import { Position } from '../../../../../../../editor/common/core/position.js'; -import { BaseToken } from '../../codecs/base/baseToken.js'; -import { ProviderInstanceManagerBase, TProviderClass } from '../providerInstanceManagerBase.js'; -import { registerThemingParticipant } from '../../../../../../../platform/theme/common/themeService.js'; -import { FrontMatterHeader } from '../../codecs/base/markdownExtensionsCodec/tokens/frontMatterHeader.js'; -import { ReactiveDecorationBase, TChangedDecorator } from './decorations/utils/reactiveDecorationBase.js'; -import { DecorationBase, TDecorationClass } from './decorations/utils/decorationBase.js'; - -/** - * Prompt tokens that are decorated by this provider. - */ -type TDecoratedToken = FrontMatterHeader; - -/** - * List of all supported decorations. - */ -const SUPPORTED_DECORATIONS: readonly TDecorationClass[] = Object.freeze([ - FrontMatterDecoration, -]); - -/** - * Prompt syntax decorations provider for text models. - */ -export class PromptDecorator extends ProviderInstanceBase { - /** - * Currently active decorations. - */ - private readonly decorations: DecorationBase[] = []; - - constructor( - model: ITextModel, - @IPromptsService promptsService: IPromptsService, - ) { - super(model, promptsService); - - this.watchCursorPosition(); - } - - protected override async onPromptSettled( - _error?: Error, - ): Promise { - // by the time the promise above completes, either this object - // or the text model might be already has been disposed - if (this.isDisposed || this.model.isDisposed()) { - return; - } - - this.addDecorations(); - - return; - } - - /** - * Get the current cursor position inside an active editor. - * Note! Currently not implemented because the provider is disabled, and - * we need to do some refactoring to get accurate cursor position. - */ - private get cursorPosition(): Position | null { - if (this.model.isDisposed()) { - return null; - } - - return null; - } - - /** - * Watch editor cursor position and update reactive decorations accordingly. - */ - private watchCursorPosition(): this { - const interval = setInterval(() => { - const { cursorPosition } = this; - - const changedDecorations: TChangedDecorator[] = []; - for (const decoration of this.decorations) { - if ((decoration instanceof ReactiveDecorationBase) === false) { - continue; - } - - if (decoration.setCursorPosition(cursorPosition) === true) { - changedDecorations.push(decoration); - } - } - - if (changedDecorations.length === 0) { - return; - } - - this.changeModelDecorations(changedDecorations); - }, 25); - - this._register(toDisposable(() => { - clearInterval(interval); - })); - - return this; - } - - /** - * Update existing decorations. - */ - private changeModelDecorations( - decorations: readonly TChangedDecorator[], - ): this { - this.model.changeDecorations((accessor) => { - for (const decoration of decorations) { - decoration.change(accessor); - } - }); - - return this; - } - - /** - * Add decorations for all prompt tokens. - */ - private addDecorations(): this { - this.model.changeDecorations((accessor) => { - const { tokens } = this.parser; - - // remove all existing decorations - for (const decoration of this.decorations.splice(0)) { - decoration.remove(accessor); - } - - // then add new decorations based on the current tokens - for (const token of tokens) { - for (const Decoration of SUPPORTED_DECORATIONS) { - if (Decoration.handles(token) === false) { - continue; - } - - this.decorations.push( - new Decoration(accessor, token), - ); - break; - } - } - }); - - return this; - } - - /** - * Remove all existing decorations. - */ - private removeAllDecorations(): this { - if (this.decorations.length === 0) { - return this; - } - - this.model.changeDecorations((accessor) => { - for (const decoration of this.decorations.splice(0)) { - decoration.remove(accessor); - } - }); - - return this; - } - - public override dispose(): void { - if (this.isDisposed) { - return; - } - - this.removeAllDecorations(); - super.dispose(); - } - - /** - * Returns a string representation of this object. - */ - public override toString(): string { - return `text-model-prompt-decorator:${this.model.uri.path}`; - } -} - -/** - * Register CSS styles of the supported decorations. - */ -registerThemingParticipant((_theme, collector) => { - for (const Decoration of SUPPORTED_DECORATIONS) { - for (const [className, styles] of Object.entries(Decoration.cssStyles)) { - collector.addRule(`.monaco-editor ${className} { ${styles.join(' ')} }`); - } - } -}); - -/** - * Provider for prompt syntax decorators on text models. - */ -export class PromptDecorationsProviderInstanceManager extends ProviderInstanceManagerBase { - protected override get InstanceClass(): TProviderClass { - return PromptDecorator; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/types.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/types.ts deleted file mode 100644 index fb3ef81b122..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/types.ts +++ /dev/null @@ -1,48 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { IRange } from '../../../../../../../editor/common/core/range.js'; -import { ModelDecorationOptions } from '../../../../../../../editor/common/model/textModel.js'; - -/** - * Decoration object. - */ -export interface ITextModelDecoration { - /** - * Range of the decoration. - */ - range: IRange; - - /** - * Associated decoration options. - */ - options: ModelDecorationOptions; -} - -/** - * Decoration CSS class names. - */ -export enum DecorationClassNames { - /** - * CSS class name for `default` prompt syntax decoration. - */ - Default = 'prompt-decoration', - - /** - * CSS class name for `file reference` prompt syntax decoration. - */ - FileReference = DecorationClassNames.Default, -} - -/** - * Decoration CSS class modifiers. - */ -export enum CssClassModifiers { - /** - * CSS class modifier for `active` state of - * a `reactive` prompt syntax decoration. - */ - Inactive = '.prompt-decoration-inactive', -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/promptHeaderDiagnosticsProvider.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/promptHeaderDiagnosticsProvider.ts deleted file mode 100644 index b8bca008829..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/promptHeaderDiagnosticsProvider.ts +++ /dev/null @@ -1,234 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { IPromptsService } from '../service/promptsService.js'; -import { ProviderInstanceBase } from './providerInstanceBase.js'; -import { ITextModel } from '../../../../../../editor/common/model.js'; -import { assertNever } from '../../../../../../base/common/assert.js'; -import { CancellationToken } from '../../../../../../base/common/cancellation.js'; -import { ProviderInstanceManagerBase, TProviderClass } from './providerInstanceManagerBase.js'; -import { TDiagnostic, PromptMetadataError, PromptMetadataWarning } from '../parsers/promptHeader/diagnostics.js'; -import { IMarkerData, IMarkerService, MarkerSeverity } from '../../../../../../platform/markers/common/markers.js'; -import { PromptHeader } from '../parsers/promptHeader/promptHeader.js'; -import { PromptToolsMetadata } from '../parsers/promptHeader/metadata/tools.js'; -import { PromptModelMetadata } from '../parsers/promptHeader/metadata/model.js'; -import { ModeHeader } from '../parsers/promptHeader/modeHeader.js'; -import { ILanguageModelChatMetadata, ILanguageModelsService } from '../../languageModels.js'; -import { ILanguageModelToolsService } from '../../languageModelToolsService.js'; -import { localize } from '../../../../../../nls.js'; -import { ChatModeKind } from '../../constants.js'; -import { IChatMode, IChatModeService } from '../../chatModes.js'; -import { PromptModeMetadata } from '../parsers/promptHeader/metadata/mode.js'; -import { Iterable } from '../../../../../../base/common/iterator.js'; - -/** - * Unique ID of the markers provider class. - */ -const MARKERS_OWNER_ID = 'prompts-header-diagnostics-provider'; - -/** - * Prompt header diagnostics provider for an individual text model - * of a prompt file. - */ -class PromptHeaderDiagnosticsProvider extends ProviderInstanceBase { - constructor( - model: ITextModel, - @IPromptsService promptsService: IPromptsService, - @IMarkerService private readonly markerService: IMarkerService, - @ILanguageModelsService private readonly languageModelsService: ILanguageModelsService, - @ILanguageModelToolsService private readonly languageModelToolsService: ILanguageModelToolsService, - @IChatModeService private readonly chatModeService: IChatModeService, - ) { - super(model, promptsService); - this._register(languageModelsService.onDidChangeLanguageModels(() => { - this.onPromptSettled(undefined, CancellationToken.None); - })); - this._register(languageModelToolsService.onDidChangeTools(() => { - this.onPromptSettled(undefined, CancellationToken.None); - })); - this._register(chatModeService.onDidChangeChatModes(() => { - this.onPromptSettled(undefined, CancellationToken.None); - })); - } - - /** - * Update diagnostic markers for the current editor. - */ - protected override async onPromptSettled( - _error: Error | undefined, - token: CancellationToken, - ): Promise { - - const { header } = this.parser; - if (header === undefined) { - this.markerService.remove(MARKERS_OWNER_ID, [this.model.uri]); - return; - } - - // header parsing process is separate from the prompt parsing one, hence - // apply markers only after the header is settled and so has diagnostics - const completed = await header.settled; - if (!completed || token.isCancellationRequested) { - return; - } - - const markers: IMarkerData[] = []; - for (const diagnostic of header.diagnostics) { - markers.push(toMarker(diagnostic)); - } - - if (header instanceof PromptHeader) { - const mode = this.validateMode(header.metadataUtility.mode, markers); - this.validateTools(header.metadataUtility.tools, mode?.kind, markers); - this.validateModel(header.metadataUtility.model, mode?.kind, markers); - } else if (header instanceof ModeHeader) { - this.validateTools(header.metadataUtility.tools, ChatModeKind.Agent, markers); - this.validateModel(header.metadataUtility.model, ChatModeKind.Agent, markers); - - } - - if (markers.length === 0) { - this.markerService.remove(MARKERS_OWNER_ID, [this.model.uri]); - return; - } - - this.markerService.changeOne( - MARKERS_OWNER_ID, - this.model.uri, - markers, - ); - return; - } - validateModel(modelNode: PromptModelMetadata | undefined, modeKind: string | ChatModeKind | undefined, markers: IMarkerData[]) { - if (!modelNode || modelNode.value === undefined) { - return; - } - const languageModes = this.languageModelsService.getLanguageModelIds(); - if (languageModes.length === 0) { - // likely the service is not initialized yet - return; - } - const modelMetadata = this.findModelByName(languageModes, modelNode.value); - if (!modelMetadata) { - markers.push({ - message: localize('promptHeaderDiagnosticsProvider.modelNotFound', "Unknown model '{0}'", modelNode.value), - severity: MarkerSeverity.Warning, - ...modelNode.range, - }); - } else if (modeKind === ChatModeKind.Agent && !ILanguageModelChatMetadata.suitableForAgentMode(modelMetadata)) { - markers.push({ - message: localize('promptHeaderDiagnosticsProvider.modelNotSuited', "Model '{0}' is not suited for agent mode", modelNode.value), - severity: MarkerSeverity.Warning, - ...modelNode.range, - }); - } - - } - findModelByName(languageModes: string[], modelName: string): ILanguageModelChatMetadata | undefined { - for (const model of languageModes) { - const metadata = this.languageModelsService.lookupLanguageModel(model); - if (metadata && metadata.isUserSelectable !== false && ILanguageModelChatMetadata.matchesQualifiedName(modelName, metadata)) { - return metadata; - } - } - return undefined; - } - - validateTools(tools: PromptToolsMetadata | undefined, modeKind: string | ChatModeKind | undefined, markers: IMarkerData[]) { - if (!tools || tools.value === undefined || modeKind === ChatModeKind.Ask || modeKind === ChatModeKind.Edit) { - return; - } - const toolNames = new Set(tools.value); - if (toolNames.size === 0) { - return; - } - for (const tool of this.languageModelToolsService.getTools()) { - toolNames.delete(tool.toolReferenceName ?? tool.displayName); - } - for (const toolSet of this.languageModelToolsService.toolSets.get()) { - toolNames.delete(toolSet.referenceName); - } - - for (const toolName of toolNames) { - const range = tools.getToolRange(toolName); - if (range) { - markers.push({ - message: localize('promptHeaderDiagnosticsProvider.toolNotFound', "Unknown tool '{0}'", toolName), - severity: MarkerSeverity.Warning, - ...range, - }); - } - } - } - - validateMode(modeNode: PromptModeMetadata | undefined, markers: IMarkerData[]): IChatMode | undefined { - if (!modeNode || modeNode.value === undefined) { - return; - } - - const modeValue = modeNode.value; - const modes = this.chatModeService.getModes(); - const availableModes = []; - - // Check if mode exists in builtin or custom modes - for (const mode of Iterable.concat(modes.builtin, modes.custom)) { - if (mode.name === modeValue) { - return mode; - } - availableModes.push(mode.name); // collect all available mode names - } - - markers.push({ - message: localize('promptHeaderDiagnosticsProvider.modeNotFound', "Unknown mode '{0}'. Available modes: {1}", modeValue, availableModes.join(', ')), - severity: MarkerSeverity.Warning, - ...modeNode.range, - }); - return undefined; - - } - - /** - * Returns a string representation of this object. - */ - public override toString(): string { - return `prompt-header-diagnostics:${this.model.uri.path}`; - } -} - -/** - * Convert a provided diagnostic object into a marker data object. - */ -function toMarker(diagnostic: TDiagnostic): IMarkerData { - if (diagnostic instanceof PromptMetadataWarning) { - return { - message: diagnostic.message, - severity: MarkerSeverity.Warning, - ...diagnostic.range, - }; - } - - if (diagnostic instanceof PromptMetadataError) { - return { - message: diagnostic.message, - severity: MarkerSeverity.Error, - ...diagnostic.range, - }; - } - - assertNever( - diagnostic, - `Unknown prompt metadata diagnostic type '${diagnostic}'.`, - ); -} - -/** - * The class that manages creation and disposal of {@link PromptHeaderDiagnosticsProvider} - * classes for each specific editor text model. - */ -export class PromptHeaderDiagnosticsInstanceManager extends ProviderInstanceManagerBase { - protected override get InstanceClass(): TProviderClass { - return PromptHeaderDiagnosticsProvider; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/promptLinkDiagnosticsProvider.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/promptLinkDiagnosticsProvider.ts deleted file mode 100644 index 3809f67ca8f..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/promptLinkDiagnosticsProvider.ts +++ /dev/null @@ -1,98 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { IPromptsService } from '../service/promptsService.js'; -import { IPromptFileReference } from '../parsers/types.js'; -import { ProviderInstanceBase } from './providerInstanceBase.js'; -import { ITextModel } from '../../../../../../editor/common/model.js'; -import { assertDefined } from '../../../../../../base/common/types.js'; -import { ProviderInstanceManagerBase, TProviderClass } from './providerInstanceManagerBase.js'; -import { IMarkerData, IMarkerService, MarkerSeverity } from '../../../../../../platform/markers/common/markers.js'; -import { IFileService } from '../../../../../../platform/files/common/files.js'; -import { localize } from '../../../../../../nls.js'; - -/** - * Unique ID of the markers provider class. - */ -const MARKERS_OWNER_ID = 'prompt-link-diagnostics-provider'; - -/** - * Prompt links diagnostics provider for a single text model. - */ -class PromptLinkDiagnosticsProvider extends ProviderInstanceBase { - constructor( - model: ITextModel, - @IPromptsService promptsService: IPromptsService, - @IMarkerService private readonly markerService: IMarkerService, - @IFileService private readonly fileService: IFileService - ) { - super(model, promptsService); - } - - /** - * Update diagnostic markers for the current editor. - */ - protected override async onPromptSettled(): Promise { - // clean up all previously added markers - this.markerService.remove(MARKERS_OWNER_ID, [this.model.uri]); - - const markers: IMarkerData[] = []; - - const stats = await this.fileService.resolveAll(this.parser.references.map(ref => ({ resource: ref.uri }))); - for (let i = 0; i < stats.length; i++) { - if (!stats[i].success) { - markers.push(toMarker(this.parser.references[i], localize('fileNotFound', 'File not found.'))); - } - } - - this.markerService.changeOne( - MARKERS_OWNER_ID, - this.model.uri, - markers, - ); - } - - /** - * Returns a string representation of this object. - */ - public override toString(): string { - return `prompt-link-diagnostics:${this.model.uri.path}`; - } -} - -/** - * Convert a prompt link with an issue to a marker data. - * - * @throws - * - if there is no link issue (e.g., `topError` undefined) - * - if there is no link range to highlight (e.g., `linkRange` undefined) - * - if the original error is of `NotPromptFile` type - we don't want to - * show diagnostic markers for non-prompt file links in the prompts - */ -function toMarker(link: IPromptFileReference, message: string): IMarkerData { - const { linkRange } = link; - - assertDefined( - linkRange, - 'Link range must to be defined.', - ); - - - return { - message: message, - severity: MarkerSeverity.Warning, - ...linkRange, - }; -} - -/** - * The class that manages creation and disposal of {@link PromptLinkDiagnosticsProvider} - * classes for each specific editor text model. - */ -export class PromptLinkDiagnosticsInstanceManager extends ProviderInstanceManagerBase { - protected override get InstanceClass(): TProviderClass { - return PromptLinkDiagnosticsProvider; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/providerInstanceBase.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/providerInstanceBase.ts deleted file mode 100644 index da51052ab58..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/providerInstanceBase.ts +++ /dev/null @@ -1,54 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { IPromptsService, TSharedPrompt } from '../service/promptsService.js'; -import { ITextModel } from '../../../../../../editor/common/model.js'; -import { ObservableDisposable } from '../utils/observableDisposable.js'; -import { CancellationToken, CancellationTokenSource } from '../../../../../../base/common/cancellation.js'; - -/** - * Abstract base class for all reusable prompt file providers. - */ -export abstract class ProviderInstanceBase extends ObservableDisposable { - /** - * Function that is called when the prompt parser is settled. - */ - protected abstract onPromptSettled(error: Error | undefined, token: CancellationToken): Promise; - - /** - * Returns a string representation of this object. - */ - public abstract override toString(): string; - - /** - * The prompt parser instance. - */ - protected readonly parser: TSharedPrompt; - - constructor( - protected readonly model: ITextModel, - @IPromptsService promptsService: IPromptsService, - ) { - super(); - - this.parser = promptsService.getSyntaxParserFor(model); - - this._register( - this.parser.onDispose(this.dispose.bind(this)), - ); - - let cancellationSource = new CancellationTokenSource(); - this._register( - this.parser.onSettled((error) => { - cancellationSource.dispose(true); - cancellationSource = new CancellationTokenSource(); - - this.onPromptSettled(error, cancellationSource.token); - }), - ); - - this.parser.start(); - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/providerInstanceManagerBase.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/providerInstanceManagerBase.ts deleted file mode 100644 index 36ff2339b49..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/providerInstanceManagerBase.ts +++ /dev/null @@ -1,176 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { ProviderInstanceBase } from './providerInstanceBase.js'; -import { assert } from '../../../../../../base/common/assert.js'; -import { ITextModel } from '../../../../../../editor/common/model.js'; -import { assertDefined } from '../../../../../../base/common/types.js'; -import { Disposable } from '../../../../../../base/common/lifecycle.js'; -import { ObjectCache } from '../utils/objectCache.js'; -import { INSTRUCTIONS_LANGUAGE_ID, MODE_LANGUAGE_ID, PROMPT_LANGUAGE_ID } from '../promptTypes.js'; -import { IModelService } from '../../../../../../editor/common/services/model.js'; -import { PromptsConfig } from '../config/config.js'; -import { IEditorService } from '../../../../../services/editor/common/editorService.js'; -import { IDiffEditor, IEditor, IEditorModel } from '../../../../../../editor/common/editorCommon.js'; -import { IInstantiationService } from '../../../../../../platform/instantiation/common/instantiation.js'; -import { IConfigurationService } from '../../../../../../platform/configuration/common/configuration.js'; - -/** - * Type for a text editor that is used for reusable prompt files. - */ -export interface IPromptFileEditor extends IEditor { - readonly getModel: () => ITextModel; -} - -/** - * Type for a class that can create a new provider instance. - */ -export type TProviderClass = new (editor: ITextModel, ...args: any[]) => TInstance; - -/** - * A generic base class that manages creation and disposal of {@link TInstance} - * objects for each specific editor object that is used for reusable prompt files. - */ -export abstract class ProviderInstanceManagerBase extends Disposable { - /** - * Currently available {@link TInstance} instances. - */ - private readonly instances: ObjectCache; - - /** - * Class object of the managed {@link TInstance}. - */ - protected abstract get InstanceClass(): TProviderClass; - - constructor( - @IModelService modelService: IModelService, - @IEditorService editorService: IEditorService, - @IInstantiationService instantiationService: IInstantiationService, - @IConfigurationService configService: IConfigurationService, - ) { - super(); - - // cache of managed instances - this.instances = this._register( - new ObjectCache((model: ITextModel) => { - assert( - model.isDisposed() === false, - 'Text model must not be disposed.', - ); - - // sanity check - the new TS/JS discrepancies regarding fields initialization - // logic mean that this can be `undefined` during runtime while defined in TS - assertDefined( - this.InstanceClass, - 'Instance class field must be defined.', - ); - - const instance: TInstance = instantiationService.createInstance( - this.InstanceClass, - model, - ); - - // this is a sanity check and the contract of the object cache, - // we must return a non-disposed object from this factory function - instance.assertNotDisposed( - 'Created instance must not be disposed.', - ); - - return instance; - }), - ); - - // if the feature is disabled, do not create any providers - if (PromptsConfig.enabled(configService) === false) { - return; - } - - // subscribe to changes of the active editor - this._register(editorService.onDidActiveEditorChange(() => { - const { activeTextEditorControl } = editorService; - if (activeTextEditorControl === undefined) { - return; - } - - this.handleNewEditor(activeTextEditorControl); - })); - - // handle existing visible text editors - editorService - .visibleTextEditorControls - .forEach(this.handleNewEditor.bind(this)); - - // subscribe to "language change" events for all models - this._register( - modelService.onModelLanguageChanged((event) => { - const { model, oldLanguageId } = event; - - // if language is set to `prompt` or `instructions` language, handle that model - if (isPromptFileModel(model)) { - this.instances.get(model); - return; - } - - // if the language is changed away from `prompt` or `instructions`, - // remove and dispose provider for this model - if (isPromptFile(oldLanguageId)) { - this.instances.remove(model, true); - return; - } - }), - ); - } - - /** - * Initialize a new {@link TInstance} for the given editor. - */ - private handleNewEditor(editor: IEditor | IDiffEditor): this { - const model = editor.getModel(); - if (model === null) { - return this; - } - - if (isPromptFileModel(model) === false) { - return this; - } - - // note! calling `get` also creates a provider if it does not exist; - // and the provider is auto-removed when the editor is disposed - this.instances.get(model); - - return this; - } -} - -/** - * Check if provided language ID is one of the prompt file languages. - */ -function isPromptFile(languageId: string): boolean { - return [ - PROMPT_LANGUAGE_ID, - INSTRUCTIONS_LANGUAGE_ID, - MODE_LANGUAGE_ID, - ].includes(languageId); -} - -/** - * Check if a provided model is used for prompt files. - */ -function isPromptFileModel(model: IEditorModel): model is ITextModel { - // we support only `text editors` for now so filter out `diff` ones - if ('modified' in model || 'model' in model) { - return false; - } - - if (model.isDisposed()) { - return false; - } - - if (isPromptFile(model.getLanguageId()) === false) { - return false; - } - - return true; -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/basePromptParser.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/basePromptParser.ts deleted file mode 100644 index 9fe72885d03..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/basePromptParser.ts +++ /dev/null @@ -1,728 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { TopError } from './topError.js'; -import { ChatModeKind } from '../../constants.js'; -import { TMetadata } from './promptHeader/headerBase.js'; -import { ModeHeader } from './promptHeader/modeHeader.js'; -import { URI } from '../../../../../../base/common/uri.js'; -import { PromptToken } from '../codecs/tokens/promptToken.js'; -import * as path from '../../../../../../base/common/path.js'; -import { ChatPromptCodec } from '../codecs/chatPromptCodec.js'; -import { FileReference } from '../codecs/tokens/fileReference.js'; -import { ChatPromptDecoder } from '../codecs/chatPromptDecoder.js'; -import { assertDefined } from '../../../../../../base/common/types.js'; -import { Emitter } from '../../../../../../base/common/event.js'; -import { DeferredPromise } from '../../../../../../base/common/async.js'; -import { InstructionsHeader } from './promptHeader/instructionsHeader.js'; -import { ILogService } from '../../../../../../platform/log/common/log.js'; -import { PromptVariable, PromptVariableWithData } from '../codecs/tokens/promptVariable.js'; -import type { IPromptContentsProvider } from '../contentProviders/types.js'; -import type { TPromptReference, ITopError, TVariableReference } from './types.js'; -import { type IDisposable } from '../../../../../../base/common/lifecycle.js'; -import { assert, assertNever } from '../../../../../../base/common/assert.js'; -import { basename, dirname, joinPath } from '../../../../../../base/common/resources.js'; -import { BaseToken } from '../codecs/base/baseToken.js'; -import { VSBufferReadableStream } from '../../../../../../base/common/buffer.js'; -import { type IRange, Range } from '../../../../../../editor/common/core/range.js'; -import { PromptHeader, type TPromptMetadata } from './promptHeader/promptHeader.js'; -import { ObservableDisposable } from '../utils/observableDisposable.js'; -import { INSTRUCTIONS_LANGUAGE_ID, MODE_LANGUAGE_ID, PROMPT_LANGUAGE_ID } from '../promptTypes.js'; -import { LinesDecoder } from '../codecs/base/linesCodec/linesDecoder.js'; -import { IInstantiationService } from '../../../../../../platform/instantiation/common/instantiation.js'; -import { MarkdownLink } from '../codecs/base/markdownCodec/tokens/markdownLink.js'; -import { MarkdownToken } from '../codecs/base/markdownCodec/tokens/markdownToken.js'; -import { FrontMatterHeader } from '../codecs/base/markdownExtensionsCodec/tokens/frontMatterHeader.js'; -import { OpenFailed, NotPromptFile, RecursiveReference, FolderReference, ResolveError } from '../../promptFileReferenceErrors.js'; -import { type IPromptContentsProviderOptions } from '../contentProviders/promptContentsProviderBase.js'; -import { CancellationToken } from '../../../../../../base/common/cancellation.js'; -import { IWorkbenchEnvironmentService } from '../../../../../services/environment/common/environmentService.js'; -import { Schemas } from '../../../../../../base/common/network.js'; - -/** - * Options of the {@link BasePromptParser} class. - */ -export interface IBasePromptParserOptions { -} - -export type IPromptParserOptions = IBasePromptParserOptions & IPromptContentsProviderOptions; - - -/** - * Error conditions that may happen during the file reference resolution. - */ -export type TErrorCondition = OpenFailed | RecursiveReference | FolderReference | NotPromptFile; - -/** - * Base prompt parser class that provides a common interface for all - * prompt parsers that are responsible for parsing chat prompt syntax. - */ -export class BasePromptParser extends ObservableDisposable { - /** - * Options passed to the constructor. - */ - protected readonly options: IBasePromptParserOptions; - - /** - * List of all tokens that were parsed from the prompt contents so far. - */ - public get tokens(): readonly BaseToken[] { - return [...this.receivedTokens]; - } - /** - * Private field behind the readonly {@link tokens} property. - */ - private receivedTokens: BaseToken[] = []; - - /** - * List of file references in the prompt file. - */ - private readonly _references: TPromptReference[] = []; - - /** - * List of variable references in the prompt file. - */ - private readonly _variableReferences: TVariableReference[] = []; - - /** - * Reference to the prompt header object that holds metadata associated - * with the prompt. - */ - private promptHeader?: PromptHeader | InstructionsHeader | ModeHeader | undefined; - - /** - * Reference to the prompt header object that holds metadata associated - * with the prompt. - */ - public get header(): PromptHeader | InstructionsHeader | ModeHeader | undefined { - return this.promptHeader; - } - - /** - * Get contents of the prompt body. - */ - public async getBody(): Promise { - const startLineNumber = (this.header !== undefined) - ? this.header.range.endLineNumber + 1 - : 1; - - const decoder = new LinesDecoder( - await this.promptContentsProvider.contents, - ); - - const tokens = (await decoder.consumeAll()) - .filter(({ range }) => { - return (range.startLineNumber >= startLineNumber); - }); - - return BaseToken.render(tokens); - } - - /** - * Get the full contents of the prompt, including the header - */ - public async getFullContent(): Promise { - const decoder = new LinesDecoder( - await this.promptContentsProvider.contents, - ); - const tokens = await decoder.consumeAll(); - return BaseToken.render(tokens); - } - - /** - * The event is fired when lines or their content change. - */ - private readonly _onUpdate = this._register(new Emitter()); - /** - * Subscribe to the event that is fired the parser state or contents - * changes, including changes in the possible prompt child references. - */ - public readonly onUpdate = this._onUpdate.event; - - /** - * Event that is fired when the current prompt parser is settled. - */ - private readonly _onSettled = this._register(new Emitter()); - - /** - * Event that is fired when the current prompt parser is settled. - */ - public onSettled( - callback: (error?: Error) => void, - ): IDisposable { - const disposable = this._onSettled.event(callback); - const streamEnded = (this.stream?.ended && (this.stream.isDisposed === false)); - - // if already in the error state or stream has already ended, - // invoke the callback immediately but asynchronously - if (streamEnded || this.errorCondition) { - setTimeout(callback.bind(undefined, this.errorCondition)); - - return disposable; - } - - return disposable; - } - - /** - * If failed to parse prompt contents, this property has - * an error object that describes the failure reason. - */ - private _errorCondition?: ResolveError; - - /** - * If file reference resolution fails, this attribute will be set - * to an error instance that describes the error condition. - */ - public get errorCondition(): ResolveError | undefined { - return this._errorCondition; - } - - /** - * Whether file references resolution failed. - * Set to `undefined` if the `resolve` method hasn't been ever called yet. - */ - public get resolveFailed(): boolean | undefined { - if (!this.firstParseResult.gotFirstResult) { - return undefined; - } - - return !!this._errorCondition; - } - - /** - * The promise is resolved when at least one parse result (a stream or - * an error) has been received from the prompt contents provider. - */ - private readonly firstParseResult = new FirstParseResult(); - - /** - * Returned promise is resolved when the parser process is settled. - * The settled state means that the prompt parser stream exists and - * has ended, or an error condition has been set in case of failure. - * - * Furthermore, this function can be called multiple times and will - * block until the latest prompt contents parsing logic is settled - * (e.g., for every `onContentChanged` event of the prompt source). - */ - public async settled(): Promise { - assert( - this.started, - 'Cannot wait on the parser that did not start yet.', - ); - - await this.firstParseResult.promise; - - if (this.errorCondition) { - return false; - } - - // by the time when the `firstParseResult` promise is resolved, - // this object may have been already disposed, hence noop - if (this.isDisposed) { - return false; - } - - assertDefined( - this.stream, - 'No stream reference found.', - ); - - const completed = await this.stream.settled; - - // if prompt header exists, also wait for it to be settled - if (this.promptHeader) { - const headerCompleted = await this.promptHeader.settled; - if (!headerCompleted) { - return false; - } - } - - return completed; - } - - constructor( - private readonly promptContentsProvider: TContentsProvider, - options: IBasePromptParserOptions, - @IInstantiationService protected readonly instantiationService: IInstantiationService, - @IWorkbenchEnvironmentService private readonly envService: IWorkbenchEnvironmentService, - @ILogService protected readonly logService: ILogService, - ) { - super(); - - this.options = options; - - this._register( - this.promptContentsProvider.onContentChanged((streamOrError) => { - // process the received message - this.onContentsChanged(streamOrError); - - // indicate that we've received at least one `onContentChanged` event - this.firstParseResult.end(); - }), - ); - - // dispose self when contents provider is disposed - this._register( - this.promptContentsProvider.onDispose(this.dispose.bind(this)), - ); - } - - /** - * The latest received stream of prompt tokens, if any. - */ - private stream: ChatPromptDecoder | undefined; - - /** - * Handler the event event that is triggered when prompt contents change. - * - * @param streamOrError Either a binary stream of file contents, or an error object - * that was generated during the reference resolve attempt. - * @param seenReferences List of parent references that we've have already seen - * during the process of traversing the references tree. It's - * used to prevent the tree navigation to fall into an infinite - * references recursion. - */ - private onContentsChanged( - streamOrError: VSBufferReadableStream | ResolveError - ): void { - // dispose and cleanup the previously received stream - // object or an error condition, if any received yet - this.stream?.dispose(); - delete this.stream; - delete this._errorCondition; - this.receivedTokens = []; - - // cleanup current prompt header object - this.promptHeader?.dispose(); - delete this.promptHeader; - - // dispose all currently existing references - this.disposeReferences(); - - // if an error received, set up the error condition and stop - if (streamOrError instanceof ResolveError) { - this._errorCondition = streamOrError; - this._onUpdate.fire(); - - // when error received fire the 'onSettled' event immediately - this._onSettled.fire(streamOrError); - - return; - } - - // decode the byte stream to a stream of prompt tokens - this.stream = ChatPromptCodec.decode(streamOrError); - - /** - * !NOTE! The order of event subscriptions below is critical here because - * the `data` event is also starts the stream, hence changing - * the order of event subscriptions can lead to race conditions. - * See {@link ReadableStreamEvents} for more info. - */ - - // on error or stream end, dispose the stream and fire the update event - this.stream.on('error', this.onStreamEnd.bind(this, this.stream)); - this.stream.on('end', this.onStreamEnd.bind(this, this.stream)); - - // when some tokens received, process and store the references - this.stream.on('data', (token) => { - // store all markdown and prompt token references - if ((token instanceof MarkdownToken) || (token instanceof PromptToken)) { - this.receivedTokens.push(token); - } - - // if a prompt header token received, create a new prompt header instance - if (token instanceof FrontMatterHeader) { - return this.createHeader(token); - } - - // try to convert a prompt variable with data token into a file reference - if (token instanceof PromptVariableWithData) { - try { - if (token.name === 'file') { - this.handleLinkToken(FileReference.from(token)); - } - } catch (error) { - // the `FileReference.from` call might throw if the `PromptVariableWithData` token - // can not be converted into a valid `#file` reference, hence we ignore the error - } - } else if (token instanceof PromptVariable) { - this.handleVariableToken(token); - } - - // note! the `isURL` is a simple check and needs to be improved to truly - // handle only file references, ignoring broken URLs or references - if (token instanceof MarkdownLink && !token.isURL) { - this.handleLinkToken(token); - } - }); - - // calling `start` on a disposed stream throws, so we warn and return instead - if (this.stream.isDisposed) { - this.logService.warn( - `[prompt parser][${basename(this.uri)}] cannot start stream that has been already disposed, aborting`, - ); - - return; - } - - // start receiving data on the stream - this.stream.start(); - } - - /** - * Create header object base on the target prompt file language ID. - * The language ID is important here, because it defines what type - * of metadata is valid for a prompt file and what type of related - * diagnostics we would show to the user. - */ - private createHeader(headerToken: FrontMatterHeader): void { - const { languageId } = this.promptContentsProvider; - - if (languageId === PROMPT_LANGUAGE_ID) { - this.promptHeader = new PromptHeader(headerToken, languageId); - } - - if (languageId === INSTRUCTIONS_LANGUAGE_ID) { - this.promptHeader = new InstructionsHeader(headerToken, languageId); - } - - if (languageId === MODE_LANGUAGE_ID) { - this.promptHeader = new ModeHeader(headerToken, languageId); - } - - this.promptHeader?.start(); - } - - /** - * Handle a new reference token inside prompt contents. - */ - private handleLinkToken(token: FileReference | MarkdownLink): this { - - let referenceUri: URI; - if (path.isAbsolute(token.path)) { - referenceUri = URI.file(token.path); - if (this.envService.remoteAuthority) { - referenceUri = referenceUri.with({ - scheme: Schemas.vscodeRemote, - authority: this.envService.remoteAuthority, - }); - } - } else { - referenceUri = joinPath(dirname(this.uri), token.path); - } - this._references.push(new PromptReference(referenceUri, token)); - - this._onUpdate.fire(); - - return this; - } - - private handleVariableToken(token: PromptVariable): this { - - this._variableReferences.push({ name: token.name, range: token.range }); - - this._onUpdate.fire(); - - return this; - } - - /** - * Handle the `stream` end event. - * - * @param stream The stream that has ended. - * @param error Optional error object if stream ended with an error. - */ - private onStreamEnd( - stream: ChatPromptDecoder, - error?: Error, - ): this { - // decoders can fire the 'end' event also when they are get disposed, - // but because we dispose them when a new stream is received, we can - // safely ignore the event in this case - if (stream.isDisposed === true) { - return this; - } - - if (error) { - this.logService.warn( - `[prompt parser][${basename(this.uri)}] received an error on the chat prompt decoder stream: ${error}`, - ); - } - - this._onUpdate.fire(); - this._onSettled.fire(error); - - return this; - } - - - private disposeReferences(): void { - - - this._references.length = 0; - this._variableReferences.length = 0; - } - - /** - * Private attribute to track if the {@link start} - * method has been already called at least once. - */ - private started: boolean = false; - - /** - * Start the prompt parser. - */ - public start(token?: CancellationToken): this { - // if already started, nothing to do - if (this.started) { - return this; - } - this.started = true; - - - // if already in the error state that could be set - // in the constructor, then nothing to do - if (this.errorCondition) { - return this; - } - - this.promptContentsProvider.start(token); - return this; - } - - /** - * Associated URI of the prompt. - */ - public get uri(): URI { - return this.promptContentsProvider.uri; - } - - /** - * Get a list of immediate child references of the prompt. - */ - public get references(): readonly TPromptReference[] { - return [...this._references]; - } - - /** - * Get a list of variable references of the prompt. - */ - public get variableReferences(): readonly TVariableReference[] { - return [...this._variableReferences]; - } - - /** - * Valid metadata records defined in the prompt header. - */ - public get metadata(): TMetadata | null { - const { promptType } = this.promptContentsProvider; - if (promptType === 'non-prompt') { - return null; - } - - if (this.header === undefined) { - return { promptType }; - } - - if (this.header instanceof InstructionsHeader || this.header instanceof ModeHeader) { - return { promptType, ...this.header.metadata }; - } - - const { tools, mode, description, model } = this.header.metadata; - - const result: Partial = {}; - - if (description !== undefined) { - result.description = description; - } - - if (tools !== undefined && mode !== ChatModeKind.Ask && mode !== ChatModeKind.Edit) { - result.tools = tools; - // Preserve custom mode if specified, otherwise default to Agent - result.mode = mode || ChatModeKind.Agent; - } else if (mode !== undefined) { - result.mode = mode; - } - - if (model !== undefined) { - result.model = model; - } - - return { promptType, ...result }; - } - - /** - * The top most error of the current reference or any of its - * possible child reference errors. - */ - public get topError(): ITopError | undefined { - if (this.errorCondition) { - return new TopError({ - errorSubject: 'root', - errorsCount: 1, - originalError: this.errorCondition, - }); - } - - return undefined; - } - - /** - * Returns a string representation of this object. - */ - public override toString(): string { - return `prompt:${this.uri.path}`; - } - - /** - * @inheritdoc - */ - public override dispose(): void { - if (this.isDisposed) { - return; - } - - this.disposeReferences(); - - this.stream?.dispose(); - delete this.stream; - - this.promptHeader?.dispose(); - delete this.promptHeader; - - super.dispose(); - } -} - -/** - * Prompt reference object represents any reference inside prompt text - * contents. For instance the file variable(`#file:/path/to/file.md`) or - * a markdown link(`[#file:file.md](/path/to/file.md)`). - */ -export class PromptReference implements TPromptReference { - - - constructor( - public readonly uri: URI, - public readonly token: FileReference | MarkdownLink, - ) { - } - - /** - * Get the range of the `link` part of the reference. - */ - public get linkRange(): IRange | undefined { - // `#file:` references - if (this.token instanceof FileReference) { - return this.token.dataRange; - } - - // `markdown link` references - if (this.token instanceof MarkdownLink) { - return this.token.linkRange; - } - - return undefined; - } - - /** - * Type of the reference, - either a prompt `#file` variable, - * or a `markdown link` reference (`[caption](/path/to/file.md)`). - */ - public get type(): 'file' { - if (this.token instanceof FileReference) { - return 'file'; - } - - if (this.token instanceof MarkdownLink) { - return 'file'; - } - - assertNever( - this.token, - `Unknown token type '${this.token}'.`, - ); - } - - /** - * Subtype of the reference, - either a prompt `#file` variable, - * or a `markdown link` reference (`[caption](/path/to/file.md)`). - */ - public get subtype(): 'prompt' | 'markdown' { - if (this.token instanceof FileReference) { - return 'prompt'; - } - - if (this.token instanceof MarkdownLink) { - return 'markdown'; - } - - assertNever( - this.token, - `Unknown token type '${this.token}'.`, - ); - } - - public get range(): Range { - return this.token.range; - } - - public get path(): string { - return this.token.path; - } - - public get text(): string { - return this.token.text; - } - - /** - * Returns a string representation of this object. - */ - public toString(): string { - return `prompt-reference/${this.type}:${this.subtype}/${this.token}`; - } -} - -/** - * A tiny utility object that helps us to track existence - * of at least one parse result from the content provider. - */ -class FirstParseResult extends DeferredPromise { - /** - * Private attribute to track if we have - * received at least one result. - */ - private _gotResult = false; - - /** - * Whether we've received at least one result. - */ - public get gotFirstResult(): boolean { - return this._gotResult; - } - - /** - * Get underlying promise reference. - */ - public get promise(): Promise { - return this.p; - } - - /** - * Complete the underlying promise. - */ - public end(): void { - this._gotResult = true; - super.complete(void 0) - .catch(() => { - // the complete method is never fails - // so we can ignore the error here - }); - - return; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/filePromptParser.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/filePromptParser.ts deleted file mode 100644 index c73efe10884..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/filePromptParser.ts +++ /dev/null @@ -1,37 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { URI } from '../../../../../../base/common/uri.js'; -import { ILogService } from '../../../../../../platform/log/common/log.js'; -import { BasePromptParser, IPromptParserOptions } from './basePromptParser.js'; -import { FilePromptContentProvider } from '../contentProviders/filePromptContentsProvider.js'; -import { IInstantiationService } from '../../../../../../platform/instantiation/common/instantiation.js'; -import { IWorkbenchEnvironmentService } from '../../../../../services/environment/common/environmentService.js'; - -/** - * Class capable of parsing prompt syntax out of a provided file, - * including all the nested child file references it may have. - */ -export class FilePromptParser extends BasePromptParser { - constructor( - uri: URI, - options: IPromptParserOptions, - @IInstantiationService instantiationService: IInstantiationService, - @IWorkbenchEnvironmentService envService: IWorkbenchEnvironmentService, - @ILogService logService: ILogService, - ) { - const contentsProvider = instantiationService.createInstance(FilePromptContentProvider, uri, options); - super(contentsProvider, options, instantiationService, envService, logService); - - this._register(contentsProvider); - } - - /** - * Returns a string representation of this object. - */ - public override toString(): string { - return `file-prompt:${this.uri.path}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/diagnostics.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/diagnostics.ts deleted file mode 100644 index 754293f6dcf..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/diagnostics.ts +++ /dev/null @@ -1,47 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { Range } from '../../../../../../../editor/common/core/range.js'; - -/** - * List of all currently supported diagnostic types. - */ -export type TDiagnostic = PromptMetadataWarning | PromptMetadataError; - -/** - * Diagnostics object that hold information about some issue - * related to the prompt header metadata. - */ -export abstract class PromptMetadataDiagnostic { - constructor( - public readonly range: Range, - public readonly message: string, - ) { } - - /** - * String representation of the diagnostic object. - */ - public abstract toString(): string; -} - -/** - * Diagnostics object that hold information about some - * non-fatal issue related to the prompt header metadata. - */ -export class PromptMetadataWarning extends PromptMetadataDiagnostic { - public override toString(): string { - return `warning(${this.message})${this.range}`; - } -} - -/** - * Diagnostics object that hold information about some - * fatal issue related to the prompt header metadata. - */ -export class PromptMetadataError extends PromptMetadataDiagnostic { - public override toString(): string { - return `error(${this.message})${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/headerBase.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/headerBase.ts deleted file mode 100644 index eedebeabeed..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/headerBase.ts +++ /dev/null @@ -1,264 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { type TModeMetadata } from './modeHeader.js'; -import { localize } from '../../../../../../../nls.js'; -import { type TPromptMetadata } from './promptHeader.js'; -import { type IMetadataRecord } from './metadata/base/record.js'; -import { type TInstructionsMetadata } from './instructionsHeader.js'; -import { Range } from '../../../../../../../editor/common/core/range.js'; -import { Disposable } from '../../../../../../../base/common/lifecycle.js'; -import { ObjectStream } from '../../codecs/base/utils/objectStream.js'; -import { PromptMetadataError, PromptMetadataWarning, type TDiagnostic } from './diagnostics.js'; -import { SimpleToken } from '../../codecs/base/simpleCodec/tokens/tokens.js'; -import { FrontMatterRecord } from '../../codecs/base/frontMatterCodec/tokens/index.js'; -import { FrontMatterHeader } from '../../codecs/base/markdownExtensionsCodec/tokens/frontMatterHeader.js'; -import { FrontMatterDecoder, type TFrontMatterToken } from '../../codecs/base/frontMatterCodec/frontMatterDecoder.js'; -import { PromptDescriptionMetadata } from './metadata/description.js'; - -/** - * A metadata utility class "dehydrated" into a plain data object with - * semi-primitive record values (string, boolean, string[], boolean[], etc.). - */ -export type TDehydrated = { - [K in keyof T]: T[K] extends IMetadataRecord ? (U extends undefined ? undefined : NonNullable) : undefined; -}; - -/** - * Metadata defined in the prompt header. - */ -export interface IHeaderMetadata { - /** - * Description metadata in the prompt header. - */ - description: PromptDescriptionMetadata; -} - -/** - * Metadata for prompt/instruction/mode files. - */ -export type THeaderMetadata = Partial>; - -/** - * Metadata defined in the header of prompt/instruction/mode files. - */ -export type TMetadata = TPromptMetadata | TModeMetadata | TInstructionsMetadata; - -/** - * Base class for prompt/instruction/mode headers. - */ -export abstract class HeaderBase< - TMetadata extends IHeaderMetadata, -> extends Disposable { - /** - * Underlying decoder for a Front Matter header. - */ - private readonly stream: FrontMatterDecoder; - - /** - * Metadata records. - */ - protected readonly meta: Partial; - - /** - * Data object with all header's metadata records. - */ - public get metadata(): Partial> { - const result: Partial> = {}; - - for (const [entryName, entryValue] of Object.entries(this.meta)) { - if (entryValue?.value === undefined) { - continue; - } - - // note! we have to resort to `Object.assign()` here because - // the `Object.entries()` call looses type information - Object.assign(result, { - [entryName]: entryValue.value, - }); - } - - return result; - } - - /** - * A copy of metadata object with utility classes as values - * for each of prompt header's record. - * - * Please use {@link metadata} instead if all you need to read is - * the plain "data" object representation of valid metadata records. - */ - public get metadataUtility(): Partial { - return { ...this.meta }; - } - - /** - * List of all unique metadata record names. - */ - private readonly recordNames: Set; - - /** - * List of all issues found while parsing the prompt header. - */ - protected readonly issues: TDiagnostic[]; - - /** - * List of all diagnostic issues found while parsing - * the prompt header. - */ - public get diagnostics(): readonly TDiagnostic[] { - return this.issues; - } - - /** - * Full range of the header in the original document. - */ - public get range(): Range { - return this.token.range; - } - - constructor( - public readonly token: FrontMatterHeader, - public readonly languageId: string, - ) { - super(); - - this.issues = []; - this.meta = {}; - this.recordNames = new Set(); - - this.stream = this._register( - new FrontMatterDecoder( - ObjectStream.fromArray([...token.contentToken.children]), - ), - ); - this.stream.onData(this.onData.bind(this)); - this.stream.onError(this.onError.bind(this)); - } - - /** - * Process a front matter record token, which includes: - * - validation of the record and whether it is compatible with other header records - * - adding validation-related diagnostic messages to the {@link issues} list - * - setting associated utility class for the record on the {@link meta} object - * - * @returns a boolean flag that indicates whether the token was handled and therefore - * should not be processed any further. - */ - protected abstract handleToken( - token: FrontMatterRecord, - ): boolean; - - /** - * Process front matter tokens, converting them into - * well-known prompt metadata records. - */ - private onData(token: TFrontMatterToken): void { - // we currently expect only front matter 'records' for - // the prompt metadata, hence add diagnostics for all - // other tokens and ignore them - if ((token instanceof FrontMatterRecord) === false) { - // unless its a simple token, in which case we just ignore it - if (token instanceof SimpleToken) { - return; - } - - this.issues.push( - new PromptMetadataError( - token.range, - localize( - 'prompt.header.diagnostics.unexpected-token', - "Unexpected token '{0}'.", - token.text, - ), - ), - ); - - return; - } - - const recordName = token.nameToken.text; - - // if we already have a record with this name, - // add a warning diagnostic and ignore it - if (this.recordNames.has(recordName)) { - this.issues.push( - new PromptMetadataWarning( - token.range, - localize( - 'prompt.header.metadata.diagnostics.duplicate-record', - "Duplicate property '{0}' will be ignored.", - recordName, - ), - ), - ); - - return; - } - this.recordNames.add(recordName); - - // if the record might be a "description" metadata - // add it to the list of parsed metadata records - if (PromptDescriptionMetadata.isDescriptionRecord(token)) { - const metadata = new PromptDescriptionMetadata(token, this.languageId); - - this.issues.push(...metadata.validate()); - this.meta.description = metadata; - this.recordNames.add(recordName); - return; - } - - // pipe the token to the actual implementation class - // that might to handle it based on the token type - if (this.handleToken(token)) { - return; - } - - // all other records are "unknown" ones - this.issues.push( - new PromptMetadataWarning( - token.range, - localize( - 'prompt.header.metadata.diagnostics.unknown-record', - "Unknown property '{0}' will be ignored.", - recordName, - ), - ), - ); - } - - /** - * Process errors from the underlying front matter decoder. - */ - private onError(error: Error): void { - this.issues.push( - new PromptMetadataError( - this.token.range, - localize( - 'prompt.header.diagnostics.parsing-error', - "Failed to parse prompt header: {0}", - error.message, - ), - ), - ); - } - - /** - * Promise that resolves when parsing process of - * the prompt header completes. - */ - public get settled(): Promise { - return this.stream.settled; - } - - /** - * Starts the parsing process of the prompt header. - */ - public start(): this { - this.stream.start(); - - return this; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/instructionsHeader.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/instructionsHeader.ts deleted file mode 100644 index 1f13901f960..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/instructionsHeader.ts +++ /dev/null @@ -1,44 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { PromptApplyToMetadata } from './metadata/applyTo.js'; -import { HeaderBase, IHeaderMetadata, type TDehydrated } from './headerBase.js'; -import { PromptsType } from '../../promptTypes.js'; -import { FrontMatterRecord } from '../../codecs/base/frontMatterCodec/tokens/index.js'; - -/** - * Metadata utility object for instruction files. - */ -interface IInstructionsMetadata extends IHeaderMetadata { - /** - * Chat 'applyTo' metadata in the prompt header. - */ - applyTo: PromptApplyToMetadata; -} - -/** - * Metadata for instruction files. - */ -export type TInstructionsMetadata = Partial> & { promptType: PromptsType.instructions }; - -/** - * Header object for instruction files. - */ -export class InstructionsHeader extends HeaderBase { - protected override handleToken(token: FrontMatterRecord): boolean { - // if the record might be a "applyTo" metadata - // add it to the list of parsed metadata records - if (PromptApplyToMetadata.isApplyToRecord(token)) { - const metadata = new PromptApplyToMetadata(token, this.languageId); - - this.issues.push(...metadata.validate()); - this.meta.applyTo = metadata; - - return true; - } - - return false; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/applyTo.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/applyTo.ts deleted file mode 100644 index 7f374c0f108..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/applyTo.ts +++ /dev/null @@ -1,122 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { PromptStringMetadata } from './base/string.js'; -import { localize } from '../../../../../../../../nls.js'; -import { INSTRUCTIONS_LANGUAGE_ID } from '../../../promptTypes.js'; -import { isEmptyPattern, parse, splitGlobAware } from '../../../../../../../../base/common/glob.js'; -import { PromptMetadataDiagnostic, PromptMetadataError, PromptMetadataWarning } from '../diagnostics.js'; -import { FrontMatterRecord, FrontMatterToken } from '../../../codecs/base/frontMatterCodec/tokens/index.js'; - -/** - * Name of the metadata record in the prompt header. - */ -const RECORD_NAME = 'applyTo'; - -/** - * Prompt `applyTo` metadata record inside the prompt header. - */ -export class PromptApplyToMetadata extends PromptStringMetadata { - constructor( - recordToken: FrontMatterRecord, - languageId: string, - ) { - super(RECORD_NAME, recordToken, languageId); - } - - public override get recordName(): string { - return RECORD_NAME; - } - - public override validate(): readonly PromptMetadataDiagnostic[] { - super.validate(); - - // if we don't have a value token, validation must - // has failed already so nothing to do more - if (this.valueToken === undefined) { - return this.issues; - } - - // the applyTo metadata makes sense only for 'instruction' prompts - if (this.languageId !== INSTRUCTIONS_LANGUAGE_ID) { - this.issues.push( - new PromptMetadataError( - this.range, - localize( - 'prompt.header.metadata.string.diagnostics.invalid-language', - "The '{0}' header property is only valid in instruction files.", - this.recordName, - ), - ), - ); - - delete this.valueToken; - return this.issues; - } - - const { cleanText } = this.valueToken; - - // warn user if specified glob pattern is not valid - if (this.isValidGlob(cleanText) === false) { - this.issues.push( - new PromptMetadataWarning( - this.valueToken.range, - localize( - 'prompt.header.metadata.applyTo.diagnostics.non-valid-glob', - "Invalid glob pattern '{0}'.", - cleanText, - ), - ), - ); - - delete this.valueToken; - return this.issues; - } - - return this.issues; - } - - /** - * Check if a provided string contains a valid glob pattern. - */ - private isValidGlob( - pattern: string, - ): boolean { - try { - const patterns = splitGlobAware(pattern, ','); - if (patterns.length === 0) { - return false; - } - for (const pattern of patterns) { - - const globPattern = parse(pattern); - if (isEmptyPattern(globPattern)) { - return false; - } - } - return true; - } catch (_error) { - return false; - } - } - - /** - * Check if a provided front matter token is a metadata record - * with name equal to `applyTo`. - */ - public static isApplyToRecord( - token: FrontMatterToken, - ): boolean { - if ((token instanceof FrontMatterRecord) === false) { - return false; - } - - if (token.nameToken.text === RECORD_NAME) { - return true; - } - - return false; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/base/enum.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/base/enum.ts deleted file mode 100644 index 71b7d9e7a4f..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/base/enum.ts +++ /dev/null @@ -1,84 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { PromptStringMetadata } from './string.js'; -import { localize } from '../../../../../../../../../nls.js'; -import { assert } from '../../../../../../../../../base/common/assert.js'; -import { isOneOf } from '../../../../../../../../../base/common/types.js'; -import { PromptMetadataDiagnostic, PromptMetadataError } from '../../diagnostics.js'; -import { FrontMatterSequence } from '../../../../codecs/base/frontMatterCodec/tokens/frontMatterSequence.js'; -import { FrontMatterRecord, FrontMatterString } from '../../../../codecs/base/frontMatterCodec/tokens/index.js'; - -/** - * Enum type is the special case of the {@link PromptStringMetadata string} - * type that can take only a well-defined set of {@link validValues}. - */ -export abstract class PromptEnumMetadata< - TValidValues extends string = string, -> extends PromptStringMetadata { - constructor( - private readonly validValues: readonly TValidValues[], - expectedRecordName: string, - recordToken: FrontMatterRecord, - languageId: string, - ) { - super(expectedRecordName, recordToken, languageId); - } - - /** - * Valid enum value or 'undefined'. - */ - private enumValue: TValidValues | undefined; - /** - * Valid enum value or 'undefined'. - */ - public override get value(): TValidValues | undefined { - return this.enumValue; - } - - /** - * Validate the metadata record has an allowed value. - */ - public override validate(): readonly PromptMetadataDiagnostic[] { - super.validate(); - - if (this.valueToken === undefined) { - return this.issues; - } - - // sanity check for our expectations about the validate call - assert( - this.valueToken instanceof FrontMatterString - || this.valueToken instanceof FrontMatterSequence, - `Record token must be 'string', got '${this.valueToken}'.`, - ); - - const { cleanText } = this.valueToken; - if (isOneOf(cleanText, this.validValues)) { - this.enumValue = cleanText; - - return this.issues; - } - - this.issues.push( - new PromptMetadataError( - this.valueToken.range, - localize( - 'prompt.header.metadata.enum.diagnostics.invalid-value', - "The property '{0}' must be one of {1}, got '{2}'.", - this.recordName, - this.validValues - .map((value) => { - return `'${value}'`; - }).join(' | '), - cleanText, - ), - ), - ); - - delete this.valueToken; - return this.issues; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/base/record.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/base/record.ts deleted file mode 100644 index aa7a4660f66..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/base/record.ts +++ /dev/null @@ -1,108 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { assert } from '../../../../../../../../../base/common/assert.js'; -import { Range } from '../../../../../../../../../editor/common/core/range.js'; -import { PromptMetadataDiagnostic, PromptMetadataError, PromptMetadataWarning } from '../../diagnostics.js'; -import { FrontMatterRecord } from '../../../../codecs/base/frontMatterCodec/tokens/index.js'; - -/** - * Supported primitive types for metadata values in a prompt header. - */ -type TMetadataPrimitive = string | boolean; - -/** - * Supported metadata values in a prompt header. - */ -type TMetadataValue = TMetadataPrimitive | TMetadataPrimitive[]; - -/** - * Interface for a generic metadata record in the prompt header. - */ -export interface IMetadataRecord { - /** - * Value of a metadata record. If the value is not defined, it usually - * means that a record is present but its value is not set or valid. - */ - readonly value: T | undefined; -} - -/** - * Abstract class for all metadata records in the prompt header. - */ -export abstract class PromptMetadataRecord implements IMetadataRecord { - /** - * Private field for tracking all diagnostic issues - * related to this metadata record. - */ - protected readonly issues: PromptMetadataDiagnostic[]; - - /** - * Full range of the metadata's record text in the prompt header. - */ - public get range(): Range { - return this.recordToken.range; - } - - constructor( - protected readonly expectedRecordName: string, - protected readonly recordToken: FrontMatterRecord, - protected readonly languageId: string, - ) { - // validate that the record name has the expected name - const recordName = recordToken.nameToken.text; - assert( - recordName === expectedRecordName, - `Record name must be '${expectedRecordName}', got '${recordName}'.`, - ); - - this.issues = []; - } - - /** - * Name of the metadata record. - */ - public get recordName(): string { - return this.recordToken.nameToken.text; - } - - /** - * Validate the metadata record and collect all issues - * related to its content. - */ - public abstract validate(): readonly PromptMetadataDiagnostic[]; - - /** - * List of all diagnostic issues related to this metadata record. - */ - public get diagnostics(): readonly PromptMetadataDiagnostic[] { - return this.issues; - } - - /** - * Get the value of the metadata record. - */ - public abstract get value(): TValue | undefined; - - /** - * List of all `error` issue diagnostics. - */ - public get errorDiagnostics(): readonly PromptMetadataError[] { - return this.diagnostics - .filter((diagnostic) => { - return (diagnostic instanceof PromptMetadataError); - }); - } - - /** - * List of all `warning` issue diagnostics. - */ - public get warningDiagnostics(): readonly PromptMetadataWarning[] { - return this.diagnostics - .filter((diagnostic) => { - return (diagnostic instanceof PromptMetadataWarning); - }); - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/base/string.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/base/string.ts deleted file mode 100644 index 87423ab25b2..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/base/string.ts +++ /dev/null @@ -1,73 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { PromptMetadataRecord } from './record.js'; -import { localize } from '../../../../../../../../../nls.js'; -import { PromptMetadataDiagnostic, PromptMetadataError } from '../../diagnostics.js'; -import { FrontMatterSequence } from '../../../../codecs/base/frontMatterCodec/tokens/frontMatterSequence.js'; -import { FrontMatterRecord, FrontMatterString } from '../../../../codecs/base/frontMatterCodec/tokens/index.js'; -import { Range } from '../../../../../../../../../editor/common/core/range.js'; - - -/** - * Base class for all metadata records with a `string` value. - */ -export abstract class PromptStringMetadata extends PromptMetadataRecord { - /** - * Value token reference of the record. - */ - protected valueToken: FrontMatterString | FrontMatterSequence | undefined; - - /** - * String value of a metadata record. - */ - public override get value(): string | undefined { - return this.valueToken?.cleanText; - } - - public get valueRange(): Range | undefined { - return this.valueToken?.range; - } - - constructor( - expectedRecordName: string, - recordToken: FrontMatterRecord, - languageId: string, - ) { - super(expectedRecordName, recordToken, languageId); - } - - /** - * Validate the metadata record has a 'string' value. - */ - public override validate(): readonly PromptMetadataDiagnostic[] { - const { valueToken } = this.recordToken; - - // validate that the record value is a string or a generic sequence - // of tokens that can be interpreted as a string without quotes - const isString = (valueToken instanceof FrontMatterString); - const isSequence = (valueToken instanceof FrontMatterSequence); - if (isString || isSequence) { - this.valueToken = valueToken; - return this.issues; - } - - this.issues.push( - new PromptMetadataError( - valueToken.range, - localize( - 'prompt.header.metadata.string.diagnostics.invalid-value-type', - "The property '{0}' must be of type '{1}', got '{2}'.", - this.recordName, - 'string', - valueToken.valueTypeName.toString(), - ), - ), - ); - - delete this.valueToken; - return this.issues; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/description.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/description.ts deleted file mode 100644 index aaa8fa3a640..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/description.ts +++ /dev/null @@ -1,46 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { PromptStringMetadata } from './base/string.js'; -import { FrontMatterRecord, FrontMatterToken } from '../../../codecs/base/frontMatterCodec/tokens/index.js'; - -/** - * Name of the metadata record in the prompt header. - */ -const RECORD_NAME = 'description'; - -/** - * Prompt `description` metadata record inside the prompt header. - */ -export class PromptDescriptionMetadata extends PromptStringMetadata { - public override get recordName(): string { - return RECORD_NAME; - } - - constructor( - recordToken: FrontMatterRecord, - languageId: string, - ) { - super(RECORD_NAME, recordToken, languageId); - } - - /** - * Check if a provided front matter token is a metadata record - * with name equal to `description`. - */ - public static isDescriptionRecord( - token: FrontMatterToken, - ): boolean { - if ((token instanceof FrontMatterRecord) === false) { - return false; - } - - if (token.nameToken.text === RECORD_NAME) { - return true; - } - - return false; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/mode.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/mode.ts deleted file mode 100644 index b12728f7e75..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/mode.ts +++ /dev/null @@ -1,47 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { PromptStringMetadata } from './base/string.js'; -import { FrontMatterRecord, FrontMatterToken } from '../../../codecs/base/frontMatterCodec/tokens/index.js'; - -/** - * Name of the metadata record in the prompt header. - */ -const RECORD_NAME = 'mode'; - -/** - * Prompt `mode` metadata record inside the prompt header. - * Now supports both built-in modes (ask, edit, agent) and custom mode IDs. - */ -export class PromptModeMetadata extends PromptStringMetadata { - constructor( - recordToken: FrontMatterRecord, - languageId: string, - ) { - super( - RECORD_NAME, - recordToken, - languageId, - ); - } - - /** - * Check if a provided front matter token is a metadata record - * with name equal to `mode`. - */ - public static isModeRecord( - token: FrontMatterToken, - ): boolean { - if ((token instanceof FrontMatterRecord) === false) { - return false; - } - - if (token.nameToken.text === RECORD_NAME) { - return true; - } - - return false; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/model.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/model.ts deleted file mode 100644 index aa559e57fc0..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/model.ts +++ /dev/null @@ -1,41 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { FrontMatterRecord, FrontMatterToken } from '../../../codecs/base/frontMatterCodec/tokens/index.js'; -import { PromptStringMetadata } from './base/string.js'; - -/** - * Name of the metadata record in the prompt header. - */ -const RECORD_NAME = 'model'; - -export class PromptModelMetadata extends PromptStringMetadata { - public override get recordName(): string { - return RECORD_NAME; - } - - constructor( - recordToken: FrontMatterRecord, - languageId: string, - ) { - super(RECORD_NAME, recordToken, languageId); - } - - /** - * Check if a provided front matter token is a metadata record - * with name equal to `description`. - */ - public static isModelRecord(token: FrontMatterToken): boolean { - if ((token instanceof FrontMatterRecord) === false) { - return false; - } - - if (token.nameToken.text === RECORD_NAME) { - return true; - } - - return false; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/tools.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/tools.ts deleted file mode 100644 index ad52e5faebe..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/tools.ts +++ /dev/null @@ -1,182 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { PromptMetadataRecord } from './base/record.js'; -import { localize } from '../../../../../../../../nls.js'; -import { PromptMetadataDiagnostic, PromptMetadataError, PromptMetadataWarning } from '../diagnostics.js'; -import { FrontMatterSequence } from '../../../codecs/base/frontMatterCodec/tokens/frontMatterSequence.js'; -import { FrontMatterArray, FrontMatterRecord, FrontMatterString, FrontMatterToken, FrontMatterValueToken } from '../../../codecs/base/frontMatterCodec/tokens/index.js'; -import { Range } from '../../../../../../../../editor/common/core/range.js'; - -/** - * Name of the metadata record in the prompt header. - */ -const RECORD_NAME = 'tools'; - -/** - * Prompt `tools` metadata record inside the prompt header. - */ -export class PromptToolsMetadata extends PromptMetadataRecord { - - /** - * List of all valid tool names that were found in - * this metadata record. - */ - public override get value(): string[] | undefined { - if (this.validToolNames === undefined) { - return []; - } - - return [...this.validToolNames.keys()]; - } - - public override get recordName(): string { - return RECORD_NAME; - } - - /** - * Value token reference of the record. - */ - protected valueToken: FrontMatterArray | undefined; - - /** - * List of all valid tool names that were found in - * this metadata record. - */ - private validToolNames: Map | undefined; - - - - constructor( - recordToken: FrontMatterRecord, - languageId: string, - ) { - super(RECORD_NAME, recordToken, languageId); - } - - /** - * Validate the metadata record and collect all issues - * related to its content. - */ - public override validate(): readonly PromptMetadataDiagnostic[] { - const { valueToken } = this.recordToken; - - // validate that the record value is an array - if ((valueToken instanceof FrontMatterArray) === false) { - this.issues.push( - new PromptMetadataError( - valueToken.range, - localize( - 'prompt.header.metadata.tools.diagnostics.invalid-value-type', - "Must be an array of tool names, got '{0}'.", - valueToken.valueTypeName.toString(), - ), - ), - ); - - delete this.valueToken; - return this.issues; - } - - this.valueToken = valueToken; - - // validate that all array items - this.validToolNames = new Map(); - for (const item of this.valueToken.items) { - this.issues.push( - ...this.validateToolName(item, this.validToolNames), - ); - } - - return this.issues; - } - - public getToolRange(toolName: string): Range | undefined { - return this.validToolNames?.get(toolName); - } - - /** - * Validate an individual provided value token that is used - * for a tool name. - */ - private validateToolName( - valueToken: FrontMatterValueToken, - validToolNames: Map, - ): readonly PromptMetadataDiagnostic[] { - const issues: PromptMetadataDiagnostic[] = []; - - // tool name must be a quoted or an unquoted 'string' - if ( - (valueToken instanceof FrontMatterString) === false && - (valueToken instanceof FrontMatterSequence) === false - ) { - issues.push( - new PromptMetadataWarning( - valueToken.range, - localize( - 'prompt.header.metadata.tools.diagnostics.invalid-tool-name-type', - "Unexpected tool name '{0}', expected a string literal.", - valueToken.text - ), - ), - ); - - return issues; - } - - const cleanToolName = valueToken.cleanText.trim(); - // the tool name should not be empty - if (cleanToolName.length === 0) { - issues.push( - new PromptMetadataWarning( - valueToken.range, - localize( - 'prompt.header.metadata.tools.diagnostics.empty-tool-name', - "Tool name cannot be empty.", - ), - ), - ); - - return issues; - } - - // the tool name should not be duplicated - if (validToolNames.has(cleanToolName)) { - issues.push( - new PromptMetadataWarning( - valueToken.range, - localize( - 'prompt.header.metadata.tools.diagnostics.duplicate-tool-name', - "Duplicate tool name '{0}'.", - cleanToolName, - ), - ), - ); - - return issues; - } - - validToolNames.set(cleanToolName, valueToken.range); - return issues; - } - - /** - * Check if a provided front matter token is a metadata record - * with name equal to `tools`. - */ - public static isToolsRecord( - token: FrontMatterToken, - ): boolean { - if ((token instanceof FrontMatterRecord) === false) { - return false; - } - - if (token.nameToken.text === RECORD_NAME) { - return true; - } - - return false; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/modeHeader.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/modeHeader.ts deleted file mode 100644 index 94307163bf6..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/modeHeader.ts +++ /dev/null @@ -1,56 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { HeaderBase, IHeaderMetadata, type TDehydrated } from './headerBase.js'; -import { PromptsType } from '../../promptTypes.js'; -import { FrontMatterRecord } from '../../codecs/base/frontMatterCodec/tokens/index.js'; -import { PromptModelMetadata } from './metadata/model.js'; -import { PromptToolsMetadata } from './metadata/tools.js'; - -/** - * Metadata utility object for mode files. - */ -interface IModeMetadata extends IHeaderMetadata { - /** - * Tools metadata in the mode header. - */ - tools: PromptToolsMetadata; - - /** - * Chat model metadata in the mode header. - */ - model: PromptModelMetadata; -} - -/** - * Metadata for mode files. - */ -export type TModeMetadata = Partial> & { promptType: PromptsType.mode }; - -/** - * Header object for mode files. - */ -export class ModeHeader extends HeaderBase { - protected override handleToken(token: FrontMatterRecord): boolean { - // if the record might be a "tools" metadata - // add it to the list of parsed metadata records - if (PromptToolsMetadata.isToolsRecord(token)) { - const metadata = new PromptToolsMetadata(token, this.languageId); - - this.issues.push(...metadata.validate()); - this.meta.tools = metadata; - return true; - } - if (PromptModelMetadata.isModelRecord(token)) { - const metadata = new PromptModelMetadata(token, this.languageId); - - this.issues.push(...metadata.validate()); - this.meta.model = metadata; - - return true; - } - return false; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/promptHeader.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/promptHeader.ts deleted file mode 100644 index 250e8382372..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/promptHeader.ts +++ /dev/null @@ -1,103 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { ChatModeKind } from '../../../constants.js'; -import { localize } from '../../../../../../../nls.js'; -import { PromptMetadataWarning } from './diagnostics.js'; -import { HeaderBase, IHeaderMetadata, type TDehydrated } from './headerBase.js'; -import { PromptsType } from '../../promptTypes.js'; -import { FrontMatterRecord } from '../../codecs/base/frontMatterCodec/tokens/index.js'; -import { PromptModelMetadata } from './metadata/model.js'; -import { PromptToolsMetadata } from './metadata/tools.js'; -import { PromptModeMetadata } from './metadata/mode.js'; - -/** - * Metadata utility object for prompt files. - */ -export interface IPromptMetadata extends IHeaderMetadata { - /** - * Tools metadata in the prompt header. - */ - tools: PromptToolsMetadata; - - /** - * Chat mode metadata in the prompt header. - */ - mode: PromptModeMetadata; - - /** - * Chat model metadata in the prompt header. - */ - model: PromptModelMetadata; -} - -/** - * Metadata for prompt files. - */ -export type TPromptMetadata = Partial> & { promptType: PromptsType.prompt }; - -/** - * Header object for prompt files. - */ -export class PromptHeader extends HeaderBase { - protected override handleToken(token: FrontMatterRecord): boolean { - // if the record might be a "tools" metadata - // add it to the list of parsed metadata records - if (PromptToolsMetadata.isToolsRecord(token)) { - const metadata = new PromptToolsMetadata(token, this.languageId); - - this.issues.push(...metadata.validate()); - this.meta.tools = metadata; - - this.validateToolsAndModeCompatibility(); - return true; - } - - // if the record might be a "mode" metadata - // add it to the list of parsed metadata records - if (PromptModeMetadata.isModeRecord(token)) { - const metadata = new PromptModeMetadata(token, this.languageId); - - this.issues.push(...metadata.validate()); - this.meta.mode = metadata; - - this.validateToolsAndModeCompatibility(); - return true; - } - - if (PromptModelMetadata.isModelRecord(token)) { - const metadata = new PromptModelMetadata(token, this.languageId); - - this.issues.push(...metadata.validate()); - this.meta.model = metadata; - - return true; - } - - return false; - } - - /** - * Validate that the `tools` and `mode` metadata are compatible - * with each other. If not, add a warning diagnostic. - */ - private validateToolsAndModeCompatibility(): void { - const { tools, mode } = this.meta; - const modeValue = mode?.value; - - if (tools !== undefined && (modeValue === ChatModeKind.Edit || modeValue === ChatModeKind.Ask)) { - this.issues.push( - new PromptMetadataWarning( - tools.range, - localize( - 'prompt.header.metadata.mode.diagnostics.incompatible-with-tools', - "Tools can not be used in '{0}' mode and will be ignored.", - modeValue - ), - ), - ); - } - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptParser.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptParser.ts deleted file mode 100644 index 99df26fd5e9..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptParser.ts +++ /dev/null @@ -1,72 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { URI } from '../../../../../../base/common/uri.js'; -import { IPromptContentsProvider } from '../contentProviders/types.js'; -import { ILogService } from '../../../../../../platform/log/common/log.js'; -import { BasePromptParser, IPromptParserOptions } from './basePromptParser.js'; -import { IModelService } from '../../../../../../editor/common/services/model.js'; -import { TextModelContentsProvider } from '../contentProviders/textModelContentsProvider.js'; -import { FilePromptContentProvider } from '../contentProviders/filePromptContentsProvider.js'; -import { IInstantiationService } from '../../../../../../platform/instantiation/common/instantiation.js'; -import { IPromptContentsProviderOptions } from '../contentProviders/promptContentsProviderBase.js'; -import { IWorkbenchEnvironmentService } from '../../../../../services/environment/common/environmentService.js'; - -/** - * Get prompt contents provider object based on the prompt type. - */ -function getContentsProvider( - uri: URI, - options: IPromptContentsProviderOptions, - modelService: IModelService, - instaService: IInstantiationService -): IPromptContentsProvider { - const model = modelService.getModel(uri); - if (model) { - return instaService.createInstance(TextModelContentsProvider, model, options); - } - return instaService.createInstance(FilePromptContentProvider, uri, options); -} - -/** - * General prompt parser class that automatically infers a prompt - * contents provider type by the type of provided prompt URI. - */ -export class PromptParser extends BasePromptParser { - /** - * Underlying prompt contents provider instance. - */ - private readonly contentsProvider: IPromptContentsProvider; - - constructor( - uri: URI, - options: IPromptParserOptions, - @ILogService logService: ILogService, - @IModelService modelService: IModelService, - @IInstantiationService instaService: IInstantiationService, - @IWorkbenchEnvironmentService envService: IWorkbenchEnvironmentService, - ) { - const contentsProvider = getContentsProvider(uri, options, modelService, instaService); - - super( - contentsProvider, - options, - instaService, - envService, - logService, - ); - - this.contentsProvider = this._register(contentsProvider); - } - - /** - * Returns a string representation of this object. - */ - public override toString(): string { - const { sourceName } = this.contentsProvider; - - return `prompt-parser:${sourceName}:${this.uri.path}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/textModelPromptParser.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/textModelPromptParser.ts deleted file mode 100644 index bb2b7060abb..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/textModelPromptParser.ts +++ /dev/null @@ -1,42 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { ITextModel } from '../../../../../../editor/common/model.js'; -import { ILogService } from '../../../../../../platform/log/common/log.js'; -import { BasePromptParser, IPromptParserOptions } from './basePromptParser.js'; -import { TextModelContentsProvider } from '../contentProviders/textModelContentsProvider.js'; -import { IInstantiationService } from '../../../../../../platform/instantiation/common/instantiation.js'; -import { IWorkbenchEnvironmentService } from '../../../../../services/environment/common/environmentService.js'; - -/** - * Class capable of parsing prompt syntax out of a provided text model, - * including all the nested child file references it may have. - */ -export class TextModelPromptParser extends BasePromptParser { - constructor( - model: ITextModel, - options: IPromptParserOptions, - @IInstantiationService instantiationService: IInstantiationService, - @IWorkbenchEnvironmentService envService: IWorkbenchEnvironmentService, - @ILogService logService: ILogService, - ) { - const contentsProvider = instantiationService.createInstance( - TextModelContentsProvider, - model, - options, - ); - - super(contentsProvider, options, instantiationService, envService, logService); - - this._register(contentsProvider); - } - - /** - * Returns a string representation of this object. - */ - public override toString(): string { - return `text-model-prompt:${this.uri.path}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/topError.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/topError.ts deleted file mode 100644 index b6281d3f42b..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/topError.ts +++ /dev/null @@ -1,102 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { ITopError } from './types.js'; -import { localize } from '../../../../../../nls.js'; -import { assert } from '../../../../../../base/common/assert.js'; -import { assertDefined } from '../../../../../../base/common/types.js'; -import { OpenFailed, RecursiveReference, FailedToResolveContentsStream } from '../../promptFileReferenceErrors.js'; - -/** - * The top-most error of the reference tree. - */ -export class TopError implements ITopError { - public readonly originalError: ITopError['originalError']; - public readonly errorSubject: ITopError['errorSubject']; - public readonly errorsCount: ITopError['errorsCount']; - public readonly parentUri: ITopError['parentUri']; - - constructor( - options: Omit, - ) { - this.originalError = options.originalError; - this.errorSubject = options.errorSubject; - this.errorsCount = options.errorsCount; - this.parentUri = options.parentUri; - } - - public get localizedMessage(): string { - const { originalError, parentUri, errorSubject: subject, errorsCount } = this; - - assert( - errorsCount >= 1, - `Error count must be at least 1, got '${errorsCount}'.`, - ); - - // a note about how many more link issues are there - const moreIssuesLabel = (errorsCount > 1) - ? localize('workbench.reusable-prompts.top-error.more-issues-label', "\n(+{0} more issues)", errorsCount - 1) - : ''; - - if (subject === 'root') { - if (originalError instanceof OpenFailed) { - return localize( - 'workbench.reusable-prompts.top-error.open-failed', - "Cannot open '{0}'.{1}", - originalError.uri.path, - moreIssuesLabel, - ); - } - - if (originalError instanceof FailedToResolveContentsStream) { - return localize( - 'workbench.reusable-prompts.top-error.cannot-read', - "Cannot read '{0}'.{1}", - originalError.uri.path, - moreIssuesLabel, - ); - } - - if (originalError instanceof RecursiveReference) { - return localize( - 'workbench.reusable-prompts.top-error.recursive-reference', - "Recursion to itself.", - ); - } - - return originalError.message + moreIssuesLabel; - } - - // a sanity check - because the error subject is not `root`, the parent must set - assertDefined( - parentUri, - 'Parent URI must be defined for error of non-root link.', - ); - - const errorMessageStart = (subject === 'child') - ? localize( - 'workbench.reusable-prompts.top-error.child.direct', - "Contains", - ) - : localize( - 'workbench.reusable-prompts.top-error.child.indirect', - "Indirectly referenced prompt '{0}' contains", - parentUri.path, - ); - - const linkIssueName = (originalError instanceof RecursiveReference) - ? localize('recursive', "recursive") - : localize('broken', "broken"); - - return localize( - 'workbench.reusable-prompts.top-error.child.final-message', - "{0} a {1} link to '{2}' that will be ignored.{3}", - errorMessageStart, - linkIssueName, - originalError.uri.path, - moreIssuesLabel, - ); - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/types.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/types.ts deleted file mode 100644 index 0a3d693c61d..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/types.ts +++ /dev/null @@ -1,116 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { URI } from '../../../../../../base/common/uri.js'; -import { ResolveError } from '../../promptFileReferenceErrors.js'; -import { IRange, Range } from '../../../../../../editor/common/core/range.js'; - -/** - * A resolve error with a parent prompt URI, if any. - */ -export interface IResolveError { - /** - * Original error instance. - */ - readonly originalError: ResolveError; - - /** - * URI of the parent that references this error. - */ - readonly parentUri?: URI; -} - -/** - * Top most error of the reference tree. - */ -export interface ITopError extends IResolveError { - /** - * Where does the error belong to: - * - * - `root` - the error is the top most error of the entire tree - * - `child` - the error is a child of the root error - * - `indirect-child` - the error is a child of a child of the root error - */ - readonly errorSubject: 'root' | 'child' | 'indirect-child'; - - /** - * Total number of all errors in the references tree, including the error - * of the current reference and all possible errors of its children. - */ - readonly errorsCount: number; - - /** - * Localized error message. - */ - readonly localizedMessage: string; -} - -/** - * Base interface for a generic prompt reference. - */ -interface IPromptReferenceBase { - /** - * Type of the prompt reference. E.g., `file`, `http`, `image`, etc. - */ - readonly type: string; - - /** - * Subtype of the prompt reference. For instance a `file` reference - * can be a `markdown link` or a prompt `#file:` variable reference. - */ - readonly subtype: string; - - /** - * URI component of the associated with this reference. - */ - readonly uri: URI; - - /** - * The full range of the prompt reference in the source text, - * including the {@link linkRange} and any additional - * parts the reference may contain (e.g., the `#file:` prefix). - */ - readonly range: Range; - - /** - * Range of the link part that the reference points to. - */ - readonly linkRange: IRange | undefined; - - /** - * Text of the reference as it appears in the source. - */ - readonly text: string; - - /** - * Original link path as it appears in the source. - */ - readonly path: string; - -} - -/** - * The special case of the {@link IPromptReferenceBase} that pertains - * to a file resource on the disk. - */ -export interface IPromptFileReference extends IPromptReferenceBase { - readonly type: 'file'; - - /** - * Subtype of a file reference, - either a prompt `#file` variable, - * or a `markdown link` (e.g., `[caption](/path/to/file.md)`). - */ - readonly subtype: 'prompt' | 'markdown'; -} - -/** - * List of all known prompt reference types. - */ -export type TPromptReference = IPromptFileReference; - -export type TVariableReference = { - readonly name: string; - readonly range: Range; -}; diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsService.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsService.ts index ab3b4887aa4..f1eef78e834 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsService.ts +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsService.ts @@ -6,10 +6,8 @@ import { ChatModeKind } from '../../constants.js'; import { URI } from '../../../../../../base/common/uri.js'; import { Event } from '../../../../../../base/common/event.js'; -import { TMetadata } from '../parsers/promptHeader/headerBase.js'; import { ITextModel } from '../../../../../../editor/common/model.js'; import { IDisposable } from '../../../../../../base/common/lifecycle.js'; -import { TextModelPromptParser } from '../parsers/textModelPromptParser.js'; import { CancellationToken } from '../../../../../../base/common/cancellation.js'; import { PromptsType } from '../promptTypes.js'; import { createDecorator } from '../../../../../../platform/instantiation/common/instantiation.js'; @@ -48,32 +46,6 @@ export interface IPromptPath { readonly type: PromptsType; } -/** - * Type for a shared prompt parser instance returned by the {@link IPromptsService}. - * Because the parser is shared, we omit the `dispose` method from - * the original type so the caller cannot dispose it prematurely - */ -export type TSharedPrompt = Omit; - -/** - * Metadata node object in a hierarchical tree of prompt references. - */ -export interface IMetadata { - /** - * URI of a prompt file. - */ - readonly uri: URI; - - /** - * Metadata of the prompt file. - */ - readonly metadata: TMetadata | null; - - /** - * List of metadata for each valid child prompt reference. - */ - readonly children?: readonly IMetadata[]; -} export interface ICustomChatMode { /** @@ -159,12 +131,6 @@ export type TCombinedToolsMetadata = ICombinedAgentToolsMetadata | ICombinedNonA export interface IPromptsService extends IDisposable { readonly _serviceBrand: undefined; - /** - * Get a prompt syntax parser for the provided text model. - * See {@link TextModelPromptParser} for more info on the parser API. - */ - getSyntaxParserFor(model: ITextModel): TSharedPrompt & { isDisposed: false }; - /** * The parsed prompt file for the provided text model. * @param textModel Returns the parsed prompt file. @@ -207,12 +173,6 @@ export interface IPromptsService extends IDisposable { */ getCustomChatModes(token: CancellationToken): Promise; - /** - * Parses the provided URI - * @param uris - */ - parse(uri: URI, type: PromptsType, token: CancellationToken): Promise; - /** * Parses the provided URI * @param uris @@ -232,15 +192,6 @@ export interface IChatPromptSlashCommand { readonly promptPath?: IPromptPath; } - -export interface IPromptParserResult { - readonly uri: URI; - readonly metadata: TMetadata | null; - readonly fileReferences: readonly URI[]; - readonly variableReferences: readonly IVariableReference[]; - readonly header?: IPromptHeader; -} - export interface IPromptHeader { readonly node: YamlNode | undefined; readonly errors: YamlParseError[]; diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts index 07579a4d003..50032a655c1 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts @@ -4,29 +4,24 @@ *--------------------------------------------------------------------------------------------*/ import { localize } from '../../../../../../nls.js'; -import { getLanguageIdForPromptsType, getPromptsTypeForLanguageId, PROMPT_LANGUAGE_ID, PromptsType } from '../promptTypes.js'; -import { PromptParser } from '../parsers/promptParser.js'; +import { getPromptsTypeForLanguageId, PROMPT_LANGUAGE_ID, PromptsType } from '../promptTypes.js'; import { type URI } from '../../../../../../base/common/uri.js'; -import { assert } from '../../../../../../base/common/assert.js'; import { basename } from '../../../../../../base/common/path.js'; import { PromptFilesLocator } from '../utils/promptFilesLocator.js'; import { Disposable } from '../../../../../../base/common/lifecycle.js'; import { Event } from '../../../../../../base/common/event.js'; import { type ITextModel } from '../../../../../../editor/common/model.js'; -import { ObjectCache } from '../utils/objectCache.js'; import { ILogService } from '../../../../../../platform/log/common/log.js'; -import { TextModelPromptParser } from '../parsers/textModelPromptParser.js'; import { ILabelService } from '../../../../../../platform/label/common/label.js'; import { IModelService } from '../../../../../../editor/common/services/model.js'; import { CancellationToken } from '../../../../../../base/common/cancellation.js'; import { IInstantiationService } from '../../../../../../platform/instantiation/common/instantiation.js'; import { IUserDataProfileService } from '../../../../../services/userDataProfile/common/userDataProfile.js'; -import type { IChatPromptSlashCommand, ICustomChatMode, IPromptParserResult, IPromptPath, IPromptsService, TPromptsStorage } from './promptsService.js'; +import type { IChatPromptSlashCommand, ICustomChatMode, IPromptPath, IPromptsService, TPromptsStorage } from './promptsService.js'; import { getCleanPromptName, PROMPT_FILE_EXTENSION } from '../config/promptFileLocations.js'; import { ILanguageService } from '../../../../../../editor/common/languages/language.js'; import { PromptsConfig } from '../config/config.js'; import { IConfigurationService } from '../../../../../../platform/configuration/common/configuration.js'; -import { PositionOffsetTransformer } from '../../../../../../editor/common/core/text/positionToOffset.js'; import { NewPromptsParser, ParsedPromptFile } from './newPromptsParser.js'; import { IFileService } from '../../../../../../platform/files/common/files.js'; import { ResourceMap } from '../../../../../../base/common/map.js'; @@ -40,11 +35,6 @@ import { IVariableReference } from '../../chatModes.js'; export class PromptsService extends Disposable implements IPromptsService { public declare readonly _serviceBrand: undefined; - /** - * Cache of text model content prompt parsers. - */ - private readonly cache: ObjectCache; - /** * Prompt files locator utility. */ @@ -77,36 +67,6 @@ export class PromptsService extends Disposable implements IPromptsService { this.fileLocator = this._register(this.instantiationService.createInstance(PromptFilesLocator)); - // the factory function below creates a new prompt parser object - // for the provided model, if no active non-disposed parser exists - this.cache = this._register( - new ObjectCache((model) => { - assert( - model.isDisposed() === false, - 'Text model must not be disposed.', - ); - - /** - * Note! When/if shared with "file" prompts, the `seenReferences` array below must be taken into account. - * Otherwise consumers will either see incorrect failing or incorrect successful results, based on their - * use case, timing of their calls to the {@link getSyntaxParserFor} function, and state of this service. - */ - const parser: TextModelPromptParser = instantiationService.createInstance( - TextModelPromptParser, - model, - { allowNonPromptFiles: true, languageId: undefined, updateOnChange: true }, - ).start(); - - // this is a sanity check and the contract of the object cache, - // we must return a non-disposed object from this factory function - parser.assertNotDisposed( - 'Created prompt parser must not be disposed.', - ); - - return parser; - }) - ); - this._register(this.modelService.onModelRemoved((model) => { this.parsedPromptFileCache.delete(model.uri); })); @@ -132,21 +92,6 @@ export class PromptsService extends Disposable implements IPromptsService { } - /** - * @throws {Error} if: - * - the provided model is disposed - * - newly created parser is disposed immediately on initialization. - * See factory function in the {@link constructor} for more info. - */ - public getSyntaxParserFor(model: ITextModel): TextModelPromptParser & { isDisposed: false } { - assert( - model.isDisposed() === false, - 'Cannot create a prompt syntax parser for a disposed model.', - ); - - return this.cache.get(model); - } - public getParsedPromptFile(textModel: ITextModel): ParsedPromptFile { const cached = this.parsedPromptFileCache.get(textModel.uri); if (cached && cached[0] === textModel.getVersionId()) { @@ -255,7 +200,7 @@ export class PromptsService extends Disposable implements IPromptsService { private async computeCustomChatModes(token: CancellationToken): Promise { const modeFiles = await this.listPromptFiles(PromptsType.mode, token); - const metadataList = await Promise.all( + const customChatModes = await Promise.all( modeFiles.map(async ({ uri }): Promise => { const ast = await this.parseNew(uri, token); @@ -281,37 +226,7 @@ export class PromptsService extends Disposable implements IPromptsService { }) ); - - return metadataList; - } - - public async parse(uri: URI, type: PromptsType, token: CancellationToken): Promise { - let parser: PromptParser | undefined; - try { - const languageId = getLanguageIdForPromptsType(type); - parser = this.instantiationService.createInstance(PromptParser, uri, { allowNonPromptFiles: true, languageId, updateOnChange: false }).start(token); - const completed = await parser.settled(); - if (!completed) { - throw new Error(localize('promptParser.notCompleted', "Prompt parser for {0} did not complete.", uri.toString())); - } - const fullContent = await parser.getFullContent(); - const transformer = new PositionOffsetTransformer(fullContent); - const variableReferences = parser.variableReferences.map(ref => { - return { - name: ref.name, - range: transformer.getOffsetRange(ref.range) - }; - }).sort((a, b) => b.range.start - a.range.start); // in reverse order - // make a copy, to avoid leaking the parser instance - return { - uri: parser.uri, - metadata: parser.metadata, - variableReferences, - fileReferences: parser.references.map(ref => ref.uri), - }; - } finally { - parser?.dispose(); - } + return customChatModes; } public async parseNew(uri: URI, token: CancellationToken): Promise { diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/utils/objectCache.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/utils/objectCache.ts deleted file mode 100644 index 443d6e44c94..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/utils/objectCache.ts +++ /dev/null @@ -1,153 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { Disposable, DisposableMap } from '../../../../../../base/common/lifecycle.js'; -import { ObservableDisposable, assertNotDisposed } from './observableDisposable.js'; - -/** - * Generic cache for object instances. Guarantees to return only non-disposed - * objects from the {@linkcode get} method. If a requested object is not yet - * in the cache or is disposed already, the {@linkcode factory} callback is - * called to create a new object. - * - * @throws if {@linkcode factory} callback returns a disposed object. - * - * ## Examples - * - * ```typescript - * // a class that will be used as a cache key; the key can be of any - * // non-nullable type, including primitives like `string` or `number`, - * // but in this case we use an object pointer as a key - * class KeyObject {} - * - * // a class for testing purposes - * class TestObject extends ObservableDisposable { - * constructor( - * public readonly id: KeyObject, - * ) {} - * }; - * - * // create an object cache instance providing it a factory function that - * // is responsible for creating new objects based on the provided key if - * // the cache does not contain the requested object yet or an existing - * // object is already disposed - * const cache = new ObjectCache((key) => { - * // create a new test object based on the provided key - * return new TestObject(key); - * }); - * - * // create two keys - * const key1 = new KeyObject(); - * const key2 = new KeyObject(); - * - * // get an object from the cache by its key - * const object1 = cache.get(key1); // returns a new test object - * - * // validate that the new object has the correct key - * assert( - * object1.id === key1, - * 'Object 1 must have correct ID.', - * ); - * - * // returns the same cached test object - * const object2 = cache.get(key1); - * - * // validate that the same exact object is returned from the cache - * assert( - * object1 === object2, - * 'Object 2 the same cached object as object 1.', - * ); - * - * // returns a new test object - * const object3 = cache.get(key2); - * - * // validate that the new object has the correct key - * assert( - * object3.id === key2, - * 'Object 3 must have correct ID.', - * ); - * - * assert( - * object3 !== object1, - * 'Object 3 must be a new object.', - * ); - * ``` - */ -export class ObjectCache< - TValue extends ObservableDisposable, - TKey extends NonNullable = string, -> extends Disposable { - private readonly cache: DisposableMap = - this._register(new DisposableMap()); - - constructor( - private readonly factory: (key: TKey) => TValue & { isDisposed: false }, - ) { - super(); - } - - /** - * Get an existing object from the cache. If a requested object is not yet - * in the cache or is disposed already, the {@linkcode factory} callback is - * called to create a new object. - * - * @throws if {@linkcode factory} callback returns a disposed object. - * @param key - ID of the object in the cache - */ - public get(key: TKey): TValue & { isDisposed: false } { - let object = this.cache.get(key); - - // if object is already disposed, remove it from the cache - if (object?.isDisposed) { - this.cache.deleteAndLeak(key); - object = undefined; - } - - // if object exists and is not disposed, return it - if (object) { - // must always hold true due to the check above - assertNotDisposed( - object, - 'Object must not be disposed.', - ); - - return object; - } - - // create a new object by calling the factory - object = this.factory(key); - - // newly created object must not be disposed - assertNotDisposed( - object, - 'Newly created object must not be disposed.', - ); - - // remove it from the cache automatically on dispose - object.addDisposables( - object.onDispose(() => { - this.cache.deleteAndLeak(key); - })); - this.cache.set(key, object); - - return object; - } - - /** - * Remove an object from the cache by its key. - * - * @param key ID of the object to remove. - * @param dispose Whether the removed object must be disposed. - */ - public remove(key: TKey, dispose: boolean): this { - if (dispose) { - this.cache.deleteAndDispose(key); - return this; - } - - this.cache.deleteAndLeak(key); - return this; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/utils/observableDisposable.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/utils/observableDisposable.ts deleted file mode 100644 index a0e6a70e733..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/utils/observableDisposable.ts +++ /dev/null @@ -1,89 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { Disposable, DisposableStore, IDisposable, toDisposable } from '../../../../../../base/common/lifecycle.js'; - -/** -* @deprecated do not use this, https://github.com/microsoft/vscode/issues/248366 - */ -export abstract class ObservableDisposable extends Disposable { - /** - * Underlying disposables store this object relies on. - */ - private readonly store = this._register(new DisposableStore()); - - /** - * Check if the current object is already has been disposed. - */ - public get isDisposed(): boolean { - return this.store.isDisposed; - } - - /** - * The event is fired when this object is disposed. - * Note! Executes the callback immediately if already disposed. - * - * @param callback The callback function to be called on updates. - */ - public onDispose(callback: () => void): IDisposable { - // if already disposed, execute the callback immediately - if (this.isDisposed) { - const timeoutHandle = setTimeout(callback); - - return toDisposable(() => { - clearTimeout(timeoutHandle); - }); - } - - return this.store.add(toDisposable(callback)); - } - - /** - * Adds disposable object(s) to the list of disposables - * that will be disposed with this object. - */ - public addDisposables(...disposables: IDisposable[]): this { - for (const disposable of disposables) { - this.store.add(disposable); - } - - return this; - } - - /** - * Assert that the current object was not yet disposed. - * - * @throws If the current object was already disposed. - * @param error Error message or error object to throw if assertion fails. - */ - public assertNotDisposed( - error: string | Error, - ): asserts this is TNotDisposed { - assertNotDisposed(this, error); - } -} - -/** - * @deprecated do not use this, https://github.com/microsoft/vscode/issues/248366 - */ -type TNotDisposed = TObject & { isDisposed: false }; - -/** - * @deprecated do not use this, https://github.com/microsoft/vscode/issues/248366 - */ -export function assertNotDisposed( - object: TObject, - error: string | Error, -): asserts object is TNotDisposed { - if (!object.isDisposed) { - return; - } - - const errorToThrow = typeof error === 'string' - ? new Error(error) - : error; - - throw errorToThrow; -} diff --git a/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/frontMatterDecoder/frontMatterBoolean.test.ts b/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/frontMatterDecoder/frontMatterBoolean.test.ts deleted file mode 100644 index 8d32c30d851..00000000000 --- a/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/frontMatterDecoder/frontMatterBoolean.test.ts +++ /dev/null @@ -1,317 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import assert from 'assert'; -import { Range } from '../../../../../../../../../editor/common/core/range.js'; -import { Word } from '../../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/tokens.js'; -import { randomBoolean } from '../../../../../../../../../base/test/common/testUtils.js'; -import { FrontMatterBoolean } from '../../../../../../common/promptSyntax/codecs/base/frontMatterCodec/tokens/index.js'; -import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../../../../../../base/test/common/utils.js'; -import { FrontMatterSequence } from '../../../../../../common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterSequence.js'; - -suite('FrontMatterBoolean', () => { - ensureNoDisposablesAreLeakedInTestSuite(); - - suite('equals()', () => { - suite('base case', () => { - test('true', () => { - // both values should yield the same result - const booleanText = (randomBoolean()) - ? 'true' - : 'TRUE'; - - const boolean = new FrontMatterBoolean( - new Word( - new Range(1, 1, 1, 5), - booleanText, - ), - ); - - const other = new FrontMatterBoolean( - new Word( - new Range(1, 1, 1, 5), - booleanText, - ), - ); - - assert.strictEqual( - boolean.value, - true, - 'Must have correct boolean value.', - ); - - assert( - boolean.equals(other), - 'Booleans must be equal.', - ); - }); - - test('false', () => { - // both values should yield the same result - const booleanText = (randomBoolean()) - ? 'false' - : 'FALSE'; - - const boolean = new FrontMatterBoolean( - new Word( - new Range(5, 15, 5, 15 + 6), - booleanText, - ), - ); - - const other = new FrontMatterBoolean( - new Word( - new Range(5, 15, 5, 15 + 6), - booleanText, - ), - ); - - assert.strictEqual( - boolean.value, - false, - 'Must have correct boolean value.', - ); - - assert( - boolean.equals(other), - 'Booleans must be equal.', - ); - }); - }); - - suite('non-boolean token', () => { - suite('word token', () => { - test('true', () => { - // both values should yield the same result - const booleanText = (randomBoolean()) - ? 'true' - : 'TRUE'; - - const boolean = new FrontMatterBoolean( - new Word( - new Range(1, 1, 1, 5), - booleanText, - ), - ); - - const other = new Word( - new Range(1, 1, 1, 5), - booleanText, - ); - - assert( - boolean.equals(other) === false, - 'Booleans must not be equal.', - ); - }); - - test('false', () => { - // both values should yield the same result - const booleanText = (randomBoolean()) - ? 'false' - : 'FALSE'; - - const boolean = new FrontMatterBoolean( - new Word( - new Range(1, 2, 1, 2 + 6), - booleanText, - ), - ); - - const other = new Word( - new Range(1, 2, 1, 2 + 6), - booleanText, - ); - - assert( - boolean.equals(other) === false, - 'Booleans must not be equal.', - ); - }); - }); - - suite('sequence token', () => { - test('true', () => { - // both values should yield the same result - const booleanText = (randomBoolean()) - ? 'true' - : 'TRUE'; - - const boolean = new FrontMatterBoolean( - new Word( - new Range(1, 1, 1, 5), - booleanText, - ), - ); - - const other = new FrontMatterSequence([ - new Word( - new Range(1, 1, 1, 5), - booleanText, - ), - ]); - - assert( - boolean.equals(other) === false, - 'Booleans must not be equal.', - ); - }); - - test('false', () => { - // both values should yield the same result - const booleanText = (randomBoolean()) - ? 'false' - : 'FALSE'; - - const boolean = new FrontMatterBoolean( - new Word( - new Range(1, 2, 1, 2 + 6), - booleanText, - ), - ); - - const other = new FrontMatterSequence([ - new Word( - new Range(1, 2, 1, 2 + 6), - booleanText, - ), - ]); - - assert( - boolean.equals(other) === false, - 'Booleans must not be equal.', - ); - }); - }); - }); - - suite('different range', () => { - test('true', () => { - // both values should yield the same result - const booleanText = (randomBoolean()) - ? 'true' - : 'TRUE'; - - const boolean = new FrontMatterBoolean( - new Word( - new Range(1, 2, 1, 2 + 4), - booleanText, - ), - ); - - const other = new FrontMatterBoolean( - new Word( - new Range(3, 2, 3, 2 + 4), - booleanText, - ), - ); - - assert( - boolean.equals(other) === false, - 'Booleans must not be equal.', - ); - }); - - test('false', () => { - // both values should yield the same result - const booleanText = (randomBoolean()) - ? 'false' - : 'FALSE'; - - const boolean = new FrontMatterBoolean( - new Word( - new Range(5, 15, 5, 15 + 5), - booleanText, - ), - ); - - const other = new FrontMatterBoolean( - new Word( - new Range(4, 15, 4, 15 + 5), - booleanText, - ), - ); - - assert( - boolean.equals(other) === false, - 'Booleans must not be equal.', - ); - }); - }); - - suite('different text', () => { - test('true', () => { - const boolean = new FrontMatterBoolean( - new Word( - new Range(1, 1, 1, 5), - 'true', - ), - ); - - const other = new FrontMatterBoolean( - new Word( - new Range(1, 1, 1, 5), - 'True', - ), - ); - - assert( - boolean.equals(other) === false, - 'Booleans must not be equal.', - ); - }); - - test('false', () => { - const boolean = new FrontMatterBoolean( - new Word( - new Range(5, 15, 5, 15 + 6), - 'FALSE', - ), - ); - - const other = new FrontMatterBoolean( - new Word( - new Range(5, 15, 5, 15 + 6), - 'false', - ), - ); - - assert( - boolean.equals(other) === false, - 'Booleans must not be equal.', - ); - }); - }); - - test('throws if cannot be converted to a boolean', () => { - assert.throws(() => { - new FrontMatterBoolean( - new Word( - new Range(1, 1, 1, 5), - 'true1', - ), - ); - }); - - assert.throws(() => { - new FrontMatterBoolean( - new Word( - new Range(2, 5, 2, 5 + 6), - 'fal se', - ), - ); - }); - - assert.throws(() => { - new FrontMatterBoolean( - new Word( - new Range(20, 4, 20, 4 + 1), - '1', - ), - ); - }); - }); - }); -}); diff --git a/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/frontMatterDecoder/frontMatterDecoder.test.ts b/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/frontMatterDecoder/frontMatterDecoder.test.ts deleted file mode 100644 index b4f78b9d19b..00000000000 --- a/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/frontMatterDecoder/frontMatterDecoder.test.ts +++ /dev/null @@ -1,415 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - - -import { Range } from '../../../../../../../../../editor/common/core/range.js'; -import { TestDecoder } from '../utils/testDecoder.js'; -import { VSBuffer } from '../../../../../../../../../base/common/buffer.js'; -import { newWriteableStream } from '../../../../../../../../../base/common/stream.js'; -import { NewLine } from '../../../../../../common/promptSyntax/codecs/base/linesCodec/tokens/newLine.js'; -import { DoubleQuote } from '../../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/doubleQuote.js'; -import { type TSimpleDecoderToken } from '../../../../../../common/promptSyntax/codecs/base/simpleCodec/simpleDecoder.js'; -import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../../../../../../base/test/common/utils.js'; -import { LeftBracket, RightBracket } from '../../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/brackets.js'; -import { FrontMatterDecoder } from '../../../../../../common/promptSyntax/codecs/base/frontMatterCodec/frontMatterDecoder.js'; -import { FrontMatterSequence } from '../../../../../../common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterSequence.js'; -import { ExclamationMark, Quote, Tab, Word, Space, Colon, VerticalTab, Comma, Dash } from '../../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/tokens.js'; -import { FrontMatterBoolean, FrontMatterString, FrontMatterArray, FrontMatterRecord, FrontMatterRecordDelimiter, FrontMatterRecordName } from '../../../../../../common/promptSyntax/codecs/base/frontMatterCodec/tokens/index.js'; - -/** - * Front Matter decoder for testing purposes. - */ -export class TestFrontMatterDecoder extends TestDecoder { - constructor() { - const stream = newWriteableStream(null); - const decoder = new FrontMatterDecoder(stream); - - super(stream, decoder); - } -} - -suite('FrontMatterDecoder', () => { - const disposables = ensureNoDisposablesAreLeakedInTestSuite(); - - test('produces expected tokens', async () => { - const test = disposables.add(new TestFrontMatterDecoder()); - - await test.run( - [ - 'just: "write some yaml "', - 'write-some :\t[ \' just\t \', "yaml!", true, , ,]', - 'anotherField \t\t\t : FALSE ', - ], - [ - // first record - new FrontMatterRecord([ - new FrontMatterRecordName([ - new Word(new Range(1, 1, 1, 1 + 4), 'just'), - ]), - new FrontMatterRecordDelimiter([ - new Colon(new Range(1, 5, 1, 6)), - new Space(new Range(1, 6, 1, 7)), - ]), - new FrontMatterString([ - new DoubleQuote(new Range(1, 7, 1, 8)), - new Word(new Range(1, 8, 1, 8 + 5), 'write'), - new Space(new Range(1, 13, 1, 14)), - new Word(new Range(1, 14, 1, 14 + 4), 'some'), - new Space(new Range(1, 18, 1, 19)), - new Word(new Range(1, 19, 1, 19 + 4), 'yaml'), - new Space(new Range(1, 23, 1, 24)), - new DoubleQuote(new Range(1, 24, 1, 25)), - ]), - ]), - new NewLine(new Range(1, 25, 1, 26)), - // second record - new FrontMatterRecord([ - new FrontMatterRecordName([ - new Word(new Range(2, 1, 2, 1 + 5), 'write'), - new Dash(new Range(2, 6, 2, 7)), - new Word(new Range(2, 7, 2, 7 + 4), 'some'), - ]), - new FrontMatterRecordDelimiter([ - new Colon(new Range(2, 12, 2, 13)), - new Tab(new Range(2, 13, 2, 14)), - ]), - new FrontMatterArray([ - new LeftBracket(new Range(2, 14, 2, 15)), - new FrontMatterString([ - new Quote(new Range(2, 16, 2, 17)), - new Space(new Range(2, 17, 2, 18)), - new Word(new Range(2, 18, 2, 18 + 4), 'just'), - new Tab(new Range(2, 22, 2, 23)), - new Space(new Range(2, 23, 2, 24)), - new Quote(new Range(2, 24, 2, 25)), - ]), - new FrontMatterString([ - new DoubleQuote(new Range(2, 28, 2, 29)), - new Word(new Range(2, 29, 2, 29 + 4), 'yaml'), - new ExclamationMark(new Range(2, 33, 2, 34)), - new DoubleQuote(new Range(2, 34, 2, 35)), - ]), - new FrontMatterBoolean( - new Word(new Range(2, 37, 2, 37 + 4), 'true'), - ), - new RightBracket(new Range(2, 46, 2, 47)), - ]), - ]), - new NewLine(new Range(2, 47, 2, 48)), - // third record - new FrontMatterRecord([ - new FrontMatterRecordName([ - new Word(new Range(3, 1, 3, 1 + 12), 'anotherField'), - ]), - new FrontMatterRecordDelimiter([ - new Colon(new Range(3, 19, 3, 20)), - new Space(new Range(3, 20, 3, 21)), - ]), - new FrontMatterBoolean( - new Word(new Range(3, 22, 3, 22 + 5), 'FALSE'), - ), - ]), - new Space(new Range(3, 27, 3, 28)), - ]); - }); - - suite('record', () => { - suite('values', () => { - test('unquoted string', async () => { - const test = disposables.add(new TestFrontMatterDecoder()); - - await test.run( - [ - 'just: write some yaml ', - 'anotherField \t\t : fal\v \t', - ], - [ - // first record - new FrontMatterRecord([ - new FrontMatterRecordName([ - new Word(new Range(1, 1, 1, 1 + 4), 'just'), - ]), - new FrontMatterRecordDelimiter([ - new Colon(new Range(1, 5, 1, 6)), - new Space(new Range(1, 6, 1, 7)), - ]), - new FrontMatterSequence([ - new Word(new Range(1, 7, 1, 7 + 5), 'write'), - new Space(new Range(1, 12, 1, 13)), - new Word(new Range(1, 13, 1, 13 + 4), 'some'), - new Space(new Range(1, 17, 1, 18)), - new Word(new Range(1, 18, 1, 18 + 4), 'yaml'), - ]), - ]), - new Space(new Range(1, 22, 1, 23)), - new NewLine(new Range(1, 23, 1, 24)), - // second record - new FrontMatterRecord([ - new FrontMatterRecordName([ - new Word(new Range(2, 1, 2, 1 + 12), 'anotherField'), - ]), - new FrontMatterRecordDelimiter([ - new Colon(new Range(2, 17, 2, 18)), - new Space(new Range(2, 18, 2, 19)), - ]), - new FrontMatterSequence([ - new Word(new Range(2, 20, 2, 20 + 3), 'fal'), - ]), - ]), - new VerticalTab(new Range(2, 23, 2, 24)), - new Space(new Range(2, 24, 2, 25)), - new Tab(new Range(2, 25, 2, 26)), - ]); - }); - - test('quoted string', async () => { - const test = disposables.add(new TestFrontMatterDecoder()); - - await test.run( - [ - `just\t:\t'\vdo\tsome\ntesting, please\v' `, - 'anotherField \t\t :\v\v"fal\nse"', - ], - [ - // first record - new FrontMatterRecord([ - new FrontMatterRecordName([ - new Word(new Range(1, 1, 1, 1 + 4), 'just'), - ]), - new FrontMatterRecordDelimiter([ - new Colon(new Range(1, 6, 1, 7)), - new Tab(new Range(1, 7, 1, 8)), - ]), - new FrontMatterString([ - new Quote(new Range(1, 8, 1, 9)), - new VerticalTab(new Range(1, 9, 1, 10)), - new Word(new Range(1, 10, 1, 10 + 2), 'do'), - new Tab(new Range(1, 12, 1, 13)), - new Word(new Range(1, 13, 1, 13 + 4), 'some'), - new NewLine(new Range(1, 17, 1, 18)), - new Word(new Range(2, 1, 2, 1 + 7), 'testing'), - new Comma(new Range(2, 8, 2, 9)), - new Space(new Range(2, 9, 2, 10)), - new Word(new Range(2, 10, 2, 10 + 6), 'please'), - new VerticalTab(new Range(2, 16, 2, 17)), - new Quote(new Range(2, 17, 2, 18)), - ]), - ]), - new Space(new Range(2, 18, 2, 19)), - new NewLine(new Range(2, 19, 2, 20)), - // second record - new FrontMatterRecord([ - new FrontMatterRecordName([ - new Word(new Range(3, 1, 3, 1 + 12), 'anotherField'), - ]), - new FrontMatterRecordDelimiter([ - new Colon(new Range(3, 17, 3, 18)), - new VerticalTab(new Range(3, 18, 3, 19)), - ]), - new FrontMatterString([ - new DoubleQuote(new Range(3, 20, 3, 21)), - new Word(new Range(3, 21, 3, 21 + 3), 'fal'), - new NewLine(new Range(3, 24, 3, 25)), - new Word(new Range(4, 1, 4, 1 + 2), 'se'), - new DoubleQuote(new Range(4, 3, 4, 4)), - ]), - ]), - ]); - }); - - test('boolean', async () => { - const test = disposables.add(new TestFrontMatterDecoder()); - - await test.run( - [ - 'anotherField \t\t : FALSE ', - 'my-field: true\t ', - ], - [ - // first record - new FrontMatterRecord([ - new FrontMatterRecordName([ - new Word(new Range(1, 1, 1, 1 + 12), 'anotherField'), - ]), - new FrontMatterRecordDelimiter([ - new Colon(new Range(1, 17, 1, 18)), - new Space(new Range(1, 18, 1, 19)), - ]), - new FrontMatterBoolean( - new Word( - new Range(1, 20, 1, 20 + 5), - 'FALSE', - ), - ), - ]), - new Space(new Range(1, 25, 1, 26)), - new NewLine(new Range(1, 26, 1, 27)), - // second record - new FrontMatterRecord([ - new FrontMatterRecordName([ - new Word(new Range(2, 1, 2, 1 + 2), 'my'), - new Dash(new Range(2, 3, 2, 4)), - new Word(new Range(2, 4, 2, 4 + 5), 'field'), - ]), - new FrontMatterRecordDelimiter([ - new Colon(new Range(2, 9, 2, 10)), - new Space(new Range(2, 10, 2, 11)), - ]), - new FrontMatterBoolean( - new Word( - new Range(2, 11, 2, 11 + 4), - 'true', - ), - ), - ]), - new Tab(new Range(2, 15, 2, 16)), - new Space(new Range(2, 16, 2, 17)), - ]); - }); - - suite('array', () => { - test('empty', async () => { - const test = disposables.add(new TestFrontMatterDecoder()); - - await test.run( - [ - `tools\v:\t []`, - 'anotherField \t\t :\v\v"fal\nse"', - ], - [ - // first record - new FrontMatterRecord([ - new FrontMatterRecordName([ - new Word(new Range(1, 1, 1, 1 + 5), 'tools'), - ]), - new FrontMatterRecordDelimiter([ - new Colon(new Range(1, 7, 1, 8)), - new Tab(new Range(1, 8, 1, 9)), - ]), - new FrontMatterArray([ - new LeftBracket(new Range(1, 10, 1, 11)), - new RightBracket(new Range(1, 11, 1, 12)), - ]), - ]), - new NewLine(new Range(1, 12, 1, 13)), - // second record - new FrontMatterRecord([ - new FrontMatterRecordName([ - new Word(new Range(2, 1, 2, 1 + 12), 'anotherField'), - ]), - new FrontMatterRecordDelimiter([ - new Colon(new Range(2, 17, 2, 18)), - new VerticalTab(new Range(2, 18, 2, 19)), - ]), - new FrontMatterString([ - new DoubleQuote(new Range(2, 20, 2, 21)), - new Word(new Range(2, 21, 2, 21 + 3), 'fal'), - new NewLine(new Range(2, 24, 2, 25)), - new Word(new Range(3, 1, 3, 1 + 2), 'se'), - new DoubleQuote(new Range(3, 3, 3, 4)), - ]), - ]), - ]); - }); - - test('mixed values', async () => { - const test = disposables.add(new TestFrontMatterDecoder()); - - await test.run( - [ - `tools\v:\t [true , 'toolName', some-tool]`, - ], - [ - // first record - new FrontMatterRecord([ - new FrontMatterRecordName([ - new Word(new Range(1, 1, 1, 1 + 5), 'tools'), - ]), - new FrontMatterRecordDelimiter([ - new Colon(new Range(1, 7, 1, 8)), - new Tab(new Range(1, 8, 1, 9)), - ]), - new FrontMatterArray([ - new LeftBracket(new Range(1, 10, 1, 11)), - // first array value - new FrontMatterBoolean( - new Word( - new Range(1, 11, 1, 11 + 4), - 'true', - ), - ), - // second array value - new FrontMatterString([ - new Quote(new Range(1, 18, 1, 19)), - new Word(new Range(1, 19, 1, 19 + 8), 'toolName'), - new Quote(new Range(1, 27, 1, 28)), - ]), - // third array value - new FrontMatterSequence([ - new Word(new Range(1, 30, 1, 30 + 4), 'some'), - new Dash(new Range(1, 34, 1, 35)), - new Word(new Range(1, 35, 1, 35 + 4), 'tool'), - ]), - new RightBracket(new Range(1, 39, 1, 40)), - ]), - ]), - ]); - }); - - test('redundant commas', async () => { - const test = disposables.add(new TestFrontMatterDecoder()); - - await test.run( - [ - `tools\v:\t [true ,, 'toolName', , , some-tool ,]`, - ], - [ - // first record - new FrontMatterRecord([ - new FrontMatterRecordName([ - new Word(new Range(1, 1, 1, 1 + 5), 'tools'), - ]), - new FrontMatterRecordDelimiter([ - new Colon(new Range(1, 7, 1, 8)), - new Tab(new Range(1, 8, 1, 9)), - ]), - new FrontMatterArray([ - new LeftBracket(new Range(1, 10, 1, 11)), - // first array value - new FrontMatterBoolean( - new Word( - new Range(1, 11, 1, 11 + 4), - 'true', - ), - ), - // second array value - new FrontMatterString([ - new Quote(new Range(1, 19, 1, 20)), - new Word(new Range(1, 20, 1, 20 + 8), 'toolName'), - new Quote(new Range(1, 28, 1, 29)), - ]), - // third array value - new FrontMatterSequence([ - new Word(new Range(1, 35, 1, 35 + 4), 'some'), - new Dash(new Range(1, 39, 1, 40)), - new Word(new Range(1, 40, 1, 40 + 4), 'tool'), - ]), - new RightBracket(new Range(1, 47, 1, 48)), - ]), - ]), - ]); - }); - }); - }); - }); - - test('empty', async () => { - const test = disposables.add( - new TestFrontMatterDecoder(), - ); - - await test.run('', []); - }); -}); diff --git a/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/frontMatterDecoder/frontMatterRecord.test.ts b/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/frontMatterDecoder/frontMatterRecord.test.ts deleted file mode 100644 index fba57e45209..00000000000 --- a/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/frontMatterDecoder/frontMatterRecord.test.ts +++ /dev/null @@ -1,183 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import assert from 'assert'; -import { Range } from '../../../../../../../../../editor/common/core/range.js'; -import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../../../../../../base/test/common/utils.js'; -import { FrontMatterSequence } from '../../../../../../common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterSequence.js'; -import { Colon, LeftBracket, Quote, RightBracket, Space, Tab, VerticalTab, Word } from '../../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/tokens.js'; -import { FrontMatterArray, FrontMatterBoolean, FrontMatterRecord, FrontMatterRecordDelimiter, FrontMatterRecordName, FrontMatterString } from '../../../../../../common/promptSyntax/codecs/base/frontMatterCodec/tokens/index.js'; - -suite('FrontMatterBoolean', () => { - ensureNoDisposablesAreLeakedInTestSuite(); - - suite('trimValueEnd()', () => { - test('trims space tokens at the end of record\'s value', () => { - const recordName = new FrontMatterRecordName([ - new Word( - new Range(4, 10, 4, 10 + 3), - 'key', - ), - ]); - - const recordDelimiter = new FrontMatterRecordDelimiter([ - new Colon(new Range(4, 14, 4, 15)), - new VerticalTab(new Range(4, 15, 4, 16)), - ]); - - const recordValue = new FrontMatterSequence([ - new Word(new Range(4, 18, 4, 18 + 10), 'some-value'), - new VerticalTab(new Range(4, 28, 4, 29)), - new Tab(new Range(4, 29, 4, 30)), - new Space(new Range(4, 30, 4, 31)), - new Tab(new Range(4, 31, 4, 32)), - ]); - - const record = new FrontMatterRecord([ - recordName, recordDelimiter, recordValue, - ]); - - const trimmed = record.trimValueEnd(); - assert.deepStrictEqual( - trimmed, - [ - new VerticalTab(new Range(4, 28, 4, 29)), - new Tab(new Range(4, 29, 4, 30)), - new Space(new Range(4, 30, 4, 31)), - new Tab(new Range(4, 31, 4, 32)), - ], - 'Must return correct trimmed list of spacing tokens.', - ); - - assert( - record.range.equalsRange( - new Range(4, 10, 4, 28), - ), - 'Must correctly update token range.', - ); - }); - - suite('does not trim non-sequence value tokens', () => { - test('boolean', () => { - const recordName = new FrontMatterRecordName([ - new Word( - new Range(4, 10, 4, 10 + 3), - 'yke', - ), - ]); - - const recordDelimiter = new FrontMatterRecordDelimiter([ - new Colon(new Range(4, 14, 4, 15)), - new VerticalTab(new Range(4, 15, 4, 16)), - ]); - - const recordValue = new FrontMatterBoolean( - new Word(new Range(4, 18, 4, 18 + 4), 'true'), - ); - - const record = new FrontMatterRecord([ - recordName, recordDelimiter, recordValue, - ]); - - const trimmed = record.trimValueEnd(); - assert.deepStrictEqual( - trimmed, - [], - 'Must return empty list of trimmed spacing tokens.', - ); - - assert( - record.range.equalsRange( - new Range(4, 10, 4, 22), - ), - 'Must not update token range.', - ); - }); - - test('quoted string', () => { - const recordName = new FrontMatterRecordName([ - new Word( - new Range(4, 10, 4, 10 + 3), - 'eyk', - ), - ]); - - const recordDelimiter = new FrontMatterRecordDelimiter([ - new Colon(new Range(4, 14, 4, 15)), - new VerticalTab(new Range(4, 15, 4, 16)), - ]); - - const recordValue = new FrontMatterString([ - new Quote(new Range(4, 18, 4, 19)), - new Word(new Range(4, 19, 4, 19 + 10), 'some text'), - new Quote(new Range(4, 29, 4, 30)), - ]); - - const record = new FrontMatterRecord([ - recordName, recordDelimiter, recordValue, - ]); - - const trimmed = record.trimValueEnd(); - assert.deepStrictEqual( - trimmed, - [], - 'Must return empty list of trimmed spacing tokens.', - ); - - assert( - record.range.equalsRange( - new Range(4, 10, 4, 30), - ), - 'Must not update token range.', - ); - }); - - test('array', () => { - const recordName = new FrontMatterRecordName([ - new Word( - new Range(4, 10, 4, 10 + 3), - 'yek', - ), - ]); - - const recordDelimiter = new FrontMatterRecordDelimiter([ - new Colon(new Range(4, 14, 4, 15)), - new VerticalTab(new Range(4, 15, 4, 16)), - ]); - - const recordValue = new FrontMatterArray([ - new LeftBracket(new Range(4, 18, 4, 19)), - new FrontMatterString([ - new Quote(new Range(4, 18, 4, 19)), - new Word(new Range(4, 19, 4, 19 + 10), 'some text'), - new Quote(new Range(4, 29, 4, 30)), - ]), - new FrontMatterBoolean( - new Word(new Range(4, 34, 4, 34 + 4), 'true'), - ), - new RightBracket(new Range(4, 38, 4, 39)), - ]); - - const record = new FrontMatterRecord([ - recordName, recordDelimiter, recordValue, - ]); - - const trimmed = record.trimValueEnd(); - assert.deepStrictEqual( - trimmed, - [], - 'Must return empty list of trimmed spacing tokens.', - ); - - assert( - record.range.equalsRange( - new Range(4, 10, 4, 39), - ), - 'Must not update token range.', - ); - }); - }); - }); -}); diff --git a/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/frontMatterDecoder/frontMatterSequence.test.ts b/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/frontMatterDecoder/frontMatterSequence.test.ts deleted file mode 100644 index 68602622bba..00000000000 --- a/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/frontMatterDecoder/frontMatterSequence.test.ts +++ /dev/null @@ -1,106 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import assert from 'assert'; -import { Range } from '../../../../../../../../../editor/common/core/range.js'; -import { FrontMatterValueToken } from '../../../../../../common/promptSyntax/codecs/base/frontMatterCodec/tokens/index.js'; -import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../../../../../../base/test/common/utils.js'; -import { Space, Tab, VerticalTab, Word } from '../../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/tokens.js'; -import { FrontMatterSequence } from '../../../../../../common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterSequence.js'; - -suite('FrontMatterSequence', () => { - ensureNoDisposablesAreLeakedInTestSuite(); - - test('extends \'FrontMatterValueToken\'', () => { - const sequence = new FrontMatterSequence([ - new Word( - new Range(1, 1, 1, 5), - 'test', - ), - ]); - - assert( - sequence instanceof FrontMatterValueToken, - 'Must extend FrontMatterValueToken class.', - ); - }); - - suite('trimEnd()', () => { - test('trims space tokens at the end of the sequence', () => { - const sequence = new FrontMatterSequence([ - new Word(new Range(4, 18, 4, 18 + 10), 'some-value'), - new Space(new Range(4, 28, 4, 29)), - new Space(new Range(4, 29, 4, 30)), - new VerticalTab(new Range(4, 30, 4, 31)), - new Tab(new Range(4, 31, 4, 32)), - new Space(new Range(4, 32, 4, 33)), - ]); - - const trimmed = sequence.trimEnd(); - assert.deepStrictEqual( - trimmed, - [ - new Space(new Range(4, 28, 4, 29)), - new Space(new Range(4, 29, 4, 30)), - new VerticalTab(new Range(4, 30, 4, 31)), - new Tab(new Range(4, 31, 4, 32)), - new Space(new Range(4, 32, 4, 33)), - ], - 'Must return correct trimmed list of spacing tokens.', - ); - - assert( - sequence.range.equalsRange( - new Range(4, 18, 4, 28), - ), - 'Must correctly update token range.', - ); - }); - - test('remains functional if only spacing tokens were present', () => { - const sequence = new FrontMatterSequence([ - new Space(new Range(4, 28, 4, 29)), - new Space(new Range(4, 29, 4, 30)), - new VerticalTab(new Range(4, 30, 4, 31)), - new Tab(new Range(4, 31, 4, 32)), - new Space(new Range(4, 32, 4, 33)), - ]); - - const trimmed = sequence.trimEnd(); - assert.deepStrictEqual( - trimmed, - [ - new Space(new Range(4, 28, 4, 29)), - new Space(new Range(4, 29, 4, 30)), - new VerticalTab(new Range(4, 30, 4, 31)), - new Tab(new Range(4, 31, 4, 32)), - new Space(new Range(4, 32, 4, 33)), - ], - 'Must return correct trimmed list of spacing tokens.', - ); - - assert( - sequence.range.equalsRange( - new Range(4, 28, 4, 28), - ), - 'Must correctly update token range.', - ); - - assert.deepStrictEqual( - sequence.children, - [ - new Word(new Range(4, 28, 4, 28), ''), - ], - 'Must contain a single empty token.', - ); - }); - }); - - test('throws if no tokens provided', () => { - assert.throws(() => { - new FrontMatterSequence([]); - }); - }); -}); diff --git a/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/linesDecoder.test.ts b/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/linesDecoder.test.ts deleted file mode 100644 index b9ac4916923..00000000000 --- a/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/linesDecoder.test.ts +++ /dev/null @@ -1,256 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import assert from 'assert'; -import { Range } from '../../../../../../../../editor/common/core/range.js'; -import { VSBuffer } from '../../../../../../../../base/common/buffer.js'; -import { DisposableStore } from '../../../../../../../../base/common/lifecycle.js'; -import { Line } from '../../../../../common/promptSyntax/codecs/base/linesCodec/tokens/line.js'; -import { TestDecoder, TTokensConsumeMethod } from './utils/testDecoder.js'; -import { NewLine } from '../../../../../common/promptSyntax/codecs/base/linesCodec/tokens/newLine.js'; -import { newWriteableStream, WriteableStream } from '../../../../../../../../base/common/stream.js'; -import { CarriageReturn } from '../../../../../common/promptSyntax/codecs/base/linesCodec/tokens/carriageReturn.js'; -import { LinesDecoder, TLineToken } from '../../../../../common/promptSyntax/codecs/base/linesCodec/linesDecoder.js'; -import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../../../../../base/test/common/utils.js'; - -/** - * Note! This decoder is also often used to test common logic of abstract {@link BaseDecoder} - * class, because the {@link LinesDecoder} is one of the simplest non-abstract decoders we have. - */ -suite('LinesDecoder', () => { - const disposables = ensureNoDisposablesAreLeakedInTestSuite(); - - /** - * Test the core logic with specific method of consuming - * tokens that are produced by a lines decoder instance. - */ - suite('core logic', () => { - testLinesDecoder('async-generator', disposables); - testLinesDecoder('consume-all-method', disposables); - testLinesDecoder('on-data-event', disposables); - }); - - suite('settled promise', () => { - test('throws if accessed on not-yet-started decoder instance', () => { - const test = disposables.add(new TestLinesDecoder()); - - assert.throws( - () => { - // testing the field access that throws here, so - // its OK to not use the returned value afterwards - // eslint-disable-next-line local/code-no-unused-expressions - test.decoder.settled; - }, - [ - 'Cannot get `settled` promise of a stream that has not been started.', - 'Please call `start()` first.', - ].join(' '), - ); - }); - }); - - suite('start', () => { - test('throws if the decoder object is already `disposed`', () => { - const test = disposables.add(new TestLinesDecoder()); - const { decoder } = test; - decoder.dispose(); - - assert.throws( - decoder.start.bind(decoder), - 'Cannot start stream that has already disposed.', - ); - }); - - test('throws if the decoder object is already `ended`', async () => { - const inputStream = newWriteableStream(null); - const test = disposables.add(new TestLinesDecoder(inputStream)); - const { decoder } = test; - - setTimeout(() => { - test.sendData([ - 'hello', - 'world :wave:', - ]); - }, 5); - - const receivedTokens = await decoder.start() - .consumeAll(); - - // a basic sanity check for received tokens - assert.strictEqual( - receivedTokens.length, - 3, - 'Must produce the correct number of tokens.', - ); - - // validate that calling `start()` after stream has ended throws - assert.throws( - decoder.start.bind(decoder), - 'Cannot start stream that has already ended.', - ); - }); - }); -}); - - -/** - * A reusable test utility that asserts that a `LinesDecoder` instance - * correctly decodes `inputData` into a stream of `TLineToken` tokens. - * - * ## Examples - * - * ```typescript - * // create a new test utility instance - * const test = disposables.add(new TestLinesDecoder()); - * - * // run the test - * await test.run( - * ' hello world\n', - * [ - * new Line(1, ' hello world'), - * new NewLine(new Range(1, 13, 1, 14)), - * ], - * ); - */ -export class TestLinesDecoder extends TestDecoder { - constructor( - inputStream?: WriteableStream, - ) { - const stream = (inputStream) - ? inputStream - : newWriteableStream(null); - - const decoder = new LinesDecoder(stream); - - super(stream, decoder); - } -} - -/** - * Common reusable test utility to validate {@link LinesDecoder} logic with - * the provided {@link tokensConsumeMethod} way of consuming decoder-produced tokens. - * - * @throws if a test fails, please see thrown error for failure details. - * @param tokensConsumeMethod The way to consume tokens produced by the decoder. - * @param disposables Test disposables store. - */ -function testLinesDecoder( - tokensConsumeMethod: TTokensConsumeMethod, - disposables: Pick, -) { - suite(tokensConsumeMethod, () => { - suite('produces expected tokens', () => { - test('input starts with line data', async () => { - const test = disposables.add(new TestLinesDecoder()); - - await test.run( - ' hello world\nhow are you doing?\n\n 😊 \r', - [ - new Line(1, ' hello world'), - new NewLine(new Range(1, 13, 1, 14)), - new Line(2, 'how are you doing?'), - new NewLine(new Range(2, 19, 2, 20)), - new Line(3, ''), - new NewLine(new Range(3, 1, 3, 2)), - new Line(4, ' 😊 '), - new NewLine(new Range(4, 5, 4, 6)), - ], - ); - }); - - test('standalone \\r is treated as new line', async () => { - const test = disposables.add(new TestLinesDecoder()); - - await test.run( - ' hello world\nhow are you doing?\n\n 😊 \r ', - [ - new Line(1, ' hello world'), - new NewLine(new Range(1, 13, 1, 14)), - new Line(2, 'how are you doing?'), - new NewLine(new Range(2, 19, 2, 20)), - new Line(3, ''), - new NewLine(new Range(3, 1, 3, 2)), - new Line(4, ' 😊 '), - new NewLine(new Range(4, 5, 4, 6)), - new Line(5, ' '), - ], - ); - }); - - test('input starts with a new line', async () => { - const test = disposables.add(new TestLinesDecoder()); - - await test.run( - '\nsome text on this line\n\n\nanother 💬 on this line\r\n🤫\n', - [ - new Line(1, ''), - new NewLine(new Range(1, 1, 1, 2)), - new Line(2, 'some text on this line'), - new NewLine(new Range(2, 23, 2, 24)), - new Line(3, ''), - new NewLine(new Range(3, 1, 3, 2)), - new Line(4, ''), - new NewLine(new Range(4, 1, 4, 2)), - new Line(5, 'another 💬 on this line'), - new CarriageReturn(new Range(5, 24, 5, 25)), - new NewLine(new Range(5, 25, 5, 26)), - new Line(6, '🤫'), - new NewLine(new Range(6, 3, 6, 4)), - ], - ); - }); - - test('input starts and ends with multiple new lines', async () => { - const test = disposables.add(new TestLinesDecoder()); - - await test.run( - '\n\n\r\nciao! 🗯️\t💭 💥 come\tva?\n\n\n\n\n', - [ - new Line(1, ''), - new NewLine(new Range(1, 1, 1, 2)), - new Line(2, ''), - new NewLine(new Range(2, 1, 2, 2)), - new Line(3, ''), - new CarriageReturn(new Range(3, 1, 3, 2)), - new NewLine(new Range(3, 2, 3, 3)), - new Line(4, 'ciao! 🗯️\t💭 💥 come\tva?'), - new NewLine(new Range(4, 25, 4, 26)), - new Line(5, ''), - new NewLine(new Range(5, 1, 5, 2)), - new Line(6, ''), - new NewLine(new Range(6, 1, 6, 2)), - new Line(7, ''), - new NewLine(new Range(7, 1, 7, 2)), - new Line(8, ''), - new NewLine(new Range(8, 1, 8, 2)), - ], - ); - }); - - test('single carriage return is treated as new line', async () => { - const test = disposables.add(new TestLinesDecoder()); - - await test.run( - '\r\rhaalo! 💥💥 how\'re you?\r ?!\r\n\r\n ', - [ - new Line(1, ''), - new NewLine(new Range(1, 1, 1, 2)), - new Line(2, ''), - new NewLine(new Range(2, 1, 2, 2)), - new Line(3, 'haalo! 💥💥 how\'re you?'), - new NewLine(new Range(3, 24, 3, 25)), - new Line(4, ' ?!'), - new CarriageReturn(new Range(4, 4, 4, 5)), - new NewLine(new Range(4, 5, 4, 6)), - new Line(5, ''), - new CarriageReturn(new Range(5, 1, 5, 2)), - new NewLine(new Range(5, 2, 5, 3)), - new Line(6, ' '), - ], - ); - }); - }); - }); -} diff --git a/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/markdownDecoder.test.ts b/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/markdownDecoder.test.ts deleted file mode 100644 index 37a3331f796..00000000000 --- a/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/markdownDecoder.test.ts +++ /dev/null @@ -1,937 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import assert from 'assert'; -import { TestDecoder } from './utils/testDecoder.js'; -import { Range } from '../../../../../../../../editor/common/core/range.js'; -import { VSBuffer } from '../../../../../../../../base/common/buffer.js'; -import { newWriteableStream } from '../../../../../../../../base/common/stream.js'; -import { Tab } from '../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/tab.js'; -import { Word } from '../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/word.js'; -import { Dash } from '../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/dash.js'; -import { Space } from '../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/space.js'; -import { Slash } from '../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/slash.js'; -import { NewLine } from '../../../../../common/promptSyntax/codecs/base/linesCodec/tokens/newLine.js'; -import { FormFeed } from '../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/formFeed.js'; -import { VerticalTab } from '../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/verticalTab.js'; -import { MarkdownLink } from '../../../../../common/promptSyntax/codecs/base/markdownCodec/tokens/markdownLink.js'; -import { CarriageReturn } from '../../../../../common/promptSyntax/codecs/base/linesCodec/tokens/carriageReturn.js'; -import { MarkdownImage } from '../../../../../common/promptSyntax/codecs/base/markdownCodec/tokens/markdownImage.js'; -import { ExclamationMark } from '../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/exclamationMark.js'; -import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../../../../../base/test/common/utils.js'; -import { MarkdownComment } from '../../../../../common/promptSyntax/codecs/base/markdownCodec/tokens/markdownComment.js'; -import { LeftBracket, RightBracket } from '../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/brackets.js'; -import { MarkdownDecoder, TMarkdownToken } from '../../../../../common/promptSyntax/codecs/base/markdownCodec/markdownDecoder.js'; -import { LeftParenthesis, RightParenthesis } from '../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/parentheses.js'; -import { LeftAngleBracket, RightAngleBracket } from '../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/angleBrackets.js'; - -/** - * A reusable test utility that asserts that a `TestMarkdownDecoder` instance - * correctly decodes `inputData` into a stream of `TMarkdownToken` tokens. - * - * ## Examples - * - * ```typescript - * // create a new test utility instance - * const test = testDisposables.add(new TestMarkdownDecoder()); - * - * // run the test - * await test.run( - * ' hello [world](/etc/hosts)!', - * [ - * new Space(new Range(1, 1, 1, 2)), - * new Word(new Range(1, 2, 1, 7), 'hello'), - * new Space(new Range(1, 7, 1, 8)), - * new MarkdownLink(1, 8, '[world]', '(/etc/hosts)'), - * new Word(new Range(1, 27, 1, 28), '!'), - * new NewLine(new Range(1, 28, 1, 29)), - * ], - * ); - */ -export class TestMarkdownDecoder extends TestDecoder { - constructor() { - const stream = newWriteableStream(null); - - super(stream, new MarkdownDecoder(stream)); - } -} - -suite('MarkdownDecoder', () => { - const testDisposables = ensureNoDisposablesAreLeakedInTestSuite(); - - suite('general', () => { - test('base cases', async () => { - const test = testDisposables.add( - new TestMarkdownDecoder(), - ); - - await test.run( - [ - // basic text - ' hello world', - // text with markdown link and special characters in the filename - 'how are\t you [caption text](./some/file/path/refer🎨nce.md)?\v', - // empty line - '', - // markdown link with special characters in the link caption and path - '[(example!)](another/path/with[-and-]-chars/folder)\t ', - // markdown link `#file` variable in the caption and with absolute path - '\t[#file:something.txt](/absolute/path/to/something.txt)', - // text with a commented out markdown link - '\v\f machines must suffer', - ], - [ - // first line - new Space(new Range(1, 1, 1, 2)), - new Word(new Range(1, 2, 1, 7), 'hello'), - new Space(new Range(1, 7, 1, 8)), - new Word(new Range(1, 8, 1, 13), 'world'), - new NewLine(new Range(1, 13, 1, 14)), - // second line - new Word(new Range(2, 1, 2, 4), 'how'), - new Space(new Range(2, 4, 2, 5)), - new Word(new Range(2, 5, 2, 8), 'are'), - new Tab(new Range(2, 8, 2, 9)), - new Space(new Range(2, 9, 2, 10)), - new Word(new Range(2, 10, 2, 13), 'you'), - new Space(new Range(2, 13, 2, 14)), - new MarkdownLink(2, 14, '[caption text]', '(./some/file/path/refer🎨nce.md)'), - new Word(new Range(2, 60, 2, 61), '?'), - new VerticalTab(new Range(2, 61, 2, 62)), - new NewLine(new Range(2, 62, 2, 63)), - // third line - new NewLine(new Range(3, 1, 3, 2)), - // fourth line - new MarkdownLink(4, 1, '[(example!)]', '(another/path/with[-and-]-chars/folder)'), - new Tab(new Range(4, 52, 4, 53)), - new Space(new Range(4, 53, 4, 54)), - new NewLine(new Range(4, 54, 4, 55)), - // fifth line - new Tab(new Range(5, 1, 5, 2)), - new MarkdownLink(5, 2, '[#file:something.txt]', '(/absolute/path/to/something.txt)'), - new NewLine(new Range(5, 56, 5, 57)), - // sixth line - new VerticalTab(new Range(6, 1, 6, 2)), - new FormFeed(new Range(6, 2, 6, 3)), - new Space(new Range(6, 3, 6, 4)), - new Word(new Range(6, 4, 6, 12), 'machines'), - new Space(new Range(6, 12, 6, 13)), - new Word(new Range(6, 13, 6, 17), 'must'), - new Space(new Range(6, 17, 6, 18)), - new MarkdownComment(new Range(6, 18, 6, 18 + 41), ''), - new Space(new Range(6, 59, 6, 60)), - new Word(new Range(6, 60, 6, 66), 'suffer'), - ], - ); - }); - - test('nuanced', async () => { - const test = testDisposables.add( - new TestMarkdownDecoder(), - ); - - const inputLines = [ - // tests that the link caption contain a chat prompt `#file:` reference, while - // the file path can contain other `graphical characters` - '\v\t[#file:./another/path/to/file.txt](./real/file!path/file◆name.md)', - // tests that the link file path contain a chat prompt `#file:` reference, - // `spaces`, `emojies`, and other `graphical characters` - ' [reference ∘ label](/absolute/pa th/to-#file:file.txt/f🥸⚡️le.md)', - // tests that link caption and file path can contain `parentheses`, `spaces`, and - // `emojies` - '\f[!(hello)!](./w(())rld/nice-🦚-filen(a).git))\n\t', - // tests that the link caption can be empty, while the file path can contain `square brackets` - '[](./s[]me/pa[h!) ', - ]; - - await test.run( - inputLines, - [ - // `1st` line - new VerticalTab(new Range(1, 1, 1, 2)), - new Tab(new Range(1, 2, 1, 3)), - new MarkdownLink(1, 3, '[#file:./another/path/to/file.txt]', '(./real/file!path/file◆name.md)'), - new NewLine(new Range(1, 68, 1, 69)), - // `2nd` line - new Space(new Range(2, 1, 2, 2)), - new MarkdownLink(2, 2, '[reference ∘ label]', '(/absolute/pa th/to-#file:file.txt/f🥸⚡️le.md)'), - new NewLine(new Range(2, 67, 2, 68)), - // `3rd` line - new FormFeed(new Range(3, 1, 3, 2)), - new MarkdownLink(3, 2, '[!(hello)!]', '(./w(())rld/nice-🦚-filen(a).git)'), - new RightParenthesis(new Range(3, 50, 3, 51)), - new NewLine(new Range(3, 51, 3, 52)), - // `4th` line - new Tab(new Range(4, 1, 4, 2)), - new NewLine(new Range(4, 2, 4, 3)), - // `5th` line - new MarkdownLink(5, 1, '[]', '(./s[]me/pa[h!)'), - new Space(new Range(5, 24, 5, 25)), - ], - ); - }); - }); - - suite('links', () => { - suite('broken', () => { - test('invalid', async () => { - const test = testDisposables.add( - new TestMarkdownDecoder(), - ); - - const inputLines = [ - // incomplete link reference with empty caption - '[ ](./real/file path/file⇧name.md', - // space between caption and reference is disallowed - '[link text] (./file path/name.txt)', - ]; - - await test.run( - inputLines, - [ - // `1st` line - new LeftBracket(new Range(1, 1, 1, 2)), - new Space(new Range(1, 2, 1, 3)), - new RightBracket(new Range(1, 3, 1, 4)), - new LeftParenthesis(new Range(1, 4, 1, 5)), - new Word(new Range(1, 5, 1, 5 + 1), '.'), - new Slash(new Range(1, 6, 1, 7)), - new Word(new Range(1, 7, 1, 7 + 4), 'real'), - new Slash(new Range(1, 11, 1, 12)), - new Word(new Range(1, 12, 1, 12 + 4), 'file'), - new Space(new Range(1, 16, 1, 17)), - new Word(new Range(1, 17, 1, 17 + 4), 'path'), - new Slash(new Range(1, 21, 1, 22)), - new Word(new Range(1, 22, 1, 22 + 12), 'file⇧name.md'), - new NewLine(new Range(1, 34, 1, 35)), - // `2nd` line - new LeftBracket(new Range(2, 1, 2, 2)), - new Word(new Range(2, 2, 2, 2 + 4), 'link'), - new Space(new Range(2, 6, 2, 7)), - new Word(new Range(2, 7, 2, 7 + 4), 'text'), - new RightBracket(new Range(2, 11, 2, 12)), - new Space(new Range(2, 12, 2, 13)), - new LeftParenthesis(new Range(2, 13, 2, 14)), - new Word(new Range(2, 14, 2, 14 + 1), '.'), - new Slash(new Range(2, 15, 2, 16)), - new Word(new Range(2, 16, 2, 16 + 4), 'file'), - new Space(new Range(2, 20, 2, 21)), - new Word(new Range(2, 21, 2, 21 + 4), 'path'), - new Slash(new Range(2, 25, 2, 26)), - new Word(new Range(2, 26, 2, 26 + 8), 'name.txt'), - new RightParenthesis(new Range(2, 34, 2, 35)), - ], - ); - }); - - suite('stop characters inside caption/reference (new lines)', () => { - for (const StopCharacter of [CarriageReturn, NewLine]) { - let characterName = ''; - - if (StopCharacter === CarriageReturn) { - characterName = '\\r'; - } - if (StopCharacter === NewLine) { - characterName = '\\n'; - } - - assert( - characterName !== '', - 'The "characterName" must be set, got "empty line".', - ); - - test(`stop character - "${characterName}"`, async () => { - const test = testDisposables.add( - new TestMarkdownDecoder(), - ); - - const inputLines = [ - // stop character inside link caption - `[haa${StopCharacter.symbol}loů](./real/💁/name.txt)`, - // stop character inside link reference - `[ref text](/etc/pat${StopCharacter.symbol}h/to/file.md)`, - // stop character between line caption and link reference is disallowed - `[text]${StopCharacter.symbol}(/etc/ path/main.mdc)`, - ]; - - - await test.run( - inputLines, - [ - // `1st` input line - new LeftBracket(new Range(1, 1, 1, 2)), - new Word(new Range(1, 2, 1, 2 + 3), 'haa'), - new NewLine(new Range(1, 5, 1, 6)), // a single CR token is treated as a `new line` - new Word(new Range(2, 1, 2, 1 + 3), 'loů'), - new RightBracket(new Range(2, 4, 2, 5)), - new LeftParenthesis(new Range(2, 5, 2, 6)), - new Word(new Range(2, 6, 2, 6 + 1), '.'), - new Slash(new Range(2, 7, 2, 8)), - new Word(new Range(2, 8, 2, 8 + 4), 'real'), - new Slash(new Range(2, 12, 2, 13)), - new Word(new Range(2, 13, 2, 13 + 2), '💁'), - new Slash(new Range(2, 15, 2, 16)), - new Word(new Range(2, 16, 2, 16 + 8), 'name.txt'), - new RightParenthesis(new Range(2, 24, 2, 25)), - new NewLine(new Range(2, 25, 2, 26)), - // `2nd` input line - new LeftBracket(new Range(3, 1, 3, 2)), - new Word(new Range(3, 2, 3, 2 + 3), 'ref'), - new Space(new Range(3, 5, 3, 6)), - new Word(new Range(3, 6, 3, 6 + 4), 'text'), - new RightBracket(new Range(3, 10, 3, 11)), - new LeftParenthesis(new Range(3, 11, 3, 12)), - new Slash(new Range(3, 12, 3, 13)), - new Word(new Range(3, 13, 3, 13 + 3), 'etc'), - new Slash(new Range(3, 16, 3, 17)), - new Word(new Range(3, 17, 3, 17 + 3), 'pat'), - new NewLine(new Range(3, 20, 3, 21)), // a single CR token is treated as a `new line` - new Word(new Range(4, 1, 4, 1 + 1), 'h'), - new Slash(new Range(4, 2, 4, 3)), - new Word(new Range(4, 3, 4, 3 + 2), 'to'), - new Slash(new Range(4, 5, 4, 6)), - new Word(new Range(4, 6, 4, 6 + 7), 'file.md'), - new RightParenthesis(new Range(4, 13, 4, 14)), - new NewLine(new Range(4, 14, 4, 15)), - // `3nd` input line - new LeftBracket(new Range(5, 1, 5, 2)), - new Word(new Range(5, 2, 5, 2 + 4), 'text'), - new RightBracket(new Range(5, 6, 5, 7)), - new NewLine(new Range(5, 7, 5, 8)), // a single CR token is treated as a `new line` - new LeftParenthesis(new Range(6, 1, 6, 2)), - new Slash(new Range(6, 2, 6, 3)), - new Word(new Range(6, 3, 6, 3 + 3), 'etc'), - new Slash(new Range(6, 6, 6, 7)), - new Space(new Range(6, 7, 6, 8)), - new Word(new Range(6, 8, 6, 8 + 4), 'path'), - new Slash(new Range(6, 12, 6, 13)), - new Word(new Range(6, 13, 6, 13 + 8), 'main.mdc'), - new RightParenthesis(new Range(6, 21, 6, 22)), - ], - ); - }); - } - }); - - /** - * Same as above but these stop characters do not move the caret to the next line. - */ - suite('stop characters inside caption/reference (same line)', () => { - for (const StopCharacter of [VerticalTab, FormFeed]) { - let characterName = ''; - - if (StopCharacter === VerticalTab) { - characterName = '\\v'; - } - if (StopCharacter === FormFeed) { - characterName = '\\f'; - } - - assert( - characterName !== '', - 'The "characterName" must be set, got "empty line".', - ); - - test(`stop character - "${characterName}"`, async () => { - const test = testDisposables.add( - new TestMarkdownDecoder(), - ); - - const inputLines = [ - // stop character inside link caption - `[haa${StopCharacter.symbol}loů](./real/💁/name.txt)`, - // stop character inside link reference - `[ref text](/etc/pat${StopCharacter.symbol}h/to/file.md)`, - // stop character between line caption and link reference is disallowed - `[text]${StopCharacter.symbol}(/etc/ path/file.md)`, - ]; - - - await test.run( - inputLines, - [ - // `1st` input line - new LeftBracket(new Range(1, 1, 1, 2)), - new Word(new Range(1, 2, 1, 2 + 3), 'haa'), - new StopCharacter(new Range(1, 5, 1, 6)), // <- stop character - new Word(new Range(1, 6, 1, 6 + 3), 'loů'), - new RightBracket(new Range(1, 9, 1, 10)), - new LeftParenthesis(new Range(1, 10, 1, 11)), - new Word(new Range(1, 11, 1, 11 + 1), '.'), - new Slash(new Range(1, 12, 1, 13)), - new Word(new Range(1, 13, 1, 13 + 4), 'real'), - new Slash(new Range(1, 17, 1, 18)), - new Word(new Range(1, 18, 1, 18 + 2), '💁'), - new Slash(new Range(1, 20, 1, 21)), - new Word(new Range(1, 21, 1, 21 + 8), 'name.txt'), - new RightParenthesis(new Range(1, 29, 1, 30)), - new NewLine(new Range(1, 30, 1, 31)), - // `2nd` input line - new LeftBracket(new Range(2, 1, 2, 2)), - new Word(new Range(2, 2, 2, 2 + 3), 'ref'), - new Space(new Range(2, 5, 2, 6)), - new Word(new Range(2, 6, 2, 6 + 4), 'text'), - new RightBracket(new Range(2, 10, 2, 11)), - new LeftParenthesis(new Range(2, 11, 2, 12)), - new Slash(new Range(2, 12, 2, 13)), - new Word(new Range(2, 13, 2, 13 + 3), 'etc'), - new Slash(new Range(2, 16, 2, 17)), - new Word(new Range(2, 17, 2, 17 + 3), 'pat'), - new StopCharacter(new Range(2, 20, 2, 21)), // <- stop character - new Word(new Range(2, 21, 2, 21 + 1), 'h'), - new Slash(new Range(2, 22, 2, 23)), - new Word(new Range(2, 23, 2, 23 + 2), 'to'), - new Slash(new Range(2, 25, 2, 26)), - new Word(new Range(2, 26, 2, 26 + 7), 'file.md'), - new RightParenthesis(new Range(2, 33, 2, 34)), - new NewLine(new Range(2, 34, 2, 35)), - // `3nd` input line - new LeftBracket(new Range(3, 1, 3, 2)), - new Word(new Range(3, 2, 3, 2 + 4), 'text'), - new RightBracket(new Range(3, 6, 3, 7)), - new StopCharacter(new Range(3, 7, 3, 8)), // <- stop character - new LeftParenthesis(new Range(3, 8, 3, 9)), - new Slash(new Range(3, 9, 3, 10)), - new Word(new Range(3, 10, 3, 10 + 3), 'etc'), - new Slash(new Range(3, 13, 3, 14)), - new Space(new Range(3, 14, 3, 15)), - new Word(new Range(3, 15, 3, 15 + 4), 'path'), - new Slash(new Range(3, 19, 3, 20)), - new Word(new Range(3, 20, 3, 20 + 7), 'file.md'), - new RightParenthesis(new Range(3, 27, 3, 28)), - ], - ); - }); - } - }); - }); - }); - - - suite('images', () => { - suite('general', () => { - test('base cases', async () => { - const test = testDisposables.add( - new TestMarkdownDecoder(), - ); - - const inputData = [ - '\t![alt text](./some/path/to/file.jpg) ', - 'plain text \f![label](./image.png)\v and more text', - '![](/var/images/default) following text', - ]; - - await test.run( - inputData, - [ - // `1st` - new Tab(new Range(1, 1, 1, 2)), - new MarkdownImage(1, 2, '![alt text]', '(./some/path/to/file.jpg)'), - new Space(new Range(1, 38, 1, 39)), - new NewLine(new Range(1, 39, 1, 40)), - // `2nd` - new Word(new Range(2, 1, 2, 6), 'plain'), - new Space(new Range(2, 6, 2, 7)), - new Word(new Range(2, 7, 2, 11), 'text'), - new Space(new Range(2, 11, 2, 12)), - new FormFeed(new Range(2, 12, 2, 13)), - new MarkdownImage(2, 13, '![label]', '(./image.png)'), - new VerticalTab(new Range(2, 34, 2, 35)), - new Space(new Range(2, 35, 2, 36)), - new Word(new Range(2, 36, 2, 39), 'and'), - new Space(new Range(2, 39, 2, 40)), - new Word(new Range(2, 40, 2, 44), 'more'), - new Space(new Range(2, 44, 2, 45)), - new Word(new Range(2, 45, 2, 49), 'text'), - new NewLine(new Range(2, 49, 2, 50)), - // `3rd` - new MarkdownImage(3, 1, '![]', '(/var/images/default)'), - new Space(new Range(3, 25, 3, 26)), - new Word(new Range(3, 26, 3, 35), 'following'), - new Space(new Range(3, 35, 3, 36)), - new Word(new Range(3, 36, 3, 40), 'text'), - ], - ); - }); - - test('nuanced', async () => { - const test = testDisposables.add( - new TestMarkdownDecoder(), - ); - - const inputData = [ - '\t![](./s☻me/path/to/file.jpeg) ', - 'raw text \f![(/1.png)](./image-🥸.png)\v and more text', - // '![](/var/images/default) following text', - ]; - - await test.run( - inputData, - [ - // `1st` - new Tab(new Range(1, 1, 1, 2)), - new MarkdownImage(1, 2, '![]', '(./s☻me/path/to/file.jpeg)'), - new Space(new Range(1, 47, 1, 48)), - new NewLine(new Range(1, 48, 1, 49)), - // `2nd` - new Word(new Range(2, 1, 2, 4), 'raw'), - new Space(new Range(2, 4, 2, 5)), - new Word(new Range(2, 5, 2, 9), 'text'), - new Space(new Range(2, 9, 2, 10)), - new FormFeed(new Range(2, 10, 2, 11)), - new MarkdownImage(2, 11, '![(/1.png)]', '(./image-🥸.png)'), - new VerticalTab(new Range(2, 38, 2, 39)), - new Space(new Range(2, 39, 2, 40)), - new Word(new Range(2, 40, 2, 43), 'and'), - new Space(new Range(2, 43, 2, 44)), - new Word(new Range(2, 44, 2, 48), 'more'), - new Space(new Range(2, 48, 2, 49)), - new Word(new Range(2, 49, 2, 53), 'text'), - ], - ); - }); - }); - - suite('broken', () => { - test('invalid', async () => { - const test = testDisposables.add( - new TestMarkdownDecoder(), - ); - - const inputLines = [ - // incomplete link reference with empty caption - '![ ](./real/file path/file★name.webp', - // space between caption and reference is disallowed - '\f![link text] (./file path/name.jpg)', - // new line inside the link reference - '\v![ ](./file\npath/name.jpeg )', - ]; - - await test.run( - inputLines, - [ - // `1st` line - new ExclamationMark(new Range(1, 1, 1, 2)), - new LeftBracket(new Range(1, 2, 1, 3)), - new Space(new Range(1, 3, 1, 4)), - new RightBracket(new Range(1, 4, 1, 5)), - new LeftParenthesis(new Range(1, 5, 1, 6)), - new Word(new Range(1, 6, 1, 6 + 1), '.'), - new Slash(new Range(1, 7, 1, 8)), - new Word(new Range(1, 8, 1, 8 + 4), 'real'), - new Slash(new Range(1, 12, 1, 13)), - new Word(new Range(1, 13, 1, 13 + 4), 'file'), - new Space(new Range(1, 17, 1, 18)), - new Word(new Range(1, 18, 1, 18 + 4), 'path'), - new Slash(new Range(1, 22, 1, 23)), - new Word(new Range(1, 23, 1, 23 + 14), 'file★name.webp'), - new NewLine(new Range(1, 37, 1, 38)), - // `2nd` line - new FormFeed(new Range(2, 1, 2, 2)), - new ExclamationMark(new Range(2, 2, 2, 3)), - new LeftBracket(new Range(2, 3, 2, 4)), - new Word(new Range(2, 4, 2, 4 + 4), 'link'), - new Space(new Range(2, 8, 2, 9)), - new Word(new Range(2, 9, 2, 9 + 4), 'text'), - new RightBracket(new Range(2, 13, 2, 14)), - new Space(new Range(2, 14, 2, 15)), - new LeftParenthesis(new Range(2, 15, 2, 16)), - new Word(new Range(2, 16, 2, 16 + 1), '.'), - new Slash(new Range(2, 17, 2, 18)), - new Word(new Range(2, 18, 2, 18 + 4), 'file'), - new Space(new Range(2, 22, 2, 23)), - new Word(new Range(2, 23, 2, 23 + 4), 'path'), - new Slash(new Range(2, 27, 2, 28)), - new Word(new Range(2, 28, 2, 28 + 8), 'name.jpg'), - new RightParenthesis(new Range(2, 36, 2, 37)), - new NewLine(new Range(2, 37, 2, 38)), - // `3rd` line - new VerticalTab(new Range(3, 1, 3, 2)), - new ExclamationMark(new Range(3, 2, 3, 3)), - new LeftBracket(new Range(3, 3, 3, 4)), - new Space(new Range(3, 4, 3, 5)), - new RightBracket(new Range(3, 5, 3, 6)), - new LeftParenthesis(new Range(3, 6, 3, 7)), - new Word(new Range(3, 7, 3, 7 + 1), '.'), - new Slash(new Range(3, 8, 3, 9)), - new Word(new Range(3, 9, 3, 9 + 4), 'file'), - new NewLine(new Range(3, 13, 3, 14)), - new Word(new Range(4, 1, 4, 1 + 4), 'path'), - new Slash(new Range(4, 5, 4, 6)), - new Word(new Range(4, 6, 4, 6 + 9), 'name.jpeg'), - new Space(new Range(4, 15, 4, 16)), - new RightParenthesis(new Range(4, 16, 4, 17)), - ], - ); - }); - - suite('stop characters inside caption/reference (new lines)', () => { - for (const StopCharacter of [CarriageReturn, NewLine]) { - let characterName = ''; - - if (StopCharacter === CarriageReturn) { - characterName = '\\r'; - } - if (StopCharacter === NewLine) { - characterName = '\\n'; - } - - assert( - characterName !== '', - 'The "characterName" must be set, got "empty line".', - ); - - test(`stop character - "${characterName}"`, async () => { - const test = testDisposables.add( - new TestMarkdownDecoder(), - ); - - const inputLines = [ - // stop character inside link caption - `![haa${StopCharacter.symbol}loů](./real/💁/name.png)`, - // stop character inside link reference - `![ref text](/etc/pat${StopCharacter.symbol}h/to/file.webp)`, - // stop character between line caption and link reference is disallowed - `![text]${StopCharacter.symbol}(/etc/ path/file.jpeg)`, - ]; - - - await test.run( - inputLines, - [ - // `1st` input line - new ExclamationMark(new Range(1, 1, 1, 2)), - new LeftBracket(new Range(1, 2, 1, 3)), - new Word(new Range(1, 3, 1, 3 + 3), 'haa'), - new NewLine(new Range(1, 6, 1, 7)), // a single CR token is treated as a `new line` - new Word(new Range(2, 1, 2, 1 + 3), 'loů'), - new RightBracket(new Range(2, 4, 2, 5)), - new LeftParenthesis(new Range(2, 5, 2, 6)), - new Word(new Range(2, 6, 2, 6 + 1), '.'), - new Slash(new Range(2, 7, 2, 8)), - new Word(new Range(2, 8, 2, 8 + 4), 'real'), - new Slash(new Range(2, 12, 2, 13)), - new Word(new Range(2, 13, 2, 13 + 2), '💁'), - new Slash(new Range(2, 15, 2, 16)), - new Word(new Range(2, 16, 2, 16 + 8), 'name.png'), - new RightParenthesis(new Range(2, 24, 2, 25)), - new NewLine(new Range(2, 25, 2, 26)), - // `2nd` input line - new ExclamationMark(new Range(3, 1, 3, 2)), - new LeftBracket(new Range(3, 2, 3, 3)), - new Word(new Range(3, 3, 3, 3 + 3), 'ref'), - new Space(new Range(3, 6, 3, 7)), - new Word(new Range(3, 7, 3, 7 + 4), 'text'), - new RightBracket(new Range(3, 11, 3, 12)), - new LeftParenthesis(new Range(3, 12, 3, 13)), - new Slash(new Range(3, 13, 3, 14)), - new Word(new Range(3, 14, 3, 14 + 3), 'etc'), - new Slash(new Range(3, 17, 3, 18)), - new Word(new Range(3, 18, 3, 18 + 3), 'pat'), - new NewLine(new Range(3, 21, 3, 22)), // a single CR token is treated as a `new line` - new Word(new Range(4, 1, 4, 1 + 1), 'h'), - new Slash(new Range(4, 2, 4, 3)), - new Word(new Range(4, 3, 4, 3 + 2), 'to'), - new Slash(new Range(4, 5, 4, 6)), - new Word(new Range(4, 6, 4, 6 + 9), 'file.webp'), - new RightParenthesis(new Range(4, 15, 4, 16)), - new NewLine(new Range(4, 16, 4, 17)), - // `3nd` input line - new ExclamationMark(new Range(5, 1, 5, 2)), - new LeftBracket(new Range(5, 2, 5, 3)), - new Word(new Range(5, 3, 5, 3 + 4), 'text'), - new RightBracket(new Range(5, 7, 5, 8)), - new NewLine(new Range(5, 8, 5, 9)), // a single CR token is treated as a `new line` - new LeftParenthesis(new Range(6, 1, 6, 2)), - new Slash(new Range(6, 2, 6, 3)), - new Word(new Range(6, 3, 6, 3 + 3), 'etc'), - new Slash(new Range(6, 6, 6, 7)), - new Space(new Range(6, 7, 6, 8)), - new Word(new Range(6, 8, 6, 8 + 4), 'path'), - new Slash(new Range(6, 12, 6, 13)), - new Word(new Range(6, 13, 6, 13 + 9), 'file.jpeg'), - new RightParenthesis(new Range(6, 22, 6, 23)), - ], - ); - }); - } - }); - - /** - * Same as above but these stop characters do not move the caret to the next line. - */ - suite('stop characters inside caption/reference (same line)', () => { - for (const stopCharacter of [VerticalTab, FormFeed]) { - let characterName = ''; - - if (stopCharacter === VerticalTab) { - characterName = '\\v'; - } - if (stopCharacter === FormFeed) { - characterName = '\\f'; - } - - assert( - characterName !== '', - 'The "characterName" must be set, got "empty line".', - ); - - test(`stop character - "${characterName}"`, async () => { - const test = testDisposables.add( - new TestMarkdownDecoder(), - ); - - const inputLines = [ - // stop character inside link caption - `![haa${stopCharacter.symbol}loů](./real/💁/name)`, - // stop character inside link reference - `![ref text](/etc/pat${stopCharacter.symbol}h/to/file.webp)`, - // stop character between line caption and link reference is disallowed - `![text]${stopCharacter.symbol}(/etc/ path/image.gif)`, - ]; - - - await test.run( - inputLines, - [ - // `1st` input line - new ExclamationMark(new Range(1, 1, 1, 2)), - new LeftBracket(new Range(1, 2, 1, 3)), - new Word(new Range(1, 3, 1, 3 + 3), 'haa'), - new stopCharacter(new Range(1, 6, 1, 7)), // <- stop character - new Word(new Range(1, 7, 1, 7 + 3), 'loů'), - new RightBracket(new Range(1, 10, 1, 11)), - new LeftParenthesis(new Range(1, 11, 1, 12)), - new Word(new Range(1, 12, 1, 12 + 1), '.'), - new Slash(new Range(1, 13, 1, 14)), - new Word(new Range(1, 14, 1, 14 + 4), 'real'), - new Slash(new Range(1, 18, 1, 19)), - new Word(new Range(1, 19, 1, 19 + 2), '💁'), - new Slash(new Range(1, 21, 1, 22)), - new Word(new Range(1, 22, 1, 22 + 4), 'name'), - new RightParenthesis(new Range(1, 26, 1, 27)), - new NewLine(new Range(1, 27, 1, 28)), - // `2nd` input line - new ExclamationMark(new Range(2, 1, 2, 2)), - new LeftBracket(new Range(2, 2, 2, 3)), - new Word(new Range(2, 3, 2, 3 + 3), 'ref'), - new Space(new Range(2, 6, 2, 7)), - new Word(new Range(2, 7, 2, 7 + 4), 'text'), - new RightBracket(new Range(2, 11, 2, 12)), - new LeftParenthesis(new Range(2, 12, 2, 13)), - new Slash(new Range(2, 13, 2, 14)), - new Word(new Range(2, 14, 2, 14 + 3), 'etc'), - new Slash(new Range(2, 17, 2, 18)), - new Word(new Range(2, 18, 2, 18 + 3), 'pat'), - new stopCharacter(new Range(2, 21, 2, 22)), // <- stop character - new Word(new Range(2, 22, 2, 22 + 1), 'h'), - new Slash(new Range(2, 23, 2, 24)), - new Word(new Range(2, 24, 2, 24 + 2), 'to'), - new Slash(new Range(2, 26, 2, 27)), - new Word(new Range(2, 27, 2, 27 + 9), 'file.webp'), - new RightParenthesis(new Range(2, 36, 2, 37)), - new NewLine(new Range(2, 37, 2, 38)), - // `3nd` input line - new ExclamationMark(new Range(3, 1, 3, 2)), - new LeftBracket(new Range(3, 2, 3, 3)), - new Word(new Range(3, 3, 3, 3 + 4), 'text'), - new RightBracket(new Range(3, 7, 3, 8)), - new stopCharacter(new Range(3, 8, 3, 9)), // <- stop character - new LeftParenthesis(new Range(3, 9, 3, 10)), - new Slash(new Range(3, 10, 3, 11)), - new Word(new Range(3, 11, 3, 11 + 3), 'etc'), - new Slash(new Range(3, 14, 3, 15)), - new Space(new Range(3, 15, 3, 16)), - new Word(new Range(3, 16, 3, 16 + 4), 'path'), - new Slash(new Range(3, 20, 3, 21)), - new Word(new Range(3, 21, 3, 21 + 9), 'image.gif'), - new RightParenthesis(new Range(3, 30, 3, 31)), - ], - ); - }); - } - }); - }); - }); - - suite('comments', () => { - suite('general', () => { - test('base cases', async () => { - const test = testDisposables.add( - new TestMarkdownDecoder(), - ); - - const inputData = [ - // comment with text inside it - '\t', - // comment with a link inside - 'some text and more text ', - // comment new lines inside it - ' usual text follows', - // an empty comment - '\t\t', - // comment that was not closed properly - 'haalo\t'), - new NewLine(new Range(1, 22, 1, 23)), - // `2nd` - new Word(new Range(2, 1, 2, 5), 'some'), - new Space(new Range(2, 5, 2, 6)), - new Word(new Range(2, 6, 2, 10), 'text'), - new MarkdownComment(new Range(2, 10, 2, 10 + 46), ''), - new Space(new Range(2, 56, 2, 57)), - new Word(new Range(2, 57, 2, 60), 'and'), - new Space(new Range(2, 60, 2, 61)), - new Word(new Range(2, 61, 2, 65), 'more'), - new Space(new Range(2, 65, 2, 66)), - new Word(new Range(2, 66, 2, 70), 'text'), - new Space(new Range(2, 70, 2, 71)), - new NewLine(new Range(2, 71, 2, 72)), - // `3rd` - new MarkdownComment(new Range(3, 1, 3 + 3, 1 + 13), ''), - new Space(new Range(6, 14, 6, 15)), - new Word(new Range(6, 15, 6, 15 + 5), 'usual'), - new Space(new Range(6, 20, 6, 21)), - new Word(new Range(6, 21, 6, 21 + 4), 'text'), - new Space(new Range(6, 25, 6, 26)), - new Word(new Range(6, 26, 6, 26 + 7), 'follows'), - new NewLine(new Range(6, 33, 6, 34)), - // `4rd` - new Tab(new Range(7, 1, 7, 2)), - new MarkdownComment(new Range(7, 2, 7, 2 + 7), ''), - new Tab(new Range(7, 9, 7, 10)), - new NewLine(new Range(7, 10, 7, 11)), - // `5th` - new Word(new Range(8, 1, 8, 6), 'haalo'), - new Tab(new Range(8, 6, 8, 7)), - new MarkdownComment(new Range(8, 7, 8, 7 + 40), '>', - // comment contains `<[]>` brackets and `!` - '\t\t', - // comment contains `\t\t', - // comment contains `'), - new RightAngleBracket(new Range(1, 19, 1, 20)), - new NewLine(new Range(1, 20, 1, 21)), - // `2nd` - new MarkdownComment(new Range(2, 1, 2, 1 + 21), ''), - new Tab(new Range(2, 22, 2, 23)), - new Tab(new Range(2, 23, 2, 24)), - new NewLine(new Range(2, 24, 2, 25)), - // `3rd` - new VerticalTab(new Range(3, 1, 3, 2)), - new MarkdownComment(new Range(3, 2, 3 + 3, 1 + 7), ''), - new Tab(new Range(6, 8, 6, 9)), - new Tab(new Range(6, 9, 6, 10)), - new NewLine(new Range(6, 10, 6, 11)), - // `4rd` - new Space(new Range(7, 1, 7, 2)), - // note! comment does not have correct closing `-->`, hence the comment extends - // to the end of the text, and therefore includes the \t\v\f and space at the end - new MarkdownComment(new Range(7, 2, 8, 1 + 12), ' ', - ' < !-- світ -->\t', - '\v\f', - '`, hence the comment extends - // to the end of the text, and therefore includes the `space` at the end - new MarkdownComment(new Range(4, 1, 4, 1 + 15), '