diff --git a/.eslint-plugin-local/code-no-observable-get-in-reactive-context.ts b/.eslint-plugin-local/code-no-observable-get-in-reactive-context.ts new file mode 100644 index 00000000000..699b1994595 --- /dev/null +++ b/.eslint-plugin-local/code-no-observable-get-in-reactive-context.ts @@ -0,0 +1,145 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as eslint from 'eslint'; +import { TSESTree } from '@typescript-eslint/utils'; +import * as ESTree from 'estree'; +import * as visitorKeys from 'eslint-visitor-keys'; + +export = new class NoObservableGetInReactiveContext implements eslint.Rule.RuleModule { + meta: eslint.Rule.RuleMetaData = { + type: 'problem', + docs: { + description: 'Disallow calling .get() on observables inside reactive contexts in favor of .read(undefined).', + }, + fixable: 'code', + }; + + create(context: eslint.Rule.RuleContext): eslint.Rule.RuleListener { + return { + 'CallExpression': (node: any) => { + const callExpression = node as TSESTree.CallExpression; + + if (!isReactiveFunctionWithReader(callExpression.callee)) { + return; + } + + const functionArg = callExpression.arguments.find(arg => + arg.type === 'ArrowFunctionExpression' || arg.type === 'FunctionExpression' + ) as TSESTree.ArrowFunctionExpression | TSESTree.FunctionExpression | undefined; + + if (!functionArg) { + return; + } + + const readerName = getReaderParameterName(functionArg); + if (!readerName) { + return; + } + + checkFunctionForObservableGetCalls(functionArg, readerName, context); + } + }; + } +}; + +function checkFunctionForObservableGetCalls( + fn: TSESTree.ArrowFunctionExpression | TSESTree.FunctionExpression, + readerName: string, + context: eslint.Rule.RuleContext +) { + const visited = new Set(); + + function traverse(node: TSESTree.Node) { + if (visited.has(node)) { + return; + } + visited.add(node); + + if (node.type === 'CallExpression' && isObservableGetCall(node)) { + // Flag .get() calls since we're always in a reactive context here + context.report({ + node: node as any as ESTree.Node, + message: `Observable '.get()' should not be used in reactive context. Use '.read(${readerName})' instead to properly track dependencies or '.read(undefined)' to be explicit about an untracked read.`, + fix: (fixer) => { + const memberExpression = node.callee as TSESTree.MemberExpression; + return fixer.replaceText(node as any, `${context.getSourceCode().getText(memberExpression.object as any)}.read(undefined)`); + } + }); + } + + walkChildren(node, traverse); + } + + if (fn.body) { + traverse(fn.body); + } +} + +function isObservableGetCall(node: TSESTree.CallExpression): boolean { + // Look for pattern: something.get() + if (node.callee.type === 'MemberExpression' && + node.callee.property.type === 'Identifier' && + node.callee.property.name === 'get' && + node.arguments.length === 0) { + + // This is a .get() call with no arguments, which is likely an observable + return true; + } + return false; +} + +const reactiveFunctions = new Set([ + 'derived', + 'derivedDisposable', + 'derivedHandleChanges', + 'derivedOpts', + 'derivedWithSetter', + 'derivedWithStore', + 'autorun', + 'autorunOpts', + 'autorunHandleChanges', + 'autorunSelfDisposable', + 'autorunDelta', + 'autorunWithStore', + 'autorunWithStoreHandleChanges', + 'autorunIterableDelta' +]); + +function getReaderParameterName(fn: TSESTree.ArrowFunctionExpression | TSESTree.FunctionExpression): string | null { + if (fn.params.length === 0) { + return null; + } + const firstParam = fn.params[0]; + if (firstParam.type === 'Identifier') { + // Accept any parameter name as a potential reader parameter + // since reactive functions should always have the reader as the first parameter + return firstParam.name; + } + return null; +} + +function isReactiveFunctionWithReader(callee: TSESTree.Node): boolean { + if (callee.type === 'Identifier') { + return reactiveFunctions.has(callee.name); + } + return false; +} + +function walkChildren(node: TSESTree.Node, cb: (child: TSESTree.Node) => void) { + const keys = visitorKeys.KEYS[node.type] || []; + for (const key of keys) { + const child = (node as any)[key]; + if (Array.isArray(child)) { + for (const item of child) { + if (item && typeof item === 'object' && item.type) { + cb(item); + } + } + } else if (child && typeof child === 'object' && child.type) { + cb(child); + } + } +} diff --git a/.eslint-plugin-local/code-no-reader-after-await.ts b/.eslint-plugin-local/code-no-reader-after-await.ts new file mode 100644 index 00000000000..b8c39969c5b --- /dev/null +++ b/.eslint-plugin-local/code-no-reader-after-await.ts @@ -0,0 +1,168 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as eslint from 'eslint'; +import { TSESTree } from '@typescript-eslint/utils'; +import * as ESTree from 'estree'; + +export = new class NoReaderAfterAwait implements eslint.Rule.RuleModule { + create(context: eslint.Rule.RuleContext): eslint.Rule.RuleListener { + return { + 'CallExpression': (node: any) => { + const callExpression = node as TSESTree.CallExpression; + + if (!isFunctionWithReader(callExpression.callee)) { + return; + } + + const functionArg = callExpression.arguments.find(arg => + arg.type === 'ArrowFunctionExpression' || arg.type === 'FunctionExpression' + ) as TSESTree.ArrowFunctionExpression | TSESTree.FunctionExpression | undefined; + + if (!functionArg) { + return; + } + + const readerName = getReaderParameterName(functionArg); + if (!readerName) { + return; + } + + checkFunctionForAwaitBeforeReader(functionArg, readerName, context); + } + }; + } +}; + +function checkFunctionForAwaitBeforeReader( + fn: TSESTree.ArrowFunctionExpression | TSESTree.FunctionExpression, + readerName: string, + context: eslint.Rule.RuleContext +) { + const awaitPositions: { line: number; column: number }[] = []; + const visited = new Set(); + + function collectPositions(node: TSESTree.Node) { + if (visited.has(node)) { + return; + } + visited.add(node); + + if (node.type === 'AwaitExpression') { + awaitPositions.push({ + line: node.loc?.start.line || 0, + column: node.loc?.start.column || 0 + }); + } else if (node.type === 'CallExpression' && isReaderMethodCall(node, readerName)) { + if (awaitPositions.length > 0) { + const methodName = getMethodName(node); + context.report({ + node: node as any as ESTree.Node, + message: `Reader method '${methodName}' should not be called after 'await'. The reader becomes invalid after async operations.` + }); + } + } + + // Safely traverse known node types only + switch (node.type) { + case 'BlockStatement': + node.body.forEach(stmt => collectPositions(stmt)); + break; + case 'ExpressionStatement': + collectPositions(node.expression); + break; + case 'VariableDeclaration': + node.declarations.forEach(decl => { + if (decl.init) { collectPositions(decl.init); } + }); + break; + case 'AwaitExpression': + if (node.argument) { collectPositions(node.argument); } + break; + case 'CallExpression': + node.arguments.forEach(arg => collectPositions(arg)); + break; + case 'IfStatement': + collectPositions(node.test); + collectPositions(node.consequent); + if (node.alternate) { collectPositions(node.alternate); } + break; + case 'TryStatement': + collectPositions(node.block); + if (node.handler) { collectPositions(node.handler.body); } + if (node.finalizer) { collectPositions(node.finalizer); } + break; + case 'ReturnStatement': + if (node.argument) { collectPositions(node.argument); } + break; + case 'BinaryExpression': + case 'LogicalExpression': + collectPositions(node.left); + collectPositions(node.right); + break; + case 'MemberExpression': + collectPositions(node.object); + if (node.computed) { collectPositions(node.property); } + break; + case 'AssignmentExpression': + collectPositions(node.left); + collectPositions(node.right); + break; + } + } + + if (fn.body) { + collectPositions(fn.body); + } +} + +function getMethodName(callExpression: TSESTree.CallExpression): string { + if (callExpression.callee.type === 'MemberExpression' && + callExpression.callee.property.type === 'Identifier') { + return callExpression.callee.property.name; + } + return 'read'; +} + +function isReaderMethodCall(node: TSESTree.CallExpression, readerName: string): boolean { + if (node.callee.type === 'MemberExpression') { + // Pattern 1: reader.read() or reader.readObservable() + if (node.callee.object.type === 'Identifier' && + node.callee.object.name === readerName && + node.callee.property.type === 'Identifier') { + return ['read', 'readObservable'].includes(node.callee.property.name); + } + + // Pattern 2: observable.read(reader) or observable.readObservable(reader) + if (node.callee.property.type === 'Identifier' && + ['read', 'readObservable'].includes(node.callee.property.name)) { + // Check if the reader is passed as the first argument + return node.arguments.length > 0 && + node.arguments[0].type === 'Identifier' && + node.arguments[0].name === readerName; + } + } + return false; +} + +const readerFunctions = new Set(['derived', 'autorun', 'autorunOpts', 'autorunHandleChanges', 'autorunSelfDisposable']); + +function getReaderParameterName(fn: TSESTree.ArrowFunctionExpression | TSESTree.FunctionExpression): string | null { + if (fn.params.length === 0) { + return null; + } + const firstParam = fn.params[0]; + if (firstParam.type === 'Identifier') { + return firstParam.name; + } + return null; +} + +function isFunctionWithReader(callee: TSESTree.Node): boolean { + if (callee.type === 'Identifier') { + return readerFunctions.has(callee.name); + } + return false; +} diff --git a/.eslint-plugin-local/tests/code-no-observable-get-in-reactive-context-test.ts b/.eslint-plugin-local/tests/code-no-observable-get-in-reactive-context-test.ts new file mode 100644 index 00000000000..fd92a45e22b --- /dev/null +++ b/.eslint-plugin-local/tests/code-no-observable-get-in-reactive-context-test.ts @@ -0,0 +1,202 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +// Test file to verify the code-no-observable-get-in-reactive-context ESLint rule works correctly + +import { observableValue, derived, autorun } from '../../src/vs/base/common/observable.js'; + +export function testValidUsage() { + const obs = observableValue('test', 0); + + // Valid: Using .read(reader) in derived + const validDerived = derived(reader => { + const value = obs.read(reader); + return value * 2; + }); + + // Valid: Using .read(reader) in autorun + autorun(rdr => { + const value = validDerived.read(rdr); + console.log('Value:', value); + }); + + // Valid: Using .get() outside reactive context + const outsideValue = obs.get(); + console.log('Outside value:', outsideValue); +} + +export function testInvalidUsage() { + const obs = observableValue('test', 0); + + // Invalid: Using .get() in derived instead of .read(reader) + const invalidDerived = derived(rdr => { + // This should use obs.read(reader) instead + // eslint-disable-next-line local/code-no-observable-get-in-reactive-context + const value = obs.get(); + // Use reader for something valid to avoid unused var warning + const validValue = obs.read(rdr); + + obs.read(undefined); + + return value * 2 + validValue; + }); + + // Invalid: Using .get() in autorun instead of .read(reader) + autorun(reader => { + // This should use invalidDerived.read(reader) instead + // eslint-disable-next-line local/code-no-observable-get-in-reactive-context + const value = invalidDerived.get(); + // Use reader for something valid to avoid unused var warning + const validValue = obs.read(reader); + console.log('Value:', value, validValue); + }); + + // Invalid: Using .get() in derivedWithStore + derived(reader => { + // eslint-disable-next-line local/code-no-observable-get-in-reactive-context + const value = obs.get(); + reader.store.add({ dispose: () => { } }); + return value; + }); +} + +export function testComplexCases() { + const obs1 = observableValue('test1', 0); + const obs2 = observableValue('test2', 10); + + // Invalid: Using .get() in conditional within derived + derived(reader => { + const initial = obs1.read(reader); + + if (initial > 0) { + // eslint-disable-next-line local/code-no-observable-get-in-reactive-context + return obs2.get(); + } + + return initial; + }); + + // Invalid: Using .get() in nested function call within autorun + autorun(reader => { + const process = () => { + // eslint-disable-next-line local/code-no-observable-get-in-reactive-context + return obs1.get() + obs2.get(); + }; + + // Use reader for something valid to avoid unused var warning + const validValue = obs1.read(reader); + const result = process(); + console.log('Result:', result, validValue); + }); + + // Invalid: Using .get() in try-catch within derived + derived(reader => { + try { + // eslint-disable-next-line local/code-no-observable-get-in-reactive-context + const value = obs1.get(); + // Use reader for something valid to avoid unused var warning + const validValue = obs2.read(reader); + return value * 2 + validValue; + } catch (e) { + return obs2.read(reader); + } + }); +} + +export function testValidComplexCases() { + const obs1 = observableValue('test1', 0); + const obs2 = observableValue('test2', 10); + + // Valid: Proper usage with .read(reader) + derived(reader => { + const value1 = obs1.read(reader); + const value2 = obs2.read(undefined); + + if (value1 > 0) { + return value2; + } + + return value1; + }); + + // Valid: Using .get() outside reactive context + function processValues() { + const val1 = obs1.get(); + const val2 = obs2.get(); + return val1 + val2; + } + + // Valid: Mixed usage - .read(reader) inside reactive, .get() outside + autorun(reader => { + const reactiveValue = obs1.read(reader); + const outsideValue = processValues(); + console.log('Values:', reactiveValue, outsideValue); + }); +} + +export function testEdgeCases() { + const obs = observableValue('test', 0); + + // Valid: Function with no reader parameter + derived(() => { + const value = obs.get(); + return value; + }); + + // Invalid: Function with differently named parameter (now also flagged) + derived(_someOtherName => { + // eslint-disable-next-line local/code-no-observable-get-in-reactive-context + const value = obs.get(); + return value; + }); + + // Invalid: Correctly named reader parameter + derived(reader => { + // eslint-disable-next-line local/code-no-observable-get-in-reactive-context + const value = obs.get(); + // Use reader for something valid to avoid unused var warning + const validValue = obs.read(reader); + return value + validValue; + }); +} + +export function testQuickFixScenarios() { + const obs = observableValue('test', 0); + const obs2 = observableValue('test2', 10); + + // These examples show what the quick fix should transform: + + // Example 1: Simple case with 'reader' parameter name + derived(_reader => { + const value = obs.read(undefined); // This should be the auto-fix result + return value; + }); + + // Example 2: Different parameter name + derived(rdr => { + // Before fix: obs2.get() + // After fix: obs2.read(rdr) + const value = obs2.read(rdr); // This should be the auto-fix result + return value; + }); + + // Example 3: Complex expression + derived(ctx => { + // Before fix: (someCondition ? obs : obs2).get() + // After fix: (someCondition ? obs : obs2).read(ctx) + const someCondition = true; + const value = (someCondition ? obs : obs2).read(ctx); // This should be the auto-fix result + return value; + }); + + // Example 4: Multiple calls in same function + autorun(reader => { + // Before fix: obs.get() and obs2.get() + // After fix: obs.read(reader) and obs2.read(reader) + const val1 = obs.read(reader); // This should be the auto-fix result + const val2 = obs2.read(reader); // This should be the auto-fix result + console.log(val1, val2); + }); +} diff --git a/.eslint-plugin-local/tests/code-no-reader-after-await-test.ts b/.eslint-plugin-local/tests/code-no-reader-after-await-test.ts new file mode 100644 index 00000000000..2dc0f457a7f --- /dev/null +++ b/.eslint-plugin-local/tests/code-no-reader-after-await-test.ts @@ -0,0 +1,86 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +// Test file to verify the code-no-reader-after-await ESLint rule works correctly + +import { observableValue, derived, autorun } from '../../src/vs/base/common/observable.js'; + +export function testValidUsage() { + const obs = observableValue('test', 0); + + const validDerived = derived(reader => { + const value = obs.read(reader); + return value * 2; + }); + + autorun(reader => { + const value = validDerived.read(reader); + console.log('Value:', value); + }); +} + +export function testInvalidUsage() { + const obs = observableValue('test', 0); + + const invalidDerived = derived(async reader => { + await Promise.resolve(); + // eslint-disable-next-line local/code-no-reader-after-await + const value = obs.read(reader); + return value * 2; + }); + + autorun(async reader => { + await Promise.resolve(); + // eslint-disable-next-line local/code-no-reader-after-await + const value = invalidDerived.read(reader); + console.log('Value:', value); + }); + + autorun(async reader => { + await Promise.resolve(); + // eslint-disable-next-line local/code-no-reader-after-await + const value = reader.readObservable(obs); + console.log('Value:', value); + }); +} + +export function testComplexCases() { + const obs = observableValue('test', 0); + + derived(async reader => { + const initial = obs.read(reader); + + if (initial > 0) { + await Promise.resolve(); + } + + // eslint-disable-next-line local/code-no-reader-after-await + const final = obs.read(reader); + return final; + }); + + autorun(async reader => { + try { + await Promise.resolve(); + } catch (e) { + } finally { + // eslint-disable-next-line local/code-no-reader-after-await + const value = obs.read(reader); + console.log(value); + } + }); +} + +export function testValidComplexCases() { + const obs = observableValue('test', 0); + + derived(async reader => { + const value1 = obs.read(reader); + const value2 = reader.readObservable(obs); + const result = value1 + value2; + await Promise.resolve(result); + return result; + }); +} diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index a17472617fc..4c7aff83c70 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -61,8 +61,7 @@ You MUST check compilation output before running ANY script or declaring work co - Start the task if it's not already running in the background ### TypeScript validation steps -- Use run test tool or `scripts/test.sh` (`scripts\test.bat` on Windows) for unit tests (add `--grep ` to filter tests) -- Use `scripts/test-integration.sh` (or `scripts\test-integration.bat` on Windows) for integration tests +- Use the run test tool if you need to run tests. If that tool is not available, then you can use `scripts/test.sh` (or `scripts\test.bat` on Windows) for unit tests (add `--grep ` to filter tests) or `scripts/test-integration.sh` (or `scripts\test-integration.bat` on Windows) for integration tests (integration tests end with .integrationTest.ts or are in /extensions/). - Use `npm run valid-layers-check` to check for layering issues ## Coding Guidelines @@ -92,7 +91,8 @@ We use tabs, not spaces. - Use "double quotes" for strings shown to the user that need to be externalized (localized) - Use 'single quotes' otherwise -- All strings visible to the user need to be externalized +- All strings visible to the user need to be externalized using the `vs/nls` module +- Externalized strings must not use string concatenation. Use placeholders instead (`{0}`). ### UI labels - Use title-style capitalization for command labels, buttons and menu items (each word is capitalized). diff --git a/.github/instructions/disposable.instructions.md b/.github/instructions/disposable.instructions.md new file mode 100644 index 00000000000..06e1b62f7e3 --- /dev/null +++ b/.github/instructions/disposable.instructions.md @@ -0,0 +1,20 @@ +--- +description: Guidelines for writing code using IDisposable +--- + +Core symbols: +* `IDisposable` + * `dispose(): void` - dispose the object +* `Disposable` (implements `IDisposable`) - base class for disposable objects + * `this._store: DisposableStore` + * `this._register(t: T): T` + * Try to immediately register created disposables! E.g. `const someDisposable = this._register(new SomeDisposable())` +* `DisposableStore` (implements `IDisposable`) + * `add(t: T): T` + * `clear()` +* `toDisposable(fn: () => void): IDisposable` - helper to create a disposable from a function + +* `MutableDisposable` (implements `IDisposable`) + * `value: IDisposable | undefined` + * `clear()` + * A value that enters a mutable disposable (at least once) will be disposed the latest when the mutable disposable is disposed (or when the value is replaced or cleared). diff --git a/.github/instructions/learnings.instructions.md b/.github/instructions/learnings.instructions.md new file mode 100644 index 00000000000..78a9f52a06e --- /dev/null +++ b/.github/instructions/learnings.instructions.md @@ -0,0 +1,32 @@ +--- +applyTo: ** +description: This document describes how to deal with learnings that you make. (meta instruction) +--- + +This document describes how to deal with learnings that you make. +It is a meta-instruction file. + +Structure of learnings: +* Each instruction file has a "Learnings" section. +* Each learning has a counter that indicates how often that learning was useful (initially 1). +* Each learning has a 1-4 sentences description of the learning. + +Example: +```markdown +## Learnings +* Prefer `const` over `let` whenever possible (1) +* Avoid `any` type (3) +``` + +When the user tells you "learn!", you should: +* extract a learning from the recent conversation + * identify the problem that you created + * identify why it was a problem + * identify how you were told to fix it/how the user fixed it +* create a learning (1-4 sentences) from that + * Write this out to the user and reflect over these sentences + * then, add the reflected learning to the "Learnings" section of the most appropriate instruction file + + + Important: Whenever a learning was really useful, increase the counter!! + When a learning was not useful and just caused more problems, decrease the counter. diff --git a/.github/instructions/observables.instructions.md b/.github/instructions/observables.instructions.md new file mode 100644 index 00000000000..2aedc290be1 --- /dev/null +++ b/.github/instructions/observables.instructions.md @@ -0,0 +1,72 @@ +--- +description: Guidelines for writing code using observables and deriveds. +--- + +```ts +class MyService extends Disposable { + private _myData1 = observableValue(/* always put `this` here */ this, /* initial value*/ 0); + private _myData2 = observableValue(/* always put `this` here */ this, /* initial value*/ 42); + + // Deriveds can combine/derive from other observables/deriveds + private _myDerivedData = derived(this, reader => { + // Use observable.read(reader) to access the value and track the dependency. + return this._myData1.read(reader) * this._myData2.read(reader); + }); + + private _myDerivedDataWithLifetime = derived(this, reader => { + // The reader.store will get cleared just before the derived is re-evaluated or gets unsubscribed. + return reader.store.add(new SomeDisposable(this._myDerivedData.read(reader))); + }); + + constructor() { + this._register(autorun((reader) => { // like mobx autorun, they run immediately and on change + const data = this._myData1.read(reader); // but you only get the data if you pass in the reader! + + console.log(data); + + // also has reader.store + })) + } + + getData(): number { + return this._myData1.get(); // use get if you don't have a reader, but try to avoid it since the dependency is not tracked. + } + + setData1() { + this._myData1.set(42, undefined); // use set to update the value. The second paramater is the transaction, which is undefined here. + } + + setData2() { + transaction(tx => { + // you can use transaction to batch updates, so they are only notified once. + // Whenever multiple observables are synchronously updated together, use transaction! + this._myData1.set(42, tx); + this._myData2.set(43, tx); + }); + } +} +``` + + +Most important symbols: +* `observableValue` +* `disposableObservableValue` +* `derived` +* `autorun` +* `transaction` +* `observableFromEvent` +* `observableSignalFromEvent` +* `observableSignal(...): IObservable` - use `.trigger(tx)` to trigger a change + + +* Check src\vs\base\common\observableInternal\index.ts for a list of all observable utitilies + + +* Important learnings: + * [1] Avoid glitches + * [2] **Choose the right observable value type:** + * Use `observableValue(owner, initialValue)` for regular values + * Use `disposableObservableValue(owner, initialValue)` when storing disposable values - it automatically disposes the previous value when a new one is set, and disposes the current value when the observable itself is disposed (similar to `MutableDisposable` behavior) + * [3] **Choose the right event observable pattern:** + * Use `observableFromEvent(owner, event, valueComputer)` when you need to track a computed value that changes with the event, and you want updates only when the computed value actually changes + * Use `observableSignalFromEvent(owner, event)` when you need to force re-computation every time the event fires, regardless of value stability. This is important when the computed value might not change but dependent computations need fresh context (e.g., workspace folder changes where the folder array reference might be the same but file path calculations need to be refreshed) diff --git a/.github/instructions/tree-widgets.instructions.md b/.github/instructions/tree-widgets.instructions.md new file mode 100644 index 00000000000..b5df37f75e6 --- /dev/null +++ b/.github/instructions/tree-widgets.instructions.md @@ -0,0 +1,157 @@ +--- +description: Use when asked to consume workbench tree widgets in VS Code. +--- + +# Workbench Tree Widgets Overview + +**Location**: `src/vs/platform/list/browser/listService.ts` +**Type**: Platform Services +**Layer**: Platform + +## Purpose + +The Workbench Tree Widgets provide high-level, workbench-integrated tree components that extend the base tree implementations with VS Code-specific functionality like context menus, keyboard navigation, theming, accessibility, and dependency injection integration. These widgets serve as the primary tree components used throughout the VS Code workbench for file explorers, debug views, search results, and other hierarchical data presentations. + +## Scope + +### Included Functionality +- **Context Integration**: Automatic context key management, focus handling, and VS Code theme integration +- **Resource Navigation**: Built-in support for opening files and resources with proper editor integration +- **Accessibility**: Complete accessibility provider integration with screen reader support +- **Keyboard Navigation**: Smart keyboard navigation with search-as-you-type functionality +- **Multi-selection**: Configurable multi-selection behavior with platform-appropriate modifier keys +- **Dependency Injection**: Full integration with VS Code's service container for automatic service injection +- **Configuration**: Automatic integration with user settings for tree behavior customization + +### Integration Points +- **IInstantiationService**: For service injection and component creation +- **IContextKeyService**: For managing focus, selection, and tree state context keys +- **IListService**: For registering trees and managing workbench list lifecycle +- **IConfigurationService**: For reading tree configuration settings +- **Resource Navigators**: For handling file/resource opening with proper editor integration + +### Out of Scope +- Low-level tree rendering and virtualization (handled by base tree classes) +- Data management and async loading logic (provided by data sources) +- Custom styling beyond workbench theming integration + +## Architecture + +### Key Classes & Interfaces + +- **WorkbenchTreeInternals**: Encapsulates common workbench functionality across all tree types +- **ResourceNavigator**: Handles file/resource opening with proper editor integration +- **IOpenEvent**: Event interface for resource opening with editor options +- **IWorkbench*TreeOptions**: Configuration interfaces extending base options with workbench features +- **IResourceNavigatorOptions**: Configuration for resource opening behavior + +### Key Files + +- **`src/vs/platform/list/browser/listService.ts`**: Contains all workbench tree widget implementations, shared workbench functionality (`WorkbenchTreeInternals`), and configuration utilities + - `src/vs/platform/list/browser/test/listService.test.ts`: Unit tests for workbench trees +- **`src/vs/base/browser/ui/tree/objectTree.ts`**: Base implementation for static trees and compressible trees + - `src/vs/base/test/browser/ui/tree/objectTree.test.ts`: Base tree tests +- **`src/vs/base/browser/ui/tree/asyncDataTree.ts`**: Base implementation for async trees with lazy loading support + - `src/vs/base/test/browser/ui/tree/asyncDataTree.test.ts`: Async tree tests +- **`src/vs/base/browser/ui/tree/dataTree.ts`**: Base implementation for data-driven trees with explicit data sources + - `src/vs/base/test/browser/ui/tree/dataTree.test.ts`: Data tree tests +- **`src/vs/base/browser/ui/tree/abstractTree.ts`**: Base tree foundation +- **`src/vs/base/browser/ui/tree/tree.ts`**: Core interfaces and types + +## Development Guidelines + +### Choosing the Right Tree Widget + +1. **WorkbenchObjectTree**: Use for simple, static hierarchical data that doesn't change frequently + ```typescript + // Example: Timeline items, loaded scripts + const tree = instantiationService.createInstance( + WorkbenchObjectTree, + 'TimelineView', container, delegate, renderers, options + ); + ``` + +2. **WorkbenchAsyncDataTree**: Use for dynamic data that loads asynchronously + ```typescript + // Example: Debug variables, file contents + const tree = instantiationService.createInstance( + WorkbenchAsyncDataTree, + 'VariablesView', container, delegate, renderers, dataSource, options + ); + ``` + +3. **WorkbenchCompressible*Tree**: Use when you need path compression for deep hierarchies + ```typescript + // Example: File explorer, call stack + const tree = instantiationService.createInstance( + WorkbenchCompressibleAsyncDataTree, + 'FileExplorer', container, delegate, compressionDelegate, renderers, dataSource, options + ); + ``` + +### Construction Pattern + +**Always use IInstantiationService.createInstance()** to ensure proper dependency injection: + +```typescript +constructor( + @IInstantiationService private instantiationService: IInstantiationService +) { + this.tree = this.instantiationService.createInstance( + WorkbenchAsyncDataTree, + 'UniqueTreeId', // Used for settings and context keys + container, // DOM container element + delegate, // IListVirtualDelegate for item height/template + renderers, // Array of tree renderers + dataSource, // Data source (async trees only) + options // Tree configuration options + ); +} +``` + +### Required Options + +All workbench trees require an **accessibilityProvider**: +```typescript +const options: IWorkbenchAsyncDataTreeOptions = { + accessibilityProvider: { + getAriaLabel: (element: T) => element.name, + getRole: () => 'treeitem' + } + // ... other options +}; +``` + +### Common Configuration Patterns + +```typescript +// Standard tree setup with search, identity, and navigation +const options = { + accessibilityProvider: new MyAccessibilityProvider(), + identityProvider: { getId: (element) => element.id }, + keyboardNavigationLabelProvider: { + getKeyboardNavigationLabel: (element) => element.name + }, + multipleSelectionController: { + isSelectionSingleChangeEvent: (e) => e.ctrlKey || e.metaKey, + isSelectionRangeChangeEvent: (e) => e.shiftKey + }, + overrideStyles: this.getLocationBasedColors().listOverrideStyles +}; +``` + +### Lifecycle Management + +- **Always register trees as disposables** in the containing component +- **Use the tree's `setInput()` method** to provide initial data +- **Always call `layout()` when the container initializes and when its size changes** +- **Handle selection and open events** through the tree's event system + +### Performance Considerations + +- Use **compression** for deep hierarchies to reduce DOM nodes +- Implement **efficient data sources** that avoid unnecessary data fetching +- Consider **virtualization settings** for large datasets +- Use **identity providers** for efficient updates and state preservation + +--- diff --git a/.github/prompts/component.prompt.md b/.github/prompts/component.prompt.md new file mode 100644 index 00000000000..9aa2eb6d176 --- /dev/null +++ b/.github/prompts/component.prompt.md @@ -0,0 +1,58 @@ +--- +mode: agent +description: 'Help author a component specification for an agent.' +tools: ['edit', 'search', 'usages', 'vscodeAPI', 'fetch', 'extensions', 'todos'] +--- + + +Your goal is to create a component overview in markdown given the context provided by the user. The overview should include a brief description of the component, its main features, an architectural diagram and layout of important code files and their relationships. The purpose of this overview is to enable a developer to attach it to a feature request and ensure the agent has enough context to make correct code changes without breaking functionality. + + + +# [Component Name] Overview + +**Location**: `src/vs/[path/to/component]` +**Type**: [Service/Contribution/Extension/API/etc.] +**Layer (if applicable)**: [base/platform/editor/workbench/code/server] + +## Purpose + +Brief description of what this component does and why it exists. + +## Scope +- What functionality is included +- What is explicitly out of scope +- Integration points with other components + +## Architecture + +### High-Level Design +[Architectural diagram or description of key patterns used] + +### Key Classes & Interfaces +- **[ClassName]**: Brief description of responsibility +- **[InterfaceName]**: Purpose and main methods +- **[ServiceName]**: Service responsibilities + +### Key Files +List all the key files and a brief description of their purpose: +- **`src/vs/[path/to/component]/[filename.ts]`**: [Purpose and main exports] +- **`src/vs/[path/to/component]/[service.ts]`**: [Service implementation details] +- **`src/vs/[path/to/component]/[contribution.ts]`**: [Workbench contributions] + +## Development Guidelines + +- Reserve a section for any specific development practices or patterns relevant to this component. These will be edited by a developer or agent as needed. + +--- + + + +- **Create** a new overview file if one is not specified: `.components/[component-name].md` +- **Fill** each section with component-specific details +- **Gather** information from the attached context and use available tools if needed to complete your understanding +- **Ask** the user for clarification if you cannot fill out a section with accurate information +- **Use complete file paths** from repository root (e.g., `src/vs/workbench/services/example/browser/exampleService.ts`) +- **Keep** descriptions concise but comprehensive +- **Use file references** instead of code snippets when making references to code as otherwise the code may become outdated + diff --git a/.github/prompts/data.prompt.md b/.github/prompts/data.prompt.md new file mode 100644 index 00000000000..527e8b440ec --- /dev/null +++ b/.github/prompts/data.prompt.md @@ -0,0 +1,50 @@ +--- +mode: agent +description: 'Answer data questions by querying telemetry docs and Kusto data.' +tools: ['edit', 'search', 'extensions', 'fetch', 'usages', 'runCommands', 'todos', 'kusto'] +--- + + +Your goal is to answer questions about VS Code telemetry data, events, properties, and related documentation by using the context from the vscode-telemetry-docs repository. + + + +Before answering any telemetry-related questions: + +1. **Check for Kusto tool**: Verify that the `kusto` tool is available for querying telemetry data + - If the Kusto tool is not available, inform the user to install the "Azure MCP Server" VS Code extension + - This extension provides access to Kusto/Azure Data Explorer functionality needed for telemetry queries + +2. **Check for telemetry docs**: First verify that the `vscode-telemetry-docs/` folder exists in the workspace + - If it doesn't exist, inform the user to run `npm run mixin-telemetry-docs` to clone the telemetry documentation + - Wait for the user to run this command before proceeding + +3. **Read context**: Once the folder exists, read the file `vscode-telemetry-docs/.github/copilot-instructions.md` to understand: + - The structure and purpose of the telemetry documentation + - How to navigate and interpret the telemetry data + - Key concepts and terminology used in VS Code telemetry + +4. **Run actual queries**: Don't just describe what could be queried - actually execute Kusto queries to provide real data and insights: + - Use the appropriate Kusto cluster and database for the data type + - Always include proper time filtering to limit data volume + - Default to a rolling 28-day window if no specific timeframe is requested + - Format and present the query results clearly to answer the user's question + +5. **Use proper time windows**: When no specific timeframe is provided: + - Default to a rolling 28-day window (standard practice in VS Code telemetry) + - Use full day boundaries to avoid partial day data + - Follow the time filtering patterns from the telemetry documentation + +6. **Be specific**: Reference specific files, sections, or examples from the telemetry docs when possible to support your answers + +7. **Stay focused**: Keep answers focused on telemetry-related topics and data questions + + + +Your response should include: +- The actual Kusto query executed (formatted nicely) +- Real query results with data to answer the user's question +- Interpretation and analysis of the results +- References to specific documentation files when applicable +- Additional context or insights from the telemetry data + diff --git a/.github/prompts/fixIssueNo.prompt.md b/.github/prompts/fixIssueNo.prompt.md new file mode 100644 index 00000000000..6438f4614fe --- /dev/null +++ b/.github/prompts/fixIssueNo.prompt.md @@ -0,0 +1,8 @@ +--- +mode: Plan +tools: ['runCommands', 'runTasks', 'runNotebooks', 'search', 'new', 'usages', 'vscodeAPI', 'problems', 'testFailure', 'openSimpleBrowser', 'fetch', 'githubRepo', 'todos', 'runTests', 'get_issue', 'get_issue_comments', 'get_me', 'get_pull_request', 'get_pull_request_diff', 'get_pull_request_files'] +--- + +The user has given you a Github issue number. Use the `get_issue` to retrieve its details. Understand the issue and propose a solution to solve it. + +NEVER share any thinking process or status updates before you have your solution. diff --git a/.github/prompts/update-instructions.prompt.md b/.github/prompts/update-instructions.prompt.md new file mode 100644 index 00000000000..8ba53454822 --- /dev/null +++ b/.github/prompts/update-instructions.prompt.md @@ -0,0 +1,17 @@ +--- +mode: agent +--- + +Read the changes introduced on the current branch, including BOTH: + +1. Uncommitted workspace modifications (staged and unstaged) +2. Committed changes that are on the current HEAD but not yet in the default upstream branch (e.g. `origin/main`) + +Guidance: + +- First, capture uncommitted diffs (equivalent of `git diff` and `git diff --cached`). +- Then, determine the merge base with the default branch (assume `origin/main` unless configured otherwise) using `git merge-base HEAD origin/main` and diff (`git diff ...HEAD`) to include committed-but-unpushed work. + +After understanding all of these changes, read every instruction file under `.github/instructions` and assess whether any instruction is invalidated. If so, propose minimal, necessary wording updates. If no updates are needed, respond exactly with: `No updates needed`. + +Be concise and conservative: only suggest changes that are absolutely necessary. diff --git a/.github/workflows/copilot-setup-steps.yml b/.github/workflows/copilot-setup-steps.yml new file mode 100644 index 00000000000..b520069a389 --- /dev/null +++ b/.github/workflows/copilot-setup-steps.yml @@ -0,0 +1,260 @@ +name: "Copilot Setup Steps" + +# Automatically run the setup steps when they are changed to allow for easy validation, and +# allow manual testing through the repository's "Actions" tab +on: + workflow_dispatch: + push: + paths: + - .github/workflows/copilot-setup-steps.yml + pull_request: + paths: + - .github/workflows/copilot-setup-steps.yml + +jobs: + # The job MUST be called `copilot-setup-steps` or it will not be picked up by Copilot. + copilot-setup-steps: + runs-on: vscode-large-runners + + # Set the permissions to the lowest permissions possible needed for your steps. + # Copilot will be given its own token for its operations. + permissions: + # If you want to clone the repository as part of your setup steps, for example to install dependencies, you'll need the `contents: read` permission. If you don't clone the repository in your setup steps, Copilot will do this for you automatically after the steps complete. + contents: read + + # You can define any steps you want, and they will run before the agent starts. + # If you do not check out your code, Copilot will do this for you. + steps: + - name: Checkout microsoft/vscode + uses: actions/checkout@v5 + + - name: Setup Node.js + uses: actions/setup-node@v5 + with: + node-version-file: .nvmrc + + - name: Setup system services + run: | + set -e + # Start X server + ./build/azure-pipelines/linux/apt-retry.sh sudo apt-get update + ./build/azure-pipelines/linux/apt-retry.sh sudo apt-get install -y pkg-config \ + xvfb \ + libgtk-3-0 \ + libxkbfile-dev \ + libkrb5-dev \ + libgbm1 \ + rpm + sudo cp build/azure-pipelines/linux/xvfb.init /etc/init.d/xvfb + sudo chmod +x /etc/init.d/xvfb + sudo update-rc.d xvfb defaults + sudo service xvfb start + + - name: Prepare node_modules cache key + run: mkdir -p .build && node build/azure-pipelines/common/computeNodeModulesCacheKey.js linux x64 $(node -p process.arch) > .build/packagelockhash + + - name: Restore node_modules cache + id: cache-node-modules + uses: actions/cache/restore@v4 + with: + path: .build/node_modules_cache + key: "node_modules-linux-${{ hashFiles('.build/packagelockhash') }}" + + - name: Extract node_modules cache + if: steps.cache-node-modules.outputs.cache-hit == 'true' + run: tar -xzf .build/node_modules_cache/cache.tgz + + - name: Install build dependencies + if: steps.cache-node-modules.outputs.cache-hit != 'true' + working-directory: build + run: | + set -e + + for i in {1..5}; do # try 5 times + npm ci && break + if [ $i -eq 5 ]; then + echo "Npm install failed too many times" >&2 + exit 1 + fi + echo "Npm install failed $i, trying again..." + done + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Install dependencies + if: steps.cache-node-modules.outputs.cache-hit != 'true' + run: | + set -e + + source ./build/azure-pipelines/linux/setup-env.sh + + for i in {1..5}; do # try 5 times + npm ci && break + if [ $i -eq 5 ]; then + echo "Npm install failed too many times" >&2 + exit 1 + fi + echo "Npm install failed $i, trying again..." + done + env: + npm_config_arch: x64 + VSCODE_ARCH: x64 + ELECTRON_SKIP_BINARY_DOWNLOAD: 1 + PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Create node_modules archive + if: steps.cache-node-modules.outputs.cache-hit != 'true' + run: | + set -e + node build/azure-pipelines/common/listNodeModules.js .build/node_modules_list.txt + mkdir -p .build/node_modules_cache + tar -czf .build/node_modules_cache/cache.tgz --files-from .build/node_modules_list.txt + + - name: Create .build folder + run: mkdir -p .build + + - name: Prepare built-in extensions cache key + run: node build/azure-pipelines/common/computeBuiltInDepsCacheKey.js > .build/builtindepshash + + - name: Restore built-in extensions cache + id: cache-builtin-extensions + uses: actions/cache/restore@v4 + with: + enableCrossOsArchive: true + path: .build/builtInExtensions + key: "builtin-extensions-${{ hashFiles('.build/builtindepshash') }}" + + - name: Download built-in extensions + if: steps.cache-builtin-extensions.outputs.cache-hit != 'true' + run: node build/lib/builtInExtensions.js + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # - name: Transpile client and extensions + # run: npm run gulp transpile-client-esbuild transpile-extensions + + - name: Download Electron and Playwright + run: | + set -e + + for i in {1..3}; do # try 3 times (matching retryCountOnTaskFailure: 3) + if npm exec -- npm-run-all -lp "electron x64" "playwright-install"; then + echo "Download successful on attempt $i" + break + fi + + if [ $i -eq 3 ]; then + echo "Download failed after 3 attempts" >&2 + exit 1 + fi + + echo "Download failed on attempt $i, retrying..." + sleep 5 # optional: add a small delay between retries + done + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # - name: 🧪 Run unit tests (Electron) + # if: ${{ inputs.electron_tests }} + # timeout-minutes: 15 + # run: ./scripts/test.sh --tfs "Unit Tests" + # env: + # DISPLAY: ":10" + + # - name: 🧪 Run unit tests (node.js) + # if: ${{ inputs.electron_tests }} + # timeout-minutes: 15 + # run: npm run test-node + + # - name: 🧪 Run unit tests (Browser, Chromium) + # if: ${{ inputs.browser_tests }} + # timeout-minutes: 30 + # run: npm run test-browser-no-install -- --browser chromium --tfs "Browser Unit Tests" + # env: + # DEBUG: "*browser*" + + # - name: Build integration tests + # run: | + # set -e + # npm run gulp \ + # compile-extension:configuration-editing \ + # compile-extension:css-language-features-server \ + # compile-extension:emmet \ + # compile-extension:git \ + # compile-extension:github-authentication \ + # compile-extension:html-language-features-server \ + # compile-extension:ipynb \ + # compile-extension:notebook-renderers \ + # compile-extension:json-language-features-server \ + # compile-extension:markdown-language-features \ + # compile-extension-media \ + # compile-extension:microsoft-authentication \ + # compile-extension:typescript-language-features \ + # compile-extension:vscode-api-tests \ + # compile-extension:vscode-colorize-tests \ + # compile-extension:vscode-colorize-perf-tests \ + # compile-extension:vscode-test-resolver + + # - name: 🧪 Run integration tests (Electron) + # if: ${{ inputs.electron_tests }} + # timeout-minutes: 20 + # run: ./scripts/test-integration.sh --tfs "Integration Tests" + # env: + # DISPLAY: ":10" + + # - name: 🧪 Run integration tests (Browser, Chromium) + # if: ${{ inputs.browser_tests }} + # timeout-minutes: 20 + # run: ./scripts/test-web-integration.sh --browser chromium + + # - name: 🧪 Run integration tests (Remote) + # if: ${{ inputs.remote_tests }} + # timeout-minutes: 20 + # run: ./scripts/test-remote-integration.sh + # env: + # DISPLAY: ":10" + + # - name: Compile smoke tests + # working-directory: test/smoke + # run: npm run compile + + # - name: Compile extensions for smoke tests + # run: npm run gulp compile-extension-media + + # - name: Diagnostics before smoke test run (processes, max_user_watches, number of opened file handles) + # run: | + # set -e + # ps -ef + # cat /proc/sys/fs/inotify/max_user_watches + # lsof | wc -l + # continue-on-error: true + # if: always() + + # - name: 🧪 Run smoke tests (Electron) + # if: ${{ inputs.electron_tests }} + # timeout-minutes: 20 + # run: npm run smoketest-no-compile -- --tracing + # env: + # DISPLAY: ":10" + + # - name: 🧪 Run smoke tests (Browser, Chromium) + # if: ${{ inputs.browser_tests }} + # timeout-minutes: 20 + # run: npm run smoketest-no-compile -- --web --tracing --headless + + # - name: 🧪 Run smoke tests (Remote) + # if: ${{ inputs.remote_tests }} + # timeout-minutes: 20 + # run: npm run smoketest-no-compile -- --remote --tracing + # env: + # DISPLAY: ":10" + + # - name: Diagnostics after smoke test run (processes, max_user_watches, number of opened file handles) + # run: | + # set -e + # ps -ef + # cat /proc/sys/fs/inotify/max_user_watches + # lsof | wc -l + # continue-on-error: true + # if: always() diff --git a/.gitignore b/.gitignore index 62394c60784..92971a7a573 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,4 @@ product.overrides.json *.snap.actual *.tsbuildinfo .vscode-test +vscode-telemetry-docs/ diff --git a/.npmrc b/.npmrc index c8d1ede6bc4..749c3c543e2 100644 --- a/.npmrc +++ b/.npmrc @@ -1,6 +1,6 @@ disturl="https://electronjs.org/headers" target="37.3.1" -ms_build_id="12342881" +ms_build_id="12404162" runtime="electron" build_from_source="true" legacy-peer-deps="true" diff --git a/build/azure-pipelines/alpine/cli-build-alpine.yml b/build/azure-pipelines/alpine/cli-build-alpine.yml index 95dee012c06..6180a7efd10 100644 --- a/build/azure-pipelines/alpine/cli-build-alpine.yml +++ b/build/azure-pipelines/alpine/cli-build-alpine.yml @@ -12,6 +12,8 @@ parameters: default: false steps: + - template: ../common/checkout.yml@self + - task: NodeTool@0 inputs: versionSource: fromFile diff --git a/build/azure-pipelines/alpine/product-build-alpine.yml b/build/azure-pipelines/alpine/product-build-alpine.yml index 303db76fd0f..aa5dd83b64a 100644 --- a/build/azure-pipelines/alpine/product-build-alpine.yml +++ b/build/azure-pipelines/alpine/product-build-alpine.yml @@ -1,4 +1,6 @@ steps: + - template: ../common/checkout.yml@self + - task: NodeTool@0 inputs: versionSource: fromFile diff --git a/build/azure-pipelines/common/checkout.yml b/build/azure-pipelines/common/checkout.yml new file mode 100644 index 00000000000..6f57a9ad9b4 --- /dev/null +++ b/build/azure-pipelines/common/checkout.yml @@ -0,0 +1,5 @@ +steps: + - checkout: self + fetchDepth: 1 + fetchTags: false + displayName: Checkout microsoft/vscode diff --git a/build/azure-pipelines/common/publish-artifact.yml b/build/azure-pipelines/common/publish-artifact.yml index b18dc8d4c7f..ba4d9f13355 100644 --- a/build/azure-pipelines/common/publish-artifact.yml +++ b/build/azure-pipelines/common/publish-artifact.yml @@ -18,9 +18,6 @@ parameters: - name: sbomPackageVersion type: string default: "" - - name: isProduction - type: boolean - default: true - name: condition type: string default: succeeded() @@ -80,7 +77,6 @@ steps: targetPath: ${{ parameters.targetPath }} artifactName: $(ARTIFACT_NAME) sbomEnabled: ${{ parameters.sbomEnabled }} - isProduction: ${{ parameters.isProduction }} ${{ if ne(parameters.sbomBuildDropPath, '') }}: sbomBuildDropPath: ${{ parameters.sbomBuildDropPath }} ${{ if ne(parameters.sbomPackageName, '') }}: diff --git a/build/azure-pipelines/darwin/cli-build-darwin.yml b/build/azure-pipelines/darwin/cli-build-darwin.yml index 730918f5da1..fe44f2827fd 100644 --- a/build/azure-pipelines/darwin/cli-build-darwin.yml +++ b/build/azure-pipelines/darwin/cli-build-darwin.yml @@ -12,6 +12,8 @@ parameters: default: false steps: + - template: ../common/checkout.yml@self + - task: NodeTool@0 inputs: versionSource: fromFile @@ -71,9 +73,7 @@ steps: targetPath: $(Build.ArtifactStagingDirectory)/unsigned_vscode_cli_darwin_x64_cli.zip artifactName: unsigned_vscode_cli_darwin_x64_cli displayName: Publish unsigned_vscode_cli_darwin_x64_cli artifact - sbomBuildDropPath: $(Build.ArtifactStagingDirectory)/cli - sbomPackageName: "VS Code macOS x64 CLI (unsigned)" - sbomPackageVersion: $(Build.SourceVersion) + sbomEnabled: false - ${{ if eq(parameters.VSCODE_BUILD_MACOS_ARM64, true) }}: - template: ../common/publish-artifact.yml@self @@ -81,6 +81,4 @@ steps: targetPath: $(Build.ArtifactStagingDirectory)/unsigned_vscode_cli_darwin_arm64_cli.zip artifactName: unsigned_vscode_cli_darwin_arm64_cli displayName: Publish unsigned_vscode_cli_darwin_arm64_cli artifact - sbomBuildDropPath: $(Build.ArtifactStagingDirectory)/cli - sbomPackageName: "VS Code macOS arm64 CLI (unsigned)" - sbomPackageVersion: $(Build.SourceVersion) + sbomEnabled: false diff --git a/build/azure-pipelines/darwin/product-build-darwin-cli-sign.yml b/build/azure-pipelines/darwin/product-build-darwin-cli-sign.yml index 505534093d0..a622547e2de 100644 --- a/build/azure-pipelines/darwin/product-build-darwin-cli-sign.yml +++ b/build/azure-pipelines/darwin/product-build-darwin-cli-sign.yml @@ -5,6 +5,8 @@ parameters: type: boolean steps: + - template: ../common/checkout.yml@self + - task: NodeTool@0 inputs: versionSource: fromFile diff --git a/build/azure-pipelines/darwin/product-build-darwin-test.yml b/build/azure-pipelines/darwin/product-build-darwin-test.yml index 3d1dfdf8ea3..f2b5e697c4d 100644 --- a/build/azure-pipelines/darwin/product-build-darwin-test.yml +++ b/build/azure-pipelines/darwin/product-build-darwin-test.yml @@ -132,7 +132,6 @@ steps: ${{ else }}: artifactName: crash-dump-macos-$(VSCODE_ARCH)-${{ parameters.VSCODE_TEST_ARTIFACT_NAME }}-$(System.JobAttempt) displayName: "Publish Crash Reports" - isProduction: false sbomEnabled: false continueOnError: true condition: failed() @@ -147,7 +146,6 @@ steps: ${{ else }}: artifactName: node-modules-macos-$(VSCODE_ARCH)-${{ parameters.VSCODE_TEST_ARTIFACT_NAME }}-$(System.JobAttempt) displayName: "Publish Node Modules" - isProduction: false sbomEnabled: false continueOnError: true condition: failed() @@ -160,7 +158,6 @@ steps: ${{ else }}: artifactName: logs-macos-$(VSCODE_ARCH)-${{ parameters.VSCODE_TEST_ARTIFACT_NAME }}-$(System.JobAttempt) displayName: "Publish Log Files" - isProduction: false sbomEnabled: false continueOnError: true condition: succeededOrFailed() diff --git a/build/azure-pipelines/darwin/product-build-darwin-universal.yml b/build/azure-pipelines/darwin/product-build-darwin-universal.yml index 60ce6a82954..b1153dc8c8b 100644 --- a/build/azure-pipelines/darwin/product-build-darwin-universal.yml +++ b/build/azure-pipelines/darwin/product-build-darwin-universal.yml @@ -1,4 +1,6 @@ steps: + - template: ../common/checkout.yml@self + - task: NodeTool@0 inputs: versionSource: fromFile diff --git a/build/azure-pipelines/darwin/product-build-darwin.yml b/build/azure-pipelines/darwin/product-build-darwin.yml index 7d2849681c5..dfb8284426a 100644 --- a/build/azure-pipelines/darwin/product-build-darwin.yml +++ b/build/azure-pipelines/darwin/product-build-darwin.yml @@ -17,6 +17,8 @@ parameters: default: "" steps: + - template: ../common/checkout.yml@self + - task: NodeTool@0 inputs: versionSource: fromFile diff --git a/build/azure-pipelines/linux/cli-build-linux.yml b/build/azure-pipelines/linux/cli-build-linux.yml index b29c4259433..c79c00ecf89 100644 --- a/build/azure-pipelines/linux/cli-build-linux.yml +++ b/build/azure-pipelines/linux/cli-build-linux.yml @@ -15,6 +15,8 @@ parameters: type: string steps: + - template: ../common/checkout.yml@self + - task: NodeTool@0 inputs: versionSource: fromFile diff --git a/build/azure-pipelines/linux/product-build-linux-test.yml b/build/azure-pipelines/linux/product-build-linux-test.yml index 4e882b78d25..e4dbfecd91b 100644 --- a/build/azure-pipelines/linux/product-build-linux-test.yml +++ b/build/azure-pipelines/linux/product-build-linux-test.yml @@ -147,7 +147,6 @@ steps: ${{ else }}: artifactName: crash-dump-linux-$(VSCODE_ARCH)-${{ parameters.VSCODE_TEST_ARTIFACT_NAME }}-$(System.JobAttempt) displayName: "Publish Crash Reports" - isProduction: false sbomEnabled: false continueOnError: true condition: failed() @@ -162,7 +161,6 @@ steps: ${{ else }}: artifactName: node-modules-linux-$(VSCODE_ARCH)-${{ parameters.VSCODE_TEST_ARTIFACT_NAME }}-$(System.JobAttempt) displayName: "Publish Node Modules" - isProduction: false sbomEnabled: false continueOnError: true condition: failed() @@ -175,7 +173,6 @@ steps: ${{ else }}: artifactName: logs-linux-$(VSCODE_ARCH)-${{ parameters.VSCODE_TEST_ARTIFACT_NAME }}-$(System.JobAttempt) displayName: "Publish Log Files" - isProduction: false sbomEnabled: false continueOnError: true condition: succeededOrFailed() diff --git a/build/azure-pipelines/linux/product-build-linux.yml b/build/azure-pipelines/linux/product-build-linux.yml index f1e92a60c1f..40d9fb7f383 100644 --- a/build/azure-pipelines/linux/product-build-linux.yml +++ b/build/azure-pipelines/linux/product-build-linux.yml @@ -22,6 +22,8 @@ parameters: default: "" steps: + - template: ../common/checkout.yml@self + - task: NodeTool@0 inputs: versionSource: fromFile diff --git a/build/azure-pipelines/product-build.yml b/build/azure-pipelines/product-build.yml index c0a346333c1..efc0af98b51 100644 --- a/build/azure-pipelines/product-build.yml +++ b/build/azure-pipelines/product-build.yml @@ -265,8 +265,7 @@ extends: - ${{ if eq(parameters.VSCODE_BUILD_MACOS, true) }}: - job: CLIMacOSX64 pool: - name: Azure Pipelines - image: macOS-13 + name: AcesShared os: macOS variables: # todo@connor4312 to diagnose build flakes @@ -282,8 +281,7 @@ extends: - ${{ if and(eq(variables['VSCODE_CIBUILD'], false), eq(parameters.VSCODE_BUILD_MACOS_ARM64, true)) }}: - job: CLIMacOSARM64 pool: - name: Azure Pipelines - image: macOS-13 + name: AcesShared os: macOS variables: # todo@connor4312 to diagnose build flakes @@ -676,6 +674,9 @@ extends: displayName: Publish Build steps: - template: build/azure-pipelines/product-publish.yml@self + parameters: + VSCODE_QUALITY: ${{ variables.VSCODE_QUALITY }} + VSCODE_SCHEDULEDBUILD: ${{ variables.VSCODE_SCHEDULEDBUILD }} - ${{ if and(parameters.VSCODE_RELEASE, eq(variables['VSCODE_PRIVATE_BUILD'], false)) }}: - stage: ApproveRelease @@ -691,12 +692,10 @@ extends: - name: skipComponentGovernanceDetection value: true - - ${{ if or(and(parameters.VSCODE_RELEASE, eq(variables['VSCODE_PRIVATE_BUILD'], false)), and(in(parameters.VSCODE_QUALITY, 'insider', 'exploration'), eq(variables['VSCODE_SCHEDULEDBUILD'], true))) }}: - stage: Release dependsOn: - Publish - - ${{ if and(parameters.VSCODE_RELEASE, eq(variables['VSCODE_PRIVATE_BUILD'], false)) }}: - - ApproveRelease + - ApproveRelease pool: name: 1es-ubuntu-22.04-x64 os: linux diff --git a/build/azure-pipelines/product-compile.yml b/build/azure-pipelines/product-compile.yml index 19e96932301..f6b2358b7e1 100644 --- a/build/azure-pipelines/product-compile.yml +++ b/build/azure-pipelines/product-compile.yml @@ -1,4 +1,6 @@ steps: + - template: ./common/checkout.yml@self + - task: NodeTool@0 inputs: versionSource: fromFile diff --git a/build/azure-pipelines/product-publish.yml b/build/azure-pipelines/product-publish.yml index 90cd06c5459..239d6e854b4 100644 --- a/build/azure-pipelines/product-publish.yml +++ b/build/azure-pipelines/product-publish.yml @@ -1,4 +1,12 @@ +parameters: + - name: VSCODE_QUALITY + type: string + - name: VSCODE_SCHEDULEDBUILD + type: boolean + steps: + - template: ./common/checkout.yml@self + - task: NodeTool@0 inputs: versionSource: fromFile @@ -93,3 +101,11 @@ steps: displayName: Publish the artifacts processed for this stage attempt sbomEnabled: false condition: always() + + - ${{ if and(in(parameters.VSCODE_QUALITY, 'insider', 'exploration'), eq(parameters.VSCODE_SCHEDULEDBUILD, true)) }}: + - script: node build/azure-pipelines/common/releaseBuild.js + env: + AZURE_TENANT_ID: "$(AZURE_TENANT_ID)" + AZURE_CLIENT_ID: "$(AZURE_CLIENT_ID)" + AZURE_ID_TOKEN: "$(AZURE_ID_TOKEN)" + displayName: Release build diff --git a/build/azure-pipelines/product-release.yml b/build/azure-pipelines/product-release.yml index d7b51aa8a92..bb54d50fdda 100644 --- a/build/azure-pipelines/product-release.yml +++ b/build/azure-pipelines/product-release.yml @@ -3,6 +3,8 @@ parameters: type: boolean steps: + - template: ./common/checkout.yml@self + - task: NodeTool@0 inputs: versionSource: fromFile diff --git a/build/azure-pipelines/web/product-build-web.yml b/build/azure-pipelines/web/product-build-web.yml index e35af2b87aa..a372a0e8bae 100644 --- a/build/azure-pipelines/web/product-build-web.yml +++ b/build/azure-pipelines/web/product-build-web.yml @@ -1,4 +1,6 @@ steps: + - template: ../common/checkout.yml@self + - task: NodeTool@0 inputs: versionSource: fromFile diff --git a/build/azure-pipelines/win32/cli-build-win32.yml b/build/azure-pipelines/win32/cli-build-win32.yml index 1914cb7cf6c..51484abb7c0 100644 --- a/build/azure-pipelines/win32/cli-build-win32.yml +++ b/build/azure-pipelines/win32/cli-build-win32.yml @@ -12,6 +12,8 @@ parameters: type: string steps: + - template: ../common/checkout.yml@self + - task: NodeTool@0 inputs: versionSource: fromFile @@ -74,9 +76,7 @@ steps: targetPath: $(Build.ArtifactStagingDirectory)/unsigned_vscode_cli_win32_arm64_cli.zip artifactName: unsigned_vscode_cli_win32_arm64_cli displayName: Publish unsigned_vscode_cli_win32_arm64_cli artifact - sbomBuildDropPath: $(Build.ArtifactStagingDirectory)/cli - sbomPackageName: "VS Code Windows arm64 CLI (unsigned)" - sbomPackageVersion: $(Build.SourceVersion) + sbomEnabled: false - ${{ if eq(parameters.VSCODE_BUILD_WIN32, true) }}: - template: ../common/publish-artifact.yml@self @@ -84,6 +84,4 @@ steps: targetPath: $(Build.ArtifactStagingDirectory)/unsigned_vscode_cli_win32_x64_cli.zip artifactName: unsigned_vscode_cli_win32_x64_cli displayName: Publish unsigned_vscode_cli_win32_x64_cli artifact - sbomBuildDropPath: $(Build.ArtifactStagingDirectory)/cli - sbomPackageName: "VS Code Windows x64 CLI (unsigned)" - sbomPackageVersion: $(Build.SourceVersion) + sbomEnabled: false diff --git a/build/azure-pipelines/win32/product-build-win32-cli-sign.yml b/build/azure-pipelines/win32/product-build-win32-cli-sign.yml index 520931ffa48..1b0224f6cae 100644 --- a/build/azure-pipelines/win32/product-build-win32-cli-sign.yml +++ b/build/azure-pipelines/win32/product-build-win32-cli-sign.yml @@ -5,6 +5,8 @@ parameters: type: boolean steps: + - template: ../common/checkout.yml@self + - task: NodeTool@0 displayName: "Use Node.js" inputs: diff --git a/build/azure-pipelines/win32/product-build-win32-test.yml b/build/azure-pipelines/win32/product-build-win32-test.yml index 7d5222e347f..154ddcf4485 100644 --- a/build/azure-pipelines/win32/product-build-win32-test.yml +++ b/build/azure-pipelines/win32/product-build-win32-test.yml @@ -149,7 +149,6 @@ steps: artifactName: crash-dump-windows-$(VSCODE_ARCH)-$(System.JobAttempt) ${{ else }}: artifactName: crash-dump-windows-$(VSCODE_ARCH)-${{ parameters.VSCODE_TEST_ARTIFACT_NAME }}-$(System.JobAttempt) - isProduction: false sbomEnabled: false displayName: "Publish Crash Reports" continueOnError: true @@ -164,7 +163,6 @@ steps: artifactName: node-modules-windows-$(VSCODE_ARCH)-$(System.JobAttempt) ${{ else }}: artifactName: node-modules-windows-$(VSCODE_ARCH)-${{ parameters.VSCODE_TEST_ARTIFACT_NAME }}-$(System.JobAttempt) - isProduction: false sbomEnabled: false displayName: "Publish Node Modules" continueOnError: true @@ -177,7 +175,6 @@ steps: artifactName: logs-windows-$(VSCODE_ARCH)-$(System.JobAttempt) ${{ else }}: artifactName: logs-windows-$(VSCODE_ARCH)-${{ parameters.VSCODE_TEST_ARTIFACT_NAME }}-$(System.JobAttempt) - isProduction: false sbomEnabled: false displayName: "Publish Log Files" continueOnError: true diff --git a/build/azure-pipelines/win32/product-build-win32.yml b/build/azure-pipelines/win32/product-build-win32.yml index a0b040c52c2..ad3ddb573f1 100644 --- a/build/azure-pipelines/win32/product-build-win32.yml +++ b/build/azure-pipelines/win32/product-build-win32.yml @@ -19,6 +19,8 @@ parameters: default: "" steps: + - template: ../common/checkout.yml@self + - task: NodeTool@0 inputs: versionSource: fromFile diff --git a/build/azure-pipelines/win32/sdl-scan-win32.yml b/build/azure-pipelines/win32/sdl-scan-win32.yml index ba60b881392..dba656eff53 100644 --- a/build/azure-pipelines/win32/sdl-scan-win32.yml +++ b/build/azure-pipelines/win32/sdl-scan-win32.yml @@ -5,6 +5,8 @@ parameters: type: string steps: + - template: ../common/checkout.yml@self + - task: NodeTool@0 inputs: versionSource: fromFile diff --git a/build/npm/mixin-telemetry-docs.mjs b/build/npm/mixin-telemetry-docs.mjs new file mode 100644 index 00000000000..fe8a6aec446 --- /dev/null +++ b/build/npm/mixin-telemetry-docs.mjs @@ -0,0 +1,34 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import { execSync } from 'child_process'; +import { join, resolve } from 'path'; +import { existsSync, rmSync } from 'fs'; +import { fileURLToPath } from 'url'; + +const rootPath = resolve(fileURLToPath(import.meta.url), '..', '..', '..'); +const telemetryDocsPath = join(rootPath, 'vscode-telemetry-docs'); +const repoUrl = 'https://github.com/microsoft/vscode-telemetry-docs'; + +console.log('Cloning vscode-telemetry-docs repository...'); + +// Remove existing directory if it exists +if (existsSync(telemetryDocsPath)) { + console.log('Removing existing vscode-telemetry-docs directory...'); + rmSync(telemetryDocsPath, { recursive: true, force: true }); +} + +try { + // Clone the repository (shallow clone of main branch only) + console.log(`Cloning ${repoUrl} to ${telemetryDocsPath}...`); + execSync(`git clone --depth 1 --branch main --single-branch ${repoUrl} vscode-telemetry-docs`, { + cwd: rootPath, + stdio: 'inherit' + }); + + console.log('Successfully cloned vscode-telemetry-docs repository.'); +} catch (error) { + console.error('Failed to clone vscode-telemetry-docs repository:', error.message); + process.exit(1); +} diff --git a/eslint.config.js b/eslint.config.js index e54236de108..3da54df165c 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -86,6 +86,8 @@ export default tseslint.config( 'local/code-no-unexternalized-strings': 'warn', 'local/code-must-use-super-dispose': 'warn', 'local/code-declare-service-brand': 'warn', + 'local/code-no-reader-after-await': 'warn', + 'local/code-no-observable-get-in-reactive-context': 'warn', 'local/code-no-deep-import-of-internal': ['error', { '.*Internal': true, 'searchExtTypesInternal': false }], 'local/code-layering': [ 'warn', diff --git a/extensions/git/src/util.ts b/extensions/git/src/util.ts index a4c5036255f..730c76909b7 100644 --- a/extensions/git/src/util.ts +++ b/extensions/git/src/util.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { Event, Disposable, EventEmitter, SourceControlHistoryItemRef, l10n, workspace, Uri, DiagnosticSeverity, env } from 'vscode'; -import { dirname, sep, relative } from 'path'; +import { dirname, normalize, sep, relative } from 'path'; import { Readable } from 'stream'; import { promises as fs, createReadStream } from 'fs'; import byline from 'byline'; @@ -299,10 +299,16 @@ function normalizePath(path: string): string { // Windows & Mac are currently being handled // as case insensitive file systems in VS Code. if (isWindows || isMacintosh) { - return path.toLowerCase(); + path = path.toLowerCase(); } - return path; + // Remove trailing separator + if (path.charAt(path.length - 1) === sep) { + path = path.substring(0, path.length - 1); + } + + // Normalize the path + return normalize(path); } export function isDescendant(parent: string, descendant: string): boolean { @@ -310,11 +316,16 @@ export function isDescendant(parent: string, descendant: string): boolean { return true; } + // Normalize the paths + parent = normalizePath(parent); + descendant = normalizePath(descendant); + + // Ensure parent ends with separator if (parent.charAt(parent.length - 1) !== sep) { parent += sep; } - return normalizePath(descendant).startsWith(normalizePath(parent)); + return descendant.startsWith(parent); } export function pathEquals(a: string, b: string): boolean { diff --git a/package.json b/package.json index 6ad06e6a153..2ca4e30bf17 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "code-oss-dev", "version": "1.105.0", - "distro": "8103a60ed6fb457dbf0ba180a67be3341dce4a23", + "distro": "0d6f2bd5a49ea5bc63f49f0c09b23d4f7f8b97be", "author": { "name": "Microsoft Corporation" }, @@ -38,6 +38,7 @@ "7z": "7z", "update-grammars": "node build/npm/update-all-grammars.mjs", "update-localization-extension": "node build/npm/update-localization-extension.js", + "mixin-telemetry-docs": "node build/npm/mixin-telemetry-docs.mjs", "smoketest": "node build/lib/preLaunch.js && cd test/smoke && npm run compile && node test/index.js", "smoketest-no-compile": "cd test/smoke && node test/index.js", "download-builtin-extensions": "node build/lib/builtInExtensions.js", diff --git a/src/vs/base/browser/markdownRenderer.ts b/src/vs/base/browser/markdownRenderer.ts index 71af7a09743..87ec4b8ec64 100644 --- a/src/vs/base/browser/markdownRenderer.ts +++ b/src/vs/base/browser/markdownRenderer.ts @@ -16,7 +16,7 @@ import { FileAccess, Schemas } from '../common/network.js'; import { cloneAndChange } from '../common/objects.js'; import { dirname, resolvePath } from '../common/resources.js'; import { escape } from '../common/strings.js'; -import { URI } from '../common/uri.js'; +import { URI, UriComponents } from '../common/uri.js'; import * as DOM from './dom.js'; import * as domSanitize from './domSanitize.js'; import { convertTagToPlaintext } from './domSanitize.js'; @@ -151,7 +151,7 @@ export function renderMarkdown(markdown: IMarkdownString, options: MarkdownRende } const renderedContent = document.createElement('div'); - const sanitizerConfig = getDomSanitizerConfig(markdown.isTrusted ?? false, options.sanitizerConfig ?? {}); + const sanitizerConfig = getDomSanitizerConfig(markdown, options.sanitizerConfig ?? {}); domSanitize.safeSetInnerHtml(renderedContent, renderedMarkdown, sanitizerConfig); // Rewrite links and images before potentially inserting them into the real dom @@ -437,12 +437,17 @@ function resolveWithBaseUri(baseUri: URI, href: string): string { } } +type MdStrConfig = { + readonly isTrusted?: boolean | MarkdownStringTrustedOptions; + readonly baseUri?: UriComponents; +}; + function sanitizeRenderedMarkdown( renderedMarkdown: string, - isTrusted: boolean | MarkdownStringTrustedOptions, + originalMdStrConfig: MdStrConfig, options: MarkdownSanitizerConfig = {}, ): TrustedHTML { - const sanitizerConfig = getDomSanitizerConfig(isTrusted, options); + const sanitizerConfig = getDomSanitizerConfig(originalMdStrConfig, options); return domSanitize.sanitizeHtml(renderedMarkdown, sanitizerConfig); } @@ -508,7 +513,8 @@ export const allowedMarkdownHtmlAttributes = Object.freeze unescapeInfo.get(m) ?? m) .trim(); diff --git a/src/vs/base/common/cancellation.ts b/src/vs/base/common/cancellation.ts index c5b8133ae6c..3be4a90a103 100644 --- a/src/vs/base/common/cancellation.ts +++ b/src/vs/base/common/cancellation.ts @@ -146,3 +146,61 @@ export function cancelOnDispose(store: DisposableStore): CancellationToken { store.add({ dispose() { source.cancel(); } }); return source.token; } + +/** + * A pool that aggregates multiple cancellation tokens. The pool's own token + * (accessible via `pool.token`) is cancelled only after every token added + * to the pool has been cancelled. Adding tokens after the pool token has + * been cancelled has no effect. + */ +export class CancellationTokenPool { + + private readonly _source = new CancellationTokenSource(); + private readonly _listeners = new DisposableStore(); + + private _total: number = 0; + private _cancelled: number = 0; + private _isDone: boolean = false; + + get token(): CancellationToken { + return this._source.token; + } + + /** + * Add a token to the pool. If the token is already cancelled it is counted + * immediately. Tokens added after the pool token has been cancelled are ignored. + */ + add(token: CancellationToken): void { + if (this._isDone) { + return; + } + + this._total++; + + if (token.isCancellationRequested) { + this._cancelled++; + this._check(); + return; + } + + const d = token.onCancellationRequested(() => { + d.dispose(); + this._cancelled++; + this._check(); + }); + this._listeners.add(d); + } + + private _check(): void { + if (!this._isDone && this._total > 0 && this._total === this._cancelled) { + this._isDone = true; + this._listeners.dispose(); + this._source.cancel(); + } + } + + dispose(): void { + this._listeners.dispose(); + this._source.dispose(); + } +} diff --git a/src/vs/base/common/numbers.ts b/src/vs/base/common/numbers.ts index 89c9f183e6d..326a9b2f2c3 100644 --- a/src/vs/base/common/numbers.ts +++ b/src/vs/base/common/numbers.ts @@ -99,65 +99,6 @@ export function isPointWithinTriangle( return u >= 0 && v >= 0 && u + v < 1; } -/** - * Function to get a (pseudo)random integer from a provided `max`...[`min`] range. - * Both `min` and `max` values are inclusive. The `min` value is optional (defaults to `0`). - * - * @throws in the next cases: - * - if provided `min` or `max` is not a number - * - if provided `min` or `max` is not finite - * - if provided `min` is larger than `max` value - * - * ## Examples - * - * Specifying a `max` value only uses `0` as the `min` value by default: - * - * ```typescript - * // get a random integer between 0 and 10 - * const randomInt = randomInt(10); - * - * assert( - * randomInt >= 0, - * 'Should be greater than or equal to 0.', - * ); - * - * assert( - * randomInt <= 10, - * 'Should be less than or equal to 10.', - * ); - * ``` - * * Specifying both `max` and `min` values: - * - * ```typescript - * // get a random integer between 5 and 8 - * const randomInt = randomInt(8, 5); - * - * assert( - * randomInt >= 5, - * 'Should be greater than or equal to 5.', - * ); - * - * assert( - * randomInt <= 8, - * 'Should be less than or equal to 8.', - * ); - * ``` - */ -export function randomInt(max: number, min: number = 0): number { - assert(!isNaN(min), '"min" param is not a number.'); - assert(!isNaN(max), '"max" param is not a number.'); - - assert(isFinite(max), '"max" param is not finite.'); - assert(isFinite(min), '"min" param is not finite.'); - - assert(max > min, `"max"(${max}) param should be greater than "min"(${min}).`); - - const delta = max - min; - const randomFloat = delta * Math.random(); - - return Math.round(min + randomFloat); -} - export function randomChance(p: number): boolean { assert(p >= 0 && p <= 1, 'p must be between 0 and 1'); return Math.random() < p; diff --git a/src/vs/base/test/browser/markdownRenderer.test.ts b/src/vs/base/test/browser/markdownRenderer.test.ts index f1174905b02..785f69925fa 100644 --- a/src/vs/base/test/browser/markdownRenderer.test.ts +++ b/src/vs/base/test/browser/markdownRenderer.test.ts @@ -256,6 +256,27 @@ suite('MarkdownRenderer', () => { assert.strictEqual(result.innerHTML, `

command1 command2

`); }); + test('Should remove relative links if there is no base url', () => { + const md = new MarkdownString(`[text](./foo) bar`, { + isTrusted: true, + supportHtml: true, + }); + + const result = store.add(renderMarkdown(md)).element; + assert.strictEqual(result.innerHTML, `

text bar

`); + }); + + test('Should support relative links if baseurl is set', () => { + const md = new MarkdownString(`[text](./foo) bar`, { + isTrusted: true, + supportHtml: true, + }); + md.baseUri = URI.parse('https://example.com/path/'); + + const result = store.add(renderMarkdown(md)).element; + assert.strictEqual(result.innerHTML, `

text bar

`); + }); + suite('PlaintextMarkdownRender', () => { test('test code, blockquote, heading, list, listitem, paragraph, table, tablerow, tablecell, strong, em, br, del, text are rendered plaintext', () => { diff --git a/src/vs/base/test/common/cancellation.test.ts b/src/vs/base/test/common/cancellation.test.ts index 38ed33f8d62..7e5ad1060fd 100644 --- a/src/vs/base/test/common/cancellation.test.ts +++ b/src/vs/base/test/common/cancellation.test.ts @@ -3,7 +3,7 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ import assert from 'assert'; -import { CancellationToken, CancellationTokenSource } from '../../common/cancellation.js'; +import { CancellationToken, CancellationTokenSource, CancellationTokenPool } from '../../common/cancellation.js'; import { ensureNoDisposablesAreLeakedInTestSuite } from './utils.js'; suite('CancellationToken', function () { @@ -125,3 +125,155 @@ suite('CancellationToken', function () { parent.dispose(); }); }); + +suite('CancellationTokenPool', function () { + + const store = ensureNoDisposablesAreLeakedInTestSuite(); + + test('empty pool token is not cancelled', function () { + const pool = new CancellationTokenPool(); + store.add(pool); + + assert.strictEqual(pool.token.isCancellationRequested, false); + }); + + test('pool token cancels when all tokens are cancelled', function () { + const pool = new CancellationTokenPool(); + store.add(pool); + + const source1 = new CancellationTokenSource(); + const source2 = new CancellationTokenSource(); + const source3 = new CancellationTokenSource(); + + pool.add(source1.token); + pool.add(source2.token); + pool.add(source3.token); + + assert.strictEqual(pool.token.isCancellationRequested, false); + + source1.cancel(); + assert.strictEqual(pool.token.isCancellationRequested, false); + + source2.cancel(); + assert.strictEqual(pool.token.isCancellationRequested, false); + + source3.cancel(); + assert.strictEqual(pool.token.isCancellationRequested, true); + + source1.dispose(); + source2.dispose(); + source3.dispose(); + }); + + test('pool token fires cancellation event when all tokens are cancelled', function () { + return new Promise(resolve => { + const pool = new CancellationTokenPool(); + store.add(pool); + + const source1 = new CancellationTokenSource(); + const source2 = new CancellationTokenSource(); + + pool.add(source1.token); + pool.add(source2.token); + + store.add(pool.token.onCancellationRequested(() => resolve())); + + source1.cancel(); + source2.cancel(); + + source1.dispose(); + source2.dispose(); + }); + }); + + test('adding already cancelled token counts immediately', function () { + const pool = new CancellationTokenPool(); + store.add(pool); + + const source1 = new CancellationTokenSource(); + const source2 = new CancellationTokenSource(); + + source1.cancel(); // Cancel before adding to pool + + pool.add(source1.token); + assert.strictEqual(pool.token.isCancellationRequested, true); // 1 of 1 cancelled, so pool is cancelled + + pool.add(source2.token); // Adding after pool is done should have no effect + assert.strictEqual(pool.token.isCancellationRequested, true); + + source2.cancel(); // This should have no effect since pool is already done + assert.strictEqual(pool.token.isCancellationRequested, true); + + source1.dispose(); + source2.dispose(); + }); + + test('adding single already cancelled token cancels pool immediately', function () { + const pool = new CancellationTokenPool(); + store.add(pool); + + const source = new CancellationTokenSource(); + source.cancel(); + + pool.add(source.token); + assert.strictEqual(pool.token.isCancellationRequested, true); // 1 of 1 cancelled + + source.dispose(); + }); + + test('adding token after pool is done has no effect', function () { + const pool = new CancellationTokenPool(); + store.add(pool); + + const source1 = new CancellationTokenSource(); + const source2 = new CancellationTokenSource(); + + pool.add(source1.token); + source1.cancel(); // Pool should be done now + + assert.strictEqual(pool.token.isCancellationRequested, true); + + // Adding another token should have no effect + pool.add(source2.token); + source2.cancel(); + + assert.strictEqual(pool.token.isCancellationRequested, true); + + source1.dispose(); + source2.dispose(); + }); + + test('single token pool behaviour', function () { + const pool = new CancellationTokenPool(); + store.add(pool); + + const source = new CancellationTokenSource(); + pool.add(source.token); + + assert.strictEqual(pool.token.isCancellationRequested, false); + + source.cancel(); + assert.strictEqual(pool.token.isCancellationRequested, true); + + source.dispose(); + }); + + test('pool with only cancelled tokens', function () { + const pool = new CancellationTokenPool(); + store.add(pool); + + const source1 = new CancellationTokenSource(); + const source2 = new CancellationTokenSource(); + + source1.cancel(); + source2.cancel(); + + pool.add(source1.token); + pool.add(source2.token); + + assert.strictEqual(pool.token.isCancellationRequested, true); + + source1.dispose(); + source2.dispose(); + }); +}); diff --git a/src/vs/base/test/common/numbers.test.ts b/src/vs/base/test/common/numbers.test.ts index 94c07090585..e21844eea98 100644 --- a/src/vs/base/test/common/numbers.test.ts +++ b/src/vs/base/test/common/numbers.test.ts @@ -5,7 +5,7 @@ import assert from 'assert'; import { ensureNoDisposablesAreLeakedInTestSuite } from './utils.js'; -import { isPointWithinTriangle, randomInt } from '../../common/numbers.js'; +import { isPointWithinTriangle } from '../../common/numbers.js'; suite('isPointWithinTriangle', () => { ensureNoDisposablesAreLeakedInTestSuite(); @@ -25,176 +25,3 @@ suite('isPointWithinTriangle', () => { assert.ok(result); }); }); - -suite('randomInt', () => { - ensureNoDisposablesAreLeakedInTestSuite(); - - /** - * Test helper that allows to run a test on the `randomInt()` - * utility with specified `max` and `min` values. - */ - const testRandomIntUtil = (max: number, min: number | undefined, testName: string) => { - suite(testName, () => { - let i = 0; - while (++i < 5) { - test(`should generate random boolean attempt#${i}`, async () => { - let iterations = 100; - while (iterations-- > 0) { - const int = randomInt(max, min); - - assert( - int <= max, - `Expected ${int} to be less than or equal to ${max}.` - ); - assert( - int >= (min ?? 0), - `Expected ${int} to be greater than or equal to ${min ?? 0}.`, - ); - } - }); - } - - test('should include min and max', async () => { - let iterations = 125; - const results = []; - while (iterations-- > 0) { - results.push(randomInt(max, min)); - } - - assert( - results.includes(max), - `Expected ${results} to include ${max}.`, - ); - assert( - results.includes(min ?? 0), - `Expected ${results} to include ${min ?? 0}.`, - ); - }); - }); - }; - - suite('positive numbers', () => { - testRandomIntUtil(4, 2, 'max: 4, min: 2'); - testRandomIntUtil(4, 0, 'max: 4, min: 0'); - testRandomIntUtil(4, undefined, 'max: 4, min: undefined'); - testRandomIntUtil(1, 0, 'max: 0, min: 0'); - }); - - suite('negative numbers', () => { - testRandomIntUtil(-2, -5, 'max: -2, min: -5'); - testRandomIntUtil(0, -5, 'max: 0, min: -5'); - testRandomIntUtil(0, -1, 'max: 0, min: -1'); - }); - - suite('split numbers', () => { - testRandomIntUtil(3, -1, 'max: 3, min: -1'); - testRandomIntUtil(2, -2, 'max: 2, min: -2'); - testRandomIntUtil(1, -3, 'max: 2, min: -2'); - }); - - suite('errors', () => { - test('should throw if "min" is == "max" #1', () => { - assert.throws(() => { - randomInt(200, 200); - }, `"max"(200) param should be greater than "min"(200)."`); - }); - - test('should throw if "min" is == "max" #2', () => { - assert.throws(() => { - randomInt(2, 2); - }, `"max"(2) param should be greater than "min"(2)."`); - }); - - test('should throw if "min" is == "max" #3', () => { - assert.throws(() => { - randomInt(0); - }, `"max"(0) param should be greater than "min"(0)."`); - }); - - test('should throw if "min" is > "max" #1', () => { - assert.throws(() => { - randomInt(2, 3); - }, `"max"(2) param should be greater than "min"(3)."`); - }); - - test('should throw if "min" is > "max" #2', () => { - assert.throws(() => { - randomInt(999, 2000); - }, `"max"(999) param should be greater than "min"(2000)."`); - }); - - test('should throw if "min" is > "max" #3', () => { - assert.throws(() => { - randomInt(0, 1); - }, `"max"(0) param should be greater than "min"(1)."`); - }); - - test('should throw if "min" is > "max" #4', () => { - assert.throws(() => { - randomInt(-5, 2); - }, `"max"(-5) param should be greater than "min"(2)."`); - }); - - test('should throw if "min" is > "max" #5', () => { - assert.throws(() => { - randomInt(-4, 0); - }, `"max"(-4) param should be greater than "min"(0)."`); - }); - - test('should throw if "min" is > "max" #6', () => { - assert.throws(() => { - randomInt(-4); - }, `"max"(-4) param should be greater than "min"(0)."`); - }); - - test('should throw if "max" is `NaN`', () => { - assert.throws(() => { - randomInt(NaN); - }, `"max" param is not a number."`); - }); - - test('should throw if "min" is `NaN`', () => { - assert.throws(() => { - randomInt(4, NaN); - }, `"min" param is not a number."`); - }); - - suite('infinite arguments', () => { - test('should throw if "max" is infinite [Infinity]', () => { - assert.throws(() => { - randomInt(Infinity); - }, `"max" param is not finite."`); - }); - - test('should throw if "max" is infinite [-Infinity]', () => { - assert.throws(() => { - randomInt(-Infinity); - }, `"max" param is not finite."`); - }); - - test('should throw if "max" is infinite [+Infinity]', () => { - assert.throws(() => { - randomInt(+Infinity); - }, `"max" param is not finite."`); - }); - - test('should throw if "min" is infinite [Infinity]', () => { - assert.throws(() => { - randomInt(Infinity, Infinity); - }, `"max" param is not finite."`); - }); - - test('should throw if "min" is infinite [-Infinity]', () => { - assert.throws(() => { - randomInt(Infinity, -Infinity); - }, `"max" param is not finite."`); - }); - - test('should throw if "min" is infinite [+Infinity]', () => { - assert.throws(() => { - randomInt(Infinity, +Infinity); - }, `"max" param is not finite."`); - }); - }); - }); -}); diff --git a/src/vs/base/test/common/yaml.test.ts b/src/vs/base/test/common/yaml.test.ts index cba621bf023..be4ced9d65a 100644 --- a/src/vs/base/test/common/yaml.test.ts +++ b/src/vs/base/test/common/yaml.test.ts @@ -209,9 +209,9 @@ suite('YAML Parser', () => { ' database:', ' host:localhost', ' port: 5432', - ' credentials:', - ' username:admin', - ' password: secret123' + ' abcde123456:', + ' logger12:admin', + ' memory12: a23123112' ], { type: 'object', start: pos(0, 0), end: pos(6, 25), properties: [ @@ -232,16 +232,16 @@ suite('YAML Parser', () => { value: { type: 'number', start: pos(3, 10), end: pos(3, 14), value: 5432 } }, { - key: { type: 'string', start: pos(4, 4), end: pos(4, 15), value: 'credentials' }, + key: { type: 'string', start: pos(4, 4), end: pos(4, 15), value: 'abcde123456' }, value: { type: 'object', start: pos(5, 6), end: pos(6, 25), properties: [ { - key: { type: 'string', start: pos(5, 6), end: pos(5, 14), value: 'username' }, + key: { type: 'string', start: pos(5, 6), end: pos(5, 14), value: 'logger12' }, value: { type: 'string', start: pos(5, 15), end: pos(5, 20), value: 'admin' } }, { - key: { type: 'string', start: pos(6, 6), end: pos(6, 14), value: 'password' }, - value: { type: 'string', start: pos(6, 16), end: pos(6, 25), value: 'secret123' } + key: { type: 'string', start: pos(6, 6), end: pos(6, 14), value: 'memory12' }, + value: { type: 'string', start: pos(6, 16), end: pos(6, 25), value: 'a23123112' } } ] } diff --git a/src/vs/editor/browser/widget/diffEditor/components/diffEditorViewZones/diffEditorViewZones.ts b/src/vs/editor/browser/widget/diffEditor/components/diffEditorViewZones/diffEditorViewZones.ts index cc9a23d23b3..a5b5d8c9540 100644 --- a/src/vs/editor/browser/widget/diffEditor/components/diffEditorViewZones/diffEditorViewZones.ts +++ b/src/vs/editor/browser/widget/diffEditor/components/diffEditorViewZones/diffEditorViewZones.ts @@ -395,8 +395,8 @@ export class DiffEditorViewZones extends Disposable { this._register(autorun(reader => { /** @description update scroll modified */ const newScrollTopModified = this._originalScrollTop.read(reader) - - (this._originalScrollOffsetAnimated.get() - this._modifiedScrollOffsetAnimated.read(reader)) - - (this._originalTopPadding.get() - this._modifiedTopPadding.read(reader)); + - (this._originalScrollOffsetAnimated.read(undefined) - this._modifiedScrollOffsetAnimated.read(reader)) + - (this._originalTopPadding.read(undefined) - this._modifiedTopPadding.read(reader)); if (newScrollTopModified !== this._editors.modified.getScrollTop()) { this._editors.modified.setScrollTop(newScrollTopModified, ScrollType.Immediate); } @@ -405,8 +405,8 @@ export class DiffEditorViewZones extends Disposable { this._register(autorun(reader => { /** @description update scroll original */ const newScrollTopOriginal = this._modifiedScrollTop.read(reader) - - (this._modifiedScrollOffsetAnimated.get() - this._originalScrollOffsetAnimated.read(reader)) - - (this._modifiedTopPadding.get() - this._originalTopPadding.read(reader)); + - (this._modifiedScrollOffsetAnimated.read(undefined) - this._originalScrollOffsetAnimated.read(reader)) + - (this._modifiedTopPadding.read(undefined) - this._originalTopPadding.read(reader)); if (newScrollTopOriginal !== this._editors.original.getScrollTop()) { this._editors.original.setScrollTop(newScrollTopOriginal, ScrollType.Immediate); } @@ -419,8 +419,8 @@ export class DiffEditorViewZones extends Disposable { let deltaOrigToMod = 0; if (m) { - const trueTopOriginal = this._editors.original.getTopForLineNumber(m.lineRangeMapping.original.startLineNumber, true) - this._originalTopPadding.get(); - const trueTopModified = this._editors.modified.getTopForLineNumber(m.lineRangeMapping.modified.startLineNumber, true) - this._modifiedTopPadding.get(); + const trueTopOriginal = this._editors.original.getTopForLineNumber(m.lineRangeMapping.original.startLineNumber, true) - this._originalTopPadding.read(undefined); + const trueTopModified = this._editors.modified.getTopForLineNumber(m.lineRangeMapping.modified.startLineNumber, true) - this._modifiedTopPadding.read(undefined); deltaOrigToMod = trueTopModified - trueTopOriginal; } @@ -438,9 +438,9 @@ export class DiffEditorViewZones extends Disposable { } if (this._editors.modified.hasTextFocus()) { - this._originalScrollOffset.set(this._modifiedScrollOffset.get() - deltaOrigToMod, undefined, true); + this._originalScrollOffset.set(this._modifiedScrollOffset.read(undefined) - deltaOrigToMod, undefined, true); } else { - this._modifiedScrollOffset.set(this._originalScrollOffset.get() + deltaOrigToMod, undefined, true); + this._modifiedScrollOffset.set(this._originalScrollOffset.read(undefined) + deltaOrigToMod, undefined, true); } })); } diff --git a/src/vs/editor/browser/widget/diffEditor/diffEditorViewModel.ts b/src/vs/editor/browser/widget/diffEditor/diffEditorViewModel.ts index 660c989c145..c87f163a8d7 100644 --- a/src/vs/editor/browser/widget/diffEditor/diffEditorViewModel.ts +++ b/src/vs/editor/browser/widget/diffEditor/diffEditorViewModel.ts @@ -40,7 +40,7 @@ export class DiffEditorViewModel extends Disposable implements IDiffEditorViewMo } else { // Reset state transaction(tx => { - for (const r of this._unchangedRegions.get()?.regions || []) { + for (const r of this._unchangedRegions.read(undefined)?.regions || []) { r.collapseAll(tx); } }); @@ -121,7 +121,7 @@ export class DiffEditorViewModel extends Disposable implements IDiffEditorViewMo if (touching.length > 1) { didChange = true; const sumLineCount = touching.reduce((sum, r) => sum + r.lineCount, 0); - const r = new UnchangedRegion(touching[0].originalLineNumber, touching[0].modifiedLineNumber, sumLineCount, touching[0].visibleLineCountTop.get(), touching[touching.length - 1].visibleLineCountBottom.get()); + const r = new UnchangedRegion(touching[0].originalLineNumber, touching[0].modifiedLineNumber, sumLineCount, touching[0].visibleLineCountTop.read(undefined), touching[touching.length - 1].visibleLineCountBottom.read(undefined)); newRanges.push(r); } else { newRanges.push(touching[0]); @@ -315,7 +315,7 @@ export class DiffEditorViewModel extends Disposable implements IDiffEditorViewMo const state = DiffState.fromDiffResult(result); this._diff.set(state, tx); this._isDiffUpToDate.set(true, tx); - const currentSyncedMovedText = this.movedTextToCompare.get(); + const currentSyncedMovedText = this.movedTextToCompare.read(undefined); this.movedTextToCompare.set(currentSyncedMovedText ? this._lastDiff.moves.find(m => m.lineRangeMapping.modified.intersect(currentSyncedMovedText.lineRangeMapping.modified)) : undefined, tx); }); })); diff --git a/src/vs/editor/browser/widget/diffEditor/diffEditorWidget.ts b/src/vs/editor/browser/widget/diffEditor/diffEditorWidget.ts index 026b0850887..8fd39f2945d 100644 --- a/src/vs/editor/browser/widget/diffEditor/diffEditorWidget.ts +++ b/src/vs/editor/browser/widget/diffEditor/diffEditorWidget.ts @@ -294,7 +294,7 @@ export class DiffEditorWidget extends DelegatingEditor implements IDiffEditor { this._diffModel, this._options, this, - () => isUpdatingViewZones || unchangedRangesFeature.get().isUpdatingHiddenAreas, + () => isUpdatingViewZones || unchangedRangesFeature.read(undefined).isUpdatingHiddenAreas, origViewZoneIdsToIgnore, modViewZoneIdsToIgnore ) diff --git a/src/vs/editor/browser/widget/diffEditor/features/gutterFeature.ts b/src/vs/editor/browser/widget/diffEditor/features/gutterFeature.ts index 6033069a413..97d8e60d70e 100644 --- a/src/vs/editor/browser/widget/diffEditor/features/gutterFeature.ts +++ b/src/vs/editor/browser/widget/diffEditor/features/gutterFeature.ts @@ -260,7 +260,7 @@ class DiffToolBar extends Disposable implements IGutterItemView { overflowBehavior: { maxItems: this._isSmall.read(reader) ? 1 : 3 }, hiddenItemStrategy: HiddenItemStrategy.Ignore, actionRunner: store.add(new ActionRunnerWithContext(() => { - const item = this._item.get(); + const item = this._item.read(undefined); const mapping = item.mapping; return { mapping, diff --git a/src/vs/editor/browser/widget/diffEditor/features/movedBlocksLinesFeature.ts b/src/vs/editor/browser/widget/diffEditor/features/movedBlocksLinesFeature.ts index 04e0c61e024..a2df0125cd5 100644 --- a/src/vs/editor/browser/widget/diffEditor/features/movedBlocksLinesFeature.ts +++ b/src/vs/editor/browser/widget/diffEditor/features/movedBlocksLinesFeature.ts @@ -244,7 +244,7 @@ export class MovedBlocksLinesFeature extends Disposable { } } - if (movedText !== m.movedTextToCompare.get()) { + if (movedText !== m.movedTextToCompare.read(undefined)) { m.movedTextToCompare.set(undefined, undefined); } m.setActiveMovedText(movedText); diff --git a/src/vs/editor/common/core/ranges/lineRange.ts b/src/vs/editor/common/core/ranges/lineRange.ts index 0a3eb6e94df..518dc2f824d 100644 --- a/src/vs/editor/common/core/ranges/lineRange.ts +++ b/src/vs/editor/common/core/ranges/lineRange.ts @@ -5,7 +5,7 @@ import { BugIndicatingError } from '../../../../base/common/errors.js'; import { OffsetRange } from './offsetRange.js'; -import { Range } from '../range.js'; +import { IRange, Range } from '../range.js'; import { findFirstIdxMonotonousOrArrLen, findLastIdxMonotonous, findLastMonotonous } from '../../../../base/common/arraysFind.js'; import { Comparator, compareBy, numberComparator } from '../../../../base/common/arrays.js'; @@ -17,11 +17,11 @@ export class LineRange { return new LineRange(startLineNumber, startLineNumber + length); } - public static fromRange(range: Range): LineRange { + public static fromRange(range: IRange): LineRange { return new LineRange(range.startLineNumber, range.endLineNumber); } - public static fromRangeInclusive(range: Range): LineRange { + public static fromRangeInclusive(range: IRange): LineRange { return new LineRange(range.startLineNumber, range.endLineNumber + 1); } diff --git a/src/vs/editor/common/viewModel/monospaceLineBreaksComputer.ts b/src/vs/editor/common/viewModel/monospaceLineBreaksComputer.ts index 00632b25780..6013b248c7d 100644 --- a/src/vs/editor/common/viewModel/monospaceLineBreaksComputer.ts +++ b/src/vs/editor/common/viewModel/monospaceLineBreaksComputer.ts @@ -42,8 +42,8 @@ export class MonospaceLineBreaksComputerFactory implements ILineBreaksComputerFa for (let i = 0, len = requests.length; i < len; i++) { const injectedText = injectedTexts[i]; const previousLineBreakData = previousBreakingData[i]; - if (previousLineBreakData && !previousLineBreakData.injectionOptions && !injectedText) { - result[i] = createLineBreaksFromPreviousLineBreaks(this.classifier, previousLineBreakData, requests[i], tabSize, wrappingColumn, columnsForFullWidthChar, wrappingIndent, wordBreak); + if (previousLineBreakData && !previousLineBreakData.injectionOptions && !injectedText && !wrapOnEscapedLineFeeds) { + result[i] = createLineBreaksFromPreviousLineBreaks(this.classifier, previousLineBreakData, requests[i], tabSize, wrappingColumn, columnsForFullWidthChar, wrappingIndent, wordBreak, wrapOnEscapedLineFeeds); } else { result[i] = createLineBreaks(this.classifier, requests[i], injectedText, tabSize, wrappingColumn, columnsForFullWidthChar, wrappingIndent, wordBreak, wrapOnEscapedLineFeeds); } @@ -101,7 +101,7 @@ class WrappingCharacterClassifier extends CharacterClassifier { let arrPool1: number[] = []; let arrPool2: number[] = []; -function createLineBreaksFromPreviousLineBreaks(classifier: WrappingCharacterClassifier, previousBreakingData: ModelLineProjectionData, lineText: string, tabSize: number, firstLineBreakColumn: number, columnsForFullWidthChar: number, wrappingIndent: WrappingIndent, wordBreak: 'normal' | 'keepAll'): ModelLineProjectionData | null { +function createLineBreaksFromPreviousLineBreaks(classifier: WrappingCharacterClassifier, previousBreakingData: ModelLineProjectionData, lineText: string, tabSize: number, firstLineBreakColumn: number, columnsForFullWidthChar: number, wrappingIndent: WrappingIndent, wordBreak: 'normal' | 'keepAll', wrapOnEscapedLineFeeds: boolean): ModelLineProjectionData | null { if (firstLineBreakColumn === -1) { return null; } @@ -185,6 +185,11 @@ function createLineBreaksFromPreviousLineBreaks(classifier: WrappingCharacterCla visibleColumn += charWidth; + // literal \n shall trigger a softwrap + if (wrapOnEscapedLineFeeds && isEscapedLineBreakAtPosition(lineText, i)) { + visibleColumn += breakingColumn; + } + // check if adding character at `i` will go over the breaking column if (visibleColumn > breakingColumn) { // We need to break at least before character at `i`: @@ -435,14 +440,7 @@ function createLineBreaks(classifier: WrappingCharacterClassifier, _lineText: st visibleColumn += charWidth; // literal \n shall trigger a softwrap - if ( - wrapOnEscapedLineFeeds - && i >= 2 - && (i < 3 || lineText.charAt(i - 3) !== '\\') - && lineText.charAt(i - 2) === '\\' - && lineText.charAt(i - 1) === 'n' - && lineText.includes('"') - ) { + if (wrapOnEscapedLineFeeds && isEscapedLineBreakAtPosition(lineText, i)) { visibleColumn += breakingColumn; } @@ -496,6 +494,20 @@ function tabCharacterWidth(visibleColumn: number, tabSize: number): number { return (tabSize - (visibleColumn % tabSize)); } +/** + * Checks if the current position in the text should trigger a soft wrap due to escaped line feeds. + * This handles the wrapOnEscapedLineFeeds feature which allows \n sequences in strings to trigger wrapping. + */ +function isEscapedLineBreakAtPosition(lineText: string, i: number): boolean { + return ( + i >= 2 + && (i < 3 || lineText.charAt(i - 3) !== '\\') + && lineText.charAt(i - 2) === '\\' + && lineText.charAt(i - 1) === 'n' + && lineText.includes('"') + ); +} + /** * Kinsoku Shori : Don't break after a leading character, like an open bracket * Kinsoku Shori : Don't break before a trailing character, like a period diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptToken.ts b/src/vs/editor/contrib/floatingMenu/browser/floatingMenu.contribution.ts similarity index 51% rename from src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptToken.ts rename to src/vs/editor/contrib/floatingMenu/browser/floatingMenu.contribution.ts index 021b8d5a425..014050a2119 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptToken.ts +++ b/src/vs/editor/contrib/floatingMenu/browser/floatingMenu.contribution.ts @@ -3,9 +3,8 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { BaseToken } from '../base/baseToken.js'; +import './floatingMenu.css'; +import { registerEditorContribution, EditorContributionInstantiation } from '../../../browser/editorExtensions.js'; +import { FloatingEditorToolbar } from './floatingMenu.js'; -/** - * Common base token that all chatbot `prompt` tokens should inherit from. - */ -export abstract class PromptToken extends BaseToken { } +registerEditorContribution(FloatingEditorToolbar.ID, FloatingEditorToolbar, EditorContributionInstantiation.AfterFirstRender); diff --git a/src/vs/editor/contrib/floatingMenu/browser/floatingMenu.css b/src/vs/editor/contrib/floatingMenu/browser/floatingMenu.css new file mode 100644 index 00000000000..1a1d169bca4 --- /dev/null +++ b/src/vs/editor/contrib/floatingMenu/browser/floatingMenu.css @@ -0,0 +1,38 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +.floating-menu-overlay-widget { + padding: 0px; + color: var(--vscode-button-foreground); + background-color: var(--vscode-button-background); + border-radius: 2px; + border: 1px solid var(--vscode-contrastBorder); + display: flex; + align-items: center; + z-index: 10; + box-shadow: 0 2px 8px var(--vscode-widget-shadow); + overflow: hidden; + + .action-item > .action-label { + padding: 5px; + font-size: 12px; + } + + .action-item > .action-label.codicon { + color: var(--vscode-button-foreground); + } + + .action-item:first-child > .action-label { + padding-left: 7px; + } + + .action-item:last-child > .action-label { + padding-right: 7px; + } + + .action-item .action-label.separator { + background-color: var(--vscode-menu-separatorBackground); + } +} diff --git a/src/vs/editor/contrib/floatingMenu/browser/floatingMenu.ts b/src/vs/editor/contrib/floatingMenu/browser/floatingMenu.ts new file mode 100644 index 00000000000..149d98ce26e --- /dev/null +++ b/src/vs/editor/contrib/floatingMenu/browser/floatingMenu.ts @@ -0,0 +1,73 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { h } from '../../../../base/browser/dom.js'; +import { Disposable } from '../../../../base/common/lifecycle.js'; +import { autorun, constObservable, observableFromEvent } from '../../../../base/common/observable.js'; +import { HiddenItemStrategy, MenuWorkbenchToolBar } from '../../../../platform/actions/browser/toolbar.js'; +import { IMenuService, MenuId } from '../../../../platform/actions/common/actions.js'; +import { IInstantiationService } from '../../../../platform/instantiation/common/instantiation.js'; +import { ICodeEditor, OverlayWidgetPositionPreference } from '../../../browser/editorBrowser.js'; +import { observableCodeEditor } from '../../../browser/observableCodeEditor.js'; +import { IEditorContribution } from '../../../common/editorCommon.js'; + +export class FloatingEditorToolbar extends Disposable implements IEditorContribution { + static readonly ID = 'editor.contrib.floatingToolbar'; + + constructor( + editor: ICodeEditor, + @IInstantiationService instantiationService: IInstantiationService, + @IMenuService menuService: IMenuService + ) { + super(); + + const editorObs = this._register(observableCodeEditor(editor)); + + const menu = this._register(menuService.createMenu(MenuId.EditorContent, editor.contextKeyService)); + const menuIsEmptyObs = observableFromEvent(this, menu.onDidChange, () => menu.getActions().length === 0); + + this._register(autorun(reader => { + const menuIsEmpty = menuIsEmptyObs.read(reader); + if (menuIsEmpty) { + return; + } + + const container = h('div.floating-menu-overlay-widget'); + + // Set height explicitly to ensure that the floating menu element + // is rendered in the lower right corner at the correct position. + container.root.style.height = '28px'; + + // Toolbar + const toolbar = instantiationService.createInstance(MenuWorkbenchToolBar, container.root, MenuId.EditorContent, { + hiddenItemStrategy: HiddenItemStrategy.Ignore, + menuOptions: { + shouldForwardArgs: true + }, + telemetrySource: 'editor.overlayToolbar', + toolbarOptions: { + primaryGroup: () => true, + useSeparatorsInPrimaryActions: true + }, + }); + + reader.store.add(toolbar); + reader.store.add(autorun(reader => { + const model = editorObs.model.read(reader); + toolbar.context = model?.uri; + })); + + // Overlay widget + reader.store.add(editorObs.createOverlayWidget({ + allowEditorOverflow: false, + domNode: container.root, + minContentWidthInPx: constObservable(0), + position: constObservable({ + preference: OverlayWidgetPositionPreference.BOTTOM_RIGHT_CORNER + }) + })); + })); + } +} diff --git a/src/vs/editor/contrib/inlineCompletions/browser/controller/inlineCompletionsController.ts b/src/vs/editor/contrib/inlineCompletions/browser/controller/inlineCompletionsController.ts index 479d7058565..5508ab07ba8 100644 --- a/src/vs/editor/contrib/inlineCompletions/browser/controller/inlineCompletionsController.ts +++ b/src/vs/editor/contrib/inlineCompletions/browser/controller/inlineCompletionsController.ts @@ -162,7 +162,7 @@ export class InlineCompletionsController extends Disposable { if (!model) { return; } const state = model.state.read(reader); if (!state) { return; } - if (!this._focusIsInEditorOrMenu.get()) { return; } + if (!this._focusIsInEditorOrMenu.read(undefined)) { return; } // This controller is in focus, hence reject others. // However if we display a NES that relates to another edit then trigger NES on that related controller @@ -172,7 +172,7 @@ export class InlineCompletionsController extends Disposable { continue; } else if (nextEditUri && isEqual(nextEditUri, ctrl.editor.getModel()?.uri)) { // The next edit in other edito is related to this controller, trigger it. - ctrl.model.get()?.trigger(); + ctrl.model.read(undefined)?.trigger(); } else { ctrl.reject(); } @@ -193,9 +193,9 @@ export class InlineCompletionsController extends Disposable { continue; } // Find the nes from another editor that points to this. - const state = ctrl.model.get()?.state.get(); + const state = ctrl.model.read(undefined)?.state.read(undefined); if (state?.kind === 'inlineEdit' && isEqual(state.nextEditUri, uri)) { - ctrl.model.get()?.stop('automatic'); + ctrl.model.read(undefined)?.stop('automatic'); } } })); @@ -250,16 +250,16 @@ export class InlineCompletionsController extends Disposable { this._register(autorun(reader => { const isFocused = this._focusIsInEditorOrMenu.read(reader); - const model = this.model.get(); + const model = this.model.read(undefined); if (isFocused) { // If this model already has an NES for another editor, then leave as is // Else stop other models. - const state = model?.state?.get(); + const state = model?.state.read(undefined); if (!state || state.kind !== 'inlineEdit' || !state.nextEditUri) { transaction(tx => { for (const ctrl of InlineCompletionsController._instances) { if (ctrl !== this) { - ctrl.model.get()?.stop('automatic', tx); + ctrl.model.read(undefined)?.stop('automatic', tx); } } }); @@ -276,7 +276,7 @@ export class InlineCompletionsController extends Disposable { } if (!model) { return; } - if (model.state.get()?.inlineCompletion?.isFromExplicitRequest && model.inlineEditAvailable.get()) { + if (model.state.read(undefined)?.inlineCompletion?.isFromExplicitRequest && model.inlineEditAvailable.read(undefined)) { // dont hide inline edits on blur when requested explicitly return; } diff --git a/src/vs/editor/contrib/inlineCompletions/browser/model/inlineCompletionsModel.ts b/src/vs/editor/contrib/inlineCompletions/browser/model/inlineCompletionsModel.ts index 89608790221..d97b62b30f2 100644 --- a/src/vs/editor/contrib/inlineCompletions/browser/model/inlineCompletionsModel.ts +++ b/src/vs/editor/contrib/inlineCompletions/browser/model/inlineCompletionsModel.ts @@ -356,9 +356,9 @@ export class InlineCompletionsModel extends Disposable { this._textModelVersionId.read(reader); // Refetch on text change - const suggestWidgetInlineCompletions = this._source.suggestWidgetInlineCompletions.get(); + const suggestWidgetInlineCompletions = this._source.suggestWidgetInlineCompletions.read(undefined); let suggestItem = this._selectedSuggestItem.read(reader); - if (this._shouldShowOnSuggestConflict.get()) { + if (this._shouldShowOnSuggestConflict.read(undefined)) { suggestItem = undefined; } if (suggestWidgetInlineCompletions && !suggestItem) { @@ -402,7 +402,7 @@ export class InlineCompletionsModel extends Disposable { includeInlineCompletions: !changeSummary.onlyRequestInlineEdits, includeInlineEdits: this._inlineEditsEnabled.read(reader), requestIssuedDateTime: requestInfo.startTime, - earliestShownDateTime: requestInfo.startTime + (changeSummary.inlineCompletionTriggerKind === InlineCompletionTriggerKind.Explicit || this.inAcceptFlow.get() ? 0 : this._minShowDelay.get()), + earliestShownDateTime: requestInfo.startTime + (changeSummary.inlineCompletionTriggerKind === InlineCompletionTriggerKind.Explicit || this.inAcceptFlow.read(undefined) ? 0 : this._minShowDelay.read(undefined)), }; if (context.triggerKind === InlineCompletionTriggerKind.Automatic && changeSummary.textChange) { @@ -417,10 +417,10 @@ export class InlineCompletionsModel extends Disposable { } } - const itemToPreserveCandidate = this.selectedInlineCompletion.get() ?? this._inlineCompletionItems.get()?.inlineEdit; + const itemToPreserveCandidate = this.selectedInlineCompletion.read(undefined) ?? this._inlineCompletionItems.read(undefined)?.inlineEdit; const itemToPreserve = changeSummary.preserveCurrentCompletion || itemToPreserveCandidate?.forwardStable ? itemToPreserveCandidate : undefined; - const userJumpedToActiveCompletion = this._jumpedToId.map(jumpedTo => !!jumpedTo && jumpedTo === this._inlineCompletionItems.get()?.inlineEdit?.semanticId); + const userJumpedToActiveCompletion = this._jumpedToId.map(jumpedTo => !!jumpedTo && jumpedTo === this._inlineCompletionItems.read(undefined)?.inlineEdit?.semanticId); const providers = changeSummary.provider ? { providers: [changeSummary.provider], label: 'single:' + changeSummary.provider.providerId?.toString() } diff --git a/src/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/components/gutterIndicatorView.ts b/src/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/components/gutterIndicatorView.ts index adc1e1239fd..d76a3fa99aa 100644 --- a/src/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/components/gutterIndicatorView.ts +++ b/src/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/components/gutterIndicatorView.ts @@ -237,7 +237,7 @@ export class InlineEditsGutterIndicator extends Disposable { const idealIconWidth = 22; const minimalIconWidth = 16; // codicon size const iconWidth = (pillRect: Rect) => { - const availableWidth = this._availableWidthForIcon.get()(pillRect.bottom + this._editorObs.editor.getScrollTop()) - gutterViewPortPadding; + const availableWidth = this._availableWidthForIcon.read(undefined)(pillRect.bottom + this._editorObs.editor.getScrollTop()) - gutterViewPortPadding; return Math.max(Math.min(availableWidth, idealIconWidth), minimalIconWidth); }; diff --git a/src/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/inlineEditsViewProducer.ts b/src/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/inlineEditsViewProducer.ts index 15aaec60501..cbfaa632b25 100644 --- a/src/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/inlineEditsViewProducer.ts +++ b/src/vs/editor/contrib/inlineCompletions/browser/view/inlineEdits/inlineEditsViewProducer.ts @@ -33,7 +33,7 @@ export class InlineEditsViewAndDiffProducer extends Disposable { // TODO: This c const textModel = this._editor.getModel(); if (!textModel) { return undefined; } - const editOffset = model.inlineEditState.get()?.inlineCompletion.updatedEdit; + const editOffset = model.inlineEditState.read(undefined)?.inlineCompletion.updatedEdit; if (!editOffset) { return undefined; } const edits = editOffset.replacements.map(e => { @@ -47,7 +47,7 @@ export class InlineEditsViewAndDiffProducer extends Disposable { // TODO: This c const diffEdits = new TextEdit(edits); const text = new TextModelText(textModel); - return new InlineEditWithChanges(text, diffEdits, model.primaryPosition.get(), inlineEdit.commands, inlineEdit.inlineCompletion); + return new InlineEditWithChanges(text, diffEdits, model.primaryPosition.read(undefined), inlineEdit.commands, inlineEdit.inlineCompletion); }); private readonly _inlineEditModel = derived(this, reader => { diff --git a/src/vs/editor/contrib/middleScroll/browser/middleScrollController.ts b/src/vs/editor/contrib/middleScroll/browser/middleScrollController.ts index c885a89d209..edcdca4e00d 100644 --- a/src/vs/editor/contrib/middleScroll/browser/middleScrollController.ts +++ b/src/vs/editor/contrib/middleScroll/browser/middleScrollController.ts @@ -47,7 +47,7 @@ export class MiddleScrollController extends Disposable implements IEditorContrib ); reader.store.add(this._editor.onMouseDown(e => { - const session = scrollingSession.get(); + const session = scrollingSession.read(undefined); if (session) { scrollingSession.set(undefined, undefined); return; @@ -75,7 +75,7 @@ export class MiddleScrollController extends Disposable implements IEditorContrib }, undefined); store.add(this._editor.onMouseUp(e => { - const session = scrollingSession.get(); + const session = scrollingSession.read(undefined); if (session && session.didScroll) { // Only cancel session on release if the user scrolled during it scrollingSession.set(undefined, undefined); @@ -101,7 +101,7 @@ export class MiddleScrollController extends Disposable implements IEditorContrib const frameDurationMs = curTime - lastTime; lastTime = curTime; - const mouseDelta = session.mouseDeltaAfterThreshold.get(); + const mouseDelta = session.mouseDeltaAfterThreshold.read(undefined); // scroll by mouse delta every 32ms const factor = frameDurationMs / 32; diff --git a/src/vs/editor/editor.all.ts b/src/vs/editor/editor.all.ts index 1f42a726b7a..916cc40a980 100644 --- a/src/vs/editor/editor.all.ts +++ b/src/vs/editor/editor.all.ts @@ -63,6 +63,7 @@ import './contrib/wordOperations/browser/wordOperations.js'; import './contrib/wordPartOperations/browser/wordPartOperations.js'; import './contrib/readOnlyMessage/browser/contribution.js'; import './contrib/diffEditorBreadcrumbs/browser/contribution.js'; +import './contrib/floatingMenu/browser/floatingMenu.contribution.js'; // Load up these strings even in VSCode, even if they are not used // in order to get them translated diff --git a/src/vs/editor/standalone/browser/standaloneThemeService.ts b/src/vs/editor/standalone/browser/standaloneThemeService.ts index 8a697f653ff..253e79405de 100644 --- a/src/vs/editor/standalone/browser/standaloneThemeService.ts +++ b/src/vs/editor/standalone/browser/standaloneThemeService.ts @@ -263,7 +263,11 @@ export class StandaloneThemeService extends Disposable implements IStandaloneThe })); addMatchMediaChangeListener(mainWindow, '(forced-colors: active)', () => { + // Update theme selection for auto-detecting high contrast this._onOSSchemeChanged(); + // Always rebuild the generated CSS so that the `forced-color-adjust: none` + // rule is added/removed reactively when the OS forced colors state changes. + this._updateThemeOrColorMap(); }); } @@ -399,6 +403,13 @@ export class StandaloneThemeService extends Disposable implements IStandaloneThe const colorMap = this._colorMapOverride || this._theme.tokenTheme.getColorMap(); ruleCollector.addRule(generateTokensCSSForColorMap(colorMap)); + // If the OS has forced-colors active, disable forced color adjustment for + // Monaco editor elements so that VS Code's built-in high contrast themes + // (hc-black / hc-light) are used instead of the OS forcing system colors. + if (mainWindow.matchMedia(`(forced-colors: active)`).matches) { + ruleCollector.addRule(`.monaco-editor, .monaco-diff-editor, .monaco-component { forced-color-adjust: none; }`); + } + this._themeCSS = cssRules.join('\n'); this._updateCSS(); diff --git a/src/vs/editor/test/common/viewModel/monospaceLineBreaksComputer.test.ts b/src/vs/editor/test/common/viewModel/monospaceLineBreaksComputer.test.ts index 850e19bc4ad..7861bb31c5b 100644 --- a/src/vs/editor/test/common/viewModel/monospaceLineBreaksComputer.test.ts +++ b/src/vs/editor/test/common/viewModel/monospaceLineBreaksComputer.test.ts @@ -317,4 +317,43 @@ suite('Editor ViewModel - MonospaceLineBreaksComputer', () => { const factory = new MonospaceLineBreaksComputerFactory(EditorOptions.wordWrapBreakBeforeCharacters.defaultValue, EditorOptions.wordWrapBreakAfterCharacters.defaultValue); assertLineBreaks(factory, 4, 8, '你好1111', WrappingIndent.Same, 'keepAll'); }); + + test('issue wrapOnEscapedLineFeeds: should work correctly after editor resize', () => { + const factory = new MonospaceLineBreaksComputerFactory(EditorOptions.wordWrapBreakBeforeCharacters.defaultValue, EditorOptions.wordWrapBreakAfterCharacters.defaultValue); + + // Test text with escaped line feeds - simulates a JSON string with \n + // The \n should trigger a soft wrap when wrapOnEscapedLineFeeds is enabled + const text = '"Short text with\\nescaped newline and more text after"'; + + // First, compute line breaks with wrapOnEscapedLineFeeds enabled at initial width + const initialBreakData = getLineBreakData(factory, 4, 30, 2, WrappingIndent.None, 'normal', true, text, null); + const initialAnnotatedText = toAnnotatedText(text, initialBreakData); + + // Verify the escaped \n triggers a wrap in the initial case + assert.ok(initialAnnotatedText.includes('with\\n'), 'Initial case should wrap at escaped line feeds'); + + // Now simulate editor resize by computing line breaks with different width using previous data + // This triggers createLineBreaksFromPreviousLineBreaks which has the bug + const resizedBreakData = getLineBreakData(factory, 4, 35, 2, WrappingIndent.None, 'normal', true, text, initialBreakData); + const resizedAnnotatedText = toAnnotatedText(text, resizedBreakData); + + // Compute fresh line breaks at the new width (without using previous data) + // This uses createLineBreaks which correctly handles wrapOnEscapedLineFeeds + const freshBreakData = getLineBreakData(factory, 4, 35, 2, WrappingIndent.None, 'normal', true, text, null); + const freshAnnotatedText = toAnnotatedText(text, freshBreakData); + + // Fresh computation should still wrap at escaped line feeds + assert.ok(freshAnnotatedText.includes('with\\n'), 'Fresh computation should wrap at escaped line feeds'); + + // BUG DEMONSTRATION: Incremental computation after resize doesn't handle escaped line feeds + // The two results should be identical, but they're not due to the bug + assert.strictEqual( + resizedAnnotatedText, + freshAnnotatedText, + `Bug: Incremental and fresh computations differ for escaped line feeds.\n` + + `Incremental (resize): ${resizedAnnotatedText}\n` + + `Fresh computation: ${freshAnnotatedText}\n` + + `The incremental path (createLineBreaksFromPreviousLineBreaks) doesn't handle wrapOnEscapedLineFeeds` + ); + }); }); diff --git a/src/vs/platform/mcp/common/mcpGalleryManifest.ts b/src/vs/platform/mcp/common/mcpGalleryManifest.ts index 759f09c5b86..9c108ab7780 100644 --- a/src/vs/platform/mcp/common/mcpGalleryManifest.ts +++ b/src/vs/platform/mcp/common/mcpGalleryManifest.ts @@ -8,6 +8,7 @@ import { createDecorator } from '../../instantiation/common/instantiation.js'; export const enum McpGalleryResourceType { McpServersQueryService = 'McpServersQueryService', + McpServersSearchService = 'McpServersSearchService', McpServerWebUri = 'McpServerWebUriTemplate', McpServerResourceUri = 'McpServerResourceUriTemplate', McpServerNamedResourceUri = 'McpServerNamedResourceUriTemplate', diff --git a/src/vs/platform/mcp/common/mcpGalleryManifestService.ts b/src/vs/platform/mcp/common/mcpGalleryManifestService.ts index 43080ff3ccb..a366e79c1f6 100644 --- a/src/vs/platform/mcp/common/mcpGalleryManifestService.ts +++ b/src/vs/platform/mcp/common/mcpGalleryManifestService.ts @@ -51,6 +51,10 @@ export class McpGalleryManifestService extends Disposable implements IMcpGallery } if (isProductGalleryUrl) { + resources.push({ + id: `${serversUrl}/search`, + type: McpGalleryResourceType.McpServersSearchService + }); resources.push({ id: `${serversUrl}/by-name/{name}`, type: McpGalleryResourceType.McpServerNamedResourceUri diff --git a/src/vs/platform/mcp/common/mcpGalleryService.ts b/src/vs/platform/mcp/common/mcpGalleryService.ts index 7c5017f21d9..6743182b7b6 100644 --- a/src/vs/platform/mcp/common/mcpGalleryService.ts +++ b/src/vs/platform/mcp/common/mcpGalleryService.ts @@ -44,18 +44,19 @@ interface IMcpRegistryInfo { } interface IGitHubInfo { - readonly 'name': string; - readonly 'name_with_owner': string; - readonly 'is_in_organization'?: boolean; - readonly 'license'?: string; - readonly 'opengraph_image_url'?: string; - readonly 'owner_avatar_url'?: string; - readonly 'primary_language'?: string; - readonly 'primary_language_color'?: string; - readonly 'pushed_at'?: string; - readonly 'stargazer_count'?: number; - readonly 'topics'?: readonly string[]; - readonly 'uses_custom_opengraph_image'?: boolean; + readonly name: string; + readonly name_with_owner: string; + readonly display_name?: string; + readonly is_in_organization?: boolean; + readonly license?: string; + readonly opengraph_image_url?: string; + readonly owner_avatar_url?: string; + readonly primary_language?: string; + readonly primary_language_color?: string; + readonly pushed_at?: string; + readonly stargazer_count?: number; + readonly topics?: readonly string[]; + readonly uses_custom_opengraph_image?: boolean; } interface IRawGalleryMcpServerMetaData { @@ -336,6 +337,10 @@ export class McpGalleryService extends Disposable implements IMcpGalleryService displayName = nameParts[nameParts.length - 1].split('-').map(s => uppercaseFirstLetter(s)).join(' '); } + if (githubInfo?.display_name) { + displayName = githubInfo.display_name; + } + const icon: { light: string; dark: string } | undefined = githubInfo?.owner_avatar_url ? { light: githubInfo.owner_avatar_url, dark: githubInfo.owner_avatar_url @@ -406,7 +411,7 @@ export class McpGalleryService extends Disposable implements IMcpGalleryService } private async queryRawGalleryMcpServers(query: Query, mcpGalleryManifest: IMcpGalleryManifest, token: CancellationToken): Promise { - const mcpGalleryUrl = this.getMcpGalleryUrl(mcpGalleryManifest); + const mcpGalleryUrl = query.searchText ? this.getSearchUrl(mcpGalleryManifest) : this.getMcpGalleryUrl(mcpGalleryManifest); if (!mcpGalleryUrl) { return { servers: [] }; } @@ -428,7 +433,7 @@ export class McpGalleryService extends Disposable implements IMcpGalleryService } if (query.searchText) { const text = encodeURIComponent(query.searchText); - url += `&search=${text}`; + url += `&q=${text}`; } const context = await this.requestService.request({ @@ -574,6 +579,10 @@ export class McpGalleryService extends Disposable implements IMcpGalleryService return format2(namedResourceUriTemplate, { name }); } + private getSearchUrl(mcpGalleryManifest: IMcpGalleryManifest): string | undefined { + return getMcpGalleryManifestResourceUri(mcpGalleryManifest, McpGalleryResourceType.McpServersSearchService); + } + private getWebUrl(name: string, mcpGalleryManifest: IMcpGalleryManifest): string | undefined { const resourceUriTemplate = getMcpGalleryManifestResourceUri(mcpGalleryManifest, McpGalleryResourceType.McpServerWebUri); if (!resourceUriTemplate) { diff --git a/src/vs/platform/remote/browser/browserSocketFactory.ts b/src/vs/platform/remote/browser/browserSocketFactory.ts index 0f01fc06138..923db9458b4 100644 --- a/src/vs/platform/remote/browser/browserSocketFactory.ts +++ b/src/vs/platform/remote/browser/browserSocketFactory.ts @@ -7,7 +7,7 @@ import * as dom from '../../../base/browser/dom.js'; import { RunOnceScheduler } from '../../../base/common/async.js'; import { VSBuffer } from '../../../base/common/buffer.js'; import { Emitter, Event } from '../../../base/common/event.js'; -import { Disposable, IDisposable } from '../../../base/common/lifecycle.js'; +import { Disposable, DisposableStore, IDisposable } from '../../../base/common/lifecycle.js'; import { ISocket, SocketCloseEvent, SocketCloseEventType, SocketDiagnostics, SocketDiagnosticsEventType } from '../../../base/parts/ipc/common/ipc.net.js'; import { ISocketFactory } from '../common/remoteSocketFactoryService.js'; import { RemoteAuthorityResolverError, RemoteAuthorityResolverErrorCode, RemoteConnectionType, WebSocketRemoteConnection } from '../common/remoteAuthorityResolver.js'; @@ -282,11 +282,12 @@ export class BrowserSocketFactory implements ISocketFactory((resolve, reject) => { const webSocketSchema = (/^https:/.test(mainWindow.location.href) ? 'wss' : 'ws'); const socket = this._webSocketFactory.create(`${webSocketSchema}://${(/:/.test(host) && !/\[/.test(host)) ? `[${host}]` : host}:${port}${path}?${query}&skipWebSocketFrames=false`, debugLabel); - const errorListener = socket.onError(reject); - socket.onOpen(() => { - errorListener.dispose(); + const disposables = new DisposableStore(); + disposables.add(socket.onError(reject)); + disposables.add(socket.onOpen(() => { + disposables.dispose(); resolve(new BrowserSocket(socket, debugLabel)); - }); + })); }); } } diff --git a/src/vs/workbench/api/browser/mainThreadAuthentication.ts b/src/vs/workbench/api/browser/mainThreadAuthentication.ts index ff002feb111..8e754163168 100644 --- a/src/vs/workbench/api/browser/mainThreadAuthentication.ts +++ b/src/vs/workbench/api/browser/mainThreadAuthentication.ts @@ -11,10 +11,12 @@ import { ExtHostAuthenticationShape, ExtHostContext, MainContext, MainThreadAuth import { IDialogService, IPromptButton } from '../../../platform/dialogs/common/dialogs.js'; import Severity from '../../../base/common/severity.js'; import { INotificationService } from '../../../platform/notification/common/notification.js'; +import { ActivationKind, IExtensionService } from '../../services/extensions/common/extensions.js'; import { ITelemetryService } from '../../../platform/telemetry/common/telemetry.js'; import { Emitter, Event } from '../../../base/common/event.js'; import { IAuthenticationAccessService } from '../../services/authentication/browser/authenticationAccessService.js'; import { IAuthenticationUsageService } from '../../services/authentication/browser/authenticationUsageService.js'; +import { getAuthenticationProviderActivationEvent } from '../../services/authentication/browser/authenticationService.js'; import { URI, UriComponents } from '../../../base/common/uri.js'; import { IOpenerService } from '../../../platform/opener/common/opener.js'; import { CancellationError } from '../../../base/common/errors.js'; @@ -116,6 +118,7 @@ export class MainThreadAuthentication extends Disposable implements MainThreadAu @IAuthenticationUsageService private readonly authenticationUsageService: IAuthenticationUsageService, @IDialogService private readonly dialogService: IDialogService, @INotificationService private readonly notificationService: INotificationService, + @IExtensionService private readonly extensionService: IExtensionService, @ITelemetryService private readonly telemetryService: ITelemetryService, @IOpenerService private readonly openerService: IOpenerService, @ILogService private readonly logService: ILogService, @@ -200,6 +203,12 @@ export class MainThreadAuthentication extends Disposable implements MainThreadAu } } + async $ensureProvider(id: string): Promise { + if (!this.authenticationService.isAuthenticationProviderRegistered(id)) { + return await this.extensionService.activateByEvent(getAuthenticationProviderActivationEvent(id), ActivationKind.Immediate); + } + } + async $sendDidChangeSessions(providerId: string, event: AuthenticationSessionsChangeEvent): Promise { const obj = this._registrations.get(providerId); if (obj instanceof Emitter) { diff --git a/src/vs/workbench/api/common/extHost.protocol.ts b/src/vs/workbench/api/common/extHost.protocol.ts index 12e3fcdcdf4..277ddeb381c 100644 --- a/src/vs/workbench/api/common/extHost.protocol.ts +++ b/src/vs/workbench/api/common/extHost.protocol.ts @@ -189,6 +189,7 @@ export interface AuthenticationGetSessionOptions { export interface MainThreadAuthenticationShape extends IDisposable { $registerAuthenticationProvider(id: string, label: string, supportsMultipleAccounts: boolean, supportedAuthorizationServers?: UriComponents[], supportsChallenges?: boolean): Promise; $unregisterAuthenticationProvider(id: string): Promise; + $ensureProvider(id: string): Promise; $sendDidChangeSessions(providerId: string, event: AuthenticationSessionsChangeEvent): Promise; $getSession(providerId: string, scopeListOrRequest: ReadonlyArray | IAuthenticationWwwAuthenticateRequest, extensionId: string, extensionName: string, options: AuthenticationGetSessionOptions): Promise; $getAccounts(providerId: string): Promise>; diff --git a/src/vs/workbench/api/common/extHostAuthentication.ts b/src/vs/workbench/api/common/extHostAuthentication.ts index dd829afab53..39814f50421 100644 --- a/src/vs/workbench/api/common/extHostAuthentication.ts +++ b/src/vs/workbench/api/common/extHostAuthentication.ts @@ -112,12 +112,14 @@ export class ExtHostAuthentication implements ExtHostAuthenticationShape { } return await this._getSessionTaskSingler.getOrCreate(singlerKey, async () => { + await this._proxy.$ensureProvider(providerId); const extensionName = requestingExtension.displayName || requestingExtension.name; return this._proxy.$getSession(providerId, scopesOrRequest, extensionId, extensionName, options); }); } async getAccounts(providerId: string) { + await this._proxy.$ensureProvider(providerId); return await this._proxy.$getAccounts(providerId); } diff --git a/src/vs/workbench/api/common/extHostChatAgents2.ts b/src/vs/workbench/api/common/extHostChatAgents2.ts index a8e1a259907..edb6b09fd47 100644 --- a/src/vs/workbench/api/common/extHostChatAgents2.ts +++ b/src/vs/workbench/api/common/extHostChatAgents2.ts @@ -599,7 +599,7 @@ export class ExtHostChatAgents2 extends Disposable implements ExtHostChatAgentsS responseIsIncomplete: true }; } - if (errorDetails?.responseIsRedacted || errorDetails?.isQuotaExceeded || errorDetails?.confirmationButtons || errorDetails?.code) { + if (errorDetails?.responseIsRedacted || errorDetails?.isQuotaExceeded || errorDetails?.isRateLimited || errorDetails?.confirmationButtons || errorDetails?.code) { checkProposedApiEnabled(agent.extension, 'chatParticipantPrivate'); } @@ -613,7 +613,8 @@ export class ExtHostChatAgents2 extends Disposable implements ExtHostChatAgentsS } const isQuotaExceeded = e instanceof Error && e.name === 'ChatQuotaExceeded'; - return { errorDetails: { message: toErrorMessage(e), responseIsIncomplete: true, isQuotaExceeded } }; + const isRateLimited = e instanceof Error && e.name === 'ChatRateLimited'; + return { errorDetails: { message: toErrorMessage(e), responseIsIncomplete: true, isQuotaExceeded, isRateLimited } }; } finally { if (inFlightRequest) { diff --git a/src/vs/workbench/api/common/extHostTypeConverters.ts b/src/vs/workbench/api/common/extHostTypeConverters.ts index 4e6f9cc9b86..6c41f9582fd 100644 --- a/src/vs/workbench/api/common/extHostTypeConverters.ts +++ b/src/vs/workbench/api/common/extHostTypeConverters.ts @@ -65,6 +65,7 @@ import { CommandsConverter } from './extHostCommands.js'; import { getPrivateApiFor } from './extHostTestingPrivateApi.js'; import * as types from './extHostTypes.js'; import { LanguageModelDataPart, LanguageModelPromptTsxPart, LanguageModelTextPart } from './extHostTypes.js'; +import { IChatRequestModeInstructions } from '../../contrib/chat/common/chatModel.js'; export namespace Command { @@ -3129,7 +3130,7 @@ export namespace ChatAgentRequest { model, editedFileEvents: request.editedFileEvents, modeInstructions: request.modeInstructions?.content, - modeInstructionsToolReferences: request.modeInstructions?.toolReferences?.map(ChatLanguageModelToolReference.to), + modeInstructions2: ChatRequestModeInstructions.to(request.modeInstructions), }; if (!isProposedApiEnabled(extension, 'chatParticipantPrivate')) { @@ -3259,6 +3260,16 @@ export namespace ChatLanguageModelToolReference { } } +export namespace ChatRequestModeInstructions { + export function to(mode: IChatRequestModeInstructions | undefined): vscode.ChatRequestModeInstructions | undefined { + return mode ? { + content: mode.content, + toolReferences: mode.toolReferences?.map(ref => ChatLanguageModelToolReference.to(ref)) ?? [], + metadata: mode.metadata + } : undefined; + } +} + export namespace ChatAgentCompletionItem { export function from(item: vscode.ChatCompletionItem, commandsConverter: CommandsConverter, disposables: DisposableStore): extHostProtocol.IChatAgentCompletionItem { return { diff --git a/src/vs/workbench/browser/parts/editor/editor.contribution.ts b/src/vs/workbench/browser/parts/editor/editor.contribution.ts index 9157f122f09..7fdea24ffcd 100644 --- a/src/vs/workbench/browser/parts/editor/editor.contribution.ts +++ b/src/vs/workbench/browser/parts/editor/editor.contribution.ts @@ -59,8 +59,6 @@ import { inQuickPickContext, getQuickNavigateHandler } from '../../quickaccess.j import { KeybindingsRegistry, KeybindingWeight } from '../../../../platform/keybinding/common/keybindingsRegistry.js'; import { ContextKeyExpr, ContextKeyExpression } from '../../../../platform/contextkey/common/contextkey.js'; import { isMacintosh } from '../../../../base/common/platform.js'; -import { EditorContributionInstantiation, registerEditorContribution } from '../../../../editor/browser/editorExtensions.js'; -import { FloatingEditorClickMenu } from '../../codeeditor.js'; import { WorkbenchPhase, registerWorkbenchContribution2 } from '../../../common/contributions.js'; import { EditorAutoSave } from './editorAutoSave.js'; import { IQuickAccessRegistry, Extensions as QuickAccessExtensions } from '../../../../platform/quickinput/common/quickAccess.js'; @@ -136,8 +134,6 @@ registerWorkbenchContribution2(EditorStatusContribution.ID, EditorStatusContribu registerWorkbenchContribution2(UntitledTextEditorWorkingCopyEditorHandler.ID, UntitledTextEditorWorkingCopyEditorHandler, WorkbenchPhase.BlockRestore); registerWorkbenchContribution2(DynamicEditorConfigurations.ID, DynamicEditorConfigurations, WorkbenchPhase.BlockRestore); -registerEditorContribution(FloatingEditorClickMenu.ID, FloatingEditorClickMenu, EditorContributionInstantiation.AfterFirstRender); - //#endregion //#region Quick Access diff --git a/src/vs/workbench/browser/parts/titlebar/commandCenterControl.ts b/src/vs/workbench/browser/parts/titlebar/commandCenterControl.ts index 408d242fbb2..3ff05bb7454 100644 --- a/src/vs/workbench/browser/parts/titlebar/commandCenterControl.ts +++ b/src/vs/workbench/browser/parts/titlebar/commandCenterControl.ts @@ -142,6 +142,7 @@ class CommandCenterCenterViewItem extends BaseActionViewItem { super.render(container); container.classList.toggle('command-center-quick-pick'); container.role = 'button'; + container.setAttribute('aria-description', this.getTooltip()); const action = this.action; // icon (search) diff --git a/src/vs/workbench/contrib/accessibility/browser/accessibleView.ts b/src/vs/workbench/contrib/accessibility/browser/accessibleView.ts index 21be375a627..6ce21ebf240 100644 --- a/src/vs/workbench/contrib/accessibility/browser/accessibleView.ts +++ b/src/vs/workbench/contrib/accessibility/browser/accessibleView.ts @@ -25,6 +25,7 @@ import { IModelService } from '../../../../editor/common/services/model.js'; import { ITextModelContentProvider, ITextModelService } from '../../../../editor/common/services/resolverService.js'; import { AccessibilityHelpNLS } from '../../../../editor/common/standaloneStrings.js'; import { CodeActionController } from '../../../../editor/contrib/codeAction/browser/codeActionController.js'; +import { FloatingEditorToolbar } from '../../../../editor/contrib/floatingMenu/browser/floatingMenu.js'; import { localize } from '../../../../nls.js'; import { AccessibleContentProvider, AccessibleViewProviderId, AccessibleViewType, ExtensionContentProvider, IAccessibleViewService, IAccessibleViewSymbol, isIAccessibleViewContentProvider } from '../../../../platform/accessibility/browser/accessibleView.js'; import { ACCESSIBLE_VIEW_SHOWN_STORAGE_PREFIX, IAccessibilityService } from '../../../../platform/accessibility/common/accessibility.js'; @@ -134,7 +135,8 @@ export class AccessibleView extends Disposable implements ITextModelContentProvi this._container.classList.add('hide'); } const codeEditorWidgetOptions: ICodeEditorWidgetOptions = { - contributions: EditorExtensionsRegistry.getEditorContributions().filter(c => c.id !== CodeActionController.ID && c.id !== FloatingEditorClickMenu.ID) + contributions: EditorExtensionsRegistry.getEditorContributions() + .filter(c => c.id !== CodeActionController.ID && c.id !== FloatingEditorClickMenu.ID && c.id !== FloatingEditorToolbar.ID) }; const titleBar = document.createElement('div'); titleBar.classList.add('accessible-view-title-bar'); diff --git a/src/vs/workbench/contrib/chat/browser/actions/chatAccessibilityHelp.ts b/src/vs/workbench/contrib/chat/browser/actions/chatAccessibilityHelp.ts index 213ecddbb47..f0e329ba490 100644 --- a/src/vs/workbench/contrib/chat/browser/actions/chatAccessibilityHelp.ts +++ b/src/vs/workbench/contrib/chat/browser/actions/chatAccessibilityHelp.ts @@ -5,7 +5,6 @@ import { ICodeEditor } from '../../../../../editor/browser/editorBrowser.js'; import { ServicesAccessor } from '../../../../../editor/browser/editorExtensions.js'; -import { ICodeEditorService } from '../../../../../editor/browser/services/codeEditorService.js'; import { AccessibleDiffViewerNext } from '../../../../../editor/browser/widget/diffEditor/commands.js'; import { localize } from '../../../../../nls.js'; import { AccessibleContentProvider, AccessibleViewProviderId, AccessibleViewType } from '../../../../../platform/accessibility/browser/accessibleView.js'; @@ -25,8 +24,7 @@ export class PanelChatAccessibilityHelp implements IAccessibleViewImplementation readonly type = AccessibleViewType.Help; readonly when = ContextKeyExpr.and(ChatContextKeys.location.isEqualTo(ChatAgentLocation.Chat), ChatContextKeys.inQuickChat.negate(), ChatContextKeys.chatModeKind.isEqualTo(ChatModeKind.Ask), ContextKeyExpr.or(ChatContextKeys.inChatSession, ChatContextKeys.isResponse, ChatContextKeys.isRequest)); getProvider(accessor: ServicesAccessor) { - const codeEditor = accessor.get(ICodeEditorService).getActiveCodeEditor() || accessor.get(ICodeEditorService).getFocusedCodeEditor(); - return getChatAccessibilityHelpProvider(accessor, codeEditor ?? undefined, 'panelChat'); + return getChatAccessibilityHelpProvider(accessor, undefined, 'panelChat'); } } @@ -36,8 +34,7 @@ export class QuickChatAccessibilityHelp implements IAccessibleViewImplementation readonly type = AccessibleViewType.Help; readonly when = ContextKeyExpr.and(ChatContextKeys.inQuickChat, ContextKeyExpr.or(ChatContextKeys.inChatSession, ChatContextKeys.isResponse, ChatContextKeys.isRequest)); getProvider(accessor: ServicesAccessor) { - const codeEditor = accessor.get(ICodeEditorService).getActiveCodeEditor() || accessor.get(ICodeEditorService).getFocusedCodeEditor(); - return getChatAccessibilityHelpProvider(accessor, codeEditor ?? undefined, 'quickChat'); + return getChatAccessibilityHelpProvider(accessor, undefined, 'quickChat'); } } @@ -47,8 +44,7 @@ export class EditsChatAccessibilityHelp implements IAccessibleViewImplementation readonly type = AccessibleViewType.Help; readonly when = ContextKeyExpr.and(ChatContextKeyExprs.inEditingMode, ChatContextKeys.inChatInput); getProvider(accessor: ServicesAccessor) { - const codeEditor = accessor.get(ICodeEditorService).getActiveCodeEditor() || accessor.get(ICodeEditorService).getFocusedCodeEditor(); - return getChatAccessibilityHelpProvider(accessor, codeEditor ?? undefined, 'editsView'); + return getChatAccessibilityHelpProvider(accessor, undefined, 'editsView'); } } @@ -58,8 +54,7 @@ export class AgentChatAccessibilityHelp implements IAccessibleViewImplementation readonly type = AccessibleViewType.Help; readonly when = ContextKeyExpr.and(ChatContextKeys.chatModeKind.isEqualTo(ChatModeKind.Agent), ChatContextKeys.inChatInput); getProvider(accessor: ServicesAccessor) { - const codeEditor = accessor.get(ICodeEditorService).getActiveCodeEditor() || accessor.get(ICodeEditorService).getFocusedCodeEditor(); - return getChatAccessibilityHelpProvider(accessor, codeEditor ?? undefined, 'agentView'); + return getChatAccessibilityHelpProvider(accessor, undefined, 'agentView'); } } @@ -134,7 +129,7 @@ export function getAccessibilityHelpText(type: 'panelChat' | 'inlineChat' | 'qui export function getChatAccessibilityHelpProvider(accessor: ServicesAccessor, editor: ICodeEditor | undefined, type: 'panelChat' | 'inlineChat' | 'quickChat' | 'editsView' | 'agentView'): AccessibleContentProvider | undefined { const widgetService = accessor.get(IChatWidgetService); const keybindingService = accessor.get(IKeybindingService); - const inputEditor: ICodeEditor | undefined = type === 'panelChat' || type === 'editsView' || type === 'quickChat' ? widgetService.lastFocusedWidget?.inputEditor : editor; + const inputEditor: ICodeEditor | undefined = widgetService.lastFocusedWidget?.inputEditor; if (!inputEditor) { return; @@ -152,8 +147,10 @@ export function getChatAccessibilityHelpProvider(accessor: ServicesAccessor, edi { type: AccessibleViewType.Help }, () => helpText, () => { - if (type === 'panelChat' && cachedPosition) { - inputEditor.setPosition(cachedPosition); + if (type === 'quickChat' || type === 'editsView' || type === 'agentView' || type === 'panelChat') { + if (cachedPosition) { + inputEditor.setPosition(cachedPosition); + } inputEditor.focus(); } else if (type === 'inlineChat') { @@ -161,9 +158,6 @@ export function getChatAccessibilityHelpProvider(accessor: ServicesAccessor, edi const ctrl = <{ focus(): void } | undefined>editor?.getContribution(INLINE_CHAT_ID); ctrl?.focus(); - } else if (type === 'quickChat' || type === 'editsView' || type === 'agentView') { - // For quickChat, editsView, and agentView, restore focus to the chat widget input - widgetService.lastFocusedWidget?.focusInput(); } }, type === 'panelChat' ? AccessibilityVerbositySettingId.Chat : AccessibilityVerbositySettingId.InlineChat, diff --git a/src/vs/workbench/contrib/chat/browser/actions/chatActions.ts b/src/vs/workbench/contrib/chat/browser/actions/chatActions.ts index fdc5b2ba208..c54230c5cdc 100644 --- a/src/vs/workbench/contrib/chat/browser/actions/chatActions.ts +++ b/src/vs/workbench/contrib/chat/browser/actions/chatActions.ts @@ -738,6 +738,9 @@ export function registerChatActions() { for (const provider of providers) { const sessions = await chatSessionsService.provideChatSessionItems(provider.type, cancellationToken.token); + if (!sessions?.length) { + continue; + } providerNSessions.push(...sessions.map(session => ({ providerType: provider.type, session }))); } @@ -1690,6 +1693,7 @@ export class CopilotTitleBarMenuRendering extends Disposable implements IWorkben const chatSentiment = chatEntitlementService.sentiment; const chatQuotaExceeded = chatEntitlementService.quotas.chat?.percentRemaining === 0; const signedOut = chatEntitlementService.entitlement === ChatEntitlement.Unknown; + const anonymous = chatEntitlementService.anonymous; const free = chatEntitlementService.entitlement === ChatEntitlement.Free; const isAuxiliaryWindow = windowId !== mainWindow.vscodeWindowId; @@ -1697,7 +1701,7 @@ export class CopilotTitleBarMenuRendering extends Disposable implements IWorkben let primaryActionTitle = isAuxiliaryWindow ? localize('openChat', "Open Chat") : localize('toggleChat', "Toggle Chat"); let primaryActionIcon = Codicon.chatSparkle; if (chatSentiment.installed && !chatSentiment.disabled) { - if (signedOut) { + if (signedOut && !anonymous) { primaryActionId = CHAT_SETUP_ACTION_ID; primaryActionTitle = localize('signInToChatSetup', "Sign in to use AI features..."); primaryActionIcon = Codicon.chatSparkleError; @@ -1715,7 +1719,8 @@ export class CopilotTitleBarMenuRendering extends Disposable implements IWorkben }, Event.any( chatEntitlementService.onDidChangeSentiment, chatEntitlementService.onDidChangeQuotaExceeded, - chatEntitlementService.onDidChangeEntitlement + chatEntitlementService.onDidChangeEntitlement, + chatEntitlementService.onDidChangeAnonymous )); // Reduces flicker a bit on reload/restart diff --git a/src/vs/workbench/contrib/chat/browser/actions/chatExecuteActions.ts b/src/vs/workbench/contrib/chat/browser/actions/chatExecuteActions.ts index 262713c22f8..36c9e7b9bc5 100644 --- a/src/vs/workbench/contrib/chat/browser/actions/chatExecuteActions.ts +++ b/src/vs/workbench/contrib/chat/browser/actions/chatExecuteActions.ts @@ -39,7 +39,7 @@ import { ChatRequestVariableSet, isChatRequestFileEntry } from '../../common/cha import { ChatAgentLocation, ChatConfiguration, ChatModeKind, } from '../../common/constants.js'; import { ILanguageModelChatMetadata } from '../../common/languageModels.js'; import { ILanguageModelToolsService } from '../../common/languageModelToolsService.js'; -import { IChatWidget, IChatWidgetService } from '../chat.js'; +import { IChatWidget, IChatWidgetService, showChatWidgetInViewOrEditor } from '../chat.js'; import { getEditingSessionContext } from '../chatEditing/chatEditingActions.js'; import { IChatEditorOptions } from '../chatEditor.js'; import { ACTION_ID_NEW_CHAT, CHAT_CATEGORY, handleCurrentEditingSession, handleModeSwitch } from './chatActions.js'; @@ -177,7 +177,10 @@ export class ChatSubmitAction extends SubmitAction { tooltip: localize('sendToRemoteAgent', "Send to coding agent"), }, keybinding: { - when: ChatContextKeys.inChatInput, + when: ContextKeyExpr.and( + ChatContextKeys.inChatInput, + ChatContextKeys.withinEditSessionDiff.negate(), + ), primary: KeyCode.Enter, weight: KeybindingWeight.EditorContrib }, @@ -186,7 +189,10 @@ export class ChatSubmitAction extends SubmitAction { id: MenuId.ChatExecuteSecondary, group: 'group_1', order: 1, - when: ContextKeyExpr.and(menuCondition, ChatContextKeys.lockedToCodingAgent.negate()), + when: ContextKeyExpr.or( + ChatContextKeys.withinEditSessionDiff, + ContextKeyExpr.and(menuCondition, ChatContextKeys.lockedToCodingAgent.negate()) + ), }, { id: MenuId.ChatExecute, @@ -194,6 +200,7 @@ export class ChatSubmitAction extends SubmitAction { when: ContextKeyExpr.and( whenNotInProgress, menuCondition, + ChatContextKeys.withinEditSessionDiff.negate(), ), group: 'navigation', }] @@ -201,6 +208,78 @@ export class ChatSubmitAction extends SubmitAction { } } +export class ChatDelegateToEditSessionAction extends Action2 { + static readonly ID = 'workbench.action.chat.delegateToEditSession'; + + constructor() { + super({ + id: ChatDelegateToEditSessionAction.ID, + title: localize2('interactive.submit.panel.label', "Send to Edit Session"), + f1: false, + category: CHAT_CATEGORY, + icon: Codicon.commentDiscussion, + keybinding: { + when: ContextKeyExpr.and( + ChatContextKeys.inChatInput, + ChatContextKeys.withinEditSessionDiff, + ), + primary: KeyCode.Enter, + weight: KeybindingWeight.EditorContrib + }, + menu: [ + { + id: MenuId.ChatExecute, + order: 4, + when: ContextKeyExpr.and( + whenNotInProgress, + ChatContextKeys.withinEditSessionDiff, + ), + group: 'navigation', + }, + { + id: MenuId.ChatExecuteSecondary, + group: 'group_1', + order: 1, + when: ContextKeyExpr.and( + whenNotInProgress, + ChatContextKeys.filePartOfEditSession, + ), + } + ] + }); + } + + override async run(accessor: ServicesAccessor, ...args: any[]): Promise { + const context: IChatExecuteActionContext | undefined = args[0]; + const widgetService = accessor.get(IChatWidgetService); + const instantiationService = accessor.get(IInstantiationService); + const inlineWidget = context?.widget ?? widgetService.lastFocusedWidget; + const locationData = inlineWidget?.locationData; + + if (inlineWidget && locationData?.type === ChatAgentLocation.EditorInline && locationData.delegateSessionId) { + const sessionWidget = widgetService.getWidgetBySessionId(locationData.delegateSessionId); + + if (sessionWidget) { + await instantiationService.invokeFunction(showChatWidgetInViewOrEditor, sessionWidget); + sessionWidget.attachmentModel.addContext({ + id: 'vscode.delegate.inline', + kind: 'file', + modelDescription: `User's chat context`, + name: 'delegate-inline', + value: { range: locationData.wholeRange, uri: locationData.document }, + }); + sessionWidget.acceptInput(inlineWidget.getInput(), { + noCommandDetection: true, + enableImplicitContext: false, + }); + + inlineWidget.setInput(''); + locationData.close(); + } + } + } +} + export const ToggleAgentModeActionId = 'workbench.action.chat.toggleAgentMode'; export interface IToggleChatModeArgs { @@ -1027,6 +1106,7 @@ export class CancelEdit extends Action2 { export function registerChatExecuteActions() { registerAction2(ChatSubmitAction); + registerAction2(ChatDelegateToEditSessionAction); registerAction2(ChatEditingSessionSubmitAction); registerAction2(SubmitWithoutDispatchingAction); registerAction2(CancelAction); diff --git a/src/vs/workbench/contrib/chat/browser/chat.contribution.ts b/src/vs/workbench/contrib/chat/browser/chat.contribution.ts index c150cdf5bd8..35c967ad64b 100644 --- a/src/vs/workbench/contrib/chat/browser/chat.contribution.ts +++ b/src/vs/workbench/contrib/chat/browser/chat.contribution.ts @@ -10,6 +10,7 @@ import { Disposable } from '../../../../base/common/lifecycle.js'; import { Schemas } from '../../../../base/common/network.js'; import { isMacintosh } from '../../../../base/common/platform.js'; import { assertDefined } from '../../../../base/common/types.js'; +import product from '../../../../platform/product/common/product.js'; import { registerEditorFeature } from '../../../../editor/common/editorFeatures.js'; import * as nls from '../../../../nls.js'; import { AccessibleViewRegistry } from '../../../../platform/accessibility/browser/accessibleViewRegistry.js'; @@ -581,6 +582,7 @@ configurationRegistry.registerConfiguration({ }, [PromptsConfig.PROMPT_FILES_SUGGEST_KEY]: { type: 'object', + scope: ConfigurationScope.RESOURCE, title: nls.localize( 'chat.promptFilesRecommendations.title', "Prompt File Recommendations", @@ -638,7 +640,7 @@ configurationRegistry.registerConfiguration({ }, [ChatConfiguration.ThinkingStyle]: { type: 'string', - default: 'fixedScrolling', + default: product.quality === 'insider' ? 'fixedScrolling' : 'collapsed', enum: ['collapsed', 'collapsedPreview', 'expanded', 'none', 'collapsedPerItem', 'fixedScrolling'], enumDescriptions: [ nls.localize('chat.agent.thinkingMode.collapsed', "Thinking parts will be collapsed by default."), diff --git a/src/vs/workbench/contrib/chat/browser/chat.ts b/src/vs/workbench/contrib/chat/browser/chat.ts index e1a1b85f8c8..fce0ee035e2 100644 --- a/src/vs/workbench/contrib/chat/browser/chat.ts +++ b/src/vs/workbench/contrib/chat/browser/chat.ts @@ -8,21 +8,24 @@ import { Event } from '../../../../base/common/event.js'; import { IDisposable } from '../../../../base/common/lifecycle.js'; import { URI } from '../../../../base/common/uri.js'; import { ICodeEditor } from '../../../../editor/browser/editorBrowser.js'; +import { ServicesAccessor } from '../../../../editor/browser/editorExtensions.js'; import { Selection } from '../../../../editor/common/core/selection.js'; import { EditDeltaInfo } from '../../../../editor/common/textModelEditSource.js'; import { MenuId } from '../../../../platform/actions/common/actions.js'; import { IContextKeyService } from '../../../../platform/contextkey/common/contextkey.js'; import { createDecorator } from '../../../../platform/instantiation/common/instantiation.js'; +import { IEditorGroupsService } from '../../../services/editor/common/editorGroupsService.js'; import { IWorkbenchLayoutService } from '../../../services/layout/browser/layoutService.js'; import { IViewsService } from '../../../services/views/common/viewsService.js'; import { IChatAgentCommand, IChatAgentData } from '../common/chatAgents.js'; import { IChatResponseModel } from '../common/chatModel.js'; import { IParsedChatRequest } from '../common/chatParserTypes.js'; import { CHAT_PROVIDER_ID } from '../common/chatParticipantContribTypes.js'; -import { IChatElicitationRequest, IChatSendRequestOptions } from '../common/chatService.js'; +import { IChatElicitationRequest, IChatLocationData, IChatSendRequestOptions } from '../common/chatService.js'; import { IChatRequestViewModel, IChatResponseViewModel, IChatViewModel } from '../common/chatViewModel.js'; import { ChatAgentLocation, ChatModeKind } from '../common/constants.js'; import { ChatAttachmentModel } from './chatAttachmentModel.js'; +import { ChatEditorInput } from './chatEditorInput.js'; import { ChatInputPart } from './chatInputPart.js'; import { ChatViewPane } from './chatViewPane.js'; import { IChatViewState, IChatWidgetContrib } from './chatWidget.js'; @@ -47,6 +50,22 @@ export interface IChatWidgetService { getWidgetsByLocations(location: ChatAgentLocation): ReadonlyArray; } +export async function showChatWidgetInViewOrEditor(accessor: ServicesAccessor, widget: IChatWidget) { + if ('viewId' in widget.viewContext) { + await accessor.get(IViewsService).openView(widget.location); + } else { + for (const group of accessor.get(IEditorGroupsService).groups) { + for (const editor of group.editors) { + if (editor instanceof ChatEditorInput && editor.sessionId === widget.viewModel?.sessionId) { + group.openEditor(editor); + return; + } + } + } + } +} + + export async function showChatView(viewsService: IViewsService): Promise { return (await viewsService.openView(ChatViewId))?.widget; } @@ -176,6 +195,7 @@ export type IChatWidgetViewContext = IChatViewViewContext | IChatResourceViewCon export interface IChatAcceptInputOptions { noCommandDetection?: boolean; isVoiceInput?: boolean; + enableImplicitContext?: boolean; // defaults to true } export interface IChatWidget { @@ -198,6 +218,7 @@ export interface IChatWidget { readonly scopedContextKeyService: IContextKeyService; readonly input: ChatInputPart; readonly attachmentModel: ChatAttachmentModel; + readonly locationData?: IChatLocationData; readonly supportsChangingModes: boolean; diff --git a/src/vs/workbench/contrib/chat/browser/chatContentParts/chatAnonymousRateLimitedPart.ts b/src/vs/workbench/contrib/chat/browser/chatContentParts/chatAnonymousRateLimitedPart.ts new file mode 100644 index 00000000000..c56338e78d3 --- /dev/null +++ b/src/vs/workbench/contrib/chat/browser/chatContentParts/chatAnonymousRateLimitedPart.ts @@ -0,0 +1,59 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { $, append } from '../../../../../base/browser/dom.js'; +import { Button } from '../../../../../base/browser/ui/button/button.js'; +import { WorkbenchActionExecutedClassification, WorkbenchActionExecutedEvent } from '../../../../../base/common/actions.js'; +import { Codicon } from '../../../../../base/common/codicons.js'; +import { Disposable, IDisposable } from '../../../../../base/common/lifecycle.js'; +import { ThemeIcon } from '../../../../../base/common/themables.js'; +import { localize } from '../../../../../nls.js'; +import { ICommandService } from '../../../../../platform/commands/common/commands.js'; +import { ITelemetryService } from '../../../../../platform/telemetry/common/telemetry.js'; +import { defaultButtonStyles } from '../../../../../platform/theme/browser/defaultStyles.js'; +import { IChatErrorDetailsPart, IChatRendererContent } from '../../common/chatViewModel.js'; +import { IChatContentPart } from './chatContentParts.js'; + +export class ChatAnonymousRateLimitedPart extends Disposable implements IChatContentPart { + + readonly domNode: HTMLElement; + + constructor( + private readonly content: IChatErrorDetailsPart, + @ICommandService private readonly commandService: ICommandService, + @ITelemetryService private readonly telemetryService: ITelemetryService + ) { + super(); + + this.domNode = $('.chat-rate-limited-error-widget'); + + const icon = append(this.domNode, $('span')); + icon.classList.add(...ThemeIcon.asClassNameArray(Codicon.warning)); + + const messageContainer = append(this.domNode, $('.chat-rate-limited-error-message')); + + const message = append(messageContainer, $('div')); + message.textContent = localize('anonymousRateLimited', "You have reached the limit for using chat without signing in. Sign in to access your benefits or sign up for free to unlock 50 premium requests per month and access to more models."); + + const signInButton = this._register(new Button(messageContainer, { ...defaultButtonStyles, supportIcons: true })); + signInButton.label = localize('signInToContinue', "Sign in to Continue"); + signInButton.element.classList.add('chat-rate-limited-error-button'); + + this._register(signInButton.onDidClick(async () => { + const commandId = 'workbench.action.chat.triggerSetup'; + this.telemetryService.publicLog2('workbenchActionExecuted', { id: commandId, from: 'chat-response' }); + + await this.commandService.executeCommand(commandId); + })); + } + + hasSameContent(other: IChatRendererContent): boolean { + return other.kind === this.content.kind && !!other.errorDetails.isRateLimited; + } + + addDisposable(disposable: IDisposable): void { + this._register(disposable); + } +} diff --git a/src/vs/workbench/contrib/chat/browser/chatContentParts/chatAttachmentsContentPart.ts b/src/vs/workbench/contrib/chat/browser/chatContentParts/chatAttachmentsContentPart.ts index aefdbc56bf6..131cbf43732 100644 --- a/src/vs/workbench/contrib/chat/browser/chatContentParts/chatAttachmentsContentPart.ts +++ b/src/vs/workbench/contrib/chat/browser/chatContentParts/chatAttachmentsContentPart.ts @@ -16,25 +16,41 @@ import { IChatRequestVariableEntry, isElementVariableEntry, isImageVariableEntry import { ChatResponseReferencePartStatusKind, IChatContentReference } from '../../common/chatService.js'; import { DefaultChatAttachmentWidget, ElementChatAttachmentWidget, FileAttachmentWidget, ImageAttachmentWidget, NotebookCellOutputChatAttachmentWidget, PasteAttachmentWidget, PromptFileAttachmentWidget, PromptTextAttachmentWidget, SCMHistoryItemAttachmentWidget, SCMHistoryItemChangeAttachmentWidget, SCMHistoryItemChangeRangeAttachmentWidget, ToolSetOrToolItemAttachmentWidget } from '../chatAttachmentWidgets.js'; +export interface IChatAttachmentsContentPartOptions { + readonly variables: IChatRequestVariableEntry[]; + readonly contentReferences?: ReadonlyArray; + readonly domNode?: HTMLElement; + readonly limit?: number; +} + export class ChatAttachmentsContentPart extends Disposable { private readonly attachedContextDisposables = this._register(new DisposableStore()); private readonly _onDidChangeVisibility = this._register(new Emitter()); private readonly _contextResourceLabels: ResourceLabels; + private _showingAll = false; + + private readonly variables: IChatRequestVariableEntry[]; + private readonly contentReferences: ReadonlyArray; + private readonly limit?: number; + public readonly domNode: HTMLElement | undefined; public contextMenuHandler?: (attachment: IChatRequestVariableEntry, event: MouseEvent) => void; constructor( - private readonly variables: IChatRequestVariableEntry[], - private readonly contentReferences: ReadonlyArray = [], - public readonly domNode: HTMLElement | undefined = dom.$('.chat-attached-context'), + options: IChatAttachmentsContentPartOptions, @IInstantiationService private readonly instantiationService: IInstantiationService, ) { super(); + this.variables = options.variables; + this.contentReferences = options.contentReferences ?? []; + this.limit = options.limit; + this.domNode = options.domNode ?? dom.$('.chat-attached-context'); + this._contextResourceLabels = this._register(this.instantiationService.createInstance(ResourceLabels, { onDidChangeVisibility: this._onDidChangeVisibility.event })); - this.initAttachedContext(domNode); - if (!domNode.childElementCount) { + this.initAttachedContext(this.domNode); + if (!this.domNode.childElementCount) { this.domNode = undefined; } } @@ -44,75 +60,130 @@ export class ChatAttachmentsContentPart extends Disposable { this.attachedContextDisposables.clear(); const hoverDelegate = this.attachedContextDisposables.add(createInstantHoverDelegate()); - for (const attachment of this.variables) { - const resource = URI.isUri(attachment.value) ? attachment.value : attachment.value && typeof attachment.value === 'object' && 'uri' in attachment.value && URI.isUri(attachment.value.uri) ? attachment.value.uri : undefined; - const range = attachment.value && typeof attachment.value === 'object' && 'range' in attachment.value && Range.isIRange(attachment.value.range) ? attachment.value.range : undefined; - const correspondingContentReference = this.contentReferences.find((ref) => (typeof ref.reference === 'object' && 'variableName' in ref.reference && ref.reference.variableName === attachment.name) || (URI.isUri(ref.reference) && basename(ref.reference.path) === attachment.name)); - const isAttachmentOmitted = correspondingContentReference?.options?.status?.kind === ChatResponseReferencePartStatusKind.Omitted; - const isAttachmentPartialOrOmitted = isAttachmentOmitted || correspondingContentReference?.options?.status?.kind === ChatResponseReferencePartStatusKind.Partial; + const visibleAttachments = this.getVisibleAttachments(); + const hasMoreAttachments = this.limit && this.variables.length > this.limit && !this._showingAll; - let widget; - if (attachment.kind === 'tool' || attachment.kind === 'toolset') { - widget = this.instantiationService.createInstance(ToolSetOrToolItemAttachmentWidget, attachment, undefined, { shouldFocusClearButton: false, supportsDeletion: false }, container, this._contextResourceLabels, hoverDelegate); - } else if (isElementVariableEntry(attachment)) { - widget = this.instantiationService.createInstance(ElementChatAttachmentWidget, attachment, undefined, { shouldFocusClearButton: false, supportsDeletion: false }, container, this._contextResourceLabels, hoverDelegate); - } else if (isImageVariableEntry(attachment)) { - attachment.omittedState = isAttachmentPartialOrOmitted ? OmittedState.Full : attachment.omittedState; - widget = this.instantiationService.createInstance(ImageAttachmentWidget, resource, attachment, undefined, { shouldFocusClearButton: false, supportsDeletion: false }, container, this._contextResourceLabels, hoverDelegate); - } else if (isPromptFileVariableEntry(attachment)) { - if (attachment.automaticallyAdded) { - continue; - } - widget = this.instantiationService.createInstance(PromptFileAttachmentWidget, attachment, undefined, { shouldFocusClearButton: false, supportsDeletion: false }, container, this._contextResourceLabels, hoverDelegate); - } else if (isPromptTextVariableEntry(attachment)) { - if (attachment.automaticallyAdded) { - continue; - } - widget = this.instantiationService.createInstance(PromptTextAttachmentWidget, attachment, undefined, { shouldFocusClearButton: false, supportsDeletion: false }, container, this._contextResourceLabels, hoverDelegate); - } else if (resource && (attachment.kind === 'file' || attachment.kind === 'directory')) { - widget = this.instantiationService.createInstance(FileAttachmentWidget, resource, range, attachment, correspondingContentReference, undefined, { shouldFocusClearButton: false, supportsDeletion: false }, container, this._contextResourceLabels, hoverDelegate); - } else if (isPasteVariableEntry(attachment)) { - widget = this.instantiationService.createInstance(PasteAttachmentWidget, attachment, undefined, { shouldFocusClearButton: false, supportsDeletion: false }, container, this._contextResourceLabels, hoverDelegate); - } else if (resource && isNotebookOutputVariableEntry(attachment)) { - widget = this.instantiationService.createInstance(NotebookCellOutputChatAttachmentWidget, resource, attachment, undefined, { shouldFocusClearButton: false, supportsDeletion: false }, container, this._contextResourceLabels, hoverDelegate); - } else if (isSCMHistoryItemVariableEntry(attachment)) { - widget = this.instantiationService.createInstance(SCMHistoryItemAttachmentWidget, attachment, undefined, { shouldFocusClearButton: false, supportsDeletion: false }, container, this._contextResourceLabels, hoverDelegate); - } else if (isSCMHistoryItemChangeVariableEntry(attachment)) { - widget = this.instantiationService.createInstance(SCMHistoryItemChangeAttachmentWidget, attachment, undefined, { shouldFocusClearButton: false, supportsDeletion: false }, container, this._contextResourceLabels, hoverDelegate); - } else if (isSCMHistoryItemChangeRangeVariableEntry(attachment)) { - widget = this.instantiationService.createInstance(SCMHistoryItemChangeRangeAttachmentWidget, attachment, undefined, { shouldFocusClearButton: false, supportsDeletion: false }, container, this._contextResourceLabels, hoverDelegate); - } else { - widget = this.instantiationService.createInstance(DefaultChatAttachmentWidget, resource, range, attachment, correspondingContentReference, undefined, { shouldFocusClearButton: false, supportsDeletion: false }, container, this._contextResourceLabels, hoverDelegate); - } + for (const attachment of visibleAttachments) { + this.renderAttachment(attachment, container, hoverDelegate); + } - let ariaLabel: string | null = null; - - if (isAttachmentPartialOrOmitted) { - widget.element.classList.add('warning'); - } - const description = correspondingContentReference?.options?.status?.description; - if (isAttachmentPartialOrOmitted) { - ariaLabel = `${ariaLabel}${description ? ` ${description}` : ''}`; - for (const selector of ['.monaco-icon-suffix-container', '.monaco-icon-name-container']) { - const element = widget.label.element.querySelector(selector); - if (element) { - element.classList.add('warning'); - } - } - } - - this._register(dom.addDisposableListener(widget.element, 'contextmenu', e => this.contextMenuHandler?.(attachment, e))); - - if (this.attachedContextDisposables.isDisposed) { - widget.dispose(); - return; - } - - if (ariaLabel) { - widget.element.ariaLabel = ariaLabel; - } - - this.attachedContextDisposables.add(widget); + if (hasMoreAttachments) { + this.renderShowMoreButton(container); } } + + private getVisibleAttachments(): IChatRequestVariableEntry[] { + if (!this.limit || this._showingAll) { + return this.variables; + } + return this.variables.slice(0, this.limit); + } + + private renderShowMoreButton(container: HTMLElement) { + const remainingCount = this.variables.length - (this.limit ?? 0); + + // Create a button that looks like the attachment pills + const showMoreButton = dom.$('div.chat-attached-context-attachment.chat-attachments-show-more-button'); + showMoreButton.setAttribute('role', 'button'); + showMoreButton.setAttribute('tabindex', '0'); + showMoreButton.style.cursor = 'pointer'; + + // Add pill icon (ellipsis) + const pillIcon = dom.$('div.chat-attached-context-pill', {}, dom.$('span.codicon.codicon-ellipsis')); + + // Add text label + const textLabel = dom.$('span.chat-attached-context-custom-text'); + textLabel.textContent = `${remainingCount} more`; + + showMoreButton.appendChild(pillIcon); + showMoreButton.appendChild(textLabel); + + // Add click and keyboard event handlers + const clickHandler = () => { + this._showingAll = true; + this.initAttachedContext(container); + }; + + this.attachedContextDisposables.add(dom.addDisposableListener(showMoreButton, 'click', clickHandler)); + this.attachedContextDisposables.add(dom.addDisposableListener(showMoreButton, 'keydown', (e) => { + if (e.key === 'Enter' || e.key === ' ') { + e.preventDefault(); + clickHandler(); + } + })); + + container.appendChild(showMoreButton); + this.attachedContextDisposables.add({ dispose: () => showMoreButton.remove() }); + } + + private renderAttachment(attachment: IChatRequestVariableEntry, container: HTMLElement, hoverDelegate: any) { + const resource = URI.isUri(attachment.value) ? attachment.value : attachment.value && typeof attachment.value === 'object' && 'uri' in attachment.value && URI.isUri(attachment.value.uri) ? attachment.value.uri : undefined; + const range = attachment.value && typeof attachment.value === 'object' && 'range' in attachment.value && Range.isIRange(attachment.value.range) ? attachment.value.range : undefined; + const correspondingContentReference = this.contentReferences.find((ref) => (typeof ref.reference === 'object' && 'variableName' in ref.reference && ref.reference.variableName === attachment.name) || (URI.isUri(ref.reference) && basename(ref.reference.path) === attachment.name)); + const isAttachmentOmitted = correspondingContentReference?.options?.status?.kind === ChatResponseReferencePartStatusKind.Omitted; + const isAttachmentPartialOrOmitted = isAttachmentOmitted || correspondingContentReference?.options?.status?.kind === ChatResponseReferencePartStatusKind.Partial; + + let widget; + if (attachment.kind === 'tool' || attachment.kind === 'toolset') { + widget = this.instantiationService.createInstance(ToolSetOrToolItemAttachmentWidget, attachment, undefined, { shouldFocusClearButton: false, supportsDeletion: false }, container, this._contextResourceLabels, hoverDelegate); + } else if (isElementVariableEntry(attachment)) { + widget = this.instantiationService.createInstance(ElementChatAttachmentWidget, attachment, undefined, { shouldFocusClearButton: false, supportsDeletion: false }, container, this._contextResourceLabels, hoverDelegate); + } else if (isImageVariableEntry(attachment)) { + attachment.omittedState = isAttachmentPartialOrOmitted ? OmittedState.Full : attachment.omittedState; + widget = this.instantiationService.createInstance(ImageAttachmentWidget, resource, attachment, undefined, { shouldFocusClearButton: false, supportsDeletion: false }, container, this._contextResourceLabels, hoverDelegate); + } else if (isPromptFileVariableEntry(attachment)) { + if (attachment.automaticallyAdded) { + return; // Skip automatically added prompt files + } + widget = this.instantiationService.createInstance(PromptFileAttachmentWidget, attachment, undefined, { shouldFocusClearButton: false, supportsDeletion: false }, container, this._contextResourceLabels, hoverDelegate); + } else if (isPromptTextVariableEntry(attachment)) { + if (attachment.automaticallyAdded) { + return; // Skip automatically added prompt text + } + widget = this.instantiationService.createInstance(PromptTextAttachmentWidget, attachment, undefined, { shouldFocusClearButton: false, supportsDeletion: false }, container, this._contextResourceLabels, hoverDelegate); + } else if (resource && (attachment.kind === 'file' || attachment.kind === 'directory')) { + widget = this.instantiationService.createInstance(FileAttachmentWidget, resource, range, attachment, correspondingContentReference, undefined, { shouldFocusClearButton: false, supportsDeletion: false }, container, this._contextResourceLabels, hoverDelegate); + } else if (isPasteVariableEntry(attachment)) { + widget = this.instantiationService.createInstance(PasteAttachmentWidget, attachment, undefined, { shouldFocusClearButton: false, supportsDeletion: false }, container, this._contextResourceLabels, hoverDelegate); + } else if (resource && isNotebookOutputVariableEntry(attachment)) { + widget = this.instantiationService.createInstance(NotebookCellOutputChatAttachmentWidget, resource, attachment, undefined, { shouldFocusClearButton: false, supportsDeletion: false }, container, this._contextResourceLabels, hoverDelegate); + } else if (isSCMHistoryItemVariableEntry(attachment)) { + widget = this.instantiationService.createInstance(SCMHistoryItemAttachmentWidget, attachment, undefined, { shouldFocusClearButton: false, supportsDeletion: false }, container, this._contextResourceLabels, hoverDelegate); + } else if (isSCMHistoryItemChangeVariableEntry(attachment)) { + widget = this.instantiationService.createInstance(SCMHistoryItemChangeAttachmentWidget, attachment, undefined, { shouldFocusClearButton: false, supportsDeletion: false }, container, this._contextResourceLabels, hoverDelegate); + } else if (isSCMHistoryItemChangeRangeVariableEntry(attachment)) { + widget = this.instantiationService.createInstance(SCMHistoryItemChangeRangeAttachmentWidget, attachment, undefined, { shouldFocusClearButton: false, supportsDeletion: false }, container, this._contextResourceLabels, hoverDelegate); + } else { + widget = this.instantiationService.createInstance(DefaultChatAttachmentWidget, resource, range, attachment, correspondingContentReference, undefined, { shouldFocusClearButton: false, supportsDeletion: false }, container, this._contextResourceLabels, hoverDelegate); + } + + let ariaLabel: string | null = null; + + if (isAttachmentPartialOrOmitted) { + widget.element.classList.add('warning'); + } + const description = correspondingContentReference?.options?.status?.description; + if (isAttachmentPartialOrOmitted) { + ariaLabel = `${ariaLabel}${description ? ` ${description}` : ''}`; + for (const selector of ['.monaco-icon-suffix-container', '.monaco-icon-name-container']) { + const element = widget.label.element.querySelector(selector); + if (element) { + element.classList.add('warning'); + } + } + } + + this._register(dom.addDisposableListener(widget.element, 'contextmenu', e => this.contextMenuHandler?.(attachment, e))); + + if (this.attachedContextDisposables.isDisposed) { + widget.dispose(); + return; + } + + if (ariaLabel) { + widget.element.ariaLabel = ariaLabel; + } + + this.attachedContextDisposables.add(widget); + } } diff --git a/src/vs/workbench/contrib/chat/browser/chatContentParts/chatPullRequestContentPart.ts b/src/vs/workbench/contrib/chat/browser/chatContentParts/chatPullRequestContentPart.ts index 2d8d045c1f7..d3df1823d78 100644 --- a/src/vs/workbench/contrib/chat/browser/chatContentParts/chatPullRequestContentPart.ts +++ b/src/vs/workbench/contrib/chat/browser/chatContentParts/chatPullRequestContentPart.ts @@ -16,6 +16,7 @@ import { ThemeIcon } from '../../../../../base/common/themables.js'; import { localize } from '../../../../../nls.js'; import { addDisposableListener } from '../../../../../base/browser/dom.js'; import { IOpenerService } from '../../../../../platform/opener/common/opener.js'; +import { renderMarkdown } from '../../../../../base/browser/markdownRenderer.js'; export class ChatPullRequestContentPart extends Disposable implements IChatContentPart { public readonly domNode: HTMLElement; @@ -41,7 +42,8 @@ export class ChatPullRequestContentPart extends Disposable implements IChatConte const descriptionElement = dom.append(contentContainer, dom.$('.description')); const descriptionWrapper = dom.append(descriptionElement, dom.$('.description-wrapper')); - descriptionWrapper.textContent = this.pullRequestContent.description; + const markdown = this._register(renderMarkdown({ value: this.pullRequestContent.description })); + dom.append(descriptionWrapper, markdown.element); const seeMoreContainer = dom.append(descriptionElement, dom.$('.see-more')); const seeMore: HTMLAnchorElement = dom.append(seeMoreContainer, dom.$('a')); diff --git a/src/vs/workbench/contrib/chat/browser/chatContentParts/chatQuotaExceededPart.ts b/src/vs/workbench/contrib/chat/browser/chatContentParts/chatQuotaExceededPart.ts index c2d815cf30c..af11573e53d 100644 --- a/src/vs/workbench/contrib/chat/browser/chatContentParts/chatQuotaExceededPart.ts +++ b/src/vs/workbench/contrib/chat/browser/chatContentParts/chatQuotaExceededPart.ts @@ -26,20 +26,23 @@ import { IChatContentPart } from './chatContentParts.js'; const $ = dom.$; /** - * Once the sign up button is clicked, and the retry button has been shown, it should be shown every time. + * Once the sign up button is clicked, and the retry + * button has been shown, it should be shown every time. */ let shouldShowRetryButton = false; /** - * Once the 'retry' button is clicked, the wait warning should be shown every time. + * Once the 'retry' button is clicked, the wait warning + * should be shown every time. */ let shouldShowWaitWarning = false; export class ChatQuotaExceededPart extends Disposable implements IChatContentPart { - public readonly domNode: HTMLElement; + + readonly domNode: HTMLElement; private readonly _onDidChangeHeight = this._register(new Emitter()); - public readonly onDidChangeHeight = this._onDidChangeHeight.event; + readonly onDidChangeHeight = this._onDidChangeHeight.event; constructor( element: IChatResponseViewModel, @@ -63,17 +66,15 @@ export class ChatQuotaExceededPart extends Disposable implements IChatContentPar const markdownContent = renderer.render(new MarkdownString(errorDetails.message)); dom.append(messageContainer, markdownContent.element); - let button1Label = ''; + let primaryButtonLabel: string | undefined; switch (chatEntitlementService.entitlement) { case ChatEntitlement.Pro: case ChatEntitlement.ProPlus: - button1Label = localize('enableAdditionalUsage', "Manage paid premium requests"); + primaryButtonLabel = localize('enableAdditionalUsage', "Manage Paid Premium Requests"); break; case ChatEntitlement.Free: - button1Label = localize('upgradeToCopilotPro', "Upgrade to GitHub Copilot Pro"); + primaryButtonLabel = localize('upgradeToCopilotPro', "Upgrade to GitHub Copilot Pro"); break; - default: - button1Label = ''; } let hasAddedWaitWarning = false; @@ -93,14 +94,16 @@ export class ChatQuotaExceededPart extends Disposable implements IChatContentPar } hasAddedRetryButton = true; - const button2 = this._register(new Button(messageContainer, { + const retryButton = this._register(new Button(messageContainer, { buttonBackground: undefined, buttonForeground: asCssVariable(textLinkForeground) })); - button2.element.classList.add('chat-quota-error-secondary-button'); - button2.label = localize('clickToContinue', "Click to retry."); + retryButton.element.classList.add('chat-quota-error-secondary-button'); + retryButton.label = localize('clickToContinue', "Click to Retry"); + this._onDidChangeHeight.fire(); - this._register(button2.onDidClick(() => { + + this._register(retryButton.onDidClick(() => { const widget = chatWidgetService.getWidgetBySessionId(element.sessionId); if (!widget) { return; @@ -113,11 +116,12 @@ export class ChatQuotaExceededPart extends Disposable implements IChatContentPar })); }; - if (button1Label) { - const button1 = this._register(new Button(messageContainer, { ...defaultButtonStyles, supportIcons: true })); - button1.label = button1Label; - button1.element.classList.add('chat-quota-error-button'); - this._register(button1.onDidClick(async () => { + if (primaryButtonLabel) { + const primaryButton = this._register(new Button(messageContainer, { ...defaultButtonStyles, supportIcons: true })); + primaryButton.label = primaryButtonLabel; + primaryButton.element.classList.add('chat-quota-error-button'); + + this._register(primaryButton.onDidClick(async () => { const commandId = chatEntitlementService.entitlement === ChatEntitlement.Free ? 'workbench.action.chat.upgradePlan' : 'workbench.action.chat.manageOverages'; telemetryService.publicLog2('workbenchActionExecuted', { id: commandId, from: 'chat-response' }); await commandService.executeCommand(commandId); diff --git a/src/vs/workbench/contrib/chat/browser/chatContentParts/chatThinkingContentPart.ts b/src/vs/workbench/contrib/chat/browser/chatContentParts/chatThinkingContentPart.ts index 0ba19b7d39b..9d94eb25aa8 100644 --- a/src/vs/workbench/contrib/chat/browser/chatContentParts/chatThinkingContentPart.ts +++ b/src/vs/workbench/contrib/chat/browser/chatContentParts/chatThinkingContentPart.ts @@ -18,13 +18,17 @@ import { ButtonWithIcon } from '../../../../../base/browser/ui/button/button.js' import { Codicon } from '../../../../../base/common/codicons.js'; import { ThemeIcon } from '../../../../../base/common/themables.js'; +function stripImSepMarkers(text: string): string { + return text.replace(/<\|im_sep\|>(\*{4,})?/g, ''); +} + function extractTextFromPart(content: IChatThinkingPart): string { const raw = Array.isArray(content.value) ? content.value.join('') : (content.value || ''); - return raw.replace(/<\|im_sep\|>\*{4,}/g, '').trim(); + return stripImSepMarkers(raw).trim(); } function extractTitleFromThinkingContent(content: string): string | undefined { - const headerMatch = content.match(/^\*\*([^*]+)\*\*\s*\n\n/); + const headerMatch = content.match(/^\*\*([^*]+)\*\*\s*/); return headerMatch ? headerMatch[1].trim() : undefined; } @@ -47,6 +51,8 @@ export class ChatThinkingContentPart extends ChatCollapsibleContentPart implemen private fixedContainer: HTMLElement | undefined; private headerButton: ButtonWithIcon | undefined; private caret: HTMLElement | undefined; + private lastExtractedTitle: string | undefined; + private hasMultipleItems: boolean = false; constructor( content: IChatThinkingPart, @@ -68,6 +74,9 @@ export class ChatThinkingContentPart extends ChatCollapsibleContentPart implemen this.fixedScrollingMode = mode === 'fixedScrolling'; this.currentTitle = extractedTitle; + if (extractedTitle !== this.defaultTitle) { + this.lastExtractedTitle = extractedTitle; + } this.currentThinkingValue = this.parseContent(initialText); if (mode === 'expanded' || mode === 'collapsedPreview' || mode === 'fixedScrolling') { this.setExpanded(true); @@ -99,7 +108,7 @@ export class ChatThinkingContentPart extends ChatCollapsibleContentPart implemen } private parseContent(content: string): string { - let cleaned = content.replace(/<\|im_sep\|>\*{4,}/g, '').trim(); + let cleaned = stripImSepMarkers(content).trim(); if (this.perItemCollapsedMode) { cleaned = cleaned.replace(/^\*\*[^*]+\*\*\s*\n+(?:\s*\n)*/, '').trim(); } @@ -240,17 +249,47 @@ export class ChatThinkingContentPart extends ChatCollapsibleContentPart implemen } } - // if title is present now (e.g., arrived mid-stream), update the header label - if (!this.fixedScrollingMode) { - const maybeTitle = extractTitleFromThinkingContent(raw); - if (maybeTitle && maybeTitle !== this.currentTitle) { - this.setTitle(maybeTitle); - this.currentTitle = maybeTitle; - } + const extractedTitle = extractTitleFromThinkingContent(raw); + if (!extractedTitle || extractedTitle === this.currentTitle) { + return; + } + this.lastExtractedTitle = extractedTitle; + + if (this.fixedScrollingMode && this.headerButton) { + const label = localize('chat.thinking.fixed.progress.withHeader', 'Thinking: {0}{1}', this.lastExtractedTitle, (!this.perItemCollapsedMode && this.hasMultipleItems) ? '...' : ''); + this.headerButton.label = label; + } else if (!this.perItemCollapsedMode) { + const label = localize('chat.thinking.progress.withHeader', '{0}{1}', this.lastExtractedTitle, (!this.perItemCollapsedMode && this.hasMultipleItems) ? '...' : ''); + this.setTitle(label); + this.currentTitle = label; + } else { + this.setTitle(this.lastExtractedTitle); + this.currentTitle = this.lastExtractedTitle; } } public finalizeTitleIfDefault(): void { + if (this.fixedScrollingMode) { + let finalLabel: string; + if (this.lastExtractedTitle) { + finalLabel = localize('chat.thinking.fixed.done.withHeader', 'Done: {0}{1}', this.lastExtractedTitle, (!this.perItemCollapsedMode && this.hasMultipleItems) ? '...' : ''); + } else { + finalLabel = localize('chat.thinking.fixed.done.generic', 'Thought for a few seconds'); + } + if (this.headerButton) { + this.headerButton.label = finalLabel; + this.headerButton.icon = Codicon.passFilled; + } + + this.currentTitle = finalLabel; + + if (this.fixedContainer) { + this.fixedContainer.classList.toggle('finished', true); + this.setFixedCollapsedState(true); + } + return; + } + if (this.currentTitle === this.defaultTitle) { const suffix = localize('chat.thinking.fixed.done.generic', 'Thought for a few seconds'); this.setTitle(suffix); @@ -274,6 +313,7 @@ export class ChatThinkingContentPart extends ChatCollapsibleContentPart implemen // makes a new text container. when we update, we now update this container. public setupThinkingContainer(content: IChatThinkingPart, context: IChatContentPartRenderContext) { + this.hasMultipleItems = true; if (this.perItemCollapsedMode) { this.createThinkingItemContainer(); } else { diff --git a/src/vs/workbench/contrib/chat/browser/chatContentParts/chatToolInputOutputContentPart.ts b/src/vs/workbench/contrib/chat/browser/chatContentParts/chatToolInputOutputContentPart.ts index 0525980d6f1..b52a35e54ee 100644 --- a/src/vs/workbench/contrib/chat/browser/chatContentParts/chatToolInputOutputContentPart.ts +++ b/src/vs/workbench/contrib/chat/browser/chatContentParts/chatToolInputOutputContentPart.ts @@ -103,6 +103,7 @@ export class ChatCollapsibleInputOutputContentPart extends Disposable { @IContextKeyService private readonly contextKeyService: IContextKeyService, @IInstantiationService private readonly _instantiationService: IInstantiationService, @IContextMenuService private readonly _contextMenuService: IContextMenuService, + @IFileService private readonly _fileService: IFileService, ) { super(); this._currentWidth = width; @@ -220,19 +221,30 @@ export class ChatCollapsibleInputOutputContentPart extends Disposable { dom.h('.chat-collapsible-io-resource-actions@actions'), ]); - const entries = parts.map((part): IChatRequestVariableEntry => { + this.fillInResourceGroup(parts, el.items, el.actions).then(() => this._onDidChangeHeight.fire()); + + container.appendChild(el.root); + return el.root; + } + + private async fillInResourceGroup(parts: IChatCollapsibleIODataPart[], itemsContainer: HTMLElement, actionsContainer: HTMLElement) { + const entries = await Promise.all(parts.map(async (part): Promise => { if (part.mimeType && getAttachableImageExtension(part.mimeType)) { - return { kind: 'image', id: generateUuid(), name: basename(part.uri), value: part.value, mimeType: part.mimeType, isURL: false, references: [{ kind: 'reference', reference: part.uri }] }; + const value = part.value ?? await this._fileService.readFile(part.uri).then(f => f.value.buffer, () => undefined); + return { kind: 'image', id: generateUuid(), name: basename(part.uri), value, mimeType: part.mimeType, isURL: false, references: [{ kind: 'reference', reference: part.uri }] }; } else { return { kind: 'file', id: generateUuid(), name: basename(part.uri), fullName: part.uri.path, value: part.uri }; } - }); + })); const attachments = this._register(this._instantiationService.createInstance( ChatAttachmentsContentPart, - entries, - undefined, - undefined, + { + variables: entries, + limit: 5, + contentReferences: undefined, + domNode: undefined + } )); attachments.contextMenuHandler = (attachment, event) => { @@ -251,19 +263,17 @@ export class ChatCollapsibleInputOutputContentPart extends Disposable { } }; - el.items.appendChild(attachments.domNode!); + itemsContainer.appendChild(attachments.domNode!); - const toolbar = this._register(this._instantiationService.createInstance(MenuWorkbenchToolBar, el.actions, MenuId.ChatToolOutputResourceToolbar, { + const toolbar = this._register(this._instantiationService.createInstance(MenuWorkbenchToolBar, actionsContainer, MenuId.ChatToolOutputResourceToolbar, { menuOptions: { shouldForwardArgs: true, }, })); toolbar.context = { parts } satisfies IChatToolOutputResourceToolbarContext; - - container.appendChild(el.root); - return el.root; } + private addCodeBlock(part: IChatCollapsibleIOCodePart, container: HTMLElement) { const data: ICodeBlockData = { languageId: part.languageId, diff --git a/src/vs/workbench/contrib/chat/browser/chatContentParts/media/chatConfirmationWidget.css b/src/vs/workbench/contrib/chat/browser/chatContentParts/media/chatConfirmationWidget.css index e0f4a3cc455..a1ae81f9076 100644 --- a/src/vs/workbench/contrib/chat/browser/chatContentParts/media/chatConfirmationWidget.css +++ b/src/vs/workbench/contrib/chat/browser/chatContentParts/media/chatConfirmationWidget.css @@ -43,6 +43,14 @@ } .chat-confirmation-widget-container { + position: relative; + + > .monaco-progress-container.active { + top: 0px; + border-top-left-radius: 10px; + border-top-right-radius: 10px; + } + .chat-collapsible-top-level-resource-group { margin-top: -12px; margin-bottom: 12px; diff --git a/src/vs/workbench/contrib/chat/browser/chatContentParts/media/chatPullRequestContent.css b/src/vs/workbench/contrib/chat/browser/chatContentParts/media/chatPullRequestContent.css index df72d9464d9..345cdae7c23 100644 --- a/src/vs/workbench/contrib/chat/browser/chatContentParts/media/chatPullRequestContent.css +++ b/src/vs/workbench/contrib/chat/browser/chatContentParts/media/chatPullRequestContent.css @@ -35,6 +35,10 @@ border-bottom: 1px solid var(--vscode-chat-requestBorder); } + p { + margin: 0px; + } + .description .see-more { display: none; position: absolute; diff --git a/src/vs/workbench/contrib/chat/browser/chatContentParts/toolInvocationParts/chatTerminalToolProgressPart.ts b/src/vs/workbench/contrib/chat/browser/chatContentParts/toolInvocationParts/chatTerminalToolProgressPart.ts index a565e12c3b0..02ad59b19fb 100644 --- a/src/vs/workbench/contrib/chat/browser/chatContentParts/toolInvocationParts/chatTerminalToolProgressPart.ts +++ b/src/vs/workbench/contrib/chat/browser/chatContentParts/toolInvocationParts/chatTerminalToolProgressPart.ts @@ -5,7 +5,7 @@ import { h } from '../../../../../../base/browser/dom.js'; import { Codicon } from '../../../../../../base/common/codicons.js'; -import { MarkdownString } from '../../../../../../base/common/htmlContent.js'; +import { isMarkdownString, MarkdownString } from '../../../../../../base/common/htmlContent.js'; import { MarkdownRenderer } from '../../../../../../editor/browser/widget/markdownRenderer/browser/markdownRenderer.js'; import { IInstantiationService } from '../../../../../../platform/instantiation/common/instantiation.js'; import { IOpenerService } from '../../../../../../platform/opener/common/opener.js'; @@ -22,9 +22,9 @@ import { BaseChatToolInvocationSubPart } from './chatToolInvocationSubPart.js'; import '../media/chatTerminalToolProgressPart.css'; import { TerminalContribSettingId } from '../../../../terminal/terminalContribExports.js'; import { ConfigurationTarget } from '../../../../../../platform/configuration/common/configuration.js'; -import { matchesScheme, Schemas } from '../../../../../../base/common/network.js'; import type { ICodeBlockRenderOptions } from '../../codeBlockPart.js'; import { ChatConfiguration } from '../../../common/constants.js'; +import { CommandsRegistry } from '../../../../../../platform/commands/common/commands.js'; export class ChatTerminalToolProgressPart extends BaseChatToolInvocationSubPart { public readonly domNode: HTMLElement; @@ -45,7 +45,6 @@ export class ChatTerminalToolProgressPart extends BaseChatToolInvocationSubPart codeBlockModelCollection: CodeBlockModelCollection, @IInstantiationService instantiationService: IInstantiationService, @IOpenerService openerService: IOpenerService, - @IPreferencesService preferencesService: IPreferencesService, ) { super(toolInvocation); @@ -72,7 +71,10 @@ export class ChatTerminalToolProgressPart extends BaseChatToolInvocationSubPart if (toolInvocation.pastTenseMessage) { pastTenseMessage = `${typeof toolInvocation.pastTenseMessage === 'string' ? toolInvocation.pastTenseMessage : toolInvocation.pastTenseMessage.value}`; } - const markdownContent = new MarkdownString(pastTenseMessage, { supportThemeIcons: true }); + const markdownContent = new MarkdownString(pastTenseMessage, { + supportThemeIcons: true, + isTrusted: isMarkdownString(toolInvocation.pastTenseMessage) ? toolInvocation.pastTenseMessage.isTrusted : false, + }); const chatMarkdownContent: IChatMarkdownContent = { kind: 'markdownContent', content: markdownContent, @@ -86,50 +88,7 @@ export class ChatTerminalToolProgressPart extends BaseChatToolInvocationSubPart wordWrap: 'on' } }; - this.markdownPart = this._register(instantiationService.createInstance(ChatMarkdownContentPart, chatMarkdownContent, context, editorPool, false, codeBlockStartIndex, renderer, { - actionHandler: (content) => { - if (matchesScheme(content, Schemas.https)) { - openerService.open(content); - return; - } - const [type, scopeRaw] = content.split('_'); - switch (type) { - case 'settings': { - if (scopeRaw === 'global') { - preferencesService.openSettings({ - query: `@id:${ChatConfiguration.GlobalAutoApprove}` - }); - } else { - const scope = parseInt(scopeRaw); - const target = !isNaN(scope) ? scope as ConfigurationTarget : undefined; - const options: IOpenSettingsOptions = { - jsonEditor: true, - revealSetting: { - key: TerminalContribSettingId.AutoApprove - } - }; - switch (target) { - case ConfigurationTarget.APPLICATION: preferencesService.openApplicationSettings(options); break; - case ConfigurationTarget.USER: - case ConfigurationTarget.USER_LOCAL: preferencesService.openUserSettings(options); break; - case ConfigurationTarget.USER_REMOTE: preferencesService.openRemoteSettings(options); break; - case ConfigurationTarget.WORKSPACE: - case ConfigurationTarget.WORKSPACE_FOLDER: preferencesService.openWorkspaceSettings(options); break; - default: { - // Fallback if something goes wrong - preferencesService.openSettings({ - target: ConfigurationTarget.USER, - query: `@id:${TerminalContribSettingId.AutoApprove}`, - }); - break; - } - } - } - break; - } - } - }, - }, currentWidthDelegate(), codeBlockModelCollection, { codeBlockRenderOptions })); + this.markdownPart = this._register(instantiationService.createInstance(ChatMarkdownContentPart, chatMarkdownContent, context, editorPool, false, codeBlockStartIndex, renderer, {}, currentWidthDelegate(), codeBlockModelCollection, { codeBlockRenderOptions })); this._register(this.markdownPart.onDidChangeHeight(() => this._onDidChangeHeight.fire())); elements.message.append(this.markdownPart.domNode); @@ -137,3 +96,40 @@ export class ChatTerminalToolProgressPart extends BaseChatToolInvocationSubPart this.domNode = progressPart.domNode; } } + +export const openTerminalSettingsLinkCommandId = '_chat.openTerminalSettingsLink'; + +CommandsRegistry.registerCommand(openTerminalSettingsLinkCommandId, async (accessor, scopeRaw: string) => { + const preferencesService = accessor.get(IPreferencesService); + + if (scopeRaw === 'global') { + preferencesService.openSettings({ + query: `@id:${ChatConfiguration.GlobalAutoApprove}` + }); + } else { + const scope = parseInt(scopeRaw); + const target = !isNaN(scope) ? scope as ConfigurationTarget : undefined; + const options: IOpenSettingsOptions = { + jsonEditor: true, + revealSetting: { + key: TerminalContribSettingId.AutoApprove + } + }; + switch (target) { + case ConfigurationTarget.APPLICATION: preferencesService.openApplicationSettings(options); break; + case ConfigurationTarget.USER: + case ConfigurationTarget.USER_LOCAL: preferencesService.openUserSettings(options); break; + case ConfigurationTarget.USER_REMOTE: preferencesService.openRemoteSettings(options); break; + case ConfigurationTarget.WORKSPACE: + case ConfigurationTarget.WORKSPACE_FOLDER: preferencesService.openWorkspaceSettings(options); break; + default: { + // Fallback if something goes wrong + preferencesService.openSettings({ + target: ConfigurationTarget.USER, + query: `@id:${TerminalContribSettingId.AutoApprove}`, + }); + break; + } + } + } +}); diff --git a/src/vs/workbench/contrib/chat/browser/chatEditing/chatEditingModifiedDocumentEntry.ts b/src/vs/workbench/contrib/chat/browser/chatEditing/chatEditingModifiedDocumentEntry.ts index daebbf892a5..aebc71b8754 100644 --- a/src/vs/workbench/contrib/chat/browser/chatEditing/chatEditingModifiedDocumentEntry.ts +++ b/src/vs/workbench/contrib/chat/browser/chatEditing/chatEditingModifiedDocumentEntry.ts @@ -9,7 +9,7 @@ import { ITransaction, autorun, transaction } from '../../../../../base/common/o import { assertType } from '../../../../../base/common/types.js'; import { URI } from '../../../../../base/common/uri.js'; import { getCodeEditor } from '../../../../../editor/browser/editorBrowser.js'; -import { TextEdit } from '../../../../../editor/common/languages.js'; +import { Location, TextEdit } from '../../../../../editor/common/languages.js'; import { ILanguageService } from '../../../../../editor/common/languages/language.js'; import { ITextModel } from '../../../../../editor/common/model.js'; import { SingleModelEditStackElement } from '../../../../../editor/common/model/editStack.js'; @@ -188,6 +188,10 @@ export class ChatEditingModifiedDocumentEntry extends AbstractChatEditingModifie }; } + public override hasModificationAt(location: Location): boolean { + return location.uri.toString() === this.modifiedModel.uri.toString() && this._textModelChangeService.hasHunkAt(location.range); + } + async restoreFromSnapshot(snapshot: ISnapshotEntry, restoreToDisk = true) { this._stateObs.set(snapshot.state, undefined); await this._textModelChangeService.resetDocumentValues(snapshot.original, restoreToDisk ? snapshot.current : undefined); diff --git a/src/vs/workbench/contrib/chat/browser/chatEditing/chatEditingModifiedFileEntry.ts b/src/vs/workbench/contrib/chat/browser/chatEditing/chatEditingModifiedFileEntry.ts index b23b06910c7..b562faab058 100644 --- a/src/vs/workbench/contrib/chat/browser/chatEditing/chatEditingModifiedFileEntry.ts +++ b/src/vs/workbench/contrib/chat/browser/chatEditing/chatEditingModifiedFileEntry.ts @@ -10,7 +10,7 @@ import { Schemas } from '../../../../../base/common/network.js'; import { clamp } from '../../../../../base/common/numbers.js'; import { autorun, derived, IObservable, ITransaction, observableValue, observableValueOpts, transaction } from '../../../../../base/common/observable.js'; import { URI } from '../../../../../base/common/uri.js'; -import { TextEdit } from '../../../../../editor/common/languages.js'; +import { Location, TextEdit } from '../../../../../editor/common/languages.js'; import { localize } from '../../../../../nls.js'; import { IConfigurationService } from '../../../../../platform/configuration/common/configuration.js'; import { IFileService } from '../../../../../platform/files/common/files.js'; @@ -149,11 +149,11 @@ export abstract class AbstractChatEditingModifiedFileEntry extends Disposable im if (inProgress === false && !this.reviewMode.read(r)) { // AUTO accept mode (when request is done) - const acceptTimeout = this._autoAcceptTimeout.get() * 1000; + const acceptTimeout = this._autoAcceptTimeout.read(undefined) * 1000; const future = Date.now() + acceptTimeout; const update = () => { - const reviewMode = this.reviewMode.get(); + const reviewMode = this.reviewMode.read(undefined); if (reviewMode) { // switched back to review mode this._autoAcceptCtrl.set(undefined, undefined); @@ -182,6 +182,8 @@ export abstract class AbstractChatEditingModifiedFileEntry extends Disposable im } } + public abstract hasModificationAt(location: Location): boolean; + acquire() { this._refCounter++; return this; diff --git a/src/vs/workbench/contrib/chat/browser/chatEditing/chatEditingModifiedNotebookEntry.ts b/src/vs/workbench/contrib/chat/browser/chatEditing/chatEditingModifiedNotebookEntry.ts index bc1f20958fd..3dc39736b9f 100644 --- a/src/vs/workbench/contrib/chat/browser/chatEditing/chatEditingModifiedNotebookEntry.ts +++ b/src/vs/workbench/contrib/chat/browser/chatEditing/chatEditingModifiedNotebookEntry.ts @@ -18,7 +18,7 @@ import { LineRange } from '../../../../../editor/common/core/ranges/lineRange.js import { Range } from '../../../../../editor/common/core/range.js'; import { nullDocumentDiff } from '../../../../../editor/common/diff/documentDiffProvider.js'; import { DetailedLineRangeMapping, RangeMapping } from '../../../../../editor/common/diff/rangeMapping.js'; -import { TextEdit } from '../../../../../editor/common/languages.js'; +import { Location, TextEdit } from '../../../../../editor/common/languages.js'; import { ITextModel } from '../../../../../editor/common/model.js'; import { IModelService } from '../../../../../editor/common/services/model.js'; import { ITextModelService } from '../../../../../editor/common/services/resolverService.js'; @@ -197,6 +197,10 @@ export class ChatEditingModifiedNotebookEntry extends AbstractChatEditingModifie this._register(this.modifiedModel.onDidChangeContent(this.mirrorNotebookEdits, this)); } + public override hasModificationAt(location: Location): boolean { + return this.cellEntryMap.get(location.uri)?.hasModificationAt(location.range) ?? false; + } + initializeModelsFromDiffImpl(cellsDiffInfo: CellDiffInfo[]) { this.cellEntryMap.forEach(entry => entry.dispose()); this.cellEntryMap.clear(); @@ -998,7 +1002,7 @@ export class ChatEditingModifiedNotebookEntry extends AbstractChatEditingModifie if (this.modifiedModel.cells.indexOf(cell) === -1) { return; } - const diffs = this.cellsDiffInfo.get().slice(); + const diffs = this.cellsDiffInfo.read(undefined).slice(); const index = this.modifiedModel.cells.indexOf(cell); let entry = diffs.find(entry => entry.modifiedCellIndex === index); if (!entry) { @@ -1007,13 +1011,13 @@ export class ChatEditingModifiedNotebookEntry extends AbstractChatEditingModifie } const entryIndex = diffs.indexOf(entry); entry.diff.set(cellEntry.diffInfo.read(r), undefined); - if (cellEntry.diffInfo.get().identical && entry.type === 'modified') { + if (cellEntry.diffInfo.read(undefined).identical && entry.type === 'modified') { entry = { ...entry, type: 'unchanged', }; } - if (!cellEntry.diffInfo.get().identical && entry.type === 'unchanged') { + if (!cellEntry.diffInfo.read(undefined).identical && entry.type === 'unchanged') { entry = { ...entry, type: 'modified', diff --git a/src/vs/workbench/contrib/chat/browser/chatEditing/chatEditingSession.ts b/src/vs/workbench/contrib/chat/browser/chatEditing/chatEditingSession.ts index 267571f8e44..e21b73357da 100644 --- a/src/vs/workbench/contrib/chat/browser/chatEditing/chatEditingSession.ts +++ b/src/vs/workbench/contrib/chat/browser/chatEditing/chatEditingSession.ts @@ -111,7 +111,6 @@ export class ChatEditingSession extends Disposable implements IChatEditingSessio private readonly _entriesObs = observableValue(this, []); public get entries(): IObservable { - this._assertNotDisposed(); return this._entriesObs; } diff --git a/src/vs/workbench/contrib/chat/browser/chatEditing/chatEditingTextModelChangeService.ts b/src/vs/workbench/contrib/chat/browser/chatEditing/chatEditingTextModelChangeService.ts index 8fcd12e5419..186d7f0748f 100644 --- a/src/vs/workbench/contrib/chat/browser/chatEditing/chatEditingTextModelChangeService.ts +++ b/src/vs/workbench/contrib/chat/browser/chatEditing/chatEditingTextModelChangeService.ts @@ -15,7 +15,8 @@ import { assertType } from '../../../../../base/common/types.js'; import { URI } from '../../../../../base/common/uri.js'; import { EditOperation, ISingleEditOperation } from '../../../../../editor/common/core/editOperation.js'; import { StringEdit } from '../../../../../editor/common/core/edits/stringEdit.js'; -import { Range } from '../../../../../editor/common/core/range.js'; +import { IRange, Range } from '../../../../../editor/common/core/range.js'; +import { LineRange } from '../../../../../editor/common/core/ranges/lineRange.js'; import { IDocumentDiff, nullDocumentDiff } from '../../../../../editor/common/diff/documentDiffProvider.js'; import { DetailedLineRangeMapping } from '../../../../../editor/common/diff/rangeMapping.js'; import { TextEdit, VersionedExtensionId } from '../../../../../editor/common/languages.js'; @@ -23,7 +24,7 @@ import { IModelDeltaDecoration, ITextModel, ITextSnapshot, MinimapPosition, Over import { ModelDecorationOptions } from '../../../../../editor/common/model/textModel.js'; import { offsetEditFromContentChanges, offsetEditFromLineRangeMapping, offsetEditToEditOperations } from '../../../../../editor/common/model/textModelStringEdit.js'; import { IEditorWorkerService } from '../../../../../editor/common/services/editorWorker.js'; -import { TextModelEditSource, EditSources } from '../../../../../editor/common/textModelEditSource.js'; +import { EditSources, TextModelEditSource } from '../../../../../editor/common/textModelEditSource.js'; import { IModelContentChangedEvent } from '../../../../../editor/common/textModelEvents.js'; import { AccessibilitySignal, IAccessibilitySignalService } from '../../../../../platform/accessibilitySignal/browser/accessibilitySignalService.js'; import { editorSelectionBackground } from '../../../../../platform/theme/common/colorRegistry.js'; @@ -417,6 +418,11 @@ export class ChatEditingTextModelChangeService extends Disposable { } } + public hasHunkAt(range: IRange) { + // return true if the range overlaps a diff range + return this._diffInfo.get().changes.some(c => c.modified.intersectsStrict(LineRange.fromRangeInclusive(range))); + } + private async _updateDiffInfo(): Promise { if (this.originalModel.isDisposed() || this.modifiedModel.isDisposed() || this._store.isDisposed) { diff --git a/src/vs/workbench/contrib/chat/browser/chatEditing/notebook/chatEditingNotebookCellEntry.ts b/src/vs/workbench/contrib/chat/browser/chatEditing/notebook/chatEditingNotebookCellEntry.ts index 44186fcb9ec..5a61bece089 100644 --- a/src/vs/workbench/contrib/chat/browser/chatEditing/notebook/chatEditingNotebookCellEntry.ts +++ b/src/vs/workbench/contrib/chat/browser/chatEditing/notebook/chatEditingNotebookCellEntry.ts @@ -6,6 +6,7 @@ import { Disposable, DisposableStore } from '../../../../../../base/common/lifecycle.js'; import { IObservable, observableValue, transaction } from '../../../../../../base/common/observable.js'; import { URI } from '../../../../../../base/common/uri.js'; +import { IRange } from '../../../../../../editor/common/core/range.js'; import { IDocumentDiff } from '../../../../../../editor/common/diff/documentDiffProvider.js'; import { DetailedLineRangeMapping } from '../../../../../../editor/common/diff/rangeMapping.js'; import { TextEdit } from '../../../../../../editor/common/languages.js'; @@ -75,6 +76,10 @@ export class ChatEditingNotebookCellEntry extends Disposable { } + public hasModificationAt(range: IRange): boolean { + return this._textModelChangeService.hasHunkAt(range); + } + public clearCurrentEditLineDecoration() { if (this.modifiedModel.isDisposed()) { return; diff --git a/src/vs/workbench/contrib/chat/browser/chatEditing/notebook/chatEditingNotebookEditorIntegration.ts b/src/vs/workbench/contrib/chat/browser/chatEditing/notebook/chatEditingNotebookEditorIntegration.ts index 2b6f1f1c40c..d8486785aaf 100644 --- a/src/vs/workbench/contrib/chat/browser/chatEditing/notebook/chatEditingNotebookEditorIntegration.ts +++ b/src/vs/workbench/contrib/chat/browser/chatEditing/notebook/chatEditingNotebookEditorIntegration.ts @@ -215,7 +215,7 @@ class ChatEditingNotebookEditorWidgetIntegration extends Disposable implements I onDidChangeVisibleRanges.read(r); if (!changes.length) { this.cellEditorIntegrations.forEach(({ diff }) => { - diff.set({ ...diff.get(), ...nullDocumentDiff }, undefined); + diff.set({ ...diff.read(undefined), ...nullDocumentDiff }, undefined); }); return; } @@ -258,7 +258,7 @@ class ChatEditingNotebookEditorWidgetIntegration extends Disposable implements I const currentDiff = this.cellEditorIntegrations.get(cell); if (currentDiff) { // Do not unnecessarily trigger a change event - if (!areDocumentDiff2Equal(currentDiff.diff.get(), diff)) { + if (!areDocumentDiff2Equal(currentDiff.diff.read(undefined), diff)) { currentDiff.diff.set(diff, undefined); } } else { diff --git a/src/vs/workbench/contrib/chat/browser/chatInputPart.ts b/src/vs/workbench/contrib/chat/browser/chatInputPart.ts index 1b9259e7430..155dd2f3ef1 100644 --- a/src/vs/workbench/contrib/chat/browser/chatInputPart.ts +++ b/src/vs/workbench/contrib/chat/browser/chatInputPart.ts @@ -89,7 +89,7 @@ import { ILanguageModelChatMetadata, ILanguageModelChatMetadataAndIdentifier, IL import { ILanguageModelToolsService } from '../common/languageModelToolsService.js'; import { PromptsType } from '../common/promptSyntax/promptTypes.js'; import { IPromptsService } from '../common/promptSyntax/service/promptsService.js'; -import { CancelAction, ChatEditingSessionSubmitAction, ChatOpenModelPickerActionId, ChatSubmitAction, IChatExecuteActionContext, OpenModePickerAction } from './actions/chatExecuteActions.js'; +import { CancelAction, ChatDelegateToEditSessionAction, ChatEditingSessionSubmitAction, ChatOpenModelPickerActionId, ChatSubmitAction, IChatExecuteActionContext, OpenModePickerAction } from './actions/chatExecuteActions.js'; import { ImplicitContextAttachmentWidget } from './attachments/implicitContextAttachment.js'; import { IChatWidget } from './chat.js'; import { ChatAttachmentModel } from './chatAttachmentModel.js'; @@ -173,11 +173,15 @@ export class ChatInputPart extends Disposable implements IHistoryNavigationWidge readonly selectedToolsModel: ChatSelectedTools; + public getAttachedContext(sessionId: string) { + const contextArr = new ChatRequestVariableSet(); + contextArr.add(...this.attachmentModel.attachments); + return contextArr; + } + public getAttachedAndImplicitContext(sessionId: string): ChatRequestVariableSet { - const contextArr = new ChatRequestVariableSet(); - - contextArr.add(...this.attachmentModel.attachments); + const contextArr = this.getAttachedContext(sessionId); if ((this.implicitContext?.enabled && this.implicitContext?.value) || (isLocation(this.implicitContext?.value) && this.configurationService.getValue('chat.implicitContext.suggestedContext'))) { const implicitChatVariables = this.implicitContext.toBaseEntries(); @@ -280,6 +284,8 @@ export class ChatInputPart extends Disposable implements IHistoryNavigationWidge */ private promptFileAttached: IContextKey; private chatModeKindKey: IContextKey; + private withinEditSessionKey: IContextKey; + private filePartOfEditSessionKey: IContextKey; private modelWidget: ModelPickerActionItem | undefined; private modeWidget: ModePickerActionItem | undefined; @@ -315,13 +321,15 @@ export class ChatInputPart extends Disposable implements IHistoryNavigationWidge const mode = this._currentModeObservable.get(); const modeId: 'ask' | 'agent' | 'edit' | 'custom' | undefined = mode.isBuiltin ? this.currentModeKind : 'custom'; + const modeInstructions = mode.modeInstructions?.get(); return { kind: this.currentModeKind, isBuiltin: mode.isBuiltin, - instructions: { - content: mode.body?.get(), - toolReferences: mode.variableReferences ? this.toolService.toToolReferences(mode.variableReferences.get()) : undefined - }, + modeInstructions: modeInstructions ? { + content: modeInstructions.content, + toolReferences: this.toolService.toToolReferences(modeInstructions.toolReferences), + metadata: modeInstructions.metadata, + } : undefined, modeId: modeId, applyCodeBlockSuggestionId: undefined, }; @@ -456,6 +464,9 @@ export class ChatInputPart extends Disposable implements IHistoryNavigationWidge this.inputEditorHasFocus = ChatContextKeys.inputHasFocus.bindTo(contextKeyService); this.promptFileAttached = ChatContextKeys.hasPromptFile.bindTo(contextKeyService); this.chatModeKindKey = ChatContextKeys.chatModeKind.bindTo(contextKeyService); + this.withinEditSessionKey = ChatContextKeys.withinEditSessionDiff.bindTo(contextKeyService); + this.filePartOfEditSessionKey = ChatContextKeys.filePartOfEditSession.bindTo(contextKeyService); + const chatToolCount = ChatContextKeys.chatToolCount.bindTo(contextKeyService); this._register(autorun(reader => { @@ -516,6 +527,11 @@ export class ChatInputPart extends Disposable implements IHistoryNavigationWidge })); } + public setIsWithinEditSession(inInsideDiff: boolean, isFilePartOfEditSession: boolean) { + this.withinEditSessionKey.set(inInsideDiff); + this.filePartOfEditSessionKey.set(isFilePartOfEditSession); + } + private getSelectedModelStorageKey(): string { return `chat.currentLanguageModel.${this.location}`; } @@ -1238,7 +1254,7 @@ export class ChatInputPart extends Disposable implements IHistoryNavigationWidge hiddenItemStrategy: HiddenItemStrategy.Ignore, // keep it lean when hiding items and avoid a "..." overflow menu actionViewItemProvider: (action, options) => { if (this.location === ChatAgentLocation.Chat || this.location === ChatAgentLocation.EditorInline) { - if ((action.id === ChatSubmitAction.ID || action.id === CancelAction.ID || action.id === ChatEditingSessionSubmitAction.ID) && action instanceof MenuItemAction) { + if ((action.id === ChatSubmitAction.ID || action.id === CancelAction.ID || action.id === ChatEditingSessionSubmitAction.ID || action.id === ChatDelegateToEditSessionAction.ID) && action instanceof MenuItemAction) { const dropdownAction = this.instantiationService.createInstance(MenuItemAction, { id: 'chat.moreExecuteActions', title: localize('notebook.moreExecuteActionsLabel', "More..."), icon: Codicon.chevronDown }, undefined, undefined, undefined, undefined); return this.instantiationService.createInstance(ChatSubmitDropdownActionItem, action, dropdownAction, { ...options, menuAsChild: false }); } diff --git a/src/vs/workbench/contrib/chat/browser/chatListRenderer.ts b/src/vs/workbench/contrib/chat/browser/chatListRenderer.ts index 99e438866b2..b53d2dcd35c 100644 --- a/src/vs/workbench/contrib/chat/browser/chatListRenderer.ts +++ b/src/vs/workbench/contrib/chat/browser/chatListRenderer.ts @@ -89,6 +89,8 @@ import { ChatMarkdownDecorationsRenderer } from './chatMarkdownDecorationsRender import { ChatMarkdownRenderer } from './chatMarkdownRenderer.js'; import { ChatEditorOptions } from './chatOptions.js'; import { ChatCodeBlockContentProvider, CodeBlockPart } from './codeBlockPart.js'; +import { ChatAnonymousRateLimitedPart } from './chatContentParts/chatAnonymousRateLimitedPart.js'; +import { IChatEntitlementService } from '../../../services/chat/common/chatEntitlementService.js'; const $ = dom.$; @@ -210,6 +212,7 @@ export class ChatListItemRenderer extends Disposable implements ITreeRenderer('chat.agent.thinkingStyle') === 'collapsedPreview') { this._currentThinkingPart.collapseContent(); } @@ -1277,6 +1280,9 @@ export class ChatListItemRenderer extends Disposable implements ITreeRenderer this.updateItemHeight(templateData))); return renderedError; + } else if (content.errorDetails.isRateLimited && this.chatEntitlementService.anonymous) { + const renderedError = this.instantiationService.createInstance(ChatAnonymousRateLimitedPart, content); + return renderedError; } else if (content.errorDetails.confirmationButtons && isLast) { const level = content.errorDetails.level ?? ChatErrorLevel.Error; const errorConfirmation = this.instantiationService.createInstance(ChatErrorConfirmationContentPart, level, new MarkdownString(content.errorDetails.message), content, content.errorDetails.confirmationButtons, this.renderer, context); @@ -1451,7 +1457,11 @@ export class ChatListItemRenderer extends Disposable implements ITreeRenderer | undefined, templateData: IChatListItemTemplate) { - return this.instantiationService.createInstance(ChatAttachmentsContentPart, variables, contentReferences, undefined); + return this.instantiationService.createInstance(ChatAttachmentsContentPart, { + variables, + contentReferences, + domNode: undefined + }); } private renderTextEdit(context: IChatContentPartRenderContext, chatTextEdit: IChatTextEditGroup, templateData: IChatListItemTemplate): IChatContentPart { @@ -1527,6 +1537,12 @@ export class ChatListItemRenderer extends Disposable implements ITreeRenderer { await this._extensionService.whenInstalledExtensionsRegistered(); const contribution = this._contributions.get(chatViewType); if (contribution && !this._isContributionAvailable(contribution)) { @@ -428,7 +428,7 @@ export class ChatSessionsService extends Disposable implements IChatSessionsServ public async provideChatSessionItems(chatSessionType: string, token: CancellationToken): Promise { if (!(await this.canResolveItemProvider(chatSessionType))) { - throw Error(`Can not find provider for ${chatSessionType}`); + return []; } const provider = this._itemsProviders.get(chatSessionType); diff --git a/src/vs/workbench/contrib/chat/browser/chatSetup.ts b/src/vs/workbench/contrib/chat/browser/chatSetup.ts index 2876cb62695..cd7a72193a3 100644 --- a/src/vs/workbench/contrib/chat/browser/chatSetup.ts +++ b/src/vs/workbench/contrib/chat/browser/chatSetup.ts @@ -759,7 +759,7 @@ class ChatSetup { createWorkbenchDialogOptions({ type: 'none', extraClasses: ['chat-setup-dialog'], - detail: ' ', // workaround allowing us to render the message in large + detail: this.getDialogDetail(), icon: Codicon.copilotLarge, alignment: DialogContentsAlignment.Vertical, cancelId: buttons.length - 1, @@ -815,6 +815,10 @@ class ChatSetup { } private getDialogTitle(options?: { forceSignInDialog?: boolean }): string { + if (this.chatEntitlementService.anonymous) { + return localize('enableMore', "Enable more AI features"); + } + if (this.context.state.entitlement === ChatEntitlement.Unknown || options?.forceSignInDialog) { return localize('signIn', "Sign in to use GitHub Copilot"); } @@ -822,6 +826,14 @@ class ChatSetup { return localize('startUsing', "Start using GitHub Copilot"); } + private getDialogDetail(): string { + if (this.chatEntitlementService.anonymous) { + return localize('enableMoreAnonymous', "Sign in to get access to more AI features like multiple chat models, AI code reviews and remote agents."); + } + + return ' '; // workaround allowing us to render the message in large + } + private createDialogFooter(disposables: DisposableStore): HTMLElement { const element = $('.chat-setup-dialog-footer'); diff --git a/src/vs/workbench/contrib/chat/browser/chatStatus.ts b/src/vs/workbench/contrib/chat/browser/chatStatus.ts index ffd7c78f2e4..2aa90df3e6a 100644 --- a/src/vs/workbench/contrib/chat/browser/chatStatus.ts +++ b/src/vs/workbench/contrib/chat/browser/chatStatus.ts @@ -44,6 +44,8 @@ import { Codicon } from '../../../../base/common/codicons.js'; import { URI } from '../../../../base/common/uri.js'; import { IInlineCompletionsService } from '../../../../editor/browser/services/inlineCompletionsService.js'; import { IChatSessionsService } from '../common/chatSessionsService.js'; +import { MarkdownRenderer } from '../../../../editor/browser/widget/markdownRenderer/browser/markdownRenderer.js'; +import { MarkdownString } from '../../../../base/common/htmlContent.js'; const gaugeForeground = registerColor('gauge.foreground', { dark: inputValidationInfoBorder, @@ -102,6 +104,9 @@ const defaultChat = { nextEditSuggestionsSetting: product.defaultChatAgent?.nextEditSuggestionsSetting ?? '', manageSettingsUrl: product.defaultChatAgent?.manageSettingsUrl ?? '', manageOverageUrl: product.defaultChatAgent?.manageOverageUrl ?? '', + provider: product.defaultChatAgent?.provider ?? { default: { id: '', name: '' }, enterprise: { id: '', name: '' }, apple: { id: '', name: '' }, google: { id: '', name: '' } }, + termsStatementUrl: product.defaultChatAgent?.termsStatementUrl ?? '', + privacyStatementUrl: product.defaultChatAgent?.privacyStatementUrl ?? '' }; export class ChatStatusBarEntry extends Disposable implements IWorkbenchContribution { @@ -179,10 +184,6 @@ export class ChatStatusBarEntry extends Disposable implements IWorkbenchContribu let ariaLabel = localize('chatStatus', "Copilot Status"); let kind: StatusbarEntryKind | undefined; - // Check if there are any chat sessions in progress - const inProgress = this.chatSessionsService.getInProgress(); - const hasInProgressSessions = inProgress.some(item => item.count > 0); - if (isNewUser(this.chatEntitlementService)) { const entitlement = this.chatEntitlementService.entitlement; @@ -202,6 +203,7 @@ export class ChatStatusBarEntry extends Disposable implements IWorkbenchContribu } else { const chatQuotaExceeded = this.chatEntitlementService.quotas.chat?.percentRemaining === 0; const completionsQuotaExceeded = this.chatEntitlementService.quotas.completions?.percentRemaining === 0; + const chatSessionsInProgressCount = this.chatSessionsService.getInProgress().reduce((total, item) => total + item.count, 0); // Disabled if (this.chatEntitlementService.sentiment.disabled || this.chatEntitlementService.sentiment.untrusted) { @@ -209,11 +211,21 @@ export class ChatStatusBarEntry extends Disposable implements IWorkbenchContribu ariaLabel = localize('copilotDisabledStatus', "Copilot Disabled"); } + // Sessions in progress + else if (chatSessionsInProgressCount > 0) { + text = `$(loading~spin)`; + if (chatSessionsInProgressCount > 1) { + ariaLabel = localize('chatSessionsInProgressStatus', "{0} chat sessions in progress", chatSessionsInProgressCount); + } else { + ariaLabel = localize('chatSessionInProgressStatus', "1 chat session in progress"); + } + } + // Signed out else if (this.chatEntitlementService.entitlement === ChatEntitlement.Unknown) { const signedOutWarning = localize('notSignedIntoCopilot', "Signed out"); - text = `$(copilot-not-connected) ${signedOutWarning}`; + text = `${this.chatEntitlementService.anonymous ? '$(copilot)' : '$(copilot-not-connected)'} ${signedOutWarning}`; ariaLabel = signedOutWarning; kind = 'prominent'; } @@ -247,14 +259,6 @@ export class ChatStatusBarEntry extends Disposable implements IWorkbenchContribu } } - // Show progress indicator when chat sessions are in progress - if (hasInProgressSessions) { - text = `$(loading~spin)\u00A0${text}`; - // Update aria label to include progress information - const sessionCount = inProgress.reduce((total, item) => total + item.count, 0); - ariaLabel = `${ariaLabel}, ${sessionCount} chat session${sessionCount === 1 ? '' : 's'} in progress`; - } - const baseResult = { name: localize('chatStatus', "Copilot Status"), text, @@ -345,7 +349,8 @@ class ChatStatusDashboard extends Disposable { @ITelemetryService private readonly telemetryService: ITelemetryService, @ITextResourceConfigurationService private readonly textResourceConfigurationService: ITextResourceConfigurationService, @IInlineCompletionsService private readonly inlineCompletionsService: IInlineCompletionsService, - @IChatSessionsService private readonly chatSessionsService: IChatSessionsService + @IChatSessionsService private readonly chatSessionsService: IChatSessionsService, + @IInstantiationService private readonly instantiationService: IInstantiationService, ) { super(); } @@ -414,12 +419,24 @@ class ChatStatusDashboard extends Disposable { })(); } + // Anonymous Indicator + else if (this.chatEntitlementService.anonymous && this.chatEntitlementService.sentiment.installed) { + addSeparator(localize('anonymousTitle', "Copilot Usage")); + + this.createQuotaIndicator(this.element, disposables, undefined, localize('completionsLabel', "Code completions"), false); + this.createQuotaIndicator(this.element, disposables, undefined, localize('chatsLabel', "Chat messages"), false); + + this.element.appendChild($('div.description', undefined, localize('anonymousFooter', "Sign in to increase allowance."))); + } + // Chat sessions { let chatSessionsElement: HTMLElement | undefined; + const updateStatus = () => { const inProgress = this.chatSessionsService.getInProgress(); if (inProgress.some(item => item.count > 0)) { + addSeparator(localize('chatSessionsTitle', "Chat Sessions"), toAction({ id: 'workbench.view.chat.status.sessions', label: localize('viewChatSessionsLabel', "View Chat Sessions"), @@ -430,22 +447,17 @@ class ChatStatusDashboard extends Disposable { for (const { displayName, count } of inProgress) { if (count > 0) { - let lowerCaseName = displayName.toLocaleLowerCase(); - // Very specific case for providers that end in session/sessions to ensure we pluralize correctly - if (lowerCaseName.endsWith('session') || lowerCaseName.endsWith('sessions')) { - lowerCaseName = lowerCaseName.replace(/session$|sessions$/g, count > 1 ? 'sessions' : 'session'); - } - const text = localize('inProgressChatSession', "$(loading~spin) {0} {1} in progress", count, lowerCaseName); + const text = localize('inProgressChatSession', "$(loading~spin) {0} in progress", displayName); chatSessionsElement = this.element.appendChild($('div.description')); const parts = renderLabelWithIcons(text); chatSessionsElement.append(...parts); } } - } - else { + } else { chatSessionsElement?.remove(); } }; + updateStatus(); disposables.add(this.chatSessionsService.onDidChangeInProgress(updateStatus)); } @@ -497,14 +509,21 @@ class ChatStatusDashboard extends Disposable { // New to Copilot / Signed out { const newUser = isNewUser(this.chatEntitlementService); + const anonymousUser = this.chatEntitlementService.anonymous; const disabled = this.chatEntitlementService.sentiment.disabled || this.chatEntitlementService.sentiment.untrusted; const signedOut = this.chatEntitlementService.entitlement === ChatEntitlement.Unknown; if (newUser || signedOut || disabled) { addSeparator(); - let descriptionText: string; - if (newUser) { + let descriptionText: string | MarkdownString; + let descriptionClass = '.description'; + if (newUser && anonymousUser) { + descriptionText = new MarkdownString(localize('activateDescriptionAnonymous', "By continuing with {0} Copilot, you agree to {1}'s [Terms]({2}) and [Privacy Statement]({3})", defaultChat.provider.default.name, defaultChat.provider.default.name, defaultChat.termsStatementUrl, defaultChat.privacyStatementUrl), { isTrusted: true }); + descriptionClass = `${descriptionClass}.terms`; + } else if (newUser) { descriptionText = localize('activateDescription', "Set up Copilot to use AI features."); + } else if (anonymousUser) { + descriptionText = localize('enableMoreDescription', "Sign in to enable more Copilot AI features."); } else if (disabled) { descriptionText = localize('enableDescription', "Enable Copilot to use AI features."); } else { @@ -514,17 +533,29 @@ class ChatStatusDashboard extends Disposable { let buttonLabel: string; if (newUser) { buttonLabel = localize('activateCopilotButton', "Set up Copilot"); + } else if (anonymousUser) { + buttonLabel = localize('enableMoreCopilotButton', "Enable more AI Features"); } else if (disabled) { buttonLabel = localize('enableCopilotButton', "Enable Copilot"); } else { buttonLabel = localize('signInToUseCopilotButton', "Sign in to use Copilot"); } - this.element.appendChild($('div.description', undefined, descriptionText)); + let setupArgs: { forceAnonymous: boolean } | undefined = undefined; + if (newUser && anonymousUser) { + setupArgs = { forceAnonymous: true }; + } + + if (typeof descriptionText === 'string') { + this.element.appendChild($(`div${descriptionClass}`, undefined, descriptionText)); + } else { + const markdown = this.instantiationService.createInstance(MarkdownRenderer, {}); + this.element.appendChild($(`div${descriptionClass}`, undefined, disposables.add(markdown.render(descriptionText)).element)); + } const button = disposables.add(new Button(this.element, { ...defaultButtonStyles, hoverDelegate: nativeHoverDelegate })); button.label = buttonLabel; - disposables.add(button.onDidClick(() => this.runCommandAndClose('workbench.action.chat.triggerSetup'))); + disposables.add(button.onDidClick(() => this.runCommandAndClose('workbench.action.chat.triggerSetup', undefined, setupArgs))); } } @@ -579,18 +610,18 @@ class ChatStatusDashboard extends Disposable { } } - private runCommandAndClose(commandOrFn: string | Function): void { + private runCommandAndClose(commandOrFn: string | Function, ...args: any[]): void { if (typeof commandOrFn === 'function') { - commandOrFn(); + commandOrFn(...args); } else { this.telemetryService.publicLog2('workbenchActionExecuted', { id: commandOrFn, from: 'chat-status' }); - this.commandService.executeCommand(commandOrFn); + this.commandService.executeCommand(commandOrFn, ...args); } this.hoverService.hideHover(true); } - private createQuotaIndicator(container: HTMLElement, disposables: DisposableStore, quota: IQuotaSnapshot, label: string, supportsOverage: boolean): (quota: IQuotaSnapshot) => void { + private createQuotaIndicator(container: HTMLElement, disposables: DisposableStore, quota: IQuotaSnapshot | undefined, label: string, supportsOverage: boolean): (quota: IQuotaSnapshot) => void { const quotaValue = $('span.quota-value'); const quotaBit = $('div.quota-bit'); const overageLabel = $('span.overage-label'); @@ -614,18 +645,20 @@ class ChatStatusDashboard extends Disposable { disposables.add(manageOverageButton.onDidClick(() => this.runCommandAndClose(() => this.openerService.open(URI.parse(defaultChat.manageOverageUrl))))); } - const update = (quota: IQuotaSnapshot) => { + const update = (quota: IQuotaSnapshot | undefined) => { quotaIndicator.classList.remove('error'); quotaIndicator.classList.remove('warning'); let usedPercentage: number; - if (quota.unlimited) { + if (!quota || quota.unlimited) { usedPercentage = 0; } else { usedPercentage = Math.max(0, 100 - quota.percentRemaining); } - if (quota.unlimited) { + if (!quota) { + quotaValue.textContent = localize('quotaLimited', "Limited"); + } else if (quota.unlimited) { quotaValue.textContent = localize('quotaUnlimited', "Included"); } else if (quota.overageCount) { quotaValue.textContent = localize('quotaDisplayWithOverage', "+{0} requests", this.quotaOverageFormatter.value.format(quota.overageCount)); @@ -642,7 +675,7 @@ class ChatStatusDashboard extends Disposable { } if (supportsOverage) { - if (quota.overageEnabled) { + if (quota?.overageEnabled) { overageLabel.textContent = localize('additionalUsageEnabled', "Additional paid premium requests enabled."); } else { overageLabel.textContent = localize('additionalUsageDisabled', "Additional paid premium requests disabled."); diff --git a/src/vs/workbench/contrib/chat/browser/chatWidget.ts b/src/vs/workbench/contrib/chat/browser/chatWidget.ts index 4f9ea5ec353..bb2fa61b5ed 100644 --- a/src/vs/workbench/contrib/chat/browser/chatWidget.ts +++ b/src/vs/workbench/contrib/chat/browser/chatWidget.ts @@ -6,17 +6,19 @@ import * as dom from '../../../../base/browser/dom.js'; import { IMouseWheelEvent } from '../../../../base/browser/mouseEvent.js'; import { Button } from '../../../../base/browser/ui/button/button.js'; +import { IListRenderer, IListVirtualDelegate } from '../../../../base/browser/ui/list/list.js'; import { ITreeContextMenuEvent, ITreeElement } from '../../../../base/browser/ui/tree/tree.js'; import { disposableTimeout, timeout } from '../../../../base/common/async.js'; import { CancellationToken } from '../../../../base/common/cancellation.js'; import { Codicon } from '../../../../base/common/codicons.js'; +import { fromNow, fromNowByDay } from '../../../../base/common/date.js'; import { toErrorMessage } from '../../../../base/common/errorMessage.js'; import { Emitter, Event } from '../../../../base/common/event.js'; import { FuzzyScore } from '../../../../base/common/filters.js'; import { IMarkdownString, MarkdownString } from '../../../../base/common/htmlContent.js'; import { Iterable } from '../../../../base/common/iterator.js'; -import { combinedDisposable, Disposable, DisposableStore, IDisposable, MutableDisposable, toDisposable } from '../../../../base/common/lifecycle.js'; import { KeyCode } from '../../../../base/common/keyCodes.js'; +import { combinedDisposable, Disposable, DisposableStore, IDisposable, MutableDisposable, toDisposable } from '../../../../base/common/lifecycle.js'; import { ResourceSet } from '../../../../base/common/map.js'; import { Schemas } from '../../../../base/common/network.js'; import { autorun, observableFromEvent, observableValue } from '../../../../base/common/observable.js'; @@ -26,24 +28,29 @@ import { isDefined } from '../../../../base/common/types.js'; import { URI } from '../../../../base/common/uri.js'; import { ICodeEditor } from '../../../../editor/browser/editorBrowser.js'; import { ICodeEditorService } from '../../../../editor/browser/services/codeEditorService.js'; +import { OffsetRange } from '../../../../editor/common/core/ranges/offsetRange.js'; import { localize } from '../../../../nls.js'; +import { MenuWorkbenchToolBar } from '../../../../platform/actions/browser/toolbar.js'; import { MenuId } from '../../../../platform/actions/common/actions.js'; -import { fromNowByDay, fromNow } from '../../../../base/common/date.js'; import { IConfigurationService } from '../../../../platform/configuration/common/configuration.js'; import { ContextKeyExpr, IContextKey, IContextKeyService } from '../../../../platform/contextkey/common/contextkey.js'; import { IContextMenuService } from '../../../../platform/contextview/browser/contextView.js'; import { ITextResourceEditorInput } from '../../../../platform/editor/common/editor.js'; import { IInstantiationService } from '../../../../platform/instantiation/common/instantiation.js'; import { ServiceCollection } from '../../../../platform/instantiation/common/serviceCollection.js'; -import { WorkbenchObjectTree, WorkbenchList } from '../../../../platform/list/browser/listService.js'; -import { IListRenderer, IListVirtualDelegate } from '../../../../base/browser/ui/list/list.js'; +import { WorkbenchList, WorkbenchObjectTree } from '../../../../platform/list/browser/listService.js'; import { ILogService } from '../../../../platform/log/common/log.js'; import { bindContextKey } from '../../../../platform/observable/common/platformObservableUtils.js'; +import product from '../../../../platform/product/common/product.js'; import { ITelemetryService } from '../../../../platform/telemetry/common/telemetry.js'; import { buttonSecondaryBackground, buttonSecondaryForeground, buttonSecondaryHoverBackground } from '../../../../platform/theme/common/colorRegistry.js'; import { asCssVariable } from '../../../../platform/theme/common/colorUtils.js'; import { IThemeService } from '../../../../platform/theme/common/themeService.js'; import { IWorkspaceContextService, WorkbenchState } from '../../../../platform/workspace/common/workspace.js'; +import { EditorResourceAccessor } from '../../../../workbench/common/editor.js'; +import { IEditorService } from '../../../../workbench/services/editor/common/editorService.js'; +import { IChatEntitlementService } from '../../../services/chat/common/chatEntitlementService.js'; +import { IViewsService } from '../../../services/views/common/viewsService.js'; import { checkModeOption } from '../common/chat.js'; import { IChatAgentCommand, IChatAgentData, IChatAgentService } from '../common/chatAgents.js'; import { ChatContextKeys } from '../common/chatContextKeys.js'; @@ -65,26 +72,21 @@ import { ILanguageModelToolsService, IToolData, ToolSet } from '../common/langua import { ComputeAutomaticInstructions } from '../common/promptSyntax/computeAutomaticInstructions.js'; import { PromptsConfig } from '../common/promptSyntax/config/config.js'; import { PromptsType } from '../common/promptSyntax/promptTypes.js'; +import { ParsedPromptFile, PromptHeader } from '../common/promptSyntax/service/newPromptsParser.js'; import { IPromptsService } from '../common/promptSyntax/service/promptsService.js'; import { handleModeSwitch } from './actions/chatActions.js'; -import { ChatTreeItem, IChatAcceptInputOptions, IChatAccessibilityService, IChatCodeBlockInfo, IChatFileTreeInfo, IChatListItemRendererOptions, IChatWidget, IChatWidgetService, IChatWidgetViewContext, IChatWidgetViewOptions, ChatViewId } from './chat.js'; +import { ChatTreeItem, ChatViewId, IChatAcceptInputOptions, IChatAccessibilityService, IChatCodeBlockInfo, IChatFileTreeInfo, IChatListItemRendererOptions, IChatWidget, IChatWidgetService, IChatWidgetViewContext, IChatWidgetViewOptions } from './chat.js'; import { ChatAccessibilityProvider } from './chatAccessibilityProvider.js'; import { ChatAttachmentModel } from './chatAttachmentModel.js'; import { ChatTodoListWidget } from './chatContentParts/chatTodoListWidget.js'; import { ChatInputPart, IChatInputStyles } from './chatInputPart.js'; import { ChatListDelegate, ChatListItemRenderer, IChatListItemTemplate, IChatRendererDelegate } from './chatListRenderer.js'; import { ChatEditorOptions } from './chatOptions.js'; +import { ChatViewPane } from './chatViewPane.js'; import './media/chat.css'; import './media/chatAgentHover.css'; import './media/chatViewWelcome.css'; import { ChatViewWelcomePart, IChatSuggestedPrompts, IChatViewWelcomeContent } from './viewsWelcome/chatViewWelcomeController.js'; -import { ChatViewPane } from './chatViewPane.js'; -import { IViewsService } from '../../../services/views/common/viewsService.js'; -import product from '../../../../platform/product/common/product.js'; -import { IChatEntitlementService } from '../../../services/chat/common/chatEntitlementService.js'; -import { MenuWorkbenchToolBar } from '../../../../platform/actions/browser/toolbar.js'; -import { ParsedPromptFile, PromptHeader } from '../common/promptSyntax/service/newPromptsParser.js'; -import { OffsetRange } from '../../../../editor/common/core/ranges/offsetRange.js'; const $ = dom.$; @@ -260,6 +262,7 @@ export class ChatWidget extends Disposable implements IChatWidget { private readonly inputPartDisposable: MutableDisposable = this._register(new MutableDisposable()); private readonly inlineInputPartDisposable: MutableDisposable = this._register(new MutableDisposable()); + private readonly timeoutDisposable: MutableDisposable = this._register(new MutableDisposable()); private inputContainer!: HTMLElement; private focusedInputDOM!: HTMLElement; private editorOptions!: ChatEditorOptions; @@ -391,12 +394,17 @@ export class ChatWidget extends Disposable implements IChatWidget { return localize('chatDisclaimer', "AI responses may be inaccurate."); } + get locationData() { + return this._location.resolveData?.(); + } + constructor( location: ChatAgentLocation | IChatWidgetLocationOptions, _viewContext: IChatWidgetViewContext | undefined, private readonly viewOptions: IChatWidgetViewOptions, private readonly styles: IChatWidgetStyles, @ICodeEditorService private readonly codeEditorService: ICodeEditorService, + @IEditorService private readonly editorService: IEditorService, @IConfigurationService private readonly configurationService: IConfigurationService, @IContextKeyService private readonly contextKeyService: IContextKeyService, @IInstantiationService private readonly instantiationService: IInstantiationService, @@ -936,11 +944,9 @@ export class ChatWidget extends Disposable implements IChatWidget { if (this.contextKeyService.contextMatchesRules(this.chatSetupTriggerContext)) { welcomeContent = this.getNewWelcomeViewContent(); this.container.classList.add('new-welcome-view'); - } - else if (expEmptyState) { + } else if (expEmptyState) { welcomeContent = this.getWelcomeViewContent(additionalMessage, expEmptyState); - } - else { + } else { const tips = this.input.currentModeKind === ChatModeKind.Ask ? new MarkdownString(localize('chatWidget.tips', "{0} or type {1} to attach context\n\n{2} to chat with extensions\n\nType {3} to use commands", '$(attach)', '#', '$(mention)', '/'), { supportThemeIcons: true }) : new MarkdownString(localize('chatWidget.tips.withoutParticipants', "{0} or type {1} to attach context", '$(attach)', '#'), { supportThemeIcons: true }); @@ -1242,8 +1248,12 @@ export class ChatWidget extends Disposable implements IChatWidget { } private getPromptFileSuggestions(): IChatSuggestedPrompts[] { + // Get the current workspace folder context if available + const activeEditor = this.editorService.activeEditor; + const resource = activeEditor ? EditorResourceAccessor.getOriginalUri(activeEditor) : undefined; + // Get the prompt file suggestions configuration - const suggestions = PromptsConfig.getPromptFilesRecommendationsValue(this.configurationService); + const suggestions = PromptsConfig.getPromptFilesRecommendationsValue(this.configurationService, resource); if (!suggestions) { return []; } @@ -1400,13 +1410,13 @@ export class ChatWidget extends Disposable implements IChatWidget { this.input.setVisible(visible); if (visible) { - this._register(disposableTimeout(() => { + this.timeoutDisposable.value = disposableTimeout(() => { // Progressive rendering paused while hidden, so start it up again. // Do it after a timeout because the container is not visible yet (it should be but offsetHeight returns 0 here) if (this._visible) { this.onDidChangeItems(true); } - }, 0)); + }, 0); if (!wasVisible) { dom.scheduleAtNextAnimationFrame(dom.getWindow(this.listContainer), () => { @@ -1796,7 +1806,7 @@ export class ChatWidget extends Disposable implements IChatWidget { renderWorkingSet: this.viewOptions.enableWorkingSet === 'explicit', supportsChangingModes: this.viewOptions.supportsChangingModes, dndContainer: this.viewOptions.dndContainer, - widgetViewKindTag: this.getWidgetViewKindTag() + widgetViewKindTag: this.getWidgetViewKindTag(), }; if (this.viewModel?.editing) { @@ -2217,7 +2227,7 @@ export class ChatWidget extends Disposable implements IChatWidget { const requestId = this.chatAccessibilityService.acceptRequest(); const requestInputs: IChatRequestInputOptions = { input: !query ? editorValue : query.query, - attachedContext: this.input.getAttachedAndImplicitContext(this.viewModel.sessionId), + attachedContext: options?.enableImplicitContext === false ? this.input.getAttachedContext(this.viewModel.sessionId) : this.input.getAttachedAndImplicitContext(this.viewModel.sessionId), }; const isUserQuery = !query; diff --git a/src/vs/workbench/contrib/chat/browser/media/chat.css b/src/vs/workbench/contrib/chat/browser/media/chat.css index 67bcf14c5a8..2f9b001b99e 100644 --- a/src/vs/workbench/contrib/chat/browser/media/chat.css +++ b/src/vs/workbench/contrib/chat/browser/media/chat.css @@ -1335,7 +1335,8 @@ have to be updated for changes to the rules above, or to support more deeply nes } .chat-notification-widget .chat-warning-codicon .codicon-warning, -.chat-quota-error-widget .codicon-warning { +.chat-quota-error-widget .codicon-warning, +.chat-rate-limited-error-widget .codicon-warning { color: var(--vscode-notificationsWarningIcon-foreground) !important; /* Have to override default styles which apply to all lists */ } @@ -1834,7 +1835,8 @@ have to be updated for changes to the rules above, or to support more deeply nes .chat-notification-widget, .chat-summary-list, .chat-used-context-list, -.chat-quota-error-widget { +.chat-quota-error-widget, +.chat-rate-limited-error-widget { border: 1px solid var(--vscode-chat-requestBorder); border-radius: 4px; margin-bottom: 8px; @@ -2138,7 +2140,8 @@ have to be updated for changes to the rules above, or to support more deeply nes display: initial; } -.chat-quota-error-widget { +.chat-quota-error-widget, +.chat-rate-limited-error-widget { padding: 8px 12px; display: flex; gap: 6px; @@ -2148,7 +2151,8 @@ have to be updated for changes to the rules above, or to support more deeply nes padding: 2px 11px; } - .chat-quota-error-button { + .chat-quota-error-button, + .chat-rate-limited-error-button { margin-top: 6px; margin-bottom: 2px; } @@ -2168,7 +2172,8 @@ have to be updated for changes to the rules above, or to support more deeply nes margin-top: 2px; } - .chat-quota-error-message { + .chat-quota-error-message, + .chat-rate-limited-error-message { .rendered-markdown p { margin: 0px; } @@ -2885,6 +2890,8 @@ have to be updated for changes to the rules above, or to support more deeply nes display: flex; align-items: center; width: 100%; + border: none; + outline: none; &:hover { background: var(--vscode-toolbar-hoverBackground); @@ -3008,3 +3015,24 @@ have to be updated for changes to the rules above, or to support more deeply nes .editor-instance .chat-todo-list-widget { background-color: var(--vscode-editor-background); } + +/* Show more attachments button styling */ +.chat-attachments-show-more-button { + opacity: 0.8; + transition: opacity 0.2s ease; +} + +.chat-attachments-show-more-button:hover { + opacity: 1; + background-color: var(--vscode-list-hoverBackground) !important; +} + +.chat-attachments-show-more-button:focus { + outline: 1px solid var(--vscode-focusBorder); + outline-offset: -1px; +} + +.chat-attachments-show-more-button .chat-attached-context-custom-text { + font-style: italic; + color: var(--vscode-descriptionForeground); +} diff --git a/src/vs/workbench/contrib/chat/browser/media/chatStatus.css b/src/vs/workbench/contrib/chat/browser/media/chatStatus.css index 05d5035fa47..14fdd0522b3 100644 --- a/src/vs/workbench/contrib/chat/browser/media/chatStatus.css +++ b/src/vs/workbench/contrib/chat/browser/media/chatStatus.css @@ -30,6 +30,13 @@ .chat-status-bar-entry-tooltip div.description { font-size: 11px; color: var(--vscode-descriptionForeground); + display: flex; + align-items: center; + gap: 3px; +} + +.chat-status-bar-entry-tooltip div.description.terms { + max-width: 250px; } .chat-status-bar-entry-tooltip .monaco-button { diff --git a/src/vs/workbench/contrib/chat/common/chatAgents.ts b/src/vs/workbench/contrib/chat/common/chatAgents.ts index 45a8b1d2109..ad2eb8b8d4e 100644 --- a/src/vs/workbench/contrib/chat/common/chatAgents.ts +++ b/src/vs/workbench/contrib/chat/common/chatAgents.ts @@ -10,7 +10,7 @@ import { Emitter, Event } from '../../../../base/common/event.js'; import { IMarkdownString } from '../../../../base/common/htmlContent.js'; import { Iterable } from '../../../../base/common/iterator.js'; import { Disposable, IDisposable, toDisposable } from '../../../../base/common/lifecycle.js'; -import { revive } from '../../../../base/common/marshalling.js'; +import { revive, Revived } from '../../../../base/common/marshalling.js'; import { IObservable, observableValue } from '../../../../base/common/observable.js'; import { equalsIgnoreCase } from '../../../../base/common/strings.js'; import { ThemeIcon } from '../../../../base/common/themables.js'; @@ -141,7 +141,7 @@ export interface IChatAgentRequest { isParticipantDetected?: boolean; variables: IChatRequestVariableData; location: ChatAgentLocation; - locationData?: IChatLocationData; + locationData?: Revived; acceptedConfirmationData?: any[]; rejectedConfirmationData?: any[]; userSelectedModelId?: string; diff --git a/src/vs/workbench/contrib/chat/common/chatContextKeys.ts b/src/vs/workbench/contrib/chat/common/chatContextKeys.ts index a790d51d1bb..405421ca596 100644 --- a/src/vs/workbench/contrib/chat/common/chatContextKeys.ts +++ b/src/vs/workbench/contrib/chat/common/chatContextKeys.ts @@ -45,6 +45,8 @@ export namespace ChatContextKeys { * True when the chat widget is locked to the coding agent session. */ export const lockedToCodingAgent = new RawContextKey('lockedToCodingAgent', false, { type: 'boolean', description: localize('lockedToCodingAgent', "True when the chat widget is locked to the coding agent session.") }); + export const withinEditSessionDiff = new RawContextKey('withinEditSessionDiff', false, { type: 'boolean', description: localize('withinEditSessionDiff', "True when the chat widget dispatches to the edit session chat.") }); + export const filePartOfEditSession = new RawContextKey('filePartOfEditSession', false, { type: 'boolean', description: localize('filePartOfEditSession', "True when the chat widget is within a file with an edit session.") }); export const extensionParticipantRegistered = new RawContextKey('chatPanelExtensionParticipantRegistered', false, { type: 'boolean', description: localize('chatPanelExtensionParticipantRegistered', "True when a default chat participant is registered for the panel from an extension.") }); export const panelParticipantRegistered = new RawContextKey('chatPanelParticipantRegistered', false, { type: 'boolean', description: localize('chatParticipantRegistered', "True when a default chat participant is registered for the panel.") }); diff --git a/src/vs/workbench/contrib/chat/common/chatEditingService.ts b/src/vs/workbench/contrib/chat/common/chatEditingService.ts index 707f8f6a8c0..0366481090c 100644 --- a/src/vs/workbench/contrib/chat/common/chatEditingService.ts +++ b/src/vs/workbench/contrib/chat/common/chatEditingService.ts @@ -8,7 +8,7 @@ import { Event } from '../../../../base/common/event.js'; import { IDisposable } from '../../../../base/common/lifecycle.js'; import { IObservable, IReader } from '../../../../base/common/observable.js'; import { URI } from '../../../../base/common/uri.js'; -import { TextEdit } from '../../../../editor/common/languages.js'; +import { Location, TextEdit } from '../../../../editor/common/languages.js'; import { ITextModel } from '../../../../editor/common/model.js'; import { localize } from '../../../../nls.js'; import { RawContextKey } from '../../../../platform/contextkey/common/contextkey.js'; @@ -272,6 +272,7 @@ export interface IModifiedFileEntry { readonly linesRemoved?: IObservable; getEditorIntegration(editor: IEditorPane): IModifiedFileEntryEditorIntegration; + hasModificationAt(location: Location): boolean; } export interface IChatEditingSessionStream { diff --git a/src/vs/workbench/contrib/chat/common/chatModel.ts b/src/vs/workbench/contrib/chat/common/chatModel.ts index d86965601cb..94ec229bf63 100644 --- a/src/vs/workbench/contrib/chat/common/chatModel.ts +++ b/src/vs/workbench/contrib/chat/common/chatModel.ts @@ -215,14 +215,15 @@ const defaultChatResponseModelChangeReason: ChatResponseModelChangeReason = { re export interface IChatRequestModeInfo { kind: ChatModeKind | undefined; // is undefined in case of modeId == 'apply' isBuiltin: boolean; - instructions: IChatRequestModeInstructions | undefined; + modeInstructions: IChatRequestModeInstructions | undefined; modeId: 'ask' | 'agent' | 'edit' | 'custom' | 'applyCodeBlock' | undefined; applyCodeBlockSuggestionId: EditSuggestionId | undefined; } export interface IChatRequestModeInstructions { - readonly content: string | undefined; - readonly toolReferences: readonly ChatRequestToolReferenceEntry[] | undefined; + readonly content: string; + readonly toolReferences: readonly ChatRequestToolReferenceEntry[]; + readonly metadata?: Record; } export interface IChatRequestModelParameters { diff --git a/src/vs/workbench/contrib/chat/common/chatModes.ts b/src/vs/workbench/contrib/chat/common/chatModes.ts index 40b84d443a1..fba7f476521 100644 --- a/src/vs/workbench/contrib/chat/common/chatModes.ts +++ b/src/vs/workbench/contrib/chat/common/chatModes.ts @@ -98,8 +98,7 @@ export class ChatModeService extends Disposable implements IChatModeService { description: cachedMode.description, tools: cachedMode.customTools, model: cachedMode.model, - body: cachedMode.body || '', - variableReferences: cachedMode.variableReferences || [], + modeInstructions: cachedMode.modeInstructions ?? { content: cachedMode.body ?? '', toolReferences: [] }, }; const instance = new CustomChatMode(customChatMode); this._customModeInstances.set(uri.toString(), instance); @@ -200,8 +199,8 @@ export interface IChatModeData { readonly kind: ChatModeKind; readonly customTools?: readonly string[]; readonly model?: string; - readonly body?: string; - readonly variableReferences?: readonly IVariableReference[]; + readonly modeInstructions?: IChatModeInstructions; + readonly body?: string; /* deprecated */ readonly uri?: URI; } @@ -214,8 +213,7 @@ export interface IChatMode { readonly kind: ChatModeKind; readonly customTools?: IObservable; readonly model?: IObservable; - readonly body?: IObservable; - readonly variableReferences?: IObservable; + readonly modeInstructions?: IObservable; readonly uri?: IObservable; } @@ -224,6 +222,12 @@ export interface IVariableReference { readonly range: IOffsetRange; } +export interface IChatModeInstructions { + readonly content: string; + readonly toolReferences: readonly IVariableReference[]; + readonly metadata?: Record; +} + function isCachedChatModeData(data: unknown): data is IChatModeData { if (typeof data !== 'object' || data === null) { return false; @@ -235,8 +239,7 @@ function isCachedChatModeData(data: unknown): data is IChatModeData { typeof mode.kind === 'string' && (mode.description === undefined || typeof mode.description === 'string') && (mode.customTools === undefined || Array.isArray(mode.customTools)) && - (mode.body === undefined || typeof mode.body === 'string') && - (mode.variableReferences === undefined || Array.isArray(mode.variableReferences)) && + (mode.modeInstructions === undefined || (typeof mode.modeInstructions === 'object' && mode.modeInstructions !== null)) && (mode.model === undefined || typeof mode.model === 'string') && (mode.uri === undefined || (typeof mode.uri === 'object' && mode.uri !== null)); } @@ -244,8 +247,7 @@ function isCachedChatModeData(data: unknown): data is IChatModeData { export class CustomChatMode implements IChatMode { private readonly _descriptionObservable: ISettableObservable; private readonly _customToolsObservable: ISettableObservable; - private readonly _bodyObservable: ISettableObservable; - private readonly _variableReferencesObservable: ISettableObservable; + private readonly _modeInstructions: ISettableObservable; private readonly _uriObservable: ISettableObservable; private readonly _modelObservable: ISettableObservable; @@ -268,12 +270,8 @@ export class CustomChatMode implements IChatMode { return this._modelObservable; } - get body(): IObservable { - return this._bodyObservable; - } - - get variableReferences(): IObservable { - return this._variableReferencesObservable; + get modeInstructions(): IObservable { + return this._modeInstructions; } get uri(): IObservable { @@ -294,8 +292,7 @@ export class CustomChatMode implements IChatMode { this._descriptionObservable = observableValue('description', customChatMode.description); this._customToolsObservable = observableValue('customTools', customChatMode.tools); this._modelObservable = observableValue('model', customChatMode.model); - this._bodyObservable = observableValue('body', customChatMode.body); - this._variableReferencesObservable = observableValue('variableReferences', customChatMode.variableReferences); + this._modeInstructions = observableValue('_modeInstructions', customChatMode.modeInstructions); this._uriObservable = observableValue('uri', customChatMode.uri); } @@ -308,8 +305,7 @@ export class CustomChatMode implements IChatMode { this._descriptionObservable.set(newData.description, tx); this._customToolsObservable.set(newData.tools, tx); this._modelObservable.set(newData.model, tx); - this._bodyObservable.set(newData.body, tx); - this._variableReferencesObservable.set(newData.variableReferences, tx); + this._modeInstructions.set(newData.modeInstructions, tx); this._uriObservable.set(newData.uri, tx); }); } @@ -322,8 +318,7 @@ export class CustomChatMode implements IChatMode { kind: this.kind, customTools: this.customTools.get(), model: this.model.get(), - body: this.body.get(), - variableReferences: this.variableReferences.get(), + modeInstructions: this.modeInstructions.get(), uri: this.uri.get() }; } diff --git a/src/vs/workbench/contrib/chat/common/chatService.ts b/src/vs/workbench/contrib/chat/common/chatService.ts index 06875920f15..d1f86c2d4f4 100644 --- a/src/vs/workbench/contrib/chat/common/chatService.ts +++ b/src/vs/workbench/contrib/chat/common/chatService.ts @@ -19,6 +19,7 @@ import { createDecorator } from '../../../../platform/instantiation/common/insta import { ICellEditOperation } from '../../notebook/common/notebookCommon.js'; import { IWorkspaceSymbol } from '../../search/common/search.js'; import { IChatAgentCommand, IChatAgentData, IChatAgentResult, UserSelectedTools } from './chatAgents.js'; +import { IChatEditingSession } from './chatEditingService.js'; import { ChatModel, IChatModel, IChatRequestModeInfo, IChatRequestModel, IChatRequestVariableData, IChatResponseModel, IExportableChatData, ISerializableChatData } from './chatModel.js'; import { IParsedChatRequest } from './chatParserTypes.js'; import { IChatParserContext } from './chatRequestParser.js'; @@ -50,6 +51,7 @@ export interface IChatResponseErrorDetails { responseIsFiltered?: boolean; responseIsRedacted?: boolean; isQuotaExceeded?: boolean; + isRateLimited?: boolean; level?: ChatErrorLevel; confirmationButtons?: IChatResponseErrorDetailsConfirmationButton[]; code?: string; @@ -639,6 +641,8 @@ export interface IChatEditorLocationData { document: URI; selection: ISelection; wholeRange: IRange; + close: () => void; + delegateSessionId: string | undefined; } export interface IChatNotebookLocationData { @@ -695,6 +699,7 @@ export interface IChatService { isPersistedSessionEmpty(sessionId: string): boolean; loadSessionFromContent(data: IExportableChatData | ISerializableChatData | URI): IChatModel | undefined; loadSessionForResource(resource: URI, location: ChatAgentLocation, token: CancellationToken): Promise; + readonly editingSessions: IChatEditingSession[]; /** * Returns whether the request was accepted. diff --git a/src/vs/workbench/contrib/chat/common/chatServiceImpl.ts b/src/vs/workbench/contrib/chat/common/chatServiceImpl.ts index 994b6822f9c..f12b9eef578 100644 --- a/src/vs/workbench/contrib/chat/common/chatServiceImpl.ts +++ b/src/vs/workbench/contrib/chat/common/chatServiceImpl.ts @@ -14,6 +14,7 @@ import { Disposable, DisposableMap, DisposableStore, IDisposable, MutableDisposa import { revive } from '../../../../base/common/marshalling.js'; import { autorun, derived, IObservable, ObservableMap } from '../../../../base/common/observable.js'; import { StopWatch } from '../../../../base/common/stopwatch.js'; +import { isDefined } from '../../../../base/common/types.js'; import { URI } from '../../../../base/common/uri.js'; import { OffsetRange } from '../../../../editor/common/core/ranges/offsetRange.js'; import { localize } from '../../../../nls.js'; @@ -162,6 +163,10 @@ export class ChatService extends Disposable implements IChatService { }); } + public get editingSessions() { + return [...this._sessionModels.values()].map(v => v.editingSession).filter(isDefined); + } + isEnabled(location: ChatAgentLocation): boolean { return this.chatAgentService.getContributedDefaultAgent(location) !== undefined; } @@ -800,7 +805,7 @@ export class ChatService extends Disposable implements IChatService { rejectedConfirmationData: options?.rejectedConfirmationData, userSelectedModelId: options?.userSelectedModelId, userSelectedTools: options?.userSelectedTools?.get(), - modeInstructions: options?.modeInfo?.instructions, + modeInstructions: options?.modeInfo?.modeInstructions, editedFileEvents: request.editedFileEvents } satisfies IChatAgentRequest; }; diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/asyncDecoder.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/asyncDecoder.ts deleted file mode 100644 index be10b50ebba..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/asyncDecoder.ts +++ /dev/null @@ -1,93 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { Disposable } from '../../../../../../../base/common/lifecycle.js'; -import { BaseDecoder } from './baseDecoder.js'; - -/** - * Asynchronous iterator wrapper for a decoder. - */ -export class AsyncDecoder, K extends NonNullable = NonNullable> extends Disposable { - // Buffer of messages that have been decoded but not yet consumed. - private readonly messages: T[] = []; - - /** - * A transient promise that is resolved when a new event - * is received. Used in the situation when there is no new - * data available and decoder stream did not finish yet, - * hence we need to wait until new event is received. - */ - private resolveOnNewEvent?: (value: void) => void; - - /** - * @param decoder The decoder instance to wrap. - * - * Note! Assumes ownership of the `decoder` object, hence will `dispose` - * it when the decoder stream is ended. - */ - constructor( - private readonly decoder: BaseDecoder, - ) { - super(); - - this._register(decoder); - } - - /** - * Async iterator implementation. - */ - async *[Symbol.asyncIterator](): AsyncIterator { - // callback is called when `data` or `end` event is received - const callback = (data?: T) => { - if (data !== undefined) { - this.messages.push(data); - } else { - this.decoder.removeListener('data', callback); - this.decoder.removeListener('end', callback); - } - - // is the promise resolve callback is present, - // then call it and remove the reference - if (this.resolveOnNewEvent) { - this.resolveOnNewEvent(); - delete this.resolveOnNewEvent; - } - }; - - /** - * !NOTE! The order of event subscriptions below is critical here because - * the `data` event is also starts the stream, hence changing - * the order of event subscriptions can lead to race conditions. - * See {@link ReadableStreamEvents} for more info. - */ - - this.decoder.on('end', callback); - this.decoder.on('data', callback); - - // start flowing the decoder stream - this.decoder.start(); - - while (true) { - const maybeMessage = this.messages.shift(); - if (maybeMessage !== undefined) { - yield maybeMessage; - continue; - } - - // if no data available and stream ended, we're done - if (this.decoder.ended) { - this.dispose(); - - return null; - } - - // stream isn't ended so wait for the new - // `data` or `end` event to be received - await new Promise((resolve) => { - this.resolveOnNewEvent = resolve; - }); - } - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/baseDecoder.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/baseDecoder.ts deleted file mode 100644 index 76d1745d3c6..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/baseDecoder.ts +++ /dev/null @@ -1,361 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { Emitter } from '../../../../../../../base/common/event.js'; -import { ReadableStream } from '../../../../../../../base/common/stream.js'; -import { DeferredPromise } from '../../../../../../../base/common/async.js'; -import { AsyncDecoder } from './asyncDecoder.js'; -import { assert, assertNever } from '../../../../../../../base/common/assert.js'; -import { DisposableMap, IDisposable } from '../../../../../../../base/common/lifecycle.js'; -import { ObservableDisposable } from '../../utils/observableDisposable.js'; - -/** - * Event names of {@link ReadableStream} stream. - */ -export type TStreamListenerNames = 'data' | 'error' | 'end'; - -/** - * Base decoder class that can be used to convert stream messages data type - * from one type to another. For instance, a stream of binary data can be - * "decoded" into a stream of well defined objects. - * Intended to be a part of "codec" implementation rather than used directly. - */ -export abstract class BaseDecoder< - T extends NonNullable, - K extends NonNullable = NonNullable, -> extends ObservableDisposable implements ReadableStream { - /** - * Private attribute to track if the stream has ended. - */ - private _ended = false; - - protected readonly _onData = this._register(new Emitter()); - private readonly _onEnd = this._register(new Emitter()); - private readonly _onError = this._register(new Emitter()); - - /** - * A store of currently registered event listeners. - */ - private readonly _listeners: DisposableMap< - TStreamListenerNames, - DisposableMap - > = this._register(new DisposableMap()); - - /** - * This method is called when a new incoming data - * is received from the input stream. - */ - protected abstract onStreamData(data: K): void; - - /** - * @param stream The input stream to decode. - */ - constructor( - protected readonly stream: ReadableStream, - ) { - super(); - } - - /** - * Private attribute to track if the stream has started. - */ - private started = false; - - /** - * Promise that resolves when the stream has ended, either by - * receiving the `end` event or by a disposal, but not when - * the `error` event is received alone. - * The promise is true if the stream has ended, and false - * if the stream has been disposed without ending. - */ - private settledPromise = new DeferredPromise(); - - /** - * Promise that resolves when the stream has ended, either by - * receiving the `end` event or by a disposal, but not when - * the `error` event is received alone. - * The promise is true if the stream has ended, and false - * if the stream has been disposed without ending. - * - * @throws If the stream was not yet started to prevent this - * promise to block the consumer calls indefinitely. - */ - public get settled(): Promise { - // if the stream has not started yet, the promise might - // block the consumer calls indefinitely if they forget - // to call the `start()` method, or if the call happens - // after await on the `settled` promise; to forbid this - // confusion, we require the stream to be started first - assert( - this.started, - [ - 'Cannot get `settled` promise of a stream that has not been started.', - 'Please call `start()` first.', - ].join(' '), - ); - - return this.settledPromise.p; - } - - /** - * Start receiving data from the stream. - * @throws if the decoder stream has already ended. - */ - public start(): this { - assert( - this._ended === false, - 'Cannot start stream that has already ended.', - ); - assert( - this.isDisposed === false, - 'Cannot start stream that has already disposed.', - ); - - // if already started, nothing to do - if (this.started) { - return this; - } - this.started = true; - - /** - * !NOTE! The order of event subscriptions is critical here because - * the `data` event is also starts the stream, hence changing - * the order of event subscriptions can lead to race conditions. - * See {@link ReadableStreamEvents} for more info. - */ - this.stream.on('end', this.onStreamEnd.bind(this)); - this.stream.on('error', this.onStreamError.bind(this)); - this.stream.on('data', this.tryOnStreamData.bind(this)); - - // this allows to compose decoders together, - if a decoder - // instance is passed as a readable stream to this decoder, - // then we need to call `start` on it too - if (this.stream instanceof BaseDecoder) { - this.stream.start(); - } - - return this; - } - - /** - * Check if the decoder has been ended hence has - * no more data to produce. - */ - public get ended(): boolean { - return this._ended; - } - - /** - * Automatically catch and dispatch errors thrown inside `onStreamData`. - */ - private tryOnStreamData(data: K): void { - try { - this.onStreamData(data); - } catch (error) { - this.onStreamError(error); - } - } - - public on(event: 'data', callback: (data: T) => void): void; - public on(event: 'error', callback: (err: Error) => void): void; - public on(event: 'end', callback: () => void): void; - public on(event: TStreamListenerNames, callback: unknown): void { - if (event === 'data') { - return this.onData(callback as (data: T) => void); - } - - if (event === 'error') { - return this.onError(callback as (error: Error) => void); - } - - if (event === 'end') { - return this.onEnd(callback as () => void); - } - - assertNever(event, `Invalid event name '${event}'`); - } - - /** - * Add listener for the `data` event. - * @throws if the decoder stream has already ended. - */ - public onData(callback: (data: T) => void): void { - assert( - !this.ended, - 'Cannot subscribe to the `data` event because the decoder stream has already ended.', - ); - - let currentListeners = this._listeners.get('data'); - - if (!currentListeners) { - currentListeners = new DisposableMap(); - this._listeners.set('data', currentListeners); - } - - currentListeners.set(callback, this._onData.event(callback)); - } - - /** - * Add listener for the `error` event. - * @throws if the decoder stream has already ended. - */ - public onError(callback: (error: Error) => void): void { - assert( - !this.ended, - 'Cannot subscribe to the `error` event because the decoder stream has already ended.', - ); - - let currentListeners = this._listeners.get('error'); - - if (!currentListeners) { - currentListeners = new DisposableMap(); - this._listeners.set('error', currentListeners); - } - - currentListeners.set(callback, this._onError.event(callback)); - } - - /** - * Add listener for the `end` event. - * @throws if the decoder stream has already ended. - */ - public onEnd(callback: () => void): void { - assert( - !this.ended, - 'Cannot subscribe to the `end` event because the decoder stream has already ended.', - ); - - let currentListeners = this._listeners.get('end'); - - if (!currentListeners) { - currentListeners = new DisposableMap(); - this._listeners.set('end', currentListeners); - } - - currentListeners.set(callback, this._onEnd.event(callback)); - } - - /** - * Pauses the stream. - */ - public pause(): void { - this.stream.pause(); - } - - /** - * Resumes the stream if it has been paused. - * @throws if the decoder stream has already ended. - */ - public resume(): void { - assert( - this.ended === false, - 'Cannot resume the stream because it has already ended.', - ); - - this.stream.resume(); - } - - /** - * Destroys(disposes) the stream. - */ - public destroy(): void { - this.dispose(); - } - - /** - * Removes a previously-registered event listener for a specified event. - * - * Note! - * - the callback function must be the same as the one that was used when - * registering the event listener as it is used as an identifier to - * remove the listener - * - this method is idempotent and results in no-op if the listener is - * not found, therefore passing incorrect `callback` function may - * result in silent unexpected behavior - */ - public removeListener(eventName: TStreamListenerNames, callback: Function): void { - const listeners = this._listeners.get(eventName); - if (listeners === undefined) { - return; - } - - for (const [listener] of listeners) { - if (listener !== callback) { - continue; - } - - listeners.deleteAndDispose(listener); - } - } - - /** - * This method is called when the input stream ends. - */ - protected onStreamEnd(): void { - if (this._ended) { - return; - } - - this._ended = true; - this._onEnd.fire(); - this.settledPromise.complete(this._ended); - } - - /** - * This method is called when the input stream emits an error. - * We re-emit the error here by default, but subclasses can - * override this method to handle the error differently. - */ - private onStreamError(error: Error): void { - this._onError.fire(error); - } - - /** - * Consume all messages from the stream, blocking until the stream finishes. - * @throws if the decoder stream has already ended. - */ - public async consumeAll(): Promise { - assert( - !this._ended, - 'Cannot consume all messages of the stream that has already ended.', - ); - - const messages = []; - - for await (const maybeMessage of this) { - if (maybeMessage === null) { - break; - } - - messages.push(maybeMessage); - } - - return messages; - } - - /** - * Async iterator interface for the decoder. - * @throws if the decoder stream has already ended. - */ - [Symbol.asyncIterator](): AsyncIterator { - assert( - !this._ended, - 'Cannot iterate on messages of the stream that has already ended.', - ); - - const asyncDecoder = this._register(new AsyncDecoder(this)); - - return asyncDecoder[Symbol.asyncIterator](); - } - - public override dispose(): void { - this.settledPromise.complete(this.ended); - - this._listeners.clearAndDisposeAll(); - this.stream.destroy(); - - super.dispose(); - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/baseToken.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/baseToken.ts deleted file mode 100644 index ba81a78738e..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/baseToken.ts +++ /dev/null @@ -1,150 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { assert } from '../../../../../../../base/common/assert.js'; -import { IRange, Range } from '../../../../../../../editor/common/core/range.js'; - -/** - * Base class for all tokens with a `range` that reflects - * token position in the original text. - */ -export abstract class BaseToken { - constructor( - private tokenRange: Range, - ) { } - - /** - * Range of the token in the original text. - */ - public get range(): Range { - return this.tokenRange; - } - - /** - * Text representation of the token. - */ - public abstract get text(): TText; - - /** - * Check if this token has the same range as another one. - */ - public sameRange(other: Range): boolean { - return this.range.equalsRange(other); - } - - /** - * Returns a string representation of the token. - */ - public abstract toString(): string; - - /** - * Check if this token is equal to another one. - */ - public equals(other: BaseToken): other is typeof this { - if (other.constructor !== this.constructor) { - return false; - } - - if (this.text.length !== other.text.length) { - return false; - } - - if (this.text !== other.text) { - return false; - } - - return this.sameRange(other.range); - } - - /** - * Change `range` of the token with provided range components. - */ - public withRange(components: Partial): this { - this.tokenRange = new Range( - components.startLineNumber ?? this.range.startLineNumber, - components.startColumn ?? this.range.startColumn, - components.endLineNumber ?? this.range.endLineNumber, - components.endColumn ?? this.range.endColumn, - ); - - return this; - } - - /** - * Collapse range of the token to its start position. - * See {@link Range.collapseToStart} for more details. - */ - public collapseRangeToStart(): this { - this.tokenRange = this.tokenRange.collapseToStart(); - - return this; - } - - /** - * Render a list of tokens into a string. - */ - public static render( - tokens: readonly BaseToken[], - delimiter: string = '', - ): string { - return tokens.map(token => token.text).join(delimiter); - } - - /** - * Returns the full range of a list of tokens in which the first token is - * used as the start of a tokens sequence and the last token reflects the end. - * - * @throws if: - * - provided {@link tokens} list is empty - * - the first token start number is greater than the start line of the last token - * - if the first and last token are on the same line, the first token start column must - * be smaller than the start column of the last token - */ - public static fullRange(tokens: readonly BaseToken[]): Range { - assert( - tokens.length > 0, - 'Cannot get full range for an empty list of tokens.', - ); - - const firstToken = tokens[0]; - const lastToken = tokens[tokens.length - 1]; - - // sanity checks for the full range we would construct - assert( - firstToken.range.startLineNumber <= lastToken.range.startLineNumber, - 'First token must start on previous or the same line as the last token.', - ); - - if ((firstToken !== lastToken) && (firstToken.range.startLineNumber === lastToken.range.startLineNumber)) { - assert( - firstToken.range.endColumn <= lastToken.range.startColumn, - [ - 'First token must end at least on previous or the same column as the last token.', - `First token: ${firstToken}; Last token: ${lastToken}.`, - ].join('\n'), - ); - } - - return new Range( - firstToken.range.startLineNumber, - firstToken.range.startColumn, - lastToken.range.endLineNumber, - lastToken.range.endColumn, - ); - } - - /** - * Shorten version of the {@link text} property. - */ - public shortText( - maxLength: number = 32, - ): string { - if (this.text.length <= maxLength) { - return this.text; - } - - return `${this.text.slice(0, maxLength - 1)}...`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/compositeToken.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/compositeToken.ts deleted file mode 100644 index e4eb1470a03..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/compositeToken.ts +++ /dev/null @@ -1,63 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from './baseToken.js'; - -/** - * Composite token consists of a list of other tokens. - * Composite token consists of a list of other tokens. - */ -export abstract class CompositeToken< - TTokens extends readonly BaseToken[], -> extends BaseToken { - /** - * Reference to the list of child tokens. - */ - protected readonly childTokens: [...TTokens]; - - constructor( - tokens: TTokens, - ) { - super(BaseToken.fullRange(tokens)); - - this.childTokens = [...tokens]; - } - - public override get text(): string { - return BaseToken.render(this.childTokens); - } - - /** - * Tokens that this composite token consists of. - */ - public get children(): TTokens { - return this.childTokens; - } - - /** - * Check if this token is equal to another one, - * including all of its child tokens. - */ - public override equals(other: BaseToken): other is typeof this { - if (super.equals(other) === false) { - return false; - } - - if (this.children.length !== other.children.length) { - return false; - } - - for (let i = 0; i < this.children.length; i++) { - const childToken = this.children[i]; - const otherChildToken = other.children[i]; - - if (childToken.equals(otherChildToken) === false) { - return false; - } - } - - return true; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/constants.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/constants.ts deleted file mode 100644 index 779e53a1ef1..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/constants.ts +++ /dev/null @@ -1,16 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { NewLine } from '../linesCodec/tokens/newLine.js'; -import { CarriageReturn } from '../linesCodec/tokens/carriageReturn.js'; -import { FormFeed, SpacingToken } from '../simpleCodec/tokens/tokens.js'; - -/** - * List of valid "space" tokens that are valid between different - * records of a Front Matter header. - */ -export const VALID_INTER_RECORD_SPACING_TOKENS = Object.freeze([ - SpacingToken, CarriageReturn, NewLine, FormFeed, -]); diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/frontMatterDecoder.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/frontMatterDecoder.ts deleted file mode 100644 index 704e10f1754..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/frontMatterDecoder.ts +++ /dev/null @@ -1,156 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { Word } from '../simpleCodec/tokens/tokens.js'; -import { assert } from '../../../../../../../../base/common/assert.js'; -import { VSBuffer } from '../../../../../../../../base/common/buffer.js'; -import { VALID_INTER_RECORD_SPACING_TOKENS } from './constants.js'; -import { ReadableStream } from '../../../../../../../../base/common/stream.js'; -import { FrontMatterToken, FrontMatterRecord } from './tokens/index.js'; -import { BaseDecoder } from '../baseDecoder.js'; -import { SimpleDecoder, type TSimpleDecoderToken } from '../simpleCodec/simpleDecoder.js'; -import { ObjectStream } from '../utils/objectStream.js'; -import { PartialFrontMatterRecordNameWithDelimiter } from './parsers/frontMatterRecord/frontMatterRecordNameWithDelimiter.js'; -import { PartialFrontMatterRecord } from './parsers/frontMatterRecord/frontMatterRecord.js'; -import { PartialFrontMatterRecordName } from './parsers/frontMatterRecord/frontMatterRecordName.js'; -import { FrontMatterParserFactory } from './parsers/frontMatterParserFactory.js'; - -/** - * Tokens produced by this decoder. - */ -export type TFrontMatterToken = FrontMatterRecord | TSimpleDecoderToken; - -/** - * Decoder capable of parsing Front Matter contents from a sequence of simple tokens. - */ -export class FrontMatterDecoder extends BaseDecoder { - /** - * Current parser reference responsible for parsing a specific sequence - * of tokens into a standalone token. - */ - private current?: PartialFrontMatterRecordName | PartialFrontMatterRecordNameWithDelimiter | PartialFrontMatterRecord; - - private readonly parserFactory: FrontMatterParserFactory; - - constructor( - stream: ReadableStream | ObjectStream, - ) { - if (stream instanceof ObjectStream) { - super(stream); - } else { - super(new SimpleDecoder(stream)); - } - this.parserFactory = new FrontMatterParserFactory(); - } - - protected override onStreamData(token: TSimpleDecoderToken): void { - if (this.current !== undefined) { - const acceptResult = this.current.accept(token); - const { result, wasTokenConsumed } = acceptResult; - - if (result === 'failure') { - this.reEmitCurrentTokens(); - - if (wasTokenConsumed === false) { - this._onData.fire(token); - } - - delete this.current; - return; - } - - const { nextParser } = acceptResult; - - if (nextParser instanceof FrontMatterToken) { - // front matter record token is the spacial case - because it can - // contain trailing space tokens, we want to emit "trimmed" record - // token and the trailing spaces tokens separately - const trimmedTokens = (nextParser instanceof FrontMatterRecord) - ? nextParser.trimValueEnd() - : []; - - this._onData.fire(nextParser); - - // re-emit all trailing space tokens if present - for (const trimmedToken of trimmedTokens) { - this._onData.fire(trimmedToken); - } - - if (wasTokenConsumed === false) { - this._onData.fire(token); - } - - delete this.current; - return; - } - - this.current = nextParser; - if (wasTokenConsumed === false) { - this._onData.fire(token); - } - - return; - } - - // a word token starts a new record - if (token instanceof Word) { - this.current = this.parserFactory.createRecordName(token); - return; - } - - // re-emit all "space" tokens immediately as all of them - // are valid while we are not in the "record parsing" mode - for (const ValidToken of VALID_INTER_RECORD_SPACING_TOKENS) { - if (token instanceof ValidToken) { - this._onData.fire(token); - return; - } - } - - // unexpected token type, re-emit existing tokens and continue - this.reEmitCurrentTokens(); - } - - protected override onStreamEnd(): void { - try { - if (this.current === undefined) { - return; - } - - assert( - this.current instanceof PartialFrontMatterRecord, - 'Only partial front matter records can be processed on stream end.', - ); - - const record = this.current.asRecordToken(); - const trimmedTokens = record.trimValueEnd(); - - this._onData.fire(record); - - for (const trimmedToken of trimmedTokens) { - this._onData.fire(trimmedToken); - } - } catch (_error) { - this.reEmitCurrentTokens(); - } finally { - delete this.current; - super.onStreamEnd(); - } - } - - /** - * Re-emit tokens accumulated so far in the current parser object. - */ - protected reEmitCurrentTokens(): void { - if (this.current === undefined) { - return; - } - - for (const token of this.current.tokens) { - this._onData.fire(token); - } - delete this.current; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterArray.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterArray.ts deleted file mode 100644 index df84fa012c7..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterArray.ts +++ /dev/null @@ -1,197 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { assert } from '../../../../../../../../../base/common/assert.js'; -import { type PartialFrontMatterValue } from './frontMatterValue.js'; -import { FrontMatterArray } from '../tokens/frontMatterArray.js'; -import { assertDefined } from '../../../../../../../../../base/common/types.js'; -import { VALID_INTER_RECORD_SPACING_TOKENS } from '../constants.js'; -import { FrontMatterValueToken } from '../tokens/frontMatterToken.js'; -import { FrontMatterSequence } from '../tokens/frontMatterSequence.js'; -import { TSimpleDecoderToken } from '../../simpleCodec/simpleDecoder.js'; -import { Comma, LeftBracket, RightBracket } from '../../simpleCodec/tokens/tokens.js'; -import { assertNotConsumed, ParserBase, TAcceptTokenResult } from '../../simpleCodec/parserBase.js'; -import { type FrontMatterParserFactory } from './frontMatterParserFactory.js'; - -/** - * List of tokens that can go in-between array items - * and array brackets. - */ -const VALID_DELIMITER_TOKENS = Object.freeze([ - ...VALID_INTER_RECORD_SPACING_TOKENS, - Comma, -]); - -/** - * Responsible for parsing an array syntax (or "inline sequence" - * in YAML terms), e.g. `[1, '2', true, 2.54]` -*/ -export class PartialFrontMatterArray extends ParserBase { - /** - * Current parser reference responsible for parsing an array "value". - */ - private currentValueParser?: PartialFrontMatterValue; - - /** - * Whether an array item is allowed in the current position of the token - * sequence. E.g., items are allowed after a command or a open bracket, - * but not immediately after another item in the array. - */ - private arrayItemAllowed = true; - - constructor( - private readonly factory: FrontMatterParserFactory, - private readonly startToken: LeftBracket, - ) { - super([startToken]); - } - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - if (this.currentValueParser !== undefined) { - const acceptResult = this.currentValueParser.accept(token); - const { result, wasTokenConsumed } = acceptResult; - - if (result === 'failure') { - this.isConsumed = true; - - return { - result: 'failure', - wasTokenConsumed, - }; - } - - const { nextParser } = acceptResult; - - if (nextParser instanceof FrontMatterValueToken) { - this.currentTokens.push(nextParser); - delete this.currentValueParser; - - // if token was not consume, call the `accept()` method - // recursively so that the current parser can re-process - // the token (e.g., a comma or a closing square bracket) - if (wasTokenConsumed === false) { - return this.accept(token); - } - - return { - result: 'success', - nextParser: this, - wasTokenConsumed, - }; - } - - this.currentValueParser = nextParser; - return { - result: 'success', - nextParser: this, - wasTokenConsumed, - }; - } - - if (token instanceof RightBracket) { - // sanity check in case this block moves around - // to a different place in the code - assert( - this.currentValueParser === undefined, - `Unexpected end of array. Last value is not finished.`, - ); - - this.currentTokens.push(token); - - this.isConsumed = true; - return { - result: 'success', - nextParser: this.asArrayToken(), - wasTokenConsumed: true, - }; - } - - // iterate until a valid value start token is found - for (const ValidToken of VALID_DELIMITER_TOKENS) { - if (token instanceof ValidToken) { - this.currentTokens.push(token); - - if ((this.arrayItemAllowed === false) && token instanceof Comma) { - this.arrayItemAllowed = true; - } - - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } - } - - // is an array item value is allowed at this position, create a new - // value parser and start the value parsing process using it - if (this.arrayItemAllowed === true) { - this.currentValueParser = this.factory.createValue( - (currentToken) => { - // comma or a closing square bracket must stop the parsing - // process of the value represented by a generic sequence of tokens - return ( - (currentToken instanceof RightBracket) - || (currentToken instanceof Comma) - ); - }, - ); - this.arrayItemAllowed = false; - - return this.accept(token); - } - - // in all other cases fail because of the unexpected token type - this.isConsumed = true; - return { - result: 'failure', - wasTokenConsumed: false, - }; - } - - /** - * Convert current parser into a {@link FrontMatterArray} token, - * if possible. - * - * @throws if the last token in the accumulated token list - * is not a closing bracket ({@link RightBracket}). - */ - public asArrayToken(): FrontMatterArray { - const endToken = this.currentTokens[this.currentTokens.length - 1]; - - assertDefined( - endToken, - 'No tokens found.', - ); - - assert( - endToken instanceof RightBracket, - 'Cannot find a closing bracket of the array.', - ); - - const valueTokens: FrontMatterValueToken[] = []; - for (const currentToken of this.currentTokens) { - if ((currentToken instanceof FrontMatterValueToken) === false) { - continue; - } - - // the generic sequence tokens can have trailing spacing tokens, - // hence trim them to ensure the array contains only "clean" values - if (currentToken instanceof FrontMatterSequence) { - currentToken.trimEnd(); - } - - valueTokens.push(currentToken); - } - - this.isConsumed = true; - return new FrontMatterArray([ - this.startToken, - ...valueTokens, - endToken, - ]); - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterParserFactory.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterParserFactory.ts deleted file mode 100644 index 0be65371449..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterParserFactory.ts +++ /dev/null @@ -1,41 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../../baseToken.js'; -import { LeftBracket } from '../../simpleCodec/tokens/brackets.js'; -import { Word } from '../../simpleCodec/tokens/word.js'; -import { FrontMatterRecordDelimiter, FrontMatterRecordName } from '../tokens/frontMatterRecord.js'; -import { TQuoteToken } from '../tokens/frontMatterString.js'; -import { PartialFrontMatterArray } from './frontMatterArray.js'; -import { PartialFrontMatterRecord } from './frontMatterRecord/frontMatterRecord.js'; -import { PartialFrontMatterRecordName } from './frontMatterRecord/frontMatterRecordName.js'; -import { PartialFrontMatterRecordNameWithDelimiter, TNameStopToken } from './frontMatterRecord/frontMatterRecordNameWithDelimiter.js'; -import { PartialFrontMatterSequence } from './frontMatterSequence.js'; -import { PartialFrontMatterString } from './frontMatterString.js'; -import { PartialFrontMatterValue } from './frontMatterValue.js'; - -export class FrontMatterParserFactory { - createRecord(tokens: [FrontMatterRecordName, FrontMatterRecordDelimiter]): PartialFrontMatterRecord { - return new PartialFrontMatterRecord(this, tokens); - } - createRecordName(startToken: Word): PartialFrontMatterRecordName { - return new PartialFrontMatterRecordName(this, startToken); - } - createRecordNameWithDelimiter(tokens: readonly [FrontMatterRecordName, TNameStopToken]): PartialFrontMatterRecordNameWithDelimiter { - return new PartialFrontMatterRecordNameWithDelimiter(this, tokens); - } - createArray(startToken: LeftBracket) { - return new PartialFrontMatterArray(this, startToken); - } - createValue(shouldStop: (token: BaseToken) => boolean): PartialFrontMatterValue { - return new PartialFrontMatterValue(this, shouldStop); - } - createString(startToken: TQuoteToken): PartialFrontMatterString { - return new PartialFrontMatterString(startToken); - } - createSequence(shouldStop: (token: BaseToken) => boolean): PartialFrontMatterSequence { - return new PartialFrontMatterSequence(shouldStop); - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterRecord/frontMatterRecord.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterRecord/frontMatterRecord.ts deleted file mode 100644 index e4afee055b5..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterRecord/frontMatterRecord.ts +++ /dev/null @@ -1,210 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../../../baseToken.js'; -import { NewLine } from '../../../linesCodec/tokens/newLine.js'; -import { PartialFrontMatterValue } from '../frontMatterValue.js'; -import { assertNever } from '../../../../../../../../../../base/common/assert.js'; -import { assertDefined } from '../../../../../../../../../../base/common/types.js'; -import { PartialFrontMatterSequence } from '../frontMatterSequence.js'; -import { CarriageReturn } from '../../../linesCodec/tokens/carriageReturn.js'; -import { type TSimpleDecoderToken } from '../../../simpleCodec/simpleDecoder.js'; -import { Word, FormFeed, SpacingToken } from '../../../simpleCodec/tokens/tokens.js'; -import { assertNotConsumed, ParserBase, type TAcceptTokenResult } from '../../../simpleCodec/parserBase.js'; -import { FrontMatterValueToken, FrontMatterRecordName, FrontMatterRecordDelimiter, FrontMatterRecord } from '../../tokens/index.js'; -import { type FrontMatterParserFactory } from '../frontMatterParserFactory.js'; - -/** - * Type of a next parser that can be returned by {@link PartialFrontMatterRecord}. - */ -type TNextParser = PartialFrontMatterRecord | FrontMatterRecord; - -/** - * Parser for a `record` inside a Front Matter header. - * - * * E.g., `name: 'value'` in the example below: - * - * ``` - * --- - * name: 'value' - * isExample: true - * --- - * ``` - */ -export class PartialFrontMatterRecord extends ParserBase { - /** - * Token that represents the 'name' part of the record. - */ - private readonly recordNameToken: FrontMatterRecordName; - - /** - * Token that represents the 'delimiter' part of the record. - */ - private readonly recordDelimiterToken: FrontMatterRecordDelimiter; - - constructor( - private readonly factory: FrontMatterParserFactory, - tokens: [FrontMatterRecordName, FrontMatterRecordDelimiter], - ) { - super(tokens); - this.recordNameToken = tokens[0]; - this.recordDelimiterToken = tokens[1]; - } - - /** - * Current parser reference responsible for parsing the "value" part of the record. - */ - private valueParser?: PartialFrontMatterValue | PartialFrontMatterSequence; - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - if (this.valueParser !== undefined) { - const acceptResult = this.valueParser.accept(token); - const { result, wasTokenConsumed } = acceptResult; - - if (result === 'failure') { - this.isConsumed = true; - - return { - result: 'failure', - wasTokenConsumed, - }; - } - - const { nextParser } = acceptResult; - - if (nextParser instanceof FrontMatterValueToken) { - this.currentTokens.push(nextParser); - delete this.valueParser; - - this.isConsumed = true; - try { - return { - result: 'success', - nextParser: new FrontMatterRecord([ - this.recordNameToken, - this.recordDelimiterToken, - nextParser, - ]), - wasTokenConsumed, - }; - } catch (_error) { - return { - result: 'failure', - wasTokenConsumed, - }; - } - } - - this.valueParser = nextParser; - return { - result: 'success', - nextParser: this, - wasTokenConsumed, - }; - } - - // iterate until the first non-space token is found - if (token instanceof SpacingToken) { - this.currentTokens.push(token); - - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } - - // if token can start a "value" sequence, parse the value - if (PartialFrontMatterValue.isValueStartToken(token)) { - this.valueParser = this.factory.createValue(shouldEndTokenSequence); - - return this.accept(token); - } - - // in all other cases, collect all the subsequent tokens into - // a "sequence of tokens" until a new line is found - this.valueParser = this.factory.createSequence( - shouldEndTokenSequence, - ); - - // if we reached this "generic sequence" parser point, but the current token is - // already of a type that stops such sequence, we must have accumulated some - // spacing tokens, hence pass those to the parser and end the sequence immediately - if (shouldEndTokenSequence(token)) { - const spaceTokens = this.currentTokens - .slice(this.startTokensCount); - - // if no space tokens accumulated at all, create an "empty" one this is needed - // to ensure that the parser always has at least one token hence it can have - // a valid range and can be interpreted as a real "value" token of the record - if (spaceTokens.length === 0) { - spaceTokens.push( - Word.newOnLine( - '', - token.range.startLineNumber, - token.range.startColumn, - ), - ); - } - - this.valueParser.addTokens(spaceTokens); - - return { - result: 'success', - nextParser: this.asRecordToken(), - wasTokenConsumed: false, - }; - } - - // otherwise use the "generic sequence" parser moving on - return this.accept(token); - } - - /** - * Convert current parser into a {@link FrontMatterRecord} token. - * - * @throws if no current parser is present, or it is not of the {@link PartialFrontMatterValue} - * or {@link PartialFrontMatterSequence} types - */ - public asRecordToken(): FrontMatterRecord { - assertDefined( - this.valueParser, - 'Current value parser must be defined.' - ); - - if ( - (this.valueParser instanceof PartialFrontMatterValue) - || (this.valueParser instanceof PartialFrontMatterSequence) - ) { - const valueToken = this.valueParser.asSequenceToken(); - this.currentTokens.push(valueToken); - - this.isConsumed = true; - return new FrontMatterRecord([ - this.recordNameToken, - this.recordDelimiterToken, - valueToken, - ]); - } - - assertNever( - this.valueParser, - `Unexpected value parser '${this.valueParser}'.`, - ); - } -} - -/** - * Callback to check if a current token should end a - * record value that is a generic sequence of tokens. - */ -function shouldEndTokenSequence(token: BaseToken): token is (NewLine | CarriageReturn | FormFeed) { - return ( - (token instanceof NewLine) - || (token instanceof CarriageReturn) - || (token instanceof FormFeed) - ); -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterRecord/frontMatterRecordName.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterRecord/frontMatterRecordName.ts deleted file mode 100644 index 84c94b75211..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterRecord/frontMatterRecordName.ts +++ /dev/null @@ -1,74 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { type TSimpleDecoderToken } from '../../../simpleCodec/simpleDecoder.js'; -import { FrontMatterRecordName, type TRecordNameToken } from '../../tokens/index.js'; -import { Colon, Word, Dash, SpacingToken } from '../../../simpleCodec/tokens/tokens.js'; -import { type PartialFrontMatterRecordNameWithDelimiter } from './frontMatterRecordNameWithDelimiter.js'; -import { assertNotConsumed, ParserBase, type TAcceptTokenResult } from '../../../simpleCodec/parserBase.js'; -import { type FrontMatterParserFactory } from '../frontMatterParserFactory.js'; - -/** - * Tokens that can be used inside a record name. - */ -const VALID_NAME_TOKENS = [Word, Dash]; - -/** - * Type of a next parser that can be returned by {@link PartialFrontMatterRecordName}. - */ -type TNextParser = PartialFrontMatterRecordName | PartialFrontMatterRecordNameWithDelimiter; - -/** - * Parser for a `name` part of a Front Matter record. - * - * E.g., `'name'` in the example below: - * - * ``` - * name: 'value' - * ``` - */ -export class PartialFrontMatterRecordName extends ParserBase { - constructor( - private readonly factory: FrontMatterParserFactory, - startToken: Word, - ) { - super([startToken]); - } - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - for (const ValidToken of VALID_NAME_TOKENS) { - if (token instanceof ValidToken) { - this.currentTokens.push(token); - - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } - } - - // once name is followed by a "space" token or a "colon", we have the full - // record name hence can transition to the next parser - if ((token instanceof Colon) || (token instanceof SpacingToken)) { - const recordName = new FrontMatterRecordName(this.currentTokens); - - this.isConsumed = true; - return { - result: 'success', - nextParser: this.factory.createRecordNameWithDelimiter([recordName, token]), - wasTokenConsumed: true, - }; - } - - // in all other cases fail due to the unexpected token type for a record name - this.isConsumed = true; - return { - result: 'failure', - wasTokenConsumed: false, - }; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterRecord/frontMatterRecordNameWithDelimiter.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterRecord/frontMatterRecordNameWithDelimiter.ts deleted file mode 100644 index 59735860631..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterRecord/frontMatterRecordNameWithDelimiter.ts +++ /dev/null @@ -1,105 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { assert } from '../../../../../../../../../../base/common/assert.js'; -import { type PartialFrontMatterRecord } from './frontMatterRecord.js'; -import { Colon, SpacingToken } from '../../../simpleCodec/tokens/tokens.js'; -import { type TSimpleDecoderToken } from '../../../simpleCodec/simpleDecoder.js'; -import { FrontMatterRecordName, FrontMatterRecordDelimiter } from '../../tokens/index.js'; -import { assertNotConsumed, ParserBase, type TAcceptTokenResult } from '../../../simpleCodec/parserBase.js'; -import { type FrontMatterParserFactory } from '../frontMatterParserFactory.js'; - -/** - * Type for tokens that stop a front matter record name sequence. - */ -export type TNameStopToken = Colon | SpacingToken; - -/** - * Type for the next parser that can be returned by {@link PartialFrontMatterRecordNameWithDelimiter}. - */ -type TNextParser = PartialFrontMatterRecordNameWithDelimiter | PartialFrontMatterRecord; - -/** - * Parser for a record `name` with the `: ` delimiter. - * - * * E.g., `name:` in the example below: - * - * ``` - * name: 'value' - * ``` - */ -export class PartialFrontMatterRecordNameWithDelimiter extends ParserBase< - FrontMatterRecordName | TNameStopToken, - TNextParser -> { - constructor( - private readonly factory: FrontMatterParserFactory, - tokens: readonly [FrontMatterRecordName, TNameStopToken], - ) { - super([...tokens]); - } - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - const previousToken = this.currentTokens[this.currentTokens.length - 1]; - const isSpacingToken = (token instanceof SpacingToken); - - // delimiter must always be a `:` followed by a "space" character - // once we encounter that sequence, we can transition to the next parser - if (isSpacingToken && (previousToken instanceof Colon)) { - const recordDelimiter = new FrontMatterRecordDelimiter([ - previousToken, - token, - ]); - - const recordName = this.currentTokens[0]; - - // sanity check - assert( - recordName instanceof FrontMatterRecordName, - `Expected a front matter record name, got '${recordName}'.`, - ); - - this.isConsumed = true; - return { - result: 'success', - nextParser: this.factory.createRecord( - [recordName, recordDelimiter], - ), - wasTokenConsumed: true, - }; - } - - // allow some spacing before the colon delimiter - if (token instanceof SpacingToken) { - this.currentTokens.push(token); - - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } - - // include the colon delimiter - if (token instanceof Colon) { - this.currentTokens.push(token); - - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } - - // otherwise fail due to the unexpected token type between - // record name and record name delimiter tokens - this.isConsumed = true; - return { - result: 'failure', - wasTokenConsumed: false, - }; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterSequence.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterSequence.ts deleted file mode 100644 index 0d91c32528c..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterSequence.ts +++ /dev/null @@ -1,78 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../../baseToken.js'; -import { FrontMatterSequence } from '../tokens/frontMatterSequence.js'; -import { TSimpleDecoderToken } from '../../simpleCodec/simpleDecoder.js'; -import { assertNotConsumed, ParserBase, TAcceptTokenResult } from '../../simpleCodec/parserBase.js'; - -/** - * Parser responsible for parsing a "generic sequence of tokens" - * of an arbitrary length in a Front Matter header. - */ -export class PartialFrontMatterSequence extends ParserBase< - TSimpleDecoderToken, - PartialFrontMatterSequence | FrontMatterSequence -> { - constructor( - /** - * Callback function that is called to check if the current token - * should stop the parsing process of the current generic "value" - * sequence of arbitrary tokens by returning `true`. - * - * When this happens, the parser *will not consume* the token that - * was passed to the `shouldStop` callback or to its `accept` method. - * On the other hand, the parser will be "consumed" hence using it - * to process other tokens will yield an error. - */ - private readonly shouldStop: (token: BaseToken) => boolean, - ) { - super([]); - } - - @assertNotConsumed - public accept( - token: TSimpleDecoderToken, - ): TAcceptTokenResult { - - // collect all tokens until an end of the sequence is found - if (this.shouldStop(token)) { - this.isConsumed = true; - - return { - result: 'success', - nextParser: this.asSequenceToken(), - wasTokenConsumed: false, - }; - } - - this.currentTokens.push(token); - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } - - /** - * Add provided tokens to the list of the current parsed tokens. - */ - public addTokens( - tokens: readonly TSimpleDecoderToken[], - ): this { - this.currentTokens.push(...tokens); - - return this; - } - - /** - * Convert the current parser into a {@link FrontMatterSequence} token. - */ - public asSequenceToken(): FrontMatterSequence { - this.isConsumed = true; - - return new FrontMatterSequence(this.currentTokens); - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterString.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterString.ts deleted file mode 100644 index d0ed92746d6..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterString.ts +++ /dev/null @@ -1,69 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { assert } from '../../../../../../../../../base/common/assert.js'; -import { SimpleToken } from '../../simpleCodec/tokens/tokens.js'; -import { assertDefined } from '../../../../../../../../../base/common/types.js'; -import { TSimpleDecoderToken } from '../../simpleCodec/simpleDecoder.js'; -import { FrontMatterString, type TQuoteToken } from '../tokens/frontMatterString.js'; -import { assertNotConsumed, ParserBase, type TAcceptTokenResult } from '../../simpleCodec/parserBase.js'; - -/** - * Parser responsible for parsing a string value. - */ -export class PartialFrontMatterString extends ParserBase> { - constructor( - private readonly startToken: TQuoteToken, - ) { - super([startToken]); - } - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult> { - this.currentTokens.push(token); - - // iterate until a `matching end quote` is found - if ((token instanceof SimpleToken) && (this.startToken.sameType(token))) { - return { - result: 'success', - nextParser: this.asStringToken(), - wasTokenConsumed: true, - }; - } - - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } - - /** - * Convert the current parser into a {@link FrontMatterString} token, - * if possible. - * - * @throws if the first and last tokens are not quote tokens of the same type. - */ - public asStringToken(): FrontMatterString { - const endToken = this.currentTokens[this.currentTokens.length - 1]; - - assertDefined( - endToken, - `No matching end token found.`, - ); - - assert( - this.startToken.sameType(endToken), - `String starts with \`${this.startToken.text}\`, but ends with \`${endToken.text}\`.`, - ); - - return new FrontMatterString([ - this.startToken, - ...this.currentTokens - .slice(1, this.currentTokens.length - 1), - endToken, - ]); - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterValue.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterValue.ts deleted file mode 100644 index 710767aed40..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/parsers/frontMatterValue.ts +++ /dev/null @@ -1,184 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../../baseToken.js'; -import { type PartialFrontMatterArray } from './frontMatterArray.js'; -import { type PartialFrontMatterString } from './frontMatterString.js'; -import { asBoolean, FrontMatterBoolean } from '../tokens/frontMatterBoolean.js'; -import { FrontMatterValueToken } from '../tokens/frontMatterToken.js'; -import { PartialFrontMatterSequence } from './frontMatterSequence.js'; -import { FrontMatterSequence } from '../tokens/frontMatterSequence.js'; -import { TSimpleDecoderToken } from '../../simpleCodec/simpleDecoder.js'; -import { Word, Quote, DoubleQuote, LeftBracket } from '../../simpleCodec/tokens/tokens.js'; -import { assertNotConsumed, ParserBase, TAcceptTokenResult } from '../../simpleCodec/parserBase.js'; -import { type FrontMatterParserFactory } from './frontMatterParserFactory.js'; - -/** - * List of tokens that can start a "value" sequence. - * - * - {@link Word} - can be a `boolean` value - * - {@link Quote}, {@link DoubleQuote} - can start a `string` value - * - {@link LeftBracket} - can start an `array` value - */ -export const VALID_VALUE_START_TOKENS = Object.freeze([ - Quote, - DoubleQuote, - LeftBracket, -]); - -/** - * Type alias for a token that can start a "value" sequence. - */ -type TValueStartToken = InstanceType; - -/** - * Parser responsible for parsing a "value" sequence in a Front Matter header. - */ -export class PartialFrontMatterValue extends ParserBase { - /** - * Current parser reference responsible for parsing - * a specific "value" sequence. - */ - private currentValueParser?: PartialFrontMatterString | PartialFrontMatterArray | PartialFrontMatterSequence; - - /** - * Get the tokens that were accumulated so far. - */ - public override get tokens(): readonly TSimpleDecoderToken[] { - if (this.currentValueParser === undefined) { - return []; - } - - return this.currentValueParser.tokens; - } - - constructor( - private readonly factory: FrontMatterParserFactory, - /** - * Callback function to pass to the {@link PartialFrontMatterSequence} - * if the current "value" sequence is not of a specific type. - */ - private readonly shouldStop: (token: BaseToken) => boolean, - ) { - super(); - } - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - if (this.currentValueParser !== undefined) { - const acceptResult = this.currentValueParser.accept(token); - const { result, wasTokenConsumed } = acceptResult; - - // current value parser is consumed with its child value parser - this.isConsumed = this.currentValueParser.consumed; - - if (result === 'success') { - const { nextParser } = acceptResult; - - if (nextParser instanceof FrontMatterValueToken) { - return { - result: 'success', - nextParser, - wasTokenConsumed, - }; - } - - this.currentValueParser = nextParser; - return { - result: 'success', - nextParser: this, - wasTokenConsumed, - }; - } - - return { - result: 'failure', - wasTokenConsumed, - }; - } - - // if the first token represents a `quote` character, try to parse a string value - if ((token instanceof Quote) || (token instanceof DoubleQuote)) { - this.currentValueParser = this.factory.createString(token); - - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } - - // if the first token represents a `[` character, try to parse an array value - if (token instanceof LeftBracket) { - this.currentValueParser = this.factory.createArray(token); - - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } - - // if the first token represents a `word` try to parse a boolean - const maybeBoolean = FrontMatterBoolean.tryFromToken(token); - if (maybeBoolean !== null) { - this.isConsumed = true; - - return { - result: 'success', - nextParser: maybeBoolean, - wasTokenConsumed: true, - }; - } - - // in all other cases, collect all the subsequent tokens into - // a generic sequence of tokens until stopped by the `this.shouldStop` - // callback or the call to the 'this.asSequenceToken' method - this.currentValueParser = this.factory.createSequence(this.shouldStop); - - return this.accept(token); - } - - /** - * Check if provided token can be a start of a "value" sequence. - * See {@link VALID_VALUE_START_TOKENS} for the list of valid tokens. - */ - public static isValueStartToken( - token: BaseToken, - ): token is TValueStartToken | Word<'true' | 'false'> { - for (const ValidToken of VALID_VALUE_START_TOKENS) { - if (token instanceof ValidToken) { - return true; - } - } - - if ((token instanceof Word) && (asBoolean(token) !== null)) { - return true; - } - - return false; - } - - /** - * Check if the current 'value' sequence does not have a specific type - * and is represented by a generic sequence of tokens ({@link PartialFrontMatterSequence}). - */ - public get isSequence(): boolean { - if (this.currentValueParser === undefined) { - return false; - } - - return (this.currentValueParser instanceof PartialFrontMatterSequence); - } - - /** - * Convert current parser into a generic sequence of tokens. - */ - public asSequenceToken(): FrontMatterSequence { - this.isConsumed = true; - - return new FrontMatterSequence(this.tokens); - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterArray.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterArray.ts deleted file mode 100644 index b8b01e42c79..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterArray.ts +++ /dev/null @@ -1,43 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../../baseToken.js'; -import { LeftBracket, RightBracket } from '../../simpleCodec/tokens/tokens.js'; -import { FrontMatterValueToken, type TValueTypeName } from './frontMatterToken.js'; - -/** - * Token that represents an `array` value in a Front Matter header. - */ -export class FrontMatterArray extends FrontMatterValueToken<'array', [ - LeftBracket, - ...FrontMatterValueToken[], - RightBracket, -]> { - /** - * Name of the `array` value type. - */ - public override readonly valueTypeName = 'array'; - - /** - * List of the array items. - */ - public get items(): readonly FrontMatterValueToken[] { - const result = []; - - for (const token of this.children) { - if (token instanceof FrontMatterValueToken) { - result.push(token); - } - } - - return result; - } - - public override toString(): string { - const itemsString = BaseToken.render(this.items, ', '); - - return `front-matter-array(${itemsString})${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterBoolean.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterBoolean.ts deleted file mode 100644 index 3eae6d114ed..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterBoolean.ts +++ /dev/null @@ -1,88 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../../baseToken.js'; -import { Word } from '../../simpleCodec/tokens/tokens.js'; -import { FrontMatterValueToken } from './frontMatterToken.js'; -import { assertDefined } from '../../../../../../../../../base/common/types.js'; - -/** - * Token that represents a `boolean` value in a Front Matter header. - */ -export class FrontMatterBoolean extends FrontMatterValueToken<'boolean', readonly [Word]> { - /** - * Name of the `boolean` value type. - */ - public override readonly valueTypeName = 'boolean'; - - /** - * Value of the `boolean` token. - */ - public readonly value: boolean; - - /** - * @throws if provided {@link Word} cannot be converted to a `boolean` value. - */ - constructor(token: Word) { - const value = asBoolean(token); - assertDefined( - value, - `Cannot convert '${token}' to a boolean value.`, - ); - - super([token]); - - this.value = value; - } - - /** - * Try creating a {@link FrontMatterBoolean} out of provided token. - * Unlike the constructor, this method does not throw, returning - * a 'null' value on failure instead. - */ - public static tryFromToken( - token: BaseToken, - ): FrontMatterBoolean | null { - if (token instanceof Word === false) { - return null; - } - - try { - return new FrontMatterBoolean(token); - } catch (_error) { - // noop - return null; - } - } - - public override equals(other: BaseToken): other is typeof this { - if (super.equals(other) === false) { - return false; - } - - return this.value === other.value; - } - - public override toString(): string { - return `front-matter-boolean(${this.shortText()})${this.range}`; - } -} - -/** - * Try to convert a {@link Word} token to a `boolean` value. - */ -export function asBoolean( - token: Word, -): boolean | null { - if (token.text.toLowerCase() === 'true') { - return true; - } - - if (token.text.toLowerCase() === 'false') { - return false; - } - - return null; -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterRecord.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterRecord.ts deleted file mode 100644 index dc498c6d667..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterRecord.ts +++ /dev/null @@ -1,118 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../../baseToken.js'; -import { FrontMatterSequence } from './frontMatterSequence.js'; -import { Colon, Word, Dash, SpacingToken } from '../../simpleCodec/tokens/tokens.js'; -import { FrontMatterToken, FrontMatterValueToken, type TValueTypeName } from './frontMatterToken.js'; - -/** - * Type for tokens that can be used inside a record name. - */ -export type TNameToken = Word | Dash; - -/** - * Token representing a `record name` inside a Front Matter record. - * - * E.g., `name` in the example below: - * - * ``` - * --- - * name: 'value' - * --- - * ``` - */ -export class FrontMatterRecordName extends FrontMatterToken { - public override toString(): string { - return `front-matter-record-name(${this.shortText()})${this.range}`; - } -} - -/** - * Token representing a delimiter of a record inside a Front Matter header. - * - * E.g., `: ` in the example below: - * - * ``` - * --- - * name: 'value' - * --- - * ``` - */ -export class FrontMatterRecordDelimiter extends FrontMatterToken { - public override toString(): string { - return `front-matter-delimiter(${this.shortText()})${this.range}`; - } -} - -/** - * Token representing a `record` inside a Front Matter header. - * - * E.g., `name: 'value'` in the example below: - * - * ``` - * --- - * name: 'value' - * --- - * ``` - */ -export class FrontMatterRecord extends FrontMatterToken< - readonly [FrontMatterRecordName, FrontMatterRecordDelimiter, FrontMatterValueToken] -> { - /** - * Token that represent `name` of the record. - * - * E.g., `tools` in the example below: - * - * ``` - * --- - * tools: ['value'] - * --- - * ``` - */ - public get nameToken(): FrontMatterRecordName { - return this.children[0]; - } - - /** - * Token that represent `value` of the record. - * - * E.g., `['value']` in the example below: - * - * ``` - * --- - * tools: ['value'] - * --- - * ``` - */ - public get valueToken(): FrontMatterValueToken { - return this.children[2]; - } - - /** - * Trim spacing tokens at the end of the record. - */ - public trimValueEnd(): readonly SpacingToken[] { - const { valueToken } = this; - - // only the "generic sequence" value tokens can hold - // some spacing tokens at the end of them - if ((valueToken instanceof FrontMatterSequence) === false) { - return []; - } - - const trimmedTokens = valueToken.trimEnd(); - // update the current range to reflect the current trimmed value - this.withRange( - BaseToken.fullRange(this.children), - ); - - return trimmedTokens; - } - - public override toString(): string { - return `front-matter-record(${this.shortText()})${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterSequence.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterSequence.ts deleted file mode 100644 index dcb6f70d5a3..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterSequence.ts +++ /dev/null @@ -1,79 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../../baseToken.js'; -import { FrontMatterValueToken } from './frontMatterToken.js'; -import { Word, SpacingToken } from '../../simpleCodec/tokens/tokens.js'; -import { type TSimpleDecoderToken } from '../../simpleCodec/simpleDecoder.js'; - - -/** - * Token represents a generic sequence of tokens in a Front Matter header. - */ -export class FrontMatterSequence extends FrontMatterValueToken { - /** - * @override Because this token represent a generic sequence of tokens, - * the type name is represented by the sequence of tokens itself - */ - public override get valueTypeName(): this { - return this; - } - - /** - * Text of the sequence value. The method exists to provide a - * consistent interface with {@link FrontMatterString} token. - * - * Note! that this method does not automatically trim spacing tokens - * in the sequence. If you need to get a trimmed value, call - * {@link trimEnd} method first. - */ - public get cleanText(): string { - return this.text; - } - - /** - * Trim spacing tokens at the end of the sequence. - */ - public trimEnd(): readonly SpacingToken[] { - const trimmedTokens = []; - - // iterate the tokens list from the end to the start, collecting - // all the spacing tokens we encounter until we reach a non-spacing token - let lastNonSpace = this.childTokens.length - 1; - while (lastNonSpace >= 0) { - const token = this.childTokens[lastNonSpace]; - - if (token instanceof SpacingToken) { - trimmedTokens.push(token); - lastNonSpace--; - - continue; - } - - break; - } - this.childTokens.length = lastNonSpace + 1; - - // if there are only spacing tokens were present add a single - // empty token to the sequence, so it has something to work with - if (this.childTokens.length === 0) { - this.collapseRangeToStart(); - this.childTokens.push(new Word(this.range, '')); - } - - // update the current range to reflect the current trimmed value - this.withRange( - BaseToken.fullRange(this.childTokens), - ); - - // trimmed tokens are collected starting from the end, - // moving to the start, hence reverse them before returning - return trimmedTokens.reverse(); - } - - public override toString(): string { - return `front-matter-sequence(${this.shortText()})${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterString.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterString.ts deleted file mode 100644 index 8a7283d1098..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterString.ts +++ /dev/null @@ -1,39 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../../baseToken.js'; -import { FrontMatterValueToken } from './frontMatterToken.js'; -import { Quote, DoubleQuote } from '../../simpleCodec/tokens/tokens.js'; - -/** - * Type for any quote token that can be used to wrap a string. - */ -export type TQuoteToken = Quote | DoubleQuote; - -/** - * Token that represents a string value in a Front Matter header. - */ -export class FrontMatterString extends FrontMatterValueToken< - 'quoted-string', - readonly [TQuote, ...BaseToken[], TQuote] -> { - /** - * Name of the `string` value type. - */ - public override readonly valueTypeName = 'quoted-string'; - - /** - * Text of the string value without the wrapping quotes. - */ - public get cleanText(): string { - return BaseToken.render( - this.children.slice(1, this.children.length - 1), - ); - } - - public override toString(): string { - return `front-matter-string(${this.shortText()})${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterToken.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterToken.ts deleted file mode 100644 index d85f85230d7..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterToken.ts +++ /dev/null @@ -1,33 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../../baseToken.js'; -import { CompositeToken } from '../../compositeToken.js'; -import { FrontMatterSequence } from './frontMatterSequence.js'; - -/** - * Base class for all tokens inside a Front Matter header. - */ -export abstract class FrontMatterToken< - TTokens extends readonly BaseToken[] = readonly BaseToken[], -> extends CompositeToken { } - -/** - * List of all currently supported value types. - */ -export type TValueTypeName = 'quoted-string' | 'boolean' | 'array' | FrontMatterSequence; - -/** - * Base class for all tokens that represent a `value` inside a Front Matter header. - */ -export abstract class FrontMatterValueToken< - TTypeName extends TValueTypeName = TValueTypeName, - TTokens extends readonly BaseToken[] = readonly BaseToken[], -> extends FrontMatterToken { - /** - * Type name of the `value` represented by this token. - */ - public abstract readonly valueTypeName: TTypeName; -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/index.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/index.ts deleted file mode 100644 index 033a1250fa5..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/frontMatterCodec/tokens/index.ts +++ /dev/null @@ -1,15 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -export { FrontMatterArray } from './frontMatterArray.js'; -export { FrontMatterString } from './frontMatterString.js'; -export { FrontMatterBoolean } from './frontMatterBoolean.js'; -export { FrontMatterToken, FrontMatterValueToken } from './frontMatterToken.js'; -export { - FrontMatterRecordName, - FrontMatterRecordDelimiter, - FrontMatterRecord, - type TNameToken as TRecordNameToken, -} from './frontMatterRecord.js'; diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/linesCodec/linesDecoder.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/linesCodec/linesDecoder.ts deleted file mode 100644 index 5d9ad249d6f..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/linesCodec/linesDecoder.ts +++ /dev/null @@ -1,237 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { Line } from './tokens/line.js'; -import { Range } from '../../../../../../../../editor/common/core/range.js'; -import { NewLine } from './tokens/newLine.js'; -import { assert } from '../../../../../../../../base/common/assert.js'; -import { CarriageReturn } from './tokens/carriageReturn.js'; -import { VSBuffer } from '../../../../../../../../base/common/buffer.js'; -import { assertDefined } from '../../../../../../../../base/common/types.js'; -import { BaseDecoder } from '../baseDecoder.js'; - -/** - * Any line break token type. - */ -export type TLineBreakToken = CarriageReturn | NewLine; - -/** - * Tokens produced by the {@link LinesDecoder}. - */ -export type TLineToken = Line | TLineBreakToken; - -/** - * The `decoder` part of the `LinesCodec` and is able to transform - * data from a binary stream into a stream of text lines(`Line`). - */ -export class LinesDecoder extends BaseDecoder { - /** - * Buffered received data yet to be processed. - */ - private buffer: VSBuffer = VSBuffer.alloc(0); - - /** - * The last emitted `Line` token, if any. The value is used - * to correctly emit remaining line range in the `onStreamEnd` - * method when underlying input stream ends and `buffer` still - * contains some data that must be emitted as the last line. - */ - private lastEmittedLine?: Line; - - /** - * Process data received from the input stream. - */ - protected override onStreamData(chunk: VSBuffer): void { - this.buffer = VSBuffer.concat([this.buffer, chunk]); - - this.processData(false); - } - - /** - * Process buffered data. - * - * @param streamEnded Flag that indicates if the input stream has ended, - * which means that is the last call of this method. - * @throws If internal logic implementation error is detected. - */ - private processData( - streamEnded: boolean, - ): void { - // iterate over each line of the data buffer, emitting each line - // as a `Line` token followed by a `NewLine` token, if applies - while (this.buffer.byteLength > 0) { - // get line number based on a previously emitted line, if any - const lineNumber = this.lastEmittedLine - ? this.lastEmittedLine.range.startLineNumber + 1 - : 1; - - // find the `\r`, `\n`, or `\r\n` tokens in the data - const endOfLineTokens = this.findEndOfLineTokens( - lineNumber, - streamEnded, - ); - const firstToken: (NewLine | CarriageReturn | undefined) = endOfLineTokens[0]; - - // if no end-of-the-line tokens found, stop the current processing - // attempt because we either (1) need more data to be received or - // (2) the stream has ended; in the case (2) remaining data must - // be emitted as the last line - if (firstToken === undefined) { - // (2) if `streamEnded`, we need to emit the whole remaining - // data as the last line immediately - if (streamEnded) { - this.emitLine(lineNumber, this.buffer.slice(0)); - } - - break; - } - - // emit the line found in the data as the `Line` token - this.emitLine(lineNumber, this.buffer.slice(0, firstToken.range.startColumn - 1)); - - // must always hold true as the `emitLine` above sets this - assertDefined( - this.lastEmittedLine, - 'No last emitted line found.', - ); - - // Note! A standalone `\r` token case is not a well-defined case, and - // was primarily used by old Mac OSx systems which treated it as - // a line ending (same as `\n`). Hence for backward compatibility - // with those systems, we treat it as a new line token as well. - // We do that by replacing standalone `\r` token with `\n` one. - if ((endOfLineTokens.length === 1) && (firstToken instanceof CarriageReturn)) { - endOfLineTokens.splice(0, 1, new NewLine(firstToken.range)); - } - - // emit the end-of-the-line tokens - let startColumn = this.lastEmittedLine.range.endColumn; - for (const token of endOfLineTokens) { - const byteLength = token.byte.byteLength; - const endColumn = startColumn + byteLength; - // emit the token updating its column start/end numbers based on - // the emitted line text length and previous end-of-the-line token - this._onData.fire(token.withRange({ startColumn, endColumn })); - // shorten the data buffer by the length of the token - this.buffer = this.buffer.slice(byteLength); - // update the start column for the next token - startColumn = endColumn; - } - } - - // if the stream has ended, assert that the input data buffer is now empty - // otherwise we have a logic error and leaving some buffered data behind - if (streamEnded) { - assert( - this.buffer.byteLength === 0, - 'Expected the input data buffer to be empty when the stream ends.', - ); - } - } - - /** - * Find the end of line tokens in the data buffer. - * Can return: - * - [`\r`, `\n`] tokens if the sequence is found - * - [`\r`] token if only the carriage return is found - * - [`\n`] token if only the newline is found - * - an `empty array` if no end of line tokens found - */ - private findEndOfLineTokens( - lineNumber: number, - streamEnded: boolean, - ): (CarriageReturn | NewLine)[] { - const result = []; - - // find the first occurrence of the carriage return and newline tokens - const carriageReturnIndex = this.buffer.indexOf(CarriageReturn.byte); - const newLineIndex = this.buffer.indexOf(NewLine.byte); - - // if the `\r` comes before the `\n`(if `\n` present at all) - if (carriageReturnIndex >= 0 && ((carriageReturnIndex < newLineIndex) || (newLineIndex === -1))) { - // add the carriage return token first - result.push( - new CarriageReturn(new Range( - lineNumber, - (carriageReturnIndex + 1), - lineNumber, - (carriageReturnIndex + 1) + CarriageReturn.byte.byteLength, - )), - ); - - // if the `\r\n` sequence - if (newLineIndex === carriageReturnIndex + 1) { - // add the newline token to the result - result.push( - new NewLine(new Range( - lineNumber, - (newLineIndex + 1), - lineNumber, - (newLineIndex + 1) + NewLine.byte.byteLength, - )), - ); - } - - // either `\r` or `\r\n` cases found; if we have the `\r` token, we can return - // the end-of-line tokens only, if the `\r` is followed by at least one more - // character (it could be a `\n` or any other character), or if the stream has - // ended (which means the `\r` is at the end of the line) - if ((this.buffer.byteLength > carriageReturnIndex + 1) || streamEnded) { - return result; - } - - // in all other cases, return the empty array (no lend-of-line tokens found) - return []; - } - - // no `\r`, but there is `\n` - if (newLineIndex >= 0) { - result.push( - new NewLine(new Range( - lineNumber, - (newLineIndex + 1), - lineNumber, - (newLineIndex + 1) + NewLine.byte.byteLength, - )), - ); - } - - // neither `\r` nor `\n` found, no end of line found at all - return result; - } - - /** - * Emit a provided line as the `Line` token to the output stream. - */ - private emitLine( - lineNumber: number, // Note! 1-based indexing - lineBytes: VSBuffer, - ): void { - - const line = new Line(lineNumber, lineBytes.toString()); - this._onData.fire(line); - - // store the last emitted line so we can use it when we need - // to send the remaining line in the `onStreamEnd` method - this.lastEmittedLine = line; - - // shorten the data buffer by the length of the line emitted - this.buffer = this.buffer.slice(lineBytes.byteLength); - } - - /** - * Handle the end of the input stream - if the buffer still has some data, - * emit it as the last available line token before firing the `onEnd` event. - */ - protected override onStreamEnd(): void { - // if the input data buffer is not empty when the input stream ends, emit - // the remaining data as the last line before firing the `onEnd` event - if (this.buffer.byteLength > 0) { - this.processData(true); - } - - super.onStreamEnd(); - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/linesCodec/tokens/carriageReturn.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/linesCodec/tokens/carriageReturn.ts deleted file mode 100644 index 46b3031829e..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/linesCodec/tokens/carriageReturn.ts +++ /dev/null @@ -1,44 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { VSBuffer } from '../../../../../../../../../base/common/buffer.js'; -import { SimpleToken } from '../../simpleCodec/tokens/simpleToken.js'; - -/** - * Token that represent a `carriage return` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class CarriageReturn extends SimpleToken<'\r'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '\r' = '\r'; - - /** - * The byte representation of the {@link symbol}. - */ - public static readonly byte = VSBuffer.fromString(CarriageReturn.symbol); - - /** - * The byte representation of the token. - */ - public get byte(): VSBuffer { - return CarriageReturn.byte; - } - - /** - * Return text representation of the token. - */ - public override get text(): '\r' { - return CarriageReturn.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `CR${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/linesCodec/tokens/line.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/linesCodec/tokens/line.ts deleted file mode 100644 index 2fa0afef6ff..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/linesCodec/tokens/line.ts +++ /dev/null @@ -1,48 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../../baseToken.js'; -import { assert } from '../../../../../../../../../base/common/assert.js'; -import { Range } from '../../../../../../../../../editor/common/core/range.js'; - -/** - * Token representing a line of text with a `range` which - * reflects the line's position in the original data. - */ -export class Line extends BaseToken { - constructor( - // the line index - // Note! 1-based indexing - lineNumber: number, - // the line contents - public readonly text: string, - ) { - assert( - !isNaN(lineNumber), - `The line number must not be a NaN.`, - ); - - assert( - lineNumber > 0, - `The line number must be >= 1, got "${lineNumber}".`, - ); - - super( - new Range( - lineNumber, - 1, - lineNumber, - text.length + 1, - ), - ); - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `line("${this.shortText()}")${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/linesCodec/tokens/newLine.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/linesCodec/tokens/newLine.ts deleted file mode 100644 index 6d95751721d..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/linesCodec/tokens/newLine.ts +++ /dev/null @@ -1,44 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { VSBuffer } from '../../../../../../../../../base/common/buffer.js'; -import { SimpleToken } from '../../simpleCodec/tokens/simpleToken.js'; - -/** - * A token that represent a `new line` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class NewLine extends SimpleToken<'\n'> { - /** - * The underlying symbol of the `NewLine` token. - */ - public static override readonly symbol: '\n' = '\n'; - - /** - * The byte representation of the {@link symbol}. - */ - public static readonly byte = VSBuffer.fromString(NewLine.symbol); - - /** - * Return text representation of the token. - */ - public override get text(): '\n' { - return NewLine.symbol; - } - - /** - * The byte representation of the token. - */ - public get byte(): VSBuffer { - return NewLine.byte; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `newline${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/markdownDecoder.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/markdownDecoder.ts deleted file mode 100644 index 67bbda1f2ec..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/markdownDecoder.ts +++ /dev/null @@ -1,136 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { MarkdownToken } from './tokens/markdownToken.js'; -import { VSBuffer } from '../../../../../../../../base/common/buffer.js'; -import { LeftBracket } from '../simpleCodec/tokens/brackets.js'; -import { PartialMarkdownImage } from './parsers/markdownImage.js'; -import { ReadableStream } from '../../../../../../../../base/common/stream.js'; -import { TSimpleDecoderToken } from '../simpleCodec/simpleDecoder.js'; -import { LeftAngleBracket } from '../simpleCodec/tokens/angleBrackets.js'; -import { ExclamationMark } from '../simpleCodec/tokens/exclamationMark.js'; -import { BaseDecoder } from '../baseDecoder.js'; -import { MarkdownCommentStart, PartialMarkdownCommentStart } from './parsers/markdownComment.js'; -import { MarkdownExtensionsDecoder } from '../markdownExtensionsCodec/markdownExtensionsDecoder.js'; -import { MarkdownLinkCaption, PartialMarkdownLink, PartialMarkdownLinkCaption } from './parsers/markdownLink.js'; - -/** - * Tokens produced by this decoder. - */ -export type TMarkdownToken = MarkdownToken | TSimpleDecoderToken; - -/** - * Decoder capable of parsing markdown entities (e.g., links) from a sequence of simple tokens. - */ -export class MarkdownDecoder extends BaseDecoder { - /** - * Current parser object that is responsible for parsing a sequence of tokens into - * some markdown entity. Set to `undefined` when no parsing is in progress at the moment. - */ - private current?: - PartialMarkdownLinkCaption | MarkdownLinkCaption | PartialMarkdownLink | - PartialMarkdownCommentStart | MarkdownCommentStart | - PartialMarkdownImage; - - constructor( - stream: ReadableStream, - ) { - super(new MarkdownExtensionsDecoder(stream)); - } - - protected override onStreamData(token: TSimpleDecoderToken): void { - // `markdown links` start with `[` character, so here we can - // initiate the process of parsing a markdown link - if (token instanceof LeftBracket && !this.current) { - this.current = new PartialMarkdownLinkCaption(token); - - return; - } - - // `markdown comments` start with `<` character, so here we can - // initiate the process of parsing a markdown comment - if (token instanceof LeftAngleBracket && !this.current) { - this.current = new PartialMarkdownCommentStart(token); - - return; - } - - // `markdown image links` start with `!` character, so here we can - // initiate the process of parsing a markdown image - if (token instanceof ExclamationMark && !this.current) { - this.current = new PartialMarkdownImage(token); - - return; - } - - // if current parser was not initiated before, - we are not inside a sequence - // of tokens we care about, therefore re-emit the token immediately and continue - if (!this.current) { - this._onData.fire(token); - return; - } - - // if there is a current parser object, submit the token to it - // so it can progress with parsing the tokens sequence - const parseResult = this.current.accept(token); - if (parseResult.result === 'success') { - const { nextParser } = parseResult; - - // if got a fully parsed out token back, emit it and reset - // the current parser object so a new parsing process can start - if (nextParser instanceof MarkdownToken) { - this._onData.fire(nextParser); - delete this.current; - } else { - // otherwise, update the current parser object - this.current = nextParser; - } - } else { - // if failed to parse a sequence of a tokens as a single markdown - // entity (e.g., a link), re-emit the tokens accumulated so far - // then reset the current parser object - for (const currentToken of this.current.tokens) { - this._onData.fire(currentToken); - } - - delete this.current; - } - - // if token was not consumed by the parser, call `onStreamData` again - // so the token is properly handled by the decoder in the case when a - // new sequence starts with this token - if (!parseResult.wasTokenConsumed) { - this.onStreamData(token); - } - } - - protected override onStreamEnd(): void { - // if the stream has ended and there is a current incomplete parser - // object present, handle the remaining parser object - if (this.current) { - // if a `markdown comment` does not have an end marker `-->` - // it is still a comment that extends to the end of the file - // so re-emit the current parser as a comment token - if (this.current instanceof MarkdownCommentStart) { - this._onData.fire(this.current.asMarkdownComment()); - delete this.current; - this.onStreamEnd(); - - return; - } - - // in all other cases, re-emit existing parser tokens - const { tokens } = this.current; - - for (const token of [...tokens]) { - this._onData.fire(token); - } - - delete this.current; - } - - super.onStreamEnd(); - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/parsers/markdownComment.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/parsers/markdownComment.ts deleted file mode 100644 index a017649f8b8..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/parsers/markdownComment.ts +++ /dev/null @@ -1,173 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { Range } from '../../../../../../../../../editor/common/core/range.js'; -import { BaseToken } from '../../baseToken.js'; -import { Dash } from '../../simpleCodec/tokens/dash.js'; -import { assert } from '../../../../../../../../../base/common/assert.js'; -import { MarkdownComment } from '../tokens/markdownComment.js'; -import { TSimpleDecoderToken } from '../../simpleCodec/simpleDecoder.js'; -import { ExclamationMark } from '../../simpleCodec/tokens/exclamationMark.js'; -import { LeftAngleBracket, RightAngleBracket } from '../../simpleCodec/tokens/angleBrackets.js'; -import { assertNotConsumed, ParserBase, TAcceptTokenResult } from '../../simpleCodec/parserBase.js'; - -/** - * The parser responsible for parsing the ``. If it does, - * then the parser transitions to the {@link MarkdownComment} token. - */ -export class MarkdownCommentStart extends ParserBase { - constructor(tokens: [LeftAngleBracket, ExclamationMark, Dash, Dash]) { - super(tokens); - } - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - // if received `>` while current token sequence ends with `--`, - // then this is the end of the comment sequence - if (token instanceof RightAngleBracket && this.endsWithDashes) { - this.currentTokens.push(token); - - return { - result: 'success', - nextParser: this.asMarkdownComment(), - wasTokenConsumed: true, - }; - } - - this.currentTokens.push(token); - - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } - - /** - * Convert the current token sequence into a {@link MarkdownComment} token. - * - * Note! that this method marks the current parser object as "consumed" - * hence it should not be used after this method is called. - */ - public asMarkdownComment(): MarkdownComment { - this.isConsumed = true; - - return new MarkdownComment( - this.range, - BaseToken.render(this.currentTokens), - ); - } - - /** - * Get range of current token sequence. - */ - private get range(): Range { - const firstToken = this.currentTokens[0]; - const lastToken = this.currentTokens[this.currentTokens.length - 1]; - - const range = new Range( - firstToken.range.startLineNumber, - firstToken.range.startColumn, - lastToken.range.endLineNumber, - lastToken.range.endColumn, - ); - - return range; - } - - /** - * Whether the current token sequence ends with two dashes. - */ - private get endsWithDashes(): boolean { - const lastToken = this.currentTokens[this.currentTokens.length - 1]; - if (!(lastToken instanceof Dash)) { - return false; - } - - const secondLastToken = this.currentTokens[this.currentTokens.length - 2]; - if (!(secondLastToken instanceof Dash)) { - return false; - } - - return true; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/parsers/markdownImage.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/parsers/markdownImage.ts deleted file mode 100644 index 5684f48de4a..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/parsers/markdownImage.ts +++ /dev/null @@ -1,99 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { MarkdownLink } from '../tokens/markdownLink.js'; -import { MarkdownImage } from '../tokens/markdownImage.js'; -import { LeftBracket } from '../../simpleCodec/tokens/brackets.js'; -import { TSimpleDecoderToken } from '../../simpleCodec/simpleDecoder.js'; -import { ExclamationMark } from '../../simpleCodec/tokens/exclamationMark.js'; -import { assertNotConsumed, ParserBase, TAcceptTokenResult } from '../../simpleCodec/parserBase.js'; -import { MarkdownLinkCaption, PartialMarkdownLink, PartialMarkdownLinkCaption } from './markdownLink.js'; - -/** - * The parser responsible for parsing the `markdown image` sequence of characters. - * E.g., `![alt text](./path/to/image.jpeg)` syntax. - */ -export class PartialMarkdownImage extends ParserBase { - /** - * Current active parser instance, if in the mode of actively parsing the markdown link sequence. - */ - private markdownLinkParser: PartialMarkdownLinkCaption | MarkdownLinkCaption | PartialMarkdownLink | undefined; - - constructor(token: ExclamationMark) { - super([token]); - } - - /** - * Get all currently available tokens of the `markdown link` sequence. - */ - public override get tokens(): readonly TSimpleDecoderToken[] { - const linkTokens = this.markdownLinkParser?.tokens ?? []; - - return [ - ...this.currentTokens, - ...linkTokens, - ]; - } - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - // on the first call we expect a character that begins `markdown link` sequence - // hence we initiate the markdown link parsing process, otherwise we fail - if (!this.markdownLinkParser) { - if (token instanceof LeftBracket) { - this.markdownLinkParser = new PartialMarkdownLinkCaption(token); - - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } - - return { - result: 'failure', - wasTokenConsumed: false, - }; - } - - // handle subsequent tokens next - - const acceptResult = this.markdownLinkParser.accept(token); - const { result, wasTokenConsumed } = acceptResult; - - if (result === 'success') { - const { nextParser } = acceptResult; - - // if full markdown link was parsed out, the process completes - if (nextParser instanceof MarkdownLink) { - this.isConsumed = true; - - const firstToken = this.currentTokens[0]; - return { - result, - wasTokenConsumed, - nextParser: new MarkdownImage( - firstToken.range.startLineNumber, - firstToken.range.startColumn, - `${firstToken.text}${nextParser.caption}`, - nextParser.reference, - ), - }; - } - - // otherwise save new link parser reference and continue - this.markdownLinkParser = nextParser; - return { - result, - wasTokenConsumed, - nextParser: this, - }; - } - - // return the failure result - this.isConsumed = true; - return acceptResult; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/parsers/markdownLink.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/parsers/markdownLink.ts deleted file mode 100644 index 641ddd14b9b..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/parsers/markdownLink.ts +++ /dev/null @@ -1,211 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../../baseToken.js'; -import { MarkdownLink } from '../tokens/markdownLink.js'; -import { NewLine } from '../../linesCodec/tokens/newLine.js'; -import { assert } from '../../../../../../../../../base/common/assert.js'; -import { FormFeed } from '../../simpleCodec/tokens/formFeed.js'; -import { VerticalTab } from '../../simpleCodec/tokens/verticalTab.js'; -import { TSimpleDecoderToken } from '../../simpleCodec/simpleDecoder.js'; -import { CarriageReturn } from '../../linesCodec/tokens/carriageReturn.js'; -import { LeftBracket, RightBracket } from '../../simpleCodec/tokens/brackets.js'; -import { ParserBase, TAcceptTokenResult } from '../../simpleCodec/parserBase.js'; -import { LeftParenthesis, RightParenthesis } from '../../simpleCodec/tokens/parentheses.js'; - -/** - * List of characters that are not allowed in links so stop a markdown link sequence abruptly. - */ -const MARKDOWN_LINK_STOP_CHARACTERS: readonly string[] = [CarriageReturn, NewLine, VerticalTab, FormFeed] - .map((token) => { return token.symbol; }); - -/** - * The parser responsible for parsing a `markdown link caption` part of a markdown - * link (e.g., the `[caption text]` part of the `[caption text](./some/path)` link). - * - * The parsing process starts with single `[` token and collects all tokens until - * the first `]` token is encountered. In this successful case, the parser transitions - * into the {@link MarkdownLinkCaption} parser type which continues the general - * parsing process of the markdown link. - * - * Otherwise, if one of the stop characters defined in the {@link MARKDOWN_LINK_STOP_CHARACTERS} - * is encountered before the `]` token, the parsing process is aborted which is communicated to - * the caller by returning a `failure` result. In this case, the caller is assumed to be responsible - * for re-emitting the {@link tokens} accumulated so far as standalone entities since they are no - * longer represent a coherent token entity of a larger size. - */ -export class PartialMarkdownLinkCaption extends ParserBase { - constructor(token: LeftBracket) { - super([token]); - } - - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - // any of stop characters is are breaking a markdown link caption sequence - if (MARKDOWN_LINK_STOP_CHARACTERS.includes(token.text)) { - return { - result: 'failure', - wasTokenConsumed: false, - }; - } - - // the `]` character ends the caption of a markdown link - if (token instanceof RightBracket) { - return { - result: 'success', - nextParser: new MarkdownLinkCaption([...this.tokens, token]), - wasTokenConsumed: true, - }; - } - - // otherwise, include the token in the sequence - // and keep the current parser object instance - this.currentTokens.push(token); - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } -} - -/** - * The parser responsible for transitioning from a {@link PartialMarkdownLinkCaption} - * parser to the {@link PartialMarkdownLink} one, therefore serves a parser glue between - * the `[caption]` and the `(./some/path)` parts of the `[caption](./some/path)` link. - * - * The only successful case of this parser is the `(` token that initiated the process - * of parsing the `reference` part of a markdown link and in this case the parser - * transitions into the `PartialMarkdownLink` parser type. - * - * Any other character is considered a failure result. In this case, the caller is assumed - * to be responsible for re-emitting the {@link tokens} accumulated so far as standalone - * entities since they are no longer represent a coherent token entity of a larger size. - */ -export class MarkdownLinkCaption extends ParserBase { - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - // the `(` character starts the link part of a markdown link - // that is the only character that can follow the caption - if (token instanceof LeftParenthesis) { - return { - result: 'success', - wasTokenConsumed: true, - nextParser: new PartialMarkdownLink([...this.tokens], token), - }; - } - - return { - result: 'failure', - wasTokenConsumed: false, - }; - } -} - -/** - * The parser responsible for parsing a `link reference` part of a markdown link - * (e.g., the `(./some/path)` part of the `[caption text](./some/path)` link). - * - * The parsing process starts with tokens that represent the `[caption]` part of a markdown - * link, followed by the `(` token. The parser collects all subsequent tokens until final closing - * parenthesis (`)`) is encountered (*\*see [1] below*). In this successful case, the parser object - * transitions into the {@link MarkdownLink} token type which signifies the end of the entire - * parsing process of the link text. - * - * Otherwise, if one of the stop characters defined in the {@link MARKDOWN_LINK_STOP_CHARACTERS} - * is encountered before the final `)` token, the parsing process is aborted which is communicated to - * the caller by returning a `failure` result. In this case, the caller is assumed to be responsible - * for re-emitting the {@link tokens} accumulated so far as standalone entities since they are no - * longer represent a coherent token entity of a larger size. - * - * `[1]` The `reference` part of the markdown link can contain any number of nested parenthesis, e.g., - * `[caption](/some/p(th/file.md)` is a valid markdown link and a valid folder name, hence number - * of open parenthesis must match the number of closing ones and the path sequence is considered - * to be complete as soon as this requirement is met. Therefore the `final` word is used in - * the description comments above to highlight this important detail. - */ -export class PartialMarkdownLink extends ParserBase { - /** - * Number of open parenthesis in the sequence. - * See comment in the {@link accept} method for more details. - */ - private openParensCount: number = 1; - - constructor( - protected readonly captionTokens: TSimpleDecoderToken[], - token: LeftParenthesis, - ) { - super([token]); - } - - public override get tokens(): readonly TSimpleDecoderToken[] { - return [...this.captionTokens, ...this.currentTokens]; - } - - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - // markdown links allow for nested parenthesis inside the link reference part, but - // the number of open parenthesis must match the number of closing parenthesis, e.g.: - // - `[caption](/some/p()th/file.md)` is a valid markdown link - // - `[caption](/some/p(th/file.md)` is an invalid markdown link - // hence we use the `openParensCount` variable to keep track of the number of open - // parenthesis encountered so far; then upon encountering a closing parenthesis we - // decrement the `openParensCount` and if it reaches 0 - we consider the link reference - // to be complete - - if (token instanceof LeftParenthesis) { - this.openParensCount += 1; - } - - if (token instanceof RightParenthesis) { - this.openParensCount -= 1; - - // sanity check! this must alway hold true because we return a complete markdown - // link as soon as we encounter matching number of closing parenthesis, hence - // we must never have `openParensCount` that is less than 0 - assert( - this.openParensCount >= 0, - `Unexpected right parenthesis token encountered: '${token}'.`, - ); - - // the markdown link is complete as soon as we get the same number of closing parenthesis - if (this.openParensCount === 0) { - const { startLineNumber, startColumn } = this.captionTokens[0].range; - - // create link caption string - const caption = BaseToken.render(this.captionTokens); - - // create link reference string - this.currentTokens.push(token); - const reference = BaseToken.render(this.currentTokens); - - // return complete markdown link object - return { - result: 'success', - wasTokenConsumed: true, - nextParser: new MarkdownLink( - startLineNumber, - startColumn, - caption, - reference, - ), - }; - } - } - - // any of stop characters is are breaking a markdown link reference sequence - if (MARKDOWN_LINK_STOP_CHARACTERS.includes(token.text)) { - return { - result: 'failure', - wasTokenConsumed: false, - }; - } - - // the rest of the tokens can be included in the sequence - this.currentTokens.push(token); - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/tokens/markdownComment.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/tokens/markdownComment.ts deleted file mode 100644 index 8eadff3589a..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/tokens/markdownComment.ts +++ /dev/null @@ -1,40 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { Range } from '../../../../../../../../../editor/common/core/range.js'; -import { MarkdownToken } from './markdownToken.js'; -import { assert } from '../../../../../../../../../base/common/assert.js'; - -/** - * A token that represent a `markdown comment` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class MarkdownComment extends MarkdownToken { - constructor( - range: Range, - public readonly text: string, - ) { - assert( - text.startsWith('`. - */ - public get hasEndMarker(): boolean { - return this.text.endsWith('-->'); - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `md-comment("${this.shortText()}")${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/tokens/markdownImage.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/tokens/markdownImage.ts deleted file mode 100644 index cab120f53db..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/tokens/markdownImage.ts +++ /dev/null @@ -1,125 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { MarkdownToken } from './markdownToken.js'; -import { IRange, Range } from '../../../../../../../../../editor/common/core/range.js'; -import { assert } from '../../../../../../../../../base/common/assert.js'; - -/** - * A token that represent a `markdown image` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class MarkdownImage extends MarkdownToken { - /** - * Check if this `markdown image link` points to a valid URL address. - */ - public readonly isURL: boolean; - - constructor( - /** - * The starting line number of the image (1-based indexing). - */ - lineNumber: number, - /** - * The starting column number of the image (1-based indexing). - */ - columnNumber: number, - /** - * The caption of the image, including the `!` and `square brackets`. - */ - private readonly caption: string, - /** - * The reference of the image, including the parentheses. - */ - private readonly reference: string, - ) { - assert( - !isNaN(lineNumber), - `The line number must not be a NaN.`, - ); - - assert( - lineNumber > 0, - `The line number must be >= 1, got "${lineNumber}".`, - ); - - assert( - columnNumber > 0, - `The column number must be >= 1, got "${columnNumber}".`, - ); - - assert( - caption[0] === '!', - `The caption must start with '!' character, got "${caption}".`, - ); - - assert( - caption[1] === '[' && caption[caption.length - 1] === ']', - `The caption must be enclosed in square brackets, got "${caption}".`, - ); - - assert( - reference[0] === '(' && reference[reference.length - 1] === ')', - `The reference must be enclosed in parentheses, got "${reference}".`, - ); - - super( - new Range( - lineNumber, - columnNumber, - lineNumber, - columnNumber + caption.length + reference.length, - ), - ); - - // set up the `isURL` flag based on the current - try { - new URL(this.path); - this.isURL = true; - } catch { - this.isURL = false; - } - } - - public override get text(): string { - return `${this.caption}${this.reference}`; - } - - /** - * Returns the `reference` part of the link without enclosing parentheses. - */ - public get path(): string { - return this.reference.slice(1, this.reference.length - 1); - } - - /** - * Get the range of the `link part` of the token. - */ - public get linkRange(): IRange | undefined { - if (this.path.length === 0) { - return undefined; - } - - const { range } = this; - - // note! '+1' for openning `(` of the link - const startColumn = range.startColumn + this.caption.length + 1; - const endColumn = startColumn + this.path.length; - - return new Range( - range.startLineNumber, - startColumn, - range.endLineNumber, - endColumn, - ); - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `md-image("${this.shortText()}")${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/tokens/markdownLink.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/tokens/markdownLink.ts deleted file mode 100644 index a963628675d..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/tokens/markdownLink.ts +++ /dev/null @@ -1,120 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { MarkdownToken } from './markdownToken.js'; -import { IRange, Range } from '../../../../../../../../../editor/common/core/range.js'; -import { assert } from '../../../../../../../../../base/common/assert.js'; - -/** - * A token that represent a `markdown link` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class MarkdownLink extends MarkdownToken { - /** - * Check if this `markdown link` points to a valid URL address. - */ - public readonly isURL: boolean; - - constructor( - /** - * The starting line number of the link (1-based indexing). - */ - lineNumber: number, - /** - * The starting column number of the link (1-based indexing). - */ - columnNumber: number, - /** - * The caption of the original link, including the square brackets. - */ - public readonly caption: string, - /** - * The reference of the original link, including the parentheses. - */ - public readonly reference: string, - ) { - assert( - !isNaN(lineNumber), - `The line number must not be a NaN.`, - ); - - assert( - lineNumber > 0, - `The line number must be >= 1, got "${lineNumber}".`, - ); - - assert( - columnNumber > 0, - `The column number must be >= 1, got "${columnNumber}".`, - ); - - assert( - caption[0] === '[' && caption[caption.length - 1] === ']', - `The caption must be enclosed in square brackets, got "${caption}".`, - ); - - assert( - reference[0] === '(' && reference[reference.length - 1] === ')', - `The reference must be enclosed in parentheses, got "${reference}".`, - ); - - super( - new Range( - lineNumber, - columnNumber, - lineNumber, - columnNumber + caption.length + reference.length, - ), - ); - - // set up the `isURL` flag based on the current - try { - new URL(this.path); - this.isURL = true; - } catch { - this.isURL = false; - } - } - - public override get text(): string { - return `${this.caption}${this.reference}`; - } - - /** - * Returns the `reference` part of the link without enclosing parentheses. - */ - public get path(): string { - return this.reference.slice(1, this.reference.length - 1); - } - - /** - * Get the range of the `link part` of the token. - */ - public get linkRange(): IRange | undefined { - if (this.path.length === 0) { - return undefined; - } - - const { range } = this; - - // note! '+1' for opening `(` of the link - const startColumn = range.startColumn + this.caption.length + 1; - const endColumn = startColumn + this.path.length; - - return new Range( - range.startLineNumber, - startColumn, - range.endLineNumber, - endColumn, - ); - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `md-link("${this.shortText()}")${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/tokens/markdownToken.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/tokens/markdownToken.ts deleted file mode 100644 index fc1935d081b..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownCodec/tokens/markdownToken.ts +++ /dev/null @@ -1,12 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../../baseToken.js'; - -/** - * Common base token that all `markdown` tokens should - * inherit from. - */ -export abstract class MarkdownToken extends BaseToken { } diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/markdownExtensionsDecoder.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/markdownExtensionsDecoder.ts deleted file mode 100644 index b5ff3d2b9f9..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/markdownExtensionsDecoder.ts +++ /dev/null @@ -1,119 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { VSBuffer } from '../../../../../../../../base/common/buffer.js'; -import { ReadableStream } from '../../../../../../../../base/common/stream.js'; -import { BaseDecoder } from '../baseDecoder.js'; -import { MarkdownExtensionsToken } from './tokens/markdownExtensionsToken.js'; -import { SimpleDecoder, TSimpleDecoderToken } from '../simpleCodec/simpleDecoder.js'; -import { PartialFrontMatterHeader, PartialFrontMatterStartMarker } from './parsers/frontMatterHeader.js'; - -/** - * Tokens produced by this decoder. - */ -export type TMarkdownExtensionsToken = MarkdownExtensionsToken | TSimpleDecoderToken; - -/** - * Decoder responsible for decoding extensions of markdown syntax, - * e.g., a `Front Matter` header, etc. - */ -export class MarkdownExtensionsDecoder extends BaseDecoder { - /** - * Current parser object that is responsible for parsing a sequence of tokens into - * some markdown entity. Set to `undefined` when no parsing is in progress at the moment. - */ - private current?: PartialFrontMatterStartMarker | PartialFrontMatterHeader; - - constructor( - stream: ReadableStream, - ) { - super(new SimpleDecoder(stream)); - } - - protected override onStreamData(token: TSimpleDecoderToken): void { - // front matter headers start with a `-` at the first column of the first line - if ((this.current === undefined) && PartialFrontMatterStartMarker.mayStartHeader(token)) { - this.current = new PartialFrontMatterStartMarker(token); - - return; - } - - // if current parser is not initiated, - we are not inside a sequence of tokens - // we care about, therefore re-emit the token immediately and continue - if (this.current === undefined) { - this._onData.fire(token); - return; - } - - // if there is a current parser object, submit the token to it - // so it can progress with parsing the tokens sequence - const parseResult = this.current.accept(token); - if (parseResult.result === 'success') { - const { nextParser } = parseResult; - - // if got a fully parsed out token back, emit it and reset - // the current parser object so a new parsing process can start - if (nextParser instanceof MarkdownExtensionsToken) { - this._onData.fire(nextParser); - delete this.current; - } else { - // otherwise, update the current parser object - this.current = nextParser; - } - } else { - // if failed to parse a sequence of a tokens as a single markdown - // entity (e.g., a link), re-emit the tokens accumulated so far - // then reset the currently initialized parser object - this.reEmitCurrentTokens(); - } - - // if token was not consumed by the parser, call `onStreamData` again - // so the token is properly handled by the decoder in the case when a - // new sequence starts with this token - if (!parseResult.wasTokenConsumed) { - this.onStreamData(token); - } - } - - protected override onStreamEnd(): void { - try { - if (this.current === undefined) { - return; - } - - // if current parser can be converted into a valid Front Matter - // header, then emit it and reset the current parser object - if (this.current instanceof PartialFrontMatterHeader) { - this._onData.fire( - this.current.asFrontMatterHeader(), - ); - delete this.current; - return; - } - - } catch { - // if failed to convert current parser object to a token, - // re-emit the tokens accumulated so far - this.reEmitCurrentTokens(); - } finally { - delete this.current; - super.onStreamEnd(); - } - } - - /** - * Re-emit tokens accumulated so far in the current parser object. - */ - protected reEmitCurrentTokens(): void { - if (this.current === undefined) { - return; - } - - for (const token of this.current.tokens) { - this._onData.fire(token); - } - delete this.current; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/parsers/frontMatterHeader.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/parsers/frontMatterHeader.ts deleted file mode 100644 index a63c911174c..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/parsers/frontMatterHeader.ts +++ /dev/null @@ -1,345 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { Dash } from '../../simpleCodec/tokens/dash.js'; -import { NewLine } from '../../linesCodec/tokens/newLine.js'; -import { FrontMatterHeader } from '../tokens/frontMatterHeader.js'; -import { assertDefined } from '../../../../../../../../../base/common/types.js'; -import { TSimpleDecoderToken } from '../../simpleCodec/simpleDecoder.js'; -import { assert, assertNever } from '../../../../../../../../../base/common/assert.js'; -import { CarriageReturn } from '../../linesCodec/tokens/carriageReturn.js'; -import { FrontMatterMarker, TMarkerToken } from '../tokens/frontMatterMarker.js'; -import { assertNotConsumed, IAcceptTokenSuccess, ParserBase, TAcceptTokenResult } from '../../simpleCodec/parserBase.js'; - -/** - * Parses the start marker of a Front Matter header. - */ -export class PartialFrontMatterStartMarker extends ParserBase { - constructor(token: Dash) { - const { range } = token; - - assert( - range.startLineNumber === 1, - `Front Matter header must start at the first line, but it starts at line #${range.startLineNumber}.`, - ); - - assert( - range.startColumn === 1, - `Front Matter header must start at the beginning of the line, but it starts at ${range.startColumn}.`, - ); - - super([token]); - } - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - const previousToken = this.currentTokens[this.currentTokens.length - 1]; - - // collect a sequence of dash tokens that may end with a CR token - if ((token instanceof Dash) || (token instanceof CarriageReturn)) { - // a dash or CR tokens can go only after another dash token - if ((previousToken instanceof Dash) === false) { - this.isConsumed = true; - - return { - result: 'failure', - wasTokenConsumed: false, - }; - } - - this.currentTokens.push(token); - - return { - result: 'success', - wasTokenConsumed: true, - nextParser: this, - }; - } - - // stop collecting dash tokens when a new line token is encountered - if (token instanceof NewLine) { - this.isConsumed = true; - - return { - result: 'success', - wasTokenConsumed: true, - nextParser: new PartialFrontMatterHeader( - FrontMatterMarker.fromTokens([ - ...this.currentTokens, - token, - ]), - ), - }; - } - - // any other token is invalid for the `start marker` - this.isConsumed = true; - return { - result: 'failure', - wasTokenConsumed: false, - }; - } - - /** - * Check if provided dash token can be a start of a Front Matter header. - */ - public static mayStartHeader(token: TSimpleDecoderToken): token is Dash { - return (token instanceof Dash) - && (token.range.startLineNumber === 1) - && (token.range.startColumn === 1); - } -} - -/** - * Parses a Front Matter header that already has a start marker - * and possibly some content that follows. - */ -export class PartialFrontMatterHeader extends ParserBase { - /** - * Parser instance for the end marker of the Front Matter header. - */ - private maybeEndMarker?: PartialFrontMatterEndMarker; - - constructor( - public readonly startMarker: FrontMatterMarker, - ) { - super([]); - } - - public override get tokens(): readonly TSimpleDecoderToken[] { - const endMarkerTokens = (this.maybeEndMarker !== undefined) - ? this.maybeEndMarker.tokens - : []; - - return [ - ...this.startMarker.tokens, - ...this.currentTokens, - ...endMarkerTokens, - ]; - } - - /** - * Convert the current token sequence into a {@link FrontMatterHeader} token. - * - * Note! that this method marks the current parser object as "consumed" - * hence it should not be used after this method is called. - */ - public asFrontMatterHeader(): FrontMatterHeader { - assertDefined( - this.maybeEndMarker, - 'Cannot convert to Front Matter header token without an end marker.', - ); - - assert( - this.maybeEndMarker.dashCount === this.startMarker.dashTokens.length, - [ - 'Start and end markers must have the same number of dashes', - `, got ${this.startMarker.dashTokens.length} / ${this.maybeEndMarker.dashCount}.`, - ].join(''), - ); - - this.isConsumed = true; - - return FrontMatterHeader.fromTokens( - this.startMarker.tokens, - this.currentTokens, - this.maybeEndMarker.tokens, - ); - } - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - // if in the mode of parsing the end marker sequence, forward - // the token to the current end marker parser instance - if (this.maybeEndMarker !== undefined) { - return this.acceptEndMarkerToken(token); - } - - // collect all tokens until a `dash token at the beginning of a line` is found - if (((token instanceof Dash) === false) || (token.range.startColumn !== 1)) { - this.currentTokens.push(token); - - return { - result: 'success', - wasTokenConsumed: true, - nextParser: this, - }; - } - - // a dash token at the beginning of the line might be a start of the `end marker` - // sequence of the front matter header, hence initialize appropriate parser object - assert( - this.maybeEndMarker === undefined, - `End marker parser must not be present.`, - ); - this.maybeEndMarker = new PartialFrontMatterEndMarker(token); - - return { - result: 'success', - wasTokenConsumed: true, - nextParser: this, - }; - } - - /** - * When a end marker parser is present, we pass all tokens to it - * until it is completes the parsing process(either success or failure). - */ - private acceptEndMarkerToken( - token: TSimpleDecoderToken, - ): TAcceptTokenResult { - assertDefined( - this.maybeEndMarker, - `Partial end marker parser must be initialized.`, - ); - - // if we have a partial end marker, we are in the process of parsing - // the end marker, so just pass the token to it and return - const acceptResult = this.maybeEndMarker.accept(token); - const { result, wasTokenConsumed } = acceptResult; - - if (result === 'success') { - const { nextParser } = acceptResult; - const endMarkerParsingComplete = (nextParser instanceof FrontMatterMarker); - - if (endMarkerParsingComplete === false) { - return { - result: 'success', - wasTokenConsumed, - nextParser: this, - }; - } - - const endMarker = nextParser; - - // start and end markers must have the same number of dashes, hence - // if they don't match, we would like to continue parsing the header - // until we find an end marker with the same number of dashes - if (endMarker.dashTokens.length !== this.startMarker.dashTokens.length) { - return this.handleEndMarkerParsingFailure( - endMarker.tokens, - wasTokenConsumed, - ); - } - - this.isConsumed = true; - return { - result: 'success', - wasTokenConsumed: true, - nextParser: FrontMatterHeader.fromTokens( - this.startMarker.tokens, - this.currentTokens, - this.maybeEndMarker.tokens, - ), - }; - } - - // if failed to parse the end marker, we would like to continue parsing - // the header until we find a valid end marker - if (result === 'failure') { - return this.handleEndMarkerParsingFailure( - this.maybeEndMarker.tokens, - wasTokenConsumed, - ); - } - - assertNever( - result, - `Unexpected result '${result}' while parsing the end marker.`, - ); - } - - /** - * On failure to parse the end marker, we need to continue parsing - * the header because there might be another valid end marker in - * the stream of tokens. Therefore we copy over the end marker tokens - * into the list of "content" tokens and reset the end marker parser. - */ - private handleEndMarkerParsingFailure( - tokens: readonly TSimpleDecoderToken[], - wasTokenConsumed: boolean, - ): IAcceptTokenSuccess { - this.currentTokens.push(...tokens); - delete this.maybeEndMarker; - - return { - result: 'success', - wasTokenConsumed, - nextParser: this, - }; - } -} - -/** - * Parser the end marker sequence of a Front Matter header. - */ -class PartialFrontMatterEndMarker extends ParserBase { - constructor(token: Dash) { - const { range } = token; - - assert( - range.startColumn === 1, - `Front Matter header must start at the beginning of the line, but it starts at ${range.startColumn}.`, - ); - - super([token]); - } - - /** - * Number of dashes in the marker. - */ - public get dashCount(): number { - return this.tokens - .filter((token) => { return token instanceof Dash; }) - .length; - } - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - const previousToken = this.currentTokens[this.currentTokens.length - 1]; - - // collect a sequence of dash tokens that may end with a CR token - if ((token instanceof Dash) || (token instanceof CarriageReturn)) { - // a dash or CR tokens can go only after another dash token - if ((previousToken instanceof Dash) === false) { - this.isConsumed = true; - - return { - result: 'failure', - wasTokenConsumed: false, - }; - } - - this.currentTokens.push(token); - - return { - result: 'success', - wasTokenConsumed: true, - nextParser: this, - }; - } - - // stop collecting dash tokens when a new line token is encountered - if (token instanceof NewLine) { - this.isConsumed = true; - this.currentTokens.push(token); - - return { - result: 'success', - wasTokenConsumed: true, - nextParser: FrontMatterMarker.fromTokens([ - ...this.currentTokens, - ]), - }; - } - - // any other token is invalid for the `start marker` - this.isConsumed = true; - return { - result: 'failure', - wasTokenConsumed: false, - }; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/tokens/frontMatterHeader.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/tokens/frontMatterHeader.ts deleted file mode 100644 index b5b3903b20a..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/tokens/frontMatterHeader.ts +++ /dev/null @@ -1,79 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { Text } from '../../textToken.js'; -import { Range } from '../../../../../../../../../editor/common/core/range.js'; -import { BaseToken } from '../../baseToken.js'; -import { MarkdownExtensionsToken } from './markdownExtensionsToken.js'; -import { TSimpleDecoderToken } from '../../simpleCodec/simpleDecoder.js'; -import { FrontMatterMarker, TMarkerToken } from './frontMatterMarker.js'; - -/** - * Token that represents a `Front Matter` header in a text. - */ -export class FrontMatterHeader extends MarkdownExtensionsToken { - constructor( - range: Range, - public readonly startMarker: FrontMatterMarker, - public readonly content: Text, - public readonly endMarker: FrontMatterMarker, - ) { - super(range); - } - - /** - * Return complete text representation of the token. - */ - public get text(): string { - const text: string[] = [ - this.startMarker.text, - this.content.text, - this.endMarker.text, - ]; - - return text.join(''); - } - - /** - * Range of the content of the Front Matter header. - */ - public get contentRange(): Range { - return this.content.range; - } - - /** - * Content token of the Front Matter header. - */ - public get contentToken(): Text { - return this.content; - } - - /** - * Create new instance of the token from the given tokens. - */ - public static fromTokens( - startMarkerTokens: readonly TMarkerToken[], - contentTokens: readonly TSimpleDecoderToken[], - endMarkerTokens: readonly TMarkerToken[], - ): FrontMatterHeader { - const range = BaseToken.fullRange( - [...startMarkerTokens, ...endMarkerTokens], - ); - - return new FrontMatterHeader( - range, - FrontMatterMarker.fromTokens(startMarkerTokens), - new Text(contentTokens), - FrontMatterMarker.fromTokens(endMarkerTokens), - ); - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `frontmatter("${this.shortText()}")${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/tokens/frontMatterMarker.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/tokens/frontMatterMarker.ts deleted file mode 100644 index 14813b3116b..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/tokens/frontMatterMarker.ts +++ /dev/null @@ -1,59 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { Range } from '../../../../../../../../../editor/common/core/range.js'; -import { BaseToken } from '../../baseToken.js'; -import { Dash } from '../../simpleCodec/tokens/dash.js'; -import { NewLine } from '../../linesCodec/tokens/newLine.js'; -import { MarkdownExtensionsToken } from './markdownExtensionsToken.js'; -import { CarriageReturn } from '../../linesCodec/tokens/carriageReturn.js'; - -/** - * Type for tokens inside a Front Matter header marker. - */ -export type TMarkerToken = Dash | CarriageReturn | NewLine; - -/** - * Marker for the start and end of a Front Matter header. - */ -export class FrontMatterMarker extends MarkdownExtensionsToken { - /** - * Returns complete text representation of the token. - */ - public get text(): string { - return BaseToken.render(this.tokens); - } - - /** - * List of {@link Dash} tokens in the marker. - */ - public get dashTokens(): readonly Dash[] { - return this.tokens - .filter((token) => { return token instanceof Dash; }); - } - - constructor( - range: Range, - public readonly tokens: readonly TMarkerToken[], - ) { - super(range); - } - - /** - * Create new instance of the token from a provided - * list of tokens. - */ - public static fromTokens( - tokens: readonly TMarkerToken[], - ): FrontMatterMarker { - const range = BaseToken.fullRange(tokens); - - return new FrontMatterMarker(range, tokens); - } - - public toString(): string { - return `frontmatter-marker(${this.dashTokens.length}:${this.range})`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/tokens/markdownExtensionsToken.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/tokens/markdownExtensionsToken.ts deleted file mode 100644 index 82046eb2b4d..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/markdownExtensionsCodec/tokens/markdownExtensionsToken.ts +++ /dev/null @@ -1,11 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { MarkdownToken } from '../../markdownCodec/tokens/markdownToken.js'; - -/** - * Base class for all tokens produced by the `MarkdownExtensionsDecoder`. - */ -export abstract class MarkdownExtensionsToken extends MarkdownToken { } diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/parserBase.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/parserBase.ts deleted file mode 100644 index 84515dc1d6c..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/parserBase.ts +++ /dev/null @@ -1,137 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../baseToken.js'; -import { assert } from '../../../../../../../../base/common/assert.js'; - -/** - * Common interface for a result of accepting a next token - * in a sequence. - */ -export interface IAcceptTokenResult { - /** - * The result type of accepting a next token in a sequence. - */ - result: 'success' | 'failure'; - - /** - * Whether the token to accept was consumed by the parser - * during the accept operation. - */ - wasTokenConsumed: boolean; -} - -/** - * Successful result of accepting a next token in a sequence. - */ -export interface IAcceptTokenSuccess extends IAcceptTokenResult { - result: 'success'; - nextParser: T; -} - -/** - * Failure result of accepting a next token in a sequence. - */ -export interface IAcceptTokenFailure extends IAcceptTokenResult { - result: 'failure'; -} - -/** - * The result of operation of accepting a next token in a sequence. - */ -export type TAcceptTokenResult = IAcceptTokenSuccess | IAcceptTokenFailure; - -/** - * An abstract parser class that is able to parse a sequence of - * tokens into a new single entity. - */ -export abstract class ParserBase { - /** - * Whether the parser object was "consumed" and should not be used anymore. - */ - protected isConsumed: boolean = false; - - /** - * Whether the parser object was "consumed" hence must not be used anymore. - */ - public get consumed(): boolean { - return this.isConsumed; - } - - /** - * Number of tokens at the initialization of the current parser. - */ - protected readonly startTokensCount: number; - - constructor( - /** - * Set of tokens that were accumulated so far. - */ - protected readonly currentTokens: TToken[] = [], - ) { - this.startTokensCount = this.currentTokens.length; - } - - /** - * Get the tokens that were accumulated so far. - */ - public get tokens(): readonly TToken[] { - return this.currentTokens; - } - - /** - * Accept a new token returning parsing result: - * - successful result must include the next parser object or a fully parsed out token - * - failure result must indicate that the token was not consumed - * - * @param token The token to accept. - * @returns The parsing result. - */ - public abstract accept(token: TToken): TAcceptTokenResult; - - /** - * A helper method that validates that the current parser object was not yet consumed, - * hence can still be used to accept new tokens in the parsing process. - * - * @throws if the parser object is already consumed. - */ - protected assertNotConsumed(): void { - assert( - this.isConsumed === false, - `The parser object is already consumed and should not be used anymore.`, - ); - } -} - -/** - * Decorator that validates that the current parser object was not yet consumed, - * hence can still be used to accept new tokens in the parsing process. - * - * @throws the resulting decorated method throws if the parser object was already consumed. - */ -export function assertNotConsumed>( - _target: T, - propertyKey: 'accept', - descriptor: PropertyDescriptor, -): PropertyDescriptor { - // store the original method reference - const originalMethod = descriptor.value; - - // validate that the current parser object was not yet consumed - // before invoking the original accept method - descriptor.value = function ( - this: T, - ...args: Parameters - ): ReturnType { - assert( - this.isConsumed === false, - `The parser object is already consumed and should not be used anymore.`, - ); - - return originalMethod.apply(this, args); - }; - - return descriptor; -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/simpleDecoder.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/simpleDecoder.ts deleted file mode 100644 index 9addf4df2ca..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/simpleDecoder.ts +++ /dev/null @@ -1,132 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { NewLine } from '../linesCodec/tokens/newLine.js'; -import { CarriageReturn } from '../linesCodec/tokens/carriageReturn.js'; -import { LinesDecoder, TLineBreakToken, TLineToken } from '../linesCodec/linesDecoder.js'; -import { - At, - Tab, - Word, - Hash, - Dash, - Colon, - Slash, - Space, - Quote, - Comma, - FormFeed, - DollarSign, - DoubleQuote, - VerticalTab, - type TBracket, - LeftBracket, - RightBracket, - type TCurlyBrace, - LeftCurlyBrace, - RightCurlyBrace, - ExclamationMark, - type TParenthesis, - LeftParenthesis, - RightParenthesis, - type TAngleBracket, - LeftAngleBracket, - RightAngleBracket, -} from './tokens/tokens.js'; -import { ISimpleTokenClass, SimpleToken } from './tokens/simpleToken.js'; -import { VSBuffer } from '../../../../../../../../base/common/buffer.js'; -import { BaseDecoder } from '../baseDecoder.js'; -import { ReadableStream } from '../../../../../../../../base/common/stream.js'; - -/** - * Type for all simple tokens. - */ -export type TSimpleToken = Space | Tab | VerticalTab | At | Quote | DoubleQuote - | CarriageReturn | NewLine | FormFeed | TBracket | TAngleBracket | TCurlyBrace - | TParenthesis | Colon | Hash | Dash | ExclamationMark | Slash | DollarSign | Comma - | TLineBreakToken; - -/** -* Type of tokens emitted by this decoder. -*/ -export type TSimpleDecoderToken = TSimpleToken | Word; - -/** - * List of well-known distinct tokens that this decoder emits (excluding - * the word stop characters defined below). Everything else is considered - * an arbitrary "text" sequence and is emitted as a single {@link Word} token. - */ -export const WELL_KNOWN_TOKENS: readonly ISimpleTokenClass[] = Object.freeze([ - LeftParenthesis, RightParenthesis, LeftBracket, RightBracket, LeftCurlyBrace, RightCurlyBrace, - LeftAngleBracket, RightAngleBracket, Space, Tab, VerticalTab, FormFeed, Colon, Hash, Dash, - ExclamationMark, At, Slash, DollarSign, Quote, DoubleQuote, Comma, -]); - -/** - * A {@link Word} sequence stops when one of the well-known tokens are encountered. - * Note! the `\r` and `\n` are excluded from the list because this decoder based on - * the {@link LinesDecoder} which emits {@link Line} tokens without them. - */ -const WORD_STOP_CHARACTERS: readonly string[] = Object.freeze( - WELL_KNOWN_TOKENS.map(token => token.symbol), -); - -/** - * A decoder that can decode a stream of `Line`s into a stream - * of simple token, - `Word`, `Space`, `Tab`, `NewLine`, etc. - */ -export class SimpleDecoder extends BaseDecoder { - constructor( - stream: ReadableStream, - ) { - super(new LinesDecoder(stream)); - } - - protected override onStreamData(line: TLineToken): void { - // re-emit new line tokens immediately - if (line instanceof CarriageReturn || line instanceof NewLine) { - this._onData.fire(line); - - return; - } - - // loop through the text separating it into `Word` and `well-known` tokens - const lineText = line.text.split(''); - let i = 0; - while (i < lineText.length) { - // index is 0-based, but column numbers are 1-based - const columnNumber = i + 1; - const character = lineText[i]; - - // check if the current character is a well-known token - const tokenConstructor = WELL_KNOWN_TOKENS - .find((wellKnownToken) => { - return wellKnownToken.symbol === character; - }); - - // if it is a well-known token, emit it and continue to the next one - if (tokenConstructor) { - this._onData.fire(SimpleToken.newOnLine(line, columnNumber, tokenConstructor)); - - i++; - continue; - } - - // otherwise, it is an arbitrary "text" sequence of characters, - // that needs to be collected into a single `Word` token, hence - // read all the characters until a stop character is encountered - let word = ''; - while (i < lineText.length && !(WORD_STOP_CHARACTERS.includes(lineText[i]))) { - word += lineText[i]; - i++; - } - - // emit a "text" sequence of characters as a single `Word` token - this._onData.fire( - Word.newOnLine(word, line, columnNumber), - ); - } - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/angleBrackets.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/angleBrackets.ts deleted file mode 100644 index d57acedc1c3..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/angleBrackets.ts +++ /dev/null @@ -1,61 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { SimpleToken } from './simpleToken.js'; - -/** - * A token that represent a `<` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class LeftAngleBracket extends SimpleToken<'<'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '<' = '<'; - - /** - * Return text representation of the token. - */ - public override get text(): '<' { - return LeftAngleBracket.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `left-angle-bracket${this.range}`; - } -} - -/** - * A token that represent a `>` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class RightAngleBracket extends SimpleToken<'>'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '>' = '>'; - - /** - * Return text representation of the token. - */ - public override get text(): '>' { - return RightAngleBracket.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `right-angle-bracket${this.range}`; - } -} - -/** - * General angle bracket token type. - */ -export type TAngleBracket = LeftAngleBracket | RightAngleBracket; diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/at.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/at.ts deleted file mode 100644 index 25c44433afd..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/at.ts +++ /dev/null @@ -1,31 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { SimpleToken } from './simpleToken.js'; - -/** - * A token that represent a `@` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class At extends SimpleToken<'@'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '@' = '@'; - - /** - * Return text representation of the token. - */ - public override get text(): '@' { - return At.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `at${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/brackets.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/brackets.ts deleted file mode 100644 index 491cb2a588c..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/brackets.ts +++ /dev/null @@ -1,61 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { SimpleToken } from './simpleToken.js'; - -/** - * A token that represent a `[` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class LeftBracket extends SimpleToken<'['> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '[' = '['; - - /** - * Return text representation of the token. - */ - public override get text(): '[' { - return LeftBracket.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `left-bracket${this.range}`; - } -} - -/** - * A token that represent a `]` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class RightBracket extends SimpleToken<']'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: ']' = ']'; - - /** - * Return text representation of the token. - */ - public override get text(): ']' { - return RightBracket.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `right-bracket${this.range}`; - } -} - -/** - * General bracket token type. - */ -export type TBracket = LeftBracket | RightBracket; diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/colon.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/colon.ts deleted file mode 100644 index a04b3e853df..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/colon.ts +++ /dev/null @@ -1,31 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { SimpleToken } from './simpleToken.js'; - -/** - * A token that represent a `:` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class Colon extends SimpleToken<':'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: ':' = ':'; - - /** - * Return text representation of the token. - */ - public override get text(): ':' { - return Colon.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `colon${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/comma.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/comma.ts deleted file mode 100644 index ce5df1748ca..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/comma.ts +++ /dev/null @@ -1,31 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { SimpleToken } from './simpleToken.js'; - -/** - * A token that represent a `,` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class Comma extends SimpleToken<','> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: ',' = ','; - - /** - * Return text representation of the token. - */ - public override get text(): ',' { - return Comma.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `comma${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/curlyBraces.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/curlyBraces.ts deleted file mode 100644 index 9c13e501d27..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/curlyBraces.ts +++ /dev/null @@ -1,61 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { SimpleToken } from './simpleToken.js'; - -/** - * A token that represent a `{` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class LeftCurlyBrace extends SimpleToken<'{'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '{' = '{'; - - /** - * Return text representation of the token. - */ - public override get text(): '{' { - return LeftCurlyBrace.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `left-curly-brace${this.range}`; - } -} - -/** - * A token that represent a `}` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class RightCurlyBrace extends SimpleToken<'}'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '}' = '}'; - - /** - * Return text representation of the token. - */ - public override get text(): '}' { - return RightCurlyBrace.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `right-curly-brace${this.range}`; - } -} - -/** - * General curly brace token type. - */ -export type TCurlyBrace = LeftCurlyBrace | RightCurlyBrace; diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/dash.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/dash.ts deleted file mode 100644 index a77b441ad26..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/dash.ts +++ /dev/null @@ -1,31 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { SimpleToken } from './simpleToken.js'; - -/** - * A token that represent a `-` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class Dash extends SimpleToken<'-'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '-' = '-'; - - /** - * Return text representation of the token. - */ - public override get text(): '-' { - return Dash.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `dash${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/dollarSign.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/dollarSign.ts deleted file mode 100644 index 5b81d0afda8..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/dollarSign.ts +++ /dev/null @@ -1,31 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { SimpleToken } from './simpleToken.js'; - -/** - * A token that represent a `$` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class DollarSign extends SimpleToken<'$'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '$' = '$'; - - /** - * Return text representation of the token. - */ - public override get text(): '$' { - return DollarSign.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `dollarSign${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/doubleQuote.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/doubleQuote.ts deleted file mode 100644 index ba041a03ed5..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/doubleQuote.ts +++ /dev/null @@ -1,40 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../../baseToken.js'; -import { SimpleToken } from './simpleToken.js'; - -/** - * A token that represent a `"` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class DoubleQuote extends SimpleToken<'"'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '"' = '"'; - - /** - * Return text representation of the token. - */ - public override get text(): '"' { - return DoubleQuote.symbol; - } - - /** - * Checks if the provided token is of the same type - * as the current one. - */ - public sameType(other: BaseToken): other is typeof this { - return (other instanceof this.constructor); - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `double-quote${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/exclamationMark.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/exclamationMark.ts deleted file mode 100644 index 32675fdf8a3..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/exclamationMark.ts +++ /dev/null @@ -1,31 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { SimpleToken } from './simpleToken.js'; - -/** - * A token that represent a `!` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class ExclamationMark extends SimpleToken<'!'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '!' = '!'; - - /** - * Return text representation of the token. - */ - public override get text(): '!' { - return ExclamationMark.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `exclamation-mark${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/formFeed.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/formFeed.ts deleted file mode 100644 index df5b8b0d446..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/formFeed.ts +++ /dev/null @@ -1,31 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { SimpleToken } from './simpleToken.js'; - -/** - * Token that represent a `form feed` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class FormFeed extends SimpleToken<'\f'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '\f' = '\f'; - - /** - * Return text representation of the token. - */ - public override get text(): '\f' { - return FormFeed.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `formfeed${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/hash.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/hash.ts deleted file mode 100644 index f499085859d..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/hash.ts +++ /dev/null @@ -1,31 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { SimpleToken } from './simpleToken.js'; - -/** - * A token that represent a `#` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class Hash extends SimpleToken<'#'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '#' = '#'; - - /** - * Return text representation of the token. - */ - public override get text(): '#' { - return Hash.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `hash${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/parentheses.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/parentheses.ts deleted file mode 100644 index b7ef4f5bc2a..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/parentheses.ts +++ /dev/null @@ -1,61 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { SimpleToken } from './simpleToken.js'; - -/** - * A token that represent a `(` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class LeftParenthesis extends SimpleToken<'('> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '(' = '('; - - /** - * Return text representation of the token. - */ - public override get text(): '(' { - return LeftParenthesis.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `left-parenthesis${this.range}`; - } -} - -/** - * A token that represent a `)` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class RightParenthesis extends SimpleToken<')'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: ')' = ')'; - - /** - * Return text representation of the token. - */ - public override get text(): ')' { - return RightParenthesis.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `right-parenthesis${this.range}`; - } -} - -/** - * General parenthesis token type. - */ -export type TParenthesis = LeftParenthesis | RightParenthesis; diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/quote.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/quote.ts deleted file mode 100644 index 3c262f67b49..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/quote.ts +++ /dev/null @@ -1,40 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../../baseToken.js'; -import { SimpleToken } from './simpleToken.js'; - -/** - * A token that represent a `'` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class Quote extends SimpleToken<`'`> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '\'' = '\''; - - /** - * Return text representation of the token. - */ - public override get text(): '\'' { - return Quote.symbol; - } - - /** - * Checks if the provided token is of the same type - * as the current one. - */ - public sameType(other: BaseToken): other is Quote { - return (other instanceof this.constructor); - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `quote${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/simpleToken.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/simpleToken.ts deleted file mode 100644 index 5a611bb1ddd..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/simpleToken.ts +++ /dev/null @@ -1,59 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { Range } from '../../../../../../../../../editor/common/core/range.js'; -import { BaseToken } from '../../baseToken.js'; -import { Line } from '../../linesCodec/tokens/line.js'; - -/** - * Interface for a class that can be instantiated into a {@link SimpleToken}. - */ -export interface ISimpleTokenClass> { - /** - * Character representing the token. - */ - readonly symbol: string; - - /** - * Constructor for the token. - */ - new(...args: any[]): TSimpleToken; -} - -/** - * Base class for all "simple" tokens with a `range`. - * A simple token is the one that represents a single character. - */ -export abstract class SimpleToken extends BaseToken { - /** - * The underlying symbol of the token. - */ - public static readonly symbol: string; - - /** - * Create new token instance with range inside - * the given `Line` at the given `column number`. - */ - public static newOnLine>( - line: Line, - atColumnNumber: number, - Constructor: ISimpleTokenClass, - ): TSimpleToken { - const { range } = line; - - return new Constructor(new Range( - range.startLineNumber, - atColumnNumber, - range.startLineNumber, - atColumnNumber + Constructor.symbol.length, - )); - } -} - -/** - * Base class for all tokens that represent some form of - * a spacing character, e.g. 'space', 'tab', etc. - */ -export abstract class SpacingToken extends SimpleToken { } diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/slash.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/slash.ts deleted file mode 100644 index 04d382bc89f..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/slash.ts +++ /dev/null @@ -1,31 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { SimpleToken } from './simpleToken.js'; - -/** - * A token that represent a `/` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class Slash extends SimpleToken<'/'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '/' = '/'; - - /** - * Return text representation of the token. - */ - public override get text(): '/' { - return Slash.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `slash${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/space.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/space.ts deleted file mode 100644 index 07eed9f58b2..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/space.ts +++ /dev/null @@ -1,31 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { SpacingToken } from './simpleToken.js'; - -/** - * A token that represent a `space` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class Space extends SpacingToken<' '> { - /** - * The underlying symbol of the `Space` token. - */ - public static override readonly symbol: ' ' = ' '; - - /** - * Return text representation of the token. - */ - public override get text(): ' ' { - return Space.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `space${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/tab.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/tab.ts deleted file mode 100644 index 718a01df813..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/tab.ts +++ /dev/null @@ -1,31 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { SpacingToken } from './simpleToken.js'; - -/** - * A token that represent a `tab` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class Tab extends SpacingToken<'\t'> { - /** - * The underlying symbol of the token. - */ - public static override readonly symbol: '\t' = '\t'; - - /** - * Return text representation of the token. - */ - public override get text(): '\t' { - return Tab.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `tab${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/tokens.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/tokens.ts deleted file mode 100644 index ee25fb83ddf..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/tokens.ts +++ /dev/null @@ -1,25 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -export { At } from './at.js'; -export { Tab } from './tab.js'; -export { Dash } from './dash.js'; -export { Hash } from './hash.js'; -export { Word } from './word.js'; -export { Colon } from './colon.js'; -export { Quote } from './quote.js'; -export { Slash } from './slash.js'; -export { Space } from './space.js'; -export { Comma } from './comma.js'; -export { FormFeed } from './formFeed.js'; -export { DollarSign } from './dollarSign.js'; -export { VerticalTab } from './verticalTab.js'; -export { DoubleQuote } from './doubleQuote.js'; -export { ExclamationMark } from './exclamationMark.js'; -export { SimpleToken, SpacingToken } from './simpleToken.js'; -export { type TBracket, LeftBracket, RightBracket } from './brackets.js'; -export { type TCurlyBrace, LeftCurlyBrace, RightCurlyBrace } from './curlyBraces.js'; -export { type TParenthesis, LeftParenthesis, RightParenthesis } from './parentheses.js'; -export { type TAngleBracket, LeftAngleBracket, RightAngleBracket } from './angleBrackets.js'; diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/verticalTab.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/verticalTab.ts deleted file mode 100644 index 4b7b4241433..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/verticalTab.ts +++ /dev/null @@ -1,31 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { SpacingToken } from './simpleToken.js'; - -/** - * Token that represent a `vertical tab` with a `range`. The `range` - * value reflects the position of the token in the original data. - */ -export class VerticalTab extends SpacingToken<'\v'> { - /** - * The underlying symbol of the `VerticalTab` token. - */ - public static override readonly symbol: '\v' = '\v'; - - /** - * Return text representation of the token. - */ - public override get text(): '\v' { - return VerticalTab.symbol; - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `vtab${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/word.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/word.ts deleted file mode 100644 index bea656895fc..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/simpleCodec/tokens/word.ts +++ /dev/null @@ -1,60 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { BaseToken } from '../../baseToken.js'; -import { Line } from '../../linesCodec/tokens/line.js'; -import { Range } from '../../../../../../../../../editor/common/core/range.js'; - -/** - * A token that represent a word - a set of continuous - * characters without stop characters, like a `space`, - * a `tab`, or a `new line`. - */ -export class Word extends BaseToken { - constructor( - /** - * The word range. - */ - range: Range, - - /** - * The string value of the word. - */ - public readonly text: TText, - ) { - super(range); - } - - /** - * Create new `Word` token with the given `text` and the range - * inside the given `Line` at the specified `column number`. - */ - public static newOnLine( - text: string, - line: Line | number, - atColumnNumber: number, - ): Word { - const startLineNumber = (typeof line === 'number') - ? line - : line.range.startLineNumber; - - const range = new Range( - startLineNumber, atColumnNumber, - startLineNumber, atColumnNumber + text.length - ); - - return new Word( - range, - text, - ); - } - - /** - * Returns a string representation of the token. - */ - public override toString(): string { - return `word("${this.shortText()}")${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/textToken.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/textToken.ts deleted file mode 100644 index ecb4c61c94e..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/textToken.ts +++ /dev/null @@ -1,19 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { type BaseToken } from './baseToken.js'; -import { CompositeToken } from './compositeToken.js'; - -/** - * Tokens that represent a sequence of tokens that does not - * hold an additional meaning in the text. - */ -export class Text< - TTokens extends readonly BaseToken[] = readonly BaseToken[], -> extends CompositeToken { - public override toString(): string { - return `text(${this.shortText()})${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/utils/objectStream.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/utils/objectStream.ts deleted file mode 100644 index 3c8476c2d56..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/utils/objectStream.ts +++ /dev/null @@ -1,224 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { assertNever } from '../../../../../../../../base/common/assert.js'; -import { CancellationToken } from '../../../../../../../../base/common/cancellation.js'; -import { ObservableDisposable } from '../../../utils/observableDisposable.js'; -import { newWriteableStream, ReadableStream, WriteableStream } from '../../../../../../../../base/common/stream.js'; - - -/** - * A readable stream of provided objects. - */ -export class ObjectStream extends ObservableDisposable implements ReadableStream { - /** - * Flag that indicates whether the stream has ended. - */ - private ended: boolean = false; - - /** - * Underlying writable stream instance. - */ - private readonly stream: WriteableStream; - - /** - * Interval reference that is used to periodically send - * objects to the stream in the background. - */ - private timeoutHandle: Timeout | undefined; - - constructor( - private readonly data: Generator, - private readonly cancellationToken?: CancellationToken, - ) { - super(); - - this.stream = newWriteableStream(null); - - if (cancellationToken?.isCancellationRequested) { - this.end(); - return; - } - - // send a first batch of data immediately - this.send(true); - } - - /** - * Starts process of sending data to the stream. - * - * @param stopAfterFirstSend whether to continue sending data to the stream - * or stop sending after the first batch of data is sent instead - */ - public send( - stopAfterFirstSend: boolean = false, - ): void { - // this method can be called asynchronously by the `setTimeout` utility below, hence - // the state of the cancellation token or the stream itself might have changed by that time - if (this.cancellationToken?.isCancellationRequested || this.ended) { - this.end(); - - return; - } - - this.sendData() - .then(() => { - if (this.cancellationToken?.isCancellationRequested || this.ended) { - this.end(); - - return; - } - - if (stopAfterFirstSend === true) { - this.stopStream(); - return; - } - - this.timeoutHandle = setTimeout(this.send.bind(this)); - }) - .catch((error) => { - this.stream.error(error); - this.dispose(); - }); - } - - /** - * Stop the data sending loop. - */ - public stopStream(): this { - if (this.timeoutHandle === undefined) { - return this; - } - - clearTimeout(this.timeoutHandle); - this.timeoutHandle = undefined; - - return this; - } - - /** - * Sends a provided number of objects to the stream. - */ - private async sendData( - objectsCount: number = 25, - ): Promise { - // send up to 'objectsCount' objects at a time - while (objectsCount > 0) { - try { - const next = this.data.next(); - if (next.done || this.cancellationToken?.isCancellationRequested) { - this.end(); - - return; - } - - await this.stream.write(next.value); - objectsCount--; - } catch (error) { - this.stream.error(error); - this.dispose(); - return; - } - } - } - - /** - * Ends the stream and stops sending data objects. - */ - private end(): this { - if (this.ended) { - return this; - } - this.ended = true; - - this.stopStream(); - this.stream.end(); - return this; - } - - public pause(): void { - this.stopStream(); - this.stream.pause(); - - return; - } - - public resume(): void { - this.send(); - this.stream.resume(); - - return; - } - - public destroy(): void { - this.dispose(); - } - - public removeListener(event: string, callback: (...args: any[]) => void): void { - this.stream.removeListener(event, callback); - - return; - } - - public on(event: 'data', callback: (data: T) => void): void; - public on(event: 'error', callback: (err: Error) => void): void; - public on(event: 'end', callback: () => void): void; - public on(event: 'data' | 'error' | 'end', callback: (...args: any[]) => void): void { - if (event === 'data') { - this.stream.on(event, callback); - // this is the convention of the readable stream, - when - // the `data` event is registered, the stream is started - this.send(); - - return; - } - - if (event === 'error') { - this.stream.on(event, callback); - return; - } - - if (event === 'end') { - this.stream.on(event, callback); - return; - } - - assertNever( - event, - `Unexpected event name '${event}'.`, - ); - } - - /** - * Cleanup send interval and destroy the stream. - */ - public override dispose(): void { - this.stopStream(); - this.stream.destroy(); - - super.dispose(); - } - - /** - * Create new instance of the stream from a provided array. - */ - public static fromArray( - array: T[], - cancellationToken?: CancellationToken, - ): ObjectStream { - return new ObjectStream(arrayToGenerator(array), cancellationToken); - } -} - -/** - * Create a generator out of a provided array. - */ -export function arrayToGenerator>(array: T[]): Generator { - return (function* (): Generator { - for (const item of array) { - yield item; - } - })(); -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/utils/objectStreamFromTextModel.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/utils/objectStreamFromTextModel.ts deleted file mode 100644 index 2a9b2173b5d..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/base/utils/objectStreamFromTextModel.ts +++ /dev/null @@ -1,46 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { ITextModel } from '../../../../../../../../editor/common/model.js'; -import { ObjectStream } from './objectStream.js'; -import { VSBuffer } from '../../../../../../../../base/common/buffer.js'; -import { CancellationToken } from '../../../../../../../../base/common/cancellation.js'; - -/** - * Create new instance of the stream from a provided text model. - */ -export function objectStreamFromTextModel( - model: ITextModel, - cancellationToken?: CancellationToken, -): ObjectStream { - return new ObjectStream(modelToGenerator(model), cancellationToken); -} - -/** - * Create a generator out of a provided text model. - */ -function modelToGenerator(model: ITextModel): Generator { - return (function* (): Generator { - const totalLines = model.getLineCount(); - let currentLine = 1; - - while (currentLine <= totalLines) { - if (model.isDisposed()) { - return undefined; - } - - yield VSBuffer.fromString( - model.getLineContent(currentLine), - ); - if (currentLine !== totalLines) { - yield VSBuffer.fromString( - model.getEOL(), - ); - } - - currentLine++; - } - })(); -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/chatPromptCodec.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/chatPromptCodec.ts deleted file mode 100644 index da27950bf92..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/chatPromptCodec.ts +++ /dev/null @@ -1,73 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { VSBuffer } from '../../../../../../base/common/buffer.js'; -import { ReadableStream } from '../../../../../../base/common/stream.js'; -import { ChatPromptDecoder, TChatPromptToken } from './chatPromptDecoder.js'; - -/** - * A codec is an object capable of encoding/decoding a stream of data transforming its messages. - * Useful for abstracting a data transfer or protocol logic on top of a stream of bytes. - * - * For instance, if protocol messages need to be transferred over `TCP` connection, a codec that - * encodes the messages into a sequence of bytes before sending it to a network socket. Likewise, - * on the other end of the connection, the same codec can decode the sequence of bytes back into - * a sequence of the protocol messages. - */ -export interface ICodec { - /** - * Encode a stream of `K`s into a stream of `T`s. - */ - encode: (value: ReadableStream) => ReadableStream; - - /** - * Decode a stream of `T`s into a stream of `K`s. - */ - decode: (value: ReadableStream) => ReadableStream; -} - - -/** - * `ChatPromptCodec` type is a `ICodec` with specific types for - * stream messages and return types of the `encode`/`decode` functions. - * @see {@link ICodec} - */ -interface IChatPromptCodec extends ICodec { - /** - * Decode a stream of `VSBuffer`s into a stream of `TChatPromptToken`s. - * - * @see {@link TChatPromptToken} - * @see {@link VSBuffer} - * @see {@link ChatPromptDecoder} - */ - decode: (value: ReadableStream) => ChatPromptDecoder; -} - -/** - * Codec that is capable to encode and decode tokens of an AI chatbot prompt message. - */ -export const ChatPromptCodec: IChatPromptCodec = Object.freeze({ - /** - * Encode a stream of `TChatPromptToken`s into a stream of `VSBuffer`s. - * - * @see {@link ReadableStream} - * @see {@link VSBuffer} - */ - encode: (_stream: ReadableStream): ReadableStream => { - throw new Error('The `encode` method is not implemented.'); - }, - - /** - * Decode a of `VSBuffer`s into a readable of `TChatPromptToken`s. - * - * @see {@link TChatPromptToken} - * @see {@link VSBuffer} - * @see {@link ChatPromptDecoder} - * @see {@link ReadableStream} - */ - decode: (stream: ReadableStream): ChatPromptDecoder => { - return new ChatPromptDecoder(stream); - }, -}); diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/chatPromptDecoder.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/chatPromptDecoder.ts deleted file mode 100644 index 5ac4a138fbe..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/chatPromptDecoder.ts +++ /dev/null @@ -1,202 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { PromptToken } from './tokens/promptToken.js'; -import { PromptAtMention } from './tokens/promptAtMention.js'; -import { VSBuffer } from '../../../../../../base/common/buffer.js'; -import { PromptSlashCommand } from './tokens/promptSlashCommand.js'; -import { ReadableStream } from '../../../../../../base/common/stream.js'; -import { PartialPromptAtMention } from './parsers/promptAtMentionParser.js'; -import { PromptTemplateVariable } from './tokens/promptTemplateVariable.js'; -import { assert, assertNever } from '../../../../../../base/common/assert.js'; -import { PartialPromptSlashCommand } from './parsers/promptSlashCommandParser.js'; -import { BaseDecoder } from './base/baseDecoder.js'; -import { PromptVariable, PromptVariableWithData } from './tokens/promptVariable.js'; -import { At } from './base/simpleCodec/tokens/at.js'; -import { Hash } from './base/simpleCodec/tokens/hash.js'; -import { Slash } from './base/simpleCodec/tokens/slash.js'; -import { DollarSign } from './base/simpleCodec/tokens/dollarSign.js'; -import { PartialPromptVariableName, PartialPromptVariableWithData } from './parsers/promptVariableParser.js'; -import { MarkdownDecoder, TMarkdownToken } from './base/markdownCodec/markdownDecoder.js'; -import { PartialPromptTemplateVariable, PartialPromptTemplateVariableStart, TPromptTemplateVariableParser } from './parsers/promptTemplateVariableParser.js'; - -/** - * Tokens produced by this decoder. - */ -export type TChatPromptToken = TMarkdownToken | (PromptVariable | PromptVariableWithData) - | PromptAtMention | PromptSlashCommand | PromptTemplateVariable; - -/** - * Decoder for the common chatbot prompt message syntax. - * For instance, the file references `#file:./path/file.md` are handled by this decoder. - */ -export class ChatPromptDecoder extends BaseDecoder { - /** - * Currently active parser object that is used to parse a well-known sequence of - * tokens, for instance, a `#file:/path/to/file.md` link that consists of `hash`, - * `word`, and `colon` tokens sequence plus the `file path` part that follows. - */ - private current?: (PartialPromptVariableName | PartialPromptVariableWithData) - | PartialPromptAtMention | PartialPromptSlashCommand - | TPromptTemplateVariableParser; - - constructor( - stream: ReadableStream, - ) { - super(new MarkdownDecoder(stream)); - } - - protected override onStreamData(token: TMarkdownToken): void { - // prompt `#variables` always start with the `#` character, hence - // initiate a parser object if we encounter respective token and - // there is no active parser object present at the moment - if ((token instanceof Hash) && !this.current) { - this.current = new PartialPromptVariableName(token); - - return; - } - - // prompt `@mentions` always start with the `@` character, hence - // initiate a parser object if we encounter respective token and - // there is no active parser object present at the moment - if ((token instanceof At) && !this.current) { - this.current = new PartialPromptAtMention(token); - - return; - } - - // prompt `/commands` always start with the `/` character, hence - // initiate a parser object if we encounter respective token and - // there is no active parser object present at the moment - if ((token instanceof Slash) && !this.current) { - this.current = new PartialPromptSlashCommand(token); - - return; - } - - // prompt `${template:variables}` always start with the `$` character, - // hence initiate a parser object if we encounter respective token and - // there is no active parser object present at the moment - if ((token instanceof DollarSign) && !this.current) { - this.current = new PartialPromptTemplateVariableStart(token); - - return; - } - - // if current parser was not yet initiated, - we are in the general "text" - // parsing mode, therefore re-emit the token immediately and continue - if (!this.current) { - this._onData.fire(token); - return; - } - - // if there is a current parser object, submit the token to it - // so it can progress with parsing the tokens sequence - const parseResult = this.current.accept(token); - - // process the parse result next - switch (parseResult.result) { - // in the case of success there might be 2 cases: - // 1) parsing fully completed and an instance of `PromptToken` is returned back, - // in this case, emit the parsed token (e.g., a `link`) and reset the current - // parser object reference so a new parsing process can be initiated next - // 2) parsing is still in progress and the next parser object is returned, hence - // we need to replace the current parser object with a new one and continue - case 'success': { - const { nextParser } = parseResult; - - if (nextParser instanceof PromptToken) { - this._onData.fire(nextParser); - delete this.current; - } else { - this.current = nextParser; - } - - break; - } - // in the case of failure, reset the current parser object - case 'failure': { - // if failed to parse a sequence of a tokens, re-emit the tokens accumulated - // so far then reset the current parser object - this.reEmitCurrentTokens(); - break; - } - } - - // if token was not consumed by the parser, call `onStreamData` again - // so the token is properly handled by the decoder in the case when a - // new sequence starts with this token - if (!parseResult.wasTokenConsumed) { - this.onStreamData(token); - } - } - - protected override onStreamEnd(): void { - try { - // if there is no currently active parser object present, nothing to do - if (this.current === undefined) { - return; - } - - // otherwise try to convert unfinished parser object to a token - - if (this.current instanceof PartialPromptVariableName) { - this._onData.fire(this.current.asPromptVariable()); - return; - } - - if (this.current instanceof PartialPromptVariableWithData) { - this._onData.fire(this.current.asPromptVariableWithData()); - return; - } - - if (this.current instanceof PartialPromptAtMention) { - this._onData.fire(this.current.asPromptAtMention()); - return; - } - - if (this.current instanceof PartialPromptSlashCommand) { - this._onData.fire(this.current.asPromptSlashCommand()); - return; - } - - assert( - (this.current instanceof PartialPromptTemplateVariableStart) === false, - 'Incomplete template variable token.', - ); - - if (this.current instanceof PartialPromptTemplateVariable) { - this._onData.fire(this.current.asPromptTemplateVariable()); - return; - } - - assertNever( - this.current, - `Unknown parser object '${this.current}'`, - ); - } catch (_error) { - // if failed to convert current parser object to a token, - // re-emit the tokens accumulated so far - this.reEmitCurrentTokens(); - } finally { - delete this.current; - super.onStreamEnd(); - } - } - - /** - * Re-emit tokens accumulated so far in the current parser object. - */ - protected reEmitCurrentTokens(): void { - if (this.current === undefined) { - return; - } - - for (const token of this.current.tokens) { - this._onData.fire(token); - } - delete this.current; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/parsers/promptAtMentionParser.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/parsers/promptAtMentionParser.ts deleted file mode 100644 index 2173ba438ad..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/parsers/promptAtMentionParser.ts +++ /dev/null @@ -1,121 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { PromptAtMention } from '../tokens/promptAtMention.js'; -import { assert } from '../../../../../../../base/common/assert.js'; -import { Range } from '../../../../../../../editor/common/core/range.js'; -import { BaseToken } from '../base/baseToken.js'; -import { At } from '../base/simpleCodec/tokens/at.js'; -import { Tab } from '../base/simpleCodec/tokens/tab.js'; -import { Hash } from '../base/simpleCodec/tokens/hash.js'; -import { Space } from '../base/simpleCodec/tokens/space.js'; -import { Colon } from '../base/simpleCodec/tokens/colon.js'; -import { NewLine } from '../base/linesCodec/tokens/newLine.js'; -import { FormFeed } from '../base/simpleCodec/tokens/formFeed.js'; -import { VerticalTab } from '../base/simpleCodec/tokens/verticalTab.js'; -import { TSimpleDecoderToken } from '../base/simpleCodec/simpleDecoder.js'; -import { CarriageReturn } from '../base/linesCodec/tokens/carriageReturn.js'; -import { ExclamationMark } from '../base/simpleCodec/tokens/exclamationMark.js'; -import { LeftBracket, RightBracket } from '../base/simpleCodec/tokens/brackets.js'; -import { LeftAngleBracket, RightAngleBracket } from '../base/simpleCodec/tokens/angleBrackets.js'; -import { assertNotConsumed, ParserBase, TAcceptTokenResult } from '../base/simpleCodec/parserBase.js'; - -/** - * List of characters that terminate the prompt at-mention sequence. - */ -export const STOP_CHARACTERS: readonly string[] = [Space, Tab, NewLine, CarriageReturn, VerticalTab, FormFeed, At, Colon, Hash] - .map((token) => { return token.symbol; }); - -/** - * List of characters that cannot be in an at-mention name (excluding the {@link STOP_CHARACTERS}). - */ -export const INVALID_NAME_CHARACTERS: readonly string[] = [ExclamationMark, LeftAngleBracket, RightAngleBracket, LeftBracket, RightBracket] - .map((token) => { return token.symbol; }); - -/** - * The parser responsible for parsing a `prompt @mention` sequences. - * E.g., `@workspace` or `@github` participant mention. - */ -export class PartialPromptAtMention extends ParserBase { - constructor(token: At) { - super([token]); - } - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - // if a `stop` character is encountered, finish the parsing process - if (STOP_CHARACTERS.includes(token.text)) { - try { - // if it is possible to convert current parser to `PromptAtMention`, return success result - return { - result: 'success', - nextParser: this.asPromptAtMention(), - wasTokenConsumed: false, - }; - } catch (error) { - // otherwise fail - return { - result: 'failure', - wasTokenConsumed: false, - }; - } finally { - // in any case this is an end of the parsing process - this.isConsumed = true; - } - } - - // variables cannot have {@link INVALID_NAME_CHARACTERS} in their names - if (INVALID_NAME_CHARACTERS.includes(token.text)) { - this.isConsumed = true; - - return { - result: 'failure', - wasTokenConsumed: false, - }; - } - - // otherwise it is a valid name character, so add it to the list of - // the current tokens and continue the parsing process - this.currentTokens.push(token); - - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } - - /** - * Try to convert current parser instance into a fully-parsed {@link PromptAtMention} token. - * - * @throws if sequence of tokens received so far do not constitute a valid prompt variable, - * for instance, if there is only `1` starting `@` token is available. - */ - public asPromptAtMention(): PromptAtMention { - // if there is only one token before the stop character - // must be the starting `@` one), then fail - assert( - this.currentTokens.length > 1, - 'Cannot create a prompt @mention out of incomplete token sequence.', - ); - - const firstToken = this.currentTokens[0]; - const lastToken = this.currentTokens[this.currentTokens.length - 1]; - - // render the characters above into strings, excluding the starting `@` character - const nameTokens = this.currentTokens.slice(1); - const atMentionName = BaseToken.render(nameTokens); - - return new PromptAtMention( - new Range( - firstToken.range.startLineNumber, - firstToken.range.startColumn, - lastToken.range.endLineNumber, - lastToken.range.endColumn, - ), - atMentionName, - ); - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/parsers/promptSlashCommandParser.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/parsers/promptSlashCommandParser.ts deleted file mode 100644 index 557f5e74379..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/parsers/promptSlashCommandParser.ts +++ /dev/null @@ -1,122 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { assert } from '../../../../../../../base/common/assert.js'; -import { PromptSlashCommand } from '../tokens/promptSlashCommand.js'; -import { Range } from '../../../../../../../editor/common/core/range.js'; -import { BaseToken } from '../base/baseToken.js'; -import { At } from '../base/simpleCodec/tokens/at.js'; -import { Tab } from '../base/simpleCodec/tokens/tab.js'; -import { Hash } from '../base/simpleCodec/tokens/hash.js'; -import { Slash } from '../base/simpleCodec/tokens/slash.js'; -import { Space } from '../base/simpleCodec/tokens/space.js'; -import { Colon } from '../base/simpleCodec/tokens/colon.js'; -import { NewLine } from '../base/linesCodec/tokens/newLine.js'; -import { FormFeed } from '../base/simpleCodec/tokens/formFeed.js'; -import { VerticalTab } from '../base/simpleCodec/tokens/verticalTab.js'; -import { TSimpleDecoderToken } from '../base/simpleCodec/simpleDecoder.js'; -import { CarriageReturn } from '../base/linesCodec/tokens/carriageReturn.js'; -import { ExclamationMark } from '../base/simpleCodec/tokens/exclamationMark.js'; -import { LeftBracket, RightBracket } from '../base/simpleCodec/tokens/brackets.js'; -import { LeftAngleBracket, RightAngleBracket } from '../base/simpleCodec/tokens/angleBrackets.js'; -import { assertNotConsumed, ParserBase, TAcceptTokenResult } from '../base/simpleCodec/parserBase.js'; - -/** - * List of characters that terminate the prompt at-mention sequence. - */ -export const STOP_CHARACTERS: readonly string[] = [Space, Tab, NewLine, CarriageReturn, VerticalTab, FormFeed, Colon, At, Hash, Slash] - .map((token) => { return token.symbol; }); - -/** - * List of characters that cannot be in an at-mention name (excluding the {@link STOP_CHARACTERS}). - */ -export const INVALID_NAME_CHARACTERS: readonly string[] = [ExclamationMark, LeftAngleBracket, RightAngleBracket, LeftBracket, RightBracket] - .map((token) => { return token.symbol; }); - -/** - * The parser responsible for parsing a `prompt /command` sequences. - * E.g., `/search` or `/explain` command. - */ -export class PartialPromptSlashCommand extends ParserBase { - constructor(token: Slash) { - super([token]); - } - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - // if a `stop` character is encountered, finish the parsing process - if (STOP_CHARACTERS.includes(token.text)) { - try { - // if it is possible to convert current parser to `PromptSlashCommand`, return success result - return { - result: 'success', - nextParser: this.asPromptSlashCommand(), - wasTokenConsumed: false, - }; - } catch (error) { - // otherwise fail - return { - result: 'failure', - wasTokenConsumed: false, - }; - } finally { - // in any case this is an end of the parsing process - this.isConsumed = true; - } - } - - // variables cannot have {@link INVALID_NAME_CHARACTERS} in their names - if (INVALID_NAME_CHARACTERS.includes(token.text)) { - this.isConsumed = true; - - return { - result: 'failure', - wasTokenConsumed: false, - }; - } - - // otherwise it is a valid name character, so add it to the list of - // the current tokens and continue the parsing process - this.currentTokens.push(token); - - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } - - /** - * Try to convert current parser instance into a fully-parsed {@link PromptSlashCommand} token. - * - * @throws if sequence of tokens received so far do not constitute a valid prompt variable, - * for instance, if there is only `1` starting `/` token is available. - */ - public asPromptSlashCommand(): PromptSlashCommand { - // if there is only one token before the stop character - // must be the starting `/` one), then fail - assert( - this.currentTokens.length > 1, - 'Cannot create a prompt /command out of incomplete token sequence.', - ); - - const firstToken = this.currentTokens[0]; - const lastToken = this.currentTokens[this.currentTokens.length - 1]; - - // render the characters above into strings, excluding the starting `/` character - const nameTokens = this.currentTokens.slice(1); - const atMentionName = BaseToken.render(nameTokens); - - return new PromptSlashCommand( - new Range( - firstToken.range.startLineNumber, - firstToken.range.startColumn, - lastToken.range.endLineNumber, - lastToken.range.endColumn, - ), - atMentionName, - ); - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/parsers/promptTemplateVariableParser.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/parsers/promptTemplateVariableParser.ts deleted file mode 100644 index b97ad0fd479..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/parsers/promptTemplateVariableParser.ts +++ /dev/null @@ -1,148 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { assert } from '../../../../../../../base/common/assert.js'; -import { PromptTemplateVariable } from '../tokens/promptTemplateVariable.js'; -import { BaseToken } from '../base/baseToken.js'; -import { TSimpleDecoderToken } from '../base/simpleCodec/simpleDecoder.js'; -import { DollarSign, LeftCurlyBrace, RightCurlyBrace } from '../base/simpleCodec/tokens/tokens.js'; -import { assertNotConsumed, ParserBase, TAcceptTokenResult } from '../base/simpleCodec/parserBase.js'; - -/** - * Parsers of the `${variable}` token sequence in a prompt text. - */ -export type TPromptTemplateVariableParser = PartialPromptTemplateVariableStart | PartialPromptTemplateVariable; - -/** - * Parser that handles start sequence of a `${variable}` token sequence in - * a prompt text. Transitions to {@link PartialPromptTemplateVariable} parser - * as soon as the `${` character sequence is found. - */ -export class PartialPromptTemplateVariableStart extends ParserBase { - constructor(token: DollarSign) { - super([token]); - } - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - if (token instanceof LeftCurlyBrace) { - this.currentTokens.push(token); - - this.isConsumed = true; - return { - result: 'success', - nextParser: new PartialPromptTemplateVariable(this.currentTokens), - wasTokenConsumed: true, - }; - } - - return { - result: 'failure', - wasTokenConsumed: false, - }; - } -} - -/** - * Parser that handles a partial `${variable}` token sequence in a prompt text. - */ -export class PartialPromptTemplateVariable extends ParserBase { - constructor(tokens: (DollarSign | LeftCurlyBrace)[]) { - super(tokens); - } - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - // template variables are terminated by the `}` character - if (token instanceof RightCurlyBrace) { - this.currentTokens.push(token); - - this.isConsumed = true; - return { - result: 'success', - nextParser: this.asPromptTemplateVariable(), - wasTokenConsumed: true, - }; - } - - // otherwise it is a valid name character, so add it to the list of - // the current tokens and continue the parsing process - this.currentTokens.push(token); - - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } - - /** - * Returns a string representation of the prompt template variable - * contents, if any is present. - */ - private get contents(): string { - const contentTokens: TSimpleDecoderToken[] = []; - - // template variables are surrounded by `${}`, hence we need to have - // at least `${` plus one character for the contents to be non-empty - if (this.currentTokens.length < 3) { - return ''; - } - - // collect all tokens besides the first two (`${`) and a possible `}` at the end - for (let i = 2; i < this.currentTokens.length; i++) { - const token = this.currentTokens[i]; - const isLastToken = (i === this.currentTokens.length - 1); - - if ((token instanceof RightCurlyBrace) && (isLastToken === true)) { - break; - } - - contentTokens.push(token); - } - - return BaseToken.render(contentTokens); - } - - /** - * Try to convert current parser instance into a {@link PromptTemplateVariable} token. - * - * @throws if: - * - current tokens sequence cannot be converted to a valid template variable token - */ - public asPromptTemplateVariable(): PromptTemplateVariable { - const firstToken = this.currentTokens[0]; - const secondToken = this.currentTokens[1]; - const lastToken = this.currentTokens[this.currentTokens.length - 1]; - - // template variables are surrounded by `${}`, hence we need - // to have at least 3 tokens in the list for a valid one - assert( - this.currentTokens.length >= 3, - 'Prompt template variable should have at least 3 tokens.', - ); - - // a complete template variable must end with a `}` - assert( - lastToken instanceof RightCurlyBrace, - 'Last token is not a "}".', - ); - - // sanity checks of the first and second tokens - assert( - firstToken instanceof DollarSign, - 'First token must be a "$".', - ); - assert( - secondToken instanceof LeftCurlyBrace, - 'Second token must be a "{".', - ); - - return new PromptTemplateVariable( - BaseToken.fullRange(this.currentTokens), - this.contents, - ); - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/parsers/promptVariableParser.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/parsers/promptVariableParser.ts deleted file mode 100644 index ef68e78ec2d..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/parsers/promptVariableParser.ts +++ /dev/null @@ -1,252 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { assert } from '../../../../../../../base/common/assert.js'; -import { Range } from '../../../../../../../editor/common/core/range.js'; -import { BaseToken } from '../base/baseToken.js'; -import { PromptVariable, PromptVariableWithData } from '../tokens/promptVariable.js'; -import { At } from '../base/simpleCodec/tokens/at.js'; -import { Tab } from '../base/simpleCodec/tokens/tab.js'; -import { Hash } from '../base/simpleCodec/tokens/hash.js'; -import { Space } from '../base/simpleCodec/tokens/space.js'; -import { Colon } from '../base/simpleCodec/tokens/colon.js'; -import { NewLine } from '../base/linesCodec/tokens/newLine.js'; -import { FormFeed } from '../base/simpleCodec/tokens/formFeed.js'; -import { VerticalTab } from '../base/simpleCodec/tokens/verticalTab.js'; -import { TSimpleDecoderToken } from '../base/simpleCodec/simpleDecoder.js'; -import { CarriageReturn } from '../base/linesCodec/tokens/carriageReturn.js'; -import { ExclamationMark } from '../base/simpleCodec/tokens/exclamationMark.js'; -import { LeftBracket, RightBracket } from '../base/simpleCodec/tokens/brackets.js'; -import { LeftAngleBracket, RightAngleBracket } from '../base/simpleCodec/tokens/angleBrackets.js'; -import { assertNotConsumed, ParserBase, TAcceptTokenResult } from '../base/simpleCodec/parserBase.js'; - -/** - * List of characters that terminate the prompt variable sequence. - */ -export const STOP_CHARACTERS: readonly string[] = [Space, Tab, NewLine, CarriageReturn, VerticalTab, FormFeed, Hash, At] - .map((token) => { return token.symbol; }); - -/** - * List of characters that cannot be in a variable name (excluding the {@link STOP_CHARACTERS}). - */ -export const INVALID_NAME_CHARACTERS: readonly string[] = [Hash, Colon, ExclamationMark, LeftAngleBracket, RightAngleBracket, LeftBracket, RightBracket] - .map((token) => { return token.symbol; }); - -/** - * The parser responsible for parsing a `prompt variable name`. - * E.g., `#selection` or `#codebase` variable. If the `:` character follows - * the variable name, the parser transitions to {@link PartialPromptVariableWithData} - * that is also able to parse the `data` part of the variable. E.g., the `#file` part - * of the `#file:/path/to/something.md` sequence. - */ -export class PartialPromptVariableName extends ParserBase { - constructor(token: Hash) { - super([token]); - } - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - // if a `stop` character is encountered, finish the parsing process - if (STOP_CHARACTERS.includes(token.text)) { - try { - // if it is possible to convert current parser to `PromptVariable`, return success result - return { - result: 'success', - nextParser: this.asPromptVariable(), - wasTokenConsumed: false, - }; - } catch (error) { - // otherwise fail - return { - result: 'failure', - wasTokenConsumed: false, - }; - } finally { - // in any case this is an end of the parsing process - this.isConsumed = true; - } - } - - // if a `:` character is encountered, we might transition to {@link PartialPromptVariableWithData} - if (token instanceof Colon) { - this.isConsumed = true; - - // if there is only one token before the `:` character, it must be the starting - // `#` symbol, therefore fail because there is no variable name present - if (this.currentTokens.length <= 1) { - return { - result: 'failure', - wasTokenConsumed: false, - }; - } - - // otherwise, if there are more characters after `#` available, - // we have a variable name, so we can transition to {@link PromptVariableWithData} - return { - result: 'success', - nextParser: new PartialPromptVariableWithData([...this.currentTokens, token]), - wasTokenConsumed: true, - }; - } - - // variables cannot have {@link INVALID_NAME_CHARACTERS} in their names - if (INVALID_NAME_CHARACTERS.includes(token.text)) { - this.isConsumed = true; - - return { - result: 'failure', - wasTokenConsumed: false, - }; - } - - // otherwise, a valid name character, so add it to the list of - // the current tokens and continue the parsing process - this.currentTokens.push(token); - - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } - - /** - * Try to convert current parser instance into a fully-parsed {@link PromptVariable} token. - * - * @throws if sequence of tokens received so far do not constitute a valid prompt variable, - * for instance, if there is only `1` starting `#` token is available. - */ - public asPromptVariable(): PromptVariable { - // if there is only one token before the stop character - // must be the starting `#` one), then fail - assert( - this.currentTokens.length > 1, - 'Cannot create a prompt variable out of incomplete token sequence.', - ); - - const firstToken = this.currentTokens[0]; - const lastToken = this.currentTokens[this.currentTokens.length - 1]; - - // render the characters above into strings, excluding the starting `#` character - const variableNameTokens = this.currentTokens.slice(1); - const variableName = BaseToken.render(variableNameTokens); - - return new PromptVariable( - new Range( - firstToken.range.startLineNumber, - firstToken.range.startColumn, - lastToken.range.endLineNumber, - lastToken.range.endColumn, - ), - variableName, - ); - } -} - -/** - * The parser responsible for parsing a `prompt variable name` with `data`. - * E.g., the `/path/to/something.md` part of the `#file:/path/to/something.md` sequence. - */ -export class PartialPromptVariableWithData extends ParserBase { - - constructor(tokens: readonly TSimpleDecoderToken[]) { - const firstToken = tokens[0]; - const lastToken = tokens[tokens.length - 1]; - - // sanity checks of our expectations about the tokens list - assert( - tokens.length > 2, - `Tokens list must contain at least 3 items, got '${tokens.length}'.`, - ); - assert( - firstToken instanceof Hash, - `The first token must be a '#', got '${firstToken} '.`, - ); - assert( - lastToken instanceof Colon, - `The last token must be a ':', got '${lastToken} '.`, - ); - - super([...tokens]); - } - - @assertNotConsumed - public accept(token: TSimpleDecoderToken): TAcceptTokenResult { - // if a `stop` character is encountered, finish the parsing process - if (STOP_CHARACTERS.includes(token.text)) { - // in any case, success of failure below, this is an end of the parsing process - this.isConsumed = true; - - const firstToken = this.currentTokens[0]; - const lastToken = this.currentTokens[this.currentTokens.length - 1]; - - // tokens representing variable name without the `#` character at the start and - // the `:` data separator character at the end - const variableNameTokens = this.currentTokens.slice(1, this.startTokensCount - 1); - // tokens representing variable data without the `:` separator character at the start - const variableDataTokens = this.currentTokens.slice(this.startTokensCount); - // compute the full range of the variable token - const fullRange = new Range( - firstToken.range.startLineNumber, - firstToken.range.startColumn, - lastToken.range.endLineNumber, - lastToken.range.endColumn, - ); - - // render the characters above into strings - const variableName = BaseToken.render(variableNameTokens); - const variableData = BaseToken.render(variableDataTokens); - - return { - result: 'success', - nextParser: new PromptVariableWithData( - fullRange, - variableName, - variableData, - ), - wasTokenConsumed: false, - }; - } - - // otherwise, token is a valid data character - the data can contain almost any character, - // including `:` and `#`, hence add it to the list of the current tokens and continue - this.currentTokens.push(token); - - return { - result: 'success', - nextParser: this, - wasTokenConsumed: true, - }; - } - - /** - * Try to convert current parser instance into a fully-parsed {@link asPromptVariableWithData} token. - */ - public asPromptVariableWithData(): PromptVariableWithData { - // tokens representing variable name without the `#` character at the start and - // the `:` data separator character at the end - const variableNameTokens = this.currentTokens.slice(1, this.startTokensCount - 1); - // tokens representing variable data without the `:` separator character at the start - const variableDataTokens = this.currentTokens.slice(this.startTokensCount); - - // render the characters above into strings - const variableName = BaseToken.render(variableNameTokens); - const variableData = BaseToken.render(variableDataTokens); - - const firstToken = this.currentTokens[0]; - const lastToken = this.currentTokens[this.currentTokens.length - 1]; - - return new PromptVariableWithData( - new Range( - firstToken.range.startLineNumber, - firstToken.range.startColumn, - lastToken.range.endLineNumber, - lastToken.range.endColumn, - ), - variableName, - variableData, - ); - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/fileReference.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/fileReference.ts deleted file mode 100644 index 78c4f7cb6ea..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/fileReference.ts +++ /dev/null @@ -1,50 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - - -import { PromptVariableWithData } from './promptVariable.js'; -import { assert } from '../../../../../../../base/common/assert.js'; -import { IRange, Range } from '../../../../../../../editor/common/core/range.js'; - -/** - * Name of the variable. - */ -const VARIABLE_NAME: string = 'file'; - -/** - * Object represents a file reference token inside a chatbot prompt. - */ -export class FileReference extends PromptVariableWithData { - constructor( - range: Range, - public readonly path: string, - ) { - super(range, VARIABLE_NAME, path); - } - - /** - * Create a {@link FileReference} from a {@link PromptVariableWithData} instance. - * @throws if variable name is not equal to {@link VARIABLE_NAME}. - */ - public static from(variable: PromptVariableWithData): FileReference { - assert( - variable.name === VARIABLE_NAME, - `Variable name must be '${VARIABLE_NAME}', got '${variable.name}'.`, - ); - - return new FileReference( - variable.range, - variable.data, - ); - } - - /** - * Get the range of the `link` part of the token (e.g., - * the `/path/to/file.md` part of `#file:/path/to/file.md`). - */ - public get linkRange(): IRange | undefined { - return super.dataRange; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptAtMention.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptAtMention.ts deleted file mode 100644 index 43bd8005235..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptAtMention.ts +++ /dev/null @@ -1,52 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { PromptToken } from './promptToken.js'; -import { assert } from '../../../../../../../base/common/assert.js'; -import { Range } from '../../../../../../../editor/common/core/range.js'; -import { INVALID_NAME_CHARACTERS, STOP_CHARACTERS } from '../parsers/promptVariableParser.js'; - -/** - * All prompt at-mentions start with `@` character. - */ -const START_CHARACTER: string = '@'; - -/** - * Represents a `@mention` token in a prompt text. - */ -export class PromptAtMention extends PromptToken { - constructor( - range: Range, - /** - * The name of a mention, excluding the `@` character at the start. - */ - public readonly name: string, - ) { - // sanity check of characters used in the provided mention name - for (const character of name) { - assert( - (INVALID_NAME_CHARACTERS.includes(character) === false) && - (STOP_CHARACTERS.includes(character) === false), - `Mention 'name' cannot contain character '${character}', got '${name}'.`, - ); - } - - super(range); - } - - /** - * Get full text of the token. - */ - public get text(): string { - return `${START_CHARACTER}${this.name}`; - } - - /** - * Return a string representation of the token. - */ - public override toString(): string { - return `${this.text}${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptSlashCommand.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptSlashCommand.ts deleted file mode 100644 index 1069d4a7a89..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptSlashCommand.ts +++ /dev/null @@ -1,42 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { PromptToken } from './promptToken.js'; -import { Range } from '../../../../../../../editor/common/core/range.js'; - -/** - * All prompt at-mentions start with `/` character. - */ -const START_CHARACTER: string = '/'; - -/** - * Represents a `/command` token in a prompt text. - */ -export class PromptSlashCommand extends PromptToken { - constructor( - range: Range, - /** - * The name of a command, excluding the `/` character at the start. - */ - public readonly name: string, - ) { - - super(range); - } - - /** - * Get full text of the token. - */ - public get text(): string { - return `${START_CHARACTER}${this.name}`; - } - - /** - * Return a string representation of the token. - */ - public override toString(): string { - return `${this.text}${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptTemplateVariable.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptTemplateVariable.ts deleted file mode 100644 index be3051f2c88..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptTemplateVariable.ts +++ /dev/null @@ -1,44 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { PromptToken } from './promptToken.js'; -import { Range } from '../../../../../../../editor/common/core/range.js'; -import { DollarSign } from '../base/simpleCodec/tokens/dollarSign.js'; -import { LeftCurlyBrace, RightCurlyBrace } from '../base/simpleCodec/tokens/curlyBraces.js'; - -/** - * Represents a `${variable}` token in a prompt text. - */ -export class PromptTemplateVariable extends PromptToken { - constructor( - range: Range, - /** - * The contents of the template variable, excluding - * the surrounding `${}` characters. - */ - public readonly contents: string, - ) { - super(range); - } - - /** - * Get full text of the token. - */ - public get text(): string { - return [ - DollarSign.symbol, - LeftCurlyBrace.symbol, - this.contents, - RightCurlyBrace.symbol, - ].join(''); - } - - /** - * Return a string representation of the token. - */ - public override toString(): string { - return `${this.text}${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptVariable.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptVariable.ts deleted file mode 100644 index ec5dd2e7a36..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/codecs/tokens/promptVariable.ts +++ /dev/null @@ -1,103 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { PromptToken } from './promptToken.js'; -import { IRange, Range } from '../../../../../../../editor/common/core/range.js'; - -/** - * All prompt variables start with `#` character. - */ -const START_CHARACTER: string = '#'; - -/** - * Character that separates name of a prompt variable from its data. - */ -const DATA_SEPARATOR: string = ':'; - -/** - * Represents a `#variable` token in a prompt text. - */ -export class PromptVariable extends PromptToken { - constructor( - range: Range, - /** - * The name of a prompt variable, excluding the `#` character at the start. - */ - public readonly name: string, - ) { - - super(range); - } - - /** - * Get full text of the token. - */ - public get text(): string { - return `${START_CHARACTER}${this.name}`; - } - - /** - * Return a string representation of the token. - */ - public override toString(): string { - return `${this.text}${this.range}`; - } -} - -/** - * Represents a {@link PromptVariable} with additional data token in a prompt text. - * (e.g., `#variable:/path/to/file.md`) - */ -export class PromptVariableWithData extends PromptVariable { - constructor( - fullRange: Range, - /** - * The name of the variable, excluding the starting `#` character. - */ - name: string, - - /** - * The data of the variable, excluding the starting {@link DATA_SEPARATOR} character. - */ - public readonly data: string, - ) { - super(fullRange, name); - } - - /** - * Get full text of the token. - */ - public override get text(): string { - return `${START_CHARACTER}${this.name}${DATA_SEPARATOR}${this.data}`; - } - - /** - * Range of the `data` part of the variable. - */ - public get dataRange(): IRange | undefined { - const { range } = this; - - // calculate the start column number of the `data` part of the variable - const dataStartColumn = range.startColumn + - START_CHARACTER.length + this.name.length + - DATA_SEPARATOR.length; - - // create `range` of the `data` part of the variable - const result = new Range( - range.startLineNumber, - dataStartColumn, - range.endLineNumber, - range.endColumn, - ); - - // if the resulting range is empty, return `undefined` - // because there is no `data` part present in the variable - if (result.isEmpty()) { - return undefined; - } - - return result; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/config/config.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/config/config.ts index a8e88f5e659..4144dd7b5e7 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/config/config.ts +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/config/config.ts @@ -5,6 +5,7 @@ import type { IConfigurationService } from '../../../../../../platform/configuration/common/configuration.js'; import { ContextKeyExpr } from '../../../../../../platform/contextkey/common/contextkey.js'; +import { URI } from '../../../../../../base/common/uri.js'; import { PromptsType } from '../promptTypes.js'; import { INSTRUCTIONS_DEFAULT_SOURCE_FOLDER, PROMPT_DEFAULT_SOURCE_FOLDER, getPromptFileDefaultLocation } from './promptFileLocations.js'; @@ -165,54 +166,52 @@ export namespace PromptsConfig { /** * Get value of the prompt file recommendations configuration setting. + * @param configService Configuration service instance + * @param resource Optional resource URI to get workspace folder-specific settings * @see {@link PROMPT_FILES_SUGGEST_KEY}. */ - export function getPromptFilesRecommendationsValue(configService: IConfigurationService): Record | undefined { - const configValue = configService.getValue(PromptsConfig.PROMPT_FILES_SUGGEST_KEY); + export function getPromptFilesRecommendationsValue(configService: IConfigurationService, resource?: URI): Record | undefined { + // Get the merged configuration value (VS Code automatically merges all levels: default → user → workspace → folder) + const configValue = configService.getValue(PromptsConfig.PROMPT_FILES_SUGGEST_KEY, { resource }); - if (configValue === undefined || configValue === null || Array.isArray(configValue)) { + if (!configValue || typeof configValue !== 'object' || Array.isArray(configValue)) { return undefined; } - // note! this would be also true for `null` and `array`, - // but those cases are already handled above - if (typeof configValue === 'object') { - const suggestions: Record = {}; + const suggestions: Record = {}; - for (const [promptName, value] of Object.entries(configValue)) { - const cleanPromptName = promptName.trim(); + for (const [promptName, value] of Object.entries(configValue)) { + const cleanPromptName = promptName.trim(); - // Skip empty prompt names - if (!cleanPromptName) { - continue; - } - - // Accept boolean values directly - if (typeof value === 'boolean') { - suggestions[cleanPromptName] = value; - continue; - } - - // Accept string values as when clauses - if (typeof value === 'string') { - const cleanValue = value.trim(); - if (cleanValue) { - suggestions[cleanPromptName] = cleanValue; - } - continue; - } - - // Convert other truthy/falsy values to boolean - const booleanValue = asBoolean(value); - if (booleanValue !== undefined) { - suggestions[cleanPromptName] = booleanValue; - } + // Skip empty prompt names + if (!cleanPromptName) { + continue; } - return suggestions; + // Accept boolean values directly + if (typeof value === 'boolean') { + suggestions[cleanPromptName] = value; + continue; + } + + // Accept string values as when clauses + if (typeof value === 'string') { + const cleanValue = value.trim(); + if (cleanValue) { + suggestions[cleanPromptName] = cleanValue; + } + continue; + } + + // Convert other truthy/falsy values to boolean + const booleanValue = asBoolean(value); + if (booleanValue !== undefined) { + suggestions[cleanPromptName] = booleanValue; + } } - return undefined; + // Return undefined if no valid suggestions were found + return Object.keys(suggestions).length > 0 ? suggestions : undefined; } } diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/contentProviders/filePromptContentsProvider.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/contentProviders/filePromptContentsProvider.ts deleted file mode 100644 index 47705213282..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/contentProviders/filePromptContentsProvider.ts +++ /dev/null @@ -1,166 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { PROMPT_LANGUAGE_ID } from '../promptTypes.js'; -import { IPromptContentsProvider } from './types.js'; -import { URI } from '../../../../../../base/common/uri.js'; -import { assert } from '../../../../../../base/common/assert.js'; -import { CancellationError } from '../../../../../../base/common/errors.js'; -import { VSBufferReadableStream } from '../../../../../../base/common/buffer.js'; -import { CancellationToken } from '../../../../../../base/common/cancellation.js'; -import { IModelService } from '../../../../../../editor/common/services/model.js'; -import { ILanguageService } from '../../../../../../editor/common/languages/language.js'; -import { isPromptOrInstructionsFile } from '../config/promptFileLocations.js'; -import { IPromptContentsProviderOptions, PromptContentsProviderBase } from './promptContentsProviderBase.js'; -import { OpenFailed, NotPromptFile, ResolveError, FolderReference } from '../../promptFileReferenceErrors.js'; -import { FileChangesEvent, FileChangeType, IFileService } from '../../../../../../platform/files/common/files.js'; - -/** - * Prompt contents provider for a file on the disk referenced by - * a provided {@link URI}. - */ -export class FilePromptContentProvider extends PromptContentsProviderBase implements IPromptContentsProvider { - public override get sourceName(): string { - return 'file'; - } - - public override get languageId(): string { - if (this.options.languageId) { - return this.options.languageId; - } - - const model = this.modelService.getModel(this.uri); - - if (model !== null) { - return model.getLanguageId(); - } - - const inferredId = this.languageService - .guessLanguageIdByFilepathOrFirstLine(this.uri); - - if (inferredId !== null) { - return inferredId; - } - - // fallback to the default prompt language ID - return PROMPT_LANGUAGE_ID; - } - - constructor( - public readonly uri: URI, - options: IPromptContentsProviderOptions, - @IFileService private readonly fileService: IFileService, - @IModelService private readonly modelService: IModelService, - @ILanguageService private readonly languageService: ILanguageService, - ) { - super(options); - - if (options.updateOnChange) { - // make sure the object is updated on file changes - this._register( - this.fileService.onDidFilesChange((event) => { - // if file was added or updated, forward the event to - // the `getContentsStream()` produce a new stream for file contents - if (event.contains(this.uri, FileChangeType.ADDED, FileChangeType.UPDATED)) { - // we support only full file parsing right now because - // the event doesn't contain a list of changed lines - this.onChangeEmitter.fire('full'); - return; - } - - // if file was deleted, forward the event to - // the `getContentsStream()` produce an error - if (event.contains(this.uri, FileChangeType.DELETED)) { - this.onChangeEmitter.fire(event); - return; - } - }), - ); - } - } - - /** - * Creates a stream of lines from the file based on the changes listed in - * the provided event. - * - * @param event - event that describes the changes in the file; `'full'` is - * the special value that means that all contents have changed - * @param cancellationToken - token that cancels this operation - */ - protected async getContentsStream( - _event: FileChangesEvent | 'full', - cancellationToken?: CancellationToken, - ): Promise { - assert( - !cancellationToken?.isCancellationRequested, - new CancellationError(), - ); - - // get the binary stream of the file contents - let fileStream; - try { - // ensure that the referenced URI points to a file before - // trying to get a stream for its contents - const info = await this.fileService.resolve(this.uri); - - // validate that the cancellation was not yet requested - assert( - !cancellationToken?.isCancellationRequested, - new CancellationError(), - ); - - assert( - info.isFile, - new FolderReference(this.uri), - ); - - const { allowNonPromptFiles } = this.options; - - // if URI doesn't point to a prompt file, don't try to resolve it, - // unless the `allowNonPromptFiles` option is set to `true` - if ((allowNonPromptFiles !== true) && (isPromptOrInstructionsFile(this.uri) === false)) { - throw new NotPromptFile(this.uri); - } - - fileStream = await this.fileService.readFileStream(this.uri); - - // after the promise above complete, this object can be already disposed or - // the cancellation could be requested, in that case destroy the stream and - // throw cancellation error - if (this.isDisposed || cancellationToken?.isCancellationRequested) { - fileStream.value.destroy(); - throw new CancellationError(); - } - - return fileStream.value; - } catch (error) { - if ((error instanceof ResolveError) || (error instanceof CancellationError)) { - throw error; - } - - throw new OpenFailed(this.uri, error); - } - } - - public override createNew( - promptContentsSource: { uri: URI }, - options: IPromptContentsProviderOptions, - ): IPromptContentsProvider { - return new FilePromptContentProvider( - promptContentsSource.uri, - options, - this.fileService, - this.modelService, - this.languageService, - ); - } - - /** - * String representation of this object. - */ - public override toString(): string { - return `file-prompt-contents-provider:${this.uri.path}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/contentProviders/promptContentsProviderBase.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/contentProviders/promptContentsProviderBase.ts deleted file mode 100644 index 7c9e58dc3d3..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/contentProviders/promptContentsProviderBase.ts +++ /dev/null @@ -1,196 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { assert } from '../../../../../../base/common/assert.js'; -import { VSBufferReadableStream } from '../../../../../../base/common/buffer.js'; -import { CancellationToken } from '../../../../../../base/common/cancellation.js'; -import { cancelPreviousCalls } from '../../../../../../base/common/decorators/cancelPreviousCalls.js'; -import { CancellationError } from '../../../../../../base/common/errors.js'; -import { Emitter } from '../../../../../../base/common/event.js'; -import { URI } from '../../../../../../base/common/uri.js'; -import { FailedToResolveContentsStream, ResolveError } from '../../promptFileReferenceErrors.js'; -import { INSTRUCTIONS_LANGUAGE_ID, MODE_LANGUAGE_ID, PROMPT_LANGUAGE_ID, PromptsType } from '../promptTypes.js'; -import { ObservableDisposable } from '../utils/observableDisposable.js'; -import { IPromptContentsProvider } from './types.js'; - -/** - * Options of the {@link PromptContentsProviderBase} class. - */ -export interface IPromptContentsProviderOptions { - /** - * Whether to allow files that don't have usual prompt - * file extension to be treated as a prompt file. - */ - readonly allowNonPromptFiles: boolean; - - /** - * Language ID to use for the prompt contents. If not set, the language ID will be inferred from the file. - */ - readonly languageId: string | undefined; - - /** - * If set to `true`, the contents provider will listen for updates and retrigger a parse. - */ - readonly updateOnChange: boolean; -} - - -/** - * Base class for prompt contents providers. Classes that extend this one are responsible to: - * - * - implement the {@link getContentsStream} method to provide the contents stream - * of a prompt; this method should throw a `ResolveError` or its derivative if the contents - * cannot be parsed for any reason - * - fire a {@link TChangeEvent} event on the {@link onChangeEmitter} event when - * prompt contents change - * - misc: - * - provide the {@link uri} property that represents the URI of a prompt that - * the contents are for - * - implement the {@link toString} method to return a string representation of this - * provider type to aid with debugging/tracing - */ -export abstract class PromptContentsProviderBase< - TChangeEvent extends NonNullable, -> extends ObservableDisposable implements IPromptContentsProvider { - public abstract readonly uri: URI; - public abstract createNew(promptContentsSource: { uri: URI }, options: IPromptContentsProviderOptions): IPromptContentsProvider; - public abstract override toString(): string; - public abstract get languageId(): string; - public abstract get sourceName(): string; - - /** - * Prompt contents stream. - */ - public get contents(): Promise { - return this.getContentsStream('full'); - } - - /** - * Prompt type used to determine how to interpret file contents. - */ - public get promptType(): PromptsType | 'non-prompt' { - const { languageId } = this; - - if (languageId === PROMPT_LANGUAGE_ID) { - return PromptsType.prompt; - } - - if (languageId === INSTRUCTIONS_LANGUAGE_ID) { - return PromptsType.instructions; - } - - if (languageId === MODE_LANGUAGE_ID) { - return PromptsType.mode; - } - - return 'non-prompt'; - } - - /** - * Function to get contents stream for the provider. This function should - * throw a `ResolveError` or its derivative if the contents cannot be parsed. - * - * @param changesEvent The event that triggered the change. The special - * `'full'` value means that everything has changed hence entire prompt - * contents need to be re-parsed from scratch. - */ - protected abstract getContentsStream( - changesEvent: TChangeEvent | 'full', - cancellationToken?: CancellationToken, - ): Promise; - - /** - * Internal event emitter for the prompt contents change event. Classes that extend - * this abstract class are responsible to use this emitter to fire the contents change - * event when the prompt contents get modified. - */ - protected readonly onChangeEmitter = this._register(new Emitter()); - - /** - * Options passed to the constructor - */ - protected readonly options: IPromptContentsProviderOptions; - - constructor( - options: IPromptContentsProviderOptions, - ) { - super(); - - this.options = options; - } - - /** - * Event emitter for the prompt contents change event. - * See {@link onContentChanged} for more details. - */ - private readonly onContentChangedEmitter = this._register(new Emitter()); - - /** - * Event that fires when the prompt contents change. The event is either - * a `VSBufferReadableStream` stream with changed contents or an instance of - * the `ResolveError` class representing a parsing failure case. - * - * `Note!` this field is meant to be used by the external consumers of the prompt - * contents provider that the classes that extend this abstract class. - * Please use the {@link onChangeEmitter} event to provide a change - * event in your prompt contents implementation instead. - */ - public readonly onContentChanged = this.onContentChangedEmitter.event; - - /** - * Internal common implementation of the event that should be fired when - * prompt contents change. - */ - @cancelPreviousCalls - private onContentsChanged( - event: TChangeEvent | 'full', - cancellationToken?: CancellationToken, - ): this { - const promise = (cancellationToken?.isCancellationRequested) - ? Promise.reject(new CancellationError()) - : this.getContentsStream(event, cancellationToken); - - promise - .then((stream) => { - if (cancellationToken?.isCancellationRequested || this.isDisposed) { - stream.destroy(); - throw new CancellationError(); - } - - this.onContentChangedEmitter.fire(stream); - }) - .catch((error) => { - if (error instanceof ResolveError) { - this.onContentChangedEmitter.fire(error); - - return; - } - - this.onContentChangedEmitter.fire( - new FailedToResolveContentsStream(this.uri, error), - ); - }); - - return this; - } - - /** - * Start producing the prompt contents data. - */ - public start(token?: CancellationToken): this { - assert( - !this.isDisposed, - 'Cannot start contents provider that was already disposed.', - ); - - // `'full'` means "everything has changed" - this.onContentsChanged('full', token); - - // subscribe to the change event emitted by a child class - this._register(this.onChangeEmitter.event(this.onContentsChanged, this)); - - return this; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/contentProviders/textModelContentsProvider.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/contentProviders/textModelContentsProvider.ts deleted file mode 100644 index aea65908ccf..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/contentProviders/textModelContentsProvider.ts +++ /dev/null @@ -1,93 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { VSBufferReadableStream } from '../../../../../../base/common/buffer.js'; -import { CancellationToken } from '../../../../../../base/common/cancellation.js'; -import { URI } from '../../../../../../base/common/uri.js'; -import { ITextModel } from '../../../../../../editor/common/model.js'; -import { TextModel } from '../../../../../../editor/common/model/textModel.js'; -import { IModelContentChangedEvent } from '../../../../../../editor/common/textModelEvents.js'; -import { IInstantiationService } from '../../../../../../platform/instantiation/common/instantiation.js'; -import { objectStreamFromTextModel } from '../codecs/base/utils/objectStreamFromTextModel.js'; -import { FilePromptContentProvider } from './filePromptContentsProvider.js'; -import { IPromptContentsProviderOptions, PromptContentsProviderBase } from './promptContentsProviderBase.js'; -import { IPromptContentsProvider } from './types.js'; - -/** - * Prompt contents provider for a {@link ITextModel} instance. - */ -export class TextModelContentsProvider extends PromptContentsProviderBase { - /** - * URI component of the prompt associated with this contents provider. - */ - public get uri(): URI { - return this.model.uri; - } - - public override get sourceName(): string { - return 'text-model'; - } - - public override get languageId(): string { - return this.options.languageId ?? this.model.getLanguageId(); - } - - constructor( - private readonly model: ITextModel, - options: IPromptContentsProviderOptions, - @IInstantiationService private readonly instantiationService: IInstantiationService, - ) { - super(options); - - this._register(this.model.onWillDispose(this.dispose.bind(this))); - if (options.updateOnChange) { - this._register(this.model.onDidChangeContent(this.onChangeEmitter.fire.bind(this.onChangeEmitter))); - } - } - - /** - * Creates a stream of binary data from the text model based on the changes - * listed in the provided event. - * - * Note! this method implements a basic logic which does not take into account - * the `_event` argument for incremental updates. This needs to be improved. - * - * @param _event - event that describes the changes in the text model; `'full'` is - * the special value that means that all contents have changed - * @param cancellationToken - token that cancels this operation - */ - protected override async getContentsStream( - _event: IModelContentChangedEvent | 'full', - cancellationToken?: CancellationToken, - ): Promise { - return objectStreamFromTextModel(this.model, cancellationToken); - } - - public override createNew( - promptContentsSource: TextModel | { uri: URI }, - options: IPromptContentsProviderOptions, - ): IPromptContentsProvider { - if (promptContentsSource instanceof TextModel) { - return this.instantiationService.createInstance( - TextModelContentsProvider, - promptContentsSource, - options, - ); - } - - return this.instantiationService.createInstance( - FilePromptContentProvider, - promptContentsSource.uri, - options, - ); - } - - /** - * String representation of this object. - */ - public override toString(): string { - return `text-model-prompt-contents-provider:${this.uri.path}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/contentProviders/types.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/contentProviders/types.ts deleted file mode 100644 index a26eaa43330..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/contentProviders/types.ts +++ /dev/null @@ -1,70 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { URI } from '../../../../../../base/common/uri.js'; -import { Event } from '../../../../../../base/common/event.js'; -import { ResolveError } from '../../promptFileReferenceErrors.js'; -import { IDisposable } from '../../../../../../base/common/lifecycle.js'; -import { VSBufferReadableStream } from '../../../../../../base/common/buffer.js'; -import { PromptsType } from '../promptTypes.js'; -import { CancellationToken } from '../../../../../../base/common/cancellation.js'; -import { IPromptContentsProviderOptions } from './promptContentsProviderBase.js'; - -/** - * Interface for a prompt contents provider. Prompt contents providers are - * responsible for providing contents of a prompt as a byte streams and - * allow to subscribe to the change events of the prompt contents. - */ -export interface IPromptContentsProvider extends IDisposable { - /** - * URI component of the prompt associated with this contents provider. - */ - readonly uri: URI; - - /** - * Language ID of the prompt contents. - */ - readonly languageId: string; - - /** - * Prompt type used to determine how to interpret file contents. - */ - readonly promptType: PromptsType | 'non-prompt'; - - /** - * Prompt contents stream. - */ - readonly contents: Promise; - - /** - * Prompt contents source name. - */ - readonly sourceName: string; - - /** - * Event that fires when the prompt contents change. The event is either a - * {@linkcode VSBufferReadableStream} stream with changed contents or - * an instance of the {@linkcode ResolveError} error. - */ - readonly onContentChanged: Event; - - /** - * Subscribe to `onDispose` event of the contents provider. - */ - readonly onDispose: Event; - - /** - * Start the contents provider to produce the underlying contents. - */ - start(token?: CancellationToken): this; - - /** - * Create a new instance of prompt contents provider. - */ - createNew( - promptContentsSource: { uri: URI }, - options: IPromptContentsProviderOptions, - ): IPromptContentsProvider; -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/frontMatterDecoration.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/frontMatterDecoration.ts deleted file mode 100644 index d07720eff5b..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/frontMatterDecoration.ts +++ /dev/null @@ -1,120 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { Position } from '../../../../../../../../editor/common/core/position.js'; -import { localize } from '../../../../../../../../nls.js'; -import { contrastBorder, editorBackground } from '../../../../../../../../platform/theme/common/colorRegistry.js'; -import { asCssVariable, ColorIdentifier, darken, registerColor } from '../../../../../../../../platform/theme/common/colorUtils.js'; -import { BaseToken } from '../../../codecs/base/baseToken.js'; -import { FrontMatterHeader } from '../../../codecs/base/markdownExtensionsCodec/tokens/frontMatterHeader.js'; -import { CssClassModifiers } from '../types.js'; -import { FrontMatterMarkerDecoration } from './frontMatterMarkerDecoration.js'; -import { ReactiveDecorationBase } from './utils/reactiveDecorationBase.js'; -import { IReactiveDecorationClassNames, TAddAccessor, TDecorationStyles } from './utils/types.js'; - -/** - * Decoration CSS class names. - */ -export enum CssClassNames { - Main = '.prompt-front-matter-decoration', - Inline = '.prompt-front-matter-decoration-inline', - MainInactive = `${CssClassNames.Main}${CssClassModifiers.Inactive}`, - InlineInactive = `${CssClassNames.Inline}${CssClassModifiers.Inactive}`, -} - -/** - * Main background color of `active` Front Matter header block. - */ -export const BACKGROUND_COLOR: ColorIdentifier = registerColor( - 'prompt.frontMatter.background', - { dark: darken(editorBackground, 0.2), light: darken(editorBackground, 0.05), hcDark: contrastBorder, hcLight: contrastBorder }, - localize('chat.prompt.frontMatter.background.description', "Background color of a Front Matter header block."), -); - -/** - * Background color of `inactive` Front Matter header block. - */ -export const INACTIVE_BACKGROUND_COLOR: ColorIdentifier = registerColor( - 'prompt.frontMatter.inactiveBackground', - { dark: darken(editorBackground, 0.1), light: darken(editorBackground, 0.025), hcDark: contrastBorder, hcLight: contrastBorder }, - localize('chat.prompt.frontMatter.inactiveBackground.description', "Background color of an inactive Front Matter header block."), -); - -/** - * CSS styles for the decoration. - */ -export const CSS_STYLES = { - [CssClassNames.Main]: [ - `background-color: ${asCssVariable(BACKGROUND_COLOR)};`, - 'z-index: -1;', // this is required to allow for selections to appear above the decoration background - ], - [CssClassNames.MainInactive]: [ - `background-color: ${asCssVariable(INACTIVE_BACKGROUND_COLOR)};`, - ], - [CssClassNames.InlineInactive]: [ - 'color: var(--vscode-disabledForeground);', - ], - ...FrontMatterMarkerDecoration.cssStyles, -}; - -/** - * Editor decoration for the Front Matter header token inside a prompt. - */ -export class FrontMatterDecoration extends ReactiveDecorationBase { - constructor( - accessor: TAddAccessor, - token: FrontMatterHeader, - ) { - super(accessor, token); - - this.childDecorators.push( - new FrontMatterMarkerDecoration(accessor, token.startMarker), - new FrontMatterMarkerDecoration(accessor, token.endMarker), - ); - } - - public override setCursorPosition( - position: Position | null | undefined, - ): this is { readonly changed: true } { - const result = super.setCursorPosition(position); - - for (const marker of this.childDecorators) { - if ((marker instanceof FrontMatterMarkerDecoration) === false) { - continue; - } - - // activate/deactivate markers based on the active state - // of the main Front Matter header decoration - marker.activate(this.active); - } - - return result; - } - - protected override get classNames(): IReactiveDecorationClassNames { - return CssClassNames; - } - - protected override get isWholeLine(): boolean { - return true; - } - - protected override get description(): string { - return 'Front Matter header decoration.'; - } - - public static get cssStyles(): TDecorationStyles { - return CSS_STYLES; - } - - /** - * Whether current decoration class can decorate provided token. - */ - public static handles( - token: BaseToken, - ): token is FrontMatterHeader { - return token instanceof FrontMatterHeader; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/frontMatterMarkerDecoration.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/frontMatterMarkerDecoration.ts deleted file mode 100644 index 2a4a174c9f4..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/frontMatterMarkerDecoration.ts +++ /dev/null @@ -1,56 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { CssClassModifiers } from '../types.js'; -import { TDecorationStyles, IReactiveDecorationClassNames } from './utils/types.js'; -import { FrontMatterMarker } from '../../../codecs/base/markdownExtensionsCodec/tokens/frontMatterMarker.js'; -import { ReactiveDecorationBase } from './utils/reactiveDecorationBase.js'; - -/** - * Decoration CSS class names. - */ -export enum CssClassNames { - Main = '.prompt-front-matter-decoration-marker', - Inline = '.prompt-front-matter-decoration-marker-inline', - MainInactive = `${CssClassNames.Main}${CssClassModifiers.Inactive}`, - InlineInactive = `${CssClassNames.Inline}${CssClassModifiers.Inactive}`, -} - -/** - * Editor decoration for a `marker` token of a Front Matter header. - */ -export class FrontMatterMarkerDecoration extends ReactiveDecorationBase { - /** - * Activate/deactivate the decoration. - */ - public activate(state: boolean): this { - const position = (state === true) - ? this.token.range.getStartPosition() - : null; - - this.setCursorPosition(position); - - return this; - } - - protected override get classNames(): IReactiveDecorationClassNames { - return CssClassNames; - } - - protected override get description(): string { - return 'Marker decoration of a Front Matter header.'; - } - - public static get cssStyles(): TDecorationStyles { - return { - [CssClassNames.Inline]: [ - 'color: var(--vscode-disabledForeground);', - ], - [CssClassNames.InlineInactive]: [ - 'opacity: 0.25;', - ], - }; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/utils/decorationBase.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/utils/decorationBase.ts deleted file mode 100644 index 8f28ec33247..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/utils/decorationBase.ts +++ /dev/null @@ -1,127 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { Range } from '../../../../../../../../../editor/common/core/range.js'; -import { IMarkdownString } from '../../../../../../../../../base/common/htmlContent.js'; -import { BaseToken } from '../../../../codecs/base/baseToken.js'; -import { TrackedRangeStickiness } from '../../../../../../../../../editor/common/model.js'; -import type { TAddAccessor, TChangeAccessor, TDecorationStyles, TRemoveAccessor } from './types.js'; -import { ModelDecorationOptions } from '../../../../../../../../../editor/common/model/textModel.js'; - -/** - * Base class for all editor decorations. - */ -export abstract class DecorationBase< - TPromptToken extends BaseToken, - TCssClassName extends string = string, -> { - /** - * Description of the decoration. - */ - protected abstract get description(): string; - - /** - * Default CSS class name of the decoration. - */ - protected abstract get className(): TCssClassName; - - /** - * Inline CSS class name of the decoration. - */ - protected abstract get inlineClassName(): TCssClassName; - - /** - * Indicates whether the decoration spans the whole line(s). - */ - protected get isWholeLine(): boolean { - return false; - } - - /** - * Hover message of the decoration. - */ - protected get hoverMessage(): IMarkdownString | IMarkdownString[] | null { - return null; - } - - /** - * ID of editor decoration it was registered with. - */ - public readonly id: string; - - constructor( - accessor: TAddAccessor, - protected readonly token: TPromptToken, - ) { - this.id = accessor.addDecoration(this.range, this.decorationOptions); - } - - /** - * Range of the decoration. - */ - public get range(): Range { - return this.token.range; - } - - /** - * Changes the decoration in the editor. - */ - public change( - accessor: TChangeAccessor, - ): this { - accessor.changeDecorationOptions( - this.id, - this.decorationOptions, - ); - - return this; - } - - /** - * Removes associated editor decoration(s). - */ - public remove( - accessor: TRemoveAccessor, - ): this { - accessor.removeDecoration(this.id); - - return this; - } - - /** - * Get editor decoration options for this decorator. - */ - private get decorationOptions(): ModelDecorationOptions { - return ModelDecorationOptions.createDynamic({ - description: this.description, - hoverMessage: this.hoverMessage, - className: this.className, - inlineClassName: this.inlineClassName, - isWholeLine: this.isWholeLine, - stickiness: TrackedRangeStickiness.NeverGrowsWhenTypingAtEdges, - shouldFillLineOnLineBreak: true, - }); - } -} - -/** - * Type of a generic decoration class. - */ -export type TDecorationClass = { - new( - accessor: TAddAccessor, - token: TPromptToken, - ): DecorationBase; - - /** - * CSS styles for the decoration. - */ - readonly cssStyles: TDecorationStyles; - - /** - * Whether the decoration class handles the provided token. - */ - handles(token: BaseToken): token is TPromptToken; -}; diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/utils/reactiveDecorationBase.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/utils/reactiveDecorationBase.ts deleted file mode 100644 index 42533aea5fe..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/utils/reactiveDecorationBase.ts +++ /dev/null @@ -1,162 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { DecorationBase } from './decorationBase.js'; -import { Position } from '../../../../../../../../../editor/common/core/position.js'; -import { BaseToken } from '../../../../codecs/base/baseToken.js'; -import type { IReactiveDecorationClassNames, TAddAccessor, TChangeAccessor, TRemoveAccessor } from './types.js'; - -/** - * Base class for all reactive editor decorations. A reactive decoration - * is a decoration that can change its appearance based on current cursor - * position in the editor, hence can "react" to the user's actions. - */ -export abstract class ReactiveDecorationBase< - TPromptToken extends BaseToken, - TCssClassName extends string = string, -> extends DecorationBase { - /** - * CSS class names of the decoration. - */ - protected abstract get classNames(): IReactiveDecorationClassNames; - - /** - * A list of child decorators that are part of this decoration. - * For instance a Front Matter header decoration can have child - * decorators for each of the header's `---` markers. - */ - protected readonly childDecorators: DecorationBase[]; - - /** - * Whether the decoration has changed since the last {@link change}. - */ - public get changed(): boolean { - // if any of the child decorators changed, this object is also - // considered to be changed - for (const marker of this.childDecorators) { - if ((marker instanceof ReactiveDecorationBase) === false) { - continue; - } - - if (marker.changed === true) { - return true; - } - } - - return this.didChange; - } - - constructor( - accessor: TAddAccessor, - token: TPromptToken, - ) { - super(accessor, token); - - this.childDecorators = []; - } - - /** - * Current position of cursor in the editor. - */ - private cursorPosition?: Position | null; - - /** - * Private field for the {@link changed} property. - */ - private didChange = true; - - /** - * Whether cursor is currently inside the decoration range. - */ - protected get active(): boolean { - return true; - - /** - * Temporarily disable until we have a proper way to get - * the cursor position inside active editor. - */ - /** - * if (!this.cursorPosition) { - * return false; - * } - * - * // when cursor is at the end of a range, the range considered to - * // not contain the position, but we want to include it - * const atEnd = (this.range.endLineNumber === this.cursorPosition.lineNumber) - * && (this.range.endColumn === this.cursorPosition.column); - * - * return atEnd || this.range.containsPosition(this.cursorPosition); - */ - } - - /** - * Set cursor position and update {@link changed} property if needed. - */ - public setCursorPosition( - position: Position | null | undefined, - ): this is { readonly changed: true } { - if (this.cursorPosition === position) { - return false; - } - - if (this.cursorPosition && position) { - if (this.cursorPosition.equals(position)) { - return false; - } - } - - const wasActive = this.active; - this.cursorPosition = position; - this.didChange = (wasActive !== this.active); - - return this.changed; - } - - public override change( - accessor: TChangeAccessor, - ): this { - if (this.didChange === false) { - return this; - } - - super.change(accessor); - this.didChange = false; - - for (const marker of this.childDecorators) { - marker.change(accessor); - } - - return this; - } - - public override remove( - accessor: TRemoveAccessor, - ): this { - super.remove(accessor); - - for (const marker of this.childDecorators) { - marker.remove(accessor); - } - - return this; - } - - protected override get className(): TCssClassName { - return (this.active) - ? this.classNames.Main - : this.classNames.MainInactive; - } - - protected override get inlineClassName(): TCssClassName { - return (this.active) - ? this.classNames.Inline - : this.classNames.InlineInactive; - } -} - -/** - * Type for a decorator with {@link ReactiveDecorationBase.changed changed} property set to `true`. - */ -export type TChangedDecorator = ReactiveDecorationBase & { readonly changed: true }; diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/utils/types.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/utils/types.ts deleted file mode 100644 index 1ca7e270d3f..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/decorations/utils/types.ts +++ /dev/null @@ -1,56 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { IModelDecorationsChangeAccessor } from '../../../../../../../../../editor/common/model.js'; - -/** - * CSS class names of a `reactive` decoration. - */ -export interface IReactiveDecorationClassNames { - /** - * Main, default CSS class name of the decoration. - */ - readonly Main: T; - - /** - * CSS class name of the decoration for the `inline`(text) styles. - */ - readonly Inline: T; - - /** - * main CSS class name of the decoration for the `inactive` - * decoration state. - */ - readonly MainInactive: T; - - /** - * CSS class name of the decoration for the `inline`(text) - * styles when decoration is in the `inactive` state. - */ - readonly InlineInactive: T; -} - -/** - * CSS styles for a decoration to be registered with editor. - */ -export type TDecorationStyles = { - readonly [key in TClassNames]: readonly string[]; -}; - -/** - * A model decorations accessor that can be used to `add` a decoration. - */ -export type TAddAccessor = Pick; - -/** - * A model decorations accessor that can be used to `change` a decoration. - */ -export type TChangeAccessor = Pick; - -/** - * A model decorations accessor that can be used to `remove` a decoration. - */ -export type TRemoveAccessor = Pick; - diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/promptDecorationsProvider.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/promptDecorationsProvider.ts deleted file mode 100644 index d7a9d9b235f..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/promptDecorationsProvider.ts +++ /dev/null @@ -1,205 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { IPromptsService } from '../../service/promptsService.js'; -import { ProviderInstanceBase } from '../providerInstanceBase.js'; -import { ITextModel } from '../../../../../../../editor/common/model.js'; -import { FrontMatterDecoration } from './decorations/frontMatterDecoration.js'; -import { toDisposable } from '../../../../../../../base/common/lifecycle.js'; -import { Position } from '../../../../../../../editor/common/core/position.js'; -import { BaseToken } from '../../codecs/base/baseToken.js'; -import { ProviderInstanceManagerBase, TProviderClass } from '../providerInstanceManagerBase.js'; -import { registerThemingParticipant } from '../../../../../../../platform/theme/common/themeService.js'; -import { FrontMatterHeader } from '../../codecs/base/markdownExtensionsCodec/tokens/frontMatterHeader.js'; -import { ReactiveDecorationBase, TChangedDecorator } from './decorations/utils/reactiveDecorationBase.js'; -import { DecorationBase, TDecorationClass } from './decorations/utils/decorationBase.js'; - -/** - * Prompt tokens that are decorated by this provider. - */ -type TDecoratedToken = FrontMatterHeader; - -/** - * List of all supported decorations. - */ -const SUPPORTED_DECORATIONS: readonly TDecorationClass[] = Object.freeze([ - FrontMatterDecoration, -]); - -/** - * Prompt syntax decorations provider for text models. - */ -export class PromptDecorator extends ProviderInstanceBase { - /** - * Currently active decorations. - */ - private readonly decorations: DecorationBase[] = []; - - constructor( - model: ITextModel, - @IPromptsService promptsService: IPromptsService, - ) { - super(model, promptsService); - - this.watchCursorPosition(); - } - - protected override async onPromptSettled( - _error?: Error, - ): Promise { - // by the time the promise above completes, either this object - // or the text model might be already has been disposed - if (this.isDisposed || this.model.isDisposed()) { - return; - } - - this.addDecorations(); - - return; - } - - /** - * Get the current cursor position inside an active editor. - * Note! Currently not implemented because the provider is disabled, and - * we need to do some refactoring to get accurate cursor position. - */ - private get cursorPosition(): Position | null { - if (this.model.isDisposed()) { - return null; - } - - return null; - } - - /** - * Watch editor cursor position and update reactive decorations accordingly. - */ - private watchCursorPosition(): this { - const interval = setInterval(() => { - const { cursorPosition } = this; - - const changedDecorations: TChangedDecorator[] = []; - for (const decoration of this.decorations) { - if ((decoration instanceof ReactiveDecorationBase) === false) { - continue; - } - - if (decoration.setCursorPosition(cursorPosition) === true) { - changedDecorations.push(decoration); - } - } - - if (changedDecorations.length === 0) { - return; - } - - this.changeModelDecorations(changedDecorations); - }, 25); - - this._register(toDisposable(() => { - clearInterval(interval); - })); - - return this; - } - - /** - * Update existing decorations. - */ - private changeModelDecorations( - decorations: readonly TChangedDecorator[], - ): this { - this.model.changeDecorations((accessor) => { - for (const decoration of decorations) { - decoration.change(accessor); - } - }); - - return this; - } - - /** - * Add decorations for all prompt tokens. - */ - private addDecorations(): this { - this.model.changeDecorations((accessor) => { - const { tokens } = this.parser; - - // remove all existing decorations - for (const decoration of this.decorations.splice(0)) { - decoration.remove(accessor); - } - - // then add new decorations based on the current tokens - for (const token of tokens) { - for (const Decoration of SUPPORTED_DECORATIONS) { - if (Decoration.handles(token) === false) { - continue; - } - - this.decorations.push( - new Decoration(accessor, token), - ); - break; - } - } - }); - - return this; - } - - /** - * Remove all existing decorations. - */ - private removeAllDecorations(): this { - if (this.decorations.length === 0) { - return this; - } - - this.model.changeDecorations((accessor) => { - for (const decoration of this.decorations.splice(0)) { - decoration.remove(accessor); - } - }); - - return this; - } - - public override dispose(): void { - if (this.isDisposed) { - return; - } - - this.removeAllDecorations(); - super.dispose(); - } - - /** - * Returns a string representation of this object. - */ - public override toString(): string { - return `text-model-prompt-decorator:${this.model.uri.path}`; - } -} - -/** - * Register CSS styles of the supported decorations. - */ -registerThemingParticipant((_theme, collector) => { - for (const Decoration of SUPPORTED_DECORATIONS) { - for (const [className, styles] of Object.entries(Decoration.cssStyles)) { - collector.addRule(`.monaco-editor ${className} { ${styles.join(' ')} }`); - } - } -}); - -/** - * Provider for prompt syntax decorators on text models. - */ -export class PromptDecorationsProviderInstanceManager extends ProviderInstanceManagerBase { - protected override get InstanceClass(): TProviderClass { - return PromptDecorator; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/types.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/types.ts deleted file mode 100644 index fb3ef81b122..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/decorationsProvider/types.ts +++ /dev/null @@ -1,48 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { IRange } from '../../../../../../../editor/common/core/range.js'; -import { ModelDecorationOptions } from '../../../../../../../editor/common/model/textModel.js'; - -/** - * Decoration object. - */ -export interface ITextModelDecoration { - /** - * Range of the decoration. - */ - range: IRange; - - /** - * Associated decoration options. - */ - options: ModelDecorationOptions; -} - -/** - * Decoration CSS class names. - */ -export enum DecorationClassNames { - /** - * CSS class name for `default` prompt syntax decoration. - */ - Default = 'prompt-decoration', - - /** - * CSS class name for `file reference` prompt syntax decoration. - */ - FileReference = DecorationClassNames.Default, -} - -/** - * Decoration CSS class modifiers. - */ -export enum CssClassModifiers { - /** - * CSS class modifier for `active` state of - * a `reactive` prompt syntax decoration. - */ - Inactive = '.prompt-decoration-inactive', -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/promptHeaderAutocompletion.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/promptHeaderAutocompletion.ts index 61bbbcc7754..4ed5bc6869e 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/promptHeaderAutocompletion.ts +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/promptHeaderAutocompletion.ts @@ -18,6 +18,7 @@ import { ALL_PROMPTS_LANGUAGE_SELECTOR, getPromptsTypeForLanguageId, PromptsType import { IPromptsService } from '../service/promptsService.js'; import { Iterable } from '../../../../../../base/common/iterator.js'; import { PromptHeader } from '../service/newPromptsParser.js'; +import { getValidAttributeNames } from '../service/promptValidator.js'; export class PromptHeaderAutocompletion extends Disposable implements CompletionItemProvider { /** @@ -91,7 +92,7 @@ export class PromptHeaderAutocompletion extends Disposable implements Completion ): Promise { const suggestions: CompletionItem[] = []; - const supportedProperties = this.getSupportedProperties(promptType); + const supportedProperties = new Set(getValidAttributeNames(promptType, false)); this.removeUsedProperties(supportedProperties, model, headerRange, position); const getInsertText = (property: string): string => { @@ -133,7 +134,7 @@ export class PromptHeaderAutocompletion extends Disposable implements Completion const lineContent = model.getLineContent(position.lineNumber); const property = lineContent.substring(0, colonPosition.column - 1).trim(); - if (!this.getSupportedProperties(promptType).has(property)) { + if (!getValidAttributeNames(promptType, true).includes(property)) { return undefined; } @@ -166,17 +167,6 @@ export class PromptHeaderAutocompletion extends Disposable implements Completion return { suggestions }; } - private getSupportedProperties(promptType: string): Set { - switch (promptType) { - case PromptsType.instructions: - return new Set(['applyTo', 'description']); - case PromptsType.prompt: - return new Set(['mode', 'tools', 'description', 'model']); - default: - return new Set(['tools', 'description', 'model']); - } - } - private removeUsedProperties(properties: Set, model: ITextModel, headerRange: Range, position: Position): void { for (let i = headerRange.startLineNumber; i <= headerRange.endLineNumber; i++) { if (i !== position.lineNumber) { diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/promptHeaderDiagnosticsProvider.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/promptHeaderDiagnosticsProvider.ts deleted file mode 100644 index b8bca008829..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/promptHeaderDiagnosticsProvider.ts +++ /dev/null @@ -1,234 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { IPromptsService } from '../service/promptsService.js'; -import { ProviderInstanceBase } from './providerInstanceBase.js'; -import { ITextModel } from '../../../../../../editor/common/model.js'; -import { assertNever } from '../../../../../../base/common/assert.js'; -import { CancellationToken } from '../../../../../../base/common/cancellation.js'; -import { ProviderInstanceManagerBase, TProviderClass } from './providerInstanceManagerBase.js'; -import { TDiagnostic, PromptMetadataError, PromptMetadataWarning } from '../parsers/promptHeader/diagnostics.js'; -import { IMarkerData, IMarkerService, MarkerSeverity } from '../../../../../../platform/markers/common/markers.js'; -import { PromptHeader } from '../parsers/promptHeader/promptHeader.js'; -import { PromptToolsMetadata } from '../parsers/promptHeader/metadata/tools.js'; -import { PromptModelMetadata } from '../parsers/promptHeader/metadata/model.js'; -import { ModeHeader } from '../parsers/promptHeader/modeHeader.js'; -import { ILanguageModelChatMetadata, ILanguageModelsService } from '../../languageModels.js'; -import { ILanguageModelToolsService } from '../../languageModelToolsService.js'; -import { localize } from '../../../../../../nls.js'; -import { ChatModeKind } from '../../constants.js'; -import { IChatMode, IChatModeService } from '../../chatModes.js'; -import { PromptModeMetadata } from '../parsers/promptHeader/metadata/mode.js'; -import { Iterable } from '../../../../../../base/common/iterator.js'; - -/** - * Unique ID of the markers provider class. - */ -const MARKERS_OWNER_ID = 'prompts-header-diagnostics-provider'; - -/** - * Prompt header diagnostics provider for an individual text model - * of a prompt file. - */ -class PromptHeaderDiagnosticsProvider extends ProviderInstanceBase { - constructor( - model: ITextModel, - @IPromptsService promptsService: IPromptsService, - @IMarkerService private readonly markerService: IMarkerService, - @ILanguageModelsService private readonly languageModelsService: ILanguageModelsService, - @ILanguageModelToolsService private readonly languageModelToolsService: ILanguageModelToolsService, - @IChatModeService private readonly chatModeService: IChatModeService, - ) { - super(model, promptsService); - this._register(languageModelsService.onDidChangeLanguageModels(() => { - this.onPromptSettled(undefined, CancellationToken.None); - })); - this._register(languageModelToolsService.onDidChangeTools(() => { - this.onPromptSettled(undefined, CancellationToken.None); - })); - this._register(chatModeService.onDidChangeChatModes(() => { - this.onPromptSettled(undefined, CancellationToken.None); - })); - } - - /** - * Update diagnostic markers for the current editor. - */ - protected override async onPromptSettled( - _error: Error | undefined, - token: CancellationToken, - ): Promise { - - const { header } = this.parser; - if (header === undefined) { - this.markerService.remove(MARKERS_OWNER_ID, [this.model.uri]); - return; - } - - // header parsing process is separate from the prompt parsing one, hence - // apply markers only after the header is settled and so has diagnostics - const completed = await header.settled; - if (!completed || token.isCancellationRequested) { - return; - } - - const markers: IMarkerData[] = []; - for (const diagnostic of header.diagnostics) { - markers.push(toMarker(diagnostic)); - } - - if (header instanceof PromptHeader) { - const mode = this.validateMode(header.metadataUtility.mode, markers); - this.validateTools(header.metadataUtility.tools, mode?.kind, markers); - this.validateModel(header.metadataUtility.model, mode?.kind, markers); - } else if (header instanceof ModeHeader) { - this.validateTools(header.metadataUtility.tools, ChatModeKind.Agent, markers); - this.validateModel(header.metadataUtility.model, ChatModeKind.Agent, markers); - - } - - if (markers.length === 0) { - this.markerService.remove(MARKERS_OWNER_ID, [this.model.uri]); - return; - } - - this.markerService.changeOne( - MARKERS_OWNER_ID, - this.model.uri, - markers, - ); - return; - } - validateModel(modelNode: PromptModelMetadata | undefined, modeKind: string | ChatModeKind | undefined, markers: IMarkerData[]) { - if (!modelNode || modelNode.value === undefined) { - return; - } - const languageModes = this.languageModelsService.getLanguageModelIds(); - if (languageModes.length === 0) { - // likely the service is not initialized yet - return; - } - const modelMetadata = this.findModelByName(languageModes, modelNode.value); - if (!modelMetadata) { - markers.push({ - message: localize('promptHeaderDiagnosticsProvider.modelNotFound', "Unknown model '{0}'", modelNode.value), - severity: MarkerSeverity.Warning, - ...modelNode.range, - }); - } else if (modeKind === ChatModeKind.Agent && !ILanguageModelChatMetadata.suitableForAgentMode(modelMetadata)) { - markers.push({ - message: localize('promptHeaderDiagnosticsProvider.modelNotSuited', "Model '{0}' is not suited for agent mode", modelNode.value), - severity: MarkerSeverity.Warning, - ...modelNode.range, - }); - } - - } - findModelByName(languageModes: string[], modelName: string): ILanguageModelChatMetadata | undefined { - for (const model of languageModes) { - const metadata = this.languageModelsService.lookupLanguageModel(model); - if (metadata && metadata.isUserSelectable !== false && ILanguageModelChatMetadata.matchesQualifiedName(modelName, metadata)) { - return metadata; - } - } - return undefined; - } - - validateTools(tools: PromptToolsMetadata | undefined, modeKind: string | ChatModeKind | undefined, markers: IMarkerData[]) { - if (!tools || tools.value === undefined || modeKind === ChatModeKind.Ask || modeKind === ChatModeKind.Edit) { - return; - } - const toolNames = new Set(tools.value); - if (toolNames.size === 0) { - return; - } - for (const tool of this.languageModelToolsService.getTools()) { - toolNames.delete(tool.toolReferenceName ?? tool.displayName); - } - for (const toolSet of this.languageModelToolsService.toolSets.get()) { - toolNames.delete(toolSet.referenceName); - } - - for (const toolName of toolNames) { - const range = tools.getToolRange(toolName); - if (range) { - markers.push({ - message: localize('promptHeaderDiagnosticsProvider.toolNotFound', "Unknown tool '{0}'", toolName), - severity: MarkerSeverity.Warning, - ...range, - }); - } - } - } - - validateMode(modeNode: PromptModeMetadata | undefined, markers: IMarkerData[]): IChatMode | undefined { - if (!modeNode || modeNode.value === undefined) { - return; - } - - const modeValue = modeNode.value; - const modes = this.chatModeService.getModes(); - const availableModes = []; - - // Check if mode exists in builtin or custom modes - for (const mode of Iterable.concat(modes.builtin, modes.custom)) { - if (mode.name === modeValue) { - return mode; - } - availableModes.push(mode.name); // collect all available mode names - } - - markers.push({ - message: localize('promptHeaderDiagnosticsProvider.modeNotFound', "Unknown mode '{0}'. Available modes: {1}", modeValue, availableModes.join(', ')), - severity: MarkerSeverity.Warning, - ...modeNode.range, - }); - return undefined; - - } - - /** - * Returns a string representation of this object. - */ - public override toString(): string { - return `prompt-header-diagnostics:${this.model.uri.path}`; - } -} - -/** - * Convert a provided diagnostic object into a marker data object. - */ -function toMarker(diagnostic: TDiagnostic): IMarkerData { - if (diagnostic instanceof PromptMetadataWarning) { - return { - message: diagnostic.message, - severity: MarkerSeverity.Warning, - ...diagnostic.range, - }; - } - - if (diagnostic instanceof PromptMetadataError) { - return { - message: diagnostic.message, - severity: MarkerSeverity.Error, - ...diagnostic.range, - }; - } - - assertNever( - diagnostic, - `Unknown prompt metadata diagnostic type '${diagnostic}'.`, - ); -} - -/** - * The class that manages creation and disposal of {@link PromptHeaderDiagnosticsProvider} - * classes for each specific editor text model. - */ -export class PromptHeaderDiagnosticsInstanceManager extends ProviderInstanceManagerBase { - protected override get InstanceClass(): TProviderClass { - return PromptHeaderDiagnosticsProvider; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/promptLinkDiagnosticsProvider.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/promptLinkDiagnosticsProvider.ts deleted file mode 100644 index 3809f67ca8f..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/promptLinkDiagnosticsProvider.ts +++ /dev/null @@ -1,98 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { IPromptsService } from '../service/promptsService.js'; -import { IPromptFileReference } from '../parsers/types.js'; -import { ProviderInstanceBase } from './providerInstanceBase.js'; -import { ITextModel } from '../../../../../../editor/common/model.js'; -import { assertDefined } from '../../../../../../base/common/types.js'; -import { ProviderInstanceManagerBase, TProviderClass } from './providerInstanceManagerBase.js'; -import { IMarkerData, IMarkerService, MarkerSeverity } from '../../../../../../platform/markers/common/markers.js'; -import { IFileService } from '../../../../../../platform/files/common/files.js'; -import { localize } from '../../../../../../nls.js'; - -/** - * Unique ID of the markers provider class. - */ -const MARKERS_OWNER_ID = 'prompt-link-diagnostics-provider'; - -/** - * Prompt links diagnostics provider for a single text model. - */ -class PromptLinkDiagnosticsProvider extends ProviderInstanceBase { - constructor( - model: ITextModel, - @IPromptsService promptsService: IPromptsService, - @IMarkerService private readonly markerService: IMarkerService, - @IFileService private readonly fileService: IFileService - ) { - super(model, promptsService); - } - - /** - * Update diagnostic markers for the current editor. - */ - protected override async onPromptSettled(): Promise { - // clean up all previously added markers - this.markerService.remove(MARKERS_OWNER_ID, [this.model.uri]); - - const markers: IMarkerData[] = []; - - const stats = await this.fileService.resolveAll(this.parser.references.map(ref => ({ resource: ref.uri }))); - for (let i = 0; i < stats.length; i++) { - if (!stats[i].success) { - markers.push(toMarker(this.parser.references[i], localize('fileNotFound', 'File not found.'))); - } - } - - this.markerService.changeOne( - MARKERS_OWNER_ID, - this.model.uri, - markers, - ); - } - - /** - * Returns a string representation of this object. - */ - public override toString(): string { - return `prompt-link-diagnostics:${this.model.uri.path}`; - } -} - -/** - * Convert a prompt link with an issue to a marker data. - * - * @throws - * - if there is no link issue (e.g., `topError` undefined) - * - if there is no link range to highlight (e.g., `linkRange` undefined) - * - if the original error is of `NotPromptFile` type - we don't want to - * show diagnostic markers for non-prompt file links in the prompts - */ -function toMarker(link: IPromptFileReference, message: string): IMarkerData { - const { linkRange } = link; - - assertDefined( - linkRange, - 'Link range must to be defined.', - ); - - - return { - message: message, - severity: MarkerSeverity.Warning, - ...linkRange, - }; -} - -/** - * The class that manages creation and disposal of {@link PromptLinkDiagnosticsProvider} - * classes for each specific editor text model. - */ -export class PromptLinkDiagnosticsInstanceManager extends ProviderInstanceManagerBase { - protected override get InstanceClass(): TProviderClass { - return PromptLinkDiagnosticsProvider; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/providerInstanceBase.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/providerInstanceBase.ts deleted file mode 100644 index da51052ab58..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/providerInstanceBase.ts +++ /dev/null @@ -1,54 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { IPromptsService, TSharedPrompt } from '../service/promptsService.js'; -import { ITextModel } from '../../../../../../editor/common/model.js'; -import { ObservableDisposable } from '../utils/observableDisposable.js'; -import { CancellationToken, CancellationTokenSource } from '../../../../../../base/common/cancellation.js'; - -/** - * Abstract base class for all reusable prompt file providers. - */ -export abstract class ProviderInstanceBase extends ObservableDisposable { - /** - * Function that is called when the prompt parser is settled. - */ - protected abstract onPromptSettled(error: Error | undefined, token: CancellationToken): Promise; - - /** - * Returns a string representation of this object. - */ - public abstract override toString(): string; - - /** - * The prompt parser instance. - */ - protected readonly parser: TSharedPrompt; - - constructor( - protected readonly model: ITextModel, - @IPromptsService promptsService: IPromptsService, - ) { - super(); - - this.parser = promptsService.getSyntaxParserFor(model); - - this._register( - this.parser.onDispose(this.dispose.bind(this)), - ); - - let cancellationSource = new CancellationTokenSource(); - this._register( - this.parser.onSettled((error) => { - cancellationSource.dispose(true); - cancellationSource = new CancellationTokenSource(); - - this.onPromptSettled(error, cancellationSource.token); - }), - ); - - this.parser.start(); - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/providerInstanceManagerBase.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/providerInstanceManagerBase.ts deleted file mode 100644 index 36ff2339b49..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/languageProviders/providerInstanceManagerBase.ts +++ /dev/null @@ -1,176 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { ProviderInstanceBase } from './providerInstanceBase.js'; -import { assert } from '../../../../../../base/common/assert.js'; -import { ITextModel } from '../../../../../../editor/common/model.js'; -import { assertDefined } from '../../../../../../base/common/types.js'; -import { Disposable } from '../../../../../../base/common/lifecycle.js'; -import { ObjectCache } from '../utils/objectCache.js'; -import { INSTRUCTIONS_LANGUAGE_ID, MODE_LANGUAGE_ID, PROMPT_LANGUAGE_ID } from '../promptTypes.js'; -import { IModelService } from '../../../../../../editor/common/services/model.js'; -import { PromptsConfig } from '../config/config.js'; -import { IEditorService } from '../../../../../services/editor/common/editorService.js'; -import { IDiffEditor, IEditor, IEditorModel } from '../../../../../../editor/common/editorCommon.js'; -import { IInstantiationService } from '../../../../../../platform/instantiation/common/instantiation.js'; -import { IConfigurationService } from '../../../../../../platform/configuration/common/configuration.js'; - -/** - * Type for a text editor that is used for reusable prompt files. - */ -export interface IPromptFileEditor extends IEditor { - readonly getModel: () => ITextModel; -} - -/** - * Type for a class that can create a new provider instance. - */ -export type TProviderClass = new (editor: ITextModel, ...args: any[]) => TInstance; - -/** - * A generic base class that manages creation and disposal of {@link TInstance} - * objects for each specific editor object that is used for reusable prompt files. - */ -export abstract class ProviderInstanceManagerBase extends Disposable { - /** - * Currently available {@link TInstance} instances. - */ - private readonly instances: ObjectCache; - - /** - * Class object of the managed {@link TInstance}. - */ - protected abstract get InstanceClass(): TProviderClass; - - constructor( - @IModelService modelService: IModelService, - @IEditorService editorService: IEditorService, - @IInstantiationService instantiationService: IInstantiationService, - @IConfigurationService configService: IConfigurationService, - ) { - super(); - - // cache of managed instances - this.instances = this._register( - new ObjectCache((model: ITextModel) => { - assert( - model.isDisposed() === false, - 'Text model must not be disposed.', - ); - - // sanity check - the new TS/JS discrepancies regarding fields initialization - // logic mean that this can be `undefined` during runtime while defined in TS - assertDefined( - this.InstanceClass, - 'Instance class field must be defined.', - ); - - const instance: TInstance = instantiationService.createInstance( - this.InstanceClass, - model, - ); - - // this is a sanity check and the contract of the object cache, - // we must return a non-disposed object from this factory function - instance.assertNotDisposed( - 'Created instance must not be disposed.', - ); - - return instance; - }), - ); - - // if the feature is disabled, do not create any providers - if (PromptsConfig.enabled(configService) === false) { - return; - } - - // subscribe to changes of the active editor - this._register(editorService.onDidActiveEditorChange(() => { - const { activeTextEditorControl } = editorService; - if (activeTextEditorControl === undefined) { - return; - } - - this.handleNewEditor(activeTextEditorControl); - })); - - // handle existing visible text editors - editorService - .visibleTextEditorControls - .forEach(this.handleNewEditor.bind(this)); - - // subscribe to "language change" events for all models - this._register( - modelService.onModelLanguageChanged((event) => { - const { model, oldLanguageId } = event; - - // if language is set to `prompt` or `instructions` language, handle that model - if (isPromptFileModel(model)) { - this.instances.get(model); - return; - } - - // if the language is changed away from `prompt` or `instructions`, - // remove and dispose provider for this model - if (isPromptFile(oldLanguageId)) { - this.instances.remove(model, true); - return; - } - }), - ); - } - - /** - * Initialize a new {@link TInstance} for the given editor. - */ - private handleNewEditor(editor: IEditor | IDiffEditor): this { - const model = editor.getModel(); - if (model === null) { - return this; - } - - if (isPromptFileModel(model) === false) { - return this; - } - - // note! calling `get` also creates a provider if it does not exist; - // and the provider is auto-removed when the editor is disposed - this.instances.get(model); - - return this; - } -} - -/** - * Check if provided language ID is one of the prompt file languages. - */ -function isPromptFile(languageId: string): boolean { - return [ - PROMPT_LANGUAGE_ID, - INSTRUCTIONS_LANGUAGE_ID, - MODE_LANGUAGE_ID, - ].includes(languageId); -} - -/** - * Check if a provided model is used for prompt files. - */ -function isPromptFileModel(model: IEditorModel): model is ITextModel { - // we support only `text editors` for now so filter out `diff` ones - if ('modified' in model || 'model' in model) { - return false; - } - - if (model.isDisposed()) { - return false; - } - - if (isPromptFile(model.getLanguageId()) === false) { - return false; - } - - return true; -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/basePromptParser.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/basePromptParser.ts deleted file mode 100644 index 9fe72885d03..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/basePromptParser.ts +++ /dev/null @@ -1,728 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { TopError } from './topError.js'; -import { ChatModeKind } from '../../constants.js'; -import { TMetadata } from './promptHeader/headerBase.js'; -import { ModeHeader } from './promptHeader/modeHeader.js'; -import { URI } from '../../../../../../base/common/uri.js'; -import { PromptToken } from '../codecs/tokens/promptToken.js'; -import * as path from '../../../../../../base/common/path.js'; -import { ChatPromptCodec } from '../codecs/chatPromptCodec.js'; -import { FileReference } from '../codecs/tokens/fileReference.js'; -import { ChatPromptDecoder } from '../codecs/chatPromptDecoder.js'; -import { assertDefined } from '../../../../../../base/common/types.js'; -import { Emitter } from '../../../../../../base/common/event.js'; -import { DeferredPromise } from '../../../../../../base/common/async.js'; -import { InstructionsHeader } from './promptHeader/instructionsHeader.js'; -import { ILogService } from '../../../../../../platform/log/common/log.js'; -import { PromptVariable, PromptVariableWithData } from '../codecs/tokens/promptVariable.js'; -import type { IPromptContentsProvider } from '../contentProviders/types.js'; -import type { TPromptReference, ITopError, TVariableReference } from './types.js'; -import { type IDisposable } from '../../../../../../base/common/lifecycle.js'; -import { assert, assertNever } from '../../../../../../base/common/assert.js'; -import { basename, dirname, joinPath } from '../../../../../../base/common/resources.js'; -import { BaseToken } from '../codecs/base/baseToken.js'; -import { VSBufferReadableStream } from '../../../../../../base/common/buffer.js'; -import { type IRange, Range } from '../../../../../../editor/common/core/range.js'; -import { PromptHeader, type TPromptMetadata } from './promptHeader/promptHeader.js'; -import { ObservableDisposable } from '../utils/observableDisposable.js'; -import { INSTRUCTIONS_LANGUAGE_ID, MODE_LANGUAGE_ID, PROMPT_LANGUAGE_ID } from '../promptTypes.js'; -import { LinesDecoder } from '../codecs/base/linesCodec/linesDecoder.js'; -import { IInstantiationService } from '../../../../../../platform/instantiation/common/instantiation.js'; -import { MarkdownLink } from '../codecs/base/markdownCodec/tokens/markdownLink.js'; -import { MarkdownToken } from '../codecs/base/markdownCodec/tokens/markdownToken.js'; -import { FrontMatterHeader } from '../codecs/base/markdownExtensionsCodec/tokens/frontMatterHeader.js'; -import { OpenFailed, NotPromptFile, RecursiveReference, FolderReference, ResolveError } from '../../promptFileReferenceErrors.js'; -import { type IPromptContentsProviderOptions } from '../contentProviders/promptContentsProviderBase.js'; -import { CancellationToken } from '../../../../../../base/common/cancellation.js'; -import { IWorkbenchEnvironmentService } from '../../../../../services/environment/common/environmentService.js'; -import { Schemas } from '../../../../../../base/common/network.js'; - -/** - * Options of the {@link BasePromptParser} class. - */ -export interface IBasePromptParserOptions { -} - -export type IPromptParserOptions = IBasePromptParserOptions & IPromptContentsProviderOptions; - - -/** - * Error conditions that may happen during the file reference resolution. - */ -export type TErrorCondition = OpenFailed | RecursiveReference | FolderReference | NotPromptFile; - -/** - * Base prompt parser class that provides a common interface for all - * prompt parsers that are responsible for parsing chat prompt syntax. - */ -export class BasePromptParser extends ObservableDisposable { - /** - * Options passed to the constructor. - */ - protected readonly options: IBasePromptParserOptions; - - /** - * List of all tokens that were parsed from the prompt contents so far. - */ - public get tokens(): readonly BaseToken[] { - return [...this.receivedTokens]; - } - /** - * Private field behind the readonly {@link tokens} property. - */ - private receivedTokens: BaseToken[] = []; - - /** - * List of file references in the prompt file. - */ - private readonly _references: TPromptReference[] = []; - - /** - * List of variable references in the prompt file. - */ - private readonly _variableReferences: TVariableReference[] = []; - - /** - * Reference to the prompt header object that holds metadata associated - * with the prompt. - */ - private promptHeader?: PromptHeader | InstructionsHeader | ModeHeader | undefined; - - /** - * Reference to the prompt header object that holds metadata associated - * with the prompt. - */ - public get header(): PromptHeader | InstructionsHeader | ModeHeader | undefined { - return this.promptHeader; - } - - /** - * Get contents of the prompt body. - */ - public async getBody(): Promise { - const startLineNumber = (this.header !== undefined) - ? this.header.range.endLineNumber + 1 - : 1; - - const decoder = new LinesDecoder( - await this.promptContentsProvider.contents, - ); - - const tokens = (await decoder.consumeAll()) - .filter(({ range }) => { - return (range.startLineNumber >= startLineNumber); - }); - - return BaseToken.render(tokens); - } - - /** - * Get the full contents of the prompt, including the header - */ - public async getFullContent(): Promise { - const decoder = new LinesDecoder( - await this.promptContentsProvider.contents, - ); - const tokens = await decoder.consumeAll(); - return BaseToken.render(tokens); - } - - /** - * The event is fired when lines or their content change. - */ - private readonly _onUpdate = this._register(new Emitter()); - /** - * Subscribe to the event that is fired the parser state or contents - * changes, including changes in the possible prompt child references. - */ - public readonly onUpdate = this._onUpdate.event; - - /** - * Event that is fired when the current prompt parser is settled. - */ - private readonly _onSettled = this._register(new Emitter()); - - /** - * Event that is fired when the current prompt parser is settled. - */ - public onSettled( - callback: (error?: Error) => void, - ): IDisposable { - const disposable = this._onSettled.event(callback); - const streamEnded = (this.stream?.ended && (this.stream.isDisposed === false)); - - // if already in the error state or stream has already ended, - // invoke the callback immediately but asynchronously - if (streamEnded || this.errorCondition) { - setTimeout(callback.bind(undefined, this.errorCondition)); - - return disposable; - } - - return disposable; - } - - /** - * If failed to parse prompt contents, this property has - * an error object that describes the failure reason. - */ - private _errorCondition?: ResolveError; - - /** - * If file reference resolution fails, this attribute will be set - * to an error instance that describes the error condition. - */ - public get errorCondition(): ResolveError | undefined { - return this._errorCondition; - } - - /** - * Whether file references resolution failed. - * Set to `undefined` if the `resolve` method hasn't been ever called yet. - */ - public get resolveFailed(): boolean | undefined { - if (!this.firstParseResult.gotFirstResult) { - return undefined; - } - - return !!this._errorCondition; - } - - /** - * The promise is resolved when at least one parse result (a stream or - * an error) has been received from the prompt contents provider. - */ - private readonly firstParseResult = new FirstParseResult(); - - /** - * Returned promise is resolved when the parser process is settled. - * The settled state means that the prompt parser stream exists and - * has ended, or an error condition has been set in case of failure. - * - * Furthermore, this function can be called multiple times and will - * block until the latest prompt contents parsing logic is settled - * (e.g., for every `onContentChanged` event of the prompt source). - */ - public async settled(): Promise { - assert( - this.started, - 'Cannot wait on the parser that did not start yet.', - ); - - await this.firstParseResult.promise; - - if (this.errorCondition) { - return false; - } - - // by the time when the `firstParseResult` promise is resolved, - // this object may have been already disposed, hence noop - if (this.isDisposed) { - return false; - } - - assertDefined( - this.stream, - 'No stream reference found.', - ); - - const completed = await this.stream.settled; - - // if prompt header exists, also wait for it to be settled - if (this.promptHeader) { - const headerCompleted = await this.promptHeader.settled; - if (!headerCompleted) { - return false; - } - } - - return completed; - } - - constructor( - private readonly promptContentsProvider: TContentsProvider, - options: IBasePromptParserOptions, - @IInstantiationService protected readonly instantiationService: IInstantiationService, - @IWorkbenchEnvironmentService private readonly envService: IWorkbenchEnvironmentService, - @ILogService protected readonly logService: ILogService, - ) { - super(); - - this.options = options; - - this._register( - this.promptContentsProvider.onContentChanged((streamOrError) => { - // process the received message - this.onContentsChanged(streamOrError); - - // indicate that we've received at least one `onContentChanged` event - this.firstParseResult.end(); - }), - ); - - // dispose self when contents provider is disposed - this._register( - this.promptContentsProvider.onDispose(this.dispose.bind(this)), - ); - } - - /** - * The latest received stream of prompt tokens, if any. - */ - private stream: ChatPromptDecoder | undefined; - - /** - * Handler the event event that is triggered when prompt contents change. - * - * @param streamOrError Either a binary stream of file contents, or an error object - * that was generated during the reference resolve attempt. - * @param seenReferences List of parent references that we've have already seen - * during the process of traversing the references tree. It's - * used to prevent the tree navigation to fall into an infinite - * references recursion. - */ - private onContentsChanged( - streamOrError: VSBufferReadableStream | ResolveError - ): void { - // dispose and cleanup the previously received stream - // object or an error condition, if any received yet - this.stream?.dispose(); - delete this.stream; - delete this._errorCondition; - this.receivedTokens = []; - - // cleanup current prompt header object - this.promptHeader?.dispose(); - delete this.promptHeader; - - // dispose all currently existing references - this.disposeReferences(); - - // if an error received, set up the error condition and stop - if (streamOrError instanceof ResolveError) { - this._errorCondition = streamOrError; - this._onUpdate.fire(); - - // when error received fire the 'onSettled' event immediately - this._onSettled.fire(streamOrError); - - return; - } - - // decode the byte stream to a stream of prompt tokens - this.stream = ChatPromptCodec.decode(streamOrError); - - /** - * !NOTE! The order of event subscriptions below is critical here because - * the `data` event is also starts the stream, hence changing - * the order of event subscriptions can lead to race conditions. - * See {@link ReadableStreamEvents} for more info. - */ - - // on error or stream end, dispose the stream and fire the update event - this.stream.on('error', this.onStreamEnd.bind(this, this.stream)); - this.stream.on('end', this.onStreamEnd.bind(this, this.stream)); - - // when some tokens received, process and store the references - this.stream.on('data', (token) => { - // store all markdown and prompt token references - if ((token instanceof MarkdownToken) || (token instanceof PromptToken)) { - this.receivedTokens.push(token); - } - - // if a prompt header token received, create a new prompt header instance - if (token instanceof FrontMatterHeader) { - return this.createHeader(token); - } - - // try to convert a prompt variable with data token into a file reference - if (token instanceof PromptVariableWithData) { - try { - if (token.name === 'file') { - this.handleLinkToken(FileReference.from(token)); - } - } catch (error) { - // the `FileReference.from` call might throw if the `PromptVariableWithData` token - // can not be converted into a valid `#file` reference, hence we ignore the error - } - } else if (token instanceof PromptVariable) { - this.handleVariableToken(token); - } - - // note! the `isURL` is a simple check and needs to be improved to truly - // handle only file references, ignoring broken URLs or references - if (token instanceof MarkdownLink && !token.isURL) { - this.handleLinkToken(token); - } - }); - - // calling `start` on a disposed stream throws, so we warn and return instead - if (this.stream.isDisposed) { - this.logService.warn( - `[prompt parser][${basename(this.uri)}] cannot start stream that has been already disposed, aborting`, - ); - - return; - } - - // start receiving data on the stream - this.stream.start(); - } - - /** - * Create header object base on the target prompt file language ID. - * The language ID is important here, because it defines what type - * of metadata is valid for a prompt file and what type of related - * diagnostics we would show to the user. - */ - private createHeader(headerToken: FrontMatterHeader): void { - const { languageId } = this.promptContentsProvider; - - if (languageId === PROMPT_LANGUAGE_ID) { - this.promptHeader = new PromptHeader(headerToken, languageId); - } - - if (languageId === INSTRUCTIONS_LANGUAGE_ID) { - this.promptHeader = new InstructionsHeader(headerToken, languageId); - } - - if (languageId === MODE_LANGUAGE_ID) { - this.promptHeader = new ModeHeader(headerToken, languageId); - } - - this.promptHeader?.start(); - } - - /** - * Handle a new reference token inside prompt contents. - */ - private handleLinkToken(token: FileReference | MarkdownLink): this { - - let referenceUri: URI; - if (path.isAbsolute(token.path)) { - referenceUri = URI.file(token.path); - if (this.envService.remoteAuthority) { - referenceUri = referenceUri.with({ - scheme: Schemas.vscodeRemote, - authority: this.envService.remoteAuthority, - }); - } - } else { - referenceUri = joinPath(dirname(this.uri), token.path); - } - this._references.push(new PromptReference(referenceUri, token)); - - this._onUpdate.fire(); - - return this; - } - - private handleVariableToken(token: PromptVariable): this { - - this._variableReferences.push({ name: token.name, range: token.range }); - - this._onUpdate.fire(); - - return this; - } - - /** - * Handle the `stream` end event. - * - * @param stream The stream that has ended. - * @param error Optional error object if stream ended with an error. - */ - private onStreamEnd( - stream: ChatPromptDecoder, - error?: Error, - ): this { - // decoders can fire the 'end' event also when they are get disposed, - // but because we dispose them when a new stream is received, we can - // safely ignore the event in this case - if (stream.isDisposed === true) { - return this; - } - - if (error) { - this.logService.warn( - `[prompt parser][${basename(this.uri)}] received an error on the chat prompt decoder stream: ${error}`, - ); - } - - this._onUpdate.fire(); - this._onSettled.fire(error); - - return this; - } - - - private disposeReferences(): void { - - - this._references.length = 0; - this._variableReferences.length = 0; - } - - /** - * Private attribute to track if the {@link start} - * method has been already called at least once. - */ - private started: boolean = false; - - /** - * Start the prompt parser. - */ - public start(token?: CancellationToken): this { - // if already started, nothing to do - if (this.started) { - return this; - } - this.started = true; - - - // if already in the error state that could be set - // in the constructor, then nothing to do - if (this.errorCondition) { - return this; - } - - this.promptContentsProvider.start(token); - return this; - } - - /** - * Associated URI of the prompt. - */ - public get uri(): URI { - return this.promptContentsProvider.uri; - } - - /** - * Get a list of immediate child references of the prompt. - */ - public get references(): readonly TPromptReference[] { - return [...this._references]; - } - - /** - * Get a list of variable references of the prompt. - */ - public get variableReferences(): readonly TVariableReference[] { - return [...this._variableReferences]; - } - - /** - * Valid metadata records defined in the prompt header. - */ - public get metadata(): TMetadata | null { - const { promptType } = this.promptContentsProvider; - if (promptType === 'non-prompt') { - return null; - } - - if (this.header === undefined) { - return { promptType }; - } - - if (this.header instanceof InstructionsHeader || this.header instanceof ModeHeader) { - return { promptType, ...this.header.metadata }; - } - - const { tools, mode, description, model } = this.header.metadata; - - const result: Partial = {}; - - if (description !== undefined) { - result.description = description; - } - - if (tools !== undefined && mode !== ChatModeKind.Ask && mode !== ChatModeKind.Edit) { - result.tools = tools; - // Preserve custom mode if specified, otherwise default to Agent - result.mode = mode || ChatModeKind.Agent; - } else if (mode !== undefined) { - result.mode = mode; - } - - if (model !== undefined) { - result.model = model; - } - - return { promptType, ...result }; - } - - /** - * The top most error of the current reference or any of its - * possible child reference errors. - */ - public get topError(): ITopError | undefined { - if (this.errorCondition) { - return new TopError({ - errorSubject: 'root', - errorsCount: 1, - originalError: this.errorCondition, - }); - } - - return undefined; - } - - /** - * Returns a string representation of this object. - */ - public override toString(): string { - return `prompt:${this.uri.path}`; - } - - /** - * @inheritdoc - */ - public override dispose(): void { - if (this.isDisposed) { - return; - } - - this.disposeReferences(); - - this.stream?.dispose(); - delete this.stream; - - this.promptHeader?.dispose(); - delete this.promptHeader; - - super.dispose(); - } -} - -/** - * Prompt reference object represents any reference inside prompt text - * contents. For instance the file variable(`#file:/path/to/file.md`) or - * a markdown link(`[#file:file.md](/path/to/file.md)`). - */ -export class PromptReference implements TPromptReference { - - - constructor( - public readonly uri: URI, - public readonly token: FileReference | MarkdownLink, - ) { - } - - /** - * Get the range of the `link` part of the reference. - */ - public get linkRange(): IRange | undefined { - // `#file:` references - if (this.token instanceof FileReference) { - return this.token.dataRange; - } - - // `markdown link` references - if (this.token instanceof MarkdownLink) { - return this.token.linkRange; - } - - return undefined; - } - - /** - * Type of the reference, - either a prompt `#file` variable, - * or a `markdown link` reference (`[caption](/path/to/file.md)`). - */ - public get type(): 'file' { - if (this.token instanceof FileReference) { - return 'file'; - } - - if (this.token instanceof MarkdownLink) { - return 'file'; - } - - assertNever( - this.token, - `Unknown token type '${this.token}'.`, - ); - } - - /** - * Subtype of the reference, - either a prompt `#file` variable, - * or a `markdown link` reference (`[caption](/path/to/file.md)`). - */ - public get subtype(): 'prompt' | 'markdown' { - if (this.token instanceof FileReference) { - return 'prompt'; - } - - if (this.token instanceof MarkdownLink) { - return 'markdown'; - } - - assertNever( - this.token, - `Unknown token type '${this.token}'.`, - ); - } - - public get range(): Range { - return this.token.range; - } - - public get path(): string { - return this.token.path; - } - - public get text(): string { - return this.token.text; - } - - /** - * Returns a string representation of this object. - */ - public toString(): string { - return `prompt-reference/${this.type}:${this.subtype}/${this.token}`; - } -} - -/** - * A tiny utility object that helps us to track existence - * of at least one parse result from the content provider. - */ -class FirstParseResult extends DeferredPromise { - /** - * Private attribute to track if we have - * received at least one result. - */ - private _gotResult = false; - - /** - * Whether we've received at least one result. - */ - public get gotFirstResult(): boolean { - return this._gotResult; - } - - /** - * Get underlying promise reference. - */ - public get promise(): Promise { - return this.p; - } - - /** - * Complete the underlying promise. - */ - public end(): void { - this._gotResult = true; - super.complete(void 0) - .catch(() => { - // the complete method is never fails - // so we can ignore the error here - }); - - return; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/filePromptParser.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/filePromptParser.ts deleted file mode 100644 index c73efe10884..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/filePromptParser.ts +++ /dev/null @@ -1,37 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { URI } from '../../../../../../base/common/uri.js'; -import { ILogService } from '../../../../../../platform/log/common/log.js'; -import { BasePromptParser, IPromptParserOptions } from './basePromptParser.js'; -import { FilePromptContentProvider } from '../contentProviders/filePromptContentsProvider.js'; -import { IInstantiationService } from '../../../../../../platform/instantiation/common/instantiation.js'; -import { IWorkbenchEnvironmentService } from '../../../../../services/environment/common/environmentService.js'; - -/** - * Class capable of parsing prompt syntax out of a provided file, - * including all the nested child file references it may have. - */ -export class FilePromptParser extends BasePromptParser { - constructor( - uri: URI, - options: IPromptParserOptions, - @IInstantiationService instantiationService: IInstantiationService, - @IWorkbenchEnvironmentService envService: IWorkbenchEnvironmentService, - @ILogService logService: ILogService, - ) { - const contentsProvider = instantiationService.createInstance(FilePromptContentProvider, uri, options); - super(contentsProvider, options, instantiationService, envService, logService); - - this._register(contentsProvider); - } - - /** - * Returns a string representation of this object. - */ - public override toString(): string { - return `file-prompt:${this.uri.path}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/diagnostics.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/diagnostics.ts deleted file mode 100644 index 754293f6dcf..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/diagnostics.ts +++ /dev/null @@ -1,47 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { Range } from '../../../../../../../editor/common/core/range.js'; - -/** - * List of all currently supported diagnostic types. - */ -export type TDiagnostic = PromptMetadataWarning | PromptMetadataError; - -/** - * Diagnostics object that hold information about some issue - * related to the prompt header metadata. - */ -export abstract class PromptMetadataDiagnostic { - constructor( - public readonly range: Range, - public readonly message: string, - ) { } - - /** - * String representation of the diagnostic object. - */ - public abstract toString(): string; -} - -/** - * Diagnostics object that hold information about some - * non-fatal issue related to the prompt header metadata. - */ -export class PromptMetadataWarning extends PromptMetadataDiagnostic { - public override toString(): string { - return `warning(${this.message})${this.range}`; - } -} - -/** - * Diagnostics object that hold information about some - * fatal issue related to the prompt header metadata. - */ -export class PromptMetadataError extends PromptMetadataDiagnostic { - public override toString(): string { - return `error(${this.message})${this.range}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/headerBase.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/headerBase.ts deleted file mode 100644 index eedebeabeed..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/headerBase.ts +++ /dev/null @@ -1,264 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { type TModeMetadata } from './modeHeader.js'; -import { localize } from '../../../../../../../nls.js'; -import { type TPromptMetadata } from './promptHeader.js'; -import { type IMetadataRecord } from './metadata/base/record.js'; -import { type TInstructionsMetadata } from './instructionsHeader.js'; -import { Range } from '../../../../../../../editor/common/core/range.js'; -import { Disposable } from '../../../../../../../base/common/lifecycle.js'; -import { ObjectStream } from '../../codecs/base/utils/objectStream.js'; -import { PromptMetadataError, PromptMetadataWarning, type TDiagnostic } from './diagnostics.js'; -import { SimpleToken } from '../../codecs/base/simpleCodec/tokens/tokens.js'; -import { FrontMatterRecord } from '../../codecs/base/frontMatterCodec/tokens/index.js'; -import { FrontMatterHeader } from '../../codecs/base/markdownExtensionsCodec/tokens/frontMatterHeader.js'; -import { FrontMatterDecoder, type TFrontMatterToken } from '../../codecs/base/frontMatterCodec/frontMatterDecoder.js'; -import { PromptDescriptionMetadata } from './metadata/description.js'; - -/** - * A metadata utility class "dehydrated" into a plain data object with - * semi-primitive record values (string, boolean, string[], boolean[], etc.). - */ -export type TDehydrated = { - [K in keyof T]: T[K] extends IMetadataRecord ? (U extends undefined ? undefined : NonNullable) : undefined; -}; - -/** - * Metadata defined in the prompt header. - */ -export interface IHeaderMetadata { - /** - * Description metadata in the prompt header. - */ - description: PromptDescriptionMetadata; -} - -/** - * Metadata for prompt/instruction/mode files. - */ -export type THeaderMetadata = Partial>; - -/** - * Metadata defined in the header of prompt/instruction/mode files. - */ -export type TMetadata = TPromptMetadata | TModeMetadata | TInstructionsMetadata; - -/** - * Base class for prompt/instruction/mode headers. - */ -export abstract class HeaderBase< - TMetadata extends IHeaderMetadata, -> extends Disposable { - /** - * Underlying decoder for a Front Matter header. - */ - private readonly stream: FrontMatterDecoder; - - /** - * Metadata records. - */ - protected readonly meta: Partial; - - /** - * Data object with all header's metadata records. - */ - public get metadata(): Partial> { - const result: Partial> = {}; - - for (const [entryName, entryValue] of Object.entries(this.meta)) { - if (entryValue?.value === undefined) { - continue; - } - - // note! we have to resort to `Object.assign()` here because - // the `Object.entries()` call looses type information - Object.assign(result, { - [entryName]: entryValue.value, - }); - } - - return result; - } - - /** - * A copy of metadata object with utility classes as values - * for each of prompt header's record. - * - * Please use {@link metadata} instead if all you need to read is - * the plain "data" object representation of valid metadata records. - */ - public get metadataUtility(): Partial { - return { ...this.meta }; - } - - /** - * List of all unique metadata record names. - */ - private readonly recordNames: Set; - - /** - * List of all issues found while parsing the prompt header. - */ - protected readonly issues: TDiagnostic[]; - - /** - * List of all diagnostic issues found while parsing - * the prompt header. - */ - public get diagnostics(): readonly TDiagnostic[] { - return this.issues; - } - - /** - * Full range of the header in the original document. - */ - public get range(): Range { - return this.token.range; - } - - constructor( - public readonly token: FrontMatterHeader, - public readonly languageId: string, - ) { - super(); - - this.issues = []; - this.meta = {}; - this.recordNames = new Set(); - - this.stream = this._register( - new FrontMatterDecoder( - ObjectStream.fromArray([...token.contentToken.children]), - ), - ); - this.stream.onData(this.onData.bind(this)); - this.stream.onError(this.onError.bind(this)); - } - - /** - * Process a front matter record token, which includes: - * - validation of the record and whether it is compatible with other header records - * - adding validation-related diagnostic messages to the {@link issues} list - * - setting associated utility class for the record on the {@link meta} object - * - * @returns a boolean flag that indicates whether the token was handled and therefore - * should not be processed any further. - */ - protected abstract handleToken( - token: FrontMatterRecord, - ): boolean; - - /** - * Process front matter tokens, converting them into - * well-known prompt metadata records. - */ - private onData(token: TFrontMatterToken): void { - // we currently expect only front matter 'records' for - // the prompt metadata, hence add diagnostics for all - // other tokens and ignore them - if ((token instanceof FrontMatterRecord) === false) { - // unless its a simple token, in which case we just ignore it - if (token instanceof SimpleToken) { - return; - } - - this.issues.push( - new PromptMetadataError( - token.range, - localize( - 'prompt.header.diagnostics.unexpected-token', - "Unexpected token '{0}'.", - token.text, - ), - ), - ); - - return; - } - - const recordName = token.nameToken.text; - - // if we already have a record with this name, - // add a warning diagnostic and ignore it - if (this.recordNames.has(recordName)) { - this.issues.push( - new PromptMetadataWarning( - token.range, - localize( - 'prompt.header.metadata.diagnostics.duplicate-record', - "Duplicate property '{0}' will be ignored.", - recordName, - ), - ), - ); - - return; - } - this.recordNames.add(recordName); - - // if the record might be a "description" metadata - // add it to the list of parsed metadata records - if (PromptDescriptionMetadata.isDescriptionRecord(token)) { - const metadata = new PromptDescriptionMetadata(token, this.languageId); - - this.issues.push(...metadata.validate()); - this.meta.description = metadata; - this.recordNames.add(recordName); - return; - } - - // pipe the token to the actual implementation class - // that might to handle it based on the token type - if (this.handleToken(token)) { - return; - } - - // all other records are "unknown" ones - this.issues.push( - new PromptMetadataWarning( - token.range, - localize( - 'prompt.header.metadata.diagnostics.unknown-record', - "Unknown property '{0}' will be ignored.", - recordName, - ), - ), - ); - } - - /** - * Process errors from the underlying front matter decoder. - */ - private onError(error: Error): void { - this.issues.push( - new PromptMetadataError( - this.token.range, - localize( - 'prompt.header.diagnostics.parsing-error', - "Failed to parse prompt header: {0}", - error.message, - ), - ), - ); - } - - /** - * Promise that resolves when parsing process of - * the prompt header completes. - */ - public get settled(): Promise { - return this.stream.settled; - } - - /** - * Starts the parsing process of the prompt header. - */ - public start(): this { - this.stream.start(); - - return this; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/instructionsHeader.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/instructionsHeader.ts deleted file mode 100644 index 1f13901f960..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/instructionsHeader.ts +++ /dev/null @@ -1,44 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { PromptApplyToMetadata } from './metadata/applyTo.js'; -import { HeaderBase, IHeaderMetadata, type TDehydrated } from './headerBase.js'; -import { PromptsType } from '../../promptTypes.js'; -import { FrontMatterRecord } from '../../codecs/base/frontMatterCodec/tokens/index.js'; - -/** - * Metadata utility object for instruction files. - */ -interface IInstructionsMetadata extends IHeaderMetadata { - /** - * Chat 'applyTo' metadata in the prompt header. - */ - applyTo: PromptApplyToMetadata; -} - -/** - * Metadata for instruction files. - */ -export type TInstructionsMetadata = Partial> & { promptType: PromptsType.instructions }; - -/** - * Header object for instruction files. - */ -export class InstructionsHeader extends HeaderBase { - protected override handleToken(token: FrontMatterRecord): boolean { - // if the record might be a "applyTo" metadata - // add it to the list of parsed metadata records - if (PromptApplyToMetadata.isApplyToRecord(token)) { - const metadata = new PromptApplyToMetadata(token, this.languageId); - - this.issues.push(...metadata.validate()); - this.meta.applyTo = metadata; - - return true; - } - - return false; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/applyTo.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/applyTo.ts deleted file mode 100644 index 7f374c0f108..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/applyTo.ts +++ /dev/null @@ -1,122 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { PromptStringMetadata } from './base/string.js'; -import { localize } from '../../../../../../../../nls.js'; -import { INSTRUCTIONS_LANGUAGE_ID } from '../../../promptTypes.js'; -import { isEmptyPattern, parse, splitGlobAware } from '../../../../../../../../base/common/glob.js'; -import { PromptMetadataDiagnostic, PromptMetadataError, PromptMetadataWarning } from '../diagnostics.js'; -import { FrontMatterRecord, FrontMatterToken } from '../../../codecs/base/frontMatterCodec/tokens/index.js'; - -/** - * Name of the metadata record in the prompt header. - */ -const RECORD_NAME = 'applyTo'; - -/** - * Prompt `applyTo` metadata record inside the prompt header. - */ -export class PromptApplyToMetadata extends PromptStringMetadata { - constructor( - recordToken: FrontMatterRecord, - languageId: string, - ) { - super(RECORD_NAME, recordToken, languageId); - } - - public override get recordName(): string { - return RECORD_NAME; - } - - public override validate(): readonly PromptMetadataDiagnostic[] { - super.validate(); - - // if we don't have a value token, validation must - // has failed already so nothing to do more - if (this.valueToken === undefined) { - return this.issues; - } - - // the applyTo metadata makes sense only for 'instruction' prompts - if (this.languageId !== INSTRUCTIONS_LANGUAGE_ID) { - this.issues.push( - new PromptMetadataError( - this.range, - localize( - 'prompt.header.metadata.string.diagnostics.invalid-language', - "The '{0}' header property is only valid in instruction files.", - this.recordName, - ), - ), - ); - - delete this.valueToken; - return this.issues; - } - - const { cleanText } = this.valueToken; - - // warn user if specified glob pattern is not valid - if (this.isValidGlob(cleanText) === false) { - this.issues.push( - new PromptMetadataWarning( - this.valueToken.range, - localize( - 'prompt.header.metadata.applyTo.diagnostics.non-valid-glob', - "Invalid glob pattern '{0}'.", - cleanText, - ), - ), - ); - - delete this.valueToken; - return this.issues; - } - - return this.issues; - } - - /** - * Check if a provided string contains a valid glob pattern. - */ - private isValidGlob( - pattern: string, - ): boolean { - try { - const patterns = splitGlobAware(pattern, ','); - if (patterns.length === 0) { - return false; - } - for (const pattern of patterns) { - - const globPattern = parse(pattern); - if (isEmptyPattern(globPattern)) { - return false; - } - } - return true; - } catch (_error) { - return false; - } - } - - /** - * Check if a provided front matter token is a metadata record - * with name equal to `applyTo`. - */ - public static isApplyToRecord( - token: FrontMatterToken, - ): boolean { - if ((token instanceof FrontMatterRecord) === false) { - return false; - } - - if (token.nameToken.text === RECORD_NAME) { - return true; - } - - return false; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/base/enum.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/base/enum.ts deleted file mode 100644 index 71b7d9e7a4f..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/base/enum.ts +++ /dev/null @@ -1,84 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { PromptStringMetadata } from './string.js'; -import { localize } from '../../../../../../../../../nls.js'; -import { assert } from '../../../../../../../../../base/common/assert.js'; -import { isOneOf } from '../../../../../../../../../base/common/types.js'; -import { PromptMetadataDiagnostic, PromptMetadataError } from '../../diagnostics.js'; -import { FrontMatterSequence } from '../../../../codecs/base/frontMatterCodec/tokens/frontMatterSequence.js'; -import { FrontMatterRecord, FrontMatterString } from '../../../../codecs/base/frontMatterCodec/tokens/index.js'; - -/** - * Enum type is the special case of the {@link PromptStringMetadata string} - * type that can take only a well-defined set of {@link validValues}. - */ -export abstract class PromptEnumMetadata< - TValidValues extends string = string, -> extends PromptStringMetadata { - constructor( - private readonly validValues: readonly TValidValues[], - expectedRecordName: string, - recordToken: FrontMatterRecord, - languageId: string, - ) { - super(expectedRecordName, recordToken, languageId); - } - - /** - * Valid enum value or 'undefined'. - */ - private enumValue: TValidValues | undefined; - /** - * Valid enum value or 'undefined'. - */ - public override get value(): TValidValues | undefined { - return this.enumValue; - } - - /** - * Validate the metadata record has an allowed value. - */ - public override validate(): readonly PromptMetadataDiagnostic[] { - super.validate(); - - if (this.valueToken === undefined) { - return this.issues; - } - - // sanity check for our expectations about the validate call - assert( - this.valueToken instanceof FrontMatterString - || this.valueToken instanceof FrontMatterSequence, - `Record token must be 'string', got '${this.valueToken}'.`, - ); - - const { cleanText } = this.valueToken; - if (isOneOf(cleanText, this.validValues)) { - this.enumValue = cleanText; - - return this.issues; - } - - this.issues.push( - new PromptMetadataError( - this.valueToken.range, - localize( - 'prompt.header.metadata.enum.diagnostics.invalid-value', - "The property '{0}' must be one of {1}, got '{2}'.", - this.recordName, - this.validValues - .map((value) => { - return `'${value}'`; - }).join(' | '), - cleanText, - ), - ), - ); - - delete this.valueToken; - return this.issues; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/base/record.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/base/record.ts deleted file mode 100644 index aa7a4660f66..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/base/record.ts +++ /dev/null @@ -1,108 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { assert } from '../../../../../../../../../base/common/assert.js'; -import { Range } from '../../../../../../../../../editor/common/core/range.js'; -import { PromptMetadataDiagnostic, PromptMetadataError, PromptMetadataWarning } from '../../diagnostics.js'; -import { FrontMatterRecord } from '../../../../codecs/base/frontMatterCodec/tokens/index.js'; - -/** - * Supported primitive types for metadata values in a prompt header. - */ -type TMetadataPrimitive = string | boolean; - -/** - * Supported metadata values in a prompt header. - */ -type TMetadataValue = TMetadataPrimitive | TMetadataPrimitive[]; - -/** - * Interface for a generic metadata record in the prompt header. - */ -export interface IMetadataRecord { - /** - * Value of a metadata record. If the value is not defined, it usually - * means that a record is present but its value is not set or valid. - */ - readonly value: T | undefined; -} - -/** - * Abstract class for all metadata records in the prompt header. - */ -export abstract class PromptMetadataRecord implements IMetadataRecord { - /** - * Private field for tracking all diagnostic issues - * related to this metadata record. - */ - protected readonly issues: PromptMetadataDiagnostic[]; - - /** - * Full range of the metadata's record text in the prompt header. - */ - public get range(): Range { - return this.recordToken.range; - } - - constructor( - protected readonly expectedRecordName: string, - protected readonly recordToken: FrontMatterRecord, - protected readonly languageId: string, - ) { - // validate that the record name has the expected name - const recordName = recordToken.nameToken.text; - assert( - recordName === expectedRecordName, - `Record name must be '${expectedRecordName}', got '${recordName}'.`, - ); - - this.issues = []; - } - - /** - * Name of the metadata record. - */ - public get recordName(): string { - return this.recordToken.nameToken.text; - } - - /** - * Validate the metadata record and collect all issues - * related to its content. - */ - public abstract validate(): readonly PromptMetadataDiagnostic[]; - - /** - * List of all diagnostic issues related to this metadata record. - */ - public get diagnostics(): readonly PromptMetadataDiagnostic[] { - return this.issues; - } - - /** - * Get the value of the metadata record. - */ - public abstract get value(): TValue | undefined; - - /** - * List of all `error` issue diagnostics. - */ - public get errorDiagnostics(): readonly PromptMetadataError[] { - return this.diagnostics - .filter((diagnostic) => { - return (diagnostic instanceof PromptMetadataError); - }); - } - - /** - * List of all `warning` issue diagnostics. - */ - public get warningDiagnostics(): readonly PromptMetadataWarning[] { - return this.diagnostics - .filter((diagnostic) => { - return (diagnostic instanceof PromptMetadataWarning); - }); - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/base/string.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/base/string.ts deleted file mode 100644 index 87423ab25b2..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/base/string.ts +++ /dev/null @@ -1,73 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { PromptMetadataRecord } from './record.js'; -import { localize } from '../../../../../../../../../nls.js'; -import { PromptMetadataDiagnostic, PromptMetadataError } from '../../diagnostics.js'; -import { FrontMatterSequence } from '../../../../codecs/base/frontMatterCodec/tokens/frontMatterSequence.js'; -import { FrontMatterRecord, FrontMatterString } from '../../../../codecs/base/frontMatterCodec/tokens/index.js'; -import { Range } from '../../../../../../../../../editor/common/core/range.js'; - - -/** - * Base class for all metadata records with a `string` value. - */ -export abstract class PromptStringMetadata extends PromptMetadataRecord { - /** - * Value token reference of the record. - */ - protected valueToken: FrontMatterString | FrontMatterSequence | undefined; - - /** - * String value of a metadata record. - */ - public override get value(): string | undefined { - return this.valueToken?.cleanText; - } - - public get valueRange(): Range | undefined { - return this.valueToken?.range; - } - - constructor( - expectedRecordName: string, - recordToken: FrontMatterRecord, - languageId: string, - ) { - super(expectedRecordName, recordToken, languageId); - } - - /** - * Validate the metadata record has a 'string' value. - */ - public override validate(): readonly PromptMetadataDiagnostic[] { - const { valueToken } = this.recordToken; - - // validate that the record value is a string or a generic sequence - // of tokens that can be interpreted as a string without quotes - const isString = (valueToken instanceof FrontMatterString); - const isSequence = (valueToken instanceof FrontMatterSequence); - if (isString || isSequence) { - this.valueToken = valueToken; - return this.issues; - } - - this.issues.push( - new PromptMetadataError( - valueToken.range, - localize( - 'prompt.header.metadata.string.diagnostics.invalid-value-type', - "The property '{0}' must be of type '{1}', got '{2}'.", - this.recordName, - 'string', - valueToken.valueTypeName.toString(), - ), - ), - ); - - delete this.valueToken; - return this.issues; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/description.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/description.ts deleted file mode 100644 index aaa8fa3a640..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/description.ts +++ /dev/null @@ -1,46 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { PromptStringMetadata } from './base/string.js'; -import { FrontMatterRecord, FrontMatterToken } from '../../../codecs/base/frontMatterCodec/tokens/index.js'; - -/** - * Name of the metadata record in the prompt header. - */ -const RECORD_NAME = 'description'; - -/** - * Prompt `description` metadata record inside the prompt header. - */ -export class PromptDescriptionMetadata extends PromptStringMetadata { - public override get recordName(): string { - return RECORD_NAME; - } - - constructor( - recordToken: FrontMatterRecord, - languageId: string, - ) { - super(RECORD_NAME, recordToken, languageId); - } - - /** - * Check if a provided front matter token is a metadata record - * with name equal to `description`. - */ - public static isDescriptionRecord( - token: FrontMatterToken, - ): boolean { - if ((token instanceof FrontMatterRecord) === false) { - return false; - } - - if (token.nameToken.text === RECORD_NAME) { - return true; - } - - return false; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/mode.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/mode.ts deleted file mode 100644 index b12728f7e75..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/mode.ts +++ /dev/null @@ -1,47 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { PromptStringMetadata } from './base/string.js'; -import { FrontMatterRecord, FrontMatterToken } from '../../../codecs/base/frontMatterCodec/tokens/index.js'; - -/** - * Name of the metadata record in the prompt header. - */ -const RECORD_NAME = 'mode'; - -/** - * Prompt `mode` metadata record inside the prompt header. - * Now supports both built-in modes (ask, edit, agent) and custom mode IDs. - */ -export class PromptModeMetadata extends PromptStringMetadata { - constructor( - recordToken: FrontMatterRecord, - languageId: string, - ) { - super( - RECORD_NAME, - recordToken, - languageId, - ); - } - - /** - * Check if a provided front matter token is a metadata record - * with name equal to `mode`. - */ - public static isModeRecord( - token: FrontMatterToken, - ): boolean { - if ((token instanceof FrontMatterRecord) === false) { - return false; - } - - if (token.nameToken.text === RECORD_NAME) { - return true; - } - - return false; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/model.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/model.ts deleted file mode 100644 index aa559e57fc0..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/model.ts +++ /dev/null @@ -1,41 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { FrontMatterRecord, FrontMatterToken } from '../../../codecs/base/frontMatterCodec/tokens/index.js'; -import { PromptStringMetadata } from './base/string.js'; - -/** - * Name of the metadata record in the prompt header. - */ -const RECORD_NAME = 'model'; - -export class PromptModelMetadata extends PromptStringMetadata { - public override get recordName(): string { - return RECORD_NAME; - } - - constructor( - recordToken: FrontMatterRecord, - languageId: string, - ) { - super(RECORD_NAME, recordToken, languageId); - } - - /** - * Check if a provided front matter token is a metadata record - * with name equal to `description`. - */ - public static isModelRecord(token: FrontMatterToken): boolean { - if ((token instanceof FrontMatterRecord) === false) { - return false; - } - - if (token.nameToken.text === RECORD_NAME) { - return true; - } - - return false; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/tools.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/tools.ts deleted file mode 100644 index ad52e5faebe..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/metadata/tools.ts +++ /dev/null @@ -1,182 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { PromptMetadataRecord } from './base/record.js'; -import { localize } from '../../../../../../../../nls.js'; -import { PromptMetadataDiagnostic, PromptMetadataError, PromptMetadataWarning } from '../diagnostics.js'; -import { FrontMatterSequence } from '../../../codecs/base/frontMatterCodec/tokens/frontMatterSequence.js'; -import { FrontMatterArray, FrontMatterRecord, FrontMatterString, FrontMatterToken, FrontMatterValueToken } from '../../../codecs/base/frontMatterCodec/tokens/index.js'; -import { Range } from '../../../../../../../../editor/common/core/range.js'; - -/** - * Name of the metadata record in the prompt header. - */ -const RECORD_NAME = 'tools'; - -/** - * Prompt `tools` metadata record inside the prompt header. - */ -export class PromptToolsMetadata extends PromptMetadataRecord { - - /** - * List of all valid tool names that were found in - * this metadata record. - */ - public override get value(): string[] | undefined { - if (this.validToolNames === undefined) { - return []; - } - - return [...this.validToolNames.keys()]; - } - - public override get recordName(): string { - return RECORD_NAME; - } - - /** - * Value token reference of the record. - */ - protected valueToken: FrontMatterArray | undefined; - - /** - * List of all valid tool names that were found in - * this metadata record. - */ - private validToolNames: Map | undefined; - - - - constructor( - recordToken: FrontMatterRecord, - languageId: string, - ) { - super(RECORD_NAME, recordToken, languageId); - } - - /** - * Validate the metadata record and collect all issues - * related to its content. - */ - public override validate(): readonly PromptMetadataDiagnostic[] { - const { valueToken } = this.recordToken; - - // validate that the record value is an array - if ((valueToken instanceof FrontMatterArray) === false) { - this.issues.push( - new PromptMetadataError( - valueToken.range, - localize( - 'prompt.header.metadata.tools.diagnostics.invalid-value-type', - "Must be an array of tool names, got '{0}'.", - valueToken.valueTypeName.toString(), - ), - ), - ); - - delete this.valueToken; - return this.issues; - } - - this.valueToken = valueToken; - - // validate that all array items - this.validToolNames = new Map(); - for (const item of this.valueToken.items) { - this.issues.push( - ...this.validateToolName(item, this.validToolNames), - ); - } - - return this.issues; - } - - public getToolRange(toolName: string): Range | undefined { - return this.validToolNames?.get(toolName); - } - - /** - * Validate an individual provided value token that is used - * for a tool name. - */ - private validateToolName( - valueToken: FrontMatterValueToken, - validToolNames: Map, - ): readonly PromptMetadataDiagnostic[] { - const issues: PromptMetadataDiagnostic[] = []; - - // tool name must be a quoted or an unquoted 'string' - if ( - (valueToken instanceof FrontMatterString) === false && - (valueToken instanceof FrontMatterSequence) === false - ) { - issues.push( - new PromptMetadataWarning( - valueToken.range, - localize( - 'prompt.header.metadata.tools.diagnostics.invalid-tool-name-type', - "Unexpected tool name '{0}', expected a string literal.", - valueToken.text - ), - ), - ); - - return issues; - } - - const cleanToolName = valueToken.cleanText.trim(); - // the tool name should not be empty - if (cleanToolName.length === 0) { - issues.push( - new PromptMetadataWarning( - valueToken.range, - localize( - 'prompt.header.metadata.tools.diagnostics.empty-tool-name', - "Tool name cannot be empty.", - ), - ), - ); - - return issues; - } - - // the tool name should not be duplicated - if (validToolNames.has(cleanToolName)) { - issues.push( - new PromptMetadataWarning( - valueToken.range, - localize( - 'prompt.header.metadata.tools.diagnostics.duplicate-tool-name', - "Duplicate tool name '{0}'.", - cleanToolName, - ), - ), - ); - - return issues; - } - - validToolNames.set(cleanToolName, valueToken.range); - return issues; - } - - /** - * Check if a provided front matter token is a metadata record - * with name equal to `tools`. - */ - public static isToolsRecord( - token: FrontMatterToken, - ): boolean { - if ((token instanceof FrontMatterRecord) === false) { - return false; - } - - if (token.nameToken.text === RECORD_NAME) { - return true; - } - - return false; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/modeHeader.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/modeHeader.ts deleted file mode 100644 index 94307163bf6..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/modeHeader.ts +++ /dev/null @@ -1,56 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { HeaderBase, IHeaderMetadata, type TDehydrated } from './headerBase.js'; -import { PromptsType } from '../../promptTypes.js'; -import { FrontMatterRecord } from '../../codecs/base/frontMatterCodec/tokens/index.js'; -import { PromptModelMetadata } from './metadata/model.js'; -import { PromptToolsMetadata } from './metadata/tools.js'; - -/** - * Metadata utility object for mode files. - */ -interface IModeMetadata extends IHeaderMetadata { - /** - * Tools metadata in the mode header. - */ - tools: PromptToolsMetadata; - - /** - * Chat model metadata in the mode header. - */ - model: PromptModelMetadata; -} - -/** - * Metadata for mode files. - */ -export type TModeMetadata = Partial> & { promptType: PromptsType.mode }; - -/** - * Header object for mode files. - */ -export class ModeHeader extends HeaderBase { - protected override handleToken(token: FrontMatterRecord): boolean { - // if the record might be a "tools" metadata - // add it to the list of parsed metadata records - if (PromptToolsMetadata.isToolsRecord(token)) { - const metadata = new PromptToolsMetadata(token, this.languageId); - - this.issues.push(...metadata.validate()); - this.meta.tools = metadata; - return true; - } - if (PromptModelMetadata.isModelRecord(token)) { - const metadata = new PromptModelMetadata(token, this.languageId); - - this.issues.push(...metadata.validate()); - this.meta.model = metadata; - - return true; - } - return false; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/promptHeader.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/promptHeader.ts deleted file mode 100644 index 250e8382372..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptHeader/promptHeader.ts +++ /dev/null @@ -1,103 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { ChatModeKind } from '../../../constants.js'; -import { localize } from '../../../../../../../nls.js'; -import { PromptMetadataWarning } from './diagnostics.js'; -import { HeaderBase, IHeaderMetadata, type TDehydrated } from './headerBase.js'; -import { PromptsType } from '../../promptTypes.js'; -import { FrontMatterRecord } from '../../codecs/base/frontMatterCodec/tokens/index.js'; -import { PromptModelMetadata } from './metadata/model.js'; -import { PromptToolsMetadata } from './metadata/tools.js'; -import { PromptModeMetadata } from './metadata/mode.js'; - -/** - * Metadata utility object for prompt files. - */ -export interface IPromptMetadata extends IHeaderMetadata { - /** - * Tools metadata in the prompt header. - */ - tools: PromptToolsMetadata; - - /** - * Chat mode metadata in the prompt header. - */ - mode: PromptModeMetadata; - - /** - * Chat model metadata in the prompt header. - */ - model: PromptModelMetadata; -} - -/** - * Metadata for prompt files. - */ -export type TPromptMetadata = Partial> & { promptType: PromptsType.prompt }; - -/** - * Header object for prompt files. - */ -export class PromptHeader extends HeaderBase { - protected override handleToken(token: FrontMatterRecord): boolean { - // if the record might be a "tools" metadata - // add it to the list of parsed metadata records - if (PromptToolsMetadata.isToolsRecord(token)) { - const metadata = new PromptToolsMetadata(token, this.languageId); - - this.issues.push(...metadata.validate()); - this.meta.tools = metadata; - - this.validateToolsAndModeCompatibility(); - return true; - } - - // if the record might be a "mode" metadata - // add it to the list of parsed metadata records - if (PromptModeMetadata.isModeRecord(token)) { - const metadata = new PromptModeMetadata(token, this.languageId); - - this.issues.push(...metadata.validate()); - this.meta.mode = metadata; - - this.validateToolsAndModeCompatibility(); - return true; - } - - if (PromptModelMetadata.isModelRecord(token)) { - const metadata = new PromptModelMetadata(token, this.languageId); - - this.issues.push(...metadata.validate()); - this.meta.model = metadata; - - return true; - } - - return false; - } - - /** - * Validate that the `tools` and `mode` metadata are compatible - * with each other. If not, add a warning diagnostic. - */ - private validateToolsAndModeCompatibility(): void { - const { tools, mode } = this.meta; - const modeValue = mode?.value; - - if (tools !== undefined && (modeValue === ChatModeKind.Edit || modeValue === ChatModeKind.Ask)) { - this.issues.push( - new PromptMetadataWarning( - tools.range, - localize( - 'prompt.header.metadata.mode.diagnostics.incompatible-with-tools', - "Tools can not be used in '{0}' mode and will be ignored.", - modeValue - ), - ), - ); - } - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptParser.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptParser.ts deleted file mode 100644 index 99df26fd5e9..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/promptParser.ts +++ /dev/null @@ -1,72 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { URI } from '../../../../../../base/common/uri.js'; -import { IPromptContentsProvider } from '../contentProviders/types.js'; -import { ILogService } from '../../../../../../platform/log/common/log.js'; -import { BasePromptParser, IPromptParserOptions } from './basePromptParser.js'; -import { IModelService } from '../../../../../../editor/common/services/model.js'; -import { TextModelContentsProvider } from '../contentProviders/textModelContentsProvider.js'; -import { FilePromptContentProvider } from '../contentProviders/filePromptContentsProvider.js'; -import { IInstantiationService } from '../../../../../../platform/instantiation/common/instantiation.js'; -import { IPromptContentsProviderOptions } from '../contentProviders/promptContentsProviderBase.js'; -import { IWorkbenchEnvironmentService } from '../../../../../services/environment/common/environmentService.js'; - -/** - * Get prompt contents provider object based on the prompt type. - */ -function getContentsProvider( - uri: URI, - options: IPromptContentsProviderOptions, - modelService: IModelService, - instaService: IInstantiationService -): IPromptContentsProvider { - const model = modelService.getModel(uri); - if (model) { - return instaService.createInstance(TextModelContentsProvider, model, options); - } - return instaService.createInstance(FilePromptContentProvider, uri, options); -} - -/** - * General prompt parser class that automatically infers a prompt - * contents provider type by the type of provided prompt URI. - */ -export class PromptParser extends BasePromptParser { - /** - * Underlying prompt contents provider instance. - */ - private readonly contentsProvider: IPromptContentsProvider; - - constructor( - uri: URI, - options: IPromptParserOptions, - @ILogService logService: ILogService, - @IModelService modelService: IModelService, - @IInstantiationService instaService: IInstantiationService, - @IWorkbenchEnvironmentService envService: IWorkbenchEnvironmentService, - ) { - const contentsProvider = getContentsProvider(uri, options, modelService, instaService); - - super( - contentsProvider, - options, - instaService, - envService, - logService, - ); - - this.contentsProvider = this._register(contentsProvider); - } - - /** - * Returns a string representation of this object. - */ - public override toString(): string { - const { sourceName } = this.contentsProvider; - - return `prompt-parser:${sourceName}:${this.uri.path}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/textModelPromptParser.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/textModelPromptParser.ts deleted file mode 100644 index bb2b7060abb..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/textModelPromptParser.ts +++ /dev/null @@ -1,42 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { ITextModel } from '../../../../../../editor/common/model.js'; -import { ILogService } from '../../../../../../platform/log/common/log.js'; -import { BasePromptParser, IPromptParserOptions } from './basePromptParser.js'; -import { TextModelContentsProvider } from '../contentProviders/textModelContentsProvider.js'; -import { IInstantiationService } from '../../../../../../platform/instantiation/common/instantiation.js'; -import { IWorkbenchEnvironmentService } from '../../../../../services/environment/common/environmentService.js'; - -/** - * Class capable of parsing prompt syntax out of a provided text model, - * including all the nested child file references it may have. - */ -export class TextModelPromptParser extends BasePromptParser { - constructor( - model: ITextModel, - options: IPromptParserOptions, - @IInstantiationService instantiationService: IInstantiationService, - @IWorkbenchEnvironmentService envService: IWorkbenchEnvironmentService, - @ILogService logService: ILogService, - ) { - const contentsProvider = instantiationService.createInstance( - TextModelContentsProvider, - model, - options, - ); - - super(contentsProvider, options, instantiationService, envService, logService); - - this._register(contentsProvider); - } - - /** - * Returns a string representation of this object. - */ - public override toString(): string { - return `text-model-prompt:${this.uri.path}`; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/topError.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/topError.ts deleted file mode 100644 index b6281d3f42b..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/topError.ts +++ /dev/null @@ -1,102 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { ITopError } from './types.js'; -import { localize } from '../../../../../../nls.js'; -import { assert } from '../../../../../../base/common/assert.js'; -import { assertDefined } from '../../../../../../base/common/types.js'; -import { OpenFailed, RecursiveReference, FailedToResolveContentsStream } from '../../promptFileReferenceErrors.js'; - -/** - * The top-most error of the reference tree. - */ -export class TopError implements ITopError { - public readonly originalError: ITopError['originalError']; - public readonly errorSubject: ITopError['errorSubject']; - public readonly errorsCount: ITopError['errorsCount']; - public readonly parentUri: ITopError['parentUri']; - - constructor( - options: Omit, - ) { - this.originalError = options.originalError; - this.errorSubject = options.errorSubject; - this.errorsCount = options.errorsCount; - this.parentUri = options.parentUri; - } - - public get localizedMessage(): string { - const { originalError, parentUri, errorSubject: subject, errorsCount } = this; - - assert( - errorsCount >= 1, - `Error count must be at least 1, got '${errorsCount}'.`, - ); - - // a note about how many more link issues are there - const moreIssuesLabel = (errorsCount > 1) - ? localize('workbench.reusable-prompts.top-error.more-issues-label', "\n(+{0} more issues)", errorsCount - 1) - : ''; - - if (subject === 'root') { - if (originalError instanceof OpenFailed) { - return localize( - 'workbench.reusable-prompts.top-error.open-failed', - "Cannot open '{0}'.{1}", - originalError.uri.path, - moreIssuesLabel, - ); - } - - if (originalError instanceof FailedToResolveContentsStream) { - return localize( - 'workbench.reusable-prompts.top-error.cannot-read', - "Cannot read '{0}'.{1}", - originalError.uri.path, - moreIssuesLabel, - ); - } - - if (originalError instanceof RecursiveReference) { - return localize( - 'workbench.reusable-prompts.top-error.recursive-reference', - "Recursion to itself.", - ); - } - - return originalError.message + moreIssuesLabel; - } - - // a sanity check - because the error subject is not `root`, the parent must set - assertDefined( - parentUri, - 'Parent URI must be defined for error of non-root link.', - ); - - const errorMessageStart = (subject === 'child') - ? localize( - 'workbench.reusable-prompts.top-error.child.direct', - "Contains", - ) - : localize( - 'workbench.reusable-prompts.top-error.child.indirect', - "Indirectly referenced prompt '{0}' contains", - parentUri.path, - ); - - const linkIssueName = (originalError instanceof RecursiveReference) - ? localize('recursive', "recursive") - : localize('broken', "broken"); - - return localize( - 'workbench.reusable-prompts.top-error.child.final-message', - "{0} a {1} link to '{2}' that will be ignored.{3}", - errorMessageStart, - linkIssueName, - originalError.uri.path, - moreIssuesLabel, - ); - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/types.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/types.ts deleted file mode 100644 index 0a3d693c61d..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/parsers/types.ts +++ /dev/null @@ -1,116 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { URI } from '../../../../../../base/common/uri.js'; -import { ResolveError } from '../../promptFileReferenceErrors.js'; -import { IRange, Range } from '../../../../../../editor/common/core/range.js'; - -/** - * A resolve error with a parent prompt URI, if any. - */ -export interface IResolveError { - /** - * Original error instance. - */ - readonly originalError: ResolveError; - - /** - * URI of the parent that references this error. - */ - readonly parentUri?: URI; -} - -/** - * Top most error of the reference tree. - */ -export interface ITopError extends IResolveError { - /** - * Where does the error belong to: - * - * - `root` - the error is the top most error of the entire tree - * - `child` - the error is a child of the root error - * - `indirect-child` - the error is a child of a child of the root error - */ - readonly errorSubject: 'root' | 'child' | 'indirect-child'; - - /** - * Total number of all errors in the references tree, including the error - * of the current reference and all possible errors of its children. - */ - readonly errorsCount: number; - - /** - * Localized error message. - */ - readonly localizedMessage: string; -} - -/** - * Base interface for a generic prompt reference. - */ -interface IPromptReferenceBase { - /** - * Type of the prompt reference. E.g., `file`, `http`, `image`, etc. - */ - readonly type: string; - - /** - * Subtype of the prompt reference. For instance a `file` reference - * can be a `markdown link` or a prompt `#file:` variable reference. - */ - readonly subtype: string; - - /** - * URI component of the associated with this reference. - */ - readonly uri: URI; - - /** - * The full range of the prompt reference in the source text, - * including the {@link linkRange} and any additional - * parts the reference may contain (e.g., the `#file:` prefix). - */ - readonly range: Range; - - /** - * Range of the link part that the reference points to. - */ - readonly linkRange: IRange | undefined; - - /** - * Text of the reference as it appears in the source. - */ - readonly text: string; - - /** - * Original link path as it appears in the source. - */ - readonly path: string; - -} - -/** - * The special case of the {@link IPromptReferenceBase} that pertains - * to a file resource on the disk. - */ -export interface IPromptFileReference extends IPromptReferenceBase { - readonly type: 'file'; - - /** - * Subtype of a file reference, - either a prompt `#file` variable, - * or a `markdown link` (e.g., `[caption](/path/to/file.md)`). - */ - readonly subtype: 'prompt' | 'markdown'; -} - -/** - * List of all known prompt reference types. - */ -export type TPromptReference = IPromptFileReference; - -export type TVariableReference = { - readonly name: string; - readonly range: Range; -}; diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptValidator.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptValidator.ts index ead5fc60b43..763d42960ba 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptValidator.ts +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptValidator.ts @@ -17,13 +17,14 @@ import { ChatModeKind } from '../../constants.js'; import { ILanguageModelChatMetadata, ILanguageModelsService } from '../../languageModels.js'; import { ILanguageModelToolsService } from '../../languageModelToolsService.js'; import { getPromptsTypeForLanguageId, PromptsType } from '../promptTypes.js'; -import { IHeaderAttribute, ParsedPromptFile } from './newPromptsParser.js'; +import { IArrayValue, IHeaderAttribute, ParsedPromptFile } from './newPromptsParser.js'; import { PromptsConfig } from '../config/config.js'; import { Disposable, DisposableStore, toDisposable } from '../../../../../../base/common/lifecycle.js'; import { Delayer } from '../../../../../../base/common/async.js'; import { ResourceMap } from '../../../../../../base/common/map.js'; import { IFileService } from '../../../../../../platform/files/common/files.js'; import { IPromptsService } from './promptsService.js'; +import { ILabelService } from '../../../../../../platform/label/common/label.js'; const MARKERS_OWNER_ID = 'prompts-diagnostics-provider'; @@ -33,6 +34,7 @@ export class PromptValidator { @ILanguageModelToolsService private readonly languageModelToolsService: ILanguageModelToolsService, @IChatModeService private readonly chatModeService: IChatModeService, @IFileService private readonly fileService: IFileService, + @ILabelService private readonly labelService: ILabelService ) { } public async validate(promptAST: ParsedPromptFile, promptType: PromptsType, report: (markers: IMarkerData) => void): Promise { @@ -58,12 +60,13 @@ export class PromptValidator { fileReferenceChecks.push((async () => { try { const exists = await this.fileService.exists(resolved); - if (!exists) { - report(toMarker(localize('promptValidator.fileNotFound', "File '{0}' not found.", ref.content), ref.range, MarkerSeverity.Warning)); + if (exists) { + return; } } catch { - report(toMarker(localize('promptValidator.fileNotFound', "File '{0}' not found.", ref.content), ref.range, MarkerSeverity.Warning)); } + const loc = this.labelService.getUriLabel(resolved); + report(toMarker(localize('promptValidator.fileNotFound', "File '{0}' not found at '{1}'.", ref.content, loc), ref.range, MarkerSeverity.Warning)); })()); } @@ -85,19 +88,20 @@ export class PromptValidator { if (!header) { return; } - const validAttributeNames = getValidAttributeNames(promptType); + const validAttributeNames = getValidAttributeNames(promptType, true); const attributes = header.attributes; for (const attribute of attributes) { if (!validAttributeNames.includes(attribute.key)) { + const supportedNames = getValidAttributeNames(promptType, false).join(', '); switch (promptType) { case PromptsType.prompt: - report(toMarker(localize('promptValidator.unknownAttribute.prompt', "Attribute '{0}' is not supported in prompt files. Supported: {1}", attribute.key, validAttributeNames.join(', ')), attribute.range, MarkerSeverity.Warning)); + report(toMarker(localize('promptValidator.unknownAttribute.prompt', "Attribute '{0}' is not supported in prompt files. Supported: {1}.", attribute.key, supportedNames), attribute.range, MarkerSeverity.Warning)); break; case PromptsType.mode: - report(toMarker(localize('promptValidator.unknownAttribute.mode', "Attribute '{0}' is not supported in mode files. Supported: {1}", attribute.key, validAttributeNames.join(', ')), attribute.range, MarkerSeverity.Warning)); + report(toMarker(localize('promptValidator.unknownAttribute.mode', "Attribute '{0}' is not supported in mode files. Supported: {1}.", attribute.key, supportedNames), attribute.range, MarkerSeverity.Warning)); break; case PromptsType.instructions: - report(toMarker(localize('promptValidator.unknownAttribute.instructions', "Attribute '{0}' is not supported in instructions files. Supported: {1}", attribute.key, validAttributeNames.join(', ')), attribute.range, MarkerSeverity.Warning)); + report(toMarker(localize('promptValidator.unknownAttribute.instructions', "Attribute '{0}' is not supported in instructions files. Supported: {1}.", attribute.key, supportedNames), attribute.range, MarkerSeverity.Warning)); break; } } @@ -112,6 +116,7 @@ export class PromptValidator { } case PromptsType.instructions: this.validateApplyTo(attributes, report); + this.validateExcludeMode(attributes, report); break; case PromptsType.mode: @@ -215,16 +220,24 @@ export class PromptValidator { } if (modeKind !== ChatModeKind.Agent) { report(toMarker(localize('promptValidator.toolsOnlyInAgent', "The 'tools' attribute is only supported in agent mode. Attribute will be ignored."), attribute.range, MarkerSeverity.Warning)); - - } - if (attribute.value.type !== 'array') { - report(toMarker(localize('promptValidator.toolsMustBeArray', "The 'tools' attribute must be an array."), attribute.value.range, MarkerSeverity.Error)); - return; } - if (attribute.value.items.length > 0) { + switch (attribute.value.type) { + case 'array': + this.validateToolsArray(attribute.value, report); + break; + case 'object': + //this.validateToolsObject(attribute.value, report); + break; + default: + report(toMarker(localize('promptValidator.toolsMustBeArrayOrMap', "The 'tools' attribute must be an array."), attribute.value.range, MarkerSeverity.Error)); + } + } + + private validateToolsArray(valueItem: IArrayValue, report: (markers: IMarkerData) => void) { + if (valueItem.items.length > 0) { const available = this.getAvailableToolAndToolSetNames(); - for (const item of attribute.value.items) { + for (const item of valueItem.items) { if (item.type !== 'string') { report(toMarker(localize('promptValidator.eachToolMustBeString', "Each tool name in the 'tools' attribute must be a string."), item.range, MarkerSeverity.Error)); } else if (item.value && !available.has(item.value)) { @@ -277,17 +290,36 @@ export class PromptValidator { report(toMarker(localize('promptValidator.applyToMustBeValidGlob', "The 'applyTo' attribute must be a valid glob pattern."), attribute.value.range, MarkerSeverity.Error)); } } + + private validateExcludeMode(attributes: IHeaderAttribute[], report: (markers: IMarkerData) => void): undefined { + const attribute = attributes.find(attr => attr.key === 'excludeMode'); + if (!attribute) { + return; + } + if (attribute.value.type !== 'array') { + report(toMarker(localize('promptValidator.excludeModeMustBeArray', "The 'excludeMode' attribute must be an array."), attribute.value.range, MarkerSeverity.Error)); + return; + } + } } -function getValidAttributeNames(promptType: PromptsType): string[] { - switch (promptType) { - case PromptsType.prompt: - return ['description', 'model', 'tools', 'mode']; - case PromptsType.instructions: - return ['description', 'applyTo']; - case PromptsType.mode: - return ['description', 'model', 'tools']; - } +const validAttributeNames = { + [PromptsType.prompt]: ['description', 'model', 'tools', 'mode'], + [PromptsType.instructions]: ['description', 'applyTo', 'excludeMode'], + [PromptsType.mode]: ['description', 'model', 'tools', 'advancedOptions'] +}; +const validAttributeNamesNoExperimental = { + [PromptsType.prompt]: validAttributeNames[PromptsType.prompt].filter(name => !isExperimentalAttribute(name)), + [PromptsType.instructions]: validAttributeNames[PromptsType.instructions].filter(name => !isExperimentalAttribute(name)), + [PromptsType.mode]: validAttributeNames[PromptsType.mode].filter(name => !isExperimentalAttribute(name)) +}; + +export function getValidAttributeNames(promptType: PromptsType, includeExperimental: boolean): string[] { + return includeExperimental ? validAttributeNames[promptType] : validAttributeNamesNoExperimental[promptType]; +} + +export function isExperimentalAttribute(attributeName: string): boolean { + return attributeName === 'advancedOptions' || attributeName === 'excludeMode'; } function toMarker(message: string, range: Range, severity = MarkerSeverity.Error): IMarkerData { diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsService.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsService.ts index ab3b4887aa4..945eb7fa5f6 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsService.ts +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsService.ts @@ -6,15 +6,13 @@ import { ChatModeKind } from '../../constants.js'; import { URI } from '../../../../../../base/common/uri.js'; import { Event } from '../../../../../../base/common/event.js'; -import { TMetadata } from '../parsers/promptHeader/headerBase.js'; import { ITextModel } from '../../../../../../editor/common/model.js'; import { IDisposable } from '../../../../../../base/common/lifecycle.js'; -import { TextModelPromptParser } from '../parsers/textModelPromptParser.js'; import { CancellationToken } from '../../../../../../base/common/cancellation.js'; import { PromptsType } from '../promptTypes.js'; import { createDecorator } from '../../../../../../platform/instantiation/common/instantiation.js'; import { YamlNode, YamlParseError } from '../../../../../../base/common/yaml.js'; -import { IVariableReference } from '../../chatModes.js'; +import { IChatModeInstructions } from '../../chatModes.js'; import { ParsedPromptFile } from './newPromptsParser.js'; /** @@ -48,32 +46,6 @@ export interface IPromptPath { readonly type: PromptsType; } -/** - * Type for a shared prompt parser instance returned by the {@link IPromptsService}. - * Because the parser is shared, we omit the `dispose` method from - * the original type so the caller cannot dispose it prematurely - */ -export type TSharedPrompt = Omit; - -/** - * Metadata node object in a hierarchical tree of prompt references. - */ -export interface IMetadata { - /** - * URI of a prompt file. - */ - readonly uri: URI; - - /** - * Metadata of the prompt file. - */ - readonly metadata: TMetadata | null; - - /** - * List of metadata for each valid child prompt reference. - */ - readonly children?: readonly IMetadata[]; -} export interface ICustomChatMode { /** @@ -102,14 +74,9 @@ export interface ICustomChatMode { readonly model?: string; /** - * Contents of the custom chat mode file body. + * Contents of the custom chat mode file body and other mode instructions. */ - readonly body: string; - - /** - * References to variables without a type in the mode body. These could be tools or toolsets. - */ - readonly variableReferences: readonly IVariableReference[]; + readonly modeInstructions: IChatModeInstructions; } /** @@ -159,12 +126,6 @@ export type TCombinedToolsMetadata = ICombinedAgentToolsMetadata | ICombinedNonA export interface IPromptsService extends IDisposable { readonly _serviceBrand: undefined; - /** - * Get a prompt syntax parser for the provided text model. - * See {@link TextModelPromptParser} for more info on the parser API. - */ - getSyntaxParserFor(model: ITextModel): TSharedPrompt & { isDisposed: false }; - /** * The parsed prompt file for the provided text model. * @param textModel Returns the parsed prompt file. @@ -207,12 +168,6 @@ export interface IPromptsService extends IDisposable { */ getCustomChatModes(token: CancellationToken): Promise; - /** - * Parses the provided URI - * @param uris - */ - parse(uri: URI, type: PromptsType, token: CancellationToken): Promise; - /** * Parses the provided URI * @param uris @@ -232,15 +187,6 @@ export interface IChatPromptSlashCommand { readonly promptPath?: IPromptPath; } - -export interface IPromptParserResult { - readonly uri: URI; - readonly metadata: TMetadata | null; - readonly fileReferences: readonly URI[]; - readonly variableReferences: readonly IVariableReference[]; - readonly header?: IPromptHeader; -} - export interface IPromptHeader { readonly node: YamlNode | undefined; readonly errors: YamlParseError[]; diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts index 07579a4d003..0bbef549c15 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts @@ -4,35 +4,30 @@ *--------------------------------------------------------------------------------------------*/ import { localize } from '../../../../../../nls.js'; -import { getLanguageIdForPromptsType, getPromptsTypeForLanguageId, PROMPT_LANGUAGE_ID, PromptsType } from '../promptTypes.js'; -import { PromptParser } from '../parsers/promptParser.js'; +import { getPromptsTypeForLanguageId, PROMPT_LANGUAGE_ID, PromptsType } from '../promptTypes.js'; import { type URI } from '../../../../../../base/common/uri.js'; -import { assert } from '../../../../../../base/common/assert.js'; import { basename } from '../../../../../../base/common/path.js'; import { PromptFilesLocator } from '../utils/promptFilesLocator.js'; import { Disposable } from '../../../../../../base/common/lifecycle.js'; import { Event } from '../../../../../../base/common/event.js'; import { type ITextModel } from '../../../../../../editor/common/model.js'; -import { ObjectCache } from '../utils/objectCache.js'; import { ILogService } from '../../../../../../platform/log/common/log.js'; -import { TextModelPromptParser } from '../parsers/textModelPromptParser.js'; import { ILabelService } from '../../../../../../platform/label/common/label.js'; import { IModelService } from '../../../../../../editor/common/services/model.js'; import { CancellationToken } from '../../../../../../base/common/cancellation.js'; import { IInstantiationService } from '../../../../../../platform/instantiation/common/instantiation.js'; import { IUserDataProfileService } from '../../../../../services/userDataProfile/common/userDataProfile.js'; -import type { IChatPromptSlashCommand, ICustomChatMode, IPromptParserResult, IPromptPath, IPromptsService, TPromptsStorage } from './promptsService.js'; +import type { IChatPromptSlashCommand, ICustomChatMode, IPromptPath, IPromptsService, TPromptsStorage } from './promptsService.js'; import { getCleanPromptName, PROMPT_FILE_EXTENSION } from '../config/promptFileLocations.js'; import { ILanguageService } from '../../../../../../editor/common/languages/language.js'; import { PromptsConfig } from '../config/config.js'; import { IConfigurationService } from '../../../../../../platform/configuration/common/configuration.js'; -import { PositionOffsetTransformer } from '../../../../../../editor/common/core/text/positionToOffset.js'; import { NewPromptsParser, ParsedPromptFile } from './newPromptsParser.js'; import { IFileService } from '../../../../../../platform/files/common/files.js'; import { ResourceMap } from '../../../../../../base/common/map.js'; import { CancellationError } from '../../../../../../base/common/errors.js'; import { OffsetRange } from '../../../../../../editor/common/core/ranges/offsetRange.js'; -import { IVariableReference } from '../../chatModes.js'; +import { IChatModeInstructions, IVariableReference } from '../../chatModes.js'; /** * Provides prompt services. @@ -40,11 +35,6 @@ import { IVariableReference } from '../../chatModes.js'; export class PromptsService extends Disposable implements IPromptsService { public declare readonly _serviceBrand: undefined; - /** - * Cache of text model content prompt parsers. - */ - private readonly cache: ObjectCache; - /** * Prompt files locator utility. */ @@ -77,36 +67,6 @@ export class PromptsService extends Disposable implements IPromptsService { this.fileLocator = this._register(this.instantiationService.createInstance(PromptFilesLocator)); - // the factory function below creates a new prompt parser object - // for the provided model, if no active non-disposed parser exists - this.cache = this._register( - new ObjectCache((model) => { - assert( - model.isDisposed() === false, - 'Text model must not be disposed.', - ); - - /** - * Note! When/if shared with "file" prompts, the `seenReferences` array below must be taken into account. - * Otherwise consumers will either see incorrect failing or incorrect successful results, based on their - * use case, timing of their calls to the {@link getSyntaxParserFor} function, and state of this service. - */ - const parser: TextModelPromptParser = instantiationService.createInstance( - TextModelPromptParser, - model, - { allowNonPromptFiles: true, languageId: undefined, updateOnChange: true }, - ).start(); - - // this is a sanity check and the contract of the object cache, - // we must return a non-disposed object from this factory function - parser.assertNotDisposed( - 'Created prompt parser must not be disposed.', - ); - - return parser; - }) - ); - this._register(this.modelService.onModelRemoved((model) => { this.parsedPromptFileCache.delete(model.uri); })); @@ -132,21 +92,6 @@ export class PromptsService extends Disposable implements IPromptsService { } - /** - * @throws {Error} if: - * - the provided model is disposed - * - newly created parser is disposed immediately on initialization. - * See factory function in the {@link constructor} for more info. - */ - public getSyntaxParserFor(model: ITextModel): TextModelPromptParser & { isDisposed: false } { - assert( - model.isDisposed() === false, - 'Cannot create a prompt syntax parser for a disposed model.', - ); - - return this.cache.get(model); - } - public getParsedPromptFile(textModel: ITextModel): ParsedPromptFile { const cached = this.parsedPromptFileCache.get(textModel.uri); if (cached && cached[0] === textModel.getVersionId()) { @@ -255,63 +200,49 @@ export class PromptsService extends Disposable implements IPromptsService { private async computeCustomChatModes(token: CancellationToken): Promise { const modeFiles = await this.listPromptFiles(PromptsType.mode, token); - const metadataList = await Promise.all( + const customChatModes = await Promise.all( modeFiles.map(async ({ uri }): Promise => { const ast = await this.parseNew(uri, token); - const variableReferences: IVariableReference[] = []; - let body = ''; + let metadata: any | undefined; + if (ast.header) { + const advanced = ast.header.getAttribute('advancedOptions'); + if (advanced && advanced.value.type === 'object') { + metadata = {}; + for (const [key, value] of Object.entries(advanced.value)) { + if (['string', 'number', 'boolean'].includes(value.type)) { + metadata[key] = value; + } + } + } + } + const toolReferences: IVariableReference[] = []; if (ast.body) { const bodyOffset = ast.body.offset; const bodyVarRefs = ast.body.variableReferences; for (let i = bodyVarRefs.length - 1; i >= 0; i--) { // in reverse order const { name, offset } = bodyVarRefs[i]; const range = new OffsetRange(offset - bodyOffset, offset - bodyOffset + name.length + 1); - variableReferences.push({ name, range }); + toolReferences.push({ name, range }); } - body = ast.body.getContent(); } + const modeInstructions = { + content: ast.body?.getContent() ?? '', + toolReferences, + metadata, + } satisfies IChatModeInstructions; + const name = getCleanPromptName(uri); if (!ast.header) { - return { uri, name, body, variableReferences }; + return { uri, name, modeInstructions }; } const { description, model, tools } = ast.header; - return { uri, name, description, model, tools, body, variableReferences }; + return { uri, name, description, model, tools, modeInstructions }; }) ); - - return metadataList; - } - - public async parse(uri: URI, type: PromptsType, token: CancellationToken): Promise { - let parser: PromptParser | undefined; - try { - const languageId = getLanguageIdForPromptsType(type); - parser = this.instantiationService.createInstance(PromptParser, uri, { allowNonPromptFiles: true, languageId, updateOnChange: false }).start(token); - const completed = await parser.settled(); - if (!completed) { - throw new Error(localize('promptParser.notCompleted', "Prompt parser for {0} did not complete.", uri.toString())); - } - const fullContent = await parser.getFullContent(); - const transformer = new PositionOffsetTransformer(fullContent); - const variableReferences = parser.variableReferences.map(ref => { - return { - name: ref.name, - range: transformer.getOffsetRange(ref.range) - }; - }).sort((a, b) => b.range.start - a.range.start); // in reverse order - // make a copy, to avoid leaking the parser instance - return { - uri: parser.uri, - metadata: parser.metadata, - variableReferences, - fileReferences: parser.references.map(ref => ref.uri), - }; - } finally { - parser?.dispose(); - } + return customChatModes; } public async parseNew(uri: URI, token: CancellationToken): Promise { diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/utils/objectCache.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/utils/objectCache.ts deleted file mode 100644 index 443d6e44c94..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/utils/objectCache.ts +++ /dev/null @@ -1,153 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { Disposable, DisposableMap } from '../../../../../../base/common/lifecycle.js'; -import { ObservableDisposable, assertNotDisposed } from './observableDisposable.js'; - -/** - * Generic cache for object instances. Guarantees to return only non-disposed - * objects from the {@linkcode get} method. If a requested object is not yet - * in the cache or is disposed already, the {@linkcode factory} callback is - * called to create a new object. - * - * @throws if {@linkcode factory} callback returns a disposed object. - * - * ## Examples - * - * ```typescript - * // a class that will be used as a cache key; the key can be of any - * // non-nullable type, including primitives like `string` or `number`, - * // but in this case we use an object pointer as a key - * class KeyObject {} - * - * // a class for testing purposes - * class TestObject extends ObservableDisposable { - * constructor( - * public readonly id: KeyObject, - * ) {} - * }; - * - * // create an object cache instance providing it a factory function that - * // is responsible for creating new objects based on the provided key if - * // the cache does not contain the requested object yet or an existing - * // object is already disposed - * const cache = new ObjectCache((key) => { - * // create a new test object based on the provided key - * return new TestObject(key); - * }); - * - * // create two keys - * const key1 = new KeyObject(); - * const key2 = new KeyObject(); - * - * // get an object from the cache by its key - * const object1 = cache.get(key1); // returns a new test object - * - * // validate that the new object has the correct key - * assert( - * object1.id === key1, - * 'Object 1 must have correct ID.', - * ); - * - * // returns the same cached test object - * const object2 = cache.get(key1); - * - * // validate that the same exact object is returned from the cache - * assert( - * object1 === object2, - * 'Object 2 the same cached object as object 1.', - * ); - * - * // returns a new test object - * const object3 = cache.get(key2); - * - * // validate that the new object has the correct key - * assert( - * object3.id === key2, - * 'Object 3 must have correct ID.', - * ); - * - * assert( - * object3 !== object1, - * 'Object 3 must be a new object.', - * ); - * ``` - */ -export class ObjectCache< - TValue extends ObservableDisposable, - TKey extends NonNullable = string, -> extends Disposable { - private readonly cache: DisposableMap = - this._register(new DisposableMap()); - - constructor( - private readonly factory: (key: TKey) => TValue & { isDisposed: false }, - ) { - super(); - } - - /** - * Get an existing object from the cache. If a requested object is not yet - * in the cache or is disposed already, the {@linkcode factory} callback is - * called to create a new object. - * - * @throws if {@linkcode factory} callback returns a disposed object. - * @param key - ID of the object in the cache - */ - public get(key: TKey): TValue & { isDisposed: false } { - let object = this.cache.get(key); - - // if object is already disposed, remove it from the cache - if (object?.isDisposed) { - this.cache.deleteAndLeak(key); - object = undefined; - } - - // if object exists and is not disposed, return it - if (object) { - // must always hold true due to the check above - assertNotDisposed( - object, - 'Object must not be disposed.', - ); - - return object; - } - - // create a new object by calling the factory - object = this.factory(key); - - // newly created object must not be disposed - assertNotDisposed( - object, - 'Newly created object must not be disposed.', - ); - - // remove it from the cache automatically on dispose - object.addDisposables( - object.onDispose(() => { - this.cache.deleteAndLeak(key); - })); - this.cache.set(key, object); - - return object; - } - - /** - * Remove an object from the cache by its key. - * - * @param key ID of the object to remove. - * @param dispose Whether the removed object must be disposed. - */ - public remove(key: TKey, dispose: boolean): this { - if (dispose) { - this.cache.deleteAndDispose(key); - return this; - } - - this.cache.deleteAndLeak(key); - return this; - } -} diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/utils/observableDisposable.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/utils/observableDisposable.ts deleted file mode 100644 index a0e6a70e733..00000000000 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/utils/observableDisposable.ts +++ /dev/null @@ -1,89 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { Disposable, DisposableStore, IDisposable, toDisposable } from '../../../../../../base/common/lifecycle.js'; - -/** -* @deprecated do not use this, https://github.com/microsoft/vscode/issues/248366 - */ -export abstract class ObservableDisposable extends Disposable { - /** - * Underlying disposables store this object relies on. - */ - private readonly store = this._register(new DisposableStore()); - - /** - * Check if the current object is already has been disposed. - */ - public get isDisposed(): boolean { - return this.store.isDisposed; - } - - /** - * The event is fired when this object is disposed. - * Note! Executes the callback immediately if already disposed. - * - * @param callback The callback function to be called on updates. - */ - public onDispose(callback: () => void): IDisposable { - // if already disposed, execute the callback immediately - if (this.isDisposed) { - const timeoutHandle = setTimeout(callback); - - return toDisposable(() => { - clearTimeout(timeoutHandle); - }); - } - - return this.store.add(toDisposable(callback)); - } - - /** - * Adds disposable object(s) to the list of disposables - * that will be disposed with this object. - */ - public addDisposables(...disposables: IDisposable[]): this { - for (const disposable of disposables) { - this.store.add(disposable); - } - - return this; - } - - /** - * Assert that the current object was not yet disposed. - * - * @throws If the current object was already disposed. - * @param error Error message or error object to throw if assertion fails. - */ - public assertNotDisposed( - error: string | Error, - ): asserts this is TNotDisposed { - assertNotDisposed(this, error); - } -} - -/** - * @deprecated do not use this, https://github.com/microsoft/vscode/issues/248366 - */ -type TNotDisposed = TObject & { isDisposed: false }; - -/** - * @deprecated do not use this, https://github.com/microsoft/vscode/issues/248366 - */ -export function assertNotDisposed( - object: TObject, - error: string | Error, -): asserts object is TNotDisposed { - if (!object.isDisposed) { - return; - } - - const errorToThrow = typeof error === 'string' - ? new Error(error) - : error; - - throw errorToThrow; -} diff --git a/src/vs/workbench/contrib/chat/common/tools/tools.ts b/src/vs/workbench/contrib/chat/common/tools/tools.ts index b702400ea21..e1e6d5a7b3d 100644 --- a/src/vs/workbench/contrib/chat/common/tools/tools.ts +++ b/src/vs/workbench/contrib/chat/common/tools/tools.ts @@ -29,7 +29,7 @@ export class BuiltinToolsContribution extends Disposable implements IWorkbenchCo // Check if write-only mode is enabled for the todo tool const writeOnlyMode = this.configurationService.getValue(TodoListToolWriteOnlySettingId) === true; const todoToolData = createManageTodoListToolData(writeOnlyMode); - const manageTodoListTool = instantiationService.createInstance(ManageTodoListTool, writeOnlyMode); + const manageTodoListTool = this._register(instantiationService.createInstance(ManageTodoListTool, writeOnlyMode)); this._register(toolsService.registerTool(todoToolData, manageTodoListTool)); // Register the confirmation tool diff --git a/src/vs/workbench/contrib/chat/test/common/chatModeService.test.ts b/src/vs/workbench/contrib/chat/test/common/chatModeService.test.ts index 09d2629d456..9b17fbdd988 100644 --- a/src/vs/workbench/contrib/chat/test/common/chatModeService.test.ts +++ b/src/vs/workbench/contrib/chat/test/common/chatModeService.test.ts @@ -110,8 +110,8 @@ suite('ChatModeService', () => { name: 'Test Mode', description: 'A test custom mode', tools: ['tool1', 'tool2'], - body: 'Custom mode body', - variableReferences: [] + modeInstructions: { content: 'Custom mode body', toolReferences: [] } + }; promptsService.setCustomModes([customMode]); @@ -129,7 +129,7 @@ suite('ChatModeService', () => { assert.strictEqual(testMode.description.get(), customMode.description); assert.strictEqual(testMode.kind, ChatModeKind.Agent); assert.deepStrictEqual(testMode.customTools?.get(), customMode.tools); - assert.strictEqual(testMode.body?.get(), customMode.body); + assert.deepStrictEqual(testMode.modeInstructions?.get(), customMode.modeInstructions); assert.strictEqual(testMode.uri?.get().toString(), customMode.uri.toString()); }); @@ -144,8 +144,7 @@ suite('ChatModeService', () => { name: 'Test Mode', description: 'A test custom mode', tools: [], - body: 'Custom mode body', - variableReferences: [] + modeInstructions: { content: 'Custom mode body', toolReferences: [] }, }; promptsService.setCustomModes([customMode]); @@ -162,8 +161,7 @@ suite('ChatModeService', () => { name: 'Findable Mode', description: 'A findable custom mode', tools: [], - body: 'Findable mode body', - variableReferences: [] + modeInstructions: { content: 'Findable mode body', toolReferences: [] }, }; promptsService.setCustomModes([customMode]); @@ -185,9 +183,8 @@ suite('ChatModeService', () => { name: 'Initial Mode', description: 'Initial description', tools: ['tool1'], - body: 'Initial body', + modeInstructions: { content: 'Initial body', toolReferences: [] }, model: 'gpt-4', - variableReferences: [] }; promptsService.setCustomModes([initialMode]); @@ -202,7 +199,7 @@ suite('ChatModeService', () => { ...initialMode, description: 'Updated description', tools: ['tool1', 'tool2'], - body: 'Updated body', + modeInstructions: { content: 'Updated body', toolReferences: [] }, model: 'Updated model' }; @@ -218,7 +215,7 @@ suite('ChatModeService', () => { // But the observable properties should be updated assert.strictEqual(updatedCustomMode.description.get(), 'Updated description'); assert.deepStrictEqual(updatedCustomMode.customTools?.get(), ['tool1', 'tool2']); - assert.strictEqual(updatedCustomMode.body?.get(), 'Updated body'); + assert.deepStrictEqual(updatedCustomMode.modeInstructions?.get(), { content: 'Updated body', toolReferences: [] }); assert.strictEqual(updatedCustomMode.model?.get(), 'Updated model'); }); @@ -228,8 +225,7 @@ suite('ChatModeService', () => { name: 'Mode 1', description: 'First mode', tools: [], - body: 'Mode 1 body', - variableReferences: [] + modeInstructions: { content: 'Mode 1 body', toolReferences: [] }, }; const mode2: ICustomChatMode = { @@ -237,8 +233,7 @@ suite('ChatModeService', () => { name: 'Mode 2', description: 'Second mode', tools: [], - body: 'Mode 2 body', - variableReferences: [] + modeInstructions: { content: 'Mode 2 body', toolReferences: [] }, }; // Add both modes diff --git a/src/vs/workbench/contrib/chat/test/common/mockChatService.ts b/src/vs/workbench/contrib/chat/test/common/mockChatService.ts index dd2c2aa4085..16241e2867b 100644 --- a/src/vs/workbench/contrib/chat/test/common/mockChatService.ts +++ b/src/vs/workbench/contrib/chat/test/common/mockChatService.ts @@ -16,6 +16,7 @@ export class MockChatService implements IChatService { requestInProgressObs = observableValue('name', false); edits2Enabled: boolean = false; _serviceBrand: undefined; + editingSessions = []; transferredSessionData: IChatTransferredSessionData | undefined; onDidSubmitRequest: Event<{ chatSessionId: string }> = Event.None; diff --git a/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/frontMatterDecoder/frontMatterBoolean.test.ts b/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/frontMatterDecoder/frontMatterBoolean.test.ts deleted file mode 100644 index 8d32c30d851..00000000000 --- a/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/frontMatterDecoder/frontMatterBoolean.test.ts +++ /dev/null @@ -1,317 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import assert from 'assert'; -import { Range } from '../../../../../../../../../editor/common/core/range.js'; -import { Word } from '../../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/tokens.js'; -import { randomBoolean } from '../../../../../../../../../base/test/common/testUtils.js'; -import { FrontMatterBoolean } from '../../../../../../common/promptSyntax/codecs/base/frontMatterCodec/tokens/index.js'; -import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../../../../../../base/test/common/utils.js'; -import { FrontMatterSequence } from '../../../../../../common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterSequence.js'; - -suite('FrontMatterBoolean', () => { - ensureNoDisposablesAreLeakedInTestSuite(); - - suite('equals()', () => { - suite('base case', () => { - test('true', () => { - // both values should yield the same result - const booleanText = (randomBoolean()) - ? 'true' - : 'TRUE'; - - const boolean = new FrontMatterBoolean( - new Word( - new Range(1, 1, 1, 5), - booleanText, - ), - ); - - const other = new FrontMatterBoolean( - new Word( - new Range(1, 1, 1, 5), - booleanText, - ), - ); - - assert.strictEqual( - boolean.value, - true, - 'Must have correct boolean value.', - ); - - assert( - boolean.equals(other), - 'Booleans must be equal.', - ); - }); - - test('false', () => { - // both values should yield the same result - const booleanText = (randomBoolean()) - ? 'false' - : 'FALSE'; - - const boolean = new FrontMatterBoolean( - new Word( - new Range(5, 15, 5, 15 + 6), - booleanText, - ), - ); - - const other = new FrontMatterBoolean( - new Word( - new Range(5, 15, 5, 15 + 6), - booleanText, - ), - ); - - assert.strictEqual( - boolean.value, - false, - 'Must have correct boolean value.', - ); - - assert( - boolean.equals(other), - 'Booleans must be equal.', - ); - }); - }); - - suite('non-boolean token', () => { - suite('word token', () => { - test('true', () => { - // both values should yield the same result - const booleanText = (randomBoolean()) - ? 'true' - : 'TRUE'; - - const boolean = new FrontMatterBoolean( - new Word( - new Range(1, 1, 1, 5), - booleanText, - ), - ); - - const other = new Word( - new Range(1, 1, 1, 5), - booleanText, - ); - - assert( - boolean.equals(other) === false, - 'Booleans must not be equal.', - ); - }); - - test('false', () => { - // both values should yield the same result - const booleanText = (randomBoolean()) - ? 'false' - : 'FALSE'; - - const boolean = new FrontMatterBoolean( - new Word( - new Range(1, 2, 1, 2 + 6), - booleanText, - ), - ); - - const other = new Word( - new Range(1, 2, 1, 2 + 6), - booleanText, - ); - - assert( - boolean.equals(other) === false, - 'Booleans must not be equal.', - ); - }); - }); - - suite('sequence token', () => { - test('true', () => { - // both values should yield the same result - const booleanText = (randomBoolean()) - ? 'true' - : 'TRUE'; - - const boolean = new FrontMatterBoolean( - new Word( - new Range(1, 1, 1, 5), - booleanText, - ), - ); - - const other = new FrontMatterSequence([ - new Word( - new Range(1, 1, 1, 5), - booleanText, - ), - ]); - - assert( - boolean.equals(other) === false, - 'Booleans must not be equal.', - ); - }); - - test('false', () => { - // both values should yield the same result - const booleanText = (randomBoolean()) - ? 'false' - : 'FALSE'; - - const boolean = new FrontMatterBoolean( - new Word( - new Range(1, 2, 1, 2 + 6), - booleanText, - ), - ); - - const other = new FrontMatterSequence([ - new Word( - new Range(1, 2, 1, 2 + 6), - booleanText, - ), - ]); - - assert( - boolean.equals(other) === false, - 'Booleans must not be equal.', - ); - }); - }); - }); - - suite('different range', () => { - test('true', () => { - // both values should yield the same result - const booleanText = (randomBoolean()) - ? 'true' - : 'TRUE'; - - const boolean = new FrontMatterBoolean( - new Word( - new Range(1, 2, 1, 2 + 4), - booleanText, - ), - ); - - const other = new FrontMatterBoolean( - new Word( - new Range(3, 2, 3, 2 + 4), - booleanText, - ), - ); - - assert( - boolean.equals(other) === false, - 'Booleans must not be equal.', - ); - }); - - test('false', () => { - // both values should yield the same result - const booleanText = (randomBoolean()) - ? 'false' - : 'FALSE'; - - const boolean = new FrontMatterBoolean( - new Word( - new Range(5, 15, 5, 15 + 5), - booleanText, - ), - ); - - const other = new FrontMatterBoolean( - new Word( - new Range(4, 15, 4, 15 + 5), - booleanText, - ), - ); - - assert( - boolean.equals(other) === false, - 'Booleans must not be equal.', - ); - }); - }); - - suite('different text', () => { - test('true', () => { - const boolean = new FrontMatterBoolean( - new Word( - new Range(1, 1, 1, 5), - 'true', - ), - ); - - const other = new FrontMatterBoolean( - new Word( - new Range(1, 1, 1, 5), - 'True', - ), - ); - - assert( - boolean.equals(other) === false, - 'Booleans must not be equal.', - ); - }); - - test('false', () => { - const boolean = new FrontMatterBoolean( - new Word( - new Range(5, 15, 5, 15 + 6), - 'FALSE', - ), - ); - - const other = new FrontMatterBoolean( - new Word( - new Range(5, 15, 5, 15 + 6), - 'false', - ), - ); - - assert( - boolean.equals(other) === false, - 'Booleans must not be equal.', - ); - }); - }); - - test('throws if cannot be converted to a boolean', () => { - assert.throws(() => { - new FrontMatterBoolean( - new Word( - new Range(1, 1, 1, 5), - 'true1', - ), - ); - }); - - assert.throws(() => { - new FrontMatterBoolean( - new Word( - new Range(2, 5, 2, 5 + 6), - 'fal se', - ), - ); - }); - - assert.throws(() => { - new FrontMatterBoolean( - new Word( - new Range(20, 4, 20, 4 + 1), - '1', - ), - ); - }); - }); - }); -}); diff --git a/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/frontMatterDecoder/frontMatterDecoder.test.ts b/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/frontMatterDecoder/frontMatterDecoder.test.ts deleted file mode 100644 index b4f78b9d19b..00000000000 --- a/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/frontMatterDecoder/frontMatterDecoder.test.ts +++ /dev/null @@ -1,415 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - - -import { Range } from '../../../../../../../../../editor/common/core/range.js'; -import { TestDecoder } from '../utils/testDecoder.js'; -import { VSBuffer } from '../../../../../../../../../base/common/buffer.js'; -import { newWriteableStream } from '../../../../../../../../../base/common/stream.js'; -import { NewLine } from '../../../../../../common/promptSyntax/codecs/base/linesCodec/tokens/newLine.js'; -import { DoubleQuote } from '../../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/doubleQuote.js'; -import { type TSimpleDecoderToken } from '../../../../../../common/promptSyntax/codecs/base/simpleCodec/simpleDecoder.js'; -import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../../../../../../base/test/common/utils.js'; -import { LeftBracket, RightBracket } from '../../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/brackets.js'; -import { FrontMatterDecoder } from '../../../../../../common/promptSyntax/codecs/base/frontMatterCodec/frontMatterDecoder.js'; -import { FrontMatterSequence } from '../../../../../../common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterSequence.js'; -import { ExclamationMark, Quote, Tab, Word, Space, Colon, VerticalTab, Comma, Dash } from '../../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/tokens.js'; -import { FrontMatterBoolean, FrontMatterString, FrontMatterArray, FrontMatterRecord, FrontMatterRecordDelimiter, FrontMatterRecordName } from '../../../../../../common/promptSyntax/codecs/base/frontMatterCodec/tokens/index.js'; - -/** - * Front Matter decoder for testing purposes. - */ -export class TestFrontMatterDecoder extends TestDecoder { - constructor() { - const stream = newWriteableStream(null); - const decoder = new FrontMatterDecoder(stream); - - super(stream, decoder); - } -} - -suite('FrontMatterDecoder', () => { - const disposables = ensureNoDisposablesAreLeakedInTestSuite(); - - test('produces expected tokens', async () => { - const test = disposables.add(new TestFrontMatterDecoder()); - - await test.run( - [ - 'just: "write some yaml "', - 'write-some :\t[ \' just\t \', "yaml!", true, , ,]', - 'anotherField \t\t\t : FALSE ', - ], - [ - // first record - new FrontMatterRecord([ - new FrontMatterRecordName([ - new Word(new Range(1, 1, 1, 1 + 4), 'just'), - ]), - new FrontMatterRecordDelimiter([ - new Colon(new Range(1, 5, 1, 6)), - new Space(new Range(1, 6, 1, 7)), - ]), - new FrontMatterString([ - new DoubleQuote(new Range(1, 7, 1, 8)), - new Word(new Range(1, 8, 1, 8 + 5), 'write'), - new Space(new Range(1, 13, 1, 14)), - new Word(new Range(1, 14, 1, 14 + 4), 'some'), - new Space(new Range(1, 18, 1, 19)), - new Word(new Range(1, 19, 1, 19 + 4), 'yaml'), - new Space(new Range(1, 23, 1, 24)), - new DoubleQuote(new Range(1, 24, 1, 25)), - ]), - ]), - new NewLine(new Range(1, 25, 1, 26)), - // second record - new FrontMatterRecord([ - new FrontMatterRecordName([ - new Word(new Range(2, 1, 2, 1 + 5), 'write'), - new Dash(new Range(2, 6, 2, 7)), - new Word(new Range(2, 7, 2, 7 + 4), 'some'), - ]), - new FrontMatterRecordDelimiter([ - new Colon(new Range(2, 12, 2, 13)), - new Tab(new Range(2, 13, 2, 14)), - ]), - new FrontMatterArray([ - new LeftBracket(new Range(2, 14, 2, 15)), - new FrontMatterString([ - new Quote(new Range(2, 16, 2, 17)), - new Space(new Range(2, 17, 2, 18)), - new Word(new Range(2, 18, 2, 18 + 4), 'just'), - new Tab(new Range(2, 22, 2, 23)), - new Space(new Range(2, 23, 2, 24)), - new Quote(new Range(2, 24, 2, 25)), - ]), - new FrontMatterString([ - new DoubleQuote(new Range(2, 28, 2, 29)), - new Word(new Range(2, 29, 2, 29 + 4), 'yaml'), - new ExclamationMark(new Range(2, 33, 2, 34)), - new DoubleQuote(new Range(2, 34, 2, 35)), - ]), - new FrontMatterBoolean( - new Word(new Range(2, 37, 2, 37 + 4), 'true'), - ), - new RightBracket(new Range(2, 46, 2, 47)), - ]), - ]), - new NewLine(new Range(2, 47, 2, 48)), - // third record - new FrontMatterRecord([ - new FrontMatterRecordName([ - new Word(new Range(3, 1, 3, 1 + 12), 'anotherField'), - ]), - new FrontMatterRecordDelimiter([ - new Colon(new Range(3, 19, 3, 20)), - new Space(new Range(3, 20, 3, 21)), - ]), - new FrontMatterBoolean( - new Word(new Range(3, 22, 3, 22 + 5), 'FALSE'), - ), - ]), - new Space(new Range(3, 27, 3, 28)), - ]); - }); - - suite('record', () => { - suite('values', () => { - test('unquoted string', async () => { - const test = disposables.add(new TestFrontMatterDecoder()); - - await test.run( - [ - 'just: write some yaml ', - 'anotherField \t\t : fal\v \t', - ], - [ - // first record - new FrontMatterRecord([ - new FrontMatterRecordName([ - new Word(new Range(1, 1, 1, 1 + 4), 'just'), - ]), - new FrontMatterRecordDelimiter([ - new Colon(new Range(1, 5, 1, 6)), - new Space(new Range(1, 6, 1, 7)), - ]), - new FrontMatterSequence([ - new Word(new Range(1, 7, 1, 7 + 5), 'write'), - new Space(new Range(1, 12, 1, 13)), - new Word(new Range(1, 13, 1, 13 + 4), 'some'), - new Space(new Range(1, 17, 1, 18)), - new Word(new Range(1, 18, 1, 18 + 4), 'yaml'), - ]), - ]), - new Space(new Range(1, 22, 1, 23)), - new NewLine(new Range(1, 23, 1, 24)), - // second record - new FrontMatterRecord([ - new FrontMatterRecordName([ - new Word(new Range(2, 1, 2, 1 + 12), 'anotherField'), - ]), - new FrontMatterRecordDelimiter([ - new Colon(new Range(2, 17, 2, 18)), - new Space(new Range(2, 18, 2, 19)), - ]), - new FrontMatterSequence([ - new Word(new Range(2, 20, 2, 20 + 3), 'fal'), - ]), - ]), - new VerticalTab(new Range(2, 23, 2, 24)), - new Space(new Range(2, 24, 2, 25)), - new Tab(new Range(2, 25, 2, 26)), - ]); - }); - - test('quoted string', async () => { - const test = disposables.add(new TestFrontMatterDecoder()); - - await test.run( - [ - `just\t:\t'\vdo\tsome\ntesting, please\v' `, - 'anotherField \t\t :\v\v"fal\nse"', - ], - [ - // first record - new FrontMatterRecord([ - new FrontMatterRecordName([ - new Word(new Range(1, 1, 1, 1 + 4), 'just'), - ]), - new FrontMatterRecordDelimiter([ - new Colon(new Range(1, 6, 1, 7)), - new Tab(new Range(1, 7, 1, 8)), - ]), - new FrontMatterString([ - new Quote(new Range(1, 8, 1, 9)), - new VerticalTab(new Range(1, 9, 1, 10)), - new Word(new Range(1, 10, 1, 10 + 2), 'do'), - new Tab(new Range(1, 12, 1, 13)), - new Word(new Range(1, 13, 1, 13 + 4), 'some'), - new NewLine(new Range(1, 17, 1, 18)), - new Word(new Range(2, 1, 2, 1 + 7), 'testing'), - new Comma(new Range(2, 8, 2, 9)), - new Space(new Range(2, 9, 2, 10)), - new Word(new Range(2, 10, 2, 10 + 6), 'please'), - new VerticalTab(new Range(2, 16, 2, 17)), - new Quote(new Range(2, 17, 2, 18)), - ]), - ]), - new Space(new Range(2, 18, 2, 19)), - new NewLine(new Range(2, 19, 2, 20)), - // second record - new FrontMatterRecord([ - new FrontMatterRecordName([ - new Word(new Range(3, 1, 3, 1 + 12), 'anotherField'), - ]), - new FrontMatterRecordDelimiter([ - new Colon(new Range(3, 17, 3, 18)), - new VerticalTab(new Range(3, 18, 3, 19)), - ]), - new FrontMatterString([ - new DoubleQuote(new Range(3, 20, 3, 21)), - new Word(new Range(3, 21, 3, 21 + 3), 'fal'), - new NewLine(new Range(3, 24, 3, 25)), - new Word(new Range(4, 1, 4, 1 + 2), 'se'), - new DoubleQuote(new Range(4, 3, 4, 4)), - ]), - ]), - ]); - }); - - test('boolean', async () => { - const test = disposables.add(new TestFrontMatterDecoder()); - - await test.run( - [ - 'anotherField \t\t : FALSE ', - 'my-field: true\t ', - ], - [ - // first record - new FrontMatterRecord([ - new FrontMatterRecordName([ - new Word(new Range(1, 1, 1, 1 + 12), 'anotherField'), - ]), - new FrontMatterRecordDelimiter([ - new Colon(new Range(1, 17, 1, 18)), - new Space(new Range(1, 18, 1, 19)), - ]), - new FrontMatterBoolean( - new Word( - new Range(1, 20, 1, 20 + 5), - 'FALSE', - ), - ), - ]), - new Space(new Range(1, 25, 1, 26)), - new NewLine(new Range(1, 26, 1, 27)), - // second record - new FrontMatterRecord([ - new FrontMatterRecordName([ - new Word(new Range(2, 1, 2, 1 + 2), 'my'), - new Dash(new Range(2, 3, 2, 4)), - new Word(new Range(2, 4, 2, 4 + 5), 'field'), - ]), - new FrontMatterRecordDelimiter([ - new Colon(new Range(2, 9, 2, 10)), - new Space(new Range(2, 10, 2, 11)), - ]), - new FrontMatterBoolean( - new Word( - new Range(2, 11, 2, 11 + 4), - 'true', - ), - ), - ]), - new Tab(new Range(2, 15, 2, 16)), - new Space(new Range(2, 16, 2, 17)), - ]); - }); - - suite('array', () => { - test('empty', async () => { - const test = disposables.add(new TestFrontMatterDecoder()); - - await test.run( - [ - `tools\v:\t []`, - 'anotherField \t\t :\v\v"fal\nse"', - ], - [ - // first record - new FrontMatterRecord([ - new FrontMatterRecordName([ - new Word(new Range(1, 1, 1, 1 + 5), 'tools'), - ]), - new FrontMatterRecordDelimiter([ - new Colon(new Range(1, 7, 1, 8)), - new Tab(new Range(1, 8, 1, 9)), - ]), - new FrontMatterArray([ - new LeftBracket(new Range(1, 10, 1, 11)), - new RightBracket(new Range(1, 11, 1, 12)), - ]), - ]), - new NewLine(new Range(1, 12, 1, 13)), - // second record - new FrontMatterRecord([ - new FrontMatterRecordName([ - new Word(new Range(2, 1, 2, 1 + 12), 'anotherField'), - ]), - new FrontMatterRecordDelimiter([ - new Colon(new Range(2, 17, 2, 18)), - new VerticalTab(new Range(2, 18, 2, 19)), - ]), - new FrontMatterString([ - new DoubleQuote(new Range(2, 20, 2, 21)), - new Word(new Range(2, 21, 2, 21 + 3), 'fal'), - new NewLine(new Range(2, 24, 2, 25)), - new Word(new Range(3, 1, 3, 1 + 2), 'se'), - new DoubleQuote(new Range(3, 3, 3, 4)), - ]), - ]), - ]); - }); - - test('mixed values', async () => { - const test = disposables.add(new TestFrontMatterDecoder()); - - await test.run( - [ - `tools\v:\t [true , 'toolName', some-tool]`, - ], - [ - // first record - new FrontMatterRecord([ - new FrontMatterRecordName([ - new Word(new Range(1, 1, 1, 1 + 5), 'tools'), - ]), - new FrontMatterRecordDelimiter([ - new Colon(new Range(1, 7, 1, 8)), - new Tab(new Range(1, 8, 1, 9)), - ]), - new FrontMatterArray([ - new LeftBracket(new Range(1, 10, 1, 11)), - // first array value - new FrontMatterBoolean( - new Word( - new Range(1, 11, 1, 11 + 4), - 'true', - ), - ), - // second array value - new FrontMatterString([ - new Quote(new Range(1, 18, 1, 19)), - new Word(new Range(1, 19, 1, 19 + 8), 'toolName'), - new Quote(new Range(1, 27, 1, 28)), - ]), - // third array value - new FrontMatterSequence([ - new Word(new Range(1, 30, 1, 30 + 4), 'some'), - new Dash(new Range(1, 34, 1, 35)), - new Word(new Range(1, 35, 1, 35 + 4), 'tool'), - ]), - new RightBracket(new Range(1, 39, 1, 40)), - ]), - ]), - ]); - }); - - test('redundant commas', async () => { - const test = disposables.add(new TestFrontMatterDecoder()); - - await test.run( - [ - `tools\v:\t [true ,, 'toolName', , , some-tool ,]`, - ], - [ - // first record - new FrontMatterRecord([ - new FrontMatterRecordName([ - new Word(new Range(1, 1, 1, 1 + 5), 'tools'), - ]), - new FrontMatterRecordDelimiter([ - new Colon(new Range(1, 7, 1, 8)), - new Tab(new Range(1, 8, 1, 9)), - ]), - new FrontMatterArray([ - new LeftBracket(new Range(1, 10, 1, 11)), - // first array value - new FrontMatterBoolean( - new Word( - new Range(1, 11, 1, 11 + 4), - 'true', - ), - ), - // second array value - new FrontMatterString([ - new Quote(new Range(1, 19, 1, 20)), - new Word(new Range(1, 20, 1, 20 + 8), 'toolName'), - new Quote(new Range(1, 28, 1, 29)), - ]), - // third array value - new FrontMatterSequence([ - new Word(new Range(1, 35, 1, 35 + 4), 'some'), - new Dash(new Range(1, 39, 1, 40)), - new Word(new Range(1, 40, 1, 40 + 4), 'tool'), - ]), - new RightBracket(new Range(1, 47, 1, 48)), - ]), - ]), - ]); - }); - }); - }); - }); - - test('empty', async () => { - const test = disposables.add( - new TestFrontMatterDecoder(), - ); - - await test.run('', []); - }); -}); diff --git a/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/frontMatterDecoder/frontMatterRecord.test.ts b/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/frontMatterDecoder/frontMatterRecord.test.ts deleted file mode 100644 index fba57e45209..00000000000 --- a/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/frontMatterDecoder/frontMatterRecord.test.ts +++ /dev/null @@ -1,183 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import assert from 'assert'; -import { Range } from '../../../../../../../../../editor/common/core/range.js'; -import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../../../../../../base/test/common/utils.js'; -import { FrontMatterSequence } from '../../../../../../common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterSequence.js'; -import { Colon, LeftBracket, Quote, RightBracket, Space, Tab, VerticalTab, Word } from '../../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/tokens.js'; -import { FrontMatterArray, FrontMatterBoolean, FrontMatterRecord, FrontMatterRecordDelimiter, FrontMatterRecordName, FrontMatterString } from '../../../../../../common/promptSyntax/codecs/base/frontMatterCodec/tokens/index.js'; - -suite('FrontMatterBoolean', () => { - ensureNoDisposablesAreLeakedInTestSuite(); - - suite('trimValueEnd()', () => { - test('trims space tokens at the end of record\'s value', () => { - const recordName = new FrontMatterRecordName([ - new Word( - new Range(4, 10, 4, 10 + 3), - 'key', - ), - ]); - - const recordDelimiter = new FrontMatterRecordDelimiter([ - new Colon(new Range(4, 14, 4, 15)), - new VerticalTab(new Range(4, 15, 4, 16)), - ]); - - const recordValue = new FrontMatterSequence([ - new Word(new Range(4, 18, 4, 18 + 10), 'some-value'), - new VerticalTab(new Range(4, 28, 4, 29)), - new Tab(new Range(4, 29, 4, 30)), - new Space(new Range(4, 30, 4, 31)), - new Tab(new Range(4, 31, 4, 32)), - ]); - - const record = new FrontMatterRecord([ - recordName, recordDelimiter, recordValue, - ]); - - const trimmed = record.trimValueEnd(); - assert.deepStrictEqual( - trimmed, - [ - new VerticalTab(new Range(4, 28, 4, 29)), - new Tab(new Range(4, 29, 4, 30)), - new Space(new Range(4, 30, 4, 31)), - new Tab(new Range(4, 31, 4, 32)), - ], - 'Must return correct trimmed list of spacing tokens.', - ); - - assert( - record.range.equalsRange( - new Range(4, 10, 4, 28), - ), - 'Must correctly update token range.', - ); - }); - - suite('does not trim non-sequence value tokens', () => { - test('boolean', () => { - const recordName = new FrontMatterRecordName([ - new Word( - new Range(4, 10, 4, 10 + 3), - 'yke', - ), - ]); - - const recordDelimiter = new FrontMatterRecordDelimiter([ - new Colon(new Range(4, 14, 4, 15)), - new VerticalTab(new Range(4, 15, 4, 16)), - ]); - - const recordValue = new FrontMatterBoolean( - new Word(new Range(4, 18, 4, 18 + 4), 'true'), - ); - - const record = new FrontMatterRecord([ - recordName, recordDelimiter, recordValue, - ]); - - const trimmed = record.trimValueEnd(); - assert.deepStrictEqual( - trimmed, - [], - 'Must return empty list of trimmed spacing tokens.', - ); - - assert( - record.range.equalsRange( - new Range(4, 10, 4, 22), - ), - 'Must not update token range.', - ); - }); - - test('quoted string', () => { - const recordName = new FrontMatterRecordName([ - new Word( - new Range(4, 10, 4, 10 + 3), - 'eyk', - ), - ]); - - const recordDelimiter = new FrontMatterRecordDelimiter([ - new Colon(new Range(4, 14, 4, 15)), - new VerticalTab(new Range(4, 15, 4, 16)), - ]); - - const recordValue = new FrontMatterString([ - new Quote(new Range(4, 18, 4, 19)), - new Word(new Range(4, 19, 4, 19 + 10), 'some text'), - new Quote(new Range(4, 29, 4, 30)), - ]); - - const record = new FrontMatterRecord([ - recordName, recordDelimiter, recordValue, - ]); - - const trimmed = record.trimValueEnd(); - assert.deepStrictEqual( - trimmed, - [], - 'Must return empty list of trimmed spacing tokens.', - ); - - assert( - record.range.equalsRange( - new Range(4, 10, 4, 30), - ), - 'Must not update token range.', - ); - }); - - test('array', () => { - const recordName = new FrontMatterRecordName([ - new Word( - new Range(4, 10, 4, 10 + 3), - 'yek', - ), - ]); - - const recordDelimiter = new FrontMatterRecordDelimiter([ - new Colon(new Range(4, 14, 4, 15)), - new VerticalTab(new Range(4, 15, 4, 16)), - ]); - - const recordValue = new FrontMatterArray([ - new LeftBracket(new Range(4, 18, 4, 19)), - new FrontMatterString([ - new Quote(new Range(4, 18, 4, 19)), - new Word(new Range(4, 19, 4, 19 + 10), 'some text'), - new Quote(new Range(4, 29, 4, 30)), - ]), - new FrontMatterBoolean( - new Word(new Range(4, 34, 4, 34 + 4), 'true'), - ), - new RightBracket(new Range(4, 38, 4, 39)), - ]); - - const record = new FrontMatterRecord([ - recordName, recordDelimiter, recordValue, - ]); - - const trimmed = record.trimValueEnd(); - assert.deepStrictEqual( - trimmed, - [], - 'Must return empty list of trimmed spacing tokens.', - ); - - assert( - record.range.equalsRange( - new Range(4, 10, 4, 39), - ), - 'Must not update token range.', - ); - }); - }); - }); -}); diff --git a/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/frontMatterDecoder/frontMatterSequence.test.ts b/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/frontMatterDecoder/frontMatterSequence.test.ts deleted file mode 100644 index 68602622bba..00000000000 --- a/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/frontMatterDecoder/frontMatterSequence.test.ts +++ /dev/null @@ -1,106 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import assert from 'assert'; -import { Range } from '../../../../../../../../../editor/common/core/range.js'; -import { FrontMatterValueToken } from '../../../../../../common/promptSyntax/codecs/base/frontMatterCodec/tokens/index.js'; -import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../../../../../../base/test/common/utils.js'; -import { Space, Tab, VerticalTab, Word } from '../../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/tokens.js'; -import { FrontMatterSequence } from '../../../../../../common/promptSyntax/codecs/base/frontMatterCodec/tokens/frontMatterSequence.js'; - -suite('FrontMatterSequence', () => { - ensureNoDisposablesAreLeakedInTestSuite(); - - test('extends \'FrontMatterValueToken\'', () => { - const sequence = new FrontMatterSequence([ - new Word( - new Range(1, 1, 1, 5), - 'test', - ), - ]); - - assert( - sequence instanceof FrontMatterValueToken, - 'Must extend FrontMatterValueToken class.', - ); - }); - - suite('trimEnd()', () => { - test('trims space tokens at the end of the sequence', () => { - const sequence = new FrontMatterSequence([ - new Word(new Range(4, 18, 4, 18 + 10), 'some-value'), - new Space(new Range(4, 28, 4, 29)), - new Space(new Range(4, 29, 4, 30)), - new VerticalTab(new Range(4, 30, 4, 31)), - new Tab(new Range(4, 31, 4, 32)), - new Space(new Range(4, 32, 4, 33)), - ]); - - const trimmed = sequence.trimEnd(); - assert.deepStrictEqual( - trimmed, - [ - new Space(new Range(4, 28, 4, 29)), - new Space(new Range(4, 29, 4, 30)), - new VerticalTab(new Range(4, 30, 4, 31)), - new Tab(new Range(4, 31, 4, 32)), - new Space(new Range(4, 32, 4, 33)), - ], - 'Must return correct trimmed list of spacing tokens.', - ); - - assert( - sequence.range.equalsRange( - new Range(4, 18, 4, 28), - ), - 'Must correctly update token range.', - ); - }); - - test('remains functional if only spacing tokens were present', () => { - const sequence = new FrontMatterSequence([ - new Space(new Range(4, 28, 4, 29)), - new Space(new Range(4, 29, 4, 30)), - new VerticalTab(new Range(4, 30, 4, 31)), - new Tab(new Range(4, 31, 4, 32)), - new Space(new Range(4, 32, 4, 33)), - ]); - - const trimmed = sequence.trimEnd(); - assert.deepStrictEqual( - trimmed, - [ - new Space(new Range(4, 28, 4, 29)), - new Space(new Range(4, 29, 4, 30)), - new VerticalTab(new Range(4, 30, 4, 31)), - new Tab(new Range(4, 31, 4, 32)), - new Space(new Range(4, 32, 4, 33)), - ], - 'Must return correct trimmed list of spacing tokens.', - ); - - assert( - sequence.range.equalsRange( - new Range(4, 28, 4, 28), - ), - 'Must correctly update token range.', - ); - - assert.deepStrictEqual( - sequence.children, - [ - new Word(new Range(4, 28, 4, 28), ''), - ], - 'Must contain a single empty token.', - ); - }); - }); - - test('throws if no tokens provided', () => { - assert.throws(() => { - new FrontMatterSequence([]); - }); - }); -}); diff --git a/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/linesDecoder.test.ts b/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/linesDecoder.test.ts deleted file mode 100644 index b9ac4916923..00000000000 --- a/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/linesDecoder.test.ts +++ /dev/null @@ -1,256 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import assert from 'assert'; -import { Range } from '../../../../../../../../editor/common/core/range.js'; -import { VSBuffer } from '../../../../../../../../base/common/buffer.js'; -import { DisposableStore } from '../../../../../../../../base/common/lifecycle.js'; -import { Line } from '../../../../../common/promptSyntax/codecs/base/linesCodec/tokens/line.js'; -import { TestDecoder, TTokensConsumeMethod } from './utils/testDecoder.js'; -import { NewLine } from '../../../../../common/promptSyntax/codecs/base/linesCodec/tokens/newLine.js'; -import { newWriteableStream, WriteableStream } from '../../../../../../../../base/common/stream.js'; -import { CarriageReturn } from '../../../../../common/promptSyntax/codecs/base/linesCodec/tokens/carriageReturn.js'; -import { LinesDecoder, TLineToken } from '../../../../../common/promptSyntax/codecs/base/linesCodec/linesDecoder.js'; -import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../../../../../base/test/common/utils.js'; - -/** - * Note! This decoder is also often used to test common logic of abstract {@link BaseDecoder} - * class, because the {@link LinesDecoder} is one of the simplest non-abstract decoders we have. - */ -suite('LinesDecoder', () => { - const disposables = ensureNoDisposablesAreLeakedInTestSuite(); - - /** - * Test the core logic with specific method of consuming - * tokens that are produced by a lines decoder instance. - */ - suite('core logic', () => { - testLinesDecoder('async-generator', disposables); - testLinesDecoder('consume-all-method', disposables); - testLinesDecoder('on-data-event', disposables); - }); - - suite('settled promise', () => { - test('throws if accessed on not-yet-started decoder instance', () => { - const test = disposables.add(new TestLinesDecoder()); - - assert.throws( - () => { - // testing the field access that throws here, so - // its OK to not use the returned value afterwards - // eslint-disable-next-line local/code-no-unused-expressions - test.decoder.settled; - }, - [ - 'Cannot get `settled` promise of a stream that has not been started.', - 'Please call `start()` first.', - ].join(' '), - ); - }); - }); - - suite('start', () => { - test('throws if the decoder object is already `disposed`', () => { - const test = disposables.add(new TestLinesDecoder()); - const { decoder } = test; - decoder.dispose(); - - assert.throws( - decoder.start.bind(decoder), - 'Cannot start stream that has already disposed.', - ); - }); - - test('throws if the decoder object is already `ended`', async () => { - const inputStream = newWriteableStream(null); - const test = disposables.add(new TestLinesDecoder(inputStream)); - const { decoder } = test; - - setTimeout(() => { - test.sendData([ - 'hello', - 'world :wave:', - ]); - }, 5); - - const receivedTokens = await decoder.start() - .consumeAll(); - - // a basic sanity check for received tokens - assert.strictEqual( - receivedTokens.length, - 3, - 'Must produce the correct number of tokens.', - ); - - // validate that calling `start()` after stream has ended throws - assert.throws( - decoder.start.bind(decoder), - 'Cannot start stream that has already ended.', - ); - }); - }); -}); - - -/** - * A reusable test utility that asserts that a `LinesDecoder` instance - * correctly decodes `inputData` into a stream of `TLineToken` tokens. - * - * ## Examples - * - * ```typescript - * // create a new test utility instance - * const test = disposables.add(new TestLinesDecoder()); - * - * // run the test - * await test.run( - * ' hello world\n', - * [ - * new Line(1, ' hello world'), - * new NewLine(new Range(1, 13, 1, 14)), - * ], - * ); - */ -export class TestLinesDecoder extends TestDecoder { - constructor( - inputStream?: WriteableStream, - ) { - const stream = (inputStream) - ? inputStream - : newWriteableStream(null); - - const decoder = new LinesDecoder(stream); - - super(stream, decoder); - } -} - -/** - * Common reusable test utility to validate {@link LinesDecoder} logic with - * the provided {@link tokensConsumeMethod} way of consuming decoder-produced tokens. - * - * @throws if a test fails, please see thrown error for failure details. - * @param tokensConsumeMethod The way to consume tokens produced by the decoder. - * @param disposables Test disposables store. - */ -function testLinesDecoder( - tokensConsumeMethod: TTokensConsumeMethod, - disposables: Pick, -) { - suite(tokensConsumeMethod, () => { - suite('produces expected tokens', () => { - test('input starts with line data', async () => { - const test = disposables.add(new TestLinesDecoder()); - - await test.run( - ' hello world\nhow are you doing?\n\n 😊 \r', - [ - new Line(1, ' hello world'), - new NewLine(new Range(1, 13, 1, 14)), - new Line(2, 'how are you doing?'), - new NewLine(new Range(2, 19, 2, 20)), - new Line(3, ''), - new NewLine(new Range(3, 1, 3, 2)), - new Line(4, ' 😊 '), - new NewLine(new Range(4, 5, 4, 6)), - ], - ); - }); - - test('standalone \\r is treated as new line', async () => { - const test = disposables.add(new TestLinesDecoder()); - - await test.run( - ' hello world\nhow are you doing?\n\n 😊 \r ', - [ - new Line(1, ' hello world'), - new NewLine(new Range(1, 13, 1, 14)), - new Line(2, 'how are you doing?'), - new NewLine(new Range(2, 19, 2, 20)), - new Line(3, ''), - new NewLine(new Range(3, 1, 3, 2)), - new Line(4, ' 😊 '), - new NewLine(new Range(4, 5, 4, 6)), - new Line(5, ' '), - ], - ); - }); - - test('input starts with a new line', async () => { - const test = disposables.add(new TestLinesDecoder()); - - await test.run( - '\nsome text on this line\n\n\nanother 💬 on this line\r\n🤫\n', - [ - new Line(1, ''), - new NewLine(new Range(1, 1, 1, 2)), - new Line(2, 'some text on this line'), - new NewLine(new Range(2, 23, 2, 24)), - new Line(3, ''), - new NewLine(new Range(3, 1, 3, 2)), - new Line(4, ''), - new NewLine(new Range(4, 1, 4, 2)), - new Line(5, 'another 💬 on this line'), - new CarriageReturn(new Range(5, 24, 5, 25)), - new NewLine(new Range(5, 25, 5, 26)), - new Line(6, '🤫'), - new NewLine(new Range(6, 3, 6, 4)), - ], - ); - }); - - test('input starts and ends with multiple new lines', async () => { - const test = disposables.add(new TestLinesDecoder()); - - await test.run( - '\n\n\r\nciao! 🗯️\t💭 💥 come\tva?\n\n\n\n\n', - [ - new Line(1, ''), - new NewLine(new Range(1, 1, 1, 2)), - new Line(2, ''), - new NewLine(new Range(2, 1, 2, 2)), - new Line(3, ''), - new CarriageReturn(new Range(3, 1, 3, 2)), - new NewLine(new Range(3, 2, 3, 3)), - new Line(4, 'ciao! 🗯️\t💭 💥 come\tva?'), - new NewLine(new Range(4, 25, 4, 26)), - new Line(5, ''), - new NewLine(new Range(5, 1, 5, 2)), - new Line(6, ''), - new NewLine(new Range(6, 1, 6, 2)), - new Line(7, ''), - new NewLine(new Range(7, 1, 7, 2)), - new Line(8, ''), - new NewLine(new Range(8, 1, 8, 2)), - ], - ); - }); - - test('single carriage return is treated as new line', async () => { - const test = disposables.add(new TestLinesDecoder()); - - await test.run( - '\r\rhaalo! 💥💥 how\'re you?\r ?!\r\n\r\n ', - [ - new Line(1, ''), - new NewLine(new Range(1, 1, 1, 2)), - new Line(2, ''), - new NewLine(new Range(2, 1, 2, 2)), - new Line(3, 'haalo! 💥💥 how\'re you?'), - new NewLine(new Range(3, 24, 3, 25)), - new Line(4, ' ?!'), - new CarriageReturn(new Range(4, 4, 4, 5)), - new NewLine(new Range(4, 5, 4, 6)), - new Line(5, ''), - new CarriageReturn(new Range(5, 1, 5, 2)), - new NewLine(new Range(5, 2, 5, 3)), - new Line(6, ' '), - ], - ); - }); - }); - }); -} diff --git a/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/markdownDecoder.test.ts b/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/markdownDecoder.test.ts deleted file mode 100644 index 37a3331f796..00000000000 --- a/src/vs/workbench/contrib/chat/test/common/promptSyntax/codecs/base/markdownDecoder.test.ts +++ /dev/null @@ -1,937 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import assert from 'assert'; -import { TestDecoder } from './utils/testDecoder.js'; -import { Range } from '../../../../../../../../editor/common/core/range.js'; -import { VSBuffer } from '../../../../../../../../base/common/buffer.js'; -import { newWriteableStream } from '../../../../../../../../base/common/stream.js'; -import { Tab } from '../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/tab.js'; -import { Word } from '../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/word.js'; -import { Dash } from '../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/dash.js'; -import { Space } from '../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/space.js'; -import { Slash } from '../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/slash.js'; -import { NewLine } from '../../../../../common/promptSyntax/codecs/base/linesCodec/tokens/newLine.js'; -import { FormFeed } from '../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/formFeed.js'; -import { VerticalTab } from '../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/verticalTab.js'; -import { MarkdownLink } from '../../../../../common/promptSyntax/codecs/base/markdownCodec/tokens/markdownLink.js'; -import { CarriageReturn } from '../../../../../common/promptSyntax/codecs/base/linesCodec/tokens/carriageReturn.js'; -import { MarkdownImage } from '../../../../../common/promptSyntax/codecs/base/markdownCodec/tokens/markdownImage.js'; -import { ExclamationMark } from '../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/exclamationMark.js'; -import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../../../../../base/test/common/utils.js'; -import { MarkdownComment } from '../../../../../common/promptSyntax/codecs/base/markdownCodec/tokens/markdownComment.js'; -import { LeftBracket, RightBracket } from '../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/brackets.js'; -import { MarkdownDecoder, TMarkdownToken } from '../../../../../common/promptSyntax/codecs/base/markdownCodec/markdownDecoder.js'; -import { LeftParenthesis, RightParenthesis } from '../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/parentheses.js'; -import { LeftAngleBracket, RightAngleBracket } from '../../../../../common/promptSyntax/codecs/base/simpleCodec/tokens/angleBrackets.js'; - -/** - * A reusable test utility that asserts that a `TestMarkdownDecoder` instance - * correctly decodes `inputData` into a stream of `TMarkdownToken` tokens. - * - * ## Examples - * - * ```typescript - * // create a new test utility instance - * const test = testDisposables.add(new TestMarkdownDecoder()); - * - * // run the test - * await test.run( - * ' hello [world](/etc/hosts)!', - * [ - * new Space(new Range(1, 1, 1, 2)), - * new Word(new Range(1, 2, 1, 7), 'hello'), - * new Space(new Range(1, 7, 1, 8)), - * new MarkdownLink(1, 8, '[world]', '(/etc/hosts)'), - * new Word(new Range(1, 27, 1, 28), '!'), - * new NewLine(new Range(1, 28, 1, 29)), - * ], - * ); - */ -export class TestMarkdownDecoder extends TestDecoder { - constructor() { - const stream = newWriteableStream(null); - - super(stream, new MarkdownDecoder(stream)); - } -} - -suite('MarkdownDecoder', () => { - const testDisposables = ensureNoDisposablesAreLeakedInTestSuite(); - - suite('general', () => { - test('base cases', async () => { - const test = testDisposables.add( - new TestMarkdownDecoder(), - ); - - await test.run( - [ - // basic text - ' hello world', - // text with markdown link and special characters in the filename - 'how are\t you [caption text](./some/file/path/refer🎨nce.md)?\v', - // empty line - '', - // markdown link with special characters in the link caption and path - '[(example!)](another/path/with[-and-]-chars/folder)\t ', - // markdown link `#file` variable in the caption and with absolute path - '\t[#file:something.txt](/absolute/path/to/something.txt)', - // text with a commented out markdown link - '\v\f machines must suffer', - ], - [ - // first line - new Space(new Range(1, 1, 1, 2)), - new Word(new Range(1, 2, 1, 7), 'hello'), - new Space(new Range(1, 7, 1, 8)), - new Word(new Range(1, 8, 1, 13), 'world'), - new NewLine(new Range(1, 13, 1, 14)), - // second line - new Word(new Range(2, 1, 2, 4), 'how'), - new Space(new Range(2, 4, 2, 5)), - new Word(new Range(2, 5, 2, 8), 'are'), - new Tab(new Range(2, 8, 2, 9)), - new Space(new Range(2, 9, 2, 10)), - new Word(new Range(2, 10, 2, 13), 'you'), - new Space(new Range(2, 13, 2, 14)), - new MarkdownLink(2, 14, '[caption text]', '(./some/file/path/refer🎨nce.md)'), - new Word(new Range(2, 60, 2, 61), '?'), - new VerticalTab(new Range(2, 61, 2, 62)), - new NewLine(new Range(2, 62, 2, 63)), - // third line - new NewLine(new Range(3, 1, 3, 2)), - // fourth line - new MarkdownLink(4, 1, '[(example!)]', '(another/path/with[-and-]-chars/folder)'), - new Tab(new Range(4, 52, 4, 53)), - new Space(new Range(4, 53, 4, 54)), - new NewLine(new Range(4, 54, 4, 55)), - // fifth line - new Tab(new Range(5, 1, 5, 2)), - new MarkdownLink(5, 2, '[#file:something.txt]', '(/absolute/path/to/something.txt)'), - new NewLine(new Range(5, 56, 5, 57)), - // sixth line - new VerticalTab(new Range(6, 1, 6, 2)), - new FormFeed(new Range(6, 2, 6, 3)), - new Space(new Range(6, 3, 6, 4)), - new Word(new Range(6, 4, 6, 12), 'machines'), - new Space(new Range(6, 12, 6, 13)), - new Word(new Range(6, 13, 6, 17), 'must'), - new Space(new Range(6, 17, 6, 18)), - new MarkdownComment(new Range(6, 18, 6, 18 + 41), ''), - new Space(new Range(6, 59, 6, 60)), - new Word(new Range(6, 60, 6, 66), 'suffer'), - ], - ); - }); - - test('nuanced', async () => { - const test = testDisposables.add( - new TestMarkdownDecoder(), - ); - - const inputLines = [ - // tests that the link caption contain a chat prompt `#file:` reference, while - // the file path can contain other `graphical characters` - '\v\t[#file:./another/path/to/file.txt](./real/file!path/file◆name.md)', - // tests that the link file path contain a chat prompt `#file:` reference, - // `spaces`, `emojies`, and other `graphical characters` - ' [reference ∘ label](/absolute/pa th/to-#file:file.txt/f🥸⚡️le.md)', - // tests that link caption and file path can contain `parentheses`, `spaces`, and - // `emojies` - '\f[!(hello)!](./w(())rld/nice-🦚-filen(a).git))\n\t', - // tests that the link caption can be empty, while the file path can contain `square brackets` - '[](./s[]me/pa[h!) ', - ]; - - await test.run( - inputLines, - [ - // `1st` line - new VerticalTab(new Range(1, 1, 1, 2)), - new Tab(new Range(1, 2, 1, 3)), - new MarkdownLink(1, 3, '[#file:./another/path/to/file.txt]', '(./real/file!path/file◆name.md)'), - new NewLine(new Range(1, 68, 1, 69)), - // `2nd` line - new Space(new Range(2, 1, 2, 2)), - new MarkdownLink(2, 2, '[reference ∘ label]', '(/absolute/pa th/to-#file:file.txt/f🥸⚡️le.md)'), - new NewLine(new Range(2, 67, 2, 68)), - // `3rd` line - new FormFeed(new Range(3, 1, 3, 2)), - new MarkdownLink(3, 2, '[!(hello)!]', '(./w(())rld/nice-🦚-filen(a).git)'), - new RightParenthesis(new Range(3, 50, 3, 51)), - new NewLine(new Range(3, 51, 3, 52)), - // `4th` line - new Tab(new Range(4, 1, 4, 2)), - new NewLine(new Range(4, 2, 4, 3)), - // `5th` line - new MarkdownLink(5, 1, '[]', '(./s[]me/pa[h!)'), - new Space(new Range(5, 24, 5, 25)), - ], - ); - }); - }); - - suite('links', () => { - suite('broken', () => { - test('invalid', async () => { - const test = testDisposables.add( - new TestMarkdownDecoder(), - ); - - const inputLines = [ - // incomplete link reference with empty caption - '[ ](./real/file path/file⇧name.md', - // space between caption and reference is disallowed - '[link text] (./file path/name.txt)', - ]; - - await test.run( - inputLines, - [ - // `1st` line - new LeftBracket(new Range(1, 1, 1, 2)), - new Space(new Range(1, 2, 1, 3)), - new RightBracket(new Range(1, 3, 1, 4)), - new LeftParenthesis(new Range(1, 4, 1, 5)), - new Word(new Range(1, 5, 1, 5 + 1), '.'), - new Slash(new Range(1, 6, 1, 7)), - new Word(new Range(1, 7, 1, 7 + 4), 'real'), - new Slash(new Range(1, 11, 1, 12)), - new Word(new Range(1, 12, 1, 12 + 4), 'file'), - new Space(new Range(1, 16, 1, 17)), - new Word(new Range(1, 17, 1, 17 + 4), 'path'), - new Slash(new Range(1, 21, 1, 22)), - new Word(new Range(1, 22, 1, 22 + 12), 'file⇧name.md'), - new NewLine(new Range(1, 34, 1, 35)), - // `2nd` line - new LeftBracket(new Range(2, 1, 2, 2)), - new Word(new Range(2, 2, 2, 2 + 4), 'link'), - new Space(new Range(2, 6, 2, 7)), - new Word(new Range(2, 7, 2, 7 + 4), 'text'), - new RightBracket(new Range(2, 11, 2, 12)), - new Space(new Range(2, 12, 2, 13)), - new LeftParenthesis(new Range(2, 13, 2, 14)), - new Word(new Range(2, 14, 2, 14 + 1), '.'), - new Slash(new Range(2, 15, 2, 16)), - new Word(new Range(2, 16, 2, 16 + 4), 'file'), - new Space(new Range(2, 20, 2, 21)), - new Word(new Range(2, 21, 2, 21 + 4), 'path'), - new Slash(new Range(2, 25, 2, 26)), - new Word(new Range(2, 26, 2, 26 + 8), 'name.txt'), - new RightParenthesis(new Range(2, 34, 2, 35)), - ], - ); - }); - - suite('stop characters inside caption/reference (new lines)', () => { - for (const StopCharacter of [CarriageReturn, NewLine]) { - let characterName = ''; - - if (StopCharacter === CarriageReturn) { - characterName = '\\r'; - } - if (StopCharacter === NewLine) { - characterName = '\\n'; - } - - assert( - characterName !== '', - 'The "characterName" must be set, got "empty line".', - ); - - test(`stop character - "${characterName}"`, async () => { - const test = testDisposables.add( - new TestMarkdownDecoder(), - ); - - const inputLines = [ - // stop character inside link caption - `[haa${StopCharacter.symbol}loů](./real/💁/name.txt)`, - // stop character inside link reference - `[ref text](/etc/pat${StopCharacter.symbol}h/to/file.md)`, - // stop character between line caption and link reference is disallowed - `[text]${StopCharacter.symbol}(/etc/ path/main.mdc)`, - ]; - - - await test.run( - inputLines, - [ - // `1st` input line - new LeftBracket(new Range(1, 1, 1, 2)), - new Word(new Range(1, 2, 1, 2 + 3), 'haa'), - new NewLine(new Range(1, 5, 1, 6)), // a single CR token is treated as a `new line` - new Word(new Range(2, 1, 2, 1 + 3), 'loů'), - new RightBracket(new Range(2, 4, 2, 5)), - new LeftParenthesis(new Range(2, 5, 2, 6)), - new Word(new Range(2, 6, 2, 6 + 1), '.'), - new Slash(new Range(2, 7, 2, 8)), - new Word(new Range(2, 8, 2, 8 + 4), 'real'), - new Slash(new Range(2, 12, 2, 13)), - new Word(new Range(2, 13, 2, 13 + 2), '💁'), - new Slash(new Range(2, 15, 2, 16)), - new Word(new Range(2, 16, 2, 16 + 8), 'name.txt'), - new RightParenthesis(new Range(2, 24, 2, 25)), - new NewLine(new Range(2, 25, 2, 26)), - // `2nd` input line - new LeftBracket(new Range(3, 1, 3, 2)), - new Word(new Range(3, 2, 3, 2 + 3), 'ref'), - new Space(new Range(3, 5, 3, 6)), - new Word(new Range(3, 6, 3, 6 + 4), 'text'), - new RightBracket(new Range(3, 10, 3, 11)), - new LeftParenthesis(new Range(3, 11, 3, 12)), - new Slash(new Range(3, 12, 3, 13)), - new Word(new Range(3, 13, 3, 13 + 3), 'etc'), - new Slash(new Range(3, 16, 3, 17)), - new Word(new Range(3, 17, 3, 17 + 3), 'pat'), - new NewLine(new Range(3, 20, 3, 21)), // a single CR token is treated as a `new line` - new Word(new Range(4, 1, 4, 1 + 1), 'h'), - new Slash(new Range(4, 2, 4, 3)), - new Word(new Range(4, 3, 4, 3 + 2), 'to'), - new Slash(new Range(4, 5, 4, 6)), - new Word(new Range(4, 6, 4, 6 + 7), 'file.md'), - new RightParenthesis(new Range(4, 13, 4, 14)), - new NewLine(new Range(4, 14, 4, 15)), - // `3nd` input line - new LeftBracket(new Range(5, 1, 5, 2)), - new Word(new Range(5, 2, 5, 2 + 4), 'text'), - new RightBracket(new Range(5, 6, 5, 7)), - new NewLine(new Range(5, 7, 5, 8)), // a single CR token is treated as a `new line` - new LeftParenthesis(new Range(6, 1, 6, 2)), - new Slash(new Range(6, 2, 6, 3)), - new Word(new Range(6, 3, 6, 3 + 3), 'etc'), - new Slash(new Range(6, 6, 6, 7)), - new Space(new Range(6, 7, 6, 8)), - new Word(new Range(6, 8, 6, 8 + 4), 'path'), - new Slash(new Range(6, 12, 6, 13)), - new Word(new Range(6, 13, 6, 13 + 8), 'main.mdc'), - new RightParenthesis(new Range(6, 21, 6, 22)), - ], - ); - }); - } - }); - - /** - * Same as above but these stop characters do not move the caret to the next line. - */ - suite('stop characters inside caption/reference (same line)', () => { - for (const StopCharacter of [VerticalTab, FormFeed]) { - let characterName = ''; - - if (StopCharacter === VerticalTab) { - characterName = '\\v'; - } - if (StopCharacter === FormFeed) { - characterName = '\\f'; - } - - assert( - characterName !== '', - 'The "characterName" must be set, got "empty line".', - ); - - test(`stop character - "${characterName}"`, async () => { - const test = testDisposables.add( - new TestMarkdownDecoder(), - ); - - const inputLines = [ - // stop character inside link caption - `[haa${StopCharacter.symbol}loů](./real/💁/name.txt)`, - // stop character inside link reference - `[ref text](/etc/pat${StopCharacter.symbol}h/to/file.md)`, - // stop character between line caption and link reference is disallowed - `[text]${StopCharacter.symbol}(/etc/ path/file.md)`, - ]; - - - await test.run( - inputLines, - [ - // `1st` input line - new LeftBracket(new Range(1, 1, 1, 2)), - new Word(new Range(1, 2, 1, 2 + 3), 'haa'), - new StopCharacter(new Range(1, 5, 1, 6)), // <- stop character - new Word(new Range(1, 6, 1, 6 + 3), 'loů'), - new RightBracket(new Range(1, 9, 1, 10)), - new LeftParenthesis(new Range(1, 10, 1, 11)), - new Word(new Range(1, 11, 1, 11 + 1), '.'), - new Slash(new Range(1, 12, 1, 13)), - new Word(new Range(1, 13, 1, 13 + 4), 'real'), - new Slash(new Range(1, 17, 1, 18)), - new Word(new Range(1, 18, 1, 18 + 2), '💁'), - new Slash(new Range(1, 20, 1, 21)), - new Word(new Range(1, 21, 1, 21 + 8), 'name.txt'), - new RightParenthesis(new Range(1, 29, 1, 30)), - new NewLine(new Range(1, 30, 1, 31)), - // `2nd` input line - new LeftBracket(new Range(2, 1, 2, 2)), - new Word(new Range(2, 2, 2, 2 + 3), 'ref'), - new Space(new Range(2, 5, 2, 6)), - new Word(new Range(2, 6, 2, 6 + 4), 'text'), - new RightBracket(new Range(2, 10, 2, 11)), - new LeftParenthesis(new Range(2, 11, 2, 12)), - new Slash(new Range(2, 12, 2, 13)), - new Word(new Range(2, 13, 2, 13 + 3), 'etc'), - new Slash(new Range(2, 16, 2, 17)), - new Word(new Range(2, 17, 2, 17 + 3), 'pat'), - new StopCharacter(new Range(2, 20, 2, 21)), // <- stop character - new Word(new Range(2, 21, 2, 21 + 1), 'h'), - new Slash(new Range(2, 22, 2, 23)), - new Word(new Range(2, 23, 2, 23 + 2), 'to'), - new Slash(new Range(2, 25, 2, 26)), - new Word(new Range(2, 26, 2, 26 + 7), 'file.md'), - new RightParenthesis(new Range(2, 33, 2, 34)), - new NewLine(new Range(2, 34, 2, 35)), - // `3nd` input line - new LeftBracket(new Range(3, 1, 3, 2)), - new Word(new Range(3, 2, 3, 2 + 4), 'text'), - new RightBracket(new Range(3, 6, 3, 7)), - new StopCharacter(new Range(3, 7, 3, 8)), // <- stop character - new LeftParenthesis(new Range(3, 8, 3, 9)), - new Slash(new Range(3, 9, 3, 10)), - new Word(new Range(3, 10, 3, 10 + 3), 'etc'), - new Slash(new Range(3, 13, 3, 14)), - new Space(new Range(3, 14, 3, 15)), - new Word(new Range(3, 15, 3, 15 + 4), 'path'), - new Slash(new Range(3, 19, 3, 20)), - new Word(new Range(3, 20, 3, 20 + 7), 'file.md'), - new RightParenthesis(new Range(3, 27, 3, 28)), - ], - ); - }); - } - }); - }); - }); - - - suite('images', () => { - suite('general', () => { - test('base cases', async () => { - const test = testDisposables.add( - new TestMarkdownDecoder(), - ); - - const inputData = [ - '\t![alt text](./some/path/to/file.jpg) ', - 'plain text \f![label](./image.png)\v and more text', - '![](/var/images/default) following text', - ]; - - await test.run( - inputData, - [ - // `1st` - new Tab(new Range(1, 1, 1, 2)), - new MarkdownImage(1, 2, '![alt text]', '(./some/path/to/file.jpg)'), - new Space(new Range(1, 38, 1, 39)), - new NewLine(new Range(1, 39, 1, 40)), - // `2nd` - new Word(new Range(2, 1, 2, 6), 'plain'), - new Space(new Range(2, 6, 2, 7)), - new Word(new Range(2, 7, 2, 11), 'text'), - new Space(new Range(2, 11, 2, 12)), - new FormFeed(new Range(2, 12, 2, 13)), - new MarkdownImage(2, 13, '![label]', '(./image.png)'), - new VerticalTab(new Range(2, 34, 2, 35)), - new Space(new Range(2, 35, 2, 36)), - new Word(new Range(2, 36, 2, 39), 'and'), - new Space(new Range(2, 39, 2, 40)), - new Word(new Range(2, 40, 2, 44), 'more'), - new Space(new Range(2, 44, 2, 45)), - new Word(new Range(2, 45, 2, 49), 'text'), - new NewLine(new Range(2, 49, 2, 50)), - // `3rd` - new MarkdownImage(3, 1, '![]', '(/var/images/default)'), - new Space(new Range(3, 25, 3, 26)), - new Word(new Range(3, 26, 3, 35), 'following'), - new Space(new Range(3, 35, 3, 36)), - new Word(new Range(3, 36, 3, 40), 'text'), - ], - ); - }); - - test('nuanced', async () => { - const test = testDisposables.add( - new TestMarkdownDecoder(), - ); - - const inputData = [ - '\t![](./s☻me/path/to/file.jpeg) ', - 'raw text \f![(/1.png)](./image-🥸.png)\v and more text', - // '![](/var/images/default) following text', - ]; - - await test.run( - inputData, - [ - // `1st` - new Tab(new Range(1, 1, 1, 2)), - new MarkdownImage(1, 2, '![]', '(./s☻me/path/to/file.jpeg)'), - new Space(new Range(1, 47, 1, 48)), - new NewLine(new Range(1, 48, 1, 49)), - // `2nd` - new Word(new Range(2, 1, 2, 4), 'raw'), - new Space(new Range(2, 4, 2, 5)), - new Word(new Range(2, 5, 2, 9), 'text'), - new Space(new Range(2, 9, 2, 10)), - new FormFeed(new Range(2, 10, 2, 11)), - new MarkdownImage(2, 11, '![(/1.png)]', '(./image-🥸.png)'), - new VerticalTab(new Range(2, 38, 2, 39)), - new Space(new Range(2, 39, 2, 40)), - new Word(new Range(2, 40, 2, 43), 'and'), - new Space(new Range(2, 43, 2, 44)), - new Word(new Range(2, 44, 2, 48), 'more'), - new Space(new Range(2, 48, 2, 49)), - new Word(new Range(2, 49, 2, 53), 'text'), - ], - ); - }); - }); - - suite('broken', () => { - test('invalid', async () => { - const test = testDisposables.add( - new TestMarkdownDecoder(), - ); - - const inputLines = [ - // incomplete link reference with empty caption - '![ ](./real/file path/file★name.webp', - // space between caption and reference is disallowed - '\f![link text] (./file path/name.jpg)', - // new line inside the link reference - '\v![ ](./file\npath/name.jpeg )', - ]; - - await test.run( - inputLines, - [ - // `1st` line - new ExclamationMark(new Range(1, 1, 1, 2)), - new LeftBracket(new Range(1, 2, 1, 3)), - new Space(new Range(1, 3, 1, 4)), - new RightBracket(new Range(1, 4, 1, 5)), - new LeftParenthesis(new Range(1, 5, 1, 6)), - new Word(new Range(1, 6, 1, 6 + 1), '.'), - new Slash(new Range(1, 7, 1, 8)), - new Word(new Range(1, 8, 1, 8 + 4), 'real'), - new Slash(new Range(1, 12, 1, 13)), - new Word(new Range(1, 13, 1, 13 + 4), 'file'), - new Space(new Range(1, 17, 1, 18)), - new Word(new Range(1, 18, 1, 18 + 4), 'path'), - new Slash(new Range(1, 22, 1, 23)), - new Word(new Range(1, 23, 1, 23 + 14), 'file★name.webp'), - new NewLine(new Range(1, 37, 1, 38)), - // `2nd` line - new FormFeed(new Range(2, 1, 2, 2)), - new ExclamationMark(new Range(2, 2, 2, 3)), - new LeftBracket(new Range(2, 3, 2, 4)), - new Word(new Range(2, 4, 2, 4 + 4), 'link'), - new Space(new Range(2, 8, 2, 9)), - new Word(new Range(2, 9, 2, 9 + 4), 'text'), - new RightBracket(new Range(2, 13, 2, 14)), - new Space(new Range(2, 14, 2, 15)), - new LeftParenthesis(new Range(2, 15, 2, 16)), - new Word(new Range(2, 16, 2, 16 + 1), '.'), - new Slash(new Range(2, 17, 2, 18)), - new Word(new Range(2, 18, 2, 18 + 4), 'file'), - new Space(new Range(2, 22, 2, 23)), - new Word(new Range(2, 23, 2, 23 + 4), 'path'), - new Slash(new Range(2, 27, 2, 28)), - new Word(new Range(2, 28, 2, 28 + 8), 'name.jpg'), - new RightParenthesis(new Range(2, 36, 2, 37)), - new NewLine(new Range(2, 37, 2, 38)), - // `3rd` line - new VerticalTab(new Range(3, 1, 3, 2)), - new ExclamationMark(new Range(3, 2, 3, 3)), - new LeftBracket(new Range(3, 3, 3, 4)), - new Space(new Range(3, 4, 3, 5)), - new RightBracket(new Range(3, 5, 3, 6)), - new LeftParenthesis(new Range(3, 6, 3, 7)), - new Word(new Range(3, 7, 3, 7 + 1), '.'), - new Slash(new Range(3, 8, 3, 9)), - new Word(new Range(3, 9, 3, 9 + 4), 'file'), - new NewLine(new Range(3, 13, 3, 14)), - new Word(new Range(4, 1, 4, 1 + 4), 'path'), - new Slash(new Range(4, 5, 4, 6)), - new Word(new Range(4, 6, 4, 6 + 9), 'name.jpeg'), - new Space(new Range(4, 15, 4, 16)), - new RightParenthesis(new Range(4, 16, 4, 17)), - ], - ); - }); - - suite('stop characters inside caption/reference (new lines)', () => { - for (const StopCharacter of [CarriageReturn, NewLine]) { - let characterName = ''; - - if (StopCharacter === CarriageReturn) { - characterName = '\\r'; - } - if (StopCharacter === NewLine) { - characterName = '\\n'; - } - - assert( - characterName !== '', - 'The "characterName" must be set, got "empty line".', - ); - - test(`stop character - "${characterName}"`, async () => { - const test = testDisposables.add( - new TestMarkdownDecoder(), - ); - - const inputLines = [ - // stop character inside link caption - `![haa${StopCharacter.symbol}loů](./real/💁/name.png)`, - // stop character inside link reference - `![ref text](/etc/pat${StopCharacter.symbol}h/to/file.webp)`, - // stop character between line caption and link reference is disallowed - `![text]${StopCharacter.symbol}(/etc/ path/file.jpeg)`, - ]; - - - await test.run( - inputLines, - [ - // `1st` input line - new ExclamationMark(new Range(1, 1, 1, 2)), - new LeftBracket(new Range(1, 2, 1, 3)), - new Word(new Range(1, 3, 1, 3 + 3), 'haa'), - new NewLine(new Range(1, 6, 1, 7)), // a single CR token is treated as a `new line` - new Word(new Range(2, 1, 2, 1 + 3), 'loů'), - new RightBracket(new Range(2, 4, 2, 5)), - new LeftParenthesis(new Range(2, 5, 2, 6)), - new Word(new Range(2, 6, 2, 6 + 1), '.'), - new Slash(new Range(2, 7, 2, 8)), - new Word(new Range(2, 8, 2, 8 + 4), 'real'), - new Slash(new Range(2, 12, 2, 13)), - new Word(new Range(2, 13, 2, 13 + 2), '💁'), - new Slash(new Range(2, 15, 2, 16)), - new Word(new Range(2, 16, 2, 16 + 8), 'name.png'), - new RightParenthesis(new Range(2, 24, 2, 25)), - new NewLine(new Range(2, 25, 2, 26)), - // `2nd` input line - new ExclamationMark(new Range(3, 1, 3, 2)), - new LeftBracket(new Range(3, 2, 3, 3)), - new Word(new Range(3, 3, 3, 3 + 3), 'ref'), - new Space(new Range(3, 6, 3, 7)), - new Word(new Range(3, 7, 3, 7 + 4), 'text'), - new RightBracket(new Range(3, 11, 3, 12)), - new LeftParenthesis(new Range(3, 12, 3, 13)), - new Slash(new Range(3, 13, 3, 14)), - new Word(new Range(3, 14, 3, 14 + 3), 'etc'), - new Slash(new Range(3, 17, 3, 18)), - new Word(new Range(3, 18, 3, 18 + 3), 'pat'), - new NewLine(new Range(3, 21, 3, 22)), // a single CR token is treated as a `new line` - new Word(new Range(4, 1, 4, 1 + 1), 'h'), - new Slash(new Range(4, 2, 4, 3)), - new Word(new Range(4, 3, 4, 3 + 2), 'to'), - new Slash(new Range(4, 5, 4, 6)), - new Word(new Range(4, 6, 4, 6 + 9), 'file.webp'), - new RightParenthesis(new Range(4, 15, 4, 16)), - new NewLine(new Range(4, 16, 4, 17)), - // `3nd` input line - new ExclamationMark(new Range(5, 1, 5, 2)), - new LeftBracket(new Range(5, 2, 5, 3)), - new Word(new Range(5, 3, 5, 3 + 4), 'text'), - new RightBracket(new Range(5, 7, 5, 8)), - new NewLine(new Range(5, 8, 5, 9)), // a single CR token is treated as a `new line` - new LeftParenthesis(new Range(6, 1, 6, 2)), - new Slash(new Range(6, 2, 6, 3)), - new Word(new Range(6, 3, 6, 3 + 3), 'etc'), - new Slash(new Range(6, 6, 6, 7)), - new Space(new Range(6, 7, 6, 8)), - new Word(new Range(6, 8, 6, 8 + 4), 'path'), - new Slash(new Range(6, 12, 6, 13)), - new Word(new Range(6, 13, 6, 13 + 9), 'file.jpeg'), - new RightParenthesis(new Range(6, 22, 6, 23)), - ], - ); - }); - } - }); - - /** - * Same as above but these stop characters do not move the caret to the next line. - */ - suite('stop characters inside caption/reference (same line)', () => { - for (const stopCharacter of [VerticalTab, FormFeed]) { - let characterName = ''; - - if (stopCharacter === VerticalTab) { - characterName = '\\v'; - } - if (stopCharacter === FormFeed) { - characterName = '\\f'; - } - - assert( - characterName !== '', - 'The "characterName" must be set, got "empty line".', - ); - - test(`stop character - "${characterName}"`, async () => { - const test = testDisposables.add( - new TestMarkdownDecoder(), - ); - - const inputLines = [ - // stop character inside link caption - `![haa${stopCharacter.symbol}loů](./real/💁/name)`, - // stop character inside link reference - `![ref text](/etc/pat${stopCharacter.symbol}h/to/file.webp)`, - // stop character between line caption and link reference is disallowed - `![text]${stopCharacter.symbol}(/etc/ path/image.gif)`, - ]; - - - await test.run( - inputLines, - [ - // `1st` input line - new ExclamationMark(new Range(1, 1, 1, 2)), - new LeftBracket(new Range(1, 2, 1, 3)), - new Word(new Range(1, 3, 1, 3 + 3), 'haa'), - new stopCharacter(new Range(1, 6, 1, 7)), // <- stop character - new Word(new Range(1, 7, 1, 7 + 3), 'loů'), - new RightBracket(new Range(1, 10, 1, 11)), - new LeftParenthesis(new Range(1, 11, 1, 12)), - new Word(new Range(1, 12, 1, 12 + 1), '.'), - new Slash(new Range(1, 13, 1, 14)), - new Word(new Range(1, 14, 1, 14 + 4), 'real'), - new Slash(new Range(1, 18, 1, 19)), - new Word(new Range(1, 19, 1, 19 + 2), '💁'), - new Slash(new Range(1, 21, 1, 22)), - new Word(new Range(1, 22, 1, 22 + 4), 'name'), - new RightParenthesis(new Range(1, 26, 1, 27)), - new NewLine(new Range(1, 27, 1, 28)), - // `2nd` input line - new ExclamationMark(new Range(2, 1, 2, 2)), - new LeftBracket(new Range(2, 2, 2, 3)), - new Word(new Range(2, 3, 2, 3 + 3), 'ref'), - new Space(new Range(2, 6, 2, 7)), - new Word(new Range(2, 7, 2, 7 + 4), 'text'), - new RightBracket(new Range(2, 11, 2, 12)), - new LeftParenthesis(new Range(2, 12, 2, 13)), - new Slash(new Range(2, 13, 2, 14)), - new Word(new Range(2, 14, 2, 14 + 3), 'etc'), - new Slash(new Range(2, 17, 2, 18)), - new Word(new Range(2, 18, 2, 18 + 3), 'pat'), - new stopCharacter(new Range(2, 21, 2, 22)), // <- stop character - new Word(new Range(2, 22, 2, 22 + 1), 'h'), - new Slash(new Range(2, 23, 2, 24)), - new Word(new Range(2, 24, 2, 24 + 2), 'to'), - new Slash(new Range(2, 26, 2, 27)), - new Word(new Range(2, 27, 2, 27 + 9), 'file.webp'), - new RightParenthesis(new Range(2, 36, 2, 37)), - new NewLine(new Range(2, 37, 2, 38)), - // `3nd` input line - new ExclamationMark(new Range(3, 1, 3, 2)), - new LeftBracket(new Range(3, 2, 3, 3)), - new Word(new Range(3, 3, 3, 3 + 4), 'text'), - new RightBracket(new Range(3, 7, 3, 8)), - new stopCharacter(new Range(3, 8, 3, 9)), // <- stop character - new LeftParenthesis(new Range(3, 9, 3, 10)), - new Slash(new Range(3, 10, 3, 11)), - new Word(new Range(3, 11, 3, 11 + 3), 'etc'), - new Slash(new Range(3, 14, 3, 15)), - new Space(new Range(3, 15, 3, 16)), - new Word(new Range(3, 16, 3, 16 + 4), 'path'), - new Slash(new Range(3, 20, 3, 21)), - new Word(new Range(3, 21, 3, 21 + 9), 'image.gif'), - new RightParenthesis(new Range(3, 30, 3, 31)), - ], - ); - }); - } - }); - }); - }); - - suite('comments', () => { - suite('general', () => { - test('base cases', async () => { - const test = testDisposables.add( - new TestMarkdownDecoder(), - ); - - const inputData = [ - // comment with text inside it - '\t', - // comment with a link inside - 'some text and more text ', - // comment new lines inside it - ' usual text follows', - // an empty comment - '\t\t', - // comment that was not closed properly - 'haalo\t'), - new NewLine(new Range(1, 22, 1, 23)), - // `2nd` - new Word(new Range(2, 1, 2, 5), 'some'), - new Space(new Range(2, 5, 2, 6)), - new Word(new Range(2, 6, 2, 10), 'text'), - new MarkdownComment(new Range(2, 10, 2, 10 + 46), ''), - new Space(new Range(2, 56, 2, 57)), - new Word(new Range(2, 57, 2, 60), 'and'), - new Space(new Range(2, 60, 2, 61)), - new Word(new Range(2, 61, 2, 65), 'more'), - new Space(new Range(2, 65, 2, 66)), - new Word(new Range(2, 66, 2, 70), 'text'), - new Space(new Range(2, 70, 2, 71)), - new NewLine(new Range(2, 71, 2, 72)), - // `3rd` - new MarkdownComment(new Range(3, 1, 3 + 3, 1 + 13), ''), - new Space(new Range(6, 14, 6, 15)), - new Word(new Range(6, 15, 6, 15 + 5), 'usual'), - new Space(new Range(6, 20, 6, 21)), - new Word(new Range(6, 21, 6, 21 + 4), 'text'), - new Space(new Range(6, 25, 6, 26)), - new Word(new Range(6, 26, 6, 26 + 7), 'follows'), - new NewLine(new Range(6, 33, 6, 34)), - // `4rd` - new Tab(new Range(7, 1, 7, 2)), - new MarkdownComment(new Range(7, 2, 7, 2 + 7), ''), - new Tab(new Range(7, 9, 7, 10)), - new NewLine(new Range(7, 10, 7, 11)), - // `5th` - new Word(new Range(8, 1, 8, 6), 'haalo'), - new Tab(new Range(8, 6, 8, 7)), - new MarkdownComment(new Range(8, 7, 8, 7 + 40), '>', - // comment contains `<[]>` brackets and `!` - '\t\t', - // comment contains `\t\t', - // comment contains `'), - new RightAngleBracket(new Range(1, 19, 1, 20)), - new NewLine(new Range(1, 20, 1, 21)), - // `2nd` - new MarkdownComment(new Range(2, 1, 2, 1 + 21), ''), - new Tab(new Range(2, 22, 2, 23)), - new Tab(new Range(2, 23, 2, 24)), - new NewLine(new Range(2, 24, 2, 25)), - // `3rd` - new VerticalTab(new Range(3, 1, 3, 2)), - new MarkdownComment(new Range(3, 2, 3 + 3, 1 + 7), ''), - new Tab(new Range(6, 8, 6, 9)), - new Tab(new Range(6, 9, 6, 10)), - new NewLine(new Range(6, 10, 6, 11)), - // `4rd` - new Space(new Range(7, 1, 7, 2)), - // note! comment does not have correct closing `-->`, hence the comment extends - // to the end of the text, and therefore includes the \t\v\f and space at the end - new MarkdownComment(new Range(7, 2, 8, 1 + 12), ' ', - ' < !-- світ -->\t', - '\v\f', - '`, hence the comment extends - // to the end of the text, and therefore includes the `space` at the end - new MarkdownComment(new Range(4, 1, 4, 1 + 15), '