diff --git a/src/vs/workbench/contrib/chat/browser/chat.contribution.ts b/src/vs/workbench/contrib/chat/browser/chat.contribution.ts index d3fc00b00e6..74d3285f678 100644 --- a/src/vs/workbench/contrib/chat/browser/chat.contribution.ts +++ b/src/vs/workbench/contrib/chat/browser/chat.contribution.ts @@ -707,7 +707,7 @@ configurationRegistry.registerConfiguration({ [PromptsConfig.USE_AGENT_SKILLS]: { type: 'boolean', title: nls.localize('chat.useAgentSkills.title', "Use Agent skills",), - markdownDescription: nls.localize('chat.useAgentSkills.description', "Controls whether skills are provided as specialized capabilities to the chat requests. Skills are loaded from `.github/skills`, `.claude/skills`, and `~/.claude/skills`. The language model can load these skills on-demand if the `read` tool is available. Learn more about [Agent Skills](https://aka.ms/vscode-agent-skills).",), + markdownDescription: nls.localize('chat.useAgentSkills.description', "Controls whether skills are provided as specialized capabilities to the chat requests. Skills are loaded from `.github/skills`, `~/.copilot/skills`, `.claude/skills`, and `~/.claude/skills`. The language model can load these skills on-demand if the `read` tool is available. Learn more about [Agent Skills](https://aka.ms/vscode-agent-skills).",), default: false, restricted: true, disallowConfigurationDefault: true, diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/config/promptFileLocations.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/config/promptFileLocations.ts index 675e845bcd2..5240b09f7c9 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/config/promptFileLocations.ts +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/config/promptFileLocations.ts @@ -57,15 +57,16 @@ export const AGENTS_SOURCE_FOLDER = '.github/agents'; * Default agent skills workspace source folders. */ export const DEFAULT_AGENT_SKILLS_WORKSPACE_FOLDERS = [ - '.github/skills', - '.claude/skills' + { path: '.github/skills', type: 'github-workspace' }, + { path: '.claude/skills', type: 'claude-workspace' } ] as const; /** * Default agent skills user home source folders. */ export const DEFAULT_AGENT_SKILLS_USER_HOME_FOLDERS = [ - '.claude/skills' + { path: '.copilot/skills', type: 'copilot-personal' }, + { path: '.claude/skills', type: 'claude-personal' } ] as const; /** diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts index 06d8bada73c..8067559def9 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/service/promptsServiceImpl.ts @@ -23,6 +23,7 @@ import { ILabelService } from '../../../../../../platform/label/common/label.js' import { ILogService } from '../../../../../../platform/log/common/log.js'; import { IFilesConfigurationService } from '../../../../../services/filesConfiguration/common/filesConfigurationService.js'; import { IStorageService, StorageScope, StorageTarget } from '../../../../../../platform/storage/common/storage.js'; +import { ITelemetryService } from '../../../../../../platform/telemetry/common/telemetry.js'; import { IUserDataProfileService } from '../../../../../services/userDataProfile/common/userDataProfile.js'; import { IVariableReference } from '../../chatModes.js'; import { PromptsConfig } from '../config/config.js'; @@ -95,7 +96,8 @@ export class PromptsService extends Disposable implements IPromptsService { @IFilesConfigurationService private readonly filesConfigService: IFilesConfigurationService, @IStorageService private readonly storageService: IStorageService, @IExtensionService private readonly extensionService: IExtensionService, - @IDefaultAccountService private readonly defaultAccountService: IDefaultAccountService + @IDefaultAccountService private readonly defaultAccountService: IDefaultAccountService, + @ITelemetryService private readonly telemetryService: ITelemetryService ) { super(); @@ -619,26 +621,90 @@ export class PromptsService extends Disposable implements IPromptsService { const previewFeaturesEnabled = defaultAccount?.chat_preview_features_enabled ?? true; if (useAgentSkills && previewFeaturesEnabled) { const result: IAgentSkill[] = []; - const process = async (uri: URI, type: 'personal' | 'project'): Promise => { + const seenNames = new Set(); + const skillTypes = new Map(); + let skippedMissingName = 0; + let skippedDuplicateName = 0; + let skippedParseFailed = 0; + + const process = async (uri: URI, skillType: string, scopeType: 'personal' | 'project'): Promise => { try { const parsedFile = await this.parseNew(uri, token); const name = parsedFile.header?.name; - if (name) { - const sanitizedName = this.truncateAgentSkillName(name, uri); - const sanitizedDescription = this.truncateAgentSkillDescription(parsedFile.header?.description, uri); - result.push({ uri, type, name: sanitizedName, description: sanitizedDescription } satisfies IAgentSkill); - } else { + if (!name) { + skippedMissingName++; this.logger.error(`[findAgentSkills] Agent skill file missing name attribute: ${uri}`); + return; } + + const sanitizedName = this.truncateAgentSkillName(name, uri); + + // Check for duplicate names + if (seenNames.has(sanitizedName)) { + skippedDuplicateName++; + this.logger.warn(`[findAgentSkills] Skipping duplicate agent skill name: ${sanitizedName} at ${uri}`); + return; + } + + seenNames.add(sanitizedName); + const sanitizedDescription = this.truncateAgentSkillDescription(parsedFile.header?.description, uri); + result.push({ uri, type: scopeType, name: sanitizedName, description: sanitizedDescription } satisfies IAgentSkill); + + // Track skill type + skillTypes.set(skillType, (skillTypes.get(skillType) || 0) + 1); } catch (e) { + skippedParseFailed++; this.logger.error(`[findAgentSkills] Failed to parse Agent skill file: ${uri}`, e instanceof Error ? e.message : String(e)); } }; const workspaceSkills = await this.fileLocator.findAgentSkillsInWorkspace(token); - await Promise.all(workspaceSkills.map(uri => process(uri, 'project'))); + await Promise.all(workspaceSkills.map(({ uri, type }) => process(uri, type, 'project'))); const userSkills = await this.fileLocator.findAgentSkillsInUserHome(token); - await Promise.all(userSkills.map(uri => process(uri, 'personal'))); + await Promise.all(userSkills.map(({ uri, type }) => process(uri, type, 'personal'))); + + // Send telemetry about skill usage + type AgentSkillsFoundEvent = { + totalSkillsFound: number; + claudePersonal: number; + claudeWorkspace: number; + copilotPersonal: number; + githubWorkspace: number; + customPersonal: number; + customWorkspace: number; + skippedDuplicateName: number; + skippedMissingName: number; + skippedParseFailed: number; + }; + + type AgentSkillsFoundClassification = { + totalSkillsFound: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'Total number of agent skills found.' }; + claudePersonal: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'Number of Claude personal skills.' }; + claudeWorkspace: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'Number of Claude workspace skills.' }; + copilotPersonal: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'Number of Copilot personal skills.' }; + githubWorkspace: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'Number of GitHub workspace skills.' }; + customPersonal: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'Number of custom personal skills.' }; + customWorkspace: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'Number of custom workspace skills.' }; + skippedDuplicateName: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'Number of skills skipped due to duplicate names.' }; + skippedMissingName: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'Number of skills skipped due to missing name attribute.' }; + skippedParseFailed: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'Number of skills skipped due to parse failures.' }; + owner: 'pwang347'; + comment: 'Tracks agent skill usage, discovery, and skipped files.'; + }; + + this.telemetryService.publicLog2('agentSkillsFound', { + totalSkillsFound: result.length, + claudePersonal: skillTypes.get('claude-personal') ?? 0, + claudeWorkspace: skillTypes.get('claude-workspace') ?? 0, + copilotPersonal: skillTypes.get('copilot-personal') ?? 0, + githubWorkspace: skillTypes.get('github-workspace') ?? 0, + customPersonal: skillTypes.get('custom-personal') ?? 0, + customWorkspace: skillTypes.get('custom-workspace') ?? 0, + skippedDuplicateName, + skippedMissingName, + skippedParseFailed + }); + return result; } return undefined; diff --git a/src/vs/workbench/contrib/chat/common/promptSyntax/utils/promptFilesLocator.ts b/src/vs/workbench/contrib/chat/common/promptSyntax/utils/promptFilesLocator.ts index e8649d42f81..b7a2dd24365 100644 --- a/src/vs/workbench/contrib/chat/common/promptSyntax/utils/promptFilesLocator.ts +++ b/src/vs/workbench/contrib/chat/common/promptSyntax/utils/promptFilesLocator.ts @@ -395,13 +395,13 @@ export class PromptFilesLocator { * Searches for skills in all default directories in the workspace. * Each skill is stored in its own subdirectory with a SKILL.md file. */ - public async findAgentSkillsInWorkspace(token: CancellationToken): Promise { + public async findAgentSkillsInWorkspace(token: CancellationToken): Promise> { const workspace = this.workspaceService.getWorkspace(); - const allResults: URI[] = []; + const allResults: Array<{ uri: URI; type: string }> = []; for (const folder of workspace.folders) { - for (const skillsFolder of DEFAULT_AGENT_SKILLS_WORKSPACE_FOLDERS) { - const results = await this.findAgentSkillsInFolder(folder.uri, skillsFolder, token); - allResults.push(...results); + for (const { path, type } of DEFAULT_AGENT_SKILLS_WORKSPACE_FOLDERS) { + const results = await this.findAgentSkillsInFolder(folder.uri, path, token); + allResults.push(...results.map(uri => ({ uri, type }))); } } return allResults; @@ -411,12 +411,12 @@ export class PromptFilesLocator { * Searches for skills in all default directories in the home folder. * Each skill is stored in its own subdirectory with a SKILL.md file. */ - public async findAgentSkillsInUserHome(token: CancellationToken): Promise { + public async findAgentSkillsInUserHome(token: CancellationToken): Promise> { const userHome = await this.pathService.userHome(); - const allResults: URI[] = []; - for (const skillsFolder of DEFAULT_AGENT_SKILLS_USER_HOME_FOLDERS) { - const results = await this.findAgentSkillsInFolder(userHome, skillsFolder, token); - allResults.push(...results); + const allResults: Array<{ uri: URI; type: string }> = []; + for (const { path, type } of DEFAULT_AGENT_SKILLS_USER_HOME_FOLDERS) { + const results = await this.findAgentSkillsInFolder(userHome, path, token); + allResults.push(...results.map(uri => ({ uri, type }))); } return allResults; } diff --git a/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/promptsService.test.ts b/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/promptsService.test.ts index 024d62d78a6..e39deb1a92f 100644 --- a/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/promptsService.test.ts +++ b/src/vs/workbench/contrib/chat/test/common/promptSyntax/service/promptsService.test.ts @@ -1367,12 +1367,22 @@ suite('PromptsService', () => { path: '/home/user/.claude/skills/not-a-skill/other-file.md', contents: ['Not a skill file'], }, + { + path: '/home/user/.copilot/skills/copilot-skill-1/SKILL.md', + contents: [ + '---', + 'name: "Copilot Skill 1"', + 'description: "A Copilot skill for testing"', + '---', + 'This is Copilot skill 1 content', + ], + }, ]); const result = await service.findAgentSkills(CancellationToken.None); assert.ok(result, 'Should return results when agent skills are enabled'); - assert.strictEqual(result.length, 3, 'Should find 3 skills total'); + assert.strictEqual(result.length, 4, 'Should find 4 skills total'); // Check project skills (both from .github/skills and .claude/skills) const projectSkills = result.filter(skill => skill.type === 'project'); @@ -1390,12 +1400,17 @@ suite('PromptsService', () => { // Check personal skills const personalSkills = result.filter(skill => skill.type === 'personal'); - assert.strictEqual(personalSkills.length, 1, 'Should find 1 personal skill'); + assert.strictEqual(personalSkills.length, 2, 'Should find 2 personal skills'); - const personalSkill1 = personalSkills[0]; - assert.strictEqual(personalSkill1.name, 'Personal Skill 1'); + const personalSkill1 = personalSkills.find(skill => skill.name === 'Personal Skill 1'); + assert.ok(personalSkill1, 'Should find Personal Skill 1'); assert.strictEqual(personalSkill1.description, 'A personal skill for testing'); assert.strictEqual(personalSkill1.uri.path, '/home/user/.claude/skills/personal-skill-1/SKILL.md'); + + const copilotSkill1 = personalSkills.find(skill => skill.name === 'Copilot Skill 1'); + assert.ok(copilotSkill1, 'Should find Copilot Skill 1'); + assert.strictEqual(copilotSkill1.description, 'A Copilot skill for testing'); + assert.strictEqual(copilotSkill1.uri.path, '/home/user/.copilot/skills/copilot-skill-1/SKILL.md'); }); test('should handle parsing errors gracefully', async () => {