Support reasoning effort in custom agent files (#329263)

* Add reasoning effort support for custom agents

* Register reasoning effort in agent language service

* Fix custom agent reasoning effort typing

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* update

---------

Co-authored-by: Martin Aeschlimann <martinae@microsoft.com>
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
Mark S.
2026-08-27 17:52:20 +00:00
committed by GitHub
co-authored by Copilot Martin Aeschlimann
parent b99d111f22
commit 2f343ff4ef
8 changed files with 123 additions and 7 deletions
@@ -17,6 +17,7 @@ description: "<required>" # For agent picker and subagent discovery
name: "Agent Name" # Optional, defaults to filename
tools: [search, web] # Optional: aliases, MCP (<server>/*), extension tools
model: "Claude Sonnet 4" # Optional, uses picker default; supports array for fallback
reasoning-effort: "high" # Optional: low, medium, high, xhigh, or max
argument-hint: "Task..." # Optional, input guidance
agents: [agent1, agent2] # Optional, restrict allowed subagents by name (omit = all, [] = none)
user-invocable: true # Optional, show in agent picker (default: true)
@@ -45,6 +46,16 @@ hooks: # Optional, inline hooks for this agent's lifecycle
model: ['Claude Sonnet 4.5 (copilot)', 'GPT-5 (copilot)'] # First available model is used
```
### Reasoning Effort
Use `reasoning-effort` to set the reasoning level for the custom agent's model:
```yaml
reasoning-effort: high # low, medium, high, xhigh, or max
```
The selected model must support the configured level. If omitted, the runtime resolves the effort from the model configuration and, when applicable, the parent agent.
## Tools
Sources: built-in aliases, specific tools, MCP servers (`<server>/*`), extension tools.
@@ -114,6 +114,13 @@ function toStringEnv(env: Record<string, string | number | null>): Record<string
// Custom agents
// ---------------------------------------------------------------------------
const customAgentReasoningEfforts = ['low', 'medium', 'high', 'xhigh', 'max'] as const satisfies readonly NonNullable<CustomAgentConfig['reasoningEffort']>[];
type CustomAgentReasoningEffort = (typeof customAgentReasoningEfforts)[number];
function isCustomAgentReasoningEffort(value: string | undefined): value is CustomAgentReasoningEffort {
return customAgentReasoningEfforts.some(reasoningEffort => reasoningEffort === value);
}
/**
* Converts parsed plugin agents into the SDK's `customAgents` config.
*
@@ -122,6 +129,7 @@ function toStringEnv(env: Record<string, string | number | null>): Record<string
* - `description` is forwarded verbatim.
* - `tools` is forwarded as the SDK's allow-list; an empty / missing array
* becomes `null` so the SDK grants the agent access to all tools.
* - `reasoning-effort` is forwarded when it is a supported runtime value.
* - `prompt` is the markdown body that follows the frontmatter (or the
* full file content when there is no frontmatter).
*/
@@ -146,6 +154,7 @@ export async function toSdkCustomAgents(agents: readonly INamedPluginResource[],
const description = md.getStringValue('description');
const tools = md.getStringArrayValue('tools');
const skills = md.getStringArrayValue('skills');
const reasoningEffort = md.getStringValue('reasoning-effort');
let infer = md.getBooleanValue('infer');
const disableModelInvocation = md.getBooleanValue('disable-model-invocation');
if (infer === undefined && disableModelInvocation === true) {
@@ -161,6 +170,7 @@ export async function toSdkCustomAgents(agents: readonly INamedPluginResource[],
name,
...(description ? { description } : {}),
...(model ? { model } : {}),
...(isCustomAgentReasoningEffort(reasoningEffort) ? { reasoningEffort } : {}),
tools: tools && tools.length > 0 ? tools : null,
...(skills !== undefined ? { skills } : {}),
...(infer !== undefined ? { infer } : {}),
@@ -250,6 +250,59 @@ suite('copilotPluginConverters', () => {
}]);
});
test('parses supported reasoning-effort values from frontmatter', async () => {
const reasoningEfforts = ['low', 'medium', 'high', 'xhigh', 'max'] as const;
const agents: INamedPluginResource[] = [];
for (const reasoningEffort of reasoningEfforts) {
const agentUri = URI.from({ scheme: Schemas.inMemory, path: `/agents/${reasoningEffort}.md` });
await fileService.writeFile(agentUri, VSBuffer.fromString([
'---',
`name: ${reasoningEffort}`,
`reasoning-effort: ${reasoningEffort}`,
'---',
'Body.',
].join('\n')));
agents.push({ uri: agentUri, name: reasoningEffort });
}
const result = await toSdkCustomAgents(agents, fileService);
assert.deepStrictEqual(result, reasoningEfforts.map(reasoningEffort => ({
name: reasoningEffort,
reasoningEffort,
tools: null,
prompt: 'Body.',
})));
});
test('omits missing or unsupported reasoning-effort values', async () => {
const missingUri = URI.from({ scheme: Schemas.inMemory, path: '/agents/missing-effort.md' });
const unsupportedUri = URI.from({ scheme: Schemas.inMemory, path: '/agents/unsupported-effort.md' });
await fileService.writeFile(missingUri, VSBuffer.fromString([
'---',
'name: missing-effort',
'---',
'Body.',
].join('\n')));
await fileService.writeFile(unsupportedUri, VSBuffer.fromString([
'---',
'name: unsupported-effort',
'reasoning-effort: extreme',
'---',
'Body.',
].join('\n')));
const result = await toSdkCustomAgents([
{ uri: missingUri, name: 'missing-effort' },
{ uri: unsupportedUri, name: 'unsupported-effort' },
], fileService);
assert.deepStrictEqual(result, [
{ name: 'missing-effort', tools: null, prompt: 'Body.' },
{ name: 'unsupported-effort', tools: null, prompt: 'Body.' },
]);
});
test('parses skills and infer from frontmatter', async () => {
const agentUri = URI.from({ scheme: Schemas.inMemory, path: '/agents/skilled.md' });
await fileService.writeFile(agentUri, VSBuffer.fromString([
@@ -120,6 +120,17 @@ export const customAgentAttributes: Record<string, IAttributeDefinition> = {
type: 'scalar | sequence',
description: localize('promptHeader.agent.model', 'Specify the model that runs this custom agent. Can also be a list of models. The first available model will be used.'),
},
[PromptHeaderAttributes.reasoningEffort]: {
type: 'scalar',
description: localize('promptHeader.agent.reasoningEffort', 'Specify the reasoning effort used by this custom agent.'),
enums: [
{ name: 'low' },
{ name: 'medium' },
{ name: 'high' },
{ name: 'xhigh' },
{ name: 'max' },
],
},
[PromptHeaderAttributes.tools]: {
type: 'scalar | sequence',
description: localize('promptHeader.agent.tools', 'The set of tools that the custom agent has access to.'),
@@ -1058,11 +1058,11 @@ function isTrueOrFalse(value: IValue): boolean {
const allAttributeNames: Record<PromptsType, string[]> = {
[PromptsType.prompt]: [PromptHeaderAttributes.name, PromptHeaderAttributes.description, PromptHeaderAttributes.model, PromptHeaderAttributes.tools, PromptHeaderAttributes.mode, PromptHeaderAttributes.agent, PromptHeaderAttributes.argumentHint],
[PromptsType.instructions]: [PromptHeaderAttributes.name, PromptHeaderAttributes.description, PromptHeaderAttributes.applyTo, PromptHeaderAttributes.excludeAgent],
[PromptsType.agent]: [PromptHeaderAttributes.name, PromptHeaderAttributes.description, PromptHeaderAttributes.model, PromptHeaderAttributes.tools, PromptHeaderAttributes.advancedOptions, PromptHeaderAttributes.handOffs, PromptHeaderAttributes.argumentHint, PromptHeaderAttributes.target, PromptHeaderAttributes.infer, PromptHeaderAttributes.agents, PromptHeaderAttributes.hooks, PromptHeaderAttributes.userInvocable, PromptHeaderAttributes.disableModelInvocation, GithubPromptHeaderAttributes.github],
[PromptsType.agent]: [PromptHeaderAttributes.name, PromptHeaderAttributes.description, PromptHeaderAttributes.model, PromptHeaderAttributes.reasoningEffort, PromptHeaderAttributes.tools, PromptHeaderAttributes.advancedOptions, PromptHeaderAttributes.handOffs, PromptHeaderAttributes.argumentHint, PromptHeaderAttributes.target, PromptHeaderAttributes.infer, PromptHeaderAttributes.agents, PromptHeaderAttributes.hooks, PromptHeaderAttributes.userInvocable, PromptHeaderAttributes.disableModelInvocation, GithubPromptHeaderAttributes.github],
[PromptsType.skill]: [PromptHeaderAttributes.name, PromptHeaderAttributes.description, PromptHeaderAttributes.license, PromptHeaderAttributes.compatibility, PromptHeaderAttributes.metadata, PromptHeaderAttributes.argumentHint, PromptHeaderAttributes.userInvocable, PromptHeaderAttributes.disableModelInvocation, PromptHeaderAttributes.context],
[PromptsType.hook]: [], // hooks are JSON files, not markdown with YAML frontmatter
};
const githubCopilotAgentAttributeNames = [PromptHeaderAttributes.name, PromptHeaderAttributes.description, PromptHeaderAttributes.tools, PromptHeaderAttributes.target, GithubPromptHeaderAttributes.mcpServers, GithubPromptHeaderAttributes.github, PromptHeaderAttributes.infer];
const githubCopilotAgentAttributeNames = [PromptHeaderAttributes.name, PromptHeaderAttributes.description, PromptHeaderAttributes.tools, PromptHeaderAttributes.target, PromptHeaderAttributes.model, PromptHeaderAttributes.reasoningEffort, GithubPromptHeaderAttributes.mcpServers, GithubPromptHeaderAttributes.github, PromptHeaderAttributes.infer];
const recommendedAttributeNames: Record<PromptsType, string[]> = {
[PromptsType.prompt]: allAttributeNames[PromptsType.prompt].filter(name => !isNonRecommendedAttribute(name)),
[PromptsType.instructions]: allAttributeNames[PromptsType.instructions].filter(name => !isNonRecommendedAttribute(name)),
@@ -68,6 +68,7 @@ export namespace PromptHeaderAttributes {
export const agent = 'agent';
export const mode = 'mode';
export const model = 'model';
export const reasoningEffort = 'reasoning-effort';
export const applyTo = 'applyTo';
export const paths = 'paths';
export const tools = 'tools';
@@ -145,6 +145,7 @@ suite('PromptHeaderAutocompletion', () => {
{ label: 'hooks', result: 'hooks:\n ${1|SessionStart,SessionEnd,UserPromptSubmit,PreToolUse,PostToolUse,PreCompact,SubagentStart,SubagentStop,Stop,ErrorOccurred|}:\n - type: command\n command: "$2"' },
{ label: 'model', result: 'model: ${0:MAE 4 (olama)}' },
{ label: 'name', result: 'name: $0' },
{ label: 'reasoning-effort', result: 'reasoning-effort: ${0:low}' },
{ label: 'target', result: 'target: ${0:vscode}' },
{ label: 'tools', result: 'tools: ${0:[]}' },
{ label: 'user-invocable', result: 'user-invocable: ${0:true}' },
@@ -167,6 +168,24 @@ suite('PromptHeaderAutocompletion', () => {
].sort(sortByLabel));
});
test('complete reasoning effort attribute value', async () => {
const content = [
'---',
'description: "Test"',
'reasoning-effort: |',
'---',
].join('\n');
const actual = await getCompletions(content, PromptsType.agent);
assert.deepStrictEqual(actual.sort(sortByLabel), [
{ label: 'high', result: 'reasoning-effort: high' },
{ label: 'low', result: 'reasoning-effort: low' },
{ label: 'max', result: 'reasoning-effort: max' },
{ label: 'medium', result: 'reasoning-effort: medium' },
{ label: 'xhigh', result: 'reasoning-effort: xhigh' },
].sort(sortByLabel));
});
test('complete model attribute value with partial input', async () => {
const content = [
'---',
@@ -546,6 +546,17 @@ suite('PromptValidator', () => {
);
});
test('reasoning effort is supported in agent file', async () => {
const content = [
'---',
'description: "Test"',
'reasoning-effort: low',
'---',
].join('\n');
const markers = await validate(content, PromptsType.agent);
assert.deepStrictEqual(markers, []);
});
test('unknown attribute in agent file', async () => {
const content = [
'---',
@@ -557,7 +568,7 @@ suite('PromptValidator', () => {
assert.deepStrictEqual(
markers.map(m => ({ severity: m.severity, message: m.message, tags: m.tags })),
[
{ severity: MarkerSeverity.Hint, message: `Attribute 'applyTo' is not supported in VS Code agent files. Supported: agents, argument-hint, description, disable-model-invocation, github, handoffs, hooks, model, name, target, tools, user-invocable.`, tags: [MarkerTag.Unnecessary] },
{ severity: MarkerSeverity.Hint, message: `Attribute 'applyTo' is not supported in VS Code agent files. Supported: agents, argument-hint, description, disable-model-invocation, github, handoffs, hooks, model, name, reasoning-effort, target, tools, user-invocable.`, tags: [MarkerTag.Unnecessary] },
]
);
});
@@ -705,13 +716,14 @@ suite('PromptValidator', () => {
assert.deepStrictEqual(markers, [], 'Expected no validation issues for github-copilot target');
});
test('github-copilot agent warns about model and handoffs attributes', async () => {
test('github-copilot agent warns about handoffs attribute', async () => {
const content = [
'---',
'name: "GitHubAgent"',
'description: "GitHub Copilot agent"',
'target: github-copilot',
'model: MAE 4.1',
'reasoning-effort: high',
`tools: ['shell', 'edit']`,
`handoffs:`,
' - label: Test',
@@ -723,9 +735,8 @@ suite('PromptValidator', () => {
const markers = await validate(content, PromptsType.agent);
const messages = markers.map(m => m.message);
assert.deepStrictEqual(messages, [
'Attribute \'model\' is not supported in custom GitHub Copilot agent files. Supported: description, github, infer, mcp-servers, name, target, tools.',
'Attribute \'handoffs\' is not supported in custom GitHub Copilot agent files. Supported: description, github, infer, mcp-servers, name, target, tools.',
], 'Model and handoffs are not validated for github-copilot target');
'Attribute \'handoffs\' is not supported in custom GitHub Copilot agent files. Supported: description, github, infer, mcp-servers, model, name, reasoning-effort, target, tools.',
], 'Only handoffs is unsupported for github-copilot target, model and reasoning-effort are supported');
});
test('github-copilot agent does not validate variable references', async () => {