diff --git a/src/vs/workbench/api/common/extHostLanguageModels.ts b/src/vs/workbench/api/common/extHostLanguageModels.ts
index 9325afc0185..b6bcbfdbb52 100644
--- a/src/vs/workbench/api/common/extHostLanguageModels.ts
+++ b/src/vs/workbench/api/common/extHostLanguageModels.ts
@@ -364,7 +364,7 @@ export class ExtHostLanguageModels implements ExtHostLanguageModelsShape {
 		}
 
 		for (const [modelIdentifier, modelData] of this._localModels) {
-			if (modelData.metadata.isDefaultForLocation[ChatAgentLocation.Chat]) {
+			if (modelData.metadata.isDefaultForLocation[ChatAgentLocation.Chat] && !modelData.metadata.targetChatSessionType) {
 				defaultModelId = modelIdentifier;
 				break;
 			}