diff --git a/src/vs/workbench/contrib/chat/common/languageModels.ts b/src/vs/workbench/contrib/chat/common/languageModels.ts
index 01cc12ca05b..2538c1efc41 100644
--- a/src/vs/workbench/contrib/chat/common/languageModels.ts
+++ b/src/vs/workbench/contrib/chat/common/languageModels.ts
@@ -363,13 +363,16 @@ export class LanguageModelsService implements ILanguageModelsService {
 			this._logService.warn(`[LM] Cannot update model picker preference for unknown model ${modelIdentifier}`);
 			return;
 		}
-		delete this._modelPickerUserPreferences[modelIdentifier];
-		if (model.isUserSelectable !== showInModelPicker) {
-			this._modelPickerUserPreferences[modelIdentifier] = showInModelPicker;
+
+		this._modelPickerUserPreferences[modelIdentifier] = showInModelPicker;
+		if (showInModelPicker === model.isUserSelectable) {
+			delete this._modelPickerUserPreferences[modelIdentifier];
+			this._storageService.store('chatModelPickerPreferences', this._modelPickerUserPreferences, StorageScope.PROFILE, StorageTarget.USER);
+		} else if (model.isUserSelectable !== showInModelPicker) {
 			this._storageService.store('chatModelPickerPreferences', this._modelPickerUserPreferences, StorageScope.PROFILE, StorageTarget.USER);
-			this._onLanguageModelChange.fire();
-			this._logService.trace(`[LM] Updated model picker preference for ${modelIdentifier} to ${showInModelPicker}`);
 		}
+		this._onLanguageModelChange.fire();
+		this._logService.trace(`[LM] Updated model picker preference for ${modelIdentifier} to ${showInModelPicker}`);
 	}
 
 	getVendors(): IUserFriendlyLanguageModel[] {