diff --git a/src/vs/editor/contrib/hover/browser/hover.ts b/src/vs/editor/contrib/hover/browser/hover.ts
index 699102e1c6e..c895d1c9888 100644
--- a/src/vs/editor/contrib/hover/browser/hover.ts
+++ b/src/vs/editor/contrib/hover/browser/hover.ts
@@ -106,7 +106,7 @@ export class ModesHoverController implements editorCommon.IEditorContribution {
 		var targetType = mouseEvent.target.type;
 		var stopKey = platform.isMacintosh ? 'metaKey' : 'ctrlKey';
 
-		if (this._isMouseDown || (this._hoverClicked && this._contentWidget.isColorPickerVisible())) {
+		if (this._isMouseDown && this._hoverClicked && this._contentWidget.isColorPickerVisible()) {
 			return;
 		}
 
@@ -139,7 +139,7 @@ export class ModesHoverController implements editorCommon.IEditorContribution {
 	}
 
 	private _hideWidgets(): void {
-		if (this._isMouseDown) {
+		if (this._isMouseDown && this._hoverClicked && this._contentWidget.isColorPickerVisible()) {
 			return;
 		}