diff --git a/build/win32/code.iss b/build/win32/code.iss
index f52a1a784b7..a1847c512ec 100644
--- a/build/win32/code.iss
+++ b/build/win32/code.iss
@@ -1534,9 +1534,8 @@ begin
   if CurStep = ssPostInstall then
   begin
 #ifdef AppxPackageName
-    if not WizardIsTaskSelected('addcontextmenufiles') then begin
-      RegDeleteKeyIncludingSubkeys({#EnvironmentRootKey}, 'Software\Classes\{#RegValueName}ContextMenu');
-    end else begin
+    // Remove the old context menu registry keys for insiders
+    if QualityIsInsiders() and WizardIsTaskSelected('addcontextmenufiles') then begin
       RegDeleteKeyIncludingSubkeys({#EnvironmentRootKey}, 'Software\Classes\*\shell\{#RegValueName}');
       RegDeleteKeyIncludingSubkeys({#EnvironmentRootKey}, 'Software\Classes\directory\shell\{#RegValueName}');
       RegDeleteKeyIncludingSubkeys({#EnvironmentRootKey}, 'Software\Classes\directory\background\shell\{#RegValueName}');