From 20601293fe535f83091d7ca73bae5947ce28f47f Mon Sep 17 00:00:00 2001 From: Sandeep Somavarapu Date: Sat, 24 Oct 2020 21:15:33 +0200 Subject: [PATCH] Enable syncing extensions storage - Implement logic to sync extension storage - Register keys to sync provided by extension --- .../userDataSync/common/extensionsMerge.ts | 74 ++++++++++++++++++- .../userDataSync/common/extensionsSync.ts | 63 ++++++++++++++-- .../userDataSync/common/userDataSync.ts | 1 + .../api/browser/mainThreadStorage.ts | 10 ++- .../workbench/api/common/extHost.protocol.ts | 2 + .../api/common/extHostExtensionService.ts | 4 +- src/vs/workbench/api/common/extHostMemento.ts | 14 +++- src/vs/workbench/api/common/extHostStorage.ts | 5 ++ 8 files changed, 156 insertions(+), 17 deletions(-) diff --git a/src/vs/platform/userDataSync/common/extensionsMerge.ts b/src/vs/platform/userDataSync/common/extensionsMerge.ts index c67ce0ff249..1202bbbba36 100644 --- a/src/vs/platform/userDataSync/common/extensionsMerge.ts +++ b/src/vs/platform/userDataSync/common/extensionsMerge.ts @@ -5,7 +5,8 @@ import { ISyncExtension } from 'vs/platform/userDataSync/common/userDataSync'; import { IExtensionIdentifier } from 'vs/platform/extensions/common/extensions'; -import { deepClone } from 'vs/base/common/objects'; +import { deepClone, equals } from 'vs/base/common/objects'; +import { IStringDictionary } from 'vs/base/common/collections'; export interface IMergeResult { added: ISyncExtension[]; @@ -72,6 +73,13 @@ export function merge(localExtensions: ISyncExtension[], remoteExtensions: ISync const baseToLocal = compare(lastSyncExtensionsMap, localExtensionsMap, ignoredExtensionsSet); const baseToRemote = compare(lastSyncExtensionsMap, remoteExtensionsMap, ignoredExtensionsSet); + const mergeAndUpdate = (key: string): void => { + const extension = remoteExtensionsMap.get(key)!; + extension.state = mergeExtensionState(localExtensionsMap.get(key)?.state, extension.state, lastSyncExtensionsMap?.get(key)?.state); + updated.push(massageOutgoingExtension(extension, key)); + newRemoteExtensionsMap.set(key, extension); + }; + // Remotely removed extension. for (const key of baseToRemote.removed.values()) { const e = localExtensionsMap.get(key); @@ -86,7 +94,7 @@ export function merge(localExtensions: ISyncExtension[], remoteExtensions: ISync if (baseToLocal.added.has(key)) { // Is different from local to remote if (localToRemote.updated.has(key)) { - updated.push(massageOutgoingExtension(remoteExtensionsMap.get(key)!, key)); + mergeAndUpdate(key); } } else { // Add only installed extension to local @@ -100,7 +108,7 @@ export function merge(localExtensions: ISyncExtension[], remoteExtensions: ISync // Remotely updated extensions for (const key of baseToRemote.updated.values()) { // Update in local always - updated.push(massageOutgoingExtension(remoteExtensionsMap.get(key)!, key)); + mergeAndUpdate(key); } // Locally added extensions @@ -165,7 +173,7 @@ function compare(from: Map | null, to: Map | null, to: Map | undefined, remote: IStringDictionary | undefined, base: IStringDictionary | undefined): IStringDictionary | undefined { + if (!local && !remote && !base) { + return undefined; + } + if (local && !remote && !base) { + return local; + } + if (remote && !local && !base) { + return remote; + } + + local = local || {}; + const merged: IStringDictionary = deepClone(local); + if (remote) { + const baseToRemote = base ? compareExtensionState(base, remote) : { added: Object.keys(remote).reduce((r, k) => { r.add(k); return r; }, new Set()), removed: new Set(), updated: new Set() }; + const baseToLocal = base ? compareExtensionState(base, local) : { added: Object.keys(local).reduce((r, k) => { r.add(k); return r; }, new Set()), removed: new Set(), updated: new Set() }; + // Added/Updated in remote + for (const key of [...baseToRemote.added.values(), ...baseToRemote.updated.values()]) { + merged[key] = remote[key]; + } + // Removed in remote + for (const key of baseToRemote.removed.values()) { + // Not updated in local + if (!baseToLocal.updated.has(key)) { + delete merged[key]; + } + } + } + + return merged; +} + +function compareExtensionState(from: IStringDictionary, to: IStringDictionary): { added: Set, removed: Set, updated: Set } { + const fromKeys = Object.keys(from); + const toKeys = Object.keys(to); + const added = toKeys.filter(key => fromKeys.indexOf(key) === -1).reduce((r, key) => { r.add(key); return r; }, new Set()); + const removed = fromKeys.filter(key => toKeys.indexOf(key) === -1).reduce((r, key) => { r.add(key); return r; }, new Set()); + const updated: Set = new Set(); + + for (const key of fromKeys) { + if (removed.has(key)) { + continue; + } + const value1 = from[key]; + const value2 = to[key]; + if (!equals(value1, value2)) { + updated.add(key); + } + } + + return { added, removed, updated }; +} + +function isSameExtensionState(a: IStringDictionary = {}, b: IStringDictionary = {}): boolean { + const { added, removed, updated } = compareExtensionState(a, b); + return added.size === 0 && removed.size === 0 && updated.size === 0; +} + // massage incoming extension - add optional properties function massageIncomingExtension(extension: ISyncExtension): ISyncExtension { return { ...extension, ...{ disabled: !!extension.disabled, installed: !!extension.installed } }; diff --git a/src/vs/platform/userDataSync/common/extensionsSync.ts b/src/vs/platform/userDataSync/common/extensionsSync.ts index c65ee377e1b..eb3a2825926 100644 --- a/src/vs/platform/userDataSync/common/extensionsSync.ts +++ b/src/vs/platform/userDataSync/common/extensionsSync.ts @@ -21,9 +21,12 @@ import { URI } from 'vs/base/common/uri'; import { format } from 'vs/base/common/jsonFormatter'; import { applyEdits } from 'vs/base/common/jsonEdit'; import { compare } from 'vs/base/common/strings'; -import { IStorageService } from 'vs/platform/storage/common/storage'; +import { IStorageService, StorageScope } from 'vs/platform/storage/common/storage'; import { CancellationToken } from 'vs/base/common/cancellation'; import { IIgnoredExtensionsManagementService } from 'vs/platform/userDataSync/common/ignoredExtensions'; +import { IStorageKeysSyncRegistryService } from 'vs/platform/userDataSync/common/storageKeys'; +import { getErrorMessage } from 'vs/base/common/errors'; +import { forEach } from 'vs/base/common/collections'; interface IExtensionResourceMergeResult extends IAcceptResult { readonly added: ISyncExtension[]; @@ -79,7 +82,8 @@ export class ExtensionsSynchroniser extends AbstractSynchroniser implements IUse protected readonly version: number = 3; */ /* Version 4: Change settings from `sync.${setting}` to `settingsSync.{setting}` */ - protected readonly version: number = 4; + /* Version 5: Introduce extension state */ + protected readonly version: number = 5; protected isEnabled(): boolean { return super.isEnabled() && this.extensionGalleryService.isEnabled(); } private readonly previewResource: URI = this.extUri.joinPath(this.syncPreviewFolder, 'extensions.json'); @@ -90,7 +94,7 @@ export class ExtensionsSynchroniser extends AbstractSynchroniser implements IUse constructor( @IEnvironmentService environmentService: IEnvironmentService, @IFileService fileService: IFileService, - @IStorageService storageService: IStorageService, + @IStorageService private readonly storageService: IStorageService, @IUserDataSyncStoreService userDataSyncStoreService: IUserDataSyncStoreService, @IUserDataSyncBackupStoreService userDataSyncBackupStoreService: IUserDataSyncBackupStoreService, @IExtensionManagementService private readonly extensionManagementService: IExtensionManagementService, @@ -101,6 +105,7 @@ export class ExtensionsSynchroniser extends AbstractSynchroniser implements IUse @IConfigurationService configurationService: IConfigurationService, @IUserDataSyncResourceEnablementService userDataSyncResourceEnablementService: IUserDataSyncResourceEnablementService, @ITelemetryService telemetryService: ITelemetryService, + @IStorageKeysSyncRegistryService private readonly storageKeysSyncRegistryService: IStorageKeysSyncRegistryService, ) { super(SyncResource.Extensions, fileService, environmentService, storageService, userDataSyncStoreService, userDataSyncBackupStoreService, userDataSyncResourceEnablementService, telemetryService, logService, configurationService); this._register( @@ -354,8 +359,13 @@ export class ExtensionsSynchroniser extends AbstractSynchroniser implements IUse await Promise.all([...added, ...updated].map(async e => { const installedExtension = installedExtensions.filter(installed => areSameExtensions(installed.identifier, e.identifier))[0]; - // Builtin Extension: Sync only enablement state + // Builtin Extension: Sync enablement and state if (installedExtension && installedExtension.isBuiltin) { + if (e.state) { + const extensionState = JSON.parse(this.storageService.get(e.identifier.id, StorageScope.GLOBAL) || '{}'); + forEach(e.state, ({ key, value }) => extensionState[key] = value); + this.storageService.store(e.identifier.id, JSON.stringify(extensionState), StorageScope.GLOBAL); + } if (e.disabled) { this.logService.trace(`${this.syncResourceLogLabel}: Disabling extension...`, e.identifier.id); await this.extensionEnablementService.disableExtension(e.identifier); @@ -369,9 +379,19 @@ export class ExtensionsSynchroniser extends AbstractSynchroniser implements IUse return; } - const extension = await this.extensionGalleryService.getCompatibleExtension(e.identifier, e.version); + const extension = await this.extensionGalleryService.getCompatibleExtension(e.identifier); if (extension) { try { + /* Update extension state only if + * extension is not installed or + * installed extension is same version as synced version + */ + if (e.state && (!installedExtension || installedExtension.manifest.version === e.version)) { + const extensionState = JSON.parse(this.storageService.get(extension.identifier.id, StorageScope.GLOBAL) || '{}'); + forEach(e.state, ({ key, value }) => extensionState[key] = value); + this.storageService.store(e.identifier.id, JSON.stringify(extensionState), StorageScope.GLOBAL); + } + if (e.disabled) { this.logService.trace(`${this.syncResourceLogLabel}: Disabling extension...`, e.identifier.id, extension.version); await this.extensionEnablementService.disableExtension(extension.identifier); @@ -381,8 +401,9 @@ export class ExtensionsSynchroniser extends AbstractSynchroniser implements IUse await this.extensionEnablementService.enableExtension(extension.identifier); this.logService.info(`${this.syncResourceLogLabel}: Enabled extension`, e.identifier.id, extension.version); } + // Install only if the extension does not exist - if (!installedExtension || installedExtension.manifest.version !== extension.version) { + if (!installedExtension) { this.logService.trace(`${this.syncResourceLogLabel}: Installing extension...`, e.identifier.id, extension.version); await this.extensionManagementService.installFromGallery(extension); this.logService.info(`${this.syncResourceLogLabel}: Installed extension.`, e.identifier.id, extension.version); @@ -420,14 +441,23 @@ export class ExtensionsSynchroniser extends AbstractSynchroniser implements IUse private getLocalExtensions(installedExtensions: ILocalExtension[]): ISyncExtension[] { const disabledExtensions = this.extensionEnablementService.getDisabledExtensions(); return installedExtensions - .map(({ identifier, isBuiltin }) => { - const syncExntesion: ISyncExtension = { identifier }; + .map(({ identifier, isBuiltin, manifest }) => { + const syncExntesion: ISyncExtension = { identifier, version: manifest.version }; if (disabledExtensions.some(disabledExtension => areSameExtensions(disabledExtension, identifier))) { syncExntesion.disabled = true; } if (!isBuiltin) { syncExntesion.installed = true; } + const keys = this.storageKeysSyncRegistryService.getExtensioStorageKeys({ id: identifier.id, version: manifest.version }); + try { + const extensionStorageValue = this.storageService.get(identifier.id, StorageScope.GLOBAL); + syncExntesion.state = keys.length && extensionStorageValue + ? JSON.parse(extensionStorageValue, (key, value) => !key || keys.includes(key) ? value : undefined) + : undefined; + } catch (error) { + this.logService.info(`${this.syncResourceLogLabel}: Error while parsing extension state`, getErrorMessage(error)); + } return syncExntesion; }); } @@ -440,6 +470,7 @@ export class ExtensionsInitializer extends AbstractInitializer { @IExtensionManagementService private readonly extensionManagementService: IExtensionManagementService, @IExtensionGalleryService private readonly galleryService: IExtensionGalleryService, @IGlobalExtensionEnablementService private readonly extensionEnablementService: IGlobalExtensionEnablementService, + @IStorageService private readonly storageService: IStorageService, @IFileService fileService: IFileService, @IEnvironmentService environmentService: IEnvironmentService, @IUserDataSyncLogService logService: IUserDataSyncLogService, @@ -455,15 +486,19 @@ export class ExtensionsInitializer extends AbstractInitializer { } const installedExtensions = await this.extensionManagementService.getInstalled(); + const newExtensionsToSync = new Map(); + const installedExtensionsToSync: ISyncExtension[] = []; const toInstall: { names: string[], uuids: string[] } = { names: [], uuids: [] }; const toDisable: IExtensionIdentifier[] = []; for (const extension of remoteExtensions) { if (installedExtensions.some(i => areSameExtensions(i.identifier, extension.identifier))) { + installedExtensionsToSync.push(extension); if (extension.disabled) { toDisable.push(extension.identifier); } } else { if (extension.installed) { + newExtensionsToSync.set(extension.identifier.id.toLowerCase(), extension); if (extension.identifier.uuid) { toInstall.uuids.push(extension.identifier.uuid); } else { @@ -477,6 +512,10 @@ export class ExtensionsInitializer extends AbstractInitializer { const galleryExtensions = (await this.galleryService.query({ ids: toInstall.uuids, names: toInstall.names, pageSize: toInstall.uuids.length + toInstall.names.length }, CancellationToken.None)).firstPage; for (const galleryExtension of galleryExtensions) { try { + const extensionToSync = newExtensionsToSync.get(galleryExtension.identifier.id.toLowerCase())!; + if (extensionToSync.state) { + this.storageService.store(extensionToSync.identifier.id, JSON.stringify(extensionToSync.state), StorageScope.GLOBAL); + } this.logService.trace(`Installing extension...`, galleryExtension.identifier.id); await this.extensionManagementService.installFromGallery(galleryExtension); this.logService.info(`Installed extension.`, galleryExtension.identifier.id); @@ -493,6 +532,14 @@ export class ExtensionsInitializer extends AbstractInitializer { this.logService.info(`Enabled extension`, identifier.id); } } + + for (const extensionToSync of installedExtensionsToSync) { + if (extensionToSync.state) { + const extensionState = JSON.parse(this.storageService.get(extensionToSync.identifier.id, StorageScope.GLOBAL) || '{}'); + forEach(extensionToSync.state, ({ key, value }) => extensionState[key] = value); + this.storageService.store(extensionToSync.identifier.id, JSON.stringify(extensionState), StorageScope.GLOBAL); + } + } } } diff --git a/src/vs/platform/userDataSync/common/userDataSync.ts b/src/vs/platform/userDataSync/common/userDataSync.ts index e654f0dc867..9d9b1fcede7 100644 --- a/src/vs/platform/userDataSync/common/userDataSync.ts +++ b/src/vs/platform/userDataSync/common/userDataSync.ts @@ -289,6 +289,7 @@ export interface ISyncExtension { version?: string; disabled?: boolean; installed?: boolean; + state?: IStringDictionary; } export interface IStorageValue { diff --git a/src/vs/workbench/api/browser/mainThreadStorage.ts b/src/vs/workbench/api/browser/mainThreadStorage.ts index 7bc3904963b..57abf0e86a5 100644 --- a/src/vs/workbench/api/browser/mainThreadStorage.ts +++ b/src/vs/workbench/api/browser/mainThreadStorage.ts @@ -7,20 +7,24 @@ import { IStorageService, StorageScope } from 'vs/platform/storage/common/storag import { MainThreadStorageShape, MainContext, IExtHostContext, ExtHostStorageShape, ExtHostContext } from '../common/extHost.protocol'; import { extHostNamedCustomer } from 'vs/workbench/api/common/extHostCustomers'; import { IDisposable } from 'vs/base/common/lifecycle'; +import { IExtensionIdWithVersion, IStorageKeysSyncRegistryService } from 'vs/platform/userDataSync/common/storageKeys'; @extHostNamedCustomer(MainContext.MainThreadStorage) export class MainThreadStorage implements MainThreadStorageShape { private readonly _storageService: IStorageService; + private readonly _storageKeysSyncRegistryService: IStorageKeysSyncRegistryService; private readonly _proxy: ExtHostStorageShape; private readonly _storageListener: IDisposable; private readonly _sharedStorageKeysToWatch: Map = new Map(); constructor( extHostContext: IExtHostContext, - @IStorageService storageService: IStorageService + @IStorageService storageService: IStorageService, + @IStorageKeysSyncRegistryService storageKeysSyncRegistryService: IStorageKeysSyncRegistryService, ) { this._storageService = storageService; + this._storageKeysSyncRegistryService = storageKeysSyncRegistryService; this._proxy = extHostContext.getProxy(ExtHostContext.ExtHostStorage); this._storageListener = this._storageService.onDidChangeStorage(e => { @@ -68,4 +72,8 @@ export class MainThreadStorage implements MainThreadStorageShape { } return Promise.resolve(undefined); } + + $registerExtensionStorageKeysToSync(extension: IExtensionIdWithVersion, keys: string[]): void { + this._storageKeysSyncRegistryService.registerExtensionStorageKeys(extension, keys); + } } diff --git a/src/vs/workbench/api/common/extHost.protocol.ts b/src/vs/workbench/api/common/extHost.protocol.ts index bbb93b68860..77ef6577821 100644 --- a/src/vs/workbench/api/common/extHost.protocol.ts +++ b/src/vs/workbench/api/common/extHost.protocol.ts @@ -57,6 +57,7 @@ import { Dto } from 'vs/base/common/types'; import { ISerializableEnvironmentVariableCollection } from 'vs/workbench/contrib/terminal/common/environmentVariable'; import { DebugConfigurationProviderTriggerKind } from 'vs/workbench/api/common/extHostTypes'; import { IAccessibilityInformation } from 'vs/platform/accessibility/common/accessibility'; +import { IExtensionIdWithVersion } from 'vs/platform/userDataSync/common/storageKeys'; export interface IEnvironment { isExtensionDevelopmentDebug: boolean; @@ -571,6 +572,7 @@ export interface MainThreadStatusBarShape extends IDisposable { export interface MainThreadStorageShape extends IDisposable { $getValue(shared: boolean, key: string): Promise; $setValue(shared: boolean, key: string, value: object): Promise; + $registerExtensionStorageKeysToSync(extension: IExtensionIdWithVersion, keys: string[]): void; } export interface MainThreadTelemetryShape extends IDisposable { diff --git a/src/vs/workbench/api/common/extHostExtensionService.ts b/src/vs/workbench/api/common/extHostExtensionService.ts index 311b529e5ad..180ece0ff96 100644 --- a/src/vs/workbench/api/common/extHostExtensionService.ts +++ b/src/vs/workbench/api/common/extHostExtensionService.ts @@ -371,8 +371,8 @@ export abstract class AbstractExtHostExtensionService extends Disposable impleme private _loadExtensionContext(extensionDescription: IExtensionDescription): Promise { - const globalState = new ExtensionMemento(extensionDescription.identifier.value, true, this._storage); - const workspaceState = new ExtensionMemento(extensionDescription.identifier.value, false, this._storage); + const globalState = new ExtensionMemento(extensionDescription, true, this._storage); + const workspaceState = new ExtensionMemento(extensionDescription, false, this._storage); const extensionMode = extensionDescription.isUnderDevelopment ? (this._initData.environment.extensionTestsLocationURI ? ExtensionMode.Test : ExtensionMode.Development) : ExtensionMode.Production; diff --git a/src/vs/workbench/api/common/extHostMemento.ts b/src/vs/workbench/api/common/extHostMemento.ts index c74a5cffe5f..b37d4e0ea60 100644 --- a/src/vs/workbench/api/common/extHostMemento.ts +++ b/src/vs/workbench/api/common/extHostMemento.ts @@ -6,10 +6,12 @@ import type * as vscode from 'vscode'; import { IDisposable } from 'vs/base/common/lifecycle'; import { ExtHostStorage } from 'vs/workbench/api/common/extHostStorage'; +import { IExtensionDescription } from 'vs/platform/extensions/common/extensions'; export class ExtensionMemento implements vscode.Memento { private readonly _id: string; + private readonly _version: string; private readonly _shared: boolean; private readonly _storage: ExtHostStorage; @@ -17,8 +19,16 @@ export class ExtensionMemento implements vscode.Memento { private _value?: { [n: string]: any; }; private readonly _storageListener: IDisposable; - constructor(id: string, global: boolean, storage: ExtHostStorage) { - this._id = id; + private _syncKeys: string[] = []; + get syncKeys(): ReadonlyArray { return Object.freeze(this._syncKeys); } + set syncKeys(syncKeys: ReadonlyArray) { + this._syncKeys = [...syncKeys]; + this._storage.registerExtensionStorageKeysToSync({ id: this._id, version: this._version }, this._syncKeys); + } + + constructor(extensionDescription: IExtensionDescription, global: boolean, storage: ExtHostStorage) { + this._id = extensionDescription.identifier.value; + this._version = extensionDescription.version; this._shared = global; this._storage = storage; diff --git a/src/vs/workbench/api/common/extHostStorage.ts b/src/vs/workbench/api/common/extHostStorage.ts index 8c87a3ea33c..0c38bb51095 100644 --- a/src/vs/workbench/api/common/extHostStorage.ts +++ b/src/vs/workbench/api/common/extHostStorage.ts @@ -7,6 +7,7 @@ import { MainContext, MainThreadStorageShape, ExtHostStorageShape } from './extH import { Emitter } from 'vs/base/common/event'; import { IExtHostRpcService } from 'vs/workbench/api/common/extHostRpcService'; import { createDecorator } from 'vs/platform/instantiation/common/instantiation'; +import { IExtensionIdWithVersion } from 'vs/platform/userDataSync/common/storageKeys'; export interface IStorageChangeEvent { shared: boolean; @@ -27,6 +28,10 @@ export class ExtHostStorage implements ExtHostStorageShape { this._proxy = mainContext.getProxy(MainContext.MainThreadStorage); } + registerExtensionStorageKeysToSync(extension: IExtensionIdWithVersion, keys: string[]): void { + this._proxy.$registerExtensionStorageKeysToSync(extension, keys); + } + getValue(shared: boolean, key: string, defaultValue?: T): Promise { return this._proxy.$getValue(shared, key).then(value => value || defaultValue); }