From 4273d9e3d7dbb1d7f8dbd2c1fff0876f7300c6c3 Mon Sep 17 00:00:00 2001 From: Greyson Parrelli Date: Thu, 7 Nov 2024 12:37:06 -0500 Subject: [PATCH] Convert StorageSyncHelper to kotlin. --- .../securesms/jobs/StorageSyncJob.kt | 6 +- .../securesms/storage/StorageSyncHelper.java | 357 ------------------ .../securesms/storage/StorageSyncHelper.kt | 300 +++++++++++++++ .../storage/StorageSyncValidations.java | 16 +- .../storage/StorageSyncHelperTest.java | 30 +- 5 files changed, 326 insertions(+), 383 deletions(-) delete mode 100644 app/src/main/java/org/thoughtcrime/securesms/storage/StorageSyncHelper.java create mode 100644 app/src/main/java/org/thoughtcrime/securesms/storage/StorageSyncHelper.kt diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/StorageSyncJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/StorageSyncJob.kt index b9d818da32..d5ea6f15dd 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/StorageSyncJob.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/StorageSyncJob.kt @@ -221,14 +221,14 @@ class StorageSyncJob private constructor(parameters: Parameters) : BaseJob(param var localStorageIdsBeforeMerge = getAllLocalStorageIds(self) var idDifference = StorageSyncHelper.findIdDifference(remoteManifest.storageIds, localStorageIdsBeforeMerge) - if (idDifference.hasTypeMismatches() && SignalStore.account.isPrimaryDevice) { + if (idDifference.hasTypeMismatches && SignalStore.account.isPrimaryDevice) { Log.w(TAG, "[Remote Sync] Found type mismatches in the ID sets! Scheduling a force push after this sync completes.") needsForcePush = true } Log.i(TAG, "[Remote Sync] Pre-Merge ID Difference :: $idDifference") - if (idDifference.localOnlyIds.size > 0) { + if (idDifference.localOnlyIds.isNotEmpty()) { val updated = SignalDatabase.recipients.removeStorageIdsFromLocalOnlyUnregisteredRecipients(idDifference.localOnlyIds) if (updated > 0) { @@ -375,7 +375,7 @@ class StorageSyncJob private constructor(parameters: Parameters) : BaseJob(param CallLinkRecordProcessor().process(records.callLinkRecords, StorageSyncHelper.KEY_GENERATOR) } - private fun getAllLocalStorageIds(self: Recipient): List { + private fun getAllLocalStorageIds(self: Recipient): List { return SignalDatabase.recipients.getContactStorageSyncIds() + listOf(StorageId.forAccount(self.storageId)) + SignalDatabase.unknownStorageIds.allUnknownIds diff --git a/app/src/main/java/org/thoughtcrime/securesms/storage/StorageSyncHelper.java b/app/src/main/java/org/thoughtcrime/securesms/storage/StorageSyncHelper.java deleted file mode 100644 index d9c1daf74a..0000000000 --- a/app/src/main/java/org/thoughtcrime/securesms/storage/StorageSyncHelper.java +++ /dev/null @@ -1,357 +0,0 @@ -package org.thoughtcrime.securesms.storage; - -import android.content.Context; - -import androidx.annotation.NonNull; -import androidx.annotation.Nullable; -import androidx.annotation.VisibleForTesting; - -import com.annimon.stream.Collectors; -import com.annimon.stream.Stream; - -import org.signal.core.util.Base64; -import org.signal.core.util.SetUtil; -import org.signal.core.util.logging.Log; -import org.thoughtcrime.securesms.components.settings.app.subscription.InAppPaymentsRepository; -import org.thoughtcrime.securesms.database.RecipientTable; -import org.thoughtcrime.securesms.database.SignalDatabase; -import org.thoughtcrime.securesms.database.model.InAppPaymentSubscriberRecord; -import org.thoughtcrime.securesms.database.model.RecipientRecord; -import org.thoughtcrime.securesms.dependencies.AppDependencies; -import org.thoughtcrime.securesms.jobs.RetrieveProfileAvatarJob; -import org.thoughtcrime.securesms.jobs.StorageSyncJob; -import org.thoughtcrime.securesms.keyvalue.AccountValues; -import org.thoughtcrime.securesms.keyvalue.PhoneNumberPrivacyValues.PhoneNumberDiscoverabilityMode; -import org.thoughtcrime.securesms.keyvalue.SignalStore; -import org.thoughtcrime.securesms.payments.Entropy; -import org.thoughtcrime.securesms.recipients.Recipient; -import org.thoughtcrime.securesms.util.TextSecurePreferences; -import org.thoughtcrime.securesms.util.Util; -import org.whispersystems.signalservice.api.push.UsernameLinkComponents; -import org.whispersystems.signalservice.api.storage.SignalAccountRecord; -import org.whispersystems.signalservice.api.storage.SignalContactRecord; -import org.whispersystems.signalservice.api.storage.SignalStorageManifest; -import org.whispersystems.signalservice.api.storage.SignalStorageRecord; -import org.whispersystems.signalservice.api.storage.StorageId; -import org.whispersystems.signalservice.api.util.OptionalUtil; -import org.whispersystems.signalservice.api.util.UuidUtil; -import org.whispersystems.signalservice.internal.storage.protos.AccountRecord; -import org.whispersystems.signalservice.internal.storage.protos.OptionalBool; - -import java.util.Arrays; -import java.util.Collection; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.TimeUnit; - -import okio.ByteString; - -public final class StorageSyncHelper { - - private static final String TAG = Log.tag(StorageSyncHelper.class); - - public static final StorageKeyGenerator KEY_GENERATOR = () -> Util.getSecretBytes(16); - - private static StorageKeyGenerator keyGenerator = KEY_GENERATOR; - - private static final long REFRESH_INTERVAL = TimeUnit.HOURS.toMillis(2); - - /** - * Given a list of all the local and remote keys you know about, this will return a result telling - * you which keys are exclusively remote and which are exclusively local. - * - * @param remoteIds All remote keys available. - * @param localIds All local keys available. - * @return An object describing which keys are exclusive to the remote data set and which keys are - * exclusive to the local data set. - */ - public static @NonNull IdDifferenceResult findIdDifference(@NonNull Collection remoteIds, - @NonNull Collection localIds) - { - Map remoteByRawId = Stream.of(remoteIds).collect(Collectors.toMap(id -> Base64.encodeWithPadding(id.getRaw()), id -> id)); - Map localByRawId = Stream.of(localIds).collect(Collectors.toMap(id -> Base64.encodeWithPadding(id.getRaw()), id -> id)); - - boolean hasTypeMismatch = remoteByRawId.size() != remoteIds.size() || localByRawId.size() != localIds.size(); - - Set remoteOnlyRawIds = SetUtil.difference(remoteByRawId.keySet(), localByRawId.keySet()); - Set localOnlyRawIds = SetUtil.difference(localByRawId.keySet(), remoteByRawId.keySet()); - Set sharedRawIds = SetUtil.intersection(localByRawId.keySet(), remoteByRawId.keySet()); - - for (String rawId : sharedRawIds) { - StorageId remote = Objects.requireNonNull(remoteByRawId.get(rawId)); - StorageId local = Objects.requireNonNull(localByRawId.get(rawId)); - - if (remote.getType() != local.getType()) { - remoteOnlyRawIds.remove(rawId); - localOnlyRawIds.remove(rawId); - hasTypeMismatch = true; - Log.w(TAG, "Remote type " + remote.getType() + " did not match local type " + local.getType() + "!"); - } - } - - List remoteOnlyKeys = Stream.of(remoteOnlyRawIds).map(remoteByRawId::get).toList(); - List localOnlyKeys = Stream.of(localOnlyRawIds).map(localByRawId::get).toList(); - - return new IdDifferenceResult(remoteOnlyKeys, localOnlyKeys, hasTypeMismatch); - } - - public static @NonNull byte[] generateKey() { - return keyGenerator.generate(); - } - - @VisibleForTesting - static void setTestKeyGenerator(@Nullable StorageKeyGenerator testKeyGenerator) { - keyGenerator = testKeyGenerator != null ? testKeyGenerator : KEY_GENERATOR; - } - - public static boolean profileKeyChanged(StorageRecordUpdate update) { - return !OptionalUtil.byteArrayEquals(update.getOld().getProfileKey(), update.getNew().getProfileKey()); - } - - public static SignalStorageRecord buildAccountRecord(@NonNull Context context, @NonNull Recipient self) { - RecipientTable recipientTable = SignalDatabase.recipients(); - RecipientRecord record = recipientTable.getRecordForSync(self.getId()); - List pinned = Stream.of(SignalDatabase.threads().getPinnedRecipientIds()) - .map(recipientTable::getRecordForSync) - .toList(); - - final OptionalBool storyViewReceiptsState = SignalStore.story().getViewedReceiptsEnabled() ? OptionalBool.ENABLED - : OptionalBool.DISABLED; - - if (self.getStorageId() == null || (record != null && record.getStorageId() == null)) { - Log.w(TAG, "[buildAccountRecord] No storageId for self or record! Generating. (Self: " + (self.getStorageId() != null) + ", Record: " + (record != null && record.getStorageId() != null) + ")"); - SignalDatabase.recipients().updateStorageId(self.getId(), generateKey()); - self = Recipient.self().fresh(); - record = recipientTable.getRecordForSync(self.getId()); - } - - if (record == null) { - Log.w(TAG, "[buildAccountRecord] Could not find a RecipientRecord for ourselves! ID: " + self.getId()); - } else if (!Arrays.equals(record.getStorageId(), self.getStorageId())) { - Log.w(TAG, "[buildAccountRecord] StorageId on RecipientRecord did not match self! ID: " + self.getId()); - } - - byte[] storageId = record != null && record.getStorageId() != null ? record.getStorageId() : self.getStorageId(); - - SignalAccountRecord.Builder account = new SignalAccountRecord.Builder(storageId, record != null ? record.getSyncExtras().getStorageProto() : null) - .setProfileKey(self.getProfileKey()) - .setGivenName(self.getProfileName().getGivenName()) - .setFamilyName(self.getProfileName().getFamilyName()) - .setAvatarUrlPath(self.getProfileAvatar()) - .setNoteToSelfArchived(record != null && record.getSyncExtras().isArchived()) - .setNoteToSelfForcedUnread(record != null && record.getSyncExtras().isForcedUnread()) - .setTypingIndicatorsEnabled(TextSecurePreferences.isTypingIndicatorsEnabled(context)) - .setReadReceiptsEnabled(TextSecurePreferences.isReadReceiptsEnabled(context)) - .setSealedSenderIndicatorsEnabled(TextSecurePreferences.isShowUnidentifiedDeliveryIndicatorsEnabled(context)) - .setLinkPreviewsEnabled(SignalStore.settings().isLinkPreviewsEnabled()) - .setUnlistedPhoneNumber(SignalStore.phoneNumberPrivacy().getPhoneNumberDiscoverabilityMode() == PhoneNumberDiscoverabilityMode.NOT_DISCOVERABLE) - .setPhoneNumberSharingMode(StorageSyncModels.localToRemotePhoneNumberSharingMode(SignalStore.phoneNumberPrivacy().getPhoneNumberSharingMode())) - .setPinnedConversations(StorageSyncModels.localToRemotePinnedConversations(pinned)) - .setPreferContactAvatars(SignalStore.settings().isPreferSystemContactPhotos()) - .setPayments(SignalStore.payments().mobileCoinPaymentsEnabled(), Optional.ofNullable(SignalStore.payments().getPaymentsEntropy()).map(Entropy::getBytes).orElse(null)) - .setPrimarySendsSms(false) - .setUniversalExpireTimer(SignalStore.settings().getUniversalExpireTimer()) - .setDefaultReactions(SignalStore.emoji().getReactions()) - .setSubscriber(StorageSyncModels.localToRemoteSubscriber(InAppPaymentsRepository.getSubscriber(InAppPaymentSubscriberRecord.Type.DONATION))) - .setBackupsSubscriber(StorageSyncModels.localToRemoteSubscriber(InAppPaymentsRepository.getSubscriber(InAppPaymentSubscriberRecord.Type.BACKUP))) - .setDisplayBadgesOnProfile(SignalStore.inAppPayments().getDisplayBadgesOnProfile()) - .setSubscriptionManuallyCancelled(InAppPaymentsRepository.isUserManuallyCancelled(InAppPaymentSubscriberRecord.Type.DONATION)) - .setKeepMutedChatsArchived(SignalStore.settings().shouldKeepMutedChatsArchived()) - .setHasSetMyStoriesPrivacy(SignalStore.story().getUserHasBeenNotifiedAboutStories()) - .setHasViewedOnboardingStory(SignalStore.story().getUserHasViewedOnboardingStory()) - .setStoriesDisabled(SignalStore.story().isFeatureDisabled()) - .setStoryViewReceiptsState(storyViewReceiptsState) - .setHasSeenGroupStoryEducationSheet(SignalStore.story().getUserHasSeenGroupStoryEducationSheet()) - .setUsername(SignalStore.account().getUsername()) - .setHasCompletedUsernameOnboarding(SignalStore.uiHints().hasCompletedUsernameOnboarding()); - - UsernameLinkComponents linkComponents = SignalStore.account().getUsernameLink(); - if (linkComponents != null) { - account.setUsernameLink(new AccountRecord.UsernameLink.Builder() - .entropy(ByteString.of(linkComponents.getEntropy())) - .serverId(UuidUtil.toByteString(linkComponents.getServerId())) - .color(StorageSyncModels.localToRemoteUsernameColor(SignalStore.misc().getUsernameQrCodeColorScheme())) - .build()); - } else { - account.setUsernameLink(null); - } - - return SignalStorageRecord.forAccount(account.build()); - } - - public static void applyAccountStorageSyncUpdates(@NonNull Context context, @NonNull Recipient self, @NonNull SignalAccountRecord updatedRecord, boolean fetchProfile) { - SignalAccountRecord localRecord = buildAccountRecord(context, self).getAccount().get(); - applyAccountStorageSyncUpdates(context, self, new StorageRecordUpdate<>(localRecord, updatedRecord), fetchProfile); - } - - public static void applyAccountStorageSyncUpdates(@NonNull Context context, @NonNull Recipient self, @NonNull StorageRecordUpdate update, boolean fetchProfile) { - SignalDatabase.recipients().applyStorageSyncAccountUpdate(update); - - TextSecurePreferences.setReadReceiptsEnabled(context, update.getNew().isReadReceiptsEnabled()); - TextSecurePreferences.setTypingIndicatorsEnabled(context, update.getNew().isTypingIndicatorsEnabled()); - TextSecurePreferences.setShowUnidentifiedDeliveryIndicatorsEnabled(context, update.getNew().isSealedSenderIndicatorsEnabled()); - SignalStore.settings().setLinkPreviewsEnabled(update.getNew().isLinkPreviewsEnabled()); - SignalStore.phoneNumberPrivacy().setPhoneNumberDiscoverabilityMode(update.getNew().isPhoneNumberUnlisted() ? PhoneNumberDiscoverabilityMode.NOT_DISCOVERABLE : PhoneNumberDiscoverabilityMode.DISCOVERABLE); - SignalStore.phoneNumberPrivacy().setPhoneNumberSharingMode(StorageSyncModels.remoteToLocalPhoneNumberSharingMode(update.getNew().getPhoneNumberSharingMode())); - SignalStore.settings().setPreferSystemContactPhotos(update.getNew().isPreferContactAvatars()); - SignalStore.payments().setEnabledAndEntropy(update.getNew().getPayments().isEnabled(), Entropy.fromBytes(update.getNew().getPayments().getEntropy().orElse(null))); - SignalStore.settings().setUniversalExpireTimer(update.getNew().getUniversalExpireTimer()); - SignalStore.emoji().setReactions(update.getNew().getDefaultReactions()); - SignalStore.inAppPayments().setDisplayBadgesOnProfile(update.getNew().isDisplayBadgesOnProfile()); - SignalStore.settings().setKeepMutedChatsArchived(update.getNew().isKeepMutedChatsArchived()); - SignalStore.story().setUserHasBeenNotifiedAboutStories(update.getNew().hasSetMyStoriesPrivacy()); - SignalStore.story().setUserHasViewedOnboardingStory(update.getNew().hasViewedOnboardingStory()); - SignalStore.story().setFeatureDisabled(update.getNew().isStoriesDisabled()); - SignalStore.story().setUserHasSeenGroupStoryEducationSheet(update.getNew().hasSeenGroupStoryEducationSheet()); - SignalStore.uiHints().setHasCompletedUsernameOnboarding(update.getNew().hasCompletedUsernameOnboarding()); - - if (update.getNew().getStoryViewReceiptsState() == OptionalBool.UNSET) { - SignalStore.story().setViewedReceiptsEnabled(update.getNew().isReadReceiptsEnabled()); - } else { - SignalStore.story().setViewedReceiptsEnabled(update.getNew().getStoryViewReceiptsState() == OptionalBool.ENABLED); - } - - if (update.getNew().getStoryViewReceiptsState() == OptionalBool.UNSET) { - SignalStore.story().setViewedReceiptsEnabled(update.getNew().isReadReceiptsEnabled()); - } else { - SignalStore.story().setViewedReceiptsEnabled(update.getNew().getStoryViewReceiptsState() == OptionalBool.ENABLED); - } - - InAppPaymentSubscriberRecord remoteSubscriber = StorageSyncModels.remoteToLocalSubscriber(update.getNew().getSubscriber(), InAppPaymentSubscriberRecord.Type.DONATION); - if (remoteSubscriber != null) { - InAppPaymentsRepository.setSubscriber(remoteSubscriber); - } - - if (update.getNew().isSubscriptionManuallyCancelled() && !update.getOld().isSubscriptionManuallyCancelled()) { - SignalStore.inAppPayments().updateLocalStateForManualCancellation(InAppPaymentSubscriberRecord.Type.DONATION); - } - - if (fetchProfile && update.getNew().getAvatarUrlPath().isPresent()) { - AppDependencies.getJobManager().add(new RetrieveProfileAvatarJob(self, update.getNew().getAvatarUrlPath().get())); - } - - if (!update.getNew().getUsername().equals(update.getOld().getUsername())) { - SignalStore.account().setUsername(update.getNew().getUsername()); - SignalStore.account().setUsernameSyncState(AccountValues.UsernameSyncState.IN_SYNC); - SignalStore.account().setUsernameSyncErrorCount(0); - } - - if (update.getNew().getUsernameLink() != null) { - SignalStore.account().setUsernameLink( - new UsernameLinkComponents( - update.getNew().getUsernameLink().entropy.toByteArray(), - UuidUtil.parseOrThrow(update.getNew().getUsernameLink().serverId.toByteArray()) - ) - ); - SignalStore.misc().setUsernameQrCodeColorScheme(StorageSyncModels.remoteToLocalUsernameColor(update.getNew().getUsernameLink().color)); - } - } - - public static void scheduleSyncForDataChange() { - if (!SignalStore.registration().isRegistrationComplete()) { - Log.d(TAG, "Registration still ongoing. Ignore sync request."); - return; - } - AppDependencies.getJobManager().add(new StorageSyncJob()); - } - - public static void scheduleRoutineSync() { - long timeSinceLastSync = System.currentTimeMillis() - SignalStore.storageService().getLastSyncTime(); - - if (timeSinceLastSync > REFRESH_INTERVAL) { - Log.d(TAG, "Scheduling a sync. Last sync was " + timeSinceLastSync + " ms ago."); - scheduleSyncForDataChange(); - } else { - Log.d(TAG, "No need for sync. Last sync was " + timeSinceLastSync + " ms ago."); - } - } - - public static final class IdDifferenceResult { - private final List remoteOnlyIds; - private final List localOnlyIds; - private final boolean hasTypeMismatches; - - private IdDifferenceResult(@NonNull List remoteOnlyIds, - @NonNull List localOnlyIds, - boolean hasTypeMismatches) - { - this.remoteOnlyIds = remoteOnlyIds; - this.localOnlyIds = localOnlyIds; - this.hasTypeMismatches = hasTypeMismatches; - } - - public @NonNull List getRemoteOnlyIds() { - return remoteOnlyIds; - } - - public @NonNull List getLocalOnlyIds() { - return localOnlyIds; - } - - /** - * @return True if there exist some keys that have matching raw ID's but different types, - * otherwise false. - */ - public boolean hasTypeMismatches() { - return hasTypeMismatches; - } - - public boolean isEmpty() { - return remoteOnlyIds.isEmpty() && localOnlyIds.isEmpty(); - } - - @Override - public @NonNull String toString() { - return "remoteOnly: " + remoteOnlyIds.size() + ", localOnly: " + localOnlyIds.size() + ", hasTypeMismatches: " + hasTypeMismatches; - } - } - - public static final class WriteOperationResult { - private final SignalStorageManifest manifest; - private final List inserts; - private final List deletes; - - public WriteOperationResult(@NonNull SignalStorageManifest manifest, - @NonNull List inserts, - @NonNull List deletes) - { - this.manifest = manifest; - this.inserts = inserts; - this.deletes = deletes; - } - - public @NonNull SignalStorageManifest getManifest() { - return manifest; - } - - public @NonNull List getInserts() { - return inserts; - } - - public @NonNull List getDeletes() { - return deletes; - } - - public boolean isEmpty() { - return inserts.isEmpty() && deletes.isEmpty(); - } - - @Override - public @NonNull String toString() { - if (isEmpty()) { - return "Empty"; - } else { - return String.format(Locale.ENGLISH, - "ManifestVersion: %d, Total Keys: %d, Inserts: %d, Deletes: %d", - manifest.getVersion(), - manifest.getStorageIds().size(), - inserts.size(), - deletes.size()); - } - } - } -} diff --git a/app/src/main/java/org/thoughtcrime/securesms/storage/StorageSyncHelper.kt b/app/src/main/java/org/thoughtcrime/securesms/storage/StorageSyncHelper.kt new file mode 100644 index 0000000000..03e04562c1 --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/storage/StorageSyncHelper.kt @@ -0,0 +1,300 @@ +package org.thoughtcrime.securesms.storage + +import android.content.Context +import androidx.annotation.VisibleForTesting +import okio.ByteString.Companion.toByteString +import org.signal.core.util.Base64.encodeWithPadding +import org.signal.core.util.logging.Log +import org.thoughtcrime.securesms.components.settings.app.subscription.InAppPaymentsRepository.getSubscriber +import org.thoughtcrime.securesms.components.settings.app.subscription.InAppPaymentsRepository.isUserManuallyCancelled +import org.thoughtcrime.securesms.components.settings.app.subscription.InAppPaymentsRepository.setSubscriber +import org.thoughtcrime.securesms.database.SignalDatabase +import org.thoughtcrime.securesms.database.model.InAppPaymentSubscriberRecord +import org.thoughtcrime.securesms.database.model.RecipientRecord +import org.thoughtcrime.securesms.dependencies.AppDependencies +import org.thoughtcrime.securesms.jobs.RetrieveProfileAvatarJob +import org.thoughtcrime.securesms.jobs.StorageSyncJob +import org.thoughtcrime.securesms.keyvalue.AccountValues +import org.thoughtcrime.securesms.keyvalue.PhoneNumberPrivacyValues.PhoneNumberDiscoverabilityMode +import org.thoughtcrime.securesms.keyvalue.SignalStore +import org.thoughtcrime.securesms.payments.Entropy +import org.thoughtcrime.securesms.recipients.Recipient +import org.thoughtcrime.securesms.recipients.Recipient.Companion.self +import org.thoughtcrime.securesms.util.TextSecurePreferences +import org.thoughtcrime.securesms.util.Util +import org.whispersystems.signalservice.api.push.UsernameLinkComponents +import org.whispersystems.signalservice.api.storage.SignalAccountRecord +import org.whispersystems.signalservice.api.storage.SignalContactRecord +import org.whispersystems.signalservice.api.storage.SignalStorageManifest +import org.whispersystems.signalservice.api.storage.SignalStorageRecord +import org.whispersystems.signalservice.api.storage.StorageId +import org.whispersystems.signalservice.api.util.OptionalUtil.byteArrayEquals +import org.whispersystems.signalservice.api.util.UuidUtil +import org.whispersystems.signalservice.api.util.toByteArray +import org.whispersystems.signalservice.internal.storage.protos.AccountRecord +import org.whispersystems.signalservice.internal.storage.protos.OptionalBool +import java.util.Optional +import java.util.concurrent.TimeUnit + +object StorageSyncHelper { + private val TAG = Log.tag(StorageSyncHelper::class.java) + + val KEY_GENERATOR: StorageKeyGenerator = StorageKeyGenerator { Util.getSecretBytes(16) } + + private var keyGenerator = KEY_GENERATOR + + private val REFRESH_INTERVAL = TimeUnit.HOURS.toMillis(2) + + /** + * Given a list of all the local and remote keys you know about, this will return a result telling + * you which keys are exclusively remote and which are exclusively local. + * + * @param remoteIds All remote keys available. + * @param localIds All local keys available. + * @return An object describing which keys are exclusive to the remote data set and which keys are + * exclusive to the local data set. + */ + @JvmStatic + fun findIdDifference( + remoteIds: Collection, + localIds: Collection + ): IdDifferenceResult { + val remoteByRawId: Map = remoteIds.associateBy { encodeWithPadding(it.raw) } + val localByRawId: Map = localIds.associateBy { encodeWithPadding(it.raw) } + + var hasTypeMismatch = remoteByRawId.size != remoteIds.size || localByRawId.size != localIds.size + + val remoteOnlyRawIds: MutableSet = (remoteByRawId.keys - localByRawId.keys).toMutableSet() + val localOnlyRawIds: MutableSet = (localByRawId.keys - remoteByRawId.keys).toMutableSet() + val sharedRawIds: Set = localByRawId.keys.intersect(remoteByRawId.keys) + + for (rawId in sharedRawIds) { + val remote = remoteByRawId[rawId]!! + val local = localByRawId[rawId]!! + + if (remote.type != local.type) { + remoteOnlyRawIds.remove(rawId) + localOnlyRawIds.remove(rawId) + hasTypeMismatch = true + Log.w(TAG, "Remote type ${remote.type} did not match local type ${local.type}!") + } + } + + val remoteOnlyKeys = remoteOnlyRawIds.mapNotNull { remoteByRawId[it] } + val localOnlyKeys = localOnlyRawIds.mapNotNull { localByRawId[it] } + + return IdDifferenceResult(remoteOnlyKeys, localOnlyKeys, hasTypeMismatch) + } + + @JvmStatic + fun generateKey(): ByteArray { + return keyGenerator.generate() + } + + @JvmStatic + @VisibleForTesting + fun setTestKeyGenerator(testKeyGenerator: StorageKeyGenerator?) { + keyGenerator = testKeyGenerator ?: KEY_GENERATOR + } + + @JvmStatic + fun profileKeyChanged(update: StorageRecordUpdate): Boolean { + return !byteArrayEquals(update.old.profileKey, update.new.profileKey) + } + + @JvmStatic + fun buildAccountRecord(context: Context, self: Recipient): SignalStorageRecord { + var self = self + var selfRecord: RecipientRecord? = SignalDatabase.recipients.getRecordForSync(self.id) + val pinned: List = SignalDatabase.threads.getPinnedRecipientIds() + .mapNotNull { SignalDatabase.recipients.getRecordForSync(it) } + + val storyViewReceiptsState = if (SignalStore.story.viewedReceiptsEnabled) { + OptionalBool.ENABLED + } else { + OptionalBool.DISABLED + } + + if (self.storageId == null || (selfRecord != null && selfRecord.storageId == null)) { + Log.w(TAG, "[buildAccountRecord] No storageId for self or record! Generating. (Self: ${self.storageId != null}, Record: ${selfRecord?.storageId != null})") + SignalDatabase.recipients.updateStorageId(self.id, generateKey()) + self = self().fresh() + selfRecord = SignalDatabase.recipients.getRecordForSync(self.id) + } + + if (selfRecord == null) { + Log.w(TAG, "[buildAccountRecord] Could not find a RecipientRecord for ourselves! ID: ${self.id}") + } else if (!selfRecord.storageId.contentEquals(self.storageId)) { + Log.w(TAG, "[buildAccountRecord] StorageId on RecipientRecord did not match self! ID: ${self.id}") + } + + val storageId = selfRecord?.storageId ?: self.storageId + + val account = SignalAccountRecord.Builder(storageId, selfRecord?.syncExtras?.storageProto) + .setProfileKey(self.profileKey) + .setGivenName(self.profileName.givenName) + .setFamilyName(self.profileName.familyName) + .setAvatarUrlPath(self.profileAvatar) + .setNoteToSelfArchived(selfRecord != null && selfRecord.syncExtras.isArchived) + .setNoteToSelfForcedUnread(selfRecord != null && selfRecord.syncExtras.isForcedUnread) + .setTypingIndicatorsEnabled(TextSecurePreferences.isTypingIndicatorsEnabled(context)) + .setReadReceiptsEnabled(TextSecurePreferences.isReadReceiptsEnabled(context)) + .setSealedSenderIndicatorsEnabled(TextSecurePreferences.isShowUnidentifiedDeliveryIndicatorsEnabled(context)) + .setLinkPreviewsEnabled(SignalStore.settings.isLinkPreviewsEnabled) + .setUnlistedPhoneNumber(SignalStore.phoneNumberPrivacy.phoneNumberDiscoverabilityMode == PhoneNumberDiscoverabilityMode.NOT_DISCOVERABLE) + .setPhoneNumberSharingMode(StorageSyncModels.localToRemotePhoneNumberSharingMode(SignalStore.phoneNumberPrivacy.phoneNumberSharingMode)) + .setPinnedConversations(StorageSyncModels.localToRemotePinnedConversations(pinned)) + .setPreferContactAvatars(SignalStore.settings.isPreferSystemContactPhotos) + .setPayments(SignalStore.payments.mobileCoinPaymentsEnabled(), Optional.ofNullable(SignalStore.payments.paymentsEntropy).map { obj: Entropy -> obj.bytes }.orElse(null)) + .setPrimarySendsSms(false) + .setUniversalExpireTimer(SignalStore.settings.universalExpireTimer) + .setDefaultReactions(SignalStore.emoji.reactions) + .setSubscriber(StorageSyncModels.localToRemoteSubscriber(getSubscriber(InAppPaymentSubscriberRecord.Type.DONATION))) + .setBackupsSubscriber(StorageSyncModels.localToRemoteSubscriber(getSubscriber(InAppPaymentSubscriberRecord.Type.BACKUP))) + .setDisplayBadgesOnProfile(SignalStore.inAppPayments.getDisplayBadgesOnProfile()) + .setSubscriptionManuallyCancelled(isUserManuallyCancelled(InAppPaymentSubscriberRecord.Type.DONATION)) + .setKeepMutedChatsArchived(SignalStore.settings.shouldKeepMutedChatsArchived()) + .setHasSetMyStoriesPrivacy(SignalStore.story.userHasBeenNotifiedAboutStories) + .setHasViewedOnboardingStory(SignalStore.story.userHasViewedOnboardingStory) + .setStoriesDisabled(SignalStore.story.isFeatureDisabled) + .setStoryViewReceiptsState(storyViewReceiptsState) + .setHasSeenGroupStoryEducationSheet(SignalStore.story.userHasSeenGroupStoryEducationSheet) + .setUsername(SignalStore.account.username) + .setHasCompletedUsernameOnboarding(SignalStore.uiHints.hasCompletedUsernameOnboarding()) + + val linkComponents = SignalStore.account.usernameLink + if (linkComponents != null) { + account.setUsernameLink( + AccountRecord.UsernameLink.Builder() + .entropy(linkComponents.entropy.toByteString()) + .serverId(linkComponents.serverId.toByteArray().toByteString()) + .color(StorageSyncModels.localToRemoteUsernameColor(SignalStore.misc.usernameQrCodeColorScheme)) + .build() + ) + } else { + account.setUsernameLink(null) + } + + return SignalStorageRecord.forAccount(account.build()) + } + + @JvmStatic + fun applyAccountStorageSyncUpdates(context: Context, self: Recipient, updatedRecord: SignalAccountRecord, fetchProfile: Boolean) { + val localRecord = buildAccountRecord(context, self).account.get() + applyAccountStorageSyncUpdates(context, self, StorageRecordUpdate(localRecord, updatedRecord), fetchProfile) + } + + @JvmStatic + fun applyAccountStorageSyncUpdates(context: Context, self: Recipient, update: StorageRecordUpdate, fetchProfile: Boolean) { + SignalDatabase.recipients.applyStorageSyncAccountUpdate(update) + + TextSecurePreferences.setReadReceiptsEnabled(context, update.new.isReadReceiptsEnabled) + TextSecurePreferences.setTypingIndicatorsEnabled(context, update.new.isTypingIndicatorsEnabled) + TextSecurePreferences.setShowUnidentifiedDeliveryIndicatorsEnabled(context, update.new.isSealedSenderIndicatorsEnabled) + SignalStore.settings.isLinkPreviewsEnabled = update.new.isLinkPreviewsEnabled + SignalStore.phoneNumberPrivacy.phoneNumberDiscoverabilityMode = if (update.new.isPhoneNumberUnlisted) PhoneNumberDiscoverabilityMode.NOT_DISCOVERABLE else PhoneNumberDiscoverabilityMode.DISCOVERABLE + SignalStore.phoneNumberPrivacy.phoneNumberSharingMode = StorageSyncModels.remoteToLocalPhoneNumberSharingMode(update.new.phoneNumberSharingMode) + SignalStore.settings.isPreferSystemContactPhotos = update.new.isPreferContactAvatars + SignalStore.payments.setEnabledAndEntropy(update.new.payments.isEnabled, Entropy.fromBytes(update.new.payments.entropy.orElse(null))) + SignalStore.settings.universalExpireTimer = update.new.universalExpireTimer + SignalStore.emoji.reactions = update.new.defaultReactions + SignalStore.inAppPayments.setDisplayBadgesOnProfile(update.new.isDisplayBadgesOnProfile) + SignalStore.settings.setKeepMutedChatsArchived(update.new.isKeepMutedChatsArchived) + SignalStore.story.userHasBeenNotifiedAboutStories = update.new.hasSetMyStoriesPrivacy() + SignalStore.story.userHasViewedOnboardingStory = update.new.hasViewedOnboardingStory() + SignalStore.story.isFeatureDisabled = update.new.isStoriesDisabled + SignalStore.story.userHasSeenGroupStoryEducationSheet = update.new.hasSeenGroupStoryEducationSheet() + SignalStore.uiHints.setHasCompletedUsernameOnboarding(update.new.hasCompletedUsernameOnboarding()) + + if (update.new.storyViewReceiptsState == OptionalBool.UNSET) { + SignalStore.story.viewedReceiptsEnabled = update.new.isReadReceiptsEnabled + } else { + SignalStore.story.viewedReceiptsEnabled = update.new.storyViewReceiptsState == OptionalBool.ENABLED + } + + if (update.new.storyViewReceiptsState == OptionalBool.UNSET) { + SignalStore.story.viewedReceiptsEnabled = update.new.isReadReceiptsEnabled + } else { + SignalStore.story.viewedReceiptsEnabled = update.new.storyViewReceiptsState == OptionalBool.ENABLED + } + + val remoteSubscriber = StorageSyncModels.remoteToLocalSubscriber(update.new.subscriber, InAppPaymentSubscriberRecord.Type.DONATION) + if (remoteSubscriber != null) { + setSubscriber(remoteSubscriber) + } + + if (update.new.isSubscriptionManuallyCancelled && !update.old.isSubscriptionManuallyCancelled) { + SignalStore.inAppPayments.updateLocalStateForManualCancellation(InAppPaymentSubscriberRecord.Type.DONATION) + } + + if (fetchProfile && update.new.avatarUrlPath.isPresent) { + AppDependencies.jobManager.add(RetrieveProfileAvatarJob(self, update.new.avatarUrlPath.get())) + } + + if (update.new.username != update.old.username) { + SignalStore.account.username = update.new.username + SignalStore.account.usernameSyncState = AccountValues.UsernameSyncState.IN_SYNC + SignalStore.account.usernameSyncErrorCount = 0 + } + + if (update.new.usernameLink != null) { + SignalStore.account.usernameLink = UsernameLinkComponents( + update.new.usernameLink!!.entropy.toByteArray(), + UuidUtil.parseOrThrow(update.new.usernameLink!!.serverId.toByteArray()) + ) + + SignalStore.misc.usernameQrCodeColorScheme = StorageSyncModels.remoteToLocalUsernameColor(update.new.usernameLink!!.color) + } + } + + @JvmStatic + fun scheduleSyncForDataChange() { + if (!SignalStore.registration.isRegistrationComplete) { + Log.d(TAG, "Registration still ongoing. Ignore sync request.") + return + } + AppDependencies.jobManager.add(StorageSyncJob()) + } + + @JvmStatic + fun scheduleRoutineSync() { + val timeSinceLastSync = System.currentTimeMillis() - SignalStore.storageService.lastSyncTime + + if (timeSinceLastSync > REFRESH_INTERVAL) { + Log.d(TAG, "Scheduling a sync. Last sync was $timeSinceLastSync ms ago.") + scheduleSyncForDataChange() + } else { + Log.d(TAG, "No need for sync. Last sync was $timeSinceLastSync ms ago.") + } + } + + class IdDifferenceResult( + @JvmField val remoteOnlyIds: List, + @JvmField val localOnlyIds: List, + val hasTypeMismatches: Boolean + ) { + val isEmpty: Boolean + get() = remoteOnlyIds.isEmpty() && localOnlyIds.isEmpty() + + override fun toString(): String { + return "remoteOnly: ${remoteOnlyIds.size}, localOnly: ${localOnlyIds.size}, hasTypeMismatches: $hasTypeMismatches" + } + } + + class WriteOperationResult( + @JvmField val manifest: SignalStorageManifest, + @JvmField val inserts: List, + @JvmField val deletes: List + ) { + val isEmpty: Boolean + get() = inserts.isEmpty() && deletes.isEmpty() + + override fun toString(): String { + return if (isEmpty) { + "Empty" + } else { + "ManifestVersion: ${manifest.version}, Total Keys: ${manifest.storageIds.size}, Inserts: ${inserts.size}, Deletes: ${deletes.size}" + } + } + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/storage/StorageSyncValidations.java b/app/src/main/java/org/thoughtcrime/securesms/storage/StorageSyncValidations.java index 981e56f515..b0b9a2538e 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/storage/StorageSyncValidations.java +++ b/app/src/main/java/org/thoughtcrime/securesms/storage/StorageSyncValidations.java @@ -31,12 +31,12 @@ public final class StorageSyncValidations { boolean forcePushPending, @NonNull Recipient self) { - validateManifestAndInserts(result.getManifest(), result.getInserts(), self); + validateManifestAndInserts(result.manifest, result.inserts, self); - if (result.getDeletes().size() > 0) { - Set allSetEncoded = Stream.of(result.getManifest().getStorageIds()).map(StorageId::getRaw).map(Base64::encodeWithPadding).collect(Collectors.toSet()); + if (result.deletes.size() > 0) { + Set allSetEncoded = Stream.of(result.manifest.getStorageIds()).map(StorageId::getRaw).map(Base64::encodeWithPadding).collect(Collectors.toSet()); - for (byte[] delete : result.getDeletes()) { + for (byte[] delete : result.deletes) { String encoded = Base64.encodeWithPadding(delete); if (allSetEncoded.contains(encoded)) { throw new DeletePresentInFullIdSetError(); @@ -49,7 +49,7 @@ public final class StorageSyncValidations { return; } - if (result.getManifest().getVersion() != previousManifest.getVersion() + 1) { + if (result.manifest.getVersion() != previousManifest.getVersion() + 1) { throw new IncorrectManifestVersionError(); } @@ -59,13 +59,13 @@ public final class StorageSyncValidations { } Set previousIds = Stream.of(previousManifest.getStorageIds()).map(id -> ByteBuffer.wrap(id.getRaw())).collect(Collectors.toSet()); - Set newIds = Stream.of(result.getManifest().getStorageIds()).map(id -> ByteBuffer.wrap(id.getRaw())).collect(Collectors.toSet()); + Set newIds = Stream.of(result.manifest.getStorageIds()).map(id -> ByteBuffer.wrap(id.getRaw())).collect(Collectors.toSet()); Set manifestInserts = SetUtil.difference(newIds, previousIds); Set manifestDeletes = SetUtil.difference(previousIds, newIds); - Set declaredInserts = Stream.of(result.getInserts()).map(r -> ByteBuffer.wrap(r.getId().getRaw())).collect(Collectors.toSet()); - Set declaredDeletes = Stream.of(result.getDeletes()).map(ByteBuffer::wrap).collect(Collectors.toSet()); + Set declaredInserts = Stream.of(result.inserts).map(r -> ByteBuffer.wrap(r.getId().getRaw())).collect(Collectors.toSet()); + Set declaredDeletes = Stream.of(result.deletes).map(ByteBuffer::wrap).collect(Collectors.toSet()); if (declaredInserts.size() > manifestInserts.size()) { Log.w(TAG, "DeclaredInserts: " + declaredInserts.size() + ", ManifestInserts: " + manifestInserts.size()); diff --git a/app/src/test/java/org/thoughtcrime/securesms/storage/StorageSyncHelperTest.java b/app/src/test/java/org/thoughtcrime/securesms/storage/StorageSyncHelperTest.java index c6c28199bf..4ef06fa1f3 100644 --- a/app/src/test/java/org/thoughtcrime/securesms/storage/StorageSyncHelperTest.java +++ b/app/src/test/java/org/thoughtcrime/securesms/storage/StorageSyncHelperTest.java @@ -72,25 +72,25 @@ public final class StorageSyncHelperTest { @Test public void findIdDifference_allOverlap() { IdDifferenceResult result = StorageSyncHelper.findIdDifference(keyListOf(1, 2, 3), keyListOf(1, 2, 3)); - assertTrue(result.getLocalOnlyIds().isEmpty()); - assertTrue(result.getRemoteOnlyIds().isEmpty()); - assertFalse(result.hasTypeMismatches()); + assertTrue(result.localOnlyIds.isEmpty()); + assertTrue(result.remoteOnlyIds.isEmpty()); + assertFalse(result.getHasTypeMismatches()); } @Test public void findIdDifference_noOverlap() { IdDifferenceResult result = StorageSyncHelper.findIdDifference(keyListOf(1, 2, 3), keyListOf(4, 5, 6)); - assertContentsEqual(keyListOf(1, 2, 3), result.getRemoteOnlyIds()); - assertContentsEqual(keyListOf(4, 5, 6), result.getLocalOnlyIds()); - assertFalse(result.hasTypeMismatches()); + assertContentsEqual(keyListOf(1, 2, 3), result.remoteOnlyIds); + assertContentsEqual(keyListOf(4, 5, 6), result.localOnlyIds); + assertFalse(result.getHasTypeMismatches()); } @Test public void findIdDifference_someOverlap() { IdDifferenceResult result = StorageSyncHelper.findIdDifference(keyListOf(1, 2, 3), keyListOf(2, 3, 4)); - assertContentsEqual(keyListOf(1), result.getRemoteOnlyIds()); - assertContentsEqual(keyListOf(4), result.getLocalOnlyIds()); - assertFalse(result.hasTypeMismatches()); + assertContentsEqual(keyListOf(1), result.remoteOnlyIds); + assertContentsEqual(keyListOf(4), result.localOnlyIds); + assertFalse(result.getHasTypeMismatches()); } @Test @@ -104,9 +104,9 @@ public final class StorageSyncHelperTest { put(200, 1); }})); - assertTrue(result.getLocalOnlyIds().isEmpty()); - assertTrue(result.getRemoteOnlyIds().isEmpty()); - assertTrue(result.hasTypeMismatches()); + assertTrue(result.localOnlyIds.isEmpty()); + assertTrue(result.remoteOnlyIds.isEmpty()); + assertTrue(result.getHasTypeMismatches()); } @Test @@ -122,9 +122,9 @@ public final class StorageSyncHelperTest { put(400, 1); }})); - assertContentsEqual(Arrays.asList(StorageId.forType(byteArray(300), 1)), result.getRemoteOnlyIds()); - assertContentsEqual(Arrays.asList(StorageId.forType(byteArray(400), 1)), result.getLocalOnlyIds()); - assertTrue(result.hasTypeMismatches()); + assertContentsEqual(Arrays.asList(StorageId.forType(byteArray(300), 1)), result.remoteOnlyIds); + assertContentsEqual(Arrays.asList(StorageId.forType(byteArray(400), 1)), result.localOnlyIds); + assertTrue(result.getHasTypeMismatches()); } @Test