From 882a11c4207e0deb4190aef2e2d9f92e98d6108e Mon Sep 17 00:00:00 2001 From: Cody Henthorne Date: Fri, 6 Jun 2025 10:47:32 -0400 Subject: [PATCH] Calculate remote backup media quota usage locally. --- .../securesms/backup/v2/BackupRepository.kt | 9 ---- .../remote/RemoteBackupsSettingsViewModel.kt | 22 +++++++-- .../securesms/database/AttachmentTable.kt | 49 +++++++++++++++++++ .../securesms/database/DatabaseObserverExt.kt | 40 +++++++++++++++ .../securesms/jobs/BackupMessagesJob.kt | 12 +---- .../jobs/CopyAttachmentToArchiveJob.kt | 4 -- .../securesms/keyvalue/BackupValues.kt | 5 +- .../securesms/keyvalue/RegistrationValues.kt | 2 + .../data/QuickRegistrationRepository.kt | 3 +- .../ui/restore/RemoteRestoreViewModel.kt | 4 +- .../ui/restore/RestoreViaQrViewModel.kt | 2 +- .../org/signal/core/util/ByteExtensions.kt | 4 ++ 12 files changed, 119 insertions(+), 37 deletions(-) create mode 100644 app/src/main/java/org/thoughtcrime/securesms/database/DatabaseObserverExt.kt diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt index 95c1182aaa..06a965715f 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt @@ -1045,14 +1045,6 @@ object BackupRepository { } } - fun getRemoteBackupUsedSpace(): NetworkResult { - return initBackupAndFetchAuth() - .then { credential -> - SignalNetwork.archive.getBackupInfo(SignalStore.account.requireAci(), credential.mediaBackupAccess) - .map { it.usedSpace } - } - } - /** * If backups are enabled, sync with the network. Otherwise, return a 404. * Used in instrumentation tests. @@ -1433,7 +1425,6 @@ object BackupRepository { return initBackupAndFetchAuth() .then { credential -> SignalNetwork.archive.getBackupInfo(SignalStore.account.requireAci(), credential.mediaBackupAccess).map { - SignalStore.backup.usedBackupMediaSpace = it.usedSpace ?: 0L "${it.backupDir!!.urlEncode()}/${it.mediaDir!!.urlEncode()}" } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/backups/remote/RemoteBackupsSettingsViewModel.kt b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/backups/remote/RemoteBackupsSettingsViewModel.kt index aebe6ff8e4..082effae45 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/backups/remote/RemoteBackupsSettingsViewModel.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/backups/remote/RemoteBackupsSettingsViewModel.kt @@ -25,6 +25,7 @@ import org.signal.core.util.billing.BillingPurchaseResult import org.signal.core.util.bytes import org.signal.core.util.logging.Log import org.signal.core.util.money.FiatMoney +import org.signal.core.util.throttleLatest import org.signal.donations.InAppPaymentType import org.thoughtcrime.securesms.backup.ArchiveUploadProgress import org.thoughtcrime.securesms.backup.DeletionState @@ -39,6 +40,7 @@ import org.thoughtcrime.securesms.components.settings.app.subscription.InAppPaym import org.thoughtcrime.securesms.components.settings.app.subscription.RecurringInAppPaymentRepository import org.thoughtcrime.securesms.database.InAppPaymentTable import org.thoughtcrime.securesms.database.SignalDatabase +import org.thoughtcrime.securesms.database.attachmentUpdates import org.thoughtcrime.securesms.database.model.InAppPaymentSubscriberRecord import org.thoughtcrime.securesms.dependencies.AppDependencies import org.thoughtcrime.securesms.jobs.BackupMessagesJob @@ -63,7 +65,6 @@ class RemoteBackupsSettingsViewModel : ViewModel() { RemoteBackupsSettingsState( backupsEnabled = SignalStore.backup.areBackupsEnabled, lastBackupTimestamp = SignalStore.backup.lastBackupTime, - backupMediaSize = SignalStore.backup.totalBackupSize, backupsFrequency = SignalStore.backup.backupFrequency, canBackUpUsingCellular = SignalStore.backup.backupWithCellular, canRestoreUsingCellular = SignalStore.backup.restoreWithCellular @@ -77,6 +78,10 @@ class RemoteBackupsSettingsViewModel : ViewModel() { val restoreState: StateFlow = _restoreState init { + viewModelScope.launch(Dispatchers.IO) { + _state.update { it.copy(backupMediaSize = SignalDatabase.attachments.getEstimatedArchiveMediaSize()) } + } + viewModelScope.launch(Dispatchers.IO) { SignalStore.backup.deletionStateFlow.collectLatest { refresh() @@ -91,6 +96,16 @@ class RemoteBackupsSettingsViewModel : ViewModel() { } } + viewModelScope.launch(Dispatchers.IO) { + AppDependencies + .databaseObserver + .attachmentUpdates() + .throttleLatest(5.seconds) + .collectLatest { + _state.update { it.copy(backupMediaSize = SignalDatabase.attachments.getEstimatedArchiveMediaSize()) } + } + } + viewModelScope.launch(Dispatchers.IO) { val restoreProgress = MediaRestoreProgressBanner() @@ -222,7 +237,7 @@ class RemoteBackupsSettingsViewModel : ViewModel() { backupsEnabled = SignalStore.backup.areBackupsEnabled, backupState = RemoteBackupsSettingsState.BackupState.Loading, lastBackupTimestamp = SignalStore.backup.lastBackupTime, - backupMediaSize = SignalStore.backup.totalBackupSize, + backupMediaSize = SignalDatabase.attachments.getEstimatedArchiveMediaSize(), backupsFrequency = SignalStore.backup.backupFrequency, canBackUpUsingCellular = SignalStore.backup.backupWithCellular, canRestoreUsingCellular = SignalStore.backup.restoreWithCellular @@ -392,7 +407,4 @@ class RemoteBackupsSettingsViewModel : ViewModel() { } } } - - private fun refreshLocalState() { - } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt b/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt index 16475ad981..af4c6c808c 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt @@ -688,6 +688,8 @@ class AttachmentTable( .where("$DATA_FILE = ?", dataFile) .run() } + + AppDependencies.databaseObserver.notifyAttachmentUpdatedObservers() } /** @@ -709,6 +711,8 @@ class AttachmentTable( .where("$ARCHIVE_TRANSFER_STATE != ? AND $DATA_FILE = ?", ArchiveTransferState.PERMANENT_FAILURE.value, dataFile) .run() } + + AppDependencies.databaseObserver.notifyAttachmentUpdatedObservers() } /** @@ -2586,6 +2590,51 @@ class AttachmentTable( .readToList { AttachmentId(it.requireLong(ID)) } } + fun getEstimatedArchiveMediaSize(): Long { + val estimatedThumbnailCount = readableDatabase + .select("COUNT(DISTINCT $REMOTE_DIGEST)") + .from(TABLE_NAME) + .where( + """ + $DATA_FILE NOT NULL AND + $REMOTE_DIGEST NOT NULL AND + $TRANSFER_STATE = $TRANSFER_PROGRESS_DONE AND + $ARCHIVE_TRANSFER_STATE != ${ArchiveTransferState.PERMANENT_FAILURE.value} AND + ($CONTENT_TYPE LIKE 'image/%' OR $CONTENT_TYPE LIKE 'video/%') + """ + ) + .run() + .readToSingleLong(0L) + + val uploadedAttachmentBytes = readableDatabase + .rawQuery( + """ + SELECT $DATA_SIZE + FROM ( + SELECT DISTINCT $REMOTE_DIGEST, $DATA_SIZE + FROM $TABLE_NAME + WHERE + $DATA_FILE NOT NULL AND + $REMOTE_DIGEST NOT NULL AND + $TRANSFER_STATE = $TRANSFER_PROGRESS_DONE AND + $ARCHIVE_TRANSFER_STATE != ${ArchiveTransferState.PERMANENT_FAILURE.value} + ) + """ + ) + .readToList { it.requireLong(DATA_SIZE) } + .sumOf { + val paddedSize = PaddingInputStream.getPaddedSize(it) + val clientEncryptedSize = AttachmentCipherStreamUtil.getCiphertextLength(paddedSize) + val serverEncryptedSize = AttachmentCipherStreamUtil.getCiphertextLength(clientEncryptedSize) + + serverEncryptedSize + } + + val estimatedUploadedThumbnailBytes = RemoteConfig.backupMaxThumbnailFileSize.inWholeBytes * estimatedThumbnailCount + + return uploadedAttachmentBytes + estimatedUploadedThumbnailBytes + } + private fun getTransferFile(db: SQLiteDatabase, attachmentId: AttachmentId): File? { return db .select(TRANSFER_FILE) diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/DatabaseObserverExt.kt b/app/src/main/java/org/thoughtcrime/securesms/database/DatabaseObserverExt.kt new file mode 100644 index 0000000000..dbada52bc3 --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/database/DatabaseObserverExt.kt @@ -0,0 +1,40 @@ +/* + * Copyright 2025 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.thoughtcrime.securesms.database + +import kotlinx.coroutines.channels.awaitClose +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.callbackFlow + +/** + * Observe attachment deletions. + */ +fun DatabaseObserver.attachmentDeletions(): Flow { + return observe { registerAttachmentDeletedObserver(it) } +} + +/** + * Observe attachment updates. + */ +fun DatabaseObserver.attachmentUpdates(): Flow { + return observe { registerAttachmentUpdatedObserver(it) } +} + +/** + * Helper to register flow-ize database observer + */ +private fun DatabaseObserver.observe(registerObserver: DatabaseObserver.(listener: DatabaseObserver.Observer) -> Unit): Flow { + return callbackFlow { + val listener = DatabaseObserver.Observer { + trySend(Unit) + } + + this@observe.registerObserver(listener) + awaitClose { + this@observe.unregisterObserver(listener) + } + } +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/BackupMessagesJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/BackupMessagesJob.kt index 8a19998f41..6c022d3b63 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/BackupMessagesJob.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/BackupMessagesJob.kt @@ -177,17 +177,7 @@ class BackupMessagesJob private constructor( } SignalStore.backup.lastBackupTime = System.currentTimeMillis() - SignalStore.backup.usedBackupMediaSpace = when (val result = BackupRepository.getRemoteBackupUsedSpace()) { - is NetworkResult.Success -> result.result ?: 0 - is NetworkResult.NetworkError -> SignalStore.backup.usedBackupMediaSpace // TODO [backup] enqueue a secondary job to fetch the latest number -- no need to fail this one - is NetworkResult.StatusCodeError -> { - Log.w(TAG, "Failed to get used space: ${result.code}") - SignalStore.backup.usedBackupMediaSpace - } - - is NetworkResult.ApplicationError -> throw result.throwable - } - stopwatch.split("used-space") + stopwatch.split("save-meta") stopwatch.stop(TAG) if (isCanceled) { diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/CopyAttachmentToArchiveJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/CopyAttachmentToArchiveJob.kt index b62cb1085a..66d3f15638 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/CopyAttachmentToArchiveJob.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/CopyAttachmentToArchiveJob.kt @@ -15,8 +15,6 @@ import org.thoughtcrime.securesms.jobmanager.impl.NetworkConstraint import org.thoughtcrime.securesms.jobs.protos.CopyAttachmentToArchiveJobData import org.thoughtcrime.securesms.keyvalue.SignalStore import org.whispersystems.signalservice.api.NetworkResult -import org.whispersystems.signalservice.api.crypto.AttachmentCipherStreamUtil -import org.whispersystems.signalservice.internal.crypto.PaddingInputStream import java.lang.RuntimeException import java.util.concurrent.TimeUnit @@ -155,8 +153,6 @@ class CopyAttachmentToArchiveJob private constructor(private val attachmentId: A Log.d(TAG, "[$attachmentId] Refusing to enqueue thumb for canceled upload.") } - SignalStore.backup.usedBackupMediaSpace += AttachmentCipherStreamUtil.getCiphertextLength(PaddingInputStream.getPaddedSize(attachment.size)) - ArchiveUploadProgress.onAttachmentFinished(attachmentId) } diff --git a/app/src/main/java/org/thoughtcrime/securesms/keyvalue/BackupValues.kt b/app/src/main/java/org/thoughtcrime/securesms/keyvalue/BackupValues.kt index a31bc1438e..83f5bd59b9 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/keyvalue/BackupValues.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/keyvalue/BackupValues.kt @@ -34,7 +34,6 @@ class BackupValues(store: KeyValueStore) : SignalStoreValues(store) { private const val KEY_MEDIA_CDN_READ_CREDENTIALS = "backup.mediaCdnReadCredentials" private const val KEY_MEDIA_CDN_READ_CREDENTIALS_TIMESTAMP = "backup.mediaCdnReadCredentialsTimestamp" private const val KEY_RESTORE_STATE = "backup.restoreState" - private const val KEY_BACKUP_USED_MEDIA_SPACE = "backup.usedMediaSpace" private const val KEY_BACKUP_LAST_PROTO_SIZE = "backup.lastProtoSize" private const val KEY_BACKUP_TIER = "backup.backupTier" private const val KEY_BACKUP_TIER_INTERNAL_OVERRIDE = "backup.backupTier.internalOverride" @@ -85,7 +84,7 @@ class BackupValues(store: KeyValueStore) : SignalStoreValues(store) { override fun getKeysToIncludeInBackup(): List = emptyList() var cachedMediaCdnPath: String? by stringValue(KEY_CDN_MEDIA_PATH, null) - var usedBackupMediaSpace: Long by longValue(KEY_BACKUP_USED_MEDIA_SPACE, 0L) + var lastBackupProtoSize: Long by longValue(KEY_BACKUP_LAST_PROTO_SIZE, 0L) private val deletionStateValue = enumValue(KEY_BACKUP_DELETION_STATE, DeletionState.NONE, DeletionState.serializer) @@ -223,8 +222,6 @@ class BackupValues(store: KeyValueStore) : SignalStoreValues(store) { */ var archiveUploadState: ArchiveUploadProgressState? by protoValue(KEY_ARCHIVE_UPLOAD_STATE, ArchiveUploadProgressState.ADAPTER) - val totalBackupSize: Long get() = lastBackupProtoSize + usedBackupMediaSpace - /** True if the user backs up media, otherwise false. */ val backsUpMedia: Boolean @JvmName("backsUpMedia") diff --git a/app/src/main/java/org/thoughtcrime/securesms/keyvalue/RegistrationValues.kt b/app/src/main/java/org/thoughtcrime/securesms/keyvalue/RegistrationValues.kt index 832fcb0f06..a770e5339e 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/keyvalue/RegistrationValues.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/keyvalue/RegistrationValues.kt @@ -20,6 +20,7 @@ class RegistrationValues internal constructor(store: KeyValueStore) : SignalStor private const val SESSION_ID = "registration.session_id" private const val LOCAL_REGISTRATION_DATA = "registration.local_registration_data" private const val RESTORE_METHOD_TOKEN = "registration.restore_method_token" + private const val RESTORE_BACKUP_MEDIA_SIZE = "registration.restore_backup_media_size" private const val IS_OTHER_DEVICE_ANDROID = "registration.is_other_device_android" private const val RESTORING_ON_NEW_DEVICE = "registration.restoring_on_new_device" @@ -72,6 +73,7 @@ class RegistrationValues internal constructor(store: KeyValueStore) : SignalStor var isOtherDeviceAndroid: Boolean by booleanValue(IS_OTHER_DEVICE_ANDROID, false) var restoreMethodToken: String? by stringValue(RESTORE_METHOD_TOKEN, null) + var restoreBackupMediaSize: Long by longValue(RESTORE_BACKUP_MEDIA_SIZE, 0L) @get:JvmName("isRestoringOnNewDevice") var restoringOnNewDevice: Boolean by booleanValue(RESTORING_ON_NEW_DEVICE, false) diff --git a/app/src/main/java/org/thoughtcrime/securesms/registrationv3/data/QuickRegistrationRepository.kt b/app/src/main/java/org/thoughtcrime/securesms/registrationv3/data/QuickRegistrationRepository.kt index f2bfdcc3ca..aefd796c86 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/registrationv3/data/QuickRegistrationRepository.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/registrationv3/data/QuickRegistrationRepository.kt @@ -18,6 +18,7 @@ import org.signal.libsignal.protocol.InvalidKeyException import org.signal.libsignal.protocol.ecc.Curve import org.signal.registration.proto.RegistrationProvisionMessage import org.thoughtcrime.securesms.backup.v2.MessageBackupTier +import org.thoughtcrime.securesms.database.SignalDatabase import org.thoughtcrime.securesms.dependencies.AppDependencies import org.thoughtcrime.securesms.keyvalue.SignalStore import org.thoughtcrime.securesms.net.SignalNetwork @@ -85,7 +86,7 @@ object QuickRegistrationRepository { MessageBackupTier.FREE -> RegistrationProvisionMessage.Tier.FREE null -> null }, - backupSizeBytes = SignalStore.backup.totalBackupSize.takeIf { it > 0 }, + backupSizeBytes = SignalDatabase.attachments.getEstimatedArchiveMediaSize().takeIf { it > 0 }, restoreMethodToken = restoreMethodToken, aciIdentityKeyPublic = SignalStore.account.aciIdentityKey.publicKey.serialize().toByteString(), aciIdentityKeyPrivate = SignalStore.account.aciIdentityKey.privateKey.serialize().toByteString(), diff --git a/app/src/main/java/org/thoughtcrime/securesms/registrationv3/ui/restore/RemoteRestoreViewModel.kt b/app/src/main/java/org/thoughtcrime/securesms/registrationv3/ui/restore/RemoteRestoreViewModel.kt index 0e54a9471c..efa595c1fb 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/registrationv3/ui/restore/RemoteRestoreViewModel.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/registrationv3/ui/restore/RemoteRestoreViewModel.kt @@ -44,7 +44,7 @@ class RemoteRestoreViewModel(isOnlyRestoreOption: Boolean) : ViewModel() { isRemoteRestoreOnlyOption = isOnlyRestoreOption, backupTier = SignalStore.backup.backupTier, backupTime = SignalStore.backup.lastBackupTime, - backupSize = SignalStore.backup.totalBackupSize.bytes + backupSize = SignalStore.registration.restoreBackupMediaSize.bytes ) ) @@ -64,7 +64,7 @@ class RemoteRestoreViewModel(isOnlyRestoreOption: Boolean) : ViewModel() { loadState = ScreenState.LoadState.LOADED, backupTier = SignalStore.backup.backupTier, backupTime = SignalStore.backup.lastBackupTime, - backupSize = SignalStore.backup.totalBackupSize.bytes + backupSize = SignalStore.registration.restoreBackupMediaSize.bytes ) } else { if (SignalStore.backup.isBackupTierRestored || SignalStore.backup.lastBackupTime == 0L) { diff --git a/app/src/main/java/org/thoughtcrime/securesms/registrationv3/ui/restore/RestoreViaQrViewModel.kt b/app/src/main/java/org/thoughtcrime/securesms/registrationv3/ui/restore/RestoreViaQrViewModel.kt index a9d582ef01..0e98f77be9 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/registrationv3/ui/restore/RestoreViaQrViewModel.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/registrationv3/ui/restore/RestoreViaQrViewModel.kt @@ -157,10 +157,10 @@ class RestoreViaQrViewModel : ViewModel() { if (result is SecondaryProvisioningCipher.RegistrationProvisionResult.Success) { Log.i(TAG, "Saving restore method token: ***${result.message.restoreMethodToken.takeLast(4)}") SignalStore.registration.restoreMethodToken = result.message.restoreMethodToken + SignalStore.registration.restoreBackupMediaSize = result.message.backupSizeBytes ?: 0 SignalStore.registration.isOtherDeviceAndroid = result.message.platform == RegistrationProvisionMessage.Platform.ANDROID SignalStore.backup.lastBackupTime = result.message.backupTimestampMs ?: 0 - SignalStore.backup.usedBackupMediaSpace = result.message.backupSizeBytes ?: 0 SignalStore.backup.backupTier = when (result.message.tier) { RegistrationProvisionMessage.Tier.FREE -> MessageBackupTier.FREE RegistrationProvisionMessage.Tier.PAID -> MessageBackupTier.PAID diff --git a/core-util-jvm/src/main/java/org/signal/core/util/ByteExtensions.kt b/core-util-jvm/src/main/java/org/signal/core/util/ByteExtensions.kt index 5fcc0d58cc..2d08a01f48 100644 --- a/core-util-jvm/src/main/java/org/signal/core/util/ByteExtensions.kt +++ b/core-util-jvm/src/main/java/org/signal/core/util/ByteExtensions.kt @@ -112,6 +112,10 @@ class ByteSize(val bytes: Long) { return ByteSize(this.inWholeBytes - other.inWholeBytes) } + operator fun times(other: Long): ByteSize { + return ByteSize(this.inWholeBytes * other) + } + enum class Size(val label: String) { BYTE("B"), KIBIBYTE("KB"),