From c5753b96fffbe7ac6768211a49c552c6facd9c0f Mon Sep 17 00:00:00 2001 From: Greyson Parrelli Date: Fri, 19 Sep 2025 13:40:28 -0400 Subject: [PATCH] Update BackupMediaSnapshot to be based on attachments in backup frames. --- .../database/BackupMediaSnapshotTableTest.kt | 109 +++++++----------- .../securesms/backup/v2/BackupRepository.kt | 42 +++++-- .../backup/v2/util/FrameExtensions.kt | 103 +++++++++++++++++ .../internal/backup/InternalBackupStatsTab.kt | 15 ++- .../securesms/database/AttachmentTable.kt | 80 ++++++++----- .../database/BackupMediaSnapshotTable.kt | 103 ++++++++--------- .../securesms/jobs/BackupMessagesJob.kt | 77 ++++++++++--- app/src/main/res/drawable/qrcode_logo.png | Bin 5676 -> 7024 bytes 8 files changed, 343 insertions(+), 186 deletions(-) create mode 100644 app/src/main/java/org/thoughtcrime/securesms/backup/v2/util/FrameExtensions.kt diff --git a/app/src/androidTest/java/org/thoughtcrime/securesms/database/BackupMediaSnapshotTableTest.kt b/app/src/androidTest/java/org/thoughtcrime/securesms/database/BackupMediaSnapshotTableTest.kt index 328a57d9dd..c6f8412e05 100644 --- a/app/src/androidTest/java/org/thoughtcrime/securesms/database/BackupMediaSnapshotTableTest.kt +++ b/app/src/androidTest/java/org/thoughtcrime/securesms/database/BackupMediaSnapshotTableTest.kt @@ -10,7 +10,7 @@ import org.junit.runner.RunWith import org.signal.core.util.count import org.signal.core.util.readToSingleInt import org.thoughtcrime.securesms.backup.v2.ArchivedMediaObject -import org.thoughtcrime.securesms.database.BackupMediaSnapshotTable.ArchiveMediaItem +import org.thoughtcrime.securesms.database.BackupMediaSnapshotTable.MediaEntry import org.thoughtcrime.securesms.testing.SignalActivityRule @RunWith(AndroidJUnit4::class) @@ -21,7 +21,7 @@ class BackupMediaSnapshotTableTest { @Test fun givenAnEmptyTable_whenIWriteToTable_thenIExpectEmptyTable() { - SignalDatabase.backupMediaSnapshots.writeFullSizePendingMediaObjects(generateArchiveMediaItemSequence(count = 100)) + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(generateArchiveMediaItemSequence(count = 100)) val count = getCountForLatestSnapshot(includeThumbnails = true) @@ -30,7 +30,7 @@ class BackupMediaSnapshotTableTest { @Test fun givenAnEmptyTable_whenIWriteToTableAndCommit_thenIExpectFilledTable() { - SignalDatabase.backupMediaSnapshots.writeFullSizePendingMediaObjects(generateArchiveMediaItemSequence(count = 100)) + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(generateArchiveMediaItemSequence(count = 100)) SignalDatabase.backupMediaSnapshots.commitPendingRows() val count = getCountForLatestSnapshot(includeThumbnails = false) @@ -43,8 +43,8 @@ class BackupMediaSnapshotTableTest { val inputCount = 100 val countWithThumbnails = inputCount * 2 - SignalDatabase.backupMediaSnapshots.writeFullSizePendingMediaObjects(generateArchiveMediaItemSequence(count = inputCount)) - SignalDatabase.backupMediaSnapshots.writeThumbnailPendingMediaObjects(generateArchiveMediaItemSequence(count = inputCount)) + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(generateArchiveMediaItemSequence(count = inputCount)) + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(generateArchiveMediaItemSequence(count = inputCount, thumbnail = true)) SignalDatabase.backupMediaSnapshots.commitPendingRows() val count = getCountForLatestSnapshot(includeThumbnails = true) @@ -52,40 +52,16 @@ class BackupMediaSnapshotTableTest { assertThat(count).isEqualTo(countWithThumbnails) } - @Test - fun givenAnEmptyTable_whenIWriteToTableAndCommitQuotes_thenIExpectFilledTableWithNoThumbnails() { - val inputCount = 100 - - SignalDatabase.backupMediaSnapshots.writeFullSizePendingMediaObjects(generateArchiveMediaItemSequence(count = inputCount, quote = true)) - SignalDatabase.backupMediaSnapshots.commitPendingRows() - - val count = getCountForLatestSnapshot(includeThumbnails = true) - - assertThat(count).isEqualTo(inputCount) - } - - @Test - fun givenAnEmptyTable_whenIWriteToTableAndCommitNonMedia_thenIExpectFilledTableWithNoThumbnails() { - val inputCount = 100 - - SignalDatabase.backupMediaSnapshots.writeFullSizePendingMediaObjects(generateArchiveMediaItemSequence(count = inputCount, contentType = "text/plain")) - SignalDatabase.backupMediaSnapshots.commitPendingRows() - - val count = getCountForLatestSnapshot(includeThumbnails = true) - - assertThat(count).isEqualTo(inputCount) - } - @Test fun givenAFilledTable_whenIReinsertObjects_thenIExpectUncommittedOverrides() { val initialCount = 100 val additionalCount = 25 - SignalDatabase.backupMediaSnapshots.writeFullSizePendingMediaObjects(generateArchiveMediaItemSequence(count = initialCount)) + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(generateArchiveMediaItemSequence(count = initialCount)) SignalDatabase.backupMediaSnapshots.commitPendingRows() // This relies on how the sequence of mediaIds is generated in tests -- the ones we generate here will have the mediaIds as the ones we generated above - SignalDatabase.backupMediaSnapshots.writeFullSizePendingMediaObjects(generateArchiveMediaItemSequence(count = additionalCount)) + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(generateArchiveMediaItemSequence(count = additionalCount)) val pendingCount = getCountForPending(includeThumbnails = false) val latestVersionCount = getCountForLatestSnapshot(includeThumbnails = false) @@ -99,11 +75,11 @@ class BackupMediaSnapshotTableTest { val initialCount = 100 val additionalCount = 25 - SignalDatabase.backupMediaSnapshots.writeFullSizePendingMediaObjects(generateArchiveMediaItemSequence(count = initialCount)) + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(generateArchiveMediaItemSequence(count = initialCount)) SignalDatabase.backupMediaSnapshots.commitPendingRows() // This relies on how the sequence of mediaIds is generated in tests -- the ones we generate here will have the mediaIds as the ones we generated above - SignalDatabase.backupMediaSnapshots.writeFullSizePendingMediaObjects(generateArchiveMediaItemSequence(count = additionalCount)) + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(generateArchiveMediaItemSequence(count = additionalCount)) SignalDatabase.backupMediaSnapshots.commitPendingRows() val pendingCount = getCountForPending(includeThumbnails = false) @@ -120,10 +96,10 @@ class BackupMediaSnapshotTableTest { val initialCount = 100 val additionalCount = 25 - SignalDatabase.backupMediaSnapshots.writeFullSizePendingMediaObjects(generateArchiveMediaItemSequence(count = initialCount)) + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(generateArchiveMediaItemSequence(count = initialCount)) SignalDatabase.backupMediaSnapshots.commitPendingRows() - SignalDatabase.backupMediaSnapshots.writeFullSizePendingMediaObjects(generateArchiveMediaItemSequence(count = additionalCount)) + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(generateArchiveMediaItemSequence(count = additionalCount)) SignalDatabase.backupMediaSnapshots.commitPendingRows() val page = SignalDatabase.backupMediaSnapshots.getPageOfOldMediaObjects(pageSize = 1_000) @@ -146,7 +122,7 @@ class BackupMediaSnapshotTableTest { createArchiveMediaObject(seed = 2, cdn = 2) ) - SignalDatabase.backupMediaSnapshots.writeFullSizePendingMediaObjects(localData.asSequence()) + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(localData) SignalDatabase.backupMediaSnapshots.commitPendingRows() val mismatches = SignalDatabase.backupMediaSnapshots.getMediaObjectsWithNonMatchingCdn(remoteData) @@ -165,7 +141,7 @@ class BackupMediaSnapshotTableTest { createArchiveMediaObject(seed = 2, cdn = 99) ) - SignalDatabase.backupMediaSnapshots.writeFullSizePendingMediaObjects(localData.asSequence()) + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(localData) SignalDatabase.backupMediaSnapshots.commitPendingRows() val mismatches = SignalDatabase.backupMediaSnapshots.getMediaObjectsWithNonMatchingCdn(remoteData) @@ -187,7 +163,7 @@ class BackupMediaSnapshotTableTest { createArchiveMediaObject(seed = 2, cdn = 2) ) - SignalDatabase.backupMediaSnapshots.writeFullSizePendingMediaObjects(localData.asSequence()) + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(localData) SignalDatabase.backupMediaSnapshots.commitPendingRows() val notFound = SignalDatabase.backupMediaSnapshots.getMediaObjectsThatCantBeFound(remoteData) @@ -206,7 +182,7 @@ class BackupMediaSnapshotTableTest { createArchiveMediaObject(seed = 3, cdn = 2) ) - SignalDatabase.backupMediaSnapshots.writeFullSizePendingMediaObjects(localData.asSequence()) + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(localData) SignalDatabase.backupMediaSnapshots.commitPendingRows() val notFound = SignalDatabase.backupMediaSnapshots.getMediaObjectsThatCantBeFound(remoteData) @@ -223,7 +199,7 @@ class BackupMediaSnapshotTableTest { @Test fun getCurrentSnapshotVersion_singleCommit() { - SignalDatabase.backupMediaSnapshots.writeFullSizePendingMediaObjects(generateArchiveMediaItemSequence(count = 100)) + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(generateArchiveMediaItemSequence(count = 100)) SignalDatabase.backupMediaSnapshots.commitPendingRows() val version = SignalDatabase.backupMediaSnapshots.getCurrentSnapshotVersion() @@ -235,15 +211,12 @@ class BackupMediaSnapshotTableTest { fun getMediaObjectsLastSeenOnCdnBeforeSnapshotVersion_noneMarkedSeen() { val initialCount = 100 - SignalDatabase.backupMediaSnapshots.writeFullSizePendingMediaObjects(generateArchiveMediaItemSequence(count = initialCount)) - SignalDatabase.backupMediaSnapshots.writeThumbnailPendingMediaObjects(generateArchiveMediaItemSequence(count = initialCount)) + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(generateArchiveMediaItemSequence(count = initialCount)) SignalDatabase.backupMediaSnapshots.commitPendingRows() val notSeenCount = SignalDatabase.backupMediaSnapshots.getMediaObjectsLastSeenOnCdnBeforeSnapshotVersion(1).count - val expectedOldCountIncludingThumbnails = initialCount * 2 - - assertThat(notSeenCount).isEqualTo(expectedOldCountIncludingThumbnails) + assertThat(notSeenCount).isEqualTo(initialCount) } @Test @@ -251,23 +224,25 @@ class BackupMediaSnapshotTableTest { val initialCount = 100 val markSeenCount = 25 - val itemsToCommit = generateArchiveMediaItemSequence(count = initialCount) - SignalDatabase.backupMediaSnapshots.writeFullSizePendingMediaObjects(itemsToCommit) - SignalDatabase.backupMediaSnapshots.writeThumbnailPendingMediaObjects(itemsToCommit) + val fullSizeItems = generateArchiveMediaItemSequence(count = initialCount, thumbnail = false) + val thumbnailItems = generateArchiveMediaItemSequence(count = initialCount, thumbnail = true) + + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(fullSizeItems) + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(thumbnailItems) + SignalDatabase.backupMediaSnapshots.commitPendingRows() - val normalIdsToMarkSeen = itemsToCommit.take(markSeenCount).map { it.mediaId }.toList() - val thumbnailIdsToMarkSeen = itemsToCommit.take(markSeenCount).map { it.thumbnailMediaId }.toList() - val allItemsToMarkSeen = normalIdsToMarkSeen + thumbnailIdsToMarkSeen + val fullSizeIdsToMarkSeen = fullSizeItems.take(markSeenCount).map { it.mediaId }.toList() + val thumbnailIdsToMarkSeen = thumbnailItems.take(markSeenCount).map { it.mediaId }.toList() - SignalDatabase.backupMediaSnapshots.markSeenOnRemote(allItemsToMarkSeen, 1) + SignalDatabase.backupMediaSnapshots.markSeenOnRemote(fullSizeIdsToMarkSeen, 1) + SignalDatabase.backupMediaSnapshots.markSeenOnRemote(thumbnailIdsToMarkSeen, 1) val notSeenCount = SignalDatabase.backupMediaSnapshots.getMediaObjectsLastSeenOnCdnBeforeSnapshotVersion(1).count - val expectedOldCount = initialCount - markSeenCount - val expectedOldCountIncludingThumbnails = expectedOldCount * 2 + val expectedOldCount = (initialCount * 2) - (markSeenCount * 2) - assertThat(notSeenCount).isEqualTo(expectedOldCountIncludingThumbnails) + assertThat(notSeenCount).isEqualTo(expectedOldCount) } private fun getTotalItemCount(includeThumbnails: Boolean): Int { @@ -317,28 +292,30 @@ class BackupMediaSnapshotTableTest { .readToSingleInt(0) } - private fun generateArchiveMediaItemSequence(count: Int, quote: Boolean = false, contentType: String = "image/jpeg"): Sequence { + private fun generateArchiveMediaItemSequence(count: Int, thumbnail: Boolean = false): Collection { return (1..count) - .asSequence() - .map { createArchiveMediaItem(it, quote = quote, contentType = contentType) } + .map { createArchiveMediaItem(it, thumbnail = thumbnail) } + .toList() } - private fun createArchiveMediaItem(seed: Int, cdn: Int = 0, quote: Boolean = false, contentType: String = "image/jpeg"): ArchiveMediaItem { - return ArchiveMediaItem( - mediaId = "media_id_$seed", - thumbnailMediaId = "thumbnail_media_id_$seed", + private fun createArchiveMediaItem(seed: Int, thumbnail: Boolean = false, cdn: Int = 0): MediaEntry { + return MediaEntry( + mediaId = mediaId(seed, thumbnail), cdn = cdn, plaintextHash = Util.toByteArray(seed), remoteKey = Util.toByteArray(seed), - quote = quote, - contentType = contentType + isThumbnail = thumbnail ) } - private fun createArchiveMediaObject(seed: Int, cdn: Int = 0): ArchivedMediaObject { + private fun createArchiveMediaObject(seed: Int, thumbnail: Boolean = false, cdn: Int = 0): ArchivedMediaObject { return ArchivedMediaObject( - mediaId = "media_id_$seed", + mediaId = mediaId(seed, thumbnail), cdn = cdn ) } + + fun mediaId(seed: Int, thumbnail: Boolean): String { + return "media_id_${seed}_$thumbnail" + } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt index 304c28adf4..68820f7004 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt @@ -72,6 +72,7 @@ import org.thoughtcrime.securesms.backup.v2.processor.RecipientArchiveProcessor import org.thoughtcrime.securesms.backup.v2.processor.StickerArchiveProcessor import org.thoughtcrime.securesms.backup.v2.proto.BackupDebugInfo import org.thoughtcrime.securesms.backup.v2.proto.BackupInfo +import org.thoughtcrime.securesms.backup.v2.proto.Frame import org.thoughtcrime.securesms.backup.v2.stream.BackupExportWriter import org.thoughtcrime.securesms.backup.v2.stream.BackupImportReader import org.thoughtcrime.securesms.backup.v2.stream.EncryptedBackupReader @@ -80,6 +81,7 @@ import org.thoughtcrime.securesms.backup.v2.stream.PlainTextBackupReader import org.thoughtcrime.securesms.backup.v2.stream.PlainTextBackupWriter import org.thoughtcrime.securesms.backup.v2.ui.BackupAlert import org.thoughtcrime.securesms.backup.v2.ui.subscription.MessageBackupsType +import org.thoughtcrime.securesms.backup.v2.util.ArchiveAttachmentInfo import org.thoughtcrime.securesms.components.settings.app.AppSettingsActivity import org.thoughtcrime.securesms.components.settings.app.subscription.RecurringInAppPaymentRepository import org.thoughtcrime.securesms.crypto.AttachmentSecretProvider @@ -722,13 +724,18 @@ object BackupRepository { append = { main.write(it) } ) + val maxBufferSize = 10_000 + var totalAttachmentCount = 0 + val attachmentInfos: MutableSet = mutableSetOf() + export( currentTime = System.currentTimeMillis(), isLocal = true, writer = writer, progressEmitter = localBackupProgressEmitter, cancellationSignal = cancellationSignal, - forTransfer = false + forTransfer = false, + extraFrameOperation = null ) { dbSnapshot -> val localArchivableAttachments = dbSnapshot .attachmentTable @@ -764,7 +771,7 @@ object BackupRepository { currentTime: Long, progressEmitter: ExportProgressListener? = null, cancellationSignal: () -> Boolean = { false }, - extraExportOperations: ((SignalDatabase) -> Unit)? + extraFrameOperation: ((Frame) -> Unit)? ) { val writer = EncryptedBackupWriter.createForSignalBackup( key = messageBackupKey, @@ -782,7 +789,8 @@ object BackupRepository { forTransfer = false, progressEmitter = progressEmitter, cancellationSignal = cancellationSignal, - extraExportOperations = extraExportOperations + extraFrameOperation = extraFrameOperation, + endingExportOperation = null ) } @@ -811,7 +819,8 @@ object BackupRepository { forTransfer = true, progressEmitter = progressEmitter, cancellationSignal = cancellationSignal, - extraExportOperations = null + extraFrameOperation = null, + endingExportOperation = null ) } @@ -825,8 +834,7 @@ object BackupRepository { currentTime: Long = System.currentTimeMillis(), forTransfer: Boolean = false, progressEmitter: ExportProgressListener? = null, - cancellationSignal: () -> Boolean = { false }, - extraExportOperations: ((SignalDatabase) -> Unit)? = null + cancellationSignal: () -> Boolean = { false } ) { val writer: BackupExportWriter = if (plaintext) { PlainTextBackupWriter(outputStream) @@ -846,7 +854,8 @@ object BackupRepository { forTransfer = forTransfer, progressEmitter = progressEmitter, cancellationSignal = cancellationSignal, - extraExportOperations = extraExportOperations + extraFrameOperation = null, + endingExportOperation = null ) } @@ -868,7 +877,8 @@ object BackupRepository { forTransfer: Boolean, progressEmitter: ExportProgressListener?, cancellationSignal: () -> Boolean, - extraExportOperations: ((SignalDatabase) -> Unit)? + extraFrameOperation: ((Frame) -> Unit)?, + endingExportOperation: ((SignalDatabase) -> Unit)? ) { val eventTimer = EventTimer() val mainDbName = if (isLocal) LOCAL_MAIN_DB_SNAPSHOT_NAME else REMOTE_MAIN_DB_SNAPSHOT_NAME @@ -906,8 +916,9 @@ object BackupRepository { // We're using a snapshot, so the transaction is more for perf than correctness dbSnapshot.rawWritableDatabase.withinTransaction { progressEmitter?.onAccount() - AccountDataArchiveProcessor.export(dbSnapshot, signalStoreSnapshot) { - writer.write(it) + AccountDataArchiveProcessor.export(dbSnapshot, signalStoreSnapshot) { frame -> + writer.write(frame) + extraFrameOperation?.invoke(frame) eventTimer.emit("account") frameCount++ } @@ -919,6 +930,7 @@ object BackupRepository { progressEmitter?.onRecipient() RecipientArchiveProcessor.export(dbSnapshot, signalStoreSnapshot, exportState, selfRecipientId, selfAci) { writer.write(it) + extraFrameOperation?.invoke(it) eventTimer.emit("recipient") frameCount++ } @@ -930,6 +942,7 @@ object BackupRepository { progressEmitter?.onThread() ChatArchiveProcessor.export(dbSnapshot, exportState) { frame -> writer.write(frame) + extraFrameOperation?.invoke(frame) eventTimer.emit("thread") frameCount++ } @@ -940,6 +953,7 @@ object BackupRepository { progressEmitter?.onCall() AdHocCallArchiveProcessor.export(dbSnapshot, exportState) { frame -> writer.write(frame) + extraFrameOperation?.invoke(frame) eventTimer.emit("call") frameCount++ } @@ -951,6 +965,7 @@ object BackupRepository { progressEmitter?.onSticker() StickerArchiveProcessor.export(dbSnapshot) { frame -> writer.write(frame) + extraFrameOperation?.invoke(frame) eventTimer.emit("sticker-pack") frameCount++ } @@ -962,6 +977,7 @@ object BackupRepository { progressEmitter?.onNotificationProfile() NotificationProfileProcessor.export(dbSnapshot, exportState) { frame -> writer.write(frame) + extraFrameOperation?.invoke(frame) eventTimer.emit("notification-profile") frameCount++ } @@ -973,6 +989,7 @@ object BackupRepository { progressEmitter?.onChatFolder() ChatFolderProcessor.export(dbSnapshot, exportState) { frame -> writer.write(frame) + extraFrameOperation?.invoke(frame) eventTimer.emit("chat-folder") frameCount++ } @@ -986,6 +1003,7 @@ object BackupRepository { progressEmitter?.onMessage(0, approximateMessageCount) ChatItemArchiveProcessor.export(dbSnapshot, exportState, selfRecipientId, cancellationSignal) { frame -> writer.write(frame) + extraFrameOperation?.invoke(frame) eventTimer.emit("message") frameCount++ @@ -1001,7 +1019,7 @@ object BackupRepository { } } - extraExportOperations?.invoke(dbSnapshot) + endingExportOperation?.invoke(dbSnapshot) Log.d(TAG, "[export] totalFrames: $frameCount | ${eventTimer.stop().summary}") } finally { @@ -2071,7 +2089,7 @@ object BackupRepository { val messageBackupKey = SignalStore.backup.messageBackupKey Log.i(TAG, "[remoteRestore] Fetching SVRB data") - val svrBAuth = when (val result = BackupRepository.getSvrBAuth()) { + val svrBAuth = when (val result = getSvrBAuth()) { is NetworkResult.Success -> result.result is NetworkResult.NetworkError -> return RemoteRestoreResult.NetworkError.logW(TAG, "[remoteRestore] Network error when getting SVRB auth.", result.getCause()) is NetworkResult.StatusCodeError -> return RemoteRestoreResult.NetworkError.logW(TAG, "[remoteRestore] Status code error when getting SVRB auth.", result.getCause()) diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/util/FrameExtensions.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/util/FrameExtensions.kt new file mode 100644 index 0000000000..f94cc54dfc --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/util/FrameExtensions.kt @@ -0,0 +1,103 @@ +/* + * Copyright 2025 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.thoughtcrime.securesms.backup.v2.util + +import okio.ByteString +import org.thoughtcrime.securesms.attachments.Cdn +import org.thoughtcrime.securesms.backup.v2.proto.AccountData +import org.thoughtcrime.securesms.backup.v2.proto.Chat +import org.thoughtcrime.securesms.backup.v2.proto.ChatItem +import org.thoughtcrime.securesms.backup.v2.proto.FilePointer +import org.thoughtcrime.securesms.backup.v2.proto.Frame +import org.whispersystems.signalservice.api.backup.MediaName + +fun Frame.getAllReferencedArchiveAttachmentInfos(): Set { + val infos: MutableSet = mutableSetOf() + when { + this.account != null -> infos += this.account.getAllReferencedArchiveAttachmentInfos() + this.chat != null -> infos += this.chat.getAllReferencedArchiveAttachmentInfos() + this.chatItem != null -> infos += this.chatItem.getAllReferencedArchiveAttachmentInfos() + } + return infos.toSet() +} + +private fun AccountData.getAllReferencedArchiveAttachmentInfos(): Set { + val info = this.accountSettings?.defaultChatStyle?.wallpaperPhoto?.toArchiveAttachmentInfo() + + return if (info != null) { + setOf(info) + } else { + emptySet() + } +} + +private fun Chat.getAllReferencedArchiveAttachmentInfos(): Set { + val info = this.style?.wallpaperPhoto?.toArchiveAttachmentInfo() + + return if (info != null) { + setOf(info) + } else { + emptySet() + } +} + +private fun ChatItem.getAllReferencedArchiveAttachmentInfos(): Set { + var out: MutableSet? = null + + // The user could have many chat items, and most will not have attachments. To avoid allocating unnecessary sets, we do this little trick. + // (Note: emptySet() returns a constant under the hood, so that's fine) + fun appendToOutput(item: ArchiveAttachmentInfo) { + if (out == null) { + out = mutableSetOf() + } + + out.add(item) + } + + this.contactMessage?.contact?.avatar?.toArchiveAttachmentInfo()?.let { appendToOutput(it) } + this.directStoryReplyMessage?.textReply?.longText?.toArchiveAttachmentInfo()?.let { appendToOutput(it) } + this.standardMessage?.attachments?.mapNotNull { it.pointer?.toArchiveAttachmentInfo() }?.forEach { appendToOutput(it) } + this.standardMessage?.quote?.attachments?.mapNotNull { it.thumbnail?.pointer?.toArchiveAttachmentInfo(forQuote = true) }?.forEach { appendToOutput(it) } + this.standardMessage?.linkPreview?.mapNotNull { it.image?.toArchiveAttachmentInfo() }?.forEach { appendToOutput(it) } + this.standardMessage?.longText?.toArchiveAttachmentInfo()?.let { appendToOutput(it) } + this.stickerMessage?.sticker?.data_?.toArchiveAttachmentInfo()?.let { appendToOutput(it) } + this.viewOnceMessage?.attachment?.pointer?.toArchiveAttachmentInfo()?.let { appendToOutput(it) } + + this.revisions.forEach { revision -> + revision.getAllReferencedArchiveAttachmentInfos().forEach { appendToOutput(it) } + } + + return out ?: emptySet() +} + +private fun FilePointer.toArchiveAttachmentInfo(forQuote: Boolean = false): ArchiveAttachmentInfo? { + if (this.locatorInfo?.key == null) { + return null + } + + if (this.locatorInfo.plaintextHash == null) { + return null + } + + return ArchiveAttachmentInfo( + plaintextHash = this.locatorInfo.plaintextHash, + remoteKey = this.locatorInfo.key, + cdn = this.locatorInfo.mediaTierCdnNumber ?: Cdn.CDN_0.cdnNumber, + contentType = this.contentType, + forQuote = forQuote + ) +} + +data class ArchiveAttachmentInfo( + val plaintextHash: ByteString, + val remoteKey: ByteString, + val cdn: Int, + val contentType: String?, + val forQuote: Boolean +) { + val fullSizeMediaName: MediaName get() = MediaName.fromPlaintextHashAndRemoteKey(plaintextHash.toByteArray(), remoteKey.toByteArray()) + val thumbnailMediaName: MediaName get() = MediaName.fromPlaintextHashAndRemoteKeyForThumbnail(plaintextHash.toByteArray(), remoteKey.toByteArray()) +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupStatsTab.kt b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupStatsTab.kt index 56541e7578..5aa2e8bb48 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupStatsTab.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupStatsTab.kt @@ -43,11 +43,6 @@ fun InternalBackupStatsTab(stats: InternalBackupPlaygroundViewModel.StatsState, label = "${stats.attachmentStats.totalUniqueMediaNames}" ) - Rows.TextRow( - text = "Total eligible for upload rows", - label = "${stats.attachmentStats.totalEligibleForUploadRows}" - ) - Rows.TextRow( text = "Total unique media names eligible for upload ⭐", label = "${stats.attachmentStats.totalUniqueMediaNamesEligibleForUpload}" @@ -73,6 +68,16 @@ fun InternalBackupStatsTab(stats: InternalBackupPlaygroundViewModel.StatsState, label = "${stats.attachmentStats.pendingAttachmentUploadBytes} (~${stats.attachmentStats.pendingAttachmentUploadBytes.bytes.toUnitString()})" ) + Rows.TextRow( + text = "Last snapshot full-size count ⭐", + label = "${stats.attachmentStats.lastSnapshotFullSizeCount}" + ) + + Rows.TextRow( + text = "Last snapshot thumbnail count ⭐", + label = "${stats.attachmentStats.lastSnapshotThumbnailCount}" + ) + Rows.TextRow( text = "Uploaded attachment bytes ⭐", label = "${stats.attachmentStats.uploadedAttachmentBytes} (~${stats.attachmentStats.uploadedAttachmentBytes.bytes.toUnitString()})" diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt b/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt index dd23804baf..871f6eee11 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt @@ -431,34 +431,27 @@ class AttachmentTable( } /** - * Returns a cursor (with just the plaintextHash+remoteKey+archive_cdn) for all full-size attachments that are slated to be included in the current archive upload. - * Used for snapshotting data in [BackupMediaSnapshotTable]. + * Returns a list that has any permanently-failed thumbnails removed. */ - fun getFullSizeAttachmentsThatWillBeIncludedInArchive(): Cursor { - return readableDatabase - .select(DATA_HASH_END, REMOTE_KEY, ARCHIVE_CDN, QUOTE, CONTENT_TYPE) - .from("$TABLE_NAME LEFT JOIN ${MessageTable.TABLE_NAME} ON $TABLE_NAME.$MESSAGE_ID = ${MessageTable.TABLE_NAME}.${MessageTable.ID}") - .where(buildAttachmentsThatNeedUploadQuery(transferStateFilter = "$ARCHIVE_TRANSFER_STATE != ${ArchiveTransferState.PERMANENT_FAILURE.value}")) - .run() - } + fun filterPermanentlyFailedThumbnails(entries: Set): Set { + val entriesByMediaName: MutableMap = entries + .associateBy { MediaName.fromPlaintextHashAndRemoteKeyForThumbnail(it.plaintextHash, it.remoteKey).name } + .toMutableMap() - /** - * Returns a cursor (with just the plaintextHash+remoteKey+archive_cdn) for all thumbnail attachments that are slated to be included in the current archive upload. - * Used for snapshotting data in [BackupMediaSnapshotTable]. - */ - fun getThumbnailAttachmentsThatWillBeIncludedInArchive(): Cursor { - return readableDatabase - .select(DATA_HASH_END, REMOTE_KEY, ARCHIVE_CDN, QUOTE, CONTENT_TYPE) - .from("$TABLE_NAME LEFT JOIN ${MessageTable.TABLE_NAME} ON $TABLE_NAME.$MESSAGE_ID = ${MessageTable.TABLE_NAME}.${MessageTable.ID}") - .where( - """ - ${buildAttachmentsThatNeedUploadQuery(transferStateFilter = "$ARCHIVE_THUMBNAIL_TRANSFER_STATE != ${ArchiveTransferState.PERMANENT_FAILURE.value}")} AND - $QUOTE = 0 AND - ($CONTENT_TYPE LIKE 'image/%' OR $CONTENT_TYPE LIKE 'video/%') AND - $CONTENT_TYPE != 'image/svg+xml' - """ - ) + readableDatabase + .select(DATA_HASH_END, REMOTE_KEY) + .from(TABLE_NAME) + .where("$DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL AND $ARCHIVE_THUMBNAIL_TRANSFER_STATE = ${ArchiveTransferState.PERMANENT_FAILURE.value}") .run() + .forEach { cursor -> + val hashEnd = cursor.requireNonNullString(DATA_HASH_END) + val remoteKey = cursor.requireNonNullString(REMOTE_KEY) + val thumbnailMediaName = MediaName.fromPlaintextHashAndRemoteKeyForThumbnail(Base64.decode(hashEnd), Base64.decode(remoteKey)).name + + entriesByMediaName.remove(thumbnailMediaName) + } + + return entriesByMediaName.values.toSet() } fun hasData(attachmentId: AttachmentId): Boolean { @@ -566,6 +559,25 @@ class AttachmentTable( .flatten() } + fun getLocalArchivableAttachment(plaintextHash: String, remoteKey: String): LocalArchivableAttachment? { + return readableDatabase + .select(*PROJECTION) + .from(TABLE_NAME) + .where("$DATA_HASH_END = ? AND $REMOTE_KEY = ?") + .orderBy("$ID DESC") + .limit(1) + .run() + .readToSingleObject { + LocalArchivableAttachment( + file = File(it.requireNonNullString(DATA_FILE)), + random = it.requireNonNullBlob(DATA_RANDOM), + size = it.requireLong(DATA_SIZE), + remoteKey = Base64.decode(it.requireNonNullString(REMOTE_KEY)), + plaintextHash = Base64.decode(it.requireNonNullString(DATA_HASH_END)) + ) + } + } + fun getLocalArchivableAttachments(): List { return readableDatabase .select(*PROJECTION) @@ -3214,7 +3226,7 @@ class AttachmentTable( .select(*PROJECTION) .from(TABLE_NAME) .where("$REMOTE_KEY NOT NULL AND $DATA_HASH_END NOT NULL") - .groupBy(DATA_HASH_END) + .groupBy("$DATA_HASH_END, $REMOTE_KEY") .run() .forEach { cursor -> val remoteKey = Base64.decode(cursor.requireNonNullString(REMOTE_KEY)) @@ -3239,7 +3251,6 @@ class AttachmentTable( fun debugGetAttachmentStats(): DebugAttachmentStats { val totalAttachmentRows = readableDatabase.count().from(TABLE_NAME).run().readToSingleLong(0) - val totalEligibleForUploadRows = getFullSizeAttachmentsThatWillBeIncludedInArchive().count val totalUniqueDataFiles = readableDatabase.select("COUNT(DISTINCT $DATA_FILE)").from(TABLE_NAME).run().readToSingleLong(0) val totalUniqueMediaNames = readableDatabase.query("SELECT COUNT(*) FROM (SELECT DISTINCT $DATA_HASH_END, $REMOTE_KEY FROM $TABLE_NAME WHERE $DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL)").readToSingleLong(0) @@ -3309,15 +3320,19 @@ class AttachmentTable( val uploadedThumbnailCount = archiveStatusMediaNameThumbnailCounts.getOrDefault(ArchiveTransferState.FINISHED, 0L) val uploadedThumbnailBytes = uploadedThumbnailCount * RemoteConfig.backupMaxThumbnailFileSize.inWholeBytes + val lastSnapshotFullSizeCount = SignalDatabase.backupMediaSnapshots.debugGetFullSizeAttachmentCountForMostRecentSnapshot() + val lastSnapshotThumbnailCount = SignalDatabase.backupMediaSnapshots.debugGetThumbnailAttachmentCountForMostRecentSnapshot() + return DebugAttachmentStats( totalAttachmentRows = totalAttachmentRows, - totalEligibleForUploadRows = totalEligibleForUploadRows.toLong(), totalUniqueMediaNamesEligibleForUpload = totalUniqueMediaNamesEligibleForUpload, totalUniqueDataFiles = totalUniqueDataFiles, totalUniqueMediaNames = totalUniqueMediaNames, archiveStatusMediaNameCounts = archiveStatusMediaNameCounts, mediaNamesWithThumbnailsCount = uniqueEligibleMediaNamesWithThumbnailsCount, archiveStatusMediaNameThumbnailCounts = archiveStatusMediaNameThumbnailCounts, + lastSnapshotFullSizeCount = lastSnapshotFullSizeCount.toLong(), + lastSnapshotThumbnailCount = lastSnapshotThumbnailCount.toLong(), pendingAttachmentUploadBytes = pendingAttachmentUploadBytes, uploadedAttachmentBytes = uploadedAttachmentBytes, uploadedThumbnailBytes = uploadedThumbnailBytes @@ -3727,13 +3742,14 @@ class AttachmentTable( data class DebugAttachmentStats( val totalAttachmentRows: Long = 0L, - val totalEligibleForUploadRows: Long = 0L, val totalUniqueMediaNamesEligibleForUpload: Long = 0L, val totalUniqueDataFiles: Long = 0L, val totalUniqueMediaNames: Long = 0L, val archiveStatusMediaNameCounts: Map = emptyMap(), val mediaNamesWithThumbnailsCount: Long = 0L, val archiveStatusMediaNameThumbnailCounts: Map = emptyMap(), + val lastSnapshotFullSizeCount: Long = 0L, + val lastSnapshotThumbnailCount: Long = 0L, val pendingAttachmentUploadBytes: Long = 0L, val uploadedAttachmentBytes: Long = 0L, val uploadedThumbnailBytes: Long = 0L @@ -3747,12 +3763,13 @@ class AttachmentTable( fun prettyString(): String { return buildString { appendLine("Total attachment rows: $totalAttachmentRows") - appendLine("Total eligible for upload rows: $totalEligibleForUploadRows") appendLine("Total unique media names eligible for upload: $totalUniqueMediaNamesEligibleForUpload") appendLine("Total unique data files: $totalUniqueDataFiles") appendLine("Total unique media names: $totalUniqueMediaNames") appendLine("Media names with thumbnails count: $mediaNamesWithThumbnailsCount") appendLine("Pending attachment upload bytes: $pendingAttachmentUploadBytes") + appendLine("Last snapshot full-size count: $lastSnapshotFullSizeCount") + appendLine("Last snapshot thumbnail count : $lastSnapshotFullSizeCount") appendLine("Uploaded attachment bytes: $uploadedAttachmentBytes") appendLine("Uploaded thumbnail bytes: $uploadedThumbnailBytes") appendLine("Total upload count: $totalUploadCount") @@ -3776,10 +3793,11 @@ class AttachmentTable( fun shortPrettyString(): String { return buildString { - appendLine("Total eligible for upload rows: $totalEligibleForUploadRows") appendLine("Total unique media names eligible for upload: $totalUniqueMediaNamesEligibleForUpload") appendLine("Total unique data files: $totalUniqueDataFiles") appendLine("Total unique media names: $totalUniqueMediaNames") + appendLine("Last snapshot full-size count: $lastSnapshotFullSizeCount") + appendLine("Last snapshot thumbnail count : $lastSnapshotFullSizeCount") appendLine("Pending attachment upload bytes: $pendingAttachmentUploadBytes") if (archiveStatusMediaNameCounts.isNotEmpty()) { diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/BackupMediaSnapshotTable.kt b/app/src/main/java/org/thoughtcrime/securesms/database/BackupMediaSnapshotTable.kt index 0dfb8c4453..b1b0a68a00 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/BackupMediaSnapshotTable.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/BackupMediaSnapshotTable.kt @@ -10,10 +10,12 @@ import android.database.Cursor import androidx.annotation.VisibleForTesting import androidx.core.content.contentValuesOf import org.signal.core.util.SqlUtil +import org.signal.core.util.count import org.signal.core.util.delete import org.signal.core.util.forEach import org.signal.core.util.readToList import org.signal.core.util.readToSet +import org.signal.core.util.readToSingleInt import org.signal.core.util.readToSingleLong import org.signal.core.util.requireBoolean import org.signal.core.util.requireInt @@ -129,33 +131,43 @@ class BackupMediaSnapshotTable(context: Context, database: SignalDatabase) : Dat } /** - * Writes the set of full-size media items that are slated to be referenced in the next backup, updating their pending sync time. + * Writes a set of [MediaEntry] that are slated to be referenced in the next backup, updating their pending sync time. */ - fun writeFullSizePendingMediaObjects(mediaObjects: Sequence) { - mediaObjects - .chunked(SqlUtil.MAX_QUERY_ARGS) - .forEach { chunk -> - writePendingMediaObjectsChunk( - chunk.map { MediaEntry(it.mediaId, it.cdn, it.plaintextHash, it.remoteKey, isThumbnail = false) } - ) - } + fun writePendingMediaEntries(entries: Collection) { + if (entries.isEmpty()) { + return + } + + val values = entries.map { + contentValuesOf( + MEDIA_ID to it.mediaId, + CDN to it.cdn, + PLAINTEXT_HASH to it.plaintextHash, + REMOTE_KEY to it.remoteKey, + IS_THUMBNAIL to it.isThumbnail.toInt(), + SNAPSHOT_VERSION to UNKNOWN_VERSION, + IS_PENDING to 1 + ) + } + + SqlUtil.buildBulkInsert(TABLE_NAME, arrayOf(MEDIA_ID, CDN, PLAINTEXT_HASH, REMOTE_KEY, IS_THUMBNAIL, SNAPSHOT_VERSION, IS_PENDING), values).forEach { query -> + writableDatabase.execSQL( + query.where + + """ + ON CONFLICT($MEDIA_ID) DO UPDATE SET + $CDN = excluded.$CDN, + $PLAINTEXT_HASH = excluded.$PLAINTEXT_HASH, + $REMOTE_KEY = excluded.$REMOTE_KEY, + $IS_THUMBNAIL = excluded.$IS_THUMBNAIL, + $IS_PENDING = excluded.$IS_PENDING + """, + query.whereArgs + ) + } } /** - * Writes the set of thumbnail media items that are slated to be referenced in the next backup, updating their pending sync time. - */ - fun writeThumbnailPendingMediaObjects(mediaObjects: Sequence) { - mediaObjects - .chunked(SqlUtil.MAX_QUERY_ARGS) - .forEach { chunk -> - writePendingMediaObjectsChunk( - chunk.map { MediaEntry(it.thumbnailMediaId, it.cdn, it.plaintextHash, it.remoteKey, isThumbnail = true) } - ) - } - } - - /** - * Commits all pending entries (written via [writePendingMediaObjects]) to have a concrete [SNAPSHOT_VERSION]. The version will be 1 higher than the previous + * Commits all pending entries (written via [writePendingMediaEntries]) to have a concrete [SNAPSHOT_VERSION]. The version will be 1 higher than the previous * snapshot version. */ fun commitPendingRows() { @@ -326,37 +338,22 @@ class BackupMediaSnapshotTable(context: Context, database: SignalDatabase) : Dat .run() } - private fun writePendingMediaObjectsChunk(chunk: List) { - if (chunk.isEmpty()) { - return - } + fun debugGetFullSizeAttachmentCountForMostRecentSnapshot(): Int { + return readableDatabase + .count() + .from(TABLE_NAME) + .where("$IS_THUMBNAIL = 0 AND $SNAPSHOT_VERSION = $MAX_VERSION") + .run() + .readToSingleInt() + } - val values = chunk.map { - contentValuesOf( - MEDIA_ID to it.mediaId, - CDN to it.cdn, - PLAINTEXT_HASH to it.plaintextHash, - REMOTE_KEY to it.remoteKey, - IS_THUMBNAIL to it.isThumbnail.toInt(), - SNAPSHOT_VERSION to UNKNOWN_VERSION, - IS_PENDING to 1 - ) - } - - val query = SqlUtil.buildSingleBulkInsert(TABLE_NAME, arrayOf(MEDIA_ID, CDN, PLAINTEXT_HASH, REMOTE_KEY, IS_THUMBNAIL, SNAPSHOT_VERSION, IS_PENDING), values) - - writableDatabase.execSQL( - query.where + - """ - ON CONFLICT($MEDIA_ID) DO UPDATE SET - $CDN = excluded.$CDN, - $PLAINTEXT_HASH = excluded.$PLAINTEXT_HASH, - $REMOTE_KEY = excluded.$REMOTE_KEY, - $IS_THUMBNAIL = excluded.$IS_THUMBNAIL, - $IS_PENDING = excluded.$IS_PENDING - """, - query.whereArgs - ) + fun debugGetThumbnailAttachmentCountForMostRecentSnapshot(): Int { + return readableDatabase + .count() + .from(TABLE_NAME) + .where("$IS_THUMBNAIL != 0 AND $SNAPSHOT_VERSION = $MAX_VERSION") + .run() + .readToSingleInt() } class ArchiveMediaItem( diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/BackupMessagesJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/BackupMessagesJob.kt index fcd254689d..3886d9fd63 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/BackupMessagesJob.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/BackupMessagesJob.kt @@ -23,11 +23,13 @@ import org.signal.libsignal.net.SvrBStoreResponse import org.signal.protos.resumableuploads.ResumableUpload import org.thoughtcrime.securesms.R import org.thoughtcrime.securesms.backup.ArchiveUploadProgress -import org.thoughtcrime.securesms.backup.v2.ArchiveMediaItemIterator import org.thoughtcrime.securesms.backup.v2.ArchiveRestoreProgress import org.thoughtcrime.securesms.backup.v2.ArchiveValidator import org.thoughtcrime.securesms.backup.v2.BackupRepository import org.thoughtcrime.securesms.backup.v2.ResumableMessagesBackupUploadSpec +import org.thoughtcrime.securesms.backup.v2.util.ArchiveAttachmentInfo +import org.thoughtcrime.securesms.backup.v2.util.getAllReferencedArchiveAttachmentInfos +import org.thoughtcrime.securesms.database.BackupMediaSnapshotTable import org.thoughtcrime.securesms.database.SignalDatabase import org.thoughtcrime.securesms.dependencies.AppDependencies import org.thoughtcrime.securesms.jobmanager.Job @@ -42,8 +44,10 @@ import org.thoughtcrime.securesms.notifications.NotificationIds import org.thoughtcrime.securesms.providers.BlobProvider import org.thoughtcrime.securesms.recipients.Recipient import org.thoughtcrime.securesms.storage.StorageSyncHelper +import org.thoughtcrime.securesms.util.MediaUtil import org.thoughtcrime.securesms.util.RemoteConfig import org.whispersystems.signalservice.api.NetworkResult +import org.whispersystems.signalservice.api.backup.MediaRootBackupKey import org.whispersystems.signalservice.api.messages.AttachmentTransferProgress import org.whispersystems.signalservice.api.messages.SignalServiceAttachment import org.whispersystems.signalservice.api.svr.SvrBApi @@ -68,6 +72,7 @@ class BackupMessagesJob private constructor( companion object { private val TAG = Log.tag(BackupMessagesJob::class.java) private val FILE_REUSE_TIMEOUT = 1.hours + private const val ATTACHMENT_SNAPSHOT_BUFFER_SIZE = 10_000 const val KEY = "BackupMessagesJob" @@ -360,8 +365,11 @@ class BackupMessagesJob private constructor( val outputStream = FileOutputStream(tempBackupFile) val backupKey = SignalStore.backup.messageBackupKey + val mediaRootBackupKey = SignalStore.backup.mediaRootBackupKey val currentTime = System.currentTimeMillis() + val attachmentInfoBuffer: MutableSet = mutableSetOf() + BackupRepository.exportForSignalBackup( outputStream = outputStream, messageBackupKey = backupKey, @@ -371,8 +379,19 @@ class BackupMessagesJob private constructor( append = { tempBackupFile.appendBytes(it) }, cancellationSignal = { this.isCanceled }, currentTime = currentTime - ) { - writeMediaCursorToTemporaryTable(it, mediaBackupEnabled = SignalStore.backup.backsUpMedia) + ) { frame -> + attachmentInfoBuffer += frame.getAllReferencedArchiveAttachmentInfos() + if (attachmentInfoBuffer.size > ATTACHMENT_SNAPSHOT_BUFFER_SIZE) { + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(attachmentInfoBuffer.toFullSizeMediaEntries(mediaRootBackupKey)) + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(attachmentInfoBuffer.toThumbnailMediaEntries(mediaRootBackupKey)) + attachmentInfoBuffer.clear() + } + } + + if (attachmentInfoBuffer.isNotEmpty()) { + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(attachmentInfoBuffer.toFullSizeMediaEntries(mediaRootBackupKey)) + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(attachmentInfoBuffer.toThumbnailMediaEntries(mediaRootBackupKey)) + attachmentInfoBuffer.clear() } if (isCanceled) { @@ -422,22 +441,6 @@ class BackupMessagesJob private constructor( ) } - private fun writeMediaCursorToTemporaryTable(db: SignalDatabase, mediaBackupEnabled: Boolean) { - if (mediaBackupEnabled) { - db.attachmentTable.getFullSizeAttachmentsThatWillBeIncludedInArchive().use { - SignalDatabase.backupMediaSnapshots.writeFullSizePendingMediaObjects( - mediaObjects = ArchiveMediaItemIterator(it).asSequence() - ) - } - - db.attachmentTable.getThumbnailAttachmentsThatWillBeIncludedInArchive().use { - SignalDatabase.backupMediaSnapshots.writeThumbnailPendingMediaObjects( - mediaObjects = ArchiveMediaItemIterator(it).asSequence() - ) - } - } - } - private fun maybePostRemoteKeyMissingNotification() { if (!RemoteConfig.internalUser || !SignalStore.backup.backsUpMedia) { return @@ -457,6 +460,42 @@ class BackupMessagesJob private constructor( NotificationManagerCompat.from(context).notify(NotificationIds.INTERNAL_ERROR, notification) } + private fun Set.toFullSizeMediaEntries(mediaRootBackupKey: MediaRootBackupKey): Set { + return this + .map { + BackupMediaSnapshotTable.MediaEntry( + mediaId = it.fullSizeMediaName.toMediaId(mediaRootBackupKey).encode(), + cdn = it.cdn, + plaintextHash = it.plaintextHash.toByteArray(), + remoteKey = it.remoteKey.toByteArray(), + isThumbnail = false + ) + } + .toSet() + } + + /** + * Note: we have to remove permanently failed thumbnails here because there's no way we can know from the backup frame whether or not the thumbnail + * failed permanently independently of the attachment itself. If the attachment itself fails permanently, it's not put in the backup, so we're covered + * for full-size stuff. + */ + private fun Set.toThumbnailMediaEntries(mediaRootBackupKey: MediaRootBackupKey): Set { + return this + .filter { MediaUtil.isImageOrVideoType(it.contentType) } + .filterNot { it.forQuote } + .map { + BackupMediaSnapshotTable.MediaEntry( + mediaId = it.thumbnailMediaName.toMediaId(mediaRootBackupKey).encode(), + cdn = it.cdn, + plaintextHash = it.plaintextHash.toByteArray(), + remoteKey = it.remoteKey.toByteArray(), + isThumbnail = true + ) + } + .toSet() + .let { SignalDatabase.attachments.filterPermanentlyFailedThumbnails(it) } + } + class Factory : Job.Factory { override fun create(parameters: Parameters, serializedData: ByteArray?): BackupMessagesJob { val jobData = if (serializedData != null) { diff --git a/app/src/main/res/drawable/qrcode_logo.png b/app/src/main/res/drawable/qrcode_logo.png index 142c8a134fc19ca7678b0b16ea6c44b19bdc4d49..1c3d57989c031b9968a13cff3ef7eb4847098520 100644 GIT binary patch literal 7024 zcmeHKc{G&$+aLSBZzXDs$iysW#xfH{))5jzh%(DP491L^!IVg%WGRtSX(42nP$63p zNtU!&OImoeSe}xkdT;9K={djOIq!MSdEft-t>R_j~U9u#2;ujHJ3G z1OkyE+FQGVzbC-YNpVr|`=f7;9r)9>hvdm~B}YKn92T7s3_y9i*#H#aGw2Wqe~{_Y ztRSQ%zPQ82!9`W2V}?W5)y!SIz&Ta7cEj^TgEISx@^S59QN7SnpS%EgZ&QcEJ;B3L zOirTN?#)R-f1H-{_JwX~4llVlAJ2Qy*XGB(m$Mg>Rwp&&?h?N56K3`hngElUyQpm< zh|sFB-0?KgChuJ1npS6%3TOVAdX>Wpn9aGO>6G@ zC_eoGdAoA4`wPDIK~`%_jJ??{=|0_^>&^{U$#i{gQSdnKKE@r}mX5SvY!7RFZqg-k zQsmr9!@ffNn0SR}068z-tM@^|y2BTX^4%IzXFOltX!X)}_k4bCG5%Rr!;O}((?a%s z0!gXR&6Xmo^qOsxJ^X4({80Y?XlcfvBtXUS*7PYT}@@H z8cn;}(DH*KlkRPwHil;{NXYLMEqkBe=p&TBiCAu2J)zK3DdXc~Vg$%t9~M|RCp~c7 z7jEy?KED4Wb|K%OCsK-6wx`}`E_4BsTUh);hRqgFR@RHWH$I|STK^?7zxSz}RSW01 zu<7}6aP;^LaKt>F90^nw6G^7AC;*brWP`H@ftYRLv&qyTfCr@j{)`ZF*i>C349cLH z!?qhap`F-PKmfyj4+n7D<4mIN38La@uuU5!&G-b6fC=!(P(Cv_giGL?!BI$6$p(mpI83Rv6D5 z1_Rfjf91zyJ30NN58?h$0n`JSr$0N46TDmwjfw(-0VW9Lf>90s5z>z62i3 z-=i*DT!P6wI_O{yb7X{s@&AUB7)-#8M_y9X(AX4>#+#a&m>S{0__)789sq|6PT~@$ zAsUJOzOvL70$2pPBb!)Y)$3? zASoJyC7`hc44!0&CZJ6SMy3cfj)49J&!REtyZ<-z(&mAheQ&uvgA2yry=?lvquhYd z?^oY1gBiH;Vj0*yudr(mcb1*4YEjK3;2L;Z&)X3KzI4jItzyA3?Nz_Sqb^Dz9O8JO?? z=j%r={y#%Np}&LtD}8^<^;@ogrNF-e|DLYja{Vg>{uTK5bp4;nCHeQ`6c7U51x0`l zOD!R}o#2C(C}oSCHRRjU`$GMtbkHKfw)f&fAO>5OJ_32>CkjEM7?0>=BQ`FiDJw?E z&QOhjK%`WO)|Mpx;9K7AFg9L_dvLR=yOQ%HIMuB0-D`C?4m^Vr; zQG!>(D-R5RK(;FhSD*2{Cm0VXi^SD!%$EDwN%I0uSc}eH8b_YoA}A~<$4^Dh%NF%P zMzJy4&$w=1BAG){74PKYb)6+fEB<7K#RstB{2!@p+}Fb_-`60>KUQ%@?U*ja19BD8 zU^pl?_OxJS_wZ!Isfh@swQ-6~_ham>>~|U<4N{ z)}bs_QOgt?B@Vxwo~#n=;loJMErC0P{1EG3sb$!OC1!YzTzwP;WhZM9RkSMxRqG=) zbZkqW0>NYn@B7Aj(GDeUNB2~Hz5D#nZ=%z?M!pJ8Z@M{t{A;~-%)W_NLSm3_&dn}x z`V*Hso4bkj zC~18|xAwwps?>JuyQVN6d)(uK%rVa_o)x3#yqD$Kl$~9q+M926(qI+QnNM7<82!0B z>=Noh{js82w9`}d{59%XiuunSMTZtTML6w-^`o1MBefpR9(=LgQQIJO{EFS@5qf90 z&)q`ct-2JXI}YmpsI7Cl@Dj@eKUWG99%k+kw@Y3)NvI>?qZnOR?6iwoZ=XI@JMB=W zrI?#1d!@#Dx9xQujrfm>uLPS=YEQn2kLo{pJuFun-}J!m6~76dbesb?MRWwMQyl}tpu;#&H+x|Zq{|Fk~RHMcj zNbRt5Ex(`mz^{My#E$sLGGe~pPLJclHOJ!xe7i)Y&ek2??|AL_Kv>uW%#H)MB8ITN z`?z_|LkV|y#e@eLpMts@WMbl&C~0%_r?q6#hd3ie0U?6BA+3Vjf7`P;`*;u zqz0~xA;DW*kb@Zc8jozEk(@=5Q0mx?b{1k3Or<&v*>$Fv6e6N8dg_G!P7?@VVlAnncKkGw#CR1 zJ9$3N?8yUK7du`_xT&g*btZboUrR=R{AKOYRo1R^=8YEUzNW4AM$$Ej`Z@9=e;ALy z+a^(%vu^t}MIfm?f&iZmKWXo%a>9puU0*Ok7V|mDf2H24&UoprA+uadswp%7o~YFD z^Yh|WUa_3%!ItOJmRdQADbmll2LkI9T31xJ!S(!6S-8rpGs?BN%l+T*^{FR0Dl42@ zC+@WFP>D7W&KxLsn1^e;_n!2uVX~0oS>W(^!GdLb2BR zUAIxKBD%fnbqm!W=en=9i@n}-*(O$lZCt2)oF{Z>pxd!}HltR)NBDqJ`##Dd68`GR zZgqhik5J*+Eam;jHfZ>g!{C7#2uXTkOp&+Bp0b2fhW+K%rD^U6S%m?AR&traGnZ+> z1xsp<$4sHBk=$#)nmom|wTV z3*dL|eg9I{?%=25?#?_d%dAXH^vBA=dOWh2wx|$_-+ld7+Dxb1#Y#C>;Fx#D{Mww< zlnuiNqR&%}Fv;aDGOvWXkp4w)w09qI^16V381O6)4!t|oqXbxUQnW@9eYZTo!X*H4aX9~aB(?@*SeAgtk_N@G@0IoSDug3swN6+h=*Uwf;yeZS%Wq+iw-D?f>xx}64#G}hduCVci@93D zRo(zd-Oq+0D)KL6)ZT#1!&*8Nv(gH59;&zkF8FAorP+l<0l3ws*%=qeu2$!P^^e44 zD}3}HCH2GcH)>uqwYOOT9+^?eHo7N{dWRpboYhu2X0GmrtEpnlP{B5Os zhoNnADD7L2BOSvi%C(7RykOd1cIQNA*fyQsHm&Qi&qDpAqd9v;M$uRNP%7P&#upD0 zm3B;(NA*u$nAb0Jn|s?jpxk>fai;TV)6w|oEQ_wTjFqwE1L2P=o6m$Ag{fi3JCYq9 zXw}}v_wE^wRmzHVGj%*M)Oh~}!@UNkzfRZNa3_1csFQ1{X~cn#w}VL$?e(Q@B>nox z>>wo*(<@mARUPNP*->7NdSdFO!(Poh>PUY;`n{Vm;k0av4s0cEBEN(uoLos+++A!O za7i&prcJn@ni4v-)+gzh7%NEVT4BP~1#;*@_IPtr*J;-s()=cd+L3!tnq~MWg(Tjv zA=9fmA2T!N{L=LgKYB{}l#`-oU-d_PgI~kk`{1DClnKkBjsr4k-WrpcG%v-xTU~-u zTf<2SZw_8dS+O_u5p-1R)v`^W;dZA{Wp~gur;Szwm%_p;lq|3P_2PZOGo{E!v!7yO;f;M{;#M zqYg>7Iz`K>mYli0{ThK*x*;S=b&bq^#*0sthzXPOZR>V)8>S~>(}xrqf&;2TJ*AUI zt358+$vv$&a+Z{JYFM3N$r+O^jPH%q$!vdJyKQGAVj5GC4$NtX zZ*`a1kHv1RVb!-Wx<}58e?(O&>ficUJ-}+$^|+I%xtM4cR6OKexr_a&=xDgm)OK_6 zr-BVZz~%R2DWoXN@}pL|1tEbWcc>D>`3D=k^sfbl@VC?y9Rt@LZL{FOC4>Mz8Z-)w`n< zH8$;3+3Z*t^i${EJ_{^)`=^7DiTo=)S*yOA#;9GB)9*3$7@6 z>ob2Q>yk=r?bXN5o_ov97tE&CM{aK?V&^voB{`I61vRFb6X_G9^)i>wH|_eudY9_y zR8(a2VHkc@r+p;z`H-f<-LMSC{i27ZHXPqrA6dVi6$u$4*|k``^sd14^o>b_U4{o6 zos_c^9?G^GVyrFOZ%!GEX|I{u5T>Cm>tyiQXu5gYf>iD>PLs@%ytNZ@gV6qIe(iwg zc***t#Vo5v(?;fd1Aw2gKJ;5InQE*D+VnGbb*gG;q&$KuhVj> ty+R*;D%4PT9W#af{JCA_3I3~`n6H{d|CfRN;7v1xXya^Mw%ITCe*k)FQ#Akp literal 5676 zcmV+{7Srj8P)5m zSuw%bExNRwr;geh+m3NLB(_a(mQfi5aX@fF5e#zyRZzSC*tekS@!q@pzIX4t?^WGz zt-VCpcb{|5`QEwb>~r=$qN;R8h=>3~fP(`W4h#Yg0Qv&gs_IiAHckVs0@ehw3Rnp& z59DL*qzzo3b_FVdfJCE#iNLr(jsO~pJT};2dN6QYk$>MTB8z|pz#G6UU`b~nTZbi< zh#Uf(0-Ow-2<#uxM4uRR8i3;iX~kL*c^P;fct%yC)Bd|zB=84GnMP#p-%cuxy zidZ5tKt!eip8}5p-%PML53%-1u>7NeJFrpgS0XYX!LnjUO%N*})a}3u>=zyPNRbbf z`V@v?Kj5E$6(Vw5tq`mxhz0Dx=21N=SY{fZ!eZzFOa@viSZ2wniOIXFt`(6xfXfps zWMd?M*=;|G~Eb{P}S9C+NlU?hFAb1aszN_#K)R}7lA(k zF935@wJGkp6@!R$1110`0pGwJwp}BdnXamr#a*|gkUFtMqziDos=nmV9e|BmVZZ)O zz!QP|uc~e=yP`2dMD_vx0{H7dift`{jj|(D^)rX<6Mh)6xqLsd72=#2yBPch#^$dEX(MC5YddO#vP)&z`K)kVqF${-^90b_ts*zuK7 zz@fk}{C=ipY`*t)8+iv=2+_i@Q`?1aKuz`hB*&*Y%3N~zmV6Ycwi2qoAkRmuy$35<)eE(5Vc zbhEL|>?Da)Om<*%p-X`QF}~lppv_9D)2|7>TD3x~Qb~0uwhAh@{2svR*qN=S6v+bZ;V?0PpXnwz?np!3@-+TmQm&q z;6AGJb62V~0rvod%4nBC*wLUU<+cFlRu!?ZK+vrbUt|W*yTtP9vF)5%`s78G&DgXe zbpNzK8|@9uj96wPaBNi&3-kGW9`Oa9qQw0xEOu~F6?A;1WFhdC65FaLc4{I@d8;Yo zD-sK_u+_-Ch!y{zQhtTl@shi+Av^8PuC%mZ{;^xPJiq4<}9~I{7*<1ykS}oEDTv$Ro_9Xh8 zYf43lg&jzXGI4$r*e7Bc-GTe7pd+hQ?jhoo0_cs+*}P?J0=`rcVqvj}&qt`-Dk4q> zu|2(KtDy6$U1k7#M{L`{z(*0vcm?QE31Xdy-*40i92L?2{@9uAv^%XYK5nMI||B%OY0J=#}fsT0#V$3l^fjtO`1(*5y5* ze}s1L!JclMMU=UB8w;^=Mz3vwjE(SY?>xlJq;#1}gsn0x{C_((mnv(Nik(<(DtQDE zPJUuu;cWF6zsw+lBO04;9T6uNQVp>ek^3=6Ylir7kE&jfR^h4T#}bkA$w-wXz9%A6 z(ki^@h!rH^XwjH+GKt7&Y@@T>ltdYdV+$7T^j4fkMWi24r$v&KgpF8Ju@r?jiO8Us zD%BHsh>TQ~z+S+kA`(kmi%8!fj`3Z>Vi}7}==#(CsP8uH=tjvW8w0y{lFo#-Gs~@I z_E}$S#n|>d{WgjuD=B7dT7gP7VyT2eGoa+Lc>Wf4&C?5hTR!6|Dga1^SL;q#r9eLq5Fuc(5??d-oXf!tpuxA z*5g=wVcSBR9QKyw9kGHtls(toKCY@;9C~M9`2#XkC-%W|h!sW9*H1a@jS`Vv{Av3s zJjGd|C!Bhdoq8G4gSU@<@6_APp*BO_lrLBUUZL+jkx=4Db?bm9x-{ShWjp-+k)P>0d|q9nD~e z>OC0h)XPwP80OSl?a=984_pHr2=u}3KkgIAK5hOSI%yM;?pO-#47G_KBGOe=n?v;8 z#j-E14`f4|Kd%e^`%!H9u|(tmU`;H>GNgt5G^Ts3ZTct9j zhKyJl5hXb7&vR&DwU<~!oP~NTm!bObwo`9_L+A54 zd`DsiwMMvxV3|WFLv_MStPu{K&+4!Yeqrc=FH$-qR_!8`Siw~%9`F47b@)$*dd|i# zb=t~MeejG}-oTsXb@&g+ALG!OhuwmZp|;S1Z%`QPwD(?}s(y%X0w0=>K~>iSvEDS5 zp(1$0c@E1%tc?V7Ebs*)dPXdO*PMD8st2A-GGN#IhGC(qYAdnkJ03VpL_87GGaWh^ zstcigCL%rXjaaY4?!Cg-9S?ND$Jjq&xxq8k26g~1JM@mlyaQoa2)l{p8LE<;UuTMS+JW}E3s?j!f=|?_>)e(4C&)hr=BMx%^F&LnQao7{*LhE zr8|KEj%27q@m-aXNk>j@8{lceQ3sqFrWCj^JjqZqq>DeOswV*T6rig^Z&y0Ww#0e{ zV_g`$pF{7pC>hejgHF9k;VOV}$dEJS%_Drc+l2qBjIir}Gf7X@QQ+F$JpH)YVQ*=% zOPZb`91XykVM@VO8TUBsWk?6NtEwkX;S78kAKPDxlEcnp@v;3S=apo`F+WM#zs^kB z%N87y_Mi{{=(Kr!yXSI@)Bztl4efDVYk`XWXB9MLChg@$XWw}*!dywtMVReAg{sOeDagH!hh_#(L;(%kGad02WNzR%Y>pqG>R4>{~*QW?43*=M~l zho{Hp=-lV`V56#j1pHs_-{(Oe;M)#8@cBS8B#(_)J~R)$2OQwgdsS7J?D4k+*m@Q| zrY-~O9PD4|w3kV;^7qcssta}i)MN8I`E2A(ye1;ufRC};oWSxjtOh>8(nhWd{(lwl zh!)wz6p_C8Q%z<_4zB|fR5kSEiHI}+*JAVNVt0D41rE&XwkX2dF<7AL{vMVWvGbRt z|4Z%cvJ;qCLf;Jpreo<7^4a}G!Dq`Mjk9uBS3$#ecDbd@KI{uj!~9>ZHn&pbKtd&m z^>h_9Y-g55IJV#-5Ns-zui@5Wy0Im|Z0Fr`AG z_cn{jFfvl5iHqVU6@aRSF2<5h@92%}$q+y8R8{Y>nAnjtvBJ_LWk?1yfQ!A#A~IP- zJeQ%mkrAuoVJ( zpBP(;=M15E5BQ2_4g|Ym1?K>VxB2VE!1=1`joiwQ0!vEIGWKU7OVckq?PL;JePIbP>DIt~qZWT0kXOZ`Skr7GteTUt}z%iu|E8iT;J1?3S z^6%rFb~2QQdDwQ1H){|gG6A^Bp*jY5RYXpVTB&@*@(^N$+BL4lkP$0+%mBXb=!;zl ze2Y)u3zr9*gS{HLW_gCl)r1d`kAw`f-LMM<(*8trWVr+A8u9bDH@3y&En^cAKe4US z&hsuJ2jTOS&kgzav1FHGCWXzw->K?DQ63VJURdgb!^3pj;G!s>*tW>s((%r(!ibgm zuad!1;B3uzVzUSW-_(EOPwWA3TREQl+&PwKUerQ(qPA=+CtkDw{{b9b0;#%Vi9bS9 zBecOB;L<4N?17wb@Jt*_P{+4?iNj7NK6!%@4(SE`&7{UsM56ys{44m+Kd`^YXm^6bc7E6qBpsBvmf4<~S-a%kgq6 zBD~NxENA7-L8BaoBTms#hhD(n0?$*OnFX6LpVHYlQVBWUwq|J3%{rQ z=8zXGb~0iIwyAk9a1rK`h~zmgvC$6%uEa6}r@5$T23`Vw4D^lh{l;LKic6`}b>t-T zh?`i)I4d$Gu#tWx zmTuskK$ZbZ)UJR@5(`UsrUm$ks@@!TW!gei6%n})J3mznn}9xQsg624f?GKHVFc@r z&0}_Adv%*JKhq{vO*KW6h+K`WmOa=8d{91;8&0{PP3A$!dAeDvD9~ zRuLy8xLWIrP5Q3}9(L$YZBwren8$i-+N;hO+F=Cd0%)5_uMrsNc(xm`Ag5jO=B}#H zyD;2ItaD;2umSiga0TY2Uy)PJGKY=Z3e@ox%+VT45{wFUKm(Q;vGvxpeD5{gX=^Oz ze0_})PkzQptnSznG>ll!mSDUaww(VVwz2vq;ZDk#4!ci;*l5J|7B2w?r%j$b_1_os zr#(XZ)Ii1FC-e@Z!1jrhcp@U!cMK}C)H!PXbCDOS!+b&KVf&7=Dec_5=TkTX|fJi8YZ@ zLVS|Jm8yDup??$+!LrH49jvh(%c=Sac8S8^q;&J6lBbWV`W0UhV zv8vcVs_L8AZoyw+v2sam0yfrHjBuaz~lHXmo68p zv7huEkyEe( zY8L|%Q&}6aU*TMAQ*^G{-O*)1RX@SDL9P>#(b(0(rvnoyZ$45fVlBfWY90q((_mFl z>&P226^sa%h{$7Dr%jA#cRRLkHMfnttqGsyQi~E^f|0C0w*Po4@HK2T83jqrvDRQm zlwSaz(GqhbR6B~s;;HIO!0|yorEdY>zycoH#tk;H+u_(sqJ^qj>=;Ml2<9aB1%ikS z!koCRG9GjA7D5~&)<*1<=K?IEc{cD?M;Voh!4>4G7Q{FQKt#F%1F);&nt)YFk3?}m zRaapPgC|@6PDF-a4$k3$48fe82VjoUFno+{_^x61