diff --git a/app/src/androidTest/java/org/thoughtcrime/securesms/database/BackupMediaSnapshotTableTest.kt b/app/src/androidTest/java/org/thoughtcrime/securesms/database/BackupMediaSnapshotTableTest.kt index 328a57d9dd..c6f8412e05 100644 --- a/app/src/androidTest/java/org/thoughtcrime/securesms/database/BackupMediaSnapshotTableTest.kt +++ b/app/src/androidTest/java/org/thoughtcrime/securesms/database/BackupMediaSnapshotTableTest.kt @@ -10,7 +10,7 @@ import org.junit.runner.RunWith import org.signal.core.util.count import org.signal.core.util.readToSingleInt import org.thoughtcrime.securesms.backup.v2.ArchivedMediaObject -import org.thoughtcrime.securesms.database.BackupMediaSnapshotTable.ArchiveMediaItem +import org.thoughtcrime.securesms.database.BackupMediaSnapshotTable.MediaEntry import org.thoughtcrime.securesms.testing.SignalActivityRule @RunWith(AndroidJUnit4::class) @@ -21,7 +21,7 @@ class BackupMediaSnapshotTableTest { @Test fun givenAnEmptyTable_whenIWriteToTable_thenIExpectEmptyTable() { - SignalDatabase.backupMediaSnapshots.writeFullSizePendingMediaObjects(generateArchiveMediaItemSequence(count = 100)) + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(generateArchiveMediaItemSequence(count = 100)) val count = getCountForLatestSnapshot(includeThumbnails = true) @@ -30,7 +30,7 @@ class BackupMediaSnapshotTableTest { @Test fun givenAnEmptyTable_whenIWriteToTableAndCommit_thenIExpectFilledTable() { - SignalDatabase.backupMediaSnapshots.writeFullSizePendingMediaObjects(generateArchiveMediaItemSequence(count = 100)) + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(generateArchiveMediaItemSequence(count = 100)) SignalDatabase.backupMediaSnapshots.commitPendingRows() val count = getCountForLatestSnapshot(includeThumbnails = false) @@ -43,8 +43,8 @@ class BackupMediaSnapshotTableTest { val inputCount = 100 val countWithThumbnails = inputCount * 2 - SignalDatabase.backupMediaSnapshots.writeFullSizePendingMediaObjects(generateArchiveMediaItemSequence(count = inputCount)) - SignalDatabase.backupMediaSnapshots.writeThumbnailPendingMediaObjects(generateArchiveMediaItemSequence(count = inputCount)) + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(generateArchiveMediaItemSequence(count = inputCount)) + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(generateArchiveMediaItemSequence(count = inputCount, thumbnail = true)) SignalDatabase.backupMediaSnapshots.commitPendingRows() val count = getCountForLatestSnapshot(includeThumbnails = true) @@ -52,40 +52,16 @@ class BackupMediaSnapshotTableTest { assertThat(count).isEqualTo(countWithThumbnails) } - @Test - fun givenAnEmptyTable_whenIWriteToTableAndCommitQuotes_thenIExpectFilledTableWithNoThumbnails() { - val inputCount = 100 - - SignalDatabase.backupMediaSnapshots.writeFullSizePendingMediaObjects(generateArchiveMediaItemSequence(count = inputCount, quote = true)) - SignalDatabase.backupMediaSnapshots.commitPendingRows() - - val count = getCountForLatestSnapshot(includeThumbnails = true) - - assertThat(count).isEqualTo(inputCount) - } - - @Test - fun givenAnEmptyTable_whenIWriteToTableAndCommitNonMedia_thenIExpectFilledTableWithNoThumbnails() { - val inputCount = 100 - - SignalDatabase.backupMediaSnapshots.writeFullSizePendingMediaObjects(generateArchiveMediaItemSequence(count = inputCount, contentType = "text/plain")) - SignalDatabase.backupMediaSnapshots.commitPendingRows() - - val count = getCountForLatestSnapshot(includeThumbnails = true) - - assertThat(count).isEqualTo(inputCount) - } - @Test fun givenAFilledTable_whenIReinsertObjects_thenIExpectUncommittedOverrides() { val initialCount = 100 val additionalCount = 25 - SignalDatabase.backupMediaSnapshots.writeFullSizePendingMediaObjects(generateArchiveMediaItemSequence(count = initialCount)) + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(generateArchiveMediaItemSequence(count = initialCount)) SignalDatabase.backupMediaSnapshots.commitPendingRows() // This relies on how the sequence of mediaIds is generated in tests -- the ones we generate here will have the mediaIds as the ones we generated above - SignalDatabase.backupMediaSnapshots.writeFullSizePendingMediaObjects(generateArchiveMediaItemSequence(count = additionalCount)) + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(generateArchiveMediaItemSequence(count = additionalCount)) val pendingCount = getCountForPending(includeThumbnails = false) val latestVersionCount = getCountForLatestSnapshot(includeThumbnails = false) @@ -99,11 +75,11 @@ class BackupMediaSnapshotTableTest { val initialCount = 100 val additionalCount = 25 - SignalDatabase.backupMediaSnapshots.writeFullSizePendingMediaObjects(generateArchiveMediaItemSequence(count = initialCount)) + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(generateArchiveMediaItemSequence(count = initialCount)) SignalDatabase.backupMediaSnapshots.commitPendingRows() // This relies on how the sequence of mediaIds is generated in tests -- the ones we generate here will have the mediaIds as the ones we generated above - SignalDatabase.backupMediaSnapshots.writeFullSizePendingMediaObjects(generateArchiveMediaItemSequence(count = additionalCount)) + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(generateArchiveMediaItemSequence(count = additionalCount)) SignalDatabase.backupMediaSnapshots.commitPendingRows() val pendingCount = getCountForPending(includeThumbnails = false) @@ -120,10 +96,10 @@ class BackupMediaSnapshotTableTest { val initialCount = 100 val additionalCount = 25 - SignalDatabase.backupMediaSnapshots.writeFullSizePendingMediaObjects(generateArchiveMediaItemSequence(count = initialCount)) + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(generateArchiveMediaItemSequence(count = initialCount)) SignalDatabase.backupMediaSnapshots.commitPendingRows() - SignalDatabase.backupMediaSnapshots.writeFullSizePendingMediaObjects(generateArchiveMediaItemSequence(count = additionalCount)) + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(generateArchiveMediaItemSequence(count = additionalCount)) SignalDatabase.backupMediaSnapshots.commitPendingRows() val page = SignalDatabase.backupMediaSnapshots.getPageOfOldMediaObjects(pageSize = 1_000) @@ -146,7 +122,7 @@ class BackupMediaSnapshotTableTest { createArchiveMediaObject(seed = 2, cdn = 2) ) - SignalDatabase.backupMediaSnapshots.writeFullSizePendingMediaObjects(localData.asSequence()) + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(localData) SignalDatabase.backupMediaSnapshots.commitPendingRows() val mismatches = SignalDatabase.backupMediaSnapshots.getMediaObjectsWithNonMatchingCdn(remoteData) @@ -165,7 +141,7 @@ class BackupMediaSnapshotTableTest { createArchiveMediaObject(seed = 2, cdn = 99) ) - SignalDatabase.backupMediaSnapshots.writeFullSizePendingMediaObjects(localData.asSequence()) + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(localData) SignalDatabase.backupMediaSnapshots.commitPendingRows() val mismatches = SignalDatabase.backupMediaSnapshots.getMediaObjectsWithNonMatchingCdn(remoteData) @@ -187,7 +163,7 @@ class BackupMediaSnapshotTableTest { createArchiveMediaObject(seed = 2, cdn = 2) ) - SignalDatabase.backupMediaSnapshots.writeFullSizePendingMediaObjects(localData.asSequence()) + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(localData) SignalDatabase.backupMediaSnapshots.commitPendingRows() val notFound = SignalDatabase.backupMediaSnapshots.getMediaObjectsThatCantBeFound(remoteData) @@ -206,7 +182,7 @@ class BackupMediaSnapshotTableTest { createArchiveMediaObject(seed = 3, cdn = 2) ) - SignalDatabase.backupMediaSnapshots.writeFullSizePendingMediaObjects(localData.asSequence()) + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(localData) SignalDatabase.backupMediaSnapshots.commitPendingRows() val notFound = SignalDatabase.backupMediaSnapshots.getMediaObjectsThatCantBeFound(remoteData) @@ -223,7 +199,7 @@ class BackupMediaSnapshotTableTest { @Test fun getCurrentSnapshotVersion_singleCommit() { - SignalDatabase.backupMediaSnapshots.writeFullSizePendingMediaObjects(generateArchiveMediaItemSequence(count = 100)) + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(generateArchiveMediaItemSequence(count = 100)) SignalDatabase.backupMediaSnapshots.commitPendingRows() val version = SignalDatabase.backupMediaSnapshots.getCurrentSnapshotVersion() @@ -235,15 +211,12 @@ class BackupMediaSnapshotTableTest { fun getMediaObjectsLastSeenOnCdnBeforeSnapshotVersion_noneMarkedSeen() { val initialCount = 100 - SignalDatabase.backupMediaSnapshots.writeFullSizePendingMediaObjects(generateArchiveMediaItemSequence(count = initialCount)) - SignalDatabase.backupMediaSnapshots.writeThumbnailPendingMediaObjects(generateArchiveMediaItemSequence(count = initialCount)) + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(generateArchiveMediaItemSequence(count = initialCount)) SignalDatabase.backupMediaSnapshots.commitPendingRows() val notSeenCount = SignalDatabase.backupMediaSnapshots.getMediaObjectsLastSeenOnCdnBeforeSnapshotVersion(1).count - val expectedOldCountIncludingThumbnails = initialCount * 2 - - assertThat(notSeenCount).isEqualTo(expectedOldCountIncludingThumbnails) + assertThat(notSeenCount).isEqualTo(initialCount) } @Test @@ -251,23 +224,25 @@ class BackupMediaSnapshotTableTest { val initialCount = 100 val markSeenCount = 25 - val itemsToCommit = generateArchiveMediaItemSequence(count = initialCount) - SignalDatabase.backupMediaSnapshots.writeFullSizePendingMediaObjects(itemsToCommit) - SignalDatabase.backupMediaSnapshots.writeThumbnailPendingMediaObjects(itemsToCommit) + val fullSizeItems = generateArchiveMediaItemSequence(count = initialCount, thumbnail = false) + val thumbnailItems = generateArchiveMediaItemSequence(count = initialCount, thumbnail = true) + + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(fullSizeItems) + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(thumbnailItems) + SignalDatabase.backupMediaSnapshots.commitPendingRows() - val normalIdsToMarkSeen = itemsToCommit.take(markSeenCount).map { it.mediaId }.toList() - val thumbnailIdsToMarkSeen = itemsToCommit.take(markSeenCount).map { it.thumbnailMediaId }.toList() - val allItemsToMarkSeen = normalIdsToMarkSeen + thumbnailIdsToMarkSeen + val fullSizeIdsToMarkSeen = fullSizeItems.take(markSeenCount).map { it.mediaId }.toList() + val thumbnailIdsToMarkSeen = thumbnailItems.take(markSeenCount).map { it.mediaId }.toList() - SignalDatabase.backupMediaSnapshots.markSeenOnRemote(allItemsToMarkSeen, 1) + SignalDatabase.backupMediaSnapshots.markSeenOnRemote(fullSizeIdsToMarkSeen, 1) + SignalDatabase.backupMediaSnapshots.markSeenOnRemote(thumbnailIdsToMarkSeen, 1) val notSeenCount = SignalDatabase.backupMediaSnapshots.getMediaObjectsLastSeenOnCdnBeforeSnapshotVersion(1).count - val expectedOldCount = initialCount - markSeenCount - val expectedOldCountIncludingThumbnails = expectedOldCount * 2 + val expectedOldCount = (initialCount * 2) - (markSeenCount * 2) - assertThat(notSeenCount).isEqualTo(expectedOldCountIncludingThumbnails) + assertThat(notSeenCount).isEqualTo(expectedOldCount) } private fun getTotalItemCount(includeThumbnails: Boolean): Int { @@ -317,28 +292,30 @@ class BackupMediaSnapshotTableTest { .readToSingleInt(0) } - private fun generateArchiveMediaItemSequence(count: Int, quote: Boolean = false, contentType: String = "image/jpeg"): Sequence { + private fun generateArchiveMediaItemSequence(count: Int, thumbnail: Boolean = false): Collection { return (1..count) - .asSequence() - .map { createArchiveMediaItem(it, quote = quote, contentType = contentType) } + .map { createArchiveMediaItem(it, thumbnail = thumbnail) } + .toList() } - private fun createArchiveMediaItem(seed: Int, cdn: Int = 0, quote: Boolean = false, contentType: String = "image/jpeg"): ArchiveMediaItem { - return ArchiveMediaItem( - mediaId = "media_id_$seed", - thumbnailMediaId = "thumbnail_media_id_$seed", + private fun createArchiveMediaItem(seed: Int, thumbnail: Boolean = false, cdn: Int = 0): MediaEntry { + return MediaEntry( + mediaId = mediaId(seed, thumbnail), cdn = cdn, plaintextHash = Util.toByteArray(seed), remoteKey = Util.toByteArray(seed), - quote = quote, - contentType = contentType + isThumbnail = thumbnail ) } - private fun createArchiveMediaObject(seed: Int, cdn: Int = 0): ArchivedMediaObject { + private fun createArchiveMediaObject(seed: Int, thumbnail: Boolean = false, cdn: Int = 0): ArchivedMediaObject { return ArchivedMediaObject( - mediaId = "media_id_$seed", + mediaId = mediaId(seed, thumbnail), cdn = cdn ) } + + fun mediaId(seed: Int, thumbnail: Boolean): String { + return "media_id_${seed}_$thumbnail" + } } diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt index 304c28adf4..68820f7004 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt @@ -72,6 +72,7 @@ import org.thoughtcrime.securesms.backup.v2.processor.RecipientArchiveProcessor import org.thoughtcrime.securesms.backup.v2.processor.StickerArchiveProcessor import org.thoughtcrime.securesms.backup.v2.proto.BackupDebugInfo import org.thoughtcrime.securesms.backup.v2.proto.BackupInfo +import org.thoughtcrime.securesms.backup.v2.proto.Frame import org.thoughtcrime.securesms.backup.v2.stream.BackupExportWriter import org.thoughtcrime.securesms.backup.v2.stream.BackupImportReader import org.thoughtcrime.securesms.backup.v2.stream.EncryptedBackupReader @@ -80,6 +81,7 @@ import org.thoughtcrime.securesms.backup.v2.stream.PlainTextBackupReader import org.thoughtcrime.securesms.backup.v2.stream.PlainTextBackupWriter import org.thoughtcrime.securesms.backup.v2.ui.BackupAlert import org.thoughtcrime.securesms.backup.v2.ui.subscription.MessageBackupsType +import org.thoughtcrime.securesms.backup.v2.util.ArchiveAttachmentInfo import org.thoughtcrime.securesms.components.settings.app.AppSettingsActivity import org.thoughtcrime.securesms.components.settings.app.subscription.RecurringInAppPaymentRepository import org.thoughtcrime.securesms.crypto.AttachmentSecretProvider @@ -722,13 +724,18 @@ object BackupRepository { append = { main.write(it) } ) + val maxBufferSize = 10_000 + var totalAttachmentCount = 0 + val attachmentInfos: MutableSet = mutableSetOf() + export( currentTime = System.currentTimeMillis(), isLocal = true, writer = writer, progressEmitter = localBackupProgressEmitter, cancellationSignal = cancellationSignal, - forTransfer = false + forTransfer = false, + extraFrameOperation = null ) { dbSnapshot -> val localArchivableAttachments = dbSnapshot .attachmentTable @@ -764,7 +771,7 @@ object BackupRepository { currentTime: Long, progressEmitter: ExportProgressListener? = null, cancellationSignal: () -> Boolean = { false }, - extraExportOperations: ((SignalDatabase) -> Unit)? + extraFrameOperation: ((Frame) -> Unit)? ) { val writer = EncryptedBackupWriter.createForSignalBackup( key = messageBackupKey, @@ -782,7 +789,8 @@ object BackupRepository { forTransfer = false, progressEmitter = progressEmitter, cancellationSignal = cancellationSignal, - extraExportOperations = extraExportOperations + extraFrameOperation = extraFrameOperation, + endingExportOperation = null ) } @@ -811,7 +819,8 @@ object BackupRepository { forTransfer = true, progressEmitter = progressEmitter, cancellationSignal = cancellationSignal, - extraExportOperations = null + extraFrameOperation = null, + endingExportOperation = null ) } @@ -825,8 +834,7 @@ object BackupRepository { currentTime: Long = System.currentTimeMillis(), forTransfer: Boolean = false, progressEmitter: ExportProgressListener? = null, - cancellationSignal: () -> Boolean = { false }, - extraExportOperations: ((SignalDatabase) -> Unit)? = null + cancellationSignal: () -> Boolean = { false } ) { val writer: BackupExportWriter = if (plaintext) { PlainTextBackupWriter(outputStream) @@ -846,7 +854,8 @@ object BackupRepository { forTransfer = forTransfer, progressEmitter = progressEmitter, cancellationSignal = cancellationSignal, - extraExportOperations = extraExportOperations + extraFrameOperation = null, + endingExportOperation = null ) } @@ -868,7 +877,8 @@ object BackupRepository { forTransfer: Boolean, progressEmitter: ExportProgressListener?, cancellationSignal: () -> Boolean, - extraExportOperations: ((SignalDatabase) -> Unit)? + extraFrameOperation: ((Frame) -> Unit)?, + endingExportOperation: ((SignalDatabase) -> Unit)? ) { val eventTimer = EventTimer() val mainDbName = if (isLocal) LOCAL_MAIN_DB_SNAPSHOT_NAME else REMOTE_MAIN_DB_SNAPSHOT_NAME @@ -906,8 +916,9 @@ object BackupRepository { // We're using a snapshot, so the transaction is more for perf than correctness dbSnapshot.rawWritableDatabase.withinTransaction { progressEmitter?.onAccount() - AccountDataArchiveProcessor.export(dbSnapshot, signalStoreSnapshot) { - writer.write(it) + AccountDataArchiveProcessor.export(dbSnapshot, signalStoreSnapshot) { frame -> + writer.write(frame) + extraFrameOperation?.invoke(frame) eventTimer.emit("account") frameCount++ } @@ -919,6 +930,7 @@ object BackupRepository { progressEmitter?.onRecipient() RecipientArchiveProcessor.export(dbSnapshot, signalStoreSnapshot, exportState, selfRecipientId, selfAci) { writer.write(it) + extraFrameOperation?.invoke(it) eventTimer.emit("recipient") frameCount++ } @@ -930,6 +942,7 @@ object BackupRepository { progressEmitter?.onThread() ChatArchiveProcessor.export(dbSnapshot, exportState) { frame -> writer.write(frame) + extraFrameOperation?.invoke(frame) eventTimer.emit("thread") frameCount++ } @@ -940,6 +953,7 @@ object BackupRepository { progressEmitter?.onCall() AdHocCallArchiveProcessor.export(dbSnapshot, exportState) { frame -> writer.write(frame) + extraFrameOperation?.invoke(frame) eventTimer.emit("call") frameCount++ } @@ -951,6 +965,7 @@ object BackupRepository { progressEmitter?.onSticker() StickerArchiveProcessor.export(dbSnapshot) { frame -> writer.write(frame) + extraFrameOperation?.invoke(frame) eventTimer.emit("sticker-pack") frameCount++ } @@ -962,6 +977,7 @@ object BackupRepository { progressEmitter?.onNotificationProfile() NotificationProfileProcessor.export(dbSnapshot, exportState) { frame -> writer.write(frame) + extraFrameOperation?.invoke(frame) eventTimer.emit("notification-profile") frameCount++ } @@ -973,6 +989,7 @@ object BackupRepository { progressEmitter?.onChatFolder() ChatFolderProcessor.export(dbSnapshot, exportState) { frame -> writer.write(frame) + extraFrameOperation?.invoke(frame) eventTimer.emit("chat-folder") frameCount++ } @@ -986,6 +1003,7 @@ object BackupRepository { progressEmitter?.onMessage(0, approximateMessageCount) ChatItemArchiveProcessor.export(dbSnapshot, exportState, selfRecipientId, cancellationSignal) { frame -> writer.write(frame) + extraFrameOperation?.invoke(frame) eventTimer.emit("message") frameCount++ @@ -1001,7 +1019,7 @@ object BackupRepository { } } - extraExportOperations?.invoke(dbSnapshot) + endingExportOperation?.invoke(dbSnapshot) Log.d(TAG, "[export] totalFrames: $frameCount | ${eventTimer.stop().summary}") } finally { @@ -2071,7 +2089,7 @@ object BackupRepository { val messageBackupKey = SignalStore.backup.messageBackupKey Log.i(TAG, "[remoteRestore] Fetching SVRB data") - val svrBAuth = when (val result = BackupRepository.getSvrBAuth()) { + val svrBAuth = when (val result = getSvrBAuth()) { is NetworkResult.Success -> result.result is NetworkResult.NetworkError -> return RemoteRestoreResult.NetworkError.logW(TAG, "[remoteRestore] Network error when getting SVRB auth.", result.getCause()) is NetworkResult.StatusCodeError -> return RemoteRestoreResult.NetworkError.logW(TAG, "[remoteRestore] Status code error when getting SVRB auth.", result.getCause()) diff --git a/app/src/main/java/org/thoughtcrime/securesms/backup/v2/util/FrameExtensions.kt b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/util/FrameExtensions.kt new file mode 100644 index 0000000000..f94cc54dfc --- /dev/null +++ b/app/src/main/java/org/thoughtcrime/securesms/backup/v2/util/FrameExtensions.kt @@ -0,0 +1,103 @@ +/* + * Copyright 2025 Signal Messenger, LLC + * SPDX-License-Identifier: AGPL-3.0-only + */ + +package org.thoughtcrime.securesms.backup.v2.util + +import okio.ByteString +import org.thoughtcrime.securesms.attachments.Cdn +import org.thoughtcrime.securesms.backup.v2.proto.AccountData +import org.thoughtcrime.securesms.backup.v2.proto.Chat +import org.thoughtcrime.securesms.backup.v2.proto.ChatItem +import org.thoughtcrime.securesms.backup.v2.proto.FilePointer +import org.thoughtcrime.securesms.backup.v2.proto.Frame +import org.whispersystems.signalservice.api.backup.MediaName + +fun Frame.getAllReferencedArchiveAttachmentInfos(): Set { + val infos: MutableSet = mutableSetOf() + when { + this.account != null -> infos += this.account.getAllReferencedArchiveAttachmentInfos() + this.chat != null -> infos += this.chat.getAllReferencedArchiveAttachmentInfos() + this.chatItem != null -> infos += this.chatItem.getAllReferencedArchiveAttachmentInfos() + } + return infos.toSet() +} + +private fun AccountData.getAllReferencedArchiveAttachmentInfos(): Set { + val info = this.accountSettings?.defaultChatStyle?.wallpaperPhoto?.toArchiveAttachmentInfo() + + return if (info != null) { + setOf(info) + } else { + emptySet() + } +} + +private fun Chat.getAllReferencedArchiveAttachmentInfos(): Set { + val info = this.style?.wallpaperPhoto?.toArchiveAttachmentInfo() + + return if (info != null) { + setOf(info) + } else { + emptySet() + } +} + +private fun ChatItem.getAllReferencedArchiveAttachmentInfos(): Set { + var out: MutableSet? = null + + // The user could have many chat items, and most will not have attachments. To avoid allocating unnecessary sets, we do this little trick. + // (Note: emptySet() returns a constant under the hood, so that's fine) + fun appendToOutput(item: ArchiveAttachmentInfo) { + if (out == null) { + out = mutableSetOf() + } + + out.add(item) + } + + this.contactMessage?.contact?.avatar?.toArchiveAttachmentInfo()?.let { appendToOutput(it) } + this.directStoryReplyMessage?.textReply?.longText?.toArchiveAttachmentInfo()?.let { appendToOutput(it) } + this.standardMessage?.attachments?.mapNotNull { it.pointer?.toArchiveAttachmentInfo() }?.forEach { appendToOutput(it) } + this.standardMessage?.quote?.attachments?.mapNotNull { it.thumbnail?.pointer?.toArchiveAttachmentInfo(forQuote = true) }?.forEach { appendToOutput(it) } + this.standardMessage?.linkPreview?.mapNotNull { it.image?.toArchiveAttachmentInfo() }?.forEach { appendToOutput(it) } + this.standardMessage?.longText?.toArchiveAttachmentInfo()?.let { appendToOutput(it) } + this.stickerMessage?.sticker?.data_?.toArchiveAttachmentInfo()?.let { appendToOutput(it) } + this.viewOnceMessage?.attachment?.pointer?.toArchiveAttachmentInfo()?.let { appendToOutput(it) } + + this.revisions.forEach { revision -> + revision.getAllReferencedArchiveAttachmentInfos().forEach { appendToOutput(it) } + } + + return out ?: emptySet() +} + +private fun FilePointer.toArchiveAttachmentInfo(forQuote: Boolean = false): ArchiveAttachmentInfo? { + if (this.locatorInfo?.key == null) { + return null + } + + if (this.locatorInfo.plaintextHash == null) { + return null + } + + return ArchiveAttachmentInfo( + plaintextHash = this.locatorInfo.plaintextHash, + remoteKey = this.locatorInfo.key, + cdn = this.locatorInfo.mediaTierCdnNumber ?: Cdn.CDN_0.cdnNumber, + contentType = this.contentType, + forQuote = forQuote + ) +} + +data class ArchiveAttachmentInfo( + val plaintextHash: ByteString, + val remoteKey: ByteString, + val cdn: Int, + val contentType: String?, + val forQuote: Boolean +) { + val fullSizeMediaName: MediaName get() = MediaName.fromPlaintextHashAndRemoteKey(plaintextHash.toByteArray(), remoteKey.toByteArray()) + val thumbnailMediaName: MediaName get() = MediaName.fromPlaintextHashAndRemoteKeyForThumbnail(plaintextHash.toByteArray(), remoteKey.toByteArray()) +} diff --git a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupStatsTab.kt b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupStatsTab.kt index 56541e7578..5aa2e8bb48 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupStatsTab.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/components/settings/app/internal/backup/InternalBackupStatsTab.kt @@ -43,11 +43,6 @@ fun InternalBackupStatsTab(stats: InternalBackupPlaygroundViewModel.StatsState, label = "${stats.attachmentStats.totalUniqueMediaNames}" ) - Rows.TextRow( - text = "Total eligible for upload rows", - label = "${stats.attachmentStats.totalEligibleForUploadRows}" - ) - Rows.TextRow( text = "Total unique media names eligible for upload ⭐", label = "${stats.attachmentStats.totalUniqueMediaNamesEligibleForUpload}" @@ -73,6 +68,16 @@ fun InternalBackupStatsTab(stats: InternalBackupPlaygroundViewModel.StatsState, label = "${stats.attachmentStats.pendingAttachmentUploadBytes} (~${stats.attachmentStats.pendingAttachmentUploadBytes.bytes.toUnitString()})" ) + Rows.TextRow( + text = "Last snapshot full-size count ⭐", + label = "${stats.attachmentStats.lastSnapshotFullSizeCount}" + ) + + Rows.TextRow( + text = "Last snapshot thumbnail count ⭐", + label = "${stats.attachmentStats.lastSnapshotThumbnailCount}" + ) + Rows.TextRow( text = "Uploaded attachment bytes ⭐", label = "${stats.attachmentStats.uploadedAttachmentBytes} (~${stats.attachmentStats.uploadedAttachmentBytes.bytes.toUnitString()})" diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt b/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt index dd23804baf..871f6eee11 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/AttachmentTable.kt @@ -431,34 +431,27 @@ class AttachmentTable( } /** - * Returns a cursor (with just the plaintextHash+remoteKey+archive_cdn) for all full-size attachments that are slated to be included in the current archive upload. - * Used for snapshotting data in [BackupMediaSnapshotTable]. + * Returns a list that has any permanently-failed thumbnails removed. */ - fun getFullSizeAttachmentsThatWillBeIncludedInArchive(): Cursor { - return readableDatabase - .select(DATA_HASH_END, REMOTE_KEY, ARCHIVE_CDN, QUOTE, CONTENT_TYPE) - .from("$TABLE_NAME LEFT JOIN ${MessageTable.TABLE_NAME} ON $TABLE_NAME.$MESSAGE_ID = ${MessageTable.TABLE_NAME}.${MessageTable.ID}") - .where(buildAttachmentsThatNeedUploadQuery(transferStateFilter = "$ARCHIVE_TRANSFER_STATE != ${ArchiveTransferState.PERMANENT_FAILURE.value}")) - .run() - } + fun filterPermanentlyFailedThumbnails(entries: Set): Set { + val entriesByMediaName: MutableMap = entries + .associateBy { MediaName.fromPlaintextHashAndRemoteKeyForThumbnail(it.plaintextHash, it.remoteKey).name } + .toMutableMap() - /** - * Returns a cursor (with just the plaintextHash+remoteKey+archive_cdn) for all thumbnail attachments that are slated to be included in the current archive upload. - * Used for snapshotting data in [BackupMediaSnapshotTable]. - */ - fun getThumbnailAttachmentsThatWillBeIncludedInArchive(): Cursor { - return readableDatabase - .select(DATA_HASH_END, REMOTE_KEY, ARCHIVE_CDN, QUOTE, CONTENT_TYPE) - .from("$TABLE_NAME LEFT JOIN ${MessageTable.TABLE_NAME} ON $TABLE_NAME.$MESSAGE_ID = ${MessageTable.TABLE_NAME}.${MessageTable.ID}") - .where( - """ - ${buildAttachmentsThatNeedUploadQuery(transferStateFilter = "$ARCHIVE_THUMBNAIL_TRANSFER_STATE != ${ArchiveTransferState.PERMANENT_FAILURE.value}")} AND - $QUOTE = 0 AND - ($CONTENT_TYPE LIKE 'image/%' OR $CONTENT_TYPE LIKE 'video/%') AND - $CONTENT_TYPE != 'image/svg+xml' - """ - ) + readableDatabase + .select(DATA_HASH_END, REMOTE_KEY) + .from(TABLE_NAME) + .where("$DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL AND $ARCHIVE_THUMBNAIL_TRANSFER_STATE = ${ArchiveTransferState.PERMANENT_FAILURE.value}") .run() + .forEach { cursor -> + val hashEnd = cursor.requireNonNullString(DATA_HASH_END) + val remoteKey = cursor.requireNonNullString(REMOTE_KEY) + val thumbnailMediaName = MediaName.fromPlaintextHashAndRemoteKeyForThumbnail(Base64.decode(hashEnd), Base64.decode(remoteKey)).name + + entriesByMediaName.remove(thumbnailMediaName) + } + + return entriesByMediaName.values.toSet() } fun hasData(attachmentId: AttachmentId): Boolean { @@ -566,6 +559,25 @@ class AttachmentTable( .flatten() } + fun getLocalArchivableAttachment(plaintextHash: String, remoteKey: String): LocalArchivableAttachment? { + return readableDatabase + .select(*PROJECTION) + .from(TABLE_NAME) + .where("$DATA_HASH_END = ? AND $REMOTE_KEY = ?") + .orderBy("$ID DESC") + .limit(1) + .run() + .readToSingleObject { + LocalArchivableAttachment( + file = File(it.requireNonNullString(DATA_FILE)), + random = it.requireNonNullBlob(DATA_RANDOM), + size = it.requireLong(DATA_SIZE), + remoteKey = Base64.decode(it.requireNonNullString(REMOTE_KEY)), + plaintextHash = Base64.decode(it.requireNonNullString(DATA_HASH_END)) + ) + } + } + fun getLocalArchivableAttachments(): List { return readableDatabase .select(*PROJECTION) @@ -3214,7 +3226,7 @@ class AttachmentTable( .select(*PROJECTION) .from(TABLE_NAME) .where("$REMOTE_KEY NOT NULL AND $DATA_HASH_END NOT NULL") - .groupBy(DATA_HASH_END) + .groupBy("$DATA_HASH_END, $REMOTE_KEY") .run() .forEach { cursor -> val remoteKey = Base64.decode(cursor.requireNonNullString(REMOTE_KEY)) @@ -3239,7 +3251,6 @@ class AttachmentTable( fun debugGetAttachmentStats(): DebugAttachmentStats { val totalAttachmentRows = readableDatabase.count().from(TABLE_NAME).run().readToSingleLong(0) - val totalEligibleForUploadRows = getFullSizeAttachmentsThatWillBeIncludedInArchive().count val totalUniqueDataFiles = readableDatabase.select("COUNT(DISTINCT $DATA_FILE)").from(TABLE_NAME).run().readToSingleLong(0) val totalUniqueMediaNames = readableDatabase.query("SELECT COUNT(*) FROM (SELECT DISTINCT $DATA_HASH_END, $REMOTE_KEY FROM $TABLE_NAME WHERE $DATA_HASH_END NOT NULL AND $REMOTE_KEY NOT NULL)").readToSingleLong(0) @@ -3309,15 +3320,19 @@ class AttachmentTable( val uploadedThumbnailCount = archiveStatusMediaNameThumbnailCounts.getOrDefault(ArchiveTransferState.FINISHED, 0L) val uploadedThumbnailBytes = uploadedThumbnailCount * RemoteConfig.backupMaxThumbnailFileSize.inWholeBytes + val lastSnapshotFullSizeCount = SignalDatabase.backupMediaSnapshots.debugGetFullSizeAttachmentCountForMostRecentSnapshot() + val lastSnapshotThumbnailCount = SignalDatabase.backupMediaSnapshots.debugGetThumbnailAttachmentCountForMostRecentSnapshot() + return DebugAttachmentStats( totalAttachmentRows = totalAttachmentRows, - totalEligibleForUploadRows = totalEligibleForUploadRows.toLong(), totalUniqueMediaNamesEligibleForUpload = totalUniqueMediaNamesEligibleForUpload, totalUniqueDataFiles = totalUniqueDataFiles, totalUniqueMediaNames = totalUniqueMediaNames, archiveStatusMediaNameCounts = archiveStatusMediaNameCounts, mediaNamesWithThumbnailsCount = uniqueEligibleMediaNamesWithThumbnailsCount, archiveStatusMediaNameThumbnailCounts = archiveStatusMediaNameThumbnailCounts, + lastSnapshotFullSizeCount = lastSnapshotFullSizeCount.toLong(), + lastSnapshotThumbnailCount = lastSnapshotThumbnailCount.toLong(), pendingAttachmentUploadBytes = pendingAttachmentUploadBytes, uploadedAttachmentBytes = uploadedAttachmentBytes, uploadedThumbnailBytes = uploadedThumbnailBytes @@ -3727,13 +3742,14 @@ class AttachmentTable( data class DebugAttachmentStats( val totalAttachmentRows: Long = 0L, - val totalEligibleForUploadRows: Long = 0L, val totalUniqueMediaNamesEligibleForUpload: Long = 0L, val totalUniqueDataFiles: Long = 0L, val totalUniqueMediaNames: Long = 0L, val archiveStatusMediaNameCounts: Map = emptyMap(), val mediaNamesWithThumbnailsCount: Long = 0L, val archiveStatusMediaNameThumbnailCounts: Map = emptyMap(), + val lastSnapshotFullSizeCount: Long = 0L, + val lastSnapshotThumbnailCount: Long = 0L, val pendingAttachmentUploadBytes: Long = 0L, val uploadedAttachmentBytes: Long = 0L, val uploadedThumbnailBytes: Long = 0L @@ -3747,12 +3763,13 @@ class AttachmentTable( fun prettyString(): String { return buildString { appendLine("Total attachment rows: $totalAttachmentRows") - appendLine("Total eligible for upload rows: $totalEligibleForUploadRows") appendLine("Total unique media names eligible for upload: $totalUniqueMediaNamesEligibleForUpload") appendLine("Total unique data files: $totalUniqueDataFiles") appendLine("Total unique media names: $totalUniqueMediaNames") appendLine("Media names with thumbnails count: $mediaNamesWithThumbnailsCount") appendLine("Pending attachment upload bytes: $pendingAttachmentUploadBytes") + appendLine("Last snapshot full-size count: $lastSnapshotFullSizeCount") + appendLine("Last snapshot thumbnail count : $lastSnapshotFullSizeCount") appendLine("Uploaded attachment bytes: $uploadedAttachmentBytes") appendLine("Uploaded thumbnail bytes: $uploadedThumbnailBytes") appendLine("Total upload count: $totalUploadCount") @@ -3776,10 +3793,11 @@ class AttachmentTable( fun shortPrettyString(): String { return buildString { - appendLine("Total eligible for upload rows: $totalEligibleForUploadRows") appendLine("Total unique media names eligible for upload: $totalUniqueMediaNamesEligibleForUpload") appendLine("Total unique data files: $totalUniqueDataFiles") appendLine("Total unique media names: $totalUniqueMediaNames") + appendLine("Last snapshot full-size count: $lastSnapshotFullSizeCount") + appendLine("Last snapshot thumbnail count : $lastSnapshotFullSizeCount") appendLine("Pending attachment upload bytes: $pendingAttachmentUploadBytes") if (archiveStatusMediaNameCounts.isNotEmpty()) { diff --git a/app/src/main/java/org/thoughtcrime/securesms/database/BackupMediaSnapshotTable.kt b/app/src/main/java/org/thoughtcrime/securesms/database/BackupMediaSnapshotTable.kt index 0dfb8c4453..b1b0a68a00 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/database/BackupMediaSnapshotTable.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/database/BackupMediaSnapshotTable.kt @@ -10,10 +10,12 @@ import android.database.Cursor import androidx.annotation.VisibleForTesting import androidx.core.content.contentValuesOf import org.signal.core.util.SqlUtil +import org.signal.core.util.count import org.signal.core.util.delete import org.signal.core.util.forEach import org.signal.core.util.readToList import org.signal.core.util.readToSet +import org.signal.core.util.readToSingleInt import org.signal.core.util.readToSingleLong import org.signal.core.util.requireBoolean import org.signal.core.util.requireInt @@ -129,33 +131,43 @@ class BackupMediaSnapshotTable(context: Context, database: SignalDatabase) : Dat } /** - * Writes the set of full-size media items that are slated to be referenced in the next backup, updating their pending sync time. + * Writes a set of [MediaEntry] that are slated to be referenced in the next backup, updating their pending sync time. */ - fun writeFullSizePendingMediaObjects(mediaObjects: Sequence) { - mediaObjects - .chunked(SqlUtil.MAX_QUERY_ARGS) - .forEach { chunk -> - writePendingMediaObjectsChunk( - chunk.map { MediaEntry(it.mediaId, it.cdn, it.plaintextHash, it.remoteKey, isThumbnail = false) } - ) - } + fun writePendingMediaEntries(entries: Collection) { + if (entries.isEmpty()) { + return + } + + val values = entries.map { + contentValuesOf( + MEDIA_ID to it.mediaId, + CDN to it.cdn, + PLAINTEXT_HASH to it.plaintextHash, + REMOTE_KEY to it.remoteKey, + IS_THUMBNAIL to it.isThumbnail.toInt(), + SNAPSHOT_VERSION to UNKNOWN_VERSION, + IS_PENDING to 1 + ) + } + + SqlUtil.buildBulkInsert(TABLE_NAME, arrayOf(MEDIA_ID, CDN, PLAINTEXT_HASH, REMOTE_KEY, IS_THUMBNAIL, SNAPSHOT_VERSION, IS_PENDING), values).forEach { query -> + writableDatabase.execSQL( + query.where + + """ + ON CONFLICT($MEDIA_ID) DO UPDATE SET + $CDN = excluded.$CDN, + $PLAINTEXT_HASH = excluded.$PLAINTEXT_HASH, + $REMOTE_KEY = excluded.$REMOTE_KEY, + $IS_THUMBNAIL = excluded.$IS_THUMBNAIL, + $IS_PENDING = excluded.$IS_PENDING + """, + query.whereArgs + ) + } } /** - * Writes the set of thumbnail media items that are slated to be referenced in the next backup, updating their pending sync time. - */ - fun writeThumbnailPendingMediaObjects(mediaObjects: Sequence) { - mediaObjects - .chunked(SqlUtil.MAX_QUERY_ARGS) - .forEach { chunk -> - writePendingMediaObjectsChunk( - chunk.map { MediaEntry(it.thumbnailMediaId, it.cdn, it.plaintextHash, it.remoteKey, isThumbnail = true) } - ) - } - } - - /** - * Commits all pending entries (written via [writePendingMediaObjects]) to have a concrete [SNAPSHOT_VERSION]. The version will be 1 higher than the previous + * Commits all pending entries (written via [writePendingMediaEntries]) to have a concrete [SNAPSHOT_VERSION]. The version will be 1 higher than the previous * snapshot version. */ fun commitPendingRows() { @@ -326,37 +338,22 @@ class BackupMediaSnapshotTable(context: Context, database: SignalDatabase) : Dat .run() } - private fun writePendingMediaObjectsChunk(chunk: List) { - if (chunk.isEmpty()) { - return - } + fun debugGetFullSizeAttachmentCountForMostRecentSnapshot(): Int { + return readableDatabase + .count() + .from(TABLE_NAME) + .where("$IS_THUMBNAIL = 0 AND $SNAPSHOT_VERSION = $MAX_VERSION") + .run() + .readToSingleInt() + } - val values = chunk.map { - contentValuesOf( - MEDIA_ID to it.mediaId, - CDN to it.cdn, - PLAINTEXT_HASH to it.plaintextHash, - REMOTE_KEY to it.remoteKey, - IS_THUMBNAIL to it.isThumbnail.toInt(), - SNAPSHOT_VERSION to UNKNOWN_VERSION, - IS_PENDING to 1 - ) - } - - val query = SqlUtil.buildSingleBulkInsert(TABLE_NAME, arrayOf(MEDIA_ID, CDN, PLAINTEXT_HASH, REMOTE_KEY, IS_THUMBNAIL, SNAPSHOT_VERSION, IS_PENDING), values) - - writableDatabase.execSQL( - query.where + - """ - ON CONFLICT($MEDIA_ID) DO UPDATE SET - $CDN = excluded.$CDN, - $PLAINTEXT_HASH = excluded.$PLAINTEXT_HASH, - $REMOTE_KEY = excluded.$REMOTE_KEY, - $IS_THUMBNAIL = excluded.$IS_THUMBNAIL, - $IS_PENDING = excluded.$IS_PENDING - """, - query.whereArgs - ) + fun debugGetThumbnailAttachmentCountForMostRecentSnapshot(): Int { + return readableDatabase + .count() + .from(TABLE_NAME) + .where("$IS_THUMBNAIL != 0 AND $SNAPSHOT_VERSION = $MAX_VERSION") + .run() + .readToSingleInt() } class ArchiveMediaItem( diff --git a/app/src/main/java/org/thoughtcrime/securesms/jobs/BackupMessagesJob.kt b/app/src/main/java/org/thoughtcrime/securesms/jobs/BackupMessagesJob.kt index fcd254689d..3886d9fd63 100644 --- a/app/src/main/java/org/thoughtcrime/securesms/jobs/BackupMessagesJob.kt +++ b/app/src/main/java/org/thoughtcrime/securesms/jobs/BackupMessagesJob.kt @@ -23,11 +23,13 @@ import org.signal.libsignal.net.SvrBStoreResponse import org.signal.protos.resumableuploads.ResumableUpload import org.thoughtcrime.securesms.R import org.thoughtcrime.securesms.backup.ArchiveUploadProgress -import org.thoughtcrime.securesms.backup.v2.ArchiveMediaItemIterator import org.thoughtcrime.securesms.backup.v2.ArchiveRestoreProgress import org.thoughtcrime.securesms.backup.v2.ArchiveValidator import org.thoughtcrime.securesms.backup.v2.BackupRepository import org.thoughtcrime.securesms.backup.v2.ResumableMessagesBackupUploadSpec +import org.thoughtcrime.securesms.backup.v2.util.ArchiveAttachmentInfo +import org.thoughtcrime.securesms.backup.v2.util.getAllReferencedArchiveAttachmentInfos +import org.thoughtcrime.securesms.database.BackupMediaSnapshotTable import org.thoughtcrime.securesms.database.SignalDatabase import org.thoughtcrime.securesms.dependencies.AppDependencies import org.thoughtcrime.securesms.jobmanager.Job @@ -42,8 +44,10 @@ import org.thoughtcrime.securesms.notifications.NotificationIds import org.thoughtcrime.securesms.providers.BlobProvider import org.thoughtcrime.securesms.recipients.Recipient import org.thoughtcrime.securesms.storage.StorageSyncHelper +import org.thoughtcrime.securesms.util.MediaUtil import org.thoughtcrime.securesms.util.RemoteConfig import org.whispersystems.signalservice.api.NetworkResult +import org.whispersystems.signalservice.api.backup.MediaRootBackupKey import org.whispersystems.signalservice.api.messages.AttachmentTransferProgress import org.whispersystems.signalservice.api.messages.SignalServiceAttachment import org.whispersystems.signalservice.api.svr.SvrBApi @@ -68,6 +72,7 @@ class BackupMessagesJob private constructor( companion object { private val TAG = Log.tag(BackupMessagesJob::class.java) private val FILE_REUSE_TIMEOUT = 1.hours + private const val ATTACHMENT_SNAPSHOT_BUFFER_SIZE = 10_000 const val KEY = "BackupMessagesJob" @@ -360,8 +365,11 @@ class BackupMessagesJob private constructor( val outputStream = FileOutputStream(tempBackupFile) val backupKey = SignalStore.backup.messageBackupKey + val mediaRootBackupKey = SignalStore.backup.mediaRootBackupKey val currentTime = System.currentTimeMillis() + val attachmentInfoBuffer: MutableSet = mutableSetOf() + BackupRepository.exportForSignalBackup( outputStream = outputStream, messageBackupKey = backupKey, @@ -371,8 +379,19 @@ class BackupMessagesJob private constructor( append = { tempBackupFile.appendBytes(it) }, cancellationSignal = { this.isCanceled }, currentTime = currentTime - ) { - writeMediaCursorToTemporaryTable(it, mediaBackupEnabled = SignalStore.backup.backsUpMedia) + ) { frame -> + attachmentInfoBuffer += frame.getAllReferencedArchiveAttachmentInfos() + if (attachmentInfoBuffer.size > ATTACHMENT_SNAPSHOT_BUFFER_SIZE) { + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(attachmentInfoBuffer.toFullSizeMediaEntries(mediaRootBackupKey)) + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(attachmentInfoBuffer.toThumbnailMediaEntries(mediaRootBackupKey)) + attachmentInfoBuffer.clear() + } + } + + if (attachmentInfoBuffer.isNotEmpty()) { + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(attachmentInfoBuffer.toFullSizeMediaEntries(mediaRootBackupKey)) + SignalDatabase.backupMediaSnapshots.writePendingMediaEntries(attachmentInfoBuffer.toThumbnailMediaEntries(mediaRootBackupKey)) + attachmentInfoBuffer.clear() } if (isCanceled) { @@ -422,22 +441,6 @@ class BackupMessagesJob private constructor( ) } - private fun writeMediaCursorToTemporaryTable(db: SignalDatabase, mediaBackupEnabled: Boolean) { - if (mediaBackupEnabled) { - db.attachmentTable.getFullSizeAttachmentsThatWillBeIncludedInArchive().use { - SignalDatabase.backupMediaSnapshots.writeFullSizePendingMediaObjects( - mediaObjects = ArchiveMediaItemIterator(it).asSequence() - ) - } - - db.attachmentTable.getThumbnailAttachmentsThatWillBeIncludedInArchive().use { - SignalDatabase.backupMediaSnapshots.writeThumbnailPendingMediaObjects( - mediaObjects = ArchiveMediaItemIterator(it).asSequence() - ) - } - } - } - private fun maybePostRemoteKeyMissingNotification() { if (!RemoteConfig.internalUser || !SignalStore.backup.backsUpMedia) { return @@ -457,6 +460,42 @@ class BackupMessagesJob private constructor( NotificationManagerCompat.from(context).notify(NotificationIds.INTERNAL_ERROR, notification) } + private fun Set.toFullSizeMediaEntries(mediaRootBackupKey: MediaRootBackupKey): Set { + return this + .map { + BackupMediaSnapshotTable.MediaEntry( + mediaId = it.fullSizeMediaName.toMediaId(mediaRootBackupKey).encode(), + cdn = it.cdn, + plaintextHash = it.plaintextHash.toByteArray(), + remoteKey = it.remoteKey.toByteArray(), + isThumbnail = false + ) + } + .toSet() + } + + /** + * Note: we have to remove permanently failed thumbnails here because there's no way we can know from the backup frame whether or not the thumbnail + * failed permanently independently of the attachment itself. If the attachment itself fails permanently, it's not put in the backup, so we're covered + * for full-size stuff. + */ + private fun Set.toThumbnailMediaEntries(mediaRootBackupKey: MediaRootBackupKey): Set { + return this + .filter { MediaUtil.isImageOrVideoType(it.contentType) } + .filterNot { it.forQuote } + .map { + BackupMediaSnapshotTable.MediaEntry( + mediaId = it.thumbnailMediaName.toMediaId(mediaRootBackupKey).encode(), + cdn = it.cdn, + plaintextHash = it.plaintextHash.toByteArray(), + remoteKey = it.remoteKey.toByteArray(), + isThumbnail = true + ) + } + .toSet() + .let { SignalDatabase.attachments.filterPermanentlyFailedThumbnails(it) } + } + class Factory : Job.Factory { override fun create(parameters: Parameters, serializedData: ByteArray?): BackupMessagesJob { val jobData = if (serializedData != null) { diff --git a/app/src/main/res/drawable/qrcode_logo.png b/app/src/main/res/drawable/qrcode_logo.png index 142c8a134f..1c3d57989c 100644 Binary files a/app/src/main/res/drawable/qrcode_logo.png and b/app/src/main/res/drawable/qrcode_logo.png differ